From c0566b2b07514897e78a820c8aecae43809eafe5 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 2 Nov 2015 16:18:56 +0100 Subject: [PATCH 0001/2083] Move extension init into Crawler.crawl() --- scrapy/crawler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index bdcfa9d0c..2794a5837 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -43,9 +43,7 @@ class Crawler(object): lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) - self.extensions = ExtensionManager.from_crawler(self) - self.settings.freeze() self.crawling = False self.spider = None self.engine = None @@ -67,6 +65,9 @@ class Crawler(object): self.crawling = True try: + self.settings.freeze() + self.extensions = ExtensionManager.from_crawler(self) + self.spider = self._create_spider(*args, **kwargs) self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) From d67f292d92a23c94f731596e4a9462583f176740 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 2 Nov 2015 16:36:33 +0100 Subject: [PATCH 0002/2083] Move Spider.update_settings() into Crawler.crawl() --- scrapy/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 2794a5837..66f28e3e3 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -29,7 +29,6 @@ class Crawler(object): self.spidercls = spidercls self.settings = settings.copy() - self.spidercls.update_settings(self.settings) self.signals = SignalManager(self) self.stats = load_object(self.settings['STATS_CLASS'])(self) @@ -65,6 +64,7 @@ class Crawler(object): self.crawling = True try: + self.spidercls.update_settings(self.settings) self.settings.freeze() self.extensions = ExtensionManager.from_crawler(self) From b06a670777058f19bb249a1d192a2cea27ea0475 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 2 Nov 2015 16:47:07 +0100 Subject: [PATCH 0003/2083] Initialize spider before calling its update_settings() --- scrapy/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 66f28e3e3..3742f86cc 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -64,11 +64,11 @@ class Crawler(object): self.crawling = True try: + self.spider = self._create_spider(*args, **kwargs) self.spidercls.update_settings(self.settings) self.settings.freeze() self.extensions = ExtensionManager.from_crawler(self) - self.spider = self._create_spider(*args, **kwargs) self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) From 86c74ce53e1fca2174bf38ed75399ba298df16a4 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 2 Nov 2015 16:57:57 +0100 Subject: [PATCH 0004/2083] Allow Spider.update_settings() to be an instance method --- scrapy/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 3742f86cc..0754276f3 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -65,7 +65,7 @@ class Crawler(object): try: self.spider = self._create_spider(*args, **kwargs) - self.spidercls.update_settings(self.settings) + self.spider.update_settings(self.settings) self.settings.freeze() self.extensions = ExtensionManager.from_crawler(self) From 4e40377bcb2b88c893fb0a7e842401aab2cab896 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Tue, 3 Nov 2015 23:32:02 +0100 Subject: [PATCH 0005/2083] Allow multiple calls to Crawler.crawl() --- scrapy/crawler.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 0754276f3..d121e90a5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -64,6 +64,13 @@ class Crawler(object): self.crawling = True try: + # Support multiple calls to crawl() + if self.settings.frozen: + # Dirty hack, this should probably be more like + # self.settings = self.settings.mutable_copy() + # or maybe + # self.settings.unfreeze() + self.settings.frozen = False self.spider = self._create_spider(*args, **kwargs) self.spider.update_settings(self.settings) self.settings.freeze() From 2c68c95cadbf45e11657b00058ff29921c27bffa Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Tue, 3 Nov 2015 23:46:48 +0100 Subject: [PATCH 0006/2083] Move stats & log init into crawl() --- scrapy/crawler.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d121e90a5..10147524d 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -31,17 +31,6 @@ class Crawler(object): self.settings = settings.copy() self.signals = SignalManager(self) - self.stats = load_object(self.settings['STATS_CLASS'])(self) - - handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL')) - logging.root.addHandler(handler) - # lambda is assigned to Crawler attribute because this way it is not - # garbage collected after leaving __init__ scope - self.__remove_handler = lambda: logging.root.removeHandler(handler) - self.signals.connect(self.__remove_handler, signals.engine_stopped) - - lf_cls = load_object(self.settings['LOG_FORMATTER']) - self.logformatter = lf_cls.from_crawler(self) self.crawling = False self.spider = None @@ -74,6 +63,20 @@ class Crawler(object): self.spider = self._create_spider(*args, **kwargs) self.spider.update_settings(self.settings) self.settings.freeze() + + self.stats = load_object(self.settings['STATS_CLASS'])(self) + + handler = LogCounterHandler(self, + level=self.settings.get('LOG_LEVEL')) + logging.root.addHandler(handler) + # lambda is assigned to Crawler attribute because this way it is not + # garbage collected after leaving __init__ scope + self.__remove_handler = lambda: logging.root.removeHandler(handler) + self.signals.connect(self.__remove_handler, signals.engine_stopped) + + lf_cls = load_object(self.settings['LOG_FORMATTER']) + self.logformatter = lf_cls.from_crawler(self) + self.extensions = ExtensionManager.from_crawler(self) self.engine = self._create_engine() From aafb31d6fb86cb05ae1945685adbb615566068c4 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 16:26:55 +0100 Subject: [PATCH 0007/2083] Revert "Move stats & log init into crawl()" This reverts commit 2c68c95cadbf45e11657b00058ff29921c27bffa. --- scrapy/crawler.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 10147524d..d121e90a5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -31,6 +31,17 @@ class Crawler(object): self.settings = settings.copy() self.signals = SignalManager(self) + self.stats = load_object(self.settings['STATS_CLASS'])(self) + + handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL')) + logging.root.addHandler(handler) + # lambda is assigned to Crawler attribute because this way it is not + # garbage collected after leaving __init__ scope + self.__remove_handler = lambda: logging.root.removeHandler(handler) + self.signals.connect(self.__remove_handler, signals.engine_stopped) + + lf_cls = load_object(self.settings['LOG_FORMATTER']) + self.logformatter = lf_cls.from_crawler(self) self.crawling = False self.spider = None @@ -63,20 +74,6 @@ class Crawler(object): self.spider = self._create_spider(*args, **kwargs) self.spider.update_settings(self.settings) self.settings.freeze() - - self.stats = load_object(self.settings['STATS_CLASS'])(self) - - handler = LogCounterHandler(self, - level=self.settings.get('LOG_LEVEL')) - logging.root.addHandler(handler) - # lambda is assigned to Crawler attribute because this way it is not - # garbage collected after leaving __init__ scope - self.__remove_handler = lambda: logging.root.removeHandler(handler) - self.signals.connect(self.__remove_handler, signals.engine_stopped) - - lf_cls = load_object(self.settings['LOG_FORMATTER']) - self.logformatter = lf_cls.from_crawler(self) - self.extensions = ExtensionManager.from_crawler(self) self.engine = self._create_engine() From fc2639731953fb35fd52c374215ae40f5db80665 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 16:32:45 +0100 Subject: [PATCH 0008/2083] Revert "Allow multiple calls to Crawler.crawl()" This reverts commit 4e40377bcb2b88c893fb0a7e842401aab2cab896. --- scrapy/crawler.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d121e90a5..0754276f3 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -64,13 +64,6 @@ class Crawler(object): self.crawling = True try: - # Support multiple calls to crawl() - if self.settings.frozen: - # Dirty hack, this should probably be more like - # self.settings = self.settings.mutable_copy() - # or maybe - # self.settings.unfreeze() - self.settings.frozen = False self.spider = self._create_spider(*args, **kwargs) self.spider.update_settings(self.settings) self.settings.freeze() From 380f76d35fa0350a37f17fbb2e2065f1e527cadd Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 16:54:22 +0100 Subject: [PATCH 0009/2083] Fix tests that had multiple calls to crawl() --- tests/test_crawl.py | 18 ++++++++++++------ tests/test_downloader_handlers.py | 1 + 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 814eb30d2..35de5527d 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -61,14 +61,16 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_failure(self): - crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider) - yield crawler.crawl(n=0.5) + runner = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}) + crawler = runner.create_crawler(DelaySpider) + yield runner.crawl(crawler, n=0.5) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) # server hangs after receiving response headers - yield crawler.crawl(n=0.5, b=1) + crawler = runner.create_crawler(DelaySpider) + yield runner.crawl(crawler, n=0.5, b=1) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) @@ -131,11 +133,15 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider) - yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3) + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(DuplicateStartRequestsSpider) + yield runner.crawl(crawler, dont_filter=True, distinct_urls=2, + dupe_factor=3) self.assertEqual(crawler.spider.visited, 6) - yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4) + crawler = runner.create_crawler(DuplicateStartRequestsSpider) + yield runner.crawl(crawler, dont_filter=False, distinct_urls=3, + dupe_factor=4) self.assertEqual(crawler.spider.visited, 3) @defer.inlineCallbacks diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e4d957d8e..7feb6b9c1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -308,6 +308,7 @@ class Http11MockServerTestCase(unittest.TestCase): # download_maxsize < 100, hence the CancelledError self.assertIsInstance(failure.value, defer.CancelledError) + crawler = get_crawler(SingleRequestSpider) request.headers.setdefault('Accept-Encoding', 'gzip,deflate') request = request.replace(url='http://localhost:8998/xpayload') yield crawler.crawl(seed=request) From daec0457110e188a131271f9fce86ecb1dbd14c6 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 18:07:05 +0100 Subject: [PATCH 0010/2083] Move spider settings tests --- tests/test_crawl.py | 32 +++++++++++++++++++++++++++++++- tests/test_crawler.py | 32 -------------------------------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 35de5527d..021849f41 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -7,7 +7,10 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.http import Request -from scrapy.crawler import CrawlerRunner +from scrapy.crawler import Crawler, CrawlerRunner +from scrapy.extensions.throttle import AutoThrottle +from scrapy.settings import Settings +from scrapy.utils.spider import DefaultSpider from tests import mock from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider @@ -272,3 +275,30 @@ with multiples lines self._assert_retried(log) self.assertIn("Got response 200", str(log)) + + @defer.inlineCallbacks + def test_populate_spider_settings(self): + spider_settings = {'TEST1': 'spider', 'TEST2': 'spider', + 'AUTOTHROTTLE_ENABLED': True} + project_settings = {'TEST1': 'project', 'TEST3': 'project'} + + class CustomSettingsSpider(DefaultSpider): + custom_settings = spider_settings + + def parse(self, response): + return + + settings = Settings() + settings.setdict(project_settings, priority='project') + crawler = Crawler(CustomSettingsSpider, settings) + yield crawler.crawl() + + self.assertEqual(crawler.settings.get('TEST1'), 'spider') + self.assertEqual(crawler.settings.get('TEST2'), 'spider') + self.assertEqual(crawler.settings.get('TEST3'), 'project') + + enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] + self.assertIn(AutoThrottle, enabled_exts) + + self.assertFalse(settings.frozen) + self.assertTrue(crawler.settings.frozen) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 53a1202e3..96bf0c866 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -7,7 +7,6 @@ from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object -from scrapy.extensions.throttle import AutoThrottle class BaseCrawlerTest(unittest.TestCase): @@ -33,24 +32,6 @@ class CrawlerTestCase(BaseCrawlerTest): self.crawler.spiders self.assertEqual(len(w), 1, "Warn deprecated access only once") - def test_populate_spidercls_settings(self): - spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'} - project_settings = {'TEST1': 'project', 'TEST3': 'project'} - - class CustomSettingsSpider(DefaultSpider): - custom_settings = spider_settings - - settings = Settings() - settings.setdict(project_settings, priority='project') - crawler = Crawler(CustomSettingsSpider, settings) - - self.assertEqual(crawler.settings.get('TEST1'), 'spider') - self.assertEqual(crawler.settings.get('TEST2'), 'spider') - self.assertEqual(crawler.settings.get('TEST3'), 'project') - - self.assertFalse(settings.frozen) - self.assertTrue(crawler.settings.frozen) - def test_crawler_accepts_dict(self): crawler = Crawler(DefaultSpider, {'foo': 'bar'}) self.assertEqual(crawler.settings['foo'], 'bar') @@ -61,19 +42,6 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED') -class SpiderSettingsTestCase(unittest.TestCase): - def test_spider_custom_settings(self): - class MySpider(scrapy.Spider): - name = 'spider' - custom_settings = { - 'AUTOTHROTTLE_ENABLED': True - } - - crawler = Crawler(MySpider, {}) - enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] - self.assertIn(AutoThrottle, enabled_exts) - - class SpiderLoaderWithWrongInterface(object): def unneeded_method(self): From 2629997a2f620c54ab5b052947a0132a52042984 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Tue, 10 Nov 2015 23:48:20 +0100 Subject: [PATCH 0011/2083] Make Spider.update_settings() an instance method --- scrapy/spiders/__init__.py | 5 ++--- tests/test_spider.py | 8 +++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index c08bb964a..4b2771415 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -75,9 +75,8 @@ class Spider(object_ref): def parse(self, response): raise NotImplementedError - @classmethod - def update_settings(cls, settings): - settings.setdict(cls.custom_settings or {}, priority='spider') + def update_settings(self, settings): + settings.setdict(self.custom_settings or {}, priority='spider') @classmethod def handles_request(cls, request): diff --git a/tests/test_spider.py b/tests/test_spider.py index 4d5d4b07e..63a3d2e61 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -98,11 +98,17 @@ class SpiderTest(unittest.TestCase): self.spider_class.custom_settings = spider_settings settings = Settings(project_settings, priority='project') - self.spider_class.update_settings(settings) + spider = self.spider_class('example.com') + spider.update_settings(settings) self.assertEqual(settings.get('TEST1'), 'spider') self.assertEqual(settings.get('TEST2'), 'spider') self.assertEqual(settings.get('TEST3'), 'project') + spider_instance_settings = {'TEST1': 'spider_instance'} + spider.custom_settings = spider_instance_settings + spider.update_settings(settings) + self.assertEqual(settings.get('TEST1'), 'spider_instance') + def test_logger(self): spider = self.spider_class('example.com') with LogCapture() as l: From cfed9b6659c90e0799361911b1d72ed127edf471 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 15 Jul 2015 17:27:57 +0200 Subject: [PATCH 0012/2083] Allow passing Python objects to middleware dict settings --- scrapy/middleware.py | 5 ++++- scrapy/utils/misc.py | 7 ++++++- tests/test_middleware.py | 13 +++++++++++++ tests/test_utils_misc/__init__.py | 4 +++- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 2ef5f30e2..690ec6a55 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,4 +1,5 @@ from collections import defaultdict +from inspect import isclass import logging import pprint @@ -31,7 +32,9 @@ class MiddlewareManager(object): for clspath in mwlist: try: mwcls = load_object(clspath) - if crawler and hasattr(mwcls, 'from_crawler'): + if not isclass(mwcls): + mw = mwcls + elif crawler and hasattr(mwcls, 'from_crawler'): mw = mwcls.from_crawler(crawler) elif hasattr(mwcls, 'from_settings'): mw = mwcls.from_settings(settings) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 303a413d8..75f42cc17 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -31,10 +31,15 @@ def arg_to_iter(arg): def load_object(path): """Load an object given its absolute object path, and return it. - object can be a class, function, variable o instance. + If ``path`` is not a string, it will be returned. + + The object can be a class, function, variable, or instance. path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware' """ + if not isinstance(path, six.string_types): + return path + try: dot = path.rindex('.') except ValueError: diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b6d885330..4e3c67d20 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -91,3 +91,16 @@ class MiddlewareManagerTest(unittest.TestCase): mwman = TestMiddlewareManager.from_settings(settings) classes = [x.__class__ for x in mwman.middlewares] self.assertEqual(classes, [M1, M3]) + + def test_instances_from_settings(self): + settings = Settings() + myM3 = M3() + class InstanceTestMiddlewareManager(MiddlewareManager): + @classmethod + def _get_mwlist_from_settings(cls, settings): + return [ 'tests.test_middleware.M1', M2, myM3 ] + mwman = InstanceTestMiddlewareManager.from_settings(settings) + self.assertIsInstance(mwman.middlewares[0], M1) + self.assertIsInstance(mwman.middlewares[1], M2) + self.assertIs(mwman.middlewares[2], myM3) + diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 01460a10b..06af3c009 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -11,7 +11,9 @@ class UtilsMiscTestCase(unittest.TestCase): def test_load_object(self): obj = load_object('scrapy.utils.misc.load_object') - assert obj is load_object + self.assertIs(obj, load_object) + not_a_string = int(1000) + self.assertIs(load_object(not_a_string), not_a_string) self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') From e5b8def0b86102eacdb9a49942e88047a2250527 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 1 Jun 2015 17:39:41 +0200 Subject: [PATCH 0013/2083] Redraft SEP-021 --- sep/sep-021.rst | 338 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 289 insertions(+), 49 deletions(-) diff --git a/sep/sep-021.rst b/sep/sep-021.rst index 628a95dd2..ce500fc00 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -17,19 +17,31 @@ Scrapy currently supports many hooks and mechanisms for extending its functionality, but no single entry point for enabling and configuring them. Instead, the hooks are spread over: -* Spider middlewares (SPIDER_MIDDLEWARES) -* Downloader middlewares (DOWNLOADER_MIDDLEWARES) -* Downloader handlers (DOWNLOADER_HANDLERS) -* Item pipelines (ITEM_PIPELINES) -* Feed exporters and storages (FEED_EXPORTERS, FEED_STORAGES) -* Overrideable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) -* Generic extensions (EXTENSIONS) -* CLI commands (COMMANDS_MODULE) +* Spider middlewares (``SPIDER_MIDDLEWARES``) +* Downloader middlewares (``DOWNLOADER_MIDDLEWARES``) +* Downloader handlers (``DOWNLOADER_HANDLERS``) +* Item pipelines (``ITEM_PIPELINES``) +* Feed exporters and storages (``FEED_EXPORTERS``, ``FEED_STORAGES``) +* Overrideable components (``DUPEFILTER_CLASS``, ``STATS_CLASS``, + ``SCHEDULER``, ``SPIDER_MANAGER_CLASS``, ``ITEM_PROCESSOR``, etc.) +* Generic extensions (``EXTENSIONS``) +* CLI commands (``COMMANDS_MODULE``) + +This approach has several shortfalls: + +* Enabling an extension often requires modifying many settings, often in a + coordinated way, which is complex and error prone. +* Extension developers have little control over ensuring their library + dependencies and configuration requirements are met, especially since most + extensions never 'see' a fully-configured crawler before it starts running. +* The user is burdened with supervising potential interplay of extensions, + especially non-included ones, ranging from setting name clashes to mutually + excluding dependencies/configuration requirements. + +*Add-ons* search to remedy these shortcomings by enhancing Scrapy's extension +management, making it easy-to-use and transparent for users while giving more +configuration control to developers. -One problem of this approach is that enabling an extension often requires -modifying many settings, often in a coordinated way, which is complex and error -prone. Add-ons are meant to fix this by providing a simple mechanism for -enabling extensions. Design goals and non-goals ========================== @@ -37,8 +49,8 @@ Design goals and non-goals Goals: * simple to manage: adding or removing extensions should be just a matter of - adding or removing lines in a ``scrapy.cfg`` file -* backward compatibility with enabling extension the "old way" (ie. modifying + adding or removing lines in a configuration file +* backward compatibility with enabling extension the "old way" (i.e. modifying settings directly) Non-goals: @@ -46,62 +58,290 @@ Non-goals: * a way to publish, distribute or discover extensions (use pypi for that) -Managing add-ons -================ +User experience: managing add-ons +================================= -Add-ons are defined in the ``scrapy.cfg`` file, inside the ``[addons]`` -section. +Add-ons are enabled and configured either via Scrapy's settings, or (for add-ons +not bound to any project) in ``scrapy.cfg``. -To enable the "httpcache" addon, either shipped with Scrapy or in the Python -search path, create an entry for it in your ``scrapy.cfg``, like this:: +In the settings, add-ons can be enabled by adding either their name (for +built-in add-ons), their Python path, or their file path, to a +``INSTALLED_ADDONS`` setting. If necessary, each add-on can be configured by +providing a dictionary-valued setting with the uppercase add-on name. For +example, to enable and configure the built-in ``httpcache`` add-on and enable +(without configuring) two custom add-ons, one via Python path and one via file +path, add these entries to your settings module:: - [addons] - httpcache = + INSTALLED_ADDONS = ( + 'httpcache', + 'mymodule.filters.myfilter', + 'mymodule/filters/otherfilter.py', + ) -You may also specify the full path to an add-on (which may be either a .py file -or a folder containing __init__.py):: + HTTPCACHE = { + 'ignore_http_codes': [404, 503], + } - [addons] - mongodb_pipeline = /path/to/mongodb_pipeline.py +In ``scrapy.cfg``, add-ons are enabled and configured with one section per +add-on. The section names correspond to the entries of ``INSTALLED_ADDONS``. +The configuration from above could look like this:: + + [addon:httpcache] + ignore_http_codes = 404,503 + + [addon:mymodule.filters.myfilter] + + [addon:mymodule/filters/otherfilter.py] -Writing add-ons -=============== +Developer experience: writing add-ons +===================================== -Add-ons are Python modules that implement the following callbacks. +Add-ons are (any) Python *objects* that implement Scrapy's *add-on interface*. +The interface is enforced through ``zope.interface``. This leaves the choice of +Python object up the developer. Examples: -addon_configure ---------------- +* for a small pipeline, the add-on interface could be implemented in the same + class that also implements the ``open/close_spider`` and ``process_item`` + callbacks +* for larger add-ons, or for clearer structure, the interface could be provided + by a stand-alone module -Receives the Settings object and modifies it to enable the required components. -If it raises an exception, Scrapy will print it and exit. +The absolute minimum interface consists of just two attributes: -Examples:: +* ``NAME``: string with add-on name +* ``VERSION``: PEP-440 style version string - def addon_configure(settings): - settings.overrides['DOWNLADER_MIDDLEWARES'].update({ - 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, - }) +To be any useful, an add-on should implement at least one of the following +callback methods: + +* ``update_addons()``: adds and configures other add-ons +* ``update_settings()``: sets configuration (such as default values for this + add-on and required settings for other extensions) and enables needed + components. +* ``check_configuration()``: receives the fully-initialized ``Crawler`` + instance before it starts running, performs additional dependency and + configuration requirement checks + +Additionally, an add-on may (and should, where appropriate) provide one or more +variables that can be used for automated detection of possible dependency +clashes: + +* ``REQUIRES``: list of built-in or custom components required by this add-on, + as PEP-440 strings +* ``MODIFIES``: list of components whose functionality is affected or replaced + by this add-on (a custom HTTP cache should list ``httpcache`` here) +* ``PROVIDES``: list of components provided by this add-on (e.g. ``mongodb`` + for an extension that provides generic read/write access to a MongoDB + database, releasing other components from having to provide their own + database access methods) + +update_addons() +----------------- + +Called: +~~~~~~~ + +Shortly after initialisation of the ``Crawler`` object. + +Arguments: +~~~~~~~~~~ + +* ``config``: configuration of this add-on +* ``addons``: the add-on manager, providing methods to add and configure add-ons + +Purpose: +~~~~~~~~ + +* Configure and enable related add-ons, useful for 'umbrella add-ons' which + chain-load other add-ons based on the configuration + +Examples: +~~~~~~~~~ :: - def addon_configure(settings): + def update_addons(config, addons): + if 'httpcache' not in addons.enabled: + addons.add('httpcache', {'expiration_secs': 60}) + +or:: + + def update_addons(config, addons): + if 'otheraddon' in addons.enabled: + addons.configs['otheraddon']['some_config_name'] = True + +update_settings() +----------------- + +Called: +~~~~~~~ + +Directly after the ``update_addons()`` callback of all add-ons has been called. + +Arguments: +~~~~~~~~~~ + +* ``config``: configuration of this add-on +* ``settings``: the crawler's ``Settings`` instance containing all project + settings + +Purpose: +~~~~~~~~ + +* Modify ``settings`` to enable required components +* Expose some add-on specific configuration (``config``) into the global + settings namespace (``settings``) if necessary +* Raise exception if components can not be properly configured (e.g. on missing + dependencies); Scrapy will print this exception *and exit* (making users + explicitly acknowledge that the add-on does not work by forcing them to + disable it). + +Side note: +~~~~~~~~~~ + +The ``MiddlewareManager.from_settings()`` method will receive a slight +modification to allow directly placing Python objects instead of class paths +in the middleware dict settings. This way, add-ons can place already +instantiated components into the settings. This allows keeping configuration +as local to components as possible and avoids cluttering up the global +settings namespace. Furthermore, it allows reusing components (e.g. using +two instances of the same mongodb pipeline to write to different locations). + +Examples: +~~~~~~~~~ + +:: + + def update_settings(config, settings): + # Don't care where this module is located + settings.set['DOWNLADER_MIDDLEWARES']({ + __name__ + '.downloadermw.coolmw': 900, + }) + + # Instantiate components to not expose settings into + # the global namespace + from .pipelines import MySQLPipeline + mysqlpl = MySQLPipeline(password = config['password']) + settings.set['ITEM_PIPELINES']({ + mysqlpl: 200, + }) + +or:: + + def update_settings(config, settings): + # Assuming this class also has a process_item() method + settings.set['ITEM_PIPELINES']({ + self: 200, + }) + +or:: + + def update_settings(config, settings): try: import boto except ImportError: raise RuntimeError("boto library is required") +check_configuration() +--------------------- -crawler_ready -------------- +Called: +~~~~~~~ -``crawler_ready`` receives a Crawler object after it has been initialized and -is meant to be used to perform post-initialization checks like making sure the -extension and its dependencies were configured properly. If it raises an -exception, Scrapy will print and exit. +Shortly before the crawler starts crawling. -Examples:: +Arguments: +~~~~~~~~~~ + +* ``config``: configuration of this add-on +* ``crawler``: fully-initialized ``Crawler`` object, ready to start crawling + +Purpose: +~~~~~~~~ + +* Perform post-initialization checks like making sure the extension and its + dependencies were configured properly. +* Raise exception if a critical check failed; Scrapy will print this exception + *and exit* (see ``update_settings()`` purpose for rationale on this). + +Examples: +~~~~~~~~~ + +:: + + def check_configuration(config, crawler): + if 'some.other.addon' not in crawler.addons.enabled: + raise RuntimeError("Some other add-on required to use this add-on") + + +Implementation +============== + +A new core component, the *add-on manager*, is introduced to Scrapy. It +facilitates loading add-ons, gathering and providing information on them, +calling their callbacks at appropriate times, and performing basic checks for +dependency and configuration clashes. + +Layout +------ + +A new ``AddonManager`` class is introduced, providing methods to + +* add and remove add-ons, +* search for add-ons by name +* read enabled add-ons and their configurations from the settings module and + from ``settings.py``, +* enable and disable add-ons +* check for possible dependency incompatibilites by inspecting the collected + ``REQUIRES``, ``MODIFIES`` and ``PROVIDES`` add-on variables +* call the add-on callbacks + +Integration into start-up process +--------------------------------- + +The settings used to crawl are not complete until the spider-specific settings +have been loaded in ``Crawler.__init__()``. Add-on management follows this +approach and only starts loading add-ons when the crawler is initialised. + +Instantiation and the calls ``update_addons()`` and ``update_settings()`` happen +in ``Crawler.__init__()``. The final checks (i.e. the callback to +``check_configuration()``) is coded into the ``Crawler.crawl()`` method after +creating the engine. + +Finding add-ons +--------------- + +Add-on localisation is governed by the add-on paths given in +``INSTALLED_ADDONS`` (or by the section names if using ``scrapy.cfg``). If +nothing is found at the given path, it is tried again with ``addons.`` +prepended (i.e. pointing to the project's ``addons`` folder or module), then +with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons`` +submodule). If the object found has an ``_addon`` attribute, that attribute +will be treated as the found add-on. This allows, for example, to change the +add-on based on the Python version. + +Updating existing extensions +---------------------------- + +An ``Addon`` class is introduced that add-on developers may or may not subclass +depending on how much of the 'default functionality' they want. Naturally, it +does not provide ``NAME`` and ``VERSION``. Its default ``update_settings()`` +exposes the add-on configuration into the global settings namespace with an +appropriate name, e.g. this section from ``scrapy.cfg``:: + + [httpcache] + dir = /some/dir + +would expose ``HTTPCACHE_DIR``. + +Add-on modules will be written for all built-in extensions and placed in +``scrapy.addons``. For many default Scrapy components, it will be sufficient to +create a subclass of ``Addon`` with minor or no method modifications. The +component code remains where it is (i.e. in ``scrapy.pipelines``, etc.). + +Later, the global settings namespace could be cleaned up in a backwards +-incompatible fashion by deprecating support for the global setting names, e.g. +``HTTPCACHE_DIR``, and instead instantiate the components with the add-on +configuration in ``update_settings()``. - def crawler_ready(crawler): - if 'some.other.addon' not in crawler.extensions.enabled: - raise RuntimeError("Some other addon is required to use this addon") From d8af395d7654ba659aa29060cf3cf0ed3dfe1174 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 19 Aug 2015 16:16:53 +0200 Subject: [PATCH 0014/2083] Introduce add-ons via AddonManager and Addon base class --- docs/topics/api.rst | 11 + docs/topics/settings.rst | 9 + scrapy/addons/__init__.py | 497 ++++++++++++++++++ scrapy/interfaces.py | 23 +- scrapy/settings/__init__.py | 1 + scrapy/settings/default_settings.py | 2 + scrapy/utils/conf.py | 12 +- scrapy/utils/misc.py | 42 +- scrapy/utils/project.py | 12 + tests/test_addons/__init__.py | 388 ++++++++++++++ tests/test_addons/addonmod.py | 16 + tests/test_addons/addons.py | 40 ++ tests/test_addons/cfg.cfg | 5 + tests/test_addons/project/__init__.py | 0 tests/test_addons/project/addons/__init__.py | 0 tests/test_addons/project/addons/addonmod.py | 7 + tests/test_addons/project/addons/addonmod2.py | 7 + tests/test_addons/scrapy_addons/__init__.py | 0 tests/test_addons/scrapy_addons/addonmod.py | 7 + tests/test_addons/scrapy_addons/addonmod2.py | 7 + tests/test_addons/scrapy_addons/addonmod3.py | 7 + tests/test_utils_misc/__init__.py | 26 +- tests/test_utils_misc/testmod.py | 1 + tests/test_utils_misc/testpkg/__init__.py | 1 + tests/test_utils_misc/testpkg/submod.py | 1 + tests/test_utils_project.py | 27 + 26 files changed, 1142 insertions(+), 7 deletions(-) create mode 100644 scrapy/addons/__init__.py create mode 100644 tests/test_addons/__init__.py create mode 100644 tests/test_addons/addonmod.py create mode 100644 tests/test_addons/addons.py create mode 100644 tests/test_addons/cfg.cfg create mode 100644 tests/test_addons/project/__init__.py create mode 100644 tests/test_addons/project/addons/__init__.py create mode 100644 tests/test_addons/project/addons/addonmod.py create mode 100644 tests/test_addons/project/addons/addonmod2.py create mode 100644 tests/test_addons/scrapy_addons/__init__.py create mode 100644 tests/test_addons/scrapy_addons/addonmod.py create mode 100644 tests/test_addons/scrapy_addons/addonmod2.py create mode 100644 tests/test_addons/scrapy_addons/addonmod3.py create mode 100644 tests/test_utils_misc/testmod.py create mode 100644 tests/test_utils_misc/testpkg/__init__.py create mode 100644 tests/test_utils_misc/testpkg/submod.py create mode 100644 tests/test_utils_project.py diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 42c0133c1..0c22b3ce9 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -149,6 +149,17 @@ Settings API .. autoclass:: BaseSettings :members: +.. _topics-api-addonmanager: + +AddonManager API +================ + +.. module:: scrapy.addons + :synopsis: Add-on manager + +.. autoclass:: AddonManager + :members: + .. _topics-api-spiderloader: SpiderLoader API diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index aa0417e1a..afcb8dd21 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -585,6 +585,15 @@ some of them need to be enabled through a setting. For more information See the :ref:`extensions user guide ` and the :ref:`list of available extensions `. +.. setting:: INSTALLED_ADDONS + +INSTALLED_ADDONS +---------------- + +Default: ``()`` + +A tuple containing paths to the add-ons enabled in your project. For more +information, see :ref:`topics-addons`. .. setting:: ITEM_PIPELINES diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py new file mode 100644 index 000000000..59e59e15f --- /dev/null +++ b/scrapy/addons/__init__.py @@ -0,0 +1,497 @@ +from collections import defaultdict, Mapping +from importlib import import_module +from inspect import isclass +import os +import six +import warnings + +from pkg_resources import WorkingSet, Distribution, Requirement +import zope.interface +from zope.interface.verify import verifyObject + +from scrapy.exceptions import NotConfigured +from scrapy.interfaces import IAddon +from scrapy.settings import BaseSettings +from scrapy.utils.conf import config_from_filepath, get_config +from scrapy.utils.misc import load_module_or_object +from scrapy.utils.project import get_project_path + + +@zope.interface.implementer(IAddon) +class Addon(object): + + basic_settings = None + """``dict`` of settings that will be exported via :meth:`export_basics`.""" + + default_config = None + """``dict`` with default configuration.""" + + config_mapping = None + """``dict`` with mappings from config names to setting names. The given + setting names will be taken as given, i.e. they will be neither prefixed + nor uppercased. + """ + + component_type = None + """Component setting into which to export via :meth:`export_component`. Can + be any of the dictionary-like component setting names (e.g. + ``DOWNLOADER_MIDDLEWARES``) or any of their abbreviations in + :attr:`~scrapy.addons.COMPONENT_TYPE_ABBR`. If ``None``, + :meth:`export_component` will do nothing. + """ + + component_key = None + """Key to be used in the component dictionary setting when exporting via + :meth:`export_component`. This is only useful for the settings that have + no order, e.g. ``DOWNLOAD_HANDLERS`` or ``FEED_EXPORTERS``. + """ + + component_order = 0 + """Component order to use when not given in the add-on configuration. Has + no effect for component types that use :attr:`component_key`. + """ + + component = None + """Component to be inserted via :meth:`export_component`. This can be + anything that can be used in the dictionary-like component settings, i.e. + a class path, a class, or an instance. If ``None``, it is assumed that the + add-on itself is also provides the component interface, and ``self`` will be + used. + """ + + settings_prefix = None + """Prefix with which the add-on configuration will be exported into the + global settings namespace via :meth:`export_config`. If ``None``, + :attr:`name` will be used. If ``False``, no configuration will be exported. + """ + + def export_component(self, config, settings): + """Export the component in :attr:`component` into the dictionary-like + component setting derived from :attr:`component_type`. + + Where applicable, the order parameter of the component (i.e. the + dictionary value) will be retrieved from the ``order`` add-on + configuration value. + + :param config: Add-on configuration from which to read component order + :type config: ``dict`` + + :param settings: Settings object into which to export component + :type settings: :class:`~scrapy.settings.Settings` + """ + if self.component_type: + comp = self.component or self + if self.component_key: + # e.g. for DOWNLOAD_HANDLERS: {'http': 'myclass'} + k = self.component_key + v = comp + else: + # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} + k = comp + v = config.get('order', self.component_order) + settings.set(self.component_type, {k: v}, 'addon') + + def export_basics(self, settings): + """Export the :attr:`basic_settings` attribute into the settings object. + + All settings will be exported with ``addon`` priority (see + :ref:`topics-api-settings`). + + :param settings: Settings object into which to expose the basic settings + :type settings: :class:`~scrapy.settings.Settings` + """ + for setting, value in six.iteritems(self.basic_settings or {}): + settings.set(setting, value, 'addon') + + def export_config(self, config, settings): + """Export the add-on configuration, all keys in caps and with + :attr:`settings_prefix` or :attr:`name` prepended, into the settings + object. + + For example, the add-on configuration ``{'key': 'value'}`` will export + the setting ``ADDONNAME_KEY`` with a value of ``value``. All settings + will be exported with ``addon`` priority (see + :ref:`topics-api-settings`). + + :param config: Add-on configuration to be exposed + :type config: ``dict`` + + :param settings: Settings object into which to export the configuration + :type settings: :class:`~scrapy.settings.Settings` + """ + if self.settings_prefix is False: + return + conf = self.default_config or {} + conf.update(config) + prefix = self.settings_prefix or self.name + # Since default exported config is case-insensitive (everything will be + # uppercased), make mapped config case-insensitive as well + conf_mapping = {k.lower(): v + for k, v in six.iteritems(self.config_mapping or {})} + for key, val in six.iteritems(conf): + if key.lower() in conf_mapping: + key = conf_mapping[key.lower()] + else: + key = (prefix + '_' + key).upper() + settings.set(key, val, 'addon') + + def update_settings(self, config, settings): + """Export both the basic settings and the add-on configuration. I.e., + call :meth:`export_basics` and :meth:`export_config`. + + For more advanced add-ons, you may want to override this callback. + + :param config: Add-on configuration + :type config: ``dict`` + + :param settings: Crawler settings object + :type settings: :class:`~scrapy.settings.Settings` + """ + self.export_component(config, settings) + self.export_basics(settings) + self.export_config(config, settings) + + +class AddonManager(Mapping): + """This class facilitates loading and storing :ref:`topics-addons`. + + You can treat it like a read-only dictionary in which keys correspond to + add-on names and values correspond to the add-on objects:: + + addons = AddonManager() + # ... load some add-ons here + print addons.enabled # prints names of all enabled add-ons + print addons['TestAddon'].version # prints version of add-on with name + # 'TestAddon' + + """ + + def __init__(self): + self._addons = {} + self.configs = {} + self._disable_on_add = [] + + def __getitem__(self, name): + return self._addons[name] + + def __delitem__(self, name): + del self._addons[name] + del self.configs[name] + + def __iter__(self): + return iter(self._addons) + + def __len__(self): + return len(self._addons) + + def add(self, addon, config=None): + """Store an add-on. + + If ``addon`` is a string, it will be treated as add-on path and passed + to :meth:`get_addon`. Otherwise, ``addon`` must be a Python object + implementing or providing Scrapy's add-on interface. The interface + will be enforced through ``zope.interface``'s ``verifyObject()``. + + If ``addon`` is a class, it will be instantiated. You can avoid this + (for example if you have implemented the add-on callbacks as class + methods) by declaring -- via ``zope.interface`` -- that your class + directly *provides* ``scrapy.interfaces.IAddon``. + + :param addon: The add-on object (or path) to be stored + :type addon: Any Python object providing the add-on interface or ``str`` + + :param config: The add-on configuration dictionary + :type config: ``dict`` + """ + addon = self.get_addon(addon) + if isclass(addon) and not IAddon.providedBy(addon): + addon = addon() + if not IAddon.providedBy(addon): + zope.interface.alsoProvides(addon, IAddon) + # zope.interface's exceptions are already quite helpful. Still, should + # we catch them and log an error message? + verifyObject(IAddon, addon) + name = addon.name + if name in self: + raise ValueError("Addon '{}' already loaded".format(name)) + self._addons[name] = addon + self.configs[name] = config or {} + if name in self._disable_on_add: + self.configs[name]['_enabled'] = False + self._disable_on_add.remove(name) + + def remove(self, addon): + """Remove an add-on. + + If ``addon`` is the name of a stored add-on, that add-on will be + removed. Otherwise, you can use the argument in the same fashion as + in :meth:`add`. + + :param addon: The add-on name, object, or path to be removed + :type addon: Any Python object providing the add-on interface or ``str`` + """ + if addon in self: + del self[addon] + elif hasattr(addon, 'name') and addon.name in self: + del self[addon.name] + else: + try: + del self[self.get_addon(addon).name] + except NameError: + raise KeyError + + @staticmethod + def get_addon(path): + """Get an add-on object by its Python or file path. + + ``path`` is assumed to be either a Python or a file path of a Scrapy + add-on. If no object is found at ``path``, it is tried again first with + ``projectname.addons`` prepended (pointing to the current project's + ``addons`` folder), then with ``scrapy.addons`` prepended (poiting to + Scrapy's built-in add-ons). These convenience shortcuts will only work + with Python paths, not file paths. + + If the object or module pointed to by ``path`` has an attribute named + ``_addon`` that attribute will be assumed to be the add-on. + :meth:`get_addon` will keep following ``_addon`` attributes until it + finds an object that does not have an attribute named ``_addon``. + + :param path: Python or file path to an add-on + :type path: ``str`` + """ + if isinstance(path, six.string_types): + prefixes = ['', 'scrapy.addons.'] + try: + prefixes.insert(1, get_project_path() + '.addons.') + except NotConfigured: + warnings.warn("Unable to locate project Python path") + for prefix in prefixes: + fullpath = prefix + path + try: + obj = load_module_or_object(fullpath) + except NameError: + pass + else: + break + else: + raise NameError("Could not find add-on '%s'" % path) + else: + obj = path + if hasattr(obj, '_addon'): + obj = AddonManager.get_addon(obj._addon) + return obj + + def load_dict(self, addonsdict): + """Load add-ons and configurations from given dictionary. + + Each add-on should be an entry in the dictionary, where the key + corresponds to the add-on path. The value should be a dictionary + representing the add-on configuration. + + Example add-on dictionary:: + + addonsdict = { + 'path.to.addon1': { + 'setting1': 'value', + 'setting2': 42, + }, + 'path/to/addon2.py': { + 'addon2setting': True, + }, + } + + :param addonsdict: dictionary where keys correspond to add-on paths \ + and values correspond to their configuration + :type addonsdict: ``dict`` + """ + for addonpath, addoncfg in six.iteritems(addonsdict): + self.add(addonpath, addoncfg) + + def load_settings(self, settings): + """Load add-ons and configurations from settings object. + + This will invoke :meth:`get_addon` for every add-on path in the + ``INSTALLED_ADDONS`` setting. For each of these add-ons, the + configuration will be read from the dictionary setting whose name + matches the uppercase add-on name. + + :param settings: The :class:`~scrapy.settings.Settings` object from \ + which to read the add-on configuration + :type settings: :class:`~scrapy.settings.Settings` + """ + paths = settings.getlist('INSTALLED_ADDONS') + addons = [self.get_addon(path) for path in paths] + configs = [settings.getdict(addon.name.upper()) for addon in addons] + for a, c in zip(addons, configs): + self.add(a, c) + + def load_cfg(self, cfg=None): + """Load add-ons and configurations from given ``ConfigParser`` object or + config file path. + + Each add-on should have its own section, where the section has a name in + the form ``addon:my_addon_path``. The add-on object is searched for via + the :meth:`get_addon` method, ``my_addon_path`` can be either a Python + or a file path. + + If ``cfg`` is ``None``, ``scrapy.cfg`` will be used. + + :param cfg: ``ConfigParser`` object or config file path from which to \ + read add-on configuration + :type cfg: ``ConfigParser`` or ``str`` + """ + if cfg is None: + cfg = get_config() + elif isinstance(cfg, six.string_types): + cfg = config_from_filepath(cfg) + for secname in cfg.sections(): + if secname.startswith("addon:"): + addonkey = secname.split("addon:", 1)[1] + addoncfg = dict(cfg.items(secname)) + self.add(addonkey, addoncfg) + + def check_dependency_clashes(self): + """Check for incompatibilities in add-on dependencies. + + Add-ons can provide information about their dependencies in their + ``provides``, ``modifies`` and ``requires`` attributes. This method will + raise an ``ImportError`` if + + * a component required by an add-on is not provided by any other add-on, + or + * a component modified by an add-on is not provided by any other add-on, + or + * the same component is provided by more than one add-on, + + and warn when a component required by an add-on is modified by any other + add-on. + """ + # Collect all active add-ons and the components they provide + ws = WorkingSet('') + def add_dist(project_name, version, **kwargs): + if project_name in ws.entry_keys.get('scrapy', []): + raise ImportError("Component {} provided by multiple add-ons" + "".format(project_name)) + else: + dist = Distribution(project_name=project_name, version=version, + **kwargs) + ws.add(dist, entry='scrapy') + for name in self: + ver = self[name].version + add_dist(name, ver) + for provides_name in getattr(self[name], 'provides', []): + add_dist(provides_name, ver) + + # Collect all required and modified components + def compile_attribute_dict(attribute_name): + attrs = defaultdict(list) + for name in self: + for entry in getattr(self[name], attribute_name, []): + attrs[entry].append(name) + return attrs + modified = compile_attribute_dict('modifies') + required = compile_attribute_dict('requires') + + req_or_mod = set(required.keys()).union(modified.keys()) + for reqstr in req_or_mod: + req = Requirement.parse(reqstr) + # May raise VersionConflict. Do we want to catch it and raise + # our own exception or is it helpful enough? + if ws.find(req) is None: + raise ImportError( + "Add-ons {} require or modify missing component {}" + "".format(required[reqstr]+modified[reqstr], reqstr)) + + mod_and_req = set(required.keys()).intersection(modified.keys()) + for conflict in mod_and_req: + warnings.warn("Component '{}', required by add-ons {}, is modified " + "by add-ons {}".format(conflict, required[conflict], + modified[conflict])) + + def disable(self, addon): + """Disable an add-on, i.e. prevent its callbacks from being called. + + If you disable an add-on before it is loaded, it will be disabled as + soon as it is added to the :class:`AddonManager`. + + :param addon: Name of the add-on to be disabled + :type addon: ``str`` + """ + if addon in self: + self.configs[addon]['_enabled'] = False + else: + self._disable_on_add.append(addon) + + def enable(self, addon): + """Re-enable a disabled add-on. + + Will raise ``ValueError`` if the add-on is neither already loaded nor + marked for being disabled on adding. + + :param addon: Name of the add-on to be enabled + :type addon: ``str`` + """ + if addon in self: + self.configs[addon]['_enabled'] = True + elif addon in self._disable_on_add: + self._disable_on_add.remove(addon) + else: + raise ValueError("Add-ons need to be added before they can be " + "enabled") + + @property + def disabled(self): + """Names of disabled add-ons""" + return ([a for a in self if not self.configs[a].get('_enabled', True)] + + self._disable_on_add) + + @property + def enabled(self): + """Names of enabled add-ons""" + return [a for a in self if self.configs[a].get('_enabled', True)] + + def _call_if_exists(self, obj, cbname, *args, **kwargs): + if obj is None: + return + try: + cb = getattr(obj, cbname) + except AttributeError: + return + else: + cb(*args, **kwargs) + + def _call_addon(self, addonname, cbname, *args, **kwargs): + if self.configs[addonname].get('_enabled', True): + self._call_if_exists(self[addonname], cbname, + self.configs[addonname], *args, **kwargs) + + def update_addons(self): + """Call ``update_addons()`` of all held add-ons. + + This will also call ``update_addons()`` of all add-ons that are added + last minute during the ``update_addons()`` routine of other add-ons. + """ + called_addons = set() + while called_addons != set(self): + for name in set(self).difference(called_addons): + called_addons.add(name) + self._call_addon(name, 'update_addons', self) + + def update_settings(self, settings): + """Call ``update_settings()`` of all held add-ons. + + :param settings: The :class:`~scrapy.settings.Settings` object to be \ + updated + :type settings: :class:`~scrapy.settings.Settings` + """ + for name in self: + self._call_addon(name, 'update_settings', settings) + + def check_configuration(self, crawler): + """Call ``check_configuration()`` of all held add-ons. + + :param crawler: the fully-initialized crawler + :type crawler: :class:`~scrapy.crawler.Crawler` + """ + for name in self: + self._call_addon(name, 'check_configuration', crawler) diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index eb93c6f7e..75b72899e 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,6 +1,6 @@ -from zope.interface import Interface +import zope.interface -class ISpiderLoader(Interface): +class ISpiderLoader(zope.interface.Interface): def from_settings(settings): """Return an instance of the class for the given settings""" @@ -20,3 +20,22 @@ class ISpiderLoader(Interface): # ISpiderManager is deprecated, don't use it! # An alias is kept for backwards compatibility. ISpiderManager = ISpiderLoader + + +class IAddon(zope.interface.Interface): + """Scrapy add-on""" + + name = zope.interface.Attribute("""Add-on name""") + version = zope.interface.Attribute("""Add-on version string (PEP440)""") + + # XXX: Can methods be declared optional? I.e., can I enforce the signature + # but not the existence of a method? + + #def update_addons(config, addons): + # """Enables and configures other add-ons""" + + #def update_settings(config, settings): + # """Modifies `settings` to enable and configure required components""" + + #def check_configuration(config, crawler): + # """Performs post-initialization checks on fully configured `crawler`""" diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 342d2585e..be9f740eb 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -14,6 +14,7 @@ from . import default_settings SETTINGS_PRIORITIES = { 'default': 0, 'command': 10, + 'addon': 15, 'project': 20, 'spider': 30, 'cmdline': 40, diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8435b0354..a230750fb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -167,6 +167,8 @@ HTTPCACHE_DBM_MODULE = 'anydbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False +INSTALLED_ADDONS = () + ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index e8af90f11..5a5418104 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -82,14 +82,20 @@ def init_env(project='default', set_syspath=True): sys.path.append(projdir) -def get_config(use_closest=True): - """Get Scrapy config file as a SafeConfigParser""" - sources = get_sources(use_closest) +def config_from_filepath(sources): + """Create a SafeConfigParser and read in the given `sources`, which can be + either a filename or a list of filenames.""" cfg = SafeConfigParser() cfg.read(sources) return cfg +def get_config(use_closest=True): + """Get Scrapy config file as a SafeConfigParser""" + sources = get_sources(use_closest) + return config_from_filepath(sources) + + def get_sources(use_closest=True): xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \ os.path.expanduser('~/.config') diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 75f42cc17..9461d93e9 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,5 +1,8 @@ """Helper functions which doesn't fit anywhere else""" +import itertools +import os.path import re +import sys import hashlib from importlib import import_module from pkgutil import iter_modules @@ -56,6 +59,26 @@ def load_object(path): return obj +def load_module_or_object(path): + """Load python module or (non-module) object from given path. + + Path can be both a Python or a file path. + """ + try: + return import_module(path) + except ImportError: + pass + try: + return load_object(path) + except (ValueError, NameError, ImportError): + pass + try: + return get_module_from_filepath(path) + except ImportError: + pass + raise NameError("Could not load '%s'" % path) + + def walk_modules(path): """Loads a module and all its submodules from a the given module path and returns them. If *any* module throws an exception while importing, that @@ -78,6 +101,23 @@ def walk_modules(path): return mods +def get_module_from_filepath(path): + """Load and return a python module/package from a file path""" + path = path.rstrip("/") + if path.endswith('.py'): + path = path.rsplit('.py', 1)[0] + basefolder, modname = os.path.split(path) + # XXX: There are other ways to import modules from a full path which don't + # need to modify PYTHONPATH, see + # https://stackoverflow.com/questions/67631/ + # These methods differ between py2 and py3, and apparently the + # py3 method was deprecated in Python 3.4 + sys.path.insert(0, basefolder) + mod = import_module(modname) + sys.path.pop(0) + return mod + + def extract_regex(regex, text, encoding='utf-8'): """Extract a list of unicode strings from the given text/encoding using the following policies: @@ -118,7 +158,7 @@ def md5sum(file): m.update(d) return m.hexdigest() + def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" return True if rel is not None and 'nofollow' in rel.split() else False - diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index a15a0d90f..a1266c879 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -71,3 +71,15 @@ def get_project_settings(): settings.setdict(env_overrides, priority='project') return settings + +def get_project_path(): + """Return the Python path of the current project. + + This fails when the settings module does not live in the project's root. + """ + if not inside_project(): + raise NotConfigured("Not inside a project") + settings_module_path = os.environ.get(ENVVAR) + if not settings_module_path: + raise NotConfigured("Unable to locate project's python path") + return settings_module_path.rsplit('.', 1)[0] diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py new file mode 100644 index 000000000..c98f0ab65 --- /dev/null +++ b/tests/test_addons/__init__.py @@ -0,0 +1,388 @@ +import os.path +import six +from six.moves.configparser import SafeConfigParser +import sys +from tests import mock +import unittest +import warnings + +from pkg_resources import VersionConflict +import zope.interface +from zope.interface.verify import verifyObject +from zope.interface.exceptions import BrokenImplementation + +import scrapy.addons +from scrapy.addons import Addon, AddonManager +from scrapy.crawler import Crawler +from scrapy.interfaces import IAddon +from scrapy.settings import BaseSettings, Settings + +from . import addons +from . import addonmod + + +class AddonTest(unittest.TestCase): + + def setUp(self): + self.rawaddon = Addon() + class AddonWithAttributes(Addon): + name = 'Test' + version = '1.0' + self.testaddon = AddonWithAttributes() + + def test_interface(self): + # Raw Addon should fail exactly b/c name and version are not given + self.assertFalse(hasattr(self.rawaddon, 'name')) + self.assertFalse(hasattr(self.rawaddon, 'version')) + self.assertRaises(BrokenImplementation, verifyObject, IAddon, + self.rawaddon) + verifyObject(IAddon, self.testaddon) + + def test_export_component(self): + settings = BaseSettings({'ITEM_PIPELINES': {}}, 'default') + self.testaddon.component_type = None + self.testaddon.export_component({}, settings) + self.assertEqual(len(settings['ITEM_PIPELINES']), 0) + self.testaddon.component_type = 'ITEM_PIPELINES' + self.testaddon.component = 'test.component' + self.testaddon.export_component({}, settings) + six.assertCountEqual(self, settings['ITEM_PIPELINES'], + ['test.component']) + self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 0) + self.testaddon.component_order = 313 + self.testaddon.export_component({}, settings) + self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 313) + self.testaddon.component_type = 'DOWNLOAD_HANDLERS' + self.testaddon.component_key = 'http' + self.testaddon.export_component({}, settings) + self.assertEqual(settings['DOWNLOAD_HANDLERS']['http'], + 'test.component') + + def test_export_basics(self): + settings = BaseSettings() + self.testaddon.basic_settings = {'TESTKEY': 313, 'OTHERKEY': True} + self.testaddon.export_basics(settings) + self.assertEqual(settings['TESTKEY'], 313) + self.assertEqual(settings['OTHERKEY'], True) + self.assertEqual(settings.getpriority('TESTKEY'), 15) + + def test_export_config(self): + settings = BaseSettings() + self.testaddon.settings_prefix = None + self.testaddon.config_mapping = {'MAPPED_key': 'MAPPING_WORKED'} + self.testaddon.default_config = {'key': 55, 'defaultkey': 100} + self.testaddon.export_config({'key': 313, 'OTHERKEY': True, + 'mapped_KEY': 99}, settings) + self.assertEqual(settings['TEST_KEY'], 313) + self.assertEqual(settings['TEST_DEFAULTKEY'], 100) + self.assertEqual(settings['TEST_OTHERKEY'], True) + self.assertNotIn('MAPPED_key', settings) + self.assertNotIn('MAPPED_KEY', settings) + self.assertEqual(settings['MAPPING_WORKED'], 99) + self.assertEqual(settings.getpriority('TEST_KEY'), 15) + + self.testaddon.settings_prefix = 'PREF' + self.testaddon.export_config({'newkey': 99}, settings) + self.assertEqual(settings['PREF_NEWKEY'], 99) + + with mock.patch.object(settings, 'set') as mock_set: + self.testaddon.settings_prefix = False + self.testaddon.export_config({'thirdnewkey': 99}, settings) + self.assertEqual(mock_set.call_count, 0) + + def test_update_settings(self): + settings = BaseSettings() + settings.set('TEST_KEY1', 'default', priority='default') + settings.set('TEST_KEY2', 'project', priority='project') + self.testaddon.settings_prefix = None + self.testaddon.basic_settings = {'OTHERTEST_KEY': 'addon'} + addon_config = {'key1': 'addon', 'key2': 'addon', 'key3': 'addon'} + self.testaddon.update_settings(addon_config, settings) + self.assertEqual(settings['OTHERTEST_KEY'], 'addon') + self.assertEqual(settings['TEST_KEY1'], 'addon') + self.assertEqual(settings['TEST_KEY2'], 'project') + self.assertEqual(settings['TEST_KEY3'], 'addon') + + +class AddonManagerTest(unittest.TestCase): + + TESTCFGPATH = os.path.join(os.path.dirname(__file__), 'cfg.cfg') + ADDONMODPATH = os.path.join(os.path.dirname(__file__), 'addonmod.py') + + def setUp(self): + self.manager = AddonManager() + + def test_add(self): + manager = AddonManager() + manager.add(addonmod, {'key': 'val1'}) + manager.add('tests.test_addons.addons.GoodAddon') + six.assertCountEqual(self, manager, ['AddonModule', 'GoodAddon']) + self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) + six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) + self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + self.assertRaises(ValueError, manager.add, addonmod) + + def test_add_dont_instantiate_providing_classes(self): + class ProviderGoodAddon(addons.GoodAddon): + pass + zope.interface.directlyProvides(ProviderGoodAddon, IAddon) + manager = AddonManager() + manager.add(ProviderGoodAddon) + self.assertIs(manager['GoodAddon'], ProviderGoodAddon) + + def test_add_verifies(self): + brokenaddon = self.manager.get_addon( + 'tests.test_addons.addons.BrokenAddon') + self.assertRaises(zope.interface.exceptions.BrokenImplementation, + self.manager.add, + brokenaddon) + + def test_add_adds_missing_interface_declaration(self): + class GoodAddonWithoutDeclaration(object): + name = 'GoodAddonWithoutDeclaration' + version = '1.0' + self.manager.add(GoodAddonWithoutDeclaration) + + def test_remove(self): + manager = AddonManager() + def test_gets_removed(removearg): + manager.add(addonmod) + self.assertIn('AddonModule', manager) + manager.remove(removearg) + self.assertNotIn('AddonModule', manager) + test_gets_removed('AddonModule') + test_gets_removed(addonmod) + test_gets_removed('tests.test_addons.addonmod') + test_gets_removed(self.ADDONMODPATH) + self.assertRaises(KeyError, manager.remove, 'nonexistent') + self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) + + def test_get_addon(self): + goodaddon = self.manager.get_addon( + 'tests.test_addons.addons.GoodAddon') + self.assertIs(goodaddon, addons.GoodAddon) + + loaded_addonmod = self.manager.get_addon(self.ADDONMODPATH) + # XXX: The module is in fact imported twice under different names into + # sys.modules, is there a good assertion for module equality? + self.assertEqual(loaded_addonmod.name, addonmod.name) + + # Does not provide interface, but has _addon attribute pointing to + # GoodAddon instance + addonspath = os.path.join(os.path.dirname(__file__), 'addons.py') + goodaddon = self.manager.get_addon(addonspath) + # XXX: Again, the imported class and addons.GoodAddon are different + # since they are imported twice. How to use isInstance? + self.assertEqual(goodaddon.name, addons.GoodAddon.name) + + self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent') + + def test_get_addon_forward(self): + class SomeCls(object): + _addon = 'tests.test_addons.addons.GoodAddon' + self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon) + + def test_get_addon_nested(self): + x = addons.GoodAddon('outer') + x._addon = addons.GoodAddon('middle') + x._addon._addon = addons.GoodAddon('inner') + self.assertIs(self.manager.get_addon(x), x._addon._addon) + + @mock.patch.object(scrapy.addons, 'get_project_path', + return_value='tests.test_addons.project') + def test_get_addon_prefixes(self, get_project_path_mock): + # From python path + self.assertEqual(self.manager.get_addon('addonmod').FROM, + 'test_addons.addonmod') + + # From project 'addons' folder + self.assertEqual(self.manager.get_addon('addonmod2').FROM, + 'test_addons.project.addons.addonmod2') + # Assert prefix priority '' > 'project.addons' + self.assertEqual(self.manager.get_addon('addonmod').FROM, + 'test_addons.addonmod') + + # From scrapy's 'addons' + from . import scrapy_addons + with mock.patch.dict('sys.modules', {'scrapy.addons': scrapy_addons}): + self.assertEqual(self.manager.get_addon('addonmod3').FROM, + 'test_addons.scrapy_addons.addonmod3') + # Assert prefix priority 'project.addons' > 'scrapy.addons' + self.assertEqual(self.manager.get_addon('addonmod2').FROM, + 'test_addons.project.addons.addonmod2') + # Assert prefix priority '' > 'scrapy.addons.' + self.assertEqual(self.manager.get_addon('addonmod').FROM, + 'test_addons.addonmod') + + def test_load_dict_load_settings(self): + def _test_load_method(func, *args, **kwargs): + manager = AddonManager() + getattr(manager, func)(*args, **kwargs) + six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) + self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) + six.assertCountEqual(self, manager.configs['GoodAddon'], + ['key']) + self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') + # XXX: Check module equality, see above + self.assertEqual(manager['AddonModule'].name, addonmod.name) + self.assertIn('key', manager.configs['AddonModule']) + self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + + addonsdict = { + self.ADDONMODPATH: { + 'key': 'val1', + }, + 'tests.test_addons.addons.GoodAddon': {'key': 'val2'}, + } + _test_load_method('load_dict', addonsdict) + + settings = BaseSettings() + settings.set('INSTALLED_ADDONS', [ + self.ADDONMODPATH, + 'tests.test_addons.addons.GoodAddon', + ]) + settings.set('ADDONMODULE', {'key': 'val1'}) + settings.set('GOODADDON', {'key': 'val2'}) + _test_load_method('load_settings', settings) + + def test_load_cfg(self): + manager = AddonManager() + manager.load_cfg(self.TESTCFGPATH) + six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) + self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) + six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) + self.assertEqual(manager.configs['GoodAddon']['key'], 'val1') + # XXX: Check module equality, see above + self.assertEqual(manager['AddonModule'].name, addonmod.name) + six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) + self.assertEqual(manager.configs['AddonModule']['key'], 'val2') + + def test_enabled_disabled(self): + manager = AddonManager() + manager.add(addons.GoodAddon('FirstAddon')) + manager.add(addons.GoodAddon('SecondAddon')) + self.assertEqual(set(manager.enabled), + set(('FirstAddon', 'SecondAddon'))) + self.assertEqual(manager.disabled, []) + manager.disable('FirstAddon') + self.assertEqual(manager.enabled, ['SecondAddon']) + self.assertEqual(manager.disabled, ['FirstAddon']) + manager.enable('FirstAddon') + self.assertEqual(set(manager.enabled), + set(('FirstAddon', 'SecondAddon'))) + self.assertEqual(manager.disabled, []) + + def test_enable_before_add(self): + manager = AddonManager() + self.assertRaises(ValueError, manager.enable, 'FirstAddon') + manager.disable('FirstAddon') + manager.enable('FirstAddon') + manager.add(addons.GoodAddon('FirstAddon')) + self.assertIn('FirstAddon', manager.enabled) + + def test_disable_before_add(self): + manager = AddonManager() + manager.disable('FirstAddon') + manager.add(addons.GoodAddon('FirstAddon')) + self.assertEqual(manager.disabled, ['FirstAddon']) + + def test_callbacks(self): + first_addon = addons.GoodAddon('FirstAddon') + second_addon = addons.GoodAddon('SecondAddon') + + manager = AddonManager() + manager.add(first_addon, {'test': 'first'}) + manager.add(second_addon, {'test': 'second'}) + crawler = mock.create_autospec(Crawler) + settings = BaseSettings() + + with mock.patch.object(first_addon, 'update_addons') as ua_first, \ + mock.patch.object(second_addon, 'update_addons') as ua_second, \ + mock.patch.object(first_addon, 'update_settings') as us_first, \ + mock.patch.object(second_addon, 'update_settings') as us_second, \ + mock.patch.object(first_addon, 'check_configuration') as cc_first, \ + mock.patch.object(second_addon, 'check_configuration') as cc_second: + manager.update_addons() + ua_first.assert_called_once_with(manager.configs['FirstAddon'], + manager) + ua_second.assert_called_once_with(manager.configs['SecondAddon'], + manager) + manager.update_settings(settings) + us_first.assert_called_once_with(manager.configs['FirstAddon'], + settings) + us_second.assert_called_once_with(manager.configs['SecondAddon'], + settings) + manager.check_configuration(crawler) + cc_first.assert_called_once_with(manager.configs['FirstAddon'], + crawler) + cc_second.assert_called_once_with(manager.configs['SecondAddon'], + crawler) + self.assertEqual(ua_first.call_count, 1) + self.assertEqual(ua_second.call_count, 1) + self.assertEqual(us_first.call_count, 1) + self.assertEqual(us_second.call_count, 1) + + us_first.reset_mock() + us_second.reset_mock() + manager.disable('FirstAddon') + manager.update_settings(settings) + self.assertEqual(us_first.call_count, 0) + manager.enable('FirstAddon') + manager.update_settings(settings) + self.assertEqual(us_first.call_count, 1) + self.assertEqual(us_second.call_count, 2) + + def test_update_addons_last_minute_add(self): + class AddedAddon(addons.GoodAddon): + name = 'AddedAddon' + + class FirstAddon(addons.GoodAddon): + name = 'FirstAddon' + def update_addons(self, config, addons): + addons.add(AddedAddon()) + + manager = AddonManager() + first_addon = FirstAddon() + with mock.patch.object(first_addon, 'update_addons', + wraps=first_addon.update_addons) as ua_first, \ + mock.patch.object(AddedAddon, 'update_addons') as ua_added: + manager.add(first_addon, {'non-empty': 'dict'}) + manager.update_addons() + six.assertCountEqual(self, manager, ['FirstAddon', 'AddedAddon']) + ua_first.assert_called_once_with(manager.configs['FirstAddon'], + manager) + ua_added.assert_called_once_with(manager.configs['AddedAddon'], + manager) + + def test_check_dependency_clashes_attributes(self): + provides = addons.GoodAddon("ProvidesAddon") + provides.provides = ('test', ) + provides2 = addons.GoodAddon("ProvidesAddon2") + provides2.provides = ('test', ) + requires = addons.GoodAddon("RequiresAddon") + requires.requires = ('test', ) + requires_name = addons.GoodAddon("RequiresNameAddon") + requires_name.requires = ('ProvidesAddon', ) + requires_newer = addons.GoodAddon("RequiresNewerAddon") + requires_newer.requires = ('test>=2.0', ) + modifies = addons.GoodAddon("ModifiesAddon") + modifies.modifies = ('test', ) + + def check_with(*addons): + manager = AddonManager() + for a in addons: + manager.add(a) + return manager.check_dependency_clashes() + + self.assertRaises(ImportError, check_with, requires) + self.assertRaises(ImportError, check_with, modifies) + self.assertRaises(ImportError, check_with, provides, provides2) + self.assertRaises(VersionConflict, check_with, provides, requires_newer) + with warnings.catch_warnings(record=True) as w: + check_with(provides, modifies) + check_with(provides) + check_with(provides, requires) + check_with(provides, requires_name) + self.assertEqual(len(w), 0) + check_with(requires, provides, modifies) + self.assertEqual(len(w), 1) diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py new file mode 100644 index 000000000..8ecf4b81d --- /dev/null +++ b/tests/test_addons/addonmod.py @@ -0,0 +1,16 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = "test_addons.addonmod" + +name = "AddonModule" +version = "1.0" + +def update_settings(config, settings): + pass + +def check_configuration(config, crawler): + pass diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py new file mode 100644 index 000000000..f3442b192 --- /dev/null +++ b/tests/test_addons/addons.py @@ -0,0 +1,40 @@ +import zope.interface + +from scrapy.addons import Addon +from scrapy.interfaces import IAddon + + +class Addon(object): + FROM = 'test_addons.addons' + + +@zope.interface.declarations.implementer(IAddon) +class GoodAddon(object): + + name = 'GoodAddon' + version = '1.0' + + def __init__(self, name=None, version=None): + if name is not None: + self.name = name + if version is not None: + self.version = version + + def update_addons(self, config, addons): + pass + + def update_settings(self, config, settings): + pass + + def check_configuration(self, config, crawler): + pass + + +@zope.interface.declarations.implementer(IAddon) +class BrokenAddon(object): + + name = 'BrokenAddon' + # No version + + +_addon = GoodAddon() diff --git a/tests/test_addons/cfg.cfg b/tests/test_addons/cfg.cfg new file mode 100644 index 000000000..98c4f0f25 --- /dev/null +++ b/tests/test_addons/cfg.cfg @@ -0,0 +1,5 @@ +[addon:tests.test_addons.addons.GoodAddon] +key = val1 + +[addon:tests/test_addons/addonmod.py] +key = val2 diff --git a/tests/test_addons/project/__init__.py b/tests/test_addons/project/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_addons/project/addons/__init__.py b/tests/test_addons/project/addons/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_addons/project/addons/addonmod.py b/tests/test_addons/project/addons/addonmod.py new file mode 100644 index 000000000..66ca644f8 --- /dev/null +++ b/tests/test_addons/project/addons/addonmod.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.project.addons.addonmod' diff --git a/tests/test_addons/project/addons/addonmod2.py b/tests/test_addons/project/addons/addonmod2.py new file mode 100644 index 000000000..0dbdd70ff --- /dev/null +++ b/tests/test_addons/project/addons/addonmod2.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.project.addons.addonmod2' diff --git a/tests/test_addons/scrapy_addons/__init__.py b/tests/test_addons/scrapy_addons/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_addons/scrapy_addons/addonmod.py b/tests/test_addons/scrapy_addons/addonmod.py new file mode 100644 index 000000000..fa479aa68 --- /dev/null +++ b/tests/test_addons/scrapy_addons/addonmod.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.scrapy_addons.addonmod' diff --git a/tests/test_addons/scrapy_addons/addonmod2.py b/tests/test_addons/scrapy_addons/addonmod2.py new file mode 100644 index 000000000..da053af4a --- /dev/null +++ b/tests/test_addons/scrapy_addons/addonmod2.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.scrapy_addons.addonmod2' diff --git a/tests/test_addons/scrapy_addons/addonmod3.py b/tests/test_addons/scrapy_addons/addonmod3.py new file mode 100644 index 000000000..c64521478 --- /dev/null +++ b/tests/test_addons/scrapy_addons/addonmod3.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.scrapy_addons.addonmod3' diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 06af3c009..f33562b7d 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -3,7 +3,8 @@ import os import unittest from scrapy.item import Item, Field -from scrapy.utils.misc import load_object, arg_to_iter, walk_modules +from scrapy.utils.misc import (load_object, load_module_or_object, arg_to_iter, + walk_modules, get_module_from_filepath) __doctests__ = ['scrapy.utils.misc'] @@ -17,6 +18,15 @@ class UtilsMiscTestCase(unittest.TestCase): self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') + def test_load_module_or_object(self): + testmod = load_module_or_object(__name__ + '.testmod') + self.assertTrue(hasattr(testmod, 'TESTVAR')) + testmod = load_module_or_object( + os.path.join(os.path.dirname(__file__), 'testmod.py')) + self.assertTrue(hasattr(testmod, 'TESTVAR')) + obj = load_object('scrapy.utils.misc.load_object') + self.assertIs(obj, load_object) + def test_walk_modules(self): mods = walk_modules('tests.test_utils_misc.test_walk_modules') expected = [ @@ -57,6 +67,20 @@ class UtilsMiscTestCase(unittest.TestCase): finally: sys.path.remove(egg) + def test_get_module_from_filepath(self): + testmodpath = os.path.join(os.path.dirname(__file__), 'testmod.py') + testmod = get_module_from_filepath(testmodpath) + self.assertTrue(hasattr(testmod, 'TESTVAR')) + + testpkgpath = os.path.join(os.path.dirname(__file__), 'testpkg') + testpkg = get_module_from_filepath(testpkgpath) + self.assertTrue(hasattr(testpkg, 'TESTVAR2')) + # Check submodule access + import testpkg.submod + self.assertTrue(hasattr(testpkg.submod, 'TESTVAR3')) + self.assertIs(testpkg.submod.TESTVAR3, + load_object(testpkg.__name__ + ".submod.TESTVAR3")) + def test_arg_to_iter(self): class TestItem(Item): diff --git a/tests/test_utils_misc/testmod.py b/tests/test_utils_misc/testmod.py new file mode 100644 index 000000000..eb540335f --- /dev/null +++ b/tests/test_utils_misc/testmod.py @@ -0,0 +1 @@ +TESTVAR = True diff --git a/tests/test_utils_misc/testpkg/__init__.py b/tests/test_utils_misc/testpkg/__init__.py new file mode 100644 index 000000000..12cc2f6d9 --- /dev/null +++ b/tests/test_utils_misc/testpkg/__init__.py @@ -0,0 +1 @@ +TESTVAR2 = True diff --git a/tests/test_utils_misc/testpkg/submod.py b/tests/test_utils_misc/testpkg/submod.py new file mode 100644 index 000000000..8a07e3592 --- /dev/null +++ b/tests/test_utils_misc/testpkg/submod.py @@ -0,0 +1 @@ +TESTVAR3 = True diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py new file mode 100644 index 000000000..cea4d9950 --- /dev/null +++ b/tests/test_utils_project.py @@ -0,0 +1,27 @@ +import os +from tests import mock +import unittest + +from scrapy.exceptions import NotConfigured +from scrapy.utils.project import get_project_path, inside_project + + +class UtilsProjectTestCase(unittest.TestCase): + + @mock.patch('scrapy.utils.project.inside_project', return_value=True) + def test_get_project_path(self, mock_ip): + def _test(settingsmod, expected): + with mock.patch.dict('os.environ', + {'SCRAPY_SETTINGS_MODULE': settingsmod}): + self.assertEqual(get_project_path(), expected) + _test('project.settings', 'project') + _test('project.othername', 'project') + _test('nested.project.settings', 'nested.project') + + with mock.patch.dict('os.environ', {}, clear=True): + self.assertRaises(NotConfigured, get_project_path) + + mock_ip.return_value = False + with mock.patch.dict('os.environ', + {'SCRAPY_SETTINGS_MODULE': 'some.settings'}): + self.assertRaises(NotConfigured, get_project_path) From 07455b1883cacab141572df4310315bb53e65ec9 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 19 Aug 2015 16:17:09 +0200 Subject: [PATCH 0015/2083] Integrate add-ons into start-up process --- scrapy/cmdline.py | 6 +++++- scrapy/crawler.py | 19 ++++++++++++++----- scrapy/utils/test.py | 4 ++-- tests/test_crawl.py | 17 +++++++++++++++++ tests/test_crawler.py | 24 ++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 8 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 35050c13d..b403df570 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,6 +6,7 @@ import inspect import pkg_resources import scrapy +from scrapy.addons import AddonManager from scrapy.crawler import CrawlerProcess from scrapy.xlib import lsprofcalltree from scrapy.commands import ScrapyCommand @@ -118,6 +119,9 @@ def execute(argv=None, settings=None): conf.settings = settings # ------------------------------------------------------------------ + addons = AddonManager() + addons.load_cfg() + inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) @@ -139,7 +143,7 @@ def execute(argv=None, settings=None): opts, args = parser.parse_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) - cmd.crawler_process = CrawlerProcess(settings) + cmd.crawler_process = CrawlerProcess(settings, addons) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index bdcfa9d0c..8107a50aa 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -6,6 +6,7 @@ import warnings from twisted.internet import reactor, defer from zope.interface.verify import verifyClass, DoesNotImplement +from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.interfaces import ISpiderLoader @@ -23,7 +24,7 @@ logger = logging.getLogger(__name__) class Crawler(object): - def __init__(self, spidercls, settings=None): + def __init__(self, spidercls, settings=None, addons=None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -31,6 +32,12 @@ class Crawler(object): self.settings = settings.copy() self.spidercls.update_settings(self.settings) + self.addons = addons if addons is not None else AddonManager() + self.addons.load_settings(self.settings) + self.addons.update_addons() + self.addons.check_dependency_clashes() + self.addons.update_settings(self.settings) + self.signals = SignalManager(self) self.stats = load_object(self.settings['STATS_CLASS'])(self) @@ -69,6 +76,7 @@ class Crawler(object): try: self.spider = self._create_spider(*args, **kwargs) self.engine = self._create_engine() + self.addons.check_configuration(self) start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) yield defer.maybeDeferred(self.engine.start) @@ -111,10 +119,11 @@ class CrawlerRunner(object): ":meth:`crawl` and managed by this class." ) - def __init__(self, settings=None): + def __init__(self, settings=None, addons=None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings + self.addons = addons self.spider_loader = _get_spider_loader(settings) self._crawlers = set() self._active = set() @@ -181,7 +190,7 @@ class CrawlerRunner(object): def _create_crawler(self, spidercls): if isinstance(spidercls, six.string_types): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings) + return Crawler(spidercls, self.settings, self.addons) def stop(self): """ @@ -223,8 +232,8 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None): - super(CrawlerProcess, self).__init__(settings) + def __init__(self, settings=None, addons=None): + super(CrawlerProcess, self).__init__(settings, addons) install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings) log_scrapy_info(self.settings) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 51edfd353..0e11ec7d1 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -20,7 +20,7 @@ def assert_aws_environ(): if 'AWS_ACCESS_KEY_ID' not in os.environ: raise SkipTest("AWS keys not found") -def get_crawler(spidercls=None, settings_dict=None): +def get_crawler(spidercls=None, settings_dict=None, addons=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. @@ -28,7 +28,7 @@ def get_crawler(spidercls=None, settings_dict=None): from scrapy.crawler import CrawlerRunner from scrapy.spiders import Spider - runner = CrawlerRunner(settings_dict) + runner = CrawlerRunner(settings_dict, addons) return runner.create_crawler(spidercls or Spider) def get_pythonpath(): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 814eb30d2..7358009e9 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -6,6 +6,7 @@ from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase +from scrapy.addons import Addon, AddonManager from scrapy.http import Request from scrapy.crawler import CrawlerRunner from tests import mock @@ -266,3 +267,19 @@ with multiples lines self._assert_retried(log) self.assertIn("Got response 200", str(log)) + + @defer.inlineCallbacks + def test_abort_on_addon_failed_check(self): + class FailedCheckAddon(Addon): + name = 'FailedCheckAddon' + version = '1.0' + def check_configuration(self, config, crawler): + raise ValueError + addonmgr = AddonManager() + addonmgr.add(FailedCheckAddon()) + crawler = self.runner.create_crawler(SimpleSpider) + crawler.addons = addonmgr + # Doesn't work in 'precise' test environment: + #with self.assertRaises(ValueError): + # yield crawler.crawl() + yield self.assertFailure(crawler.crawl(), ValueError) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 53a1202e3..dfad11405 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -2,6 +2,7 @@ import warnings import unittest import scrapy +from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader @@ -51,6 +52,29 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertFalse(settings.frozen) self.assertTrue(crawler.settings.frozen) + def test_populate_addons_settings(self): + class TestAddon(Addon): + name = 'TestAddon' + version = '1.0' + addonconfig = {'TEST1': 'addon', 'TEST2': 'addon', 'TEST3': 'addon'} + class TestAddon2(Addon): + name = 'testAddon2' + version = '1.0' + addonconfig2 = {'TEST': 'addon2'} + + settings = Settings() + settings.set('TESTADDON_TEST1', 'project', priority='project') + settings.set('TESTADDON_TEST2', 'default', priority='default') + addonmgr = AddonManager() + addonmgr.add(TestAddon(), addonconfig) + addonmgr.add(TestAddon2(), addonconfig2) + crawler = Crawler(DefaultSpider, settings, addonmgr) + + self.assertEqual(crawler.settings['TESTADDON_TEST1'], 'project') + self.assertEqual(crawler.settings['TESTADDON_TEST2'], 'addon') + self.assertEqual(crawler.settings['TESTADDON_TEST3'], 'addon') + self.assertEqual(crawler.settings['TESTADDON2_TEST'], 'addon2') + def test_crawler_accepts_dict(self): crawler = Crawler(DefaultSpider, {'foo': 'bar'}) self.assertEqual(crawler.settings['foo'], 'bar') From d91647c38b9504ca514a7cfc4d6c4fdb54c3d853 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 17 Aug 2015 02:48:12 +0200 Subject: [PATCH 0016/2083] Add built-in add-ons --- scrapy/addons/__init__.py | 3 + scrapy/addons/builtins.py | 293 +++++++++++++++++++++++++++++ tests/test_addons/test_builtins.py | 42 +++++ 3 files changed, 338 insertions(+) create mode 100644 scrapy/addons/builtins.py create mode 100644 tests/test_addons/test_builtins.py diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 59e59e15f..420d46b68 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -495,3 +495,6 @@ class AddonManager(Mapping): """ for name in self: self._call_addon(name, 'check_configuration', crawler) + + +from scrapy.addons.builtins import * diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py new file mode 100644 index 000000000..ff7902afb --- /dev/null +++ b/scrapy/addons/builtins.py @@ -0,0 +1,293 @@ +import scrapy +from scrapy.addons import Addon + +__all__ = ['make_builtin_addon', + + 'depth', 'httperror', 'offsite', 'referer', 'urllength', + + 'ajaxcrawl', 'chunked', 'cookies', 'defaultheaders', + 'downloadtimeout', 'httpauth', 'httpcache', 'httpcompression', + 'httpproxy', 'metarefresh', 'redirect', 'retry', 'robotstxt', + 'stats', 'useragent', + + 'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport', + 'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump', + 'statsmailer', 'telnetconsole', + ] + + +def make_builtin_addon(addon_name, comp_type, comp, order=0, + addon_default_config=None, addon_version=None): + class ThisAddon(Addon): + name = addon_name + version = addon_version or scrapy.__version__ + component_type = comp_type + component = comp + component_order = order + default_config = addon_default_config or {} + + return ThisAddon + + +# XXX: Below are CLASSES that have lowercase names. This is in line with the +# original SEP-021 but violates PEP8. +# We might consider prepending all built-in addon names with scrapy_ or similar +# to reduce the chance of name clashes. + +# SPIDER MIDDLEWARES + +depth = make_builtin_addon( + 'depth', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.depth.DepthMiddleware', + 900, +) + +httperror = make_builtin_addon( + 'httperror', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', + 50, +) + +offsite = make_builtin_addon( + 'offsite', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', + 500, +) + +referer = make_builtin_addon( + 'referer', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.referer.RefererMiddleware', + 700, + {'enabled': True}, +) + +urllength = make_builtin_addon( + 'urllength', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', + 800, +) + + +# DOWNLOADER MIDDLEWARES + +ajaxcrawl = make_builtin_addon( + 'ajaxcrawl', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware', + 560, +) + +chunked = make_builtin_addon( + 'chunked', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware', + 830, +) + +cookies = make_builtin_addon( + 'cookies', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware', + 700, + {'enabled': True}, +) + +defaultheaders = make_builtin_addon( + 'defaultheaders', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', + 550, +) +# Assume every config entry is a header +def defaultheaders_export_config(self, config, settings): + conf = self.default_config or {} + conf.update(config) + settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon') +defaultheaders.export_config = defaultheaders_export_config + +downloadtimeout = make_builtin_addon( + 'downloadtimeout', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', + 350, +) +downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT', + 'download_timeout': 'DOWNLOAD_TIMEOUT'} + +httpauth = make_builtin_addon( + 'httpauth', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', + 300, +) + +httpcache = make_builtin_addon( + 'httpcache', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', + 900, + {'enabled': True}, +) + +httpcompression = make_builtin_addon( + 'httpcompression', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', + 590, + {'enabled': True}, +) +httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'} + +httpproxy = make_builtin_addon( + 'httpproxy', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', + 750, +) + +metarefresh = make_builtin_addon( + 'metarefresh', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', + 580, + {'enabled': True}, +) +metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'} + +redirect = make_builtin_addon( + 'redirect', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', + 600, + {'enabled': True}, +) + +retry = make_builtin_addon( + 'retry', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.retry.RetryMiddleware', + 500, + {'enabled': True}, +) + +robotstxt = make_builtin_addon( + 'robotstxt', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', + 100, + {'obey': True}, +) + +stats = make_builtin_addon( + 'stats', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.stats.DownloaderStats', + 850, +) + +useragent = make_builtin_addon( + 'useragent', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', + 400, +) +useragent.config_mapping = {'user_agent': 'USER_AGENT'} + + +# ITEM PIPELINES + + +# EXTENSIONS + +autothrottle = make_builtin_addon( + 'throttle', + 'EXTENSIONS', + 'scrapy.extensions.throttle.AutoThrottle', + 0, + {'enabled': True}, +) + +corestats = make_builtin_addon( + 'corestats', + 'EXTENSIONS' + 'scrapy.extensions.corestats.CoreStats', + 0, +) + +closespider = make_builtin_addon( + 'closespider', + 'EXTENSIONS' + 'scrapy.extensions.closespider.CloseSpider', + 0, +) + +debugger = make_builtin_addon( + 'debugger', + 'EXTENSIONS' + 'scrapy.extensions.debug.Debugger', + 0, +) + +feedexport = make_builtin_addon( + 'feedexport', + 'EXTENSIONS' + 'scrapy.extensions.feedexport.FeedExporter', + 0, +) +feedexport.settings_prefix = 'FEED' + +logstats = make_builtin_addon( + 'logstats', + 'EXTENSIONS' + 'scrapy.extensions.logstats.LogStats', + 0, +) + +memdebug = make_builtin_addon( + 'memdebug', + 'EXTENSIONS' + 'scrapy.extensions.memdebug.MemoryDebugger', + 0, + {'enabled': True}, +) + +memusage = make_builtin_addon( + 'memusage', + 'EXTENSIONS' + 'scrapy.extensions.memusage.MemoryUsage', + 0, + {'enabled': True}, +) + +spiderstate = make_builtin_addon( + 'spiderstate', + 'EXTENSIONS' + 'scrapy.extensions.spiderstate.SpiderState', + 0, +) + +stacktracedump = make_builtin_addon( + 'stacktracedump', + 'EXTENSIONS' + 'scrapy.extensions.debug.StackTraceDump', + 0, +) + +statsmailer = make_builtin_addon( + 'statsmailer', + 'EXTENSIONS' + 'scrapy.extensions.statsmailer.StatsMailer', + 0, +) + +telnetconsole = make_builtin_addon( + 'telnetconsole', + 'EXTENSIONS' + 'scrapy.telnet.TelnetConsole', + 0, +) diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py new file mode 100644 index 000000000..607c911fb --- /dev/null +++ b/tests/test_addons/test_builtins.py @@ -0,0 +1,42 @@ +import unittest + +import scrapy +import scrapy.addons +from scrapy.addons.builtins import make_builtin_addon +from scrapy.settings import Settings + + +class BuiltinAddonsTest(unittest.TestCase): + + def test_make_builtin_addon(self): + httpcache = make_builtin_addon( + 'httpcache', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', + 900, + {'enabled': True}, + ) + self.assertEqual(httpcache.name, 'httpcache') + self.assertEqual(httpcache.component_type, 'DOWNLOADER_MIDDLEWARES') + self.assertEqual(httpcache.component, 'scrapy.downloadermiddlewares.' + 'httpcache.HttpCacheMiddleware') + self.assertEqual(httpcache.component_order, 900) + self.assertEqual(httpcache.default_config, {'enabled': True}) + self.assertEqual(httpcache.version, scrapy.__version__) + httpcache = make_builtin_addon( + 'httpcache', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', + 900, + {'enabled': True}, + '99.9', + ) + self.assertEqual(httpcache.version, '99.9') + + def test_defaultheaders_export_config(self): + settings = Settings() + dh = scrapy.addons.defaultheaders() + dh.export_config({'X-Test-Header': 'val'}, settings) + self.assertIn('X-Test-Header', settings['DEFAULT_REQUEST_HEADERS']) + self.assertEqual(settings['DEFAULT_REQUEST_HEADERS']['X-Test-Header'], + 'val') From 2946b674144833254f3ec6dd6aece0156f71a21b Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Tue, 28 Jul 2015 12:58:05 +0200 Subject: [PATCH 0017/2083] Document add-ons --- docs/index.rst | 4 + docs/topics/addons.rst | 387 ++++++++++++++++++++++++++++++++++++++ scrapy/addons/__init__.py | 4 +- 3 files changed, 394 insertions(+), 1 deletion(-) create mode 100644 docs/topics/addons.rst diff --git a/docs/index.rst b/docs/index.rst index 0d21f5d40..3e8a220e9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -206,6 +206,7 @@ Extending Scrapy :hidden: topics/architecture + topics/addons topics/downloader-middleware topics/spider-middleware topics/extensions @@ -217,6 +218,9 @@ Extending Scrapy :doc:`topics/architecture` Understand the Scrapy architecture. +:doc:`topics/addons` + Enable and configure built-in and third-party extensions. + :doc:`topics/downloader-middleware` Customize how pages get requested and downloaded. diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst new file mode 100644 index 000000000..39ef286eb --- /dev/null +++ b/docs/topics/addons.rst @@ -0,0 +1,387 @@ +.. _topics-addons: + +======= +Add-ons +======= + +Scrapy's add-on system is a framework which unifies managing and configuring +components that extend Scrapy's core functionality, such as middlewares, +extensions, or pipelines. It provides users with a plug-and-play experience in +Scrapy extension management, and grants extensive configuration control to +developers. + + +Activating and configuring add-ons +================================== + +Add-ons and their configuration live in Scrapy's +:class:`~scrapy.addons.AddonManager`. During Scrapy's start-up process, and +only then, the add-on manager will read a list of enabled add-ons and their +configurations from your settings. There are two places where you can provide +the paths to add-ons you want to enable: + +* the ``INSTALLED_ADDONS`` setting, and +* the ``scrapy.cfg`` file. + +As Scrapy settings can be modified from many places, e.g. in a project's +``settings.py``, in a Spider's ``custom_settings`` attribute, or from the +command line, using the ``INSTALLED_ADDONS`` setting is the preferred way to +manage add-ons. + +The ``INSTALLED_ADDONS`` setting a tuple in which every item is a path to an +add-on. The path can be both a Python or a file path. While more precise, it is +not necessary to specify the full add-on Python path if it is either built into +Scrapy or lives in your project's ``addons`` submodule. + +The configuration of an add-on, if necessary at all, is stored as a dictionary +setting whose name is the uppercase add-on name. + +This is an example where an internal add-on and two third-party add-ons (in this +case with one requiring no configuration) are enabled/configured in a project's +``settings.py``:: + + INSTALLED_ADDONS = ( + 'httpcache', + 'path.to.some.addon', + 'path/to/other/addon.py', + ) + + HTTPCACHE = { + 'expiration_secs': 60, + 'ignore_http_codes': [404, 405], + } + + SOMEADDON = { + 'some_config': True, + } + +It is also possible to manage add-ons from ``scrapy.cfg``. While the syntax is +a little friendlier, be aware that this file, and therefore the configuration in +it, is not bound to a particular Scrapy project. While this should not pose a +problem when you use the project on your development machine only, a common +stumbling block is that ``scrapy.cfg`` is not deployed via ``scrapyd-deploy``. + +In ``scrapy.cfg``, section names, prepended with ``addon:``, replace the +dictionary keys. I.e., the configuration from above would look like this: + +.. code-block:: cfg + + [addon:httpcache] + expiration_secs = 60 + ignore_http_codes = 404,405 + + [addon:path.to.some.addon] + some_config = true + + [addon:path/to/other/addon.py] + + +Enabling and configuring add-ons within Python code +--------------------------------------------------- + +The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings +and from ``scrapy.cfg`` *at the beginning* of Scrapy's start-up process. +Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is +populated, changing the ``INSTALLED_ADDONS`` setting or any of the add-on +configuration dictionary settings will have no effect. + +If you want to enable, disable, or configure add-ons in Python code, for example +when writing your own add-on, you will have to use the +:class:`~scrapy.addons.AddonManager`. You can access the add-on manager through +either ``crawler.addons`` or, if you are writing an add-on, through the +``addons`` argument of the :meth:`update_addons` callback. The add-on manager +provides many useful methods and attributes to facilitate interacting with the +add-ons framework, e.g.: + +* an :meth:`~scrapy.addons.AddonManager.add` method to load add-ons, +* the :attr:`~scrapy.addons.AddonManager.enabled` list of enabled add-ons, +* :meth:`~scrapy.addons.AddonManager.enable` and + :meth:`~scrapy.addons.AddonManager.disable` methods, +* the :attr:`~scrapy.addons.AddonManager.configs` dictionary which holds the + configuration of all add-ons + +In this example, we ensure that the ``httpcache`` add-on is loaded, and that +its ``expiration_secs`` configuration is set to ``60``:: + + # addons is an instance of AddonManager + if 'httpcache' not in addons: + addons.add('httpcache', {'expiration_secs': 60}) + else: + addons.configs['httpcache']['expiration_secs'] = 60 + + +Writing your own add-ons +======================== + +Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*. +The interface is enforced through ``zope.interface``. This leaves the choice of +Python object up the developer. Examples: + +* for a small pipeline, the add-on interface could be implemented in the same + class that also implements the ``open/close_spider`` and ``process_item`` + callbacks +* for larger add-ons, or for clearer structure, the interface could be provided + by a stand-alone module + +The absolute minimum interface consists of two attributes: + +.. attribute:: name + + string with add-on name + +.. attribute:: version + + version string (PEP-404, e.g. ``'1.0.1'``) + +Of course, stating just these two attributes will not get you very far. Add-ons +can provide three callback methods that are called at various stages before the +crawling process: + +.. method:: update_settings(config, settings) + + This method is called during the initialization of the + :class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks + (e.g. for external Python libraries) and update the + :class:`~scrapy.settings.Settings` object as wished, e.g. enable components + for this add-on or set required configuration of other extensions. + + :param config: Configuration of this add-on + :type config: ``dict`` + + :param settings: The settings object storing Scrapy/component configuration + :type settings: :class:`~scrapy.settings.Settings` + +.. method:: check_configuration(config, crawler) + + This method is called when the :class:`~scrapy.crawler.Crawler` has been + fully initialized, immediately before it starts crawling. You can perform + additional dependency and configuration checks here. + + :param config: Configuration of this add-on + :type config: ``dict`` + + :param crawler: Fully initialized Scrapy crawler + :type crawler: :class:`~scrapy.crawler.Crawler` + +.. method:: update_addons(config, addons) + + This method is called immediately before :meth:`update_settings`, and should + be used to enable and configure other *add-ons* only. + + When using this callback, be aware that there is no guarantee in which order + the :meth:`update_addons` callbacks of enabled add-ons will be called. + Add-ons that are added to the :class:`~scrapy.addons.AddonManager` during + this callback will also have their :meth:`update_addons` method called. + + :param config: Configuration of this add-on + :type config: ``dict`` + + :param addons: Add-on manager holding all loaded add-ons + :type addons: :class:`~scrapy.addons.AddonManager` + +Additionally, add-ons may (and should, where appropriate) provide one or more +attributes that can be used for limited automated detection of possible +dependency clashes: + +.. attribute:: requires + + list of built-in or custom components needed by this add-on, as strings. + +.. attribute:: modifies + + list of built-in or custom components whose functionality is affected or + replaced by this add-on (a custom HTTP cache should list ``httpcache`` here) + +.. attribute:: provides + + list of components provided by this add-on (e.g. ``mongodb`` for an + extension that provides generic read/write access to a MongoDB database) + +The entries in the :attr:`requires` and :attr:`modifies` attributes can be add-on +names or components from other add-ons' :attr:`provides` attribute. You can +specify :pep:`440`-style information about required versions. Examples:: + + requires = ['httpcache'] + requires = ['otheraddon >= 2.0', 'yetanotheraddon'] + +The Python object or module that is pointed to by an add-on path (e.g. given in +the ``INSTALLED_ADDONS`` setting, or given to +:meth:`~scrapy.addons.AddonManager.add`) does not necessarily have to be an +add-on. Instead, it can provide an ``_addon`` attribute. This attribute can be +either an add-on or another add-on path. + + +Add-on base class +================= + +Scrapy comes with a built-in base class for add-ons which provides some +convenience functionality: + +* basic settings can be exported via :meth:`~scrapy.addons.Addon.export_basics`, + configurable via :attr:`~scrapy.addons.Addon.basic_settings`. +* a single component (e.g. an item pipeline or a downloader middleware) can be + inserted into Scrapy's settings via + :meth:`~scrapy.addons.Addon.export_component`, configurable via + :attr:`~scrapy.addons.Addon.component_type`, + :attr:`~scrapy.addons.Addon.component_key`, + :attr:`~scrapy.addons.Addon.component`, and the ``order`` key in + :attr:`~scrapy.addons.Addon.default_config`. +* the add-on configuration can be exposed into Scrapy's settings via + :meth:`~scrapy.addons.Addon.export_config`, configurable via + :attr:`~scrapy.addons.Addon.default_config`, + :attr:`~scrapy.addons.Addon.config_mapping`, and + :attr:`~scrapy.addons.Addon.settings_prefix`. + +By default, the base add-on class will expose the add-on configuration into +Scrapy's settings namespace, in caps and with the add-on name prepended. It is +easy to write your own functionality while still being able to use the +convenience functions by overwriting +:meth:`~scrapy.addons.Addon.update_settings`. + +.. module:: scrapy.addons + +.. autoclass:: Addon + :members: + + +Add-on examples +=============== + +Set some basic configuration using the :class:`Addon` base class:: + + from scrapy.addons import Addon + + class MyAddon(Addon): + name = 'myaddon' + version = '1.0' + component = 'path.to.mypipeline' + component_type = 'ITEM_PIPELINES' + component_order = 200 + basic_settings = { + 'DNSCACHE_ENABLED': False, + } + +Check dependencies:: + + from scrapy.addons import Addon + + class MyAddon(Addon): + name = 'myaddon' + version = '1.0' + + def update_settings(self, config, settings): + try: + import boto + except ImportError: + raise RuntimeError("myaddon requires the boto library") + else: + self.export_config(config, settings) + +Enable a component that lives relative to the add-on (see +:ref:`topics-api-settings`):: + + from scrapy.addons import Addon + + class MyAddon(Addon): + name = 'myaddon' + version = '1.0' + component = __name__ + '.downloadermw.coolmw' + component_type = 'DOWNLOADER_MIDDLEWARES' + component_order = 900 + +Instantiate components ad hoc:: + + from path.to.my.pipelines import MySQLPipeline + + class MyAddon(object): + name = 'myaddon' + version = '1.0' + + def update_settings(self, config, settings): + mysqlpl = MySQLPipeline(password=config['password']) + settings.set( + 'ITEM_PIPELINES', + {mysqlpl: 200}, + priority='addon', + ) + +Provide add-on interface along component interface:: + + class MyPipeline(object): + name = 'mypipeline' + version = '1.0' + + def process_item(self, item, spider): + # Do some processing here + return item + + def update_settings(self, config, settings): + settings.set( + 'ITEM_PIPELINES', + {self: 200}, + priority='addon', + ) + +Enable another addon (see :ref:`topics-api-addonmanager`):: + + class MyAddon(object): + name = 'myaddon' + version = '1.0' + + def update_addons(self, config, addons): + if 'httpcache' not in addons.enabled: + addons.add('httpcache', {'expiration_secs': 60}) + +Check configuration of fully initialized crawler (see +:ref:`topics-api-crawler`):: + + class MyAddon(object): + name = 'myaddon' + version = '1.0' + + def update_settings(self, config, settings): + settings.set('DNSCACHE_ENABLED', False, priority='addon') + + def check_configuration(self, config, crawler): + if crawler.settings.getbool('DNSCACHE_ENABLED'): + # The spider, some other add-on, or the user messed with the + # DNS cache setting + raise ValueError("myaddon is incompatible with DNS cache") + +Provide add-on interface through a module: + +.. No idea why just using '::' doesn't work for this one +.. code-block:: python + + name = 'AddonModule' + version = '1.0' + + class MyPipeline(object): + # ... + + class MyDownloaderMiddleware(object): + # ... + + def update_settings(config, settings): + settings.set( + 'ITEM_PIPELINES', + {MyPipeline(): 200}, + priority='addon', + } + settings.set( + 'DOWNLOADER_MIDDLEWARES', + {MyDownloaderMiddleware(): 800}, + priority='addon', + } + +Forward to other add-ons depending on Python version:: + + # This could be a Python module, say project/pipelines/mypipeline.py, but + # could also be done inside a class, etc. + import six + + if six.PY3: + # We're running Python 3 + _addon = 'path.to.addon' + else: + _addon = 'path.to.other.addon' diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 420d46b68..15460143c 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -156,13 +156,15 @@ class AddonManager(Mapping): """This class facilitates loading and storing :ref:`topics-addons`. You can treat it like a read-only dictionary in which keys correspond to - add-on names and values correspond to the add-on objects:: + add-on names and values correspond to the add-on objects. Add-on + configurations are saved in the :attr:`config` dictionary attribute:: addons = AddonManager() # ... load some add-ons here print addons.enabled # prints names of all enabled add-ons print addons['TestAddon'].version # prints version of add-on with name # 'TestAddon' + print addons.configs['TestAddon'] # prints configuration of 'TestAddon' """ From 25498c3c210ea5aa1a45aed02618033d832cfc1c Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 24 Aug 2015 18:04:58 +0200 Subject: [PATCH 0018/2083] Remove unused imports in add-ons --- scrapy/addons/__init__.py | 3 --- tests/test_addons/__init__.py | 4 +--- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 15460143c..b1d6e14cb 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -1,7 +1,5 @@ from collections import defaultdict, Mapping -from importlib import import_module from inspect import isclass -import os import six import warnings @@ -11,7 +9,6 @@ from zope.interface.verify import verifyObject from scrapy.exceptions import NotConfigured from scrapy.interfaces import IAddon -from scrapy.settings import BaseSettings from scrapy.utils.conf import config_from_filepath, get_config from scrapy.utils.misc import load_module_or_object from scrapy.utils.project import get_project_path diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index c98f0ab65..84870ec52 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,7 +1,5 @@ import os.path import six -from six.moves.configparser import SafeConfigParser -import sys from tests import mock import unittest import warnings @@ -15,7 +13,7 @@ import scrapy.addons from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon -from scrapy.settings import BaseSettings, Settings +from scrapy.settings import BaseSettings from . import addons from . import addonmod From 4ac6a83072f6e507f7f952b9c466419bec43364c Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 26 Aug 2015 01:43:59 +0200 Subject: [PATCH 0019/2083] Fix class signatures in Extensions docs --- docs/topics/extensions.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 847353868..b29e1802f 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -183,7 +183,7 @@ Telnet console extension .. module:: scrapy.extensions.telnet :synopsis: Telnet console -.. class:: scrapy.extensions.telnet.TelnetConsole +.. class:: TelnetConsole Provides a telnet console for getting into a Python interpreter inside the currently running Scrapy process, which can be very useful for debugging. @@ -200,7 +200,7 @@ Memory usage extension .. module:: scrapy.extensions.memusage :synopsis: Memory usage extension -.. class:: scrapy.extensions.memusage.MemoryUsage +.. class:: MemoryUsage .. note:: This extension does not work in Windows. @@ -229,7 +229,7 @@ Memory debugger extension .. module:: scrapy.extensions.memdebug :synopsis: Memory debugger extension -.. class:: scrapy.extensions.memdebug.MemoryDebugger +.. class:: MemoryDebugger An extension for debugging memory usage. It collects information about: @@ -245,7 +245,7 @@ Close spider extension .. module:: scrapy.extensions.closespider :synopsis: Close spider extension -.. class:: scrapy.extensions.closespider.CloseSpider +.. class:: CloseSpider Closes a spider automatically when some conditions are met, using a specific closing reason for each condition. @@ -316,7 +316,7 @@ StatsMailer extension .. module:: scrapy.extensions.statsmailer :synopsis: StatsMailer extension -.. class:: scrapy.extensions.statsmailer.StatsMailer +.. class:: StatsMailer This simple extension can be used to send a notification e-mail every time a domain has finished scraping, including the Scrapy stats collected. The email @@ -332,7 +332,7 @@ Debugging extensions Stack trace dump extension ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. class:: scrapy.extensions.debug.StackTraceDump +.. class:: StackTraceDump Dumps information about the running process when a `SIGQUIT`_ or `SIGUSR2`_ signal is received. The information dumped is the following: @@ -361,7 +361,7 @@ There are at least two ways to send Scrapy the `SIGQUIT`_ signal: Debugger extension ~~~~~~~~~~~~~~~~~~ -.. class:: scrapy.extensions.debug.Debugger +.. class:: Debugger Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_ signal is received. After the debugger is exited, the Scrapy process continues From 18c7f3dbe2d225166b5dbd54a3872abfc91e77b0 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 21 Aug 2015 16:29:27 +0200 Subject: [PATCH 0020/2083] Document built-in add-ons --- docs/topics/addons.rst | 123 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 39ef286eb..4dab15a2a 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -385,3 +385,126 @@ Forward to other add-ons depending on Python version:: _addon = 'path.to.addon' else: _addon = 'path.to.other.addon' + + +Built-in add-on reference +========================= + +Scrapy comes with gateway add-ons that you can use to configure the built-in +middlewares and extensions. For example, to activate and configure the +:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`, instead +of placing this in your ``settings.py``:: + + HTTPCACHE_ENABLED = True + HTTPCACHE_EXPIRATION_SECS = 60 + HTTPCACHE_IGNORE_HTTP_CODES = [404] + +you can also use the add-on framework:: + + INSTALLED_ADDONS = ( + # ..., + 'httpcache', + ) + + HTTPCACHE = { + 'expiration_secs': 60, + 'ignore_http_codes': [404], + } + +Note that you *must* enable built-in addons by placing them in your +``INSTALLED_ADDONS`` setting before you can use them for configuring built-in +components. I.e., configuring the ``HTTPCACHE`` setting will have no effect +when ``httpcache`` is not listed in ``INSTALLED_ADDONS``. + +In general, the add-on names match the lowercase name of the component, with its +type suffix removed (i.e. the add-on configuring the +:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` is called +``httpcache``), and the configuration option names match the names of the +settings they map to, with the component prefix removed (i.e. +``expiration_secs`` maps to :setting:`HTTPCACHE_EXPIRATION_SECS`, as above). +The available add-ons are: + + ++--------------------------------------+--------------------------------------+ +| Add-on | Notes | ++======================================+======================================+ +| **Spider middlewares** | ++--------------------------------------+--------------------------------------+ +| depth (:class:`~scrapy.spidermi\ | | +| ddlewares.depth.DepthMiddleware`) | | ++--------------------------------------+--------------------------------------+ +| httperror (:class:`~scrapy.spid\ | | +| ermiddlewares.httperror.HttpErrorMi\ | | +| ddleware`) | | ++--------------------------------------+--------------------------------------+ +| offsite (:class:`~scrapy.spid\ | | +| ermiddlewares.offsite.OffsiteMiddle\ | | +| ware`) | | ++--------------------------------------+--------------------------------------+ +| referer (:class:`~scrapy.spid\ | | +| ermiddlewares.referer.RefererMiddle\ | | +| ware`) | | ++--------------------------------------+--------------------------------------+ +| urllength (:class:`~scrapy.spid\ | | +| ermiddlewares.urllength.UrlLengthMi\ | | +| ddleware`) | | ++--------------------------------------+--------------------------------------+ +| **Downloader middlewares** | ++--------------------------------------+--------------------------------------+ +| ajaxcrawl (:class:`~scrapy.download\ | | +| ermiddlewares.ajaxcrawl.AjaxCrawlMi\ | | +| ddleware`) | | ++--------------------------------------+--------------------------------------+ +| chunked (:class:`~scrapy.download\ | | +| ermiddlewares.chunked.ChunkedTrans\ | | +| ferMiddleware`) | | ++--------------------------------------+--------------------------------------+ +| cookies (:class:`~scrapy.download\ | | +| ermiddlewares.cookies.CookiesMiddle\ | | +| ware`) | | ++--------------------------------------+--------------------------------------+ +| defaultheaders (:class:`~scrapy.down\| Every configuration entry is treated | +| loadermiddlewares.defaultheaders.Def\| as a default header. | +| aultHeadersMiddleware`) | | ++--------------------------------------+--------------------------------------+ +| **Extensions** | ++--------------------------------------+--------------------------------------+ +| autothrottle | Installing sets | +| (:ref:`topics-autothrottle`) | :setting:`AUTOTHROTTLE_ENABLED` to | +| | ``True``. | ++--------------------------------------+--------------------------------------+ +| corestats (:class:`~scrapy.exten\ | | +| sions.corestats.CoreStats`) | | ++--------------------------------------+--------------------------------------+ +| closespider (:class:`~scrapy.exten\ | | +| sions.closespider.CloseSpider`) | | ++--------------------------------------+--------------------------------------+ +| debugger (:class:`~scrapy.exten\ | | +| sions.debug.Debugger`) | | ++--------------------------------------+--------------------------------------+ +| feedexport (:ref:`topics-feed-expor\ | | +| ts`) | | ++--------------------------------------+--------------------------------------+ +| logstats (:class:`~scrapy.exten\ | | +| sions.logstats.LogStats`) | | ++--------------------------------------+--------------------------------------+ +| memdebug (:class:`~scrapy.exten\ | Installing sets | +| sions.memdebug.MemoryDebugger`) | :setting:`MEMDEBUG_ENABLED` to | +| | ``True``. | ++--------------------------------------+--------------------------------------+ +| memusage (:class:`~scrapy.exten\ | Installing sets | +| sions.memusage.MemoryUsage`) | :setting:`MEMUSAGE_ENABLED` to | +| | ``True``. | ++--------------------------------------+--------------------------------------+ +| spiderstate (:class:`~scrapy.exten\ | | +| sions.spiderstate.SpiderState`) | | ++--------------------------------------+--------------------------------------+ +| stacktracedump (:class:`~scrapy.ext\ | | +| ensions.debug.StackTraceDump`) | | ++--------------------------------------+--------------------------------------+ +| statsmailer (:class:`~scrapy.exten\ | | +| sions.statsmailer.StatsMailer`) | | ++--------------------------------------+--------------------------------------+ +| telnetconsole (:ref:`topics-telnet\ | | +| console`) | | ++--------------------------------------+--------------------------------------+ From d18b6a61d7db295c5f4b1800402a064ed1f40059 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Thu, 29 Oct 2015 16:34:52 +0100 Subject: [PATCH 0021/2083] Add missing AddonManager tests --- tests/test_addons/__init__.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 84870ec52..ab06023e3 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -14,6 +14,7 @@ from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon from scrapy.settings import BaseSettings +from scrapy.utils.conf import config_from_filepath from . import addons from . import addonmod @@ -244,16 +245,22 @@ class AddonManagerTest(unittest.TestCase): _test_load_method('load_settings', settings) def test_load_cfg(self): + def _check_loaded_addons(manager): + six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) + self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) + six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) + self.assertEqual(manager.configs['GoodAddon']['key'], 'val1') + # XXX: Check module equality, see above + self.assertEqual(manager['AddonModule'].name, addonmod.name) + six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) + self.assertEqual(manager.configs['AddonModule']['key'], 'val2') manager = AddonManager() manager.load_cfg(self.TESTCFGPATH) - six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) - self.assertEqual(manager.configs['GoodAddon']['key'], 'val1') - # XXX: Check module equality, see above - self.assertEqual(manager['AddonModule'].name, addonmod.name) - six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val2') + _check_loaded_addons(manager) + manager = AddonManager() + preloaded_cfg = config_from_filepath(self.TESTCFGPATH) + manager.load_cfg(preloaded_cfg) + _check_loaded_addons(manager) def test_enabled_disabled(self): manager = AddonManager() @@ -330,6 +337,11 @@ class AddonManagerTest(unittest.TestCase): self.assertEqual(us_first.call_count, 1) self.assertEqual(us_second.call_count, 2) + # This will become relevant when we let spiders implement the add-on + # interface and should be replaced with a test where + # AddonManager.spidercls = None then. + manager._call_if_exists(None, 'irrelevant') + def test_update_addons_last_minute_add(self): class AddedAddon(addons.GoodAddon): name = 'AddedAddon' From 9f7fcf5582ed61787180df20a62a301cc09fce4a Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 22:18:10 +0100 Subject: [PATCH 0022/2083] Make update_classpath() util function return non-string objects --- scrapy/utils/deprecate.py | 3 +++ tests/test_utils_deprecate.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 0fc33e0c4..9293b1480 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -1,5 +1,6 @@ """Some helpers for deprecation messages""" +import six import warnings import inspect from scrapy.exceptions import ScrapyDeprecationWarning @@ -149,6 +150,8 @@ DEPRECATION_RULES = [ def update_classpath(path): """Update a deprecated path from an object with its new location""" + if not isinstance(path, six.string_types): + return path for prefix, replacement in DEPRECATION_RULES: if path.startswith(prefix): new_path = path.replace(prefix, replacement, 1) diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 3e7236fb1..7a35c424b 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -279,3 +279,7 @@ class UpdateClassPathTest(unittest.TestCase): output = update_classpath('scrapy.unmatched.Path') self.assertEqual(output, 'scrapy.unmatched.Path') self.assertEqual(len(w), 0) + + def test_returns_nonstring(self): + for notastring in [None, True, [1, 2, 3], object()]: + self.assertEqual(update_classpath(notastring), notastring) From f7ed239fcb47663d881e72ffab4386244b215ce4 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 22:27:01 +0100 Subject: [PATCH 0023/2083] Replace INSTALLED_ADDONS tuple setting with ADDONS dictionary setting --- scrapy/addons/__init__.py | 15 ++++++++------- scrapy/settings/default_settings.py | 4 ++-- tests/test_addons/__init__.py | 26 ++++++++++++++++++++++---- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index b1d6e14cb..a1a9a388a 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -1,4 +1,4 @@ -from collections import defaultdict, Mapping +from collections import defaultdict, Mapping, OrderedDict from inspect import isclass import six import warnings @@ -9,7 +9,8 @@ from zope.interface.verify import verifyObject from scrapy.exceptions import NotConfigured from scrapy.interfaces import IAddon -from scrapy.utils.conf import config_from_filepath, get_config +from scrapy.utils.conf import (build_component_list, config_from_filepath, + get_config) from scrapy.utils.misc import load_module_or_object from scrapy.utils.project import get_project_path @@ -166,7 +167,7 @@ class AddonManager(Mapping): """ def __init__(self): - self._addons = {} + self._addons = OrderedDict() self.configs = {} self._disable_on_add = [] @@ -310,15 +311,15 @@ class AddonManager(Mapping): """Load add-ons and configurations from settings object. This will invoke :meth:`get_addon` for every add-on path in the - ``INSTALLED_ADDONS`` setting. For each of these add-ons, the - configuration will be read from the dictionary setting whose name - matches the uppercase add-on name. + ``ADDONS`` setting. For each of these add-ons, the configuration will be + read from the dictionary setting whose name matches the uppercase add-on + name. :param settings: The :class:`~scrapy.settings.Settings` object from \ which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - paths = settings.getlist('INSTALLED_ADDONS') + paths = build_component_list(settings['ADDONS']) addons = [self.get_addon(path) for path in paths] configs = [settings.getdict(addon.name.upper()) for addon in addons] for a, c in zip(addons, configs): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a230750fb..6068a1393 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -18,6 +18,8 @@ import sys from importlib import import_module from os.path import join, abspath, dirname +ADDONS = {} + AJAXCRAWL_ENABLED = False AUTOTHROTTLE_ENABLED = False @@ -167,8 +169,6 @@ HTTPCACHE_DBM_MODULE = 'anydbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False -INSTALLED_ADDONS = () - ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index ab06023e3..aa8dfbb63 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,3 +1,5 @@ +from collections import OrderedDict +import itertools import os.path import six from tests import mock @@ -236,14 +238,30 @@ class AddonManagerTest(unittest.TestCase): _test_load_method('load_dict', addonsdict) settings = BaseSettings() - settings.set('INSTALLED_ADDONS', [ - self.ADDONMODPATH, - 'tests.test_addons.addons.GoodAddon', - ]) + settings.set('ADDONS', {self.ADDONMODPATH: 0, + 'tests.test_addons.addons.GoodAddon': 0}) settings.set('ADDONMODULE', {'key': 'val1'}) settings.set('GOODADDON', {'key': 'val2'}) _test_load_method('load_settings', settings) + def test_load_dict_load_settings_order(self): + def _test_load_method(expected_order, func, *args, **kwargs): + manager = AddonManager() + getattr(manager, func)(*args, **kwargs) + self.assertEqual(list(manager.keys()), expected_order) + + # Get three addons named 0, 1, 2 + addonlist = [addons.GoodAddon(str(x)) for x in range(3)] + # Test both methods for every possible mutation + for ordered_addons in itertools.permutations(addonlist): + expected_order = [a.name for a in ordered_addons] + addonsdict = OrderedDict((a, {}) for a in ordered_addons) + _test_load_method(expected_order, 'load_dict', addonsdict) + settings = BaseSettings({ + 'ADDONS': {a: i for i, a in enumerate(ordered_addons)} + }) + _test_load_method(expected_order, 'load_settings', settings) + def test_load_cfg(self): def _check_loaded_addons(manager): six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) From b10caf91a14dcf0d0d4fe89865a53e2542588772 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 23:14:42 +0100 Subject: [PATCH 0024/2083] Drop support for add-on configuration in scrapy.cfg --- scrapy/addons/__init__.py | 28 +--------------------------- scrapy/cmdline.py | 1 - tests/test_addons/__init__.py | 20 -------------------- tests/test_addons/cfg.cfg | 5 ----- 4 files changed, 1 insertion(+), 53 deletions(-) delete mode 100644 tests/test_addons/cfg.cfg diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index a1a9a388a..40a98676d 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -9,8 +9,7 @@ from zope.interface.verify import verifyObject from scrapy.exceptions import NotConfigured from scrapy.interfaces import IAddon -from scrapy.utils.conf import (build_component_list, config_from_filepath, - get_config) +from scrapy.utils.conf import build_component_list from scrapy.utils.misc import load_module_or_object from scrapy.utils.project import get_project_path @@ -325,31 +324,6 @@ class AddonManager(Mapping): for a, c in zip(addons, configs): self.add(a, c) - def load_cfg(self, cfg=None): - """Load add-ons and configurations from given ``ConfigParser`` object or - config file path. - - Each add-on should have its own section, where the section has a name in - the form ``addon:my_addon_path``. The add-on object is searched for via - the :meth:`get_addon` method, ``my_addon_path`` can be either a Python - or a file path. - - If ``cfg`` is ``None``, ``scrapy.cfg`` will be used. - - :param cfg: ``ConfigParser`` object or config file path from which to \ - read add-on configuration - :type cfg: ``ConfigParser`` or ``str`` - """ - if cfg is None: - cfg = get_config() - elif isinstance(cfg, six.string_types): - cfg = config_from_filepath(cfg) - for secname in cfg.sections(): - if secname.startswith("addon:"): - addonkey = secname.split("addon:", 1)[1] - addoncfg = dict(cfg.items(secname)) - self.add(addonkey, addoncfg) - def check_dependency_clashes(self): """Check for incompatibilities in add-on dependencies. diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index b403df570..b7c349793 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -120,7 +120,6 @@ def execute(argv=None, settings=None): # ------------------------------------------------------------------ addons = AddonManager() - addons.load_cfg() inproject = inside_project() cmds = _get_commands_dict(settings, inproject) diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index aa8dfbb63..f10819685 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -16,7 +16,6 @@ from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon from scrapy.settings import BaseSettings -from scrapy.utils.conf import config_from_filepath from . import addons from . import addonmod @@ -107,7 +106,6 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - TESTCFGPATH = os.path.join(os.path.dirname(__file__), 'cfg.cfg') ADDONMODPATH = os.path.join(os.path.dirname(__file__), 'addonmod.py') def setUp(self): @@ -262,24 +260,6 @@ class AddonManagerTest(unittest.TestCase): }) _test_load_method(expected_order, 'load_settings', settings) - def test_load_cfg(self): - def _check_loaded_addons(manager): - six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) - self.assertEqual(manager.configs['GoodAddon']['key'], 'val1') - # XXX: Check module equality, see above - self.assertEqual(manager['AddonModule'].name, addonmod.name) - six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val2') - manager = AddonManager() - manager.load_cfg(self.TESTCFGPATH) - _check_loaded_addons(manager) - manager = AddonManager() - preloaded_cfg = config_from_filepath(self.TESTCFGPATH) - manager.load_cfg(preloaded_cfg) - _check_loaded_addons(manager) - def test_enabled_disabled(self): manager = AddonManager() manager.add(addons.GoodAddon('FirstAddon')) diff --git a/tests/test_addons/cfg.cfg b/tests/test_addons/cfg.cfg deleted file mode 100644 index 98c4f0f25..000000000 --- a/tests/test_addons/cfg.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[addon:tests.test_addons.addons.GoodAddon] -key = val1 - -[addon:tests/test_addons/addonmod.py] -key = val2 From 8e5d067af1e50a32f8930294199798535357edf2 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Sat, 7 Nov 2015 19:56:47 +0100 Subject: [PATCH 0025/2083] Drop BaseSettings.get_addon() prefix magic --- scrapy/addons/__init__.py | 27 +++---------------- tests/test_addons/__init__.py | 27 ------------------- tests/test_addons/project/__init__.py | 0 tests/test_addons/project/addons/__init__.py | 0 tests/test_addons/project/addons/addonmod.py | 7 ----- tests/test_addons/project/addons/addonmod2.py | 7 ----- tests/test_addons/scrapy_addons/__init__.py | 0 tests/test_addons/scrapy_addons/addonmod.py | 7 ----- tests/test_addons/scrapy_addons/addonmod2.py | 7 ----- tests/test_addons/scrapy_addons/addonmod3.py | 7 ----- 10 files changed, 4 insertions(+), 85 deletions(-) delete mode 100644 tests/test_addons/project/__init__.py delete mode 100644 tests/test_addons/project/addons/__init__.py delete mode 100644 tests/test_addons/project/addons/addonmod.py delete mode 100644 tests/test_addons/project/addons/addonmod2.py delete mode 100644 tests/test_addons/scrapy_addons/__init__.py delete mode 100644 tests/test_addons/scrapy_addons/addonmod.py delete mode 100644 tests/test_addons/scrapy_addons/addonmod2.py delete mode 100644 tests/test_addons/scrapy_addons/addonmod3.py diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 40a98676d..ddd18c7fb 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -7,11 +7,9 @@ from pkg_resources import WorkingSet, Distribution, Requirement import zope.interface from zope.interface.verify import verifyObject -from scrapy.exceptions import NotConfigured from scrapy.interfaces import IAddon from scrapy.utils.conf import build_component_list from scrapy.utils.misc import load_module_or_object -from scrapy.utils.project import get_project_path @zope.interface.implementer(IAddon) @@ -244,14 +242,8 @@ class AddonManager(Mapping): """Get an add-on object by its Python or file path. ``path`` is assumed to be either a Python or a file path of a Scrapy - add-on. If no object is found at ``path``, it is tried again first with - ``projectname.addons`` prepended (pointing to the current project's - ``addons`` folder), then with ``scrapy.addons`` prepended (poiting to - Scrapy's built-in add-ons). These convenience shortcuts will only work - with Python paths, not file paths. - - If the object or module pointed to by ``path`` has an attribute named - ``_addon`` that attribute will be assumed to be the add-on. + add-on. If the object or module pointed to by ``path`` has an attribute + named ``_addon`` that attribute will be assumed to be the add-on. :meth:`get_addon` will keep following ``_addon`` attributes until it finds an object that does not have an attribute named ``_addon``. @@ -259,20 +251,9 @@ class AddonManager(Mapping): :type path: ``str`` """ if isinstance(path, six.string_types): - prefixes = ['', 'scrapy.addons.'] try: - prefixes.insert(1, get_project_path() + '.addons.') - except NotConfigured: - warnings.warn("Unable to locate project Python path") - for prefix in prefixes: - fullpath = prefix + path - try: - obj = load_module_or_object(fullpath) - except NameError: - pass - else: - break - else: + obj = load_module_or_object(path) + except NameError: raise NameError("Could not find add-on '%s'" % path) else: obj = path diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index f10819685..4f1074221 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -11,7 +11,6 @@ import zope.interface from zope.interface.verify import verifyObject from zope.interface.exceptions import BrokenImplementation -import scrapy.addons from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon @@ -187,32 +186,6 @@ class AddonManagerTest(unittest.TestCase): x._addon._addon = addons.GoodAddon('inner') self.assertIs(self.manager.get_addon(x), x._addon._addon) - @mock.patch.object(scrapy.addons, 'get_project_path', - return_value='tests.test_addons.project') - def test_get_addon_prefixes(self, get_project_path_mock): - # From python path - self.assertEqual(self.manager.get_addon('addonmod').FROM, - 'test_addons.addonmod') - - # From project 'addons' folder - self.assertEqual(self.manager.get_addon('addonmod2').FROM, - 'test_addons.project.addons.addonmod2') - # Assert prefix priority '' > 'project.addons' - self.assertEqual(self.manager.get_addon('addonmod').FROM, - 'test_addons.addonmod') - - # From scrapy's 'addons' - from . import scrapy_addons - with mock.patch.dict('sys.modules', {'scrapy.addons': scrapy_addons}): - self.assertEqual(self.manager.get_addon('addonmod3').FROM, - 'test_addons.scrapy_addons.addonmod3') - # Assert prefix priority 'project.addons' > 'scrapy.addons' - self.assertEqual(self.manager.get_addon('addonmod2').FROM, - 'test_addons.project.addons.addonmod2') - # Assert prefix priority '' > 'scrapy.addons.' - self.assertEqual(self.manager.get_addon('addonmod').FROM, - 'test_addons.addonmod') - def test_load_dict_load_settings(self): def _test_load_method(func, *args, **kwargs): manager = AddonManager() diff --git a/tests/test_addons/project/__init__.py b/tests/test_addons/project/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_addons/project/addons/__init__.py b/tests/test_addons/project/addons/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_addons/project/addons/addonmod.py b/tests/test_addons/project/addons/addonmod.py deleted file mode 100644 index 66ca644f8..000000000 --- a/tests/test_addons/project/addons/addonmod.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.project.addons.addonmod' diff --git a/tests/test_addons/project/addons/addonmod2.py b/tests/test_addons/project/addons/addonmod2.py deleted file mode 100644 index 0dbdd70ff..000000000 --- a/tests/test_addons/project/addons/addonmod2.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.project.addons.addonmod2' diff --git a/tests/test_addons/scrapy_addons/__init__.py b/tests/test_addons/scrapy_addons/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_addons/scrapy_addons/addonmod.py b/tests/test_addons/scrapy_addons/addonmod.py deleted file mode 100644 index fa479aa68..000000000 --- a/tests/test_addons/scrapy_addons/addonmod.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.scrapy_addons.addonmod' diff --git a/tests/test_addons/scrapy_addons/addonmod2.py b/tests/test_addons/scrapy_addons/addonmod2.py deleted file mode 100644 index da053af4a..000000000 --- a/tests/test_addons/scrapy_addons/addonmod2.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.scrapy_addons.addonmod2' diff --git a/tests/test_addons/scrapy_addons/addonmod3.py b/tests/test_addons/scrapy_addons/addonmod3.py deleted file mode 100644 index c64521478..000000000 --- a/tests/test_addons/scrapy_addons/addonmod3.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.scrapy_addons.addonmod3' From 388c5c4b78509adaf801fb8f66d2d1fdbe1ce9d4 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 00:36:16 +0100 Subject: [PATCH 0026/2083] Fix component exporting for Addon base class --- scrapy/addons/__init__.py | 2 +- tests/test_addons/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index ddd18c7fb..48de99c1a 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -84,7 +84,7 @@ class Addon(object): # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} k = comp v = config.get('order', self.component_order) - settings.set(self.component_type, {k: v}, 'addon') + settings[self.component_type].update({k: v}, 'addon') def export_basics(self, settings): """Export the :attr:`basic_settings` attribute into the settings object. diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 4f1074221..b135dd04d 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -38,7 +38,9 @@ class AddonTest(unittest.TestCase): verifyObject(IAddon, self.testaddon) def test_export_component(self): - settings = BaseSettings({'ITEM_PIPELINES': {}}, 'default') + settings = BaseSettings({'ITEM_PIPELINES': BaseSettings(), + 'DOWNLOAD_HANDLERS': BaseSettings()}, + 'default') self.testaddon.component_type = None self.testaddon.export_component({}, settings) self.assertEqual(len(settings['ITEM_PIPELINES']), 0) From e924d382380a6bc7285b8a9713ac14588681af3b Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 01:09:23 +0100 Subject: [PATCH 0027/2083] Drop component configuration (copied from default_settings) from built-in add-ons --- scrapy/addons/builtins.py | 244 +++++------------------------ tests/test_addons/test_builtins.py | 21 +-- 2 files changed, 37 insertions(+), 228 deletions(-) diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py index ff7902afb..9babdeb6f 100644 --- a/scrapy/addons/builtins.py +++ b/scrapy/addons/builtins.py @@ -13,17 +13,14 @@ __all__ = ['make_builtin_addon', 'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport', 'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump', 'statsmailer', 'telnetconsole', - ] + ] -def make_builtin_addon(addon_name, comp_type, comp, order=0, - addon_default_config=None, addon_version=None): +def make_builtin_addon(addon_name, addon_default_config=None, + addon_version=None): class ThisAddon(Addon): name = addon_name version = addon_version or scrapy.__version__ - component_type = comp_type - component = comp - component_order = order default_config = addon_default_config or {} return ThisAddon @@ -36,73 +33,26 @@ def make_builtin_addon(addon_name, comp_type, comp, order=0, # SPIDER MIDDLEWARES -depth = make_builtin_addon( - 'depth', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.depth.DepthMiddleware', - 900, -) +depth = make_builtin_addon('depth') -httperror = make_builtin_addon( - 'httperror', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', - 50, -) +httperror = make_builtin_addon('httperror') -offsite = make_builtin_addon( - 'offsite', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', - 500, -) +offsite = make_builtin_addon('offsite') -referer = make_builtin_addon( - 'referer', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.referer.RefererMiddleware', - 700, - {'enabled': True}, -) +referer = make_builtin_addon('referer') -urllength = make_builtin_addon( - 'urllength', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', - 800, -) +urllength = make_builtin_addon('urllength') # DOWNLOADER MIDDLEWARES -ajaxcrawl = make_builtin_addon( - 'ajaxcrawl', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware', - 560, -) +ajaxcrawl = make_builtin_addon('ajaxcrawl', {'enabled': True}) -chunked = make_builtin_addon( - 'chunked', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware', - 830, -) +chunked = make_builtin_addon('chunked') -cookies = make_builtin_addon( - 'cookies', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware', - 700, - {'enabled': True}, -) +cookies = make_builtin_addon('cookies') -defaultheaders = make_builtin_addon( - 'defaultheaders', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', - 550, -) +defaultheaders = make_builtin_addon('defaultheaders') # Assume every config entry is a header def defaultheaders_export_config(self, config, settings): conf = self.default_config or {} @@ -110,92 +60,31 @@ def defaultheaders_export_config(self, config, settings): settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon') defaultheaders.export_config = defaultheaders_export_config -downloadtimeout = make_builtin_addon( - 'downloadtimeout', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', - 350, -) +downloadtimeout = make_builtin_addon('downloadtimeout') downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT', 'download_timeout': 'DOWNLOAD_TIMEOUT'} -httpauth = make_builtin_addon( - 'httpauth', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', - 300, -) +httpauth = make_builtin_addon('httpauth') -httpcache = make_builtin_addon( - 'httpcache', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', - 900, - {'enabled': True}, -) +httpcache = make_builtin_addon('httpcache', {'enabled': True}) -httpcompression = make_builtin_addon( - 'httpcompression', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', - 590, - {'enabled': True}, -) +httpcompression = make_builtin_addon('httpcompression') httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'} -httpproxy = make_builtin_addon( - 'httpproxy', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', - 750, -) +httpproxy = make_builtin_addon('httpproxy') -metarefresh = make_builtin_addon( - 'metarefresh', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', - 580, - {'enabled': True}, -) +metarefresh = make_builtin_addon('metarefresh') metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'} -redirect = make_builtin_addon( - 'redirect', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', - 600, - {'enabled': True}, -) +redirect = make_builtin_addon('redirect') -retry = make_builtin_addon( - 'retry', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.retry.RetryMiddleware', - 500, - {'enabled': True}, -) +retry = make_builtin_addon('retry') -robotstxt = make_builtin_addon( - 'robotstxt', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', - 100, - {'obey': True}, -) +robotstxt = make_builtin_addon('robotstxt', {'obey': True}) -stats = make_builtin_addon( - 'stats', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.stats.DownloaderStats', - 850, -) +stats = make_builtin_addon('stats') -useragent = make_builtin_addon( - 'useragent', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', - 400, -) +useragent = make_builtin_addon('useragent') useragent.config_mapping = {'user_agent': 'USER_AGENT'} @@ -204,90 +93,27 @@ useragent.config_mapping = {'user_agent': 'USER_AGENT'} # EXTENSIONS -autothrottle = make_builtin_addon( - 'throttle', - 'EXTENSIONS', - 'scrapy.extensions.throttle.AutoThrottle', - 0, - {'enabled': True}, -) +autothrottle = make_builtin_addon('autothrottle', {'enabled': True}) -corestats = make_builtin_addon( - 'corestats', - 'EXTENSIONS' - 'scrapy.extensions.corestats.CoreStats', - 0, -) +corestats = make_builtin_addon('corestats') -closespider = make_builtin_addon( - 'closespider', - 'EXTENSIONS' - 'scrapy.extensions.closespider.CloseSpider', - 0, -) +closespider = make_builtin_addon('closespider') -debugger = make_builtin_addon( - 'debugger', - 'EXTENSIONS' - 'scrapy.extensions.debug.Debugger', - 0, -) +debugger = make_builtin_addon('debugger') -feedexport = make_builtin_addon( - 'feedexport', - 'EXTENSIONS' - 'scrapy.extensions.feedexport.FeedExporter', - 0, -) +feedexport = make_builtin_addon('feedexport') feedexport.settings_prefix = 'FEED' -logstats = make_builtin_addon( - 'logstats', - 'EXTENSIONS' - 'scrapy.extensions.logstats.LogStats', - 0, -) +logstats = make_builtin_addon('logstats') -memdebug = make_builtin_addon( - 'memdebug', - 'EXTENSIONS' - 'scrapy.extensions.memdebug.MemoryDebugger', - 0, - {'enabled': True}, -) +memdebug = make_builtin_addon('memdebug', {'enabled': True}) -memusage = make_builtin_addon( - 'memusage', - 'EXTENSIONS' - 'scrapy.extensions.memusage.MemoryUsage', - 0, - {'enabled': True}, -) +memusage = make_builtin_addon('memusage', {'enabled': True}) -spiderstate = make_builtin_addon( - 'spiderstate', - 'EXTENSIONS' - 'scrapy.extensions.spiderstate.SpiderState', - 0, -) +spiderstate = make_builtin_addon('spiderstate') -stacktracedump = make_builtin_addon( - 'stacktracedump', - 'EXTENSIONS' - 'scrapy.extensions.debug.StackTraceDump', - 0, -) +stacktracedump = make_builtin_addon('stacktracedump') -statsmailer = make_builtin_addon( - 'statsmailer', - 'EXTENSIONS' - 'scrapy.extensions.statsmailer.StatsMailer', - 0, -) +statsmailer = make_builtin_addon('statsmailer') -telnetconsole = make_builtin_addon( - 'telnetconsole', - 'EXTENSIONS' - 'scrapy.telnet.TelnetConsole', - 0, -) +telnetconsole = make_builtin_addon('telnetconsole') diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py index 607c911fb..c89876950 100644 --- a/tests/test_addons/test_builtins.py +++ b/tests/test_addons/test_builtins.py @@ -9,28 +9,11 @@ from scrapy.settings import Settings class BuiltinAddonsTest(unittest.TestCase): def test_make_builtin_addon(self): - httpcache = make_builtin_addon( - 'httpcache', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', - 900, - {'enabled': True}, - ) + httpcache = make_builtin_addon('httpcache', {'enabled': True}) self.assertEqual(httpcache.name, 'httpcache') - self.assertEqual(httpcache.component_type, 'DOWNLOADER_MIDDLEWARES') - self.assertEqual(httpcache.component, 'scrapy.downloadermiddlewares.' - 'httpcache.HttpCacheMiddleware') - self.assertEqual(httpcache.component_order, 900) self.assertEqual(httpcache.default_config, {'enabled': True}) self.assertEqual(httpcache.version, scrapy.__version__) - httpcache = make_builtin_addon( - 'httpcache', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', - 900, - {'enabled': True}, - '99.9', - ) + httpcache = make_builtin_addon('httpcache', {'enabled': True}, '99.9') self.assertEqual(httpcache.version, '99.9') def test_defaultheaders_export_config(self): From b7b00fb95669196167a5e5f76b1f3b8e2feb7b84 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 01:18:41 +0100 Subject: [PATCH 0028/2083] PEP8ify add-ons and tests --- scrapy/addons/__init__.py | 6 ++++-- tests/test_addons/__init__.py | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 48de99c1a..169c79eac 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -323,6 +323,7 @@ class AddonManager(Mapping): """ # Collect all active add-ons and the components they provide ws = WorkingSet('') + def add_dist(project_name, version, **kwargs): if project_name in ws.entry_keys.get('scrapy', []): raise ImportError("Component {} provided by multiple add-ons" @@ -354,8 +355,9 @@ class AddonManager(Mapping): # our own exception or is it helpful enough? if ws.find(req) is None: raise ImportError( - "Add-ons {} require or modify missing component {}" - "".format(required[reqstr]+modified[reqstr], reqstr)) + "Add-ons {} require or modify missing component {}" + "".format(required[reqstr]+modified[reqstr], reqstr) + ) mod_and_req = set(required.keys()).intersection(modified.keys()) for conflict in mod_and_req: diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index b135dd04d..32ee25ad8 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -24,6 +24,7 @@ class AddonTest(unittest.TestCase): def setUp(self): self.rawaddon = Addon() + class AddonWithAttributes(Addon): name = 'Test' version = '1.0' @@ -132,7 +133,7 @@ class AddonManagerTest(unittest.TestCase): def test_add_verifies(self): brokenaddon = self.manager.get_addon( - 'tests.test_addons.addons.BrokenAddon') + 'tests.test_addons.addons.BrokenAddon') self.assertRaises(zope.interface.exceptions.BrokenImplementation, self.manager.add, brokenaddon) @@ -145,11 +146,13 @@ class AddonManagerTest(unittest.TestCase): def test_remove(self): manager = AddonManager() + def test_gets_removed(removearg): manager.add(addonmod) self.assertIn('AddonModule', manager) manager.remove(removearg) self.assertNotIn('AddonModule', manager) + test_gets_removed('AddonModule') test_gets_removed(addonmod) test_gets_removed('tests.test_addons.addonmod') @@ -158,8 +161,7 @@ class AddonManagerTest(unittest.TestCase): self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) def test_get_addon(self): - goodaddon = self.manager.get_addon( - 'tests.test_addons.addons.GoodAddon') + goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon') self.assertIs(goodaddon, addons.GoodAddon) loaded_addonmod = self.manager.get_addon(self.ADDONMODPATH) @@ -194,8 +196,7 @@ class AddonManagerTest(unittest.TestCase): getattr(manager, func)(*args, **kwargs) six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], - ['key']) + six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') # XXX: Check module equality, see above self.assertEqual(manager['AddonModule'].name, addonmod.name) @@ -284,17 +285,17 @@ class AddonManagerTest(unittest.TestCase): ua_first.assert_called_once_with(manager.configs['FirstAddon'], manager) ua_second.assert_called_once_with(manager.configs['SecondAddon'], - manager) + manager) manager.update_settings(settings) us_first.assert_called_once_with(manager.configs['FirstAddon'], settings) us_second.assert_called_once_with(manager.configs['SecondAddon'], - settings) + settings) manager.check_configuration(crawler) cc_first.assert_called_once_with(manager.configs['FirstAddon'], crawler) cc_second.assert_called_once_with(manager.configs['SecondAddon'], - crawler) + crawler) self.assertEqual(ua_first.call_count, 1) self.assertEqual(ua_second.call_count, 1) self.assertEqual(us_first.call_count, 1) @@ -321,6 +322,7 @@ class AddonManagerTest(unittest.TestCase): class FirstAddon(addons.GoodAddon): name = 'FirstAddon' + def update_addons(self, config, addons): addons.add(AddedAddon()) From 33dfb3e167f7d31e2573ab39c2b80bf7728c2b00 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 11:22:24 +0100 Subject: [PATCH 0029/2083] Remove unused project path util function --- scrapy/utils/project.py | 12 ------------ tests/test_utils_project.py | 27 --------------------------- 2 files changed, 39 deletions(-) delete mode 100644 tests/test_utils_project.py diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index a1266c879..a15a0d90f 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -71,15 +71,3 @@ def get_project_settings(): settings.setdict(env_overrides, priority='project') return settings - -def get_project_path(): - """Return the Python path of the current project. - - This fails when the settings module does not live in the project's root. - """ - if not inside_project(): - raise NotConfigured("Not inside a project") - settings_module_path = os.environ.get(ENVVAR) - if not settings_module_path: - raise NotConfigured("Unable to locate project's python path") - return settings_module_path.rsplit('.', 1)[0] diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py deleted file mode 100644 index cea4d9950..000000000 --- a/tests/test_utils_project.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -from tests import mock -import unittest - -from scrapy.exceptions import NotConfigured -from scrapy.utils.project import get_project_path, inside_project - - -class UtilsProjectTestCase(unittest.TestCase): - - @mock.patch('scrapy.utils.project.inside_project', return_value=True) - def test_get_project_path(self, mock_ip): - def _test(settingsmod, expected): - with mock.patch.dict('os.environ', - {'SCRAPY_SETTINGS_MODULE': settingsmod}): - self.assertEqual(get_project_path(), expected) - _test('project.settings', 'project') - _test('project.othername', 'project') - _test('nested.project.settings', 'nested.project') - - with mock.patch.dict('os.environ', {}, clear=True): - self.assertRaises(NotConfigured, get_project_path) - - mock_ip.return_value = False - with mock.patch.dict('os.environ', - {'SCRAPY_SETTINGS_MODULE': 'some.settings'}): - self.assertRaises(NotConfigured, get_project_path) From a65fc0db7d7d728bc8abbb37eff358870ce7a9b5 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Thu, 12 Nov 2015 18:37:26 +0100 Subject: [PATCH 0030/2083] Drop support for providing file paths as add-ons --- scrapy/utils/misc.py | 24 --------------------- tests/test_addons/__init__.py | 26 ++++++----------------- tests/test_utils_misc/__init__.py | 19 +---------------- tests/test_utils_misc/testpkg/__init__.py | 1 - tests/test_utils_misc/testpkg/submod.py | 1 - 5 files changed, 8 insertions(+), 63 deletions(-) delete mode 100644 tests/test_utils_misc/testpkg/__init__.py delete mode 100644 tests/test_utils_misc/testpkg/submod.py diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 9461d93e9..e55edd63e 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,8 +1,5 @@ """Helper functions which doesn't fit anywhere else""" -import itertools -import os.path import re -import sys import hashlib from importlib import import_module from pkgutil import iter_modules @@ -72,10 +69,6 @@ def load_module_or_object(path): return load_object(path) except (ValueError, NameError, ImportError): pass - try: - return get_module_from_filepath(path) - except ImportError: - pass raise NameError("Could not load '%s'" % path) @@ -101,23 +94,6 @@ def walk_modules(path): return mods -def get_module_from_filepath(path): - """Load and return a python module/package from a file path""" - path = path.rstrip("/") - if path.endswith('.py'): - path = path.rsplit('.py', 1)[0] - basefolder, modname = os.path.split(path) - # XXX: There are other ways to import modules from a full path which don't - # need to modify PYTHONPATH, see - # https://stackoverflow.com/questions/67631/ - # These methods differ between py2 and py3, and apparently the - # py3 method was deprecated in Python 3.4 - sys.path.insert(0, basefolder) - mod = import_module(modname) - sys.path.pop(0) - return mod - - def extract_regex(regex, text, encoding='utf-8'): """Extract a list of unicode strings from the given text/encoding using the following policies: diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 32ee25ad8..a4e278fa5 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -108,8 +108,6 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - ADDONMODPATH = os.path.join(os.path.dirname(__file__), 'addonmod.py') - def setUp(self): self.manager = AddonManager() @@ -156,7 +154,6 @@ class AddonManagerTest(unittest.TestCase): test_gets_removed('AddonModule') test_gets_removed(addonmod) test_gets_removed('tests.test_addons.addonmod') - test_gets_removed(self.ADDONMODPATH) self.assertRaises(KeyError, manager.remove, 'nonexistent') self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) @@ -164,18 +161,12 @@ class AddonManagerTest(unittest.TestCase): goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon') self.assertIs(goodaddon, addons.GoodAddon) - loaded_addonmod = self.manager.get_addon(self.ADDONMODPATH) - # XXX: The module is in fact imported twice under different names into - # sys.modules, is there a good assertion for module equality? - self.assertEqual(loaded_addonmod.name, addonmod.name) + loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod") + self.assertIs(loaded_addonmod, addonmod) - # Does not provide interface, but has _addon attribute pointing to - # GoodAddon instance addonspath = os.path.join(os.path.dirname(__file__), 'addons.py') - goodaddon = self.manager.get_addon(addonspath) - # XXX: Again, the imported class and addons.GoodAddon are different - # since they are imported twice. How to use isInstance? - self.assertEqual(goodaddon.name, addons.GoodAddon.name) + goodaddon = self.manager.get_addon("tests.test_addons.addons") + self.assertIsInstance(goodaddon, addons.GoodAddon) self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent') @@ -198,21 +189,18 @@ class AddonManagerTest(unittest.TestCase): self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') - # XXX: Check module equality, see above - self.assertEqual(manager['AddonModule'].name, addonmod.name) + self.assertEqual(manager['AddonModule'], addonmod) self.assertIn('key', manager.configs['AddonModule']) self.assertEqual(manager.configs['AddonModule']['key'], 'val1') addonsdict = { - self.ADDONMODPATH: { - 'key': 'val1', - }, + "tests.test_addons.addonmod": {'key': 'val1'}, 'tests.test_addons.addons.GoodAddon': {'key': 'val2'}, } _test_load_method('load_dict', addonsdict) settings = BaseSettings() - settings.set('ADDONS', {self.ADDONMODPATH: 0, + settings.set('ADDONS', {"tests.test_addons.addonmod": 0, 'tests.test_addons.addons.GoodAddon': 0}) settings.set('ADDONMODULE', {'key': 'val1'}) settings.set('GOODADDON', {'key': 'val2'}) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index f33562b7d..8ea8786d7 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -4,7 +4,7 @@ import unittest from scrapy.item import Item, Field from scrapy.utils.misc import (load_object, load_module_or_object, arg_to_iter, - walk_modules, get_module_from_filepath) + walk_modules) __doctests__ = ['scrapy.utils.misc'] @@ -21,9 +21,6 @@ class UtilsMiscTestCase(unittest.TestCase): def test_load_module_or_object(self): testmod = load_module_or_object(__name__ + '.testmod') self.assertTrue(hasattr(testmod, 'TESTVAR')) - testmod = load_module_or_object( - os.path.join(os.path.dirname(__file__), 'testmod.py')) - self.assertTrue(hasattr(testmod, 'TESTVAR')) obj = load_object('scrapy.utils.misc.load_object') self.assertIs(obj, load_object) @@ -67,20 +64,6 @@ class UtilsMiscTestCase(unittest.TestCase): finally: sys.path.remove(egg) - def test_get_module_from_filepath(self): - testmodpath = os.path.join(os.path.dirname(__file__), 'testmod.py') - testmod = get_module_from_filepath(testmodpath) - self.assertTrue(hasattr(testmod, 'TESTVAR')) - - testpkgpath = os.path.join(os.path.dirname(__file__), 'testpkg') - testpkg = get_module_from_filepath(testpkgpath) - self.assertTrue(hasattr(testpkg, 'TESTVAR2')) - # Check submodule access - import testpkg.submod - self.assertTrue(hasattr(testpkg.submod, 'TESTVAR3')) - self.assertIs(testpkg.submod.TESTVAR3, - load_object(testpkg.__name__ + ".submod.TESTVAR3")) - def test_arg_to_iter(self): class TestItem(Item): diff --git a/tests/test_utils_misc/testpkg/__init__.py b/tests/test_utils_misc/testpkg/__init__.py deleted file mode 100644 index 12cc2f6d9..000000000 --- a/tests/test_utils_misc/testpkg/__init__.py +++ /dev/null @@ -1 +0,0 @@ -TESTVAR2 = True diff --git a/tests/test_utils_misc/testpkg/submod.py b/tests/test_utils_misc/testpkg/submod.py deleted file mode 100644 index 8a07e3592..000000000 --- a/tests/test_utils_misc/testpkg/submod.py +++ /dev/null @@ -1 +0,0 @@ -TESTVAR3 = True From 114437c1693c85125c4275387616d398e68ec20f Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Wed, 4 Nov 2015 02:29:28 +0800 Subject: [PATCH 0031/2083] added: Doc for `scrapy.http.TextResponse.urljoin` --- docs/topics/request-response.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 75b98d3b3..ba3d697ef 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -666,6 +666,14 @@ TextResponse objects The same as :attr:`text`, but available as a method. This method is kept for backwards compatibility; please prefer ``response.text``. + .. method:: TextResponse.urljoin(url) + + Constructs an absolute url by combining the Response's base url with + a possible relative url. The base url shall be extracted from the + ```` tag, or just the Response's :attr:`url` if there is no such + tag. + + HtmlResponse objects -------------------- From 8d97f49e5e31955fb972dc63106298032ebe0e59 Mon Sep 17 00:00:00 2001 From: Harrison Gregg Date: Sat, 9 Sep 2017 13:54:48 +0630 Subject: [PATCH 0032/2083] Re-add SIGINT handler in inspect_response after shell closes --- scrapy/shell.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/shell.py b/scrapy/shell.py index 80b625633..af91d6ce0 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -164,7 +164,11 @@ class Shell(object): def inspect_response(response, spider): """Open a shell to inspect the given response""" + # Shell.start removes the SIGINT handler, so save it and re-add it after + # the shell has closed + sigint_handler = signal.getsignal(signal.SIGINT) Shell(spider.crawler).start(response=response, spider=spider) + signal.signal(signal.SIGINT, sigint_handler) def _request_deferred(request): From 412f8526029f63fbebd6dae3bad7240dee8dc090 Mon Sep 17 00:00:00 2001 From: Arvind Prasanna <1108710+aprasanna@users.noreply.github.com> Date: Tue, 6 Mar 2018 23:58:27 -0500 Subject: [PATCH 0033/2083] A few typo fixes and some grammatical enhancements --- docs/news.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 1629510b2..be4cac3f3 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1888,7 +1888,7 @@ Scrapy 0.14.3 - include egg files used by testsuite in source distribution. #118 (:commit:`c897793`) - update docstring in project template to avoid confusion with genspider command, which may be considered as an advanced feature. refs #107 (:commit:`2548dcc`) - added note to docs/topics/firebug.rst about google directory being shut down (:commit:`668e352`) -- dont discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`) +- do not discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`) - do not fail handling unicode xpaths in libxml2 backed selectors (:commit:`b830e95`) - fixed minor mistake in Request objects documentation (:commit:`bf3c9ee`) - fixed minor defect in link extractors documentation (:commit:`ba14f38`) @@ -1984,7 +1984,7 @@ Code rearranged and removed - Removed googledir project from `examples/googledir`. There's now a new example project called `dirbot` available on github: https://github.com/scrapy/dirbot - Removed support for default field values in Scrapy items (:rev:`2616`) - Removed experimental crawlspider v2 (:rev:`2632`) -- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`) +- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe filtering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`) - Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`) - Removed deprecated Execution Queue (:rev:`2704`) - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`) @@ -2054,7 +2054,7 @@ New features and improvements - Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195) - Support for overriding default request headers per spider (#181) - Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186) -- Splitted Debian package into two packages - the library and the service (#187) +- The Debian package has been split into two packages - library and the service (#187) - Scrapy log refactoring (#188) - New extension for keeping persistent spider contexts among different runs (#203) - Added `dont_redirect` request.meta key for avoiding redirects (#233) @@ -2075,7 +2075,7 @@ API changes - ``url`` and ``body`` attributes of Request objects are now read-only (#230) - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231) - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default) -- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) +- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) - Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself. - Changes to Scrapy Manager (now called "Crawler"): - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` From ac111088c6470d55e4d698247a76a4938f4a6dc6 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 20:12:29 +0530 Subject: [PATCH 0034/2083] duplicate keys handled --- scrapy/http/request/form.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index c2413b431..660f4fc96 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ import six -from six.moves.urllib.parse import urljoin, urlencode +from six.moves.urllib.parse import urljoin, urlencode, urlsplit import lxml.html from parsel.selector import create_root_node @@ -33,7 +33,30 @@ class FormRequest(Request): self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') self._set_body(querystr) else: - self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr) + if urlsplit(self.url).query: + queries = (urlsplit(self.url).query + '&' + querystr).split('&') + else: + queries = querystr.split('&') + query_dict = {} + for i in range(len(queries)): + query_list = queries[i].split('=') + query_dict[query_list[0]] = query_list[1] + querystr = '' + query_key = list(query_dict.keys()) + for i in range(len(query_dict)): + querystr += (query_key[i] + '=' + query_dict[query_key[i]]) + if i!=len(query_dict)-1: + querystr += '&' + if urlsplit(self.url).fragment: + if urlsplit(self.url).query: + self._set_url(self.url[:self.url.index('?')+1]+querystr+'#'+urlsplit(self.url).fragment) + else: + self._set_url(self.url+'?'+querystr+'#'+urlsplit(self.url).fragment) + else: + if urlsplit(self.url).query: + self._set_url(self.url[:self.url.index('?')+1]+querystr) + else: + self._set_url(self.url+'?'+querystr) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From b5e454809e240839e76f050b4587abcd80c33765 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 20:12:31 +0530 Subject: [PATCH 0035/2083] Included test --- tests/test_http_request.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 58326a384..5a7cf3918 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -273,6 +273,19 @@ class FormRequestTest(RequestTest): def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) self.assertEqual(r1.body, b'') + + def test_formdata_overrides_querystring_duplicates(self): + data1 = {'a' : '1', 'b' : '2'} + data2 = [('a','one'), ('a','two'), ('c','three')] + + fs1 = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1)) + self.assertEqual(fs1[b'a'], [b'1']) + self.assertEqual(fs1[b'b'], [b'2']) + + fs2 = _qs(self.request_class('http://www.example.com?a=1&b=2&c=3', method='GET', formdata=data2)) + self.assertEqual(fs2[b'a'], [b'two']) + self.assertEqual(fs2[b'b'], [b'2']) + self.assertEqual(fs2[b'c'], [b'three']) def test_default_encoding_bytes(self): # using default encoding (utf-8) From 1bea5d307628a11d4010cdabb87b2a29a6c3c772 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 22:35:16 +0530 Subject: [PATCH 0036/2083] Fixed error --- scrapy/http/request/form.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 660f4fc96..f08d1f626 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -34,29 +34,32 @@ class FormRequest(Request): self._set_body(querystr) else: if urlsplit(self.url).query: - queries = (urlsplit(self.url).query + '&' + querystr).split('&') + queries = (urlsplit(self.url).query).split('&') else: queries = querystr.split('&') query_dict = {} + duplicate_key_passed=[] + for i in range(len(querystr.split('&'))): + query_list = querystr.split('&')[i].split('=') + duplicate_key_passed.append(query_list[0]) for i in range(len(queries)): query_list = queries[i].split('=') - query_dict[query_list[0]] = query_list[1] - querystr = '' + if duplicate_key_passed.count(query_list[0])<=1: + query_dict[query_list[0]] = query_list[1] + query_str = '' query_key = list(query_dict.keys()) for i in range(len(query_dict)): - querystr += (query_key[i] + '=' + query_dict[query_key[i]]) - if i!=len(query_dict)-1: - querystr += '&' + query_str += (query_key[i] + '=' + query_dict[query_key[i]] + '&') if urlsplit(self.url).fragment: if urlsplit(self.url).query: - self._set_url(self.url[:self.url.index('?')+1]+querystr+'#'+urlsplit(self.url).fragment) + self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + '#' + urlsplit(self.url).fragment) else: - self._set_url(self.url+'?'+querystr+'#'+urlsplit(self.url).fragment) + self._set_url(self.url + '?' + querystr + '#' + urlsplit(self.url).fragment) else: if urlsplit(self.url).query: - self._set_url(self.url[:self.url.index('?')+1]+querystr) + self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr) else: - self._set_url(self.url+'?'+querystr) + self._set_url(self.url + '?' + querystr) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 9a4bbd6d029f61dc60594fb8bca631c3a494f855 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 23:05:58 +0530 Subject: [PATCH 0037/2083] Update form.py --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index f08d1f626..1b487cbd0 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -44,7 +44,7 @@ class FormRequest(Request): duplicate_key_passed.append(query_list[0]) for i in range(len(queries)): query_list = queries[i].split('=') - if duplicate_key_passed.count(query_list[0])<=1: + if duplicate_key_passed.count(query_list[0])==0: query_dict[query_list[0]] = query_list[1] query_str = '' query_key = list(query_dict.keys()) From 4abcdcb306d8782e6b792ad36e40391a0bcbd0f3 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 23:22:53 +0530 Subject: [PATCH 0038/2083] Update test_http_request.py --- tests/test_http_request.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5a7cf3918..ec247a02d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -276,17 +276,10 @@ class FormRequestTest(RequestTest): def test_formdata_overrides_querystring_duplicates(self): data1 = {'a' : '1', 'b' : '2'} - data2 = [('a','one'), ('a','two'), ('c','three')] - - fs1 = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1)) + fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1)) self.assertEqual(fs1[b'a'], [b'1']) self.assertEqual(fs1[b'b'], [b'2']) - fs2 = _qs(self.request_class('http://www.example.com?a=1&b=2&c=3', method='GET', formdata=data2)) - self.assertEqual(fs2[b'a'], [b'two']) - self.assertEqual(fs2[b'b'], [b'2']) - self.assertEqual(fs2[b'c'], [b'three']) - def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b'one': b'two', b'price': b'\xc2\xa3 100'} From 023290dabc55c92ea2143a98f7753abaf21f74c9 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sun, 13 Jan 2019 23:50:31 +0530 Subject: [PATCH 0039/2083] Update test_http_request.py --- tests/test_http_request.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index ec247a02d..1ef84f1d6 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -275,10 +275,10 @@ class FormRequestTest(RequestTest): self.assertEqual(r1.body, b'') def test_formdata_overrides_querystring_duplicates(self): - data1 = {'a' : '1', 'b' : '2'} - fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data1)) - self.assertEqual(fs1[b'a'], [b'1']) - self.assertEqual(fs1[b'b'], [b'2']) + data = {'a' : '1', 'b' : '2'} + fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) def test_default_encoding_bytes(self): # using default encoding (utf-8) From 9f1f4df9666f38f81adda5a41f3cac646a4f18dc Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Wed, 16 Jan 2019 22:59:41 +0530 Subject: [PATCH 0040/2083] Update test_http_request.py --- tests/test_http_request.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 1ef84f1d6..6c4cb10a8 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -275,8 +275,22 @@ class FormRequestTest(RequestTest): self.assertEqual(r1.body, b'') def test_formdata_overrides_querystring_duplicates(self): + #Without fragment data = {'a' : '1', 'b' : '2'} - fs = _qs(self.request_class('http://www.example.com?a=0&a=2&b=1', method='GET', formdata=data)) + fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + + #With fragment + data = (('a', '1'), ('b', '2')) + url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0] + fs = _qs(self.request_class(url, method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + + #Witout query + data = {'a' : '1', 'b' : '2'} + fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) From 3e67fa8fc1eaa2189cec08ce4000b3efc254ee1b Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Wed, 16 Jan 2019 23:01:38 +0530 Subject: [PATCH 0041/2083] Improved for better user readability --- scrapy/http/request/form.py | 45 ++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 1b487cbd0..fa5110382 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -33,33 +33,26 @@ class FormRequest(Request): self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') self._set_body(querystr) else: - if urlsplit(self.url).query: - queries = (urlsplit(self.url).query).split('&') + url_split = urlsplit(self.url) + formdata_key_list = [] + for k in querystr.split('&'): + formdata_key_list.append(k.split('=')[0]) + + if url_split.query: + queries = (url_split.query).split('&') + query_dict = {} + + for x in queries: + query = x.split('=') + if formdata_key_list.count(query[0])==0: + query_dict[query[0]] = query[1] + query_str = '' + for k, v in query_dict: + query_str += (k + '=' + v + '&') + + self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else '')) else: - queries = querystr.split('&') - query_dict = {} - duplicate_key_passed=[] - for i in range(len(querystr.split('&'))): - query_list = querystr.split('&')[i].split('=') - duplicate_key_passed.append(query_list[0]) - for i in range(len(queries)): - query_list = queries[i].split('=') - if duplicate_key_passed.count(query_list[0])==0: - query_dict[query_list[0]] = query_list[1] - query_str = '' - query_key = list(query_dict.keys()) - for i in range(len(query_dict)): - query_str += (query_key[i] + '=' + query_dict[query_key[i]] + '&') - if urlsplit(self.url).fragment: - if urlsplit(self.url).query: - self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + '#' + urlsplit(self.url).fragment) - else: - self._set_url(self.url + '?' + querystr + '#' + urlsplit(self.url).fragment) - else: - if urlsplit(self.url).query: - self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr) - else: - self._set_url(self.url + '?' + querystr) + self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else '')) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 6f86c93f366509ba05faf3b2cdc12f5e3ef6693a Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Wed, 16 Jan 2019 23:54:35 +0530 Subject: [PATCH 0042/2083] Increased test cases --- tests/test_http_request.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6c4cb10a8..571df0176 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -275,20 +275,27 @@ class FormRequestTest(RequestTest): self.assertEqual(r1.body, b'') def test_formdata_overrides_querystring_duplicates(self): - #Without fragment - data = {'a' : '1', 'b' : '2'} - fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) - self.assertEqual(fs[b'b'], [b'2']) - - #With fragment + #Both fragment and query in url data = (('a', '1'), ('b', '2')) url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) - #Witout query + #only query in url + data = {'a' : '1', 'b' : '2'} + fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + + #only fragment in url + data = (('a', '1'), ('b', '2')) + url = self.request_class('http://www.example.com/#fragment', method='GET', formdata=data).url.split('#')[0] + fs = _qs(self.request_class(url, method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + + #None of both in url data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) From 6be73f06c33a27cb3291a1e46d80574b4d23b10b Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Thu, 17 Jan 2019 23:50:58 +0530 Subject: [PATCH 0043/2083] Updated tests --- tests/test_http_request.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 571df0176..a17acf93b 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -277,23 +277,11 @@ class FormRequestTest(RequestTest): def test_formdata_overrides_querystring_duplicates(self): #Both fragment and query in url data = (('a', '1'), ('b', '2')) - url = self.request_class('http://www.example.com/?a=0&a=2&b=1#fragment', method='GET', formdata=data).url.split('#')[0] - fs = _qs(self.request_class(url, method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) - self.assertEqual(fs[b'b'], [b'2']) - - #only query in url - data = {'a' : '1', 'b' : '2'} - fs = _qs(self.request_class('http://www.example.com/?a=0&a=2&b=1', method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) - self.assertEqual(fs[b'b'], [b'2']) - - #only fragment in url - data = (('a', '1'), ('b', '2')) - url = self.request_class('http://www.example.com/#fragment', method='GET', formdata=data).url.split('#')[0] + url = self.request_class('http://www.example.com/?a=0&a=2&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) + self.assertEqual(fs[b'c'], [b'3']) #None of both in url data = {'a' : '1', 'b' : '2'} From a9f68acb6dd7f5aca2bee95353278e5a40c32b8b Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Thu, 17 Jan 2019 23:51:09 +0530 Subject: [PATCH 0044/2083] modified code --- scrapy/http/request/form.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index fa5110382..2305491e2 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -41,15 +41,14 @@ class FormRequest(Request): if url_split.query: queries = (url_split.query).split('&') query_dict = {} - for x in queries: - query = x.split('=') - if formdata_key_list.count(query[0])==0: - query_dict[query[0]] = query[1] + k = x.split('=')[0] + v = x.split('=')[1] + if formdata_key_list.count(k)==0: + query_dict[k] = v query_str = '' - for k, v in query_dict: + for k, v in query_dict.items(): query_str += (k + '=' + v + '&') - self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else '')) else: self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else '')) From 7dee841b8b904a3961481f7b1999d819d2c17c5d Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sat, 19 Jan 2019 13:20:01 +0530 Subject: [PATCH 0045/2083] Update form.py --- scrapy/http/request/form.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2305491e2..af4ed1793 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ import six -from six.moves.urllib.parse import urljoin, urlencode, urlsplit +from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl import lxml.html from parsel.selector import create_root_node @@ -37,21 +37,10 @@ class FormRequest(Request): formdata_key_list = [] for k in querystr.split('&'): formdata_key_list.append(k.split('=')[0]) - - if url_split.query: - queries = (url_split.query).split('&') - query_dict = {} - for x in queries: - k = x.split('=')[0] - v = x.split('=')[1] - if formdata_key_list.count(k)==0: - query_dict[k] = v - query_str = '' - for k, v in query_dict.items(): - query_str += (k + '=' + v + '&') - self._set_url(self.url[:self.url.index('?')+1] + query_str + querystr + ('#' + url_split.fragment if url_split.fragment else '')) - else: - self._set_url(self.url + '?' + querystr + ('#' + url_split.fragment if url_split.fragment else '')) + items = [] + items += [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list] + query_str = _urlencode(items, self.encoding) + self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else ''))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 722a30ac2bc35bdc7abb52d10417ed1191a3aee0 Mon Sep 17 00:00:00 2001 From: Maram Sumanth <32808381+maramsumanth@users.noreply.github.com> Date: Sat, 19 Jan 2019 13:20:05 +0530 Subject: [PATCH 0046/2083] Update test_http_request.py --- tests/test_http_request.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a17acf93b..955db9e76 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -276,10 +276,10 @@ class FormRequestTest(RequestTest): def test_formdata_overrides_querystring_duplicates(self): #Both fragment and query in url - data = (('a', '1'), ('b', '2')) - url = self.request_class('http://www.example.com/?a=0&a=2&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] + data = (('a', 'one'), ('a', 'two'), ('b', '2')) + url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) self.assertEqual(fs[b'c'], [b'3']) From 6eca6f92c6ae48bb136311308e77e82d7659cf43 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 14:59:34 +0530 Subject: [PATCH 0047/2083] Update form.py --- scrapy/http/request/form.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index af4ed1793..b6ca6ef9c 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -34,11 +34,8 @@ class FormRequest(Request): self._set_body(querystr) else: url_split = urlsplit(self.url) - formdata_key_list = [] - for k in querystr.split('&'): - formdata_key_list.append(k.split('=')[0]) - items = [] - items += [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list] + formdata_key_list = list(dict(parse_qsl(querystr)).keys()) + items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list] query_str = _urlencode(items, self.encoding) self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else ''))) From d75b61b96aad172e21c7d3aeba6f48419a63c72d Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 15:07:12 +0530 Subject: [PATCH 0048/2083] Update test_http_request.py --- tests/test_http_request.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 955db9e76..2da1cdf6d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -289,6 +289,12 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) + #GET duplicates are preserved + formdata=(('foo', 'bar'), ('foo', 'baz')) + fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data)) + self.assertEqual(set(fs[b'foo']), {b'bar', b'baz'}) + self.assertEqual(set(fs[b'a']), {b'1', b'2'}) + def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b'one': b'two', b'price': b'\xc2\xa3 100'} From fdf03a6d0daae79b9a3b421d19102873d5a75f46 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 15:12:44 +0530 Subject: [PATCH 0049/2083] correcting tests --- tests/test_http_request.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 2da1cdf6d..0c2f95262 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -289,10 +289,10 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) - #GET duplicates are preserved - formdata=(('foo', 'bar'), ('foo', 'baz')) + #Duplicate GET arguments are preserved + formdata={'foo' : 'bar'} fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data)) - self.assertEqual(set(fs[b'foo']), {b'bar', b'baz'}) + self.assertEqual(fs[b'foo'], [b'bar']) self.assertEqual(set(fs[b'a']), {b'1', b'2'}) def test_default_encoding_bytes(self): From 8831fafabc699bd2ab48e3e185712fbcd4f7cc7f Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 15:42:48 +0530 Subject: [PATCH 0050/2083] Update test_http_request.py --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 0c2f95262..5c30c9785 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -290,7 +290,7 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'b'], [b'2']) #Duplicate GET arguments are preserved - formdata={'foo' : 'bar'} + data={'foo' : 'bar'} fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data)) self.assertEqual(fs[b'foo'], [b'bar']) self.assertEqual(set(fs[b'a']), {b'1', b'2'}) From 7da460b7935940c93d5454019ea99f9f117b3e7b Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 4 Mar 2019 17:25:15 +0530 Subject: [PATCH 0051/2083] Update form.py --- scrapy/http/request/form.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b6ca6ef9c..b44d7b9a6 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ import six -from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl +from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl, urlunsplit import lxml.html from parsel.selector import create_root_node @@ -34,10 +34,10 @@ class FormRequest(Request): self._set_body(querystr) else: url_split = urlsplit(self.url) - formdata_key_list = list(dict(parse_qsl(querystr)).keys()) - items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_key_list] + formdata_keys = set(dict(parse_qsl(querystr)).keys()) + items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys] query_str = _urlencode(items, self.encoding) - self._set_url(urljoin(self.url,'?'+ (query_str + '&' if query_str else '') + querystr + ('#'+ url_split.fragment if url_split.fragment else ''))) + self._set_url(urlunsplit((url_split.scheme, url_split.netloc, url_split.path, (query_str + '&' + querystr) if query_str else querystr, url_split.fragment))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From f7bf3abbd03a10124ed648d45abbb6c8eb3218b7 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Wed, 6 Mar 2019 14:10:03 +0530 Subject: [PATCH 0052/2083] Modified code --- scrapy/http/request/form.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b44d7b9a6..a0e7f76a9 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -37,7 +37,8 @@ class FormRequest(Request): formdata_keys = set(dict(parse_qsl(querystr)).keys()) items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys] query_str = _urlencode(items, self.encoding) - self._set_url(urlunsplit((url_split.scheme, url_split.netloc, url_split.path, (query_str + '&' + querystr) if query_str else querystr, url_split.fragment))) + query = (query_str + '&' + querystr) if query_str else querystr + self._set_url(urlunsplit(url_split._replace(query = query))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 120007c0577f819a26ff795ac8b9cd6fc397df19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 13:53:47 +0100 Subject: [PATCH 0053/2083] Add a FAQ entry on how to deal with long lists of allowed domains --- docs/faq.rst | 35 +++++++++++++++++++++++++++++++ docs/topics/spider-middleware.rst | 2 ++ 2 files changed, 37 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 7a0628f88..f56d26c0a 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,6 +149,41 @@ How can I make Scrapy consume less memory? See previous question. +How can I prevent memory errors due to many allowed domains? +------------------------------------------------------------ + +If you have a spider with a long list of +:attr:`~scrapy.spiders.Spider.allowed_domains` (e.g. 50,000+), consider +replacing the default +:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware +with a :ref:`custom spider middleware ` that requires +less memory. For example: + +- If your domain names are similar enough, use your own regular expression + instead joining the strings in + :attr:`~scrapy.spiders.Spider.allowed_domains` into a complex regular + expression. + +- If you can `meet the installation requirements`_, use pyre2_ instead of + Python’s re_ to compile your URL-filtering regular expression. See + :issue:`1908`. + +See also other suggestions at `StackOverflow`_. + +.. note:: Remember to disable + :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable + your custom implementation:: + + SPIDER_MIDDLEWARES = { + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, + 'myproject.middlewares.CustomOffsiteMiddleware': 500, + } + +.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation +.. _pyre2: https://github.com/andreasvc/pyre2 +.. _re: https://docs.python.org/library/re.html +.. _StackOverflow: https://stackoverflow.com/q/36440681/939364 + Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b7e42771..80357a987 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -54,6 +54,8 @@ value. For example, if you want to disable the off-site middleware:: Finally, keep in mind that some middlewares may need to be enabled through a particular setting. See each middleware documentation for more info. +.. _custom-spider-middleware: + Writing your own spider middleware ================================== From 35f7595dbebe7012c8f66336b76625b089a1fad3 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Mon, 11 Mar 2019 23:58:37 +0530 Subject: [PATCH 0054/2083] changed variable names --- scrapy/http/request/form.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a0e7f76a9..9af2db5ff 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -28,16 +28,16 @@ class FormRequest(Request): if formdata: items = formdata.items() if isinstance(formdata, dict) else formdata - querystr = _urlencode(items, self.encoding) + form_query_str = _urlencode(items, self.encoding) if self.method == 'POST': self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') - self._set_body(querystr) + self._set_body(form_query_str) else: url_split = urlsplit(self.url) - formdata_keys = set(dict(parse_qsl(querystr)).keys()) + formdata_keys = set(dict(parse_qsl(form_query_str)).keys()) items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys] - query_str = _urlencode(items, self.encoding) - query = (query_str + '&' + querystr) if query_str else querystr + url_query_str = _urlencode(items, self.encoding) + query = (url_query_str + '&' + form_query_str) if url_query_str else form_query_str self._set_url(urlunsplit(url_split._replace(query = query))) @classmethod From 25e616fa04bf4199b9017553b3e57699c40b0683 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Fri, 15 Mar 2019 18:09:47 +0530 Subject: [PATCH 0055/2083] do not degrade JPEG files. --- scrapy/pipelines/images.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index a1457c7e9..add606a2e 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -127,7 +127,7 @@ class ImagesPipeline(FilesPipeline): raise ImageException("Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) - image, buf = self.convert_image(orig_image) + image, buf = self.convert_image(orig_image, BytesIO(response.body)) yield path, image, buf for thumb_id, size in six.iteritems(self.thumbs): @@ -135,7 +135,7 @@ class ImagesPipeline(FilesPipeline): thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_path, thumb_image, thumb_buf - def convert_image(self, image, size=None): + def convert_image(self, image, response_body, size=None): if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) @@ -152,6 +152,9 @@ class ImagesPipeline(FilesPipeline): image = image.copy() image.thumbnail(size, Image.ANTIALIAS) + if not size and image.format == 'JPEG': + return image, response_body + buf = BytesIO() image.save(buf, 'JPEG') return image, buf From 07487dd487f7a10fbf0693e378b7b66909a67d6b Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Fri, 15 Mar 2019 23:29:53 +0530 Subject: [PATCH 0056/2083] make tests work with new convert_image --- tests/test_pipeline_images.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index a7c652959..eb3347442 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -79,28 +79,28 @@ class ImagesPipelineTestCase(unittest.TestCase): SIZE = (100, 100) # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) - im = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) + im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) + converted, buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) + thumbnail, buf = self.pipeline.convert_image(converted, buf, size=(10, 25)) self.assertEqual(thumbnail.mode, 'RGB') self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) - im = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + converted, buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) - im = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') - converted, _ = self.pipeline.convert_image(im) + converted, buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) @@ -406,7 +406,7 @@ def _create_image(format, *a, **kw): buf = io.BytesIO() Image.new(*a, **kw).save(buf, format) buf.seek(0) - return Image.open(buf) + return Image.open(buf), buf if __name__ == "__main__": From 282f24c510bdc6fff0e86ca16b7d681617af25ca Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Wed, 20 Mar 2019 18:46:22 +0530 Subject: [PATCH 0057/2083] Update form.py --- scrapy/http/request/form.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 9af2db5ff..03692f063 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ import six -from six.moves.urllib.parse import urljoin, urlencode, urlsplit, parse_qsl, urlunsplit +from six.moves.urllib.parse import urljoin, urlencode, urlsplit, urlunsplit import lxml.html from parsel.selector import create_root_node @@ -33,12 +33,7 @@ class FormRequest(Request): self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') self._set_body(form_query_str) else: - url_split = urlsplit(self.url) - formdata_keys = set(dict(parse_qsl(form_query_str)).keys()) - items = [(k, v) for k, v in parse_qsl(url_split.query) if k not in formdata_keys] - url_query_str = _urlencode(items, self.encoding) - query = (url_query_str + '&' + form_query_str) if url_query_str else form_query_str - self._set_url(urlunsplit(url_split._replace(query = query))) + self._set_url(urlunsplit(urlsplit(self.url)._replace(query = form_query_str))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, From 4c89e53e684987c61857ef810374101440d5616c Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Wed, 20 Mar 2019 18:46:25 +0530 Subject: [PATCH 0058/2083] Update test_http_request.py --- tests/test_http_request.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5c30c9785..feea54a54 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -275,26 +275,17 @@ class FormRequestTest(RequestTest): self.assertEqual(r1.body, b'') def test_formdata_overrides_querystring_duplicates(self): - #Both fragment and query in url data = (('a', 'one'), ('a', 'two'), ('b', '2')) url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) - self.assertEqual(fs[b'c'], [b'3']) - #None of both in url data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) - #Duplicate GET arguments are preserved - data={'foo' : 'bar'} - fs = _qs(self.request_class('http://example.com/?foo=1&foo=2&a=1&a=2', method='GET', formdata=data)) - self.assertEqual(fs[b'foo'], [b'bar']) - self.assertEqual(set(fs[b'a']), {b'1', b'2'}) - def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b'one': b'two', b'price': b'\xc2\xa3 100'} From 72cf190145196c3054b611eef7a0eef30ac63c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Mar 2019 14:46:07 +0100 Subject: [PATCH 0059/2083] Add a FAQ entry about name collisions --- docs/faq.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 7a0628f88..8de680816 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -319,7 +319,18 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. +Running ``runspider`` I get ``error: No spider found in file: `` +-------------------------------------------------------------------------- + +This may happen if your Scrapy project has a spider module with a name that +conflicts with the name of one of the `Python standard library modules`_, such +as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed. +See :issue:`2680`. + +.. _Python standard library modules: https://docs.python.org/py-modindex.html +.. _Python package: https://pypi.org/ + .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search +.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search \ No newline at end of file From dc8310e2929d228a224f12055a26cfef71d751cd Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Tue, 26 Mar 2019 15:42:58 +0530 Subject: [PATCH 0060/2083] changed tests --- tests/test_http_request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index feea54a54..81c1a4a9e 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -274,12 +274,13 @@ class FormRequestTest(RequestTest): r1 = self.request_class("http://www.example.com", formdata={}) self.assertEqual(r1.body, b'') - def test_formdata_overrides_querystring_duplicates(self): + def test_formdata_overrides_querystring(self): data = (('a', 'one'), ('a', 'two'), ('b', '2')) url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) + self.assertNone(fs[b'c']) data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) From 213b9eb879c4c6d8ae60ffa950688a38951b3349 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Tue, 26 Mar 2019 15:59:38 +0530 Subject: [PATCH 0061/2083] Update test_http_request.py --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 81c1a4a9e..56c7d3d94 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -280,7 +280,7 @@ class FormRequestTest(RequestTest): fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) - self.assertNone(fs[b'c']) + self.assertIsNone(fs[b'c']) data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) From ae856e8ba835d1d99510e9509a462b55a757e85f Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Tue, 26 Mar 2019 16:21:52 +0530 Subject: [PATCH 0062/2083] corrected tests --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 56c7d3d94..55fa4ad22 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -280,7 +280,7 @@ class FormRequestTest(RequestTest): fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) - self.assertIsNone(fs[b'c']) + self.assertIsNone(fs.get([b'c'])) data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) From 5f2ad5377e54ecdb3059891db4887eb09e56bfc6 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Tue, 26 Mar 2019 16:46:15 +0530 Subject: [PATCH 0063/2083] fixed typo --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 55fa4ad22..89e4a8683 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -280,7 +280,7 @@ class FormRequestTest(RequestTest): fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) - self.assertIsNone(fs.get([b'c'])) + self.assertIsNone(fs.get(b'c')) data = {'a' : '1', 'b' : '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) From ca882d8d9f094244d9f1d3476fb72fab9c230765 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Wed, 27 Mar 2019 19:10:44 +0530 Subject: [PATCH 0064/2083] include test --- tests/test_pipeline_images.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index eb3347442..efa96e146 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -80,9 +80,10 @@ class ImagesPipelineTestCase(unittest.TestCase): # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, buf = self.pipeline.convert_image(im, buf) + converted, converted_buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + self.assertEqual(converted_buf.read(), buf.read()) # check that thumbnail keep image ratio thumbnail, buf = self.pipeline.convert_image(converted, buf, size=(10, 25)) From 484bd0d22a11a04ab775ac4f72c75f3ec9050d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 22 Mar 2019 16:59:29 +0100 Subject: [PATCH 0065/2083] Allow customizing export column names --- docs/topics/exporters.rst | 35 +++++++++++++++----- docs/topics/feed-exports.rst | 17 +++------- scrapy/exporters.py | 32 +++++++++++++++--- scrapy/extensions/feedexport.py | 3 +- scrapy/settings/__init__.py | 35 +++++++++++++++++++- tests/test_exporters.py | 15 ++++++++- tests/test_feedexport.py | 57 +++++++++++++++++++++++++++++++++ 7 files changed, 165 insertions(+), 29 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index f5048d2da..42a93e459 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -190,14 +190,33 @@ BaseItemExporter .. attribute:: fields_to_export - A list with the name of the fields that will be exported, or None if you - want to export all fields. Defaults to None. + Fields to export, their order [1]_ and their output names. - Some exporters (like :class:`CsvItemExporter`) respect the order of the - fields defined in this attribute. + Possible values are: - Some exporters may require fields_to_export list in order to export the - data properly when spiders return dicts (not :class:`~Item` instances). + - ``None`` (all fields [2]_, default) + + - A list of fields:: + + ['field1', 'field2'] + + - A dict [3]_ where keys are fields and values are output names:: + + {'field1': 'Field 1', 'field2': 'Field 2'} + + .. [1] Not all exporters respect the specified field order. + .. [2] If you yield items as dicts (not :class:`Item` instances), + exporters that need to know the fields to export beforehand, like + :class:`CsvItemExporter`, only export the fields found in the + first item. + .. [3] Dicts preserve insertion order since `Python 3.7`_ + (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version + of Python, use an OrderedDict_ to enforce a specific field order. + + .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html + .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation + .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html + .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict .. attribute:: export_empty_fields @@ -286,8 +305,8 @@ CsvItemExporter Exports Items in CSV format to the given file-like object. If the :attr:`fields_to_export` attribute is set, it will be used to define the - CSV columns and their order. The :attr:`export_empty_fields` attribute has - no effect on this exporter. + CSV columns, their order and their column names. The + :attr:`export_empty_fields` attribute has no effect on this exporter. :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index cf70b8aca..968cb8884 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -56,7 +56,7 @@ CSV * :setting:`FEED_FORMAT`: ``csv`` * Exporter used: :class:`~scrapy.exporters.CsvItemExporter` - * To specify columns to export and their order use + * To specify columns to export, their order and their column names, use :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this option, but it is important for CSV because unlike many other export formats CSV uses a fixed header. @@ -259,18 +259,9 @@ FEED_EXPORT_FIELDS Default: ``None`` -A list of fields to export, optional. -Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``. - -Use FEED_EXPORT_FIELDS option to define fields to export and their order. - -When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields -defined in dicts or :class:`~.Item` subclasses a spider is yielding. - -If an exporter requires a fixed set of fields (this is the case for -:ref:`CSV ` export format) and FEED_EXPORT_FIELDS -is empty or None, then Scrapy tries to infer field names from the -exported data - currently it uses field names from the first item. +Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their +order and their output names. See :attr:`BaseItemExporter.fields_to_export +` for more information. .. setting:: FEED_EXPORT_INDENT diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 695c74fec..c05acaca5 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -2,6 +2,7 @@ Item Exporters are used to export/serialize items into different formats. """ +from collections import Mapping import csv import io import sys @@ -64,6 +65,14 @@ class BaseItemExporter(object): field_iter = six.iterkeys(item.fields) else: field_iter = six.iterkeys(item) + elif isinstance(self.fields_to_export, Mapping): + if include_empty: + field_iter = self.fields_to_export.items() + else: + field_iter = ( + (x, y) for x, y in self.fields_to_export.items() + if x in item + ) else: if include_empty: field_iter = self.fields_to_export @@ -71,13 +80,22 @@ class BaseItemExporter(object): field_iter = (x for x in self.fields_to_export if x in item) for field_name in field_iter: - if field_name in item: - field = {} if isinstance(item, dict) else item.fields[field_name] - value = self.serialize_field(field, field_name, item[field_name]) + if isinstance(field_name, six.string_types): + item_field, output_field = field_name, field_name + else: + item_field, output_field = field_name + if item_field in item: + if isinstance(item, dict): + field = {} + else: + field = item.fields[item_field] + value = self.serialize_field( + field, output_field, item[item_field] + ) else: value = default_value - yield field_name, value + yield output_field, value class JsonLinesItemExporter(BaseItemExporter): @@ -259,7 +277,11 @@ class CsvItemExporter(BaseItemExporter): else: # use fields declared in Item self.fields_to_export = list(item.fields.keys()) - row = list(self._build_row(self.fields_to_export)) + if isinstance(self.fields_to_export, Mapping): + fields = self.fields_to_export.values() + else: + fields = self.fields_to_export + row = list(self._build_row(fields)) self.csv_writer.writerow(row) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index b2f7267a2..1ed476d83 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -201,7 +201,8 @@ class FeedExporter(object): raise NotConfigured self.store_empty = settings.getbool('FEED_STORE_EMPTY') self._exporting = False - self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None + self.export_fields = settings.getdictorlist('FEED_EXPORT_FIELDS') + self.export_fields = self.export_fields or None self.indent = None if settings.get('FEED_EXPORT_INDENT') is not None: self.indent = settings.getint('FEED_EXPORT_INDENT') diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 14c93bef2..69b324e84 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,7 +1,7 @@ import six import json import copy -from collections import MutableMapping +from collections import MutableMapping, OrderedDict from importlib import import_module from pprint import pformat @@ -198,6 +198,39 @@ class BaseSettings(MutableMapping): value = json.loads(value) return dict(value) + def getdictorlist(self, name, default=None): + """Get a setting value as either an ``OrderedDict`` or a list. + + If the setting is already a dict or a list, a copy of it will be + returned. + + If it is a string it will be evaluated as JSON, or as a comma-separated + list of strings as a fallback. + + For example, settings populated through environment variables will + return: + + - ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to + ``'{"key1": "value1", "key2": "value2"}'`` + + - ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'`` + + :param name: the setting name + :type name: string + + :param default: the value to return if no setting is found + :type default: any + """ + value = self.get(name, default) + if value is None: + return {} + if isinstance(value, six.string_types): + try: + return json.loads(value, object_pairs_hook=OrderedDict) + except ValueError: + return value.split(',') + return copy.deepcopy(value) + def getwithbase(self, name): """Get a composition of a dictionary-like setting and its `_BASE` counterpart. diff --git a/tests/test_exporters.py b/tests/test_exporters.py index cd72c661a..1b3dc14a1 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1,4 +1,7 @@ +# -*- coding:utf-8 -*- + from __future__ import absolute_import +from collections import OrderedDict import re import json import marshal @@ -83,6 +86,14 @@ class BaseItemExporterTest(unittest.TestCase): assert isinstance(name, six.text_type) self.assertEqual(name, u'John\xa3') + ie = self._get_exporter( + fields_to_export=OrderedDict([('name', u'å稱')]) + ) + self.assertEqual( + list(ie._get_serialized_fields(self.i)), + [(u'å稱', u'John\xa3')] + ) + def test_field_custom_serializer(self): def custom_serializer(value): return str(int(value) + 2) @@ -214,6 +225,7 @@ class MarshalItemExporterTest(BaseItemExporterTest): class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): + self.output = tempfile.TemporaryFile() return CsvItemExporter(self.output, **kwargs) def assertCsvEqual(self, first, second, msg=None): @@ -224,7 +236,8 @@ class CsvItemExporterTest(BaseItemExporterTest): return self.assertEqual(csvsplit(first), csvsplit(second), msg) def _check_output(self): - self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n') + self.output.seek(0) + self.assertCsvEqual(to_unicode(self.output.read()), u'age,name\r\n22,John\xa3\r\n') def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index eef0384cf..4f72c0ff4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from collections import OrderedDict import os import csv import json @@ -578,6 +579,62 @@ class FeedExportTest(unittest.TestCase): yield self.assertExported(items, header, rows, settings=settings, ordered=True) + # fields may be defined as a comma-separated list + header = ["foo", "baz", "hello"] + settings = {'FEED_EXPORT_FIELDS': ",".join(header)} + rows = [ + {'foo': 'bar1', 'baz': '', 'hello': ''}, + {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, + {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, + {'foo': '', 'baz': '', 'hello': 'world4'}, + ] + yield self.assertExported(items, header, rows, + settings=settings, ordered=True) + + # fields may also be defined as a JSON array + header = ["foo", "baz", "hello"] + settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} + rows = [ + {'foo': 'bar1', 'baz': '', 'hello': ''}, + {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, + {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, + {'foo': '', 'baz': '', 'hello': 'world4'}, + ] + yield self.assertExported(items, header, rows, + settings=settings, ordered=True) + + # custom output field names can be specified + header = OrderedDict(( + ("foo", "Foo"), + ("baz", "Baz"), + ("hello", "Hello"), + )) + settings = {'FEED_EXPORT_FIELDS': header} + rows = [ + {'Foo': 'bar1', 'Baz': '', 'Hello': ''}, + {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'}, + {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''}, + {'Foo': '', 'Baz': '', 'Hello': 'world4'}, + ] + yield self.assertExported(items, list(header.values()), rows, + settings=settings, ordered=True) + + # custom output field names can be specified as a JSON object + header = OrderedDict(( + ("foo", "Foo"), + ("baz", "Baz"), + ("hello", "Hello"), + )) + settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} + rows = [ + {'Foo': 'bar1', 'Baz': '', 'Hello': ''}, + {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'}, + {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''}, + {'Foo': '', 'Baz': '', 'Hello': 'world4'}, + ] + yield self.assertExported(items, list(header.values()), rows, + settings=settings, ordered=True) + @defer.inlineCallbacks def test_export_dicts(self): # When dicts are used, only keys from the first row are used as From 398639a0bfe749eee07e51249f04b2c6c93eab73 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Mon, 8 Apr 2019 12:27:36 +0530 Subject: [PATCH 0066/2083] fix test --- scrapy/pipelines/images.py | 5 +++-- tests/test_pipeline_images.py | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index add606a2e..746244dab 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -132,7 +132,7 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - thumb_image, thumb_buf = self.convert_image(image, size) + thumb_image, thumb_buf = self.convert_image(image, buf, size) yield thumb_path, thumb_image, thumb_buf def convert_image(self, image, response_body, size=None): @@ -153,7 +153,8 @@ class ImagesPipeline(FilesPipeline): image.thumbnail(size, Image.ANTIALIAS) if not size and image.format == 'JPEG': - return image, response_body + buf = BytesIO(response_body.read()) + return image, buf buf = BytesIO() image.save(buf, 'JPEG') diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index efa96e146..dde0fa030 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -83,6 +83,47 @@ class ImagesPipelineTestCase(unittest.TestCase): converted, converted_buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + + # check that we don't convert JPEGs again + buf = io.BytesIO((b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00' + b'\xff\xdb\x00C\x00\x08\x06\x06\x07\x06\x05\x08\x07\x07\x07\t\t\x08\n\x0c' + b'\x14\r\x0c\x0b\x0b\x0c\x19\x12\x13\x0f\x14\x1d\x1a\x1f\x1e\x1d\x1a\x1c\x1c ' + b'$.\' ",#\x1c\x1c(7),01444\x1f\'9=82<.342\xff\xdb\x00C\x01\t\t\t\x0c\x0b\x0c' + b'\x18\r\r\x182!\x1c!222222222222222222222222222222222222222222222222' + b'22\xff\xc0\x00\x11\x08\x00\x14\x00\x14\x03\x01"\x00\x02\x11\x01\x03\x11' + b'\x01\xff\xc4\x00\x1f\x00\x00\x01\x05\x01\x01\x01\x01\x01\x01\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4' + b'\x00\xb5\x10\x00\x02\x01\x03\x03\x02\x04\x03\x05\x05\x04\x04\x00' + b'\x00\x01}\x01\x02\x03\x00\x04\x11\x05\x12!1A\x06\x13Qa\x07"q\x142\x81' + b"\x91\xa1\x08#B\xb1\xc1\x15R\xd1\xf0$3br\x82\t\n\x16\x17\x18\x19\x1a%&'()*456" + b'789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x83\x84\x85\x86\x87\x88\x89\x8a' + b'\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6\xa7\xa8' + b'\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4\xc5\xc6' + b'\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe1\xe2\xe3' + b'\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9' + b'\xfa\xff\xc4\x00\x1f\x01\x00\x03\x01\x01\x01\x01\x01\x01\x01\x01' + b'\x01\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4' + b'\x00\xb5\x11\x00\x02\x01\x02\x04\x04\x03\x04\x07\x05\x04\x04\x00' + b'\x01\x02w\x00\x01\x02\x03\x11\x04\x05!1\x06\x12AQ\x07aq\x13"2\x81\x08' + b"\x14B\x91\xa1\xb1\xc1\t#3R\xf0\x15br\xd1\n\x16$4\xe1%\xf1\x17\x18\x19\x1a&'" + b'()*56789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x82\x83\x84\x85\x86\x87\x88' + b'\x89\x8a\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6' + b'\xa7\xa8\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4' + b'\xc5\xc6\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe2' + b'\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9' + b'\xfa\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00\xbb' + b'\xe2\x1b\xcb\x88\xe4\x90\t\x0e1\xd2\xb9\xab\x1dF\xe6\xda\xe0\xb4l\xd9' + b'5>\xaf\xac\xc7y}$k\x92\x03u\xaaL\xeb\x18VL\x9fZ+\xb9\xb9EBV]Ow\x0f\x8a\xc0' + b'\xac\x14\xbd\xa2NG_\xa5x\x8esg\xfb\xd9\x8e\xed\xc79\xa2\xa1\xd0\xf4\x8bk' + b'\x9d5e\xdeN\xe6=\xa8\xae\xa9V\xc2\xb6|\xdf2\xeep:w\xfc\x84d\x04g\x9e\xf5\xd3' + b'L\xa9\x0c\x1f*/>\xa2\x8a+\xca\x93z\x1c\x15]\xa9\xc6\xc4Q\xea\x97V\xca' + b'c\x85\xc2\xaes\x80(\xa2\x8a\xe3\x92W<\xdb\x9f\xff\xd9')) + im = Image.open(buf) + buf.seek(0) + converted, converted_buf = self.pipeline.convert_image(im, buf) + converted_buf.seek(0) + buf.seek(0) + self.assertEqual(im.format, "JPEG") self.assertEqual(converted_buf.read(), buf.read()) # check that thumbnail keep image ratio From c6769d6887b5c311c9083a5ac0349e81d1a2aea7 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sat, 13 Apr 2019 10:10:08 +0530 Subject: [PATCH 0067/2083] make suggested changes --- scrapy/pipelines/images.py | 6 ++--- tests/test_pipeline_images.py | 42 +---------------------------------- 2 files changed, 3 insertions(+), 45 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 746244dab..3450ee721 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -151,10 +151,8 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - - if not size and image.format == 'JPEG': - buf = BytesIO(response_body.read()) - return image, buf + elif image.format == 'JPEG': + return image, response_body buf = BytesIO() image.save(buf, 'JPEG') diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index dde0fa030..b419039b3 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -83,48 +83,8 @@ class ImagesPipelineTestCase(unittest.TestCase): converted, converted_buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) - # check that we don't convert JPEGs again - buf = io.BytesIO((b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00' - b'\xff\xdb\x00C\x00\x08\x06\x06\x07\x06\x05\x08\x07\x07\x07\t\t\x08\n\x0c' - b'\x14\r\x0c\x0b\x0b\x0c\x19\x12\x13\x0f\x14\x1d\x1a\x1f\x1e\x1d\x1a\x1c\x1c ' - b'$.\' ",#\x1c\x1c(7),01444\x1f\'9=82<.342\xff\xdb\x00C\x01\t\t\t\x0c\x0b\x0c' - b'\x18\r\r\x182!\x1c!222222222222222222222222222222222222222222222222' - b'22\xff\xc0\x00\x11\x08\x00\x14\x00\x14\x03\x01"\x00\x02\x11\x01\x03\x11' - b'\x01\xff\xc4\x00\x1f\x00\x00\x01\x05\x01\x01\x01\x01\x01\x01\x00' - b'\x00\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4' - b'\x00\xb5\x10\x00\x02\x01\x03\x03\x02\x04\x03\x05\x05\x04\x04\x00' - b'\x00\x01}\x01\x02\x03\x00\x04\x11\x05\x12!1A\x06\x13Qa\x07"q\x142\x81' - b"\x91\xa1\x08#B\xb1\xc1\x15R\xd1\xf0$3br\x82\t\n\x16\x17\x18\x19\x1a%&'()*456" - b'789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x83\x84\x85\x86\x87\x88\x89\x8a' - b'\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6\xa7\xa8' - b'\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4\xc5\xc6' - b'\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe1\xe2\xe3' - b'\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9' - b'\xfa\xff\xc4\x00\x1f\x01\x00\x03\x01\x01\x01\x01\x01\x01\x01\x01' - b'\x01\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4' - b'\x00\xb5\x11\x00\x02\x01\x02\x04\x04\x03\x04\x07\x05\x04\x04\x00' - b'\x01\x02w\x00\x01\x02\x03\x11\x04\x05!1\x06\x12AQ\x07aq\x13"2\x81\x08' - b"\x14B\x91\xa1\xb1\xc1\t#3R\xf0\x15br\xd1\n\x16$4\xe1%\xf1\x17\x18\x19\x1a&'" - b'()*56789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x82\x83\x84\x85\x86\x87\x88' - b'\x89\x8a\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6' - b'\xa7\xa8\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4' - b'\xc5\xc6\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe2' - b'\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9' - b'\xfa\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00\xbb' - b'\xe2\x1b\xcb\x88\xe4\x90\t\x0e1\xd2\xb9\xab\x1dF\xe6\xda\xe0\xb4l\xd9' - b'5>\xaf\xac\xc7y}$k\x92\x03u\xaaL\xeb\x18VL\x9fZ+\xb9\xb9EBV]Ow\x0f\x8a\xc0' - b'\xac\x14\xbd\xa2NG_\xa5x\x8esg\xfb\xd9\x8e\xed\xc79\xa2\xa1\xd0\xf4\x8bk' - b'\x9d5e\xdeN\xe6=\xa8\xae\xa9V\xc2\xb6|\xdf2\xeep:w\xfc\x84d\x04g\x9e\xf5\xd3' - b'L\xa9\x0c\x1f*/>\xa2\x8a+\xca\x93z\x1c\x15]\xa9\xc6\xc4Q\xea\x97V\xca' - b'c\x85\xc2\xaes\x80(\xa2\x8a\xe3\x92W<\xdb\x9f\xff\xd9')) - im = Image.open(buf) - buf.seek(0) - converted, converted_buf = self.pipeline.convert_image(im, buf) - converted_buf.seek(0) - buf.seek(0) - self.assertEqual(im.format, "JPEG") - self.assertEqual(converted_buf.read(), buf.read()) + self.assertEqual(converted_buf, buf) # check that thumbnail keep image ratio thumbnail, buf = self.pipeline.convert_image(converted, buf, size=(10, 25)) From 6039b66f42fe2c6e7708febb70939952cedaedd0 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sat, 13 Apr 2019 10:17:26 +0530 Subject: [PATCH 0068/2083] aesthetic changes --- tests/test_pipeline_images.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index b419039b3..0a2153ddb 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -87,14 +87,14 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted_buf, buf) # check that thumbnail keep image ratio - thumbnail, buf = self.pipeline.convert_image(converted, buf, size=(10, 25)) + thumbnail, _ = self.pipeline.convert_image(converted, converted_buf, size=(10, 25)) self.assertEqual(thumbnail.mode, 'RGB') self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, buf = self.pipeline.convert_image(im, buf) + converted, _ = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) @@ -102,7 +102,7 @@ class ImagesPipelineTestCase(unittest.TestCase): COLOUR = (0, 127, 255, 50) im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') - converted, buf = self.pipeline.convert_image(im, buf) + converted, _ = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) From 2a6bcdb413da0a4202fe905279d46d581368769d Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Mon, 29 Apr 2019 21:21:02 +0530 Subject: [PATCH 0069/2083] makes fix backward compatible --- scrapy/pipelines/images.py | 45 ++++++++++++++++++++++++++++++----- tests/test_pipeline_images.py | 8 +++---- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 3450ee721..ca8ac7b83 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -126,16 +126,42 @@ class ImagesPipeline(FilesPipeline): if width < self.min_width or height < self.min_height: raise ImageException("Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) + + def _is_convert_image_overriden(): + import inspect + if six.PY2: + convert_image_signature = inspect.getargspec(self.convert_image) + elif six.PY3: + convert_image_signature = inspect.getfullargspec(self.convert_image) + if 'response_body' not in convert_image_signature.args: + return True + return False - image, buf = self.convert_image(orig_image, BytesIO(response.body)) + def _warn(): + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ImagesPipeline.convert_image() method overriden in a incompatible way, ' + 'overriden method does not accept response_body attribute.', + category=ScrapyDeprecationWarning, stacklevel=1) + + convert_image_overriden = _is_convert_image_overriden() + if convert_image_overriden: + _warn() + image, buf = self.convert_image(orig_image) + else: + image, buf = self.convert_image(orig_image, response_body=BytesIO(response.body)) yield path, image, buf for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - thumb_image, thumb_buf = self.convert_image(image, buf, size) + if convert_image_overriden: + _warn() + thumb_image, thumb_buf = self.convert_image(image, size) + else: + thumb_image, thumb_buf = self.convert_image(image, size, buf) yield thumb_path, thumb_image, thumb_buf - def convert_image(self, image, response_body, size=None): + def convert_image(self, image, size=None, response_body=None): if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) @@ -151,9 +177,16 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - elif image.format == 'JPEG': - return image, response_body - + else: + if not response_body: + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, ' + 'method called without response_body attribute.', + category=ScrapyDeprecationWarning, stacklevel=1) + elif image.format == 'JPEG': + return image, response_body + buf = BytesIO() image.save(buf, 'JPEG') return image, buf diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 0a2153ddb..3fe71589d 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -80,21 +80,21 @@ class ImagesPipelineTestCase(unittest.TestCase): # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, converted_buf = self.pipeline.convert_image(im, buf) + converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) # check that we don't convert JPEGs again self.assertEqual(converted_buf, buf) # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, converted_buf, size=(10, 25)) + thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf) self.assertEqual(thumbnail.mode, 'RGB') self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im, buf) + converted, _ = self.pipeline.convert_image(im, response_body=buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) @@ -102,7 +102,7 @@ class ImagesPipelineTestCase(unittest.TestCase): COLOUR = (0, 127, 255, 50) im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') - converted, _ = self.pipeline.convert_image(im, buf) + converted, _ = self.pipeline.convert_image(im, response_body=buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) From 33925a77610293c44a169efd1c274239a75ba968 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Tue, 7 May 2019 15:44:21 +0530 Subject: [PATCH 0070/2083] test for deprecation warning --- scrapy/pipelines/images.py | 20 +++++------ tests/test_pipeline_images.py | 63 ++++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 36 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index ca8ac7b83..9776817bc 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -141,7 +141,7 @@ class ImagesPipeline(FilesPipeline): from scrapy.exceptions import ScrapyDeprecationWarning import warnings warnings.warn('ImagesPipeline.convert_image() method overriden in a incompatible way, ' - 'overriden method does not accept response_body attribute.', + 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) convert_image_overriden = _is_convert_image_overriden() @@ -162,6 +162,13 @@ class ImagesPipeline(FilesPipeline): yield thumb_path, thumb_image, thumb_buf def convert_image(self, image, size=None, response_body=None): + if not response_body: + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, ' + 'method called without response_body argument.', + category=ScrapyDeprecationWarning, stacklevel=1) + if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) @@ -177,15 +184,8 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - else: - if not response_body: - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings - warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, ' - 'method called without response_body attribute.', - category=ScrapyDeprecationWarning, stacklevel=1) - elif image.format == 'JPEG': - return image, response_body + elif response_body and image.format == 'JPEG': + return image, response_body buf = BytesIO() image.save(buf, 'JPEG') diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 3fe71589d..651691862 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -76,37 +76,48 @@ class ImagesPipelineTestCase(unittest.TestCase): 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') def test_convert_image(self): - SIZE = (100, 100) - # straigh forward case: RGB and JPEG + # tests for old API + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + + SIZE = (100, 100) + # straigh forward case: RGB and JPEG + COLOUR = (0, 127, 255) + im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) + converted, converted_buf = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + + # check that thumbnail keep image ratio + thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) + self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.size, (10, 10)) + + # transparency case: RGBA and PNG + COLOUR = (0, 127, 255, 50) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # transparency case with palette: P and PNG + COLOUR = (0, 127, 255, 50) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im = im.convert('P') + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # ensure that we recieved deprecation warnings + self.assertTrue(len(w) >= 4) + + # tests for new API + # check that we don't convert JPEGs again COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) - # check that we don't convert JPEGs again self.assertEqual(converted_buf, buf) - # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf) - self.assertEqual(thumbnail.mode, 'RGB') - self.assertEqual(thumbnail.size, (10, 10)) - - # transparency case: RGBA and PNG - COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # transparency case with palette: P and PNG - COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - im = im.convert('P') - converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - class DeprecatedImagesPipeline(ImagesPipeline): def file_key(self, url): return self.image_key(url) From 881bade2c1b2d5842f7d0bdf2648455996f814b7 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Tue, 7 May 2019 16:12:26 +0530 Subject: [PATCH 0071/2083] tests for new API --- tests/test_pipeline_images.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 651691862..1dfca5c11 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -79,12 +79,11 @@ class ImagesPipelineTestCase(unittest.TestCase): # tests for old API with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') - SIZE = (100, 100) # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) - im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, converted_buf = self.pipeline.convert_image(im) + im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) @@ -95,14 +94,14 @@ class ImagesPipelineTestCase(unittest.TestCase): # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') converted, _ = self.pipeline.convert_image(im) self.assertEqual(converted.mode, 'RGB') @@ -112,12 +111,36 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertTrue(len(w) >= 4) # tests for new API - # check that we don't convert JPEGs again + SIZE = (100, 100) + # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + # check that we don't convert JPEGs again self.assertEqual(converted_buf, buf) + # check that thumbnail keep image ratio + thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf) + self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.size, (10, 10)) + + # transparency case: RGBA and PNG + COLOUR = (0, 127, 255, 50) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im, response_body=buf) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # transparency case with palette: P and PNG + COLOUR = (0, 127, 255, 50) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im = im.convert('P') + converted, _ = self.pipeline.convert_image(im, response_body=buf) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + class DeprecatedImagesPipeline(ImagesPipeline): def file_key(self, url): return self.image_key(url) From 653ac3eebe96bdfff8128cdb1e500728215486a2 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Thu, 16 May 2019 06:20:39 +0000 Subject: [PATCH 0072/2083] makes suggested changes --- scrapy/pipelines/images.py | 1 - tests/test_pipeline_images.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 9776817bc..b95383b3b 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -155,7 +155,6 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) if convert_image_overriden: - _warn() thumb_image, thumb_buf = self.convert_image(image, size) else: thumb_image, thumb_buf = self.convert_image(image, size, buf) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 1dfca5c11..ba79dd6bd 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -108,7 +108,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # ensure that we recieved deprecation warnings - self.assertTrue(len(w) >= 4) + self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a incompatible way' in str(warning.message)]) == 4) # tests for new API SIZE = (100, 100) From 2994b624e03ec9a2cba2c024eacf549329c056de Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Fri, 17 May 2019 12:14:43 +0000 Subject: [PATCH 0073/2083] makes suggested changes --- scrapy/pipelines/images.py | 21 +++++---------------- tests/test_pipeline_images.py | 2 +- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index b95383b3b..fd34c9df6 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -15,7 +15,7 @@ except ImportError: from PIL import Image from scrapy.utils.misc import md5sum -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, get_func_args from scrapy.http import Request from scrapy.settings import Settings from scrapy.exceptions import DropItem @@ -127,25 +127,14 @@ class ImagesPipeline(FilesPipeline): raise ImageException("Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) - def _is_convert_image_overriden(): - import inspect - if six.PY2: - convert_image_signature = inspect.getargspec(self.convert_image) - elif six.PY3: - convert_image_signature = inspect.getfullargspec(self.convert_image) - if 'response_body' not in convert_image_signature.args: - return True - return False - def _warn(): from scrapy.exceptions import ScrapyDeprecationWarning import warnings - warnings.warn('ImagesPipeline.convert_image() method overriden in a incompatible way, ' + warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) - convert_image_overriden = _is_convert_image_overriden() - if convert_image_overriden: + if 'response_body' not in get_func_args(self.convert_image): _warn() image, buf = self.convert_image(orig_image) else: @@ -154,7 +143,7 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - if convert_image_overriden: + if 'response_body' not in get_func_args(self.convert_image): thumb_image, thumb_buf = self.convert_image(image, size) else: thumb_image, thumb_buf = self.convert_image(image, size, buf) @@ -164,7 +153,7 @@ class ImagesPipeline(FilesPipeline): if not response_body: from scrapy.exceptions import ScrapyDeprecationWarning import warnings - warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, ' + warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ' 'method called without response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index ba79dd6bd..ec0c87264 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -108,7 +108,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # ensure that we recieved deprecation warnings - self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a incompatible way' in str(warning.message)]) == 4) + self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a deprecated way' in str(warning.message)]) == 4) # tests for new API SIZE = (100, 100) From c8e28ec194b3730e7954a18526ac77cb44138feb Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Thu, 23 May 2019 15:04:21 +0530 Subject: [PATCH 0074/2083] makes suggested changes --- scrapy/pipelines/images.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index fd34c9df6..a3f5f9292 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -134,7 +134,8 @@ class ImagesPipeline(FilesPipeline): 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) - if 'response_body' not in get_func_args(self.convert_image): + convert_image_overriden = 'response_body' not in get_func_args(self.convert_image) + if convert_image_overriden: _warn() image, buf = self.convert_image(orig_image) else: @@ -143,7 +144,7 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - if 'response_body' not in get_func_args(self.convert_image): + if convert_image_overriden: thumb_image, thumb_buf = self.convert_image(image, size) else: thumb_image, thumb_buf = self.convert_image(image, size, buf) From 90fdefcbca89d0ef2cc81955b9fd8b8af1dff392 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sat, 25 May 2019 19:11:48 +0530 Subject: [PATCH 0075/2083] cache if convert_image has deprecated signature --- scrapy/pipelines/images.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index a3f5f9292..f709c5057 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -84,6 +84,8 @@ class ImagesPipeline(FilesPipeline): resolve('IMAGES_THUMBS'), self.THUMBS ) + self._deprecated_convert_image = None + @classmethod def from_settings(cls, settings): s3store = cls.STORE_SCHEMES['s3'] @@ -126,17 +128,17 @@ class ImagesPipeline(FilesPipeline): if width < self.min_width or height < self.min_height: raise ImageException("Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) - - def _warn(): - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings - warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' - 'overriden method does not accept response_body argument.', - category=ScrapyDeprecationWarning, stacklevel=1) - convert_image_overriden = 'response_body' not in get_func_args(self.convert_image) - if convert_image_overriden: - _warn() + if self._deprecated_convert_image is None: + self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image) + if self._deprecated_convert_image: + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' + 'overriden method does not accept response_body argument.', + category=ScrapyDeprecationWarning, stacklevel=1) + + if self._deprecated_convert_image: image, buf = self.convert_image(orig_image) else: image, buf = self.convert_image(orig_image, response_body=BytesIO(response.body)) @@ -144,7 +146,7 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - if convert_image_overriden: + if self._deprecated_convert_image: thumb_image, thumb_buf = self.convert_image(image, size) else: thumb_image, thumb_buf = self.convert_image(image, size, buf) From 20719bac5cdc4898e9f01fcf4f92aa781a9d0ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Dec 2019 15:09:43 +0100 Subject: [PATCH 0076/2083] Fix import error --- scrapy/settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index c1fff4d95..98421be18 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -224,7 +224,7 @@ class BaseSettings(MutableMapping): value = self.get(name, default) if value is None: return {} - if isinstance(value, six.string_types): + if isinstance(value, str): try: return json.loads(value, object_pairs_hook=OrderedDict) except ValueError: From 6d6243afbb16cb5b7d401a0b2ea7a174b7be71b8 Mon Sep 17 00:00:00 2001 From: leobalestri <33645316+leobalestri@users.noreply.github.com> Date: Sun, 16 Feb 2020 23:45:41 -0800 Subject: [PATCH 0077/2083] Update install.rst Minor grammar and typo fixes --- docs/intro/install.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 51b41b4d7..a08dedbd0 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -186,7 +186,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to successfully install Scrapy and its dependencies. Here are some proposed solutions: -* *(Recommended)* **Don't** use system python, install a new, updated version +* *(Recommended)* **Don't** use system python. Install a new, updated version that doesn't conflict with the rest of your system. Here's how to do it using the `homebrew`_ package manager: @@ -231,9 +231,9 @@ PyPy We recommend using the latest PyPy version. The version tested is 5.9.0. For PyPy3, only Linux installation was tested. -Most scrapy dependencides now have binary wheels for CPython, but not for PyPy. -This means that these dependecies will be built during installation. -On OS X, you are likely to face an issue with building Cryptography dependency, +Most scrapy dependencies now have binary wheels for CPython, but not for PyPy. +This means that these dependencies will be built during installation. +On OS X, you are likely to face an issue with building Cryptography dependency. The solution to this problem is described `here `_, that is to ``brew install openssl`` and then export the flags that this command From 5834088e670d93b2a63ad8afb258e687af0a9b88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 18 Feb 2020 14:18:15 +0100 Subject: [PATCH 0078/2083] Apply feedback --- scrapy/settings/__init__.py | 5 +- tests/test_feedexport.py | 109 ++++++++++++++++++------------------ 2 files changed, 56 insertions(+), 58 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 98421be18..6f5b1ef97 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -207,8 +207,7 @@ class BaseSettings(MutableMapping): If it is a string it will be evaluated as JSON, or as a comma-separated list of strings as a fallback. - For example, settings populated through environment variables will - return: + For example, settings populated from the command line will return: - ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to ``'{"key1": "value1", "key2": "value2"}'`` @@ -223,7 +222,7 @@ class BaseSettings(MutableMapping): """ value = self.get(name, default) if value is None: - return {} + return OrderedDict() if isinstance(value, str): try: return json.loads(value, object_pairs_hook=OrderedDict) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 291c47702..781cdc543 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,6 +5,7 @@ import warnings import tempfile import shutil import string +import sys from collections import OrderedDict from io import BytesIO from pathlib import Path @@ -12,6 +13,7 @@ from unittest import mock from urllib.parse import urljoin, urlparse, quote from urllib.request import pathname2url +import pytest from zope.interface.verify import verifyObject from twisted.trial import unittest from twisted.internet import defer @@ -590,78 +592,75 @@ class FeedExportTest(unittest.TestCase): yield self.assertExportedCsv(items, header, rows_csv, ordered=False) yield self.assertExportedJsonLines(items, rows_jl) - # edge case: FEED_EXPORT_FIELDS==[] means the same as default None + @defer.inlineCallbacks + def test_export_items_empty_field_list(self): + # FEED_EXPORT_FIELDS==[] means the same as default None + items = [{'foo': 'bar'}] + header = ["foo"] + rows = [{'foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': []} - yield self.assertExportedCsv(items, header, rows_csv, ordered=False) - yield self.assertExportedJsonLines(items, rows_jl, settings) + yield self.assertExportedCsv(items, header, rows, ordered=False) + yield self.assertExportedJsonLines(items, rows, settings) - # it is possible to override fields using FEED_EXPORT_FIELDS - header = ["foo", "baz", "hello"] + @defer.inlineCallbacks + def test_export_items_field_list(self): + items = [{'foo': 'bar'}] + header = ["foo", "baz"] + rows = [{'foo': 'bar', 'baz': ''}] settings = {'FEED_EXPORT_FIELDS': header} - rows = [ - {'foo': 'bar1', 'baz': '', 'hello': ''}, - {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, - {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, - {'foo': '', 'baz': '', 'hello': 'world4'}, - ] - yield self.assertExported(items, header, rows, - settings=settings, ordered=True) + yield self.assertExported(items, header, rows, settings=settings) - # fields may be defined as a comma-separated list - header = ["foo", "baz", "hello"] + @defer.inlineCallbacks + def test_export_items_comma_separated_field_list(self): + items = [{'foo': 'bar'}] + header = ["foo", "baz"] + rows = [{'foo': 'bar', 'baz': ''}] settings = {'FEED_EXPORT_FIELDS': ",".join(header)} - rows = [ - {'foo': 'bar1', 'baz': '', 'hello': ''}, - {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, - {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, - {'foo': '', 'baz': '', 'hello': 'world4'}, - ] - yield self.assertExported(items, header, rows, - settings=settings, ordered=True) + yield self.assertExported(items, header, rows, settings=settings) - # fields may also be defined as a JSON array - header = ["foo", "baz", "hello"] + @defer.inlineCallbacks + def test_export_items_json_field_list(self): + items = [{'foo': 'bar'}] + header = ["foo", "baz"] + rows = [{'foo': 'bar', 'baz': ''}] settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} - rows = [ - {'foo': 'bar1', 'baz': '', 'hello': ''}, - {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, - {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, - {'foo': '', 'baz': '', 'hello': 'world4'}, - ] - yield self.assertExported(items, header, rows, - settings=settings, ordered=True) + yield self.assertExported(items, header, rows, settings=settings) - # custom output field names can be specified + @defer.inlineCallbacks + def test_export_items_field_names(self): + items = [{'foo': 'bar'}] header = OrderedDict(( ("foo", "Foo"), - ("baz", "Baz"), - ("hello", "Hello"), )) + rows = [{'Foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': header} - rows = [ - {'Foo': 'bar1', 'Baz': '', 'Hello': ''}, - {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'}, - {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''}, - {'Foo': '', 'Baz': '', 'Hello': 'world4'}, - ] yield self.assertExported(items, list(header.values()), rows, - settings=settings, ordered=True) + settings=settings) - # custom output field names can be specified as a JSON object + @pytest.mark.skipif(sys.version_info < (3, 7), + reason='Only official in Python 3.7+') + @defer.inlineCallbacks + def test_export_items_dict_field_names(self): + items = [{'foo': 'bar'}] + header = { + 'baz': 'Baz', + 'foo': 'Foo', + } + rows = [{'Baz': '', 'Foo': 'bar'}] + settings = {'FEED_EXPORT_FIELDS': header} + yield self.assertExported(items, ['Baz', 'Foo'], rows, + settings=settings) + + @defer.inlineCallbacks + def test_export_items_json_field_names(self): + items = [{'foo': 'bar'}] header = OrderedDict(( ("foo", "Foo"), - ("baz", "Baz"), - ("hello", "Hello"), )) + rows = [{'Foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} - rows = [ - {'Foo': 'bar1', 'Baz': '', 'Hello': ''}, - {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'}, - {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''}, - {'Foo': '', 'Baz': '', 'Hello': 'world4'}, - ] yield self.assertExported(items, list(header.values()), rows, - settings=settings, ordered=True) + settings=settings) @defer.inlineCallbacks def test_export_dicts(self): @@ -697,7 +696,7 @@ class FeedExportTest(unittest.TestCase): {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} ] yield self.assertExported(items, ['foo', 'baz', 'egg'], rows, - settings=settings, ordered=True) + settings=settings) # export a subset of columns settings = {'FEED_EXPORT_FIELDS': 'egg,baz'} @@ -706,7 +705,7 @@ class FeedExportTest(unittest.TestCase): {'egg': 'spam2', 'baz': 'quux2'} ] yield self.assertExported(items, ['egg', 'baz'], rows, - settings=settings, ordered=True) + settings=settings) @defer.inlineCallbacks def test_export_encoding(self): From 231c9ddef8be9d749dcc2684f07ea9bb8bd02aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 28 Feb 2020 18:50:45 +0100 Subject: [PATCH 0079/2083] Update docs/intro/install.rst --- docs/intro/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index a08dedbd0..b71379e4d 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -186,7 +186,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to successfully install Scrapy and its dependencies. Here are some proposed solutions: -* *(Recommended)* **Don't** use system python. Install a new, updated version +* *(Recommended)* **Don't** use system Python. Install a new, updated version that doesn't conflict with the rest of your system. Here's how to do it using the `homebrew`_ package manager: From 1d77eac950966463faee411b6196f197a8d32d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 16 Apr 2020 14:57:55 +0200 Subject: [PATCH 0080/2083] Fix Flake8-reported issues --- scrapy/http/request/form.py | 2 +- tests/test_http_request.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d58d217b6..bdb6bec7a 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -33,7 +33,7 @@ class FormRequest(Request): self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') self._set_body(form_query_str) else: - self._set_url(urlunsplit(urlsplit(self.url)._replace(query = form_query_str))) + self._set_url(urlunsplit(urlsplit(self.url)._replace(query=form_query_str))) @classmethod def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, diff --git a/tests/test_http_request.py b/tests/test_http_request.py index bc34bb26d..96954d419 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -369,7 +369,7 @@ class FormRequestTest(RequestTest): def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) self.assertEqual(r1.body, b'') - + def test_formdata_overrides_querystring(self): data = (('a', 'one'), ('a', 'two'), ('b', '2')) url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] @@ -378,7 +378,7 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'b'], [b'2']) self.assertIsNone(fs.get(b'c')) - data = {'a' : '1', 'b' : '2'} + data = {'a': '1', 'b': '2'} fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) self.assertEqual(fs[b'a'], [b'1']) self.assertEqual(fs[b'b'], [b'2']) From 4605c66a80dabd64924e397580224a667cd73ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 7 May 2020 12:38:51 +0200 Subject: [PATCH 0081/2083] Fix AttributeError --- scrapy/utils/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 376c1f992..6a6d38a5c 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -114,7 +114,7 @@ def get_sources(use_closest=True): def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) - out.setdefault("fields", settings.settings.getdictorlist("FEED_EXPORT_FIELDS") or None) + out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) if settings["FEED_EXPORT_INDENT"] is None: From 9408c77a1e16df89feeab055d3530ebad66555e1 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Date: Sun, 31 May 2020 13:09:56 +0530 Subject: [PATCH 0082/2083] feat(http2): IH2EventsHandler, http2 module --- scrapy/core/http2/__init__.py | 0 scrapy/core/http2/protocol.py | 136 ++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 scrapy/core/http2/__init__.py create mode 100644 scrapy/core/http2/protocol.py diff --git a/scrapy/core/http2/__init__.py b/scrapy/core/http2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py new file mode 100644 index 000000000..4a465168f --- /dev/null +++ b/scrapy/core/http2/protocol.py @@ -0,0 +1,136 @@ +from h2.connection import H2Connection +from h2.config import H2Configuration +from h2.events import ( + ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, + StreamReset, TrailersReceived, WindowUpdated +) + +from scrapy.http import Request + +from twisted.internet.defer import maybeDeferred +from twisted.internet.protocol import Protocol + +from urllib.parse import urlparse + +from zope.interface import implementer, Interface + + +class IH2EventsHandler(Interface): + def connection_terminated(event: ConnectionTerminated): + pass + + def data_received(event: DataReceived): + pass + + def response_received(event: ResponseReceived): + pass + + def stream_ended(event: StreamEnded): + pass + + def stream_reset(event: StreamReset): + pass + + def trailers_received(event: TrailersReceived): + pass + + def window_updated(event: WindowUpdated): + pass + + +@implementer(IH2EventsHandler) +class H2ClientProtocol(Protocol): + def __init__(self): + config = H2Configuration(client_side=True) + self.conn = H2Connection(config=config) + + # List of ongoing stream id's + self.streams = [] + + def request(self, _request: Request): + url = urlparse(_request.url) + + request_headers = [ + (':method', _request.method), + (':authority', url.netloc), + (':scheme', url.scheme), + (':path', url.path), + ] + + # TODO: Check for user-agent while testing + request_headers += list(_request.headers.items()) + + # TODO: Add support for cookies here + + + + + + + def connectionMade(self): + """Called by Twisted when the connection is established. We can start + sending some data now: we should open with the connection preamble. + """ + self.conn.initiate_connection() + self.transport.write(self.conn.data_to_send()) + + def dataReceived(self, data): + events = self.conn.receive_data(data) + + self._handle_events(events) + + _data = self.conn.data_to_send() + if _data: + self.transport.write(data) + + def connectionLost(self, reason): + """Called by Twisted when the transport connection is lost. + """ + + for stream_id in self.streams: + self.conn.end_stream(stream_id) + + def _handle_events(self, events): + """Private method which acts as a bridge between the events + received from the HTTP/2 data and IH2EventsHandler + + Arguments: + events {list} -- A list of events that the remote peer + triggered by sending data + """ + for event in events: + if isinstance(event, ConnectionTerminated): + self.connection_terminated(event) + elif isinstance(event, DataReceived): + self.data_received(event) + elif isinstance(event, ResponseReceived): + self.response_received(event) + elif isinstance(event, StreamEnded): + self.stream_ended(event) + elif isinstance(event, StreamReset): + self.stream_reset(event) + elif isinstance(event, TrailersReceived): + self.trailers_received(event) + elif isinstance(event, WindowUpdated): + self.window_updated(event) + + def connection_terminated(self, event): + pass + + def data_received(self, event): + pass + + def response_received(self, event): + pass + + def stream_ended(self, event): + pass + + def stream_reset(self, event): + pass + + def trailers_received(self, event): + pass + + def window_updated(self, event): + pass From 791292334e86723a80cfd94b2877a258721f2c93 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 2 Jun 2020 09:13:31 +0530 Subject: [PATCH 0083/2083] chore(http2): Stream class --- scrapy/core/http2/protocol.py | 75 +++++++++++++++++++++++------------ scrapy/core/http2/stream.py | 44 ++++++++++++++++++++ 2 files changed, 94 insertions(+), 25 deletions(-) create mode 100644 scrapy/core/http2/stream.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 4a465168f..b95cb05ae 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -6,8 +6,9 @@ from h2.events import ( ) from scrapy.http import Request +from scrapy.core.http2.stream import Stream -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import Deferred from twisted.internet.protocol import Protocol from urllib.parse import urlparse @@ -40,31 +41,55 @@ class IH2EventsHandler(Interface): @implementer(IH2EventsHandler) class H2ClientProtocol(Protocol): + # TODO: Check for user-agent while testing + # TODO: Add support for cookies + # TODO: Handle priority updates + def __init__(self): config = H2Configuration(client_side=True) self.conn = H2Connection(config=config) - - # List of ongoing stream id's - self.streams = [] + + # ID of the next request stream + # Assuming each request stream creates a new response stream + # we increment by 2 for each new request stream created + self.next_stream_id = 1 + + # Streams are stored in a dictionary keyed off their stream IDs + self.streams = {} + + def _new_stream(self, headers): + """Instantiates a new Stream object + """ + stream = Stream(self.next_stream_id, headers) + + self.next_stream_id += 2 + + return stream def request(self, _request: Request): + """ + + Arguments: + _request {Request} -- [description] + """ url = urlparse(_request.url) - request_headers = [ - (':method', _request.method), - (':authority', url.netloc), - (':scheme', url.scheme), - (':path', url.path), - ] - - # TODO: Check for user-agent while testing - request_headers += list(_request.headers.items()) - - # TODO: Add support for cookies here + _request[":method"] = _request.method + # TODO: Make authority private class variable instead + # of parsing it from request url all requests to same + # host are multiplexed into one connection & a connection + # can have only 1 host at a time + _request[":authority"] = url.netloc + # TODO: Check if scheme can be 'http' for HTTP/2 ? + _request[":scheme"] = "https" + _request[":path"] = url.path + stream = self._new_stream(_request.headers) + d = stream.get_response() + return d def connectionMade(self): @@ -76,7 +101,6 @@ class H2ClientProtocol(Protocol): def dataReceived(self, data): events = self.conn.receive_data(data) - self._handle_events(events) _data = self.conn.data_to_send() @@ -85,9 +109,10 @@ class H2ClientProtocol(Protocol): def connectionLost(self, reason): """Called by Twisted when the transport connection is lost. - """ + """ - for stream_id in self.streams: + for stream_id in self.streams.keys(): + # TODO: Close each Stream instance in a clean manner self.conn.end_stream(stream_id) def _handle_events(self, events): @@ -114,23 +139,23 @@ class H2ClientProtocol(Protocol): elif isinstance(event, WindowUpdated): self.window_updated(event) - def connection_terminated(self, event): + def connection_terminated(self, event: ConnectionTerminated): pass - def data_received(self, event): + def data_received(self, event: DataReceived): pass - def response_received(self, event): + def response_received(self, event: ResponseReceived): pass - def stream_ended(self, event): + def stream_ended(self, event: StreamEnded): pass - def stream_reset(self, event): + def stream_reset(self, event: StreamReset): pass - def trailers_received(self, event): + def trailers_received(self, event: TrailersReceived): pass - def window_updated(self, event): + def window_updated(self, event: WindowUpdated): pass diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py new file mode 100644 index 000000000..558a6552c --- /dev/null +++ b/scrapy/core/http2/stream.py @@ -0,0 +1,44 @@ +from scrapy.http.headers import Headers + + +class Stream: + """Represents a single HTTP/2 Stream. + + Stream is a bidirectional flow of bytes within an established connection, + which may carry one or more messages. Handles the tranfer of HTTP Headers + and Data frames. + """ + + def __init__(self, stream_id, headers): + """ + Arguments: + stream_id {int} -- For one HTTP/2 connection each stream is + uniquely identified by a single integer + headers {Headers} -- HTTP request headers + """ + + # Headers received after sending the request + self.response_headers = Headers({}) + + # Headers which are send with the request + # These cannot be modified any furthur + self._request_headers = headers + + # TODO: Add canceller for the Deferred below + self._deferred_response = Deferred() + + def get_response(self): + """Simply return a Deferred which fires when response + from the asynchronous request is available + + Returns: + Deferred -- Calls the callback when the response is + avaialble + """ + return self._deferred_response + + def receive_data(self, data): + pass + + def receive_headers(self, headers): + pass From bdabc500aaa37026dfceddf94b41e80f9ce2ce6b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 6 Jun 2020 16:32:43 -0300 Subject: [PATCH 0084/2083] Update headless browser docs --- docs/topics/dynamic-content.rst | 36 ++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 495111b56..7450de4a2 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -248,8 +248,34 @@ Using a headless browser A `headless browser`_ is a special web browser that provides an API for automation. -The easiest way to use a headless browser with Scrapy is to use Selenium_, -along with `scrapy-selenium`_ for seamless integration. +Since version 2.0, it is possible to integrate libraries that use the +``async/await`` syntax. One such library is `pyppeteer`_ (an unnoficial +Python port of `puppeteer`_), which uses headless Chrome to download and +render pages. +The following is a simple snippet to illustrate its usage within Scrapy:: + + import pyppeteer + import scrapy + + class PyppeteerSpider(scrapy.Spider): + name = "pyppeteer" + start_urls = ["data:,"] # avoid making an actual upstream request + + async def parse(self, response): + browser = await pyppeteer.launch() + page = await browser.newPage() + await page.goto("https:/example.org") + title = await page.title() + yield {"title": title} + +Keep in mind that this is just a proof of concept, since it circumvents +most of the Scrapy components (middlewares, dupefilter, etc). + +There are some 3rd party projects which provider better integration: + +* https://github.com/elacuesta/scrapy-pyppeteer +* https://github.com/lopuhin/scrapy-pyppeteer +* https://github.com/clemfromspace/scrapy-puppeteer .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29 @@ -259,11 +285,11 @@ along with `scrapy-selenium`_ for seamless integration. .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser .. _JavaScript: https://en.wikipedia.org/wiki/JavaScript .. _js2xml: https://github.com/scrapinghub/js2xml +.. _puppeteer: https://pptr.dev/ +.. _pyppeteer: https://pyppeteer.github.io/pyppeteer/ .. _pytesseract: https://github.com/madmaze/pytesseract -.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium .. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash -.. _Selenium: https://www.selenium.dev/ .. _Splash: https://github.com/scrapinghub/splash .. _tabula-py: https://github.com/chezou/tabula-py .. _wget: https://www.gnu.org/software/wget/ -.. _wgrep: https://github.com/stav/wgrep \ No newline at end of file +.. _wgrep: https://github.com/stav/wgrep From 9ff9caecadf8215ce75b0ad4231b8289bca168fe Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 7 Jun 2020 14:04:53 +0530 Subject: [PATCH 0085/2083] feat(http2): support for GET requests --- scrapy/core/http2/protocol.py | 126 ++++++++++++++++++++++------------ scrapy/core/http2/stream.py | 71 +++++++++++++++---- 2 files changed, 139 insertions(+), 58 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index b95cb05ae..847e74f97 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,20 +1,19 @@ -from h2.connection import H2Connection +import logging + from h2.config import H2Configuration +from h2.connection import H2Connection from h2.events import ( - ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, - StreamReset, TrailersReceived, WindowUpdated + ConnectionTerminated, DataReceived, ResponseReceived, RemoteSettingsChanged, + StreamEnded, StreamReset, TrailersReceived, WindowUpdated ) - -from scrapy.http import Request -from scrapy.core.http2.stream import Stream - -from twisted.internet.defer import Deferred -from twisted.internet.protocol import Protocol - -from urllib.parse import urlparse - +from twisted.internet.protocol import connectionDone, Protocol from zope.interface import implementer, Interface +from scrapy.core.http2.stream import Stream +from scrapy.http import Request + +LOGGER = logging.getLogger(__name__) + class IH2EventsHandler(Interface): def connection_terminated(event: ConnectionTerminated): @@ -26,6 +25,9 @@ class IH2EventsHandler(Interface): def response_received(event: ResponseReceived): pass + def remote_settings_changed(event: RemoteSettingsChanged): + pass + def stream_ended(event: StreamEnded): pass @@ -46,7 +48,7 @@ class H2ClientProtocol(Protocol): # TODO: Handle priority updates def __init__(self): - config = H2Configuration(client_side=True) + config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) # ID of the next request stream @@ -57,60 +59,69 @@ class H2ClientProtocol(Protocol): # Streams are stored in a dictionary keyed off their stream IDs self.streams = {} - def _new_stream(self, headers): + # Boolean to keep track the connection is made + # If requests are received before connection is made + # we keep all requests in a pool and send them as the connection + # is made + self.is_connection_made = False + self._pending_request_stream_pool = [] + + def _new_stream(self, request: Request): """Instantiates a new Stream object """ - stream = Stream(self.next_stream_id, headers) - + stream = Stream(self.next_stream_id, request, self) self.next_stream_id += 2 + self.streams[stream.stream_id] = stream return stream + def _write_to_transport(self): + """ Write data to the underlying transport connection + from the HTTP2 connection instance if any + """ + data = self.conn.data_to_send() + if data: + self.transport.write(data) + def request(self, _request: Request): - """ - - Arguments: - _request {Request} -- [description] - """ - url = urlparse(_request.url) - - _request[":method"] = _request.method - - # TODO: Make authority private class variable instead - # of parsing it from request url all requests to same - # host are multiplexed into one connection & a connection - # can have only 1 host at a time - _request[":authority"] = url.netloc - - # TODO: Check if scheme can be 'http' for HTTP/2 ? - _request[":scheme"] = "https" - _request[":path"] = url.path - - stream = self._new_stream(_request.headers) + stream = self._new_stream(_request) d = stream.get_response() - return d + # If connection is not yet established then add the + # stream to pool or initiate request + if self.is_connection_made: + stream.initiate_request() + else: + self._pending_request_stream_pool.append(stream) + return d def connectionMade(self): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ + LOGGER.info("Connection made to {}".format(self.transport)) self.conn.initiate_connection() - self.transport.write(self.conn.data_to_send()) + self._write_to_transport() + + self.is_connection_made = True + + # Initiate all pending requests + for stream in self._pending_request_stream_pool: + assert isinstance(stream, Stream) + stream.initiate_request() + + self._pending_request_stream_pool.clear() def dataReceived(self, data): events = self.conn.receive_data(data) self._handle_events(events) + self._write_to_transport() - _data = self.conn.data_to_send() - if _data: - self.transport.write(data) + def connectionLost(self, reason=connectionDone): - def connectionLost(self, reason): """Called by Twisted when the transport connection is lost. """ - for stream_id in self.streams.keys(): # TODO: Close each Stream instance in a clean manner self.conn.end_stream(stream_id) @@ -138,18 +149,43 @@ class H2ClientProtocol(Protocol): self.trailers_received(event) elif isinstance(event, WindowUpdated): self.window_updated(event) + elif isinstance(event, RemoteSettingsChanged): + self.remote_settings_changed(event) + + def send_headers(self, stream_id, headers): + """ Send the headers for a given stream to the resource + Initiates a new connection hence. + + Arguments: + stream_id {int} -- Valid stream id + headers {List[Tuple[str, str]]} -- Headers of the request + """ + if stream_id in self.streams: + self.conn.send_headers(stream_id, headers, end_stream=True) + self._write_to_transport() + else: + pass def connection_terminated(self, event: ConnectionTerminated): pass def data_received(self, event: DataReceived): - pass + stream_id = event.stream_id + # TODO: Stream do not exist in self.streams dict + self.streams[stream_id].receive_data(event.data) def response_received(self, event: ResponseReceived): + stream_id = event.stream_id + # TODO: Stream do not exist in self.streams dict + self.streams[stream_id].receive_headers(event.headers) + + def remote_settings_changed(self, event: RemoteSettingsChanged): pass def stream_ended(self, event: StreamEnded): - pass + stream_id = event.stream_id + # TODO: Stream do not exist in self.streams dict + self.streams[stream_id].end_stream() def stream_reset(self, event: StreamReset): pass diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 558a6552c..d2a9f02fa 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,3 +1,8 @@ +from urllib.parse import urlparse + +from twisted.internet.defer import Deferred + +from scrapy.http import Request, Response from scrapy.http.headers import Headers @@ -5,24 +10,30 @@ class Stream: """Represents a single HTTP/2 Stream. Stream is a bidirectional flow of bytes within an established connection, - which may carry one or more messages. Handles the tranfer of HTTP Headers + which may carry one or more messages. Handles the transfer of HTTP Headers and Data frames. + + Role of this class is to + 1. Combine all the data frames """ - def __init__(self, stream_id, headers): + def __init__(self, stream_id: int, request: Request, connection): """ Arguments: stream_id {int} -- For one HTTP/2 connection each stream is uniquely identified by a single integer - headers {Headers} -- HTTP request headers + request {Request} -- HTTP request + connection {H2ClientProtocol} -- HTTP/2 connection this stream belongs to """ - # Headers received after sending the request - self.response_headers = Headers({}) + self.stream_id = stream_id + self._request = request + self._conn = connection - # Headers which are send with the request - # These cannot be modified any furthur - self._request_headers = headers + self._response_data = b"" + + # Headers received after sending the request + self._response_headers = Headers({}) # TODO: Add canceller for the Deferred below self._deferred_response = Deferred() @@ -32,13 +43,47 @@ class Stream: from the asynchronous request is available Returns: - Deferred -- Calls the callback when the response is - avaialble + Deferred -- Calls the callback passing the response """ return self._deferred_response - def receive_data(self, data): - pass + def initiate_request(self): + http2_request_headers = [] + for name, value in self._request.headers.items(): + http2_request_headers.append((name, value)) + + url = urlparse(self._request.url) + http2_request_headers += [ + (":method", self._request.method), + (":authority", url.netloc), + + # TODO: Check if scheme can be "http" for HTTP/2 ? + (":scheme", "https"), + (":path", url.path) + ] + + self._conn.send_headers(self.stream_id, http2_request_headers) + + def receive_data(self, data: bytes): + self._response_data += data def receive_headers(self, headers): - pass + for name, value in headers: + self._response_headers[name] = value + + def end_stream(self): + """Stream is ended by the resource hence no further + data or headers should be expected on this stream. + + We will call the response deferred callback passing + the response object + """ + # TODO: Set flags, certificate, ip_address + response = Response( + url=self._request.url, + status=self._response_headers[":status"], + headers=self._response_headers, + body=self._response_data, + request=self._request + ) + self._deferred_response.callback(response) From 78aa1b2bfc3eceb09f2fa3dea59811375d6ed1f8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 8 Jun 2020 11:19:15 -0300 Subject: [PATCH 0086/2083] Fix typo --- docs/topics/dynamic-content.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 7450de4a2..e244eb7ff 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -271,7 +271,7 @@ The following is a simple snippet to illustrate its usage within Scrapy:: Keep in mind that this is just a proof of concept, since it circumvents most of the Scrapy components (middlewares, dupefilter, etc). -There are some 3rd party projects which provider better integration: +There are some 3rd party projects which provide better integration: * https://github.com/elacuesta/scrapy-pyppeteer * https://github.com/lopuhin/scrapy-pyppeteer From b6c5289fb900beb552a1ab608f572d99c180b4fb Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 10 Jun 2020 12:11:49 -0300 Subject: [PATCH 0087/2083] Close page in pyppeteer example, mention asyncio reactor --- docs/topics/dynamic-content.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index e244eb7ff..56c8b6ae9 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -266,12 +266,16 @@ The following is a simple snippet to illustrate its usage within Scrapy:: page = await browser.newPage() await page.goto("https:/example.org") title = await page.title() + await page.close() yield {"title": title} +For this example to work, Scrapy needs to be running on top of the +:ref:`asyncio reactor `. + Keep in mind that this is just a proof of concept, since it circumvents most of the Scrapy components (middlewares, dupefilter, etc). -There are some 3rd party projects which provide better integration: +The following is a list of 3rd party projects which provide better integration: * https://github.com/elacuesta/scrapy-pyppeteer * https://github.com/lopuhin/scrapy-pyppeteer From d09ccf8d3b2932d9393e6ce4a20c46befdd43acf Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 13 Jun 2020 20:40:01 +0530 Subject: [PATCH 0088/2083] feat(http2): support for POST requests BREAKING CHANGES - Request is sent successfully with its Response received as well. However, the StreamEnded event is not received which do not fires the response deferred --- scrapy/core/http2/protocol.py | 108 +++++++++++++++++----------------- scrapy/core/http2/stream.py | 79 +++++++++++++++++++++++-- 2 files changed, 127 insertions(+), 60 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 847e74f97..6d926b100 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,4 +1,5 @@ import logging +from typing import Dict, List from h2.config import H2Configuration from h2.connection import H2Connection @@ -7,45 +8,19 @@ from h2.events import ( StreamEnded, StreamReset, TrailersReceived, WindowUpdated ) from twisted.internet.protocol import connectionDone, Protocol -from zope.interface import implementer, Interface from scrapy.core.http2.stream import Stream from scrapy.http import Request LOGGER = logging.getLogger(__name__) +LOGGER.debug = print -class IH2EventsHandler(Interface): - def connection_terminated(event: ConnectionTerminated): - pass - - def data_received(event: DataReceived): - pass - - def response_received(event: ResponseReceived): - pass - - def remote_settings_changed(event: RemoteSettingsChanged): - pass - - def stream_ended(event: StreamEnded): - pass - - def stream_reset(event: StreamReset): - pass - - def trailers_received(event: TrailersReceived): - pass - - def window_updated(event: WindowUpdated): - pass - - -@implementer(IH2EventsHandler) class H2ClientProtocol(Protocol): - # TODO: Check for user-agent while testing - # TODO: Add support for cookies - # TODO: Handle priority updates + # TODO: + # 1. Check for user-agent while testing + # 2. Add support for cookies + # 3. Handle priority updates def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') @@ -57,14 +32,14 @@ class H2ClientProtocol(Protocol): self.next_stream_id = 1 # Streams are stored in a dictionary keyed off their stream IDs - self.streams = {} + self.streams: Dict[int, Stream] = {} # Boolean to keep track the connection is made # If requests are received before connection is made # we keep all requests in a pool and send them as the connection # is made self.is_connection_made = False - self._pending_request_stream_pool = [] + self._pending_request_stream_pool: List[Stream] = [] def _new_stream(self, request: Request): """Instantiates a new Stream object @@ -100,29 +75,27 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ - LOGGER.info("Connection made to {}".format(self.transport)) + LOGGER.debug("Connection made to {}".format(self.transport)) self.conn.initiate_connection() self._write_to_transport() self.is_connection_made = True - # Initiate all pending requests - for stream in self._pending_request_stream_pool: - assert isinstance(stream, Stream) - stream.initiate_request() - - self._pending_request_stream_pool.clear() - def dataReceived(self, data): events = self.conn.receive_data(data) self._handle_events(events) self._write_to_transport() def connectionLost(self, reason=connectionDone): - """Called by Twisted when the transport connection is lost. """ - for stream_id in self.streams.keys(): + LOGGER.debug(f"connectionLost {reason}") + stream_ids = list(self.streams.keys()) + + for stream in self._pending_request_stream_pool: + stream_ids.remove(stream.stream_id) + + for stream_id in stream_ids: # TODO: Close each Stream instance in a clean manner self.conn.end_stream(stream_id) @@ -135,6 +108,7 @@ class H2ClientProtocol(Protocol): triggered by sending data """ for event in events: + LOGGER.debug(event) if isinstance(event, ConnectionTerminated): self.connection_terminated(event) elif isinstance(event, DataReceived): @@ -153,38 +127,62 @@ class H2ClientProtocol(Protocol): self.remote_settings_changed(event) def send_headers(self, stream_id, headers): - """ Send the headers for a given stream to the resource + """Send the headers for a given stream to the resource Initiates a new connection hence. + This function is wrapper for :func:`~h2.connection.H2Connection.send_headers` Arguments: stream_id {int} -- Valid stream id headers {List[Tuple[str, str]]} -- Headers of the request """ - if stream_id in self.streams: - self.conn.send_headers(stream_id, headers, end_stream=True) - self._write_to_transport() - else: - pass + LOGGER.debug(f'Send Headers: stream_id={stream_id} headers={headers}') + self.conn.send_headers(stream_id, headers, end_stream=False) + def send_data(self, stream_id, data): + """Send the data for a given stream to the resource. + Requires request headers to be sent at least once before this + function is called. + This function is wrapper for :func:`~h2.connection.H2Connection.send_data` + + Arguments: + stream_id {int} -- Valid stream id + data {bytes} -- The data to send on the stream. + """ + LOGGER.debug(f"Send Data: stream_id={stream_id} data={data}") + self.conn.send_data(stream_id, data, end_stream=False) + + def end_stream(self, stream_id): + """End the given stream. + This function is wrapper for :func:`~h2.connection.H2Connection.end_stream` + + Arguments: + stream_id {int} - Valid stream id + """ + LOGGER.debug(f"End Stream: stream_id={stream_id}") + self.conn.end_stream(stream_id) + + # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated): pass def data_received(self, event: DataReceived): stream_id = event.stream_id - # TODO: Stream do not exist in self.streams dict self.streams[stream_id].receive_data(event.data) def response_received(self, event: ResponseReceived): stream_id = event.stream_id - # TODO: Stream do not exist in self.streams dict self.streams[stream_id].receive_headers(event.headers) def remote_settings_changed(self, event: RemoteSettingsChanged): - pass + # TODO: handle MAX_CONCURRENT_STREAMS + # Initiate all pending requests + for stream in self._pending_request_stream_pool: + stream.initiate_request() + + self._pending_request_stream_pool.clear() def stream_ended(self, event: StreamEnded): stream_id = event.stream_id - # TODO: Stream do not exist in self.streams dict self.streams[stream_id].end_stream() def stream_reset(self, event: StreamReset): @@ -194,4 +192,6 @@ class H2ClientProtocol(Protocol): pass def window_updated(self, event: WindowUpdated): - pass + stream_id = event.stream_id + if stream_id != 0: + self.streams[stream_id].window_updated() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index d2a9f02fa..ab5a0fc88 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -23,13 +23,23 @@ class Stream: stream_id {int} -- For one HTTP/2 connection each stream is uniquely identified by a single integer request {Request} -- HTTP request - connection {H2ClientProtocol} -- HTTP/2 connection this stream belongs to + connection {H2Connection} -- HTTP/2 connection this stream belongs to. """ - self.stream_id = stream_id self._request = request - self._conn = connection + self._client_protocol = connection + self._request_body = self._request.body + self.content_length = 0 if self._request_body is None else len(self._request_body) + + # Each time we send a data frame, we will decrease value by the amount send. + self.remaining_content_length = self.content_length + + # Flag to keep track whether we have ended this stream + self.stream_ended = True + + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. self._response_data = b"" # Headers received after sending the request @@ -59,10 +69,67 @@ class Stream: # TODO: Check if scheme can be "http" for HTTP/2 ? (":scheme", "https"), - (":path", url.path) + (":path", url.path), + # ("Content-Length", str(self.content_length)) + + # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers + # are sent for request having body ] - self._conn.send_headers(self.stream_id, http2_request_headers) + self._client_protocol.send_headers(self.stream_id, http2_request_headers) + self.send_data() + + def send_data(self): + """Called immediately after the headers are sent. Here we send all the + data as part of the request. + + If the content length is 0 initially then we end the stream immediately and + wait for response data. + """ + + # TODO: + # 1. Add test for sending very large data + # 2. Add test for small data + # 3. Both (1) and (2) should be tested for + # 3.1 Large number of request + # 3.2 Small number of requests + + # Firstly, check what the flow control window is for current stream. + window_size = self._client_protocol.conn.local_flow_control_window(stream_id=self.stream_id) + + # Next, check what the maximum frame size is. + max_frame_size = self._client_protocol.conn.max_outbound_frame_size + + # We will send no more than the window size or the remaining file size + # of data in this call, whichever is smaller. + bytes_to_send = min(window_size, self.remaining_content_length) + + # We now need to send a number of data frames. + while bytes_to_send > 0: + chunk_size = min(bytes_to_send, max_frame_size) + + data_chunk_start = self.content_length - self.remaining_content_length + data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size] + + self._client_protocol.send_data(self.stream_id, data_chunk, end_stream=False) + + bytes_to_send = max(0, bytes_to_send - chunk_size) + self.remaining_content_length = max(0, self.remaining_content_length - chunk_size) + + # End the stream if no more data has to be send + if self.remaining_content_length == 0: + self._client_protocol.end_stream(self.stream_id) + else: + # TODO: Continue from here :) + pass + + def window_updated(self): + """Flow control window size was changed. + Send data that earlier could not be sent as we were + blocked behind the flow control. + """ + if self.remaining_content_length > 0 and not self.stream_ended: + self.send_data() def receive_data(self, data: bytes): self._response_data += data @@ -72,7 +139,7 @@ class Stream: self._response_headers[name] = value def end_stream(self): - """Stream is ended by the resource hence no further + """Stream is ended by the server hence no further data or headers should be expected on this stream. We will call the response deferred callback passing From d06bb12e351f61ee20c28626b003f4b59fd689f7 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 13 Jun 2020 22:29:16 +0530 Subject: [PATCH 0089/2083] refactor: move H2Connection instance to stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove all wrapper funtions made such that stream can send header/data to H2Connection as they were not necessary BREAKING CHANGES Looks like, for small set of response data the StreamEnded event is emitted and everything works well -- tested for both GET & POST request. Maybe some issue with window size and/or flow control as when the response data needs to be broken into separate chunks -- not all chunks are received everytime which leads to indefinite waiting for next data chunk and the connection is lost due to timeout. 😥 Working on setting up testing environment now. After testing is setup I'll debug the above bug furthur. --- scrapy/core/http2/protocol.py | 74 ++++++++--------------------------- scrapy/core/http2/stream.py | 16 ++++---- 2 files changed, 24 insertions(+), 66 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 6d926b100..4036dfb3e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -4,8 +4,7 @@ from typing import Dict, List from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - ConnectionTerminated, DataReceived, ResponseReceived, RemoteSettingsChanged, - StreamEnded, StreamReset, TrailersReceived, WindowUpdated + ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, StreamReset, WindowUpdated ) from twisted.internet.protocol import connectionDone, Protocol @@ -13,7 +12,6 @@ from scrapy.core.http2.stream import Stream from scrapy.http import Request LOGGER = logging.getLogger(__name__) -LOGGER.debug = print class H2ClientProtocol(Protocol): @@ -44,19 +42,27 @@ class H2ClientProtocol(Protocol): def _new_stream(self, request: Request): """Instantiates a new Stream object """ - stream = Stream(self.next_stream_id, request, self) + stream = Stream(self.next_stream_id, request, self.conn) self.next_stream_id += 2 self.streams[stream.stream_id] = stream return stream + def _send_pending_requests(self): + # TODO: handle MAX_CONCURRENT_STREAMS + # Initiate all pending requests + for stream in self._pending_request_stream_pool: + stream.initiate_request() + self._write_to_transport() + + self._pending_request_stream_pool.clear() + def _write_to_transport(self): """ Write data to the underlying transport connection from the HTTP2 connection instance if any """ data = self.conn.data_to_send() - if data: - self.transport.write(data) + self.transport.write(data) def request(self, _request: Request): stream = self._new_stream(_request) @@ -75,10 +81,11 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ - LOGGER.debug("Connection made to {}".format(self.transport)) self.conn.initiate_connection() self._write_to_transport() + self._send_pending_requests() + self.is_connection_made = True def dataReceived(self, data): @@ -89,7 +96,6 @@ class H2ClientProtocol(Protocol): def connectionLost(self, reason=connectionDone): """Called by Twisted when the transport connection is lost. """ - LOGGER.debug(f"connectionLost {reason}") stream_ids = list(self.streams.keys()) for stream in self._pending_request_stream_pool: @@ -119,47 +125,10 @@ class H2ClientProtocol(Protocol): self.stream_ended(event) elif isinstance(event, StreamReset): self.stream_reset(event) - elif isinstance(event, TrailersReceived): - self.trailers_received(event) elif isinstance(event, WindowUpdated): self.window_updated(event) - elif isinstance(event, RemoteSettingsChanged): - self.remote_settings_changed(event) - - def send_headers(self, stream_id, headers): - """Send the headers for a given stream to the resource - Initiates a new connection hence. - This function is wrapper for :func:`~h2.connection.H2Connection.send_headers` - - Arguments: - stream_id {int} -- Valid stream id - headers {List[Tuple[str, str]]} -- Headers of the request - """ - LOGGER.debug(f'Send Headers: stream_id={stream_id} headers={headers}') - self.conn.send_headers(stream_id, headers, end_stream=False) - - def send_data(self, stream_id, data): - """Send the data for a given stream to the resource. - Requires request headers to be sent at least once before this - function is called. - This function is wrapper for :func:`~h2.connection.H2Connection.send_data` - - Arguments: - stream_id {int} -- Valid stream id - data {bytes} -- The data to send on the stream. - """ - LOGGER.debug(f"Send Data: stream_id={stream_id} data={data}") - self.conn.send_data(stream_id, data, end_stream=False) - - def end_stream(self, stream_id): - """End the given stream. - This function is wrapper for :func:`~h2.connection.H2Connection.end_stream` - - Arguments: - stream_id {int} - Valid stream id - """ - LOGGER.debug(f"End Stream: stream_id={stream_id}") - self.conn.end_stream(stream_id) + else: + LOGGER.info("Received unhandled event {}".format(event)) # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated): @@ -173,14 +142,6 @@ class H2ClientProtocol(Protocol): stream_id = event.stream_id self.streams[stream_id].receive_headers(event.headers) - def remote_settings_changed(self, event: RemoteSettingsChanged): - # TODO: handle MAX_CONCURRENT_STREAMS - # Initiate all pending requests - for stream in self._pending_request_stream_pool: - stream.initiate_request() - - self._pending_request_stream_pool.clear() - def stream_ended(self, event: StreamEnded): stream_id = event.stream_id self.streams[stream_id].end_stream() @@ -188,9 +149,6 @@ class H2ClientProtocol(Protocol): def stream_reset(self, event: StreamReset): pass - def trailers_received(self, event: TrailersReceived): - pass - def window_updated(self, event: WindowUpdated): stream_id = event.stream_id if stream_id != 0: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index ab5a0fc88..b37755042 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,5 +1,6 @@ from urllib.parse import urlparse +from h2.connection import H2Connection from twisted.internet.defer import Deferred from scrapy.http import Request, Response @@ -17,7 +18,7 @@ class Stream: 1. Combine all the data frames """ - def __init__(self, stream_id: int, request: Request, connection): + def __init__(self, stream_id: int, request: Request, connection: H2Connection): """ Arguments: stream_id {int} -- For one HTTP/2 connection each stream is @@ -27,7 +28,7 @@ class Stream: """ self.stream_id = stream_id self._request = request - self._client_protocol = connection + self._conn = connection self._request_body = self._request.body self.content_length = 0 if self._request_body is None else len(self._request_body) @@ -70,13 +71,12 @@ class Stream: # TODO: Check if scheme can be "http" for HTTP/2 ? (":scheme", "https"), (":path", url.path), - # ("Content-Length", str(self.content_length)) # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers # are sent for request having body ] - self._client_protocol.send_headers(self.stream_id, http2_request_headers) + self._conn.send_headers(self.stream_id, http2_request_headers, end_stream=False) self.send_data() def send_data(self): @@ -95,10 +95,10 @@ class Stream: # 3.2 Small number of requests # Firstly, check what the flow control window is for current stream. - window_size = self._client_protocol.conn.local_flow_control_window(stream_id=self.stream_id) + window_size = self._conn.local_flow_control_window(stream_id=self.stream_id) # Next, check what the maximum frame size is. - max_frame_size = self._client_protocol.conn.max_outbound_frame_size + max_frame_size = self._conn.max_outbound_frame_size # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. @@ -111,14 +111,14 @@ class Stream: data_chunk_start = self.content_length - self.remaining_content_length data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size] - self._client_protocol.send_data(self.stream_id, data_chunk, end_stream=False) + self._conn.send_data(self.stream_id, data_chunk, end_stream=False) bytes_to_send = max(0, bytes_to_send - chunk_size) self.remaining_content_length = max(0, self.remaining_content_length - chunk_size) # End the stream if no more data has to be send if self.remaining_content_length == 0: - self._client_protocol.end_stream(self.stream_id) + self._conn.end_stream(self.stream_id) else: # TODO: Continue from here :) pass From de4a34365a2d872eb69ab2a2edfb723658e28530 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 14 Jun 2020 22:40:49 +0530 Subject: [PATCH 0090/2083] fix: large data chunk not received Every data chunk received needs to be acknowledged to - update the flow control window size - get furthur data chunks from the server --- scrapy/core/http2/protocol.py | 74 +++++++++++++++++++++--------- scrapy/core/http2/stream.py | 85 ++++++++++++++++++++++++++++++----- 2 files changed, 129 insertions(+), 30 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 4036dfb3e..167596340 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,10 +1,10 @@ import logging -from typing import Dict, List from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - ConnectionTerminated, DataReceived, ResponseReceived, StreamEnded, StreamReset, WindowUpdated + ConnectionTerminated, DataReceived, ResponseReceived, + StreamEnded, StreamReset, WindowUpdated ) from twisted.internet.protocol import connectionDone, Protocol @@ -18,33 +18,57 @@ class H2ClientProtocol(Protocol): # TODO: # 1. Check for user-agent while testing # 2. Add support for cookies - # 3. Handle priority updates + # 3. Handle priority updates (Not required) + # 4. Handle case when received events have StreamID = 0 (applied to H2Connection) + # 1 & 2: + # - Automatically handled by the Request middleware + # - request.headers will have 'Set-Cookie' value def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) + # Address of the server we are connected to + # these are updated when connection is successfully made + self.destination = None + # ID of the next request stream - # Assuming each request stream creates a new response stream - # we increment by 2 for each new request stream created + # Following the convention made by hyper-h2 each client ID + # will be odd. self.next_stream_id = 1 # Streams are stored in a dictionary keyed off their stream IDs - self.streams: Dict[int, Stream] = {} + self.streams = {} # Boolean to keep track the connection is made # If requests are received before connection is made # we keep all requests in a pool and send them as the connection # is made self.is_connection_made = False - self._pending_request_stream_pool: List[Stream] = [] + self._pending_request_stream_pool = [] + + def _stream_close_cb(self, stream_id: int): + """Called when stream is closed completely + """ + try: + del self.streams[stream_id] + except KeyError: + pass def _new_stream(self, request: Request): """Instantiates a new Stream object """ - stream = Stream(self.next_stream_id, request, self.conn) + stream_id = self.next_stream_id self.next_stream_id += 2 + stream = Stream( + stream_id=stream_id, + request=request, + connection=self.conn, + write_to_transport=self._write_to_transport, + cb_close=lambda: self._stream_close_cb(stream_id) + ) + self.streams[stream.stream_id] = stream return stream @@ -53,7 +77,6 @@ class H2ClientProtocol(Protocol): # Initiate all pending requests for stream in self._pending_request_stream_pool: stream.initiate_request() - self._write_to_transport() self._pending_request_stream_pool.clear() @@ -81,13 +104,15 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ + self.destination = self.transport.connector.getDestination() + LOGGER.info('Connection made to {}'.format(self.destination)) + self.conn.initiate_connection() self._write_to_transport() + self.is_connection_made = True self._send_pending_requests() - self.is_connection_made = True - def dataReceived(self, data): events = self.conn.receive_data(data) self._handle_events(events) @@ -95,15 +120,18 @@ class H2ClientProtocol(Protocol): def connectionLost(self, reason=connectionDone): """Called by Twisted when the transport connection is lost. + No need to write anything to transport here. """ - stream_ids = list(self.streams.keys()) + # Pop all streams which were pending and were not yet started + for stream_id in list(self.streams): + try: + self.streams[stream_id].lost_connection() + except KeyError: + pass - for stream in self._pending_request_stream_pool: - stream_ids.remove(stream.stream_id) + self.conn.close_connection() - for stream_id in stream_ids: - # TODO: Close each Stream instance in a clean manner - self.conn.end_stream(stream_id) + LOGGER.info("Connection lost with reason " + str(reason)) def _handle_events(self, events): """Private method which acts as a bridge between the events @@ -136,7 +164,7 @@ class H2ClientProtocol(Protocol): def data_received(self, event: DataReceived): stream_id = event.stream_id - self.streams[stream_id].receive_data(event.data) + self.streams[stream_id].receive_data(event.data, event.flow_controlled_length) def response_received(self, event: ResponseReceived): stream_id = event.stream_id @@ -147,9 +175,15 @@ class H2ClientProtocol(Protocol): self.streams[stream_id].end_stream() def stream_reset(self, event: StreamReset): - pass + # TODO: event.stream_id was abruptly closed + # Q. What should be the response? (Failure/Partial/???) + self.streams[event.stream_id].reset() def window_updated(self, event: WindowUpdated): stream_id = event.stream_id if stream_id != 0: - self.streams[stream_id].window_updated() + self.streams[stream_id].receive_window_update(event.delta) + else: + # TODO: + # Q. What to do when StreamID=0 ? + pass diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index b37755042..7c9cf7cf5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,3 +1,4 @@ +import logging from urllib.parse import urlparse from h2.connection import H2Connection @@ -6,6 +7,8 @@ from twisted.internet.defer import Deferred from scrapy.http import Request, Response from scrapy.http.headers import Headers +LOGGER = logging.getLogger(__name__) + class Stream: """Represents a single HTTP/2 Stream. @@ -18,17 +21,30 @@ class Stream: 1. Combine all the data frames """ - def __init__(self, stream_id: int, request: Request, connection: H2Connection): + def __init__( + self, + stream_id: int, + request: Request, + connection: H2Connection, + write_to_transport, + cb_close + ): """ Arguments: stream_id {int} -- For one HTTP/2 connection each stream is uniquely identified by a single integer request {Request} -- HTTP request connection {H2Connection} -- HTTP/2 connection this stream belongs to. + write_to_transport {callable} -- Method used to write & send data to the server + This method should be used whenever some frame is to be sent to the server. + cb_close {callable} -- Method called when this stream is closed + to notify the TCP connection instance. """ self.stream_id = stream_id self._request = request self._conn = connection + self._write_to_transport = write_to_transport + self._cb_close = cb_close self._request_body = self._request.body self.content_length = 0 if self._request_body is None else len(self._request_body) @@ -36,13 +52,20 @@ class Stream: # Each time we send a data frame, we will decrease value by the amount send. self.remaining_content_length = self.content_length - # Flag to keep track whether we have ended this stream - self.stream_ended = True + # Flag to keep track whether we have closed this stream + self.stream_closed_local = False + + # Flag to keep track whether the server has closed the stream + self.stream_closed_server = False # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. self._response_data = b"" + # The amount of data received that counts against the flow control + # window + self._response_flow_controlled_size = 0 + # Headers received after sending the request self._response_headers = Headers({}) @@ -77,6 +100,8 @@ class Stream: ] self._conn.send_headers(self.stream_id, http2_request_headers, end_stream=False) + self._write_to_transport() + self.send_data() def send_data(self): @@ -112,32 +137,59 @@ class Stream: data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size] self._conn.send_data(self.stream_id, data_chunk, end_stream=False) + self._write_to_transport() bytes_to_send = max(0, bytes_to_send - chunk_size) self.remaining_content_length = max(0, self.remaining_content_length - chunk_size) # End the stream if no more data has to be send if self.remaining_content_length == 0: + self.stream_closed_local = True self._conn.end_stream(self.stream_id) - else: - # TODO: Continue from here :) - pass - def window_updated(self): + self._write_to_transport() + + # Q. What about the rest of the data? + # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame + + def receive_window_update(self, delta): """Flow control window size was changed. Send data that earlier could not be sent as we were blocked behind the flow control. + + Arguments: + delta -- Window change delta """ - if self.remaining_content_length > 0 and not self.stream_ended: + if self.remaining_content_length > 0 and not self.stream_closed_local: self.send_data() - def receive_data(self, data: bytes): + def receive_data(self, data: bytes, flow_controlled_length: int): self._response_data += data + self._response_flow_controlled_size += flow_controlled_length + + # Acknowledge the data received + self._conn.acknowledge_received_data( + self._response_flow_controlled_size, + self.stream_id + ) def receive_headers(self, headers): for name, value in headers: self._response_headers[name] = value + def reset(self): + """Received a RST_STREAM -- forcefully reset""" + # TODO: + # Q1. Do we need to send the request again? + # Q2. What response should we send now? + self.stream_closed_server = True + self._cb_close() + + def lost_connection(self): + # TODO: Same as self.reset + self.stream_closed_server = True + self._cb_close() + def end_stream(self): """Stream is ended by the server hence no further data or headers should be expected on this stream. @@ -145,7 +197,20 @@ class Stream: We will call the response deferred callback passing the response object """ - # TODO: Set flags, certificate, ip_address + assert self.stream_closed_server is False + self.stream_closed_server = True + + self._fire_response_deferred() + self._cb_close() + + def _fire_response_deferred(self): + # TODO: + # 1. Set flags, certificate, ip_address in response + # 2. Should we fire this in case of + # 2.1 StreamReset in between when data is received partially + # 2.2 Forcefully closed the stream + + # NOTE: Presently on fired with successful response response = Response( url=self._request.url, status=self._response_headers[":status"], From 01ad8b31ab7fe86fd78a70b09a6dd61a497e0ccb Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 15 Jun 2020 05:14:00 +0530 Subject: [PATCH 0091/2083] refactor(http2): clean up - make separate function to parse http headers from Request instance --- scrapy/core/http2/protocol.py | 12 ++------ scrapy/core/http2/stream.py | 55 ++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 167596340..915284c33 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -50,10 +50,7 @@ class H2ClientProtocol(Protocol): def _stream_close_cb(self, stream_id: int): """Called when stream is closed completely """ - try: - del self.streams[stream_id] - except KeyError: - pass + self.streams.pop(stream_id, None) def _new_stream(self, request: Request): """Instantiates a new Stream object @@ -66,7 +63,7 @@ class H2ClientProtocol(Protocol): request=request, connection=self.conn, write_to_transport=self._write_to_transport, - cb_close=lambda: self._stream_close_cb(stream_id) + cb_close=self._stream_close_cb ) self.streams[stream.stream_id] = stream @@ -124,10 +121,7 @@ class H2ClientProtocol(Protocol): """ # Pop all streams which were pending and were not yet started for stream_id in list(self.streams): - try: - self.streams[stream_id].lost_connection() - except KeyError: - pass + self.streams[stream_id].lost_connection() self.conn.close_connection() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 7c9cf7cf5..a0b75850d 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -81,25 +81,27 @@ class Stream: """ return self._deferred_response - def initiate_request(self): - http2_request_headers = [] - for name, value in self._request.headers.items(): - http2_request_headers.append((name, value)) - + def _get_request_headers(self): url = urlparse(self._request.url) - http2_request_headers += [ - (":method", self._request.method), - (":authority", url.netloc), - # TODO: Check if scheme can be "http" for HTTP/2 ? - (":scheme", "https"), - (":path", url.path), + # Make sure pseudo-headers comes before all the other headers + headers = [ + (':method', self._request.method), + (':authority', url.netloc), - # TODO: Make sure 'Content-Type' and 'Content-Encoding' headers - # are sent for request having body + # TODO: Check if scheme can be 'http' for HTTP/2 ? + (':scheme', 'https'), + (':path', url.path), ] - self._conn.send_headers(self.stream_id, http2_request_headers, end_stream=False) + for name, value in self._request.headers.items(): + headers.append((name, value[0])) + + return headers + + def initiate_request(self): + headers = self._get_request_headers() + self._conn.send_headers(self.stream_id, headers, end_stream=False) self._write_to_transport() self.send_data() @@ -127,23 +129,24 @@ class Stream: # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. - bytes_to_send = min(window_size, self.remaining_content_length) + bytes_to_send_size = min(window_size, self.remaining_content_length) # We now need to send a number of data frames. - while bytes_to_send > 0: - chunk_size = min(bytes_to_send, max_frame_size) + while bytes_to_send_size > 0: + chunk_size = min(bytes_to_send_size, max_frame_size) - data_chunk_start = self.content_length - self.remaining_content_length - data_chunk = self._request_body[data_chunk_start:data_chunk_start + chunk_size] + data_chunk_start_id = self.content_length - self.remaining_content_length + data_chunk = self._request_body[data_chunk_start_id:data_chunk_start_id + chunk_size] self._conn.send_data(self.stream_id, data_chunk, end_stream=False) - self._write_to_transport() - bytes_to_send = max(0, bytes_to_send - chunk_size) - self.remaining_content_length = max(0, self.remaining_content_length - chunk_size) + bytes_to_send_size = bytes_to_send_size - chunk_size + self.remaining_content_length = self.remaining_content_length - chunk_size # End the stream if no more data has to be send - if self.remaining_content_length == 0: + if self.remaining_content_length <= 0: + self.remaining_content_length = 0 + self.stream_closed_local = True self._conn.end_stream(self.stream_id) @@ -183,12 +186,12 @@ class Stream: # Q1. Do we need to send the request again? # Q2. What response should we send now? self.stream_closed_server = True - self._cb_close() + self._cb_close(self.stream_id) def lost_connection(self): # TODO: Same as self.reset self.stream_closed_server = True - self._cb_close() + self._cb_close(self.stream_id) def end_stream(self): """Stream is ended by the server hence no further @@ -201,7 +204,7 @@ class Stream: self.stream_closed_server = True self._fire_response_deferred() - self._cb_close() + self._cb_close(self.stream_id) def _fire_response_deferred(self): # TODO: From 089dbc75e78a2da9c455f21bb3c7ebaaeb2e3582 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 17 Jun 2020 20:57:03 +0530 Subject: [PATCH 0092/2083] chore: use deque for pending request pool - Use itertools.count to generate next stream_id BREAKING CHANGES When sending data/body more than the local flow control window -- no window update occurs to send the remaining data frames. Hence, the complete body is not send resulting in no response received. --- scrapy/core/http2/protocol.py | 16 +++++++++------- scrapy/core/http2/stream.py | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 915284c33..dbc048ffa 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,4 +1,6 @@ +import itertools import logging +from collections import deque from h2.config import H2Configuration from h2.connection import H2Connection @@ -35,7 +37,7 @@ class H2ClientProtocol(Protocol): # ID of the next request stream # Following the convention made by hyper-h2 each client ID # will be odd. - self.next_stream_id = 1 + self.stream_id_count = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs self.streams = {} @@ -45,7 +47,7 @@ class H2ClientProtocol(Protocol): # we keep all requests in a pool and send them as the connection # is made self.is_connection_made = False - self._pending_request_stream_pool = [] + self._pending_request_stream_pool = deque() def _stream_close_cb(self, stream_id: int): """Called when stream is closed completely @@ -55,8 +57,7 @@ class H2ClientProtocol(Protocol): def _new_stream(self, request: Request): """Instantiates a new Stream object """ - stream_id = self.next_stream_id - self.next_stream_id += 2 + stream_id = next(self.stream_id_count) stream = Stream( stream_id=stream_id, @@ -72,11 +73,10 @@ class H2ClientProtocol(Protocol): def _send_pending_requests(self): # TODO: handle MAX_CONCURRENT_STREAMS # Initiate all pending requests - for stream in self._pending_request_stream_pool: + while len(self._pending_request_stream_pool): + stream = self._pending_request_stream_pool.popleft() stream.initiate_request() - self._pending_request_stream_pool.clear() - def _write_to_transport(self): """ Write data to the underlying transport connection from the HTTP2 connection instance if any @@ -108,6 +108,8 @@ class H2ClientProtocol(Protocol): self._write_to_transport() self.is_connection_made = True + # Send off all the pending requests + # as now we have established a proper HTTP/2 connection self._send_pending_requests() def dataReceived(self, data): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a0b75850d..c2e1adce5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -132,7 +132,7 @@ class Stream: bytes_to_send_size = min(window_size, self.remaining_content_length) # We now need to send a number of data frames. - while bytes_to_send_size > 0: + while bytes_to_send_size: chunk_size = min(bytes_to_send_size, max_frame_size) data_chunk_start_id = self.content_length - self.remaining_content_length @@ -163,7 +163,7 @@ class Stream: Arguments: delta -- Window change delta """ - if self.remaining_content_length > 0 and not self.stream_closed_local: + if self.stream_closed_local is False: self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int): From 700df3eeb7c540892c8b2910db688f1cc5d1c845 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 17 Jun 2020 21:02:14 +0530 Subject: [PATCH 0093/2083] test: mockserver with h2 protocol for tests - add Twisted[http2] in setup.py requirements - add test_protocol.py to test the current implementation BREAKING CHANGES test_download times out because of no protocol negotiated between Mockserver and HTTP/2 client --- scrapy/core/http2/test_protocol.py | 67 +++++++++++++++++++++++++++ setup.py | 6 +-- tests/test_http2_client_protocol.py | 71 +++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 scrapy/core/http2/test_protocol.py create mode 100644 tests/test_http2_client_protocol.py diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.py new file mode 100644 index 000000000..c7782a518 --- /dev/null +++ b/scrapy/core/http2/test_protocol.py @@ -0,0 +1,67 @@ +# This is simple script to test + +import json + +from twisted.internet import reactor +from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint +from twisted.internet.ssl import optionsForClientTLS + +from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.http import Request, Response, JsonRequest + +try: + with open('data.json', 'r') as f: + JSON_DATA = json.load(f) +except: + JSON_DATA = { + "data": "To test for really large amount of data -- Add data.json with lots of data.", + "why": "To test whether correct data is sent :)" + } + +# Use nghttp2 for testing whether basic setup works - for small response +HTTPBIN_AUTHORITY = u'nghttp2.org' +HTTPBIN_REQUEST_URLS = 1 * [ + Request(url='https://nghttp2.org/httpbin/get', method='GET'), + Request(url='https://nghttp2.org/httpbin/post', method='POST'), + JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA), +] + +# Use POKE_API for testing large responses +POKE_API_AUTHORITY = u'pokeapi.co' +POKE_API_REQUESTS = 15 * [ + Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'), + Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'), + Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'), + Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'), # should give 404 +] + +AUTHORITY = POKE_API_AUTHORITY +REQUEST_URLS = POKE_API_REQUESTS + +options = optionsForClientTLS( + hostname=AUTHORITY, + acceptableProtocols=[b'h2'], +) + +protocol = H2ClientProtocol() + +count_responses = 1 + + +def print_response(response): + global count_responses + assert isinstance(response, Response) + print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body))) + count_responses = count_responses + 1 + + +for request in REQUEST_URLS: + d = protocol.request(request) + d.addCallback(print_response) + +connectProtocol( + SSL4ClientEndpoint(reactor, AUTHORITY, 443, options), + protocol +) + +reactor.run() diff --git a/setup.py b/setup.py index 1b3c6771a..dafa5684a 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from os.path import dirname, join + from pkg_resources import parse_version from setuptools import setup, find_packages, __version__ as setuptools_version - with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: version = f.read().decode('ascii').strip() @@ -25,12 +25,11 @@ if has_environment_marker_platform_impl_support(): 'PyPyDispatcher>=2.1.0', ] - setup( name='Scrapy', version=version, url='https://scrapy.org', - project_urls = { + project_urls={ 'Documentation': 'https://docs.scrapy.org/', 'Source': 'https://github.com/scrapy/scrapy', 'Tracker': 'https://github.com/scrapy/scrapy/issues', @@ -69,6 +68,7 @@ setup( python_requires='>=3.5', install_requires=[ 'Twisted>=17.9.0', + 'Twisted[http2]>=17.9.0' 'cryptography>=2.0', 'cssselect>=0.9.1', 'lxml>=3.5.0', diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py new file mode 100644 index 000000000..7830f7028 --- /dev/null +++ b/tests/test_http2_client_protocol.py @@ -0,0 +1,71 @@ +import os +import shutil + +from twisted.internet import defer, reactor +from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint +from twisted.internet.ssl import optionsForClientTLS +from twisted.protocols.policies import WrappingFactory +from twisted.python.filepath import FilePath +from twisted.trial import unittest +from twisted.web import static, server + +from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.http import Request +from tests.mockserver import ssl_context_factory + + +class Http2ClientProtocolTestCase(unittest.TestCase): + scheme = 'https' + + # only used for HTTPS tests + file_key = 'keys/localhost.key' + file_certificate = 'keys/localhost.crt' + + def setUp(self): + # Start server for testing + self.path_temp = self.mktemp() + os.mkdir(self.path_temp) + FilePath(self.path_temp).child('file').setContent(b"0123456789") + r = static.File(self.path_temp) + + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.host = 'localhost' + if self.scheme is 'https': + self.port = reactor.listenSSL( + 0, self.wrapper, + ssl_context_factory(self.file_key, self.file_certificate), + interface=self.host + ) + else: + self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) + + self.port_number = self.port.getHost().port + + # Connect to the server using the custom HTTP2ClientProtocol + options = optionsForClientTLS( + hostname=self.host, + acceptableProtocols=[b'h2'] + ) + + self.protocol = H2ClientProtocol() + + connectProtocol( + endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options), + protocol=self.protocol + ) + + def getURL(self, path): + return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path) + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + shutil.rmtree(self.path_temp) + + def test_download(self): + request = Request(self.getURL('file')) + d = self.protocol.request(request) + d.addCallback(lambda response: response.body) + d.addCallback(self.assertEqual, b"0123456789") + return d From 303485a9b4fd86c5123c96c81a9401b5e323a91d Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 21 Jun 2020 00:33:34 +0530 Subject: [PATCH 0094/2083] fix(http2): POST request not sending large body --- scrapy/core/http2/protocol.py | 51 +++++++----- scrapy/core/http2/stream.py | 148 ++++++++++++++++++++++++---------- 2 files changed, 138 insertions(+), 61 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index dbc048ffa..d6134183a 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,3 +1,4 @@ +import ipaddress import itertools import logging from collections import deque @@ -5,7 +6,7 @@ from collections import deque from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - ConnectionTerminated, DataReceived, ResponseReceived, + DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) from twisted.internet.protocol import connectionDone, Protocol @@ -49,6 +50,13 @@ class H2ClientProtocol(Protocol): self.is_connection_made = False self._pending_request_stream_pool = deque() + # Some meta data of this connection + # initialized when connection is successfully made + self._metadata = { + 'certificate': None, + 'ip_address': None + } + def _stream_close_cb(self, stream_id: int): """Called when stream is closed completely """ @@ -63,6 +71,7 @@ class H2ClientProtocol(Protocol): stream_id=stream_id, request=request, connection=self.conn, + metadata=self._metadata, write_to_transport=self._write_to_transport, cb_close=self._stream_close_cb ) @@ -73,7 +82,7 @@ class H2ClientProtocol(Protocol): def _send_pending_requests(self): # TODO: handle MAX_CONCURRENT_STREAMS # Initiate all pending requests - while len(self._pending_request_stream_pool): + while self._pending_request_stream_pool: stream = self._pending_request_stream_pool.popleft() stream.initiate_request() @@ -84,6 +93,8 @@ class H2ClientProtocol(Protocol): data = self.conn.data_to_send() self.transport.write(data) + LOGGER.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address'])) + def request(self, _request: Request): stream = self._new_stream(_request) d = stream.get_response() @@ -101,17 +112,16 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ - self.destination = self.transport.connector.getDestination() + self.destination = self.transport.getPeer() LOGGER.info('Connection made to {}'.format(self.destination)) + self._metadata['certificate'] = self.transport.getPeerCertificate() + self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) + self.conn.initiate_connection() self._write_to_transport() self.is_connection_made = True - # Send off all the pending requests - # as now we have established a proper HTTP/2 connection - self._send_pending_requests() - def dataReceived(self, data): events = self.conn.receive_data(data) self._handle_events(events) @@ -123,7 +133,7 @@ class H2ClientProtocol(Protocol): """ # Pop all streams which were pending and were not yet started for stream_id in list(self.streams): - self.streams[stream_id].lost_connection() + self.streams[stream_id].close() self.conn.close_connection() @@ -139,9 +149,7 @@ class H2ClientProtocol(Protocol): """ for event in events: LOGGER.debug(event) - if isinstance(event, ConnectionTerminated): - self.connection_terminated(event) - elif isinstance(event, DataReceived): + if isinstance(event, DataReceived): self.data_received(event) elif isinstance(event, ResponseReceived): self.response_received(event) @@ -151,13 +159,12 @@ class H2ClientProtocol(Protocol): self.stream_reset(event) elif isinstance(event, WindowUpdated): self.window_updated(event) + elif isinstance(event, SettingsAcknowledged): + self.settings_acknowledged(event) else: LOGGER.info("Received unhandled event {}".format(event)) # Event handler functions starts here - def connection_terminated(self, event: ConnectionTerminated): - pass - def data_received(self, event: DataReceived): stream_id = event.stream_id self.streams[stream_id].receive_data(event.data, event.flow_controlled_length) @@ -166,20 +173,26 @@ class H2ClientProtocol(Protocol): stream_id = event.stream_id self.streams[stream_id].receive_headers(event.headers) + def settings_acknowledged(self, event: SettingsAcknowledged): + # Send off all the pending requests + # as now we have established a proper HTTP/2 connection + self._send_pending_requests() + def stream_ended(self, event: StreamEnded): stream_id = event.stream_id - self.streams[stream_id].end_stream() + self.streams[stream_id].close() def stream_reset(self, event: StreamReset): # TODO: event.stream_id was abruptly closed # Q. What should be the response? (Failure/Partial/???) - self.streams[event.stream_id].reset() + self.streams[event.stream_id].close(event) def window_updated(self, event: WindowUpdated): stream_id = event.stream_id if stream_id != 0: self.streams[stream_id].receive_window_update(event.delta) else: - # TODO: - # Q. What to do when StreamID=0 ? - pass + # Send leftover data for all the streams + for stream in self.streams.values(): + if stream.request_sent: + stream.send_data() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index c2e1adce5..e8b4471d6 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,11 +1,15 @@ import logging +from typing import Dict from urllib.parse import urlparse from h2.connection import H2Connection +from h2.events import StreamEnded +from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred -from scrapy.http import Request, Response +from scrapy.http import Request from scrapy.http.headers import Headers +from scrapy.responsetypes import responsetypes LOGGER = logging.getLogger(__name__) @@ -22,12 +26,13 @@ class Stream: """ def __init__( - self, - stream_id: int, - request: Request, - connection: H2Connection, - write_to_transport, - cb_close + self, + stream_id: int, + request: Request, + connection: H2Connection, + metadata: Dict, + write_to_transport, + cb_close ): """ Arguments: @@ -35,6 +40,7 @@ class Stream: uniquely identified by a single integer request {Request} -- HTTP request connection {H2Connection} -- HTTP/2 connection this stream belongs to. + metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection write_to_transport {callable} -- Method used to write & send data to the server This method should be used whenever some frame is to be sent to the server. cb_close {callable} -- Method called when this stream is closed @@ -43,12 +49,16 @@ class Stream: self.stream_id = stream_id self._request = request self._conn = connection + self._metadata = metadata self._write_to_transport = write_to_transport self._cb_close = cb_close self._request_body = self._request.body self.content_length = 0 if self._request_body is None else len(self._request_body) + # Flag to keep track whether this stream has initiated the request + self.request_sent = False + # Each time we send a data frame, we will decrease value by the amount send. self.remaining_content_length = self.content_length @@ -58,20 +68,30 @@ class Stream: # Flag to keep track whether the server has closed the stream self.stream_closed_server = False - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. - self._response_data = b"" - # The amount of data received that counts against the flow control # window self._response_flow_controlled_size = 0 - # Headers received after sending the request - self._response_headers = Headers({}) + # Private variable used to build the response + # this response is then converted to appropriate Response class + # passed to the response deferred callback + self._response = { + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. + 'body': b'', + + # Headers received after sending the request + 'headers': Headers({}) + } # TODO: Add canceller for the Deferred below self._deferred_response = Deferred() + def __str__(self): + return "Stream(id={})".format(self.stream_id) + + __repr__ = __str__ + def get_response(self): """Simply return a Deferred which fires when response from the asynchronous request is available @@ -104,6 +124,8 @@ class Stream: self._conn.send_headers(self.stream_id, headers, end_stream=False) self._write_to_transport() + self.request_sent = True + self.send_data() def send_data(self): @@ -112,7 +134,18 @@ class Stream: If the content length is 0 initially then we end the stream immediately and wait for response data. + + Warning: Only call this method when stream not closed from client side + and has initiated request already by sending HEADER frame. If not then + stream will be closed from client side with 499 response. + + TODO: Q. Should we instead raise ProtocolError here with a proper message? """ + if self.stream_closed_local or self.stream_closed_server: + raise StreamClosedError(self.stream_id) + elif not self.request_sent: + self.close() + return # TODO: # 1. Add test for sending very large data @@ -132,7 +165,8 @@ class Stream: bytes_to_send_size = min(window_size, self.remaining_content_length) # We now need to send a number of data frames. - while bytes_to_send_size: + data_frames_sent = 0 + while bytes_to_send_size > 0: chunk_size = min(bytes_to_send_size, max_frame_size) data_chunk_start_id = self.content_length - self.remaining_content_length @@ -140,16 +174,24 @@ class Stream: self._conn.send_data(self.stream_id, data_chunk, end_stream=False) + data_frames_sent += 1 bytes_to_send_size = bytes_to_send_size - chunk_size self.remaining_content_length = self.remaining_content_length - chunk_size - # End the stream if no more data has to be send - if self.remaining_content_length <= 0: - self.remaining_content_length = 0 + self.remaining_content_length = max(0, self.remaining_content_length) + LOGGER.debug("{} sending {}/{} data bytes ({} frames) to {}".format( + self, + self.content_length - self.remaining_content_length, self.content_length, + data_frames_sent, + self._metadata['ip_address']) + ) + # End the stream if no more data needs to be send + if self.remaining_content_length == 0: self.stream_closed_local = True self._conn.end_stream(self.stream_id) + # Write data to transport -- Empty the outstanding data self._write_to_transport() # Q. What about the rest of the data? @@ -163,11 +205,11 @@ class Stream: Arguments: delta -- Window change delta """ - if self.stream_closed_local is False: + if self.remaining_content_length > 0 and not self.stream_closed_server: self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int): - self._response_data += data + self._response['body'] += data self._response_flow_controlled_size += flow_controlled_length # Acknowledge the data received @@ -178,47 +220,69 @@ class Stream: def receive_headers(self, headers): for name, value in headers: - self._response_headers[name] = value + self._response['headers'][name] = value - def reset(self): - """Received a RST_STREAM -- forcefully reset""" - # TODO: - # Q1. Do we need to send the request again? - # Q2. What response should we send now? - self.stream_closed_server = True - self._cb_close(self.stream_id) + def close(self, event=None): + """Based on the event sent we will handle each case. - def lost_connection(self): - # TODO: Same as self.reset - self.stream_closed_server = True - self._cb_close(self.stream_id) - - def end_stream(self): - """Stream is ended by the server hence no further + event: StreamEnded + Stream is ended by the server hence no further data or headers should be expected on this stream. - We will call the response deferred callback passing the response object + + event: StreamReset + Stream reset via RST_FRAME by the upstream hence forcefully close + this stream and send TODO: ? + + event: None + No event is launched -- Hence we will simply close this stream """ + # TODO: In case of abruptly stream close + # Q1. Do we need to send the request again? + # Q2. What response should we send now? assert self.stream_closed_server is False self.stream_closed_server = True + if not isinstance(event, StreamEnded): + # TODO + # Stream was abruptly ended here + # Partial - Content-Length header not provided + pass + self._fire_response_deferred() self._cb_close(self.stream_id) - def _fire_response_deferred(self): + def _fire_response_deferred(self, flags=None): + """Builds response from the self._response dict + and fires the response deferred callback with the + generated response instance""" # TODO: # 1. Set flags, certificate, ip_address in response # 2. Should we fire this in case of # 2.1 StreamReset in between when data is received partially # 2.2 Forcefully closed the stream + # 3. Update Client Side Status Codes here - # NOTE: Presently on fired with successful response - response = Response( + response_cls = responsetypes.from_args( + headers=self._response['headers'], url=self._request.url, - status=self._response_headers[":status"], - headers=self._response_headers, - body=self._response_data, - request=self._request + body=self._response['body'] ) + + # If there is :status in headers then + # HTTP Status Code: 499 - Client Closed Request + status = self._response['headers'].get(':status', '499') + + response = response_cls( + url=self._request.url, + status=status, + headers=self._response['headers'], + body=self._response['body'], + request=self._request, + flags=flags, + certificate=self._metadata['certificate'], + ip_address=self._metadata['ip_address'] + ) + self._deferred_response.callback(response) From c74ef660c7b28cd69825a4b9843230a383f702f4 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 21 Jun 2020 09:34:23 +0530 Subject: [PATCH 0095/2083] feat: handle response for different reasons - Add StreamCloseReason enum - Send response for different cases considering download_warnsize, download_maxsize, fail_on_data_loss, connection lost, etc. --- scrapy/core/http2/protocol.py | 15 ++- scrapy/core/http2/stream.py | 209 ++++++++++++++++++++++++---------- 2 files changed, 155 insertions(+), 69 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index d6134183a..188c14c15 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -11,7 +11,7 @@ from h2.events import ( ) from twisted.internet.protocol import connectionDone, Protocol -from scrapy.core.http2.stream import Stream +from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.http import Request LOGGER = logging.getLogger(__name__) @@ -71,7 +71,7 @@ class H2ClientProtocol(Protocol): stream_id=stream_id, request=request, connection=self.conn, - metadata=self._metadata, + conn_metadata=self._metadata, write_to_transport=self._write_to_transport, cb_close=self._stream_close_cb ) @@ -133,7 +133,7 @@ class H2ClientProtocol(Protocol): """ # Pop all streams which were pending and were not yet started for stream_id in list(self.streams): - self.streams[stream_id].close() + self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST) self.conn.close_connection() @@ -180,19 +180,18 @@ class H2ClientProtocol(Protocol): def stream_ended(self, event: StreamEnded): stream_id = event.stream_id - self.streams[stream_id].close() + self.streams[stream_id].close(StreamCloseReason.ENDED) def stream_reset(self, event: StreamReset): # TODO: event.stream_id was abruptly closed # Q. What should be the response? (Failure/Partial/???) - self.streams[event.stream_id].close(event) + self.streams[event.stream_id].close(StreamCloseReason.RESET) def window_updated(self, event: WindowUpdated): stream_id = event.stream_id if stream_id != 0: - self.streams[stream_id].receive_window_update(event.delta) + self.streams[stream_id].receive_window_update() else: # Send leftover data for all the streams for stream in self.streams.values(): - if stream.request_sent: - stream.send_data() + stream.receive_window_update() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index e8b4471d6..07a4428c8 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,11 +1,15 @@ import logging +from enum import IntFlag, auto +from io import BytesIO from typing import Dict from urllib.parse import urlparse from h2.connection import H2Connection -from h2.events import StreamEnded +from h2.errors import ErrorCodes from h2.exceptions import StreamClosedError -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, CancelledError +from twisted.python.failure import Failure +from twisted.web.client import ResponseFailed from scrapy.http import Request from scrapy.http.headers import Headers @@ -14,6 +18,20 @@ from scrapy.responsetypes import responsetypes LOGGER = logging.getLogger(__name__) +class StreamCloseReason(IntFlag): + # Received a StreamEnded event + ENDED = auto() + + # Received a StreamReset event -- ended abruptly + RESET = auto() + + # Transport connection was lost + CONNECTION_LOST = auto() + + # Expected response body size is more than allowed limit + MAXSIZE_EXCEEDED = auto() + + class Stream: """Represents a single HTTP/2 Stream. @@ -26,13 +44,16 @@ class Stream: """ def __init__( - self, - stream_id: int, - request: Request, - connection: H2Connection, - metadata: Dict, - write_to_transport, - cb_close + self, + stream_id: int, + request: Request, + connection: H2Connection, + conn_metadata: Dict, + write_to_transport, + cb_close, + download_maxsize=0, + download_warnsize=0, + fail_on_data_loss=True ): """ Arguments: @@ -40,7 +61,7 @@ class Stream: uniquely identified by a single integer request {Request} -- HTTP request connection {H2Connection} -- HTTP/2 connection this stream belongs to. - metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection + conn_metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection write_to_transport {callable} -- Method used to write & send data to the server This method should be used whenever some frame is to be sent to the server. cb_close {callable} -- Method called when this stream is closed @@ -49,16 +70,24 @@ class Stream: self.stream_id = stream_id self._request = request self._conn = connection - self._metadata = metadata + self._conn_metadata = conn_metadata self._write_to_transport = write_to_transport self._cb_close = cb_close - self._request_body = self._request.body - self.content_length = 0 if self._request_body is None else len(self._request_body) + self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) + self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) + self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss) + + self.request_start_time = None + + self.content_length = 0 if self._request.body is None else len(self._request.body) # Flag to keep track whether this stream has initiated the request self.request_sent = False + # Flag to track whether we have logged about exceeding download warnsize + self._reached_warnsize = False + # Each time we send a data frame, we will decrease value by the amount send. self.remaining_content_length = self.content_length @@ -68,17 +97,17 @@ class Stream: # Flag to keep track whether the server has closed the stream self.stream_closed_server = False - # The amount of data received that counts against the flow control - # window - self._response_flow_controlled_size = 0 - # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback self._response = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. - 'body': b'', + 'body': BytesIO(), + + # The amount of data received that counts against the flow control + # window + 'flow_controlled_size': 0, # Headers received after sending the request 'headers': Headers({}) @@ -137,16 +166,8 @@ class Stream: Warning: Only call this method when stream not closed from client side and has initiated request already by sending HEADER frame. If not then - stream will be closed from client side with 499 response. - - TODO: Q. Should we instead raise ProtocolError here with a proper message? + stream will raise ProtocolError (raise by h2 state machine). """ - if self.stream_closed_local or self.stream_closed_server: - raise StreamClosedError(self.stream_id) - elif not self.request_sent: - self.close() - return - # TODO: # 1. Add test for sending very large data # 2. Add test for small data @@ -154,6 +175,9 @@ class Stream: # 3.1 Large number of request # 3.2 Small number of requests + if self.stream_closed_local: + raise StreamClosedError(self.stream_id) + # Firstly, check what the flow control window is for current stream. window_size = self._conn.local_flow_control_window(stream_id=self.stream_id) @@ -170,7 +194,7 @@ class Stream: chunk_size = min(bytes_to_send_size, max_frame_size) data_chunk_start_id = self.content_length - self.remaining_content_length - data_chunk = self._request_body[data_chunk_start_id:data_chunk_start_id + chunk_size] + data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size] self._conn.send_data(self.stream_id, data_chunk, end_stream=False) @@ -183,7 +207,7 @@ class Stream: self, self.content_length - self.remaining_content_length, self.content_length, data_frames_sent, - self._metadata['ip_address']) + self._conn_metadata['ip_address']) ) # End the stream if no more data needs to be send @@ -197,24 +221,40 @@ class Stream: # Q. What about the rest of the data? # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame - def receive_window_update(self, delta): + def receive_window_update(self): """Flow control window size was changed. Send data that earlier could not be sent as we were blocked behind the flow control. - - Arguments: - delta -- Window change delta """ - if self.remaining_content_length > 0 and not self.stream_closed_server: + if self.remaining_content_length and not self.stream_closed_server and self.request_sent: self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int): - self._response['body'] += data - self._response_flow_controlled_size += flow_controlled_length + self._response['body'].write(data) + self._response['flow_controlled_size'] += flow_controlled_length + + if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize: + # Clear buffer earlier to avoid keeping data in memory for a long time + self._response['body'].truncate(0) + self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) + return + + if self._download_warnsize \ + and self._response['flow_controlled_size'] > self._download_warnsize \ + and not self._reached_warnsize: + self._reached_warnsize = True + warning_msg = ('Received more ({bytes}) bytes than download ', + 'warn size ({warnsize}) in request {request}') + warning_args = { + 'bytes': self._response['flow_controlled_size'], + 'warnsize': self._download_warnsize, + 'request': self._request + } + LOGGER.warning(warning_msg, warning_args) # Acknowledge the data received self._conn.acknowledge_received_data( - self._response_flow_controlled_size, + self._response['flow_controlled_size'], self.stream_id ) @@ -222,35 +262,83 @@ class Stream: for name, value in headers: self._response['headers'][name] = value - def close(self, event=None): - """Based on the event sent we will handle each case. + # Check if we exceed the allowed max data size which can be received + expected_size = int(self._response['headers'].get(b'Content-Length', -1)) + if self._download_maxsize and expected_size > self._download_maxsize: + self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) + return - event: StreamEnded - Stream is ended by the server hence no further - data or headers should be expected on this stream. - We will call the response deferred callback passing - the response object + if self._download_warnsize and expected_size > self._download_warnsize: + warning_msg = ("Expected response size ({size}) larger than ", + "download warn size ({warnsize}) in request {request}.") + warning_args = { + 'size': expected_size, 'warnsize': self._download_warnsize, + 'request': self._request + } + LOGGER.warning(warning_msg, warning_args) - event: StreamReset - Stream reset via RST_FRAME by the upstream hence forcefully close - this stream and send TODO: ? + def reset_stream(self, reason=StreamCloseReason.RESET): + """Close this stream by sending a RST_FRAME to the remote peer""" + # TODO: Q. REFUSED_STREAM or CANCEL ? + if self.stream_closed_local: + raise StreamClosedError(self.stream_id) - event: None - No event is launched -- Hence we will simply close this stream + self.stream_closed_local = True + self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) + self._write_to_transport() + self.close(reason) + + def _is_data_lost(self) -> bool: + assert self.stream_closed_server + + expected_size = self._response['flow_controlled_size'] + received_body_size = int(self._response['headers'][b'Content-Length']) + + return expected_size != received_body_size + + def close(self, reason: StreamCloseReason): + """Based on the reason sent we will handle each case. """ # TODO: In case of abruptly stream close # Q1. Do we need to send the request again? # Q2. What response should we send now? - assert self.stream_closed_server is False + if self.stream_closed_server: + raise StreamClosedError(self.stream_id) + self.stream_closed_server = True - if not isinstance(event, StreamEnded): - # TODO - # Stream was abruptly ended here - # Partial - Content-Length header not provided - pass + flags = None + if b'Content-Length' not in self._response['headers']: + # Missing Content-Length - PotentialDataLoss + flags = ['partial'] + elif self._is_data_lost(): + if self._fail_on_dataloss: + self._deferred_response.errback(ResponseFailed([Failure()])) + self._cb_close(self.stream_id) + return + else: + flags = ['dataloss'] + + if reason is StreamCloseReason.ENDED: + self._fire_response_deferred(flags) + + elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST): + # Stream was abruptly ended here + self._deferred_response.errback(ResponseFailed([Failure()])) + + elif reason is StreamCloseReason.MAXSIZE_EXCEEDED: + expected_size = int(self._response['headers'].get(b'Content-Length', -1)) + error_msg = ("Cancelling download of {url}: expected response " + "size ({size}) larger than download max size ({maxsize}).") + error_args = { + 'url': self._request.url, + 'size': expected_size, + 'maxsize': self._download_maxsize + } + + LOGGER.error(error_msg, error_args) + self._deferred_response.errback(CancelledError(error_msg.format(**error_args))) - self._fire_response_deferred() self._cb_close(self.stream_id) def _fire_response_deferred(self, flags=None): @@ -258,7 +346,6 @@ class Stream: and fires the response deferred callback with the generated response instance""" # TODO: - # 1. Set flags, certificate, ip_address in response # 2. Should we fire this in case of # 2.1 StreamReset in between when data is received partially # 2.2 Forcefully closed the stream @@ -270,7 +357,7 @@ class Stream: body=self._response['body'] ) - # If there is :status in headers then + # If there is no :status in headers then # HTTP Status Code: 499 - Client Closed Request status = self._response['headers'].get(':status', '499') @@ -278,11 +365,11 @@ class Stream: url=self._request.url, status=status, headers=self._response['headers'], - body=self._response['body'], + body=self._response['body'].getvalue(), request=self._request, flags=flags, - certificate=self._metadata['certificate'], - ip_address=self._metadata['ip_address'] + certificate=self._conn_metadata['certificate'], + ip_address=self._conn_metadata['ip_address'] ) self._deferred_response.callback(response) From a97ac0adf86b67a16f09c41f618f736adb872503 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 24 Jun 2020 06:40:20 +0530 Subject: [PATCH 0096/2083] test: GET request for HTTP2Client using mockserver --- scrapy/core/http2/protocol.py | 4 +- scrapy/core/http2/stream.py | 10 +--- tests/test_http2_client_protocol.py | 71 ++++++++++------------------- 3 files changed, 30 insertions(+), 55 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 188c14c15..455d8777e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -9,6 +9,8 @@ from h2.events import ( DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) + +from twisted.internet.ssl import Certificate from twisted.internet.protocol import connectionDone, Protocol from scrapy.core.http2.stream import Stream, StreamCloseReason @@ -115,7 +117,7 @@ class H2ClientProtocol(Protocol): self.destination = self.transport.getPeer() LOGGER.info('Connection made to {}'.format(self.destination)) - self._metadata['certificate'] = self.transport.getPeerCertificate() + self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) self.conn.initiate_connection() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 07a4428c8..112ce5bcd 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -117,7 +117,7 @@ class Stream: self._deferred_response = Deferred() def __str__(self): - return "Stream(id={})".format(self.stream_id) + return "Stream(id={})".format(repr(self.stream_id)) __repr__ = __str__ @@ -299,9 +299,6 @@ class Stream: def close(self, reason: StreamCloseReason): """Based on the reason sent we will handle each case. """ - # TODO: In case of abruptly stream close - # Q1. Do we need to send the request again? - # Q2. What response should we send now? if self.stream_closed_server: raise StreamClosedError(self.stream_id) @@ -346,10 +343,7 @@ class Stream: and fires the response deferred callback with the generated response instance""" # TODO: - # 2. Should we fire this in case of - # 2.1 StreamReset in between when data is received partially - # 2.2 Forcefully closed the stream - # 3. Update Client Side Status Codes here + # 1. Update Client Side Status Codes here response_cls = responsetypes.from_args( headers=self._response['headers'], diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7830f7028..a67575d3c 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,71 +1,50 @@ -import os -import shutil +from urllib.parse import urlparse -from twisted.internet import defer, reactor +from twisted.internet import reactor from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint -from twisted.internet.ssl import optionsForClientTLS -from twisted.protocols.policies import WrappingFactory -from twisted.python.filepath import FilePath +from twisted.internet.ssl import CertificateOptions from twisted.trial import unittest -from twisted.web import static, server from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.http import Request -from tests.mockserver import ssl_context_factory +from scrapy.http import Request, Response +from tests.mockserver import MockServer class Http2ClientProtocolTestCase(unittest.TestCase): scheme = 'https' - # only used for HTTPS tests - file_key = 'keys/localhost.key' - file_certificate = 'keys/localhost.crt' - def setUp(self): # Start server for testing - self.path_temp = self.mktemp() - os.mkdir(self.path_temp) - FilePath(self.path_temp).child('file').setContent(b"0123456789") - r = static.File(self.path_temp) + self.mockserver = MockServer() + self.mockserver.__enter__() - self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) - self.host = 'localhost' - if self.scheme is 'https': - self.port = reactor.listenSSL( - 0, self.wrapper, - ssl_context_factory(self.file_key, self.file_certificate), - interface=self.host - ) + if self.scheme == 'https': + self.url = urlparse(self.mockserver.https_address) else: - self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) - - self.port_number = self.port.getHost().port - - # Connect to the server using the custom HTTP2ClientProtocol - options = optionsForClientTLS( - hostname=self.host, - acceptableProtocols=[b'h2'] - ) + self.url = urlparse(self.mockserver.http_address) self.protocol = H2ClientProtocol() - connectProtocol( - endpoint=SSL4ClientEndpoint(reactor, self.host, self.port_number, options), - protocol=self.protocol - ) + # Connect to the server using the custom HTTP2ClientProtocol + options = CertificateOptions(acceptableProtocols=[b'h2']) + endpoint = SSL4ClientEndpoint(reactor, self.url.hostname, self.url.port, options) + connectProtocol(endpoint, self.protocol) def getURL(self, path): - return "%s://%s:%d/%s" % (self.scheme, self.host, self.port_number, path) + return "{}://{}:{}/{}".format(self.url.scheme, self.url.hostname, self.url.port, path) - @defer.inlineCallbacks def tearDown(self): - yield self.port.stopListening() - shutil.rmtree(self.path_temp) + self.mockserver.__exit__(None, None, None) def test_download(self): - request = Request(self.getURL('file')) + request = Request(self.getURL('')) + + def assert_response(response: Response): + self.assertEqual(response.body, b'Scrapy mock HTTP server\n') + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + d = self.protocol.request(request) - d.addCallback(lambda response: response.body) - d.addCallback(self.assertEqual, b"0123456789") + d.addCallback(assert_response) return d From 69f6d038c0bc51a9de706890621d2ce183f79e09 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 24 Jun 2020 07:06:32 +0530 Subject: [PATCH 0097/2083] feat: TypedDict for Stream._response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - remove test_protocol.py as working testing environment is setup 🙂🙃 - Add typing_extensions as dependency to support TypedDict for python<3.8 --- scrapy/core/http2/protocol.py | 12 +----- scrapy/core/http2/stream.py | 11 ++++- scrapy/core/http2/test_protocol.py | 67 ------------------------------ setup.py | 4 +- 4 files changed, 15 insertions(+), 79 deletions(-) delete mode 100644 scrapy/core/http2/test_protocol.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 455d8777e..9b8ec6c77 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -9,9 +9,8 @@ from h2.events import ( DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) - -from twisted.internet.ssl import Certificate from twisted.internet.protocol import connectionDone, Protocol +from twisted.internet.ssl import Certificate from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.http import Request @@ -22,12 +21,7 @@ LOGGER = logging.getLogger(__name__) class H2ClientProtocol(Protocol): # TODO: # 1. Check for user-agent while testing - # 2. Add support for cookies - # 3. Handle priority updates (Not required) - # 4. Handle case when received events have StreamID = 0 (applied to H2Connection) - # 1 & 2: - # - Automatically handled by the Request middleware - # - request.headers will have 'Set-Cookie' value + # 2. Handle case when received events have StreamID = 0 (applied to H2Connection) def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') @@ -185,8 +179,6 @@ class H2ClientProtocol(Protocol): self.streams[stream_id].close(StreamCloseReason.ENDED) def stream_reset(self, event: StreamReset): - # TODO: event.stream_id was abruptly closed - # Q. What should be the response? (Failure/Partial/???) self.streams[event.stream_id].close(StreamCloseReason.RESET) def window_updated(self, event: WindowUpdated): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 112ce5bcd..023f4f4eb 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -10,11 +10,20 @@ from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred, CancelledError from twisted.python.failure import Failure from twisted.web.client import ResponseFailed +# for python < 3.8 -- typing.TypedDict is undefined +from typing_extensions import TypedDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes + +class _ResponseTypedDict(TypedDict): + body: BytesIO + flow_controlled_size: int + headers: Headers + + LOGGER = logging.getLogger(__name__) @@ -100,7 +109,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response = { + self._response: _ResponseTypedDict = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. 'body': BytesIO(), diff --git a/scrapy/core/http2/test_protocol.py b/scrapy/core/http2/test_protocol.py deleted file mode 100644 index c7782a518..000000000 --- a/scrapy/core/http2/test_protocol.py +++ /dev/null @@ -1,67 +0,0 @@ -# This is simple script to test - -import json - -from twisted.internet import reactor -from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint -from twisted.internet.ssl import optionsForClientTLS - -from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.http import Request, Response, JsonRequest - -try: - with open('data.json', 'r') as f: - JSON_DATA = json.load(f) -except: - JSON_DATA = { - "data": "To test for really large amount of data -- Add data.json with lots of data.", - "why": "To test whether correct data is sent :)" - } - -# Use nghttp2 for testing whether basic setup works - for small response -HTTPBIN_AUTHORITY = u'nghttp2.org' -HTTPBIN_REQUEST_URLS = 1 * [ - Request(url='https://nghttp2.org/httpbin/get', method='GET'), - Request(url='https://nghttp2.org/httpbin/post', method='POST'), - JsonRequest(url='https://nghttp2.org/httpbin/anything', method='POST', data=JSON_DATA), -] - -# Use POKE_API for testing large responses -POKE_API_AUTHORITY = u'pokeapi.co' -POKE_API_REQUESTS = 15 * [ - Request(url='https://pokeapi.co/api/v2/pokemon/ditto', method='GET'), - Request(url='https://pokeapi.co/api/v2/pokemon/charizard', method='GET'), - Request(url='https://pokeapi.co/api/v2/pokemon/pikachu', method='GET'), - Request(url='https://pokeapi.co/api/v2/pokemon/DoesNotExist', method='GET'), # should give 404 -] - -AUTHORITY = POKE_API_AUTHORITY -REQUEST_URLS = POKE_API_REQUESTS - -options = optionsForClientTLS( - hostname=AUTHORITY, - acceptableProtocols=[b'h2'], -) - -protocol = H2ClientProtocol() - -count_responses = 1 - - -def print_response(response): - global count_responses - assert isinstance(response, Response) - print('({})\t{}: ReponseBodySize={}'.format(count_responses, response, len(response.body))) - count_responses = count_responses + 1 - - -for request in REQUEST_URLS: - d = protocol.request(request) - d.addCallback(print_response) - -connectProtocol( - SSL4ClientEndpoint(reactor, AUTHORITY, 443, options), - protocol -) - -reactor.run() diff --git a/setup.py b/setup.py index dafa5684a..d1470df5e 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=3.5', + python_requires='>=3.5.2', install_requires=[ 'Twisted>=17.9.0', 'Twisted[http2]>=17.9.0' @@ -80,6 +80,8 @@ setup( 'w3lib>=1.17.0', 'zope.interface>=4.1.3', 'protego>=0.1.15', + 'itemadapter>=0.1.0', + 'typing_extensions>=3.7' ], extras_require=extras_require, ) From 23da8e106822cf2805e8d74c382a31bca70986f3 Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Sat, 27 Jun 2020 20:36:45 +0530 Subject: [PATCH 0098/2083] Add schemaless http proxy support --- scrapy/core/downloader/handlers/http11.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 22c9ac520..73e56c87d 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -7,7 +7,7 @@ import warnings from contextlib import suppress from io import BytesIO from time import time -from urllib.parse import urldefrag +from urllib.parse import urldefrag, urlparse from twisted.internet import defer, protocol, ssl from twisted.internet.endpoints import TCP4ClientEndpoint @@ -255,7 +255,7 @@ class ScrapyProxyAgent(Agent): bindAddress=bindAddress, pool=pool, ) - self._proxyURI = URI.fromBytes(proxyURI) + self._proxyURI = URI.fromBytes(urlparse(proxyURI)._replace(scheme=b'http').geturl()) def request(self, method, uri, headers=None, bodyProducer=None): """ From f53f06020b740b450c7555d562565a27918e8036 Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Sat, 27 Jun 2020 23:28:40 +0530 Subject: [PATCH 0099/2083] Test http schemaless proxy --- tests/test_downloader_handlers.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 51deb20f4..5854659dd 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -131,6 +131,7 @@ class ContentLengthHeaderResource(resource.Resource): A testing resource which renders itself as the value of the Content-Length header from the request. """ + def render(self, request): return request.requestHeaders.getRawHeaders(b"content-length")[0] @@ -186,6 +187,7 @@ class EmptyContentTypeHeaderResource(resource.Resource): A testing resource which renders itself as the value of request body without content-type header in response. """ + def render(self, request): request.setHeader("content-type", "") return request.content.read() @@ -733,6 +735,16 @@ class Http11ProxyTestCase(HttpProxyTestCase): timeout = yield self.assertFailure(d, error.TimeoutError) self.assertIn(domain, timeout.osError) + def test_download_with_proxy_without_http_scheme(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'http://example.com') + + http_proxy = self.getURL('').replace('http:', '') + request = Request('http://example.com', meta={'proxy': http_proxy}) + return self.download_request(request, Spider('foo')).addCallback(_test) + class HttpDownloadHandlerMock: From 690dd7f38bfd245428bdc618dcca1fbc26284df1 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 28 Jun 2020 16:35:04 +0530 Subject: [PATCH 0100/2083] test: GET & POST request test for h2 client - Remove repeated dependency Twisted from setup.py - Test for both GET & POST when - Only 1 request - Large number (=20) of requests and - Small Data (10 KB) per request - Large Data (10 MB) per request - Test when request is cancelled by the client' BREAKING CHANGES Tests raises OpenSSL.SSL.Error when run using tox. However, all tests passes when ran using `python -m unittest`. --- scrapy/core/http2/protocol.py | 4 +- scrapy/core/http2/stream.py | 66 +++--- setup.py | 1 - tests/test_http2_client_protocol.py | 321 +++++++++++++++++++++++++--- 4 files changed, 331 insertions(+), 61 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 9b8ec6c77..7fb935f10 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -129,11 +129,11 @@ class H2ClientProtocol(Protocol): """ # Pop all streams which were pending and were not yet started for stream_id in list(self.streams): - self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST) + self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST, reason) self.conn.close_connection() - LOGGER.info("Connection lost with reason " + str(reason)) + LOGGER.warning("Connection lost with reason " + str(reason)) def _handle_events(self, events): """Private method which acts as a bridge between the events diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 023f4f4eb..a26a33918 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -19,8 +19,15 @@ from scrapy.responsetypes import responsetypes class _ResponseTypedDict(TypedDict): + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. body: BytesIO + + # The amount of data received that counts against the flow control + # window flow_controlled_size: int + + # Headers received after sending the request headers: Headers @@ -40,6 +47,9 @@ class StreamCloseReason(IntFlag): # Expected response body size is more than allowed limit MAXSIZE_EXCEEDED = auto() + # When the response deferred is cancelled + CANCELLED = auto() + class Stream: """Represents a single HTTP/2 Stream. @@ -110,20 +120,16 @@ class Stream: # this response is then converted to appropriate Response class # passed to the response deferred callback self._response: _ResponseTypedDict = { - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. 'body': BytesIO(), - - # The amount of data received that counts against the flow control - # window 'flow_controlled_size': 0, - - # Headers received after sending the request 'headers': Headers({}) } - # TODO: Add canceller for the Deferred below - self._deferred_response = Deferred() + def _cancel(_): + # Close this stream as gracefully as possible :) + self.reset_stream(StreamCloseReason.CANCELLED) + + self._deferred_response = Deferred(_cancel) def __str__(self): return "Stream(id={})".format(repr(self.stream_id)) @@ -177,13 +183,6 @@ class Stream: and has initiated request already by sending HEADER frame. If not then stream will raise ProtocolError (raise by h2 state machine). """ - # TODO: - # 1. Add test for sending very large data - # 2. Add test for small data - # 3. Both (1) and (2) should be tested for - # 3.1 Large number of request - # 3.2 Small number of requests - if self.stream_closed_local: raise StreamClosedError(self.stream_id) @@ -221,7 +220,6 @@ class Stream: # End the stream if no more data needs to be send if self.remaining_content_length == 0: - self.stream_closed_local = True self._conn.end_stream(self.stream_id) # Write data to transport -- Empty the outstanding data @@ -288,7 +286,6 @@ class Stream: def reset_stream(self, reason=StreamCloseReason.RESET): """Close this stream by sending a RST_FRAME to the remote peer""" - # TODO: Q. REFUSED_STREAM or CANCEL ? if self.stream_closed_local: raise StreamClosedError(self.stream_id) @@ -305,14 +302,16 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason): + def close(self, reason: StreamCloseReason, failure=None): """Based on the reason sent we will handle each case. """ if self.stream_closed_server: raise StreamClosedError(self.stream_id) + self._cb_close(self.stream_id) self.stream_closed_server = True + # Do nothing if the response deferred was cancelled flags = None if b'Content-Length' not in self._response['headers']: # Missing Content-Length - PotentialDataLoss @@ -320,7 +319,6 @@ class Stream: elif self._is_data_lost(): if self._fail_on_dataloss: self._deferred_response.errback(ResponseFailed([Failure()])) - self._cb_close(self.stream_id) return else: flags = ['dataloss'] @@ -328,10 +326,19 @@ class Stream: if reason is StreamCloseReason.ENDED: self._fire_response_deferred(flags) - elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST): - # Stream was abruptly ended here - self._deferred_response.errback(ResponseFailed([Failure()])) + # Stream was abruptly ended here + elif reason is StreamCloseReason.CANCELLED: + # Client has cancelled the request. Remove all the data + # received and fire the response deferred with no flags set + self._response['body'].truncate(0) + self._response['headers'].clear() + self._fire_response_deferred() + elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST): + if failure is None: + self._deferred_response.errback(ResponseFailed([Failure()])) + else: + self._deferred_response.errback(failure) elif reason is StreamCloseReason.MAXSIZE_EXCEEDED: expected_size = int(self._response['headers'].get(b'Content-Length', -1)) error_msg = ("Cancelling download of {url}: expected response " @@ -345,22 +352,21 @@ class Stream: LOGGER.error(error_msg, error_args) self._deferred_response.errback(CancelledError(error_msg.format(**error_args))) - self._cb_close(self.stream_id) - def _fire_response_deferred(self, flags=None): """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" - # TODO: - # 1. Update Client Side Status Codes here + # TODO: Update Client Side Status Codes here + body = self._response['body'].getvalue() response_cls = responsetypes.from_args( headers=self._response['headers'], url=self._request.url, - body=self._response['body'] + body=body ) - # If there is no :status in headers then + # If there is no :status in headers + # (happens when client called response_deferred.cancel()) # HTTP Status Code: 499 - Client Closed Request status = self._response['headers'].get(':status', '499') @@ -368,7 +374,7 @@ class Stream: url=self._request.url, status=status, headers=self._response['headers'], - body=self._response['body'].getvalue(), + body=body, request=self._request, flags=flags, certificate=self._conn_metadata['certificate'], diff --git a/setup.py b/setup.py index d1470df5e..575c74e7f 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,6 @@ setup( ], python_requires='>=3.5.2', install_requires=[ - 'Twisted>=17.9.0', 'Twisted[http2]>=17.9.0' 'cryptography>=2.0', 'cssselect>=0.9.1', diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index a67575d3c..0f3730c9e 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,50 +1,315 @@ -from urllib.parse import urlparse +# TODO: Add test cases for +# 1. No Content Length response header +# 2. Cancel Response Deferred +import json +import os +import random +import shutil +import string from twisted.internet import reactor -from twisted.internet.endpoints import connectProtocol, SSL4ClientEndpoint -from twisted.internet.ssl import CertificateOptions -from twisted.trial import unittest +from twisted.internet.defer import inlineCallbacks, DeferredList +from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint +from twisted.internet.protocol import Factory +from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate +from twisted.trial.unittest import TestCase +from twisted.web.http import Request as TxRequest +from twisted.web.resource import Resource +from twisted.web.server import Site +from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.http import Request, Response -from tests.mockserver import MockServer +from scrapy.http import Request, Response, JsonRequest +from tests.mockserver import ssl_context_factory -class Http2ClientProtocolTestCase(unittest.TestCase): +def generate_random_string(size): + return ''.join(random.choices( + string.ascii_uppercase + string.digits, + k=size + )) + + +def make_html_body(val): + response = ''' +

Hello from HTTP2

+

{}

+'''.format(val) + return bytes(response, 'utf-8') + + +class Data: + SMALL_SIZE = 1024 * 10 # 10 KB + LARGE_SIZE = (1024 ** 2) * 10 # 10 MB + + STR_SMALL = generate_random_string(SMALL_SIZE) + STR_LARGE = generate_random_string(LARGE_SIZE) + + EXTRA_SMALL = generate_random_string(1024 * 15) + EXTRA_LARGE = generate_random_string((1024 ** 2) * 15) + + HTML_SMALL = make_html_body(STR_SMALL) + HTML_LARGE = make_html_body(STR_LARGE) + + JSON_SMALL = {'data': STR_SMALL} + JSON_LARGE = {'data': STR_LARGE} + + +class LeafResource(Resource): + isLeaf = True + + +class GetDataHtmlSmall(LeafResource): + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'text/html; charset=UTF-8') + return Data.HTML_SMALL + + +class GetDataHtmlLarge(LeafResource): + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'text/html; charset=UTF-8') + return Data.HTML_LARGE + + +class PostDataJsonMixin: + @staticmethod + def make_response(request: TxRequest, extra_data: str): + response = { + 'request-headers': {}, + 'request-body': json.loads(request.content.read()), + 'extra-data': extra_data + } + for k, v in request.requestHeaders.getAllRawHeaders(): + response['request-headers'][k.decode('utf-8')] = v[0].decode('utf-8') + + response_bytes = bytes(json.dumps(response), 'utf-8') + request.setHeader('Content-Type', 'application/json') + return response_bytes + + +class PostDataJsonSmall(LeafResource, PostDataJsonMixin): + def render_POST(self, request: TxRequest): + return self.make_response(request, Data.EXTRA_SMALL) + + +class PostDataJsonLarge(LeafResource, PostDataJsonMixin): + def render_POST(self, request: TxRequest): + return self.make_response(request, Data.EXTRA_LARGE) + + +def get_client_certificate(key_file, certificate_file): + with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: + pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) + + return PrivateCertificate.loadPEM(pem) + + +class Https2ClientProtocolTestCase(TestCase): scheme = 'https' + key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key') + certificate_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.crt') + def _init_resource(self): + self.temp_directory = self.mktemp() + os.mkdir(self.temp_directory) + r = File(self.temp_directory) + r.putChild(b'get-data-html-small', GetDataHtmlSmall()) + r.putChild(b'get-data-html-large', GetDataHtmlLarge()) + + r.putChild(b'post-data-json-small', PostDataJsonSmall()) + r.putChild(b'post-data-json-large', PostDataJsonLarge()) + return r + + @inlineCallbacks def setUp(self): + # Initialize resource tree + root = self._init_resource() + self.site = Site(root, timeout=None) + # Start server for testing - self.mockserver = MockServer() - self.mockserver.__enter__() - + self.hostname = u'localhost' if self.scheme == 'https': - self.url = urlparse(self.mockserver.https_address) + context_factory = ssl_context_factory(self.key_file, self.certificate_file) + server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) else: - self.url = urlparse(self.mockserver.http_address) + server_endpoint = TCP4ServerEndpoint(reactor, 0, interface=self.hostname) + self.server = yield server_endpoint.listen(self.site) + self.port_number = self.server.getHost().port - self.protocol = H2ClientProtocol() - - # Connect to the server using the custom HTTP2ClientProtocol - options = CertificateOptions(acceptableProtocols=[b'h2']) - endpoint = SSL4ClientEndpoint(reactor, self.url.hostname, self.url.port, options) - connectProtocol(endpoint, self.protocol) - - def getURL(self, path): - return "{}://{}:{}/{}".format(self.url.scheme, self.url.hostname, self.url.port, path) + # Connect H2 client with server + client_certificate = get_client_certificate(self.key_file, self.certificate_file) + client_options = optionsForClientTLS( + hostname=self.hostname, + trustRoot=client_certificate, + acceptableProtocols=[b'h2'] + ) + h2_client_factory = Factory.forProtocol(H2ClientProtocol) + client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) + self.client = yield client_endpoint.connect(h2_client_factory) + @inlineCallbacks def tearDown(self): - self.mockserver.__exit__(None, None, None) + yield self.client.transport.loseConnection() + yield self.client.transport.abortConnection() + yield self.server.stopListening() + shutil.rmtree(self.temp_directory) - def test_download(self): - request = Request(self.getURL('')) + def get_url(self, path): + """ + :param path: Should have / at the starting compulsorily if not empty + :return: Complete url + """ + assert len(path) > 0 and (path[0] == '/' or path[0] == '&') + return "{}://{}:{}{}".format(self.scheme, self.hostname, self.port_number, path) - def assert_response(response: Response): - self.assertEqual(response.body, b'Scrapy mock HTTP server\n') - self.assertEqual(response.status, 200) + @staticmethod + def _check_repeat(get_deferred, count): + d_list = [] + for _ in range(count): + d = get_deferred() + d_list.append(d) + + return DeferredList(d_list, fireOnOneErrback=True) + + def _check_GET( + self, + request: Request, + expected_body, + expected_status + ): + def check_response(response: Response): + self.assertEqual(response.status, expected_status) + self.assertEqual(response.body, expected_body) self.assertEqual(response.request, request) self.assertEqual(response.url, request.url) - d = self.protocol.request(request) + content_length = int(response.headers.get('Content-Length')) + self.assertEqual(len(response.body), content_length) + + d = self.client.request(request) + d.addCallback(check_response) + d.addErrback(self.fail) + return d + + def test_GET_small_body(self): + request = Request(self.get_url('/get-data-html-small')) + return self._check_GET(request, Data.HTML_SMALL, 200) + + def test_GET_large_body(self): + request = Request(self.get_url('/get-data-html-large')) + return self._check_GET(request, Data.HTML_LARGE, 200) + + def _check_GET_x20(self, *args, **kwargs): + def get_deferred(): + return self._check_GET(*args, **kwargs) + + return self._check_repeat(get_deferred, 20) + + def test_GET_small_body_x20(self): + return self._check_GET_x20( + Request(self.get_url('/get-data-html-small')), + Data.HTML_SMALL, + 200 + ) + + def test_GET_large_body_x20(self): + return self._check_GET_x20( + Request(self.get_url('/get-data-html-large')), + Data.HTML_LARGE, + 200 + ) + + def _check_POST_json( + self, + request: Request, + expected_request_body, + expected_extra_data, + expected_status: int + ): + d = self.client.request(request) + + def assert_response(response: Response): + self.assertEqual(response.status, expected_status) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + + content_length = int(response.headers.get('Content-Length')) + self.assertEqual(len(response.body), content_length) + + # Parse the body + body = json.loads(response.body.decode('utf-8')) + self.assertIn('request-body', body) + self.assertIn('extra-data', body) + self.assertIn('request-headers', body) + + request_body = body['request-body'] + self.assertEqual(request_body, expected_request_body) + + extra_data = body['extra-data'] + self.assertEqual(extra_data, expected_extra_data) + + # Check if headers were sent successfully + request_headers = body['request-headers'] + for k, v in request.headers.items(): + k_str = k.decode('utf-8') + self.assertIn(k_str, request_headers) + self.assertEqual(request_headers[k_str], v[0].decode('utf-8')) + d.addCallback(assert_response) return d + + def test_POST_small_json(self): + request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) + return self._check_POST_json( + request, + Data.JSON_SMALL, + Data.EXTRA_SMALL, + 200 + ) + + def test_POST_large_json(self): + request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) + return self._check_POST_json( + request, + Data.JSON_LARGE, + Data.EXTRA_LARGE, + 200 + ) + + def _check_POST_json_x20(self, *args, **kwargs): + def get_deferred(): + return self._check_POST_json(*args, **kwargs) + + return self._check_repeat(get_deferred, 20) + + def test_POST_small_json_x20(self): + request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) + return self._check_POST_json_x20( + request, + Data.JSON_SMALL, + Data.EXTRA_SMALL, + 200 + ) + + def test_POST_large_json_x20(self): + request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) + return self._check_POST_json_x20( + request, + Data.JSON_LARGE, + Data.EXTRA_LARGE, + 200 + ) + + def test_cancel_request(self): + request = Request(url=self.get_url('/get-data-html-large')) + + def assert_response(response: Response): + self.assertEqual(response.status, 499) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + + d = self.client.request(request) + d.addCallback(assert_response) + d.cancel() + + return d From 6387445ef519124f393a20657e66383340fb1677 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 28 Jun 2020 18:44:57 +0530 Subject: [PATCH 0101/2083] test(tox.ini): change Twisted -> Twisted[http2] --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 27d21ade2..ada211b3c 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,7 @@ deps = pyOpenSSL==16.2.0 queuelib==1.4.2 service_identity==16.0.0 - Twisted==17.9.0 + Twisted[http2]==17.9.0 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt From 23906b6bee953d9bc5dd8042e785711b11840797 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 29 Jun 2020 18:21:05 +0530 Subject: [PATCH 0102/2083] refactor: move TypedDict types to types.py - rename LOGGER -> logger - remove self._write_to_transport from Stream class and handle all transport related activities inside HTTP2ClientProtocol class --- scrapy/core/http2/protocol.py | 69 +++++++----- scrapy/core/http2/stream.py | 206 +++++++++++++++++----------------- scrapy/core/http2/types.py | 30 +++++ setup.py | 2 +- 4 files changed, 174 insertions(+), 133 deletions(-) create mode 100644 scrapy/core/http2/types.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 7fb935f10..0b3e5d304 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,6 +2,7 @@ import ipaddress import itertools import logging from collections import deque +from typing import Union, Dict from h2.config import H2Configuration from h2.connection import H2Connection @@ -9,20 +10,18 @@ from h2.events import ( DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) +from h2.exceptions import ProtocolError from twisted.internet.protocol import connectionDone, Protocol from twisted.internet.ssl import Certificate from scrapy.core.http2.stream import Stream, StreamCloseReason +from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request -LOGGER = logging.getLogger(__name__) +logger = logging.getLogger(__name__) class H2ClientProtocol(Protocol): - # TODO: - # 1. Check for user-agent while testing - # 2. Handle case when received events have StreamID = 0 (applied to H2Connection) - def __init__(self): config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) @@ -37,7 +36,7 @@ class H2ClientProtocol(Protocol): self.stream_id_count = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs - self.streams = {} + self.streams: Dict[int, Stream] = {} # Boolean to keep track the connection is made # If requests are received before connection is made @@ -46,9 +45,11 @@ class H2ClientProtocol(Protocol): self.is_connection_made = False self._pending_request_stream_pool = deque() - # Some meta data of this connection - # initialized when connection is successfully made - self._metadata = { + # Save an instance of ProtocolError raised by hyper-h2 + # We pass this instance to the streams ResponseFailed() failure + self._protocol_error: Union[None, ProtocolError] = None + + self._metadata: H2ConnectionMetadataDict = { 'certificate': None, 'ip_address': None } @@ -68,7 +69,6 @@ class H2ClientProtocol(Protocol): request=request, connection=self.conn, conn_metadata=self._metadata, - write_to_transport=self._write_to_transport, cb_close=self._stream_close_cb ) @@ -89,10 +89,10 @@ class H2ClientProtocol(Protocol): data = self.conn.data_to_send() self.transport.write(data) - LOGGER.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address'])) + logger.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address'])) - def request(self, _request: Request): - stream = self._new_stream(_request) + def request(self, request: Request): + stream = self._new_stream(request) d = stream.get_response() # If connection is not yet established then add the @@ -109,7 +109,7 @@ class H2ClientProtocol(Protocol): sending some data now: we should open with the connection preamble. """ self.destination = self.transport.getPeer() - LOGGER.info('Connection made to {}'.format(self.destination)) + logger.info('Connection made to {}'.format(self.destination)) self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) @@ -119,21 +119,37 @@ class H2ClientProtocol(Protocol): self.is_connection_made = True def dataReceived(self, data): - events = self.conn.receive_data(data) - self._handle_events(events) - self._write_to_transport() + try: + events = self.conn.receive_data(data) + self._handle_events(events) + except ProtocolError as e: + # TODO: In case of InvalidBodyLengthError -- terminate only one stream + + # Save this error as ultimately the connection will be dropped + # internally by hyper-h2. Saved error will be passed to all the streams + # closed with the connection. + self._protocol_error = e + + # We lose the transport connection here + self.transport.loseConnection() + finally: + self._write_to_transport() def connectionLost(self, reason=connectionDone): """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ # Pop all streams which were pending and were not yet started - for stream_id in list(self.streams): - self.streams[stream_id].close(StreamCloseReason.CONNECTION_LOST, reason) + # NOTE: Stream.close() pops the element from the streams dictionary + # which raises `RuntimeError: dictionary changed size during iteration` + # Hence, we copy the streams into a list. + for stream in list(self.streams.values()): + stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error) self.conn.close_connection() - LOGGER.warning("Connection lost with reason " + str(reason)) + if not reason.check(connectionDone): + logger.warning("Connection lost with reason " + str(reason)) def _handle_events(self, events): """Private method which acts as a bridge between the events @@ -144,7 +160,7 @@ class H2ClientProtocol(Protocol): triggered by sending data """ for event in events: - LOGGER.debug(event) + logger.debug(event) if isinstance(event, DataReceived): self.data_received(event) elif isinstance(event, ResponseReceived): @@ -158,16 +174,14 @@ class H2ClientProtocol(Protocol): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) else: - LOGGER.info("Received unhandled event {}".format(event)) + logger.info("Received unhandled event {}".format(event)) # Event handler functions starts here def data_received(self, event: DataReceived): - stream_id = event.stream_id - self.streams[stream_id].receive_data(event.data, event.flow_controlled_length) + self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) def response_received(self, event: ResponseReceived): - stream_id = event.stream_id - self.streams[stream_id].receive_headers(event.headers) + self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged): # Send off all the pending requests @@ -175,8 +189,7 @@ class H2ClientProtocol(Protocol): self._send_pending_requests() def stream_ended(self, event: StreamEnded): - stream_id = event.stream_id - self.streams[stream_id].close(StreamCloseReason.ENDED) + self.streams[event.stream_id].close(StreamCloseReason.ENDED) def stream_reset(self, event: StreamReset): self.streams[event.stream_id].close(StreamCloseReason.RESET) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a26a33918..da0181d52 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,7 +1,7 @@ import logging -from enum import IntFlag, auto +from enum import Enum from io import BytesIO -from typing import Dict +from typing import Callable, List from urllib.parse import urlparse from h2.connection import H2Connection @@ -10,45 +10,31 @@ from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred, CancelledError from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -# for python < 3.8 -- typing.TypedDict is undefined -from typing_extensions import TypedDict +from scrapy.core.http2.types import H2ConnectionMetadataDict, H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes - -class _ResponseTypedDict(TypedDict): - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. - body: BytesIO - - # The amount of data received that counts against the flow control - # window - flow_controlled_size: int - - # Headers received after sending the request - headers: Headers +logger = logging.getLogger(__name__) -LOGGER = logging.getLogger(__name__) - - -class StreamCloseReason(IntFlag): +class StreamCloseReason(Enum): # Received a StreamEnded event - ENDED = auto() + ENDED = 1 # Received a StreamReset event -- ended abruptly - RESET = auto() + RESET = 2 # Transport connection was lost - CONNECTION_LOST = auto() + CONNECTION_LOST = 3 # Expected response body size is more than allowed limit - MAXSIZE_EXCEEDED = auto() + MAXSIZE_EXCEEDED = 4 - # When the response deferred is cancelled - CANCELLED = auto() + # When the response deferred is cancelled by the client + # (happens when client called response_deferred.cancel()) + CANCELLED = 5 class Stream: @@ -63,34 +49,30 @@ class Stream: """ def __init__( - self, - stream_id: int, - request: Request, - connection: H2Connection, - conn_metadata: Dict, - write_to_transport, - cb_close, - download_maxsize=0, - download_warnsize=0, - fail_on_data_loss=True + self, + stream_id: int, + request: Request, + connection: H2Connection, + conn_metadata: H2ConnectionMetadataDict, + cb_close: Callable[[int], None], + download_maxsize: int = 0, + download_warnsize: int = 0, + fail_on_data_loss: bool = True ): """ Arguments: - stream_id {int} -- For one HTTP/2 connection each stream is + stream_id -- For one HTTP/2 connection each stream is uniquely identified by a single integer - request {Request} -- HTTP request - connection {H2Connection} -- HTTP/2 connection this stream belongs to. - conn_metadata {Dict} -- Reference to dictionary having metadata of HTTP/2 connection - write_to_transport {callable} -- Method used to write & send data to the server - This method should be used whenever some frame is to be sent to the server. - cb_close {callable} -- Method called when this stream is closed + request -- HTTP request + connection -- HTTP/2 connection this stream belongs to. + conn_metadata -- Reference to dictionary having metadata of HTTP/2 connection + cb_close -- Method called when this stream is closed to notify the TCP connection instance. """ self.stream_id = stream_id self._request = request self._conn = connection self._conn_metadata = conn_metadata - self._write_to_transport = write_to_transport self._cb_close = cb_close self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) @@ -119,7 +101,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: _ResponseTypedDict = { + self._response: H2ResponseDict = { 'body': BytesIO(), 'flow_controlled_size': 0, 'headers': Headers({}) @@ -136,6 +118,25 @@ class Stream: __repr__ = __str__ + @property + def _log_warnsize(self) -> bool: + """Checks if we have received data which exceeds the download warnsize + and whether we have not already logged about it. + + Returns: + True if both the above conditions hold true + False if any of the conditions is false + """ + content_length_header = int(self._response['headers'].get(b'Content-Length', -1)) + return ( + self._download_warnsize + and ( + self._response['flow_controlled_size'] > self._download_warnsize + or content_length_header > self._download_warnsize + ) + and not self._reached_warnsize + ) + def get_response(self): """Simply return a Deferred which fires when response from the asynchronous request is available @@ -166,10 +167,7 @@ class Stream: def initiate_request(self): headers = self._get_request_headers() self._conn.send_headers(self.stream_id, headers, end_stream=False) - self._write_to_transport() - self.request_sent = True - self.send_data() def send_data(self): @@ -211,20 +209,19 @@ class Stream: self.remaining_content_length = self.remaining_content_length - chunk_size self.remaining_content_length = max(0, self.remaining_content_length) - LOGGER.debug("{} sending {}/{} data bytes ({} frames) to {}".format( - self, - self.content_length - self.remaining_content_length, self.content_length, - data_frames_sent, - self._conn_metadata['ip_address']) + logger.debug( + "{stream} sending {received}/{expected} data bytes ({frames} frames) to {ip_address}".format( + stream=self, + received=self.content_length - self.remaining_content_length, + expected=self.content_length, + frames=data_frames_sent, + ip_address=self._conn_metadata['ip_address']) ) # End the stream if no more data needs to be send if self.remaining_content_length == 0: self._conn.end_stream(self.stream_id) - # Write data to transport -- Empty the outstanding data - self._write_to_transport() - # Q. What about the rest of the data? # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame @@ -240,24 +237,21 @@ class Stream: self._response['body'].write(data) self._response['flow_controlled_size'] += flow_controlled_length + # We check maxsize here in case the Content-Length header was not received if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize: - # Clear buffer earlier to avoid keeping data in memory for a long time - self._response['body'].truncate(0) self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) return - if self._download_warnsize \ - and self._response['flow_controlled_size'] > self._download_warnsize \ - and not self._reached_warnsize: + if self._log_warnsize: self._reached_warnsize = True - warning_msg = ('Received more ({bytes}) bytes than download ', - 'warn size ({warnsize}) in request {request}') + warning_msg = 'Received more ({bytes}) bytes than download ' \ + + 'warn size ({warnsize}) in request {request}' warning_args = { 'bytes': self._response['flow_controlled_size'], 'warnsize': self._download_warnsize, 'request': self._request } - LOGGER.warning(warning_msg, warning_args) + logger.warning(warning_msg.format(**warning_args)) # Acknowledge the data received self._conn.acknowledge_received_data( @@ -275,23 +269,27 @@ class Stream: self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) return - if self._download_warnsize and expected_size > self._download_warnsize: - warning_msg = ("Expected response size ({size}) larger than ", - "download warn size ({warnsize}) in request {request}.") + if self._log_warnsize: + self._reached_warnsize = True + warning_msg = 'Expected response size ({size}) larger than ' \ + + 'download warn size ({warnsize}) in request {request}' warning_args = { - 'size': expected_size, 'warnsize': self._download_warnsize, + 'size': expected_size, + 'warnsize': self._download_warnsize, 'request': self._request } - LOGGER.warning(warning_msg, warning_args) + logger.warning(warning_msg.format(**warning_args)) def reset_stream(self, reason=StreamCloseReason.RESET): """Close this stream by sending a RST_FRAME to the remote peer""" if self.stream_closed_local: raise StreamClosedError(self.stream_id) + # Clear buffer earlier to avoid keeping data in memory for a long time + self._response['body'].truncate(0) + self.stream_closed_local = True self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) - self._write_to_transport() self.close(reason) def _is_data_lost(self) -> bool: @@ -302,8 +300,11 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason, failure=None): + def close(self, reason: StreamCloseReason, error: Exception = None): """Based on the reason sent we will handle each case. + + Arguments: + reason -- One if StreamCloseReason """ if self.stream_closed_server: raise StreamClosedError(self.stream_id) @@ -311,35 +312,16 @@ class Stream: self._cb_close(self.stream_id) self.stream_closed_server = True - # Do nothing if the response deferred was cancelled flags = None if b'Content-Length' not in self._response['headers']: - # Missing Content-Length - PotentialDataLoss + # Missing Content-Length - {twisted.web.http.PotentialDataLoss} flags = ['partial'] - elif self._is_data_lost(): - if self._fail_on_dataloss: - self._deferred_response.errback(ResponseFailed([Failure()])) - return - else: - flags = ['dataloss'] - if reason is StreamCloseReason.ENDED: - self._fire_response_deferred(flags) - - # Stream was abruptly ended here - elif reason is StreamCloseReason.CANCELLED: - # Client has cancelled the request. Remove all the data - # received and fire the response deferred with no flags set - self._response['body'].truncate(0) - self._response['headers'].clear() - self._fire_response_deferred() - - elif reason in (StreamCloseReason.RESET | StreamCloseReason.CONNECTION_LOST): - if failure is None: - self._deferred_response.errback(ResponseFailed([Failure()])) - else: - self._deferred_response.errback(failure) - elif reason is StreamCloseReason.MAXSIZE_EXCEEDED: + # NOTE: Order of handling the events is important here + # As we immediately cancel the request when maxsize is exceeded while + # receiving DATA_FRAME's when we have received the headers (not + # having Content-Length) + if reason is StreamCloseReason.MAXSIZE_EXCEEDED: expected_size = int(self._response['headers'].get(b'Content-Length', -1)) error_msg = ("Cancelling download of {url}: expected response " "size ({size}) larger than download max size ({maxsize}).") @@ -349,14 +331,34 @@ class Stream: 'maxsize': self._download_maxsize } - LOGGER.error(error_msg, error_args) + logger.error(error_msg, error_args) self._deferred_response.errback(CancelledError(error_msg.format(**error_args))) - def _fire_response_deferred(self, flags=None): + elif reason is StreamCloseReason.ENDED: + self._fire_response_deferred(flags) + + # Stream was abruptly ended here + elif reason is StreamCloseReason.CANCELLED: + # Client has cancelled the request. Remove all the data + # received and fire the response deferred with no flags set + + # NOTE: The data is already flushed in Stream.reset_stream() called + # immediately when the stream needs to be cancelled + + # There maybe no :status in headers, we make + # HTTP Status Code: 499 - Client Closed Request + self._response['headers'][':status'] = '499' + self._fire_response_deferred() + + elif reason in (StreamCloseReason.RESET, StreamCloseReason.CONNECTION_LOST): + self._deferred_response.errback(ResponseFailed([ + error if error else Failure() + ])) + + def _fire_response_deferred(self, flags: List[str] = None): """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" - # TODO: Update Client Side Status Codes here body = self._response['body'].getvalue() response_cls = responsetypes.from_args( @@ -365,11 +367,7 @@ class Stream: body=body ) - # If there is no :status in headers - # (happens when client called response_deferred.cancel()) - # HTTP Status Code: 499 - Client Closed Request - status = self._response['headers'].get(':status', '499') - + status = self._response['headers'][':status'] response = response_cls( url=self._request.url, status=status, diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py new file mode 100644 index 000000000..f28bf9472 --- /dev/null +++ b/scrapy/core/http2/types.py @@ -0,0 +1,30 @@ +from io import BytesIO +from ipaddress import IPv4Address, IPv6Address +from typing import Union + +from twisted.internet.ssl import Certificate +# for python < 3.8 -- typing.TypedDict is undefined +from typing_extensions import TypedDict + +from scrapy.http.headers import Headers + + +class H2ConnectionMetadataDict(TypedDict): + """Some meta data of this connection + initialized when connection is successfully made + """ + certificate: Union[None, Certificate] + ip_address: Union[None, IPv4Address, IPv6Address] + + +class H2ResponseDict(TypedDict): + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. + body: BytesIO + + # The amount of data received that counts against the flow control + # window + flow_controlled_size: int + + # Headers received after sending the request + headers: Headers diff --git a/setup.py b/setup.py index 575c74e7f..8e50733e6 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ setup( ], python_requires='>=3.5.2', install_requires=[ - 'Twisted[http2]>=17.9.0' + 'Twisted[http2]>=17.9.0', 'cryptography>=2.0', 'cssselect>=0.9.1', 'lxml>=3.5.0', From 90a7007f8818dbc224dbe51b95f07199cb730204 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 29 Jun 2020 18:29:31 +0530 Subject: [PATCH 0103/2083] test: warnsize logs, no content header, dataloss --- tests/test_http2_client_protocol.py | 167 +++++++++++++++++++++++----- 1 file changed, 142 insertions(+), 25 deletions(-) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0f3730c9e..0a2719d23 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,26 +1,26 @@ -# TODO: Add test cases for -# 1. No Content Length response header -# 2. Cancel Response Deferred import json import os import random +import re import shutil import string +from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks, DeferredList +from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint from twisted.internet.protocol import Factory from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate +from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.http import Request as TxRequest -from twisted.web.resource import Resource -from twisted.web.server import Site +from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol from scrapy.http import Request, Response, JsonRequest -from tests.mockserver import ssl_context_factory +from scrapy.utils.python import to_bytes, to_unicode +from tests.mockserver import ssl_context_factory, LeafResource def generate_random_string(size): @@ -35,7 +35,7 @@ def make_html_body(val):

Hello from HTTP2

{}

'''.format(val) - return bytes(response, 'utf-8') + return to_bytes(response) class Data: @@ -54,9 +54,8 @@ class Data: JSON_SMALL = {'data': STR_SMALL} JSON_LARGE = {'data': STR_LARGE} - -class LeafResource(Resource): - isLeaf = True + DATALOSS = b'Dataloss Content' + NO_CONTENT_LENGTH = b'This response do not have any content-length header' class GetDataHtmlSmall(LeafResource): @@ -80,9 +79,9 @@ class PostDataJsonMixin: 'extra-data': extra_data } for k, v in request.requestHeaders.getAllRawHeaders(): - response['request-headers'][k.decode('utf-8')] = v[0].decode('utf-8') + response['request-headers'][to_unicode(k)] = to_unicode(v[0]) - response_bytes = bytes(json.dumps(response), 'utf-8') + response_bytes = to_bytes(json.dumps(response)) request.setHeader('Content-Type', 'application/json') return response_bytes @@ -97,6 +96,31 @@ class PostDataJsonLarge(LeafResource, PostDataJsonMixin): return self.make_response(request, Data.EXTRA_LARGE) +class Dataloss(LeafResource): + + def render_GET(self, request: TxRequest): + request.setHeader(b"Content-Length", b"1024") + self.deferRequest(request, 0, self._delayed_render, request) + return NOT_DONE_YET + + @staticmethod + def _delayed_render(request: TxRequest): + request.write(Data.DATALOSS) + request.finish() + + +class NoContentLengthHeader(LeafResource): + def render_GET(self, request: TxRequest): + request.requestHeaders.removeHeader('Content-Length') + self.deferRequest(request, 0, self._delayed_render, request) + return NOT_DONE_YET + + @staticmethod + def _delayed_render(request: TxRequest): + request.write(Data.NO_CONTENT_LENGTH) + request.finish() + + def get_client_certificate(key_file, certificate_file): with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -118,6 +142,9 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'post-data-json-small', PostDataJsonSmall()) r.putChild(b'post-data-json-large', PostDataJsonLarge()) + + r.putChild(b'dataloss', Dataloss()) + r.putChild(b'no-content-length-header', NoContentLengthHeader()) return r @inlineCallbacks @@ -172,10 +199,10 @@ class Https2ClientProtocolTestCase(TestCase): return DeferredList(d_list, fireOnOneErrback=True) def _check_GET( - self, - request: Request, - expected_body, - expected_status + self, + request: Request, + expected_body, + expected_status ): def check_response(response: Response): self.assertEqual(response.status, expected_status) @@ -220,11 +247,11 @@ class Https2ClientProtocolTestCase(TestCase): ) def _check_POST_json( - self, - request: Request, - expected_request_body, - expected_extra_data, - expected_status: int + self, + request: Request, + expected_request_body, + expected_extra_data, + expected_status: int ): d = self.client.request(request) @@ -237,7 +264,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(len(response.body), content_length) # Parse the body - body = json.loads(response.body.decode('utf-8')) + body = json.loads(to_unicode(response.body)) self.assertIn('request-body', body) self.assertIn('extra-data', body) self.assertIn('request-headers', body) @@ -251,11 +278,12 @@ class Https2ClientProtocolTestCase(TestCase): # Check if headers were sent successfully request_headers = body['request-headers'] for k, v in request.headers.items(): - k_str = k.decode('utf-8') + k_str = to_unicode(k) self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], v[0].decode('utf-8')) + self.assertEqual(request_headers[k_str], to_unicode(v[0])) d.addCallback(assert_response) + d.addErrback(self.fail) return d def test_POST_small_json(self): @@ -310,6 +338,95 @@ class Https2ClientProtocolTestCase(TestCase): d = self.client.request(request) d.addCallback(assert_response) + d.addErrback(self.fail) d.cancel() return d + + def test_download_maxsize_exceeded(self): + request = Request(url=self.get_url('/get-data-html-large'), meta={'download_maxsize': 1000}) + + def assert_cancelled_error(failure): + self.assertIsInstance(failure.value, CancelledError) + + d = self.client.request(request) + d.addCallback(self.fail) + d.addErrback(assert_cancelled_error) + return d + + # TODO: Test in multiple requests if one request fails due to dataloss + # remaining request do not fail (change expected behaviour) + # Can be done only when hyper-h2 don't terminate connection over + # InvalidBodyLengthError check + def test_received_dataloss_response(self): + """In case when value of Header Content-Length != len(Received Data) + ProtocolError is raised""" + request = Request(url=self.get_url('/dataloss')) + + def assert_failure(failure: Failure): + self.assertTrue(len(failure.value.reasons) > 0) + self.assertTrue(any( + isinstance(error, InvalidBodyLengthError) + for error in failure.value.reasons + )) + + d = self.client.request(request) + d.addCallback(self.fail) + d.addErrback(assert_failure) + return d + + def test_missing_content_length_header(self): + request = Request(url=self.get_url('/no-content-length-header')) + + def assert_content_length(response: Response): + self.assertEqual(response.status, 200) + self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + self.assertIn('partial', response.flags) + self.assertNotIn('Content-Length', response.headers) + + d = self.client.request(request) + d.addCallback(assert_content_length) + d.addErrback(self.fail) + return d + + @inlineCallbacks + def _check_log_warnsize( + self, + request, + warn_pattern, + expected_body + ): + with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm: + response = yield self.client.request(request) + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, expected_body) + + # Check the warning is raised only once for this request + self.assertEqual(sum( + len(re.findall(warn_pattern, log)) + for log in cm.output + ), 1) + + @inlineCallbacks + def test_log_expected_warnsize(self): + request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000}) + warn_pattern = re.compile( + r'Expected response size \(\d*\) larger than ' + r'download warn size \(1000\) in request {}'.format(request) + ) + + yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) + + @inlineCallbacks + def test_log_received_warnsize(self): + request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10}) + warn_pattern = re.compile( + r'Received more \(\d*\) bytes than download ' + r'warn size \(10\) in request {}'.format(request) + ) + + yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) From 26ab3e4137ddee3c643ae63c0709529efc698433 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 30 Jun 2020 06:44:20 +0530 Subject: [PATCH 0104/2083] feat: FIFO policy to handle large no. of requests - add required test -- test by sending 1000 requests - increase test timeout to 180 seconds to account for tests taking long time --- scrapy/core/http2/protocol.py | 87 +++++++++++++++++++---------- scrapy/core/http2/stream.py | 26 ++++++++- tests/test_http2_client_protocol.py | 51 ++++++++++++++++- 3 files changed, 128 insertions(+), 36 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 0b3e5d304..4de80c05e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -38,13 +38,16 @@ class H2ClientProtocol(Protocol): # Streams are stored in a dictionary keyed off their stream IDs self.streams: Dict[int, Stream] = {} - # Boolean to keep track the connection is made - # If requests are received before connection is made - # we keep all requests in a pool and send them as the connection - # is made - self.is_connection_made = False + # If requests are received before connection is made we keep + # all requests in a pool and send them as the connection is made self._pending_request_stream_pool = deque() + # Counter to keep track of opened stream. This counter + # is used to make sure that not more than MAX_CONCURRENT_STREAMS + # streams are opened which leads to ProtocolError + # We use simple FIFO policy to handle pending requests + self._active_streams = 0 + # Save an instance of ProtocolError raised by hyper-h2 # We pass this instance to the streams ResponseFailed() failure self._protocol_error: Union[None, ProtocolError] = None @@ -54,10 +57,46 @@ class H2ClientProtocol(Protocol): 'ip_address': None } + @property + def is_connected(self): + """Boolean to keep track of the connection status. + This is used while initiating pending streams to make sure + that we initiate stream only during active HTTP/2 Connection + """ + return bool(self.transport.connected) + + @property + def allowed_max_concurrent_streams(self) -> int: + """We keep total two streams for client (sending data) and + server side (receiving data) for a single request. To be safe + we choose the minimum. Since this value can change in event + RemoteSettingsChanged we make variable a property. + """ + return min( + self.conn.local_settings.max_concurrent_streams, + self.conn.remote_settings.max_concurrent_streams + ) + + def _send_pending_requests(self): + """Initiate all pending requests from the deque following FIFO + We make sure that at any time {allowed_max_concurrent_streams} + streams are active. + """ + while ( + self._pending_request_stream_pool + and self._active_streams < self.allowed_max_concurrent_streams + and self.is_connected + ): + self._active_streams += 1 + stream = self._pending_request_stream_pool.popleft() + stream.initiate_request() + def _stream_close_cb(self, stream_id: int): """Called when stream is closed completely """ - self.streams.pop(stream_id, None) + self.streams.pop(stream_id) + self._active_streams -= 1 + self._send_pending_requests() def _new_stream(self, request: Request): """Instantiates a new Stream object @@ -75,13 +114,6 @@ class H2ClientProtocol(Protocol): self.streams[stream.stream_id] = stream return stream - def _send_pending_requests(self): - # TODO: handle MAX_CONCURRENT_STREAMS - # Initiate all pending requests - while self._pending_request_stream_pool: - stream = self._pending_request_stream_pool.popleft() - stream.initiate_request() - def _write_to_transport(self): """ Write data to the underlying transport connection from the HTTP2 connection instance if any @@ -89,19 +121,12 @@ class H2ClientProtocol(Protocol): data = self.conn.data_to_send() self.transport.write(data) - logger.debug("Sent {} bytes to {} via transport".format(len(data), self._metadata['ip_address'])) - def request(self, request: Request): stream = self._new_stream(request) d = stream.get_response() - # If connection is not yet established then add the - # stream to pool or initiate request - if self.is_connection_made: - stream.initiate_request() - else: - self._pending_request_stream_pool.append(stream) - + # Add the stream to the request pool + self._pending_request_stream_pool.append(stream) return d def connectionMade(self): @@ -116,7 +141,6 @@ class H2ClientProtocol(Protocol): self.conn.initiate_connection() self._write_to_transport() - self.is_connection_made = True def dataReceived(self, data): try: @@ -144,7 +168,10 @@ class H2ClientProtocol(Protocol): # which raises `RuntimeError: dictionary changed size during iteration` # Hence, we copy the streams into a list. for stream in list(self.streams.values()): - stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error) + if stream.request_sent: + stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error) + else: + stream.close(StreamCloseReason.INACTIVE) self.conn.close_connection() @@ -160,7 +187,6 @@ class H2ClientProtocol(Protocol): triggered by sending data """ for event in events: - logger.debug(event) if isinstance(event, DataReceived): self.data_received(event) elif isinstance(event, ResponseReceived): @@ -174,7 +200,7 @@ class H2ClientProtocol(Protocol): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) else: - logger.info("Received unhandled event {}".format(event)) + logger.debug("Received unhandled event {}".format(event)) # Event handler functions starts here def data_received(self, event: DataReceived): @@ -184,8 +210,8 @@ class H2ClientProtocol(Protocol): self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged): - # Send off all the pending requests - # as now we have established a proper HTTP/2 connection + # Send off all the pending requests as now we have + # established a proper HTTP/2 connection self._send_pending_requests() def stream_ended(self, event: StreamEnded): @@ -195,9 +221,8 @@ class H2ClientProtocol(Protocol): self.streams[event.stream_id].close(StreamCloseReason.RESET) def window_updated(self, event: WindowUpdated): - stream_id = event.stream_id - if stream_id != 0: - self.streams[stream_id].receive_window_update() + if event.stream_id != 0: + self.streams[event.stream_id].receive_window_update() else: # Send leftover data for all the streams for stream in self.streams.values(): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index da0181d52..19b1825e4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -8,6 +8,7 @@ from h2.connection import H2Connection from h2.errors import ErrorCodes from h2.exceptions import StreamClosedError from twisted.internet.defer import Deferred, CancelledError +from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed @@ -19,6 +20,15 @@ from scrapy.responsetypes import responsetypes logger = logging.getLogger(__name__) +class InactiveStreamClosed(ConnectionClosed): + """Connection was closed without sending request headers + of the stream. This happens when a stream is waiting for other + streams to close and connection is lost.""" + + def __init__(self, request: Request): + self.request = request + + class StreamCloseReason(Enum): # Received a StreamEnded event ENDED = 1 @@ -32,10 +42,13 @@ class StreamCloseReason(Enum): # Expected response body size is more than allowed limit MAXSIZE_EXCEEDED = 4 - # When the response deferred is cancelled by the client + # Response deferred is cancelled by the client # (happens when client called response_deferred.cancel()) CANCELLED = 5 + # Connection lost and the stream was not initiated + INACTIVE = 6 + class Stream: """Represents a single HTTP/2 Stream. @@ -108,8 +121,12 @@ class Stream: } def _cancel(_): - # Close this stream as gracefully as possible :) - self.reset_stream(StreamCloseReason.CANCELLED) + # Close this stream as gracefully as possible + # Check if the stream has started + if self.request_sent: + self.reset_stream(StreamCloseReason.CANCELLED) + else: + self.close(StreamCloseReason.CANCELLED) self._deferred_response = Deferred(_cancel) @@ -355,6 +372,9 @@ class Stream: error if error else Failure() ])) + elif reason is StreamCloseReason.INACTIVE: + self._deferred_response.errback(InactiveStreamClosed(self._request)) + def _fire_response_deferred(self, flags: List[str] = None): """Builds response from the self._response dict and fires the response deferred callback with the diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0a2719d23..0cb32dda6 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -18,6 +18,7 @@ from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.core.http2.stream import InactiveStreamClosed from scrapy.http import Request, Response, JsonRequest from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource @@ -174,10 +175,14 @@ class Https2ClientProtocolTestCase(TestCase): client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) + # Increase the total time taken for each tests + self.timeout = 180 # default is 120 seconds + @inlineCallbacks def tearDown(self): - yield self.client.transport.loseConnection() - yield self.client.transport.abortConnection() + if self.client.is_connected: + yield self.client.transport.loseConnection() + yield self.client.transport.abortConnection() yield self.server.stopListening() shutil.rmtree(self.temp_directory) @@ -430,3 +435,45 @@ class Https2ClientProtocolTestCase(TestCase): ) yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) + + def test_max_concurrent_streams(self): + """Send 1000 requests to check if we can handle + very large number of request + """ + + def get_deferred(): + return self._check_GET( + Request(self.get_url('/get-data-html-small')), + Data.HTML_SMALL, + 200 + ) + + return self._check_repeat(get_deferred, 1000) + + def test_inactive_stream(self): + """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS + by default is 100. After sending the first 100 requests we close the + connection.""" + d_list = [] + + def assert_inactive_stream(failure): + self.assertIsNotNone(failure.check(InactiveStreamClosed)) + + # Send 100 request (we do not check the result) + for _ in range(100): + d = self.client.request(Request(self.get_url('/get-data-html-small'))) + d.addBoth(lambda _: None) + d_list.append(d) + + # Now send 10 extra request and save the response deferred in a list + for _ in range(10): + d = self.client.request(Request(self.get_url('/get-data-html-small'))) + d.addCallback(self.fail) + d.addErrback(assert_inactive_stream) + d_list.append(d) + + # Close the connection now to fire all the extra 10 requests errback + # with InactiveStreamClosed + self.client.transport.abortConnection() + + return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) From 50dd9271b4566785430106cfa9384d51103f73d9 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 30 Jun 2020 07:17:48 +0530 Subject: [PATCH 0105/2083] fix: disable redundant logs - while testing the job exceeded the maximum log length and was terminated - reduce the number of requests from 20 to 10 --- scrapy/core/http2/stream.py | 8 -------- tests/test_http2_client_protocol.py | 27 ++++++++++++--------------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 19b1825e4..8d0c6d94d 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -226,14 +226,6 @@ class Stream: self.remaining_content_length = self.remaining_content_length - chunk_size self.remaining_content_length = max(0, self.remaining_content_length) - logger.debug( - "{stream} sending {received}/{expected} data bytes ({frames} frames) to {ip_address}".format( - stream=self, - received=self.content_length - self.remaining_content_length, - expected=self.content_length, - frames=data_frames_sent, - ip_address=self._conn_metadata['ip_address']) - ) # End the stream if no more data needs to be send if self.remaining_content_length == 0: diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0cb32dda6..79c129d11 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -175,9 +175,6 @@ class Https2ClientProtocolTestCase(TestCase): client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) - # Increase the total time taken for each tests - self.timeout = 180 # default is 120 seconds - @inlineCallbacks def tearDown(self): if self.client.is_connected: @@ -231,21 +228,21 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(self.get_url('/get-data-html-large')) return self._check_GET(request, Data.HTML_LARGE, 200) - def _check_GET_x20(self, *args, **kwargs): + def _check_GET_x10(self, *args, **kwargs): def get_deferred(): return self._check_GET(*args, **kwargs) - return self._check_repeat(get_deferred, 20) + return self._check_repeat(get_deferred, 10) - def test_GET_small_body_x20(self): - return self._check_GET_x20( + def test_GET_small_body_x10(self): + return self._check_GET_x10( Request(self.get_url('/get-data-html-small')), Data.HTML_SMALL, 200 ) - def test_GET_large_body_x20(self): - return self._check_GET_x20( + def test_GET_large_body_x10(self): + return self._check_GET_x10( Request(self.get_url('/get-data-html-large')), Data.HTML_LARGE, 200 @@ -309,24 +306,24 @@ class Https2ClientProtocolTestCase(TestCase): 200 ) - def _check_POST_json_x20(self, *args, **kwargs): + def _check_POST_json_x10(self, *args, **kwargs): def get_deferred(): return self._check_POST_json(*args, **kwargs) - return self._check_repeat(get_deferred, 20) + return self._check_repeat(get_deferred, 10) - def test_POST_small_json_x20(self): + def test_POST_small_json_x10(self): request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) - return self._check_POST_json_x20( + return self._check_POST_json_x10( request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200 ) - def test_POST_large_json_x20(self): + def test_POST_large_json_x10(self): request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) - return self._check_POST_json_x20( + return self._check_POST_json_x10( request, Data.JSON_LARGE, Data.EXTRA_LARGE, From 7b1ad995a4996babf9a019815dd7256a1cbfa044 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 1 Jul 2020 10:45:36 +0530 Subject: [PATCH 0106/2083] test: query params, certificate & ip_address - refactor from str.format() to f-strings --- scrapy/core/http2/protocol.py | 12 ++-- scrapy/core/http2/stream.py | 46 +++++--------- scrapy/core/http2/types.py | 6 +- setup.py | 7 ++- tests/test_http2_client_protocol.py | 96 +++++++++++++++++++++++------ 5 files changed, 108 insertions(+), 59 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 4de80c05e..3438c99f0 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -122,6 +122,9 @@ class H2ClientProtocol(Protocol): self.transport.write(data) def request(self, request: Request): + if not isinstance(request, Request): + raise TypeError(f'Expected type scrapy.http.Request but received {request.__class__.__name__}') + stream = self._new_stream(request) d = stream.get_response() @@ -134,9 +137,7 @@ class H2ClientProtocol(Protocol): sending some data now: we should open with the connection preamble. """ self.destination = self.transport.getPeer() - logger.info('Connection made to {}'.format(self.destination)) - - self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) + logger.info(f'Connection made to {self.destination}') self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) self.conn.initiate_connection() @@ -200,7 +201,7 @@ class H2ClientProtocol(Protocol): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) else: - logger.debug("Received unhandled event {}".format(event)) + logger.debug(f'Received unhandled event {event}') # Event handler functions starts here def data_received(self, event: DataReceived): @@ -214,6 +215,9 @@ class H2ClientProtocol(Protocol): # established a proper HTTP/2 connection self._send_pending_requests() + # Update certificate when our HTTP/2 connection is established + self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) + def stream_ended(self, event: StreamEnded): self.streams[event.stream_id].close(StreamCloseReason.ENDED) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8d0c6d94d..f4a90a753 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -131,7 +131,7 @@ class Stream: self._deferred_response = Deferred(_cancel) def __str__(self): - return "Stream(id={})".format(repr(self.stream_id)) + return f'Stream(id={self.stream_id})' __repr__ = __str__ @@ -167,13 +167,15 @@ class Stream: url = urlparse(self._request.url) # Make sure pseudo-headers comes before all the other headers + path = url.path + if url.query: + path += '?' + url.query + headers = [ (':method', self._request.method), (':authority', url.netloc), - - # TODO: Check if scheme can be 'http' for HTTP/2 ? (':scheme', 'https'), - (':path', url.path), + (':path', path), ] for name, value in self._request.headers.items(): @@ -253,14 +255,9 @@ class Stream: if self._log_warnsize: self._reached_warnsize = True - warning_msg = 'Received more ({bytes}) bytes than download ' \ - + 'warn size ({warnsize}) in request {request}' - warning_args = { - 'bytes': self._response['flow_controlled_size'], - 'warnsize': self._download_warnsize, - 'request': self._request - } - logger.warning(warning_msg.format(**warning_args)) + warning_msg = f"Received more ({self._response['flow_controlled_size']}) bytes than download " \ + + f'warn size ({self._download_warnsize}) in request {self._request}' + logger.warning(warning_msg) # Acknowledge the data received self._conn.acknowledge_received_data( @@ -280,14 +277,9 @@ class Stream: if self._log_warnsize: self._reached_warnsize = True - warning_msg = 'Expected response size ({size}) larger than ' \ - + 'download warn size ({warnsize}) in request {request}' - warning_args = { - 'size': expected_size, - 'warnsize': self._download_warnsize, - 'request': self._request - } - logger.warning(warning_msg.format(**warning_args)) + warning_msg = f'Expected response size ({expected_size}) larger than ' \ + + f'download warn size ({self._download_warnsize}) in request {self._request}' + logger.warning(warning_msg) def reset_stream(self, reason=StreamCloseReason.RESET): """Close this stream by sending a RST_FRAME to the remote peer""" @@ -332,16 +324,10 @@ class Stream: # having Content-Length) if reason is StreamCloseReason.MAXSIZE_EXCEEDED: expected_size = int(self._response['headers'].get(b'Content-Length', -1)) - error_msg = ("Cancelling download of {url}: expected response " - "size ({size}) larger than download max size ({maxsize}).") - error_args = { - 'url': self._request.url, - 'size': expected_size, - 'maxsize': self._download_maxsize - } - - logger.error(error_msg, error_args) - self._deferred_response.errback(CancelledError(error_msg.format(**error_args))) + error_msg = f'Cancelling download of {self._request.url}: expected response ' \ + f'size ({expected_size}) larger than download max size ({self._download_maxsize}).' + logger.error(error_msg) + self._deferred_response.errback(CancelledError(error_msg)) elif reason is StreamCloseReason.ENDED: self._fire_response_deferred(flags) diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index f28bf9472..c0961cd3a 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -1,6 +1,6 @@ from io import BytesIO from ipaddress import IPv4Address, IPv6Address -from typing import Union +from typing import Union, Optional from twisted.internet.ssl import Certificate # for python < 3.8 -- typing.TypedDict is undefined @@ -13,8 +13,8 @@ class H2ConnectionMetadataDict(TypedDict): """Some meta data of this connection initialized when connection is successfully made """ - certificate: Union[None, Certificate] - ip_address: Union[None, IPv4Address, IPv6Address] + certificate: Optional[Certificate] + ip_address: Optional[Union[IPv4Address, IPv6Address]] class H2ResponseDict(TypedDict): diff --git a/setup.py b/setup.py index 8e50733e6..47c5906e4 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from os.path import dirname, join - from pkg_resources import parse_version from setuptools import setup, find_packages, __version__ as setuptools_version + with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: version = f.read().decode('ascii').strip() @@ -25,11 +25,12 @@ if has_environment_marker_platform_impl_support(): 'PyPyDispatcher>=2.1.0', ] + setup( name='Scrapy', version=version, url='https://scrapy.org', - project_urls={ + project_urls = { 'Documentation': 'https://docs.scrapy.org/', 'Source': 'https://github.com/scrapy/scrapy', 'Tracker': 'https://github.com/scrapy/scrapy/issues', @@ -83,4 +84,4 @@ setup( 'typing_extensions>=3.7' ], extras_require=extras_require, -) +) \ No newline at end of file diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 79c129d11..c6a9bd5fb 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -4,13 +4,15 @@ import random import re import shutil import string +from ipaddress import IPv4Address +from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint from twisted.internet.protocol import Factory -from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate +from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.http import Request as TxRequest @@ -21,7 +23,7 @@ from scrapy.core.http2.protocol import H2ClientProtocol from scrapy.core.http2.stream import InactiveStreamClosed from scrapy.http import Request, Response, JsonRequest from scrapy.utils.python import to_bytes, to_unicode -from tests.mockserver import ssl_context_factory, LeafResource +from tests.mockserver import ssl_context_factory, LeafResource, Status def generate_random_string(size): @@ -32,10 +34,10 @@ def generate_random_string(size): def make_html_body(val): - response = ''' + response = f'''

Hello from HTTP2

-

{}

-'''.format(val) +

{val}

+''' return to_bytes(response) @@ -122,6 +124,17 @@ class NoContentLengthHeader(LeafResource): request.finish() +class QueryParams(LeafResource): + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'application/json') + + query_params = {} + for k, v in request.args.items(): + query_params[to_unicode(k)] = to_unicode(v[0]) + + return to_bytes(json.dumps(query_params)) + + def get_client_certificate(key_file, certificate_file): with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -146,6 +159,8 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'dataloss', Dataloss()) r.putChild(b'no-content-length-header', NoContentLengthHeader()) + r.putChild(b'status', Status()) + r.putChild(b'query-params', QueryParams()) return r @inlineCallbacks @@ -189,7 +204,7 @@ class Https2ClientProtocolTestCase(TestCase): :return: Complete url """ assert len(path) > 0 and (path[0] == '/' or path[0] == '&') - return "{}://{}:{}{}".format(self.scheme, self.hostname, self.port_number, path) + return f'{self.scheme}://{self.hostname}:{self.port_number}{path}' @staticmethod def _check_repeat(get_deferred, count): @@ -210,7 +225,6 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, expected_status) self.assertEqual(response.body, expected_body) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) content_length = int(response.headers.get('Content-Length')) self.assertEqual(len(response.body), content_length) @@ -260,7 +274,6 @@ class Https2ClientProtocolTestCase(TestCase): def assert_response(response: Response): self.assertEqual(response.status, expected_status) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) content_length = int(response.headers.get('Content-Length')) self.assertEqual(len(response.body), content_length) @@ -336,7 +349,6 @@ class Https2ClientProtocolTestCase(TestCase): def assert_response(response: Response): self.assertEqual(response.status, 499) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) d = self.client.request(request) d.addCallback(assert_response) @@ -356,10 +368,6 @@ class Https2ClientProtocolTestCase(TestCase): d.addErrback(assert_cancelled_error) return d - # TODO: Test in multiple requests if one request fails due to dataloss - # remaining request do not fail (change expected behaviour) - # Can be done only when hyper-h2 don't terminate connection over - # InvalidBodyLengthError check def test_received_dataloss_response(self): """In case when value of Header Content-Length != len(Received Data) ProtocolError is raised""" @@ -384,7 +392,6 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, 200) self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) self.assertIn('partial', response.flags) self.assertNotIn('Content-Length', response.headers) @@ -404,7 +411,6 @@ class Https2ClientProtocolTestCase(TestCase): response = yield self.client.request(request) self.assertEqual(response.status, 200) self.assertEqual(response.request, request) - self.assertEqual(response.url, request.url) self.assertEqual(response.body, expected_body) # Check the warning is raised only once for this request @@ -417,8 +423,8 @@ class Https2ClientProtocolTestCase(TestCase): def test_log_expected_warnsize(self): request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000}) warn_pattern = re.compile( - r'Expected response size \(\d*\) larger than ' - r'download warn size \(1000\) in request {}'.format(request) + rf'Expected response size \(\d*\) larger than ' + rf'download warn size \(1000\) in request {request}' ) yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) @@ -427,8 +433,8 @@ class Https2ClientProtocolTestCase(TestCase): def test_log_received_warnsize(self): request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10}) warn_pattern = re.compile( - r'Received more \(\d*\) bytes than download ' - r'warn size \(10\) in request {}'.format(request) + rf'Received more \(\d*\) bytes than download ' + rf'warn size \(10\) in request {request}' ) yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) @@ -474,3 +480,55 @@ class Https2ClientProtocolTestCase(TestCase): self.client.transport.abortConnection() return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) + + def test_invalid_request_type(self): + with self.assertRaises(TypeError): + self.client.request('https://InvalidDataTypePassed.com') + + def test_query_parameters(self): + params = { + 'a': generate_random_string(20), + 'b': generate_random_string(20), + 'c': generate_random_string(20), + 'd': generate_random_string(20) + } + request = Request(self.get_url(f'/query-params?{urlencode(params)}')) + + def assert_query_params(response: Response): + data = json.loads(to_unicode(response.body)) + self.assertEqual(data, params) + + d = self.client.request(request) + d.addCallback(assert_query_params) + d.addErrback(self.fail) + + return d + + def test_status_codes(self): + def assert_response_status(response: Response, expected_status: int): + self.assertEqual(response.status, expected_status) + + d_list = [] + for status in [200, 404]: + request = Request(self.get_url(f'/status?n={status}')) + d = self.client.request(request) + d.addCallback(assert_response_status, status) + d.addErrback(self.fail) + d_list.append(d) + + return DeferredList(d_list, fireOnOneErrback=True) + + def test_response_has_correct_certificate_ip_address(self): + request = Request(self.get_url('/status?n=200')) + + def assert_metadata(response: Response): + self.assertEqual(response.request, request) + self.assertIsInstance(response.certificate, Certificate) + self.assertIsInstance(response.ip_address, IPv4Address) + self.assertEqual(str(response.ip_address), '127.0.0.1') + + d = self.client.request(request) + d.addCallback(assert_metadata) + d.addErrback(self.fail) + + return d From 7fc80671a84144ad36c3d5332aa822ef3be00781 Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Wed, 1 Jul 2020 13:32:17 +0530 Subject: [PATCH 0107/2083] Update schemaless URI support --- scrapy/core/downloader/handlers/http11.py | 12 ++++++++---- scrapy/core/downloader/webclient.py | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 73e56c87d..15de8cdbd 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -7,7 +7,7 @@ import warnings from contextlib import suppress from io import BytesIO from time import time -from urllib.parse import urldefrag, urlparse +from urllib.parse import urldefrag, urlunparse from twisted.internet import defer, protocol, ssl from twisted.internet.endpoints import TCP4ClientEndpoint @@ -255,7 +255,7 @@ class ScrapyProxyAgent(Agent): bindAddress=bindAddress, pool=pool, ) - self._proxyURI = URI.fromBytes(urlparse(proxyURI)._replace(scheme=b'http').geturl()) + self._proxyURI = URI.fromBytes(proxyURI) def request(self, method, uri, headers=None, bodyProducer=None): """ @@ -297,7 +297,7 @@ class ScrapyAgent: bindaddress = request.meta.get('bindaddress') or self._bindAddress proxy = request.meta.get('proxy') if proxy: - _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) + proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams @@ -319,9 +319,13 @@ class ScrapyAgent: pool=self._pool, ) else: + proxyScheme = b'http' if not proxyScheme else proxyScheme + proxyHost = to_bytes(proxyHost, encoding='ascii') + proxyPort = to_bytes(str(proxyPort), encoding='ascii') + proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) return self._ProxyAgent( reactor=reactor, - proxyURI=to_bytes(proxy, encoding='ascii'), + proxyURI=to_bytes(proxyURI, encoding='ascii'), connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool, diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 355045d74..af49e78ce 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,5 +1,6 @@ from time import time from urllib.parse import urlparse, urlunparse, urldefrag +from re import match from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient @@ -32,6 +33,8 @@ def _parse(url): and is ascii-only. """ url = url.strip() + if not match(r'^\w+://', url): + url = '//' + url parsed = urlparse(url) return _parsed_url_args(parsed) From 006a945214422da22f645b4416263124377adc5e Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Wed, 1 Jul 2020 13:32:58 +0530 Subject: [PATCH 0108/2083] Update schemaless http proxy test --- tests/test_downloader_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 5854659dd..9441be736 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -741,7 +741,7 @@ class Http11ProxyTestCase(HttpProxyTestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.body, b'http://example.com') - http_proxy = self.getURL('').replace('http:', '') + http_proxy = self.getURL('').replace('http://', '') request = Request('http://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) From 065b9b1170fe249fbfce51c87631e5572127df21 Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Wed, 1 Jul 2020 15:53:29 +0530 Subject: [PATCH 0109/2083] Update regex import --- scrapy/core/downloader/webclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index af49e78ce..8b6f87c3f 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,6 +1,6 @@ +import re from time import time from urllib.parse import urlparse, urlunparse, urldefrag -from re import match from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient @@ -33,7 +33,7 @@ def _parse(url): and is ascii-only. """ url = url.strip() - if not match(r'^\w+://', url): + if not re.match(r'^\w+://', url): url = '//' + url parsed = urlparse(url) return _parsed_url_args(parsed) From c361fe0d3b80ff8a9f88adc05e730d5e469db225 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 1 Jul 2020 18:14:44 +0530 Subject: [PATCH 0110/2083] feat: check for invalid hostname - Initiating requests having hostname or (ip_address, port) different from the peer to which HTTP/2 connection is made can lead to closing the whole connection and close out all the pending streams. - This change aims to fix that problem - Add required tests - Save hostname & port in H2ConnectionMetadataDict --- scrapy/core/http2/protocol.py | 26 ++++++------ scrapy/core/http2/stream.py | 50 +++++++++++++++++++--- scrapy/core/http2/types.py | 11 +++++ tests/test_http2_client_protocol.py | 64 +++++++++++++++++++++++------ 4 files changed, 118 insertions(+), 33 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 3438c99f0..5de516482 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,7 +2,7 @@ import ipaddress import itertools import logging from collections import deque -from typing import Union, Dict +from typing import Dict, Optional from h2.config import H2Configuration from h2.connection import H2Connection @@ -26,10 +26,6 @@ class H2ClientProtocol(Protocol): config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) - # Address of the server we are connected to - # these are updated when connection is successfully made - self.destination = None - # ID of the next request stream # Following the convention made by hyper-h2 each client ID # will be odd. @@ -50,11 +46,13 @@ class H2ClientProtocol(Protocol): # Save an instance of ProtocolError raised by hyper-h2 # We pass this instance to the streams ResponseFailed() failure - self._protocol_error: Union[None, ProtocolError] = None + self._protocol_error: Optional[ProtocolError] = None self._metadata: H2ConnectionMetadataDict = { 'certificate': None, - 'ip_address': None + 'ip_address': None, + 'hostname': None, + 'port': None } @property @@ -123,7 +121,7 @@ class H2ClientProtocol(Protocol): def request(self, request: Request): if not isinstance(request, Request): - raise TypeError(f'Expected type scrapy.http.Request but received {request.__class__.__name__}') + raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__name__}') stream = self._new_stream(request) d = stream.get_response() @@ -136,9 +134,11 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ - self.destination = self.transport.getPeer() - logger.info(f'Connection made to {self.destination}') - self._metadata['ip_address'] = ipaddress.ip_address(self.destination.host) + destination = self.transport.getPeer() + logger.debug('Connection made to {}'.format(destination)) + self._metadata['ip_address'] = ipaddress.ip_address(destination.host) + self._metadata['port'] = destination.port + self._metadata['hostname'] = self.transport.transport.addr[0] self.conn.initiate_connection() self._write_to_transport() @@ -148,8 +148,6 @@ class H2ClientProtocol(Protocol): events = self.conn.receive_data(data) self._handle_events(events) except ProtocolError as e: - # TODO: In case of InvalidBodyLengthError -- terminate only one stream - # Save this error as ultimately the connection will be dropped # internally by hyper-h2. Saved error will be passed to all the streams # closed with the connection. @@ -201,7 +199,7 @@ class H2ClientProtocol(Protocol): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) else: - logger.debug(f'Received unhandled event {event}') + logger.debug('Received unhandled event {}'.format(event)) # Event handler functions starts here def data_received(self, event: DataReceived): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index f4a90a753..cc751b682 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -29,6 +29,17 @@ class InactiveStreamClosed(ConnectionClosed): self.request = request +class InvalidHostname(Exception): + + def __init__(self, request: Request, expected_hostname, expected_netloc): + self.request = request + self.expected_hostname = expected_hostname + self.expected_netloc = expected_netloc + + def __str__(self): + return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}' + + class StreamCloseReason(Enum): # Received a StreamEnded event ENDED = 1 @@ -49,6 +60,10 @@ class StreamCloseReason(Enum): # Connection lost and the stream was not initiated INACTIVE = 6 + # The hostname of the request is not same as of connected peer hostname + # As a result sending this request will the end the connection + INVALID_HOSTNAME = 7 + class Stream: """Represents a single HTTP/2 Stream. @@ -163,6 +178,15 @@ class Stream: """ return self._deferred_response + def check_request_url(self) -> bool: + # Make sure that we are sending the request to the correct URL + url = urlparse(self._request.url) + return ( + url.netloc == self._conn_metadata['hostname'] + or url.netloc == f'{self._conn_metadata["hostname"]}:{self._conn_metadata["port"]}' + or url.netloc == f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}' + ) + def _get_request_headers(self): url = urlparse(self._request.url) @@ -184,10 +208,15 @@ class Stream: return headers def initiate_request(self): - headers = self._get_request_headers() - self._conn.send_headers(self.stream_id, headers, end_stream=False) - self.request_sent = True - self.send_data() + if self.check_request_url(): + headers = self._get_request_headers() + self._conn.send_headers(self.stream_id, headers, end_stream=False) + self.request_sent = True + self.send_data() + else: + # Close this stream calling the response errback + # Note that we have not sent any headers + self.close(StreamCloseReason.INVALID_HOSTNAME) def send_data(self): """Called immediately after the headers are sent. Here we send all the @@ -310,6 +339,9 @@ class Stream: if self.stream_closed_server: raise StreamClosedError(self.stream_id) + if not isinstance(reason, StreamCloseReason): + raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__name__}') + self._cb_close(self.stream_id) self.stream_closed_server = True @@ -353,6 +385,13 @@ class Stream: elif reason is StreamCloseReason.INACTIVE: self._deferred_response.errback(InactiveStreamClosed(self._request)) + elif reason is StreamCloseReason.INVALID_HOSTNAME: + self._deferred_response.errback(InvalidHostname( + self._request, + self._conn_metadata['hostname'], + f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}' + )) + def _fire_response_deferred(self, flags: List[str] = None): """Builds response from the self._response dict and fires the response deferred callback with the @@ -365,10 +404,9 @@ class Stream: body=body ) - status = self._response['headers'][':status'] response = response_cls( url=self._request.url, - status=status, + status=self._response['headers'][':status'], headers=self._response['headers'], body=body, request=self._request, diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index c0961cd3a..dd7b1187b 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -14,8 +14,19 @@ class H2ConnectionMetadataDict(TypedDict): initialized when connection is successfully made """ certificate: Optional[Certificate] + + # Address of the server we are connected to which + # is updated when HTTP/2 connection is made successfully ip_address: Optional[Union[IPv4Address, IPv6Address]] + # Name of the peer HTTP/2 connection is established + hostname: Optional[str] + + port: Optional[int] + + # Both ip_address and hostname are used by the Stream before + # initiating the request to verify that the base address + class H2ResponseDict(TypedDict): # Data received frame by frame from the server is appended diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index c6a9bd5fb..8f9fbe6fd 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -10,7 +10,7 @@ from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError -from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint, TCP4ServerEndpoint +from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.protocol import Factory from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure @@ -20,7 +20,7 @@ from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File from scrapy.core.http2.protocol import H2ClientProtocol -from scrapy.core.http2.stream import InactiveStreamClosed +from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -135,7 +135,7 @@ class QueryParams(LeafResource): return to_bytes(json.dumps(query_params)) -def get_client_certificate(key_file, certificate_file): +def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -171,19 +171,16 @@ class Https2ClientProtocolTestCase(TestCase): # Start server for testing self.hostname = u'localhost' - if self.scheme == 'https': - context_factory = ssl_context_factory(self.key_file, self.certificate_file) - server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) - else: - server_endpoint = TCP4ServerEndpoint(reactor, 0, interface=self.hostname) + context_factory = ssl_context_factory(self.key_file, self.certificate_file) + server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) self.server = yield server_endpoint.listen(self.site) self.port_number = self.server.getHost().port # Connect H2 client with server - client_certificate = get_client_certificate(self.key_file, self.certificate_file) + self.client_certificate = get_client_certificate(self.key_file, self.certificate_file) client_options = optionsForClientTLS( hostname=self.hostname, - trustRoot=client_certificate, + trustRoot=self.client_certificate, acceptableProtocols=[b'h2'] ) h2_client_factory = Factory.forProtocol(H2ClientProtocol) @@ -440,8 +437,8 @@ class Https2ClientProtocolTestCase(TestCase): yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) def test_max_concurrent_streams(self): - """Send 1000 requests to check if we can handle - very large number of request + """Send 500 requests at one to check if we can handle + very large number of request. """ def get_deferred(): @@ -451,7 +448,7 @@ class Https2ClientProtocolTestCase(TestCase): 200 ) - return self._check_repeat(get_deferred, 1000) + return self._check_repeat(get_deferred, 500) def test_inactive_stream(self): """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS @@ -524,6 +521,10 @@ class Https2ClientProtocolTestCase(TestCase): def assert_metadata(response: Response): self.assertEqual(response.request, request) self.assertIsInstance(response.certificate, Certificate) + self.assertIsNotNone(response.certificate.original) + self.assertEqual(response.certificate.getIssuer(), self.client_certificate.getIssuer()) + self.assertTrue(response.certificate.getPublicKey().matches(self.client_certificate.getPublicKey())) + self.assertIsInstance(response.ip_address, IPv4Address) self.assertEqual(str(response.ip_address), '127.0.0.1') @@ -532,3 +533,40 @@ class Https2ClientProtocolTestCase(TestCase): d.addErrback(self.fail) return d + + def _check_invalid_netloc(self, url): + request = Request(url) + + def assert_invalid_hostname(failure: Failure): + self.assertIsNotNone(failure.check(InvalidHostname)) + error_msg = str(failure.value) + self.assertIn('localhost', error_msg) + self.assertIn('127.0.0.1', error_msg) + self.assertIn(str(request), error_msg) + + d = self.client.request(request) + d.addCallback(self.fail) + d.addErrback(assert_invalid_hostname) + return d + + def test_invalid_hostname(self): + return self._check_invalid_netloc('https://notlocalhost.notlocalhostdomain') + + def test_invalid_host_port(self): + port = self.port_number + 1 + return self._check_invalid_netloc(f'https://127.0.0.1:{port}') + + def test_connection_stays_with_invalid_requests(self): + d_list = [ + self.test_invalid_hostname(), + self.test_invalid_host_port(), + self._check_GET(Request(self.get_url('/get-data-html-small')), Data.HTML_SMALL, 200), + self._check_POST_json( + JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL), + Data.JSON_SMALL, + Data.EXTRA_SMALL, + 200 + ) + ] + + return DeferredList(d_list, fireOnOneErrback=True) From 4acdc2e5d623c6330235647aaad817e77eaad800 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 1 Jul 2020 20:15:33 +0530 Subject: [PATCH 0111/2083] refactor: use __qualname__, () for large strings --- scrapy/core/http2/protocol.py | 2 +- scrapy/core/http2/stream.py | 24 +++++++++++++++--------- tests/test_http2_client_protocol.py | 9 ++------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 5de516482..a3dfdb76e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -121,7 +121,7 @@ class H2ClientProtocol(Protocol): def request(self, request: Request): if not isinstance(request, Request): - raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__name__}') + raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}') stream = self._new_stream(request) d = stream.get_response() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index cc751b682..f45ddc04e 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -146,7 +146,7 @@ class Stream: self._deferred_response = Deferred(_cancel) def __str__(self): - return f'Stream(id={self.stream_id})' + return f'Stream(id={self.stream_id!r})' __repr__ = __str__ @@ -190,11 +190,11 @@ class Stream: def _get_request_headers(self): url = urlparse(self._request.url) - # Make sure pseudo-headers comes before all the other headers path = url.path if url.query: path += '?' + url.query + # Make sure pseudo-headers comes before all the other headers headers = [ (':method', self._request.method), (':authority', url.netloc), @@ -284,8 +284,10 @@ class Stream: if self._log_warnsize: self._reached_warnsize = True - warning_msg = f"Received more ({self._response['flow_controlled_size']}) bytes than download " \ - + f'warn size ({self._download_warnsize}) in request {self._request}' + warning_msg = ( + f'Received more ({self._response["flow_controlled_size"]}) bytes than download ' + f'warn size ({self._download_warnsize}) in request {self._request}' + ) logger.warning(warning_msg) # Acknowledge the data received @@ -306,8 +308,10 @@ class Stream: if self._log_warnsize: self._reached_warnsize = True - warning_msg = f'Expected response size ({expected_size}) larger than ' \ - + f'download warn size ({self._download_warnsize}) in request {self._request}' + warning_msg = ( + f'Expected response size ({expected_size}) larger than ' + f'download warn size ({self._download_warnsize}) in request {self._request}' + ) logger.warning(warning_msg) def reset_stream(self, reason=StreamCloseReason.RESET): @@ -340,7 +344,7 @@ class Stream: raise StreamClosedError(self.stream_id) if not isinstance(reason, StreamCloseReason): - raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__name__}') + raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}') self._cb_close(self.stream_id) self.stream_closed_server = True @@ -356,8 +360,10 @@ class Stream: # having Content-Length) if reason is StreamCloseReason.MAXSIZE_EXCEEDED: expected_size = int(self._response['headers'].get(b'Content-Length', -1)) - error_msg = f'Cancelling download of {self._request.url}: expected response ' \ - f'size ({expected_size}) larger than download max size ({self._download_maxsize}).' + error_msg = ( + f'Cancelling download of {self._request.url}: expected response ' + f'size ({expected_size}) larger than download max size ({self._download_maxsize}).' + ) logger.error(error_msg) self._deferred_response.errback(CancelledError(error_msg)) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 8f9fbe6fd..ca8a629b1 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -560,13 +560,8 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [ self.test_invalid_hostname(), self.test_invalid_host_port(), - self._check_GET(Request(self.get_url('/get-data-html-small')), Data.HTML_SMALL, 200), - self._check_POST_json( - JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL), - Data.JSON_SMALL, - Data.EXTRA_SMALL, - 200 - ) + self.test_GET_small_body(), + self.test_POST_small_json() ] return DeferredList(d_list, fireOnOneErrback=True) From a94b30342a451b94e7f358f68ce1b1adc20723f9 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 6 Jul 2020 12:49:12 +0530 Subject: [PATCH 0112/2083] test: reduce test data size to 1MB --- scrapy/core/http2/stream.py | 2 -- tests/test_http2_client_protocol.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index f45ddc04e..1c856ff68 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -243,7 +243,6 @@ class Stream: bytes_to_send_size = min(window_size, self.remaining_content_length) # We now need to send a number of data frames. - data_frames_sent = 0 while bytes_to_send_size > 0: chunk_size = min(bytes_to_send_size, max_frame_size) @@ -252,7 +251,6 @@ class Stream: self._conn.send_data(self.stream_id, data_chunk, end_stream=False) - data_frames_sent += 1 bytes_to_send_size = bytes_to_send_size - chunk_size self.remaining_content_length = self.remaining_content_length - chunk_size diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index ca8a629b1..98dc98a0f 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -42,8 +42,8 @@ def make_html_body(val): class Data: - SMALL_SIZE = 1024 * 10 # 10 KB - LARGE_SIZE = (1024 ** 2) * 10 # 10 MB + SMALL_SIZE = 1024 # 1 KB + LARGE_SIZE = 1024 ** 2 # 1 MB STR_SMALL = generate_random_string(SMALL_SIZE) STR_LARGE = generate_random_string(LARGE_SIZE) From 7f5bb6b34c6a5137aa12cd4ad4f3845a8c67653f Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 6 Jul 2020 13:08:14 +0530 Subject: [PATCH 0113/2083] chore: add h2 to setup.py, tox.ini - Change log level for hpack to ERROR --- scrapy/utils/log.py | 3 +++ setup.py | 3 ++- tox.ini | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 51d276097..4e2671478 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -52,6 +52,9 @@ DEFAULT_LOGGING = { 'twisted': { 'level': 'ERROR', }, + 'hpack': { + 'level': 'ERROR', + }, } } diff --git a/setup.py b/setup.py index 47c5906e4..d872d6472 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,8 @@ setup( 'zope.interface>=4.1.3', 'protego>=0.1.15', 'itemadapter>=0.1.0', - 'typing_extensions>=3.7' + 'typing_extensions>=3.7', + 'h2>=3.2.0' ], extras_require=extras_require, ) \ No newline at end of file diff --git a/tox.ini b/tox.ini index ada211b3c..bc6314a2f 100644 --- a/tox.ini +++ b/tox.ini @@ -84,6 +84,7 @@ deps = # Extras botocore==1.3.23 Pillow==3.4.2 + h2==3.2.0 [testenv:extra-deps] deps = From 54e4228c3a22164b79db36a29a5e2d64391b592a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 6 Jul 2020 14:10:45 -0300 Subject: [PATCH 0114/2083] refactor: use protocol - H2ClientProtocol.close_stream - Fix and add missing type hints - More adjustments - Rename stream id generator - Simplify decrement --- scrapy/core/http2/protocol.py | 92 +++++++++++++++--------------- scrapy/core/http2/stream.py | 103 ++++++++++++++++------------------ 2 files changed, 93 insertions(+), 102 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index a3dfdb76e..2f177656d 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -11,32 +11,34 @@ from h2.events import ( StreamEnded, StreamReset, WindowUpdated ) from h2.exceptions import ProtocolError +from twisted.internet.defer import Deferred from twisted.internet.protocol import connectionDone, Protocol from twisted.internet.ssl import Certificate +from twisted.python.failure import Failure from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request + logger = logging.getLogger(__name__) class H2ClientProtocol(Protocol): - def __init__(self): + def __init__(self) -> None: config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) # ID of the next request stream - # Following the convention made by hyper-h2 each client ID - # will be odd. - self.stream_id_count = itertools.count(start=1, step=2) + # Following the convention made by hyper-h2 all IDs will be odd + self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs self.streams: Dict[int, Stream] = {} # If requests are received before connection is made we keep # all requests in a pool and send them as the connection is made - self._pending_request_stream_pool = deque() + self._pending_request_stream_pool: deque = deque() # Counter to keep track of opened stream. This counter # is used to make sure that not more than MAX_CONCURRENT_STREAMS @@ -48,15 +50,15 @@ class H2ClientProtocol(Protocol): # We pass this instance to the streams ResponseFailed() failure self._protocol_error: Optional[ProtocolError] = None - self._metadata: H2ConnectionMetadataDict = { + self.metadata: H2ConnectionMetadataDict = { 'certificate': None, 'ip_address': None, 'hostname': None, - 'port': None + 'port': None, } @property - def is_connected(self): + def is_connected(self) -> bool: """Boolean to keep track of the connection status. This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection @@ -75,7 +77,7 @@ class H2ClientProtocol(Protocol): self.conn.remote_settings.max_concurrent_streams ) - def _send_pending_requests(self): + def _send_pending_requests(self) -> None: """Initiate all pending requests from the deque following FIFO We make sure that at any time {allowed_max_concurrent_streams} streams are active. @@ -89,37 +91,33 @@ class H2ClientProtocol(Protocol): stream = self._pending_request_stream_pool.popleft() stream.initiate_request() - def _stream_close_cb(self, stream_id: int): - """Called when stream is closed completely + def pop_stream(self, stream_id: int) -> Stream: + """Perform cleanup when a stream is closed """ - self.streams.pop(stream_id) + stream = self.streams.pop(stream_id) self._active_streams -= 1 self._send_pending_requests() + return stream - def _new_stream(self, request: Request): + def _new_stream(self, request: Request) -> Stream: """Instantiates a new Stream object """ - stream_id = next(self.stream_id_count) - stream = Stream( - stream_id=stream_id, + stream_id=next(self._stream_id_generator), request=request, - connection=self.conn, - conn_metadata=self._metadata, - cb_close=self._stream_close_cb + protocol=self, ) - self.streams[stream.stream_id] = stream return stream - def _write_to_transport(self): + def _write_to_transport(self) -> None: """ Write data to the underlying transport connection from the HTTP2 connection instance if any """ data = self.conn.data_to_send() self.transport.write(data) - def request(self, request: Request): + def request(self, request: Request) -> Deferred: if not isinstance(request, Request): raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}') @@ -130,20 +128,20 @@ class H2ClientProtocol(Protocol): self._pending_request_stream_pool.append(stream) return d - def connectionMade(self): + def connectionMade(self) -> None: """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ destination = self.transport.getPeer() logger.debug('Connection made to {}'.format(destination)) - self._metadata['ip_address'] = ipaddress.ip_address(destination.host) - self._metadata['port'] = destination.port - self._metadata['hostname'] = self.transport.transport.addr[0] + self.metadata['ip_address'] = ipaddress.ip_address(destination.host) + self.metadata['port'] = destination.port + self.metadata['hostname'] = self.transport.transport.addr[0] self.conn.initiate_connection() self._write_to_transport() - def dataReceived(self, data): + def dataReceived(self, data: bytes) -> None: try: events = self.conn.receive_data(data) self._handle_events(events) @@ -158,32 +156,30 @@ class H2ClientProtocol(Protocol): finally: self._write_to_transport() - def connectionLost(self, reason=connectionDone): + def connectionLost(self, reason: Failure = connectionDone) -> None: """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ - # Pop all streams which were pending and were not yet started - # NOTE: Stream.close() pops the element from the streams dictionary - # which raises `RuntimeError: dictionary changed size during iteration` - # Hence, we copy the streams into a list. - for stream in list(self.streams.values()): + for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error) + stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error, from_protocol=True) else: - stream.close(StreamCloseReason.INACTIVE) + stream.close(StreamCloseReason.INACTIVE, from_protocol=True) + self._active_streams -= len(self.streams) + self.streams.clear() + self._send_pending_requests() self.conn.close_connection() if not reason.check(connectionDone): logger.warning("Connection lost with reason " + str(reason)) - def _handle_events(self, events): + def _handle_events(self, events: list) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler Arguments: - events {list} -- A list of events that the remote peer - triggered by sending data + events -- A list of events that the remote peer triggered by sending data """ for event in events: if isinstance(event, DataReceived): @@ -202,27 +198,29 @@ class H2ClientProtocol(Protocol): logger.debug('Received unhandled event {}'.format(event)) # Event handler functions starts here - def data_received(self, event: DataReceived): + def data_received(self, event: DataReceived) -> None: self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) - def response_received(self, event: ResponseReceived): + def response_received(self, event: ResponseReceived) -> None: self.streams[event.stream_id].receive_headers(event.headers) - def settings_acknowledged(self, event: SettingsAcknowledged): + def settings_acknowledged(self, event: SettingsAcknowledged) -> None: # Send off all the pending requests as now we have # established a proper HTTP/2 connection self._send_pending_requests() # Update certificate when our HTTP/2 connection is established - self._metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) + self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) - def stream_ended(self, event: StreamEnded): - self.streams[event.stream_id].close(StreamCloseReason.ENDED) + def stream_ended(self, event: StreamEnded) -> None: + stream = self.pop_stream(event.stream_id) + stream.close(StreamCloseReason.ENDED, from_protocol=True) - def stream_reset(self, event: StreamReset): - self.streams[event.stream_id].close(StreamCloseReason.RESET) + def stream_reset(self, event: StreamReset) -> None: + stream = self.pop_stream(event.stream_id) + stream.close(StreamCloseReason.RESET, from_protocol=True) - def window_updated(self, event: WindowUpdated): + def window_updated(self, event: WindowUpdated) -> None: if event.stream_id != 0: self.streams[event.stream_id].receive_window_update() else: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 1c856ff68..77cfbcfbf 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,22 +1,27 @@ import logging from enum import Enum from io import BytesIO -from typing import Callable, List +from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse -from h2.connection import H2Connection from h2.errors import ErrorCodes from h2.exceptions import StreamClosedError +from hpack import HeaderTuple from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from scrapy.core.http2.types import H2ConnectionMetadataDict, H2ResponseDict +from scrapy.core.http2.types import H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes + +if TYPE_CHECKING: + from scrapy.core.http2.protocol import H2ClientProtocol + + logger = logging.getLogger(__name__) @@ -31,12 +36,12 @@ class InactiveStreamClosed(ConnectionClosed): class InvalidHostname(Exception): - def __init__(self, request: Request, expected_hostname, expected_netloc): + def __init__(self, request: Request, expected_hostname: Optional[str], expected_netloc: Optional[str]) -> None: self.request = request self.expected_hostname = expected_hostname self.expected_netloc = expected_netloc - def __str__(self): + def __str__(self) -> str: return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}' @@ -80,28 +85,20 @@ class Stream: self, stream_id: int, request: Request, - connection: H2Connection, - conn_metadata: H2ConnectionMetadataDict, - cb_close: Callable[[int], None], + protocol: "H2ClientProtocol", download_maxsize: int = 0, download_warnsize: int = 0, fail_on_data_loss: bool = True - ): + ) -> None: """ Arguments: - stream_id -- For one HTTP/2 connection each stream is - uniquely identified by a single integer - request -- HTTP request - connection -- HTTP/2 connection this stream belongs to. - conn_metadata -- Reference to dictionary having metadata of HTTP/2 connection - cb_close -- Method called when this stream is closed - to notify the TCP connection instance. + stream_id -- Unique identifier for the stream within a single HTTP/2 connection + request -- The HTTP request associated to the stream + protocol -- Parent H2ClientProtocol instance """ - self.stream_id = stream_id - self._request = request - self._conn = connection - self._conn_metadata = conn_metadata - self._cb_close = cb_close + self.stream_id: int = stream_id + self._request: Request = request + self._protocol: "H2ClientProtocol" = protocol self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) @@ -132,7 +129,7 @@ class Stream: self._response: H2ResponseDict = { 'body': BytesIO(), 'flow_controlled_size': 0, - 'headers': Headers({}) + 'headers': Headers({}), } def _cancel(_): @@ -145,7 +142,7 @@ class Stream: self._deferred_response = Deferred(_cancel) - def __str__(self): + def __str__(self) -> str: return f'Stream(id={self.stream_id!r})' __repr__ = __str__ @@ -169,12 +166,9 @@ class Stream: and not self._reached_warnsize ) - def get_response(self): + def get_response(self) -> Deferred: """Simply return a Deferred which fires when response from the asynchronous request is available - - Returns: - Deferred -- Calls the callback passing the response """ return self._deferred_response @@ -182,12 +176,12 @@ class Stream: # Make sure that we are sending the request to the correct URL url = urlparse(self._request.url) return ( - url.netloc == self._conn_metadata['hostname'] - or url.netloc == f'{self._conn_metadata["hostname"]}:{self._conn_metadata["port"]}' - or url.netloc == f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}' + url.netloc == self._protocol.metadata['hostname'] + or url.netloc == f'{self._protocol.metadata["hostname"]}:{self._protocol.metadata["port"]}' + or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' ) - def _get_request_headers(self): + def _get_request_headers(self) -> List[Tuple[str, str]]: url = urlparse(self._request.url) path = url.path @@ -207,10 +201,10 @@ class Stream: return headers - def initiate_request(self): + def initiate_request(self) -> None: if self.check_request_url(): headers = self._get_request_headers() - self._conn.send_headers(self.stream_id, headers, end_stream=False) + self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False) self.request_sent = True self.send_data() else: @@ -218,7 +212,7 @@ class Stream: # Note that we have not sent any headers self.close(StreamCloseReason.INVALID_HOSTNAME) - def send_data(self): + def send_data(self) -> None: """Called immediately after the headers are sent. Here we send all the data as part of the request. @@ -233,10 +227,10 @@ class Stream: raise StreamClosedError(self.stream_id) # Firstly, check what the flow control window is for current stream. - window_size = self._conn.local_flow_control_window(stream_id=self.stream_id) + window_size = self._protocol.conn.local_flow_control_window(stream_id=self.stream_id) # Next, check what the maximum frame size is. - max_frame_size = self._conn.max_outbound_frame_size + max_frame_size = self._protocol.conn.max_outbound_frame_size # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. @@ -249,7 +243,7 @@ class Stream: data_chunk_start_id = self.content_length - self.remaining_content_length data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size] - self._conn.send_data(self.stream_id, data_chunk, end_stream=False) + self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False) bytes_to_send_size = bytes_to_send_size - chunk_size self.remaining_content_length = self.remaining_content_length - chunk_size @@ -258,12 +252,12 @@ class Stream: # End the stream if no more data needs to be send if self.remaining_content_length == 0: - self._conn.end_stream(self.stream_id) + self._protocol.conn.end_stream(self.stream_id) # Q. What about the rest of the data? # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame - def receive_window_update(self): + def receive_window_update(self) -> None: """Flow control window size was changed. Send data that earlier could not be sent as we were blocked behind the flow control. @@ -271,7 +265,7 @@ class Stream: if self.remaining_content_length and not self.stream_closed_server and self.request_sent: self.send_data() - def receive_data(self, data: bytes, flow_controlled_length: int): + def receive_data(self, data: bytes, flow_controlled_length: int) -> None: self._response['body'].write(data) self._response['flow_controlled_size'] += flow_controlled_length @@ -289,12 +283,12 @@ class Stream: logger.warning(warning_msg) # Acknowledge the data received - self._conn.acknowledge_received_data( + self._protocol.conn.acknowledge_received_data( self._response['flow_controlled_size'], self.stream_id ) - def receive_headers(self, headers): + def receive_headers(self, headers: List[HeaderTuple]) -> None: for name, value in headers: self._response['headers'][name] = value @@ -312,7 +306,7 @@ class Stream: ) logger.warning(warning_msg) - def reset_stream(self, reason=StreamCloseReason.RESET): + def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None: """Close this stream by sending a RST_FRAME to the remote peer""" if self.stream_closed_local: raise StreamClosedError(self.stream_id) @@ -321,7 +315,7 @@ class Stream: self._response['body'].truncate(0) self.stream_closed_local = True - self._conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) + self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) self.close(reason) def _is_data_lost(self) -> bool: @@ -332,11 +326,8 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason, error: Exception = None): + def close(self, reason: StreamCloseReason, error: Optional[Exception] = None, from_protocol: bool = False) -> None: """Based on the reason sent we will handle each case. - - Arguments: - reason -- One if StreamCloseReason """ if self.stream_closed_server: raise StreamClosedError(self.stream_id) @@ -344,7 +335,9 @@ class Stream: if not isinstance(reason, StreamCloseReason): raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}') - self._cb_close(self.stream_id) + if not from_protocol: + self._protocol.pop_stream(self.stream_id) + self.stream_closed_server = True flags = None @@ -392,11 +385,11 @@ class Stream: elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( self._request, - self._conn_metadata['hostname'], - f'{self._conn_metadata["ip_address"]}:{self._conn_metadata["port"]}' + self._protocol.metadata['hostname'], + f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' )) - def _fire_response_deferred(self, flags: List[str] = None): + def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None: """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" @@ -405,7 +398,7 @@ class Stream: response_cls = responsetypes.from_args( headers=self._response['headers'], url=self._request.url, - body=body + body=body, ) response = response_cls( @@ -415,8 +408,8 @@ class Stream: body=body, request=self._request, flags=flags, - certificate=self._conn_metadata['certificate'], - ip_address=self._conn_metadata['ip_address'] + certificate=self._protocol.metadata['certificate'], + ip_address=self._protocol.metadata['ip_address'], ) self._deferred_response.callback(response) From 1c40dfa7408026bc9ae831000a8614e8f4a9dd0d Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 7 Jul 2020 00:44:09 +0530 Subject: [PATCH 0115/2083] fix: handle CONNECTION_LOST & RESET separately --- scrapy/core/http2/protocol.py | 19 ++++++++++++------- scrapy/core/http2/stream.py | 18 ++++++++++++------ scrapy/utils/log.py | 6 +++--- tests/test_http2_client_protocol.py | 2 +- tox.ini | 2 +- 5 files changed, 29 insertions(+), 18 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 2f177656d..55dbcabec 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -20,7 +20,6 @@ from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request - logger = logging.getLogger(__name__) @@ -30,7 +29,8 @@ class H2ClientProtocol(Protocol): self.conn = H2Connection(config=config) # ID of the next request stream - # Following the convention made by hyper-h2 all IDs will be odd + # Following the convention - 'Streams initiated by a client MUST + # use odd-numbered stream identifiers' (RFC 7540) self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs @@ -160,20 +160,25 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ + errors = [] + if not reason.check(connectionDone): + logger.warning("Connection lost with reason " + str(reason)) + errors.append(reason) + + if self._protocol_error: + errors.append(self._protocol_error) + for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, self._protocol_error, from_protocol=True) + stream.close(StreamCloseReason.CONNECTION_LOST, errors, from_protocol=True) else: stream.close(StreamCloseReason.INACTIVE, from_protocol=True) self._active_streams -= len(self.streams) self.streams.clear() - self._send_pending_requests() + self._pending_request_stream_pool.clear() self.conn.close_connection() - if not reason.check(connectionDone): - logger.warning("Connection lost with reason " + str(reason)) - def _handle_events(self, events: list) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 77cfbcfbf..8b66d4b85 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -17,11 +17,9 @@ from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes - if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol - logger = logging.getLogger(__name__) @@ -326,7 +324,12 @@ class Stream: return expected_size != received_body_size - def close(self, reason: StreamCloseReason, error: Optional[Exception] = None, from_protocol: bool = False) -> None: + def close( + self, + reason: StreamCloseReason, + errors: Optional[List[Exception]] = None, + from_protocol: bool = False + ) -> None: """Based on the reason sent we will handle each case. """ if self.stream_closed_server: @@ -374,11 +377,14 @@ class Stream: self._response['headers'][':status'] = '499' self._fire_response_deferred() - elif reason in (StreamCloseReason.RESET, StreamCloseReason.CONNECTION_LOST): + elif reason is StreamCloseReason.RESET: self._deferred_response.errback(ResponseFailed([ - error if error else Failure() + Failure(f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM') ])) + elif reason is StreamCloseReason.CONNECTION_LOST: + self._deferred_response.errback(ResponseFailed(errors)) + elif reason is StreamCloseReason.INACTIVE: self._deferred_response.errback(InactiveStreamClosed(self._request)) @@ -403,7 +409,7 @@ class Stream: response = response_cls( url=self._request.url, - status=self._response['headers'][':status'], + status=int(self._response['headers'][':status']), headers=self._response['headers'], body=body, request=self._request, diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 4e2671478..9d59fdd68 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -46,15 +46,15 @@ DEFAULT_LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'loggers': { + 'hpack': { + 'level': 'ERROR', + }, 'scrapy': { 'level': 'DEBUG', }, 'twisted': { 'level': 'ERROR', }, - 'hpack': { - 'level': 'ERROR', - }, } } diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 98dc98a0f..9efca5267 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -474,7 +474,7 @@ class Https2ClientProtocolTestCase(TestCase): # Close the connection now to fire all the extra 10 requests errback # with InactiveStreamClosed - self.client.transport.abortConnection() + self.client.transport.loseConnection() return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) diff --git a/tox.ini b/tox.ini index bc6314a2f..be3b56e2d 100644 --- a/tox.ini +++ b/tox.ini @@ -83,8 +83,8 @@ deps = -rtests/requirements-py3.txt # Extras botocore==1.3.23 - Pillow==3.4.2 h2==3.2.0 + Pillow==3.4.2 [testenv:extra-deps] deps = From 2ea7d82534cafe5b25b28ef5a78e5b714767d27a Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 8 Jul 2020 18:57:13 +0530 Subject: [PATCH 0116/2083] feat: H2ClientFactory --- scrapy/core/http2/protocol.py | 35 +++++++++++++++++++++-------- scrapy/core/http2/stream.py | 29 +++++++++++++----------- scrapy/core/http2/types.py | 13 ++++++----- tests/test_http2_client_protocol.py | 8 ++++--- 4 files changed, 55 insertions(+), 30 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 55dbcabec..ee51300cc 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,29 +2,38 @@ import ipaddress import itertools import logging from collections import deque -from typing import Dict, Optional +from typing import Dict, List, Optional from h2.config import H2Configuration from h2.connection import H2Connection from h2.events import ( - DataReceived, ResponseReceived, SettingsAcknowledged, + Event, DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) from h2.exceptions import ProtocolError from twisted.internet.defer import Deferred -from twisted.internet.protocol import connectionDone, Protocol +from twisted.internet.protocol import connectionDone, Factory, Protocol from twisted.internet.ssl import Certificate from twisted.python.failure import Failure +from twisted.web.client import URI from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request +from scrapy.settings import Settings logger = logging.getLogger(__name__) class H2ClientProtocol(Protocol): - def __init__(self) -> None: + def __init__(self, uri: URI, settings: Settings) -> None: + """ + Arguments: + uri -- URI of the base url to which HTTP/2 Connection will be made. + uri is used to verify that incoming client requests have correct + base URL. + settings -- Scrapy project settings + """ config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) @@ -53,8 +62,9 @@ class H2ClientProtocol(Protocol): self.metadata: H2ConnectionMetadataDict = { 'certificate': None, 'ip_address': None, - 'hostname': None, - 'port': None, + 'uri': uri, + 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'), + 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'), } @property @@ -135,8 +145,6 @@ class H2ClientProtocol(Protocol): destination = self.transport.getPeer() logger.debug('Connection made to {}'.format(destination)) self.metadata['ip_address'] = ipaddress.ip_address(destination.host) - self.metadata['port'] = destination.port - self.metadata['hostname'] = self.transport.transport.addr[0] self.conn.initiate_connection() self._write_to_transport() @@ -179,7 +187,7 @@ class H2ClientProtocol(Protocol): self._pending_request_stream_pool.clear() self.conn.close_connection() - def _handle_events(self, events: list) -> None: + def _handle_events(self, events: List[Event]) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler @@ -232,3 +240,12 @@ class H2ClientProtocol(Protocol): # Send leftover data for all the streams for stream in self.streams.values(): stream.receive_window_update() + + +class H2ClientFactory(Factory): + def __init__(self, uri: URI, settings: Settings) -> None: + self.uri = uri + self.settings = settings + + def buildProtocol(self, addr) -> H2ClientProtocol: + return H2ClientProtocol(self.uri, self.settings) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8b66d4b85..5017f9cd4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -16,6 +16,7 @@ from scrapy.core.http2.types import H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes +from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -34,7 +35,7 @@ class InactiveStreamClosed(ConnectionClosed): class InvalidHostname(Exception): - def __init__(self, request: Request, expected_hostname: Optional[str], expected_netloc: Optional[str]) -> None: + def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None: self.request = request self.expected_hostname = expected_hostname self.expected_netloc = expected_netloc @@ -83,10 +84,7 @@ class Stream: self, stream_id: int, request: Request, - protocol: "H2ClientProtocol", - download_maxsize: int = 0, - download_warnsize: int = 0, - fail_on_data_loss: bool = True + protocol: "H2ClientProtocol" ) -> None: """ Arguments: @@ -98,9 +96,14 @@ class Stream: self._request: Request = request self._protocol: "H2ClientProtocol" = protocol - self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) - self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) - self._fail_on_dataloss = self._request.meta.get('download_fail_on_dataloss', fail_on_data_loss) + self._download_maxsize = self._request.meta.get( + 'download_maxsize', + self._protocol.metadata['default_download_maxsize'] + ) + self._download_warnsize = self._request.meta.get( + 'download_warnsize', + self._protocol.metadata['default_download_warnsize'] + ) self.request_start_time = None @@ -174,9 +177,9 @@ class Stream: # Make sure that we are sending the request to the correct URL url = urlparse(self._request.url) return ( - url.netloc == self._protocol.metadata['hostname'] - or url.netloc == f'{self._protocol.metadata["hostname"]}:{self._protocol.metadata["port"]}' - or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' + url.netloc == to_unicode(self._protocol.metadata['uri'].host) + or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc) + or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' ) def _get_request_headers(self) -> List[Tuple[str, str]]: @@ -391,8 +394,8 @@ class Stream: elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( self._request, - self._protocol.metadata['hostname'], - f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["port"]}' + to_unicode(self._protocol.metadata['uri'].host), + f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' )) def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None: diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index dd7b1187b..d2aa1a9d8 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -3,6 +3,7 @@ from ipaddress import IPv4Address, IPv6Address from typing import Union, Optional from twisted.internet.ssl import Certificate +from twisted.web.client import URI # for python < 3.8 -- typing.TypedDict is undefined from typing_extensions import TypedDict @@ -19,14 +20,16 @@ class H2ConnectionMetadataDict(TypedDict): # is updated when HTTP/2 connection is made successfully ip_address: Optional[Union[IPv4Address, IPv6Address]] - # Name of the peer HTTP/2 connection is established - hostname: Optional[str] + # URI of the peer HTTP/2 connection is made + uri: URI - port: Optional[int] - - # Both ip_address and hostname are used by the Stream before + # Both ip_address and uri are used by the Stream before # initiating the request to verify that the base address + # Variables taken from Project Settings + default_download_maxsize: int + default_download_warnsize: int + class H2ResponseDict(TypedDict): # Data received frame by frame from the server is appended diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 9efca5267..05e5f5047 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -11,17 +11,18 @@ from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint -from twisted.internet.protocol import Factory from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase +from twisted.web.client import URI from twisted.web.http import Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File -from scrapy.core.http2.protocol import H2ClientProtocol +from scrapy.core.http2.protocol import H2ClientFactory from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest +from scrapy.settings import Settings from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -183,7 +184,8 @@ class Https2ClientProtocolTestCase(TestCase): trustRoot=self.client_certificate, acceptableProtocols=[b'h2'] ) - h2_client_factory = Factory.forProtocol(H2ClientProtocol) + uri = URI.fromBytes(to_bytes(self.get_url('/'))) + h2_client_factory = H2ClientFactory(uri, Settings()) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) From 64c6af10e1b276db68ea722845621912d2023a75 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 13 Jul 2020 00:57:49 +0530 Subject: [PATCH 0117/2083] refactor: use str instead of to_unicode --- scrapy/core/http2/stream.py | 7 +++---- tests/test_http2_client_protocol.py | 29 ++++++++++++++++------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 5017f9cd4..a2f0e2aa4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -16,7 +16,6 @@ from scrapy.core.http2.types import H2ResponseDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes -from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -177,8 +176,8 @@ class Stream: # Make sure that we are sending the request to the correct URL url = urlparse(self._request.url) return ( - url.netloc == to_unicode(self._protocol.metadata['uri'].host) - or url.netloc == to_unicode(self._protocol.metadata['uri'].netloc) + url.netloc == str(self._protocol.metadata['uri'].host, 'utf-8') + or url.netloc == str(self._protocol.metadata['uri'].netloc, 'utf-8') or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' ) @@ -394,7 +393,7 @@ class Stream: elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( self._request, - to_unicode(self._protocol.metadata['uri'].host), + str(self._protocol.metadata['uri'].host, 'utf-8'), f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' )) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 05e5f5047..05f4889ef 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -23,7 +23,6 @@ from scrapy.core.http2.protocol import H2ClientFactory from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings -from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -39,7 +38,7 @@ def make_html_body(val):

Hello from HTTP2

{val}

''' - return to_bytes(response) + return bytes(response, 'utf-8') class Data: @@ -83,10 +82,11 @@ class PostDataJsonMixin: 'extra-data': extra_data } for k, v in request.requestHeaders.getAllRawHeaders(): - response['request-headers'][to_unicode(k)] = to_unicode(v[0]) + response['request-headers'][str(k, 'utf-8')] = str(v[0], 'utf-8') - response_bytes = to_bytes(json.dumps(response)) - request.setHeader('Content-Type', 'application/json') + response_bytes = bytes(json.dumps(response), 'utf-8') + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') return response_bytes @@ -127,13 +127,14 @@ class NoContentLengthHeader(LeafResource): class QueryParams(LeafResource): def render_GET(self, request: TxRequest): - request.setHeader('Content-Type', 'application/json') + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') query_params = {} for k, v in request.args.items(): - query_params[to_unicode(k)] = to_unicode(v[0]) + query_params[str(k, 'utf-8')] = str(v[0], 'utf-8') - return to_bytes(json.dumps(query_params)) + return bytes(json.dumps(query_params), 'utf-8') def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: @@ -184,7 +185,7 @@ class Https2ClientProtocolTestCase(TestCase): trustRoot=self.client_certificate, acceptableProtocols=[b'h2'] ) - uri = URI.fromBytes(to_bytes(self.get_url('/'))) + uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) h2_client_factory = H2ClientFactory(uri, Settings()) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) @@ -278,7 +279,8 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(len(response.body), content_length) # Parse the body - body = json.loads(to_unicode(response.body)) + content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + body = json.loads(str(response.body, content_encoding)) self.assertIn('request-body', body) self.assertIn('extra-data', body) self.assertIn('request-headers', body) @@ -292,9 +294,9 @@ class Https2ClientProtocolTestCase(TestCase): # Check if headers were sent successfully request_headers = body['request-headers'] for k, v in request.headers.items(): - k_str = to_unicode(k) + k_str = str(k, 'utf-8') self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], to_unicode(v[0])) + self.assertEqual(request_headers[k_str], str(v[0], 'utf-8')) d.addCallback(assert_response) d.addErrback(self.fail) @@ -494,7 +496,8 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(self.get_url(f'/query-params?{urlencode(params)}')) def assert_query_params(response: Response): - data = json.loads(to_unicode(response.body)) + content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + data = json.loads(str(response.body, content_encoding)) self.assertEqual(data, params) d = self.client.request(request) From 0770961054f24d56d219a81d9e0c467de98312c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 13 Jul 2020 16:05:57 +0200 Subject: [PATCH 0118/2083] Write a test for #4665 --- tests/test_commands.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 002237824..2e5bd6c00 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -66,9 +66,14 @@ class ProjectTest(unittest.TestCase): def proc(self, *new_args, **popen_kwargs): args = (sys.executable, '-m', 'scrapy.cmdline') + new_args - p = subprocess.Popen(args, cwd=self.cwd, env=self.env, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **popen_kwargs) + p = subprocess.Popen( + args, + cwd=popen_kwargs.pop('cwd', self.cwd), + env=self.env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **popen_kwargs, + ) def kill_proc(): p.kill() @@ -122,6 +127,32 @@ class StartprojectTest(ProjectTest): self.assertEqual(2, self.call('startproject')) self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params')) + def test_existing_project_dir(self): + project_dir = mkdtemp() + os.mkdir(os.path.join(project_dir, self.project_name)) + + p, out, err = self.proc('startproject', self.project_name, cwd=project_dir) + print(out) + print(err, file=sys.stderr) + self.assertEqual(p.returncode, 0) + + assert exists(join(abspath(project_dir), 'scrapy.cfg')) + assert exists(join(abspath(project_dir), 'testproject')) + assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py')) + assert exists(join(join(abspath(project_dir), self.project_name), 'items.py')) + assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py')) + assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py')) + assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py')) + + self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2')) + + self.assertEqual(1, self.call('startproject', self.project_name, project_dir)) + self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir)) + self.assertEqual(1, self.call('startproject', 'wrong---project---name')) + self.assertEqual(1, self.call('startproject', 'sys')) + self.assertEqual(2, self.call('startproject')) + self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params')) + def get_permissions_dict(path, renamings=None, ignore=None): renamings = renamings or tuple() From 544c1f6e390c72053f768d3992ea2d0801363b83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 13 Jul 2020 16:30:34 +0200 Subject: [PATCH 0119/2083] Fix the issue --- scrapy/commands/startproject.py | 8 ++++---- tests/test_commands.py | 34 +++++++++++++++------------------ 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e5158d993..35b58090c 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,7 +1,7 @@ import re import os import string -from importlib import import_module +from importlib.util import find_spec from os.path import join, exists, abspath from shutil import ignore_patterns, move, copy2, copystat from stat import S_IWUSR as OWNER_WRITE_PERMISSION @@ -43,10 +43,10 @@ class Command(ScrapyCommand): def _is_valid_name(self, project_name): def _module_exists(module_name): try: - import_module(module_name) - return True - except ImportError: + spec = find_spec(module_name) + except ModuleNotFoundError: return False + return spec is not None and spec.loader is not None if not re.search(r'^[_a-zA-Z]\w*$', project_name): print('Error: Project names must begin with a letter and contain' diff --git a/tests/test_commands.py b/tests/test_commands.py index 2e5bd6c00..10a3aa16c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -92,7 +92,10 @@ class ProjectTest(unittest.TestCase): class StartprojectTest(ProjectTest): def test_startproject(self): - self.assertEqual(0, self.call('startproject', self.project_name)) + p, out, err = self.proc('startproject', self.project_name) + print(out) + print(err, file=sys.stderr) + self.assertEqual(p.returncode, 0) assert exists(join(self.proj_path, 'scrapy.cfg')) assert exists(join(self.proj_path, 'testproject')) @@ -129,29 +132,22 @@ class StartprojectTest(ProjectTest): def test_existing_project_dir(self): project_dir = mkdtemp() - os.mkdir(os.path.join(project_dir, self.project_name)) + project_name = self.project_name + '_existing' + project_path = os.path.join(project_dir, project_name) + os.mkdir(project_path) - p, out, err = self.proc('startproject', self.project_name, cwd=project_dir) + p, out, err = self.proc('startproject', project_name, cwd=project_dir) print(out) print(err, file=sys.stderr) self.assertEqual(p.returncode, 0) - assert exists(join(abspath(project_dir), 'scrapy.cfg')) - assert exists(join(abspath(project_dir), 'testproject')) - assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'items.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py')) - - self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2')) - - self.assertEqual(1, self.call('startproject', self.project_name, project_dir)) - self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir)) - self.assertEqual(1, self.call('startproject', 'wrong---project---name')) - self.assertEqual(1, self.call('startproject', 'sys')) - self.assertEqual(2, self.call('startproject')) - self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params')) + assert exists(join(abspath(project_path), 'scrapy.cfg')) + assert exists(join(abspath(project_path), project_name)) + assert exists(join(join(abspath(project_path), project_name), '__init__.py')) + assert exists(join(join(abspath(project_path), project_name), 'items.py')) + assert exists(join(join(abspath(project_path), project_name), 'pipelines.py')) + assert exists(join(join(abspath(project_path), project_name), 'settings.py')) + assert exists(join(join(abspath(project_path), project_name), 'spiders', '__init__.py')) def get_permissions_dict(path, renamings=None, ignore=None): From aeaeb7385b7d1e6570bad38da4db492de8fb4206 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 14 Jul 2020 03:55:14 +0530 Subject: [PATCH 0120/2083] feat: assert negotiated protocol as h2 - implement IHandshakeListener in H2ClientProtocol to know when handshake is completed - implement IProtocolNegotiationFactory in H2ClientFactory to provide information about the acceptableProtols (h2) during NPN or ALPN protocol --- scrapy/core/http2/protocol.py | 69 ++++++++++++++++++++++------- scrapy/core/http2/stream.py | 4 +- tests/test_http2_client_protocol.py | 2 +- 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index ee51300cc..5d859d475 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -12,10 +12,12 @@ from h2.events import ( ) from h2.exceptions import ProtocolError from twisted.internet.defer import Deferred +from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory from twisted.internet.protocol import connectionDone, Factory, Protocol from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from twisted.web.client import URI +from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict @@ -25,15 +27,29 @@ from scrapy.settings import Settings logger = logging.getLogger(__name__) +class InvalidNegotiatedProtocol(ProtocolError): + + def __init__(self, negotiated_protocol: str) -> None: + self.negotiated_protocol = negotiated_protocol + + def __str__(self) -> str: + return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}' + + +@implementer(IHandshakeListener) class H2ClientProtocol(Protocol): - def __init__(self, uri: URI, settings: Settings) -> None: + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: """ Arguments: uri -- URI of the base url to which HTTP/2 Connection will be made. uri is used to verify that incoming client requests have correct base URL. settings -- Scrapy project settings + conn_lost_deferred -- Deferred fires with the reason: Failure to notify + that connection was lost """ + self._conn_lost_deferred = conn_lost_deferred + config = H2Configuration(client_side=True, header_encoding='utf-8') self.conn = H2Connection(config=config) @@ -55,9 +71,9 @@ class H2ClientProtocol(Protocol): # We use simple FIFO policy to handle pending requests self._active_streams = 0 - # Save an instance of ProtocolError raised by hyper-h2 - # We pass this instance to the streams ResponseFailed() failure - self._protocol_error: Optional[ProtocolError] = None + # Save an instance of errors raised which lead to losing the connection + # We pass these instances to the streams ResponseFailed() failure + self._conn_lost_errors: List[BaseException] = [] self.metadata: H2ConnectionMetadataDict = { 'certificate': None, @@ -136,6 +152,12 @@ class H2ClientProtocol(Protocol): # Add the stream to the request pool self._pending_request_stream_pool.append(stream) + + # If we are connection and receive a request + # There is a good chance that the connection was IDLE + # Hence, we need to initiate pending requests + if self.is_connected: + self._send_pending_requests() return d def connectionMade(self) -> None: @@ -149,6 +171,19 @@ class H2ClientProtocol(Protocol): self.conn.initiate_connection() self._write_to_transport() + def _lose_connection_with_error(self, errors: List[BaseException]): + """Helper function to lose the connection with the error sent as a + reason""" + self._conn_lost_errors += errors + self.transport.loseConnection() + + def handshakeCompleted(self): + """We close the connection with InvalidNegotiatedProtocol exception + when the connection was not made via h2 protocol""" + negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') + if negotiated_protocol != 'h2': + self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) + def dataReceived(self, data: bytes) -> None: try: events = self.conn.receive_data(data) @@ -157,10 +192,7 @@ class H2ClientProtocol(Protocol): # Save this error as ultimately the connection will be dropped # internally by hyper-h2. Saved error will be passed to all the streams # closed with the connection. - self._protocol_error = e - - # We lose the transport connection here - self.transport.loseConnection() + self._lose_connection_with_error([e]) finally: self._write_to_transport() @@ -168,17 +200,17 @@ class H2ClientProtocol(Protocol): """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ - errors = [] + # Notify the connection pool instance such that no new requests are + # sent over current connection if not reason.check(connectionDone): - logger.warning("Connection lost with reason " + str(reason)) - errors.append(reason) + self._conn_lost_errors.append(reason) - if self._protocol_error: - errors.append(self._protocol_error) + if self._conn_lost_deferred: + self._conn_lost_deferred.callback(self._conn_lost_errors) for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, errors, from_protocol=True) + stream.close(StreamCloseReason.CONNECTION_LOST, self._conn_lost_errors, from_protocol=True) else: stream.close(StreamCloseReason.INACTIVE, from_protocol=True) @@ -242,10 +274,15 @@ class H2ClientProtocol(Protocol): stream.receive_window_update() +@implementer(IProtocolNegotiationFactory) class H2ClientFactory(Factory): - def __init__(self, uri: URI, settings: Settings) -> None: + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: self.uri = uri self.settings = settings + self.conn_lost_deferred = conn_lost_deferred def buildProtocol(self, addr) -> H2ClientProtocol: - return H2ClientProtocol(self.uri, self.settings) + return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) + + def acceptableProtocols(self) -> List[bytes]: + return [b'h2'] diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a2f0e2aa4..16d54b2fc 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -44,7 +44,7 @@ class InvalidHostname(Exception): class StreamCloseReason(Enum): - # Received a StreamEnded event + # Received a StreamEnded event from the remote ENDED = 1 # Received a StreamReset event -- ended abruptly @@ -329,7 +329,7 @@ class Stream: def close( self, reason: StreamCloseReason, - errors: Optional[List[Exception]] = None, + errors: Optional[List[BaseException]] = None, from_protocol: bool = False ) -> None: """Based on the reason sent we will handle each case. diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 05f4889ef..7fcea58c5 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -9,7 +9,7 @@ from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks, DeferredList, CancelledError +from twisted.internet.defer import CancelledError, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure From 1dd27a92fa53be87c71df43bd6f3043a225a2c10 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 14 Jul 2020 17:58:22 +0530 Subject: [PATCH 0121/2083] feat: Idle Timeout for H2Connection (240s) --- scrapy/core/http2/protocol.py | 60 ++++++++++++++++++++++++----- scrapy/core/http2/stream.py | 3 +- tests/test_http2_client_protocol.py | 10 +++-- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 5d859d475..bf41a2805 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -6,15 +6,18 @@ from typing import Dict, List, Optional from h2.config import H2Configuration from h2.connection import H2Connection +from h2.errors import ErrorCodes from h2.events import ( Event, DataReceived, ResponseReceived, SettingsAcknowledged, StreamEnded, StreamReset, WindowUpdated ) from h2.exceptions import ProtocolError from twisted.internet.defer import Deferred +from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory from twisted.internet.protocol import connectionDone, Factory, Protocol from twisted.internet.ssl import Certificate +from twisted.protocols.policies import TimeoutMixin from twisted.python.failure import Failure from twisted.web.client import URI from zope.interface import implementer @@ -37,7 +40,9 @@ class InvalidNegotiatedProtocol(ProtocolError): @implementer(IHandshakeListener) -class H2ClientProtocol(Protocol): +class H2ClientProtocol(Protocol, TimeoutMixin): + IDLE_TIMEOUT = 240 + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: """ Arguments: @@ -71,6 +76,10 @@ class H2ClientProtocol(Protocol): # We use simple FIFO policy to handle pending requests self._active_streams = 0 + # Flag to keep track if settings were acknowledged by the remote + # This ensures that we have established a HTTP/2 connection + self._settings_acknowledged = False + # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure self._conn_lost_errors: List[BaseException] = [] @@ -84,12 +93,12 @@ class H2ClientProtocol(Protocol): } @property - def is_connected(self) -> bool: + def h2_connected(self) -> bool: """Boolean to keep track of the connection status. This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection """ - return bool(self.transport.connected) + return bool(self.transport.connected) and self._settings_acknowledged @property def allowed_max_concurrent_streams(self) -> int: @@ -111,7 +120,7 @@ class H2ClientProtocol(Protocol): while ( self._pending_request_stream_pool and self._active_streams < self.allowed_max_concurrent_streams - and self.is_connected + and self.h2_connected ): self._active_streams += 1 stream = self._pending_request_stream_pool.popleft() @@ -140,6 +149,9 @@ class H2ClientProtocol(Protocol): """ Write data to the underlying transport connection from the HTTP2 connection instance if any """ + # Reset the idle timeout as connection is still actively sending data + self.resetTimeout() + data = self.conn.data_to_send() self.transport.write(data) @@ -153,21 +165,23 @@ class H2ClientProtocol(Protocol): # Add the stream to the request pool self._pending_request_stream_pool.append(stream) - # If we are connection and receive a request - # There is a good chance that the connection was IDLE - # Hence, we need to initiate pending requests - if self.is_connected: - self._send_pending_requests() + # If we receive a request when connection is idle + # We need to initiate pending requests + self._send_pending_requests() return d def connectionMade(self) -> None: """Called by Twisted when the connection is established. We can start sending some data now: we should open with the connection preamble. """ + # Initialize the timeout + self.setTimeout(self.IDLE_TIMEOUT) + destination = self.transport.getPeer() logger.debug('Connection made to {}'.format(destination)) self.metadata['ip_address'] = ipaddress.ip_address(destination.host) + # Initiate H2 Connection self.conn.initiate_connection() self._write_to_transport() @@ -185,6 +199,9 @@ class H2ClientProtocol(Protocol): self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) def dataReceived(self, data: bytes) -> None: + # Reset the idle timeout as connection is still actively receiving data + self.resetTimeout() + try: events = self.conn.receive_data(data) self._handle_events(events) @@ -196,10 +213,33 @@ class H2ClientProtocol(Protocol): finally: self._write_to_transport() + def timeoutConnection(self): + """Called when the connection times out. + We lose the connection with TimeoutError""" + + # Check whether there are open streams. If there are, we're going to + # want to use the error code PROTOCOL_ERROR. If there aren't, use + # NO_ERROR. + if ( + self.conn.open_outbound_streams > 0 + or self.conn.open_inbound_streams > 0 + or self._active_streams > 0 + ): + error_code = ErrorCodes.PROTOCOL_ERROR + else: + error_code = ErrorCodes.NO_ERROR + self.conn.close_connection(error_code=error_code) + self._write_to_transport() + + self._lose_connection_with_error([TimeoutError("Hello")]) + def connectionLost(self, reason: Failure = connectionDone) -> None: """Called by Twisted when the transport connection is lost. No need to write anything to transport here. """ + # Cancel the timeout if not done yet + self.setTimeout(None) + # Notify the connection pool instance such that no new requests are # sent over current connection if not reason.check(connectionDone): @@ -250,6 +290,8 @@ class H2ClientProtocol(Protocol): self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: + self._settings_acknowledged = True + # Send off all the pending requests as now we have # established a proper HTTP/2 connection self._send_pending_requests() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 16d54b2fc..f60691945 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -134,7 +134,8 @@ class Stream: def _cancel(_): # Close this stream as gracefully as possible - # Check if the stream has started + # If the associated request is initiated we reset this stream + # else we directly call close() method if self.request_sent: self.reset_stream(StreamCloseReason.CANCELLED) else: diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7fcea58c5..2833801e7 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -9,7 +9,7 @@ from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor -from twisted.internet.defer import CancelledError, DeferredList, inlineCallbacks +from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure @@ -174,6 +174,7 @@ class Https2ClientProtocolTestCase(TestCase): # Start server for testing self.hostname = u'localhost' context_factory = ssl_context_factory(self.key_file, self.certificate_file) + server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) self.server = yield server_endpoint.listen(self.site) self.port_number = self.server.getHost().port @@ -186,17 +187,20 @@ class Https2ClientProtocolTestCase(TestCase): acceptableProtocols=[b'h2'] ) uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) - h2_client_factory = H2ClientFactory(uri, Settings()) + + self.conn_closed_deferred = Deferred() + h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) @inlineCallbacks def tearDown(self): - if self.client.is_connected: + if self.client.connected: yield self.client.transport.loseConnection() yield self.client.transport.abortConnection() yield self.server.stopListening() shutil.rmtree(self.temp_directory) + self.conn_closed_deferred = None def get_url(self, path): """ From e662762e6ab2f53ff2e31e21f34c078590f65aa9 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 15 Jul 2020 03:45:32 +0530 Subject: [PATCH 0122/2083] chore: Handle ConnectionTerminated event --- scrapy/core/http2/protocol.py | 44 +++++++++++++++++++++++++++-------- scrapy/core/http2/stream.py | 4 ++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index bf41a2805..041908116 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -2,16 +2,18 @@ import ipaddress import itertools import logging from collections import deque -from typing import Dict, List, Optional +from ipaddress import IPv4Address, IPv6Address +from typing import Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection from h2.errors import ErrorCodes from h2.events import ( - Event, DataReceived, ResponseReceived, SettingsAcknowledged, - StreamEnded, StreamReset, WindowUpdated + Event, ConnectionTerminated, DataReceived, ResponseReceived, + SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived, + WindowUpdated ) -from h2.exceptions import ProtocolError +from h2.exceptions import H2Error, ProtocolError from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory @@ -30,7 +32,7 @@ from scrapy.settings import Settings logger = logging.getLogger(__name__) -class InvalidNegotiatedProtocol(ProtocolError): +class InvalidNegotiatedProtocol(H2Error): def __init__(self, negotiated_protocol: str) -> None: self.negotiated_protocol = negotiated_protocol @@ -39,6 +41,15 @@ class InvalidNegotiatedProtocol(ProtocolError): return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}' +class RemoteTerminatedConnection(H2Error): + def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], event: ConnectionTerminated): + self.remote_ip_address = remote_ip_address + self.terminate_event = event + + def __str__(self) -> str: + return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address}' + + @implementer(IHandshakeListener) class H2ClientProtocol(Protocol, TimeoutMixin): IDLE_TIMEOUT = 240 @@ -194,8 +205,12 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def handshakeCompleted(self): """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" - negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') + negotiated_protocol = self.transport.negotiatedProtocol + if type(negotiated_protocol) is bytes: + negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') if negotiated_protocol != 'h2': + # Here we have not initiated the connection yet + # So, no need to send a GOAWAY frame to the remote self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) def dataReceived(self, data: bytes) -> None: @@ -231,7 +246,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.conn.close_connection(error_code=error_code) self._write_to_transport() - self._lose_connection_with_error([TimeoutError("Hello")]) + self._lose_connection_with_error([ + TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s") + ]) def connectionLost(self, reason: Failure = connectionDone) -> None: """Called by Twisted when the transport connection is lost. @@ -267,7 +284,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin): events -- A list of events that the remote peer triggered by sending data """ for event in events: - if isinstance(event, DataReceived): + if isinstance(event, ConnectionTerminated): + self.connection_terminated(event) + elif isinstance(event, DataReceived): self.data_received(event) elif isinstance(event, ResponseReceived): self.response_received(event) @@ -279,10 +298,15 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.window_updated(event) elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) - else: - logger.debug('Received unhandled event {}'.format(event)) + elif isinstance(event, UnknownFrameReceived): + logger.debug(f'UnknownFrameReceived: frame={event.frame}') # Event handler functions starts here + def connection_terminated(self, event: ConnectionTerminated) -> None: + self._lose_connection_with_error([ + RemoteTerminatedConnection(self.metadata['ip_address'], event) + ]) + def data_received(self, event: DataReceived) -> None: self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index f60691945..15f081cab 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -5,7 +5,7 @@ from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes -from h2.exceptions import StreamClosedError +from h2.exceptions import H2Error, StreamClosedError from hpack import HeaderTuple from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed @@ -32,7 +32,7 @@ class InactiveStreamClosed(ConnectionClosed): self.request = request -class InvalidHostname(Exception): +class InvalidHostname(H2Error): def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None: self.request = request From 316620b517207b1082dd9c5b4ebfc7fbe745e3bb Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 22 Jul 2020 13:53:46 +0530 Subject: [PATCH 0123/2083] chore: pass spider as argument for request method - download_maxsize and download_warnsize can now be extracted from the spider directly and passed to the stream - remove `partial` flag from the response as per RFC 7540 - Section 8.1.2.6 --- scrapy/core/http2/protocol.py | 12 +++++--- scrapy/core/http2/stream.py | 35 ++++++++++------------ tests/test_http2_client_protocol.py | 46 +++++++++++++++++++---------- 3 files changed, 55 insertions(+), 38 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 041908116..feb034a0c 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -28,6 +28,7 @@ from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request from scrapy.settings import Settings +from scrapy.spiders import Spider logger = logging.getLogger(__name__) @@ -71,7 +72,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # ID of the next request stream # Following the convention - 'Streams initiated by a client MUST - # use odd-numbered stream identifiers' (RFC 7540) + # use odd-numbered stream identifiers' (RFC 7540 - Section 5.1.1) self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs @@ -136,6 +137,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._active_streams += 1 stream = self._pending_request_stream_pool.popleft() stream.initiate_request() + self._write_to_transport() def pop_stream(self, stream_id: int) -> Stream: """Perform cleanup when a stream is closed @@ -145,13 +147,15 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._send_pending_requests() return stream - def _new_stream(self, request: Request) -> Stream: + def _new_stream(self, request: Request, spider: Spider) -> Stream: """Instantiates a new Stream object """ stream = Stream( stream_id=next(self._stream_id_generator), request=request, protocol=self, + download_maxsize=getattr(spider, 'download_maxsize', self.metadata['default_download_maxsize']), + download_warnsize=getattr(spider, 'download_warnsize', self.metadata['default_download_warnsize']), ) self.streams[stream.stream_id] = stream return stream @@ -166,11 +170,11 @@ class H2ClientProtocol(Protocol, TimeoutMixin): data = self.conn.data_to_send() self.transport.write(data) - def request(self, request: Request) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred: if not isinstance(request, Request): raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}') - stream = self._new_stream(request) + stream = self._new_stream(request, spider) d = stream.get_response() # Add the stream to the request pool diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 15f081cab..133196797 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -83,7 +83,9 @@ class Stream: self, stream_id: int, request: Request, - protocol: "H2ClientProtocol" + protocol: "H2ClientProtocol", + download_maxsize: int = 0, + download_warnsize: int = 0 ) -> None: """ Arguments: @@ -95,14 +97,8 @@ class Stream: self._request: Request = request self._protocol: "H2ClientProtocol" = protocol - self._download_maxsize = self._request.meta.get( - 'download_maxsize', - self._protocol.metadata['default_download_maxsize'] - ) - self._download_warnsize = self._request.meta.get( - 'download_warnsize', - self._protocol.metadata['default_download_warnsize'] - ) + self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) + self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) self.request_start_time = None @@ -346,26 +342,28 @@ class Stream: self.stream_closed_server = True - flags = None - if b'Content-Length' not in self._response['headers']: - # Missing Content-Length - {twisted.web.http.PotentialDataLoss} - flags = ['partial'] + # We do not check for Content-Length or Transfer-Encoding in response headers + # and add `partial` flag as in HTTP/1.1 as 'A request or response that includes + # a payload body can include a content-length header field' (RFC 7540 - Section 8.1.2.6) # NOTE: Order of handling the events is important here # As we immediately cancel the request when maxsize is exceeded while # receiving DATA_FRAME's when we have received the headers (not # having Content-Length) if reason is StreamCloseReason.MAXSIZE_EXCEEDED: - expected_size = int(self._response['headers'].get(b'Content-Length', -1)) + expected_size = int(self._response['headers'].get( + b'Content-Length', + self._response['flow_controlled_size']) + ) error_msg = ( - f'Cancelling download of {self._request.url}: expected response ' - f'size ({expected_size}) larger than download max size ({self._download_maxsize}).' + f'Cancelling download of {self._request.url}: received response ' + f'size ({expected_size}) larger than download max size ({self._download_maxsize})' ) logger.error(error_msg) self._deferred_response.errback(CancelledError(error_msg)) elif reason is StreamCloseReason.ENDED: - self._fire_response_deferred(flags) + self._fire_response_deferred() # Stream was abruptly ended here elif reason is StreamCloseReason.CANCELLED: @@ -398,7 +396,7 @@ class Stream: f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' )) - def _fire_response_deferred(self, flags: Optional[List[str]] = None) -> None: + def _fire_response_deferred(self) -> None: """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" @@ -416,7 +414,6 @@ class Stream: headers=self._response['headers'], body=body, request=self._request, - flags=flags, certificate=self._protocol.metadata['certificate'], ip_address=self._protocol.metadata['ip_address'], ) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 2833801e7..d0386f7f8 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -23,6 +23,7 @@ from scrapy.core.http2.protocol import H2ClientFactory from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings +from scrapy.spiders import Spider from tests.mockserver import ssl_context_factory, LeafResource, Status @@ -41,6 +42,14 @@ def make_html_body(val): return bytes(response, 'utf-8') +class DummySpider(Spider): + name = 'dummy' + start_urls = [] + + def parse(self, response): + print(response) + + class Data: SMALL_SIZE = 1024 # 1 KB LARGE_SIZE = 1024 ** 2 # 1 MB @@ -210,6 +219,9 @@ class Https2ClientProtocolTestCase(TestCase): assert len(path) > 0 and (path[0] == '/' or path[0] == '&') return f'{self.scheme}://{self.hostname}:{self.port_number}{path}' + def make_request(self, request: Request) -> Deferred: + return self.client.request(request, DummySpider()) + @staticmethod def _check_repeat(get_deferred, count): d_list = [] @@ -233,7 +245,7 @@ class Https2ClientProtocolTestCase(TestCase): content_length = int(response.headers.get('Content-Length')) self.assertEqual(len(response.body), content_length) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(check_response) d.addErrback(self.fail) return d @@ -273,7 +285,7 @@ class Https2ClientProtocolTestCase(TestCase): expected_extra_data, expected_status: int ): - d = self.client.request(request) + d = self.make_request(request) def assert_response(response: Response): self.assertEqual(response.status, expected_status) @@ -355,7 +367,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, 499) self.assertEqual(response.request, request) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_response) d.addErrback(self.fail) d.cancel() @@ -367,8 +379,13 @@ class Https2ClientProtocolTestCase(TestCase): def assert_cancelled_error(failure): self.assertIsInstance(failure.value, CancelledError) + error_pattern = re.compile( + rf'Cancelling download of {request.url}: received response ' + rf'size \(\d*\) larger than download max size \(1000\)' + ) + self.assertEqual(len(re.findall(error_pattern, str(failure.value))), 1) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(self.fail) d.addErrback(assert_cancelled_error) return d @@ -385,7 +402,7 @@ class Https2ClientProtocolTestCase(TestCase): for error in failure.value.reasons )) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(self.fail) d.addErrback(assert_failure) return d @@ -397,10 +414,9 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, 200) self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) self.assertEqual(response.request, request) - self.assertIn('partial', response.flags) self.assertNotIn('Content-Length', response.headers) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_content_length) d.addErrback(self.fail) return d @@ -413,7 +429,7 @@ class Https2ClientProtocolTestCase(TestCase): expected_body ): with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm: - response = yield self.client.request(request) + response = yield self.make_request(request) self.assertEqual(response.status, 200) self.assertEqual(response.request, request) self.assertEqual(response.body, expected_body) @@ -469,13 +485,13 @@ class Https2ClientProtocolTestCase(TestCase): # Send 100 request (we do not check the result) for _ in range(100): - d = self.client.request(Request(self.get_url('/get-data-html-small'))) + d = self.make_request(Request(self.get_url('/get-data-html-small'))) d.addBoth(lambda _: None) d_list.append(d) # Now send 10 extra request and save the response deferred in a list for _ in range(10): - d = self.client.request(Request(self.get_url('/get-data-html-small'))) + d = self.make_request(Request(self.get_url('/get-data-html-small'))) d.addCallback(self.fail) d.addErrback(assert_inactive_stream) d_list.append(d) @@ -488,7 +504,7 @@ class Https2ClientProtocolTestCase(TestCase): def test_invalid_request_type(self): with self.assertRaises(TypeError): - self.client.request('https://InvalidDataTypePassed.com') + self.make_request('https://InvalidDataTypePassed.com') def test_query_parameters(self): params = { @@ -504,7 +520,7 @@ class Https2ClientProtocolTestCase(TestCase): data = json.loads(str(response.body, content_encoding)) self.assertEqual(data, params) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_query_params) d.addErrback(self.fail) @@ -517,7 +533,7 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [] for status in [200, 404]: request = Request(self.get_url(f'/status?n={status}')) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_response_status, status) d.addErrback(self.fail) d_list.append(d) @@ -537,7 +553,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertIsInstance(response.ip_address, IPv4Address) self.assertEqual(str(response.ip_address), '127.0.0.1') - d = self.client.request(request) + d = self.make_request(request) d.addCallback(assert_metadata) d.addErrback(self.fail) @@ -553,7 +569,7 @@ class Https2ClientProtocolTestCase(TestCase): self.assertIn('127.0.0.1', error_msg) self.assertIn(str(request), error_msg) - d = self.client.request(request) + d = self.make_request(request) d.addCallback(self.fail) d.addErrback(assert_invalid_hostname) return d From 3685e99cca47a5006258cd4246f255216797641f Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 22 Jul 2020 14:47:20 +0530 Subject: [PATCH 0124/2083] test: http2 connection timeout --- tests/test_http2_client_protocol.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index d0386f7f8..746eef4d6 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -18,8 +18,9 @@ from twisted.web.client import URI from twisted.web.http import Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File +from twisted.internet.error import TimeoutError -from scrapy.core.http2.protocol import H2ClientFactory +from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings @@ -134,6 +135,11 @@ class NoContentLengthHeader(LeafResource): request.finish() +class TimeoutResponse(LeafResource): + def render_GET(self, request: TxRequest): + return NOT_DONE_YET + + class QueryParams(LeafResource): def render_GET(self, request: TxRequest): request.setHeader('Content-Type', 'application/json; charset=UTF-8') @@ -172,6 +178,7 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'no-content-length-header', NoContentLengthHeader()) r.putChild(b'status', Status()) r.putChild(b'query-params', QueryParams()) + r.putChild(b'timeout', TimeoutResponse()) return r @inlineCallbacks @@ -590,3 +597,22 @@ class Https2ClientProtocolTestCase(TestCase): ] return DeferredList(d_list, fireOnOneErrback=True) + + def test_connection_timeout(self): + request = Request(self.get_url('/timeout')) + d = self.make_request(request) + + # Update the timer to 1s to test connection timeout + self.client.setTimeout(1) + + def assert_timeout_error(failure: Failure): + for err in failure.value.reasons: + if isinstance(err, TimeoutError): + self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err)) + break + else: + self.fail() + + d.addCallback(self.fail) + d.addErrback(assert_timeout_error) + return d From 9fffb801ed3dedbb3935c811c7d61ed953ff22dc Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 8 Jul 2020 20:18:38 +0530 Subject: [PATCH 0125/2083] feat: H2Agent, H2ConnectionPool base implementation --- scrapy/core/downloader/handlers/http2.py | 55 ++++++++++++++++++++ scrapy/core/http2/agent.py | 64 ++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 scrapy/core/downloader/handlers/http2.py create mode 100644 scrapy/core/http2/agent.py diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py new file mode 100644 index 000000000..0dc06f4d8 --- /dev/null +++ b/scrapy/core/downloader/handlers/http2.py @@ -0,0 +1,55 @@ +import warnings + +from scrapy.core.downloader.tls import openssl_methods +from scrapy.core.http2.agent import H2Agent, H2ConnectionPool +from scrapy.http.request import Request +from scrapy.settings import Settings +from scrapy.utils.misc import create_instance, load_object + + +class H2DownloadHandler: + def __init__(self, settings: Settings, crawler=None): + self._crawler = crawler + + from twisted.internet import reactor + self._pool = H2ConnectionPool(reactor, settings) + + self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] + self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + # try method-aware context factory + try: + self._context_factory = create_instance( + objcls=self._context_factory_cls, + settings=settings, + crawler=crawler, + method=self._ssl_method, + ) + except TypeError: + # use context factory defaults + self._context_factory = create_instance( + objcls=self._context_factory_cls, + settings=settings, + crawler=crawler, + ) + msg = """ + '%s' does not accept `method` argument (type OpenSSL.SSL method,\ + e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" % ( + settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + warnings.warn(msg) + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler.settings, crawler) + + def download_request(self, request: Request, spider): + from twisted.internet import reactor + + agent = H2Agent(reactor, self._pool, self._context_factory) + d = agent.request(request) + + def print_result(result): + print(result) + + d.addCallback(print_result) + return d diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py new file mode 100644 index 000000000..e14e5d633 --- /dev/null +++ b/scrapy/core/http2/agent.py @@ -0,0 +1,64 @@ +from typing import Dict, Tuple + +from twisted.internet import defer +from twisted.internet.base import ReactorBase +from twisted.internet.defer import Deferred +from twisted.internet.endpoints import SSL4ClientEndpoint, optionsForClientTLS +from twisted.web.client import URI, BrowserLikePolicyForHTTPS + +from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory +from scrapy.http.request import Request +from scrapy.settings import Settings +from scrapy.utils.python import to_bytes, to_unicode + + +class H2ConnectionPool: + def __init__(self, reactor: ReactorBase, settings: Settings) -> None: + self._reactor = reactor + self.settings = settings + self._connections: Dict[Tuple, H2ClientProtocol] = {} + + def get_connection(self, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: + key = (uri.scheme, uri.host, uri.port) + conn = self._connections.get(key, None) + if conn: + return defer.succeed(conn) + return self._new_connection(key, uri, endpoint) + + def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: + factory = H2ClientFactory(uri, self.settings) + d = endpoint.connect(factory) + + def put_connection(conn: H2ClientProtocol) -> H2ClientProtocol: + self._connections[key] = conn + return conn + + d.addCallback(put_connection) + return d + + def _remove_connection(self, key) -> None: + conn = self._connections.pop(key) + conn.loseConnection() + + +class H2Agent: + def __init__( + self, reactor: ReactorBase, pool: H2ConnectionPool, + context_factory=BrowserLikePolicyForHTTPS() + ) -> None: + self._reactor = reactor + self._pool = pool + self._context_factory = context_factory + + def request(self, request: Request) -> Deferred: + uri = URI.fromBytes(to_bytes(request.url, encoding='ascii')) + # options = optionsForClientTLS(hostname=to_unicode(uri.host), acceptableProtocols=[b'h2']) + # Hacky fix: Use options instead of self._context_factory to make endpoint work for HTTP/2 + endpoint = SSL4ClientEndpoint(self._reactor, to_unicode(uri.host), uri.port, self._context_factory) + d = self._pool.get_connection(uri, endpoint) + + def cb_connected(conn: H2ClientProtocol): + return conn.request(request) + + d.addCallback(cb_connected) + return d From 8252a6f8d8930ce23bdf8a6f51038b4ce49d1968 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 15 Jul 2020 04:58:59 +0530 Subject: [PATCH 0126/2083] fix: H2Agent not able to connect via SSL - add H2WrappedContextFactory class which wraps the context factory passed to H2Agent and updates the SSL context acceptable protocols list to only h2 --- scrapy/core/downloader/handlers/http2.py | 1 + scrapy/core/http2/agent.py | 66 +++++++++++++++--------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 0dc06f4d8..c8b401e80 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -50,6 +50,7 @@ class H2DownloadHandler: def print_result(result): print(result) + return result d.addCallback(print_result) return d diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index e14e5d633..566c074c2 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,15 +1,19 @@ -from typing import Dict, Tuple +from typing import Dict, Tuple, Optional from twisted.internet import defer +from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred -from twisted.internet.endpoints import SSL4ClientEndpoint, optionsForClientTLS -from twisted.web.client import URI, BrowserLikePolicyForHTTPS +from twisted.internet.endpoints import SSL4ClientEndpoint +from twisted.python.failure import Failure +from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory +from twisted.web.iweb import IPolicyForHTTPS +from zope.interface import implementer +from zope.interface.verify import verifyObject from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory from scrapy.http.request import Request from scrapy.settings import Settings -from scrapy.utils.python import to_bytes, to_unicode class H2ConnectionPool: @@ -26,39 +30,51 @@ class H2ConnectionPool: return self._new_connection(key, uri, endpoint) def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: - factory = H2ClientFactory(uri, self.settings) + conn_lost_deferred = Deferred() + conn_lost_deferred.addCallback(self._remove_connection, key) + + factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) d = endpoint.connect(factory) - - def put_connection(conn: H2ClientProtocol) -> H2ClientProtocol: - self._connections[key] = conn - return conn - - d.addCallback(put_connection) + d.addCallback(self.put_connection, key) return d - def _remove_connection(self, key) -> None: - conn = self._connections.pop(key) - conn.loseConnection() + def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol: + self._connections[key] = conn + return conn + + def _remove_connection(self, reason: Failure, key: Tuple) -> None: + self._connections.pop(key) + + +@implementer(IPolicyForHTTPS) +class H2WrappedContextFactory: + def __init__(self, context_factory) -> None: + verifyObject(IPolicyForHTTPS, context_factory) + self._wrapped_context_factory = context_factory + + def creatorForNetloc(self, hostname, port) -> ClientTLSOptions: + options = self._wrapped_context_factory.creatorForNetloc(hostname, port) + _setAcceptableProtocols(options._ctx, [b'h2']) + return options class H2Agent: def __init__( self, reactor: ReactorBase, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS() + context_factory=BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None ) -> None: self._reactor = reactor self._pool = pool - self._context_factory = context_factory + self._context_factory = H2WrappedContextFactory(context_factory) + self._endpoint_factory = _StandardEndpointFactory( + self._reactor, self._context_factory, + connect_timeout, bind_address + ) def request(self, request: Request) -> Deferred: - uri = URI.fromBytes(to_bytes(request.url, encoding='ascii')) - # options = optionsForClientTLS(hostname=to_unicode(uri.host), acceptableProtocols=[b'h2']) - # Hacky fix: Use options instead of self._context_factory to make endpoint work for HTTP/2 - endpoint = SSL4ClientEndpoint(self._reactor, to_unicode(uri.host), uri.port, self._context_factory) + uri = URI.fromBytes(bytes(request.url, encoding='utf-8')) + endpoint = self._endpoint_factory.endpointForURI(uri) d = self._pool.get_connection(uri, endpoint) - - def cb_connected(conn: H2ClientProtocol): - return conn.request(request) - - d.addCallback(cb_connected) + d.addCallback(lambda conn: conn.request(request)) return d From 62ce842afc8ef829ffd6f164712a8a9413eb9e1d Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 15 Jul 2020 07:50:53 +0530 Subject: [PATCH 0127/2083] fix: multiple h2 connections to same uri - When multiple requests are sent to H2ConnectionPool to the same uri while the connection is in connecting state -- multiple connections were establised. - Fixed the bug using a deque of all the request deferred's which fire with the H2ClientProtocol (connection) instance when connection is established --- scrapy/core/http2/agent.py | 55 ++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 566c074c2..7a8847c38 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,11 +1,11 @@ -from typing import Dict, Tuple, Optional +from collections import deque +from typing import Deque, Dict, List, Tuple, Optional from twisted.internet import defer from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred -from twisted.internet.endpoints import SSL4ClientEndpoint -from twisted.python.failure import Failure +from twisted.internet.endpoints import HostnameEndpoint from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory from twisted.web.iweb import IPolicyForHTTPS from zope.interface import implementer @@ -20,31 +20,70 @@ class H2ConnectionPool: def __init__(self, reactor: ReactorBase, settings: Settings) -> None: self._reactor = reactor self.settings = settings + + # Store a dictionary which is used to get the respective + # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port) self._connections: Dict[Tuple, H2ClientProtocol] = {} - def get_connection(self, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: + # Save all requests that arrive before the connection is established + self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} + + def get_connection(self, uri: URI, endpoint: HostnameEndpoint) -> Deferred: key = (uri.scheme, uri.host, uri.port) + if key in self._pending_requests: + # Received a request while connecting to remote + # Create a deferred which will fire with the H2ClientProtocol + # instance + d = Deferred() + self._pending_requests[key].append(d) + return d + + # Check if we already have a connection to the remote conn = self._connections.get(key, None) if conn: + # Return this connection instance wrapped inside a deferred return defer.succeed(conn) + + # No connection is established for the given URI return self._new_connection(key, uri, endpoint) - def _new_connection(self, key: Tuple, uri: URI, endpoint: SSL4ClientEndpoint) -> Deferred: + def _new_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred: + self._pending_requests[key] = deque() + conn_lost_deferred = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) - d = endpoint.connect(factory) - d.addCallback(self.put_connection, key) + conn_d = endpoint.connect(factory) + conn_d.addCallback(self.put_connection, key) + + d = Deferred() + self._pending_requests[key].append(d) return d def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol: self._connections[key] = conn + + # Now as we have established a proper HTTP/2 connection + # we fire all the deferred's with the connection instance + pending_requests = self._pending_requests.pop(key) + while pending_requests: + d = pending_requests.popleft() + d.callback(conn) + + del pending_requests + return conn - def _remove_connection(self, reason: Failure, key: Tuple) -> None: + def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None: self._connections.pop(key) + # Call the errback of all the pending requests for this connection + pending_requests = self._pending_requests.pop(key, None) + while pending_requests: + d = pending_requests.popleft() + d.errback(errors) + @implementer(IPolicyForHTTPS) class H2WrappedContextFactory: From 031bfc9c3bed6c4dbfa4b7fcd48a8fc15f3582fb Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 22 Jul 2020 15:01:59 +0530 Subject: [PATCH 0128/2083] feat(wip): ScrapyH2Agent, ScrapyProxyH2Agent --- scrapy/core/downloader/contextfactory.py | 32 +++++ scrapy/core/downloader/handlers/http11.py | 27 +--- scrapy/core/downloader/handlers/http2.py | 161 +++++++++++++++++----- scrapy/core/http2/agent.py | 35 ++++- 4 files changed, 190 insertions(+), 65 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 452242d47..c0463cfc7 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,8 +1,12 @@ from OpenSSL import SSL +import warnings + from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer +from scrapy.core.downloader.tls import openssl_methods +from scrapy.utils.misc import create_instance, load_object from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS @@ -92,3 +96,31 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): trustRoot=platformTrust(), extraCertificateOptions={'method': self._ssl_method}, ) + + +def load_context_factory_from_settings(settings, crawler): + ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] + context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + # try method-aware context factory + try: + context_factory = create_instance( + objcls=context_factory_cls, + settings=settings, + crawler=crawler, + method=ssl_method, + ) + except TypeError: + # use context factory defaults + context_factory = create_instance( + objcls=context_factory_cls, + settings=settings, + crawler=crawler, + ) + msg = """ + '%s' does not accept `method` argument (type OpenSSL.SSL method,\ + e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" % ( + settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + warnings.warn(msg) + + return context_factory diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 22c9ac520..dac97ad29 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from zope.interface import implementer from scrapy import signals -from scrapy.core.downloader.tls import openssl_methods +from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload from scrapy.http import Headers from scrapy.responsetypes import responsetypes -from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import to_bytes, to_unicode @@ -43,29 +42,7 @@ class HTTP11DownloadHandler: self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False - self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] - self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - # try method-aware context factory - try: - self._contextFactory = create_instance( - objcls=self._contextFactoryClass, - settings=settings, - crawler=crawler, - method=self._sslMethod, - ) - except TypeError: - # use context factory defaults - self._contextFactory = create_instance( - objcls=self._contextFactoryClass, - settings=settings, - crawler=crawler, - ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) - warnings.warn(msg) + self._contextFactory = load_context_factory_from_settings(settings, crawler) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index c8b401e80..81ea78e69 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,10 +1,19 @@ import warnings +from time import time +from typing import Optional, Tuple +from urllib.parse import urldefrag -from scrapy.core.downloader.tls import openssl_methods +from twisted.internet.base import ReactorBase +from twisted.internet.error import TimeoutError +from twisted.web.client import URI + +from scrapy.core.downloader.contextfactory import load_context_factory_from_settings +from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool -from scrapy.http.request import Request +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.http import Request, Response from scrapy.settings import Settings -from scrapy.utils.misc import create_instance, load_object +from scrapy.spiders import Spider class H2DownloadHandler: @@ -13,44 +22,128 @@ class H2DownloadHandler: from twisted.internet import reactor self._pool = H2ConnectionPool(reactor, settings) - - self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] - self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - # try method-aware context factory - try: - self._context_factory = create_instance( - objcls=self._context_factory_cls, - settings=settings, - crawler=crawler, - method=self._ssl_method, - ) - except TypeError: - # use context factory defaults - self._context_factory = create_instance( - objcls=self._context_factory_cls, - settings=settings, - crawler=crawler, - ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) - warnings.warn(msg) + self._context_factory = load_context_factory_from_settings(settings, crawler) + self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') + self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') + self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') @classmethod def from_crawler(cls, crawler): return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider): + def download_request(self, request: Request, spider: Spider): + agent = ScrapyH2Agent( + context_factory=self._context_factory, + pool=self._pool, + maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), + warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), + crawler=self._crawler + ) + return agent.download_request(request, spider) + + def close(self) -> None: + self._pool.close_connections() + + +class ScrapyProxyH2Agent(H2Agent): + def __init__( + self, reactor: ReactorBase, + proxy_uri: URI, pool: H2ConnectionPool, + connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + ) -> None: + super(ScrapyProxyH2Agent, self).__init__( + reactor=reactor, + pool=pool, + connect_timeout=connect_timeout, + bind_address=bind_address + ) + self._proxy_uri = proxy_uri + + @staticmethod + def get_key(uri: URI) -> Tuple: + return "http-proxy", uri.host, uri.port + + +class ScrapyH2Agent: + _Agent = H2Agent + _ProxyAgent = ScrapyProxyH2Agent + + def __init__( + self, context_factory, + connect_timeout=10, + bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None, + maxsize: int = 0, warnsize: int = 0, + crawler=None + ) -> None: + self._context_factory = context_factory + self._connect_timeout = connect_timeout + self._bind_address = bind_address + self._pool = pool + self._maxsize = maxsize + self._warnsize = warnsize + self._crawler = crawler + + def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent: from twisted.internet import reactor + bind_address = request.meta.get('bindaddress') or self._bind_address + proxy = request.meta.get('proxy') + if proxy: + _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) + scheme = _parse(request.url)[0] + proxy_host = str(proxy_host, 'utf-8') + omit_connect_timeout = b'noconnect' in proxy_params + if omit_connect_timeout: + warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " + "If you use Crawlera, it doesn't require this mode anymore, " + "so you should update scrapy-crawlera to 1.3.0+ " + "and remove '?noconnect' from the Crawlera URL.", + ScrapyDeprecationWarning) - agent = H2Agent(reactor, self._pool, self._context_factory) - d = agent.request(request) + if scheme == b'https' and not omit_connect_timeout: + proxy_auth = request.headers.get(b'Proxy-Authorization', None) + proxy_conf = (proxy_host, proxy_port, proxy_auth) - def print_result(result): - print(result) - return result + # TODO: Return TunnelingAgent instance + else: + return self._ProxyAgent( + reactor=reactor, + proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')), + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool + ) - d.addCallback(print_result) + return self._Agent( + reactor=reactor, + context_factory=self._context_factory, + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool + ) + + def download_request(self, request: Request, spider: Spider): + from twisted.internet import reactor + timeout = request.meta.get('download_timeout') or self._connect_timeout + agent = self._get_agent(request, timeout) + + start_time = time() + d = agent.request(request, spider) + d.addCallback(self._cb_latency, request, start_time) + + timeout_cl = reactor.callLater(timeout, d.cancel) + d.addBoth(self._cb_timeout, request, timeout, timeout_cl) return d + + @staticmethod + def _cb_latency(response: Response, request: Request, start_time: float): + request.meta['download_latency'] = time() - start_time + return response + + @staticmethod + def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl): + if timeout_cl.active(): + timeout_cl.cancel() + return response + + url = urldefrag(request.url)[0] + raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 7a8847c38..c7a49fd42 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -6,7 +6,9 @@ from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOption from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint +from twisted.python.failure import Failure from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory +from twisted.web.error import SchemeNotSupported from twisted.web.iweb import IPolicyForHTTPS from zope.interface import implementer from zope.interface.verify import verifyObject @@ -14,6 +16,7 @@ from zope.interface.verify import verifyObject from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory from scrapy.http.request import Request from scrapy.settings import Settings +from scrapy.spiders import Spider class H2ConnectionPool: @@ -28,8 +31,7 @@ class H2ConnectionPool: # Save all requests that arrive before the connection is established self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} - def get_connection(self, uri: URI, endpoint: HostnameEndpoint) -> Deferred: - key = (uri.scheme, uri.host, uri.port) + def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred: if key in self._pending_requests: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol @@ -84,6 +86,15 @@ class H2ConnectionPool: d = pending_requests.popleft() d.errback(errors) + def close_connections(self) -> None: + """Close all the HTTP/2 connections and remove them from pool + + Returns: + Deferred that fires when all connections have been closed + """ + for conn in self._connections.values(): + conn.transport.loseConnection() + @implementer(IPolicyForHTTPS) class H2WrappedContextFactory: @@ -111,9 +122,21 @@ class H2Agent: connect_timeout, bind_address ) - def request(self, request: Request) -> Deferred: + def _get_endpoint(self, uri: URI): + return self._endpoint_factory.endpointForURI(uri) + + @staticmethod + def get_key(uri: URI) -> Tuple: + return uri.scheme, uri.host, uri.port + + def request(self, request: Request, spider: Spider) -> Deferred: uri = URI.fromBytes(bytes(request.url, encoding='utf-8')) - endpoint = self._endpoint_factory.endpointForURI(uri) - d = self._pool.get_connection(uri, endpoint) - d.addCallback(lambda conn: conn.request(request)) + try: + endpoint = self._get_endpoint(uri) + except SchemeNotSupported: + return defer.fail(Failure()) + + key = self.get_key(uri) + d = self._pool.get_connection(key, uri, endpoint) + d.addCallback(lambda conn: conn.request(request, spider)) return d From 92bec38591fccb523c2e643aef70a1f6cd7267ea Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 29 Jul 2020 13:43:59 +0530 Subject: [PATCH 0129/2083] feat: MethodNotAllowed405, Content-Length header - add tests to check for Content-Length header - raise MethodNotAllowed405 when remote send 'HTTP/2.0 405 Method Not Allowed' --- scrapy/core/downloader/handlers/http2.py | 15 +++----- scrapy/core/http2/agent.py | 2 +- scrapy/core/http2/protocol.py | 32 +++++++++++++---- scrapy/core/http2/stream.py | 14 ++++++-- tests/test_http2_client_protocol.py | 46 ++++++++++++++++++++++-- 5 files changed, 85 insertions(+), 24 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 81ea78e69..e9cc5ebbc 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -3,6 +3,7 @@ from time import time from typing import Optional, Tuple from urllib.parse import urldefrag +from twisted.internet.defer import Deferred from twisted.internet.base import ReactorBase from twisted.internet.error import TimeoutError from twisted.web.client import URI @@ -23,9 +24,6 @@ class H2DownloadHandler: from twisted.internet import reactor self._pool = H2ConnectionPool(reactor, settings) self._context_factory = load_context_factory_from_settings(settings, crawler) - self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') - self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') - self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') @classmethod def from_crawler(cls, crawler): @@ -35,8 +33,6 @@ class H2DownloadHandler: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, - maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), - warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), crawler=self._crawler ) return agent.download_request(request, spider) @@ -72,15 +68,12 @@ class ScrapyH2Agent: self, context_factory, connect_timeout=10, bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None, - maxsize: int = 0, warnsize: int = 0, crawler=None ) -> None: self._context_factory = context_factory self._connect_timeout = connect_timeout self._bind_address = bind_address self._pool = pool - self._maxsize = maxsize - self._warnsize = warnsize self._crawler = crawler def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent: @@ -121,7 +114,7 @@ class ScrapyH2Agent: pool=self._pool ) - def download_request(self, request: Request, spider: Spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: from twisted.internet import reactor timeout = request.meta.get('download_timeout') or self._connect_timeout agent = self._get_agent(request, timeout) @@ -135,12 +128,12 @@ class ScrapyH2Agent: return d @staticmethod - def _cb_latency(response: Response, request: Request, start_time: float): + def _cb_latency(response: Response, request: Request, start_time: float) -> Response: request.meta['download_latency'] = time() - start_time return response @staticmethod - def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl): + def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl) -> Response: if timeout_cl.active(): timeout_cl.cancel() return response diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index c7a49fd42..e62eef263 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -93,7 +93,7 @@ class H2ConnectionPool: Deferred that fires when all connections have been closed """ for conn in self._connections.values(): - conn.transport.loseConnection() + conn.transport.abortConnection() @implementer(IPolicyForHTTPS) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index feb034a0c..1ce8b6548 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -13,7 +13,7 @@ from h2.events import ( SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived, WindowUpdated ) -from h2.exceptions import H2Error, ProtocolError +from h2.exceptions import H2Error from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory @@ -39,7 +39,7 @@ class InvalidNegotiatedProtocol(H2Error): self.negotiated_protocol = negotiated_protocol def __str__(self) -> str: - return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol}' + return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}' class RemoteTerminatedConnection(H2Error): @@ -48,7 +48,15 @@ class RemoteTerminatedConnection(H2Error): self.terminate_event = event def __str__(self) -> str: - return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address}' + return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address!r}' + + +class MethodNotAllowed405(H2Error): + def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]): + self.remote_ip_address = remote_ip_address + + def __str__(self) -> str: + return f"MethodNotAllowed405: Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}" @implementer(IHandshakeListener) @@ -217,14 +225,25 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # So, no need to send a GOAWAY frame to the remote self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) + def _check_received_data(self, data: bytes) -> None: + """Checks for edge cases where the connection to remote fails + without raising an appropriate H2Error + + Arguments: + data -- Data received from the remote + """ + if data.startswith(b'HTTP/2.0 405 Method Not Allowed'): + raise MethodNotAllowed405(self.metadata['ip_address']) + def dataReceived(self, data: bytes) -> None: # Reset the idle timeout as connection is still actively receiving data self.resetTimeout() try: + self._check_received_data(data) events = self.conn.receive_data(data) self._handle_events(events) - except ProtocolError as e: + except H2Error as e: # Save this error as ultimately the connection will be dropped # internally by hyper-h2. Saved error will be passed to all the streams # closed with the connection. @@ -271,9 +290,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): for stream in self.streams.values(): if stream.request_sent: - stream.close(StreamCloseReason.CONNECTION_LOST, self._conn_lost_errors, from_protocol=True) + close_reason = StreamCloseReason.CONNECTION_LOST else: - stream.close(StreamCloseReason.INACTIVE, from_protocol=True) + close_reason = StreamCloseReason.INACTIVE + stream.close(close_reason, self._conn_lost_errors, from_protocol=True) self._active_streams -= len(self.streams) self.streams.clear() diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 133196797..5bffa67e7 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -189,12 +189,15 @@ class Stream: headers = [ (':method', self._request.method), (':authority', url.netloc), - (':scheme', 'https'), + (':scheme', self._protocol.metadata['uri'].scheme), (':path', path), ] for name, value in self._request.headers.items(): - headers.append((name, value[0])) + headers.append((str(name, 'utf-8'), str(value[0], 'utf-8'))) + + if b'Content-Length' not in self._request.headers.keys(): + headers.append(('Content-Length', str(len(self._request.body)))) return headers @@ -337,6 +340,10 @@ class Stream: if not isinstance(reason, StreamCloseReason): raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}') + # Have default value of errors as an empty list as + # some cases can add a list of exceptions + errors = errors or [] + if not from_protocol: self._protocol.pop_stream(self.stream_id) @@ -387,7 +394,8 @@ class Stream: self._deferred_response.errback(ResponseFailed(errors)) elif reason is StreamCloseReason.INACTIVE: - self._deferred_response.errback(InactiveStreamClosed(self._request)) + errors.insert(0, InactiveStreamClosed(self._request)) + self._deferred_response.errback(ResponseFailed(errors)) elif reason is StreamCloseReason.INVALID_HOSTNAME: self._deferred_response.errback(InvalidHostname( diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 746eef4d6..4926ada14 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -11,14 +11,14 @@ from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint +from twisted.internet.error import TimeoutError from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase -from twisted.web.client import URI +from twisted.web.client import ResponseFailed, URI from twisted.web.http import Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File -from twisted.internet.error import TimeoutError from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname @@ -152,6 +152,19 @@ class QueryParams(LeafResource): return bytes(json.dumps(query_params), 'utf-8') +class RequestHeaders(LeafResource): + """Sends all the headers received as a response""" + + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') + headers = {} + for k, v in request.requestHeaders.getAllRawHeaders(): + headers[str(k, 'utf-8')] = str(v[0], 'utf-8') + + return bytes(json.dumps(headers), 'utf-8') + + def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) @@ -179,6 +192,7 @@ class Https2ClientProtocolTestCase(TestCase): r.putChild(b'status', Status()) r.putChild(b'query-params', QueryParams()) r.putChild(b'timeout', TimeoutResponse()) + r.putChild(b'request-headers', RequestHeaders()) return r @inlineCallbacks @@ -488,7 +502,11 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [] def assert_inactive_stream(failure): - self.assertIsNotNone(failure.check(InactiveStreamClosed)) + self.assertIsNotNone(failure.check(ResponseFailed)) + self.assertTrue(any( + isinstance(e, InactiveStreamClosed) + for e in failure.value.reasons + )) # Send 100 request (we do not check the result) for _ in range(100): @@ -616,3 +634,25 @@ class Https2ClientProtocolTestCase(TestCase): d.addCallback(self.fail) d.addErrback(assert_timeout_error) return d + + def test_request_headers_received(self): + request = Request(self.get_url('/request-headers'), headers={ + 'header-1': 'header value 1', + 'header-2': 'header value 2' + }) + d = self.make_request(request) + + def assert_request_headers(response: Response): + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + + response_headers = json.loads(str(response.body, 'utf-8')) + self.assertIsInstance(response_headers, dict) + for k, v in request.headers.items(): + k, v = str(k, 'utf-8'), str(v[0], 'utf-8') + self.assertIn(k, response_headers) + self.assertEqual(v, response_headers[k]) + + d.addErrback(self.fail) + d.addCallback(assert_request_headers) + return d From e8342996f6273b6b60ffc26a67e90dec34e96dfd Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 29 Jul 2020 13:51:01 +0530 Subject: [PATCH 0130/2083] test: H2DownloadHandler Following tests are skipped as Content-Length header not matching the data received is considered as a ProtocolError - test_download_broken_content_cause_data_loss - test_download_broken_chunked_content_cause_data_loss - test_download_broken_content_allow_data_loss - test_download_broken_chunked_content_allow_data_loss - test_download_broken_content_allow_data_loss_via_setting - test_download_broken_chunked_content_allow_data_loss_via_setting BREAKING CHANGES The following tests currently fail - test_content_length_zero_bodyless_post_request_headers - test_host_header_seted_in_request_headers - test_download_with_maxsize_very_large_file --- tests/test_downloader_handlers.py | 108 +++++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 9 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 51deb20f4..f6add82dc 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -23,8 +23,8 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler +from scrapy.core.downloader.handlers.http2 import H2DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler - from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Headers, Request from scrapy.http.response.text import TextResponse @@ -33,7 +33,6 @@ from scrapy.spiders import Spider from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto - from tests.mockserver import MockServer, ssl_context_factory, Echo from tests.spiders import SingleRequestSpider @@ -131,6 +130,7 @@ class ContentLengthHeaderResource(resource.Resource): A testing resource which renders itself as the value of the Content-Length header from the request. """ + def render(self, request): return request.requestHeaders.getRawHeaders(b"content-length")[0] @@ -142,6 +142,7 @@ class ChunkedResource(resource.Resource): request.write(b"chunked ") request.write(b"content\n") request.finish() + reactor.callLater(0, response) return server.NOT_DONE_YET @@ -155,6 +156,7 @@ class BrokenChunkedResource(resource.Resource): # Disable terminating chunk on finish. request.chunked = False closeConnection(request) + reactor.callLater(0, response) return server.NOT_DONE_YET @@ -186,6 +188,7 @@ class EmptyContentTypeHeaderResource(resource.Resource): A testing resource which renders itself as the value of request body without content-type header in response. """ + def render(self, request): request.setHeader("content-type", "") return request.content.read() @@ -197,12 +200,12 @@ class LargeChunkedFileResource(resource.Resource): for i in range(1024): request.write(b"x" * 1024) request.finish() + reactor.callLater(0, response) return server.NOT_DONE_YET class HttpTestCase(unittest.TestCase): - scheme = 'http' download_handler_cls = HTTPDownloadHandler @@ -230,10 +233,12 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) - self.host = 'localhost' + self.host = u'localhost' if self.scheme == 'https': + # Using WrappingFactory do not enable HTTP/2 failing all the + # tests with H2DownloadHandler self.port = reactor.listenSSL( - 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile), + 0, self.site, ssl_context_factory(self.keyfile, self.certfile), interface=self.host) else: self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) @@ -330,6 +335,7 @@ class HttpTestCase(unittest.TestCase): https://github.com/kennethreitz/requests/issues/405 https://bugs.python.org/issue14721 """ + def _test(response): self.assertEqual(response.body, b'0') @@ -514,6 +520,30 @@ class Https11TestCase(Http11TestCase): yield download_handler.close() +class Https2TestCase(Https11TestCase): + scheme = 'https' + download_handler_cls = H2DownloadHandler + HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + + def test_download_broken_content_cause_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_cause_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + class Https11WrongHostnameTestCase(Http11TestCase): scheme = 'https' @@ -526,6 +556,23 @@ class Https11WrongHostnameTestCase(Http11TestCase): certfile = 'keys/example-com.cert.pem' +class Https2WrongHostnameTestCase(Https2TestCase): + tls_log_message = ( + 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' + 'subject "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' + ) + + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = 'keys/example-com.key.pem' + certfile = 'keys/example-com.cert.pem' + + class Https11InvalidDNSId(Https11TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" @@ -534,6 +581,14 @@ class Https11InvalidDNSId(Https11TestCase): self.host = '127.0.0.1' +class Https2InvalidDNSId(Https2TestCase): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super(Https2InvalidDNSId, self).setUp() + self.host = '127.0.0.1' + + class Https11InvalidDNSPattern(Https11TestCase): """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" @@ -552,6 +607,24 @@ class Https11InvalidDNSPattern(Https11TestCase): super(Https11InvalidDNSPattern, self).setUp() +class Https2InvalidDNSPattern(Https2TestCase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = 'keys/localhost.ip.key' + certfile = 'keys/localhost.ip.crt' + + def setUp(self): + try: + from service_identity.exceptions import CertificateError # noqa: F401 + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") + self.tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' + 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + ) + super(Https2InvalidDNSPattern, self).setUp() + + class Https11CustomCiphers(unittest.TestCase): scheme = 'https' download_handler_cls = HTTP11DownloadHandler @@ -565,10 +638,9 @@ class Https11CustomCiphers(unittest.TestCase): FilePath(self.tmpname).child("file").setContent(b"0123456789") r = static.File(self.tmpname) self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) self.host = 'localhost' self.port = reactor.listenSSL( - 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'), + 0, self.site, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'), interface=self.host) self.portno = self.port.getHost().port crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}) @@ -593,6 +665,11 @@ class Https11CustomCiphers(unittest.TestCase): return d +class Https2CustomCiphers(Https11CustomCiphers): + scheme = 'https' + download_handler_cls = H2DownloadHandler + + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" @@ -644,6 +721,10 @@ class Http11MockServerTestCase(unittest.TestCase): self.assertTrue(reason, 'finished') +class Http2MockServerTestCase(Http11MockServerTestCase): + """HTTP 2.0 test case with MockServer""" + + class UriResource(resource.Resource): """Return the full uri that was requested""" @@ -734,6 +815,11 @@ class Http11ProxyTestCase(HttpProxyTestCase): self.assertIn(domain, timeout.osError) +# TODO: +class Http2ProxyTestCase(Http11ProxyTestCase): + download_handler_cls = H2DownloadHandler + + class HttpDownloadHandlerMock: def __init__(self, *args, **kwargs): @@ -931,7 +1017,6 @@ class S3TestCase(unittest.TestCase): class BaseFTPTestCase(unittest.TestCase): - username = "scrapy" password = "passwd" req_meta = {"ftp_user": username, "ftp_password": password} @@ -969,6 +1054,7 @@ class BaseFTPTestCase(unittest.TestCase): def _clean(data): self.download_handler.client.transport.loseConnection() return data + deferred.addCallback(_clean) if callback: deferred.addCallback(callback) @@ -985,6 +1071,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'I have the power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) + return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): @@ -998,6 +1085,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'Moooooooooo power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']}) + return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): @@ -1007,6 +1095,7 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.status, 404) + return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): @@ -1027,6 +1116,7 @@ class BaseFTPTestCase(unittest.TestCase): with open(local_fname, "rb") as f: self.assertEqual(f.read(), b"I have the power!") os.remove(local_fname) + return self._add_test_callbacks(d, _test) @@ -1043,11 +1133,11 @@ class FTPTestCase(BaseFTPTestCase): def _test(r): self.assertEqual(r.type, ConnectionLost) + return self._add_test_callbacks(d, errback=_test) class AnonymousFTPTestCase(BaseFTPTestCase): - username = "anonymous" req_meta = {} From 19f2b4b53dd51044083a9749f00366f41ed795c7 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 29 Jul 2020 17:25:59 +0530 Subject: [PATCH 0131/2083] refactor: AcceptableProtocolsContextFactory - rename H2WrappedContextFactory to AcceptableProtocolsContextFactory - AcceptableProtocolsContextFactory accepts an argument acceptable_protocols which can be used to override the context factory priority list of protocols during ALPN or NPN --- scrapy/core/downloader/handlers/http2.py | 2 +- scrapy/core/http2/agent.py | 9 +++++---- scrapy/core/http2/stream.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index e9cc5ebbc..411e06a78 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -3,8 +3,8 @@ from time import time from typing import Optional, Tuple from urllib.parse import urldefrag -from twisted.internet.defer import Deferred from twisted.internet.base import ReactorBase +from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index e62eef263..aa51508a5 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -97,14 +97,15 @@ class H2ConnectionPool: @implementer(IPolicyForHTTPS) -class H2WrappedContextFactory: - def __init__(self, context_factory) -> None: +class AcceptableProtocolsContextFactory: + def __init__(self, context_factory, acceptable_protocols: List[bytes]) -> None: verifyObject(IPolicyForHTTPS, context_factory) self._wrapped_context_factory = context_factory + self._acceptable_protocols = acceptable_protocols def creatorForNetloc(self, hostname, port) -> ClientTLSOptions: options = self._wrapped_context_factory.creatorForNetloc(hostname, port) - _setAcceptableProtocols(options._ctx, [b'h2']) + _setAcceptableProtocols(options._ctx, self._acceptable_protocols) return options @@ -116,7 +117,7 @@ class H2Agent: ) -> None: self._reactor = reactor self._pool = pool - self._context_factory = H2WrappedContextFactory(context_factory) + self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2']) self._endpoint_factory = _StandardEndpointFactory( self._reactor, self._context_factory, connect_timeout, bind_address diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 5bffa67e7..acdd46320 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -85,7 +85,7 @@ class Stream: request: Request, protocol: "H2ClientProtocol", download_maxsize: int = 0, - download_warnsize: int = 0 + download_warnsize: int = 0, ) -> None: """ Arguments: From a3fecaf07f9edd6a1bbac1ade825c23845c7e6b1 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 30 Jul 2020 15:45:27 +0530 Subject: [PATCH 0132/2083] test: fix host-name H2DownloadHandler tests --- tests/test_downloader_handlers.py | 33 +++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index f6add82dc..4dd32b6f5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -312,16 +312,18 @@ class HttpTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): - def _test(response): - self.assertEqual(response.body, b'example.com') - self.assertEqual(request.headers.get('Host'), b'example.com') + host = self.host + ':' + str(self.portno) - request = Request(self.getURL('host'), headers={'Host': 'example.com'}) + def _test(response): + self.assertEqual(response.body, bytes(host, 'utf-8')) + self.assertEqual(request.headers.get('Host'), bytes(host, 'utf-8')) + + request = Request(self.getURL('host'), headers={'Host': host}) return self.download_request(request, Spider('foo')).addCallback(_test) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b'example.com') + d.addCallback(self.assertEqual, b'localhost') return d def test_content_length_zero_bodyless_post_request_headers(self): @@ -339,7 +341,7 @@ class HttpTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.body, b'0') - request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'}) + request = Request(self.getURL('contentlength'), method='POST') return self.download_request(request, Spider('foo')).addCallback(_test) def test_content_length_zero_bodyless_post_only_one(self): @@ -525,6 +527,25 @@ class Https2TestCase(Https11TestCase): download_handler_cls = H2DownloadHandler HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + @defer.inlineCallbacks + def test_download_with_maxsize_very_large_file(self): + with mock.patch('scrapy.core.http2.stream.logger') as logger: + request = Request(self.getURL('largechunkedfile')) + + def check(logger): + logger.error.assert_called_once_with(mock.ANY) + + d = self.download_request(request, Spider('foo', download_maxsize=1500)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + # As the error message is logged in the dataReceived callback, we + # have to give a bit of time to the reactor to process the queue + # after closing the connection. + d = defer.Deferred() + d.addCallback(check) + reactor.callLater(.1, d.callback, logger) + yield d + def test_download_broken_content_cause_data_loss(self, url='broken'): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) From d707f8b5d94856ba13bd8e76acb1efb8de377f9c Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 30 Jul 2020 18:06:21 +0530 Subject: [PATCH 0133/2083] docs: mention H2DownloadHandler in settings.rst --- docs/topics/settings.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5178f272f..670b44f3c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -620,6 +620,13 @@ handler (without replacement), place this in your ``settings.py``:: 'ftp': None, } +The default https handler uses HTTP/1.x, to use HTTP/2.0 update :setting:`DOWNLOAD_HANDLERS` +as:: + + DOWNLOAD_HANDLERS = { + 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', + } + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT @@ -697,6 +704,14 @@ Optionally, this can be set per-request basis by using the If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``, the ``ResponseFailed([_DataLoss])`` failure will be retried as usual. +.. warning:: + + This is ignored when :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` + is set as ``https`` download handler in :setting:`DOWNLOAD_HANDLERS`. In + case of data loss error the connection may be corrupted affecting other streams, + hence all streams return with the ``ResponseFailed([InvalidBodyLengthError])`` + failure. + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS From 1cc8d5829fc1b1b10fd852db693ac44dc9be0ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 6 Aug 2020 13:52:47 +0200 Subject: [PATCH 0134/2083] Remove unneeded try-except Exceptions only happen when find_spec gets a 2nd parameter. --- scrapy/commands/startproject.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 35b58090c..82ccda35e 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -42,10 +42,7 @@ class Command(ScrapyCommand): def _is_valid_name(self, project_name): def _module_exists(module_name): - try: - spec = find_spec(module_name) - except ModuleNotFoundError: - return False + spec = find_spec(module_name) return spec is not None and spec.loader is not None if not re.search(r'^[_a-zA-Z]\w*$', project_name): From 13181ba7882f2aef3ea103a9c2391fd8f87fbb44 Mon Sep 17 00:00:00 2001 From: Jose Galdos Date: Thu, 23 Jul 2020 18:45:45 -0500 Subject: [PATCH 0135/2083] Improve http status all on http error middleware --- scrapy/spidermiddlewares/httperror.py | 2 +- tests/test_spidermiddleware_httperror.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 375042340..bf908d2f7 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -32,7 +32,7 @@ class HttpErrorMiddleware: if 200 <= response.status < 300: # common case return meta = response.meta - if 'handle_httpstatus_all' in meta: + if meta.get('handle_httpstatus_all', False): return if 'handle_httpstatus_list' in meta: allowed_statuses = meta['handle_httpstatus_list'] diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e032b247c..f3e5478c4 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -139,6 +139,19 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + def test_httperror_allow_all_false(self): + crawler = get_crawler(_HttpErrorSpider) + mw = HttpErrorMiddleware.from_crawler(crawler) + request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False}) + request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True}) + res404 = self.res404.copy() + res404.request = request_httpstatus_false + res402 = self.res402.copy() + res402.request = request_httpstatus_true + + self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider) + self.assertIsNone(mw.process_spider_input(res402, self.spider)) + class TestHttpErrorMiddlewareIntegrational(TrialTestCase): def setUp(self): From e0c3019d90f187482cd84b946b83c496411ec34b Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 9 Aug 2020 16:19:35 +0530 Subject: [PATCH 0136/2083] fix: ScrapyProxyH2Agent - add required test cases BREAKING CHANGES Presently the tests (in test_downloader_handlers.py) 1. test_download_without_proxy 2. test_download_with_proxy_https_timeout collide with each other when run together. However, if both of the tests are ran individually then both pass. --- scrapy/core/downloader/handlers/http2.py | 14 ++++++--- scrapy/core/http2/agent.py | 15 ++++++---- scrapy/core/http2/stream.py | 22 +++++++++++++-- tests/test_downloader_handlers.py | 36 ++++++++++++++++++++---- 4 files changed, 70 insertions(+), 17 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 411e06a78..d6e5cd1c1 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -6,7 +6,7 @@ from urllib.parse import urldefrag from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.web.client import URI +from twisted.web.client import URI, BrowserLikePolicyForHTTPS from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse @@ -45,19 +45,24 @@ class ScrapyProxyH2Agent(H2Agent): def __init__( self, reactor: ReactorBase, proxy_uri: URI, pool: H2ConnectionPool, + context_factory=BrowserLikePolicyForHTTPS(), connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None ) -> None: super(ScrapyProxyH2Agent, self).__init__( reactor=reactor, pool=pool, + context_factory=context_factory, connect_timeout=connect_timeout, bind_address=bind_address ) self._proxy_uri = proxy_uri - @staticmethod - def get_key(uri: URI) -> Tuple: - return "http-proxy", uri.host, uri.port + def get_endpoint(self, uri: URI): + return self.endpoint_factory.endpointForURI(self._proxy_uri) + + def get_key(self, uri: URI) -> Tuple: + """We use the proxy uri instead of uri obtained from request url""" + return "http-proxy", self._proxy_uri.host, self._proxy_uri.port class ScrapyH2Agent: @@ -100,6 +105,7 @@ class ScrapyH2Agent: else: return self._ProxyAgent( reactor=reactor, + context_factory=self._context_factory, proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')), connect_timeout=timeout, bind_address=bind_address, diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index aa51508a5..f4ac29bc6 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -118,22 +118,25 @@ class H2Agent: self._reactor = reactor self._pool = pool self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2']) - self._endpoint_factory = _StandardEndpointFactory( + self.endpoint_factory = _StandardEndpointFactory( self._reactor, self._context_factory, connect_timeout, bind_address ) - def _get_endpoint(self, uri: URI): - return self._endpoint_factory.endpointForURI(uri) + def get_endpoint(self, uri: URI): + return self.endpoint_factory.endpointForURI(uri) - @staticmethod - def get_key(uri: URI) -> Tuple: + def get_key(self, uri: URI) -> Tuple: + """ + Arguments: + uri - URI obtained directly from request URL + """ return uri.scheme, uri.host, uri.port def request(self, request: Request, spider: Spider) -> Deferred: uri = URI.fromBytes(bytes(request.url, encoding='utf-8')) try: - endpoint = self._get_endpoint(uri) + endpoint = self.get_endpoint(uri) except SchemeNotSupported: return defer.fail(Failure()) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index acdd46320..40ea07b63 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -185,14 +185,32 @@ class Stream: if url.query: path += '?' + url.query + # This pseudo-header field MUST NOT be empty for "http" or "https" + # URIs; "http" or "https" URIs that do not contain a path component + # MUST include a value of '/'. The exception to this rule is an + # OPTIONS request for an "http" or "https" URI that does not include + # a path component; these MUST include a ":path" pseudo-header field + # with a value of '*' (refer RFC 7540 - Section 8.1.2.3) + if not path: + if self._request.method == 'OPTIONS': + path = path or '*' + else: + path = path or '/' + # Make sure pseudo-headers comes before all the other headers headers = [ (':method', self._request.method), (':authority', url.netloc), - (':scheme', self._protocol.metadata['uri'].scheme), - (':path', path), ] + # The ":scheme" and ":path" pseudo-header fields MUST + # be omitted for CONNECT method (refer RFC 7540 - Section 8.3) + if self._request.method != 'CONNECT': + headers += [ + (':scheme', self._protocol.metadata['uri'].scheme), + (':path', path), + ] + for name, value in self._request.headers.items(): headers.append((str(name, 'utf-8'), str(value[0], 'utf-8'))) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 4dd32b6f5..486614121 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -786,7 +786,10 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'http://example.com') + self.assertTrue( + response.body == b'http://example.com' # HTTP/1.x + or response.body == b'/' # HTTP/2 + ) http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -796,10 +799,13 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'https://example.com') + self.assertTrue( + response.body == b'http://example.com' # HTTP/1.x + or response.body == b'/' # HTTP/2 + ) http_proxy = '%s?noconnect' % self.getURL('') - request = Request('https://example.com', meta={'proxy': http_proxy}) + request = Request('http://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex(ScrapyDeprecationWarning, r'Using HTTPS proxies in the noconnect mode is deprecated'): return self.download_request(request, Spider('foo')).addCallback(_test) @@ -836,10 +842,30 @@ class Http11ProxyTestCase(HttpProxyTestCase): self.assertIn(domain, timeout.osError) -# TODO: -class Http2ProxyTestCase(Http11ProxyTestCase): +class Https2ProxyTestCase(Http11ProxyTestCase): + # only used for HTTPS tests + keyfile = 'keys/localhost.key' + certfile = 'keys/localhost.crt' + + scheme = 'https' + host = u'127.0.0.1' + download_handler_cls = H2DownloadHandler + def setUp(self): + site = server.Site(UriResource(), timeout=None) + self.port = reactor.listenSSL( + 0, site, + ssl_context_factory(self.keyfile, self.certfile), + interface=self.host + ) + self.portno = self.port.getHost().port + self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) + self.download_request = self.download_handler.download_request + + def getURL(self, path): + return f"{self.scheme}://{self.host}:{self.portno}/{path}" + class HttpDownloadHandlerMock: From c67d6dea318d6f0915ae86d46ce367b6c1e8ee51 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 11 Aug 2020 04:39:41 +0530 Subject: [PATCH 0137/2083] fix: H2 docs, NotImplementedError for H2 Tunnel --- docs/topics/settings.rst | 15 ++++---- scrapy/core/downloader/contextfactory.py | 7 ++-- scrapy/core/downloader/handlers/http2.py | 44 +++++++++++------------- scrapy/core/http2/stream.py | 5 +-- tests/test_downloader_handlers.py | 33 ++++++++++++------ 5 files changed, 54 insertions(+), 50 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 670b44f3c..bb543433c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -620,8 +620,8 @@ handler (without replacement), place this in your ``settings.py``:: 'ftp': None, } -The default https handler uses HTTP/1.x, to use HTTP/2.0 update :setting:`DOWNLOAD_HANDLERS` -as:: +The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update +:setting:`DOWNLOAD_HANDLERS` as follows:: DOWNLOAD_HANDLERS = { 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', @@ -706,11 +706,12 @@ Optionally, this can be set per-request basis by using the .. warning:: - This is ignored when :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` - is set as ``https`` download handler in :setting:`DOWNLOAD_HANDLERS`. In - case of data loss error the connection may be corrupted affecting other streams, - hence all streams return with the ``ResponseFailed([InvalidBodyLengthError])`` - failure. + This setting is ignored by the + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` + download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss + error, the corresponding HTTP/2 connection may be corrupted, affecting other + requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])`` + failure is always raised for every request that was using that connection. .. setting:: DUPEFILTER_CLASS diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index c0463cfc7..8dcba15ff 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,14 +1,13 @@ -from OpenSSL import SSL import warnings +from OpenSSL import SSL from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer -from scrapy.core.downloader.tls import openssl_methods -from scrapy.utils.misc import create_instance, load_object -from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS +from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions +from scrapy.utils.misc import create_instance, load_object @implementer(IPolicyForHTTPS) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index d6e5cd1c1..650af9778 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -6,15 +6,15 @@ from urllib.parse import urldefrag from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.web.client import URI, BrowserLikePolicyForHTTPS +from twisted.web.client import BrowserLikePolicyForHTTPS, URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.python import to_bytes class H2DownloadHandler: @@ -88,29 +88,25 @@ class ScrapyH2Agent: if proxy: _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) scheme = _parse(request.url)[0] - proxy_host = str(proxy_host, 'utf-8') - omit_connect_timeout = b'noconnect' in proxy_params - if omit_connect_timeout: - warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Crawlera, it doesn't require this mode anymore, " - "so you should update scrapy-crawlera to 1.3.0+ " - "and remove '?noconnect' from the Crawlera URL.", - ScrapyDeprecationWarning) + proxy_host = proxy_host.decode() + omit_connect_tunnel = b'noconnect' in proxy_params + if omit_connect_tunnel: + warnings.warn("Using HTTPS proxies in the noconnect mode is not supported by the " + "downloader handler. If you use Crawlera, it doesn't require this " + "mode anymore, so you should update scrapy-crawlera to 1.3.0+ " + "and remove '?noconnect' from the Crawlera URL.") - if scheme == b'https' and not omit_connect_timeout: - proxy_auth = request.headers.get(b'Proxy-Authorization', None) - proxy_conf = (proxy_host, proxy_port, proxy_auth) - - # TODO: Return TunnelingAgent instance - else: - return self._ProxyAgent( - reactor=reactor, - context_factory=self._context_factory, - proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')), - connect_timeout=timeout, - bind_address=bind_address, - pool=self._pool - ) + if scheme == b'https' and not omit_connect_tunnel: + # ToDo + raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported') + return self._ProxyAgent( + reactor=reactor, + context_factory=self._context_factory, + proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')), + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool + ) return self._Agent( reactor=reactor, diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 40ea07b63..1e136fbd5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -192,10 +192,7 @@ class Stream: # a path component; these MUST include a ":path" pseudo-header field # with a value of '*' (refer RFC 7540 - Section 8.1.2.3) if not path: - if self._request.method == 'OPTIONS': - path = path or '*' - else: - path = path or '/' + path = '*' if self._request.method == 'OPTIONS' else '/' # Make sure pseudo-headers comes before all the other headers headers = [ diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 486614121..1cedf6b10 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -315,8 +315,8 @@ class HttpTestCase(unittest.TestCase): host = self.host + ':' + str(self.portno) def _test(response): - self.assertEqual(response.body, bytes(host, 'utf-8')) - self.assertEqual(request.headers.get('Host'), bytes(host, 'utf-8')) + self.assertEqual(response.body, to_bytes(host)) + self.assertEqual(request.headers.get('Host'), to_bytes(host)) request = Request(self.getURL('host'), headers={'Host': host}) return self.download_request(request, Spider('foo')).addCallback(_test) @@ -764,6 +764,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): download_handler_cls = HTTPDownloadHandler + expected_http_proxy_request_body = b'http://example.com' def setUp(self): site = server.Site(UriResource(), timeout=None) @@ -786,10 +787,7 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertTrue( - response.body == b'http://example.com' # HTTP/1.x - or response.body == b'/' # HTTP/2 - ) + self.assertEqual(response.body, self.expected_http_proxy_request_body) http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -799,13 +797,10 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertTrue( - response.body == b'http://example.com' # HTTP/1.x - or response.body == b'/' # HTTP/2 - ) + self.assertEqual(response.body, b'https://example.com') http_proxy = '%s?noconnect' % self.getURL('') - request = Request('http://example.com', meta={'proxy': http_proxy}) + request = Request('https://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex(ScrapyDeprecationWarning, r'Using HTTPS proxies in the noconnect mode is deprecated'): return self.download_request(request, Spider('foo')).addCallback(_test) @@ -851,6 +846,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): host = u'127.0.0.1' download_handler_cls = H2DownloadHandler + expected_http_proxy_request_body = b'/' def setUp(self): site = server.Site(UriResource(), timeout=None) @@ -866,6 +862,21 @@ class Https2ProxyTestCase(Http11ProxyTestCase): def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" + def test_download_with_proxy_https_noconnect(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'/') + + http_proxy = '%s?noconnect' % self.getURL('') + request = Request('https://example.com', meta={'proxy': http_proxy}) + with self.assertWarnsRegex( + Warning, + r'Using HTTPS proxies in the noconnect mode is not supported by the ' + r'downloader handler.' + ): + return self.download_request(request, Spider('foo')).addCallback(_test) + class HttpDownloadHandlerMock: From 90f85a2b9b0cfac4a7a56a1926e9694ef7e2d299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Aug 2020 10:20:30 +0200 Subject: [PATCH 0138/2083] Enable Travis CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b403ac54c..a3bd2e199 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ dist: xenial branches: only: - master + - http2 # Remove once merged into master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: From af73f141b23aabbef208a12fe95ae63c27105fda Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 16 Aug 2020 11:26:10 +0530 Subject: [PATCH 0139/2083] refactor: move all http2 tests in separate files --- tests/test_download_handlers_http2.py | 158 ++++++++++++++++++++++++++ tests/test_downloader_handlers.py | 141 +---------------------- 2 files changed, 160 insertions(+), 139 deletions(-) create mode 100644 tests/test_download_handlers_http2.py diff --git a/tests/test_download_handlers_http2.py b/tests/test_download_handlers_http2.py new file mode 100644 index 000000000..583dc1d17 --- /dev/null +++ b/tests/test_download_handlers_http2.py @@ -0,0 +1,158 @@ +from unittest import mock + +from twisted.internet import defer, error, reactor +from twisted.trial import unittest +from twisted.web import server + +from scrapy.core.downloader.handlers.http2 import H2DownloadHandler +from scrapy.http import Request +from scrapy.spiders import Spider +from scrapy.utils.misc import create_instance +from scrapy.utils.test import get_crawler +from tests.mockserver import ssl_context_factory +from tests.test_downloader_handlers import ( + Https11TestCase, Https11CustomCiphers, + Http11MockServerTestCase, Http11ProxyTestCase, + UriResource +) + + +class Https2TestCase(Https11TestCase): + scheme = 'https' + download_handler_cls = H2DownloadHandler + HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + + @defer.inlineCallbacks + def test_download_with_maxsize_very_large_file(self): + with mock.patch('scrapy.core.http2.stream.logger') as logger: + request = Request(self.getURL('largechunkedfile')) + + def check(logger): + logger.error.assert_called_once_with(mock.ANY) + + d = self.download_request(request, Spider('foo', download_maxsize=1500)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + # As the error message is logged in the dataReceived callback, we + # have to give a bit of time to the reactor to process the queue + # after closing the connection. + d = defer.Deferred() + d.addCallback(check) + reactor.callLater(.1, d.callback, logger) + yield d + + def test_download_broken_content_cause_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_cause_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + +class Https2WrongHostnameTestCase(Https2TestCase): + tls_log_message = ( + 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' + 'subject "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' + ) + + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = 'keys/example-com.key.pem' + certfile = 'keys/example-com.cert.pem' + + +class Https2InvalidDNSId(Https2TestCase): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super(Https2InvalidDNSId, self).setUp() + self.host = '127.0.0.1' + + +class Https2InvalidDNSPattern(Https2TestCase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = 'keys/localhost.ip.key' + certfile = 'keys/localhost.ip.crt' + + def setUp(self): + try: + from service_identity.exceptions import CertificateError # noqa: F401 + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") + self.tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' + 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + ) + super(Https2InvalidDNSPattern, self).setUp() + + +class Https2CustomCiphers(Https11CustomCiphers): + scheme = 'https' + download_handler_cls = H2DownloadHandler + + +class Http2MockServerTestCase(Http11MockServerTestCase): + """HTTP 2.0 test case with MockServer""" + + +class Https2ProxyTestCase(Http11ProxyTestCase): + # only used for HTTPS tests + keyfile = 'keys/localhost.key' + certfile = 'keys/localhost.crt' + + scheme = 'https' + host = u'127.0.0.1' + + download_handler_cls = H2DownloadHandler + expected_http_proxy_request_body = b'/' + + def setUp(self): + site = server.Site(UriResource(), timeout=None) + self.port = reactor.listenSSL( + 0, site, + ssl_context_factory(self.keyfile, self.certfile), + interface=self.host + ) + self.portno = self.port.getHost().port + self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) + self.download_request = self.download_handler.download_request + + def getURL(self, path): + return f"{self.scheme}://{self.host}:{self.portno}/{path}" + + def test_download_with_proxy_https_noconnect(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'/') + + http_proxy = '%s?noconnect' % self.getURL('') + request = Request('https://example.com', meta={'proxy': http_proxy}) + with self.assertWarnsRegex( + Warning, + r'Using HTTPS proxies in the noconnect mode is not supported by the ' + r'downloader handler.' + ): + return self.download_request(request, Spider('foo')).addCallback(_test) + + @defer.inlineCallbacks + def test_download_with_proxy_https_timeout(self): + with self.assertRaises(NotImplementedError): + yield super(Https2ProxyTestCase, self).test_download_with_proxy_https_timeout() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 1cedf6b10..8b2b2c32c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -23,7 +23,6 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler -from scrapy.core.downloader.handlers.http2 import H2DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Headers, Request @@ -315,8 +314,8 @@ class HttpTestCase(unittest.TestCase): host = self.host + ':' + str(self.portno) def _test(response): - self.assertEqual(response.body, to_bytes(host)) - self.assertEqual(request.headers.get('Host'), to_bytes(host)) + self.assertEqual(response.body, host.encode()) + self.assertEqual(request.headers.get('Host'), host.encode()) request = Request(self.getURL('host'), headers={'Host': host}) return self.download_request(request, Spider('foo')).addCallback(_test) @@ -522,49 +521,6 @@ class Https11TestCase(Http11TestCase): yield download_handler.close() -class Https2TestCase(Https11TestCase): - scheme = 'https' - download_handler_cls = H2DownloadHandler - HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): - with mock.patch('scrapy.core.http2.stream.logger') as logger: - request = Request(self.getURL('largechunkedfile')) - - def check(logger): - logger.error.assert_called_once_with(mock.ANY) - - d = self.download_request(request, Spider('foo', download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - # As the error message is logged in the dataReceived callback, we - # have to give a bit of time to the reactor to process the queue - # after closing the connection. - d = defer.Deferred() - d.addCallback(check) - reactor.callLater(.1, d.callback, logger) - yield d - - def test_download_broken_content_cause_data_loss(self, url='broken'): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_chunked_content_cause_data_loss(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_content_allow_data_loss(self, url='broken'): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_chunked_content_allow_data_loss(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_broken_chunked_content_allow_data_loss_via_setting(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - - class Https11WrongHostnameTestCase(Http11TestCase): scheme = 'https' @@ -577,23 +533,6 @@ class Https11WrongHostnameTestCase(Http11TestCase): certfile = 'keys/example-com.cert.pem' -class Https2WrongHostnameTestCase(Https2TestCase): - tls_log_message = ( - 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' - 'subject "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' - ) - - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = 'keys/example-com.key.pem' - certfile = 'keys/example-com.cert.pem' - - class Https11InvalidDNSId(Https11TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" @@ -602,14 +541,6 @@ class Https11InvalidDNSId(Https11TestCase): self.host = '127.0.0.1' -class Https2InvalidDNSId(Https2TestCase): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super(Https2InvalidDNSId, self).setUp() - self.host = '127.0.0.1' - - class Https11InvalidDNSPattern(Https11TestCase): """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" @@ -628,24 +559,6 @@ class Https11InvalidDNSPattern(Https11TestCase): super(Https11InvalidDNSPattern, self).setUp() -class Https2InvalidDNSPattern(Https2TestCase): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = 'keys/localhost.ip.key' - certfile = 'keys/localhost.ip.crt' - - def setUp(self): - try: - from service_identity.exceptions import CertificateError # noqa: F401 - except ImportError: - raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' - 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' - ) - super(Https2InvalidDNSPattern, self).setUp() - - class Https11CustomCiphers(unittest.TestCase): scheme = 'https' download_handler_cls = HTTP11DownloadHandler @@ -686,11 +599,6 @@ class Https11CustomCiphers(unittest.TestCase): return d -class Https2CustomCiphers(Https11CustomCiphers): - scheme = 'https' - download_handler_cls = H2DownloadHandler - - class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" @@ -742,10 +650,6 @@ class Http11MockServerTestCase(unittest.TestCase): self.assertTrue(reason, 'finished') -class Http2MockServerTestCase(Http11MockServerTestCase): - """HTTP 2.0 test case with MockServer""" - - class UriResource(resource.Resource): """Return the full uri that was requested""" @@ -837,47 +741,6 @@ class Http11ProxyTestCase(HttpProxyTestCase): self.assertIn(domain, timeout.osError) -class Https2ProxyTestCase(Http11ProxyTestCase): - # only used for HTTPS tests - keyfile = 'keys/localhost.key' - certfile = 'keys/localhost.crt' - - scheme = 'https' - host = u'127.0.0.1' - - download_handler_cls = H2DownloadHandler - expected_http_proxy_request_body = b'/' - - def setUp(self): - site = server.Site(UriResource(), timeout=None) - self.port = reactor.listenSSL( - 0, site, - ssl_context_factory(self.keyfile, self.certfile), - interface=self.host - ) - self.portno = self.port.getHost().port - self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) - self.download_request = self.download_handler.download_request - - def getURL(self, path): - return f"{self.scheme}://{self.host}:{self.portno}/{path}" - - def test_download_with_proxy_https_noconnect(self): - def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'/') - - http_proxy = '%s?noconnect' % self.getURL('') - request = Request('https://example.com', meta={'proxy': http_proxy}) - with self.assertWarnsRegex( - Warning, - r'Using HTTPS proxies in the noconnect mode is not supported by the ' - r'downloader handler.' - ): - return self.download_request(request, Spider('foo')).addCallback(_test) - - class HttpDownloadHandlerMock: def __init__(self, *args, **kwargs): From f9f008e935c5c892c2839d354ded3ee213057d9e Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 16 Aug 2020 17:04:40 +0530 Subject: [PATCH 0140/2083] test: add typing-extensions --- .travis.yml | 2 +- setup.py | 1 + tox.ini | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e273e358d..0b55cda19 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ dist: xenial branches: only: - master - - http2 # Remove once merged into master + - http2 # ToDo: Remove once merged into master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: diff --git a/setup.py b/setup.py index 7f4ff0095..c8733ae96 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.2.0', + 'typing-extensions>=3.7', ] extras_require = {} diff --git a/tox.ini b/tox.ini index e8fdbd85d..3bf2224ec 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,7 @@ deps = -ctests/constraints.txt -rtests/requirements-py3.txt # Extras + Twisted[http2]>=17.9.0 boto3>=1.13.0 botocore>=1.3.23 Pillow>=3.4.2 @@ -64,6 +65,7 @@ deps = -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 + h2==3.2.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 @@ -72,12 +74,12 @@ deps = queuelib==1.4.2 service_identity==16.0.0 Twisted[http2]==17.9.0 + typing-extensions==3.7 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt # Extras botocore==1.3.23 - h2==3.2.0 google-cloud-storage==1.29.0 Pillow==3.4.2 From 38d361792c02ae2b25323258d070c04d8906495a Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 16 Aug 2020 17:55:16 +0530 Subject: [PATCH 0141/2083] fix: typing & pylint errors - Ignore typing check for http2 test files --- scrapy/core/downloader/handlers/http2.py | 4 ++-- scrapy/core/http2/protocol.py | 2 +- setup.cfg | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 650af9778..f2ed40f9b 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -71,8 +71,8 @@ class ScrapyH2Agent: def __init__( self, context_factory, - connect_timeout=10, - bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None, + pool: H2ConnectionPool, + connect_timeout=10, bind_address: Optional[bytes] = None, crawler=None ) -> None: self._context_factory = context_factory diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 1ce8b6548..fee391af6 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -323,7 +323,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) elif isinstance(event, UnknownFrameReceived): - logger.debug(f'UnknownFrameReceived: frame={event.frame}') + logger.debug('UnknownFrameReceived: frame={}'.format(event.frame)) # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated) -> None: diff --git a/setup.cfg b/setup.cfg index f8e7c0c91..8b70a0e60 100644 --- a/setup.cfg +++ b/setup.cfg @@ -100,6 +100,9 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True +[mypy-tests.test_download_handlers_http2] +ignore_errors = True + [mypy-tests.test_engine] ignore_errors = True @@ -109,6 +112,9 @@ ignore_errors = True [mypy-tests.test_http_request] ignore_errors = True +[mypy-tests.test_http2_client_protocol] +ignore_errors = True + [mypy-tests.test_linkextractors] ignore_errors = True From 75fe3d13657e11bdabb9b26b452c6cdacecffec7 Mon Sep 17 00:00:00 2001 From: adityaa30 Date: Mon, 17 Aug 2020 03:47:17 +0530 Subject: [PATCH 0142/2083] fix: increase timeout to 0.5 seconds - In Windows specifically the reactor was left unclean by the HostnameEndpoint due to the tearDown method of test_downloader_handlers.py::HttpTestCase due to which the following 2 tests were failing: 1. test_timeout_download_from_spider_server_hangs 2. test_timeout_download_from_spider_nodata_rcvd - Increasing the timeout fixed the test (in local) --- setup.cfg | 2 +- tests/test_downloader_handlers.py | 4 ++-- ...ad_handlers_http2.py => test_downloader_handlers_http2.py} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename tests/{test_download_handlers_http2.py => test_downloader_handlers_http2.py} (100%) diff --git a/setup.cfg b/setup.cfg index 8b70a0e60..5b267c295 100644 --- a/setup.cfg +++ b/setup.cfg @@ -100,7 +100,7 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_download_handlers_http2] +[mypy-tests.test_downloader_handlers_http2] ignore_errors = True [mypy-tests.test_engine] diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index ecac47d90..2b3fa2aca 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -287,7 +287,7 @@ class HttpTestCase(unittest.TestCase): def test_timeout_download_from_spider_nodata_rcvd(self): # client connects but no data is received spider = Spider('foo') - meta = {'download_timeout': 0.2} + meta = {'download_timeout': 0.5} request = Request(self.getURL('wait'), meta=meta) d = self.download_request(request, spider) yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) @@ -296,7 +296,7 @@ class HttpTestCase(unittest.TestCase): def test_timeout_download_from_spider_server_hangs(self): # client connects, server send headers and some body bytes but hangs spider = Spider('foo') - meta = {'download_timeout': 0.2} + meta = {'download_timeout': 0.5} request = Request(self.getURL('hang-after-headers'), meta=meta) d = self.download_request(request, spider) yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) diff --git a/tests/test_download_handlers_http2.py b/tests/test_downloader_handlers_http2.py similarity index 100% rename from tests/test_download_handlers_http2.py rename to tests/test_downloader_handlers_http2.py From a87ab71d1061585e41864d7283557bbe9823a91b Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 18 Aug 2020 04:47:09 +0530 Subject: [PATCH 0143/2083] refactor(http2): metadata for Stream - Add Note about HTTP/2 Cleartext not supported in settings.rst --- docs/topics/settings.rst | 7 +++ scrapy/core/http2/protocol.py | 2 +- scrapy/core/http2/stream.py | 73 +++++++++++++++---------------- scrapy/core/http2/types.py | 23 ++++++++++ setup.py | 2 +- tests/test_downloader_handlers.py | 2 +- 6 files changed, 68 insertions(+), 41 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d4d4f9332..0dad30b29 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -627,6 +627,13 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', } +.. note:: + + Scrapy currently does not support HTTP/2 Cleartext (h2c) since none + of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_). + +.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index fee391af6..7a3156541 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -289,7 +289,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._conn_lost_deferred.callback(self._conn_lost_errors) for stream in self.streams.values(): - if stream.request_sent: + if stream.metadata['request_sent']: close_reason = StreamCloseReason.CONNECTION_LOST else: close_reason = StreamCloseReason.INACTIVE diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 1e136fbd5..bddf50a56 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -5,14 +5,14 @@ from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes -from h2.exceptions import H2Error, StreamClosedError +from h2.exceptions import H2Error, StreamClosedError, ProtocolError from hpack import HeaderTuple from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from scrapy.core.http2.types import H2ResponseDict +from scrapy.core.http2.types import H2ResponseDict, H2StreamMetadataDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes @@ -100,24 +100,14 @@ class Stream: self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) - self.request_start_time = None - - self.content_length = 0 if self._request.body is None else len(self._request.body) - - # Flag to keep track whether this stream has initiated the request - self.request_sent = False - - # Flag to track whether we have logged about exceeding download warnsize - self._reached_warnsize = False - - # Each time we send a data frame, we will decrease value by the amount send. - self.remaining_content_length = self.content_length - - # Flag to keep track whether we have closed this stream - self.stream_closed_local = False - - # Flag to keep track whether the server has closed the stream - self.stream_closed_server = False + self.metadata: H2StreamMetadataDict = { + 'request_content_length': 0 if self._request.body is None else len(self._request.body), + 'request_sent': False, + 'reached_warnsize': False, + 'remaining_content_length': 0 if self._request.body is None else len(self._request.body), + 'stream_closed_local': False, + 'stream_closed_server': False, + } # Private variable used to build the response # this response is then converted to appropriate Response class @@ -132,7 +122,7 @@ class Stream: # Close this stream as gracefully as possible # If the associated request is initiated we reset this stream # else we directly call close() method - if self.request_sent: + if self.metadata['request_sent']: self.reset_stream(StreamCloseReason.CANCELLED) else: self.close(StreamCloseReason.CANCELLED) @@ -160,7 +150,7 @@ class Stream: self._response['flow_controlled_size'] > self._download_warnsize or content_length_header > self._download_warnsize ) - and not self._reached_warnsize + and not self.metadata['reached_warnsize'] ) def get_response(self) -> Deferred: @@ -220,7 +210,7 @@ class Stream: if self.check_request_url(): headers = self._get_request_headers() self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False) - self.request_sent = True + self.metadata['request_sent'] = True self.send_data() else: # Close this stream calling the response errback @@ -238,7 +228,7 @@ class Stream: and has initiated request already by sending HEADER frame. If not then stream will raise ProtocolError (raise by h2 state machine). """ - if self.stream_closed_local: + if self.metadata['stream_closed_local']: raise StreamClosedError(self.stream_id) # Firstly, check what the flow control window is for current stream. @@ -249,24 +239,24 @@ class Stream: # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. - bytes_to_send_size = min(window_size, self.remaining_content_length) + bytes_to_send_size = min(window_size, self.metadata['remaining_content_length']) # We now need to send a number of data frames. while bytes_to_send_size > 0: chunk_size = min(bytes_to_send_size, max_frame_size) - data_chunk_start_id = self.content_length - self.remaining_content_length + data_chunk_start_id = self.metadata['request_content_length'] - self.metadata['remaining_content_length'] data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size] self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False) bytes_to_send_size = bytes_to_send_size - chunk_size - self.remaining_content_length = self.remaining_content_length - chunk_size + self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size - self.remaining_content_length = max(0, self.remaining_content_length) + self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length']) # End the stream if no more data needs to be send - if self.remaining_content_length == 0: + if self.metadata['remaining_content_length'] == 0: self._protocol.conn.end_stream(self.stream_id) # Q. What about the rest of the data? @@ -277,7 +267,11 @@ class Stream: Send data that earlier could not be sent as we were blocked behind the flow control. """ - if self.remaining_content_length and not self.stream_closed_server and self.request_sent: + if ( + self.metadata['remaining_content_length'] + and not self.metadata['stream_closed_server'] + and self.metadata['request_sent'] + ): self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int) -> None: @@ -290,7 +284,7 @@ class Stream: return if self._log_warnsize: - self._reached_warnsize = True + self.metadata['reached_warnsize'] = True warning_msg = ( f'Received more ({self._response["flow_controlled_size"]}) bytes than download ' f'warn size ({self._download_warnsize}) in request {self._request}' @@ -314,7 +308,7 @@ class Stream: return if self._log_warnsize: - self._reached_warnsize = True + self.metadata['reached_warnsize'] = True warning_msg = ( f'Expected response size ({expected_size}) larger than ' f'download warn size ({self._download_warnsize}) in request {self._request}' @@ -323,18 +317,18 @@ class Stream: def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None: """Close this stream by sending a RST_FRAME to the remote peer""" - if self.stream_closed_local: + if self.metadata['stream_closed_local']: raise StreamClosedError(self.stream_id) # Clear buffer earlier to avoid keeping data in memory for a long time self._response['body'].truncate(0) - self.stream_closed_local = True + self.metadata['stream_closed_local'] = True self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) self.close(reason) def _is_data_lost(self) -> bool: - assert self.stream_closed_server + assert self.metadata['stream_closed_server'] expected_size = self._response['flow_controlled_size'] received_body_size = int(self._response['headers'][b'Content-Length']) @@ -349,7 +343,7 @@ class Stream: ) -> None: """Based on the reason sent we will handle each case. """ - if self.stream_closed_server: + if self.metadata['stream_closed_server']: raise StreamClosedError(self.stream_id) if not isinstance(reason, StreamCloseReason): @@ -362,7 +356,7 @@ class Stream: if not from_protocol: self._protocol.pop_stream(self.stream_id) - self.stream_closed_server = True + self.metadata['stream_closed_server'] = True # We do not check for Content-Length or Transfer-Encoding in response headers # and add `partial` flag as in HTTP/1.1 as 'A request or response that includes @@ -402,7 +396,10 @@ class Stream: elif reason is StreamCloseReason.RESET: self._deferred_response.errback(ResponseFailed([ - Failure(f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM') + Failure( + f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM', + ProtocolError + ) ])) elif reason is StreamCloseReason.CONNECTION_LOST: diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py index d2aa1a9d8..ff8d94066 100644 --- a/scrapy/core/http2/types.py +++ b/scrapy/core/http2/types.py @@ -31,6 +31,29 @@ class H2ConnectionMetadataDict(TypedDict): default_download_warnsize: int +class H2StreamMetadataDict(TypedDict): + """Metadata of an HTTP/2 connection stream + initialized when stream is instantiated + """ + + request_content_length: int + + # Flag to keep track whether the stream has initiated the request + request_sent: bool + + # Flag to track whether we have logged about exceeding download warnsize + reached_warnsize: bool + + # Each time we send a data frame, we will decrease value by the amount send. + remaining_content_length: int + + # Flag to keep track whether we have closed this stream + stream_closed_local: bool + + # Flag to keep track whether the server has closed the stream + stream_closed_server: bool + + class H2ResponseDict(TypedDict): # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. diff --git a/setup.py b/setup.py index c8733ae96..66f369a71 100644 --- a/setup.py +++ b/setup.py @@ -97,4 +97,4 @@ setup( python_requires='>=3.5.2', install_requires=install_requires, extras_require=extras_require, -) \ No newline at end of file +) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2b3fa2aca..e3777ee1d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -232,7 +232,7 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) - self.host = u'localhost' + self.host = 'localhost' if self.scheme == 'https': # Using WrappingFactory do not enable HTTP/2 failing all the # tests with H2DownloadHandler From a206ac5f6f4e5eb057ecc535556a15087b250d40 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 18 Aug 2020 07:36:00 +0530 Subject: [PATCH 0144/2083] tests: disable python 3.5 for travis and azure --- .travis.yml | 25 +++++++++++++------------ azure-pipelines.yml | 7 ++++--- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0b55cda19..6628e8e43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,18 +18,19 @@ matrix: python: 3.7 # Keep in sync with .readthedocs.yml - env: TOXENV=typing python: 3.8 - - - env: TOXENV=pinned - python: 3.5.2 - - env: TOXENV=asyncio-pinned - python: 3.5.2 # We use additional code to support 3.5.3 and earlier - - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0 - - - env: TOXENV=py - python: 3.5 - - env: TOXENV=asyncio - python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 - - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 + +# ToDo: Remove once merged into master +# - env: TOXENV=pinned +# python: 3.5.2 +# - env: TOXENV=asyncio-pinned +# python: 3.5.2 # We use additional code to support 3.5.3 and earlier +# - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0 +# +# - env: TOXENV=py +# python: 3.5 +# - env: TOXENV=asyncio +# python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 +# - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 - env: TOXENV=py python: 3.6 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 710e42090..c77c128b3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -4,9 +4,10 @@ pool: vmImage: 'windows-latest' strategy: matrix: - Python35: - python.version: '3.5' - TOXENV: windows-pinned +# ToDo: Remove once merged into master +# Python35: +# python.version: '3.5' +# TOXENV: windows-pinned Python36: python.version: '3.6' Python37: From e3233b79deee6ad283ed4316135e30ab774c1078 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 19 Aug 2020 05:10:19 +0530 Subject: [PATCH 0145/2083] refactor(h2-stream): alphabetical order of imports --- scrapy/core/downloader/handlers/http2.py | 2 +- scrapy/core/http2/stream.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index f2ed40f9b..ddd813cec 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -29,7 +29,7 @@ class H2DownloadHandler: def from_crawler(cls, crawler): return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index bddf50a56..33302421e 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -5,7 +5,7 @@ from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes -from h2.exceptions import H2Error, StreamClosedError, ProtocolError +from h2.exceptions import H2Error, ProtocolError, StreamClosedError from hpack import HeaderTuple from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed From 30eb005639b77ea94e8859e1d041dfe53048cf77 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 19 Aug 2020 06:25:04 +0530 Subject: [PATCH 0146/2083] fix: InvalidNegotiatedProtocol __str__ method --- scrapy/core/http2/agent.py | 2 +- scrapy/core/http2/protocol.py | 2 +- scrapy/core/http2/stream.py | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index f4ac29bc6..f829cc5f8 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -68,7 +68,7 @@ class H2ConnectionPool: # Now as we have established a proper HTTP/2 connection # we fire all the deferred's with the connection instance - pending_requests = self._pending_requests.pop(key) + pending_requests = self._pending_requests.pop(key, None) while pending_requests: d = pending_requests.popleft() d.callback(conn) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 7a3156541..c6f00423a 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -39,7 +39,7 @@ class InvalidNegotiatedProtocol(H2Error): self.negotiated_protocol = negotiated_protocol def __str__(self) -> str: - return f'InvalidHostname: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}' + return f'InvalidNegotiatedProtocol: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}' class RemoteTerminatedConnection(H2Error): diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 33302421e..df7470e11 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -31,6 +31,9 @@ class InactiveStreamClosed(ConnectionClosed): def __init__(self, request: Request): self.request = request + def __str__(self) -> str: + return f'InactiveStreamClosed: Connection was closed without sending the request {self.request!r}' + class InvalidHostname(H2Error): From 2f00666d749b4a736536bafc786e63be06113676 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 19 Aug 2020 07:31:52 +0530 Subject: [PATCH 0147/2083] refactor: move agents & context-factory --- scrapy/core/downloader/contextfactory.py | 22 ++++++++++++- scrapy/core/downloader/handlers/http2.py | 31 ++--------------- scrapy/core/http2/agent.py | 42 ++++++++++++++---------- 3 files changed, 49 insertions(+), 46 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 5768d8f8e..073ef16bf 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,10 +1,12 @@ import warnings from OpenSSL import SSL +from twisted.internet._sslverify import _setAcceptableProtocols from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer +from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions from scrapy.utils.misc import create_instance, load_object @@ -84,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): The default OpenSSL method is ``TLS_METHOD`` (also called ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ - def creatorForNetloc(self, hostname, port): + def creatorForNetloc(self, hostname, port): # trustRoot set to platformTrust() will use the platform's root CAs. # # This means that a website like https://www.cacert.org will be rejected @@ -97,6 +99,24 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): ) +@implementer(IPolicyForHTTPS) +class AcceptableProtocolsContextFactory: + """Context factory to used to override the acceptable protocols + to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN + negotiation. + """ + + def __init__(self, context_factory, acceptable_protocols): + verifyObject(IPolicyForHTTPS, context_factory) + self._wrapped_context_factory = context_factory + self._acceptable_protocols = acceptable_protocols + + def creatorForNetloc(self, hostname, port): + options = self._wrapped_context_factory.creatorForNetloc(hostname, port) + _setAcceptableProtocols(options._ctx, self._acceptable_protocols) + return options + + def load_context_factory_from_settings(settings, crawler): ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index ddd813cec..4be888bda 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,16 +1,15 @@ import warnings from time import time -from typing import Optional, Tuple +from typing import Optional from urllib.parse import urldefrag -from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.web.client import BrowserLikePolicyForHTTPS, URI +from twisted.web.client import URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse -from scrapy.core.http2.agent import H2Agent, H2ConnectionPool +from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider @@ -41,30 +40,6 @@ class H2DownloadHandler: self._pool.close_connections() -class ScrapyProxyH2Agent(H2Agent): - def __init__( - self, reactor: ReactorBase, - proxy_uri: URI, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None - ) -> None: - super(ScrapyProxyH2Agent, self).__init__( - reactor=reactor, - pool=pool, - context_factory=context_factory, - connect_timeout=connect_timeout, - bind_address=bind_address - ) - self._proxy_uri = proxy_uri - - def get_endpoint(self, uri: URI): - return self.endpoint_factory.endpointForURI(self._proxy_uri) - - def get_key(self, uri: URI) -> Tuple: - """We use the proxy uri instead of uri obtained from request url""" - return "http-proxy", self._proxy_uri.host, self._proxy_uri.port - - class ScrapyH2Agent: _Agent = H2Agent _ProxyAgent = ScrapyProxyH2Agent diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index f829cc5f8..d950c6cfb 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -2,17 +2,14 @@ from collections import deque from typing import Deque, Dict, List, Tuple, Optional from twisted.internet import defer -from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint from twisted.python.failure import Failure from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory from twisted.web.error import SchemeNotSupported -from twisted.web.iweb import IPolicyForHTTPS -from zope.interface import implementer -from zope.interface.verify import verifyObject +from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory from scrapy.http.request import Request from scrapy.settings import Settings @@ -96,19 +93,6 @@ class H2ConnectionPool: conn.transport.abortConnection() -@implementer(IPolicyForHTTPS) -class AcceptableProtocolsContextFactory: - def __init__(self, context_factory, acceptable_protocols: List[bytes]) -> None: - verifyObject(IPolicyForHTTPS, context_factory) - self._wrapped_context_factory = context_factory - self._acceptable_protocols = acceptable_protocols - - def creatorForNetloc(self, hostname, port) -> ClientTLSOptions: - options = self._wrapped_context_factory.creatorForNetloc(hostname, port) - _setAcceptableProtocols(options._ctx, self._acceptable_protocols) - return options - - class H2Agent: def __init__( self, reactor: ReactorBase, pool: H2ConnectionPool, @@ -144,3 +128,27 @@ class H2Agent: d = self._pool.get_connection(key, uri, endpoint) d.addCallback(lambda conn: conn.request(request, spider)) return d + + +class ScrapyProxyH2Agent(H2Agent): + def __init__( + self, reactor: ReactorBase, + proxy_uri: URI, pool: H2ConnectionPool, + context_factory=BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + ) -> None: + super(ScrapyProxyH2Agent, self).__init__( + reactor=reactor, + pool=pool, + context_factory=context_factory, + connect_timeout=connect_timeout, + bind_address=bind_address + ) + self._proxy_uri = proxy_uri + + def get_endpoint(self, uri: URI): + return self.endpoint_factory.endpointForURI(self._proxy_uri) + + def get_key(self, uri: URI) -> Tuple: + """We use the proxy uri instead of uri obtained from request url""" + return "http-proxy", self._proxy_uri.host, self._proxy_uri.port From 1432161477673152f4d2f95cd32829a81ffe3f70 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 24 Aug 2020 15:40:01 +0530 Subject: [PATCH 0148/2083] fix: bump min typing-extensions version to 3.7.4 - typing-extensions>=3.7.4 only supports TypedDict --- setup.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 607f1dadf..39482d383 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.2.0', - 'typing-extensions>=3.7', + 'typing-extensions>=3.7.4', ] extras_require = {} diff --git a/tox.ini b/tox.ini index 3f6fd1224..0a88ed8af 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,7 @@ deps = queuelib==1.4.2 service_identity==16.0.0 Twisted[http2]==17.9.0 - typing-extensions==3.7 + typing-extensions==3.7.4 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt From 8b84a65a6b2d391fdd9c49426d748751a03351f0 Mon Sep 17 00:00:00 2001 From: drs-11 Date: Tue, 25 Aug 2020 00:30:17 +0530 Subject: [PATCH 0149/2083] cleaned up code relating to issue #3689 --- scrapy/pipelines/images.py | 14 ++----- tests/test_pipeline_images.py | 72 ++++++++++++++++++----------------- 2 files changed, 41 insertions(+), 45 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index f709c5057..e265685fb 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -6,6 +6,7 @@ See documentation in topics/media-pipeline.rst import functools import hashlib import six +import warnings try: from cStringIO import StringIO as BytesIO @@ -19,6 +20,7 @@ from scrapy.utils.python import to_bytes, get_func_args from scrapy.http import Request from scrapy.settings import Settings from scrapy.exceptions import DropItem +from scrapy.exceptions import ScrapyDeprecationWarning #TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.pipelines.files import FileException, FilesPipeline @@ -132,8 +134,6 @@ class ImagesPipeline(FilesPipeline): if self._deprecated_convert_image is None: self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image) if self._deprecated_convert_image: - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) @@ -153,9 +153,7 @@ class ImagesPipeline(FilesPipeline): yield thumb_path, thumb_image, thumb_buf def convert_image(self, image, size=None, response_body=None): - if not response_body: - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings + if response_body is None: warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ' 'method called without response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) @@ -175,7 +173,7 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - elif response_body and image.format == 'JPEG': + elif response_body is not None and image.format == 'JPEG': return image, response_body buf = BytesIO() @@ -193,8 +191,6 @@ class ImagesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None): ## start of deprecation warning block (can be removed in the future) def _warn(): - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings warnings.warn('ImagesPipeline.image_key(url) and file_key(url) methods are deprecated, ' 'please use file_path(request, response=None, info=None) instead', category=ScrapyDeprecationWarning, stacklevel=1) @@ -221,8 +217,6 @@ class ImagesPipeline(FilesPipeline): def thumb_path(self, request, thumb_id, response=None, info=None): ## start of deprecation warning block (can be removed in the future) def _warn(): - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings warnings.warn('ImagesPipeline.thumb_key(url) method is deprecated, please use ' 'thumb_path(request, thumb_id, response=None, info=None) instead', category=ScrapyDeprecationWarning, stacklevel=1) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index ec0c87264..915b6a579 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -75,41 +75,7 @@ class ImagesPipelineTestCase(unittest.TestCase): info=object()), 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') - def test_convert_image(self): - # tests for old API - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - SIZE = (100, 100) - # straigh forward case: RGB and JPEG - COLOUR = (0, 127, 255) - im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) - - # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) - self.assertEqual(thumbnail.mode, 'RGB') - self.assertEqual(thumbnail.size, (10, 10)) - - # transparency case: RGBA and PNG - COLOUR = (0, 127, 255, 50) - im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # transparency case with palette: P and PNG - COLOUR = (0, 127, 255, 50) - im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) - im = im.convert('P') - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # ensure that we recieved deprecation warnings - self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a deprecated way' in str(warning.message)]) == 4) - + def test_convert_image_new(self): # tests for new API SIZE = (100, 100) # straigh forward case: RGB and JPEG @@ -207,6 +173,42 @@ class DeprecatedImagesPipelineTestCase(unittest.TestCase): self.assertEqual(len(w), 1) self.assertTrue('thumb_key(url) method is deprecated' in str(w[-1].message)) + def test_overriden_convert_image_method(self): + self.init_pipeline(ImagesPipeline) + # tests for old API + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + SIZE = (100, 100) + # straigh forward case: RGB and JPEG + COLOUR = (0, 127, 255) + im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + + # check that thumbnail keep image ratio + thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) + self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.size, (10, 10)) + + # transparency case: RGBA and PNG + COLOUR = (0, 127, 255, 50) + im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # transparency case with palette: P and PNG + COLOUR = (0, 127, 255, 50) + im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im = im.convert('P') + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # ensure that we recieved deprecation warnings + self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a deprecated way' in str(warning.message)]) == 4) + def tearDown(self): rmtree(self.tempdir) From ecec5f9e5171568c4ead64336245a1bb3be2ecfe Mon Sep 17 00:00:00 2001 From: drs-11 Date: Tue, 25 Aug 2020 02:46:44 +0530 Subject: [PATCH 0150/2083] Cleaned up code --- scrapy/pipelines/images.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index d7f437adc..47d688c62 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -12,7 +12,7 @@ from io import BytesIO from itemadapter import ItemAdapter from PIL import Image -from scrapy.exceptions import DropItem,ScrapyDeprecationWarning +from scrapy.exceptions import DropItem, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline @@ -175,7 +175,7 @@ class ImagesPipeline(FilesPipeline): image.thumbnail(size, Image.ANTIALIAS) elif response_body is not None and image.format == 'JPEG': return image, response_body - + buf = BytesIO() image.save(buf, 'JPEG') return image, buf From 450ba6b51f7cc3cd89a92e8ff4d9e105865eb862 Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 26 Aug 2020 17:20:59 +0530 Subject: [PATCH 0151/2083] fix(typo): stream -> streams, use isinstance --- scrapy/core/http2/protocol.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index c6f00423a..6647ae0b7 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -90,7 +90,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # all requests in a pool and send them as the connection is made self._pending_request_stream_pool: deque = deque() - # Counter to keep track of opened stream. This counter + # Counter to keep track of opened streams. This counter # is used to make sure that not more than MAX_CONCURRENT_STREAMS # streams are opened which leads to ProtocolError # We use simple FIFO policy to handle pending requests @@ -218,7 +218,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" negotiated_protocol = self.transport.negotiatedProtocol - if type(negotiated_protocol) is bytes: + if isinstance(negotiated_protocol, bytes): negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') if negotiated_protocol != 'h2': # Here we have not initiated the connection yet From 5e36f539e28631625862b84d8ca1c7c0134584b0 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 27 Aug 2020 15:12:22 +0530 Subject: [PATCH 0152/2083] chore: remove typing-extensions dependency --- scrapy/core/http2/protocol.py | 52 ++++++++++++++++----------- scrapy/core/http2/stream.py | 27 +++++++++++--- scrapy/core/http2/types.py | 67 ----------------------------------- setup.py | 1 - tox.ini | 1 - 5 files changed, 55 insertions(+), 93 deletions(-) delete mode 100644 scrapy/core/http2/types.py diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 6647ae0b7..0b872f6ba 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -3,7 +3,6 @@ import itertools import logging from collections import deque from ipaddress import IPv4Address, IPv6Address -from typing import Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -22,10 +21,10 @@ from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin from twisted.python.failure import Failure from twisted.web.client import URI +from typing import Dict, List, Optional, Union from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason -from scrapy.core.http2.types import H2ConnectionMetadataDict from scrapy.http import Request from scrapy.settings import Settings from scrapy.spiders import Spider @@ -90,26 +89,39 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # all requests in a pool and send them as the connection is made self._pending_request_stream_pool: deque = deque() - # Counter to keep track of opened streams. This counter - # is used to make sure that not more than MAX_CONCURRENT_STREAMS - # streams are opened which leads to ProtocolError - # We use simple FIFO policy to handle pending requests - self._active_streams = 0 - - # Flag to keep track if settings were acknowledged by the remote - # This ensures that we have established a HTTP/2 connection - self._settings_acknowledged = False - # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure self._conn_lost_errors: List[BaseException] = [] - self.metadata: H2ConnectionMetadataDict = { + # Some meta data of this connection + # initialized when connection is successfully made + self.metadata: Dict = { + # Peer certificate instance 'certificate': None, + + # Address of the server we are connected to which + # is updated when HTTP/2 connection is made successfully 'ip_address': None, + + # URI of the peer HTTP/2 connection is made 'uri': uri, + + # Both ip_address and uri are used by the Stream before + # initiating the request to verify that the base address + + # Variables taken from Project Settings 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'), 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'), + + # Counter to keep track of opened streams. This counter + # is used to make sure that not more than MAX_CONCURRENT_STREAMS + # streams are opened which leads to ProtocolError + # We use simple FIFO policy to handle pending requests + 'active_streams': 0, + + # Flag to keep track if settings were acknowledged by the remote + # This ensures that we have established a HTTP/2 connection + 'settings_acknowledged': False, } @property @@ -118,7 +130,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection """ - return bool(self.transport.connected) and self._settings_acknowledged + return bool(self.transport.connected) and self.metadata['settings_acknowledged'] @property def allowed_max_concurrent_streams(self) -> int: @@ -139,10 +151,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """ while ( self._pending_request_stream_pool - and self._active_streams < self.allowed_max_concurrent_streams + and self.metadata['active_streams'] < self.allowed_max_concurrent_streams and self.h2_connected ): - self._active_streams += 1 + self.metadata['active_streams'] += 1 stream = self._pending_request_stream_pool.popleft() stream.initiate_request() self._write_to_transport() @@ -151,7 +163,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """Perform cleanup when a stream is closed """ stream = self.streams.pop(stream_id) - self._active_streams -= 1 + self.metadata['active_streams'] -= 1 self._send_pending_requests() return stream @@ -261,7 +273,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): if ( self.conn.open_outbound_streams > 0 or self.conn.open_inbound_streams > 0 - or self._active_streams > 0 + or self.metadata['active_streams'] > 0 ): error_code = ErrorCodes.PROTOCOL_ERROR else: @@ -295,7 +307,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): close_reason = StreamCloseReason.INACTIVE stream.close(close_reason, self._conn_lost_errors, from_protocol=True) - self._active_streams -= len(self.streams) + self.metadata['active_streams'] -= len(self.streams) self.streams.clear() self._pending_request_stream_pool.clear() self.conn.close_connection() @@ -338,7 +350,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.streams[event.stream_id].receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: - self._settings_acknowledged = True + self.metadata['settings_acknowledged'] = True # Send off all the pending requests as now we have # established a proper HTTP/2 connection diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index df7470e11..e01e76ae5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,7 +1,6 @@ import logging from enum import Enum from io import BytesIO -from typing import List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse from h2.errors import ErrorCodes @@ -11,8 +10,9 @@ from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed +from typing import Dict +from typing import List, Optional, Tuple, TYPE_CHECKING -from scrapy.core.http2.types import H2ResponseDict, H2StreamMetadataDict from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes @@ -103,21 +103,40 @@ class Stream: self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) - self.metadata: H2StreamMetadataDict = { + # Metadata of an HTTP/2 connection stream + # initialized when stream is instantiated + self.metadata: Dict = { 'request_content_length': 0 if self._request.body is None else len(self._request.body), + + # Flag to keep track whether the stream has initiated the request 'request_sent': False, + + # Flag to track whether we have logged about exceeding download warnsize 'reached_warnsize': False, + + # Each time we send a data frame, we will decrease value by the amount send. 'remaining_content_length': 0 if self._request.body is None else len(self._request.body), + + # Flag to keep track whether client (self) have closed this stream 'stream_closed_local': False, + + # Flag to keep track whether the server has closed the stream 'stream_closed_server': False, } # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: H2ResponseDict = { + self._response: Dict = { + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. 'body': BytesIO(), + + # The amount of data received that counts against the + # flow control window 'flow_controlled_size': 0, + + # Headers received after sending the request 'headers': Headers({}), } diff --git a/scrapy/core/http2/types.py b/scrapy/core/http2/types.py deleted file mode 100644 index ff8d94066..000000000 --- a/scrapy/core/http2/types.py +++ /dev/null @@ -1,67 +0,0 @@ -from io import BytesIO -from ipaddress import IPv4Address, IPv6Address -from typing import Union, Optional - -from twisted.internet.ssl import Certificate -from twisted.web.client import URI -# for python < 3.8 -- typing.TypedDict is undefined -from typing_extensions import TypedDict - -from scrapy.http.headers import Headers - - -class H2ConnectionMetadataDict(TypedDict): - """Some meta data of this connection - initialized when connection is successfully made - """ - certificate: Optional[Certificate] - - # Address of the server we are connected to which - # is updated when HTTP/2 connection is made successfully - ip_address: Optional[Union[IPv4Address, IPv6Address]] - - # URI of the peer HTTP/2 connection is made - uri: URI - - # Both ip_address and uri are used by the Stream before - # initiating the request to verify that the base address - - # Variables taken from Project Settings - default_download_maxsize: int - default_download_warnsize: int - - -class H2StreamMetadataDict(TypedDict): - """Metadata of an HTTP/2 connection stream - initialized when stream is instantiated - """ - - request_content_length: int - - # Flag to keep track whether the stream has initiated the request - request_sent: bool - - # Flag to track whether we have logged about exceeding download warnsize - reached_warnsize: bool - - # Each time we send a data frame, we will decrease value by the amount send. - remaining_content_length: int - - # Flag to keep track whether we have closed this stream - stream_closed_local: bool - - # Flag to keep track whether the server has closed the stream - stream_closed_server: bool - - -class H2ResponseDict(TypedDict): - # Data received frame by frame from the server is appended - # and passed to the response Deferred when completely received. - body: BytesIO - - # The amount of data received that counts against the flow control - # window - flow_controlled_size: int - - # Headers received after sending the request - headers: Headers diff --git a/setup.py b/setup.py index 39482d383..34af7ec67 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,6 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.2.0', - 'typing-extensions>=3.7.4', ] extras_require = {} diff --git a/tox.ini b/tox.ini index 0a88ed8af..78090d6de 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,6 @@ deps = queuelib==1.4.2 service_identity==16.0.0 Twisted[http2]==17.9.0 - typing-extensions==3.7.4 w3lib==1.17.0 zope.interface==4.1.3 -rtests/requirements-py3.txt From a8aedbeb7c20c7e2ec041e49d1567a499cd3acdb Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 29 Aug 2020 12:12:18 +0530 Subject: [PATCH 0153/2083] chore: rearrange imports --- scrapy/core/http2/protocol.py | 2 +- scrapy/core/http2/stream.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 0b872f6ba..e32e2b6fe 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -3,6 +3,7 @@ import itertools import logging from collections import deque from ipaddress import IPv4Address, IPv6Address +from typing import Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -21,7 +22,6 @@ from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin from twisted.python.failure import Failure from twisted.web.client import URI -from typing import Dict, List, Optional, Union from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index e01e76ae5..ef90773b6 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -2,6 +2,7 @@ import logging from enum import Enum from io import BytesIO from urllib.parse import urlparse +from typing import Dict, List, Optional, Tuple, TYPE_CHECKING from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -10,8 +11,6 @@ from twisted.internet.defer import Deferred, CancelledError from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from typing import Dict -from typing import List, Optional, Tuple, TYPE_CHECKING from scrapy.http import Request from scrapy.http.headers import Headers From eff33a2e7950b29fd69adc40b17b7fe9b0e637c2 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 30 Aug 2020 23:54:43 +0530 Subject: [PATCH 0154/2083] fix(h2): Mockserver test uses H2DownloadHandler --- tests/test_downloader_handlers.py | 7 ++++--- tests/test_downloader_handlers_http2.py | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e3777ee1d..c6f20cb50 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -601,6 +601,7 @@ class Https11CustomCiphers(unittest.TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" + settings_dict = None def setUp(self): self.mockserver = MockServer() @@ -611,7 +612,7 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_with_content_length(self): - crawler = get_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000})) @@ -620,7 +621,7 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download(self): - crawler = get_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) yield crawler.crawl(seed=Request(url=self.mockserver.url(''))) failure = crawler.spider.meta.get('failure') self.assertTrue(failure is None) @@ -629,7 +630,7 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_gzip_response(self): - crawler = get_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) body = b'1' * 100 # PayloadResource requires body length to be 100 request = Request(self.mockserver.url('/payload'), method='POST', body=body, meta={'download_maxsize': 50}) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 583dc1d17..253646040 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -110,6 +110,11 @@ class Https2CustomCiphers(Https11CustomCiphers): class Http2MockServerTestCase(Http11MockServerTestCase): """HTTP 2.0 test case with MockServer""" + settings_dict = { + 'DOWNLOAD_HANDLERS': { + 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler' + } + } class Https2ProxyTestCase(Http11ProxyTestCase): From a41c205928aa2aa86233de0cfb694b0c7ded2297 Mon Sep 17 00:00:00 2001 From: Jose Galdos Date: Fri, 21 Aug 2020 12:16:37 -0500 Subject: [PATCH 0155/2083] Update httpstatus documentation. --- docs/topics/spider-middleware.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c6cbdba76..28645fd53 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -255,7 +255,8 @@ this:: The ``handle_httpstatus_list`` key of :attr:`Request.meta ` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` -to ``True`` if you want to allow any response code for a request. +to ``True`` if you want to allow any response code for a request, and ``False`` to +disable the effects of the ``handle_httpstatus_all`` key. Keep in mind, however, that it's usually a bad idea to handle non-200 responses, unless you really know what you're doing. From ddc26f3f8fdf766587eda55dfaf60b524eb89e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Sep 2020 11:26:07 +0200 Subject: [PATCH 0156/2083] Revert Travis CI changes --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d9a8e512a..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ dist: xenial branches: only: - master - - http2 # ToDo: Remove once merged into master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: From 6565adc471360c2f542392befb7bb9448e8f171f Mon Sep 17 00:00:00 2001 From: drs-11 Date: Wed, 2 Sep 2020 20:44:26 +0530 Subject: [PATCH 0157/2083] added test case for get_images --- scrapy/pipelines/images.py | 6 +-- tests/test_pipeline_images.py | 75 ++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 47d688c62..09194a0fe 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -18,7 +18,7 @@ from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.misc import md5sum -from scrapy.utils.python import to_bytes, get_func_args +from scrapy.utils.python import get_func_args, to_bytes class NoimagesDrop(DropItem): @@ -136,7 +136,7 @@ class ImagesPipeline(FilesPipeline): if self._deprecated_convert_image: warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.', - category=ScrapyDeprecationWarning, stacklevel=1) + category=ScrapyDeprecationWarning) if self._deprecated_convert_image: image, buf = self.convert_image(orig_image) @@ -156,7 +156,7 @@ class ImagesPipeline(FilesPipeline): if response_body is None: warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ' 'method called without response_body argument.', - category=ScrapyDeprecationWarning, stacklevel=1) + category=ScrapyDeprecationWarning, stacklevel=2) if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 6b71b64bc..8e98b8734 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -5,6 +5,7 @@ import warnings from shutil import rmtree from tempfile import mkdtemp from unittest import skipIf +from unittest.mock import patch import attr from itemadapter import ItemAdapter @@ -12,7 +13,7 @@ from twisted.trial import unittest from scrapy.http import Request, Response from scrapy.item import Field, Item -from scrapy.pipelines.images import ImagesPipeline +from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.settings import Settings from scrapy.utils.python import to_bytes @@ -93,6 +94,78 @@ class ImagesPipelineTestCase(unittest.TestCase): info=object()), 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') + def test_get_images_exception(self): + self.pipeline.min_width = 100 + self.pipeline.min_height = 100 + + _, buf1 = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + _, buf2 = _create_image('JPEG', 'RGB', (150, 50), (0, 0, 0)) + _, buf3 = _create_image('JPEG', 'RGB', (50, 150), (0, 0, 0)) + + resp1 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf1.getvalue()) + resp2 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf2.getvalue()) + resp3 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf3.getvalue()) + req = Request(url="https://dev.mydeco.com/mydeco.gif") + + with self.assertRaises(ImageException): + next(self.pipeline.get_images(response=resp1, request=req, info=object())) + with self.assertRaises(ImageException): + next(self.pipeline.get_images(response=resp2, request=req, info=object())) + with self.assertRaises(ImageException): + next(self.pipeline.get_images(response=resp3, request=req, info=object())) + + def test_get_images_new(self): + self.pipeline.min_width = 0 + self.pipeline.min_height = 0 + self.pipeline.thumbs = {'small': (20, 20)} + + orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + orig_thumb, orig_thumb_buf = _create_image('JPEG', 'RGB', (20, 20), (0, 0, 0)) + resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) + req = Request(url="https://dev.mydeco.com/mydeco.gif") + + get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object()) + + path, new_im, new_buf = next(get_images_gen) + self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(orig_im, new_im) + self.assertEqual(buf.getvalue(), new_buf.getvalue()) + + thumb_path, thumb_img, thumb_buf = next(get_images_gen) + self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(thumb_img, thumb_img) + self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue()) + + def test_get_images_old(self): + self.pipeline.thumbs = {'small': (20, 20)} + orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) + req = Request(url="https://dev.mydeco.com/mydeco.gif") + + def overridden_convert_image(image, size=None): + im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + return im, buf + + with patch.object(self.pipeline, 'convert_image', overridden_convert_image): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object()) + path, new_im, new_buf = next(get_images_gen) + self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(orig_im.mode, new_im.mode) + self.assertEqual(orig_im.getcolors(), new_im.getcolors()) + self.assertEqual(buf.getvalue(), new_buf.getvalue()) + + thumb_path, thumb_img, thumb_buf = next(get_images_gen) + self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(orig_im.mode, thumb_img.mode) + self.assertEqual(orig_im.getcolors(), thumb_img.getcolors()) + self.assertEqual(buf.getvalue(), thumb_buf.getvalue()) + + expected_warning_msg = ('ImagesPipeline.convert_image() method overriden in a deprecated way, ' + 'overriden method does not accept response_body argument.') + self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1) + def test_convert_image_old(self): # tests for old API with warnings.catch_warnings(record=True) as w: From 4d6359df2dfa3d561088faf097f6abe3f850a4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 11 Sep 2020 13:51:05 +0200 Subject: [PATCH 0158/2083] Mark HTTP/2 as experimental --- docs/topics/asyncio.rst | 11 ++++++----- docs/topics/commands.rst | 2 -- docs/topics/settings.rst | 4 ++++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index bfb430d52..c04044e8f 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -4,13 +4,14 @@ asyncio .. versionadded:: 2.0 -Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio -reactor `, you may use :mod:`asyncio` and +Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the +asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy - versions may introduce related changes without a deprecation - period or warning. +.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet + recommended for production environments. Future Scrapy versions + may introduce related changes without a deprecation period or + warning. .. _install-asyncio: diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 7de5e8121..eef6b36ff 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -598,8 +598,6 @@ Example: Register commands via setup.py entry points ------------------------------------------- -.. note:: This is an experimental feature, use with caution. - You can also add Scrapy commands from an external library by adding a ``scrapy.commands`` section in the entry points of the library ``setup.py`` file. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8794e0259..218b23c87 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -678,6 +678,10 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update Scrapy currently does not support HTTP/2 Cleartext (h2c) since none of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_). +.. warning:: HTTP/2 support in Scrapy is experimental, and not yet recommended + for production environments. Future Scrapy versions may introduce + related changes without a deprecation period or warning. + .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. setting:: DOWNLOAD_TIMEOUT From 6e8d20a07a8c1a1f3ad7b3c3d74b7d37f2b47327 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 16 Sep 2020 04:57:07 -0300 Subject: [PATCH 0159/2083] HTTP/2: add some type hints (#4785) --- scrapy/core/downloader/handlers/http2.py | 24 +++++++++++++-------- scrapy/core/http2/agent.py | 27 ++++++++++++++---------- scrapy/core/http2/protocol.py | 15 ++++++++----- scrapy/core/http2/stream.py | 7 +++--- setup.cfg | 6 ------ tests/test_downloader_handlers.py | 19 +++++++++-------- tests/test_http2_client_protocol.py | 2 +- 7 files changed, 56 insertions(+), 44 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 4be888bda..e97c31e90 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,8 +1,9 @@ import warnings from time import time -from typing import Optional +from typing import Optional, Type, TypeVar from urllib.parse import urldefrag +from twisted.internet.base import DelayedCall from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI @@ -10,14 +11,18 @@ from twisted.web.client import URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent +from scrapy.crawler import Crawler from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes +H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler") + + class H2DownloadHandler: - def __init__(self, settings: Settings, crawler=None): + def __init__(self, settings: Settings, crawler: Optional[Crawler] = None): self._crawler = crawler from twisted.internet import reactor @@ -25,14 +30,14 @@ class H2DownloadHandler: self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass: return cls(crawler.settings, crawler) def download_request(self, request: Request, spider: Spider) -> Deferred: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, - crawler=self._crawler + crawler=self._crawler, ) return agent.download_request(request, spider) @@ -47,8 +52,9 @@ class ScrapyH2Agent: def __init__( self, context_factory, pool: H2ConnectionPool, - connect_timeout=10, bind_address: Optional[bytes] = None, - crawler=None + connect_timeout: int = 10, + bind_address: Optional[bytes] = None, + crawler: Optional[Crawler] = None, ) -> None: self._context_factory = context_factory self._connect_timeout = connect_timeout @@ -80,7 +86,7 @@ class ScrapyH2Agent: proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')), connect_timeout=timeout, bind_address=bind_address, - pool=self._pool + pool=self._pool, ) return self._Agent( @@ -88,7 +94,7 @@ class ScrapyH2Agent: context_factory=self._context_factory, connect_timeout=timeout, bind_address=bind_address, - pool=self._pool + pool=self._pool, ) def download_request(self, request: Request, spider: Spider) -> Deferred: @@ -110,7 +116,7 @@ class ScrapyH2Agent: return response @staticmethod - def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl) -> Response: + def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response: if timeout_cl.active(): timeout_cl.cancel() return response diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index d950c6cfb..a142fa210 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,5 +1,5 @@ from collections import deque -from typing import Deque, Dict, List, Tuple, Optional +from typing import Deque, Dict, List, Optional, Tuple from twisted.internet import defer from twisted.internet.base import ReactorBase @@ -95,16 +95,18 @@ class H2ConnectionPool: class H2Agent: def __init__( - self, reactor: ReactorBase, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + self, + reactor: ReactorBase, + pool: H2ConnectionPool, + context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, + bind_address: Optional[bytes] = None, ) -> None: self._reactor = reactor self._pool = pool self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2']) self.endpoint_factory = _StandardEndpointFactory( - self._reactor, self._context_factory, - connect_timeout, bind_address + self._reactor, self._context_factory, connect_timeout, bind_address ) def get_endpoint(self, uri: URI): @@ -132,17 +134,20 @@ class H2Agent: class ScrapyProxyH2Agent(H2Agent): def __init__( - self, reactor: ReactorBase, - proxy_uri: URI, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + self, + reactor: ReactorBase, + proxy_uri: URI, + pool: H2ConnectionPool, + context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, + bind_address: Optional[bytes] = None, ) -> None: super(ScrapyProxyH2Agent, self).__init__( reactor=reactor, pool=pool, context_factory=context_factory, connect_timeout=connect_timeout, - bind_address=bind_address + bind_address=bind_address, ) self._proxy_uri = proxy_uri diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index e32e2b6fe..9d499596c 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -29,6 +29,7 @@ from scrapy.http import Request from scrapy.settings import Settings from scrapy.spiders import Spider + logger = logging.getLogger(__name__) @@ -42,7 +43,11 @@ class InvalidNegotiatedProtocol(H2Error): class RemoteTerminatedConnection(H2Error): - def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], event: ConnectionTerminated): + def __init__( + self, + remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], + event: ConnectionTerminated, + ) -> None: self.remote_ip_address = remote_ip_address self.terminate_event = event @@ -51,7 +56,7 @@ class RemoteTerminatedConnection(H2Error): class MethodNotAllowed405(H2Error): - def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]): + def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None: self.remote_ip_address = remote_ip_address def __str__(self) -> str: @@ -220,13 +225,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.conn.initiate_connection() self._write_to_transport() - def _lose_connection_with_error(self, errors: List[BaseException]): + def _lose_connection_with_error(self, errors: List[BaseException]) -> None: """Helper function to lose the connection with the error sent as a reason""" self._conn_lost_errors += errors self.transport.loseConnection() - def handshakeCompleted(self): + def handshakeCompleted(self) -> None: """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" negotiated_protocol = self.transport.negotiatedProtocol @@ -263,7 +268,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): finally: self._write_to_transport() - def timeoutConnection(self): + def timeoutConnection(self) -> None: """Called when the connection times out. We lose the connection with TimeoutError""" diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index ef90773b6..3ae2e8db8 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -19,6 +19,7 @@ from scrapy.responsetypes import responsetypes if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol + logger = logging.getLogger(__name__) @@ -27,7 +28,7 @@ class InactiveStreamClosed(ConnectionClosed): of the stream. This happens when a stream is waiting for other streams to close and connection is lost.""" - def __init__(self, request: Request): + def __init__(self, request: Request) -> None: self.request = request def __str__(self) -> str: @@ -139,7 +140,7 @@ class Stream: 'headers': Headers({}), } - def _cancel(_): + def _cancel(_) -> None: # Close this stream as gracefully as possible # If the associated request is initiated we reset this stream # else we directly call close() method @@ -360,7 +361,7 @@ class Stream: self, reason: StreamCloseReason, errors: Optional[List[BaseException]] = None, - from_protocol: bool = False + from_protocol: bool = False, ) -> None: """Based on the reason sent we will handle each case. """ diff --git a/setup.cfg b/setup.cfg index 94c1e7b89..8101443e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -82,9 +82,6 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_downloader_handlers_http2] -ignore_errors = True - [mypy-tests.test_engine] ignore_errors = True @@ -94,9 +91,6 @@ ignore_errors = True [mypy-tests.test_http_request] ignore_errors = True -[mypy-tests.test_http2_client_protocol] -ignore_errors = True - [mypy-tests.test_linkextractors] ignore_errors = True diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 20e31f7f7..5b4a2d270 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -2,6 +2,7 @@ import contextlib import os import shutil import tempfile +from typing import Optional, Type from unittest import mock from testfixtures import LogCapture @@ -206,7 +207,7 @@ class LargeChunkedFileResource(resource.Resource): class HttpTestCase(unittest.TestCase): scheme = 'http' - download_handler_cls = HTTPDownloadHandler + download_handler_cls: Type = HTTPDownloadHandler # only used for HTTPS tests keyfile = 'keys/localhost.key' @@ -365,7 +366,7 @@ class HttpTestCase(unittest.TestCase): class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" - download_handler_cls = HTTP10DownloadHandler + download_handler_cls: Type = HTTP10DownloadHandler class Https10TestCase(Http10TestCase): @@ -374,7 +375,7 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler def test_download_without_maxsize_limit(self): request = Request(self.getURL('file')) @@ -561,7 +562,7 @@ class Https11InvalidDNSPattern(Https11TestCase): class Https11CustomCiphers(unittest.TestCase): scheme = 'https' - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler keyfile = 'keys/localhost.key' certfile = 'keys/localhost.crt' @@ -601,7 +602,7 @@ class Https11CustomCiphers(unittest.TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" - settings_dict = None + settings_dict: Optional[dict] = None def setUp(self): self.mockserver = MockServer() @@ -668,7 +669,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): - download_handler_cls = HTTPDownloadHandler + download_handler_cls: Type = HTTPDownloadHandler expected_http_proxy_request_body = b'http://example.com' def setUp(self): @@ -721,14 +722,14 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): - download_handler_cls = HTTP10DownloadHandler + download_handler_cls: Type = HTTP10DownloadHandler def test_download_with_proxy_https_noconnect(self): raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler') class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -776,7 +777,7 @@ class S3AnonTestCase(unittest.TestCase): class S3TestCase(unittest.TestCase): - download_handler_cls = S3DownloadHandler + download_handler_cls: Type = S3DownloadHandler # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 4926ada14..d9ab553f0 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -45,7 +45,7 @@ def make_html_body(val): class DummySpider(Spider): name = 'dummy' - start_urls = [] + start_urls: list = [] def parse(self, response): print(response) From 70c82d33c00538228314dd6cef0253b70f8627e8 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Sun, 20 Sep 2020 16:24:05 +0300 Subject: [PATCH 0160/2083] httpcompression stats added (#4797) --- scrapy/downloadermiddlewares/httpcompression.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 727c41466..b34f76f21 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -37,6 +37,8 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) + spider.crawler.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + spider.crawler.stats.inc_value('httpcompression/response_count', spider=spider) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) From c22e810658b227095ea516ed61e51e6be41068ce Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Tue, 22 Sep 2020 07:47:37 +0300 Subject: [PATCH 0161/2083] httocompression tests added --- .../downloadermiddlewares/httpcompression.py | 9 ++++--- ...st_downloadermiddleware_httpcompression.py | 27 +++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index b34f76f21..ca80e9444 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -18,11 +18,14 @@ except ImportError: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" + def __init__(self, stats): + self.stats = stats + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): raise NotConfigured - return cls() + return cls(crawler.stats) def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', @@ -37,8 +40,8 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) - spider.crawler.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) - spider.crawler.stats.inc_value('httpcompression/response_count', spider=spider) + self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + self.stats.inc_value('httpcompression/response_count', spider=spider) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a806f55ce..998749c2e 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -8,6 +8,7 @@ from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip +from scrapy.utils.test import get_crawler from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -26,8 +27,10 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - self.spider = Spider('foo') - self.mw = HttpCompressionMiddleware() + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('scrapytest.org') + self.mw = HttpCompressionMiddleware(self.crawler.stats) + self.crawler.stats.open_spider(self.spider) def _getresponse(self, coding): if coding not in FORMAT: @@ -50,6 +53,13 @@ class HttpCompressionTest(TestCase): response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'}) return response + def assertStatsEqual(self, key, value): + self.assertEqual( + self.crawler.stats.get_value(key, spider=self.spider), + value, + str(self.crawler.stats.get_stats(self.spider)) + ) + def test_process_request(self): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers @@ -66,6 +76,7 @@ class HttpCompressionTest(TestCase): assert newresponse is not response assert newresponse.body.startswith(b' Date: Wed, 30 Sep 2020 14:17:30 +0200 Subject: [PATCH 0162/2083] Mention contributors in the README --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index a8f2ba52b..19faa9a87 100644 --- a/README.rst +++ b/README.rst @@ -34,9 +34,16 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. +Scrapy has contributions from `many users`_ (thanks everyone!) and is sponsored +by `Scrapinghub Ltd`_. + +.. _many users: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapinghub Ltd: https://www.scrapinghub.com/ + Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. + Requirements ============ From 0c24cdb2573958cc0fb9127c6f195d3174640ff4 Mon Sep 17 00:00:00 2001 From: D R Siddhartha Date: Sun, 4 Oct 2020 02:09:21 +0530 Subject: [PATCH 0163/2083] Improved warning messages a little --- scrapy/pipelines/images.py | 4 ++-- tests/test_pipeline_images.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index d3254bc20..48e8a7b83 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -135,7 +135,7 @@ class ImagesPipeline(FilesPipeline): if self._deprecated_convert_image is None: self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image) if self._deprecated_convert_image: - warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' + warnings.warn(f'{self.__class__.__name__}.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning) @@ -155,7 +155,7 @@ class ImagesPipeline(FilesPipeline): def convert_image(self, image, size=None, response_body=None): if response_body is None: - warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ' + warnings.warn(f'{self.__class__.__name__}.convert_image() method called in a deprecated way, ' 'method called without response_body argument.', category=ScrapyDeprecationWarning, stacklevel=2) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 380c775c4..0a294db80 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -162,7 +162,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(orig_im.getcolors(), thumb_img.getcolors()) self.assertEqual(buf.getvalue(), thumb_buf.getvalue()) - expected_warning_msg = ('ImagesPipeline.convert_image() method overriden in a deprecated way, ' + expected_warning_msg = ('.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.') self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1) @@ -199,7 +199,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # ensure that we recieved deprecation warnings - expected_warning_msg = 'ImagesPipeline.convert_image() method called in a deprecated way' + expected_warning_msg = '.convert_image() method called in a deprecated way' self.assertTrue(len([warning for warning in w if expected_warning_msg in str(warning.message)]) == 4) def test_convert_image_new(self): From 6050604f626a5ee38239ef1eff44d0c867469a3b Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Tue, 6 Oct 2020 18:59:57 +0300 Subject: [PATCH 0164/2083] httocompression/response_bytes tests added --- tests/test_downloadermiddleware_httpcompression.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 998749c2e..144fd3378 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -77,6 +77,7 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b' Date: Tue, 13 Oct 2020 18:35:06 +0300 Subject: [PATCH 0165/2083] __init__ stats parameter - optional, stats==None - covered. --- scrapy/downloadermiddlewares/httpcompression.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index ca80e9444..8980e9ca4 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -18,7 +18,7 @@ except ImportError: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, stats): + def __init__(self, stats=None): self.stats = stats @classmethod @@ -40,8 +40,9 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) - self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) - self.stats.inc_value('httpcompression/response_count', spider=spider) + if self.stats: + self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + self.stats.inc_value('httpcompression/response_count', spider=spider) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) From d32d0d27393ce55490c44d9fb039130320461865 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Tue, 13 Oct 2020 18:36:41 +0300 Subject: [PATCH 0166/2083] testcase added for HttpCompressionMiddleware with no stats --- tests/test_downloadermiddleware_httpcompression.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 144fd3378..5f8e76e0a 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -29,7 +29,7 @@ class HttpCompressionTest(TestCase): def setUp(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider('scrapytest.org') - self.mw = HttpCompressionMiddleware(self.crawler.stats) + self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) self.crawler.stats.open_spider(self.spider) def _getresponse(self, coding): @@ -79,6 +79,18 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual('httpcompression/response_count', 1) self.assertStatsEqual('httpcompression/response_bytes', 74837) + def test_process_response_gzip_no_stats(self): + mw = HttpCompressionMiddleware() + response = self._getresponse('gzip') + request = response.request + + self.assertEqual(response.headers['Content-Encoding'], b'gzip') + newresponse = mw.process_response(request, response, self.spider) + self.assertEqual(mw.stats, None) + assert newresponse is not response + assert newresponse.body.startswith(b' Date: Tue, 13 Oct 2020 18:41:58 +0300 Subject: [PATCH 0167/2083] testcase added for COMPRESSION_ENABLED setting --- ...st_downloadermiddleware_httpcompression.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5f8e76e0a..c2bcbd63c 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -6,6 +6,7 @@ from gzip import GzipFile from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS +from scrapy.exceptions import NotConfigured from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler @@ -60,6 +61,27 @@ class HttpCompressionTest(TestCase): str(self.crawler.stats.get_stats(self.spider)) ) + def test_setting_false_compression_enabled(self): + self.assertRaises( + NotConfigured, + HttpCompressionMiddleware.from_crawler, + get_crawler(settings_dict={'COMPRESSION_ENABLED': False}) + ) + + def test_setting_default_compression_enabled(self): + self.assertIsInstance( + HttpCompressionMiddleware.from_crawler(get_crawler()), + HttpCompressionMiddleware + ) + + def test_setting_true_compression_enabled(self): + self.assertIsInstance( + HttpCompressionMiddleware.from_crawler( + get_crawler(settings_dict={'COMPRESSION_ENABLED': True}) + ), + HttpCompressionMiddleware + ) + def test_process_request(self): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers From e9c3188189cffc965797b1b77fc5dc5cfa06b5cb Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 30 Oct 2020 21:23:29 +0200 Subject: [PATCH 0168/2083] Update scrapy/downloadermiddlewares/httpcompression.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/downloadermiddlewares/httpcompression.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 8980e9ca4..87f744956 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -25,7 +25,12 @@ class HttpCompressionMiddleware: def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): raise NotConfigured - return cls(crawler.stats) + try: + return cls(stats=crawler.stats) + except TypeError: + result = cls() + result.stats = crawler.stats + return result def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', From a3e53027ec35498dcb931404e02689877da6aeb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 14:16:26 +0100 Subject: [PATCH 0169/2083] Test HttpCompressionMiddleware subclasses with custom, parameterless __init__ --- .../downloadermiddlewares/httpcompression.py | 12 +++++-- ...st_downloadermiddleware_httpcompression.py | 36 ++++++++++++++++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 87f744956..1808154d2 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,9 +1,11 @@ +import warnings import zlib -from scrapy.utils.gz import gunzip +from scrapy.exceptions import NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.exceptions import NotConfigured +from scrapy.utils.deprecate import ScrapyDeprecationWarning +from scrapy.utils.gz import gunzip ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] @@ -28,6 +30,12 @@ class HttpCompressionMiddleware: try: return cls(stats=crawler.stats) except TypeError: + warnings.warn( + "HttpCompressionMiddleware subclasses must either modify " + "their '__init__' method to support a 'stats' parameter or " + "reimplement the 'from_crawler' method.", + ScrapyDeprecationWarning, + ) result = cls() result.stats = crawler.stats return result diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index c2bcbd63c..2ee87aa67 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,12 +1,13 @@ -from io import BytesIO -from unittest import TestCase, SkipTest -from os.path import join from gzip import GzipFile +from io import BytesIO +from os.path import join +from unittest import TestCase, SkipTest +from warnings import catch_warnings from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler @@ -321,3 +322,30 @@ class HttpCompressionTest(TestCase): self.assertEqual(response.body, b'') self.assertStatsEqual('httpcompression/response_count', None) self.assertStatsEqual('httpcompression/response_bytes', None) + + +class HttpCompressionSubclassTest(TestCase): + + def test_init_missing_stats(self): + class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware): + + def __init__(self): + super().__init__() + + crawler = get_crawler(Spider) + with catch_warnings(record=True) as caught_warnings: + instance = HttpCompressionMiddlewareSubclass.from_crawler(crawler) + messages = tuple( + str(warning.message) for warning in caught_warnings + if warning.category is ScrapyDeprecationWarning + ) + self.assertEqual( + messages, + ( + ( + "HttpCompressionMiddleware subclasses must either modify " + "their '__init__' method to support a 'stats' parameter " + "or reimplement the 'from_crawler' method." + ), + ) + ) From ee98771fa72aa5e109292af5d734e5c49bae64e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 16:42:32 +0100 Subject: [PATCH 0170/2083] Remove unused variable --- tests/test_downloadermiddleware_httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 2ee87aa67..38d8534ca 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -334,7 +334,7 @@ class HttpCompressionSubclassTest(TestCase): crawler = get_crawler(Spider) with catch_warnings(record=True) as caught_warnings: - instance = HttpCompressionMiddlewareSubclass.from_crawler(crawler) + HttpCompressionMiddlewareSubclass.from_crawler(crawler) messages = tuple( str(warning.message) for warning in caught_warnings if warning.category is ScrapyDeprecationWarning From bde96a5ad98caed74cbaaeaf1dfe4b215093c03f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Nov 2020 16:42:44 +0100 Subject: [PATCH 0171/2083] Ignore server-initiated events --- scrapy/core/http2/protocol.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 9d499596c..67a86bd62 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -349,10 +349,16 @@ class H2ClientProtocol(Protocol, TimeoutMixin): ]) def data_received(self, event: DataReceived) -> None: - self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) + try: + self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) + except KeyError: + logger.debug(f'Ignoring server-initiated event {event}') def response_received(self, event: ResponseReceived) -> None: - self.streams[event.stream_id].receive_headers(event.headers) + try: + self.streams[event.stream_id].receive_headers(event.headers) + except KeyError: + logger.debug(f'Ignoring server-initiated event {event}') def settings_acknowledged(self, event: SettingsAcknowledged) -> None: self.metadata['settings_acknowledged'] = True @@ -365,12 +371,20 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) def stream_ended(self, event: StreamEnded) -> None: - stream = self.pop_stream(event.stream_id) - stream.close(StreamCloseReason.ENDED, from_protocol=True) + try: + stream = self.pop_stream(event.stream_id) + except KeyError: + logger.debug(f'Ignoring server-initiated event {event}') + else: + stream.close(StreamCloseReason.ENDED, from_protocol=True) def stream_reset(self, event: StreamReset) -> None: - stream = self.pop_stream(event.stream_id) - stream.close(StreamCloseReason.RESET, from_protocol=True) + try: + stream = self.pop_stream(event.stream_id) + except KeyError: + logger.debug(f'Ignoring server-initiated event {event}') + else: + stream.close(StreamCloseReason.RESET, from_protocol=True) def window_updated(self, event: WindowUpdated) -> None: if event.stream_id != 0: From 08f5ed712f4fdafec122f887c3ef06629f35ee0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Nov 2020 17:38:18 +0100 Subject: [PATCH 0172/2083] Fix memory issue due to unexpectedly large server frames --- scrapy/core/http2/protocol.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 67a86bd62..d8d0974b8 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -13,7 +13,7 @@ from h2.events import ( SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived, WindowUpdated ) -from h2.exceptions import H2Error +from h2.exceptions import FrameTooLargeError, H2Error from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory @@ -261,6 +261,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin): events = self.conn.receive_data(data) self._handle_events(events) except H2Error as e: + if isinstance(e, FrameTooLargeError): + # hyper-h2 does not drop the connection in this scenario, we + # need to abort the connection manually. + self._conn_lost_errors += [e] + self.transport.abortConnection() + return + # Save this error as ultimately the connection will be dropped # internally by hyper-h2. Saved error will be passed to all the streams # closed with the connection. From a752fa072e1acbb233191dbf04728db1fd6712a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 23 Nov 2020 22:58:54 +0100 Subject: [PATCH 0173/2083] Implement retry request functions and mixin --- scrapy/downloadermiddlewares/retry.py | 134 ++++++++++++++++++++------ 1 file changed, 106 insertions(+), 28 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 51fe59254..023ab7d60 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -31,6 +31,105 @@ from scrapy.utils.python import global_object_name logger = logging.getLogger(__name__) +def get_retry_request( + request, + *, + reason, + spider, + max_retry_times=None, + priority_adjust=None, +): + settings = spider.crawler.settings + stats = spider.crawler.stats + retry_times = request.meta.get('retry_times', 0) + 1 + request_max_retry_times = request.meta.get( + 'max_retry_times', + max_retry_times, + ) + if request_max_retry_times is None: + request_max_retry_times = settings.getint('RETRY_TIMES') + if retry_times <= request_max_retry_times: + logger.debug( + "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider} + ) + new_request = request.copy() + new_request.meta['retry_times'] = retry_times + new_request.dont_filter = True + if priority_adjust is None: + priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') + new_request.priority = request.priority + priority_adjust + + if isinstance(reason, Exception): + reason = global_object_name(reason.__class__) + + stats.inc_value('retry/count') + stats.inc_value(f'retry/reason_count/{reason}') + return new_request + else: + stats.inc_value('retry/max_reached') + logger.error("Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider}) + return None + + +def retry_request( + request, + *, + reason, + spider, + max_retry_times=None, + priority_adjust=None, +): + new_request = get_retry_request( + request, + reason=reason, + spider=spider, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, + ) + if new_request: + return [new_request] + return [] + + +class RetrySpiderMixin: + + def get_retry_request( + self, + request, + *, + reason, + max_retry_times=None, + priority_adjust=None, + ): + return get_retry_request( + request, + reason=reason, + spider=self, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, + ) + + def retry_request( + self, + request, + *, + reason, + max_retry_times=None, + priority_adjust=None, + ): + return retry_request( + request, + reason=reason, + spider=self, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, + ) + + class RetryMiddleware: # IOError is raised by the HttpCompression middleware when trying to @@ -67,31 +166,10 @@ class RetryMiddleware: return self._retry(request, exception, spider) def _retry(self, request, reason, spider): - retries = request.meta.get('retry_times', 0) + 1 - - retry_times = self.max_retry_times - - if 'max_retry_times' in request.meta: - retry_times = request.meta['max_retry_times'] - - stats = spider.crawler.stats - if retries <= retry_times: - logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s", - {'request': request, 'retries': retries, 'reason': reason}, - extra={'spider': spider}) - retryreq = request.copy() - retryreq.meta['retry_times'] = retries - retryreq.dont_filter = True - retryreq.priority = request.priority + self.priority_adjust - - if isinstance(reason, Exception): - reason = global_object_name(reason.__class__) - - stats.inc_value('retry/count') - stats.inc_value(f'retry/reason_count/{reason}') - return retryreq - else: - stats.inc_value('retry/max_reached') - logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s", - {'request': request, 'retries': retries, 'reason': reason}, - extra={'spider': spider}) + return get_retry_request( + request, + reason=reason, + spider=spider, + max_retry_times=self.max_retry_times, + priority_adjust=self.priority_adjust, + ) From ef09e0d10fc950ac308a159988be6b50e87bd906 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 19 Nov 2020 10:35:49 -0300 Subject: [PATCH 0174/2083] Some type hints --- scrapy/__init__.py | 2 +- scrapy/commands/__init__.py | 4 +- scrapy/commands/parse.py | 6 +- scrapy/contracts/__init__.py | 127 ++++++++++++++-------------- scrapy/http/cookies.py | 8 +- scrapy/item.py | 3 +- scrapy/mail.py | 7 +- scrapy/spidermiddlewares/referer.py | 30 ++++--- scrapy/utils/httpobj.py | 11 ++- scrapy/utils/request.py | 28 +++--- scrapy/utils/response.py | 31 ++++--- scrapy/utils/trackref.py | 9 +- setup.cfg | 36 -------- 13 files changed, 154 insertions(+), 148 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 4326ca4aa..8a8065bf2 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -22,7 +22,7 @@ __all__ = [ # Scrapy and Twisted versions -__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip() +__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip() version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.')) twisted_version = (_txv.major, _txv.minor, _txv.micro) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 23ccffcd9..6e77551c6 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -3,6 +3,8 @@ Base class for Scrapy commands """ import os from optparse import OptionGroup +from typing import Any, Dict + from twisted.python import failure from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli @@ -15,7 +17,7 @@ class ScrapyCommand: crawler_process = None # default settings to be used for this command instead of global defaults - default_settings = {} + default_settings: Dict[str, Any] = {} exitcode = 0 diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 83ee074da..52118db1b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,5 +1,6 @@ import json import logging +from typing import Dict from itemadapter import is_item, ItemAdapter from w3lib.url import is_url @@ -10,6 +11,7 @@ from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output, spidercls_for_request from scrapy.exceptions import UsageError + logger = logging.getLogger(__name__) @@ -17,8 +19,8 @@ class Command(BaseRunSpiderCommand): requires_project = True spider = None - items = {} - requests = {} + items: Dict[int, list] = {} + requests: Dict[int, list] = {} first_response = None diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index db0a56e56..b47e55092 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -1,16 +1,77 @@ -import sys import re +import sys from functools import wraps from inspect import getmembers +from typing import Dict from unittest import TestCase from scrapy.http import Request -from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_spec +from scrapy.utils.spider import iterate_spider_output + + +class Contract: + """ Abstract class for contracts """ + request_cls = None + + def __init__(self, method, *args): + self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') + self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') + self.args = args + + def add_pre_hook(self, request, results): + if hasattr(self, 'pre_process'): + cb = request.callback + + @wraps(cb) + def wrapper(response, **cb_kwargs): + try: + results.startTest(self.testcase_pre) + self.pre_process(response) + results.stopTest(self.testcase_pre) + except AssertionError: + results.addFailure(self.testcase_pre, sys.exc_info()) + except Exception: + results.addError(self.testcase_pre, sys.exc_info()) + else: + results.addSuccess(self.testcase_pre) + finally: + return list(iterate_spider_output(cb(response, **cb_kwargs))) + + request.callback = wrapper + + return request + + def add_post_hook(self, request, results): + if hasattr(self, 'post_process'): + cb = request.callback + + @wraps(cb) + def wrapper(response, **cb_kwargs): + output = list(iterate_spider_output(cb(response, **cb_kwargs))) + try: + results.startTest(self.testcase_post) + self.post_process(output) + results.stopTest(self.testcase_post) + except AssertionError: + results.addFailure(self.testcase_post, sys.exc_info()) + except Exception: + results.addError(self.testcase_post, sys.exc_info()) + else: + results.addSuccess(self.testcase_post) + finally: + return output + + request.callback = wrapper + + return request + + def adjust_request_args(self, args): + return args class ContractsManager: - contracts = {} + contracts: Dict[str, Contract] = {} def __init__(self, contracts): for contract in contracts: @@ -107,66 +168,6 @@ class ContractsManager: request.errback = eb_wrapper -class Contract: - """ Abstract class for contracts """ - request_cls = None - - def __init__(self, method, *args): - self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') - self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') - self.args = args - - def add_pre_hook(self, request, results): - if hasattr(self, 'pre_process'): - cb = request.callback - - @wraps(cb) - def wrapper(response, **cb_kwargs): - try: - results.startTest(self.testcase_pre) - self.pre_process(response) - results.stopTest(self.testcase_pre) - except AssertionError: - results.addFailure(self.testcase_pre, sys.exc_info()) - except Exception: - results.addError(self.testcase_pre, sys.exc_info()) - else: - results.addSuccess(self.testcase_pre) - finally: - return list(iterate_spider_output(cb(response, **cb_kwargs))) - - request.callback = wrapper - - return request - - def add_post_hook(self, request, results): - if hasattr(self, 'post_process'): - cb = request.callback - - @wraps(cb) - def wrapper(response, **cb_kwargs): - output = list(iterate_spider_output(cb(response, **cb_kwargs))) - try: - results.startTest(self.testcase_post) - self.post_process(output) - results.stopTest(self.testcase_post) - except AssertionError: - results.addFailure(self.testcase_post, sys.exc_info()) - except Exception: - results.addError(self.testcase_post, sys.exc_info()) - else: - results.addSuccess(self.testcase_post) - finally: - return output - - request.callback = wrapper - - return request - - def adjust_request_args(self, args): - return args - - def _create_testcase(method, desc): spider = method.__self__.name diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 0c97e6999..bf4ae7b45 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,10 +1,16 @@ +import re import time -from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE +from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +# Defined in the http.cookiejar module, but undocumented: +# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 +IPV4_RE = re.compile(r"\.\d+$", re.ASCII) + + class CookieJar: def __init__(self, policy=None, check_expired_frequency=10000): self.policy = policy or DefaultCookiePolicy() diff --git a/scrapy/item.py b/scrapy/item.py index af3849302..2ccd7ad18 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -8,6 +8,7 @@ from abc import ABCMeta from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat +from typing import Dict from warnings import warn from scrapy.utils.deprecate import ScrapyDeprecationWarning @@ -75,7 +76,7 @@ class ItemMeta(_BaseItemMeta): class DictItem(MutableMapping, BaseItem): - fields = {} + fields: Dict[str, Field] = {} def __new__(cls, *args, **kwargs): if issubclass(cls, DictItem) and not issubclass(cls, Item): diff --git a/scrapy/mail.py b/scrapy/mail.py index 7d7a2c435..2a25ccd44 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -9,7 +9,7 @@ from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText -from email.utils import COMMASPACE, formatdate +from email.utils import formatdate from io import BytesIO from twisted.internet import defer, ssl @@ -21,6 +21,11 @@ from scrapy.utils.python import to_bytes logger = logging.getLogger(__name__) +# Defined in the email.utils module, but undocumented: +# https://github.com/python/cpython/blob/v3.9.0/Lib/email/utils.py#L42 +COMMASPACE = ", " + + def _to_bytes_or_none(text): if text is None: return None diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index f81041376..608c0eea5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -3,15 +3,16 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ import warnings +from typing import Tuple from urllib.parse import urlparse from w3lib.url import safe_url_string -from scrapy.http import Request, Response -from scrapy.exceptions import NotConfigured from scrapy import signals -from scrapy.utils.python import to_unicode +from scrapy.exceptions import NotConfigured +from scrapy.http import Request, Response from scrapy.utils.misc import load_object +from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url @@ -30,7 +31,8 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default" class ReferrerPolicy: - NOREFERRER_SCHEMES = LOCAL_SCHEMES + NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + name: str def referrer(self, response_url, request_url): raise NotImplementedError() @@ -88,7 +90,7 @@ class NoReferrerPolicy(ReferrerPolicy): is to be sent along with requests made from a particular request client to any origin. The header will be omitted entirely. """ - name = POLICY_NO_REFERRER + name: str = POLICY_NO_REFERRER def referrer(self, response_url, request_url): return None @@ -108,7 +110,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): This is a user agent's default behavior, if no policy is otherwise specified. """ - name = POLICY_NO_REFERRER_WHEN_DOWNGRADE + name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response_url, request_url): if not self.tls_protected(response_url) or self.tls_protected(request_url): @@ -125,7 +127,7 @@ class SameOriginPolicy(ReferrerPolicy): Cross-origin requests, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_SAME_ORIGIN + name: str = POLICY_SAME_ORIGIN def referrer(self, response_url, request_url): if self.origin(response_url) == self.origin(request_url): @@ -141,7 +143,7 @@ class OriginPolicy(ReferrerPolicy): when making both same-origin requests and cross-origin requests from a particular request client. """ - name = POLICY_ORIGIN + name: str = POLICY_ORIGIN def referrer(self, response_url, request_url): return self.origin_referrer(response_url) @@ -160,7 +162,7 @@ class StrictOriginPolicy(ReferrerPolicy): on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_STRICT_ORIGIN + name: str = POLICY_STRICT_ORIGIN def referrer(self, response_url, request_url): if ( @@ -181,7 +183,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): is sent as referrer information when making cross-origin requests from a particular request client. """ - name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN + name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response_url, request_url): origin = self.origin(response_url) @@ -208,7 +210,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN + name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response_url, request_url): origin = self.origin(response_url) @@ -234,7 +236,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): to insecure origins. Carefully consider the impact of setting such a policy for potentially sensitive documents. """ - name = POLICY_UNSAFE_URL + name: str = POLICY_UNSAFE_URL def referrer(self, response_url, request_url): return self.stripped_referrer(response_url) @@ -246,8 +248,8 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): with the addition that "Referer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') - name = POLICY_SCRAPY_DEFAULT + NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ('file', 's3') + name: str = POLICY_SCRAPY_DEFAULT _policy_classes = {p.name: p for p in ( diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index c8d4391b1..a90f1d278 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,13 +1,16 @@ """Helper functions for scrapy.http objects (Request, Response)""" -import weakref -from urllib.parse import urlparse +from typing import Union +from urllib.parse import urlparse, ParseResult +from weakref import WeakKeyDictionary + +from scrapy.http import Request, Response -_urlparse_cache = weakref.WeakKeyDictionary() +_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary() -def urlparse_cached(request_or_response): +def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult: """Return urlparse.urlparse caching the result, where the argument can be a Request or Response object """ diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 12c03d78e..66736b42f 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -4,20 +4,27 @@ scrapy.http.Request objects """ import hashlib -import weakref +from typing import Dict, Iterable, Optional, Tuple, Union from urllib.parse import urlunparse +from weakref import WeakKeyDictionary from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url +from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode -_fingerprint_cache = weakref.WeakKeyDictionary() +_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" +_fingerprint_cache = WeakKeyDictionary() -def request_fingerprint(request, include_headers=None, keep_fragments=False): +def request_fingerprint( + request: Request, + include_headers: Optional[Iterable[Union[bytes, str]]] = None, + keep_fragments: bool = False, +): """ Return the request fingerprint. @@ -49,17 +56,18 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): (for instance when handling requests with a headless browser). """ + headers: Optional[Tuple[bytes, ...]] = None if include_headers: - include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) + headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) cache = _fingerprint_cache.setdefault(request, {}) - cache_key = (include_headers, keep_fragments) + cache_key = (headers, keep_fragments) if cache_key not in cache: fp = hashlib.sha1() fp.update(to_bytes(request.method)) fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) fp.update(request.body or b'') - if include_headers: - for hdr in include_headers: + if headers: + for hdr in headers: if hdr in request.headers: fp.update(hdr) for v in request.headers.getlist(hdr): @@ -68,14 +76,14 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): return cache[cache_key] -def request_authenticate(request, username, password): +def request_authenticate(request: Request, username: str, password: str) -> None: """Autenticate the given request (in place) using the HTTP basic access authentication mechanism (RFC 2617) and the given username and password """ request.headers['Authorization'] = basic_auth_header(username, password) -def request_httprepr(request): +def request_httprepr(request: Request) -> bytes: """Return the raw HTTP representation (as bytes) of the given request. This is provided only for reference since it's not the actual stream of bytes that will be send when performing the request (that's controlled @@ -92,7 +100,7 @@ def request_httprepr(request): return s -def referer_str(request): +def referer_str(request: Request) -> Optional[str]: """ Return Referer HTTP header suitable for logging. """ referrer = request.headers.get('Referer') if referrer is None: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 99b089b6f..b3ef7b463 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,19 +3,23 @@ This module provides some useful functions for working with scrapy.http.Response objects """ import os -import weakref import webbrowser import tempfile +from typing import Any, Callable, Iterable, Optional, Tuple, Union +from weakref import WeakKeyDictionary + +import scrapy +from scrapy.http.response import Response from twisted.web import http from scrapy.utils.python import to_bytes, to_unicode from w3lib import html -_baseurl_cache = weakref.WeakKeyDictionary() +_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() -def get_base_url(response): +def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] @@ -23,10 +27,13 @@ def get_base_url(response): return _baseurl_cache[response] -_metaref_cache = weakref.WeakKeyDictionary() +_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary() -def get_meta_refresh(response, ignore_tags=('script', 'noscript')): +def get_meta_refresh( + response: "scrapy.http.response.text.TextResponse", + ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'), +) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] @@ -35,14 +42,15 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')): return _metaref_cache[response] -def response_status_message(status): +def response_status_message(status: Union[bytes, float, int, str]) -> str: """Return status code plus status text descriptive message """ - message = http.RESPONSES.get(int(status), "Unknown Status") - return f'{status} {to_unicode(message)}' + status_int = int(status) + message = http.RESPONSES.get(status_int, "Unknown Status") + return f'{status_int} {to_unicode(message)}' -def response_httprepr(response): +def response_httprepr(response: Response) -> bytes: """Return raw HTTP representation (as bytes) of the given response. This is provided only for reference, since it's not the exact stream of bytes that was received (that's not exposed by Twisted). @@ -60,7 +68,10 @@ def response_httprepr(response): return b"".join(values) -def open_in_browser(response, _openfunc=webbrowser.open): +def open_in_browser( + response: Union["scrapy.http.response.html.HtmlResponse", "scrapy.http.response.text.TextResponse"], + _openfunc: Callable[[str], Any] = webbrowser.open, +) -> Any: """Open the given response in a local web browser, populating the tag for external links to work """ diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 3e40acd69..b0c6a2424 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -9,14 +9,15 @@ and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). """ -import weakref -from time import time -from operator import itemgetter from collections import defaultdict +from operator import itemgetter +from time import time +from typing import DefaultDict +from weakref import WeakKeyDictionary NoneType = type(None) -live_refs = defaultdict(weakref.WeakKeyDictionary) +live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) class object_ref: diff --git a/setup.cfg b/setup.cfg index 0c9f6b963..89b4ec57e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,54 +10,18 @@ follow_imports = skip # FIXME: remove the following sections once the issues are solved -[mypy-scrapy] -ignore_errors = True - -[mypy-scrapy.commands] -ignore_errors = True - -[mypy-scrapy.commands.parse] -ignore_errors = True - [mypy-scrapy.downloadermiddlewares.httpproxy] ignore_errors = True -[mypy-scrapy.contracts] -ignore_errors = True - [mypy-scrapy.interfaces] ignore_errors = True -[mypy-scrapy.item] -ignore_errors = True - -[mypy-scrapy.http.cookies] -ignore_errors = True - -[mypy-scrapy.mail] -ignore_errors = True - [mypy-scrapy.pipelines.images] ignore_errors = True [mypy-scrapy.settings.default_settings] ignore_errors = True -[mypy-scrapy.spidermiddlewares.referer] -ignore_errors = True - -[mypy-scrapy.utils.httpobj] -ignore_errors = True - -[mypy-scrapy.utils.request] -ignore_errors = True - -[mypy-scrapy.utils.response] -ignore_errors = True - -[mypy-scrapy.utils.trackref] -ignore_errors = True - [mypy-tests.mocks.dummydbm] ignore_errors = True From e494a3f73318db76d7c56e65bc632cd5875b8825 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 31 Dec 2020 11:50:15 -0300 Subject: [PATCH 0175/2083] protocol attribute for h2 responses --- docs/topics/request-response.rst | 2 +- scrapy/core/http2/stream.py | 2 ++ tests/test_downloader_handlers_http2.py | 7 +++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 98906992d..48f7f4a87 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -694,7 +694,7 @@ Response objects :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` :param protocol: The protocol that was used to download the response. - For instance: "HTTP/1.0", "HTTP/1.1" + For instance: "HTTP/1.0", "HTTP/1.1", "h2" :type protocol: :class:`str` .. versionadded:: 2.0.0 diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 3ae2e8db8..e345ca79a 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -15,6 +15,7 @@ from twisted.web.client import ResponseFailed from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes +from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -458,6 +459,7 @@ class Stream: request=self._request, certificate=self._protocol.metadata['certificate'], ip_address=self._protocol.metadata['ip_address'], + protocol=to_unicode(self._protocol.transport.negotiatedProtocol), ) self._deferred_response.callback(response) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 253646040..8f7f7aee0 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -22,6 +22,13 @@ class Https2TestCase(Https11TestCase): download_handler_cls = H2DownloadHandler HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "h2") + return d + @defer.inlineCallbacks def test_download_with_maxsize_very_large_file(self): with mock.patch('scrapy.core.http2.stream.logger') as logger: From 6e7ae789f9d0b62ef5e07d16fab062ef7bdfa660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Jan 2021 11:59:38 +0100 Subject: [PATCH 0176/2083] Reuse the text from https://scrapy.org/ --- README.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 19faa9a87..551a06f6b 100644 --- a/README.rst +++ b/README.rst @@ -34,11 +34,10 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. -Scrapy has contributions from `many users`_ (thanks everyone!) and is sponsored -by `Scrapinghub Ltd`_. +Scrapy is maintained by `Scrapinghub`_ and `many other contributors`_. -.. _many users: https://github.com/scrapy/scrapy/graphs/contributors -.. _Scrapinghub Ltd: https://www.scrapinghub.com/ +.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapinghub: https://www.scrapinghub.com/ Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. From f30f53b3cc958770406628411db3d93c925db59e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 2 Feb 2021 15:03:20 +0100 Subject: [PATCH 0177/2083] =?UTF-8?q?Scrapinghub=20=E2=86=92=20Zyte?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AUTHORS | 4 +-- CODE_OF_CONDUCT.md | 2 +- README.rst | 7 ++--- docs/intro/install.rst | 1 - docs/topics/deploy.rst | 32 +++++++++++------------ docs/topics/logging.rst | 4 +-- docs/topics/practices.rst | 6 ++--- docs/topics/selectors.rst | 4 +-- scrapy/core/downloader/handlers/http11.py | 13 +++++---- 9 files changed, 38 insertions(+), 35 deletions(-) diff --git a/AUTHORS b/AUTHORS index bcaa1ecd3..9706adf42 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,8 +1,8 @@ Scrapy was brought to life by Shane Evans while hacking a scraping framework prototype for Mydeco (mydeco.com). It soon became maintained, extended and improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to -bootstrap the project. In mid-2011, Scrapinghub became the new official -maintainer. +bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new +official maintainer. Here is the list of the primary authors & contributors: diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index d1cd3e517..652460383 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -55,7 +55,7 @@ further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at opensource@scrapinghub.com. All +reported by contacting the project team at opensource@zyte.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. diff --git a/README.rst b/README.rst index bbe346522..5750e2c0f 100644 --- a/README.rst +++ b/README.rst @@ -42,10 +42,11 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. -Scrapy is maintained by `Scrapinghub`_ and `many other contributors`_. +Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other +contributors`_. .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors -.. _Scrapinghub: https://www.scrapinghub.com/ +.. _Zyte: https://www.zyte.com/ Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. @@ -95,7 +96,7 @@ Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@scrapinghub.com. +Please report unacceptable behavior to opensource@zyte.com. Companies using Scrapy ====================== diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 73d7ede42..bf919ce25 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -266,7 +266,6 @@ For details, see `Issue #2473 `_. .. _setuptools: https://pypi.python.org/pypi/setuptools .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ -.. _Scrapinghub: https://scrapinghub.com .. _Anaconda: https://docs.anaconda.com/anaconda/ .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html .. _conda-forge: https://conda-forge.org/ diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index 361914a29..961d6dc01 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -14,7 +14,7 @@ spiders come in. Popular choices for deploying Scrapy spiders are: * :ref:`Scrapyd ` (open source) -* :ref:`Scrapy Cloud ` (cloud-based) +* :ref:`Zyte Scrapy Cloud ` (cloud-based) .. _deploy-scrapyd: @@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers. .. _deploy-scrapy-cloud: -Deploying to Scrapy Cloud -========================= +Deploying to Zyte Scrapy Cloud +============================== -`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_, -the company behind Scrapy. +`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company +behind Scrapy. -Scrapy Cloud removes the need to setup and monitor servers -and provides a nice UI to manage spiders and review scraped items, -logs and stats. +Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a +nice UI to manage spiders and review scraped items, logs and stats. -To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool. -Please refer to the `Scrapy Cloud documentation`_ for more information. +To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line +tool. +Please refer to the `Zyte Scrapy Cloud documentation`_ for more information. -Scrapy Cloud is compatible with Scrapyd and one can switch between +Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between them as needed - the configuration is read from the ``scrapy.cfg`` file just like ``scrapyd-deploy``. -.. _Scrapyd: https://github.com/scrapy/scrapyd .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud +.. _Scrapyd: https://github.com/scrapy/scrapyd .. _scrapyd-client: https://github.com/scrapy/scrapyd-client -.. _shub: https://doc.scrapinghub.com/shub.html .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html -.. _Scrapinghub: https://scrapinghub.com/ +.. _shub: https://shub.readthedocs.io/en/latest/ +.. _Zyte: https://zyte.com/ +.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/ +.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 55065a1a3..c3445d40e 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -101,7 +101,7 @@ instance, which can be accessed and used like this:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['https://scrapinghub.com'] + start_urls = ['https://scrapy.org'] def parse(self, response): self.logger.info('Parse function called on %s', response.url) @@ -117,7 +117,7 @@ Python logger you want. For example:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['https://scrapinghub.com'] + start_urls = ['https://scrapy.org'] def parse(self, response): logger.info('Parse function called on %s', response.url) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cf1de1bd1..502fd5fcd 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -63,7 +63,7 @@ project as example. process = CrawlerProcess(get_project_settings()) # 'followall' is the name of one of the spiders of the project. - process.crawl('followall', domain='scrapinghub.com') + process.crawl('followall', domain='scrapy.org') process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling @@ -244,7 +244,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: super proxy that you can attach your own proxies to. * use a highly distributed downloader that circumvents bans internally, so you can just focus on parsing clean pages. One example of such downloaders is - `Crawlera`_ + `Zyte Smart Proxy Manager`_ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. @@ -254,5 +254,5 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _ProxyMesh: https://proxymesh.com/ .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders -.. _Crawlera: https://scrapinghub.com/crawlera .. _scrapoxy: https://scrapoxy.io/ +.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index b576fde91..c7ec2e0cc 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -464,10 +464,10 @@ effectively. If you are not much familiar with XPath yet, you may want to take a look first at this `XPath tutorial`_. .. note:: - Some of the tips are based on `this post from ScrapingHub's blog`_. + Some of the tips are based on `this post from Zyte's blog`_. .. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html -.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/ +.. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/ .. _topics-selectors-relative-xpaths: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a0fd837b1..513df2de9 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -303,11 +303,14 @@ class ScrapyAgent: proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams if omitConnectTunnel: - warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Crawlera, it doesn't require this mode anymore, " - "so you should update scrapy-crawlera to 1.3.0+ " - "and remove '?noconnect' from the Crawlera URL.", - ScrapyDeprecationWarning) + warnings.warn( + "Using HTTPS proxies in the noconnect mode is deprecated. " + "If you use Zyte Smart Proxy Manager (formerly Crawlera), " + "it doesn't require this mode anymore, so you should " + "update scrapy-crawlera to 1.3.0+ and remove '?noconnect' " + "from the Zyte Smart Proxy Manager URL.", + ScrapyDeprecationWarning, + ) if scheme == b'https' and not omitConnectTunnel: proxyAuth = request.headers.get(b'Proxy-Authorization', None) proxyConf = (proxyHost, proxyPort, proxyAuth) From 2ce8e0c74200481f8249bcee0f2a8a249e8fd58c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 09:09:53 +0100 Subject: [PATCH 0178/2083] Document the (hard-coded) maximum HTTP/2 frame size accepted from servers --- docs/topics/settings.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index b948dbfde..c7b59d582 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -689,10 +689,15 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update Scrapy currently does not support HTTP/2 Cleartext (h2c) since none of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_). + Also, Scrapy does not currently support specifying a maximum `frame size`_ + larger than the default value, 16384. Connections to servers that send a + larger frame will fail. + .. warning:: HTTP/2 support in Scrapy is experimental, and not yet recommended for production environments. Future Scrapy versions may introduce related changes without a deprecation period or warning. +.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. setting:: DOWNLOAD_TIMEOUT From d1024566d85e52f71f5591e53a3a9ee4867148ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 09:13:45 +0100 Subject: [PATCH 0179/2083] =?UTF-8?q?setup.py:=20Twisted=20=E2=86=92=20Twi?= =?UTF-8?q?sted[http2]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a95014c98..d1d6cc4b6 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support(): install_requires = [ - 'Twisted>=17.9.0', + 'Twisted[http2]>=17.9.0', 'cryptography>=2.0', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', From 536e749eccbdf479c9849b1868a5d95aa1210225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 09:22:02 +0100 Subject: [PATCH 0180/2083] HTTP/2: remove verbose protocol-handling logging --- scrapy/core/http2/protocol.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index d8d0974b8..36a51b898 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -218,7 +218,6 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.setTimeout(self.IDLE_TIMEOUT) destination = self.transport.getPeer() - logger.debug('Connection made to {}'.format(destination)) self.metadata['ip_address'] = ipaddress.ip_address(destination.host) # Initiate H2 Connection @@ -347,7 +346,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) elif isinstance(event, UnknownFrameReceived): - logger.debug('UnknownFrameReceived: frame={}'.format(event.frame)) + logger.warning(f'Unknown frame received: {event.frame}') # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated) -> None: @@ -357,15 +356,19 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def data_received(self, event: DataReceived) -> None: try: - self.streams[event.stream_id].receive_data(event.data, event.flow_controlled_length) + stream = self.streams[event.stream_id] except KeyError: - logger.debug(f'Ignoring server-initiated event {event}') + pass # We ignore server-initiated events + else: + stream.receive_data(event.data, event.flow_controlled_length) def response_received(self, event: ResponseReceived) -> None: try: - self.streams[event.stream_id].receive_headers(event.headers) + stream = self.streams[event.stream_id] except KeyError: - logger.debug(f'Ignoring server-initiated event {event}') + pass # We ignore server-initiated events + else: + stream.receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: self.metadata['settings_acknowledged'] = True @@ -381,7 +384,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): try: stream = self.pop_stream(event.stream_id) except KeyError: - logger.debug(f'Ignoring server-initiated event {event}') + pass # We ignore server-initiated events else: stream.close(StreamCloseReason.ENDED, from_protocol=True) @@ -389,7 +392,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): try: stream = self.pop_stream(event.stream_id) except KeyError: - logger.debug(f'Ignoring server-initiated event {event}') + pass # We ignore server-initiated events else: stream.close(StreamCloseReason.RESET, from_protocol=True) From 1a7bde0d8e142acc06a534f6daf8910fde5de06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 10:55:11 +0100 Subject: [PATCH 0181/2083] Document that HTTP/2 server pushes are ignored --- docs/topics/settings.rst | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c7b59d582..05cb4a855 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -684,21 +684,28 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', } +.. warning:: + + HTTP/2 support in Scrapy is experimental, and not yet recommended for + production environments. Future Scrapy versions may introduce related + changes without a deprecation period or warning. + .. note:: - Scrapy currently does not support HTTP/2 Cleartext (h2c) since none - of the major browsers support HTTP/2 unencrypted (refer `http2 faq`_). + Known limitations of the current HTTP/2 implementation of Scrapy include: - Also, Scrapy does not currently support specifying a maximum `frame size`_ - larger than the default value, 16384. Connections to servers that send a - larger frame will fail. + - No support for HTTP/2 Cleartext (h2c), since no major browser supports + HTTP/2 unencrypted (refer `http2 faq`_). -.. warning:: HTTP/2 support in Scrapy is experimental, and not yet recommended - for production environments. Future Scrapy versions may introduce - related changes without a deprecation period or warning. + - No setting to specify a maximum `frame size`_ larger than the default + value, 16384. Connections to servers that send a larger frame will + fail. + + - No support for `server pushes`_, which are ignored. .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption +.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 .. setting:: DOWNLOAD_TIMEOUT From 1773eaf5dc2330464cf769b7d874f24033f69707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 11:43:18 +0100 Subject: [PATCH 0182/2083] Move lists closer to their introducing paragraph --- docs/_static/custom.css | 10 +++ docs/conf.py | 5 +- docs/topics/exceptions.rst | 8 +- docs/topics/feed-exports.rst | 139 ++++++++++++++++++----------------- docs/topics/selectors.rst | 14 ++-- docs/topics/shell.rst | 45 ++++++------ 6 files changed, 121 insertions(+), 100 deletions(-) create mode 100644 docs/_static/custom.css diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 000000000..64f16939c --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,10 @@ +/* Move lists closer to their introducing paragraph */ +.rst-content .section ol p, .rst-content .section ul p { + margin-bottom: 0px; +} +.rst-content p + ol, .rst-content p + ul { + margin-top: -18px; /* Compensates margin-top: 24px of p */ +} +.rst-content dl p + ol, .rst-content dl p + ul { + margin-top: -6px; /* Compensates margin-top: 12px of p */ +} \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 543507a46..406c4d94a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -122,7 +122,6 @@ html_theme = 'sphinx_rtd_theme' import sphinx_rtd_theme html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # The style sheet to use for HTML and HTML Help pages. A file of that name # must exist either in Sphinx' static/ path, or in one of the custom paths # given in html_static_path. @@ -183,6 +182,10 @@ html_copy_source = True # Output file base name for HTML help builder. htmlhelp_basename = 'Scrapydoc' +html_css_files = [ + 'custom.css', +] + # Options for LaTeX output # ------------------------ diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 583a50ab8..e5264d641 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -64,10 +64,10 @@ NotConfigured This exception can be raised by some components to indicate that they will remain disabled. Those components include: - * Extensions - * Item pipelines - * Downloader middlewares - * Spider middlewares +- Extensions +- Item pipelines +- Downloader middlewares +- Spider middlewares The exception must be raised in the component's ``__init__`` method. diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 843ed25f9..e772a461c 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -21,10 +21,10 @@ Serialization formats For serializing the scraped data, the feed exports use the :ref:`Item exporters `. These formats are supported out of the box: - * :ref:`topics-feed-format-json` - * :ref:`topics-feed-format-jsonlines` - * :ref:`topics-feed-format-csv` - * :ref:`topics-feed-format-xml` +- :ref:`topics-feed-format-json` +- :ref:`topics-feed-format-jsonlines` +- :ref:`topics-feed-format-csv` +- :ref:`topics-feed-format-xml` But you can also extend the supported format through the :setting:`FEED_EXPORTERS` setting. @@ -34,54 +34,58 @@ But you can also extend the supported format through the JSON ---- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``json`` - * Exporter used: :class:`~scrapy.exporters.JsonItemExporter` - * See :ref:`this warning ` if you're using JSON with - large feeds. +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``json`` + +- Exporter used: :class:`~scrapy.exporters.JsonItemExporter` + +- See :ref:`this warning ` if you're using JSON with + large feeds. .. _topics-feed-format-jsonlines: JSON lines ---------- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines`` - * Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines`` +- Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter` .. _topics-feed-format-csv: CSV --- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv`` - * Exporter used: :class:`~scrapy.exporters.CsvItemExporter` - * To specify columns to export and their order use - :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this - option, but it is important for CSV because unlike many other export - formats CSV uses a fixed header. +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv`` + +- Exporter used: :class:`~scrapy.exporters.CsvItemExporter` + +- To specify columns to export and their order use + :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this + option, but it is important for CSV because unlike many other export + formats CSV uses a fixed header. .. _topics-feed-format-xml: XML --- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml`` - * Exporter used: :class:`~scrapy.exporters.XmlItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml`` +- Exporter used: :class:`~scrapy.exporters.XmlItemExporter` .. _topics-feed-format-pickle: Pickle ------ - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle`` - * Exporter used: :class:`~scrapy.exporters.PickleItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle`` +- Exporter used: :class:`~scrapy.exporters.PickleItemExporter` .. _topics-feed-format-marshal: Marshal ------- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal`` - * Exporter used: :class:`~scrapy.exporters.MarshalItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal`` +- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter` .. _topics-feed-storage: @@ -95,11 +99,11 @@ storage backend types which are defined by the URI scheme. The storages backends supported out of the box are: - * :ref:`topics-feed-storage-fs` - * :ref:`topics-feed-storage-ftp` - * :ref:`topics-feed-storage-s3` (requires botocore_) - * :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) - * :ref:`topics-feed-storage-stdout` +- :ref:`topics-feed-storage-fs` +- :ref:`topics-feed-storage-ftp` +- :ref:`topics-feed-storage-s3` (requires botocore_) +- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) +- :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are not available. For example, the S3 backend is only available if the botocore_ @@ -114,8 +118,8 @@ Storage URI parameters The storage URI can also contain parameters that get replaced when the feed is being created. These parameters are: - * ``%(time)s`` - gets replaced by a timestamp when the feed is being created - * ``%(name)s`` - gets replaced by the spider name +- ``%(time)s`` - gets replaced by a timestamp when the feed is being created +- ``%(name)s`` - gets replaced by the spider name Any other named parameter gets replaced by the spider attribute of the same name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id`` @@ -123,13 +127,13 @@ attribute the moment the feed is being created. Here are some examples to illustrate: - * Store in FTP using one directory per spider: +- Store in FTP using one directory per spider: - * ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json`` + - ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json`` - * Store in S3 using one directory per spider: +- Store in S3 using one directory per spider: - * ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` + - ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` .. _topics-feed-storage-backends: @@ -144,9 +148,9 @@ Local filesystem The feeds are stored in the local filesystem. - * URI scheme: ``file`` - * Example URI: ``file:///tmp/export.csv`` - * Required external libraries: none +- URI scheme: ``file`` +- Example URI: ``file:///tmp/export.csv`` +- Required external libraries: none Note that for the local filesystem storage (only) you can omit the scheme if you specify an absolute path like ``/tmp/export.csv``. This only works on Unix @@ -159,9 +163,9 @@ FTP The feeds are stored in a FTP server. - * URI scheme: ``ftp`` - * Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv`` - * Required external libraries: none +- URI scheme: ``ftp`` +- Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv`` +- Required external libraries: none FTP supports two different connection modes: `active or passive `_. Scrapy uses the passive connection @@ -178,23 +182,25 @@ S3 The feeds are stored on `Amazon S3`_. - * URI scheme: ``s3`` - * Example URIs: +- URI scheme: ``s3`` - * ``s3://mybucket/path/to/export.csv`` - * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` +- Example URIs: - * Required external libraries: `botocore`_ >= 1.4.87 + - ``s3://mybucket/path/to/export.csv`` + + - ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` + +- Required external libraries: `botocore`_ >= 1.4.87 The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: - * :setting:`AWS_ACCESS_KEY_ID` - * :setting:`AWS_SECRET_ACCESS_KEY` +- :setting:`AWS_ACCESS_KEY_ID` +- :setting:`AWS_SECRET_ACCESS_KEY` You can also define a custom ACL for exported feeds using this setting: - * :setting:`FEED_STORAGE_S3_ACL` +- :setting:`FEED_STORAGE_S3_ACL` This storage backend uses :ref:`delayed file delivery `. @@ -208,19 +214,20 @@ Google Cloud Storage (GCS) The feeds are stored on `Google Cloud Storage`_. - * URI scheme: ``gs`` - * Example URIs: +- URI scheme: ``gs`` - * ``gs://mybucket/path/to/export.csv`` +- Example URIs: - * Required external libraries: `google-cloud-storage`_. + - ``gs://mybucket/path/to/export.csv`` + +- Required external libraries: `google-cloud-storage`_. For more information about authentication, please refer to `Google Cloud documentation `_. You can set a *Project ID* and *Access Control List (ACL)* through the following settings: - * :setting:`FEED_STORAGE_GCS_ACL` - * :setting:`GCS_PROJECT_ID` +- :setting:`FEED_STORAGE_GCS_ACL` +- :setting:`GCS_PROJECT_ID` This storage backend uses :ref:`delayed file delivery `. @@ -234,9 +241,9 @@ Standard output The feeds are written to the standard output of the Scrapy process. - * URI scheme: ``stdout`` - * Example URI: ``stdout:`` - * Required external libraries: none +- URI scheme: ``stdout`` +- Example URI: ``stdout:`` +- Required external libraries: none .. _delayed-file-delivery: @@ -264,16 +271,16 @@ Settings These are the settings used for configuring the feed exports: - * :setting:`FEEDS` (mandatory) - * :setting:`FEED_EXPORT_ENCODING` - * :setting:`FEED_STORE_EMPTY` - * :setting:`FEED_EXPORT_FIELDS` - * :setting:`FEED_EXPORT_INDENT` - * :setting:`FEED_STORAGES` - * :setting:`FEED_STORAGE_FTP_ACTIVE` - * :setting:`FEED_STORAGE_S3_ACL` - * :setting:`FEED_EXPORTERS` - * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` +- :setting:`FEEDS` (mandatory) +- :setting:`FEED_EXPORT_ENCODING` +- :setting:`FEED_STORE_EMPTY` +- :setting:`FEED_EXPORT_FIELDS` +- :setting:`FEED_EXPORT_INDENT` +- :setting:`FEED_STORAGES` +- :setting:`FEED_STORAGE_FTP_ACTIVE` +- :setting:`FEED_STORAGE_S3_ACL` +- :setting:`FEED_EXPORTERS` +- :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index c7ec2e0cc..9caba5ee5 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -8,14 +8,14 @@ When you're scraping web pages, the most common task you need to perform is to extract data from the HTML source. There are several libraries available to achieve this, such as: - * `BeautifulSoup`_ is a very popular web scraping library among Python - programmers which constructs a Python object based on the structure of the - HTML code and also deals with bad markup reasonably well, but it has one - drawback: it's slow. +- `BeautifulSoup`_ is a very popular web scraping library among Python + programmers which constructs a Python object based on the structure of the + HTML code and also deals with bad markup reasonably well, but it has one + drawback: it's slow. - * `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic - API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard - library.) +- `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic + API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python + standard library.) Scrapy comes with its own mechanism for extracting data. They're called selectors because they "select" certain parts of the HTML document specified diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 0f46f1c87..b910fc453 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -95,20 +95,21 @@ convenience. Available Shortcuts ------------------- - * ``shelp()`` - print a help with the list of available objects and shortcuts +- ``shelp()`` - print a help with the list of available objects and + shortcuts - * ``fetch(url[, redirect=True])`` - fetch a new response from the given - URL and update all related objects accordingly. You can optionaly ask for - HTTP 3xx redirections to not be followed by passing ``redirect=False`` +- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL + and update all related objects accordingly. You can optionaly ask for HTTP + 3xx redirections to not be followed by passing ``redirect=False`` - * ``fetch(request)`` - fetch a new response from the given request and - update all related objects accordingly. +- ``fetch(request)`` - fetch a new response from the given request and update + all related objects accordingly. - * ``view(response)`` - open the given response in your local web browser, for - inspection. This will add a `\ tag`_ to the response body in order - for external links (such as images and style sheets) to display properly. - Note, however, that this will create a temporary file in your computer, - which won't be removed automatically. +- ``view(response)`` - open the given response in your local web browser, for + inspection. This will add a `\ tag`_ to the response body in order + for external links (such as images and style sheets) to display properly. + Note, however, that this will create a temporary file in your computer, + which won't be removed automatically. .. _ tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base @@ -122,21 +123,21 @@ content). Those objects are: - * ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object. +- ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object. - * ``spider`` - the Spider which is known to handle the URL, or a - :class:`~scrapy.spiders.Spider` object if there is no spider found for - the current URL +- ``spider`` - the Spider which is known to handle the URL, or a + :class:`~scrapy.spiders.Spider` object if there is no spider found for the + current URL - * ``request`` - a :class:`~scrapy.http.Request` object of the last fetched - page. You can modify this request using :meth:`~scrapy.http.Request.replace` - or fetch a new request (without leaving the shell) using the ``fetch`` - shortcut. +- ``request`` - a :class:`~scrapy.http.Request` object of the last fetched + page. You can modify this request using + :meth:`~scrapy.http.Request.replace` or fetch a new request (without + leaving the shell) using the ``fetch`` shortcut. - * ``response`` - a :class:`~scrapy.http.Response` object containing the last - fetched page +- ``response`` - a :class:`~scrapy.http.Response` object containing the last + fetched page - * ``settings`` - the current :ref:`Scrapy settings ` +- ``settings`` - the current :ref:`Scrapy settings ` Example of shell session ======================== From 4c801551fa10e4ff75f0768b903664f415a6a504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 21:11:46 +0100 Subject: [PATCH 0183/2083] Document that the bytes_received signal is not yet implemented for HTTP/2 --- docs/topics/settings.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 05cb4a855..0a4684a91 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -703,6 +703,8 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update - No support for `server pushes`_, which are ignored. + - No support for the :signal:`bytes_received` signal. + .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 From 248800328cb32f79a214289d59e927b576e21712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 21:13:43 +0100 Subject: [PATCH 0184/2083] Fix test_pinned_twisted_version --- tests/test_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 93e7311d2..5e63ebffb 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -36,7 +36,7 @@ class ScrapyUtilsTest(unittest.TestCase): ) config_parser = ConfigParser() config_parser.read(tox_config_file_path) - pattern = r'Twisted==([\d.]+)' + pattern = r'Twisted\[http2\]==([\d.]+)' match = re.search(pattern, config_parser['pinned']['deps']) pinned_twisted_version_string = match[1] From 0e4b291701baa8b74bd13bd47831dd040dd7173f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 3 Feb 2021 21:28:04 +0100 Subject: [PATCH 0185/2083] HTTP/2: fix canceling a request before a connection has been established --- scrapy/core/http2/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index e345ca79a..572dbf7aa 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -459,7 +459,7 @@ class Stream: request=self._request, certificate=self._protocol.metadata['certificate'], ip_address=self._protocol.metadata['ip_address'], - protocol=to_unicode(self._protocol.transport.negotiatedProtocol), + protocol='h2', ) self._deferred_response.callback(response) From 7b11b74c77d1535f943baebbf5c794a63d147a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 4 Feb 2021 11:08:01 +0100 Subject: [PATCH 0186/2083] Use --use-deprecated=legacy-resolver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let’s see how test results change --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index d8e900e06..ecd3aad6e 100644 --- a/tox.ini +++ b/tox.ini @@ -27,6 +27,10 @@ passenv = GCS_PROJECT_ID #allow tox virtualenv to upgrade pip/wheel/setuptools download = true +# TODO: Remove the custom install_command below +# Temporary workaround to filter out errors caused by the insanely long time +# that it takes for the new resolver to install dependencies. +install_command=python -m pip install --use-deprecated=legacy-resolver {opts} {packages} commands = py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} From 76abcedaf4f31a7c76de9e19680dd7499d9eccf4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 13:36:25 +0500 Subject: [PATCH 0187/2083] Add as_async_generator. --- scrapy/utils/asyncgen.py | 12 ++++++++++++ tests/test_utils_asyncgen.py | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/test_utils_asyncgen.py diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 7f697af5f..db2173f85 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,5 +1,17 @@ +import collections + + async def collect_asyncgen(result): results = [] async for x in result: results.append(x) return results + + +async def as_async_generator(it): + if isinstance(it, collections.abc.AsyncIterator): + async for r in it: + yield r + else: + for r in it: + yield r diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py new file mode 100644 index 000000000..9ae66c57c --- /dev/null +++ b/tests/test_utils_asyncgen.py @@ -0,0 +1,20 @@ +from twisted.trial import unittest + +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.defer import deferred_f_from_coro_f + + +class AsyncgenUtilsTest(unittest.TestCase): + @deferred_f_from_coro_f + async def test_as_async_generator(self): + ag = as_async_generator(range(42)) + results = [] + async for i in ag: + results.append(i) + self.assertEqual(results, list(range(42))) + + @deferred_f_from_coro_f + async def test_collect_asyncgen(self): + ag = as_async_generator(range(42)) + results = await collect_asyncgen(ag) + self.assertEqual(results, list(range(42))) From acff1eb4960940eb360f3cf00d5b33ed71acc351 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 14:36:38 +0500 Subject: [PATCH 0188/2083] Add aiter_errback. --- scrapy/utils/defer.py | 14 ++++++++++++++ tests/test_utils_defer.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 6db9cc117..2d02c0621 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -124,6 +124,20 @@ def iter_errback(iterable, errback, *a, **kw): errback(failure.Failure(), *a, **kw) +async def aiter_errback(aiterable, errback, *a, **kw): + """Wraps an async iterable calling an errback if an error is caught while + iterating it. Similar to scrapy.utils.defer.iter_errback() + """ + it = aiterable.__aiter__() + while True: + try: + yield await it.__anext__() + except StopAsyncIteration: + break + except Exception: + errback(failure.Failure(), *a, **kw) + + def deferred_from_coro(o): """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, defer.Deferred): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index e60242a3b..06d91c574 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -2,12 +2,15 @@ from twisted.trial import unittest from twisted.internet import reactor, defer from twisted.python.failure import Failure +from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import ( iter_errback, + aiter_errback, mustbe_deferred, process_chain, process_chain_both, process_parallel, + deferred_f_from_coro_f, ) @@ -117,3 +120,31 @@ class IterErrbackTest(unittest.TestCase): self.assertEqual(out, [0, 1, 2, 3, 4]) self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0].value, ZeroDivisionError) + + +class AiterErrbackTest(unittest.TestCase): + + @deferred_f_from_coro_f + async def test_aiter_errback_good(self): + async def itergood(): + for x in range(10): + yield x + + errors = [] + out = await collect_asyncgen(aiter_errback(itergood(), errors.append)) + self.assertEqual(out, list(range(10))) + self.assertFalse(errors) + + @deferred_f_from_coro_f + async def test_iter_errback_bad(self): + async def iterbad(): + for x in range(10): + if x == 5: + 1 / 0 + yield x + + errors = [] + out = await collect_asyncgen(aiter_errback(iterbad(), errors.append)) + self.assertEqual(out, [0, 1, 2, 3, 4]) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0].value, ZeroDivisionError) From 7e9f498e00fd36c76ac139eb285526c9e70b4054 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 14:37:27 +0500 Subject: [PATCH 0189/2083] Add MutableAsyncChain. --- scrapy/utils/python.py | 25 ++++++++++++++++ tests/test_utils_python.py | 58 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5703fd4c3..0bf9bff70 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -12,6 +12,7 @@ from functools import partial, wraps from itertools import chain from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.decorators import deprecated @@ -355,3 +356,27 @@ class MutableChain: @deprecated("scrapy.utils.python.MutableChain.__next__") def next(self): return self.__next__() + + +async def _async_chain(*iterables): + for it in iterables: + async for o in as_async_generator(it): + yield o + + +class MutableAsyncChain: + """ + Similar to MutableChain but for async iterables + """ + + def __init__(self, *args): + self.data = _async_chain(*args) + + def extend(self, *aiterables): + self.data = _async_chain(self.data, _async_chain(*aiterables)) + + def __aiter__(self): + return self + + async def __anext__(self): + return await self.data.__anext__() diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3115cc92f..58b384591 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -2,15 +2,18 @@ import functools import gc import operator import platform -import unittest from datetime import datetime from itertools import count from warnings import catch_warnings +from twisted.trial import unittest + +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, WeakKeyCache, get_func_args, to_bytes, to_unicode, - without_none_values, MutableChain) + without_none_values, MutableChain, MutableAsyncChain) __doctests__ = ['scrapy.utils.python'] @@ -32,6 +35,57 @@ class MutableChainTest(unittest.TestCase): self.assertEqual(list(m), list(range(3, 13))) +class MutableAsyncChainTest(unittest.TestCase): + @staticmethod + async def g1(): + for i in range(3): + yield i + + @staticmethod + async def g2(): + return + yield + + @staticmethod + async def g3(): + for i in range(7, 10): + yield i + + @staticmethod + async def g4(): + for i in range(3, 5): + yield i + 1 / 0 + for i in range(5, 7): + yield i + + @staticmethod + async def collect_asyncgen_exc(asyncgen): + results = [] + async for x in asyncgen: + results.append(x) + return results + + @deferred_f_from_coro_f + async def test_mutableasyncchain(self): + m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7))) + m.extend(self.g2()) + m.extend(self.g3()) + + self.assertEqual(await m.__anext__(), 0) + results = await collect_asyncgen(m) + self.assertEqual(results, list(range(1, 10))) + + @deferred_f_from_coro_f + async def test_mutableasyncchain_exc(self): + m = MutableAsyncChain(self.g1()) + m.extend(self.g4()) + m.extend(self.g3()) + + results = await collect_asyncgen(aiter_errback(m, lambda _: None)) + self.assertEqual(results, list(range(5))) + + class ToUnicodeTest(unittest.TestCase): def test_converting_an_utf8_encoded_string_to_unicode(self): self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e') From d658552f232547cc227e597b0d8489c4762eb9d2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 15:20:04 +0500 Subject: [PATCH 0190/2083] Add only_not_asyncio. --- conftest.py | 6 ++++++ pytest.ini | 1 + 2 files changed, 7 insertions(+) diff --git a/conftest.py b/conftest.py index 68b855c08..407bf9e62 100644 --- a/conftest.py +++ b/conftest.py @@ -55,5 +55,11 @@ def only_asyncio(request, reactor_pytest): pytest.skip('This test is only run with --reactor=asyncio') +@pytest.fixture(autouse=True) +def only_not_asyncio(request, reactor_pytest): + if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio': + pytest.skip('This test is only run without --reactor=asyncio') + + # Generate localhost certificate files, needed by some tests generate_keys() diff --git a/pytest.ini b/pytest.ini index d4deeb57c..416b228f9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -21,6 +21,7 @@ addopts = twisted = 1 markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed + only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed flake8-max-line-length = 119 flake8-ignore = W503 From d66d52d3ed8aab0b4f126169c2855d00f4053907 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 15:21:12 +0500 Subject: [PATCH 0191/2083] Add process_iterable_helper. --- scrapy/utils/middlewares.py | 35 +++++++++++++ tests/test_utils_middlewares.py | 87 +++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 scrapy/utils/middlewares.py create mode 100644 tests/test_utils_middlewares.py diff --git a/scrapy/utils/middlewares.py b/scrapy/utils/middlewares.py new file mode 100644 index 000000000..da28e0ddf --- /dev/null +++ b/scrapy/utils/middlewares.py @@ -0,0 +1,35 @@ +# coding: utf-8 +import inspect + + +def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): + for o in it: + if in_predicate and not in_predicate(o): + continue + if processor is not None: + o = processor(o) + if out_predicate and not out_predicate(o): + continue + yield o + + +async def process_async_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): + async for o in it: + if in_predicate and not in_predicate(o): + continue + if processor is not None: + o = processor(o) + if out_predicate and not out_predicate(o): + continue + yield o + + +def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): + """ + For each item in the iterable: skips it if in_predicate is False, applies processor, + skips the result if out_predicate is False, else yields it. + """ + if inspect.isasyncgen(it): + return process_async_iterable_helper(it, in_predicate, out_predicate, processor) + else: + return process_normal_iterable_helper(it, in_predicate, out_predicate, processor) diff --git a/tests/test_utils_middlewares.py b/tests/test_utils_middlewares.py new file mode 100644 index 000000000..d395ba1a9 --- /dev/null +++ b/tests/test_utils_middlewares.py @@ -0,0 +1,87 @@ +import collections + +from twisted.trial import unittest + +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.middlewares import process_iterable_helper + + +def predicate1(o): + return bool(o % 2) + + +def predicate2(o): + return o < 10 + + +def processor(o): + return o * 2 + + +class ProcessIterableHelperNormalTest(unittest.TestCase): + + def test_normal_in_predicate(self): + iterable1 = iter([1, 2, 3]) + iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1) + self.assertIsInstance(iterable2, collections.abc.Iterable) + list2 = list(iterable2) + self.assertEqual(list2, [1, 3]) + + def test_normal_out_predicate(self): + iterable1 = iter([1, 2, 10, 3, 15]) + iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2) + self.assertIsInstance(iterable2, collections.abc.Iterable) + list2 = list(iterable2) + self.assertEqual(list2, [1, 2, 3]) + + def test_normal_processor(self): + iterable1 = iter([1, 2, 3]) + iterable2 = process_iterable_helper(iterable1, processor=processor) + self.assertIsInstance(iterable2, collections.abc.Iterable) + list2 = list(iterable2) + self.assertEqual(list2, [2, 4, 6]) + + def test_normal_combined(self): + iterable1 = iter([1, 2, 10, 3, 6, 18, 5, 15]) + iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1, + out_predicate=predicate2, processor=processor) + self.assertIsInstance(iterable2, collections.abc.Iterable) + list2 = list(iterable2) + self.assertEqual(list2, [2, 6]) + + +class ProcessIterableHelperAsyncTest(unittest.TestCase): + + @deferred_f_from_coro_f + async def test_async_in_predicate(self): + iterable1 = as_async_generator([1, 2, 3]) + iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1) + self.assertIsInstance(iterable2, collections.abc.AsyncIterable) + list2 = await collect_asyncgen(iterable2) + self.assertEqual(list2, [1, 3]) + + @deferred_f_from_coro_f + async def test_async_out_predicate(self): + iterable1 = as_async_generator([1, 2, 10, 3, 15]) + iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2) + self.assertIsInstance(iterable2, collections.abc.AsyncIterable) + list2 = await collect_asyncgen(iterable2) + self.assertEqual(list2, [1, 2, 3]) + + @deferred_f_from_coro_f + async def test_async_processor(self): + iterable1 = as_async_generator([1, 2, 3]) + iterable2 = process_iterable_helper(iterable1, processor=processor) + self.assertIsInstance(iterable2, collections.abc.AsyncIterable) + list2 = await collect_asyncgen(iterable2) + self.assertEqual(list2, [2, 4, 6]) + + @deferred_f_from_coro_f + async def test_async_combined(self): + iterable1 = as_async_generator([1, 2, 10, 3, 6, 18, 5, 15]) + iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1, + out_predicate=predicate2, processor=processor) + self.assertIsInstance(iterable2, collections.abc.AsyncIterable) + list2 = await collect_asyncgen(iterable2) + self.assertEqual(list2, [2, 6]) From 92f2c9e308a5eda361229a8e74f74a21b9ff770a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 15:22:20 +0500 Subject: [PATCH 0192/2083] Move spider middlewares to process_iterable_helper. --- scrapy/spidermiddlewares/depth.py | 4 ++-- scrapy/spidermiddlewares/offsite.py | 32 ++++++++++++++------------- scrapy/spidermiddlewares/referer.py | 3 ++- scrapy/spidermiddlewares/urllength.py | 3 ++- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 776a6879a..73079bca9 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -3,10 +3,10 @@ Depth Spider Middleware See documentation in docs/topics/spider-middleware.rst """ - import logging from scrapy.http import Request +from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -55,4 +55,4 @@ class DepthMiddleware: if self.verbose_stats: self.stats.inc_value('request_depth_count/0', spider=spider) - return (r for r in result or () if _filter(r)) + return process_iterable_helper(result or (), in_predicate=_filter) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 6e4efda97..e7f481269 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -10,6 +10,7 @@ import warnings from scrapy import signals from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -26,21 +27,22 @@ class OffsiteMiddleware: return o def process_spider_output(self, response, result, spider): - for x in result: - if isinstance(x, Request): - if x.dont_filter or self.should_follow(x, spider): - yield x - else: - domain = urlparse_cached(x).hostname - if domain and domain not in self.domains_seen: - self.domains_seen.add(domain) - logger.debug( - "Filtered offsite request to %(domain)r: %(request)s", - {'domain': domain, 'request': x}, extra={'spider': spider}) - self.stats.inc_value('offsite/domains', spider=spider) - self.stats.inc_value('offsite/filtered', spider=spider) - else: - yield x + def in_predicate(x): + if not isinstance(x, Request): + return True + if x.dont_filter or self.should_follow(x, spider): + return True + domain = urlparse_cached(x).hostname + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {'domain': domain, 'request': x}, extra={'spider': spider}) + self.stats.inc_value('offsite/domains', spider=spider) + self.stats.inc_value('offsite/filtered', spider=spider) + return False + + return process_iterable_helper(result or (), in_predicate=in_predicate) def should_follow(self, request, spider): regex = self.host_regex diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index f81041376..91c8727e1 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -10,6 +10,7 @@ from w3lib.url import safe_url_string from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals +from scrapy.utils.middlewares import process_iterable_helper from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object from scrapy.utils.url import strip_url @@ -337,7 +338,7 @@ class RefererMiddleware: if referrer is not None: r.headers.setdefault('Referer', referrer) return r - return (_set_referer(r) for r in result or ()) + return process_iterable_helper(result or (), processor=_set_referer) def request_scheduled(self, request, spider): # check redirected request to patch "Referer" header if necessary diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 5be1f80cb..c7359fecd 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -8,6 +8,7 @@ import logging from scrapy.http import Request from scrapy.exceptions import NotConfigured +from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -34,4 +35,4 @@ class UrlLengthMiddleware: else: return True - return (r for r in result or () if _filter(r)) + return process_iterable_helper(result or (), in_predicate=_filter) From 2a1e9359caa0e16b336fdb2944ab7f4a69549896 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 5 Feb 2021 16:16:29 +0500 Subject: [PATCH 0193/2083] Add parallel_async. --- scrapy/core/scraper.py | 24 +++++++-- scrapy/utils/defer.py | 110 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 5 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 0d3e3450f..3eae71af7 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,6 +1,6 @@ """This module implements the Scraper component which parses responses and extracts information from them""" - +import collections import logging from collections import deque @@ -12,7 +12,16 @@ from scrapy import signals from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response -from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel +from scrapy.utils.defer import ( + aiter_errback, + defer_fail, + defer_succeed, + deferred_from_coro, + iter_errback, + parallel, + parallel_async, +) + from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output @@ -180,9 +189,14 @@ class Scraper: def handle_spider_output(self, result, request, response, spider): if not result: return defer_succeed(None) - it = iter_errback(result, self.handle_spider_error, request, response, spider) - dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, - request, response, spider) + if isinstance(result, collections.abc.AsyncIterable): + it = aiter_errback(result, self.handle_spider_error, request, response, spider) + dfd = deferred_from_coro(parallel_async(it, self.concurrent_items, self._process_spidermw_output, + request, response, spider)) + else: + it = iter_errback(result, self.handle_spider_error, request, response, spider) + dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, + request, response, spider) return dfd def _process_spidermw_output(self, output, request, response, spider): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 2d02c0621..bd5f9c8fc 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -75,6 +75,116 @@ def parallel(iterable, count, callable, *args, **named): return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) +class _AsyncCooperatorAdapter: + """ A class that wraps an async iterator into a normal iterator suitable + for using in Cooperator.coiterate(). As it's only needed for parallel_async(), + it calls the callable directly in the callback, instead of providing a more + generic interface. + + On the outside, this class behaves as an iterator that yields Deferreds. + Each Deferred is fired with the result of the callable which was called on + the next result from aiterator. It raises StopIteration when aiterator is + exhausted, as expected. + + Cooperator calls __next__() multiple times and waits on the Deferreds + returned from it. As async generators (since Python 3.8) don't support + awaiting on __anext__() several times in parallel, we need to serialize + this. It's done by storing the Deferreds returned from __next__() and + firing the oldest one when a result from __anext__() is available. + + The workflow: + 1. When __next__() is called for the first time, it creates a Deferred, stores it + in self.waiting_deferreds and returns it. It also makes a Deferred that will wait + for self.aiterator.__anext__() and puts it into self.anext_deferred. + 2. If __next__() is called again before self.anext_deferred fires, more Deferreds + are added to self.waiting_deferreds. + 3. When self.anext_deferred fires, it either calls _callback() or _errback(). Both + clear self.anext_deferred. + 3.1. _callback() calls the callable passing the result value that it takes, pops a + Deferred from self.waiting_deferreds, and if the callable result was a Deferred, it + chains those Deferreds so that the waiting Deferred will fire when the result + Deferred does, otherwise it fires it directly. This causes one awaiting task to + receive a result. If self.waiting_deferreds is still not empty, new __anext__() is + called and self.anext_deferred is populated. + 3.2. _errback() checks the exception class. If it's StopAsyncIteration it means + self.aiterator is exhausted and so it sets self.finished and fires all + self.waiting_deferreds. Other exceptions are propagated. + 4. If __next__() is called after __anext__() was handled, then if self.finished is + True, it raises StopIteration, otherwise it acts like in step 2, but if + self.anext_deferred is now empty is also populates it with a new __anext__(). + + Note that CooperativeTask ignores the value returned from the Deferred that it waits + for, so we fire them with None when needed. + + It may be possible to write an async iterator-aware replacement for + Cooperator/CooperativeTask and use it instead of this adapter to achieve the same + goal. + """ + def __init__(self, aiterator, callable, *callable_args, **callable_kwargs): + self.aiterator = aiterator + self.callable = callable + self.callable_args = callable_args + self.callable_kwargs = callable_kwargs + self.finished = False + self.waiting_deferreds = [] + self.anext_deferred = None + + def _callback(self, result): + # This gets called when the result from aiterator.__anext__() is available. + # It calls the callable on it and sends the result to the oldest waiting Deferred + # (by chaining if the result is a Deferred too or by firing if not). + self.anext_deferred = None + result = self.callable(result, *self.callable_args, **self.callable_kwargs) + d = self.waiting_deferreds.pop(0) + if d.called: + raise ValueError('Deferred in waiting_deferreds already called') + if isinstance(result, defer.Deferred): + result.chainDeferred(d) + else: + d.callback(None) + if self.waiting_deferreds: + self._call_anext() + + def _errback(self, failure): + # This gets called on any exceptions in aiterator.__anext__(). + # It handles StopAsyncIteration by stopping the iteration and reraises all others. + self.anext_deferred = None + failure.trap(StopAsyncIteration) + self.finished = True + for d in self.waiting_deferreds: + if d.called: + raise ValueError('Deferred in waiting_deferreds already called') + d.callback(None) + + def _call_anext(self): + # This starts waiting for the next result from aiterator. + # If aiterator is exhausted, _errback will be called. + self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) + self.anext_deferred.addCallbacks(self._callback, self._errback) + + def __iter__(self): + return self + + def __next__(self): + # This puts a new Deferred into self.waiting_deferreds and returns it. + # It also calls __anext__() if needed. + if self.finished: + raise StopIteration + d = defer.Deferred() + self.waiting_deferreds.append(d) + if not self.anext_deferred: + self._call_anext() + return d + + +def parallel_async(async_iterable, count, callable, *args, **named): + """ Like parallel but for async iterables """ + coop = task.Cooperator() + work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) + dl = defer.DeferredList([coop.coiterate(work) for _ in range(count)]) + return dl + + def process_chain(callbacks, input, *a, **kw): """Return a Deferred built by chaining the given callbacks""" d = defer.Deferred() From 2152a2a50898b91a44e6b0c282b69b7bfb8d18f0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 5 Feb 2021 16:19:52 +0500 Subject: [PATCH 0194/2083] Add main infrastructure for async callbacks. --- scrapy/core/spidermw.py | 45 ++++++++++++++++----- scrapy/utils/defer.py | 19 +++++++++ scrapy/utils/spider.py | 8 ++-- scrapy/utils/test.py | 7 ++++ tests/spiders.py | 38 ++++++++++++++++- tests/test_crawl.py | 33 +++++++++++++++ tests/test_spidermiddleware_output_chain.py | 27 +++++++++++++ 7 files changed, 161 insertions(+), 16 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 763e0cdf6..961606f29 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,6 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +import inspect from itertools import islice from twisted.python.failure import Failure @@ -11,11 +12,11 @@ from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list from scrapy.utils.defer import mustbe_deferred -from scrapy.utils.python import MutableChain +from scrapy.utils.python import MutableAsyncChain, MutableChain def _isiterable(possible_iterator): - return hasattr(possible_iterator, '__iter__') + return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__') def _fname(f): @@ -58,15 +59,31 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(response, request, spider) def _evaluate_iterable(iterable, exception_processor_index, recover_to): - try: - for r in iterable: - yield r - except Exception as ex: + def _process_exception(ex): exception_result = process_spider_exception(Failure(ex), exception_processor_index) if isinstance(exception_result, Failure): raise recover_to.extend(exception_result) + def _evaluate_normal_iterable(iterable): + try: + for r in iterable: + yield r + except Exception as ex: + _process_exception(ex) + + async def _evaluate_async_iterable(iterable): + try: + async for r in iterable: + yield r + except Exception as ex: + _process_exception(ex) + + if inspect.isasyncgen(iterable): + return _evaluate_async_iterable(iterable) + else: + return _evaluate_normal_iterable(iterable) + def process_spider_exception(_failure, start_index=0): exception = _failure.value # don't handle _InvalidOutput exception @@ -92,7 +109,11 @@ class SpiderMiddlewareManager(MiddlewareManager): def process_spider_output(result, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - recovered = MutableChain() + if inspect.isasyncgen(result): + iter_class = MutableAsyncChain + else: + iter_class = MutableChain + recovered = iter_class() method_list = islice(self.methods['process_spider_output'], start_index, None) for method_index, method in enumerate(method_list, start=start_index): @@ -113,12 +134,16 @@ class SpiderMiddlewareManager(MiddlewareManager): f"iterable, got {type(result)}") raise _InvalidOutput(msg) - return MutableChain(result, recovered) + return iter_class(result, recovered) def process_callback_output(result): - recovered = MutableChain() + if inspect.isasyncgen(result): + iter_class = MutableAsyncChain + else: + iter_class = MutableChain + recovered = iter_class() result = _evaluate_iterable(result, 0, recovered) - return MutableChain(process_spider_output(result), recovered) + return iter_class(process_spider_output(result), recovered) dfd = mustbe_deferred(process_spider_input, response) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bd5f9c8fc..554edc38c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -290,3 +290,22 @@ def maybeDeferred_coro(f, *args, **kw): return defer.fail(result) else: return defer.succeed(result) + + +def deferred_to_future(d): + """ Wraps a Deferred into a Future. Requires the asyncio reactor. + """ + return d.asFuture(asyncio.get_event_loop()) + + +def maybe_deferred_to_future(d): + """ Converts a Deferred to something that can be awaited in a callback or other user coroutine. + + If the asyncio reactor is installed, coroutines are wrapped into Futures, and only Futures can be + awaited inside them. Otherwise, coroutines are wrapped into Deferreds and Deferreds can be awaited + directly inside them. + """ + if not is_asyncio_reactor_installed(): + return d + else: + return deferred_to_future(d) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 59fc9202f..d0fd1757d 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,7 +4,6 @@ import logging from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter -from scrapy.utils.asyncgen import collect_asyncgen logger = logging.getLogger(__name__) @@ -12,14 +11,13 @@ logger = logging.getLogger(__name__) def iterate_spider_output(result): if inspect.isasyncgen(result): - d = deferred_from_coro(collect_asyncgen(result)) - d.addCallback(iterate_spider_output) - return d + return result elif inspect.iscoroutine(result): d = deferred_from_coro(result) d.addCallback(iterate_spider_output) return d - return arg_to_iter(result) + else: + return arg_to_iter(deferred_from_coro(result)) def iter_spider_classes(module): diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 24c38283a..d8fc25094 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -110,3 +110,10 @@ def mock_google_cloud_storage(): bucket_mock.blob.return_value = blob_mock return (client_mock, bucket_mock, blob_mock) + + +def get_web_client_agent_req(url): + from twisted.internet import reactor + from twisted.web.client import Agent # imports twisted.internet.reactor + agent = Agent(reactor) + return agent.request(b'GET', url.encode('utf-8')) diff --git a/tests/spiders.py b/tests/spiders.py index 106392ea6..3e0ec001b 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -14,7 +14,8 @@ from scrapy.item import Item from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.spiders.crawl import CrawlSpider, Rule -from scrapy.utils.test import get_from_asyncio_queue +from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future +from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req class MockServerSpider(Spider): @@ -148,6 +149,41 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): return reqs +class AsyncDefAsyncioGenExcSpider(SimpleSpider): + name = 'asyncdef_asyncio_gen_exc' + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {'foo': i} + if i > 5: + raise ValueError("Stopping the processing") + + +class AsyncDefDeferredDirectSpider(SimpleSpider): + name = 'asyncdef_deferred_direct' + + async def parse(self, response): + resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200")) + yield {'code': resp.code} + + +class AsyncDefDeferredWrappedSpider(SimpleSpider): + name = 'asyncdef_deferred_wrapped' + + async def parse(self, response): + resp = await deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200"))) + yield {'code': resp.code} + + +class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): + name = 'asyncdef_deferred_wrapped' + + async def parse(self, response): + resp = await maybe_deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200"))) + yield {'code': resp.code} + + class AsyncDefAsyncioGenSpider(SimpleSpider): name = 'asyncdef_asyncio_gen' diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1083c1678..cda52f0d4 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -20,12 +20,16 @@ from scrapy.utils.python import to_unicode from tests.mockserver import MockServer from tests.spiders import ( AsyncDefAsyncioGenComplexSpider, + AsyncDefAsyncioGenExcSpider, AsyncDefAsyncioGenLoopSpider, AsyncDefAsyncioGenSpider, AsyncDefAsyncioReqsReturnSpider, AsyncDefAsyncioReturnSingleElementSpider, AsyncDefAsyncioReturnSpider, AsyncDefAsyncioSpider, + AsyncDefDeferredDirectSpider, + AsyncDefDeferredMaybeWrappedSpider, + AsyncDefDeferredWrappedSpider, AsyncDefSpider, BrokenStartRequestsSpider, BytesReceivedCallbackSpider, @@ -430,6 +434,18 @@ class CrawlSpiderTestCase(TestCase): for i in range(10): self.assertIn({'foo': i}, items) + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncgen_parse_exc(self): + log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) + log = str(log) + self.assertIn("Spider error processing", log) + self.assertIn("ValueError", log) + itemcount = stats.get_value('item_scraped_count') + self.assertEqual(itemcount, 7) + for i in range(7): + self.assertIn({'foo': i}, items) + @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): @@ -449,6 +465,23 @@ class CrawlSpiderTestCase(TestCase): for req_id in range(3): self.assertIn(f"Got response 200, req_id {req_id}", str(log)) + @mark.only_not_asyncio() + @defer.inlineCallbacks + def test_async_def_deferred_direct(self): + _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) + self.assertEqual(items, [{'code': 200}]) + + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_deferred_wrapped(self): + log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) + self.assertEqual(items, [{'code': 200}]) + + @defer.inlineCallbacks + def test_async_def_deferred_maybe_wrapped(self): + _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider) + self.assertEqual(items, [{'code': 200}]) + @defer.inlineCallbacks def test_response_ssl_certificate_none(self): crawler = self.runner.create_crawler(SingleRequestSpider) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 029bf8bd6..4d1a7fcb0 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -43,6 +43,23 @@ class RecoverySpider(Spider): raise TabError() +class RecoveryAsyncGenSpider(RecoverySpider): + name = 'RecoveryAsyncGenSpider' + + async def parse(self, response): + for r in super().parse(response): + yield r + + +class RecoveryMiddleware: + def process_spider_exception(self, response, exception, spider): + spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) + return [ + {'from': 'process_spider_exception'}, + Request(response.url, meta={'dont_fail': True}, dont_filter=True), + ] + + # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method class FailProcessSpiderInputMiddleware: @@ -307,6 +324,16 @@ class TestSpiderMiddleware(TestCase): self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) self.assertIn("'item_scraped_count': 3", str(log)) + @defer.inlineCallbacks + def test_recovery_asyncgen(self): + """ + Same as test_recovery but with an async callback. + """ + log = yield self.crawl_log(RecoveryAsyncGenSpider) + self.assertIn("Middleware: TabError exception caught", str(log)) + self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) + self.assertIn("'item_scraped_count': 3", str(log)) + @defer.inlineCallbacks def test_process_spider_input_without_errback(self): """ From 58f848130145649a0191e98b182977278e2b9b6c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 5 Feb 2021 16:20:32 +0500 Subject: [PATCH 0195/2083] Update docs. --- docs/topics/asyncio.rst | 24 ++++++++++++++++++++++++ docs/topics/coroutines.rst | 12 +++--------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 91e1cca0d..4addaa178 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -39,4 +39,28 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the use it instead of the default asyncio event loop. +.. _asyncio-await-dfd: +Awaiting on Deferreds +===================== + +When the asyncio reactor isn't installed, you can await on Deferreds in the +coroutines directly. When it is installed, this is not possible anymore, due to +specifics of the Scrapy coroutine integration (the coroutines are wrapped into +asyncio Futures, not into Deferreds directly), and you need to wrap them into +Futures. Scrapy provides two helpers for this: + +.. autofunction:: scrapy.utils.defer.deferred_to_future +.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future + +If you want to write universal code that works on any reactors, +you should use ``maybe_deferred_to_future`` on all Deferreds:: + + from scrapy.utils.defer import maybe_deferred_to_future + + class MySpider(Spider): + # ... + async def parse_with_deferred(self, response): + additional_response = await maybe_deferred_to_future(treq.get('https://additional.url')) + additional_data = await maybe_deferred_to_future(treq.content(additional_response)) + # ... use response and additional_data to yield items and requests diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 3b1549bd3..279632653 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -17,15 +17,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.http.Request` callbacks. - .. note:: The callback output is not processed until the whole callback - finishes. - - As a side effect, if the callback raises an exception, none of its - output is processed. - - This is a known caveat of the current implementation that we aim to - address in a future version of Scrapy. - - The :meth:`process_item` method of :ref:`item pipelines `. @@ -92,6 +83,9 @@ This means you can use many useful Python libraries providing such code:: :mod:`asyncio` loop and to use them you need to :doc:`enable asyncio support in Scrapy`. +.. note:: If you want to ``await`` on Deferreds, you may need to + :ref:`wrap them`. + Common use cases for asynchronous code include: * requesting data from websites, databases and other services (in callbacks, From 7afcd634ab7d71a465cd9406091d371dca105789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 5 Feb 2021 13:04:54 +0100 Subject: [PATCH 0196/2083] Remove unused import --- scrapy/core/http2/stream.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 572dbf7aa..8a1b3e470 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -15,7 +15,6 @@ from twisted.web.client import ResponseFailed from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes -from scrapy.utils.python import to_unicode if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol From 8527b53e14e729504f6e382dd78ef6b0457fc7e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 5 Feb 2021 13:06:27 +0100 Subject: [PATCH 0197/2083] Revert "Use --use-deprecated=legacy-resolver" This reverts commit 7b11b74c77d1535f943baebbf5c794a63d147a13. --- tox.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tox.ini b/tox.ini index ecd3aad6e..d8e900e06 100644 --- a/tox.ini +++ b/tox.ini @@ -27,10 +27,6 @@ passenv = GCS_PROJECT_ID #allow tox virtualenv to upgrade pip/wheel/setuptools download = true -# TODO: Remove the custom install_command below -# Temporary workaround to filter out errors caused by the insanely long time -# that it takes for the new resolver to install dependencies. -install_command=python -m pip install --use-deprecated=legacy-resolver {opts} {packages} commands = py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} From 5cf403295d44bd2531ee3fe2f07057cf155e9804 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 5 Feb 2021 19:40:14 +0500 Subject: [PATCH 0198/2083] Remove a duplicate definition. --- tests/test_spidermiddleware_output_chain.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 4d1a7fcb0..088c14ca8 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -51,15 +51,6 @@ class RecoveryAsyncGenSpider(RecoverySpider): yield r -class RecoveryMiddleware: - def process_spider_exception(self, response, exception, spider): - spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) - return [ - {'from': 'process_spider_exception'}, - Request(response.url, meta={'dont_fail': True}, dont_filter=True), - ] - - # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method class FailProcessSpiderInputMiddleware: From 1e9b52c3e0e4eb51aaad37e796b3ee810fd919a6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 8 Feb 2021 22:02:03 +0500 Subject: [PATCH 0199/2083] Refactor SpiderMiddlewareManager.scrape_response. --- scrapy/core/spidermw.py | 148 +++++++++++++++++++++------------------- 1 file changed, 77 insertions(+), 71 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 763e0cdf6..289292da7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -41,86 +41,92 @@ class SpiderMiddlewareManager(MiddlewareManager): process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) - def scrape_response(self, scrape_func, response, request, spider): - - def process_spider_input(response): - for method in self.methods['process_spider_input']: - try: - result = method(response=response, spider=spider) - if result is not None: - msg = (f"Middleware {_fname(method)} must return None " - f"or raise an exception, got {type(result)}") - raise _InvalidOutput(msg) - except _InvalidOutput: - raise - except Exception: - return scrape_func(Failure(), request, spider) - return scrape_func(response, request, spider) - - def _evaluate_iterable(iterable, exception_processor_index, recover_to): + def _process_spider_input(self, scrape_func, response, request, spider): + for method in self.methods['process_spider_input']: try: - for r in iterable: - yield r + result = method(response=response, spider=spider) + if result is not None: + msg = (f"Middleware {_fname(method)} must return None " + f"or raise an exception, got {type(result)}") + raise _InvalidOutput(msg) + except _InvalidOutput: + raise + except Exception: + return scrape_func(Failure(), request, spider) + return scrape_func(response, request, spider) + + def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to): + try: + for r in iterable: + yield r + except Exception as ex: + exception_result = self._process_spider_exception(response, spider, Failure(ex), + exception_processor_index) + if isinstance(exception_result, Failure): + raise + recover_to.extend(exception_result) + + def _process_spider_exception(self, response, spider, _failure, start_index=0): + exception = _failure.value + # don't handle _InvalidOutput exception + if isinstance(exception, _InvalidOutput): + return _failure + method_list = islice(self.methods['process_spider_exception'], start_index, None) + for method_index, method in enumerate(method_list, start=start_index): + if method is None: + continue + result = method(response=response, exception=exception, spider=spider) + if _isiterable(result): + # stop exception handling by handing control over to the + # process_spider_output chain if an iterable has been returned + return self._process_spider_output(response, spider, result, method_index + 1) + elif result is None: + continue + else: + msg = (f"Middleware {_fname(method)} must return None " + f"or an iterable, got {type(result)}") + raise _InvalidOutput(msg) + return _failure + + def _process_spider_output(self, response, spider, result, start_index=0): + # items in this iterable do not need to go through the process_spider_output + # chain, they went through it already from the process_spider_exception method + recovered = MutableChain() + + method_list = islice(self.methods['process_spider_output'], start_index, None) + for method_index, method in enumerate(method_list, start=start_index): + if method is None: + continue + try: + # might fail directly if the output value is not a generator + result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result = process_spider_exception(Failure(ex), exception_processor_index) + exception_result = self._process_spider_exception(response, spider, Failure(ex), method_index + 1) if isinstance(exception_result, Failure): raise - recover_to.extend(exception_result) + return exception_result + if _isiterable(result): + result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) + else: + msg = (f"Middleware {_fname(method)} must return an " + f"iterable, got {type(result)}") + raise _InvalidOutput(msg) - def process_spider_exception(_failure, start_index=0): - exception = _failure.value - # don't handle _InvalidOutput exception - if isinstance(exception, _InvalidOutput): - return _failure - method_list = islice(self.methods['process_spider_exception'], start_index, None) - for method_index, method in enumerate(method_list, start=start_index): - if method is None: - continue - result = method(response=response, exception=exception, spider=spider) - if _isiterable(result): - # stop exception handling by handing control over to the - # process_spider_output chain if an iterable has been returned - return process_spider_output(result, method_index + 1) - elif result is None: - continue - else: - msg = (f"Middleware {_fname(method)} must return None " - f"or an iterable, got {type(result)}") - raise _InvalidOutput(msg) - return _failure + return MutableChain(result, recovered) - def process_spider_output(result, start_index=0): - # items in this iterable do not need to go through the process_spider_output - # chain, they went through it already from the process_spider_exception method - recovered = MutableChain() - - method_list = islice(self.methods['process_spider_output'], start_index, None) - for method_index, method in enumerate(method_list, start=start_index): - if method is None: - continue - try: - # might fail directly if the output value is not a generator - result = method(response=response, result=result, spider=spider) - except Exception as ex: - exception_result = process_spider_exception(Failure(ex), method_index + 1) - if isinstance(exception_result, Failure): - raise - return exception_result - if _isiterable(result): - result = _evaluate_iterable(result, method_index + 1, recovered) - else: - msg = (f"Middleware {_fname(method)} must return an " - f"iterable, got {type(result)}") - raise _InvalidOutput(msg) - - return MutableChain(result, recovered) + def _process_callback_output(self, response, spider, result): + recovered = MutableChain() + result = self._evaluate_iterable(response, spider, result, 0, recovered) + return MutableChain(self._process_spider_output(response, spider, result), recovered) + def scrape_response(self, scrape_func, response, request, spider): def process_callback_output(result): - recovered = MutableChain() - result = _evaluate_iterable(result, 0, recovered) - return MutableChain(process_spider_output(result), recovered) + return self._process_callback_output(response, spider, result) - dfd = mustbe_deferred(process_spider_input, response) + def process_spider_exception(_failure): + return self._process_spider_exception(response, spider, _failure) + + dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) return dfd From 45345ba6b508fae426223039e491dd721eaac9a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 8 Feb 2021 17:56:29 +0100 Subject: [PATCH 0200/2083] Use constraints.txt to limit pip resolver backtracking --- tests/constraints.txt | 9 ++++++++- tox.ini | 1 - 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/constraints.txt b/tests/constraints.txt index 5655ac2d3..3b30e6bb5 100644 --- a/tests/constraints.txt +++ b/tests/constraints.txt @@ -1 +1,8 @@ -Twisted!=18.4.0 \ No newline at end of file +# Request the latest known version or newer of some dependencies to prevent the +# pip dependency resolver from spending too much time backtracking. +attrs>=20.2.0 +Pillow>=8.0.1 +pytest>=6.2.1 +pytest-twisted>=1.13.1 +sybil>=2.0.0 +Twisted>=19.10.0 diff --git a/tox.ini b/tox.ini index d8e900e06..9908a4d51 100644 --- a/tox.ini +++ b/tox.ini @@ -67,7 +67,6 @@ commands = [pinned] deps = - -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 h2==3.2.0 From bb72bba1786bc6a725df24a01dffba2e6e2cb7c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 8 Feb 2021 21:51:57 +0100 Subject: [PATCH 0201/2083] tox: apply upper constraints to all non-pinned package installations --- tests/{constraints.txt => upper-constraints.txt} | 8 ++++++++ tox.ini | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) rename tests/{constraints.txt => upper-constraints.txt} (64%) diff --git a/tests/constraints.txt b/tests/upper-constraints.txt similarity index 64% rename from tests/constraints.txt rename to tests/upper-constraints.txt index 3b30e6bb5..c8c57deea 100644 --- a/tests/constraints.txt +++ b/tests/upper-constraints.txt @@ -1,8 +1,16 @@ # Request the latest known version or newer of some dependencies to prevent the # pip dependency resolver from spending too much time backtracking. attrs>=20.2.0 +Automat>=0.8.0 +itemadapter>=0.1.1 +itemloaders>=1.0.3 +lxml>=4.6.1 +parsel>=1.5.2 Pillow>=8.0.1 +pyOpenSSL>=20.0.0 pytest>=6.2.1 pytest-twisted>=1.13.1 +service_identity>=17.0.0 +six>=1.14.0 sybil>=2.0.0 Twisted>=19.10.0 diff --git a/tox.ini b/tox.ini index 9908a4d51..2dfe8987c 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,6 @@ minversion = 1.7.0 [testenv] deps = - -ctests/constraints.txt -rtests/requirements-py3.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 @@ -29,6 +28,8 @@ passenv = download = true commands = py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} +install_command = + pip install -U -ctests/upper-constraints.txt {opts} {packages} [testenv:typing] basepython = python3 @@ -90,12 +91,15 @@ deps = Pillow==4.0.0 setenv = _SCRAPY_PINNED=true +install_command = + pip install -U {opts} {packages} [testenv:pinned] deps = {[pinned]deps} lxml==3.5.0 PyDispatcher==2.0.5 +install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -107,6 +111,7 @@ deps = # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 PyDispatcher==2.0.5 +install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -123,6 +128,7 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} commands = {[testenv:asyncio]commands} +install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -138,6 +144,7 @@ deps = lxml==4.0.0 PyPyDispatcher==2.1.0 commands = {[testenv:pypy3]commands} +install_command = {[pinned]install_command} setenv = {[pinned]setenv} From 9ac5b1d021562a20b2a7d437f8bacb6754426811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 8 Feb 2021 22:31:20 +0100 Subject: [PATCH 0202/2083] Adjust test constraints --- tests/upper-constraints.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/upper-constraints.txt b/tests/upper-constraints.txt index c8c57deea..75f337856 100644 --- a/tests/upper-constraints.txt +++ b/tests/upper-constraints.txt @@ -2,12 +2,13 @@ # pip dependency resolver from spending too much time backtracking. attrs>=20.2.0 Automat>=0.8.0 +botocore>=1.20.3 itemadapter>=0.1.1 itemloaders>=1.0.3 lxml>=4.6.1 parsel>=1.5.2 Pillow>=8.0.1 -pyOpenSSL>=20.0.0 +pyOpenSSL>=17.5 # mitmproxy 4.0.4 pytest>=6.2.1 pytest-twisted>=1.13.1 service_identity>=17.0.0 From 15b501c0898150de686ee42dd4d78a411891e795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 10 Feb 2021 18:10:57 +0100 Subject: [PATCH 0203/2083] Do not force string interpolation while logging --- scrapy/core/http2/protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 36a51b898..968bfce63 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -346,7 +346,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) elif isinstance(event, UnknownFrameReceived): - logger.warning(f'Unknown frame received: {event.frame}') + logger.warning('Unknown frame received: %s', event.frame) # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated) -> None: From 67cff0e8a949a83220ae6d7d2c2ca40e6c8a8207 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 10 Feb 2021 22:44:14 +0500 Subject: [PATCH 0204/2083] Silence pylint "naked raise" error. --- scrapy/core/spidermw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 96392aae7..d3d7e5f8c 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -61,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager): exception_result = self._process_spider_exception(response, spider, Failure(ex), exception_processor_index) if isinstance(exception_result, Failure): - raise + raise # pylint: disable=E0704 recover_to.extend(exception_result) def _evaluate_normal_iterable(iterable): From 54fd371481ef0ff80cc23bfab7e7f53126a31bc3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 11 Feb 2021 14:24:11 +0500 Subject: [PATCH 0205/2083] Skip uvloop 0.15.0+ on py36. --- tests/requirements-py3.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index a86c4ae4f..21a554624 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -9,7 +9,8 @@ pytest-twisted >= 1.11 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures -uvloop; platform_system != "Windows" +uvloop < 0.15.0; platform_system != "Windows" and python_version == '3.6' +uvloop; platform_system != "Windows" and python_version > '3.6' # optional for shell wrapper tests bpython From abbbfbbb38a87b9f7259fe47493d5810b2439963 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 12 Feb 2021 22:41:16 +0500 Subject: [PATCH 0206/2083] Add tests for deferred_f_from_coro_f. --- tests/test_utils_defer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index e60242a3b..7a5f458c7 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,8 +1,10 @@ +from pytest import mark from twisted.trial import unittest from twisted.internet import reactor, defer from twisted.python.failure import Failure from scrapy.utils.defer import ( + deferred_f_from_coro_f, iter_errback, mustbe_deferred, process_chain, @@ -117,3 +119,18 @@ class IterErrbackTest(unittest.TestCase): self.assertEqual(out, [0, 1, 2, 3, 4]) self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0].value, ZeroDivisionError) + + +class AsyncDefTestsuiteTest(unittest.TestCase): + @deferred_f_from_coro_f + async def test_deferred_f_from_coro_f(self): + pass + + @deferred_f_from_coro_f + async def test_deferred_f_from_coro_f_generator(self): + yield + + @mark.xfail(reason="Checks that the test is actually executed", strict=True) + @deferred_f_from_coro_f + async def test_deferred_f_from_coro_f_xfail(self): + raise Exception("This is expected to be raised") From e80f37bd3fa0f2262e898f712541bb7a12e89110 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 17 Feb 2021 16:34:29 -0300 Subject: [PATCH 0207/2083] Test http2 agent for unsupported scheme --- tests/test_downloader_handlers_http2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 8f7f7aee0..bee9ae75b 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -3,6 +3,7 @@ from unittest import mock from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server +from twisted.web.error import SchemeNotSupported from scrapy.core.downloader.handlers.http2 import H2DownloadHandler from scrapy.http import Request @@ -48,6 +49,12 @@ class Https2TestCase(Https11TestCase): reactor.callLater(.1, d.callback, logger) yield d + @defer.inlineCallbacks + def test_unsupported_scheme(self): + request = Request("ftp://unsupported.scheme") + d = self.download_request(request, Spider("foo")) + yield self.assertFailure(d, SchemeNotSupported) + def test_download_broken_content_cause_data_loss(self, url='broken'): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) From 4418f78941fc9d89ada9ad0436ebd0ae4ece1017 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 17 Feb 2021 18:36:52 -0300 Subject: [PATCH 0208/2083] Simplify check for negotiated protocol negotiatedProtocol's type is Optional[bytes] See https://github.com/twisted/twisted/blob/twisted-20.3.0/src/twisted/protocols/tls.py#L563-L587 and https://www.pyopenssl.org/en/20.0.1/api/ssl.html#OpenSSL.SSL.Connection.get_alpn_proto_negotiated Note that OpenSSL.SSL.Connection.get_next_proto_negotiated is deprecated: https://www.pyopenssl.org/en/20.0.0/changelog.html#backward-incompatible-changes --- scrapy/core/http2/protocol.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 968bfce63..9d7da14c1 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -233,13 +233,11 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def handshakeCompleted(self) -> None: """We close the connection with InvalidNegotiatedProtocol exception when the connection was not made via h2 protocol""" - negotiated_protocol = self.transport.negotiatedProtocol - if isinstance(negotiated_protocol, bytes): - negotiated_protocol = str(self.transport.negotiatedProtocol, 'utf-8') - if negotiated_protocol != 'h2': + protocol = self.transport.negotiatedProtocol + if protocol is not None and protocol != b"h2": # Here we have not initiated the connection yet # So, no need to send a GOAWAY frame to the remote - self._lose_connection_with_error([InvalidNegotiatedProtocol(negotiated_protocol)]) + self._lose_connection_with_error([InvalidNegotiatedProtocol(protocol.decode("utf-8"))]) def _check_received_data(self, data: bytes) -> None: """Checks for edge cases where the connection to remote fails From 49af7c4c8b71f8643fe13c20261f97323117f980 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 18 Feb 2021 17:10:30 +0500 Subject: [PATCH 0209/2083] Drop pytest-twisted, use Scrapy code to install the reactor. --- conftest.py | 15 +++++++++++++++ pytest.ini | 1 - tests/requirements-py3.txt | 4 +--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/conftest.py b/conftest.py index 68b855c08..e4dd80de0 100644 --- a/conftest.py +++ b/conftest.py @@ -2,6 +2,8 @@ from pathlib import Path import pytest +from scrapy.utils.reactor import install_reactor + from tests.keys import generate_keys @@ -40,6 +42,14 @@ def pytest_collection_modifyitems(session, config, items): pass +def pytest_addoption(parser): + parser.addoption( + "--reactor", + default="default", + choices=["default", "asyncio"], + ) + + @pytest.fixture(scope='class') def reactor_pytest(request): if not request.cls: @@ -55,5 +65,10 @@ def only_asyncio(request, reactor_pytest): pytest.skip('This test is only run with --reactor=asyncio') +def pytest_configure(config): + if config.getoption("--reactor") == "asyncio": + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + + # Generate localhost certificate files, needed by some tests generate_keys() diff --git a/pytest.ini b/pytest.ini index d4deeb57c..0aae09ff5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -18,7 +18,6 @@ addopts = --ignore=docs/topics/stats.rst --ignore=docs/topics/telnetconsole.rst --ignore=docs/utils -twisted = 1 markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed flake8-max-line-length = 119 diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 21a554624..bd72c8c46 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -2,10 +2,8 @@ attrs dataclasses; python_version == '3.6' pyftpdlib -# https://github.com/pytest-dev/pytest-twisted/issues/93 -pytest != 5.4, != 5.4.1 +pytest pytest-cov -pytest-twisted >= 1.11 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures From 40eab1d473a78369702490a901b5d304a40baf3b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 18 Feb 2021 19:56:12 +0500 Subject: [PATCH 0210/2083] Drop a duplicate import. --- tests/test_utils_defer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index d220f969b..62f6ff194 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -5,14 +5,13 @@ from twisted.python.failure import Failure from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import ( + aiter_errback, deferred_f_from_coro_f, iter_errback, - aiter_errback, mustbe_deferred, process_chain, process_chain_both, process_parallel, - deferred_f_from_coro_f, ) From f9a5385146c741825d8c0b089cf8e8318dca9403 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 19 Feb 2021 21:19:21 +0500 Subject: [PATCH 0211/2083] Revert "Move spider middlewares to process_iterable_helper." This reverts commit 92f2c9e308a5eda361229a8e74f74a21b9ff770a. --- scrapy/spidermiddlewares/depth.py | 4 ++-- scrapy/spidermiddlewares/offsite.py | 32 +++++++++++++-------------- scrapy/spidermiddlewares/referer.py | 3 +-- scrapy/spidermiddlewares/urllength.py | 3 +-- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 73079bca9..776a6879a 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -3,10 +3,10 @@ Depth Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + import logging from scrapy.http import Request -from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -55,4 +55,4 @@ class DepthMiddleware: if self.verbose_stats: self.stats.inc_value('request_depth_count/0', spider=spider) - return process_iterable_helper(result or (), in_predicate=_filter) + return (r for r in result or () if _filter(r)) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index e7f481269..6e4efda97 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -10,7 +10,6 @@ import warnings from scrapy import signals from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -27,22 +26,21 @@ class OffsiteMiddleware: return o def process_spider_output(self, response, result, spider): - def in_predicate(x): - if not isinstance(x, Request): - return True - if x.dont_filter or self.should_follow(x, spider): - return True - domain = urlparse_cached(x).hostname - if domain and domain not in self.domains_seen: - self.domains_seen.add(domain) - logger.debug( - "Filtered offsite request to %(domain)r: %(request)s", - {'domain': domain, 'request': x}, extra={'spider': spider}) - self.stats.inc_value('offsite/domains', spider=spider) - self.stats.inc_value('offsite/filtered', spider=spider) - return False - - return process_iterable_helper(result or (), in_predicate=in_predicate) + for x in result: + if isinstance(x, Request): + if x.dont_filter or self.should_follow(x, spider): + yield x + else: + domain = urlparse_cached(x).hostname + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {'domain': domain, 'request': x}, extra={'spider': spider}) + self.stats.inc_value('offsite/domains', spider=spider) + self.stats.inc_value('offsite/filtered', spider=spider) + else: + yield x def should_follow(self, request, spider): regex = self.host_regex diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 91c8727e1..f81041376 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -10,7 +10,6 @@ from w3lib.url import safe_url_string from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals -from scrapy.utils.middlewares import process_iterable_helper from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object from scrapy.utils.url import strip_url @@ -338,7 +337,7 @@ class RefererMiddleware: if referrer is not None: r.headers.setdefault('Referer', referrer) return r - return process_iterable_helper(result or (), processor=_set_referer) + return (_set_referer(r) for r in result or ()) def request_scheduled(self, request, spider): # check redirected request to patch "Referer" header if necessary diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index c7359fecd..5be1f80cb 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -8,7 +8,6 @@ import logging from scrapy.http import Request from scrapy.exceptions import NotConfigured -from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -35,4 +34,4 @@ class UrlLengthMiddleware: else: return True - return process_iterable_helper(result or (), in_predicate=_filter) + return (r for r in result or () if _filter(r)) From 5fc27b1e6fc8532dc5bc08fc5abf5a0daa8ef0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 14:09:06 +0100 Subject: [PATCH 0212/2083] Remove RetrySpiderMixin and retry_request --- scrapy/downloadermiddlewares/retry.py | 55 --------------------------- 1 file changed, 55 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 023ab7d60..5963dacdf 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -75,61 +75,6 @@ def get_retry_request( return None -def retry_request( - request, - *, - reason, - spider, - max_retry_times=None, - priority_adjust=None, -): - new_request = get_retry_request( - request, - reason=reason, - spider=spider, - max_retry_times=max_retry_times, - priority_adjust=priority_adjust, - ) - if new_request: - return [new_request] - return [] - - -class RetrySpiderMixin: - - def get_retry_request( - self, - request, - *, - reason, - max_retry_times=None, - priority_adjust=None, - ): - return get_retry_request( - request, - reason=reason, - spider=self, - max_retry_times=max_retry_times, - priority_adjust=priority_adjust, - ) - - def retry_request( - self, - request, - *, - reason, - max_retry_times=None, - priority_adjust=None, - ): - return retry_request( - request, - reason=reason, - spider=self, - max_retry_times=max_retry_times, - priority_adjust=priority_adjust, - ) - - class RetryMiddleware: # IOError is raised by the HttpCompression middleware when trying to From 825462615a8df8e2274cf71c8a9bbea68c89262a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 14:09:48 +0100 Subject: [PATCH 0213/2083] =?UTF-8?q?get=5Fretry=5Frequest:=20set=20the=20?= =?UTF-8?q?default=20retry=20reason=20to=20=E2=80=9Cunspecified=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/downloadermiddlewares/retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5963dacdf..b8ead12ce 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -34,8 +34,8 @@ logger = logging.getLogger(__name__) def get_retry_request( request, *, - reason, spider, + reason='unspecified', max_retry_times=None, priority_adjust=None, ): From ec836dcc9290f50bad27c874cd0c25a87781735c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 14:15:28 +0100 Subject: [PATCH 0214/2083] Solve style issues --- scrapy/downloadermiddlewares/retry.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index b8ead12ce..5f9bc756c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -69,9 +69,12 @@ def get_retry_request( return new_request else: stats.inc_value('retry/max_reached') - logger.error("Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s", - {'request': request, 'retry_times': retry_times, 'reason': reason}, - extra={'spider': spider}) + logger.error( + "Gave up retrying %(request)s (failed %(retry_times)d times): " + "%(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider}, + ) return None From 6ab990181c6502624ceb0cca6783d99b30d30c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 14:48:03 +0100 Subject: [PATCH 0215/2083] Document get_retry_requests --- docs/topics/downloader-middleware.rst | 17 ++++++++++ docs/topics/settings.rst | 14 -------- scrapy/downloadermiddlewares/retry.py | 49 +++++++++++++++++++++++---- 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 6801adc9c..b539c23df 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -892,6 +892,11 @@ settings (see the settings documentation for more info): If :attr:`Request.meta ` has ``dont_retry`` key set to True, the request will be ignored by this middleware. +To retry requests from a spider callback, you can use the +:func:`get_retry_request` function: + +.. autofunction:: get_retry_request + RetryMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -932,6 +937,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because it is a common code used to indicate server overload. It is not included by default because HTTP specs say so. +.. setting:: RETRY_PRIORITY_ADJUST + +RETRY_PRIORITY_ADJUST +--------------------- + +Default: ``-1`` + +Adjust retry request priority relative to original request: + +- a positive priority adjust means higher priority. +- **a negative priority adjust (default) means lower priority.** + .. _topics-dlmw-robots: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0086a6c74..7c5e9ef6f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1188,20 +1188,6 @@ Adjust redirect request priority relative to original request: - **a positive priority adjust (default) means higher priority.** - a negative priority adjust means lower priority. -.. setting:: RETRY_PRIORITY_ADJUST - -RETRY_PRIORITY_ADJUST ---------------------- - -Default: ``-1`` - -Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware`` - -Adjust retry request priority relative to original request: - -- a positive priority adjust means higher priority. -- **a negative priority adjust (default) means lower priority.** - .. setting:: ROBOTSTXT_OBEY ROBOTSTXT_OBEY diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5f9bc756c..046e3ea71 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -39,16 +39,51 @@ def get_retry_request( max_retry_times=None, priority_adjust=None, ): + """ + Returns a new :class:`~scrapy.Request` object to retry the specified + request, or ``None`` if retries of the specified request have been + exhausted. + + For example, in a :class:`~scrapy.Spider` callback, you could use it as + follows:: + + def parse(self, response): + if not response.text: + new_request = get_retry_request( + response.request, + spider=self, + reason='empty', + ) + if new_request: + yield new_request + return + + *spider* is the :class:`~scrapy.Spider` instance which is asking for the + retry request. It is used to access the :ref:`settings ` + and :ref:`stats `, and to provide extra logging context (see + :func:`logging.debug`). + + *reason* is a string or an :class:`Exception` object that indicates the + reason why the request needs to be retried. It is used to name retry stats. + + *max_retry_times* is a number that determines the maximum number of times + that *request* can be retried. If not specified or ``None``, the number is + read from the :reqmeta:`max_retry_times` meta key of the request. If the + :reqmeta:`max_retry_times` meta key is not defined or ``None``, the number + is read from the :setting:`RETRY_TIMES` setting. + + *priority_adjust* is a number that determines how the priority of the new + request changes in relation to *request*. If not specified, the number is + read from the :setting:`RETRY_PRIORITY_ADJUST` setting. + """ settings = spider.crawler.settings stats = spider.crawler.stats retry_times = request.meta.get('retry_times', 0) + 1 - request_max_retry_times = request.meta.get( - 'max_retry_times', - max_retry_times, - ) - if request_max_retry_times is None: - request_max_retry_times = settings.getint('RETRY_TIMES') - if retry_times <= request_max_retry_times: + if max_retry_times is None: + max_retry_times = request.meta.get('max_retry_times') + if max_retry_times is None: + max_retry_times = settings.getint('RETRY_TIMES') + if retry_times <= max_retry_times: logger.debug( "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", {'request': request, 'retry_times': retry_times, 'reason': reason}, From 80f5003c88f8ee4c5529c1a6d7fc3981efef511d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 16:38:38 +0100 Subject: [PATCH 0216/2083] Add tests for get_retry_request --- scrapy/downloadermiddlewares/retry.py | 13 +- tests/test_downloadermiddleware_retry.py | 498 ++++++++++++++++++++--- 2 files changed, 459 insertions(+), 52 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 046e3ea71..0f24e5d28 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -10,6 +10,7 @@ Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ import logging +from inspect import isclass from twisted.internet import defer from twisted.internet.error import ( @@ -81,8 +82,8 @@ def get_retry_request( retry_times = request.meta.get('retry_times', 0) + 1 if max_retry_times is None: max_retry_times = request.meta.get('max_retry_times') - if max_retry_times is None: - max_retry_times = settings.getint('RETRY_TIMES') + if max_retry_times is None: + max_retry_times = settings.getint('RETRY_TIMES') if retry_times <= max_retry_times: logger.debug( "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", @@ -96,6 +97,8 @@ def get_retry_request( priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') new_request.priority = request.priority + priority_adjust + if isclass(reason): + reason = reason() if isinstance(reason, Exception): reason = global_object_name(reason.__class__) @@ -149,10 +152,12 @@ class RetryMiddleware: return self._retry(request, exception, spider) def _retry(self, request, reason, spider): + max_retry_times = request.meta.get('max_retry_times', self.max_retry_times) + priority_adjust = request.meta.get('priority_adjust', self.priority_adjust) return get_retry_request( request, reason=reason, spider=spider, - max_retry_times=self.max_retry_times, - priority_adjust=self.priority_adjust, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, ) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 364ce0c89..cf01a7dff 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -4,18 +4,22 @@ from twisted.internet.error import ( ConnectError, ConnectionDone, ConnectionLost, - ConnectionRefusedError, DNSLookupError, TCPTimedOutError, - TimeoutError, ) from twisted.web.client import ResponseFailed -from scrapy.downloadermiddlewares.retry import RetryMiddleware -from scrapy.spiders import Spider +from scrapy.downloadermiddlewares.retry import ( + get_retry_request, + RetryMiddleware, +) +from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +from testfixtures import LogCapture + class RetryTest(unittest.TestCase): def setUp(self): @@ -119,82 +123,480 @@ class RetryTest(unittest.TestCase): class MaxRetryTimesTest(unittest.TestCase): - def setUp(self): - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider('foo') - self.mw = RetryMiddleware.from_crawler(self.crawler) - self.mw.max_retry_times = 2 - self.invalid_url = 'http://www.scrapytest.org/invalid_url' + + invalid_url = 'http://www.scrapytest.org/invalid_url' + + def get_spider_and_middleware(self, settings=None): + crawler = get_crawler(Spider, settings or {}) + spider = crawler._create_spider('foo') + middleware = RetryMiddleware.from_crawler(crawler) + return spider, middleware def test_with_settings_zero(self): - - # SETTINGS: RETRY_TIMES = 0 - self.mw.max_retry_times = 0 - + max_retry_times = 0 + settings = {'RETRY_TIMES': max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) req = Request(self.invalid_url) - self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) + self._test_retry( + req, + DNSLookupError('foo'), + max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_metakey_zero(self): - - # SETTINGS: meta(max_retry_times) = 0 - meta_max_retry_times = 0 - - req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) - self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times) + max_retry_times = 0 + spider, middleware = self.get_spider_and_middleware() + meta = {'max_retry_times': max_retry_times} + req = Request(self.invalid_url, meta=meta) + self._test_retry( + req, + DNSLookupError('foo'), + max_retry_times, + spider=spider, + middleware=middleware, + ) def test_without_metakey(self): - - # SETTINGS: RETRY_TIMES is NON-ZERO - self.mw.max_retry_times = 5 - + max_retry_times = 5 + settings = {'RETRY_TIMES': max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) req = Request(self.invalid_url) - self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) + self._test_retry( + req, + DNSLookupError('foo'), + max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_metakey_greater(self): - - # SETINGS: RETRY_TIMES < meta(max_retry_times) - self.mw.max_retry_times = 2 meta_max_retry_times = 3 + middleware_max_retry_times = 2 req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) req2 = Request(self.invalid_url) - self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) - self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + settings = {'RETRY_TIMES': middleware_max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) + + self._test_retry( + req1, + DNSLookupError('foo'), + meta_max_retry_times, + spider=spider, + middleware=middleware, + ) + self._test_retry( + req2, + DNSLookupError('foo'), + middleware_max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_metakey_lesser(self): - - # SETINGS: RETRY_TIMES > meta(max_retry_times) - self.mw.max_retry_times = 5 meta_max_retry_times = 4 + middleware_max_retry_times = 5 req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) req2 = Request(self.invalid_url) - self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) - self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + settings = {'RETRY_TIMES': middleware_max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) + + self._test_retry( + req1, + DNSLookupError('foo'), + meta_max_retry_times, + spider=spider, + middleware=middleware, + ) + self._test_retry( + req2, + DNSLookupError('foo'), + middleware_max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_dont_retry(self): + max_retry_times = 4 + spider, middleware = self.get_spider_and_middleware() + meta = { + 'max_retry_times': max_retry_times, + 'dont_retry': True, + } + req = Request(self.invalid_url, meta=meta) + self._test_retry( + req, + DNSLookupError('foo'), + 0, + spider=spider, + middleware=middleware, + ) - # SETTINGS: meta(max_retry_times) = 4 - meta_max_retry_times = 4 - - req = Request(self.invalid_url, meta={ - 'max_retry_times': meta_max_retry_times, 'dont_retry': True - }) - - self._test_retry(req, DNSLookupError('foo'), 0) - - def _test_retry(self, req, exception, max_retry_times): + def _test_retry( + self, + req, + exception, + max_retry_times, + spider=None, + middleware=None, + ): + spider = spider or self.spider + middleware = middleware or self.mw for i in range(0, max_retry_times): - req = self.mw.process_exception(req, exception, self.spider) + req = middleware.process_exception(req, exception, spider) assert isinstance(req, Request) # discard it - req = self.mw.process_exception(req, exception, self.spider) + req = middleware.process_exception(req, exception, spider) self.assertEqual(req, None) +class GetRetryRequestTest(unittest.TestCase): + + def get_spider(self, settings=None): + crawler = get_crawler(Spider, settings or {}) + return crawler._create_spider('foo') + + def test_basic_usage(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = 1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -1) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_max_retries_reached(self): + request = Request('https://example.com') + spider = self.get_spider() + max_retry_times = 0 + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + self.assertEqual( + spider.crawler.stats.get_value('retry/max_reached'), + 1 + ) + failure_count = max_retry_times + 1 + expected_reason = "unspecified" + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) + ) + + def test_one_retry(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=1, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = 1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -1) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_two_retries(self): + spider = self.get_spider() + request = Request('https://example.com') + new_request = request + max_retry_times = 2 + for index in range(max_retry_times): + with LogCapture() as log: + new_request = get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = index+1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -expected_retry_times) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + value = spider.crawler.stats.get_value(stat) + self.assertEqual(value, expected_retry_times) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + with LogCapture() as log: + new_request = get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + self.assertEqual( + spider.crawler.stats.get_value('retry/max_reached'), + 1 + ) + failure_count = max_retry_times + 1 + expected_reason = "unspecified" + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) + ) + + def test_no_spider(self): + request = Request('https://example.com') + with self.assertRaises(TypeError): + get_retry_request(request) + + def test_max_retry_times_setting(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request, None) + + def test_max_retry_times_meta(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) + meta = {'max_retry_times': max_retry_times} + request = Request('https://example.com', meta=meta) + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request, None) + + def test_max_retry_times_argument(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) + meta = {'max_retry_times': max_retry_times + 1} + request = Request('https://example.com', meta=meta) + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + + def test_priority_adjust_setting(self): + priority_adjust = 1 + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request.priority, priority_adjust) + + def test_priority_adjust_argument(self): + priority_adjust = 1 + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust+1}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + priority_adjust=priority_adjust, + ) + self.assertEqual(new_request.priority, priority_adjust) + + def test_log_extra_retry_success(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture(attributes=('spider',)) as log: + new_request = get_retry_request( + request, + spider=spider, + ) + log.check_present(spider) + + def test_log_extra_retries_exceeded(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture(attributes=('spider',)) as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=0, + ) + log.check_present(spider) + + def test_reason_string(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = 'because' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_builtin_exception(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = NotImplementedError() + expected_reason_string = 'builtins.NotImplementedError' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_builtin_exception_class(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = NotImplementedError + expected_reason_string = 'builtins.NotImplementedError' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_custom_exception(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = IgnoreRequest() + expected_reason_string = 'scrapy.exceptions.IgnoreRequest' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_custom_exception_class(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = IgnoreRequest + expected_reason_string = 'scrapy.exceptions.IgnoreRequest' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + if __name__ == "__main__": unittest.main() From 722a33a2ac8ebc22bb7a7056598898dcb76e98a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 16:42:38 +0100 Subject: [PATCH 0217/2083] Fix style issues --- tests/test_downloadermiddleware_retry.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index cf01a7dff..61c0aaf2f 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -357,7 +357,7 @@ class GetRetryRequestTest(unittest.TestCase): self.assertIsInstance(new_request, Request) self.assertNotEqual(new_request, request) self.assertEqual(new_request.dont_filter, True) - expected_retry_times = index+1 + expected_retry_times = index + 1 self.assertEqual(new_request.meta['retry_times'], expected_retry_times) self.assertEqual(new_request.priority, -expected_retry_times) expected_reason = "unspecified" @@ -445,7 +445,7 @@ class GetRetryRequestTest(unittest.TestCase): def test_priority_adjust_argument(self): priority_adjust = 1 - spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust+1}) + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust + 1}) request = Request('https://example.com') new_request = get_retry_request( request, @@ -458,7 +458,7 @@ class GetRetryRequestTest(unittest.TestCase): request = Request('https://example.com') spider = self.get_spider() with LogCapture(attributes=('spider',)) as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, ) @@ -468,7 +468,7 @@ class GetRetryRequestTest(unittest.TestCase): request = Request('https://example.com') spider = self.get_spider() with LogCapture(attributes=('spider',)) as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, max_retry_times=0, @@ -480,7 +480,7 @@ class GetRetryRequestTest(unittest.TestCase): spider = self.get_spider() expected_reason = 'because' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, @@ -503,7 +503,7 @@ class GetRetryRequestTest(unittest.TestCase): expected_reason = NotImplementedError() expected_reason_string = 'builtins.NotImplementedError' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, @@ -528,7 +528,7 @@ class GetRetryRequestTest(unittest.TestCase): expected_reason = NotImplementedError expected_reason_string = 'builtins.NotImplementedError' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, @@ -553,7 +553,7 @@ class GetRetryRequestTest(unittest.TestCase): expected_reason = IgnoreRequest() expected_reason_string = 'scrapy.exceptions.IgnoreRequest' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, @@ -578,7 +578,7 @@ class GetRetryRequestTest(unittest.TestCase): expected_reason = IgnoreRequest expected_reason_string = 'scrapy.exceptions.IgnoreRequest' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, From 1f7665c4cfb955ca4e81d7bdb249cd88ada788c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 16:48:10 +0100 Subject: [PATCH 0218/2083] Silence a PyLint check on a mistake made for testing purposes --- tests/test_downloadermiddleware_retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 61c0aaf2f..46e525f99 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -398,7 +398,7 @@ class GetRetryRequestTest(unittest.TestCase): def test_no_spider(self): request = Request('https://example.com') with self.assertRaises(TypeError): - get_retry_request(request) + get_retry_request(request) # pylint: disable=missing-kwoa def test_max_retry_times_setting(self): max_retry_times = 0 From 6326178bc5824e8c08d84b6543de6977a16fb8d2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 22 Feb 2021 12:50:51 -0300 Subject: [PATCH 0219/2083] http2: acceptable protocol update, tests (#4994) --- scrapy/core/http2/protocol.py | 26 ++++++++++++++------------ tests/test_http2_client_protocol.py | 8 ++++++++ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 9d7da14c1..6ca69b23b 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -33,13 +33,16 @@ from scrapy.spiders import Spider logger = logging.getLogger(__name__) +PROTOCOL_NAME = b"h2" + + class InvalidNegotiatedProtocol(H2Error): - def __init__(self, negotiated_protocol: str) -> None: + def __init__(self, negotiated_protocol: bytes) -> None: self.negotiated_protocol = negotiated_protocol def __str__(self) -> str: - return f'InvalidNegotiatedProtocol: Expected h2 as negotiated protocol, received {self.negotiated_protocol!r}' + return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}") class RemoteTerminatedConnection(H2Error): @@ -52,7 +55,7 @@ class RemoteTerminatedConnection(H2Error): self.terminate_event = event def __str__(self) -> str: - return f'RemoteTerminatedConnection: Received GOAWAY frame from {self.remote_ip_address!r}' + return f'Received GOAWAY frame from {self.remote_ip_address!r}' class MethodNotAllowed405(H2Error): @@ -60,7 +63,7 @@ class MethodNotAllowed405(H2Error): self.remote_ip_address = remote_ip_address def __str__(self) -> str: - return f"MethodNotAllowed405: Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}" + return f"Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}" @implementer(IHandshakeListener) @@ -231,13 +234,12 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.transport.loseConnection() def handshakeCompleted(self) -> None: - """We close the connection with InvalidNegotiatedProtocol exception - when the connection was not made via h2 protocol""" - protocol = self.transport.negotiatedProtocol - if protocol is not None and protocol != b"h2": - # Here we have not initiated the connection yet - # So, no need to send a GOAWAY frame to the remote - self._lose_connection_with_error([InvalidNegotiatedProtocol(protocol.decode("utf-8"))]) + """ + Close the connection if it's not made via the expected protocol + """ + if self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME: + # we have not initiated the connection yet, no need to send a GOAWAY frame to the remote peer + self._lose_connection_with_error([InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)]) def _check_received_data(self, data: bytes) -> None: """Checks for edge cases where the connection to remote fails @@ -414,4 +416,4 @@ class H2ClientFactory(Factory): return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) def acceptableProtocols(self) -> List[bytes]: - return [b'h2'] + return [PROTOCOL_NAME] diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index d9ab553f0..8b2f6a11d 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,6 +5,7 @@ import re import shutil import string from ipaddress import IPv4Address +from unittest import mock from urllib.parse import urlencode from h2.exceptions import InvalidBodyLengthError @@ -381,6 +382,13 @@ class Https2ClientProtocolTestCase(TestCase): 200 ) + @inlineCallbacks + def test_invalid_negotiated_protocol(self): + with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"): + request = Request(url=self.get_url('/status?n=200')) + with self.assertRaises(ResponseFailed): + yield self.make_request(request) + def test_cancel_request(self): request = Request(url=self.get_url('/get-data-html-large')) From 7605f19ec429fd3adb6a18da8e48143f99f6bfec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 23 Feb 2021 05:54:48 +0100 Subject: [PATCH 0220/2083] HTTP/2: test 2 concurrent requests to the same domain --- tests/test_downloader_handlers_http2.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index bee9ae75b..44d45b7d8 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -73,6 +73,21 @@ class Https2TestCase(Https11TestCase): def test_download_broken_chunked_content_allow_data_loss_via_setting(self): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + def test_concurrent_requests_same_domain(self): + spider = Spider('foo') + + request1 = Request(self.getURL('file')) + d1 = self.download_request(request1, spider) + d1.addCallback(lambda r: r.body) + d1.addCallback(self.assertEqual, b"0123456789") + + request2 = Request(self.getURL('echo'), method='POST') + d2 = self.download_request(request2, spider) + d2.addCallback(lambda r: r.headers['Content-Length']) + d2.addCallback(self.assertEqual, b"79") + + return defer.DeferredList([d1, d2]) + class Https2WrongHostnameTestCase(Https2TestCase): tls_log_message = ( From bd29f32dee445c77e7f2427a3047b7c74505efb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 23 Feb 2021 06:42:28 +0100 Subject: [PATCH 0221/2083] HTTP/2: do not make conn_lost_deferred optional --- scrapy/core/http2/protocol.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 6ca69b23b..1d150b7ce 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -70,7 +70,7 @@ class MethodNotAllowed405(H2Error): class H2ClientProtocol(Protocol, TimeoutMixin): IDLE_TIMEOUT = 240 - def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None: """ Arguments: uri -- URI of the base url to which HTTP/2 Connection will be made. @@ -308,8 +308,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): if not reason.check(connectionDone): self._conn_lost_errors.append(reason) - if self._conn_lost_deferred: - self._conn_lost_deferred.callback(self._conn_lost_errors) + self._conn_lost_deferred.callback(self._conn_lost_errors) for stream in self.streams.values(): if stream.metadata['request_sent']: @@ -407,7 +406,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): @implementer(IProtocolNegotiationFactory) class H2ClientFactory(Factory): - def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Optional[Deferred] = None) -> None: + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None: self.uri = uri self.settings = settings self.conn_lost_deferred = conn_lost_deferred From 5ba31cd1a268475db1f3cc64d4e6febb477e8f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 23 Feb 2021 11:57:33 +0100 Subject: [PATCH 0222/2083] HTTP/2 stream close reason handling: Use else + assert instead of elif --- scrapy/core/http2/stream.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8a1b3e470..aa44c08ce 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -431,7 +431,8 @@ class Stream: errors.insert(0, InactiveStreamClosed(self._request)) self._deferred_response.errback(ResponseFailed(errors)) - elif reason is StreamCloseReason.INVALID_HOSTNAME: + else: + assert reason is StreamCloseReason.INVALID_HOSTNAME self._deferred_response.errback(InvalidHostname( self._request, str(self._protocol.metadata['uri'].host, 'utf-8'), From 510109420733e93db0d525302555224a8364ed5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 24 Feb 2021 07:33:39 +0100 Subject: [PATCH 0223/2083] HTTP/2: test a CONNECT request --- tests/test_downloader_handlers_http2.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 44d45b7d8..b5a40468a 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -1,5 +1,6 @@ from unittest import mock +from pytest import mark from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server @@ -88,6 +89,14 @@ class Https2TestCase(Https11TestCase): return defer.DeferredList([d1, d2]) + @mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") + def test_connect_request(self): + request = Request(self.getURL('file'), method='CONNECT') + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b'') + return d + class Https2WrongHostnameTestCase(Https2TestCase): tls_log_message = ( From a36f952198101bfdfeadf4bec079db120809dbb1 Mon Sep 17 00:00:00 2001 From: Wehzie <39304339+Wehzie@users.noreply.github.com> Date: Wed, 24 Feb 2021 08:15:44 +0100 Subject: [PATCH 0224/2083] fixed typo "an quotes.json" -> "a quotes.json" (#5005) --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 9270ff42c..740e47d0c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -464,7 +464,7 @@ The simplest way to store the scraped data is by using :ref:`Feed exports scrapy crawl quotes -O quotes.json -That will generate an ``quotes.json`` file containing all scraped items, +That will generate a ``quotes.json`` file containing all scraped items, serialized in `JSON`_. The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead From 12064d799b8f15eef770a615237932b880b594a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 24 Feb 2021 10:37:38 +0100 Subject: [PATCH 0225/2083] HTTP/2: improve header handling --- scrapy/core/http2/stream.py | 21 ++++++++--- tests/test_downloader_handlers_http2.py | 46 +++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index aa44c08ce..8a701e7c6 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -220,11 +220,24 @@ class Stream: (':path', path), ] - for name, value in self._request.headers.items(): - headers.append((str(name, 'utf-8'), str(value[0], 'utf-8'))) + content_length = str(len(self._request.body)) + headers.append(('Content-Length', content_length)) - if b'Content-Length' not in self._request.headers.keys(): - headers.append(('Content-Length', str(len(self._request.body)))) + content_length_name = self._request.headers.normkey(b'Content-Length') + for name, values in self._request.headers.items(): + for value in values: + value = str(value, 'utf-8') + if name == content_length_name: + if value != content_length: + logger.warning( + 'Ignoring bad Content-Length header %r of request %r, ' + 'sending %r instead', + value, + self._request, + content_length, + ) + continue + headers.append((str(name, 'utf-8'), value)) return headers diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index b5a40468a..7c3db5835 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -1,6 +1,8 @@ +import json from unittest import mock from pytest import mark +from testfixtures import LogCapture from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server @@ -97,6 +99,50 @@ class Https2TestCase(Https11TestCase): d.addCallback(self.assertEqual, b'') return d + def test_custom_content_length_good(self): + request = Request(self.getURL('contentlength')) + custom_content_length = str(len(request.body)) + request.headers['Content-Length'] = custom_content_length + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.text) + d.addCallback(self.assertEqual, custom_content_length) + return d + + def test_custom_content_length_bad(self): + request = Request(self.getURL('contentlength')) + actual_content_length = str(len(request.body)) + bad_content_length = str(len(request.body)+1) + request.headers['Content-Length'] = bad_content_length + log = LogCapture() + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.text) + d.addCallback(self.assertEqual, actual_content_length) + d.addCallback( + lambda _: log.check_present( + ( + 'scrapy.core.http2.stream', + 'WARNING', + f'Ignoring bad Content-Length header ' + f'{bad_content_length!r} of request {request}, sending ' + f'{actual_content_length!r} instead', + ) + ) + ) + d.addCallback( + lambda _: log.uninstall() + ) + return d + + def test_duplicate_header(self): + request = Request(self.getURL('echo')) + header, value1, value2 = 'Custom-Header', 'foo', 'bar' + request.headers.appendlist(header, value1) + request.headers.appendlist(header, value2) + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: json.loads(r.text)['headers'][header]) + d.addCallback(self.assertEqual, [value1, value2]) + return d + class Https2WrongHostnameTestCase(Https2TestCase): tls_log_message = ( From 386e2a51ae4ed7bd53374a5cadfdf380b58284ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 24 Feb 2021 10:41:01 +0100 Subject: [PATCH 0226/2083] tests/test_downloader_handlers_http2.py: fix style issue --- tests/test_downloader_handlers_http2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 7c3db5835..439778014 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -111,7 +111,7 @@ class Https2TestCase(Https11TestCase): def test_custom_content_length_bad(self): request = Request(self.getURL('contentlength')) actual_content_length = str(len(request.body)) - bad_content_length = str(len(request.body)+1) + bad_content_length = str(len(request.body) + 1) request.headers['Content-Length'] = bad_content_length log = LogCapture() d = self.download_request(request, Spider('foo')) From 3894ebb1497b32959c405201f2e010292cf65098 Mon Sep 17 00:00:00 2001 From: Djiar Date: Tue, 23 Feb 2021 15:34:53 +0100 Subject: [PATCH 0227/2083] Refactor curl_to_request_kwargs #5001 Co-authored-by: alkazaz alkazaz@kth.se Co-authored-by: swill swill@kth.se Co-authored-by: lerjevik lerjevik@kth.se Co-authored-by: aljica aljica@kth.se --- scrapy/utils/curl.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 6660b9dc0..d8b3deaa1 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -34,6 +34,26 @@ for argument in safe_to_ignore_arguments: curl_parser.add_argument(*argument, action='store_true') +def _parse_headers_and_cookies(parsed_args): + headers = [] + cookies = {} + for header in parsed_args.headers or (): + name, val = header.split(':', 1) + name = name.strip() + val = val.strip() + if name.title() == 'Cookie': + for name, morsel in SimpleCookie(val).items(): + cookies[name] = morsel.value + else: + headers.append((name, val)) + + if parsed_args.auth: + user, password = parsed_args.auth.split(':', 1) + headers.append(('Authorization', basic_auth_header(user, password))) + + return headers, cookies + + def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): """Convert a cURL command syntax to Request kwargs. @@ -70,21 +90,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): result = {'method': method.upper(), 'url': url} - headers = [] - cookies = {} - for header in parsed_args.headers or (): - name, val = header.split(':', 1) - name = name.strip() - val = val.strip() - if name.title() == 'Cookie': - for name, morsel in SimpleCookie(val).items(): - cookies[name] = morsel.value - else: - headers.append((name, val)) - - if parsed_args.auth: - user, password = parsed_args.auth.split(':', 1) - headers.append(('Authorization', basic_auth_header(user, password))) + headers, cookies = _parse_headers_and_cookies(parsed_args) if headers: result['headers'] = headers From f689615e8d61f1f651b869b7a6284ca6ca15cde7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 24 Feb 2021 12:54:56 +0100 Subject: [PATCH 0228/2083] Close files in the PerYearXmlExportPipeline documentation example --- docs/topics/exporters.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 0a0a1765a..8648daded 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -50,18 +50,19 @@ value of one of their fields:: self.year_to_exporter = {} def close_spider(self, spider): - for exporter in self.year_to_exporter.values(): + for exporter, xml_file in self.year_to_exporter.values(): exporter.finish_exporting() + xml_file.close() def _exporter_for_item(self, item): adapter = ItemAdapter(item) year = adapter['year'] if year not in self.year_to_exporter: - f = open(f'{year}.xml', 'wb') - exporter = XmlItemExporter(f) + xml_file = open(f'{year}.xml', 'wb') + exporter = XmlItemExporter(xml_file) exporter.start_exporting() - self.year_to_exporter[year] = exporter - return self.year_to_exporter[year] + self.year_to_exporter[year] = (exporter, xml_file) + return self.year_to_exporter[year][0] def process_item(self, item, spider): exporter = self._exporter_for_item(item) From 7a54580679f192c4f1b66775aaddf1bee1efe448 Mon Sep 17 00:00:00 2001 From: deepang17 <47976918+deepang17@users.noreply.github.com> Date: Sun, 28 Feb 2021 15:33:09 +0530 Subject: [PATCH 0229/2083] DOCS:Cover scrapy-bench in the documentation --- docs/topics/benchmarking.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index b01a66188..4e53900ee 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,5 +81,4 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -In the future, more cases will be added to the benchmarking suite to cover -other common scenarios. +To use it as a project for more complex Scrapy benchmarking: https://github.com/scrapy/scrapy-bench From b25616d107d88bb195f19dca4c7e886a9ba652d3 Mon Sep 17 00:00:00 2001 From: deepang17 <47976918+deepang17@users.noreply.github.com> Date: Sun, 28 Feb 2021 16:26:46 +0530 Subject: [PATCH 0230/2083] DOCS: Cover scrapy-bench in the documentation --- docs/topics/benchmarking.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 4e53900ee..b15836771 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,4 +81,5 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -To use it as a project for more complex Scrapy benchmarking: https://github.com/scrapy/scrapy-bench +To use it as a project for more complex Scrapy benchmarking: +https://github.com/scrapy/scrapy-bench From 3c5668d0db4b11836bafcc91b12aba911e71f104 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Mon, 1 Mar 2021 22:00:33 -0500 Subject: [PATCH 0231/2083] docs: Clarify there's one extension instance per spider --- docs/topics/extensions.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 519f18b63..9e86fd0fe 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -7,8 +7,7 @@ Extensions The extensions framework provides a mechanism for inserting your own custom functionality into Scrapy. -Extensions are just regular classes that are instantiated at Scrapy startup, -when extensions are initialized. +Extensions are just regular classes. Extension settings ================== @@ -27,8 +26,8 @@ Loading & activating extensions =============================== Extensions are loaded and activated at startup by instantiating a single -instance of the extension class. Therefore, all the extension initialization -code must be performed in the class ``__init__`` method. +instance of the extension class per spider being run. All the extension +initialization code must be performed in the class ``__init__`` method. To make an extension available, add it to the :setting:`EXTENSIONS` setting in your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented From 9e62355271fa39e67b06f00f5601cdb848c7894e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 2 Mar 2021 12:09:10 -0300 Subject: [PATCH 0232/2083] Allow logger/stats customization in get_retry_request --- scrapy/downloadermiddlewares/retry.py | 16 ++++++--- tests/test_downloadermiddleware_retry.py | 44 ++++++++++++++++++++---- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 0f24e5d28..8955c7e4f 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -29,7 +29,8 @@ from scrapy.utils.response import response_status_message from scrapy.core.downloader.handlers.http11 import TunnelError from scrapy.utils.python import global_object_name -logger = logging.getLogger(__name__) + +retry_logger = logging.getLogger(__name__) def get_retry_request( @@ -39,6 +40,8 @@ def get_retry_request( reason='unspecified', max_retry_times=None, priority_adjust=None, + logger=retry_logger, + stats_base_key='retry', ): """ Returns a new :class:`~scrapy.Request` object to retry the specified @@ -76,6 +79,11 @@ def get_retry_request( *priority_adjust* is a number that determines how the priority of the new request changes in relation to *request*. If not specified, the number is read from the :setting:`RETRY_PRIORITY_ADJUST` setting. + + *logger* is the logging.Logger object to be used when logging messages + + *stats_base_key* is a string to be used as the base key for the + retry-related job stats """ settings = spider.crawler.settings stats = spider.crawler.stats @@ -102,11 +110,11 @@ def get_retry_request( if isinstance(reason, Exception): reason = global_object_name(reason.__class__) - stats.inc_value('retry/count') - stats.inc_value(f'retry/reason_count/{reason}') + stats.inc_value(f'{stats_base_key}/count') + stats.inc_value(f'{stats_base_key}/reason_count/{reason}') return new_request else: - stats.inc_value('retry/max_reached') + stats.inc_value(f'{stats_base_key}/max_reached') logger.error( "Gave up retrying %(request)s (failed %(retry_times)d times): " "%(reason)s", diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 46e525f99..915bd3a3e 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,4 +1,7 @@ +import logging import unittest + +from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.error import ( ConnectError, @@ -9,17 +12,12 @@ from twisted.internet.error import ( ) from twisted.web.client import ResponseFailed -from scrapy.downloadermiddlewares.retry import ( - get_retry_request, - RetryMiddleware, -) +from scrapy.downloadermiddlewares.retry import get_retry_request, RetryMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from testfixtures import LogCapture - class RetryTest(unittest.TestCase): def setUp(self): @@ -597,6 +595,40 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + def test_custom_logger(self): + logger = logging.getLogger("custom-logger") + request = Request("https://example.com") + spider = self.get_spider() + expected_reason = "because" + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + logger=logger, + ) + log.check_present( + ( + "custom-logger", + "DEBUG", + f"Retrying {request} (failed 1 times): {expected_reason}", + ) + ) + + def test_custom_stats_key(self): + request = Request("https://example.com") + spider = self.get_spider() + expected_reason = "because" + stats_key = "custom_retry" + get_retry_request( + request, + spider=spider, + reason=expected_reason, + stats_base_key=stats_key, + ) + for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + if __name__ == "__main__": unittest.main() From 36f1dbf665abd8935c3adda9a5abfef959573cdb Mon Sep 17 00:00:00 2001 From: deepang17 <47976918+deepang17@users.noreply.github.com> Date: Tue, 2 Mar 2021 22:12:44 +0530 Subject: [PATCH 0233/2083] DOCS: Covered scrapy-bench --- .vscode/settings.json | 3 +++ docs/topics/benchmarking.rst | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..500bc7007 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.linting.pylintEnabled": true +} \ No newline at end of file diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index b15836771..3e671365b 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,5 +81,6 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -To use it as a project for more complex Scrapy benchmarking: -https://github.com/scrapy/scrapy-bench +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench + +Use scrapy-bench_ for more complex benchmarking. \ No newline at end of file From 4fe26ae9701c95a05723a79649314da03e3ddc1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 2 Mar 2021 19:46:34 +0100 Subject: [PATCH 0234/2083] Limit tests to Twisted < 21 --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index e70aef2d2..69f52bd9f 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,8 @@ deps = # Extras botocore>=1.4.87 Pillow>=4.0.0 + # Twisted 21+ causes issues in tests that use skipIf + Twisted<21 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From 3d88ac605b04f4c60ca81ce509c0c3b25cfb8cd7 Mon Sep 17 00:00:00 2001 From: deepang17 <47976918+deepang17@users.noreply.github.com> Date: Tue, 9 Mar 2021 17:19:34 +0530 Subject: [PATCH 0235/2083] FIX: Updated benchmarking.rst --- .vscode/settings.json | 3 --- docs/topics/benchmarking.rst | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 500bc7007..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.linting.pylintEnabled": true -} \ No newline at end of file diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 3e671365b..0643df6a6 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,6 +81,6 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -.. _scrapy-bench: https://github.com/scrapy/scrapy-bench +Use scrapy-bench_ for more complex benchmarking. -Use scrapy-bench_ for more complex benchmarking. \ No newline at end of file +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench \ No newline at end of file From 3bea5e1a974b7cd99f75edb4ff728a9d7163a805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 9 Mar 2021 16:19:51 +0100 Subject: [PATCH 0236/2083] Remove unused _is_data_lost method --- scrapy/core/http2/stream.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 8a701e7c6..c2a4b702f 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -362,14 +362,6 @@ class Stream: self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) self.close(reason) - def _is_data_lost(self) -> bool: - assert self.metadata['stream_closed_server'] - - expected_size = self._response['flow_controlled_size'] - received_body_size = int(self._response['headers'][b'Content-Length']) - - return expected_size != received_body_size - def close( self, reason: StreamCloseReason, From 0c160882306a8e33f92815cc12dc8979f04f8f5e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 11 Mar 2021 11:52:35 -0300 Subject: [PATCH 0237/2083] headers_received signal (#4897) --- docs/faq.rst | 11 +-- docs/topics/exceptions.rst | 7 +- docs/topics/request-response.rst | 6 +- docs/topics/signals.rst | 32 +++++++- scrapy/core/downloader/handlers/http11.py | 24 ++++++ scrapy/signals.py | 1 + tests/spiders.py | 29 ++++++++ tests/test_crawl.py | 26 ++++++- tests/test_engine.py | 85 ++++++---------------- tests/test_engine_stop_download_bytes.py | 60 +++++++++++++++ tests/test_engine_stop_download_headers.py | 56 ++++++++++++++ 11 files changed, 260 insertions(+), 77 deletions(-) create mode 100644 tests/test_engine_stop_download_bytes.py create mode 100644 tests/test_engine_stop_download_headers.py diff --git a/docs/faq.rst b/docs/faq.rst index f492dfa30..9709885f6 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -398,11 +398,12 @@ How can I cancel the download of a given response? -------------------------------------------------- In some situations, it might be useful to stop the download of a certain response. -For instance, if you only need the first part of a large response and you would like -to save resources by avoiding the download of the whole body. -In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received` -signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to -the :ref:`topics-stop-response-download` topic for additional information and examples. +For instance, sometimes you can determine whether or not you need the full contents +of a response by inspecting its headers or the first bytes of its body. In that case, +you could save resources by attaching a handler to the :class:`~scrapy.signals.bytes_received` +or :class:`~scrapy.signals.headers_received` signals and raising a +:exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the +:ref:`topics-stop-response-download` topic for additional information and examples. .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 583a50ab8..2f1517906 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -85,8 +85,8 @@ StopDownload .. exception:: StopDownload(fail=True) -Raised from a :class:`~scrapy.signals.bytes_received` signal handler to -indicate that no further bytes should be downloaded for a response. +Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received` +signal handler to indicate that no further bytes should be downloaded for a response. The ``fail`` boolean parameter controls which method will handle the resulting response: @@ -110,5 +110,6 @@ attribute. ``StopDownload(False)`` or ``StopDownload(True)`` will raise a :class:`TypeError`. -See the documentation for the :class:`~scrapy.signals.bytes_received` signal +See the documentation for the :class:`~scrapy.signals.bytes_received` and +:class:`~scrapy.signals.headers_received` signals and the :ref:`topics-stop-response-download` topic for additional information and examples. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 98906992d..37008f3e9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -432,9 +432,9 @@ The meta key is used set retry times per request. When initialized, the Stopping the download of a Response =================================== -Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a -:class:`~scrapy.signals.bytes_received` signal handler will stop the -download of a given response. See the following example:: +Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the +:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received` +signals will stop the download of a given response. See the following example:: import scrapy diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 1d99d8c28..98cfa606c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -384,6 +384,11 @@ bytes_received a possible scenario for a 25 kb response would be two signals fired with 10 kb of data, and a final one with 5 kb of data. + Handlers for this signal can stop the download of a response while it + is in progress by raising the :exc:`~scrapy.exceptions.StopDownload` + exception. Please refer to the :ref:`topics-stop-response-download` topic + for additional information and examples. + This signal does not support returning deferreds from its handlers. :param data: the data received by the download handler @@ -395,11 +400,36 @@ bytes_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.spiders.Spider` object -.. note:: Handlers of this signal can stop the download of a response while it +headers_received +~~~~~~~~~~~~~~~~ + +.. versionadded:: VERSION + +.. signal:: headers_received +.. function:: headers_received(headers, request, spider) + + Sent by the HTTP 1.1 and S3 download handlers when the response headers are + available for a given request, before downloading any additional content. + + Handlers for this signal can stop the download of a response while it is in progress by raising the :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the :ref:`topics-stop-response-download` topic for additional information and examples. + This signal does not support returning deferreds from its handlers. + + :param headers: the headers received by the download handler + :type headers: :class:`scrapy.http.headers.Headers` object + + :param body_length: expected size of the response body, in bytes + :type body_length: `int` + + :param request: the request that generated the download + :type request: :class:`~scrapy.http.Request` object + + :param spider: the spider associated with the response + :type spider: :class:`~scrapy.spiders.Spider` object + Response signals ---------------- diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 513df2de9..516a4326b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -382,6 +382,29 @@ class ScrapyAgent: return result def _cb_bodyready(self, txresponse, request): + headers_received_result = self._crawler.signals.send_catch_log( + signal=signals.headers_received, + headers=Headers(txresponse.headers.getAllRawHeaders()), + body_length=txresponse.length, + request=request, + spider=self._crawler.spider, + ) + for handler, result in headers_received_result: + if isinstance(result, Failure) and isinstance(result.value, StopDownload): + logger.debug("Download stopped for %(request)s from signal handler %(handler)s", + {"request": request, "handler": handler.__qualname__}) + txresponse._transport.stopProducing() + with suppress(AttributeError): + txresponse._transport._producer.loseConnection() + return { + "txresponse": txresponse, + "body": b"", + "flags": ["download_stopped"], + "certificate": None, + "ip_address": None, + "failure": result if result.value.fail else None, + } + # deliverBody hangs for responses without body if txresponse.length == 0: return { @@ -529,6 +552,7 @@ class _ResponseReader(protocol.Protocol): if isinstance(result, Failure) and isinstance(result.value, StopDownload): logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": self._request, "handler": handler.__qualname__}) + self.transport.stopProducing() self.transport._producer.loseConnection() failure = result if result.value.fail else None self._finish_response(flags=["download_stopped"], failure=failure) diff --git a/scrapy/signals.py b/scrapy/signals.py index c61ae6ec3..8cf2a4d93 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -17,6 +17,7 @@ request_reached_downloader = object() request_left_downloader = object() response_received = object() response_downloaded = object() +headers_received = object() bytes_received = object() item_scraped = object() item_dropped = object() diff --git a/tests/spiders.py b/tests/spiders.py index 106392ea6..7e579098a 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -390,3 +390,32 @@ class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider): def bytes_received(self, data, request, spider): self.meta["bytes_received"] = data raise StopDownload(fail=True) + + +class HeadersReceivedCallbackSpider(MetaSpider): + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + crawler.signals.connect(spider.headers_received, signals.headers_received) + return spider + + def start_requests(self): + yield Request(self.mockserver.url("/status"), errback=self.errback) + + def parse(self, response): + self.meta["response"] = response + + def errback(self, failure): + self.meta["failure"] = failure + + def headers_received(self, headers, body_length, request, spider): + self.meta["headers_received"] = headers + raise StopDownload(fail=False) + + +class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider): + + def headers_received(self, headers, body_length, request, spider): + self.meta["headers_received"] = headers + raise StopDownload(fail=True) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1083c1678..84bac9b50 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -35,6 +35,8 @@ from tests.spiders import ( DelaySpider, DuplicateStartRequestsSpider, FollowAllSpider, + HeadersReceivedCallbackSpider, + HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, ) @@ -496,7 +498,7 @@ class CrawlSpiderTestCase(TestCase): self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) @defer.inlineCallbacks - def test_stop_download_callback(self): + def test_bytes_received_stop_download_callback(self): crawler = self.runner.create_crawler(BytesReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("failure")) @@ -505,7 +507,7 @@ class CrawlSpiderTestCase(TestCase): self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length) @defer.inlineCallbacks - def test_stop_download_errback(self): + def test_bytes_received_stop_download_errback(self): crawler = self.runner.create_crawler(BytesReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("response")) @@ -518,3 +520,23 @@ class CrawlSpiderTestCase(TestCase): self.assertLess( len(crawler.spider.meta["failure"].value.response.body), crawler.spider.full_response_length) + + @defer.inlineCallbacks + def test_headers_received_stop_download_callback(self): + crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider) + yield crawler.crawl(mockserver=self.mockserver) + self.assertIsNone(crawler.spider.meta.get("failure")) + self.assertIsInstance(crawler.spider.meta["response"], Response) + self.assertEqual(crawler.spider.meta["response"].headers, crawler.spider.meta.get("headers_received")) + + @defer.inlineCallbacks + def test_headers_received_stop_download_errback(self): + crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider) + yield crawler.crawl(mockserver=self.mockserver) + self.assertIsNone(crawler.spider.meta.get("response")) + self.assertIsInstance(crawler.spider.meta["failure"], Failure) + self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) + self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) + self.assertEqual( + crawler.spider.meta["failure"].value.response.headers, + crawler.spider.meta.get("headers_received")) diff --git a/tests/test_engine.py b/tests/test_engine.py index 3629aa1aa..ef1204f94 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -19,14 +19,12 @@ from urllib.parse import urlparse import attr from itemadapter import ItemAdapter from pydispatch import dispatcher -from testfixtures import LogCapture from twisted.internet import defer, reactor from twisted.trial import unittest from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor @@ -143,6 +141,7 @@ class CrawlerRun: self.reqreached = [] self.itemerror = [] self.itemresp = [] + self.headers = {} self.bytes = defaultdict(lambda: list()) self.signals_caught = {} self.spider_class = spider_class @@ -165,6 +164,7 @@ class CrawlerRun: self.crawler = get_crawler(self.spider_class) self.crawler.signals.connect(self.item_scraped, signals.item_scraped) self.crawler.signals.connect(self.item_error, signals.item_error) + self.crawler.signals.connect(self.headers_received, signals.headers_received) self.crawler.signals.connect(self.bytes_received, signals.bytes_received) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) @@ -183,6 +183,7 @@ class CrawlerRun: if not name.startswith('_'): disconnect_all(signal) self.deferred.callback(None) + return self.crawler.stop() def geturl(self, path): return f"http://localhost:{self.portno}{path}" @@ -197,6 +198,9 @@ class CrawlerRun: def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) + def headers_received(self, headers, body_length, request, spider): + self.headers[request] = headers + def bytes_received(self, data, request, spider): self.bytes[request].append(data) @@ -220,18 +224,7 @@ class CrawlerRun: self.signals_caught[sig] = signalargs -class StopDownloadCrawlerRun(CrawlerRun): - """ - Make sure raising the StopDownload exception stops the download of the response body - """ - - def bytes_received(self, data, request, spider): - super().bytes_received(data, request, spider) - raise StopDownload(fail=False) - - class EngineTest(unittest.TestCase): - @defer.inlineCallbacks def test_crawler(self): @@ -241,8 +234,8 @@ class EngineTest(unittest.TestCase): self.run = CrawlerRun(spider) yield self.run.run() self._assert_visited_urls() - self._assert_scheduled_requests(urls_to_visit=9) - self._assert_downloaded_responses() + self._assert_scheduled_requests(count=9) + self._assert_downloaded_responses(count=9) self._assert_scraped_items() self._assert_signals_caught() self._assert_bytes_received() @@ -251,7 +244,7 @@ class EngineTest(unittest.TestCase): def test_crawler_dupefilter(self): self.run = CrawlerRun(TestDupeFilterSpider) yield self.run.run() - self._assert_scheduled_requests(urls_to_visit=8) + self._assert_scheduled_requests(count=8) self._assert_dropped_requests() @defer.inlineCallbacks @@ -267,8 +260,8 @@ class EngineTest(unittest.TestCase): urls_expected = {self.run.geturl(p) for p in must_be_visited} assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" - def _assert_scheduled_requests(self, urls_to_visit=None): - self.assertEqual(urls_to_visit, len(self.run.reqplug)) + def _assert_scheduled_requests(self, count=None): + self.assertEqual(count, len(self.run.reqplug)) paths_expected = ['/item999.html', '/item2.html', '/item1.html'] @@ -286,10 +279,10 @@ class EngineTest(unittest.TestCase): def _assert_dropped_requests(self): self.assertEqual(len(self.run.reqdropped), 1) - def _assert_downloaded_responses(self): + def _assert_downloaded_responses(self, count): # response tests - self.assertEqual(9, len(self.run.respplug)) - self.assertEqual(9, len(self.run.reqreached)) + self.assertEqual(count, len(self.run.respplug)) + self.assertEqual(count, len(self.run.reqreached)) for response, _ in self.run.respplug: if self.run.getpath(response.url) == '/item999.html': @@ -323,6 +316,13 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) + def _assert_headers_received(self): + for headers in self.run.headers.values(): + self.assertIn(b"Server", headers) + self.assertIn(b"TwistedWeb", headers[b"Server"]) + self.assertIn(b"Date", headers) + self.assertIn(b"Content-Type", headers) + def _assert_bytes_received(self): self.assertEqual(9, len(self.run.bytes)) for request, data in self.run.bytes.items(): @@ -371,6 +371,7 @@ class EngineTest(unittest.TestCase): assert signals.spider_opened in self.run.signals_caught assert signals.spider_idle in self.run.signals_caught assert signals.spider_closed in self.run.signals_caught + assert signals.headers_received in self.run.signals_caught self.assertEqual({'spider': self.run.spider}, self.run.signals_caught[signals.spider_opened]) @@ -403,48 +404,6 @@ class EngineTest(unittest.TestCase): self.assertEqual(len(e.open_spiders), 0) -class StopDownloadEngineTest(EngineTest): - - @defer.inlineCallbacks - def test_crawler(self): - for spider in TestSpider, DictItemsSpider: - self.run = StopDownloadCrawlerRun(spider) - with LogCapture() as log: - yield self.run.run() - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler" - " StopDownloadCrawlerRun.bytes_received")) - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler" - " StopDownloadCrawlerRun.bytes_received")) - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler" - " StopDownloadCrawlerRun.bytes_received")) - self._assert_visited_urls() - self._assert_scheduled_requests(urls_to_visit=9) - self._assert_downloaded_responses() - self._assert_signals_caught() - self._assert_bytes_received() - - def _assert_bytes_received(self): - self.assertEqual(9, len(self.run.bytes)) - for request, data in self.run.bytes.items(): - joined_data = b"".join(data) - self.assertTrue(len(data) == 1) # signal was fired only once - if self.run.getpath(request.url) == "/numbers": - # Received bytes are not the complete response. The exact amount depends - # on the buffer size, which can vary, so we only check that the amount - # of received bytes is strictly less than the full response. - numbers = [str(x).encode("utf8") for x in range(2**18)] - self.assertTrue(len(joined_data) < len(b"".join(numbers))) - - if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == 'runserver': start_test_site(debug=True) diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py new file mode 100644 index 000000000..0ba69e096 --- /dev/null +++ b/tests/test_engine_stop_download_bytes.py @@ -0,0 +1,60 @@ +from testfixtures import LogCapture +from twisted.internet import defer + +from scrapy.exceptions import StopDownload + +from tests.test_engine import ( + AttrsItemsSpider, + DataClassItemsSpider, + DictItemsSpider, + TestSpider, + CrawlerRun, + EngineTest, +) + + +class BytesReceivedCrawlerRun(CrawlerRun): + def bytes_received(self, data, request, spider): + super().bytes_received(data, request, spider) + raise StopDownload(fail=False) + + +class BytesReceivedEngineTest(EngineTest): + @defer.inlineCallbacks + def test_crawler(self): + for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + if spider is None: + continue + self.run = BytesReceivedCrawlerRun(spider) + with LogCapture() as log: + yield self.run.run() + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received")) + self._assert_visited_urls() + self._assert_scheduled_requests(count=9) + self._assert_downloaded_responses(count=9) + self._assert_signals_caught() + self._assert_headers_received() + self._assert_bytes_received() + + def _assert_bytes_received(self): + self.assertEqual(9, len(self.run.bytes)) + for request, data in self.run.bytes.items(): + joined_data = b"".join(data) + self.assertTrue(len(data) == 1) # signal was fired only once + if self.run.getpath(request.url) == "/numbers": + # Received bytes are not the complete response. The exact amount depends + # on the buffer size, which can vary, so we only check that the amount + # of received bytes is strictly less than the full response. + numbers = [str(x).encode("utf8") for x in range(2**18)] + self.assertTrue(len(joined_data) < len(b"".join(numbers))) diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py new file mode 100644 index 000000000..fad6643ad --- /dev/null +++ b/tests/test_engine_stop_download_headers.py @@ -0,0 +1,56 @@ +from testfixtures import LogCapture +from twisted.internet import defer + +from scrapy.exceptions import StopDownload + +from tests.test_engine import ( + AttrsItemsSpider, + DataClassItemsSpider, + DictItemsSpider, + TestSpider, + CrawlerRun, + EngineTest, +) + + +class HeadersReceivedCrawlerRun(CrawlerRun): + def headers_received(self, headers, body_length, request, spider): + super().headers_received(headers, body_length, request, spider) + raise StopDownload(fail=False) + + +class HeadersReceivedEngineTest(EngineTest): + @defer.inlineCallbacks + def test_crawler(self): + for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + if spider is None: + continue + self.run = HeadersReceivedCrawlerRun(spider) + with LogCapture() as log: + yield self.run.run() + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from" + " signal handler HeadersReceivedCrawlerRun.headers_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from signal" + " handler HeadersReceivedCrawlerRun.headers_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from" + " signal handler HeadersReceivedCrawlerRun.headers_received")) + self._assert_visited_urls() + self._assert_downloaded_responses(count=6) + self._assert_signals_caught() + self._assert_bytes_received() + self._assert_headers_received() + + def _assert_bytes_received(self): + self.assertEqual(0, len(self.run.bytes)) + + def _assert_visited_urls(self): + must_be_visited = ["/", "/redirect", "/redirected"] + urls_visited = {rp[0].url for rp in self.run.respplug} + urls_expected = {self.run.geturl(p) for p in must_be_visited} + assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" From 6e5ea7924c7e3f5dd958d13db73e04c6348c99ba Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Fri, 12 Mar 2021 11:08:41 +0600 Subject: [PATCH 0238/2083] Log skipped urls by length to INFO, add skipped stats --- scrapy/spidermiddlewares/urllength.py | 17 ++++++---- tests/test_spidermiddleware_urllength.py | 43 +++++++++++++++++++----- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 5be1f80cb..ee3cb9fd6 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -14,22 +14,27 @@ logger = logging.getLogger(__name__) class UrlLengthMiddleware: - def __init__(self, maxlength): + def __init__(self, maxlength, stats): self.maxlength = maxlength + self.stats = stats @classmethod - def from_settings(cls, settings): + def from_crawler(cls, crawler): + settings = crawler.settings maxlength = settings.getint('URLLENGTH_LIMIT') if not maxlength: raise NotConfigured - return cls(maxlength) + return cls(maxlength, crawler.stats) def process_spider_output(self, response, result, spider): def _filter(request): if isinstance(request, Request) and len(request.url) > self.maxlength: - logger.debug("Ignoring link (url length > %(maxlength)d): %(url)s ", - {'maxlength': self.maxlength, 'url': request.url}, - extra={'spider': spider}) + logger.info( + "Ignoring link (url length > %(maxlength)d): %(url)s ", + {'maxlength': self.maxlength, 'url': request.url}, + extra={'spider': spider} + ) + self.stats.inc_value('urllength/request_ignored_count', spider=spider) return False else: return True diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 5ef2b23fd..33c524627 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,20 +1,45 @@ from unittest import TestCase +from testfixtures import LogCapture + from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider +from scrapy.statscollectors import StatsCollector +from scrapy.utils.test import get_crawler class TestUrlLengthMiddleware(TestCase): - def test_process_spider_output(self): - res = Response('http://scrapytest.org') + def setUp(self): + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('foo') - short_url_req = Request('http://scrapytest.org/') - long_url_req = Request('http://scrapytest.org/this_is_a_long_url') - reqs = [short_url_req, long_url_req] + self.stats = StatsCollector(crawler) + self.stats.open_spider(self.spider) - mw = UrlLengthMiddleware(maxlength=25) - spider = Spider('foo') - out = list(mw.process_spider_output(res, reqs, spider)) - self.assertEqual(out, [short_url_req]) + self.maxlength = 25 + self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats) + + self.response = Response('http://scrapytest.org') + self.short_url_req = Request('http://scrapytest.org/') + self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url') + self.reqs = [self.short_url_req, self.long_url_req] + + def tearDown(self): + self.stats.close_spider(self.spider, '') + + def process_spider_output(self): + return list(self.mw.process_spider_output(self.response, self.reqs, self.spider)) + + def test_middleware_works(self): + self.assertEqual(self.process_spider_output(), [self.short_url_req]) + + def test_logging(self): + with LogCapture() as log: + self.process_spider_output() + + ric = self.stats.get_value('urllength/request_ignored_count', spider=self.spider) + self.assertEqual(ric, 1) + + self.assertIn(f'Ignoring link (url length > {self.maxlength})', str(log)) From d4b2b612551918647148013893da4cfa83fa2e7a Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Fri, 12 Mar 2021 16:59:37 +0600 Subject: [PATCH 0239/2083] Use from_settings for backward compatibility --- scrapy/spidermiddlewares/urllength.py | 10 ++++------ tests/test_spidermiddleware_urllength.py | 10 ++-------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index ee3cb9fd6..450d4ff40 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -14,17 +14,15 @@ logger = logging.getLogger(__name__) class UrlLengthMiddleware: - def __init__(self, maxlength, stats): + def __init__(self, maxlength): self.maxlength = maxlength - self.stats = stats @classmethod - def from_crawler(cls, crawler): - settings = crawler.settings + def from_settings(cls, settings): maxlength = settings.getint('URLLENGTH_LIMIT') if not maxlength: raise NotConfigured - return cls(maxlength, crawler.stats) + return cls(maxlength) def process_spider_output(self, response, result, spider): def _filter(request): @@ -34,7 +32,7 @@ class UrlLengthMiddleware: {'maxlength': self.maxlength, 'url': request.url}, extra={'spider': spider} ) - self.stats.inc_value('urllength/request_ignored_count', spider=spider) + spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider) return False else: return True diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 33c524627..6a72d2a8d 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -5,7 +5,6 @@ from testfixtures import LogCapture from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler @@ -14,21 +13,16 @@ class TestUrlLengthMiddleware(TestCase): def setUp(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider('foo') - - self.stats = StatsCollector(crawler) - self.stats.open_spider(self.spider) + self.stats = self.spider.crawler.stats self.maxlength = 25 - self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats) + self.mw = UrlLengthMiddleware(maxlength=self.maxlength) self.response = Response('http://scrapytest.org') self.short_url_req = Request('http://scrapytest.org/') self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url') self.reqs = [self.short_url_req, self.long_url_req] - def tearDown(self): - self.stats.close_spider(self.spider, '') - def process_spider_output(self): return list(self.mw.process_spider_output(self.response, self.reqs, self.spider)) From 0f254a6afbc3ad4a42048ea67acafdd035ba690a Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Fri, 12 Mar 2021 17:11:50 +0600 Subject: [PATCH 0240/2083] Test from_settings --- tests/test_spidermiddleware_urllength.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 6a72d2a8d..ee79c109f 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -6,17 +6,19 @@ from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +from scrapy.settings import Settings class TestUrlLengthMiddleware(TestCase): def setUp(self): + self.maxlength = 25 + settings = Settings({'URLLENGTH_LIMIT': self.maxlength}) + crawler = get_crawler(Spider) self.spider = crawler._create_spider('foo') self.stats = self.spider.crawler.stats - - self.maxlength = 25 - self.mw = UrlLengthMiddleware(maxlength=self.maxlength) + self.mw = UrlLengthMiddleware.from_settings(settings) self.response = Response('http://scrapytest.org') self.short_url_req = Request('http://scrapytest.org/') From c0f3ca193873cd4dbf4de730dcceb38967efcc16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 12 Mar 2021 14:02:48 +0100 Subject: [PATCH 0241/2083] get_retry_request: add typing information --- scrapy/downloadermiddlewares/retry.py | 28 ++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 8955c7e4f..5e49a284a 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,8 +9,8 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ -import logging -from inspect import isclass +from logging import getLogger, Logger +from typing import Optional, Union from twisted.internet import defer from twisted.internet.error import ( @@ -24,24 +24,26 @@ from twisted.internet.error import ( ) from twisted.web.client import ResponseFailed -from scrapy.exceptions import NotConfigured -from scrapy.utils.response import response_status_message from scrapy.core.downloader.handlers.http11 import TunnelError +from scrapy.exceptions import NotConfigured +from scrapy.http.request import Request +from scrapy.spiders import Spider from scrapy.utils.python import global_object_name +from scrapy.utils.response import response_status_message -retry_logger = logging.getLogger(__name__) +retry_logger = getLogger(__name__) def get_retry_request( - request, + request: Request, *, - spider, - reason='unspecified', - max_retry_times=None, - priority_adjust=None, - logger=retry_logger, - stats_base_key='retry', + spider: Spider, + reason: Union[str, Exception] = 'unspecified', + max_retry_times: Optional[int] = None, + priority_adjust: Union[int, float, None] = None, + logger: Logger = retry_logger, + stats_base_key: str = 'retry', ): """ Returns a new :class:`~scrapy.Request` object to retry the specified @@ -105,7 +107,7 @@ def get_retry_request( priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') new_request.priority = request.priority + priority_adjust - if isclass(reason): + if callable(reason): reason = reason() if isinstance(reason, Exception): reason = global_object_name(reason.__class__) From 9cc4513bd60dcebcbfc53035482eff82d2b7acc0 Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Mon, 15 Mar 2021 21:38:03 +0600 Subject: [PATCH 0242/2083] simpler stats access --- tests/test_spidermiddleware_urllength.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index ee79c109f..171f4ddfd 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -17,7 +17,7 @@ class TestUrlLengthMiddleware(TestCase): crawler = get_crawler(Spider) self.spider = crawler._create_spider('foo') - self.stats = self.spider.crawler.stats + self.stats = crawler.stats self.mw = UrlLengthMiddleware.from_settings(settings) self.response = Response('http://scrapytest.org') From 2f61d7cc034e6ba6ba6ae9ccfc3cf2f1021b857e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 15 Mar 2021 14:25:46 -0300 Subject: [PATCH 0243/2083] Remove unnecesary del statement --- scrapy/core/http2/agent.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index a142fa210..f7b0c3f99 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -70,8 +70,6 @@ class H2ConnectionPool: d = pending_requests.popleft() d.callback(conn) - del pending_requests - return conn def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None: From 42e4dbb23de238e1252eb50b059666494418588f Mon Sep 17 00:00:00 2001 From: vinayak Date: Thu, 18 Mar 2021 18:10:03 +0530 Subject: [PATCH 0244/2083] Support Python 3.9 (#4759) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update .travis.yml * Update .travis.yml * updage travis.yml * Make 3.9 support official * Upgrade mitmproxy for Python 3.9 * Restore the Pylint job * Undo unintended change to mitmproxy requirement * Enable Python 3.9 in GitHub Actions * Work around reppy’s Python version limitation * Disable tests in Windows / Python 3.9 due to a Twisted bug Co-authored-by: Adrián Chaves --- .github/workflows/checks.yml | 10 ++++++---- .github/workflows/publish.yml | 4 ++-- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 7 ++++++- .github/workflows/tests-windows.yml | 4 ++++ .readthedocs.yml | 6 +++++- setup.py | 1 + tox.ini | 4 +++- 8 files changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2748bf5fe..02c647da9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -7,19 +7,21 @@ jobs: strategy: matrix: include: - - python-version: 3.8 + - python-version: 3.9 env: TOXENV: security - - python-version: 3.8 + - python-version: 3.9 env: TOXENV: flake8 + # Pylint requires installing reppy, which does not support Python 3.9 + # https://github.com/seomoz/reppy/issues/122 - python-version: 3.8 env: TOXENV: pylint - - python-version: 3.8 + - python-version: 3.9 env: TOXENV: typing - - python-version: 3.7 # Keep in sync with .readthedocs.yml + - python-version: 3.8 # Keep in sync with .readthedocs.yml env: TOXENV: docs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index aec6b8696..b48066ea4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,10 +9,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Check Tag id: check-release-tag diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 51d27c405..4f8f7a19d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -6,7 +6,7 @@ jobs: runs-on: macos-10.15 strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 89c0334e2..df5ee9d69 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -13,6 +13,9 @@ jobs: - python-version: 3.8 env: TOXENV: py + - python-version: 3.9 + env: + TOXENV: py - python-version: pypy3 env: TOXENV: pypy3 @@ -31,10 +34,12 @@ jobs: PYPY_VERSION: 3.6-v7.2.0 # extras + # extra-deps includes reppy, which does not support Python 3.9 + # https://github.com/seomoz/reppy/issues/122 - python-version: 3.8 env: TOXENV: extra-deps - - python-version: 3.8 + - python-version: 3.9 env: TOXENV: asyncio diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index ed2e4075d..5459a845b 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -16,6 +16,10 @@ jobs: - python-version: 3.8 env: TOXENV: py + # https://twistedmatrix.com/trac/ticket/9990 + #- python-version: 3.9 + #env: + #TOXENV: py steps: - uses: actions/checkout@v2 diff --git a/.readthedocs.yml b/.readthedocs.yml index e4d3f02cc..80a1cd036 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,10 +3,14 @@ formats: all sphinx: configuration: docs/conf.py fail_on_warning: true + +build: + image: latest + python: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image - version: 3.7 # Keep in sync with .travis.yml + version: 3.8 # Keep in sync with .github/workflows/checks.yml install: - requirements: docs/requirements.txt - path: . diff --git a/setup.py b/setup.py index b5c42a3c2..cf9261271 100644 --- a/setup.py +++ b/setup.py @@ -85,6 +85,7 @@ setup( 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', diff --git a/tox.ini b/tox.ini index 69f52bd9f..9815f80f7 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,9 @@ deps = -rtests/requirements-py3.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 - mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy' + # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe + mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' + mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 From 94201612bcad7b74c60a2a7ab70f40ba87714ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 18 Mar 2021 23:35:47 +0100 Subject: [PATCH 0245/2083] Simplify the get_retry_request code example --- scrapy/downloadermiddlewares/retry.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5e49a284a..2721db7cf 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -55,14 +55,12 @@ def get_retry_request( def parse(self, response): if not response.text: - new_request = get_retry_request( + new_request_or_none = get_retry_request( response.request, spider=self, reason='empty', ) - if new_request: - yield new_request - return + return new_request_or_none *spider* is the :class:`~scrapy.Spider` instance which is asking for the retry request. It is used to access the :ref:`settings ` From 8e73e1dfb51ad6a25ba4583f000d50a12718fb88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 18 Mar 2021 23:42:29 +0100 Subject: [PATCH 0246/2083] upper-constraints.txt: restrict botocore further --- tests/upper-constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/upper-constraints.txt b/tests/upper-constraints.txt index 75f337856..2a335e533 100644 --- a/tests/upper-constraints.txt +++ b/tests/upper-constraints.txt @@ -2,7 +2,7 @@ # pip dependency resolver from spending too much time backtracking. attrs>=20.2.0 Automat>=0.8.0 -botocore>=1.20.3 +botocore>=1.20.30 itemadapter>=0.1.1 itemloaders>=1.0.3 lxml>=4.6.1 From a390b934de4b8190499c15da115709aa6424d8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 18 Mar 2021 23:53:58 +0100 Subject: [PATCH 0247/2083] Do not install mitmproxy in Python 3.9 --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 86ae951b5..e0c69350e 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,8 @@ deps = # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe - mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' + # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 + #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras From 0dad0fce72266aa7b38b536f87bab26e7f233c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 19 Mar 2021 11:13:05 +0100 Subject: [PATCH 0248/2083] Use pip<20.3 to fix ReadTheDocs builds (#5052) --- .readthedocs.yml | 1 + docs/pip.txt | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 docs/pip.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index 80a1cd036..2d781ae81 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,5 +12,6 @@ python: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image version: 3.8 # Keep in sync with .github/workflows/checks.yml install: + - requirements: docs/pip.txt - requirements: docs/requirements.txt - path: . diff --git a/docs/pip.txt b/docs/pip.txt new file mode 100644 index 000000000..095e53a0d --- /dev/null +++ b/docs/pip.txt @@ -0,0 +1,3 @@ +# In pip 20.3-21.0, the default dependency resolver causes the build in +# ReadTheDocs to fail due to memory exhaustion or timeout. +pip<20.3 From 308a58aa275aa514397351caa263e08de2f89adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 19 Mar 2021 18:39:44 +0100 Subject: [PATCH 0249/2083] Update CI to support Twisted 21.2.0 (#5027) --- scrapy/pipelines/images.py | 19 +++++++++++++------ tests/test_commands.py | 3 +++ tests/test_crawler.py | 4 ++++ tests/test_pipeline_crawl.py | 11 +++++++++++ tests/test_pipeline_images.py | 14 ++++++++++---- tests/test_pipeline_media.py | 11 +++++++++++ tox.ini | 4 +--- 7 files changed, 53 insertions(+), 13 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index aafd1d8b2..e3ab23ea5 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -9,9 +9,8 @@ from contextlib import suppress from io import BytesIO from itemadapter import ItemAdapter -from PIL import Image -from scrapy.exceptions import DropItem +from scrapy.exceptions import DropItem, NotConfigured from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline @@ -45,6 +44,14 @@ class ImagesPipeline(FilesPipeline): DEFAULT_IMAGES_RESULT_FIELD = 'images' def __init__(self, store_uri, download_func=None, settings=None): + try: + from PIL import Image + self._Image = Image + except ImportError: + raise NotConfigured( + 'ImagesPipeline requires installing Pillow 4.0.0 or later' + ) + super().__init__(store_uri, settings=settings, download_func=download_func) if isinstance(settings, dict) or settings is None: @@ -121,7 +128,7 @@ class ImagesPipeline(FilesPipeline): def get_images(self, response, request, info, *, item=None): path = self.file_path(request, response=response, info=info, item=item) - orig_image = Image.open(BytesIO(response.body)) + orig_image = self._Image.open(BytesIO(response.body)) width, height = orig_image.size if width < self.min_width or height < self.min_height: @@ -139,12 +146,12 @@ class ImagesPipeline(FilesPipeline): def convert_image(self, image, size=None): if image.format == 'PNG' and image.mode == 'RGBA': - background = Image.new('RGBA', image.size, (255, 255, 255)) + background = self._Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert('RGB') elif image.mode == 'P': image = image.convert("RGBA") - background = Image.new('RGBA', image.size, (255, 255, 255)) + background = self._Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert('RGB') elif image.mode != 'RGB': @@ -152,7 +159,7 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() - image.thumbnail(size, Image.ANTIALIAS) + image.thumbnail(size, self._Image.ANTIALIAS) buf = BytesIO() image.save(buf, 'JPEG') diff --git a/tests/test_commands.py b/tests/test_commands.py index d3ac05eac..eec1f02ee 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -17,6 +17,8 @@ from threading import Timer from unittest import skipIf from pytest import mark +from twisted import version as twisted_version +from twisted.python.versions import Version from twisted.trial import unittest import scrapy @@ -630,6 +632,7 @@ class MySpider(scrapy.Spider): @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') def test_custom_asyncio_loop_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ab113710d..dec517bb6 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -8,7 +8,9 @@ from unittest import skipIf from pytest import raises, mark from testfixtures import LogCapture +from twisted import version as twisted_version from twisted.internet import defer +from twisted.python.versions import Version from twisted.trial import unittest import scrapy @@ -358,6 +360,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") self.assertIn("Spider closed (finished)", log) @@ -366,6 +369,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly") @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows") + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 55fcfa7ba..f49fda701 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -180,7 +180,18 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) +try: + from PIL import Image # noqa: imported just to check for the import error +except ImportError: + skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' +else: + skip_pillow = None + + class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): + + skip = skip_pillow + pipeline_class = 'scrapy.pipelines.images.ImagesPipeline' store_setting_key = 'IMAGES_STORE' media_key = 'images' diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index ad138a2dc..c69cd0e4a 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -23,15 +23,16 @@ except ImportError: dataclass_field = None -skip = False try: from PIL import Image except ImportError: - skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' + skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' else: encoders = {'jpeg_encoder', 'jpeg_decoder'} if not encoders.issubset(set(Image.core.__dict__)): - skip = 'Missing JPEG encoders' + skip_pillow = 'Missing JPEG encoders' + else: + skip_pillow = None def _mocked_download_func(request, info): @@ -41,7 +42,7 @@ def _mocked_download_func(request, info): class ImagesPipelineTestCase(unittest.TestCase): - skip = skip + skip = skip_pillow def setUp(self): self.tempdir = mkdtemp() @@ -137,6 +138,8 @@ class DeprecatedImagesPipeline(ImagesPipeline): class ImagesPipelineTestCaseFieldsMixin: + skip = skip_pillow + def test_item_fields_default(self): url = 'http://www.example.com/images/1.jpg' item = self.item_class(name='item1', image_urls=[url]) @@ -221,6 +224,9 @@ class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, u class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): + + skip = skip_pillow + img_cls_attribute_names = [ # Pipeline attribute names with corresponding setting names. ("EXPIRES", "IMAGES_EXPIRES"), diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 6afd47497..893d43052 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,3 +1,5 @@ +from typing import Optional + from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure @@ -17,6 +19,14 @@ from scrapy.utils.signal import disconnect_all from scrapy import signals +try: + from PIL import Image # noqa: imported just to check for the import error +except ImportError: + skip_pillow: Optional[str] = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' +else: + skip_pillow = None + + def _mocked_download_func(request, info): response = request.meta.get('response') return response() if callable(response) else response @@ -379,6 +389,7 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): + skip = skip_pillow def setUp(self): self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func) diff --git a/tox.ini b/tox.ini index e0c69350e..6907c8906 100644 --- a/tox.ini +++ b/tox.ini @@ -19,9 +19,6 @@ deps = mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 - Pillow>=4.0.0 - # Twisted 21+ causes issues in tests that use skipIf - Twisted[http2]>=17.9.0,<21 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -124,6 +121,7 @@ deps = {[testenv]deps} reppy robotexclusionrulesparser + Pillow>=4.0.0 [testenv:asyncio] commands = From 2973d8d51abbda4839981c192a366e907e2ad39e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Mar 2021 11:24:10 +0100 Subject: [PATCH 0250/2083] Remove unnecessary reference to private parsel.Selector._default_type (#5006) --- scrapy/selector/unified.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index a25871433..08f08e8d7 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -69,7 +69,7 @@ class Selector(_ParselSelector, object_ref): raise ValueError(f'{self.__class__.__name__}.__init__() received ' 'both response and text') - st = _st(response, type or self._default_type) + st = _st(response, type) if text is not None: response = _response_from_text(text, st) From ec5a7918ec2d8888ba356f9e3295dcd1ee935884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Mar 2021 11:25:40 +0100 Subject: [PATCH 0251/2083] Include Content-Length in HTTP/1.1 responses (#5057) --- scrapy/core/downloader/handlers/http11.py | 13 +++++++++++-- tests/test_downloader_handlers.py | 7 +++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9cdadb27f..1f82751fd 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -358,10 +358,19 @@ class ScrapyAgent: request.meta['download_latency'] = time() - start_time return result + @staticmethod + def _headers_from_twisted_response(response): + headers = Headers() + if response.length is not None: + headers[b'Content-Length'] = str(response.length).encode() + for key, value in response.headers.getAllRawHeaders(): + headers[key] = value + return headers + def _cb_bodyready(self, txresponse, request): headers_received_result = self._crawler.signals.send_catch_log( signal=signals.headers_received, - headers=Headers(txresponse.headers.getAllRawHeaders()), + headers=self._headers_from_twisted_response(txresponse), body_length=txresponse.length, request=request, spider=self._crawler.spider, @@ -435,7 +444,7 @@ class ScrapyAgent: return d def _cb_bodydone(self, result, request, url): - headers = Headers(result["txresponse"].headers.getAllRawHeaders()) + headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) try: version = result["txresponse"].version diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 86d72772c..fa7d5c8a6 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -364,6 +364,13 @@ class HttpTestCase(unittest.TestCase): d.addCallback(self.assertEqual, body) return d + def test_response_header_content_length(self): + request = Request(self.getURL("file"), method=b"GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.headers[b'content-length']) + d.addCallback(self.assertEqual, b'159') + return d + class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" From 72e8cea8afb85f6704fcf6f5648b0a01f1abaab3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 22 Mar 2021 11:51:11 -0300 Subject: [PATCH 0252/2083] Avoid exceptions in is_generator_with_return_value (#4935) --- scrapy/utils/misc.py | 27 ++- ...t_return_with_argument_inside_generator.py | 167 +++++++++++++++--- 2 files changed, 161 insertions(+), 33 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 081cd33f1..5c986eedc 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -9,7 +9,6 @@ from collections import deque from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules -from textwrap import dedent from w3lib.html import replace_entities @@ -227,7 +226,8 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - tree = ast.parse(dedent(inspect.getsource(callable))) + code = re.sub(r"^[\t ]+", "", inspect.getsource(callable)) + tree = ast.parse(code) for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): _generator_callbacks_cache[callable] = True @@ -242,12 +242,23 @@ def warn_on_generator_with_return_value(spider, callable): Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None """ - if is_generator_with_return_value(callable): + try: + if is_generator_with_return_value(callable): + warnings.warn( + f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' + 'a generator and includes a "return" statement with a value ' + 'different than None. This could lead to unexpected behaviour. Please see ' + 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' + 'for details about the semantics of the "return" statement within generators', + stacklevel=2, + ) + except IndentationError: + callable_name = spider.__class__.__name__ + "." + callable.__name__ warnings.warn( - f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' - 'a generator and includes a "return" statement with a value ' - 'different than None. This could lead to unexpected behaviour. Please see ' - 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' - 'for details about the semantics of the "return" statement within generators', + f'Unable to determine whether or not "{callable_name}" is a generator with a return value. ' + 'This will not prevent your code from working, but it prevents Scrapy from detecting ' + f'potential issues in your implementation of "{callable_name}". Please, report this in the ' + 'Scrapy issue tracker (https://github.com/scrapy/scrapy/issues), ' + f'including the code of "{callable_name}"', stacklevel=2, ) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 2be38620c..1c85ca353 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,35 +1,116 @@ import unittest +import warnings +from unittest import mock -from scrapy.utils.misc import is_generator_with_return_value +from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value + + +def _indentation_error(*args, **kwargs): + raise IndentationError() + + +def top_level_return_something(): + """ +docstring + """ + url = """ +https://example.org +""" + yield url + return 1 + + +def top_level_return_none(): + """ +docstring + """ + url = """ +https://example.org +""" + yield url + return + + +def generator_that_returns_stuff(): + yield 1 + yield 2 + return 3 class UtilsMiscPy3TestCase(unittest.TestCase): - def test_generators_with_return_statements(self): - def f(): + def test_generators_return_something(self): + def f1(): yield 1 return 2 - def g(): + def g1(): yield 1 - return 'asdf' + return "asdf" - def h(): + def h1(): + yield 1 + + def helper(): + return 0 + + yield helper() + return 2 + + def i1(): + """ +docstring + """ + url = """ +https://example.org + """ + yield url + return 1 + + assert is_generator_with_return_value(top_level_return_something) + assert is_generator_with_return_value(f1) + assert is_generator_with_return_value(g1) + assert is_generator_with_return_value(h1) + assert is_generator_with_return_value(i1) + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_something) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.top_level_return_something" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, f1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.f1" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, g1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.g1" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, h1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.h1" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, i1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.i1" method is a generator', str(w[0].message)) + + def test_generators_return_none(self): + def f2(): yield 1 return None - def i(): + def g2(): yield 1 return - def j(): + def h2(): yield 1 - def k(): + def i2(): yield 1 - yield from g() + yield from generator_that_returns_stuff() - def m(): + def j2(): yield 1 def helper(): @@ -37,20 +118,56 @@ class UtilsMiscPy3TestCase(unittest.TestCase): yield helper() - def n(): - yield 1 + def k2(): + """ +docstring + """ + url = """ +https://example.org + """ + yield url + return - def helper(): - return 0 + def l2(): + return - yield helper() - return 2 + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f2) + assert not is_generator_with_return_value(g2) + assert not is_generator_with_return_value(h2) + assert not is_generator_with_return_value(i2) + assert not is_generator_with_return_value(j2) # not recursive + assert not is_generator_with_return_value(k2) # not recursive + assert not is_generator_with_return_value(l2) - assert is_generator_with_return_value(f) - assert is_generator_with_return_value(g) - assert not is_generator_with_return_value(h) - assert not is_generator_with_return_value(i) - assert not is_generator_with_return_value(j) - assert not is_generator_with_return_value(k) # not recursive - assert not is_generator_with_return_value(m) - assert is_generator_with_return_value(n) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_none) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, f2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, g2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, h2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, i2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, j2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, k2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, l2) + self.assertEqual(len(w), 0) + + @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error) + def test_indentation_error(self): + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_none) + self.assertEqual(len(w), 1) + self.assertIn('Unable to determine', str(w[0].message)) From 64d4ae1a19eda9c6bb0b6dc85e5dfff6187d0ca8 Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 22 Mar 2021 21:46:05 +0100 Subject: [PATCH 0253/2083] Update UsageError message --- scrapy/commands/crawl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index f205c40b0..0f2a21b85 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand): if len(args) < 1: raise UsageError() elif len(args) > 1: - raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") + raise UsageError("running 'scrapy crawl' with more than one spider is not supported") spname = args[0] crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) From f0e1a33225dd7c9162e277626df70284f41a427e Mon Sep 17 00:00:00 2001 From: Pratik Mahankal <53421565+pratik1500@users.noreply.github.com> Date: Tue, 23 Mar 2021 22:46:50 +0530 Subject: [PATCH 0254/2083] Sort the list of Request.meta alphabetically #5061 (#5065) --- docs/topics/request-response.rst | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a0448c5ab..c0283df01 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -363,26 +363,26 @@ are some special keys recognized by Scrapy and its built-in extensions. Those are: -* :reqmeta:`dont_redirect` -* :reqmeta:`dont_retry` -* :reqmeta:`handle_httpstatus_list` -* :reqmeta:`handle_httpstatus_all` -* :reqmeta:`dont_merge_cookies` +* :reqmeta:`bindaddress` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` +* :reqmeta:`dont_merge_cookies` +* :reqmeta:`dont_obey_robotstxt` +* :reqmeta:`dont_redirect` +* :reqmeta:`dont_retry` +* :reqmeta:`download_fail_on_dataloss` +* :reqmeta:`download_latency` +* :reqmeta:`download_maxsize` +* :reqmeta:`download_timeout` +* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) +* ``ftp_user`` (See :setting:`FTP_USER` for more info) +* :reqmeta:`handle_httpstatus_all` +* :reqmeta:`handle_httpstatus_list` +* :reqmeta:`max_retry_times` +* :reqmeta:`proxy` * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` -* :reqmeta:`bindaddress` -* :reqmeta:`dont_obey_robotstxt` -* :reqmeta:`download_timeout` -* :reqmeta:`download_maxsize` -* :reqmeta:`download_latency` -* :reqmeta:`download_fail_on_dataloss` -* :reqmeta:`proxy` -* ``ftp_user`` (See :setting:`FTP_USER` for more info) -* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * :reqmeta:`referrer_policy` -* :reqmeta:`max_retry_times` .. reqmeta:: bindaddress From c51ec1ae1cef3fd68fc512f0636b75d0f3a2da13 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Feb 2021 19:32:45 +0500 Subject: [PATCH 0255/2083] Drop process_iterable_helper, add _process_iterable_universal. --- scrapy/core/spidermw.py | 28 +++------ scrapy/spidermiddlewares/depth.py | 18 ++++-- scrapy/spidermiddlewares/offsite.py | 32 +++++----- scrapy/spidermiddlewares/referer.py | 9 ++- scrapy/spidermiddlewares/urllength.py | 9 ++- scrapy/utils/asyncgen.py | 45 ++++++++++++++ scrapy/utils/middlewares.py | 35 ----------- tests/test_utils_asyncgen.py | 22 ++++++- tests/test_utils_middlewares.py | 87 --------------------------- 9 files changed, 120 insertions(+), 165 deletions(-) delete mode 100644 scrapy/utils/middlewares.py delete mode 100644 tests/test_utils_middlewares.py diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index d3d7e5f8c..33e215971 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -10,6 +10,7 @@ from twisted.python.failure import Failure from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager +from scrapy.utils.asyncgen import _process_iterable_universal from scrapy.utils.conf import build_component_list from scrapy.utils.defer import mustbe_deferred from scrapy.utils.python import MutableAsyncChain, MutableChain @@ -57,31 +58,18 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(response, request, spider) def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to): - def _process_exception(ex): - exception_result = self._process_spider_exception(response, spider, Failure(ex), - exception_processor_index) - if isinstance(exception_result, Failure): - raise # pylint: disable=E0704 - recover_to.extend(exception_result) - - def _evaluate_normal_iterable(iterable): - try: - for r in iterable: - yield r - except Exception as ex: - _process_exception(ex) - + @_process_iterable_universal async def _evaluate_async_iterable(iterable): try: async for r in iterable: yield r except Exception as ex: - _process_exception(ex) - - if inspect.isasyncgen(iterable): - return _evaluate_async_iterable(iterable) - else: - return _evaluate_normal_iterable(iterable) + exception_result = self._process_spider_exception(response, spider, Failure(ex), + exception_processor_index) + if isinstance(exception_result, Failure): + raise + recover_to.extend(exception_result) + return _evaluate_async_iterable(iterable) def _process_spider_exception(self, response, spider, _failure, start_index=0): exception = _failure.value diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 776a6879a..973404b2b 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -7,6 +7,7 @@ See documentation in docs/topics/spider-middleware.rst import logging from scrapy.http import Request +from scrapy.utils.asyncgen import _process_iterable_universal logger = logging.getLogger(__name__) @@ -49,10 +50,15 @@ class DepthMiddleware: spider=spider) return True - # base case (depth=0) - if 'depth' not in response.meta: - response.meta['depth'] = 0 - if self.verbose_stats: - self.stats.inc_value('request_depth_count/0', spider=spider) + @_process_iterable_universal + async def process(result): + # base case (depth=0) + if 'depth' not in response.meta: + response.meta['depth'] = 0 + if self.verbose_stats: + self.stats.inc_value('request_depth_count/0', spider=spider) - return (r for r in result or () if _filter(r)) + async for r in result or (): + if _filter(r): + yield r + return process(result) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 6e4efda97..074ec7a4e 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -9,6 +9,7 @@ import warnings from scrapy import signals from scrapy.http import Request +from scrapy.utils.asyncgen import _process_iterable_universal from scrapy.utils.httpobj import urlparse_cached logger = logging.getLogger(__name__) @@ -26,21 +27,24 @@ class OffsiteMiddleware: return o def process_spider_output(self, response, result, spider): - for x in result: - if isinstance(x, Request): - if x.dont_filter or self.should_follow(x, spider): - yield x + @_process_iterable_universal + async def process(result): + async for x in result: + if isinstance(x, Request): + if x.dont_filter or self.should_follow(x, spider): + yield x + else: + domain = urlparse_cached(x).hostname + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {'domain': domain, 'request': x}, extra={'spider': spider}) + self.stats.inc_value('offsite/domains', spider=spider) + self.stats.inc_value('offsite/filtered', spider=spider) else: - domain = urlparse_cached(x).hostname - if domain and domain not in self.domains_seen: - self.domains_seen.add(domain) - logger.debug( - "Filtered offsite request to %(domain)r: %(request)s", - {'domain': domain, 'request': x}, extra={'spider': spider}) - self.stats.inc_value('offsite/domains', spider=spider) - self.stats.inc_value('offsite/filtered', spider=spider) - else: - yield x + yield x + return process(result) def should_follow(self, request, spider): regex = self.host_regex diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index f81041376..8d862d1d0 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -10,6 +10,7 @@ from w3lib.url import safe_url_string from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals +from scrapy.utils.asyncgen import _process_iterable_universal from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object from scrapy.utils.url import strip_url @@ -337,7 +338,13 @@ class RefererMiddleware: if referrer is not None: r.headers.setdefault('Referer', referrer) return r - return (_set_referer(r) for r in result or ()) + + @_process_iterable_universal + async def process(result): + async for r in result or (): + yield _set_referer(r) + + return process(result) def request_scheduled(self, request, spider): # check redirected request to patch "Referer" header if necessary diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 5be1f80cb..d40d43ff1 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -8,6 +8,7 @@ import logging from scrapy.http import Request from scrapy.exceptions import NotConfigured +from scrapy.utils.asyncgen import _process_iterable_universal logger = logging.getLogger(__name__) @@ -34,4 +35,10 @@ class UrlLengthMiddleware: else: return True - return (r for r in result or () if _filter(r)) + @_process_iterable_universal + async def process(result): + async for r in result or (): + if _filter(r): + yield r + + return process(result) diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index db2173f85..39c94ad8a 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,4 +1,6 @@ import collections +import functools +import inspect async def collect_asyncgen(result): @@ -15,3 +17,46 @@ async def as_async_generator(it): else: for r in it: yield r + + +# https://stackoverflow.com/a/66170760/113586 +def _process_iterable_universal(process_async): + """ Takes a function that takes an async iterable, args and kwargs. Returns + a function that takes any iterable, args and kwargs. + + Requires that process_async only awaits on the iterable and synchronous functions, + so it's better to use this only in the Scrapy code itself. + """ + + # If this stops working, all internal uses can be just replaced with manually-written + # process_sync functions. + + def process_sync(iterable, *args, **kwargs): + agen = process_async(as_async_generator(iterable), *args, **kwargs) + if not inspect.isasyncgen(agen): + raise ValueError(f"process_async returned wrong type {type(agen)}") + sent = None + while True: + try: + gen = agen.asend(sent) + gen.send(None) + except StopIteration as e: + sent = yield e.value + except StopAsyncIteration: + return + else: + gen.throw(RuntimeError, + f"Synchronously-called function '{process_async.__name__}' has blocked, " + f"you can't use {_process_iterable_universal.__name__} with it.") + + @functools.wraps(process_async) + def process(iterable, *args, **kwargs): + if inspect.isasyncgen(iterable): + # call process_async directly + return process_async(iterable, *args, **kwargs) + if hasattr(iterable, '__iter__'): + # convert process_async to process_sync + return process_sync(iterable, *args, **kwargs) + raise ValueError(f"Wrong iterable type {type(iterable)}") + + return process diff --git a/scrapy/utils/middlewares.py b/scrapy/utils/middlewares.py deleted file mode 100644 index da28e0ddf..000000000 --- a/scrapy/utils/middlewares.py +++ /dev/null @@ -1,35 +0,0 @@ -# coding: utf-8 -import inspect - - -def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): - for o in it: - if in_predicate and not in_predicate(o): - continue - if processor is not None: - o = processor(o) - if out_predicate and not out_predicate(o): - continue - yield o - - -async def process_async_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): - async for o in it: - if in_predicate and not in_predicate(o): - continue - if processor is not None: - o = processor(o) - if out_predicate and not out_predicate(o): - continue - yield o - - -def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): - """ - For each item in the iterable: skips it if in_predicate is False, applies processor, - skips the result if out_predicate is False, else yields it. - """ - if inspect.isasyncgen(it): - return process_async_iterable_helper(it, in_predicate, out_predicate, processor) - else: - return process_normal_iterable_helper(it, in_predicate, out_predicate, processor) diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 9ae66c57c..2f4181d3d 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,6 +1,6 @@ from twisted.trial import unittest -from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal from scrapy.utils.defer import deferred_f_from_coro_f @@ -18,3 +18,23 @@ class AsyncgenUtilsTest(unittest.TestCase): ag = as_async_generator(range(42)) results = await collect_asyncgen(ag) self.assertEqual(results, list(range(42))) + + +@_process_iterable_universal +async def process_iterable(iterable): + async for i in iterable: + yield i * 2 + + +class ProcessIterableUniversalTest(unittest.TestCase): + + def test_normal(self): + iterable = iter([1, 2, 3]) + results = list(process_iterable(iterable)) + self.assertEqual(results, [2, 4, 6]) + + @deferred_f_from_coro_f + async def test_async(self): + iterable = as_async_generator([1, 2, 3]) + results = await collect_asyncgen(process_iterable(iterable)) + self.assertEqual(results, [2, 4, 6]) diff --git a/tests/test_utils_middlewares.py b/tests/test_utils_middlewares.py deleted file mode 100644 index d395ba1a9..000000000 --- a/tests/test_utils_middlewares.py +++ /dev/null @@ -1,87 +0,0 @@ -import collections - -from twisted.trial import unittest - -from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen -from scrapy.utils.defer import deferred_f_from_coro_f -from scrapy.utils.middlewares import process_iterable_helper - - -def predicate1(o): - return bool(o % 2) - - -def predicate2(o): - return o < 10 - - -def processor(o): - return o * 2 - - -class ProcessIterableHelperNormalTest(unittest.TestCase): - - def test_normal_in_predicate(self): - iterable1 = iter([1, 2, 3]) - iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1) - self.assertIsInstance(iterable2, collections.abc.Iterable) - list2 = list(iterable2) - self.assertEqual(list2, [1, 3]) - - def test_normal_out_predicate(self): - iterable1 = iter([1, 2, 10, 3, 15]) - iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2) - self.assertIsInstance(iterable2, collections.abc.Iterable) - list2 = list(iterable2) - self.assertEqual(list2, [1, 2, 3]) - - def test_normal_processor(self): - iterable1 = iter([1, 2, 3]) - iterable2 = process_iterable_helper(iterable1, processor=processor) - self.assertIsInstance(iterable2, collections.abc.Iterable) - list2 = list(iterable2) - self.assertEqual(list2, [2, 4, 6]) - - def test_normal_combined(self): - iterable1 = iter([1, 2, 10, 3, 6, 18, 5, 15]) - iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1, - out_predicate=predicate2, processor=processor) - self.assertIsInstance(iterable2, collections.abc.Iterable) - list2 = list(iterable2) - self.assertEqual(list2, [2, 6]) - - -class ProcessIterableHelperAsyncTest(unittest.TestCase): - - @deferred_f_from_coro_f - async def test_async_in_predicate(self): - iterable1 = as_async_generator([1, 2, 3]) - iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1) - self.assertIsInstance(iterable2, collections.abc.AsyncIterable) - list2 = await collect_asyncgen(iterable2) - self.assertEqual(list2, [1, 3]) - - @deferred_f_from_coro_f - async def test_async_out_predicate(self): - iterable1 = as_async_generator([1, 2, 10, 3, 15]) - iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2) - self.assertIsInstance(iterable2, collections.abc.AsyncIterable) - list2 = await collect_asyncgen(iterable2) - self.assertEqual(list2, [1, 2, 3]) - - @deferred_f_from_coro_f - async def test_async_processor(self): - iterable1 = as_async_generator([1, 2, 3]) - iterable2 = process_iterable_helper(iterable1, processor=processor) - self.assertIsInstance(iterable2, collections.abc.AsyncIterable) - list2 = await collect_asyncgen(iterable2) - self.assertEqual(list2, [2, 4, 6]) - - @deferred_f_from_coro_f - async def test_async_combined(self): - iterable1 = as_async_generator([1, 2, 10, 3, 6, 18, 5, 15]) - iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1, - out_predicate=predicate2, processor=processor) - self.assertIsInstance(iterable2, collections.abc.AsyncIterable) - list2 = await collect_asyncgen(iterable2) - self.assertEqual(list2, [2, 6]) From a6034f942b034946538855cc53644b5a89ba081a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 22 Mar 2021 22:53:52 +0500 Subject: [PATCH 0256/2083] Add tests for _AsyncCooperatorAdapter. --- tests/test_utils_defer.py | 68 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 62f6ff194..543bbee09 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,14 +1,18 @@ +import random + from pytest import mark from twisted.trial import unittest from twisted.internet import reactor, defer from twisted.python.failure import Failure -from scrapy.utils.asyncgen import collect_asyncgen +from scrapy.utils.asyncgen import collect_asyncgen, as_async_generator from scrapy.utils.defer import ( aiter_errback, deferred_f_from_coro_f, iter_errback, + maybe_deferred_to_future, mustbe_deferred, + parallel_async, process_chain, process_chain_both, process_parallel, @@ -164,3 +168,65 @@ class AsyncDefTestsuiteTest(unittest.TestCase): @deferred_f_from_coro_f async def test_deferred_f_from_coro_f_xfail(self): raise Exception("This is expected to be raised") + + +class AsyncCooperatorTest(unittest.TestCase): + """ This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage. + + parallel_async is called with the results of a callback (so an iterable of items, requests and None, + with arbitrary delays between values), and it uses Scraper._process_spidermw_output as the callable + (so a callable that returns a Deferred for an item, which will fire after pipelines process it, and + None for everything else). The concurrent task count is the CONCURRENT_ITEMS setting. + + We want to test different concurrency values compared to the iterable length. + We also want to simulate the real usage, with arbitrary delays between getting the values + from the iterable. We also want to simulate sync and async results from the callable. + """ + CONCURRENT_ITEMS = 50 + + @staticmethod + def callable(o, results): + if random.random() < 0.4: + # simulate async processing + dfd = defer.Deferred() + dfd.addCallback(lambda _: results.append(o)) + delay = random.random() / 8 + reactor.callLater(delay, dfd.callback, None) + return dfd + else: + # simulate trivial sync processing + results.append(o) + + @staticmethod + def get_async_iterable(length): + # simulate a simple callback without delays between results + return as_async_generator(range(length)) + + @staticmethod + async def get_async_iterable_with_delays(length): + # simulate a callback with delays between some of the results + for i in range(length): + if random.random() < 0.1: + dfd = defer.Deferred() + delay = random.random() / 20 + reactor.callLater(delay, dfd.callback, None) + await maybe_deferred_to_future(dfd) + yield i + + @defer.inlineCallbacks + def test_simple(self): + for length in [20, 50, 100]: + results = [] + ait = self.get_async_iterable(length) + dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) + yield dl + self.assertEqual(list(range(length)), sorted(results)) + + @defer.inlineCallbacks + def test_delays(self): + for length in [20, 50, 100]: + results = [] + ait = self.get_async_iterable_with_delays(length) + dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) + yield dl + self.assertEqual(list(range(length)), sorted(results)) From 9c9e1a318d83b8e55125fdf7a7fb9482cb4951f8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 25 Mar 2021 11:58:39 -0300 Subject: [PATCH 0257/2083] [HTTP/1.1] Skip Content-Length header if its value is UNKNOWN_LENGTH (#5062) --- scrapy/core/downloader/handlers/http11.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1f82751fd..25cb3ec62 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -361,10 +361,9 @@ class ScrapyAgent: @staticmethod def _headers_from_twisted_response(response): headers = Headers() - if response.length is not None: + if response.length != UNKNOWN_LENGTH: headers[b'Content-Length'] = str(response.length).encode() - for key, value in response.headers.getAllRawHeaders(): - headers[key] = value + headers.update(response.headers.getAllRawHeaders()) return headers def _cb_bodyready(self, txresponse, request): From 0596f2bf6e7fc404333345d1cece01e31fabed8e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 23 Mar 2021 22:47:48 +0500 Subject: [PATCH 0258/2083] Remove not needed deferred_from_coro call. --- scrapy/core/scraper.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 3eae71af7..7eedbf33e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -16,7 +16,6 @@ from scrapy.utils.defer import ( aiter_errback, defer_fail, defer_succeed, - deferred_from_coro, iter_errback, parallel, parallel_async, @@ -191,8 +190,8 @@ class Scraper: return defer_succeed(None) if isinstance(result, collections.abc.AsyncIterable): it = aiter_errback(result, self.handle_spider_error, request, response, spider) - dfd = deferred_from_coro(parallel_async(it, self.concurrent_items, self._process_spidermw_output, - request, response, spider)) + dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output, + request, response, spider) else: it = iter_errback(result, self.handle_spider_error, request, response, spider) dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, From a97dc55c714d90378dbc8a819cae1a3a40056d6e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 25 Mar 2021 17:37:32 +0500 Subject: [PATCH 0259/2083] Add/improve docs. --- docs/topics/asyncio.rst | 1 - docs/topics/coroutines.rst | 13 ++++++++++++- docs/topics/spider-middleware.rst | 17 +++++++++++------ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 4addaa178..18712c928 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -38,7 +38,6 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to use it instead of the default asyncio event loop. - .. _asyncio-await-dfd: Awaiting on Deferreds diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 279632653..9b50d9312 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -17,6 +17,10 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.http.Request` callbacks. + .. versionchanged:: VERSION + Output of async callbacks is now processed asynchronously instead of collecting + all of it first. + - The :meth:`process_item` method of :ref:`item pipelines `. @@ -30,6 +34,13 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. +- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` + method of :ref:`spider middlewares `. + + .. versionadded:: VERSION + .. note:: This method needs to be an async generator, not just a coroutine that + returns an iterable. + Usage ===== @@ -76,7 +87,7 @@ This means you can use many useful Python libraries providing such code:: async def parse_with_asyncio(self, response): async with aiohttp.ClientSession() as session: async with session.get('https://additional.url') as additional_response: - additional_data = await r.text() + additional_data = await additional_response.text() # ... use response and additional_data to yield items and requests .. note:: Many libraries that use coroutines, such as `aio-libs`_, require the diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index fc114a63f..d09693c16 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -98,20 +98,25 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_spider_output(response, result, spider) + .. versionchanged:: VERSION + Since VERSION this can take and return an :term:`python:asynchronous + iterable`. + This method is called with the results returned from the Spider, after it has processed the response. - :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.http.Request` objects and :ref:`item object - `. + :meth:`process_spider_output` must return an iterable (normal or + asynchronous) of :class:`~scrapy.http.Request` objects and + :ref:`item objects `. :param response: the response which generated this output from the spider :type response: :class:`~scrapy.http.Response` object :param result: the result returned by the spider - :type result: an iterable of :class:`~scrapy.http.Request` objects and - :ref:`item object ` + :type result: an iterable (normal or asynchronous) of + :class:`~scrapy.http.Request` objects and :ref:`item objects + ` :param spider: the spider whose result is being processed :type spider: :class:`~scrapy.spiders.Spider` object @@ -122,7 +127,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Request` objects and :ref:`item object + iterable of :class:`~scrapy.http.Request` objects and :ref:`item objects `. If it returns ``None``, Scrapy will continue processing this exception, From f422861ef49e8d0e0c2aa4de937fb77b95e063ca Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 25 Mar 2021 20:47:40 +0500 Subject: [PATCH 0260/2083] Add more tests for spider middlewares. --- tests/test_spidermiddleware.py | 188 +++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 78e926adc..2584dec21 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,11 +1,15 @@ +import collections.abc from unittest import mock +from twisted.internet import defer from twisted.trial.unittest import TestCase from twisted.python.failure import Failure from scrapy.spiders import Spider from scrapy.http import Request, Response from scrapy.exceptions import _InvalidOutput +from scrapy.utils.asyncgen import _process_iterable_universal, as_async_generator, collect_asyncgen +from scrapy.utils.defer import deferred_from_coro from scrapy.utils.test import get_crawler from scrapy.core.spidermw import SpiderMiddlewareManager @@ -101,3 +105,187 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): result = self._scrape_response() self.assertIsInstance(result, Failure) self.assertIsInstance(result.value, ZeroDivisionError) + + +class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): + """ Helpers for testing sync, async and mixed middlewares. + + Should work for process_spider_output and, when it's supported, process_start_requests. + """ + + RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects + + @defer.inlineCallbacks + def _get_middleware_result(self, *mw_classes): + for mw_cls in mw_classes: + self.mwman._add_middleware(mw_cls()) + result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider) + return result + + def assertAsyncGeneratorNotIterable(self, o): + with self.assertRaisesRegex(TypeError, + "'(async_generator|MutableAsyncChain)' object is not iterable"): + list(o) + + @defer.inlineCallbacks + def _test_simple_base(self, *mw_classes): + result = yield self._get_middleware_result(*mw_classes) + self.assertIsInstance(result, collections.abc.Iterable) + result_list = list(result) + self.assertEqual(len(result_list), self.RESULT_COUNT) + self.assertIsInstance(result_list[0], self.ITEM_TYPE) + + @defer.inlineCallbacks + def _test_asyncgen_base(self, *mw_classes): + result = yield self._get_middleware_result(*mw_classes) + self.assertIsInstance(result, collections.abc.AsyncIterator) + result_list = yield deferred_from_coro(collect_asyncgen(result)) + self.assertEqual(len(result_list), self.RESULT_COUNT) + self.assertIsInstance(result_list[0], self.ITEM_TYPE) + + @defer.inlineCallbacks + def _test_asyncgen_fail(self, *mw_classes): + result = yield self._get_middleware_result(*mw_classes) + self.assertIsInstance(result, collections.abc.Iterable) + self.assertAsyncGeneratorNotIterable(result) + + +class ProcessSpiderOutputSimpleMiddleware: + def process_spider_output(self, response, result, spider): + for r in result: + yield r + + +class ProcessSpiderOutputAsyncGenMiddleware: + async def process_spider_output(self, response, result, spider): + async for r in as_async_generator(result): + yield r + + +class ProcessSpiderOutputUniversalMiddleware: + def process_spider_output(self, response, result, spider): + @_process_iterable_universal + async def process(result): + async for r in result: + yield r + return process(result) + + +class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): + """ process_spider_output tests for simple callbacks""" + + ITEM_TYPE = dict + MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware + MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware + MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware + + def _scrape_func(self, *args, **kwargs): + yield {'foo': 1} + yield {'foo': 2} + yield {'foo': 3} + + def test_simple(self): + """ Simple mw """ + return self._test_simple_base(self.MW_SIMPLE) + + def test_asyncgen(self): + """ Asyncgen mw """ + return self._test_asyncgen_base(self.MW_ASYNCGEN) + + def test_simple_asyncgen(self): + """ Simple mw -> asyncgen mw """ + return self._test_asyncgen_base(self.MW_ASYNCGEN, + self.MW_SIMPLE) + + def test_asyncgen_simple(self): + """ Asyncgen mw -> simple mw; cannot work """ + return self._test_asyncgen_fail(self.MW_SIMPLE, + self.MW_ASYNCGEN) + + def test_universal(self): + """ Universal mw """ + return self._test_simple_base(self.MW_UNIVERSAL) + + def test_universal_simple(self): + """ Universal mw -> simple mw """ + return self._test_simple_base(self.MW_SIMPLE, + self.MW_UNIVERSAL) + + def test_simple_universal(self): + """ Simple mw -> universal mw """ + return self._test_simple_base(self.MW_UNIVERSAL, + self.MW_SIMPLE) + + def test_universal_asyncgen(self): + """ Universal mw -> asyncgen mw """ + return self._test_asyncgen_base(self.MW_ASYNCGEN, + self.MW_UNIVERSAL) + + def test_asyncgen_universal(self): + """ Asyncgen mw -> universal mw """ + return self._test_asyncgen_base(self.MW_UNIVERSAL, + self.MW_ASYNCGEN) + + +class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): + """ process_spider_output tests for async generator callbacks """ + + async def _scrape_func(self, *args, **kwargs): + for item in super()._scrape_func(): + yield item + + def test_simple(self): + """ Simple mw; cannot work """ + return self._test_asyncgen_fail(self.MW_SIMPLE) + + @defer.inlineCallbacks + def test_simple_asyncgen(self): + """ Simple mw -> asyncgen mw; cannot work """ + result = yield self._get_middleware_result( + self.MW_ASYNCGEN, + self.MW_SIMPLE) + self.assertIsInstance(result, collections.abc.AsyncIterable) + self.assertAsyncGeneratorNotIterable(result) + + def test_universal(self): + """ Universal mw """ + return self._test_asyncgen_base(self.MW_UNIVERSAL) + + def test_universal_simple(self): + """ Universal mw -> simple mw; cannot work """ + return self._test_asyncgen_fail(self.MW_SIMPLE, + self.MW_UNIVERSAL) + + def test_simple_universal(self): + """ Simple mw -> universal mw; cannot work """ + return self._test_asyncgen_fail(self.MW_UNIVERSAL, + self.MW_SIMPLE) + + +class ProcessStartRequestsSimpleMiddleware: + def process_start_requests(self, start_requests, spider): + for r in start_requests: + yield r + + +class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): + """ process_start_requests tests for simple start_requests""" + + ITEM_TYPE = Request + MW_SIMPLE = ProcessStartRequestsSimpleMiddleware + + def _start_requests(self): + for i in range(3): + yield Request(f'https://example.com/{i}', dont_filter=True) + + @defer.inlineCallbacks + def _get_middleware_result(self, *mw_classes): + for mw_cls in mw_classes: + self.mwman._add_middleware(mw_cls()) + start_requests = iter(self._start_requests()) + results = yield self.mwman.process_start_requests(start_requests, self.spider) + return results + + def test_simple(self): + """ Simple mw """ + self._test_simple_base(self.MW_SIMPLE) From 0638d6f01c41f12311b21f06fee5c71f5d08569f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 25 Mar 2021 21:58:29 +0500 Subject: [PATCH 0261/2083] Fix handling middlewares that change sync iterables into async. --- scrapy/core/spidermw.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 33e215971..230332673 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,7 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -import inspect +import collections.abc from itertools import islice from twisted.python.failure import Failure @@ -96,11 +96,10 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_output(self, response, spider, result, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - if inspect.isasyncgen(result): - iter_class = MutableAsyncChain + if isinstance(result, collections.abc.AsyncIterator): + recovered = MutableAsyncChain() else: - iter_class = MutableChain - recovered = iter_class() + recovered = MutableChain() method_list = islice(self.methods['process_spider_output'], start_index, None) for method_index, method in enumerate(method_list, start=start_index): @@ -121,16 +120,23 @@ class SpiderMiddlewareManager(MiddlewareManager): f"iterable, got {type(result)}") raise _InvalidOutput(msg) - return iter_class(result, recovered) + # check this again as the middlewares could change "result" from sync to async + if isinstance(result, collections.abc.AsyncIterator): + return MutableAsyncChain(result, recovered) + else: + return MutableChain(result, recovered) def _process_callback_output(self, response, spider, result): - if inspect.isasyncgen(result): - iter_class = MutableAsyncChain + if isinstance(result, collections.abc.AsyncIterator): + recovered = MutableAsyncChain() else: - iter_class = MutableChain - recovered = iter_class() + recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) - return iter_class(self._process_spider_output(response, spider, result), recovered) + result = self._process_spider_output(response, spider, result) + if isinstance(result, collections.abc.AsyncIterator): + return MutableAsyncChain(result, recovered) + else: + return MutableChain(result, recovered) def scrape_response(self, scrape_func, response, request, spider): def process_callback_output(result): From 1d200258a527668186ea84f14e7c2b23b3200ab5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Fri, 26 Mar 2021 10:45:26 -0300 Subject: [PATCH 0262/2083] Adjust h2 version requirement (#5066) --- setup.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 767c6f6bf..2b60a10af 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ install_requires = [ 'zope.interface>=4.1.3', 'protego>=0.1.15', 'itemadapter>=0.1.0', - 'h2>=3.2.0', + 'h2>=3.0,<4.0', ] extras_require = {} cpython_dependencies = [ diff --git a/tox.ini b/tox.ini index 6907c8906..353977519 100644 --- a/tox.ini +++ b/tox.ini @@ -71,7 +71,7 @@ commands = deps = cryptography==2.0 cssselect==0.9.1 - h2==3.2.0 + h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 From b5f501df7bc3917af3d144bb6556a763f381cd7e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Mar 2021 20:17:41 +0500 Subject: [PATCH 0263/2083] Remove some unneeded code from _AsyncCooperatorAdapter. --- scrapy/utils/defer.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 554edc38c..ca3d79fa6 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -136,8 +136,6 @@ class _AsyncCooperatorAdapter: self.anext_deferred = None result = self.callable(result, *self.callable_args, **self.callable_kwargs) d = self.waiting_deferreds.pop(0) - if d.called: - raise ValueError('Deferred in waiting_deferreds already called') if isinstance(result, defer.Deferred): result.chainDeferred(d) else: @@ -152,8 +150,6 @@ class _AsyncCooperatorAdapter: failure.trap(StopAsyncIteration) self.finished = True for d in self.waiting_deferreds: - if d.called: - raise ValueError('Deferred in waiting_deferreds already called') d.callback(None) def _call_anext(self): @@ -162,9 +158,6 @@ class _AsyncCooperatorAdapter: self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) - def __iter__(self): - return self - def __next__(self): # This puts a new Deferred into self.waiting_deferreds and returns it. # It also calls __anext__() if needed. From 6803779eb7e408b275da62f670aba9deaf3eade9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Mar 2021 22:29:07 +0500 Subject: [PATCH 0264/2083] Add more tests for _process_iterable_universal. --- scrapy/utils/asyncgen.py | 2 +- tests/test_utils_asyncgen.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 39c94ad8a..a79552f76 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -57,6 +57,6 @@ def _process_iterable_universal(process_async): if hasattr(iterable, '__iter__'): # convert process_async to process_sync return process_sync(iterable, *args, **kwargs) - raise ValueError(f"Wrong iterable type {type(iterable)}") + raise TypeError(f"Wrong iterable type {type(iterable)}") return process diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 2f4181d3d..41993a934 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -2,6 +2,7 @@ from twisted.trial import unittest from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.test import get_web_client_agent_req class AsyncgenUtilsTest(unittest.TestCase): @@ -26,6 +27,13 @@ async def process_iterable(iterable): yield i * 2 +@_process_iterable_universal +async def process_iterable_awaiting(iterable): + async for i in iterable: + yield i * 2 + await get_web_client_agent_req('http://example.com') + + class ProcessIterableUniversalTest(unittest.TestCase): def test_normal(self): @@ -38,3 +46,22 @@ class ProcessIterableUniversalTest(unittest.TestCase): iterable = as_async_generator([1, 2, 3]) results = await collect_asyncgen(process_iterable(iterable)) self.assertEqual(results, [2, 4, 6]) + + @deferred_f_from_coro_f + async def test_blocking(self): + iterable = [1, 2, 3] + with self.assertRaisesRegex(RuntimeError, "Synchronously-called function"): + list(process_iterable_awaiting(iterable)) + + def test_invalid_iterable(self): + with self.assertRaisesRegex(TypeError, "Wrong iterable type"): + process_iterable(None) + + @deferred_f_from_coro_f + async def test_invalid_process(self): + @_process_iterable_universal + def process_iterable_invalid(iterable): + pass + + with self.assertRaisesRegex(ValueError, "process_async returned wrong type"): + list(process_iterable_invalid([])) From b247fa9982a390e1380c46e390069c6058d97921 Mon Sep 17 00:00:00 2001 From: Ricardo Amendoeira Date: Mon, 29 Mar 2021 01:48:28 +0100 Subject: [PATCH 0265/2083] Include loading settings in `Running multiple spiders in the same process` section The example in the documentation doesn't take into account the project settings --- docs/topics/practices.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cf1de1bd1..db1ed362e 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -118,6 +118,7 @@ Here is an example that runs multiple spiders simultaneously: :: import scrapy + from scrapy.utils.project import get_project_settings from scrapy.crawler import CrawlerProcess class MySpider1(scrapy.Spider): @@ -128,7 +129,8 @@ Here is an example that runs multiple spiders simultaneously: # Your second spider definition ... - process = CrawlerProcess() + settings = get_project_settings() + process = CrawlerProcess(settings) process.crawl(MySpider1) process.crawl(MySpider2) process.start() # the script will block here until all crawling jobs are finished From 90fe494ba2ab66faa49eb73914a1eae3cebacd0a Mon Sep 17 00:00:00 2001 From: Veniamin Gvozdikov Date: Thu, 1 Apr 2021 11:11:28 +0300 Subject: [PATCH 0266/2083] Rebranding, updated GA code --- docs/_templates/layout.html | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index a6f6cbda8..18a5231ee 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -3,14 +3,9 @@ {% block footer %} {{ super() }} {% endblock %} From d458ccff3b2d6df94df1aa86eeb7d2505d62f2d6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 1 Apr 2021 12:27:35 -0300 Subject: [PATCH 0267/2083] Retry request: priority_adjust cannot be float (Request.priority is int) --- scrapy/downloadermiddlewares/retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 2721db7cf..5965a1c6c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -41,7 +41,7 @@ def get_retry_request( spider: Spider, reason: Union[str, Exception] = 'unspecified', max_retry_times: Optional[int] = None, - priority_adjust: Union[int, float, None] = None, + priority_adjust: Optional[int] = None, logger: Logger = retry_logger, stats_base_key: str = 'retry', ): From 5492972d8a874e9fb7780175e4f396bc8f39378c Mon Sep 17 00:00:00 2001 From: anay2103 <55763427+anay2103@users.noreply.github.com> Date: Thu, 1 Apr 2021 20:30:48 +0300 Subject: [PATCH 0268/2083] added customized filtering examples in logging.rst (#4965) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added customized filtering examples in logging.rst * Update logging.rst * Update docs/topics/logging.rst Co-authored-by: Adrián Chaves * Update docs/topics/logging.rst Co-authored-by: Adrián Chaves * Update docs/topics/logging.rst Co-authored-by: Adrián Chaves * Update docs/topics/logging.rst Co-authored-by: Adrián Chaves * Update logging.rst Co-authored-by: Adrián Chaves --- docs/topics/logging.rst | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index c3445d40e..00806392a 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -242,6 +242,47 @@ e.g. in the spider's ``__init__`` method:: If you run this spider again then INFO messages from ``scrapy.spidermiddlewares.httperror`` logger will be gone. +You can also filter log records by :class:`~logging.LogRecord` data. For +example, you can filter log records by message content using a substring or +a regular expression. Create a :class:`logging.Filter` subclass +and equip it with a regular expression pattern to +filter out unwanted messages:: + + import logging + import re + + class ContentFilter(logging.Filter): + def filter(self, record): + match = re.search(r'\d{3} [Ee]rror, retrying', record.message) + if match: + return False + +A project-level filter may be attached to the root +handler created by Scrapy, this is a wieldy way to +filter all loggers in different parts of the project +(middlewares, spider, etc.):: + + import logging + import scrapy + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + for handler in logging.root.handlers: + handler.addFilter(ContentFilter()) + +Alternatively, you may choose a specific logger +and hide it without affecting other loggers:: + + import logging + import scrapy + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + logger = logging.getLogger('my_logger') + logger.addFilter(ContentFilter()) + scrapy.utils.log module ======================= From ad7456746961fb23beb3ebc051ee1ab0ed82a91e Mon Sep 17 00:00:00 2001 From: Kader DJEHAF Date: Thu, 1 Apr 2021 19:33:56 +0200 Subject: [PATCH 0269/2083] Fix argument type (int -> bool) (#4950) * Fix warning: Expected type 'bool', got 'int' instead * Update defer.py * Fix warning: Expected type 'bool', got 'int' instead Co-authored-by: Eugenio Lacuesta --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 7f267c800..ef2eb3ba6 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -71,7 +71,7 @@ def _urlencode(seq, enc): values = [(to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq for v in (vs if is_listlike(vs) else [vs])] - return urlencode(values, doseq=1) + return urlencode(values, doseq=True) def _get_form(response, formname, formid, formnumber, formxpath): From cc095aa8950975cf203b1d33724cc499043e3f17 Mon Sep 17 00:00:00 2001 From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com> Date: Thu, 1 Apr 2021 23:09:33 +0530 Subject: [PATCH 0270/2083] Windows pip installation guide (#4736) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added initial steps * fixing link * python3 -> python * remaining steps * steps updated * Update docs/intro/install.rst Co-authored-by: Adrián Chaves * added link to Visual Studio * removed 'install V' * Update docs/intro/install.rst Co-authored-by: Adrián Chaves Co-authored-by: Adrián Chaves --- docs/intro/install.rst | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index bf919ce25..8581dde0b 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -12,6 +12,7 @@ Supported Python versions Scrapy requires Python 3.6+, either the CPython implementation (default) or the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). +.. _intro-install-scrapy: Installing Scrapy ================= @@ -29,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with:: pip install Scrapy +We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, +to avoid conflicting with your system packages. + Note that sometimes this may require solving compilation issues for some Scrapy dependencies depending on your operating system, so be sure to check the :ref:`intro-install-platform-notes`. -We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, -to avoid conflicting with your system packages. - For more detailed and platform specifics instructions, as well as troubleshooting information, read on. @@ -117,6 +118,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: conda install -c conda-forge scrapy +To install Scrapy on Windows using ``pip``: + +.. warning:: + This installation method requires “Microsoft Visual C++†for installing some + Scrapy dependencies, which demands significantly more disk space than Anaconda. + +#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. + +#. Run the Visual Studio Installer. + +#. Under the Workloads section, select **C++ build tools**. + +#. Check the installation details and make sure following packages are selected as optional components: + + * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) + + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) + +#. Install the Visual Studio Build Tools. + +Now, you should be able to :ref:`install Scrapy ` using ``pip``. .. _intro-install-ubuntu: @@ -268,4 +290,6 @@ For details, see `Issue #2473 `_. .. _zsh: https://www.zsh.org/ .. _Anaconda: https://docs.anaconda.com/anaconda/ .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html +.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio +.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/ .. _conda-forge: https://conda-forge.org/ From 9e7cbc05ae10ab6c1335d92b4d41429ba8e2bf24 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 1 Apr 2021 15:22:51 -0300 Subject: [PATCH 0271/2083] Fix type for urlencode's doseq argument --- scrapy/commands/bench.py | 2 +- tests/spiders.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 999c987ea..6bdf9eae0 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider): def start_requests(self): qargs = {'total': self.total, 'show': self.show} - url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}' + url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}' return [scrapy.Request(url, dont_filter=True)] def parse(self, response): diff --git a/tests/spiders.py b/tests/spiders.py index 7e579098a..5b45f897e 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -45,7 +45,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") self.start_urls = [url] def parse(self, response): @@ -245,7 +245,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 From 849472535ef9490e8cc2a62cc7f1835b2bc3eaba Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 2 Apr 2021 20:20:35 +0500 Subject: [PATCH 0272/2083] Update docs. --- docs/topics/coroutines.rst | 65 +++++++++++++++++++++++++++++++ docs/topics/spider-middleware.rst | 14 ++++--- scrapy/utils/asyncgen.py | 1 + 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 9b50d9312..6a39dcb5e 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -108,3 +108,68 @@ Common use cases for asynchronous code include: :ref:`the screenshot pipeline example`). .. _aio-libs: https://github.com/aio-libs + +.. _async-spider-middlewares: + +Asynchronous spider middlewares +=============================== + +.. versionadded:: VERSION +.. note:: This currently applies to + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`. + +Middleware methods discussed here can take and return async iterables. They can +return the same type of iterable or they can take a normal one and return an +async one. If such method needs to return an async iterable it must be an async +generator, not just a coroutine that returns an iterable. + +.. autofunction:: scrapy.utils.asyncgen.as_async_generator + +In the simplest form that supports both sync and async input it can be written +like this:: + + from scrapy.utils.asyncgen import as_async_generator + + class ProcessSpiderOutputAsyncGenMiddleware: + async def process_spider_output(self, response, result, spider): + async for r in as_async_generator(result): + # ... do something with r + yield r + +If the middleware input (the callback result for ``process_spider_output``) is +an async iterable, all middlewares that process it must support it. The +built-in ones do, but the ones in your project and 3rd-party ones will need to +be updated to support it, as the code that expects a normal iterable will break +on an async one. If these middlewares receive an async iterable, they must +return one as well. On the other hand, if they receive a normal iterable, they +shouldn't break and ideally should return a normal iterable too. There can be +several possible implementations of this. + +The simplest one, always converting normal iterables to async ones, is provided +above. Because a result of a middleware method is passed to the same method of +the next middleware, it's only possible to mix middlewares with synchronous and +asynchronous implementations of the same method if all synchronous ones are +called first (which isn't always possible). + +Another option is to make separate methods for normal and async iterables and +choose one at run time:: + + from inspect import isasyncgen + + class ProcessSpiderOutputAsyncGenMiddleware: + def _normal_process_spider_output(self, response, result, spider): + # ... do something with normal result + + async def _async_process_spider_output(self, response, result, spider): + # ... do the same with async result + + def process_spider_output(self, response, result, spider): + if isasyncgen(result): + return self._async_process_spider_output(self, response, result, spider) + else: + return self._normal_process_spider_output(self, response, result, spider) + +If you are writing a middleware that you intend to publish or to use in many +projects, this is likely the best way to implement it. It may be possible to +extract common code from both methods to reduce code duplication, as in the +simplest case the only difference between them will be ``for`` vs ``async for``. diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index d09693c16..d87b89292 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -105,18 +105,20 @@ object gives you access, for example, to the :ref:`settings `. This method is called with the results returned from the Spider, after it has processed the response. - :meth:`process_spider_output` must return an iterable (normal or - asynchronous) of :class:`~scrapy.http.Request` objects and - :ref:`item objects `. + :meth:`process_spider_output` must return an iterable of + :class:`~scrapy.http.Request` objects and :ref:`item objects + `. + + .. note:: When defined as a :ref:`coroutine `, this method needs + to be an async generator, not just return an iterable. :param response: the response which generated this output from the spider :type response: :class:`~scrapy.http.Response` object :param result: the result returned by the spider - :type result: an iterable (normal or asynchronous) of - :class:`~scrapy.http.Request` objects and :ref:`item objects - ` + :type result: an iterable of :class:`~scrapy.http.Request` objects and + :ref:`item objects ` :param spider: the spider whose result is being processed :type spider: :class:`~scrapy.spiders.Spider` object diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index a79552f76..6c0bb1d10 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -11,6 +11,7 @@ async def collect_asyncgen(result): async def as_async_generator(it): + """ Wraps an iterator (sync or async) into an async generator. """ if isinstance(it, collections.abc.AsyncIterator): async for r in it: yield r From 30ed7fa349214ad11b4d13c981cbd2fc63ddaf42 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 2 Apr 2021 22:20:56 +0500 Subject: [PATCH 0273/2083] Some cleanup, make sync middlewares fail earlier. --- scrapy/core/spidermw.py | 15 +++++++-------- tests/test_spidermiddleware.py | 18 ++++-------------- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 230332673..b24ccf6ae 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -20,10 +20,6 @@ def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__') -def _fname(f): - return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" - - class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -48,7 +44,7 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - msg = (f"Middleware {_fname(method)} must return None " + msg = (f"Middleware {method.__qualname__} must return None " f"or raise an exception, got {type(result)}") raise _InvalidOutput(msg) except _InvalidOutput: @@ -88,7 +84,7 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = (f"Middleware {_fname(method)} must return None " + msg = (f"Middleware {method.__qualname__} must return None " f"or an iterable, got {type(result)}") raise _InvalidOutput(msg) return _failure @@ -96,7 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_output(self, response, spider, result, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - if isinstance(result, collections.abc.AsyncIterator): + result_async = isinstance(result, collections.abc.AsyncIterator) + if result_async: recovered = MutableAsyncChain() else: recovered = MutableChain() @@ -116,9 +113,11 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) else: - msg = (f"Middleware {_fname(method)} must return an " + msg = (f"Middleware {method.__qualname__} must return an " f"iterable, got {type(result)}") raise _InvalidOutput(msg) + if result_async and isinstance(result, collections.abc.Iterator): + raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable") # check this again as the middlewares could change "result" from sync to async if isinstance(result, collections.abc.AsyncIterator): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 2584dec21..0a6b96c0c 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -122,11 +122,6 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider) return result - def assertAsyncGeneratorNotIterable(self, o): - with self.assertRaisesRegex(TypeError, - "'(async_generator|MutableAsyncChain)' object is not iterable"): - list(o) - @defer.inlineCallbacks def _test_simple_base(self, *mw_classes): result = yield self._get_middleware_result(*mw_classes) @@ -145,9 +140,8 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): @defer.inlineCallbacks def _test_asyncgen_fail(self, *mw_classes): - result = yield self._get_middleware_result(*mw_classes) - self.assertIsInstance(result, collections.abc.Iterable) - self.assertAsyncGeneratorNotIterable(result) + with self.assertRaisesRegex(TypeError, "Synchronous .+ called with an async iterable"): + yield self._get_middleware_result(*mw_classes) class ProcessSpiderOutputSimpleMiddleware: @@ -238,14 +232,10 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): """ Simple mw; cannot work """ return self._test_asyncgen_fail(self.MW_SIMPLE) - @defer.inlineCallbacks def test_simple_asyncgen(self): """ Simple mw -> asyncgen mw; cannot work """ - result = yield self._get_middleware_result( - self.MW_ASYNCGEN, - self.MW_SIMPLE) - self.assertIsInstance(result, collections.abc.AsyncIterable) - self.assertAsyncGeneratorNotIterable(result) + return self._test_asyncgen_fail(self.MW_ASYNCGEN, + self.MW_SIMPLE) def test_universal(self): """ Universal mw """ From 7bd1d888d49d238622007e659e54af76e82bf1c1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 2 Apr 2021 23:06:29 +0500 Subject: [PATCH 0274/2083] More robust sync/async middleware mix checking. --- scrapy/core/spidermw.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index b24ccf6ae..d0d292007 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -92,8 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_output(self, response, spider, result, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - result_async = isinstance(result, collections.abc.AsyncIterator) - if result_async: + last_result_async = isinstance(result, collections.abc.AsyncIterator) + if last_result_async: recovered = MutableAsyncChain() else: recovered = MutableChain() @@ -116,11 +116,11 @@ class SpiderMiddlewareManager(MiddlewareManager): msg = (f"Middleware {method.__qualname__} must return an " f"iterable, got {type(result)}") raise _InvalidOutput(msg) - if result_async and isinstance(result, collections.abc.Iterator): + if last_result_async and isinstance(result, collections.abc.Iterator): raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable") + last_result_async = isinstance(result, collections.abc.AsyncIterator) - # check this again as the middlewares could change "result" from sync to async - if isinstance(result, collections.abc.AsyncIterator): + if last_result_async: return MutableAsyncChain(result, recovered) else: return MutableChain(result, recovered) From 9e3b868dd83a3ac93dfe04b53d5130145154f478 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 3 Apr 2021 17:04:09 +0500 Subject: [PATCH 0275/2083] Use __qualname__ in middleware handling. --- scrapy/core/downloader/middleware.py | 9 +++------ scrapy/core/spidermw.py | 10 +++------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index b0e612e43..177f3f760 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -36,8 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_request must return None, Response or " + f"Middleware {method.__qualname__} must return None, Response or " f"Request, got {response.__class__.__name__}" ) if response: @@ -55,8 +54,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_response must return Response or Request, " + f"Middleware {method.__qualname__} must return Response or Request, " f"got {type(response)}" ) if isinstance(response, Request): @@ -70,8 +68,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_exception must return None, Response or " + f"Middleware {method.__qualname__} must return None, Response or " f"Request, got {type(response)}" ) if response: diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 289292da7..e8733c4ad 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -18,10 +18,6 @@ def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') -def _fname(f): - return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" - - class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -46,7 +42,7 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - msg = (f"Middleware {_fname(method)} must return None " + msg = (f"Middleware {method.__qualname__} must return None " f"or raise an exception, got {type(result)}") raise _InvalidOutput(msg) except _InvalidOutput: @@ -83,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = (f"Middleware {_fname(method)} must return None " + msg = (f"Middleware {method.__qualname__} must return None " f"or an iterable, got {type(result)}") raise _InvalidOutput(msg) return _failure @@ -108,7 +104,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) else: - msg = (f"Middleware {_fname(method)} must return an " + msg = (f"Middleware {method.__qualname__} must return an " f"iterable, got {type(result)}") raise _InvalidOutput(msg) From a9e96f99077865bc2d844ecf0ae174db97ba6d8b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 3 Apr 2021 17:40:45 +0500 Subject: [PATCH 0276/2083] Add typing for middleware and coroutine related code. --- scrapy/core/downloader/middleware.py | 14 ++++++++----- scrapy/core/spidermw.py | 31 ++++++++++++++++++---------- scrapy/middleware.py | 11 +++++----- scrapy/utils/asyncgen.py | 5 ++++- scrapy/utils/defer.py | 23 ++++++++++++--------- scrapy/utils/python.py | 7 ++++--- 6 files changed, 56 insertions(+), 35 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index b0e612e43..441fc9fa6 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,8 +3,12 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from twisted.internet import defer +from typing import Callable, Union +from twisted.internet import defer +from twisted.python.failure import Failure + +from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager @@ -29,9 +33,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process_exception'): self.methods['process_exception'].appendleft(mw.process_exception) - def download(self, download_func, request, spider): + def download(self, download_func: Callable, request: Request, spider: Spider): @defer.inlineCallbacks - def process_request(request): + def process_request(request: Request): for method in self.methods['process_request']: response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): @@ -45,7 +49,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return (yield download_func(request=request, spider=spider)) @defer.inlineCallbacks - def process_response(response): + def process_response(response: Union[Response, Request]): if response is None: raise TypeError("Received None in process_response") elif isinstance(response, Request): @@ -64,7 +68,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response @defer.inlineCallbacks - def process_exception(failure): + def process_exception(failure: Failure): exception = failure.value for method in self.methods['process_exception']: response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 289292da7..b09adf8e2 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,25 +3,32 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +from collections.abc import Iterable, AsyncIterable from itertools import islice +from typing import Callable, Union, Any from twisted.python.failure import Failure +from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput +from scrapy.http import Response from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list from scrapy.utils.defer import mustbe_deferred from scrapy.utils.python import MutableChain -def _isiterable(possible_iterator): - return hasattr(possible_iterator, '__iter__') +def _isiterable(o): + return isinstance(o, Iterable) def _fname(f): return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" +ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] + + class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -41,7 +48,7 @@ class SpiderMiddlewareManager(MiddlewareManager): process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) - def _process_spider_input(self, scrape_func, response, request, spider): + def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider): for method in self.methods['process_spider_input']: try: result = method(response=response, spider=spider) @@ -55,7 +62,8 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to): + def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable, + exception_processor_index: int, recover_to: MutableChain): try: for r in iterable: yield r @@ -66,7 +74,7 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recover_to.extend(exception_result) - def _process_spider_exception(self, response, spider, _failure, start_index=0): + def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index=0): exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -88,7 +96,8 @@ class SpiderMiddlewareManager(MiddlewareManager): raise _InvalidOutput(msg) return _failure - def _process_spider_output(self, response, spider, result, start_index=0): + def _process_spider_output(self, response: Response, spider: Spider, + result: Iterable, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered = MutableChain() @@ -114,21 +123,21 @@ class SpiderMiddlewareManager(MiddlewareManager): return MutableChain(result, recovered) - def _process_callback_output(self, response, spider, result): + def _process_callback_output(self, response: Response, spider: Spider, result: Iterable): recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) return MutableChain(self._process_spider_output(response, spider, result), recovered) - def scrape_response(self, scrape_func, response, request, spider): - def process_callback_output(result): + def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider): + def process_callback_output(result: Iterable): return self._process_callback_output(response, spider, result) - def process_spider_exception(_failure): + def process_spider_exception(_failure: Failure): return self._process_spider_exception(response, spider, _failure) dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) return dfd - def process_start_requests(self, start_requests, spider): + def process_start_requests(self, start_requests, spider: Spider): return self._process_chain('process_start_requests', start_requests, spider) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 5040378ea..c53cfb814 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,6 +1,7 @@ -from collections import defaultdict, deque import logging import pprint +from collections import defaultdict, deque +from typing import Callable from scrapy.exceptions import NotConfigured from scrapy.utils.misc import create_instance, load_object @@ -16,7 +17,7 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods = defaultdict(deque) + self.methods: dict[str, deque[Callable]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -58,13 +59,13 @@ class MiddlewareManager: if hasattr(mw, 'close_spider'): self.methods['close_spider'].appendleft(mw.close_spider) - def _process_parallel(self, methodname, obj, *args): + def _process_parallel(self, methodname: str, obj, *args): return process_parallel(self.methods[methodname], obj, *args) - def _process_chain(self, methodname, obj, *args): + def _process_chain(self, methodname: str, obj, *args): return process_chain(self.methods[methodname], obj, *args) - def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args): + def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args): return process_chain_both(self.methods[cb_methodname], self.methods[eb_methodname], obj, *args) diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 7f697af5f..c290e376c 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,4 +1,7 @@ -async def collect_asyncgen(result): +from collections.abc import AsyncIterable + + +async def collect_asyncgen(result: AsyncIterable): results = [] async for x in result: results.append(x) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 6db9cc117..c382a00f7 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -3,16 +3,19 @@ Helper functions for dealing with Twisted deferreds """ import asyncio import inspect +from collections.abc import Coroutine from functools import wraps +from typing import Callable, Iterable, Any from twisted.internet import defer, task from twisted.python import failure +from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import is_asyncio_reactor_installed -def defer_fail(_failure): +def defer_fail(_failure: Failure): """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -47,7 +50,7 @@ def defer_result(result): return defer_succeed(result) -def mustbe_deferred(f, *args, **kw): +def mustbe_deferred(f: Callable, *args, **kw): """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -64,7 +67,7 @@ def mustbe_deferred(f, *args, **kw): return defer_result(result) -def parallel(iterable, count, callable, *args, **named): +def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named): """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -75,7 +78,7 @@ def parallel(iterable, count, callable, *args, **named): return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) -def process_chain(callbacks, input, *a, **kw): +def process_chain(callbacks: Iterable[Callable], input, *a, **kw): """Return a Deferred built by chaining the given callbacks""" d = defer.Deferred() for x in callbacks: @@ -84,7 +87,7 @@ def process_chain(callbacks, input, *a, **kw): return d -def process_chain_both(callbacks, errbacks, input, *a, **kw): +def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw): """Return a Deferred built by chaining the given callbacks and errbacks""" d = defer.Deferred() for cb, eb in zip(callbacks, errbacks): @@ -100,7 +103,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw): return d -def process_parallel(callbacks, input, *a, **kw): +def process_parallel(callbacks: Iterable[Callable], input, *a, **kw): """Return a Deferred with the output of all successful calls to the given callbacks """ @@ -110,7 +113,7 @@ def process_parallel(callbacks, input, *a, **kw): return d -def iter_errback(iterable, errback, *a, **kw): +def iter_errback(iterable: Iterable, errback: Callable, *a, **kw): """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -124,7 +127,7 @@ def iter_errback(iterable, errback, *a, **kw): errback(failure.Failure(), *a, **kw) -def deferred_from_coro(o): +def deferred_from_coro(o) -> Any: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, defer.Deferred): return o @@ -139,7 +142,7 @@ def deferred_from_coro(o): return o -def deferred_f_from_coro_f(coro_f): +def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]): """ Converts a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. @@ -151,7 +154,7 @@ def deferred_f_from_coro_f(coro_f): return f -def maybeDeferred_coro(f, *args, **kw): +def maybeDeferred_coro(f: Callable, *args, **kw): """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ try: result = f(*args, **kw) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5703fd4c3..bcc12f24f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,6 +8,7 @@ import re import sys import warnings import weakref +from collections.abc import Iterable from functools import partial, wraps from itertools import chain @@ -335,15 +336,15 @@ else: gc.collect() -class MutableChain: +class MutableChain(Iterable): """ Thin wrapper around itertools.chain, allowing to add iterables "in-place" """ - def __init__(self, *args): + def __init__(self, *args: Iterable): self.data = chain.from_iterable(args) - def extend(self, *iterables): + def extend(self, *iterables: Iterable): self.data = chain(self.data, chain.from_iterable(iterables)) def __iter__(self): From 414dd1119a7e15c612de7bd0fe560b6ca30b505f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 3 Apr 2021 17:54:55 +0500 Subject: [PATCH 0277/2083] Drop an unused import. --- scrapy/core/spidermw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index b09adf8e2..9a5305376 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,7 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -from collections.abc import Iterable, AsyncIterable +from collections.abc import Iterable from itertools import islice from typing import Callable, Union, Any From 7dc857668f16e8c52ff44662aceb32f93ae3d80e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 4 Apr 2021 16:15:33 +0500 Subject: [PATCH 0278/2083] Also some typing for Scraper. --- scrapy/core/scraper.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 0d3e3450f..4a3eff888 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -3,12 +3,14 @@ extracts information from them""" import logging from collections import deque +from collections.abc import Iterable +from typing import Union from itemadapter import is_item from twisted.internet import defer from twisted.python.failure import Failure -from scrapy import signals +from scrapy import signals, Spider from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response @@ -120,7 +122,7 @@ class Scraper: response, request, deferred = slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, result, request, spider): + def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider): """ Handle the downloaded response or failure through the spider callback/errback """ @@ -131,7 +133,7 @@ class Scraper: dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, result, request, spider): + def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider): """ Handle the different cases of request's result been a Response or a Failure """ @@ -141,7 +143,7 @@ class Scraper: dfd = self.call_spider(result, request, spider) return dfd.addErrback(self._log_download_errors, result, request, spider) - def call_spider(self, result, request, spider): + def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider): if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request @@ -156,7 +158,7 @@ class Scraper: dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) - def handle_spider_error(self, _failure, request, response, spider): + def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider): exc = _failure.value if isinstance(exc, CloseSpider): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') @@ -177,7 +179,7 @@ class Scraper: spider=spider ) - def handle_spider_output(self, result, request, response, spider): + def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider): if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, response, spider) From e7d51886ef90f5b2d4fa13911381680ac192fc37 Mon Sep 17 00:00:00 2001 From: Mayank Singhal <17mayanksinghal@gmail.com> Date: Tue, 6 Apr 2021 02:21:18 +0530 Subject: [PATCH 0279/2083] Find bash from PATH instead of /bin/bash --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index ff68bf1ae..87d5d3047 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,7 +8,7 @@ PYTHON = python SPHINXOPTS = PAPER = SOURCES = -SHELL = /bin/bash +SHELL = /usr/bin/env bash ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ -D latex_elements.papersize=$(PAPER) \ From a71d6ef29da8e3dfa906fbaa74ee1db64567f856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 6 Apr 2021 16:09:07 +0200 Subject: [PATCH 0280/2083] 2.5.0 release notes (#5028) Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> --- docs/news.rst | 193 ++++++++++++++++++++++++++++++- docs/topics/request-response.rst | 4 +- docs/topics/settings.rst | 5 +- docs/topics/signals.rst | 2 +- tests/test_webclient.py | 2 +- 5 files changed, 200 insertions(+), 6 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index d9fe897ad..0ea412e75 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,190 @@ Release notes ============= +.. _release-2.5.0: + +Scrapy 2.5.0 (2021-04-06) +------------------------- + +Highlights: + +- Official Python 3.9 support + +- Experimental :ref:`HTTP/2 support ` + +- New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function + to retry requests from spider callbacks + +- New :class:`~scrapy.signals.headers_received` signal that allows stopping + downloads early + +- New :class:`Response.protocol ` attribute + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed all code that :ref:`was deprecated in 1.7.0 <1.7-deprecations>` and + had not :ref:`already been removed in 2.4.0 <2.4-deprecation-removals>`. + (:issue:`4901`) + +- Removed support for the ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment + variable, :ref:`deprecated in 1.8.0 <1.8-deprecations>`. (:issue:`4912`) + + +Deprecations +~~~~~~~~~~~~ + +- The :mod:`scrapy.utils.py36` module is now deprecated in favor of + :mod:`scrapy.utils.asyncgen`. (:issue:`4900`) + + +New features +~~~~~~~~~~~~ + +- Experimental :ref:`HTTP/2 support ` through a new download handler + that can be assigned to the ``https`` protocol in the + :setting:`DOWNLOAD_HANDLERS` setting. + (:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`) + +- The new :func:`scrapy.downloadermiddlewares.retry.get_retry_request` + function may be used from spider callbacks or middlewares to handle the + retrying of a request beyond the scenarios that + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` supports. + (:issue:`3590`, :issue:`3685`, :issue:`4902`) + +- The new :class:`~scrapy.signals.headers_received` signal gives early access + to response headers and allows :ref:`stopping downloads + `. + (:issue:`1772`, :issue:`4897`) + +- The new :attr:`Response.protocol ` + attribute gives access to the string that identifies the protocol used to + download a response. (:issue:`4878`) + +- :ref:`Stats ` now include the following entries that indicate + the number of successes and failures in storing + :ref:`feeds `:: + + feedexport/success_count/ + feedexport/failed_count/ + + Where ```` is the feed storage backend class name, such as + :class:`~scrapy.extensions.feedexport.FileFeedStorage` or + :class:`~scrapy.extensions.feedexport.FTPFeedStorage`. + + (:issue:`3947`, :issue:`4850`) + +- The :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` spider + middleware now logs ignored URLs with ``INFO`` :ref:`logging level + ` instead of ``DEBUG``, and it now includes the following entry + into :ref:`stats ` to keep track of the number of ignored + URLs:: + + urllength/request_ignored_count + + (:issue:`5036`) + +- The + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + downloader middleware now logs the number of decompressed responses and the + total count of resulting bytes:: + + httpcompression/response_bytes + httpcompression/response_count + + (:issue:`4797`, :issue:`4799`) + + +Bug fixes +~~~~~~~~~ + +- Fixed installation on PyPy installing PyDispatcher in addition to + PyPyDispatcher, which could prevent Scrapy from working depending on which + package got imported. (:issue:`4710`, :issue:`4814`) + +- When inspecting a callback to check if it is a generator that also returns + a value, an exception is no longer raised if the callback has a docstring + with lower indentation than the following code. + (:issue:`4477`, :issue:`4935`) + +- The `Content-Length `_ + header is no longer omitted from responses when using the default, HTTP/1.1 + download handler (see :setting:`DOWNLOAD_HANDLERS`). + (:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`) + +- Setting the :reqmeta:`handle_httpstatus_all` request meta key to ``False`` + now has the same effect as not setting it at all, instead of having the + same effect as setting it to ``True``. + (:issue:`3851`, :issue:`4694`) + + +Documentation +~~~~~~~~~~~~~ + +- Added instructions to :ref:`install Scrapy in Windows using pip + `. + (:issue:`4715`, :issue:`4736`) + +- Logging documentation now includes :ref:`additional ways to filter logs + `. + (:issue:`4216`, :issue:`4257`, :issue:`4965`) + +- Covered how to deal with long lists of allowed domains in the :ref:`FAQ + `. (:issue:`2263`, :issue:`3667`) + +- Covered scrapy-bench_ in :ref:`benchmarking`. + (:issue:`4996`, :issue:`5016`) + +- Clarified that one :ref:`extension ` instance is created + per crawler. + (:issue:`5014`) + +- Fixed some errors in examples. + (:issue:`4829`, :issue:`4830`, :issue:`4907`, :issue:`4909`, + :issue:`5008`) + +- Fixed some external links, typos, and so on. + (:issue:`4892`, :issue:`4899`, :issue:`4936`, :issue:`4942`, :issue:`5005`, + :issue:`5063`) + +- The :ref:`list of Request.meta keys ` is now sorted + alphabetically. + (:issue:`5061`, :issue:`5065`) + +- Updated references to Scrapinghub, which is now called Zyte. + (:issue:`4973`, :issue:`5072`) + +- Added a mention to contributors in the README. (:issue:`4956`) + +- Reduced the top margin of lists. (:issue:`4974`) + + +Quality Assurance +~~~~~~~~~~~~~~~~~ + +- Made Python 3.9 support official (:issue:`4757`, :issue:`4759`) + +- Extended typing hints (:issue:`4895`) + +- Fixed deprecated uses of the Twisted API. + (:issue:`4940`, :issue:`4950`, :issue:`5073`) + +- Made our tests run with the new pip resolver. + (:issue:`4710`, :issue:`4814`) + +- Added tests to ensure that :ref:`coroutine support ` + is tested. (:issue:`4987`) + +- Migrated from Travis CI to GitHub Actions. (:issue:`4924`) + +- Fixed CI issues. + (:issue:`4986`, :issue:`5020`, :issue:`5022`, :issue:`5027`, :issue:`5052`, + :issue:`5053`) + +- Implemented code refactorings, style fixes and cleanups. + (:issue:`4911`, :issue:`4982`, :issue:`5001`, :issue:`5002`, :issue:`5076`) + + .. _release-2.4.1: Scrapy 2.4.1 (2020-11-17) @@ -97,6 +281,8 @@ Backward-incompatible changes (:issue:`4717`, :issue:`4823`) +.. _2.4-deprecation-removals: + Deprecation removals ~~~~~~~~~~~~~~~~~~~~ @@ -1433,6 +1619,8 @@ Deprecation removals * ``scrapy.xlib`` has been removed (:issue:`4015`) +.. _1.8-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -1789,6 +1977,8 @@ The following deprecated settings have also been removed (:issue:`3578`): * ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`) +.. _1.7-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -4184,7 +4374,7 @@ API changes - ``url`` and ``body`` attributes of Request objects are now read-only (#230) - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231) - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default) -- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) +- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) - Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself. - Changes to Scrapy Manager (now called "Crawler"): - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` @@ -4331,6 +4521,7 @@ First release of Scrapy. .. _resource: https://docs.python.org/2/library/resource.html .. _robots.txt: https://www.robotstxt.org/ .. _scrapely: https://github.com/scrapy/scrapely +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench .. _service_identity: https://service-identity.readthedocs.io/en/stable/ .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index c0283df01..500781c05 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -703,7 +703,7 @@ Response objects .. versionadded:: 2.1.0 The ``ip_address`` parameter. - .. versionadded:: VERSION + .. versionadded:: 2.5.0 The ``protocol`` parameter. .. attribute:: Response.url @@ -809,7 +809,7 @@ Response objects .. attribute:: Response.protocol - .. versionadded:: VERSION + .. versionadded:: 2.5.0 The protocol that was used to download the response. For instance: "HTTP/1.0", "HTTP/1.1" diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 9dcee64eb..f5dca824f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -677,6 +677,8 @@ handler (without replacement), place this in your ``settings.py``:: 'ftp': None, } +.. _http2: + The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update :setting:`DOWNLOAD_HANDLERS` as follows:: @@ -703,7 +705,8 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update - No support for `server pushes`_, which are ignored. - - No support for the :signal:`bytes_received` signal. + - No support for the :signal:`bytes_received` and + :signal:`headers_received` signals. .. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 98cfa606c..3d838fb63 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -403,7 +403,7 @@ bytes_received headers_received ~~~~~~~~~~~~~~~~ -.. versionadded:: VERSION +.. versionadded:: 2.5 .. signal:: headers_received .. function:: headers_received(headers, request, spider) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index f935a8689..6e4cb9b6e 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -418,7 +418,7 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): def testPayloadDisabledCipher(self): if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"): - self.skipTest("This does work in PyPy with cryptography<=2.3.1") + self.skipTest("This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1") s = "0123456789" * 10 settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) From e63188cbf753d560e43d8489c821bd6eb9fe54e9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 6 Apr 2021 19:13:32 +0500 Subject: [PATCH 0281/2083] =?UTF-8?q?Bump=20version:=202.4.1=20=E2=86=92?= =?UTF-8?q?=202.5.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 956c512cb..d9e4a2831 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.1 +current_version = 2.5.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 005119baa..437459cd9 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.4.1 +2.5.0 From 8603f9d7a5524b7709b4e8a8fc04a75a9a4f0ffe Mon Sep 17 00:00:00 2001 From: Ricardo Amendoeira Date: Tue, 6 Apr 2021 20:23:07 +0100 Subject: [PATCH 0282/2083] Apply changes to other examples in the same section. --- docs/topics/practices.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index db1ed362e..15ac520e2 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -118,8 +118,8 @@ Here is an example that runs multiple spiders simultaneously: :: import scrapy - from scrapy.utils.project import get_project_settings from scrapy.crawler import CrawlerProcess + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -143,6 +143,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: from twisted.internet import reactor from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -153,7 +154,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: ... configure_logging() - runner = CrawlerRunner() + settings = get_project_settings() + runner = CrawlerRunner(settings) runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() @@ -168,6 +170,7 @@ Same example but running the spiders sequentially by chaining the deferreds: from twisted.internet import reactor, defer from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -178,7 +181,8 @@ Same example but running the spiders sequentially by chaining the deferreds: ... configure_logging() - runner = CrawlerRunner() + settings = get_project_settings() + runner = CrawlerRunner(settings) @defer.inlineCallbacks def crawl(): From 5a75b14a5fbbbd37c14aa7317761655ac7706b70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 7 Apr 2021 12:33:37 +0200 Subject: [PATCH 0283/2083] docs: require sphinx-rtd-theme>=0.5.2 and the latest pip to prevent installing breaking docutils>=0.17 --- .readthedocs.yml | 1 - docs/pip.txt | 3 --- docs/requirements.txt | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 docs/pip.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index 2d781ae81..80a1cd036 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,6 +12,5 @@ python: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image version: 3.8 # Keep in sync with .github/workflows/checks.yml install: - - requirements: docs/pip.txt - requirements: docs/requirements.txt - path: . diff --git a/docs/pip.txt b/docs/pip.txt deleted file mode 100644 index 095e53a0d..000000000 --- a/docs/pip.txt +++ /dev/null @@ -1,3 +0,0 @@ -# In pip 20.3-21.0, the default dependency resolver causes the build in -# ReadTheDocs to fail due to memory exhaustion or timeout. -pip<20.3 diff --git a/docs/requirements.txt b/docs/requirements.txt index 3d34b47da..a0930ba1e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ Sphinx>=3.0 sphinx-hoverxref>=0.2b1 sphinx-notfound-page>=0.4 -sphinx_rtd_theme>=0.4 +sphinx-rtd-theme>=0.5.2 \ No newline at end of file From 91f81445524630843561c8a3c71b7fa0f081d783 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 20 Nov 2019 00:30:18 -0300 Subject: [PATCH 0284/2083] Remove deprecated Spider.make_requests_from_url method --- scrapy/spiders/__init__.py | 27 ++------------------------- tests/test_spider.py | 33 --------------------------------- 2 files changed, 2 insertions(+), 58 deletions(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index c13ba4b3c..d8248c606 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -4,14 +4,12 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ import logging -import warnings from typing import Optional from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider -from scrapy.utils.deprecate import method_is_overridden class Spider(object_ref): @@ -57,34 +55,13 @@ class Spider(object_ref): crawler.signals.connect(self.close, signals.spider_closed) def start_requests(self): - cls = self.__class__ if not self.start_urls and hasattr(self, 'start_url'): raise AttributeError( "Crawling could not start: 'start_urls' not found " "or empty (but found 'start_url' attribute instead, " "did you miss an 's'?)") - if method_is_overridden(cls, Spider, 'make_requests_from_url'): - warnings.warn( - "Spider.make_requests_from_url method is deprecated; it " - "won't be called in future Scrapy releases. Please " - "override Spider.start_requests method instead " - f"(see {cls.__module__}.{cls.__name__}).", - ) - for url in self.start_urls: - yield self.make_requests_from_url(url) - else: - for url in self.start_urls: - yield Request(url, dont_filter=True) - - def make_requests_from_url(self, url): - """ This method is deprecated. """ - warnings.warn( - "Spider.make_requests_from_url method is deprecated: " - "it will be removed and not be called by the default " - "Spider.start_requests method in future Scrapy releases. " - "Please override Spider.start_requests method instead." - ) - return Request(url, dont_filter=True) + for url in self.start_urls: + yield Request(url, dont_filter=True) def _parse(self, response, **kwargs): return self.parse(response, **kwargs) diff --git a/tests/test_spider.py b/tests/test_spider.py index d23543f6a..a7c3ee048 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -584,39 +584,6 @@ class DeprecationTest(unittest.TestCase): assert issubclass(CrawlSpider, Spider) assert isinstance(CrawlSpider(name='foo'), Spider) - def test_make_requests_from_url_deprecated(self): - class MySpider4(Spider): - name = 'spider1' - start_urls = ['http://example.com'] - - class MySpider5(Spider): - name = 'spider2' - start_urls = ['http://example.com'] - - def make_requests_from_url(self, url): - return Request(url + "/foo", dont_filter=True) - - with warnings.catch_warnings(record=True) as w: - # spider without overridden make_requests_from_url method - # doesn't issue a warning - spider1 = MySpider4() - self.assertEqual(len(list(spider1.start_requests())), 1) - self.assertEqual(len(w), 0) - - # spider without overridden make_requests_from_url method - # should issue a warning when called directly - request = spider1.make_requests_from_url("http://www.example.com") - self.assertTrue(isinstance(request, Request)) - self.assertEqual(len(w), 1) - - # spider with overridden make_requests_from_url issues a warning, - # but the method still works - spider2 = MySpider5() - requests = list(spider2.start_requests()) - self.assertEqual(len(requests), 1) - self.assertEqual(requests[0].url, 'http://example.com/foo') - self.assertEqual(len(w), 2) - class NoParseMethodSpiderTest(unittest.TestCase): From b6f77806b0ec414a28cfcbac3fa2d928040548d6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 9 Apr 2021 12:19:30 -0300 Subject: [PATCH 0285/2083] Engine tests: fix item class spider, add minimal type hints --- setup.cfg | 3 --- tests/test_engine.py | 24 +++++++++++------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/setup.cfg b/setup.cfg index 89b4ec57e..1fab6fe22 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,9 +43,6 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_engine] -ignore_errors = True - [mypy-tests.test_exporters] ignore_errors = True diff --git a/tests/test_engine.py b/tests/test_engine.py index ef1204f94..c406d2577 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -56,7 +56,7 @@ class TestSpider(Spider): name_re = re.compile(r"

(.*?)

", re.M) price_re = re.compile(r">Price: \$(.*?)<", re.M) - item_cls = TestItem + item_cls: type = TestItem def parse(self, response): xlink = LinkExtractor() @@ -66,15 +66,15 @@ class TestSpider(Spider): yield Request(url=link.url, callback=self.parse_item) def parse_item(self, response): - item = self.item_cls() + adapter = ItemAdapter(self.item_cls()) m = self.name_re.search(response.text) if m: - item['name'] = m.group(1) - item['url'] = response.url + adapter['name'] = m.group(1) + adapter['url'] = response.url m = self.price_re.search(response.text) if m: - item['price'] = m.group(1) - return item + adapter['price'] = m.group(1) + return adapter.item class TestDupeFilterSpider(TestSpider): @@ -87,7 +87,7 @@ class DictItemsSpider(TestSpider): class AttrsItemsSpider(TestSpider): - item_class = AttrsItem + item_cls = AttrsItem try: @@ -97,14 +97,12 @@ except ImportError: else: TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) - class DataClassItemsSpider(DictItemsSpider): + class _dataclass_spider(DictItemsSpider): def parse_item(self, response): item = super().parse_item(response) - return TestDataClass( - name=item.get('name'), - url=item.get('url'), - price=item.get('price'), - ) + return TestDataClass(**item) + + DataClassItemsSpider = _dataclass_spider class ItemZeroDivisionErrorSpider(TestSpider): From 4673f05ddec85f18a890e606a6b85282035e26be Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 9 Apr 2021 23:42:24 +0500 Subject: [PATCH 0286/2083] Cleanup of slot handling in Scraper. --- scrapy/core/scraper.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 0d3e3450f..4a3ddea10 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -82,28 +82,26 @@ class Scraper: def close_spider(self, spider): """Close a spider being scraped and release its resources""" - slot = self.slot - slot.closing = defer.Deferred() - slot.closing.addCallback(self.itemproc.close_spider) - self._check_if_closing(spider, slot) - return slot.closing + self.slot.closing = defer.Deferred() + self.slot.closing.addCallback(self.itemproc.close_spider) + self._check_if_closing(spider) + return self.slot.closing def is_idle(self): """Return True if there isn't any more spiders to process""" return not self.slot - def _check_if_closing(self, spider, slot): - if slot.closing and slot.is_idle(): - slot.closing.callback(spider) + def _check_if_closing(self, spider): + if self.slot.closing and self.slot.is_idle(): + self.slot.closing.callback(spider) def enqueue_scrape(self, response, request, spider): - slot = self.slot - dfd = slot.add_response_request(response, request) + dfd = self.slot.add_response_request(response, request) def finish_scraping(_): - slot.finish_response(response, request) - self._check_if_closing(spider, slot) - self._scrape_next(spider, slot) + self.slot.finish_response(response, request) + self._check_if_closing(spider) + self._scrape_next(spider) return _ dfd.addBoth(finish_scraping) @@ -112,12 +110,12 @@ class Scraper: {'request': request}, exc_info=failure_to_exc_info(f), extra={'spider': spider})) - self._scrape_next(spider, slot) + self._scrape_next(spider) return dfd - def _scrape_next(self, spider, slot): - while slot.queue: - response, request, deferred = slot.next_response_request_deferred() + def _scrape_next(self, spider): + while self.slot.queue: + response, request, deferred = self.slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) def _scrape(self, result, request, spider): From d8d1dc5b508832598591ae9d3d3694f77f70492d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 12 Apr 2021 10:43:02 -0300 Subject: [PATCH 0287/2083] Ignore typing warning in test --- tests/test_engine.py | 4 +--- tox.ini | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index c406d2577..b2d1d83c7 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -97,13 +97,11 @@ except ImportError: else: TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) - class _dataclass_spider(DictItemsSpider): + class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef] def parse_item(self, response): item = super().parse_item(response) return TestDataClass(**item) - DataClassItemsSpider = _dataclass_spider - class ItemZeroDivisionErrorSpider(TestSpider): custom_settings = { diff --git a/tox.ini b/tox.ini index 353977519..5b0606f8f 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,7 @@ basepython = python3 deps = mypy==0.780 commands = - mypy {posargs: scrapy tests} + mypy --show-error-codes {posargs: scrapy tests} [testenv:security] basepython = python3 From a4415e4e6fa0f9becb40968438a77f4ea262633a Mon Sep 17 00:00:00 2001 From: Mayank Singhal <17mayanksinghal@gmail.com> Date: Tue, 13 Apr 2021 17:20:55 +0530 Subject: [PATCH 0288/2083] Add DataURI download handler in DOWNLOAD_HANDLERS_BASE documentation --- docs/topics/settings.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f5dca824f..1d5babcec 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -657,6 +657,7 @@ DOWNLOAD_HANDLERS_BASE Default:: { + 'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler', 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', From 76fa2257ef0280fc82e123457c791254cc2f185e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Apr 2021 20:01:18 +0500 Subject: [PATCH 0289/2083] Add typing also for return values, other small fixes. --- scrapy/core/spidermw.py | 33 ++++++++++++++++++--------------- scrapy/middleware.py | 24 ++++++++++++++---------- scrapy/utils/defer.py | 24 ++++++++++++------------ 3 files changed, 44 insertions(+), 37 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index dc0e58095..05df8c988 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,10 +3,10 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -from collections.abc import Iterable from itertools import islice -from typing import Callable, Union, Any +from typing import Callable, Union, Any, Generator, Iterable +from twisted.internet.defer import Deferred from twisted.python.failure import Failure from scrapy import Request, Spider @@ -18,13 +18,13 @@ from scrapy.utils.defer import mustbe_deferred from scrapy.utils.python import MutableChain -def _isiterable(o): - return isinstance(o, Iterable) - - ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] +def _isiterable(o) -> bool: + return isinstance(o, Iterable) + + class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -44,7 +44,8 @@ class SpiderMiddlewareManager(MiddlewareManager): process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) - def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider): + def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, + spider: Spider) -> Any: for method in self.methods['process_spider_input']: try: result = method(response=response, spider=spider) @@ -59,7 +60,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(response, request, spider) def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable, - exception_processor_index: int, recover_to: MutableChain): + exception_processor_index: int, recover_to: MutableChain) -> Generator: try: for r in iterable: yield r @@ -70,7 +71,8 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recover_to.extend(exception_result) - def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index=0): + def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, + start_index: int = 0) -> Union[Failure, MutableChain]: exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -93,7 +95,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return _failure def _process_spider_output(self, response: Response, spider: Spider, - result: Iterable, start_index=0): + result: Iterable, start_index: int = 0) -> MutableChain: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered = MutableChain() @@ -119,21 +121,22 @@ class SpiderMiddlewareManager(MiddlewareManager): return MutableChain(result, recovered) - def _process_callback_output(self, response: Response, spider: Spider, result: Iterable): + def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) return MutableChain(self._process_spider_output(response, spider, result), recovered) - def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider): - def process_callback_output(result: Iterable): + def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, + spider: Spider) -> Deferred: + def process_callback_output(result: Iterable) -> MutableChain: return self._process_callback_output(response, spider, result) - def process_spider_exception(_failure: Failure): + def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: return self._process_spider_exception(response, spider, _failure) dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) return dfd - def process_start_requests(self, start_requests, spider: Spider): + def process_start_requests(self, start_requests, spider: Spider) -> Deferred: return self._process_chain('process_start_requests', start_requests, spider) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index c53cfb814..3f8c1cbf5 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,9 +1,13 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable +from typing import Callable, Dict, Deque +from twisted.internet import defer + +from scrapy import Spider from scrapy.exceptions import NotConfigured +from scrapy.settings import Settings from scrapy.utils.misc import create_instance, load_object from scrapy.utils.defer import process_parallel, process_chain, process_chain_both @@ -17,16 +21,16 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods: dict[str, deque[Callable]] = defaultdict(deque) + self.methods: Dict[str, Deque[Callable]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: Settings) -> list: raise NotImplementedError @classmethod - def from_settings(cls, settings, crawler=None): + def from_settings(cls, settings: Settings, crawler=None): mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] @@ -53,24 +57,24 @@ class MiddlewareManager: def from_crawler(cls, crawler): return cls.from_settings(crawler.settings, crawler) - def _add_middleware(self, mw): + def _add_middleware(self, mw) -> None: if hasattr(mw, 'open_spider'): self.methods['open_spider'].append(mw.open_spider) if hasattr(mw, 'close_spider'): self.methods['close_spider'].appendleft(mw.close_spider) - def _process_parallel(self, methodname: str, obj, *args): + def _process_parallel(self, methodname: str, obj, *args) -> defer.Deferred: return process_parallel(self.methods[methodname], obj, *args) - def _process_chain(self, methodname: str, obj, *args): + def _process_chain(self, methodname: str, obj, *args) -> defer.Deferred: return process_chain(self.methods[methodname], obj, *args) - def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args): + def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> defer.Deferred: return process_chain_both(self.methods[cb_methodname], self.methods[eb_methodname], obj, *args) - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> defer.Deferred: return self._process_parallel('open_spider', spider) - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> defer.Deferred: return self._process_parallel('close_spider', spider) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index c382a00f7..095eae94c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -5,7 +5,7 @@ import asyncio import inspect from collections.abc import Coroutine from functools import wraps -from typing import Callable, Iterable, Any +from typing import Callable, Iterable, Any, Generator from twisted.internet import defer, task from twisted.python import failure @@ -15,7 +15,7 @@ from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import is_asyncio_reactor_installed -def defer_fail(_failure: Failure): +def defer_fail(_failure: Failure) -> defer.Deferred: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -28,7 +28,7 @@ def defer_fail(_failure: Failure): return d -def defer_succeed(result): +def defer_succeed(result) -> defer.Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -41,7 +41,7 @@ def defer_succeed(result): return d -def defer_result(result): +def defer_result(result) -> defer.Deferred: if isinstance(result, defer.Deferred): return result elif isinstance(result, failure.Failure): @@ -50,7 +50,7 @@ def defer_result(result): return defer_succeed(result) -def mustbe_deferred(f: Callable, *args, **kw): +def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -67,7 +67,7 @@ def mustbe_deferred(f: Callable, *args, **kw): return defer_result(result) -def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named): +def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> defer.DeferredList: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -78,7 +78,7 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) -def process_chain(callbacks: Iterable[Callable], input, *a, **kw): +def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: """Return a Deferred built by chaining the given callbacks""" d = defer.Deferred() for x in callbacks: @@ -87,7 +87,7 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw): return d -def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw): +def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" d = defer.Deferred() for cb, eb in zip(callbacks, errbacks): @@ -103,7 +103,7 @@ def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callabl return d -def process_parallel(callbacks: Iterable[Callable], input, *a, **kw): +def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ @@ -113,7 +113,7 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw): return d -def iter_errback(iterable: Iterable, errback: Callable, *a, **kw): +def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -142,7 +142,7 @@ def deferred_from_coro(o) -> Any: return o -def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]): +def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: """ Converts a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. @@ -154,7 +154,7 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]): return f -def maybeDeferred_coro(f: Callable, *args, **kw): +def maybeDeferred_coro(f: Callable, *args, **kw) -> defer.Deferred: """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ try: result = f(*args, **kw) From 335a25675278543d4c85123bbbed99f228b26416 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 13 Apr 2021 21:05:20 +0500 Subject: [PATCH 0290/2083] Update scrapy/core/spidermw.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/core/spidermw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 05df8c988..7e58521ac 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ from itertools import islice -from typing import Callable, Union, Any, Generator, Iterable +from typing import Any, Callable, Generator, Iterable, Union from twisted.internet.defer import Deferred from twisted.python.failure import Failure From b0e75125749ec1dce14614468bf06cd8e8842649 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 13 Apr 2021 21:05:25 +0500 Subject: [PATCH 0291/2083] Update scrapy/middleware.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 3f8c1cbf5..09768b59d 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Dict, Deque +from typing import Callable, Deque, Dict from twisted.internet import defer From a8de04c823f5e10d012aaa2dd94a4f5a9f70b119 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 13 Apr 2021 21:05:30 +0500 Subject: [PATCH 0292/2083] Update scrapy/utils/defer.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/defer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 095eae94c..e1139b1d1 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -5,7 +5,7 @@ import asyncio import inspect from collections.abc import Coroutine from functools import wraps -from typing import Callable, Iterable, Any, Generator +from typing import Any, Callable, Generator, Iterable from twisted.internet import defer, task from twisted.python import failure From cef0a8b3d653d847efe32dfc2850e5992b627408 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Apr 2021 21:07:07 +0500 Subject: [PATCH 0293/2083] Import Deferred directly in scrapy/middleware.py. --- scrapy/middleware.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 09768b59d..bbec38086 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -3,7 +3,7 @@ import pprint from collections import defaultdict, deque from typing import Callable, Deque, Dict -from twisted.internet import defer +from twisted.internet.defer import Deferred from scrapy import Spider from scrapy.exceptions import NotConfigured @@ -63,18 +63,18 @@ class MiddlewareManager: if hasattr(mw, 'close_spider'): self.methods['close_spider'].appendleft(mw.close_spider) - def _process_parallel(self, methodname: str, obj, *args) -> defer.Deferred: + def _process_parallel(self, methodname: str, obj, *args) -> Deferred: return process_parallel(self.methods[methodname], obj, *args) - def _process_chain(self, methodname: str, obj, *args) -> defer.Deferred: + def _process_chain(self, methodname: str, obj, *args) -> Deferred: return process_chain(self.methods[methodname], obj, *args) - def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> defer.Deferred: + def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: return process_chain_both(self.methods[cb_methodname], self.methods[eb_methodname], obj, *args) - def open_spider(self, spider: Spider) -> defer.Deferred: + def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) - def close_spider(self, spider: Spider) -> defer.Deferred: + def close_spider(self, spider: Spider) -> Deferred: return self._process_parallel('close_spider', spider) From 08e4eaf97369ba6daa4b5d84e00fd4d36b78e00a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Apr 2021 22:41:01 +0500 Subject: [PATCH 0294/2083] Import Deferred directly in scrapy/utils/defer.py. --- scrapy/utils/defer.py | 48 ++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index e1139b1d1..b317c12a3 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -7,7 +7,9 @@ from collections.abc import Coroutine from functools import wraps from typing import Any, Callable, Generator, Iterable -from twisted.internet import defer, task +from twisted.internet import defer +from twisted.internet.defer import Deferred, DeferredList, ensureDeferred +from twisted.internet.task import Cooperator from twisted.python import failure from twisted.python.failure import Failure @@ -15,7 +17,7 @@ from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import is_asyncio_reactor_installed -def defer_fail(_failure: Failure) -> defer.Deferred: +def defer_fail(_failure: Failure) -> Deferred: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -23,12 +25,12 @@ def defer_fail(_failure: Failure) -> defer.Deferred: before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.errback, _failure) return d -def defer_succeed(result) -> defer.Deferred: +def defer_succeed(result) -> Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -36,13 +38,13 @@ def defer_succeed(result) -> defer.Deferred: before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.callback, result) return d -def defer_result(result) -> defer.Deferred: - if isinstance(result, defer.Deferred): +def defer_result(result) -> Deferred: + if isinstance(result, Deferred): return result elif isinstance(result, failure.Failure): return defer_fail(result) @@ -50,7 +52,7 @@ def defer_result(result) -> defer.Deferred: return defer_succeed(result) -def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred: +def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -67,29 +69,29 @@ def mustbe_deferred(f: Callable, *args, **kw) -> defer.Deferred: return defer_result(result) -def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> defer.DeferredList: +def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> DeferredList: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. Taken from: https://jcalderone.livejournal.com/24285.html """ - coop = task.Cooperator() + coop = Cooperator() work = (callable(elem, *args, **named) for elem in iterable) - return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) + return DeferredList([coop.coiterate(work) for _ in range(count)]) -def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: +def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks""" - d = defer.Deferred() + d = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) return d -def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: +def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" - d = defer.Deferred() + d = Deferred() for cb, eb in zip(callbacks, errbacks): d.addCallbacks( callback=cb, errback=eb, @@ -103,12 +105,12 @@ def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callabl return d -def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> defer.Deferred: +def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = defer.DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) + d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d @@ -129,16 +131,16 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: def deferred_from_coro(o) -> Any: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" - if isinstance(o, defer.Deferred): + if isinstance(o, Deferred): return o if asyncio.isfuture(o) or inspect.isawaitable(o): if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" - return defer.ensureDeferred(o) + return ensureDeferred(o) else: # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - return defer.Deferred.fromFuture(asyncio.ensure_future(o)) + return Deferred.fromFuture(asyncio.ensure_future(o)) return o @@ -154,14 +156,14 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: return f -def maybeDeferred_coro(f: Callable, *args, **kw) -> defer.Deferred: +def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ try: result = f(*args, **kw) except: # noqa: E722 - return defer.fail(failure.Failure(captureVars=defer.Deferred.debug)) + return defer.fail(failure.Failure(captureVars=Deferred.debug)) - if isinstance(result, defer.Deferred): + if isinstance(result, Deferred): return result elif asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) From 61197d3dba53cd67c41e5d23e1f0c11a864f539b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Apr 2021 18:21:58 +0500 Subject: [PATCH 0295/2083] Add/update typing, cleanup iterator/iterable inconsistencies. --- scrapy/core/scraper.py | 7 +++--- scrapy/core/spidermw.py | 36 +++++++++++++++------------- scrapy/utils/asyncgen.py | 19 +++++++-------- scrapy/utils/defer.py | 52 ++++++++++++++++++++++++---------------- scrapy/utils/python.py | 14 +++++------ 5 files changed, 71 insertions(+), 57 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8ce57af2f..0630ce625 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,10 +1,8 @@ """This module implements the Scraper component which parses responses and extracts information from them""" -import collections import logging from collections import deque -from collections.abc import Iterable -from typing import Union +from typing import AsyncGenerator, AsyncIterable, Generator, Iterable, Union from itemadapter import is_item from twisted.internet import defer @@ -188,7 +186,8 @@ class Scraper: def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider): if not result: return defer_succeed(None) - if isinstance(result, collections.abc.AsyncIterable): + it: Union[Generator, AsyncGenerator] + if isinstance(result, AsyncIterable): it = aiter_errback(result, self.handle_spider_error, request, response, spider) dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 6fcbc7492..9dd6c462d 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,9 +3,8 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -import collections.abc from itertools import islice -from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable +from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable, AsyncGenerator from twisted.internet.defer import Deferred from twisted.python.failure import Failure @@ -61,8 +60,9 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable, - exception_processor_index: int, recover_to: MutableChain) -> Generator: + def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable], + exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain] + ) -> Union[Generator, AsyncGenerator]: @_process_iterable_universal async def _evaluate_async_iterable(iterable): try: @@ -100,11 +100,13 @@ class SpiderMiddlewareManager(MiddlewareManager): return _failure def _process_spider_output(self, response: Response, spider: Spider, - result: Iterable, start_index: int = 0) -> MutableChain: + result: Union[Iterable, AsyncIterable], start_index: int = 0 + ) -> Union[MutableChain, MutableAsyncChain]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - last_result_async = isinstance(result, collections.abc.AsyncIterator) - if last_result_async: + recovered: Union[MutableChain, MutableAsyncChain] + last_result_is_async = isinstance(result, AsyncIterable) + if last_result_is_async: recovered = MutableAsyncChain() else: recovered = MutableChain() @@ -127,30 +129,32 @@ class SpiderMiddlewareManager(MiddlewareManager): msg = (f"Middleware {method.__qualname__} must return an " f"iterable, got {type(result)}") raise _InvalidOutput(msg) - if last_result_async and isinstance(result, collections.abc.Iterator): + if last_result_is_async and isinstance(result, Iterable): raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable") - last_result_async = isinstance(result, collections.abc.AsyncIterator) + last_result_is_async = isinstance(result, AsyncIterable) - if last_result_async: + if last_result_is_async: return MutableAsyncChain(result, recovered) else: - return MutableChain(result, recovered) + return MutableChain(result, recovered) # type: ignore[arg-type] - def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain: - if isinstance(result, collections.abc.AsyncIterator): + def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] + ) -> Union[MutableChain, MutableAsyncChain]: + recovered: Union[MutableChain, MutableAsyncChain] + if isinstance(result, AsyncIterable): recovered = MutableAsyncChain() else: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) result = self._process_spider_output(response, spider, result) - if isinstance(result, collections.abc.AsyncIterator): + if isinstance(result, AsyncIterable): return MutableAsyncChain(result, recovered) else: - return MutableChain(result, recovered) + return MutableChain(result, recovered) # type: ignore[arg-type] def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Deferred: - def process_callback_output(result: Iterable) -> MutableChain: + def process_callback_output(result: Union[Iterable, AsyncIterable]) -> Union[MutableChain, MutableAsyncChain]: return self._process_callback_output(response, spider, result) def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 92118ddb9..ae9a79989 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,7 +1,6 @@ -import collections import functools import inspect -from collections.abc import AsyncIterable +from typing import AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union async def collect_asyncgen(result: AsyncIterable): @@ -11,9 +10,9 @@ async def collect_asyncgen(result: AsyncIterable): return results -async def as_async_generator(it): - """ Wraps an iterator (sync or async) into an async generator. """ - if isinstance(it, collections.abc.AsyncIterator): +async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator: + """ Wraps an iterable (sync or async) into an async generator. """ + if isinstance(it, AsyncIterable): async for r in it: yield r else: @@ -22,7 +21,7 @@ async def as_async_generator(it): # https://stackoverflow.com/a/66170760/113586 -def _process_iterable_universal(process_async): +def _process_iterable_universal(process_async: Callable): """ Takes a function that takes an async iterable, args and kwargs. Returns a function that takes any iterable, args and kwargs. @@ -33,7 +32,7 @@ def _process_iterable_universal(process_async): # If this stops working, all internal uses can be just replaced with manually-written # process_sync functions. - def process_sync(iterable, *args, **kwargs): + def process_sync(iterable: Iterable, *args, **kwargs) -> Generator: agen = process_async(as_async_generator(iterable), *args, **kwargs) if not inspect.isasyncgen(agen): raise ValueError(f"process_async returned wrong type {type(agen)}") @@ -52,11 +51,11 @@ def _process_iterable_universal(process_async): f"you can't use {_process_iterable_universal.__name__} with it.") @functools.wraps(process_async) - def process(iterable, *args, **kwargs): - if inspect.isasyncgen(iterable): + def process(iterable: Union[Iterable, AsyncIterable], *args, **kwargs) -> Union[Generator, AsyncGenerator]: + if isinstance(iterable, AsyncIterable): # call process_async directly return process_async(iterable, *args, **kwargs) - if hasattr(iterable, '__iter__'): + if isinstance(iterable, Iterable): # convert process_async to process_sync return process_sync(iterable, *args, **kwargs) raise TypeError(f"Wrong iterable type {type(iterable)}") diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index f81faf6dd..39c8a85e9 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -3,9 +3,21 @@ Helper functions for dealing with Twisted deferreds """ import asyncio import inspect -from collections.abc import Coroutine +from asyncio import Future from functools import wraps -from typing import Any, Callable, Generator, Iterable +from typing import ( + Any, + AsyncGenerator, + AsyncIterable, + Callable, + Coroutine, + Generator, + Iterable, + Iterator, + List, + Optional, + Union +) from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred @@ -80,8 +92,8 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter: - """ A class that wraps an async iterator into a normal iterator suitable +class _AsyncCooperatorAdapter(Iterator): + """ A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more generic interface. @@ -125,30 +137,30 @@ class _AsyncCooperatorAdapter: Cooperator/CooperativeTask and use it instead of this adapter to achieve the same goal. """ - def __init__(self, aiterator, callable, *callable_args, **callable_kwargs): - self.aiterator = aiterator + def __init__(self, aiterable: AsyncIterable, callable: Callable, *callable_args, **callable_kwargs): + self.aiterator = aiterable.__aiter__() self.callable = callable self.callable_args = callable_args self.callable_kwargs = callable_kwargs self.finished = False - self.waiting_deferreds = [] - self.anext_deferred = None + self.waiting_deferreds: List[Deferred] = [] + self.anext_deferred: Optional[Deferred] = None - def _callback(self, result): + def _callback(self, result: Any) -> None: # This gets called when the result from aiterator.__anext__() is available. # It calls the callable on it and sends the result to the oldest waiting Deferred # (by chaining if the result is a Deferred too or by firing if not). self.anext_deferred = None result = self.callable(result, *self.callable_args, **self.callable_kwargs) d = self.waiting_deferreds.pop(0) - if isinstance(result, defer.Deferred): + if isinstance(result, Deferred): result.chainDeferred(d) else: d.callback(None) if self.waiting_deferreds: self._call_anext() - def _errback(self, failure): + def _errback(self, failure: Failure) -> None: # This gets called on any exceptions in aiterator.__anext__(). # It handles StopAsyncIteration by stopping the iteration and reraises all others. self.anext_deferred = None @@ -157,29 +169,29 @@ class _AsyncCooperatorAdapter: for d in self.waiting_deferreds: d.callback(None) - def _call_anext(self): + def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) - def __next__(self): + def __next__(self) -> Deferred: # This puts a new Deferred into self.waiting_deferreds and returns it. # It also calls __anext__() if needed. if self.finished: raise StopIteration - d = defer.Deferred() + d = Deferred() self.waiting_deferreds.append(d) if not self.anext_deferred: self._call_anext() return d -def parallel_async(async_iterable, count, callable, *args, **named): - """ Like parallel but for async iterables """ +def parallel_async(async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named) -> DeferredList: + """ Like parallel but for async iterators """ coop = Cooperator() work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) - dl = defer.DeferredList([coop.coiterate(work) for _ in range(count)]) + dl = DeferredList([coop.coiterate(work) for _ in range(count)]) return dl @@ -232,7 +244,7 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: errback(failure.Failure(), *a, **kw) -async def aiter_errback(aiterable, errback, *a, **kw): +async def aiter_errback(aiterable: AsyncIterable, errback: Callable, *a, **kw) -> AsyncGenerator: """Wraps an async iterable calling an errback if an error is caught while iterating it. Similar to scrapy.utils.defer.iter_errback() """ @@ -290,13 +302,13 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: return defer.succeed(result) -def deferred_to_future(d): +def deferred_to_future(d: Deferred) -> Future: """ Wraps a Deferred into a Future. Requires the asyncio reactor. """ return d.asFuture(asyncio.get_event_loop()) -def maybe_deferred_to_future(d): +def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: """ Converts a Deferred to something that can be awaited in a callback or other user coroutine. If the asyncio reactor is installed, coroutines are wrapped into Futures, and only Futures can be diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 8b823d174..d086347bc 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,9 +8,9 @@ import re import sys import warnings import weakref -from collections.abc import Iterable from functools import partial, wraps from itertools import chain +from typing import AsyncIterable, Iterable, Union from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator @@ -345,7 +345,7 @@ class MutableChain(Iterable): def __init__(self, *args: Iterable): self.data = chain.from_iterable(args) - def extend(self, *iterables: Iterable): + def extend(self, *iterables: Iterable) -> None: self.data = chain(self.data, chain.from_iterable(iterables)) def __iter__(self): @@ -359,22 +359,22 @@ class MutableChain(Iterable): return self.__next__() -async def _async_chain(*iterables): +async def _async_chain(*iterables: Union[Iterable, AsyncIterable]): for it in iterables: async for o in as_async_generator(it): yield o -class MutableAsyncChain: +class MutableAsyncChain(AsyncIterable): """ Similar to MutableChain but for async iterables """ - def __init__(self, *args): + def __init__(self, *args: Union[Iterable, AsyncIterable]): self.data = _async_chain(*args) - def extend(self, *aiterables): - self.data = _async_chain(self.data, _async_chain(*aiterables)) + def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None: + self.data = _async_chain(self.data, _async_chain(*iterables)) def __aiter__(self): return self From 9db01a483c729369b272235bfb6e1c66ff62d1f7 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 14 Apr 2021 19:02:38 +0500 Subject: [PATCH 0296/2083] Update scrapy/core/spidermw.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/core/spidermw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 9dd6c462d..d1fedae07 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ from itertools import islice -from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable, AsyncGenerator +from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union from twisted.internet.defer import Deferred from twisted.python.failure import Failure From 77bff0db0a6bdfca15295444f0e0eda47b35e702 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Apr 2021 19:10:13 +0500 Subject: [PATCH 0297/2083] Additional typing for scraper and a small code change. --- scrapy/core/scraper.py | 100 ++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index c760a4155..adbf8ef3d 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -3,11 +3,10 @@ extracts information from them""" import logging from collections import deque -from collections.abc import Iterable -from typing import Union +from typing import Union, Optional, Tuple, Set, Deque, Any, Iterable from itemadapter import is_item -from twisted.internet import defer +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import signals, Spider @@ -20,6 +19,9 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output +QUEUE_TUPLE = Tuple[Union[Response, Failure], Request, Deferred] + + logger = logging.getLogger(__name__) @@ -28,46 +30,46 @@ class Slot: MIN_RESPONSE_SIZE = 1024 - def __init__(self, max_active_size=5000000): + def __init__(self, max_active_size: int = 5000000): self.max_active_size = max_active_size - self.queue = deque() - self.active = set() - self.active_size = 0 - self.itemproc_size = 0 - self.closing = None + self.queue: Deque[QUEUE_TUPLE] = deque() + self.active: Set[Request] = set() + self.active_size: int = 0 + self.itemproc_size: int = 0 + self.closing: Optional[Deferred] = None - def add_response_request(self, response, request): - deferred = defer.Deferred() - self.queue.append((response, request, deferred)) - if isinstance(response, Response): - self.active_size += max(len(response.body), self.MIN_RESPONSE_SIZE) + def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred: + deferred = Deferred() + self.queue.append((result, request, deferred)) + if isinstance(result, Response): + self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) else: self.active_size += self.MIN_RESPONSE_SIZE return deferred - def next_response_request_deferred(self): + def next_response_request_deferred(self) -> QUEUE_TUPLE: response, request, deferred = self.queue.popleft() self.active.add(request) return response, request, deferred - def finish_response(self, response, request): + def finish_response(self, result: Union[Response, Failure], request: Request) -> None: self.active.remove(request) - if isinstance(response, Response): - self.active_size -= max(len(response.body), self.MIN_RESPONSE_SIZE) + if isinstance(result, Response): + self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE) else: self.active_size -= self.MIN_RESPONSE_SIZE - def is_idle(self): + def is_idle(self) -> bool: return not (self.queue or self.active) - def needs_backout(self): + def needs_backout(self) -> bool: return self.active_size > self.max_active_size class Scraper: def __init__(self, crawler): - self.slot = None + self.slot: Optional[Slot] = None self.spidermw = SpiderMiddlewareManager.from_crawler(crawler) itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR']) self.itemproc = itemproc_cls.from_crawler(crawler) @@ -76,32 +78,37 @@ class Scraper: self.signals = crawler.signals self.logformatter = crawler.logformatter - @defer.inlineCallbacks - def open_spider(self, spider): + @inlineCallbacks + def open_spider(self, spider: Spider): """Open the given spider for scraping and allocate resources for it""" self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE')) yield self.itemproc.open_spider(spider) - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> Deferred: """Close a spider being scraped and release its resources""" - self.slot.closing = defer.Deferred() + if self.slot is None: + raise RuntimeError("Scraper slot not assigned") + self.slot.closing = Deferred() self.slot.closing.addCallback(self.itemproc.close_spider) self._check_if_closing(spider) return self.slot.closing - def is_idle(self): + def is_idle(self) -> bool: """Return True if there isn't any more spiders to process""" return not self.slot - def _check_if_closing(self, spider): + def _check_if_closing(self, spider: Spider) -> None: + assert self.slot is not None # typing if self.slot.closing and self.slot.is_idle(): self.slot.closing.callback(spider) - def enqueue_scrape(self, response, request, spider): - dfd = self.slot.add_response_request(response, request) + def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: + if self.slot is None: + raise RuntimeError("Scraper slot not assigned") + dfd = self.slot.add_response_request(result, request) def finish_scraping(_): - self.slot.finish_response(response, request) + self.slot.finish_response(result, request) self._check_if_closing(spider) self._scrape_next(spider) return _ @@ -115,12 +122,13 @@ class Scraper: self._scrape_next(spider) return dfd - def _scrape_next(self, spider): + def _scrape_next(self, spider: Spider) -> None: + assert self.slot is not None # typing while self.slot.queue: response, request, deferred = self.slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider): + def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: """ Handle the downloaded response or failure through the spider callback/errback """ @@ -131,7 +139,7 @@ class Scraper: dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider): + def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: """ Handle the different cases of request's result been a Response or a Failure """ @@ -141,7 +149,7 @@ class Scraper: dfd = self.call_spider(result, request, spider) return dfd.addErrback(self._log_download_errors, result, request, spider) - def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider): + def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request @@ -156,7 +164,7 @@ class Scraper: dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) - def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider): + def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None: exc = _failure.value if isinstance(exc, CloseSpider): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') @@ -177,7 +185,7 @@ class Scraper: spider=spider ) - def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider): + def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred: if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, response, spider) @@ -185,10 +193,12 @@ class Scraper: request, response, spider) return dfd - def _process_spidermw_output(self, output, request, response, spider): + def _process_spidermw_output(self, output: Any, request: Request, response: Response, + spider: Spider) -> Optional[Deferred]: """Process each Request/Item (given in the output parameter) returned from the given spider """ + assert self.slot is not None # typing if isinstance(output, Request): self.crawler.engine.crawl(request=output, spider=spider) elif is_item(output): @@ -205,12 +215,18 @@ class Scraper: {'request': request, 'typename': typename}, extra={'spider': spider}, ) + return None - def _log_download_errors(self, spider_failure, download_failure, request, spider): + def _log_download_errors(self, spider_failure: Failure, download_failure: Failure, request: Request, + spider: Spider) -> Union[Failure, None]: """Log and silence errors that come from the engine (typically download - errors that got propagated thru here) + errors that got propagated thru here). + + spider_failure: the value passed into the errback of self.call_spider() + download_failure: the value passed into _scrape2() from + ExecutionEngine._handle_downloader_output() as "result" """ - if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest): + if not download_failure.check(IgnoreRequest): if download_failure.frames: logkws = self.logformatter.download_error(download_failure, request, spider) logger.log( @@ -230,10 +246,12 @@ class Scraper: if spider_failure is not download_failure: return spider_failure + return None - def _itemproc_finished(self, output, item, response, spider): + def _itemproc_finished(self, output: Any, item: Any, response: Response, spider: Spider) -> None: """ItemProcessor finished for the given ``item`` and returned ``output`` """ + assert self.slot is not None # typing self.slot.itemproc_size -= 1 if isinstance(output, Failure): ex = output.value From 309a637f32b0f6196eb4f77f19152527dcf3e5e7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Apr 2021 20:26:37 +0500 Subject: [PATCH 0298/2083] Small changes. --- scrapy/core/scraper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index adbf8ef3d..96aa53686 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -3,7 +3,7 @@ extracts information from them""" import logging from collections import deque -from typing import Union, Optional, Tuple, Set, Deque, Any, Iterable +from typing import Any, Deque, Iterable, Optional, Set, Tuple, Union from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks @@ -19,7 +19,7 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output -QUEUE_TUPLE = Tuple[Union[Response, Failure], Request, Deferred] +QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] logger = logging.getLogger(__name__) @@ -32,7 +32,7 @@ class Slot: def __init__(self, max_active_size: int = 5000000): self.max_active_size = max_active_size - self.queue: Deque[QUEUE_TUPLE] = deque() + self.queue: Deque[QueueTuple] = deque() self.active: Set[Request] = set() self.active_size: int = 0 self.itemproc_size: int = 0 @@ -47,7 +47,7 @@ class Slot: self.active_size += self.MIN_RESPONSE_SIZE return deferred - def next_response_request_deferred(self) -> QUEUE_TUPLE: + def next_response_request_deferred(self) -> QueueTuple: response, request, deferred = self.queue.popleft() self.active.add(request) return response, request, deferred From 7e23677b52b659b11471a63f3be9905a0bbaf995 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 20 Apr 2021 08:45:28 -0300 Subject: [PATCH 0299/2083] Engine: deprecations and type hints (#5090) --- docs/topics/telnetconsole.rst | 3 +- scrapy/core/engine.py | 419 ++++++++++--------- scrapy/core/scraper.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 2 +- scrapy/extensions/memusage.py | 6 +- scrapy/pipelines/media.py | 2 +- scrapy/shell.py | 2 +- scrapy/utils/engine.py | 3 +- tests/test_downloadermiddleware_robotstxt.py | 12 +- tests/test_engine.py | 110 ++++- 10 files changed, 336 insertions(+), 225 deletions(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 9802a34a2..832829b75 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -110,11 +110,10 @@ using the telnet console:: Execution engine status time()-engine.start_time : 8.62972998619 - engine.has_capacity() : False len(engine.downloader.active) : 16 engine.scraper.is_idle() : False engine.spider.name : followall - engine.spider_is_idle(engine.spider) : False + engine.spider_is_idle() : False engine.slot.closing : False len(engine.slot.inprogress) : 16 len(engine.slot.scheduler.dqs or []) : 0 diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 93bcdb49a..edfac87c6 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -1,51 +1,61 @@ """ -This is the Scrapy engine which controls the Scheduler, Downloader and Spiders. +This is the Scrapy engine which controls the Scheduler, Downloader and Spider. For more information see docs/topics/architecture.rst """ import logging +import warnings from time import time +from typing import Callable, Iterable, Iterator, Optional, Set, Union -from twisted.internet import defer, task +from twisted.internet.defer import Deferred, inlineCallbacks, succeed +from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper -from scrapy.exceptions import DontCloseSpider +from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning from scrapy.http import Response, Request -from scrapy.utils.misc import load_object -from scrapy.utils.reactor import CallLaterOnce +from scrapy.spiders import Spider from scrapy.utils.log import logformatter_adapter, failure_to_exc_info +from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.reactor import CallLaterOnce + logger = logging.getLogger(__name__) class Slot: - - def __init__(self, start_requests, close_if_idle, nextcall, scheduler): - self.closing = False - self.inprogress = set() # requests in progress - self.start_requests = iter(start_requests) + def __init__( + self, + start_requests: Iterable, + close_if_idle: bool, + nextcall: CallLaterOnce, + scheduler, + ) -> None: + self.closing: Optional[Deferred] = None + self.inprogress: Set[Request] = set() + self.start_requests: Optional[Iterator] = iter(start_requests) self.close_if_idle = close_if_idle self.nextcall = nextcall self.scheduler = scheduler - self.heartbeat = task.LoopingCall(nextcall.schedule) + self.heartbeat = LoopingCall(nextcall.schedule) - def add_request(self, request): + def add_request(self, request: Request) -> None: self.inprogress.add(request) - def remove_request(self, request): + def remove_request(self, request: Request) -> None: self.inprogress.remove(request) self._maybe_fire_closing() - def close(self): - self.closing = defer.Deferred() + def close(self) -> Deferred: + self.closing = Deferred() self._maybe_fire_closing() return self.closing - def _maybe_fire_closing(self): - if self.closing and not self.inprogress: + def _maybe_fire_closing(self) -> None: + if self.closing is not None and not self.inprogress: if self.nextcall: self.nextcall.cancel() if self.heartbeat.running: @@ -54,210 +64,224 @@ class Slot: class ExecutionEngine: - - def __init__(self, crawler, spider_closed_callback): + def __init__(self, crawler, spider_closed_callback: Callable) -> None: self.crawler = crawler self.settings = crawler.settings self.signals = crawler.signals self.logformatter = crawler.logformatter - self.slot = None - self.spider = None + self.slot: Optional[Slot] = None + self.spider: Optional[Spider] = None self.running = False self.paused = False - self.scheduler_cls = load_object(self.settings['SCHEDULER']) + self.scheduler_cls = load_object(crawler.settings["SCHEDULER"]) downloader_cls = load_object(self.settings['DOWNLOADER']) self.downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback = spider_closed_callback - @defer.inlineCallbacks - def start(self): - """Start the execution engine""" + @inlineCallbacks + def start(self) -> Deferred: if self.running: raise RuntimeError("Engine already running") self.start_time = time() yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True - self._closewait = defer.Deferred() + self._closewait = Deferred() yield self._closewait - def stop(self): - """Stop the execution engine gracefully""" + def stop(self) -> Deferred: + """Gracefully stop the execution engine""" + @inlineCallbacks + def _finish_stopping_engine(_) -> Deferred: + yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + self._closewait.callback(None) + if not self.running: raise RuntimeError("Engine not running") + self.running = False - dfd = self._close_all_spiders() - return dfd.addBoth(lambda _: self._finish_stopping_engine()) + dfd = self.close_spider(self.spider, reason="shutdown") if self.spider is not None else succeed(None) + return dfd.addBoth(_finish_stopping_engine) - def close(self): - """Close the execution engine gracefully. - - If it has already been started, stop it. In all cases, close all spiders - and the downloader. + def close(self) -> Deferred: + """ + Gracefully close the execution engine. + If it has already been started, stop it. In all cases, close the spider and the downloader. """ if self.running: - # Will also close spiders and downloader - return self.stop() - elif self.open_spiders: - # Will also close downloader - return self._close_all_spiders() - else: - return defer.succeed(self.downloader.close()) + return self.stop() # will also close spider and downloader + if self.spider is not None: + return self.close_spider(self.spider, reason="shutdown") # will also close downloader + return succeed(self.downloader.close()) - def pause(self): - """Pause the execution engine""" + def pause(self) -> None: self.paused = True - def unpause(self): - """Resume the execution engine""" + def unpause(self) -> None: self.paused = False - def _next_request(self, spider): - slot = self.slot - if not slot: - return + def _next_request(self) -> None: + assert self.slot is not None # typing + assert self.spider is not None # typing if self.paused: - return + return None - while not self._needs_backout(spider): - if not self._next_request_from_scheduler(spider): - break + while not self._needs_backout() and self._next_request_from_scheduler() is not None: + pass - if slot.start_requests and not self._needs_backout(spider): + if self.slot.start_requests is not None and not self._needs_backout(): try: - request = next(slot.start_requests) + request = next(self.slot.start_requests) except StopIteration: - slot.start_requests = None + self.slot.start_requests = None except Exception: - slot.start_requests = None - logger.error('Error while obtaining start requests', - exc_info=True, extra={'spider': spider}) + self.slot.start_requests = None + logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': self.spider}) else: - self.crawl(request, spider) + self.crawl(request) - if self.spider_is_idle(spider) and slot.close_if_idle: - self._spider_idle(spider) + if self.spider_is_idle() and self.slot.close_if_idle: + self._spider_idle() - def _needs_backout(self, spider): - slot = self.slot + def _needs_backout(self) -> bool: return ( not self.running - or slot.closing + or self.slot.closing # type: ignore[union-attr] or self.downloader.needs_backout() - or self.scraper.slot.needs_backout() + or self.scraper.slot.needs_backout() # type: ignore[union-attr] ) - def _next_request_from_scheduler(self, spider): - slot = self.slot - request = slot.scheduler.next_request() - if not request: - return - d = self._download(request, spider) - d.addBoth(self._handle_downloader_output, request, spider) + def _next_request_from_scheduler(self) -> Optional[Deferred]: + assert self.slot is not None # typing + assert self.spider is not None # typing + + request = self.slot.scheduler.next_request() + if request is None: + return None + + d = self._download(request, self.spider) + d.addBoth(self._handle_downloader_output, request, self.spider) d.addErrback(lambda f: logger.info('Error while handling downloader output', exc_info=failure_to_exc_info(f), - extra={'spider': spider})) - d.addBoth(lambda _: slot.remove_request(request)) + extra={'spider': self.spider})) + d.addBoth(lambda _: self.slot.remove_request(request)) d.addErrback(lambda f: logger.info('Error while removing request from slot', exc_info=failure_to_exc_info(f), - extra={'spider': spider})) - d.addBoth(lambda _: slot.nextcall.schedule()) + extra={'spider': self.spider})) + d.addBoth(lambda _: self.slot.nextcall.schedule()) d.addErrback(lambda f: logger.info('Error while scheduling new request', exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + extra={'spider': self.spider})) return d - def _handle_downloader_output(self, response, request, spider): - if not isinstance(response, (Request, Response, Failure)): - raise TypeError( - "Incorrect type: expected Request, Response or Failure, got " - f"{type(response)}: {response!r}" - ) + def _handle_downloader_output( + self, result: Union[Request, Response, Failure], request: Request, spider: Spider + ) -> Optional[Deferred]: + if not isinstance(result, (Request, Response, Failure)): + raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}") + # downloader middleware can return requests (for example, redirects) - if isinstance(response, Request): - self.crawl(response, spider) - return - # response is a Response or Failure - d = self.scraper.enqueue_scrape(response, request, spider) - d.addErrback(lambda f: logger.error('Error while enqueuing downloader output', - exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + if isinstance(result, Request): + self.crawl(result) + return None + + d = self.scraper.enqueue_scrape(result, request, spider) + d.addErrback( + lambda f: logger.error( + "Error while enqueuing downloader output", + exc_info=failure_to_exc_info(f), + extra={'spider': spider}, + ) + ) return d - def spider_is_idle(self, spider): - if not self.scraper.slot.is_idle(): - # scraper is not idle + def spider_is_idle(self, spider: Optional[Spider] = None) -> bool: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: + raise RuntimeError("Engine slot not assigned") + if not self.scraper.slot.is_idle(): # type: ignore[union-attr] return False - - if self.downloader.active: - # downloader has pending requests + if self.downloader.active: # downloader has pending requests return False - - if self.slot.start_requests is not None: - # not all start requests are handled + if self.slot.start_requests is not None: # not all start requests are handled return False - if self.slot.scheduler.has_pending_requests(): - # scheduler has pending requests return False - return True - @property - def open_spiders(self): - return [self.spider] if self.spider else [] + def crawl(self, request: Request, spider: Optional[Spider] = None) -> None: + """Inject the request into the spider <-> downloader pipeline""" + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if spider is not self.spider: + raise RuntimeError(f"The spider {spider.name!r} does not match the open spider") + if self.spider is None: + raise RuntimeError(f"No open spider to crawl: {request}") + self._schedule_request(request, self.spider) + self.slot.nextcall.schedule() # type: ignore[union-attr] - def has_capacity(self): - """Does the engine have capacity to handle more spiders""" - return not bool(self.slot) - - def crawl(self, request, spider): - if spider not in self.open_spiders: - raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}") - self.schedule(request, spider) - self.slot.nextcall.schedule() - - def schedule(self, request, spider): + def _schedule_request(self, request: Request, spider: Spider) -> None: self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider) - if not self.slot.scheduler.enqueue_request(request): + if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider) - def download(self, request, spider): - d = self._download(request, spider) - d.addBoth(self._downloaded, self.slot, request, spider) - return d + def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred: + """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" + if spider is None: + spider = self.spider + else: + warnings.warn( + "Passing a 'spider' argument to ExecutionEngine.download is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if spider is not self.spider: + logger.warning("The spider '%s' does not match the open spider", spider.name) + if spider is None: + raise RuntimeError(f"No open spider to crawl: {request}") + return self._download(request, spider).addBoth(self._downloaded, request, spider) - def _downloaded(self, response, slot, request, spider): - slot.remove_request(request) - return self.download(response, spider) if isinstance(response, Request) else response + def _downloaded( + self, result: Union[Response, Request], request: Request, spider: Spider + ) -> Union[Deferred, Response]: + assert self.slot is not None # typing + self.slot.remove_request(request) + return self.download(result, spider) if isinstance(result, Request) else result - def _download(self, request, spider): - slot = self.slot - slot.add_request(request) + def _download(self, request: Request, spider: Spider) -> Deferred: + assert self.slot is not None # typing - def _on_success(response): - if not isinstance(response, (Response, Request)): - raise TypeError( - "Incorrect type: expected Response or Request, got " - f"{type(response)}: {response!r}" - ) - if isinstance(response, Response): - if response.request is None: - response.request = request - logkws = self.logformatter.crawled(response.request, response, spider) + self.slot.add_request(request) + + def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: + if not isinstance(result, (Response, Request)): + raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}") + if isinstance(result, Response): + if result.request is None: + result.request = request + logkws = self.logformatter.crawled(result.request, result, spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) self.signals.send_catch_log( signal=signals.response_received, - response=response, - request=response.request, + response=result, + request=result.request, spider=spider, ) - return response + return result def _on_complete(_): - slot.nextcall.schedule() + self.slot.nextcall.schedule() return _ dwld = self.downloader.fetch(request, spider) @@ -265,58 +289,52 @@ class ExecutionEngine: dwld.addBoth(_on_complete) return dwld - @defer.inlineCallbacks - def open_spider(self, spider, start_requests=(), close_if_idle=True): - if not self.has_capacity(): + @inlineCallbacks + def open_spider(self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True): + if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={'spider': spider}) - nextcall = CallLaterOnce(self._next_request, spider) - scheduler = self.scheduler_cls.from_crawler(self.crawler) + nextcall = CallLaterOnce(self._next_request) + scheduler = create_instance(self.scheduler_cls, settings=None, crawler=self.crawler) start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider) - slot = Slot(start_requests, close_if_idle, nextcall, scheduler) - self.slot = slot + self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider yield scheduler.open(spider) yield self.scraper.open_spider(spider) self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) - slot.nextcall.schedule() - slot.heartbeat.start(5) + self.slot.nextcall.schedule() + self.slot.heartbeat.start(5) - def _spider_idle(self, spider): - """Called when a spider gets idle. This function is called when there - are no remaining pages to download or schedule. It can be called - multiple times. If some extension raises a DontCloseSpider exception - (in the spider_idle signal handler) the spider is not closed until the - next loop and this function is guaranteed to be called (at least) once - again for this spider. + def _spider_idle(self) -> None: """ - res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider) + Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. + It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider + exception, the spider is not closed until the next loop and this function is guaranteed to be called + (at least) once again. + """ + assert self.spider is not None # typing + res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider) if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res): - return + return None + if self.spider_is_idle(): + self.close_spider(self.spider, reason='finished') - if self.spider_is_idle(spider): - self.close_spider(spider, reason='finished') - - def close_spider(self, spider, reason='cancelled'): + def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: """Close (cancel) spider and clear all its outstanding requests""" + if self.slot is None: + raise RuntimeError("Engine slot not assigned") - slot = self.slot - if slot.closing: - return slot.closing - logger.info("Closing spider (%(reason)s)", - {'reason': reason}, - extra={'spider': spider}) + if self.slot.closing is not None: + return self.slot.closing - dfd = slot.close() + logger.info("Closing spider (%(reason)s)", {'reason': reason}, extra={'spider': spider}) - def log_failure(msg): - def errback(failure): - logger.error( - msg, - exc_info=failure_to_exc_info(failure), - extra={'spider': spider} - ) + dfd = self.slot.close() + + def log_failure(msg: str) -> Callable: + def errback(failure: Failure) -> None: + logger.error(msg, exc_info=failure_to_exc_info(failure), extra={'spider': spider}) return errback dfd.addBoth(lambda _: self.downloader.close()) @@ -325,19 +343,18 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.scraper.close_spider(spider)) dfd.addErrback(log_failure('Scraper close failure')) - dfd.addBoth(lambda _: slot.scheduler.close(reason)) + dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) dfd.addErrback(log_failure('Scheduler close failure')) dfd.addBoth(lambda _: self.signals.send_catch_log_deferred( - signal=signals.spider_closed, spider=spider, reason=reason)) + signal=signals.spider_closed, spider=spider, reason=reason, + )) dfd.addErrback(log_failure('Error while sending spider_close signal')) dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason)) dfd.addErrback(log_failure('Stats close failure')) - dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", - {'reason': reason}, - extra={'spider': spider})) + dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", {'reason': reason}, extra={'spider': spider})) dfd.addBoth(lambda _: setattr(self, 'slot', None)) dfd.addErrback(log_failure('Error while unassigning slot')) @@ -349,12 +366,26 @@ class ExecutionEngine: return dfd - def _close_all_spiders(self): - dfds = [self.close_spider(s, reason='shutdown') for s in self.open_spiders] - dlist = defer.DeferredList(dfds) - return dlist + @property + def open_spiders(self) -> list: + warnings.warn( + "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return [self.spider] if self.spider is not None else [] - @defer.inlineCallbacks - def _finish_stopping_engine(self): - yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) - self._closewait.callback(None) + def has_capacity(self) -> bool: + warnings.warn("ExecutionEngine.has_capacity is deprecated", ScrapyDeprecationWarning, stacklevel=2) + return not bool(self.slot) + + def schedule(self, request: Request, spider: Spider) -> None: + warnings.warn( + "ExecutionEngine.schedule is deprecated, please use " + "ExecutionEngine.crawl or ExecutionEngine.download instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: + raise RuntimeError("Engine slot not assigned") + self._schedule_request(request, spider) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 96aa53686..d6d6f64f9 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -200,7 +200,7 @@ class Scraper: """ assert self.slot is not None # typing if isinstance(output, Request): - self.crawler.engine.crawl(request=output, spider=spider) + self.crawler.engine.crawl(request=output) elif is_item(output): self.slot.itemproc_size += 1 dfd = self.itemproc.process_item(output, spider) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index d6da55535..e66bf177e 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -67,7 +67,7 @@ class RobotsTxtMiddleware: priority=self.DOWNLOAD_PRIORITY, meta={'dont_obey_robotstxt': True} ) - dfd = self.crawler.engine.download(robotsreq, spider) + dfd = self.crawler.engine.download(robotsreq) dfd.addCallback(self._parse_robots, netloc, spider) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 274cbdbfe..9de119a10 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -88,10 +88,8 @@ class MemoryUsage: self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) - open_spiders = self.crawler.engine.open_spiders - if open_spiders: - for spider in open_spiders: - self.crawler.engine.close_spider(spider, 'memusage_exceeded') + if self.crawler.engine.spider is not None: + self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded') else: self.crawler.stop() diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0c2ee6856..d1bccf323 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -173,7 +173,7 @@ class MediaPipeline: errback=self.media_failed, errbackArgs=(request, info)) else: self._modify_media_request(request) - dfd = self.crawler.engine.download(request, info.spider) + dfd = self.crawler.engine.download(request) dfd.addCallbacks( callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) diff --git a/scrapy/shell.py b/scrapy/shell.py index c370ccaff..f2dff2ae3 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -79,7 +79,7 @@ class Shell: spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) - self.crawler.engine.crawl(request, spider) + self.crawler.engine.crawl(request) return d def _open_spider(self, request, spider): diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 0c1cee1a0..8e3ec2c37 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -8,11 +8,10 @@ def get_engine_status(engine): """Return a report of the current engine status""" tests = [ "time()-engine.start_time", - "engine.has_capacity()", "len(engine.downloader.active)", "engine.scraper.is_idle()", "engine.spider.name", - "engine.spider_is_idle(engine.spider)", + "engine.spider_is_idle()", "engine.slot.closing", "len(engine.slot.inprogress)", "len(engine.slot.scheduler.dqs or [])", diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 858138f81..1460d88eb 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -42,7 +42,7 @@ Disallow: /some/randome/page.html """.encode('utf-8') response = TextResponse('http://site.local/robots.txt', body=ROBOTS) - def return_response(request, spider): + def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -79,7 +79,7 @@ Disallow: /some/randome/page.html crawler.settings.set('ROBOTSTXT_OBEY', True) response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2') - def return_response(request, spider): + def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -102,7 +102,7 @@ Disallow: /some/randome/page.html crawler.settings.set('ROBOTSTXT_OBEY', True) response = Response('http://site.local/robots.txt') - def return_response(request, spider): + def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -122,7 +122,7 @@ Disallow: /some/randome/page.html self.crawler.settings.set('ROBOTSTXT_OBEY', True) err = error.DNSLookupError('Robotstxt address not found') - def return_failure(request, spider): + def return_failure(request): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(err)) return deferred @@ -138,7 +138,7 @@ Disallow: /some/randome/page.html self.crawler.settings.set('ROBOTSTXT_OBEY', True) err = error.DNSLookupError('Robotstxt address not found') - def immediate_failure(request, spider): + def immediate_failure(request): deferred = Deferred() deferred.errback(failure.Failure(err)) return deferred @@ -150,7 +150,7 @@ Disallow: /some/randome/page.html def test_ignore_robotstxt_request(self): self.crawler.settings.set('ROBOTSTXT_OBEY', True) - def ignore_request(request, spider): + def ignore_request(request): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest())) return deferred diff --git a/tests/test_engine.py b/tests/test_engine.py index b2d1d83c7..c200ded90 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -13,6 +13,7 @@ module with the ``runserver`` argument:: import os import re import sys +import warnings from collections import defaultdict from urllib.parse import urlparse @@ -25,6 +26,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor @@ -382,22 +384,104 @@ class EngineTest(unittest.TestCase): yield e.close() @defer.inlineCallbacks - def test_close_spiders_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - self.assertEqual(len(e.open_spiders), 1) - yield e.close() - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_close_engine_spiders_downloader(self): + def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() - self.assertTrue(e.running) - yield e.close() - self.assertFalse(e.running) - self.assertEqual(len(e.open_spiders), 0) + yield self.assertFailure(e.start(), RuntimeError).addBoth( + lambda exc: self.assertEqual(str(exc), "Engine already running") + ) + yield e.stop() + + @defer.inlineCallbacks + def test_close_spiders_downloader(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + yield e.open_spider(TestSpider(), []) + self.assertEqual(len(e.open_spiders), 1) + yield e.close() + self.assertEqual(len(e.open_spiders), 0) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", + ) + + @defer.inlineCallbacks + def test_close_engine_spiders_downloader(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + yield e.open_spider(TestSpider(), []) + e.start() + self.assertTrue(e.running) + yield e.close() + self.assertFalse(e.running) + self.assertEqual(len(e.open_spiders), 0) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", + ) + + @defer.inlineCallbacks + def test_crawl_deprecated_spider_arg(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + spider = TestSpider() + yield e.open_spider(spider, []) + e.start() + e.crawl(Request("data:,"), spider) + yield e.close() + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", + ) + + @defer.inlineCallbacks + def test_download_deprecated_spider_arg(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + spider = TestSpider() + yield e.open_spider(spider, []) + e.start() + e.download(Request("data:,"), spider) + yield e.close() + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "Passing a 'spider' argument to ExecutionEngine.download is deprecated", + ) + + @defer.inlineCallbacks + def test_deprecated_schedule(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + spider = TestSpider() + yield e.open_spider(spider, []) + e.start() + e.schedule(Request("data:,"), spider) + yield e.close() + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "ExecutionEngine.schedule is deprecated, please use " + "ExecutionEngine.crawl or ExecutionEngine.download instead", + ) + + @defer.inlineCallbacks + def test_deprecated_has_capacity(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + self.assertTrue(e.has_capacity()) + spider = TestSpider() + yield e.open_spider(spider, []) + self.assertFalse(e.has_capacity()) + e.start() + yield e.close() + self.assertTrue(e.has_capacity()) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated") if __name__ == "__main__": From e3f81d8d5f17515b6eba135ac0db7e270ff0a9f0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 20 Apr 2021 11:46:43 -0300 Subject: [PATCH 0300/2083] Engine: remove unnecessary parameter (#5106) --- scrapy/core/engine.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index edfac87c6..7a09bafa1 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -161,7 +161,7 @@ class ExecutionEngine: return None d = self._download(request, self.spider) - d.addBoth(self._handle_downloader_output, request, self.spider) + d.addBoth(self._handle_downloader_output, request) d.addErrback(lambda f: logger.info('Error while handling downloader output', exc_info=failure_to_exc_info(f), extra={'spider': self.spider})) @@ -176,8 +176,10 @@ class ExecutionEngine: return d def _handle_downloader_output( - self, result: Union[Request, Response, Failure], request: Request, spider: Spider + self, result: Union[Request, Response, Failure], request: Request ) -> Optional[Deferred]: + assert self.spider is not None # typing + if not isinstance(result, (Request, Response, Failure)): raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}") @@ -186,12 +188,12 @@ class ExecutionEngine: self.crawl(result) return None - d = self.scraper.enqueue_scrape(result, request, spider) + d = self.scraper.enqueue_scrape(result, request, self.spider) d.addErrback( lambda f: logger.error( "Error while enqueuing downloader output", exc_info=failure_to_exc_info(f), - extra={'spider': spider}, + extra={'spider': self.spider}, ) ) return d From e779ed7d93beec36f565d33a1cc8d3e8fe6068d7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 20 Apr 2021 16:39:07 -0300 Subject: [PATCH 0301/2083] Dupefilter type hints (#5108) --- scrapy/dupefilters.py | 41 +++++++++++++++++++++++++++-------------- scrapy/utils/request.py | 2 +- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index ac5478e7c..292c68099 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,35 +1,47 @@ -import os import logging +import os +from typing import Optional, Set, Type, TypeVar +from twisted.internet.defer import Deferred + +from scrapy.http.request import Request +from scrapy.settings import BaseSettings +from scrapy.spiders import Spider from scrapy.utils.job import job_dir from scrapy.utils.request import referer_str, request_fingerprint -class BaseDupeFilter: +BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter") + +class BaseDupeFilter: @classmethod - def from_settings(cls, settings): + def from_settings(cls: Type[BaseDupeFilterTV], settings: BaseSettings) -> BaseDupeFilterTV: return cls() - def request_seen(self, request): + def request_seen(self, request: Request) -> bool: return False - def open(self): # can return deferred + def open(self) -> Optional[Deferred]: pass - def close(self, reason): # can return a deferred + def close(self, reason: str) -> Optional[Deferred]: pass - def log(self, request, spider): # log that a request has been filtered + def log(self, request: Request, spider: Spider) -> None: + """Log that a request has been filtered""" pass +RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter") + + class RFPDupeFilter(BaseDupeFilter): """Request Fingerprint duplicates filter""" - def __init__(self, path=None, debug=False): + def __init__(self, path: Optional[str] = None, debug: bool = False) -> None: self.file = None - self.fingerprints = set() + self.fingerprints: Set[str] = set() self.logdupes = True self.debug = debug self.logger = logging.getLogger(__name__) @@ -39,26 +51,27 @@ class RFPDupeFilter(BaseDupeFilter): self.fingerprints.update(x.rstrip() for x in self.file) @classmethod - def from_settings(cls, settings): + def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV: debug = settings.getbool('DUPEFILTER_DEBUG') return cls(job_dir(settings), debug) - def request_seen(self, request): + def request_seen(self, request: Request) -> bool: fp = self.request_fingerprint(request) if fp in self.fingerprints: return True self.fingerprints.add(fp) if self.file: self.file.write(fp + '\n') + return False - def request_fingerprint(self, request): + def request_fingerprint(self, request: Request) -> str: return request_fingerprint(request) - def close(self, reason): + def close(self, reason: str) -> None: if self.file: self.file.close() - def log(self, request, spider): + def log(self, request: Request, spider: Spider) -> None: if self.debug: msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)" args = {'request': request, 'referer': referer_str(request)} diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 66736b42f..541368423 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -24,7 +24,7 @@ def request_fingerprint( request: Request, include_headers: Optional[Iterable[Union[bytes, str]]] = None, keep_fragments: bool = False, -): +) -> str: """ Return the request fingerprint. From 68379197986ae3deb81a545b5fd6920ea3347094 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 26 Apr 2021 14:55:02 -0300 Subject: [PATCH 0302/2083] Add peek method to queues (#5112) --- pylintrc | 1 + scrapy/pqueues.py | 59 +++++++--- scrapy/squeues.py | 59 +++++++--- tests/test_pqueues.py | 144 +++++++++++++++++++++++ tests/test_squeues_request.py | 214 ++++++++++++++++++++++++++++++++++ 5 files changed, 447 insertions(+), 30 deletions(-) create mode 100644 tests/test_pqueues.py create mode 100644 tests/test_squeues_request.py diff --git a/pylintrc b/pylintrc index 5b6b9fab0..972bf99de 100644 --- a/pylintrc +++ b/pylintrc @@ -24,6 +24,7 @@ disable=abstract-method, consider-using-in, consider-using-set-comprehension, consider-using-sys-exit, + consider-using-with, cyclic-import, dangerous-default-value, deprecated-method, diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index a9aa6c649..b4b63e7c7 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -3,6 +3,7 @@ import logging from scrapy.utils.misc import create_instance + logger = logging.getLogger(__name__) @@ -17,8 +18,7 @@ def _path_safe(text): >>> _path_safe('some@symbol?').startswith('some_symbol_') True """ - pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' - for c in text]) + pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part unique_slot = hashlib.md5(text.encode('utf8')).hexdigest() @@ -35,6 +35,9 @@ class ScrapyPriorityQueue: * close() * __len__() + Optionally, the queue could provide a ``peek`` method, that should return the + next object to be returned by ``pop``, but without removing it from the queue. + ``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls argument, which is a class used to instantiate a new (internal) queue when a new priority is allocated. @@ -70,10 +73,12 @@ class ScrapyPriorityQueue: self.curprio = min(startprios) def qfactory(self, key): - return create_instance(self.downstream_queue_cls, - None, - self.crawler, - self.key + '/' + str(key)) + return create_instance( + self.downstream_queue_cls, + None, + self.crawler, + self.key + '/' + str(key), + ) def priority(self, request): return -request.priority @@ -99,6 +104,18 @@ class ScrapyPriorityQueue: self.curprio = min(prios) if prios else None return m + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + if self.curprio is None: + return None + queue = self.queues[self.curprio] + return queue.peek() + def close(self): active = [] for p, q in self.queues.items(): @@ -116,8 +133,7 @@ class DownloaderInterface: self.downloader = crawler.engine.downloader def stats(self, possible_slots): - return [(self._active_downloads(slot), slot) - for slot in possible_slots] + return [(self._active_downloads(slot), slot) for slot in possible_slots] def get_slot_key(self, request): return self.downloader._get_slot_key(request, None) @@ -162,10 +178,12 @@ class DownloaderAwarePriorityQueue: self.pqueues[slot] = self.pqfactory(slot, startprios) def pqfactory(self, slot, startprios=()): - return ScrapyPriorityQueue(self.crawler, - self.downstream_queue_cls, - self.key + '/' + _path_safe(slot), - startprios) + return ScrapyPriorityQueue( + self.crawler, + self.downstream_queue_cls, + self.key + '/' + _path_safe(slot), + startprios, + ) def pop(self): stats = self._downloader_interface.stats(self.pqueues) @@ -187,9 +205,22 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] queue.push(request) + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + stats = self._downloader_interface.stats(self.pqueues) + if not stats: + return None + slot = min(stats)[1] + queue = self.pqueues[slot] + return queue.peek() + def close(self): - active = {slot: queue.close() - for slot, queue in self.pqueues.items()} + active = {slot: queue.close() for slot, queue in self.pqueues.items()} self.pqueues.clear() return active diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 77ffda6f7..44898ba08 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -19,7 +19,6 @@ def _with_mkdir(queue_class): dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname, exist_ok=True) - super().__init__(path, *args, **kwargs) return DirectoriesCreated @@ -38,6 +37,20 @@ def _serializable_queue(queue_class, serialize, deserialize): if s: return deserialize(s) + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + try: + s = super().peek() + except AttributeError as ex: + raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex + if s: + return deserialize(s) + return SerializableQueue @@ -59,12 +72,21 @@ def _scrapy_serialization_queue(queue_class): def pop(self): request = super().pop() - if not request: return None + return request_from_dict(request, self.spider) - request = request_from_dict(request, self.spider) - return request + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + request = super().peek() + if not request: + return None + return request_from_dict(request, self.spider) return ScrapyRequestQueue @@ -76,6 +98,19 @@ def _scrapy_non_serialization_queue(queue_class): def from_crawler(cls, crawler, *args, **kwargs): return cls() + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + try: + s = super().peek() + except AttributeError as ex: + raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex + return s + return ScrapyRequestQueue @@ -109,17 +144,9 @@ MarshalLifoDiskQueueNonRequest = _serializable_queue( marshal.loads ) -PickleFifoDiskQueue = _scrapy_serialization_queue( - PickleFifoDiskQueueNonRequest -) -PickleLifoDiskQueue = _scrapy_serialization_queue( - PickleLifoDiskQueueNonRequest -) -MarshalFifoDiskQueue = _scrapy_serialization_queue( - MarshalFifoDiskQueueNonRequest -) -MarshalLifoDiskQueue = _scrapy_serialization_queue( - MarshalLifoDiskQueueNonRequest -) +PickleFifoDiskQueue = _scrapy_serialization_queue(PickleFifoDiskQueueNonRequest) +PickleLifoDiskQueue = _scrapy_serialization_queue(PickleLifoDiskQueueNonRequest) +MarshalFifoDiskQueue = _scrapy_serialization_queue(MarshalFifoDiskQueueNonRequest) +MarshalLifoDiskQueue = _scrapy_serialization_queue(MarshalLifoDiskQueueNonRequest) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py new file mode 100644 index 000000000..ec55033d1 --- /dev/null +++ b/tests/test_pqueues.py @@ -0,0 +1,144 @@ +import tempfile +import unittest + +import queuelib + +from scrapy.http.request import Request +from scrapy.pqueues import ScrapyPriorityQueue, DownloaderAwarePriorityQueue +from scrapy.spiders import Spider +from scrapy.squeues import FifoMemoryQueue +from scrapy.utils.test import get_crawler + +from tests.test_scheduler import MockDownloader, MockEngine + + +class PriorityQueueTest(unittest.TestCase): + def setUp(self): + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider("foo") + + def test_queue_push_pop_one(self): + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + self.assertIsNone(queue.pop()) + self.assertEqual(len(queue), 0) + req1 = Request("https://example.org/1", priority=1) + queue.push(req1) + self.assertEqual(len(queue), 1) + dequeued = queue.pop() + self.assertEqual(len(queue), 0) + self.assertEqual(dequeued.url, req1.url) + self.assertEqual(dequeued.priority, req1.priority) + self.assertEqual(queue.close(), []) + + def test_no_peek_raises(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + queue.push(Request("https://example.org")) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + queue.peek() + queue.close() + + def test_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + self.assertEqual(len(queue), 0) + self.assertIsNone(queue.peek()) + req1 = Request("https://example.org/1") + req2 = Request("https://example.org/2") + req3 = Request("https://example.org/3") + queue.push(req1) + queue.push(req2) + queue.push(req3) + self.assertEqual(len(queue), 3) + self.assertEqual(queue.peek().url, req1.url) + self.assertEqual(queue.pop().url, req1.url) + self.assertEqual(len(queue), 2) + self.assertEqual(queue.peek().url, req2.url) + self.assertEqual(queue.pop().url, req2.url) + self.assertEqual(len(queue), 1) + self.assertEqual(queue.peek().url, req3.url) + self.assertEqual(queue.pop().url, req3.url) + self.assertEqual(queue.close(), []) + + def test_queue_push_pop_priorities(self): + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]) + self.assertIsNone(queue.pop()) + self.assertEqual(len(queue), 0) + req1 = Request("https://example.org/1", priority=1) + req2 = Request("https://example.org/2", priority=2) + req3 = Request("https://example.org/3", priority=3) + queue.push(req1) + queue.push(req2) + queue.push(req3) + self.assertEqual(len(queue), 3) + dequeued = queue.pop() + self.assertEqual(len(queue), 2) + self.assertEqual(dequeued.url, req3.url) + self.assertEqual(dequeued.priority, req3.priority) + self.assertEqual(queue.close(), [-1, -2]) + + +class DownloaderAwarePriorityQueueTest(unittest.TestCase): + def setUp(self): + crawler = get_crawler(Spider) + crawler.engine = MockEngine(downloader=MockDownloader()) + self.queue = DownloaderAwarePriorityQueue.from_crawler( + crawler=crawler, + downstream_queue_cls=FifoMemoryQueue, + key="foo/bar", + ) + + def tearDown(self): + self.queue.close() + + def test_push_pop(self): + self.assertEqual(len(self.queue), 0) + self.assertIsNone(self.queue.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + self.queue.push(req1) + self.queue.push(req2) + self.queue.push(req3) + self.assertEqual(len(self.queue), 3) + self.assertEqual(self.queue.pop().url, req1.url) + self.assertEqual(len(self.queue), 2) + self.assertEqual(self.queue.pop().url, req2.url) + self.assertEqual(len(self.queue), 1) + self.assertEqual(self.queue.pop().url, req3.url) + self.assertEqual(len(self.queue), 0) + self.assertIsNone(self.queue.pop()) + + def test_no_peek_raises(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + self.queue.push(Request("https://example.org")) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + self.queue.peek() + + def test_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") + self.assertEqual(len(self.queue), 0) + req1 = Request("https://example.org/1") + req2 = Request("https://example.org/2") + req3 = Request("https://example.org/3") + self.queue.push(req1) + self.queue.push(req2) + self.queue.push(req3) + self.assertEqual(len(self.queue), 3) + self.assertEqual(self.queue.peek().url, req1.url) + self.assertEqual(self.queue.pop().url, req1.url) + self.assertEqual(len(self.queue), 2) + self.assertEqual(self.queue.peek().url, req2.url) + self.assertEqual(self.queue.pop().url, req2.url) + self.assertEqual(len(self.queue), 1) + self.assertEqual(self.queue.peek().url, req3.url) + self.assertEqual(self.queue.pop().url, req3.url) + self.assertIsNone(self.queue.peek()) diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py new file mode 100644 index 000000000..c5fcc1853 --- /dev/null +++ b/tests/test_squeues_request.py @@ -0,0 +1,214 @@ +import shutil +import tempfile +import unittest + +import queuelib + +from scrapy.squeues import ( + PickleFifoDiskQueue, + PickleLifoDiskQueue, + MarshalFifoDiskQueue, + MarshalLifoDiskQueue, + FifoMemoryQueue, + LifoMemoryQueue, +) +from scrapy.http import Request +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler + + +""" +Queues that handle requests +""" + + +class BaseQueueTestCase(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") + self.qpath = self.tempfilename() + self.qdir = self.mkdtemp() + self.crawler = get_crawler(Spider) + + def tearDown(self): + shutil.rmtree(self.tmpdir) + + def tempfilename(self): + with tempfile.NamedTemporaryFile(dir=self.tmpdir) as nf: + return nf.name + + def mkdtemp(self): + return tempfile.mkdtemp(dir=self.tmpdir) + + +class RequestQueueTestMixin: + def queue(self): + raise NotImplementedError() + + def test_one_element_with_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + req = Request("http://www.example.com") + q.push(req) + self.assertEqual(len(q), 1) + self.assertEqual(q.peek().url, req.url) + self.assertEqual(q.pop().url, req.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + q.close() + + def test_one_element_without_peek(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + req = Request("http://www.example.com") + q.push(req) + self.assertEqual(len(q), 1) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + q.peek() + self.assertEqual(q.pop().url, req.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + q.close() + + +class FifoQueueMixin(RequestQueueTestMixin): + def test_fifo_with_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + self.assertEqual(len(q), 3) + self.assertEqual(q.peek().url, req1.url) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.peek().url, req2.url) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.peek().url, req3.url) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + q.close() + + def test_fifo_without_peek(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + q.peek() + self.assertEqual(len(q), 3) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + q.close() + + +class LifoQueueMixin(RequestQueueTestMixin): + def test_lifo_with_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + self.assertEqual(len(q), 3) + self.assertEqual(q.peek().url, req3.url) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.peek().url, req2.url) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.peek().url, req1.url) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + q.close() + + def test_lifo_without_peek(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + q.peek() + self.assertEqual(len(q), 3) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + q.close() + + +class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): + def queue(self): + return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo") + + +class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): + def queue(self): + return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo") + + +class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): + def queue(self): + return MarshalFifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/fifo") + + +class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): + def queue(self): + return MarshalLifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/lifo") + + +class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): + def queue(self): + return FifoMemoryQueue.from_crawler(crawler=self.crawler) + + +class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): + def queue(self): + return LifoMemoryQueue.from_crawler(crawler=self.crawler) From ddea6b7bfa38bf5402d78350ab61e2c827ca49b5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:16:14 -0300 Subject: [PATCH 0303/2083] Scheduler: minimal interface, API docs (#3559) --- docs/index.rst | 4 + docs/topics/architecture.rst | 5 +- docs/topics/scheduler.rst | 34 ++++ docs/topics/settings.rst | 3 +- scrapy/core/engine.py | 21 ++- scrapy/core/scheduler.py | 298 +++++++++++++++++++++++++++-------- scrapy/utils/job.py | 5 +- tests/test_scheduler_base.py | 159 +++++++++++++++++++ 8 files changed, 458 insertions(+), 71 deletions(-) create mode 100644 docs/topics/scheduler.rst create mode 100644 tests/test_scheduler_base.py diff --git a/docs/index.rst b/docs/index.rst index da264fb34..433798aa8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -227,6 +227,7 @@ Extending Scrapy topics/extensions topics/api topics/signals + topics/scheduler topics/exporters @@ -248,6 +249,9 @@ Extending Scrapy :doc:`topics/signals` See all available signals and how to work with them. +:doc:`topics/scheduler` + Understand the scheduler component. + :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 074c59241..71d027c86 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -87,8 +87,9 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The Scheduler receives requests from the engine and enqueues them for feeding -them later (also to the engine) when the engine requests them. +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine +requests them. .. _component-downloader: diff --git a/docs/topics/scheduler.rst b/docs/topics/scheduler.rst new file mode 100644 index 000000000..57c24b76a --- /dev/null +++ b/docs/topics/scheduler.rst @@ -0,0 +1,34 @@ +.. _topics-scheduler: + +========= +Scheduler +========= + +.. module:: scrapy.core.scheduler + +The scheduler component receives requests from the :ref:`engine ` +and stores them into persistent and/or non-persistent data structures. +It also gets those requests and feeds them back to the engine when it +asks for a next request to be downloaded. + + +Overriding the default scheduler +================================ + +You can use your own custom scheduler class by supplying its full +Python path in the :setting:`SCHEDULER` setting. + + +Minimal scheduler interface +=========================== + +.. autoclass:: BaseScheduler + :members: + + +Default Scrapy scheduler +======================== + +.. autoclass:: Scheduler + :members: + :special-members: __len__ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1d5babcec..e4fb2baf7 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1280,7 +1280,8 @@ SCHEDULER Default: ``'scrapy.core.scheduler.Scheduler'`` -The scheduler to use for crawling. +The scheduler class to be used for crawling. +See the :ref:`topics-scheduler` topic for details. .. setting:: SCHEDULER_DEBUG diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7a09bafa1..dd3225082 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -17,6 +17,7 @@ from scrapy import signals from scrapy.core.scraper import Scraper from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning from scrapy.http import Response, Request +from scrapy.settings import BaseSettings from scrapy.spiders import Spider from scrapy.utils.log import logformatter_adapter, failure_to_exc_info from scrapy.utils.misc import create_instance, load_object @@ -73,12 +74,22 @@ class ExecutionEngine: self.spider: Optional[Spider] = None self.running = False self.paused = False - self.scheduler_cls = load_object(crawler.settings["SCHEDULER"]) + self.scheduler_cls = self._get_scheduler_class(crawler.settings) downloader_cls = load_object(self.settings['DOWNLOADER']) self.downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback = spider_closed_callback + def _get_scheduler_class(self, settings: BaseSettings) -> type: + from scrapy.core.scheduler import BaseScheduler + scheduler_cls = load_object(settings["SCHEDULER"]) + if not issubclass(scheduler_cls, BaseScheduler): + raise TypeError( + f"The provided scheduler class ({settings['SCHEDULER']})" + " does not fully implement the scheduler interface" + ) + return scheduler_cls + @inlineCallbacks def start(self) -> Deferred: if self.running: @@ -301,7 +312,8 @@ class ExecutionEngine: start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider) self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider - yield scheduler.open(spider) + if hasattr(scheduler, "open"): + yield scheduler.open(spider) yield self.scraper.open_spider(spider) self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) @@ -345,8 +357,9 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.scraper.close_spider(spider)) dfd.addErrback(log_failure('Scraper close failure')) - dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) - dfd.addErrback(log_failure('Scheduler close failure')) + if hasattr(self.slot.scheduler, "close"): + dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) + dfd.addErrback(log_failure("Scheduler close failure")) dfd.addBoth(lambda _: self.signals.send_catch_log_deferred( signal=signals.spider_closed, spider=spider, reason=reason, diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 9ce823dbc..5ba0fb63b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,42 +1,179 @@ -import os import json import logging -from os.path import join, exists +import os +from abc import abstractmethod +from os.path import exists, join +from typing import Optional, Type, TypeVar -from scrapy.utils.misc import load_object, create_instance +from twisted.internet.defer import Deferred + +from scrapy.crawler import Crawler +from scrapy.http.request import Request +from scrapy.spiders import Spider from scrapy.utils.job import job_dir +from scrapy.utils.misc import create_instance, load_object logger = logging.getLogger(__name__) -class Scheduler: +class BaseSchedulerMeta(type): """ - Scrapy Scheduler. It allows to enqueue requests and then get - a next request to download. Scheduler is also handling duplication - filtering, via dupefilter. - - Prioritization and queueing is not performed by the Scheduler. - User sets ``priority`` field for each Request, and a PriorityQueue - (defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities - to dequeue requests in a desired order. - - Scheduler uses two PriorityQueue instances, configured to work in-memory - and on-disk (optional). When on-disk queue is present, it is used by - default, and an in-memory queue is used as a fallback for cases where - a disk queue can't handle a request (can't serialize it). - - :setting:`SCHEDULER_MEMORY_QUEUE` and - :setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes - which PriorityQueue instances would be instantiated with, to keep requests - on disk and in memory respectively. - - Overall, Scheduler is an object which holds several PriorityQueue instances - (in-memory and on-disk) and implements fallback logic for them. - Also, it handles dupefilters. + Metaclass to check scheduler classes against the necessary interface """ - def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, - logunser=False, stats=None, pqclass=None, crawler=None): + def __instancecheck__(cls, instance): + return cls.__subclasscheck__(type(instance)) + + def __subclasscheck__(cls, subclass): + return ( + hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests) + and hasattr(subclass, "enqueue_request") and callable(subclass.enqueue_request) + and hasattr(subclass, "next_request") and callable(subclass.next_request) + ) + + +class BaseScheduler(metaclass=BaseSchedulerMeta): + """ + The scheduler component is responsible for storing requests received from + the engine, and feeding them back upon request (also to the engine). + + The original sources of said requests are: + + * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks + * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods + * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods + + The order in which the scheduler returns its stored requests (via the ``next_request`` method) + plays a great part in determining the order in which those requests are downloaded. + + The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with. + """ + + @classmethod + def from_crawler(cls, crawler: Crawler): + """ + Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument. + """ + return cls() + + def open(self, spider: Spider) -> Optional[Deferred]: + """ + Called when the spider is opened by the engine. It receives the spider + instance as argument and it's useful to execute initialization code. + + :param spider: the spider object for the current crawl + :type spider: :class:`~scrapy.spiders.Spider` + """ + pass + + def close(self, reason: str) -> Optional[Deferred]: + """ + Called when the spider is closed by the engine. It receives the reason why the crawl + finished as argument and it's useful to execute cleaning code. + + :param reason: a string which describes the reason why the spider was closed + :type reason: :class:`str` + """ + pass + + @abstractmethod + def has_pending_requests(self) -> bool: + """ + ``True`` if the scheduler has enqueued requests, ``False`` otherwise + """ + raise NotImplementedError() + + @abstractmethod + def enqueue_request(self, request: Request) -> bool: + """ + Process a request received by the engine. + + Return ``True`` if the request is stored correctly, ``False`` otherwise. + + If ``False``, the engine will fire a ``request_dropped`` signal, and + will not make further attempts to schedule the request at a later time. + For reference, the default Scrapy scheduler returns ``False`` when the + request is rejected by the dupefilter. + """ + raise NotImplementedError() + + @abstractmethod + def next_request(self) -> Optional[Request]: + """ + Return the next :class:`~scrapy.http.Request` to be processed, or ``None`` + to indicate that there are no requests to be considered ready at the moment. + + Returning ``None`` implies that no request from the scheduler will be sent + to the downloader in the current reactor cycle. The engine will continue + calling ``next_request`` until ``has_pending_requests`` is ``False``. + """ + raise NotImplementedError() + + +SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler") + + +class Scheduler(BaseScheduler): + """ + Default Scrapy scheduler. This implementation also handles duplication + filtering via the :setting:`dupefilter `. + + This scheduler stores requests into several priority queues (defined by the + :setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues + are backed by either memory or disk based queues (respectively defined by the + :setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings). + + Request prioritization is almost entirely delegated to the priority queue. The only + prioritization performed by this scheduler is using the disk-based queue if present + (i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based + queue if a serialization error occurs. If the disk queue is not present, the memory one + is used directly. + + :param dupefilter: An object responsible for checking and filtering duplicate requests. + The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. + :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: + any class that implements the `BaseDupeFilter` interface + + :param jobdir: The path of a directory to be used for persisting the crawl's state. + The value for the :setting:`JOBDIR` setting is used by default. + See :ref:`topics-jobs`. + :type jobdir: :class:`str` or ``None`` + + :param dqclass: A class to be used as persistent request queue. + The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. + :type dqclass: class + + :param mqclass: A class to be used as non-persistent request queue. + The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. + :type mqclass: class + + :param logunser: A boolean that indicates whether or not unserializable requests should be logged. + The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. + :type logunser: bool + + :param stats: A stats collector object to record stats about the request scheduling process. + The value for the :setting:`STATS_CLASS` setting is used by default. + :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: + any class that implements the `StatsCollector` interface + + :param pqclass: A class to be used as priority queue for requests. + The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. + :type pqclass: class + + :param crawler: The crawler object corresponding to the current crawl. + :type crawler: :class:`scrapy.crawler.Crawler` + """ + def __init__( + self, + dupefilter, + jobdir: Optional[str] = None, + dqclass=None, + mqclass=None, + logunser: bool = False, + stats=None, + pqclass=None, + crawler: Optional[Crawler] = None, + ): self.df = dupefilter self.dqdir = self._dqdir(jobdir) self.pqclass = pqclass @@ -47,34 +184,57 @@ class Scheduler: self.crawler = crawler @classmethod - def from_crawler(cls, crawler): - settings = crawler.settings - dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) - dupefilter = create_instance(dupefilter_cls, settings, crawler) - pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) - dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) - mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) - logunser = settings.getbool('SCHEDULER_DEBUG') - return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, - stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, - mqclass=mqclass, crawler=crawler) + def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV: + """ + Factory method, initializes the scheduler with arguments taken from the crawl settings + """ + dupefilter_cls = load_object(crawler.settings['DUPEFILTER_CLASS']) + return cls( + dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler), + jobdir=job_dir(crawler.settings), + dqclass=load_object(crawler.settings['SCHEDULER_DISK_QUEUE']), + mqclass=load_object(crawler.settings['SCHEDULER_MEMORY_QUEUE']), + logunser=crawler.settings.getbool('SCHEDULER_DEBUG'), + stats=crawler.stats, + pqclass=load_object(crawler.settings['SCHEDULER_PRIORITY_QUEUE']), + crawler=crawler, + ) - def has_pending_requests(self): + def has_pending_requests(self) -> bool: return len(self) > 0 - def open(self, spider): + def open(self, spider: Spider) -> Optional[Deferred]: + """ + (1) initialize the memory queue + (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory + (3) return the result of the dupefilter's ``open`` method + """ self.spider = spider self.mqs = self._mq() self.dqs = self._dq() if self.dqdir else None return self.df.open() - def close(self, reason): - if self.dqs: + def close(self, reason: str) -> Optional[Deferred]: + """ + (1) dump pending requests to disk if there is a disk queue + (2) return the result of the dupefilter's ``close`` method + """ + if self.dqs is not None: state = self.dqs.close() + assert isinstance(self.dqdir, str) self._write_dqs_state(self.dqdir, state) return self.df.close(reason) - def enqueue_request(self, request): + def enqueue_request(self, request: Request) -> bool: + """ + Unless the received request is filtered out by the Dupefilter, attempt to push + it into the disk queue, falling back to pushing it into the memory queue. + + Increment the appropriate stats, such as: ``scheduler/enqueued``, + ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``. + + Return ``True`` if the request was stored successfully, ``False`` otherwise. + """ if not request.dont_filter and self.df.request_seen(request): self.df.log(request, self.spider) return False @@ -87,24 +247,35 @@ class Scheduler: self.stats.inc_value('scheduler/enqueued', spider=self.spider) return True - def next_request(self): + def next_request(self) -> Optional[Request]: + """ + Return a :class:`~scrapy.http.Request` object from the memory queue, + falling back to the disk queue if the memory queue is empty. + Return ``None`` if there are no more enqueued requests. + + Increment the appropriate stats, such as: ``scheduler/dequeued``, + ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. + """ request = self.mqs.pop() - if request: + if request is not None: self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider) else: request = self._dqpop() - if request: + if request is not None: self.stats.inc_value('scheduler/dequeued/disk', spider=self.spider) - if request: + if request is not None: self.stats.inc_value('scheduler/dequeued', spider=self.spider) return request - def __len__(self): - return len(self.dqs) + len(self.mqs) if self.dqs else len(self.mqs) + def __len__(self) -> int: + """ + Return the total amount of enqueued requests + """ + return len(self.dqs) + len(self.mqs) if self.dqs is not None else len(self.mqs) - def _dqpush(self, request): + def _dqpush(self, request: Request) -> bool: if self.dqs is None: - return + return False try: self.dqs.push(request) except ValueError as e: # non serializable request @@ -115,18 +286,18 @@ class Scheduler: logger.warning(msg, {'request': request, 'reason': e}, exc_info=True, extra={'spider': self.spider}) self.logunser = False - self.stats.inc_value('scheduler/unserializable', - spider=self.spider) - return + self.stats.inc_value('scheduler/unserializable', spider=self.spider) + return False else: return True - def _mqpush(self, request): + def _mqpush(self, request: Request) -> None: self.mqs.push(request) - def _dqpop(self): - if self.dqs: + def _dqpop(self) -> Optional[Request]: + if self.dqs is not None: return self.dqs.pop() + return None def _mq(self): """ Create a new priority queue instance, with in-memory storage """ @@ -150,21 +321,22 @@ class Scheduler: {'queuesize': len(q)}, extra={'spider': self.spider}) return q - def _dqdir(self, jobdir): + def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: """ Return a folder name to keep disk queue state at """ - if jobdir: + if jobdir is not None: dqdir = join(jobdir, 'requests.queue') if not exists(dqdir): os.makedirs(dqdir) return dqdir + return None - def _read_dqs_state(self, dqdir): + def _read_dqs_state(self, dqdir: str) -> list: path = join(dqdir, 'active.json') if not exists(path): - return () + return [] with open(path) as f: return json.load(f) - def _write_dqs_state(self, dqdir, state): + def _write_dqs_state(self, dqdir: str, state: list) -> None: with open(join(dqdir, 'active.json'), 'w') as f: json.dump(state, f) diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 4f1e601fc..c92ef36f5 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,7 +1,10 @@ import os +from typing import Optional + +from scrapy.settings import BaseSettings -def job_dir(settings): +def job_dir(settings: BaseSettings) -> Optional[str]: path = settings['JOBDIR'] if path and not os.path.exists(path): os.makedirs(path) diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py new file mode 100644 index 000000000..bf90b4320 --- /dev/null +++ b/tests/test_scheduler_base.py @@ -0,0 +1,159 @@ +from typing import Dict, Optional +from unittest import TestCase +from urllib.parse import urljoin, urlparse + +from testfixtures import LogCapture +from twisted.internet import defer +from twisted.trial.unittest import TestCase as TwistedTestCase + +from scrapy.core.scheduler import BaseScheduler +from scrapy.crawler import CrawlerRunner +from scrapy.http import Request +from scrapy.spiders import Spider +from scrapy.utils.request import request_fingerprint + +from tests.mockserver import MockServer + + +PATHS = ["/a", "/b", "/c"] +URLS = [urljoin("https://example.org", p) for p in PATHS] + + +class MinimalScheduler: + def __init__(self) -> None: + self.requests: Dict[str, Request] = {} + + def has_pending_requests(self) -> bool: + return bool(self.requests) + + def enqueue_request(self, request: Request) -> bool: + fp = request_fingerprint(request) + if fp not in self.requests: + self.requests[fp] = request + return True + return False + + def next_request(self) -> Optional[Request]: + if self.has_pending_requests(): + fp, request = self.requests.popitem() + return request + return None + + +class SimpleScheduler(MinimalScheduler): + def open(self, spider: Spider) -> defer.Deferred: + return defer.succeed("open") + + def close(self, reason: str) -> defer.Deferred: + return defer.succeed("close") + + def __len__(self) -> int: + return len(self.requests) + + +class TestSpider(Spider): + name = "test" + + def __init__(self, mockserver, *args, **kwargs): + super().__init__(*args, **kwargs) + self.start_urls = map(mockserver.url, PATHS) + + def parse(self, response): + return {"path": urlparse(response.url).path} + + +class InterfaceCheckMixin: + def test_scheduler_class(self): + self.assertTrue(isinstance(self.scheduler, BaseScheduler)) + self.assertTrue(issubclass(self.scheduler.__class__, BaseScheduler)) + + +class BaseSchedulerTest(TestCase, InterfaceCheckMixin): + def setUp(self): + self.scheduler = BaseScheduler() + + def test_methods(self): + self.assertIsNone(self.scheduler.open(Spider("foo"))) + self.assertIsNone(self.scheduler.close("finished")) + self.assertRaises(NotImplementedError, self.scheduler.has_pending_requests) + self.assertRaises(NotImplementedError, self.scheduler.enqueue_request, Request("https://example.org")) + self.assertRaises(NotImplementedError, self.scheduler.next_request) + + +class MinimalSchedulerTest(TestCase, InterfaceCheckMixin): + def setUp(self): + self.scheduler = MinimalScheduler() + + def test_open_close(self): + with self.assertRaises(AttributeError): + self.scheduler.open(Spider("foo")) + with self.assertRaises(AttributeError): + self.scheduler.close("finished") + + def test_len(self): + with self.assertRaises(AttributeError): + self.scheduler.__len__() + with self.assertRaises(TypeError): + len(self.scheduler) + + def test_enqueue_dequeue(self): + self.assertFalse(self.scheduler.has_pending_requests()) + for url in URLS: + self.assertTrue(self.scheduler.enqueue_request(Request(url))) + self.assertFalse(self.scheduler.enqueue_request(Request(url))) + self.assertTrue(self.scheduler.has_pending_requests) + + dequeued = [] + while self.scheduler.has_pending_requests(): + request = self.scheduler.next_request() + dequeued.append(request.url) + self.assertEqual(set(dequeued), set(URLS)) + self.assertFalse(self.scheduler.has_pending_requests()) + + +class SimpleSchedulerTest(TwistedTestCase, InterfaceCheckMixin): + def setUp(self): + self.scheduler = SimpleScheduler() + + @defer.inlineCallbacks + def test_enqueue_dequeue(self): + open_result = yield self.scheduler.open(Spider("foo")) + self.assertEqual(open_result, "open") + self.assertFalse(self.scheduler.has_pending_requests()) + + for url in URLS: + self.assertTrue(self.scheduler.enqueue_request(Request(url))) + self.assertFalse(self.scheduler.enqueue_request(Request(url))) + + self.assertTrue(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), len(URLS)) + + dequeued = [] + while self.scheduler.has_pending_requests(): + request = self.scheduler.next_request() + dequeued.append(request.url) + self.assertEqual(set(dequeued), set(URLS)) + + self.assertFalse(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), 0) + + close_result = yield self.scheduler.close("") + self.assertEqual(close_result, "close") + + +class MinimalSchedulerCrawlTest(TwistedTestCase): + scheduler_cls = MinimalScheduler + + @defer.inlineCallbacks + def test_crawl(self): + with MockServer() as mockserver: + settings = {"SCHEDULER": self.scheduler_cls} + with LogCapture() as log: + yield CrawlerRunner(settings).crawl(TestSpider, mockserver) + for path in PATHS: + self.assertIn(f"{{'path': '{path}'}}", str(log)) + self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log)) + + +class SimpleSchedulerCrawlTest(MinimalSchedulerCrawlTest): + scheduler_cls = SimpleScheduler From 02ae1deaf499e79dc8ececf4d5dcea6daef0ade0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 27 Apr 2021 09:41:44 -0300 Subject: [PATCH 0304/2083] Deprecate unused squeues (#5117) --- scrapy/squeues.py | 47 +++++++++++++++++++++++++++++++++++-------- tests/test_squeues.py | 16 +++++++-------- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 44898ba08..16f7bf4b6 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -8,6 +8,7 @@ import pickle from queuelib import queue +from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.reqser import request_to_dict, request_from_dict @@ -123,30 +124,60 @@ def _pickle_serialize(obj): raise ValueError(str(e)) from e -PickleFifoDiskQueueNonRequest = _serializable_queue( +_PickleFifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads ) -PickleLifoDiskQueueNonRequest = _serializable_queue( +_PickleLifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads ) -MarshalFifoDiskQueueNonRequest = _serializable_queue( +_MarshalFifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads ) -MarshalLifoDiskQueueNonRequest = _serializable_queue( +_MarshalLifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads ) -PickleFifoDiskQueue = _scrapy_serialization_queue(PickleFifoDiskQueueNonRequest) -PickleLifoDiskQueue = _scrapy_serialization_queue(PickleLifoDiskQueueNonRequest) -MarshalFifoDiskQueue = _scrapy_serialization_queue(MarshalFifoDiskQueueNonRequest) -MarshalLifoDiskQueue = _scrapy_serialization_queue(MarshalLifoDiskQueueNonRequest) +# public queue classes +PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue) +PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue) +MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue) +MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) + + +# deprecated queue classes +_subclass_warn_message = "{cls} inherits from deprecated class {old}" +_instance_warn_message = "{cls} is deprecated" +PickleFifoDiskQueueNonRequest = create_deprecated_class( + name="PickleFifoDiskQueueNonRequest", + new_class=_PickleFifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) +PickleLifoDiskQueueNonRequest = create_deprecated_class( + name="PickleLifoDiskQueueNonRequest", + new_class=_PickleLifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) +MarshalFifoDiskQueueNonRequest = create_deprecated_class( + name="MarshalFifoDiskQueueNonRequest", + new_class=_MarshalFifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) +MarshalLifoDiskQueueNonRequest = create_deprecated_class( + name="MarshalLifoDiskQueueNonRequest", + new_class=_MarshalLifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index becacce62..acc821b83 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -3,10 +3,10 @@ import sys from queuelib.tests import test_queue as t from scrapy.squeues import ( - MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue, - MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue, - PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue, - PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue + _MarshalFifoSerializationDiskQueue, + _MarshalLifoSerializationDiskQueue, + _PickleFifoSerializationDiskQueue, + _PickleLifoSerializationDiskQueue, ) from scrapy.item import Item, Field from scrapy.http import Request @@ -53,7 +53,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): chunksize = 100000 def queue(self): - return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize) + return _MarshalFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize) class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): @@ -77,7 +77,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): chunksize = 100000 def queue(self): - return PickleFifoDiskQueue(self.qpath, chunksize=self.chunksize) + return _PickleFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize) def test_serialize_item(self): q = self.queue() @@ -155,13 +155,13 @@ class LifoDiskQueueTestMixin: class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def queue(self): - return MarshalLifoDiskQueue(self.qpath) + return _MarshalLifoSerializationDiskQueue(self.qpath) class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def queue(self): - return PickleLifoDiskQueue(self.qpath) + return _PickleLifoSerializationDiskQueue(self.qpath) def test_serialize_item(self): q = self.queue() From 4f500342c8ad4674b191e1fab0d1b2ac944d7d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Wed, 28 Apr 2021 11:57:44 +0200 Subject: [PATCH 0305/2083] Require setuptools, scrapy/cmdline.py, /setup.py and tests/test_webclient.py import pkg_resources --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 2b60a10af..b1bb64575 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'h2>=3.0,<4.0', + 'setuptools', ] extras_require = {} cpython_dependencies = [ From 19c7415aae1678631d5ca115a13094b6bd70f245 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 1 May 2021 16:34:39 -0300 Subject: [PATCH 0306/2083] Request type hints --- scrapy/downloadermiddlewares/retry.py | 2 +- scrapy/http/request/__init__.py | 69 ++++++++++++++++----------- scrapy/utils/curl.py | 2 +- 3 files changed, 42 insertions(+), 31 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5965a1c6c..f1fdc3858 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -98,7 +98,7 @@ def get_retry_request( {'request': request, 'retry_times': retry_times, 'reason': reason}, extra={'spider': spider} ) - new_request = request.copy() + new_request: Request = request.copy() new_request.meta['retry_times'] = retry_times new_request.dont_filter = True if priority_adjust is None: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 498f1b052..3cce9f501 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,22 +4,38 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +from typing import Callable, List, Optional, Type, TypeVar, Union + from w3lib.url import safe_url_string +from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers +from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax -from scrapy.http.common import obsolete_setter -from scrapy.utils.curl import curl_to_request_kwargs + + +RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") class Request(object_ref): - - def __init__(self, url, callback=None, method='GET', headers=None, body=None, - cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, flags=None, cb_kwargs=None): - + def __init__( + self, + url: str, + callback: Optional[Callable] = None, + method: str = "GET", + headers: Optional[dict] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]]=None, + meta: Optional[dict] = None, + encoding: str = "utf-8", + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict] = None, + ) -> None: self._encoding = encoding # this one has to be set first self.method = str(method).upper() self._set_url(url) @@ -44,23 +60,23 @@ class Request(object_ref): self.flags = [] if flags is None else list(flags) @property - def cb_kwargs(self): + def cb_kwargs(self) -> dict: if self._cb_kwargs is None: self._cb_kwargs = {} return self._cb_kwargs @property - def meta(self): + def meta(self) -> dict: if self._meta is None: self._meta = {} return self._meta - def _get_url(self): + def _get_url(self) -> str: return self._url - def _set_url(self, url): + def _set_url(self, url: str) -> None: if not isinstance(url, str): - raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}') + raise TypeError(f"Request url must be str, got {type(url).__name__}") s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) @@ -74,34 +90,28 @@ class Request(object_ref): url = property(_get_url, obsolete_setter(_set_url, 'url')) - def _get_body(self): + def _get_body(self) -> bytes: return self._body - def _set_body(self, body): - if body is None: - self._body = b'' - else: - self._body = to_bytes(body, self.encoding) + def _set_body(self, body: Optional[Union[str, bytes]]) -> None: + self._body = b"" if body is None else to_bytes(body, self.encoding) body = property(_get_body, obsolete_setter(_set_body, 'body')) @property - def encoding(self): + def encoding(self) -> str: return self._encoding - def __str__(self): + def __str__(self) -> str: return f"<{self.method} {self.url}>" __repr__ = __str__ - def copy(self): - """Return a copy of this Request""" + def copy(self) -> RequestTypeVar: return self.replace() - def replace(self, *args, **kwargs): - """Create a new Request with the same attributes except for those - given new values. - """ + def replace(self, *args, **kwargs) -> RequestTypeVar: + """Create a new Request with the same attributes except for those given new values""" for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']: kwargs.setdefault(x, getattr(self, x)) @@ -109,7 +119,9 @@ class Request(object_ref): return cls(*args, **kwargs) @classmethod - def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs): + def from_curl( + cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, **kwargs + ) -> RequestTypeVar: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same @@ -136,8 +148,7 @@ class Request(object_ref): To translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. - - """ + """ request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options) request_kwargs.update(kwargs) return cls(**request_kwargs) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index d8b3deaa1..74f82ad75 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -54,7 +54,7 @@ def _parse_headers_and_cookies(parsed_args): return headers, cookies -def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): +def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = True) -> dict: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command From 34b216289c31c27ad6256ff95382505a9d84adb3 Mon Sep 17 00:00:00 2001 From: Renne Rocha Date: Thu, 6 May 2021 11:34:05 -0300 Subject: [PATCH 0307/2083] Update link for reasoning value of URLLENGTH_LIMIT (#5134) --- docs/topics/settings.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e4fb2baf7..2506497e2 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1619,7 +1619,7 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. For more information about -the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html +the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 .. setting:: USER_AGENT @@ -1642,7 +1642,6 @@ case to see how to enable and use them. .. settingslist:: - .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search From cec36a9284641bb7b69a2081ad92cd7d4ee25934 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 10 May 2021 13:00:08 -0300 Subject: [PATCH 0308/2083] Refactor request to/from dict (#5130) --- docs/topics/request-response.rst | 17 ++- scrapy/http/request/__init__.py | 70 ++++++++++-- scrapy/http/request/json_request.py | 8 ++ scrapy/squeues.py | 8 +- scrapy/utils/reqser.py | 103 +++--------------- scrapy/utils/request.py | 27 ++++- ...t_utils_reqser.py => test_request_dict.py} | 76 +++++++++---- 7 files changed, 183 insertions(+), 126 deletions(-) rename tests/{test_utils_reqser.py => test_request_dict.py} (68%) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 500781c05..73b5a858f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -26,10 +26,6 @@ Request objects .. autoclass:: Request - A :class:`Request` object represents an HTTP request, which is usually - generated in the Spider and executed by the Downloader, and thus generating - a :class:`Response`. - :param url: the URL of this request If the URL is invalid, a :exc:`ValueError` exception is raised. @@ -205,6 +201,8 @@ Request objects ``failure.request.cb_kwargs`` in the request's errback. For more information, see :ref:`errback-cb_kwargs`. + .. autoattribute:: Request.attributes + .. method:: Request.copy() Return a new Request which is a copy of this Request. See also: @@ -220,6 +218,15 @@ Request objects .. automethod:: from_curl + .. automethod:: to_dict + + +Other functions related to requests +----------------------------------- + +.. autofunction:: scrapy.utils.request.request_from_dict + + .. _topics-request-response-ref-request-callback-arguments: Passing additional data to callback functions @@ -642,6 +649,8 @@ dealing with JSON requests. data into JSON format. :type dumps_kwargs: dict + .. autoattribute:: JsonRequest.attributes + JsonRequest usage example ------------------------- diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 498f1b052..ad884feac 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,17 +4,37 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +import inspect +from typing import Optional, Tuple + from w3lib.url import safe_url_string +import scrapy +from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers +from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax -from scrapy.http.common import obsolete_setter -from scrapy.utils.curl import curl_to_request_kwargs class Request(object_ref): + """Represents an HTTP request, which is usually generated in a Spider and + executed by the Downloader, thus generating a :class:`Response`. + """ + + attributes: Tuple[str, ...] = ( + "url", "callback", "method", "headers", "body", + "cookies", "meta", "encoding", "priority", + "dont_filter", "errback", "flags", "cb_kwargs", + ) + """A tuple of :class:`str` objects containing the name of all public + attributes of the class that are also keyword parameters of the + ``__init__`` method. + + Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and + :func:`~scrapy.utils.request.request_from_dict`. + """ def __init__(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, @@ -99,11 +119,8 @@ class Request(object_ref): return self.replace() def replace(self, *args, **kwargs): - """Create a new Request with the same attributes except for those - given new values. - """ - for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', - 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']: + """Create a new Request with the same attributes except for those given new values""" + for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) @@ -136,8 +153,43 @@ class Request(object_ref): To translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. - - """ + """ request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options) request_kwargs.update(kwargs) return cls(**request_kwargs) + + def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict: + """Return a dictionary containing the Request's data. + + Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. + + If a spider is given, this method will try to find out the name of the spider methods used as callback + and errback and include them in the output dict, raising an exception if they cannot be found. + """ + d = { + "url": self.url, # urls are safe (safe_string_url) + "callback": _find_method(spider, self.callback) if callable(self.callback) else self.callback, + "errback": _find_method(spider, self.errback) if callable(self.errback) else self.errback, + "headers": dict(self.headers), + } + for attr in self.attributes: + d.setdefault(attr, getattr(self, attr)) + if type(self) is not Request: + d["_class"] = self.__module__ + '.' + self.__class__.__name__ + return d + + +def _find_method(obj, func): + """Helper function for Request.to_dict""" + # Only instance methods contain ``__func__`` + if obj and hasattr(func, '__func__'): + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original function object because instance + # method objects are generated each time attribute is retrieved from instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name + raise ValueError(f"Function {func} is not an instance method in: {obj}") diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index eae3f9f6b..04e80d897 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,12 +8,16 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings +from typing import Tuple from scrapy.http.request import Request from scrapy.utils.deprecate import create_deprecated_class class JsonRequest(Request): + + attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) + def __init__(self, *args, **kwargs): dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) dumps_kwargs.setdefault('sort_keys', True) @@ -36,6 +40,10 @@ class JsonRequest(Request): self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') + @property + def dumps_kwargs(self): + return self._dumps_kwargs + def replace(self, *args, **kwargs): body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 16f7bf4b6..dff9b1350 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -9,7 +9,7 @@ import pickle from queuelib import queue from scrapy.utils.deprecate import create_deprecated_class -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy.utils.request import request_from_dict def _with_mkdir(queue_class): @@ -68,14 +68,14 @@ def _scrapy_serialization_queue(queue_class): return cls(crawler, key) def push(self, request): - request = request_to_dict(request, self.spider) + request = request.to_dict(spider=self.spider) return super().push(request) def pop(self): request = super().pop() if not request: return None - return request_from_dict(request, self.spider) + return request_from_dict(request, spider=self.spider) def peek(self): """Returns the next object to be returned by :meth:`pop`, @@ -87,7 +87,7 @@ def _scrapy_serialization_queue(queue_class): request = super().peek() if not request: return None - return request_from_dict(request, self.spider) + return request_from_dict(request, spider=self.spider) return ScrapyRequestQueue diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index d38b1bc4d..c254b9f82 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -1,95 +1,22 @@ -""" -Helper functions for serializing (and deserializing) requests. -""" -import inspect +import warnings +from typing import Optional -from scrapy.http import Request -from scrapy.utils.python import to_unicode -from scrapy.utils.misc import load_object +import scrapy +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.request import request_from_dict as _from_dict -def request_to_dict(request, spider=None): - """Convert Request object to a dict. - - If a spider is given, it will try to find out the name of the spider method - used in the callback and store that as the callback. - """ - cb = request.callback - if callable(cb): - cb = _find_method(spider, cb) - eb = request.errback - if callable(eb): - eb = _find_method(spider, eb) - d = { - 'url': to_unicode(request.url), # urls should be safe (safe_string_url) - 'callback': cb, - 'errback': eb, - 'method': request.method, - 'headers': dict(request.headers), - 'body': request.body, - 'cookies': request.cookies, - 'meta': request.meta, - '_encoding': request._encoding, - 'priority': request.priority, - 'dont_filter': request.dont_filter, - 'flags': request.flags, - 'cb_kwargs': request.cb_kwargs, - } - if type(request) is not Request: - d['_class'] = request.__module__ + '.' + request.__class__.__name__ - return d +warnings.warn( + ("Module scrapy.utils.reqser is deprecated, please use request.to_dict method" + " and/or scrapy.utils.request.request_from_dict instead"), + category=ScrapyDeprecationWarning, + stacklevel=2, +) -def request_from_dict(d, spider=None): - """Create Request object from a dict. - - If a spider is given, it will try to resolve the callbacks looking at the - spider for methods with the same name. - """ - cb = d['callback'] - if cb and spider: - cb = _get_method(spider, cb) - eb = d['errback'] - if eb and spider: - eb = _get_method(spider, eb) - request_cls = load_object(d['_class']) if '_class' in d else Request - return request_cls( - url=to_unicode(d['url']), - callback=cb, - errback=eb, - method=d['method'], - headers=d['headers'], - body=d['body'], - cookies=d['cookies'], - meta=d['meta'], - encoding=d['_encoding'], - priority=d['priority'], - dont_filter=d['dont_filter'], - flags=d.get('flags'), - cb_kwargs=d.get('cb_kwargs'), - ) +def request_to_dict(request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None) -> dict: + return request.to_dict(spider=spider) -def _find_method(obj, func): - # Only instance methods contain ``__func__`` - if obj and hasattr(func, '__func__'): - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name - raise ValueError(f"Function {func} is not an instance method in: {obj}") - - -def _get_method(obj, name): - name = str(name) - try: - return getattr(obj, name) - except AttributeError: - raise ValueError(f"Method {name!r} not found in: {obj}") +def request_from_dict(d: dict, spider: Optional["scrapy.Spider"] = None) -> "scrapy.Request": + return _from_dict(d, spider=spider) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 541368423..57dcc5f2c 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -11,8 +11,9 @@ from weakref import WeakKeyDictionary from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url -from scrapy.http import Request +from scrapy import Request, Spider from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode @@ -106,3 +107,27 @@ def referer_str(request: Request) -> Optional[str]: if referrer is None: return referrer return to_unicode(referrer, errors='replace') + + +def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: + """Create a :class:`~scrapy.Request` object from a dict. + + If a spider is given, it will try to resolve the callbacks looking at the + spider for methods with the same name. + """ + request_cls = load_object(d["_class"]) if "_class" in d else Request + kwargs = {key: value for key, value in d.items() if key in request_cls.attributes} + if d.get("callback") and spider: + kwargs["callback"] = _get_method(spider, d["callback"]) + if d.get("errback") and spider: + kwargs["errback"] = _get_method(spider, d["errback"]) + return request_cls(**kwargs) + + +def _get_method(obj, name): + """Helper function for request_from_dict""" + name = str(name) + try: + return getattr(obj, name) + except AttributeError: + raise ValueError(f"Method {name!r} not found in: {obj}") diff --git a/tests/test_utils_reqser.py b/tests/test_request_dict.py similarity index 68% rename from tests/test_utils_reqser.py rename to tests/test_request_dict.py index ee68cf6b1..5bdcb975b 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_request_dict.py @@ -1,8 +1,16 @@ +import sys import unittest +import warnings +from contextlib import suppress -from scrapy.http import Request, FormRequest -from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy import Spider, Request +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.http import FormRequest, JsonRequest +from scrapy.utils.request import request_from_dict + + +class CustomRequest(Request): + pass class RequestSerializationTest(unittest.TestCase): @@ -27,7 +35,8 @@ class RequestSerializationTest(unittest.TestCase): priority=20, meta={'a': 'b'}, cb_kwargs={'k': 'v'}, - flags=['testFlag']) + flags=['testFlag'], + ) self._assert_serializes_ok(r, spider=self.spider) def test_latin1_body(self): @@ -39,7 +48,7 @@ class RequestSerializationTest(unittest.TestCase): self._assert_serializes_ok(r) def _assert_serializes_ok(self, request, spider=None): - d = request_to_dict(request, spider=spider) + d = request.to_dict(spider=spider) request2 = request_from_dict(d, spider=spider) self._assert_same_request(request, request2) @@ -54,16 +63,21 @@ class RequestSerializationTest(unittest.TestCase): self.assertEqual(r1.cookies, r2.cookies) self.assertEqual(r1.meta, r2.meta) self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) + self.assertEqual(r1.encoding, r2.encoding) self.assertEqual(r1._encoding, r2._encoding) self.assertEqual(r1.priority, r2.priority) self.assertEqual(r1.dont_filter, r2.dont_filter) self.assertEqual(r1.flags, r2.flags) + if isinstance(r1, JsonRequest): + self.assertEqual(r1.dumps_kwargs, r2.dumps_kwargs) def test_request_class(self): - r = FormRequest("http://www.example.com") - self._assert_serializes_ok(r, spider=self.spider) - r = CustomRequest("http://www.example.com") - self._assert_serializes_ok(r, spider=self.spider) + r1 = FormRequest("http://www.example.com") + self._assert_serializes_ok(r1, spider=self.spider) + r2 = CustomRequest("http://www.example.com") + self._assert_serializes_ok(r2, spider=self.spider) + r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4}) + self._assert_serializes_ok(r3, spider=self.spider) def test_callback_serialization(self): r = Request("http://www.example.com", callback=self.spider.parse_item, @@ -75,7 +89,7 @@ class RequestSerializationTest(unittest.TestCase): callback=self.spider.parse_item_reference, errback=self.spider.handle_error_reference) self._assert_serializes_ok(r, spider=self.spider) - request_dict = request_to_dict(r, self.spider) + request_dict = r.to_dict(spider=self.spider) self.assertEqual(request_dict['callback'], 'parse_item_reference') self.assertEqual(request_dict['errback'], 'handle_error_reference') @@ -84,7 +98,7 @@ class RequestSerializationTest(unittest.TestCase): callback=self.spider._TestSpider__parse_item_reference, errback=self.spider._TestSpider__handle_error_reference) self._assert_serializes_ok(r, spider=self.spider) - request_dict = request_to_dict(r, self.spider) + request_dict = r.to_dict(spider=self.spider) self.assertEqual(request_dict['callback'], '_TestSpider__parse_item_reference') self.assertEqual(request_dict['errback'], @@ -110,18 +124,16 @@ class RequestSerializationTest(unittest.TestCase): def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) - self.assertRaises(ValueError, request_to_dict, r) - self.assertRaises(ValueError, request_to_dict, r, spider=self.spider) + self.assertRaises(ValueError, r.to_dict, spider=self.spider) def test_unserializable_callback2(self): r = Request("http://www.example.com", callback=self.spider.parse_item) - self.assertRaises(ValueError, request_to_dict, r) + self.assertRaises(ValueError, r.to_dict, spider=None) def test_unserializable_callback3(self): """Parser method is removed or replaced dynamically.""" class MySpider(Spider): - name = 'my_spider' def parse(self, response): @@ -130,7 +142,35 @@ class RequestSerializationTest(unittest.TestCase): spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) setattr(spider, 'parse', None) - self.assertRaises(ValueError, request_to_dict, r, spider=spider) + self.assertRaises(ValueError, r.to_dict, spider=spider) + + def test_callback_not_available(self): + """Callback method is not available in the spider passed to from_dict""" + spider = TestSpiderDelegation() + r = Request("http://www.example.com", callback=spider.delegated_callback) + d = r.to_dict(spider=spider) + self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo")) + + +class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest): + def _assert_serializes_ok(self, request, spider=None): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with suppress(KeyError): + del sys.modules["scrapy.utils.reqser"] # delete module to reset the deprecation warning + + from scrapy.utils.reqser import request_from_dict as _from_dict, request_to_dict as _to_dict + + request_copy = _from_dict(_to_dict(request, spider), spider) + self._assert_same_request(request, request_copy) + + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) + self.assertEqual( + "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" + " and/or scrapy.utils.request.request_from_dict instead", + str(caught[0].message), + ) class TestSpiderMixin: @@ -177,7 +217,3 @@ class TestSpider(Spider, TestSpiderMixin): def __parse_item_private(self, response): pass - - -class CustomRequest(Request): - pass From bd60c3f41fd25e7b5a413cf5c112166b813c2691 Mon Sep 17 00:00:00 2001 From: Shinichi Takayanagi Date: Tue, 11 May 2021 04:58:04 +0900 Subject: [PATCH 0309/2083] More documentation for setting spider atributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: require sphinx-rtd-theme>=0.5.2 and the latest pip to prevent installing breaking docutils>=0.17 * Update feed-exports.rst * Update feed-exports.rst * Reflects the comments * Remove redundant newline * Update docs/topics/feed-exports.rst Co-authored-by: Adrián Chaves * Apply suggestions from code review Co-authored-by: Adrián Chaves Co-authored-by: Adrián Chaves Co-authored-by: Eugenio Lacuesta --- docs/topics/feed-exports.rst | 3 +++ docs/topics/spiders.rst | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index e772a461c..26c247cdd 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -135,6 +135,9 @@ Here are some examples to illustrate: - ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` +.. note:: :ref:`Spider arguments ` become spider attributes, hence + they can also be used as storage URI parameters. + .. _topics-feed-storage-backends: diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 2056664c7..a3e9f410f 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -294,6 +294,14 @@ The above example can also be written as follows:: def start_requests(self): yield scrapy.Request(f'http://www.example.com/categories/{self.category}') +If you are :ref:`running Scrapy from a script `, you can +specify spider arguments when calling +:class:`CrawlerProcess.crawl ` or +:class:`CrawlerRunner.crawl `:: + + process = CrawlerProcess() + process.crawl(MySpider, category="electronics") + Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. If you were to set the ``start_urls`` attribute from the command line, From c5b1ee810167266fcd259f263dbfc0fe0204761a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 May 2021 09:04:53 +0200 Subject: [PATCH 0310/2083] Make Twisted[http2] installation optional (#5113) Co-authored-by: Eugenio Lacuesta --- conftest.py | 9 +++++++++ docs/topics/settings.rst | 14 ++++++++----- setup.py | 3 +-- tests/test_downloader_handlers_http2.py | 26 ++++++++++++++++++++----- tests/test_http2_client_protocol.py | 13 ++++++++----- tox.ini | 11 +++++------ 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/conftest.py b/conftest.py index e4dd80de0..4931c5a79 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,7 @@ from pathlib import Path import pytest +from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor @@ -25,6 +26,14 @@ for line in open('tests/ignores.txt'): if file_path and file_path[0] != '#': collect_ignore.append(file_path) +if not H2_ENABLED: + collect_ignore.extend( + ( + 'scrapy/core/downloader/handlers/http2.py', + *_py_files("scrapy/core/http2"), + ) + ) + @pytest.fixture() def chdir(tmpdir): diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2506497e2..0b290598f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -680,12 +680,16 @@ handler (without replacement), place this in your ``settings.py``:: .. _http2: -The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update -:setting:`DOWNLOAD_HANDLERS` as follows:: +The default HTTPS handler uses HTTP/1.1. To use HTTP/2: - DOWNLOAD_HANDLERS = { - 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', - } +#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to + enable HTTP/2 support in Twisted. + +#. Update :setting:`DOWNLOAD_HANDLERS` as follows:: + + DOWNLOAD_HANDLERS = { + 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', + } .. warning:: diff --git a/setup.py b/setup.py index b1bb64575..ed2b6e347 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support(): install_requires = [ - 'Twisted[http2]>=17.9.0', + 'Twisted>=17.9.0', 'cryptography>=2.0', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', @@ -31,7 +31,6 @@ install_requires = [ 'zope.interface>=4.1.3', 'protego>=0.1.15', 'itemadapter>=0.1.0', - 'h2>=3.0,<4.0', 'setuptools', ] extras_require = {} diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 439778014..53bb4fe92 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -1,5 +1,5 @@ import json -from unittest import mock +from unittest import mock, skipIf from pytest import mark from testfixtures import LogCapture @@ -7,8 +7,8 @@ from twisted.internet import defer, error, reactor from twisted.trial import unittest from twisted.web import server from twisted.web.error import SchemeNotSupported +from twisted.web.http import H2_ENABLED -from scrapy.core.downloader.handlers.http2 import H2DownloadHandler from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.misc import create_instance @@ -21,11 +21,17 @@ from tests.test_downloader_handlers import ( ) +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2TestCase(Https11TestCase): + scheme = 'https' - download_handler_cls = H2DownloadHandler HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + @classmethod + def setUpClass(cls): + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler + def test_protocol(self): request = Request(self.getURL("host"), method="GET") d = self.download_request(request, Spider("foo")) @@ -187,9 +193,14 @@ class Https2InvalidDNSPattern(Https2TestCase): super(Https2InvalidDNSPattern, self).setUp() +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2CustomCiphers(Https11CustomCiphers): scheme = 'https' - download_handler_cls = H2DownloadHandler + + @classmethod + def setUpClass(cls): + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler class Http2MockServerTestCase(Http11MockServerTestCase): @@ -201,6 +212,7 @@ class Http2MockServerTestCase(Http11MockServerTestCase): } +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2ProxyTestCase(Http11ProxyTestCase): # only used for HTTPS tests keyfile = 'keys/localhost.key' @@ -209,9 +221,13 @@ class Https2ProxyTestCase(Http11ProxyTestCase): scheme = 'https' host = u'127.0.0.1' - download_handler_cls = H2DownloadHandler expected_http_proxy_request_body = b'/' + @classmethod + def setUpClass(cls): + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler + def setUp(self): site = server.Site(UriResource(), timeout=None) self.port = reactor.listenSSL( diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 8b2f6a11d..677ede92b 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,10 +5,9 @@ import re import shutil import string from ipaddress import IPv4Address -from unittest import mock +from unittest import mock, skipIf from urllib.parse import urlencode -from h2.exceptions import InvalidBodyLengthError from twisted.internet import reactor from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint @@ -17,12 +16,10 @@ from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certif from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.client import ResponseFailed, URI -from twisted.web.http import Request as TxRequest +from twisted.web.http import H2_ENABLED, Request as TxRequest from twisted.web.server import Site, NOT_DONE_YET from twisted.web.static import File -from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol -from scrapy.core.http2.stream import InactiveStreamClosed, InvalidHostname from scrapy.http import Request, Response, JsonRequest from scrapy.settings import Settings from scrapy.spiders import Spider @@ -173,6 +170,7 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: return PrivateCertificate.loadPEM(pem) +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2ClientProtocolTestCase(TestCase): scheme = 'https' key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key') @@ -220,6 +218,7 @@ class Https2ClientProtocolTestCase(TestCase): uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) self.conn_closed_deferred = Deferred() + from scrapy.core.http2.protocol import H2ClientFactory h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory) @@ -426,6 +425,7 @@ class Https2ClientProtocolTestCase(TestCase): def assert_failure(failure: Failure): self.assertTrue(len(failure.value.reasons) > 0) + from h2.exceptions import InvalidBodyLengthError self.assertTrue(any( isinstance(error, InvalidBodyLengthError) for error in failure.value.reasons @@ -511,6 +511,7 @@ class Https2ClientProtocolTestCase(TestCase): def assert_inactive_stream(failure): self.assertIsNotNone(failure.check(ResponseFailed)) + from scrapy.core.http2.stream import InactiveStreamClosed self.assertTrue(any( isinstance(e, InactiveStreamClosed) for e in failure.value.reasons @@ -596,6 +597,7 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(url) def assert_invalid_hostname(failure: Failure): + from scrapy.core.http2.stream import InvalidHostname self.assertIsNotNone(failure.check(InvalidHostname)) error_msg = str(failure.value) self.assertIn('localhost', error_msg) @@ -633,6 +635,7 @@ class Https2ClientProtocolTestCase(TestCase): def assert_timeout_error(failure: Failure): for err in failure.value.reasons: + from scrapy.core.http2.protocol import H2ClientProtocol if isinstance(err, TimeoutError): self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err)) break diff --git a/tox.ini b/tox.ini index 5b0606f8f..8167aff96 100644 --- a/tox.ini +++ b/tox.ini @@ -50,6 +50,8 @@ commands = basepython = python3 deps = {[testenv]deps} + # Twisted[http2] is required to import some files + Twisted[http2]>=17.9.0 pytest-flake8 commands = py.test --flake8 {posargs:docs scrapy tests} @@ -57,12 +59,7 @@ commands = [testenv:pylint] basepython = python3 deps = - {[testenv]deps} - # Optional dependencies - boto - reppy - robotexclusionrulesparser - # Test dependencies + {[testenv:extra-deps]deps} pylint commands = pylint conftest.py docs extras scrapy setup.py tests @@ -119,9 +116,11 @@ setenv = [testenv:extra-deps] deps = {[testenv]deps} + boto reppy robotexclusionrulesparser Pillow>=4.0.0 + Twisted[http2]>=17.9.0 [testenv:asyncio] commands = From ee682af3b06d48815dbdaa27c1177b94aaf679e1 Mon Sep 17 00:00:00 2001 From: Bhavesh <35660861+Bhavesh0327@users.noreply.github.com> Date: Wed, 12 May 2021 01:53:02 +0530 Subject: [PATCH 0311/2083] [Fix] Change the truncation limit of Proxy TunnelError from 32 to 1000 (#5007) * [Fix] Change the truncation limit oof Proxy TunnelError from 32 to 64 * [Fix] Change the truncation limit for Proxy tunnel error * [Fix] flake8 check * [Fix] formatting issues * [Remove] coverage report * [Fix] truncation error issue * [Fix] formatting issues * [Remove] coverage report --- scrapy/core/downloader/handlers/http11.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 25cb3ec62..073f35891 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -98,8 +98,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): with this endpoint comes from the pool and a CONNECT has already been issued for it. """ - - _responseMatcher = re.compile(br'HTTP/1\.. (?P\d{3})(?P.{,32})') + _truncatedLength = 1000 + _responseAnswer = r'HTTP/1\.. (?P\d{3})(?P.{,' + str(_truncatedLength) + r'})' + _responseMatcher = re.compile(_responseAnswer.encode()) def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf @@ -144,7 +145,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): extra = {'status': int(respm.group('status')), 'reason': respm.group('reason').strip()} else: - extra = rcvd_bytes[:32] + extra = rcvd_bytes[:self._truncatedLength] self._tunnelReadyDeferred.errback( TunnelError('Could not open CONNECT tunnel with proxy ' f'{self._host}:{self._port} [{extra!r}]') From 52d0df5f989903d46e6de4878e1d6a0e87a2c803 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 12 May 2021 13:08:08 -0300 Subject: [PATCH 0312/2083] CaseInsensitiveDict (deprecate CaselessDict) --- scrapy/http/headers.py | 13 ++++--- scrapy/pipelines/files.py | 4 +- scrapy/utils/datatypes.py | 46 +++++++++++++++++++++++ tests/test_utils_datatypes.py | 71 +++++++++++++++++++++++++---------- 4 files changed, 107 insertions(+), 27 deletions(-) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 1a2b99b0a..dfbcf8361 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,5 +1,6 @@ from w3lib.http import headers_dict_to_raw -from scrapy.utils.datatypes import CaselessDict + +from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict from scrapy.utils.python import to_unicode @@ -76,13 +77,13 @@ class Headers(CaselessDict): return headers_dict_to_raw(self) def to_unicode_dict(self): - """ Return headers as a CaselessDict with unicode keys + """ Return headers as a CaseInsensitiveDict with unicode keys and unicode values. Multiple values are joined with ','. """ - return CaselessDict( - (to_unicode(key, encoding=self.encoding), - to_unicode(b','.join(value), encoding=self.encoding)) - for key, value in self.items()) + return CaseInsensitiveDict( + (to_unicode(key, encoding=self.encoding), to_unicode(b','.join(value), encoding=self.encoding)) + for key, value in self.items() + ) def __copy__(self): return self.__class__(self) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 13ecd4e6c..2f1a25dfc 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -23,7 +23,7 @@ from scrapy.http import Request from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import md5sum @@ -143,7 +143,7 @@ class S3FilesStore: """ Convert headers to botocore keyword agruments. """ # This is required while we need to support both boto and botocore. - mapping = CaselessDict({ + mapping = CaseInsensitiveDict({ 'Content-Type': 'ContentType', 'Cache-Control': 'CacheControl', 'Content-Disposition': 'ContentDisposition', diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index e31284a7f..ca6089e0f 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -6,14 +6,30 @@ This module must not depend on any module outside the Standard Library. """ import collections +import warnings import weakref from collections.abc import Mapping +from typing import Any, AnyStr + +from scrapy.exceptions import ScrapyDeprecationWarning class CaselessDict(dict): __slots__ = () + def __new__(cls, *args, **kwargs): + from scrapy.http.headers import Headers + + if issubclass(cls, CaselessDict) and not issubclass(cls, Headers): + warnings.warn( + "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return super().__new__(cls, *args, **kwargs) + def __init__(self, seq=None): super().__init__() if seq: @@ -63,6 +79,36 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) +class CaseInsensitiveDict(collections.UserDict): + """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups. + + It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods. + """ + + def __getitem__(self, key: AnyStr) -> Any: + return super().__getitem__(self.normkey(key)) + + def __setitem__(self, key: AnyStr, value: Any) -> None: + super().__setitem__(self.normkey(key), self.normvalue(value)) + + def __delitem__(self, key: AnyStr) -> None: + super().__delitem__(self.normkey(key)) + + def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] + return super().__contains__(self.normkey(key)) + + def normkey(self, key: AnyStr) -> AnyStr: + """Method to normalize dictionary key access""" + return key.lower() + + def normvalue(self, value: Any) -> Any: + """Method to normalize values prior to be set""" + return value + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}: {super().__repr__()}>" + + class LocalCache(collections.OrderedDict): """Dictionary with a finite number of keys. diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index e4bccf30e..c033cd537 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,26 +1,34 @@ import copy import unittest +import warnings from collections.abc import Mapping, MutableMapping +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request -from scrapy.utils.datatypes import CaselessDict, LocalCache, LocalWeakReferencedCache, SequenceExclude +from scrapy.utils.datatypes import ( + CaseInsensitiveDict, + CaselessDict, + LocalCache, + LocalWeakReferencedCache, + SequenceExclude, +) from scrapy.utils.python import garbage_collect __doctests__ = ['scrapy.utils.datatypes'] -class CaselessDictTest(unittest.TestCase): +class CaseInsensitiveDictMixin: def test_init_dict(self): seq = {'red': 1, 'black': 3} - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) def test_init_pair_sequence(self): seq = (('red', 1), ('black', 3)) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) @@ -39,7 +47,7 @@ class CaselessDictTest(unittest.TestCase): return len(self._d) seq = MyMapping(red=1, black=3) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) @@ -64,12 +72,12 @@ class CaselessDictTest(unittest.TestCase): return len(self._d) seq = MyMutableMapping(red=1, black=3) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) def test_caseless(self): - d = CaselessDict() + d = self.dict_class() d['key_Lower'] = 1 self.assertEqual(d['KEy_loWer'], 1) self.assertEqual(d.get('KEy_loWer'), 1) @@ -79,19 +87,19 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d.get('key_Lower'), 3) def test_delete(self): - d = CaselessDict({'key_lower': 1}) + d = self.dict_class({'key_lower': 1}) del d['key_LOWER'] self.assertRaises(KeyError, d.__getitem__, 'key_LOWER') self.assertRaises(KeyError, d.__getitem__, 'key_lower') def test_getdefault(self): - d = CaselessDict() + d = self.dict_class() self.assertEqual(d.get('c', 5), 5) d['c'] = 10 self.assertEqual(d.get('c', 5), 10) def test_setdefault(self): - d = CaselessDict({'a': 1, 'b': 2}) + d = self.dict_class({'a': 1, 'b': 2}) r = d.setdefault('A', 5) self.assertEqual(r, 1) @@ -104,15 +112,15 @@ class CaselessDictTest(unittest.TestCase): def test_fromkeys(self): keys = ('a', 'b') - d = CaselessDict.fromkeys(keys) + d = self.dict_class.fromkeys(keys) self.assertEqual(d['A'], None) self.assertEqual(d['B'], None) - d = CaselessDict.fromkeys(keys, 1) + d = self.dict_class.fromkeys(keys, 1) self.assertEqual(d['A'], 1) self.assertEqual(d['B'], 1) - instance = CaselessDict() + instance = self.dict_class() d = instance.fromkeys(keys) self.assertEqual(d['A'], None) self.assertEqual(d['B'], None) @@ -122,18 +130,19 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d['B'], 1) def test_contains(self): - d = CaselessDict() + d = self.dict_class() d['a'] = 1 assert 'a' in d + assert 'A' in d def test_pop(self): - d = CaselessDict() + d = self.dict_class() d['a'] = 1 self.assertEqual(d.pop('A'), 1) self.assertRaises(KeyError, d.pop, 'A') def test_normkey(self): - class MyDict(CaselessDict): + class MyDict(self.dict_class): def normkey(self, key): return key.title() @@ -142,7 +151,7 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(list(d.keys()), ['Key-One']) def test_normvalue(self): - class MyDict(CaselessDict): + class MyDict(self.dict_class): def normvalue(self, value): if value is not None: return value + 1 @@ -171,11 +180,35 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d.get('key'), 2) def test_copy(self): - h1 = CaselessDict({'header1': 'value'}) + h1 = self.dict_class({'header1': 'value'}) h2 = copy.copy(h1) self.assertEqual(h1, h2) self.assertEqual(h1.get('header1'), h2.get('header1')) - assert isinstance(h2, CaselessDict) + assert isinstance(h2, self.dict_class) + + +class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): + dict_class = CaseInsensitiveDict + + def test_repr(self): + d = self.dict_class({"foo": "bar"}) + self.assertEqual(repr(d), "") + + +class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): + dict_class = CaselessDict + + def test_deprecation_message(self): + with warnings.catch_warnings(record=True) as caught: + self.dict_class({"foo": "bar"}) + + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) + self.assertEqual( + "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", + str(caught[0].message), + ) class SequenceExcludeTest(unittest.TestCase): From 5bacc4822e80a78a2a287a5d1e417c0d3b7ddbdf Mon Sep 17 00:00:00 2001 From: Gustavo Bordin Date: Sun, 16 May 2021 18:40:30 -0300 Subject: [PATCH 0313/2083] changing dunder-str to dunder-repr --- scrapy/core/http2/stream.py | 6 ++---- scrapy/http/request/__init__.py | 4 +--- scrapy/http/response/__init__.py | 4 +--- scrapy/settings/__init__.py | 4 +--- scrapy/spiders/__init__.py | 4 +--- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index c2a4b702f..a3a9e5e1d 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -150,12 +150,10 @@ class Stream: self.close(StreamCloseReason.CANCELLED) self._deferred_response = Deferred(_cancel) - - def __str__(self) -> str: + + def __repr__(self): return f'Stream(id={self.stream_id!r})' - __repr__ = __str__ - @property def _log_warnsize(self) -> bool: """Checks if we have received data which exceeds the download warnsize diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index ad884feac..a1857c604 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -109,11 +109,9 @@ class Request(object_ref): def encoding(self): return self._encoding - def __str__(self): + def __repr__(self): return f"<{self.method} {self.url}>" - __repr__ = __str__ - def copy(self): """Return a copy of this Request""" return self.replace() diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 185a9bb67..eb80c5214 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -87,11 +87,9 @@ class Response(object_ref): body = property(_get_body, obsolete_setter(_set_body, 'body')) - def __str__(self): + def __repr__(self): return f"<{self.status} {self.url}>" - __repr__ = __str__ - def copy(self): """Return a copy of this Response""" return self.replace() diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 1fe1e6fd1..69d0476eb 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -51,11 +51,9 @@ class SettingsAttribute: self.value = value self.priority = priority - def __str__(self): + def __repr__(self): return f"" - __repr__ = __str__ - class BaseSettings(MutableMapping): """ diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index c13ba4b3c..1c079ff27 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -106,11 +106,9 @@ class Spider(object_ref): if callable(closed): return closed(reason) - def __str__(self): + def __repr__(self): return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>" - __repr__ = __str__ - # Top-level imports from scrapy.spiders.crawl import CrawlSpider, Rule From 09a07c9b4ad02959e7416c25984fa715df7bd71f Mon Sep 17 00:00:00 2001 From: Gustavo Bordin Date: Tue, 18 May 2021 19:07:28 -0300 Subject: [PATCH 0314/2083] removed whitespaces --- scrapy/core/http2/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a3a9e5e1d..780191505 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -150,7 +150,7 @@ class Stream: self.close(StreamCloseReason.CANCELLED) self._deferred_response = Deferred(_cancel) - + def __repr__(self): return f'Stream(id={self.stream_id!r})' From 23cfdb058e80a18ee2b66e1b966355f3aca426d0 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 28 May 2021 09:45:06 +0000 Subject: [PATCH 0315/2083] Reducing amount of warnings during test run (#5162) * put flake8 options into separate file to remove pytest warnings * remove ResourceLeaked warning in pypy * suppress warnings from twisted * ignore deprecation warnings here * ignore deprecation warning in tests of deprecated methods * ignore deprecation warnings here * update test classes * don`t use deprecated method call * ignore deprecation warnings here * proper warning class * more selective ignoring * Revert "don`t use deprecated method call" This reverts commit 59216ab5603c4b47574382768614ef4c39d36747. --- .flake8 | 19 +++++++++++++++++++ .gitignore | 2 ++ conftest.py | 9 +++++---- pytest.ini | 19 ++----------------- tests/test_exporters.py | 12 ++++++++---- tests/test_feedexport.py | 4 ++-- tests/test_http_response.py | 12 ++++++++---- tests/test_item.py | 22 ++++++++++++---------- tests/test_utils_deprecate.py | 2 +- tests/test_utils_python.py | 9 +++++++-- 10 files changed, 66 insertions(+), 44 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..1c503fb0b --- /dev/null +++ b/.flake8 @@ -0,0 +1,19 @@ +[flake8] + +max-line-length = 119 +ignore = W503 + +exclude = +# Exclude files that are meant to provide top-level imports +# E402: Module level import not at top of file +# F401: Module imported but unused + scrapy/__init__.py E402 + scrapy/core/downloader/handlers/http.py F401 + scrapy/http/__init__.py F401 + scrapy/linkextractors/__init__.py E402 F401 + scrapy/selector/__init__.py F401 + scrapy/spiders/__init__.py E402 F401 + + # Issues pending a review: + scrapy/utils/url.py F403 F405 + tests/test_loader.py E741 diff --git a/.gitignore b/.gitignore index 795e2605e..d77d24624 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ htmlcov/ .coverage .pytest_cache/ .coverage.* +coverage.* +test-output.* .cache/ .mypy_cache/ /tests/keys/localhost.crt diff --git a/conftest.py b/conftest.py index 4931c5a79..05b4ccdad 100644 --- a/conftest.py +++ b/conftest.py @@ -21,10 +21,11 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -for line in open('tests/ignores.txt'): - file_path = line.strip() - if file_path and file_path[0] != '#': - collect_ignore.append(file_path) +with open('tests/ignores.txt') as reader: + for line in reader: + file_path = line.strip() + if file_path and file_path[0] != '#': + collect_ignore.append(file_path) if not H2_ENABLED: collect_ignore.extend( diff --git a/pytest.ini b/pytest.ini index 0aae09ff5..6de08c78d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,20 +20,5 @@ addopts = --ignore=docs/utils markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed -flake8-max-line-length = 119 -flake8-ignore = - W503 - - # Exclude files that are meant to provide top-level imports - # E402: Module level import not at top of file - # F401: Module imported but unused - scrapy/__init__.py E402 - scrapy/core/downloader/handlers/http.py F401 - scrapy/http/__init__.py F401 - scrapy/linkextractors/__init__.py E402 F401 - scrapy/selector/__init__.py F401 - scrapy/spiders/__init__.py E402 F401 - - # Issues pending a review: - scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 +filterwarnings= + ignore::DeprecationWarning:twisted.web.test.test_webclient diff --git a/tests/test_exporters.py b/tests/test_exporters.py index ebc477e74..04bae31d3 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -6,12 +6,14 @@ import tempfile import unittest from io import BytesIO from datetime import datetime +from warnings import catch_warnings, filterwarnings import lxml.etree from itemadapter import ItemAdapter from scrapy.item import Item, Field from scrapy.utils.python import to_unicode +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exporters import ( BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, @@ -172,10 +174,12 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_binary(self): - exporter = PythonItemExporter(binary=True) - value = self.item_class(name='John\xa3', age='22') - expected = {b'name': b'John\xc2\xa3', b'age': b'22'} - self.assertEqual(expected, exporter.export_item(value)) + with catch_warnings(): + filterwarnings('ignore', category=ScrapyDeprecationWarning) + exporter = PythonItemExporter(binary=True) + value = self.item_class(name='John\xa3', age='22') + expected = {b'name': b'John\xc2\xa3', b'age': b'22'} + self.assertEqual(expected, exporter.export_item(value)) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d248824fc..df7ec4461 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -515,7 +515,7 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): class DummyBlockingFeedStorage(BlockingFeedStorage): - def __init__(self, uri): + def __init__(self, uri, *args, feed_options=None): self.path = file_uri_to_path(uri) def _store_in_thread(self, file): @@ -541,7 +541,7 @@ class LogOnStoreFileStorage: It can be used to make sure `store` method is invoked. """ - def __init__(self, uri): + def __init__(self, uri, feed_options=None): self.path = file_uri_to_path(uri) self.logger = getLogger() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index f831ef5dc..04a594d03 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,6 +1,6 @@ import unittest from unittest import mock -from warnings import catch_warnings +from warnings import catch_warnings, filterwarnings from w3lib.encoding import resolve_encoding @@ -134,7 +134,9 @@ class BaseResponseTest(unittest.TestCase): assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) - self.assertEqual(response.body_as_unicode(), body_unicode) + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + self.assertEqual(response.body_as_unicode(), body_unicode) self.assertEqual(response.text, body_unicode) def _assert_response_encoding(self, response, encoding): @@ -345,8 +347,10 @@ class TextResponseTest(BaseResponseTest): r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') # check body_as_unicode - self.assertTrue(isinstance(r1.body_as_unicode(), str)) - self.assertEqual(r1.body_as_unicode(), unicode_string) + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + self.assertTrue(isinstance(r1.body_as_unicode(), str)) + self.assertEqual(r1.body_as_unicode(), unicode_string) # check response.text self.assertTrue(isinstance(r1.text, str)) diff --git a/tests/test_item.py b/tests/test_item.py index 78d204e34..c94bb44af 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,6 +1,6 @@ import unittest from unittest import mock -from warnings import catch_warnings +from warnings import catch_warnings, filterwarnings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta @@ -328,16 +328,18 @@ class BaseItemTest(unittest.TestCase): class SubclassedItem(Item): pass - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem)) - self.assertTrue(isinstance(Item(), BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), BaseItem)) + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + self.assertTrue(isinstance(BaseItem(), BaseItem)) + self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem)) + self.assertTrue(isinstance(Item(), BaseItem)) + self.assertTrue(isinstance(SubclassedItem(), BaseItem)) - # make sure internal checks using private _BaseItem class succeed - self.assertTrue(isinstance(BaseItem(), _BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), _BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), _BaseItem)) + # make sure internal checks using private _BaseItem class succeed + self.assertTrue(isinstance(BaseItem(), _BaseItem)) + self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem)) + self.assertTrue(isinstance(Item(), _BaseItem)) + self.assertTrue(isinstance(SubclassedItem(), _BaseItem)) def test_deprecation_warning(self): """ diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 35d35b45d..e47afa266 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -108,7 +108,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): # ignore subclassing warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', ScrapyDeprecationWarning) + warnings.simplefilter('ignore', MyWarning) class UserClass(Deprecated): pass diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3115cc92f..4b3964154 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -5,8 +5,9 @@ import platform import unittest from datetime import datetime from itertools import count -from warnings import catch_warnings +from warnings import catch_warnings, filterwarnings +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, WeakKeyCache, get_func_args, to_bytes, to_unicode, @@ -160,7 +161,11 @@ class UtilsPythonTestCase(unittest.TestCase): pass _values = count() - wk = WeakKeyCache(lambda k: next(_values)) + + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + wk = WeakKeyCache(lambda k: next(_values)) + k = _Weakme() v = wk[k] self.assertEqual(v, wk[k]) From 216dd37953112e360348e19eaf8cae45a52fe87a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 11:16:40 -0300 Subject: [PATCH 0316/2083] Type hints for FormRequest --- scrapy/http/request/form.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ef2eb3ba6..4465f40ae 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,6 +5,7 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +from typing import Optional, Type, TypeVar from urllib.parse import urljoin, urlencode import lxml.html @@ -12,15 +13,18 @@ from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request +from scrapy.http.response.text import TextResponse from scrapy.utils.python import to_bytes, is_listlike from scrapy.utils.response import get_base_url +FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") + + class FormRequest(Request): valid_form_methods = ['GET', 'POST'] - def __init__(self, *args, **kwargs): - formdata = kwargs.pop('formdata', None) + def __init__(self, *args, formdata: Optional[dict] = None, **kwargs) -> None: if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' @@ -36,9 +40,19 @@ class FormRequest(Request): self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr) @classmethod - def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, - clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs): - + def from_response( + cls: Type[FormRequestTypeVar], + response: TextResponse, + formname: Optional[str] = None, + formid: Optional[str] = None, + formnumber: Optional[int] = 0, + formdata: Optional[dict] = None, + clickdata: Optional[dict] = None, + dont_click: bool = False, + formxpath: Optional[str] = None, + formcss: Optional[str] = None, + **kwargs, + ) -> FormRequestTypeVar: kwargs.setdefault('encoding', response.encoding) if formcss is not None: From c594017e518af31edc24619bfc590907ba6280c7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 11:27:21 -0300 Subject: [PATCH 0317/2083] Type hints for private functions used by FormRequest --- scrapy/http/request/form.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 4465f40ae..781e3495a 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst from typing import Optional, Type, TypeVar from urllib.parse import urljoin, urlencode -import lxml.html +from lxml.html import HTMLParser, FormElement from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace @@ -72,7 +72,7 @@ class FormRequest(Request): return cls(url=url, method=method, formdata=formdata, **kwargs) -def _get_form_url(form, url): +def _get_form_url(form: FormElement, url: Optional[str]) -> str: if url is None: action = form.get('action') if action is None: @@ -88,10 +88,15 @@ def _urlencode(seq, enc): return urlencode(values, doseq=True) -def _get_form(response, formname, formid, formnumber, formxpath): - """Find the form element """ - root = create_root_node(response.text, lxml.html.HTMLParser, - base_url=get_base_url(response)) +def _get_form( + response: TextResponse, + formname: Optional[str], + formid: Optional[str], + formnumber: Optional[int], + formxpath: Optional[str], +) -> FormElement: + """Find the wanted form element within the given response.""" + root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: raise ValueError(f"No
element found in {response}") @@ -119,8 +124,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): break raise ValueError(f'No element found with {formxpath}') - # If we get here, it means that either formname was None - # or invalid + # If we get here, it means that either formname was None or invalid if formnumber is not None: try: form = forms[formnumber] From 85f88a5710e51a3137ded72f5fdb1c3465480355 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 12:02:16 -0300 Subject: [PATCH 0318/2083] More type hints for private functions used by FormRequest --- scrapy/http/request/form.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 781e3495a..1ad878c03 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,7 +5,7 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import Optional, Type, TypeVar +from typing import List, Optional, Tuple, Type, TypeVar, Union from urllib.parse import urljoin, urlencode from lxml.html import HTMLParser, FormElement @@ -20,11 +20,13 @@ from scrapy.utils.response import get_base_url FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") +FormdataType = Optional[Union[dict, List[Tuple[str, str]]]] + class FormRequest(Request): valid_form_methods = ['GET', 'POST'] - def __init__(self, *args, formdata: Optional[dict] = None, **kwargs) -> None: + def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None: if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' @@ -46,7 +48,7 @@ class FormRequest(Request): formname: Optional[str] = None, formid: Optional[str] = None, formnumber: Optional[int] = 0, - formdata: Optional[dict] = None, + formdata: FormdataType = None, clickdata: Optional[dict] = None, dont_click: bool = False, formxpath: Optional[str] = None, @@ -60,7 +62,7 @@ class FormRequest(Request): formxpath = HTMLTranslator().css_to_xpath(formcss) form = _get_form(response, formname, formid, formnumber, formxpath) - formdata = _get_inputs(form, formdata, dont_click, clickdata, response) + formdata = _get_inputs(form, formdata, dont_click, clickdata) url = _get_form_url(form, kwargs.pop('url', None)) method = kwargs.pop('method', form.method) @@ -134,22 +136,27 @@ def _get_form( return form -def _get_inputs(form, formdata, dont_click, clickdata, response): +def _get_inputs( + form: FormElement, + formdata: FormdataType, + dont_click: bool, + clickdata: Optional[dict], +) -> List[Tuple[str, str]]: + """Return a list of key-value pairs for the inputs found in the given form.""" try: formdata_keys = dict(formdata or ()).keys() except (ValueError, TypeError): raise ValueError('formdata should be a dict or iterable of tuples') if not formdata: - formdata = () + formdata = [] inputs = form.xpath('descendant::textarea' '|descendant::select' '|descendant::input[not(@type) or @type[' ' not(re:test(., "^(?:submit|image|reset)$", "i"))' ' and (../@checked or' ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', - namespaces={ - "re": "http://exslt.org/regular-expressions"}) + namespaces={"re": "http://exslt.org/regular-expressions"}) values = [(k, '' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys] @@ -160,7 +167,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): values.append(clickable) if isinstance(formdata, dict): - formdata = formdata.items() + formdata = formdata.items() # type: ignore[assignment] values.extend((k, v) for k, v in formdata if v is not None) return values @@ -189,7 +196,7 @@ def _select_value(ele, n, v): return n, v -def _get_clickable(clickdata, form): +def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tuple[str, str]]: """ Returns the clickable element specified in clickdata, if the latter is given. If not, it returns the first @@ -201,7 +208,7 @@ def _get_clickable(clickdata, form): namespaces={"re": "http://exslt.org/regular-expressions"} )) if not clickables: - return + return None # If we don't have clickdata, we just use the first clickable element if clickdata is None: From c9fecca010a3ddddd1145f65a78dde30b5c71a72 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 12:25:26 -0300 Subject: [PATCH 0319/2083] More type hints --- scrapy/http/request/form.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 1ad878c03..c3e112041 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,10 +5,10 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import List, Optional, Tuple, Type, TypeVar, Union +from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union from urllib.parse import urljoin, urlencode -from lxml.html import HTMLParser, FormElement +from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace @@ -83,7 +83,7 @@ def _get_form_url(form: FormElement, url: Optional[str]) -> str: return urljoin(form.base_url, url) -def _urlencode(seq, enc): +def _urlencode(seq: Iterable, enc: str) -> str: values = [(to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq for v in (vs if is_listlike(vs) else [vs])] @@ -157,9 +157,11 @@ def _get_inputs( ' and (../@checked or' ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={"re": "http://exslt.org/regular-expressions"}) - values = [(k, '' if v is None else v) - for k, v in (_value(e) for e in inputs) - if k and k not in formdata_keys] + values = [ + (k, '' if v is None else v) + for k, v in (_value(e) for e in inputs) + if k and k not in formdata_keys + ] if not dont_click: clickable = _get_clickable(clickdata, form) @@ -173,7 +175,7 @@ def _get_inputs( return values -def _value(ele): +def _value(ele: HtmlElement): n = ele.name v = ele.value if ele.tag == 'select': @@ -181,7 +183,7 @@ def _value(ele): return n, v -def _select_value(ele, n, v): +def _select_value(ele: SelectElement, n: str, v: str): multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected @@ -192,7 +194,8 @@ def _select_value(ele, n, v): # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') - v = [(o.get('value') or o.text or '').strip() for o in selected_options] + values = [(o.get('value') or o.text or '').strip() for o in selected_options] + return n, values return n, v From 479260dca012b3d03221f2e6322008448d0fb8b5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 12:52:46 -0300 Subject: [PATCH 0320/2083] Type hints for Request subclasses --- scrapy/http/request/json_request.py | 17 +++++++---------- scrapy/http/request/rpc.py | 4 ++-- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 04e80d897..dba3c3a82 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,9 +8,9 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings -from typing import Tuple +from typing import Optional, Tuple -from scrapy.http.request import Request +from scrapy.http.request import Request, RequestTypeVar from scrapy.utils.deprecate import create_deprecated_class @@ -18,8 +18,8 @@ class JsonRequest(Request): attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) - def __init__(self, *args, **kwargs): - dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) + def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None: + dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault('sort_keys', True) self._dumps_kwargs = dumps_kwargs @@ -29,10 +29,8 @@ class JsonRequest(Request): if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') - elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) - if 'method' not in kwargs: kwargs['method'] = 'POST' @@ -41,23 +39,22 @@ class JsonRequest(Request): self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') @property - def dumps_kwargs(self): + def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args, **kwargs): + def replace(self, *args, **kwargs) -> RequestTypeVar: body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') - elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) return super().replace(*args, **kwargs) - def _dumps(self, data): + def _dumps(self, data: dict) -> str: """Convert to JSON """ return json.dumps(data, **self._dumps_kwargs) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index c70912e49..06d98cea5 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -5,6 +5,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ import xmlrpc.client as xmlrpclib +from typing import Optional from scrapy.http.request import Request from scrapy.utils.python import get_func_args @@ -15,8 +16,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - def __init__(self, *args, **kwargs): - encoding = kwargs.get('encoding', None) + def __init__(self, *args, encoding: Optional[str] = None, **kwargs): if 'body' not in kwargs and 'params' in kwargs: kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) kwargs['body'] = xmlrpclib.dumps(**kw) From ce6447731a91e32faf17564d21ddc3b88e582f50 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 7 Jun 2021 13:25:04 -0300 Subject: [PATCH 0321/2083] Replace return type --- scrapy/http/request/__init__.py | 4 ++-- scrapy/http/request/json_request.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 8f00c20b7..7672dec00 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -126,10 +126,10 @@ class Request(object_ref): __repr__ = __str__ - def copy(self) -> RequestTypeVar: + def copy(self) -> "Request": return self.replace() - def replace(self, *args, **kwargs) -> RequestTypeVar: + def replace(self, *args, **kwargs) -> "Request": """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index dba3c3a82..728a2a104 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -10,7 +10,7 @@ import json import warnings from typing import Optional, Tuple -from scrapy.http.request import Request, RequestTypeVar +from scrapy.http.request import Request from scrapy.utils.deprecate import create_deprecated_class @@ -42,7 +42,7 @@ class JsonRequest(Request): def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args, **kwargs) -> RequestTypeVar: + def replace(self, *args, **kwargs) -> Request: body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None From e876d8e38780a88e4c1fc060e3ead779222be83a Mon Sep 17 00:00:00 2001 From: Veniamin Gvozdikov Date: Fri, 11 Jun 2021 09:22:04 +0300 Subject: [PATCH 0322/2083] Rename scrapy-crawlera to scrapy-zyte-smartproxy (#5074) --- scrapy/core/downloader/handlers/http11.py | 6 +++--- scrapy/core/downloader/handlers/http2.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 073f35891..50486d13c 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -283,9 +283,9 @@ class ScrapyAgent: if omitConnectTunnel: warnings.warn( "Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Zyte Smart Proxy Manager (formerly Crawlera), " - "it doesn't require this mode anymore, so you should " - "update scrapy-crawlera to 1.3.0+ and remove '?noconnect' " + "If you use Zyte Smart Proxy Manager, it doesn't require " + "this mode anymore, so you should update scrapy-crawlera " + "to scrapy-zyte-smartproxy and remove '?noconnect' " "from the Zyte Smart Proxy Manager URL.", ScrapyDeprecationWarning, ) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index e97c31e90..7bb88a193 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -72,10 +72,14 @@ class ScrapyH2Agent: proxy_host = proxy_host.decode() omit_connect_tunnel = b'noconnect' in proxy_params if omit_connect_tunnel: - warnings.warn("Using HTTPS proxies in the noconnect mode is not supported by the " - "downloader handler. If you use Crawlera, it doesn't require this " - "mode anymore, so you should update scrapy-crawlera to 1.3.0+ " - "and remove '?noconnect' from the Crawlera URL.") + warnings.warn( + "Using HTTPS proxies in the noconnect mode is not " + "supported by the downloader handler. If you use Zyte " + "Smart Proxy Manager, it doesn't require this mode " + "anymore, so you should update scrapy-crawlera to " + "scrapy-zyte-smartproxy and remove '?noconnect' from the " + "Zyte Smart Proxy Manager URL." + ) if scheme == b'https' and not omit_connect_tunnel: # ToDo From 28858574d92fd2170575bea83fc6abac6b45b63a Mon Sep 17 00:00:00 2001 From: pdt1931 <56060869+pdt1931@users.noreply.github.com> Date: Fri, 11 Jun 2021 00:49:41 -0700 Subject: [PATCH 0323/2083] Add FAQ to code of Conduct (#5177) * added to CODE_OF_CONDUCT.md to include link to FAQ about the code of conduct * added to CODE_OF_CONDUCT.md to include link to FAQ about the code of conduct --- CODE_OF_CONDUCT.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 652460383..902cd523e 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -72,3 +72,6 @@ available at [http://contributor-covenant.org/version/1/4][version]. [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq From de69d967f90bc70bc07f734b93a4e7b73f2a4aa9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 11 Jun 2021 16:12:25 +0500 Subject: [PATCH 0324/2083] Fix async spider examples. --- docs/topics/coroutines.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 6a39dcb5e..67f1a4098 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -77,14 +77,16 @@ coroutines, functions that return Deferreds and functions that return :term:`awaitable objects ` such as :class:`~asyncio.Future`. This means you can use many useful Python libraries providing such code:: - class MySpider(Spider): + class MySpiderDeferred(Spider): # ... - async def parse_with_deferred(self, response): + async def parse(self, response): additional_response = await treq.get('https://additional.url') additional_data = await treq.content(additional_response) # ... use response and additional_data to yield items and requests - async def parse_with_asyncio(self, response): + class MySpiderAsyncio(Spider): + # ... + async def parse(self, response): async with aiohttp.ClientSession() as session: async with session.get('https://additional.url') as additional_response: additional_data = await additional_response.text() From 66e200423943a921326334a37777720c706ee2b5 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 14 Jun 2021 18:58:35 +0500 Subject: [PATCH 0325/2083] Fix a flake8 problem --- tests/test_http_request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index ea32c6711..b610087bd 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -381,7 +381,8 @@ class FormRequestTest(RequestTest): def test_formdata_overrides_querystring(self): data = (('a', 'one'), ('a', 'two'), ('b', '2')) - url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', method='GET', formdata=data).url.split('#')[0] + url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', + method='GET', formdata=data).url.split('#')[0] fs = _qs(self.request_class(url, method='GET', formdata=data)) self.assertEqual(set(fs[b'a']), {b'one', b'two'}) self.assertEqual(fs[b'b'], [b'2']) From 5044549c550876cc31fb2e15aaa5013e8fc333c3 Mon Sep 17 00:00:00 2001 From: Ajay Mittur Date: Mon, 14 Jun 2021 14:58:19 +0000 Subject: [PATCH 0326/2083] Update proxyScheme assignment --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 15de8cdbd..d2f9084fc 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -319,7 +319,7 @@ class ScrapyAgent: pool=self._pool, ) else: - proxyScheme = b'http' if not proxyScheme else proxyScheme + proxyScheme = proxyScheme or b'http' proxyHost = to_bytes(proxyHost, encoding='ascii') proxyPort = to_bytes(str(proxyPort), encoding='ascii') proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) From 7d653288e3d9369e4c007d223327f919b99b7737 Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Mon, 14 Jun 2021 21:39:18 +0530 Subject: [PATCH 0327/2083] Update unittest --- tests/test_downloader_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 67224ed53..9c11820e5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -768,7 +768,7 @@ class Http11ProxyTestCase(HttpProxyTestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'http://example.com') + self.assertEqual(response.body, self.expected_http_proxy_request_body) http_proxy = self.getURL('').replace('http://', '') request = Request('http://example.com', meta={'proxy': http_proxy}) From 812b4bb51855605a36bcfb166f66a121cc67f97c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Wed, 23 Jun 2021 17:09:28 +0100 Subject: [PATCH 0328/2083] CloseSpider can be raised on spider_idle signal handler --- docs/topics/signals.rst | 3 +++ scrapy/core/engine.py | 16 +++++++++++----- scrapy/utils/signal.py | 10 ++++------ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3d838fb63..ee9d95836 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -268,6 +268,9 @@ spider_idle You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to prevent the spider from being closed. + Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider` + exception to provide a custom spider closing reason. + This signal does not support returning deferreds from its handlers. :param spider: the spider which has gone idle diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index dd3225082..949d89e76 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -15,7 +15,8 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper -from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning +from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning, \ + CloseSpider from scrapy.http import Response, Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider @@ -325,14 +326,19 @@ class ExecutionEngine: Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider exception, the spider is not closed until the next loop and this function is guaranteed to be called - (at least) once again. + (at least) once again. A handler can raise CloseSpider to provide a custom closing reason """ assert self.spider is not None # typing - res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider) - if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res): + expected_ex = (DontCloseSpider, CloseSpider) + res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex) + detected_ex = {ex: x.value + for _, x in res for ex in expected_ex + if isinstance(x, Failure) and isinstance(x.value, ex)} + if DontCloseSpider in detected_ex: return None if self.spider_is_idle(): - self.close_spider(self.spider, reason='finished') + reason = detected_ex[CloseSpider].reason if CloseSpider in detected_ex else 'finished' + self.close_spider(self.spider, reason=reason) def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: """Close (cancel) spider and clear all its outstanding requests""" diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 115707182..62808f3ce 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,5 +1,5 @@ """Helper functions for working with signals""" - +import collections import logging from twisted.internet.defer import DeferredList, Deferred @@ -16,15 +16,13 @@ from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) -class _IgnoredException(Exception): - pass - - def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. """ - dont_log = (named.pop('dont_log', _IgnoredException), StopDownload) + dont_log = named.pop('dont_log', ()) + dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,) + dont_log += (StopDownload, ) spider = named.get('spider', None) responses = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): From ce445f20462463e49bd9dfd417aa7397d4c17f1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Thu, 24 Jun 2021 09:56:05 +0100 Subject: [PATCH 0329/2083] Fix typing --- scrapy/core/engine.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 949d89e76..81a7a5068 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -337,8 +337,9 @@ class ExecutionEngine: if DontCloseSpider in detected_ex: return None if self.spider_is_idle(): - reason = detected_ex[CloseSpider].reason if CloseSpider in detected_ex else 'finished' - self.close_spider(self.spider, reason=reason) + ex = detected_ex.get(CloseSpider, CloseSpider(reason='finished')) + assert isinstance(ex, CloseSpider) # typing + self.close_spider(self.spider, reason=ex.reason) def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: """Close (cancel) spider and clear all its outstanding requests""" From 73ff9ffd64742bb4d64db376851b52cfc3658fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 26 Jun 2021 08:58:29 +0200 Subject: [PATCH 0330/2083] spiders.rst: indent warnings into class descriptions --- docs/topics/spiders.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index a3e9f410f..903fbd383 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -422,10 +422,9 @@ Crawling rules It receives a :class:`Twisted Failure ` instance as first parameter. - -.. warning:: Because of its internal implementation, you must explicitly set - callbacks for new requests when writing :class:`CrawlSpider`-based spiders; - unexpected behaviour can occur otherwise. + .. warning:: Because of its internal implementation, you must explicitly set + callbacks for new requests when writing :class:`CrawlSpider`-based spiders; + unexpected behaviour can occur otherwise. .. versionadded:: 2.0 The *errback* parameter. @@ -557,10 +556,9 @@ XMLFeedSpider item IDs. It receives a list of results and the response which originated those results. It must return a list of results (items or requests). - -.. warning:: Because of its internal implementation, you must explicitly set - callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders; - unexpected behaviour can occur otherwise. + .. warning:: Because of its internal implementation, you must explicitly set + callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders; + unexpected behaviour can occur otherwise. XMLFeedSpider example From f35970778b93033ee36379e0b8b55035aacaf918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Tue, 29 Jun 2021 13:21:38 +0100 Subject: [PATCH 0331/2083] Test case for raising CloseSpider on spider idle signal handler --- tests/test_engine.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index c200ded90..dc24f50fa 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -26,7 +26,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import ScrapyDeprecationWarning, CloseSpider from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor @@ -113,6 +113,18 @@ class ItemZeroDivisionErrorSpider(TestSpider): } +class ChangeCloseReasonSpider(TestSpider): + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(*args, **kwargs) + spider._set_crawler(crawler) + crawler.signals.connect(spider.spider_idle, signals.spider_idle) + return spider + + def spider_idle(self): + raise CloseSpider(reason="custom_reason") + + def start_test_site(debug=False): root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) @@ -251,6 +263,13 @@ class EngineTest(unittest.TestCase): yield self.run.run() self._assert_items_error() + @defer.inlineCallbacks + def test_crawler_change_close_reason_on_idle(self): + self.run = CrawlerRun(ChangeCloseReasonSpider) + yield self.run.run() + self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'}, + self.run.signals_caught[signals.spider_closed]) + def _assert_visited_urls(self): must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"] From e94d3ac173d2fcd269bdd8f06b6b1633953a225e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Tue, 29 Jun 2021 13:40:43 +0100 Subject: [PATCH 0332/2083] Expanded doc for idle signal --- docs/topics/signals.rst | 6 +++++- scrapy/core/engine.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index ee9d95836..530af1e37 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -269,7 +269,11 @@ spider_idle prevent the spider from being closed. Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider` - exception to provide a custom spider closing reason. + exception to provide a custom spider closing reason. An + idle handler is the perfect place to put some code that assesses + the final spider results and update the final closing reason + accordingly (e.g. setting it to 'too_few_results' instead of + 'finished'). This signal does not support returning deferreds from its handlers. diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 81a7a5068..0b34d213d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -326,7 +326,7 @@ class ExecutionEngine: Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider exception, the spider is not closed until the next loop and this function is guaranteed to be called - (at least) once again. A handler can raise CloseSpider to provide a custom closing reason + (at least) once again. A handler can raise CloseSpider to provide a custom closing reason. """ assert self.spider is not None # typing expected_ex = (DontCloseSpider, CloseSpider) From 7597d860c86e0ddfaf8e3ce57492106c907245f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Thu, 8 Jul 2021 12:39:17 +0100 Subject: [PATCH 0333/2083] Update scrapy/core/engine.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/core/engine.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 0b34d213d..b7f91d744 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -15,8 +15,11 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper -from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning, \ - CloseSpider +from scrapy.exceptions import ( + CloseSpider, + DontCloseSpider, + ScrapyDeprecationWarning, +) from scrapy.http import Response, Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider From 6b8f694653b0eec95055d50fafde0ef2e0ec8bc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Thu, 8 Jul 2021 12:40:02 +0100 Subject: [PATCH 0334/2083] Update scrapy/core/engine.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/core/engine.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b7f91d744..b04fedce6 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -334,9 +334,12 @@ class ExecutionEngine: assert self.spider is not None # typing expected_ex = (DontCloseSpider, CloseSpider) res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex) - detected_ex = {ex: x.value - for _, x in res for ex in expected_ex - if isinstance(x, Failure) and isinstance(x.value, ex)} + detected_ex = { + ex: x.value + for _, x in res + for ex in expected_ex + if isinstance(x, Failure) and isinstance(x.value, ex) + } if DontCloseSpider in detected_ex: return None if self.spider_is_idle(): From eca641aa3da335781c373e3d50deb78837044c16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Thu, 8 Jul 2021 12:40:20 +0100 Subject: [PATCH 0335/2083] Update tests/test_engine.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- tests/test_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index dc24f50fa..92bf45f25 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -26,7 +26,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import ScrapyDeprecationWarning, CloseSpider +from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor From cb08e3644b081febecd90eec9391367a2d13ee5b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 8 Jul 2021 09:22:21 -0300 Subject: [PATCH 0336/2083] Remove trailing whitespaces --- scrapy/core/engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b04fedce6..f9de7ee23 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -17,9 +17,9 @@ from scrapy import signals from scrapy.core.scraper import Scraper from scrapy.exceptions import ( CloseSpider, - DontCloseSpider, + DontCloseSpider, ScrapyDeprecationWarning, -) +) from scrapy.http import Response, Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider From c062ed017a89e40c41140ff2782043776b26165b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 12 Jul 2021 13:34:22 -0300 Subject: [PATCH 0337/2083] [CI] fail-fast: false (#5200) --- .github/workflows/checks.yml | 1 + .github/workflows/tests-macos.yml | 1 + .github/workflows/tests-ubuntu.yml | 1 + .github/workflows/tests-windows.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 02c647da9..e7080db9a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -5,6 +5,7 @@ jobs: checks: runs-on: ubuntu-18.04 strategy: + fail-fast: false matrix: include: - python-version: 3.9 diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 4f8f7a19d..095ca1013 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -5,6 +5,7 @@ jobs: tests: runs-on: macos-10.15 strategy: + fail-fast: false matrix: python-version: [3.6, 3.7, 3.8, 3.9] diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index df5ee9d69..b42e8b127 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -5,6 +5,7 @@ jobs: tests: runs-on: ubuntu-18.04 strategy: + fail-fast: false matrix: include: - python-version: 3.7 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 5459a845b..30fda33e8 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -5,6 +5,7 @@ jobs: tests: runs-on: windows-latest strategy: + fail-fast: false matrix: include: - python-version: 3.6 From 4ddc9d6b55fa708becfd70812ea44eb0c6837638 Mon Sep 17 00:00:00 2001 From: D R Siddhartha Date: Tue, 13 Jul 2021 20:52:29 +0530 Subject: [PATCH 0338/2083] Feeds: Item Filters (#5178) --- docs/topics/feed-exports.rst | 54 +++++++++++++++ scrapy/extensions/feedexport.py | 48 ++++++++++++- tests/test_feedexport.py | 118 ++++++++++++++++++++++++++++++-- 3 files changed, 214 insertions(+), 6 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 26c247cdd..7a4b054e9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -268,6 +268,45 @@ in multiple files, with the specified maximum item count per file. That way, as soon as a file reaches the maximum item count, that file is delivered to the feed URI, allowing item delivery to start way before the end of the crawl. +.. _item-filter: + +Item filtering +============== + +.. versionadded:: VERSION + +You can filter items that you want to allow for a particular feed by using the +``item_classes`` option in :ref:`feeds options `. Only items of +the specified types will be added to the feed. + +The ``item_classes`` option is implemented by the :class:`~scrapy.extensions.feedexport.ItemFilter` +class, which is the default value of the ``item_filter`` :ref:`feed option `. + +You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s +method ``accepts`` and taking ``feed_options`` as an argument. + +For instance:: + + class MyCustomFilter: + + def __init__(self, feed_options): + self.feed_options = feed_options + + def accepts(self, item): + if "field1" in item and item["field1"] == "expected_data": + return True + return False + + +You can assign your custom filtering class to the ``item_filter`` :ref:`option of a feed `. +See :setting:`FEEDS` for examples. + +ItemFilter +---------- + +.. autoclass:: scrapy.extensions.feedexport.ItemFilter + :members: + Settings ======== @@ -311,6 +350,7 @@ For instance:: 'format': 'json', 'encoding': 'utf8', 'store_empty': False, + 'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'], 'fields': None, 'indent': 4, 'item_export_kwargs': { @@ -320,12 +360,14 @@ For instance:: '/home/user/documents/items.xml': { 'format': 'xml', 'fields': ['name', 'price'], + 'item_filter': MyCustomFilter1, 'encoding': 'latin1', 'indent': 8, }, pathlib.Path('items.csv'): { 'format': 'csv', 'fields': ['price', 'name'], + 'item_filter': 'myproject.filters.MyCustomFilter2', }, } @@ -347,6 +389,18 @@ as a fallback value if that key is not provided for a specific feed definition: - ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. +- ``item_classes``: list of :ref:`item classes ` to export. + + If undefined or empty, all items are exported. + + .. versionadded:: VERSION + +- ``item_filter``: a :ref:`filter class ` to filter items to export. + + :class:`~scrapy.extensions.feedexport.ItemFilter` is used be default. + + .. versionadded:: VERSION + - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. - ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index bec114707..89dca12f4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -46,6 +46,38 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): return builder(*preargs, uri, *args, **kwargs) +class ItemFilter: + """ + This will be used by FeedExporter to decide if an item should be allowed + to be exported to a particular feed. + + :param feed_options: feed specific options passed from FeedExporter + :type feed_options: dict + """ + + def __init__(self, feed_options): + self.feed_options = feed_options + self.item_classes = set() + + if 'item_classes' in self.feed_options: + for item_class in self.feed_options['item_classes']: + self.item_classes.add(load_object(item_class)) + + def accepts(self, item): + """ + Return ``True`` if `item` should be exported or ``False`` otherwise. + + :param item: scraped item which user wants to check if is acceptable + :type item: :ref:`Scrapy items ` + :return: `True` if accepted, `False` otherwise + :rtype: bool + """ + if self.item_classes: + return isinstance(item, tuple(self.item_classes)) + + return True # accept all items if none declared in item_classes + + class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" @@ -215,7 +247,7 @@ class FTPFeedStorage(BlockingFeedStorage): class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template): + def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template, filter): self.file = file self.exporter = exporter self.storage = storage @@ -225,6 +257,7 @@ class _FeedSlot: self.store_empty = store_empty self.uri_template = uri_template self.uri = uri + self.filter = filter # flags self.itemcount = 0 self._exporting = False @@ -255,6 +288,7 @@ class FeedExporter: self.settings = crawler.settings self.feeds = {} self.slots = [] + self.filters = {} if not self.settings['FEEDS'] and not self.settings['FEED_URI']: raise NotConfigured @@ -269,12 +303,14 @@ class FeedExporter: uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) + self.filters[uri] = self._load_filter(feed_options) # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' for uri, feed_options in self.settings.getdict('FEEDS').items(): uri = str(uri) # handle pathlib.Path objects self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) + self.filters[uri] = self._load_filter(feed_options) self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') @@ -368,6 +404,7 @@ class FeedExporter: store_empty=feed_options['store_empty'], batch_id=batch_id, uri_template=uri_template, + filter=self.filters[uri_template] ) if slot.store_empty: slot.start_exporting() @@ -376,6 +413,10 @@ class FeedExporter: def item_scraped(self, item, spider): slots = [] for slot in self.slots: + if not slot.filter.accepts(item): + slots.append(slot) # if slot doesn't accept item, continue with next slot + continue + slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 @@ -486,3 +527,8 @@ class FeedExporter: uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) return params + + def _load_filter(self, feed_options): + # load the item filter if declared else load the default filter class + item_filter_class = load_object(feed_options.get("item_filter", ItemFilter)) + return item_filter_class(feed_options) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index df7ec4461..81437b011 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -561,6 +561,10 @@ class FeedExportTestBase(ABC, unittest.TestCase): egg = scrapy.Field() baz = scrapy.Field() + class MyItem2(scrapy.Item): + foo = scrapy.Field() + hello = scrapy.Field() + def _random_temp_filename(self, inter_dir=''): chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) @@ -888,13 +892,9 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_item_classes(self): - class MyItem2(scrapy.Item): - foo = scrapy.Field() - hello = scrapy.Field() - items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), - MyItem2({'hello': 'world2', 'foo': 'bar2'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}), {'hello': 'world4', 'egg': 'spam4'}, ] @@ -929,6 +929,114 @@ class FeedExportTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings, ordered=True) + @defer.inlineCallbacks + def test_export_based_on_item_classes(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), + {'hello': 'world3', 'egg': 'spam3'}, + ] + + formats = { + 'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n', + 'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]', + 'jsonlines': ( + b'{"foo": "bar1", "egg": "spam1"}\n' + b'{"hello": "world2", "foo": "bar2"}\n' + ), + 'xml': ( + b'\n\n' + b'bar1spam1\n' + b'world2bar2\nworld3' + b'spam3\n' + ), + } + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'csv', + 'item_classes': [self.MyItem], + }, + self._random_temp_filename(): { + 'format': 'json', + 'item_classes': [self.MyItem2], + }, + self._random_temp_filename(): { + 'format': 'jsonlines', + 'item_classes': [self.MyItem, self.MyItem2], + }, + self._random_temp_filename(): { + 'format': 'xml', + }, + }, + } + + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + self.assertEqual(expected, data[fmt]) + + @defer.inlineCallbacks + def test_export_based_on_custom_filters(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), + {'hello': 'world3', 'egg': 'spam3'}, + ] + + MyItem = self.MyItem + + class CustomFilter1: + def __init__(self, feed_options): + pass + + def accepts(self, item): + return isinstance(item, MyItem) + + class CustomFilter2(scrapy.extensions.feedexport.ItemFilter): + def accepts(self, item): + if 'foo' not in item.fields: + return False + return True + + class CustomFilter3(scrapy.extensions.feedexport.ItemFilter): + def accepts(self, item): + if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1": + return True + return False + + formats = { + 'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]', + 'xml': ( + b'\n\n' + b'bar1spam1\n' + b'world2bar2\n' + ), + 'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n', + } + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'json', + 'item_filter': CustomFilter1, + }, + self._random_temp_filename(): { + 'format': 'xml', + 'item_filter': CustomFilter2, + }, + self._random_temp_filename(): { + 'format': 'jsonlines', + 'item_classes': [self.MyItem, self.MyItem2], + 'item_filter': CustomFilter3, + }, + }, + } + + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + self.assertEqual(expected, data[fmt]) + @defer.inlineCallbacks def test_export_dicts(self): # When dicts are used, only keys from the first row are used as From fcc6becc586e6f895cdeed66a579d5735d215de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=BCrkalp=20Burak=20KAYRANCIO=C4=9ELU?= Date: Wed, 14 Jul 2021 11:00:43 +0300 Subject: [PATCH 0339/2083] S3FeedStorage: allow custom endpoint (#4998) Co-authored-by: Andrey Rahmatullin --- docs/topics/feed-exports.rst | 3 ++- scrapy/extensions/feedexport.py | 7 ++++-- tests/test_feedexport.py | 44 +++++++++++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7a4b054e9..af7fce852 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -201,9 +201,10 @@ passed through the following settings: - :setting:`AWS_ACCESS_KEY_ID` - :setting:`AWS_SECRET_ACCESS_KEY` -You can also define a custom ACL for exported feeds using this setting: +You can also define a custom ACL and custom endpoint for exported feeds using this setting: - :setting:`FEED_STORAGE_S3_ACL` +- :setting:`AWS_ENDPOINT_URL` This storage backend uses :ref:`delayed file delivery `. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 89dca12f4..84a79e32d 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -150,7 +150,7 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None, acl=None, *, + def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *, feed_options=None): if not is_botocore_available(): raise NotConfigured('missing botocore library') @@ -160,11 +160,13 @@ class S3FeedStorage(BlockingFeedStorage): self.secret_key = u.password or secret_key self.keyname = u.path[1:] # remove first "/" self.acl = acl + self.endpoint_url = endpoint_url import botocore.session session = botocore.session.get_session() self.s3_client = session.create_client( 's3', aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key) + aws_secret_access_key=self.secret_key, + endpoint_url=self.endpoint_url) if feed_options and feed_options.get('overwrite', True) is False: logger.warning('S3 does not support appending to files. To ' 'suppress this warning, remove the overwrite ' @@ -178,6 +180,7 @@ class S3FeedStorage(BlockingFeedStorage): access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, + endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None, feed_options=feed_options, ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 81437b011..da0b2c786 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -326,6 +326,17 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + def test_init_with_endpoint_url(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + endpoint_url='https://example.com' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, 'https://example.com') + def test_from_crawler_without_acl(self): settings = { 'AWS_ACCESS_KEY_ID': 'access_key', @@ -340,6 +351,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) + def test_without_endpoint_url(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, None) + def test_from_crawler_with_acl(self): settings = { 'AWS_ACCESS_KEY_ID': 'access_key', @@ -355,6 +380,21 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + def test_from_crawler_with_endpoint_url(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'AWS_ENDPOINT_URL': 'https://example.com', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, 'https://example.com') + @defer.inlineCallbacks def test_store_botocore_without_acl(self): skip_if_no_boto() @@ -1917,8 +1957,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): class S3FeedStorageWithoutFeedOptions(S3FeedStorage): - def __init__(self, uri, access_key, secret_key, acl): - super().__init__(uri, access_key, secret_key, acl) + def __init__(self, uri, access_key, secret_key, acl, endpoint_url): + super().__init__(uri, access_key, secret_key, acl, endpoint_url) class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): From d7deba7e89242774ae712d87eb7bb331759a731f Mon Sep 17 00:00:00 2001 From: Marlena Chatzigrigoriou <56519084+marlenachatzigrigoriou@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:34:28 +0300 Subject: [PATCH 0340/2083] Document all import paths and use the shortest in examples (#5099) --- docs/faq.rst | 4 +- docs/intro/tutorial.rst | 32 +++++++-------- docs/topics/api.rst | 4 +- docs/topics/contracts.rst | 4 +- docs/topics/coroutines.rst | 2 +- docs/topics/debug.rst | 2 +- docs/topics/developer-tools.rst | 2 +- docs/topics/downloader-middleware.rst | 58 +++++++++++++-------------- docs/topics/dynamic-content.rst | 14 +++---- docs/topics/exporters.rst | 4 +- docs/topics/feed-exports.rst | 4 +- docs/topics/item-pipeline.rst | 6 +-- docs/topics/items.rst | 10 +++-- docs/topics/jobs.rst | 4 +- docs/topics/leaks.rst | 16 ++++---- docs/topics/logging.rst | 2 +- docs/topics/media-pipeline.rst | 6 +-- docs/topics/request-response.rst | 16 ++++---- docs/topics/selectors.rst | 8 ++-- docs/topics/settings.rst | 16 ++++---- docs/topics/shell.rst | 8 ++-- docs/topics/signals.rst | 54 ++++++++++++------------- docs/topics/spider-middleware.rst | 26 ++++++------ docs/topics/spiders.rst | 43 ++++++++++---------- 24 files changed, 175 insertions(+), 170 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 9709885f6..16903daea 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,7 +149,7 @@ How can I prevent memory errors due to many allowed domains? ------------------------------------------------------------ If you have a spider with a long list of -:attr:`~scrapy.spiders.Spider.allowed_domains` (e.g. 50,000+), consider +:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider replacing the default :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware with a :ref:`custom spider middleware ` that requires @@ -157,7 +157,7 @@ less memory. For example: - If your domain names are similar enough, use your own regular expression instead joining the strings in - :attr:`~scrapy.spiders.Spider.allowed_domains` into a complex regular + :attr:`~scrapy.Spider.allowed_domains` into a complex regular expression. - If you can `meet the installation requirements`_, use pyre2_ instead of diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 740e47d0c..438f3d6df 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -78,7 +78,7 @@ Our first Spider Spiders are classes that you define and that Scrapy uses to scrape information from a website (or a group of websites). They must subclass -:class:`~scrapy.spiders.Spider` and define the initial requests to make, +:class:`~scrapy.Spider` and define the initial requests to make, optionally how to follow links in the pages, and how to parse the downloaded page content to extract data. @@ -107,26 +107,26 @@ This is the code for our first Spider. Save it in a file named self.log(f'Saved file {filename}') -As you can see, our Spider subclasses :class:`scrapy.Spider ` +As you can see, our Spider subclasses :class:`scrapy.Spider ` and defines some attributes and methods: -* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be +* :attr:`~scrapy.Spider.name`: identifies the Spider. It must be unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of +* :meth:`~scrapy.Spider.start_requests`: must return an iterable of Requests (you can return a list of requests or write a generator function) which the Spider will begin to crawl from. Subsequent requests will be generated successively from these initial requests. -* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle +* :meth:`~scrapy.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter is an instance of :class:`~scrapy.http.TextResponse` that holds the page content and has further helpful methods to handle it. - The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting + The :meth:`~scrapy.Spider.parse` method usually parses the response, extracting the scraped data as dicts and also finding new URLs to - follow and creating new requests (:class:`~scrapy.http.Request`) from them. + follow and creating new requests (:class:`~scrapy.Request`) from them. How to run our spider --------------------- @@ -162,7 +162,7 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy schedules the :class:`scrapy.Request ` objects +Scrapy schedules the :class:`scrapy.Request ` objects returned by the ``start_requests`` method of the Spider. Upon receiving a response for each one, it instantiates :class:`~scrapy.http.Response` objects and calls the callback method associated with the request (in this case, the @@ -171,11 +171,11 @@ and calls the callback method associated with the request (in this case, the A shortcut to the start_requests method --------------------------------------- -Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method -that generates :class:`scrapy.Request ` objects from URLs, -you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute +Instead of implementing a :meth:`~scrapy.Spider.start_requests` method +that generates :class:`scrapy.Request ` objects from URLs, +you can just define a :attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This list will then be used by the default implementation -of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests +of :meth:`~scrapy.Spider.start_requests` to create the initial requests for your spider:: import scrapy @@ -194,9 +194,9 @@ for your spider:: with open(filename, 'wb') as f: f.write(response.body) -The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each +The :meth:`~scrapy.Spider.parse` method will be called to handle each of the requests for those URLs, even though we haven't explicitly told Scrapy -to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's +to do so. This happens because :meth:`~scrapy.Spider.parse` is Scrapy's default callback method, which is called for requests without an explicitly assigned callback. @@ -248,7 +248,7 @@ object: The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of -:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements +:class:`~scrapy.Selector` objects that wrap around XML/HTML elements and allow you to run further queries to fine-grain the selection or extract the data. @@ -670,7 +670,7 @@ the pagination links with the ``parse`` callback as we saw before. Here we're passing callbacks to :meth:`response.follow_all ` as positional arguments to make the code shorter; it also works for -:class:`~scrapy.http.Request`. +:class:`~scrapy.Request`. The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 445b2979f..900b19c7a 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -29,7 +29,7 @@ how you :ref:`configure the downloader middlewares .. class:: Crawler(spidercls, settings) The Crawler object must be instantiated with a - :class:`scrapy.spiders.Spider` subclass and a + :class:`scrapy.Spider` subclass and a :class:`scrapy.settings.Settings` object. .. attribute:: settings @@ -196,7 +196,7 @@ SpiderLoader API match the request's url against the domains of the spiders. :param request: queried request - :type request: :class:`~scrapy.http.Request` instance + :type request: :class:`~scrapy.Request` instance .. _topics-api-signals: diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index e61421bf1..ef296dc9e 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -37,7 +37,7 @@ This callback is tested using three built-in contracts: .. class:: CallbackKeywordArgumentsContract - This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` + This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` attribute for the sample request. It must be a valid JSON dictionary. :: @@ -88,7 +88,7 @@ override three methods: .. method:: Contract.adjust_request_args(args) This receives a ``dict`` as an argument containing default arguments - for request object. :class:`~scrapy.http.Request` is used by default, + for request object. :class:`~scrapy.Request` is used by default, but this can be changed with the ``request_cls`` attribute. If multiple contracts in chain have this attribute defined, the last one is used. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 3b1549bd3..0904637b0 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -15,7 +15,7 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): -- :class:`~scrapy.http.Request` callbacks. +- :class:`~scrapy.Request` callbacks. .. note:: The callback output is not processed until the whole callback finishes. diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index d75f17301..4d452b4df 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -36,7 +36,7 @@ Consider the following Scrapy spider below:: Basically this is a simple spider which parses two pages of items (the start_urls). Items also have a details page with additional information, so we -use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a +use the ``cb_kwargs`` functionality of :class:`~scrapy.Request` to pass a partially populated item. diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index c83b1a9d9..057b1ec62 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -274,7 +274,7 @@ In more complex websites, it could be difficult to easily reproduce the requests, as we could need to add ``headers`` or ``cookies`` to make it work. In those cases you can export the requests in `cURL `_ format, by right-clicking on each of them in the network tool and using the -:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent +:meth:`~scrapy.Request.from_curl()` method to generate an equivalent request:: from scrapy import Request diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index b539c23df..80c6c2c37 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -76,7 +76,7 @@ object gives you access, for example, to the :ref:`settings `. middleware. :meth:`process_request` should either: return ``None``, return a - :class:`~scrapy.http.Response` object, return a :class:`~scrapy.http.Request` + :class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request` object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`. If it returns ``None``, Scrapy will continue processing this request, executing all @@ -88,7 +88,7 @@ object gives you access, for example, to the :ref:`settings `. or the appropriate download function; it'll return that response. The :meth:`process_response` methods of installed middleware is always called on every response. - If it returns a :class:`~scrapy.http.Request` object, Scrapy will stop calling + If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling process_request methods and reschedule the returned request. Once the newly returned request is performed, the appropriate middleware chain will be called on the downloaded response. @@ -100,22 +100,22 @@ object gives you access, for example, to the :ref:`settings `. ignored and not logged (unlike other exceptions). :param request: the request being processed - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_response(request, response, spider) :meth:`process_response` should either: return a :class:`~scrapy.http.Response` - object, return a :class:`~scrapy.http.Request` object or + object, return a :class:`~scrapy.Request` object or raise a :exc:`~scrapy.exceptions.IgnoreRequest` exception. If it returns a :class:`~scrapy.http.Response` (it could be the same given response, or a brand-new one), that response will continue to be processed with the :meth:`process_response` of the next middleware in the chain. - If it returns a :class:`~scrapy.http.Request` object, the middleware chain is + If it returns a :class:`~scrapy.Request` object, the middleware chain is halted and the returned request is rescheduled to be downloaded in the future. This is the same behavior as if a request is returned from :meth:`process_request`. @@ -124,13 +124,13 @@ object gives you access, for example, to the :ref:`settings `. exception, it is ignored and not logged (unlike other exceptions). :param request: the request that originated the response - :type request: is a :class:`~scrapy.http.Request` object + :type request: is a :class:`~scrapy.Request` object :param response: the response being processed :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_exception(request, exception, spider) @@ -139,7 +139,7 @@ object gives you access, for example, to the :ref:`settings `. exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception) :meth:`process_exception` should return: either ``None``, - a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.http.Request` object. + a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object. If it returns ``None``, Scrapy will continue processing this exception, executing any other :meth:`process_exception` methods of installed middleware, @@ -149,19 +149,19 @@ object gives you access, for example, to the :ref:`settings `. method chain of installed middleware is started, and Scrapy won't bother calling any other :meth:`process_exception` methods of middleware. - If it returns a :class:`~scrapy.http.Request` object, the returned request is + If it returns a :class:`~scrapy.Request` object, the returned request is rescheduled to be downloaded in the future. This stops the execution of :meth:`process_exception` methods of the middleware the same as returning a response would. :param request: the request that generated the exception - :type request: is a :class:`~scrapy.http.Request` object + :type request: is a :class:`~scrapy.Request` object :param exception: the raised exception :type exception: an ``Exception`` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: from_crawler(cls, crawler) @@ -203,13 +203,13 @@ CookiesMiddleware browsers do. .. caution:: When non-UTF8 encoded byte sequences are passed to a - :class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log + :class:`~scrapy.Request`, the ``CookiesMiddleware`` will log a warning. Refer to :ref:`topics-logging-advanced-customization` to customize the logging behaviour. .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. The following settings can be used to configure the cookie middleware: @@ -258,7 +258,7 @@ web server and received cookies in :class:`~scrapy.http.Response` will **not** be merged with the existing cookies. For more detailed information see the ``cookies`` parameter in -:class:`~scrapy.http.Request`. +:class:`~scrapy.Request`. .. setting:: COOKIES_DEBUG @@ -501,7 +501,7 @@ defines the methods described below. the :signal:`open_spider ` signal. :param spider: the spider which has been opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: close_spider(spider) @@ -509,27 +509,27 @@ defines the methods described below. the :signal:`close_spider ` signal. :param spider: the spider which has been closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: retrieve_response(spider, request) Return response if present in cache, or ``None`` otherwise. :param spider: the spider which generated the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param request: the request to find cached response for - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object .. method:: store_response(spider, request, response) Store the given response in the cache. :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param request: the corresponding request the spider generated - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param response: the response to store in the cache :type response: :class:`~scrapy.http.Response` object @@ -722,7 +722,7 @@ HttpProxyMiddleware .. class:: HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the - ``proxy`` meta value for :class:`~scrapy.http.Request` objects. + ``proxy`` meta value for :class:`~scrapy.Request` objects. Like the Python standard library module :mod:`urllib.request`, it obeys the following environment variables: @@ -749,12 +749,12 @@ RedirectMiddleware .. reqmeta:: redirect_urls The urls which the request goes through (while being redirected) can be found -in the ``redirect_urls`` :attr:`Request.meta ` key. +in the ``redirect_urls`` :attr:`Request.meta ` key. .. reqmeta:: redirect_reasons The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the -``redirect_reasons`` :attr:`Request.meta ` key. For +``redirect_reasons`` :attr:`Request.meta ` key. For example: ``[301, 302, 307, 'meta refresh']``. The format of a reason depends on the middleware that handled the corresponding @@ -770,7 +770,7 @@ settings (see the settings documentation for more info): .. reqmeta:: dont_redirect -If :attr:`Request.meta ` has ``dont_redirect`` +If :attr:`Request.meta ` has ``dont_redirect`` key set to True, the request will be ignored by this middleware. If you want to handle some redirect status codes in your spider, you can @@ -783,7 +783,7 @@ responses (and pass them through to your spider) you can do this:: handle_httpstatus_list = [301, 302] The ``handle_httpstatus_list`` key of :attr:`Request.meta -` can also be used to specify which response codes to +` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` to ``True`` if you want to allow any response code for a request. @@ -889,7 +889,7 @@ settings (see the settings documentation for more info): .. reqmeta:: dont_retry -If :attr:`Request.meta ` has ``dont_retry`` key +If :attr:`Request.meta ` has ``dont_retry`` key set to True, the request will be ignored by this middleware. To retry requests from a spider callback, you can use the @@ -919,7 +919,7 @@ Default: ``2`` Maximum number of times to retry, in addition to the first download. Maximum number of retries can also be specified per-request using -:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. +:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. When initialized, the :reqmeta:`max_retry_times` meta key takes higher precedence over the :setting:`RETRY_TIMES` setting. @@ -986,7 +986,7 @@ RobotsTxtMiddleware .. reqmeta:: dont_obey_robotstxt -If :attr:`Request.meta ` has +If :attr:`Request.meta ` has ``dont_obey_robotstxt`` key set to True the request will be ignored by this middleware even if :setting:`ROBOTSTXT_OBEY` is enabled. diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 495111b56..aa32c8943 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -62,9 +62,9 @@ download the webpage with an HTTP client like curl_ or wget_ and see if the information can be found in the response they get. If they get a response with the desired data, modify your Scrapy -:class:`~scrapy.http.Request` to match that of the other HTTP client. For +:class:`~scrapy.Request` to match that of the other HTTP client. For example, try using the same user-agent string (:setting:`USER_AGENT`) or the -same :attr:`~scrapy.http.Request.headers`. +same :attr:`~scrapy.Request.headers`. If they also get a response without the desired data, you’ll need to take steps to make your request more similar to that of the web browser. See @@ -81,14 +81,14 @@ Use the :ref:`network tool ` of your web browser to see how your web browser performs the desired request, and try to reproduce that request with Scrapy. -It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP +It might be enough to yield a :class:`~scrapy.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and -form parameters (see :class:`~scrapy.http.FormRequest`) of that request. +form parameters (see :class:`~scrapy.FormRequest`) of that request. As all major browsers allow to export the requests in `cURL `_ format, Scrapy incorporates the method -:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent -:class:`~scrapy.http.Request` from a cURL command. To get more information +:meth:`~scrapy.Request.from_curl()` to generate an equivalent +:class:`~scrapy.Request` from a cURL command. To get more information visit :ref:`request from curl ` inside the network tool section. @@ -125,7 +125,7 @@ data from it depends on the type of response: If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a - :class:`~scrapy.selector.Selector` and then + :class:`~scrapy.Selector` and then :ref:`use it ` as usual:: selector = Selector(data['html']) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 8648daded..8c30122b6 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -90,7 +90,7 @@ described next. 1. Declaring a serializer in the field -------------------------------------- -If you use :class:`~.Item` you can declare a serializer in the +If you use :class:`~scrapy.Item` you can declare a serializer in the :ref:`field metadata `. The serializer must be a callable which receives a value and returns its serialized form. @@ -172,7 +172,7 @@ BaseItemExporter :param field: the field being serialized. If the source :ref:`item object ` does not define field metadata, *field* is an empty :class:`dict`. - :type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance + :type field: :class:`~scrapy.Field` object or a :class:`dict` instance :param name: the name of the field being serialized :type name: str diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index af7fce852..216a8bc52 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -674,9 +674,9 @@ The function signature should be as follows: :type params: dict :param spider: source spider of the feed items - :type spider: scrapy.spiders.Spider + :type spider: scrapy.Spider -For example, to include the :attr:`name ` of the +For example, to include the :attr:`name ` of the source spider in the feed URI: #. Define the following function somewhere in your project:: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 6287ee0ad..5351a2293 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -42,7 +42,7 @@ Each item pipeline component is a Python class that must implement the following :type item: :ref:`item object ` :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object Additionally, they may also implement the following methods: @@ -51,14 +51,14 @@ Additionally, they may also implement the following methods: This method is called when the spider is opened. :param spider: the spider which was opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: close_spider(self, spider) This method is called when the spider is closed. :param spider: the spider which was closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: from_crawler(cls, crawler) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 65bf156ac..7cd482d07 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -42,7 +42,8 @@ Item objects :class:`Item` provides a :class:`dict`-like API plus additional features that make it the most feature-complete item type: -.. class:: Item([arg]) +.. class:: scrapy.item.Item([arg]) +.. class:: scrapy.Item([arg]) :class:`Item` objects replicate the standard :class:`dict` API, including its ``__init__`` method. @@ -199,7 +200,8 @@ It's important to note that the :class:`Field` objects used to declare the item do not stay assigned as class attributes. Instead, they can be accessed through the :attr:`Item.fields` attribute. -.. class:: Field([arg]) +.. class:: scrapy.item.Field([arg]) +.. class:: scrapy.Field([arg]) The :class:`Field` class is just an alias to the built-in :class:`dict` class and doesn't provide any extra functionality or attributes. In other words, @@ -317,11 +319,11 @@ If that is not the desired behavior, use a deep copy instead. See :mod:`copy` for more information. To create a shallow copy of an item, you can either call -:meth:`~scrapy.item.Item.copy` on an existing item +:meth:`~scrapy.Item.copy` on an existing item (``product2 = product.copy()``) or instantiate your item class from an existing item (``product2 = Product(product)``). -To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead +To create a deep copy, call :meth:`~scrapy.Item.deepcopy` instead (``product2 = product.deepcopy()``). diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index d855d0133..e49f37a2f 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -74,10 +74,10 @@ on cookies. Request serialization --------------------- -For persistence to work, :class:`~scrapy.http.Request` objects must be +For persistence to work, :class:`~scrapy.Request` objects must be serializable with :mod:`pickle`, except for the ``callback`` and ``errback`` values passed to their ``__init__`` method, which must be methods of the -running :class:`~scrapy.spiders.Spider` class. +running :class:`~scrapy.Spider` class. If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index b895b95cb..477652704 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -27,7 +27,7 @@ Common causes of memory leaks It happens quite often (sometimes by accident, sometimes on purpose) that the Scrapy developer passes objects referenced in Requests (for example, using the -:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta` +:attr:`~scrapy.Request.cb_kwargs` or :attr:`~scrapy.Request.meta` attributes or the request callback function) and that effectively bounds the lifetime of those referenced objects to the lifetime of the Request. This is, by far, the most common cause of memory leaks in Scrapy projects, and a quite @@ -48,9 +48,9 @@ Too Many Requests? ------------------ By default Scrapy keeps the request queue in memory; it includes -:class:`~scrapy.http.Request` objects and all objects -referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs` -and :attr:`~scrapy.http.Request.meta`). +:class:`~scrapy.Request` objects and all objects +referenced in Request attributes (e.g. in :attr:`~scrapy.Request.cb_kwargs` +and :attr:`~scrapy.Request.meta`). While not necessarily a leak, this can take a lot of memory. Enabling :ref:`persistent job queue ` could help keeping memory usage in control. @@ -90,11 +90,11 @@ Which objects are tracked? The objects tracked by ``trackrefs`` are all from these classes (and all its subclasses): -* :class:`scrapy.http.Request` +* :class:`scrapy.Request` * :class:`scrapy.http.Response` -* :class:`scrapy.item.Item` -* :class:`scrapy.selector.Selector` -* :class:`scrapy.spiders.Spider` +* :class:`scrapy.Item` +* :class:`scrapy.Selector` +* :class:`scrapy.Spider` A real example -------------- diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 00806392a..dda04dc4d 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -93,7 +93,7 @@ path:: Logging from Spiders ==================== -Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider +Scrapy provides a :data:`~scrapy.Spider.logger` within each Spider instance, which can be accessed and used like this:: import scrapy diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 156897274..3438cb637 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -259,7 +259,7 @@ respectively), the pipeline will put the results under the respective field When using :ref:`item types ` for which fields are defined beforehand, you must define both the URLs field and the results field. For example, when using the images pipeline, items must define both the ``image_urls`` and the -``images`` field. For instance, using the :class:`~scrapy.item.Item` class:: +``images`` field. For instance, using the :class:`~scrapy.Item` class:: import scrapy @@ -424,7 +424,7 @@ See here the methods that you can override in your custom Files Pipeline: In addition to ``response``, this method receives the original :class:`request `, :class:`info ` and - :class:`item ` + :class:`item ` You can override this method to customize the download path of each file. @@ -563,7 +563,7 @@ See here the methods that you can override in your custom Images Pipeline: In addition to ``response``, this method receives the original :class:`request `, :class:`info ` and - :class:`item ` + :class:`item ` You can override this method to customize the download path of each file. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 73b5a858f..a6a3daf31 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -35,7 +35,7 @@ Request objects request (once it's downloaded) as its first parameter. For more information see :ref:`topics-request-response-ref-request-callback-arguments` below. If a Request doesn't specify a callback, the spider's - :meth:`~scrapy.spiders.Spider.parse` method will be used. + :meth:`~scrapy.Spider.parse` method will be used. Note that if exceptions are raised during processing, errback is called instead. :type callback: collections.abc.Callable @@ -60,7 +60,7 @@ Request objects .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. :type headers: dict @@ -92,7 +92,7 @@ Request objects To create a request that does not send stored cookies and does not store received cookies, set the ``dont_merge_cookies`` key to ``True`` - in :attr:`request.meta `. + in :attr:`request.meta `. Example of a request that sends manually-defined cookies and ignores cookie storage:: @@ -107,7 +107,7 @@ Request objects .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. :type cookies: dict or list @@ -495,7 +495,9 @@ fields with form data from :class:`Response` objects. .. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms -.. class:: FormRequest(url, [formdata, ...]) +.. class:: scrapy.http.request.form.FormRequest +.. class:: scrapy.http.FormRequest +.. class:: scrapy.FormRequest(url, [formdata, ...]) The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The remaining arguments are the same as for the :class:`Request` class and are @@ -694,7 +696,7 @@ Response objects :param request: the initial value of the :attr:`Response.request` attribute. This represents the :class:`Request` that generated this response. - :type request: scrapy.http.Request + :type request: scrapy.Request :param certificate: an object representing the server's SSL certificate. :type certificate: twisted.internet.ssl.Certificate @@ -920,7 +922,7 @@ TextResponse objects .. attribute:: TextResponse.selector - A :class:`~scrapy.selector.Selector` instance using the response as + A :class:`~scrapy.Selector` instance using the response as target. The selector is lazily instantiated on first access. :class:`TextResponse` objects support the following methods in addition to diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 9caba5ee5..574d4568c 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -48,7 +48,7 @@ Constructing selectors .. highlight:: python -Response objects expose a :class:`~scrapy.selector.Selector` instance +Response objects expose a :class:`~scrapy.Selector` instance on ``.selector`` attribute: >>> response.selector.xpath('//span/text()').get() @@ -62,7 +62,7 @@ more shortcuts: ``response.xpath()`` and ``response.css()``: >>> response.css('span::text').get() 'good' -Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class +Scrapy selectors are instances of :class:`~scrapy.Selector` class constructed by passing either :class:`~scrapy.http.TextResponse` object or markup as a string (in ``text`` argument). @@ -175,7 +175,7 @@ of ``None``: 'not-found' Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes -using ``.attrib`` property of a :class:`~scrapy.selector.Selector`: +using ``.attrib`` property of a :class:`~scrapy.Selector`: >>> [img.attrib['src'] for img in response.css('img')] ['image1_thumb.jpg', @@ -383,7 +383,7 @@ ID, or when selecting an unique element on a page): Using selectors with regular expressions ---------------------------------------- -:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting +:class:`~scrapy.Selector` also has a ``.re()`` method for extracting data using regular expressions. However, unlike using ``.xpath()`` or ``.css()`` methods, ``.re()`` returns a list of strings. So you can't construct nested ``.re()`` calls. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0b290598f..1290b4a5e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -67,7 +67,7 @@ Example:: Spiders (See the :ref:`topics-spiders` chapter for reference) can define their own settings that will take precedence and override the project ones. They can -do so by setting their :attr:`~scrapy.spiders.Spider.custom_settings` attribute:: +do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute:: class MySpider(scrapy.Spider): name = 'myspider' @@ -142,7 +142,7 @@ In a spider, the settings are available through ``self.settings``:: The ``settings`` attribute is set in the base Spider class after the spider is initialized. If you want to use the settings before the initialization (e.g., in your spider's ``__init__()`` method), you'll need to override the - :meth:`~scrapy.spiders.Spider.from_crawler` method. + :meth:`~scrapy.Spider.from_crawler` method. Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` attribute of the Crawler that is passed to ``from_crawler`` method in @@ -338,7 +338,7 @@ is non-zero, download delay is enforced per IP, not per domain. DEFAULT_ITEM_CLASS ------------------ -Default: ``'scrapy.item.Item'`` +Default: ``'scrapy.Item'`` The default class that will be used for instantiating items in the :ref:`the Scrapy shell `. @@ -360,7 +360,7 @@ The default headers used for Scrapy HTTP Requests. They're populated in the .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. .. setting:: DEPTH_LIMIT @@ -384,8 +384,8 @@ Default: ``0`` Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` -An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of -a :class:`~scrapy.http.Request` based on its depth. +An integer that is used to adjust the :attr:`~scrapy.Request.priority` of +a :class:`~scrapy.Request` based on its depth. The priority of a request is adjusted as follows:: @@ -816,14 +816,14 @@ The default (``RFPDupeFilter``) filters based on request fingerprint using the ``scrapy.utils.request.request_fingerprint`` function. In order to change the way duplicates are checked you could subclass ``RFPDupeFilter`` and override its ``request_fingerprint`` method. This method should accept -scrapy :class:`~scrapy.http.Request` object and return its fingerprint +scrapy :class:`~scrapy.Request` object and return its fingerprint (a string). You can disable filtering of duplicate requests by setting :setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``. Be very careful about this however, because you can get into crawling loops. It's usually a better idea to set the ``dont_filter`` parameter to -``True`` on the specific :class:`~scrapy.http.Request` that should not be +``True`` on the specific :class:`~scrapy.Request` that should not be filtered. .. setting:: DUPEFILTER_DEBUG diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index b910fc453..8c90a506c 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -118,7 +118,7 @@ Available Scrapy objects The Scrapy shell automatically creates some convenient objects from the downloaded page, like the :class:`~scrapy.http.Response` object and the -:class:`~scrapy.selector.Selector` objects (for both HTML and XML +:class:`~scrapy.Selector` objects (for both HTML and XML content). Those objects are: @@ -126,12 +126,12 @@ Those objects are: - ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object. - ``spider`` - the Spider which is known to handle the URL, or a - :class:`~scrapy.spiders.Spider` object if there is no spider found for the + :class:`~scrapy.Spider` object if there is no spider found for the current URL -- ``request`` - a :class:`~scrapy.http.Request` object of the last fetched +- ``request`` - a :class:`~scrapy.Request` object of the last fetched page. You can modify this request using - :meth:`~scrapy.http.Request.replace` or fetch a new request (without + :meth:`~scrapy.Request.replace` or fetch a new request (without leaving the shell) using the ``fetch`` shortcut. - ``response`` - a :class:`~scrapy.http.Response` object containing the last diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 530af1e37..a67cc1879 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -155,7 +155,7 @@ item_scraped :type item: :ref:`item object ` :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was scraped :type response: :class:`~scrapy.http.Response` object @@ -175,7 +175,7 @@ item_dropped :type item: :ref:`item object ` :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was dropped :type response: :class:`~scrapy.http.Response` object @@ -203,7 +203,7 @@ item_error :type response: :class:`~scrapy.http.Response` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param failure: the exception raised :type failure: twisted.python.failure.Failure @@ -223,7 +223,7 @@ spider_closed This signal supports returning deferreds from its handlers. :param spider: the spider which has been closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param reason: a string which describes the reason why the spider was closed. If it was closed because the spider has completed scraping, the reason @@ -247,7 +247,7 @@ spider_opened This signal supports returning deferreds from its handlers. :param spider: the spider which has been opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object spider_idle ~~~~~~~~~~~ @@ -278,7 +278,7 @@ spider_idle This signal does not support returning deferreds from its handlers. :param spider: the spider which has gone idle - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. note:: Scheduling some requests in your :signal:`spider_idle` handler does **not** guarantee that it can prevent the spider from being closed, @@ -303,7 +303,7 @@ spider_error :type response: :class:`~scrapy.http.Response` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object Request signals --------------- @@ -314,16 +314,16 @@ request_scheduled .. signal:: request_scheduled .. function:: request_scheduled(request, spider) - Sent when the engine schedules a :class:`~scrapy.http.Request`, to be + Sent when the engine schedules a :class:`~scrapy.Request`, to be downloaded later. This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object request_dropped ~~~~~~~~~~~~~~~ @@ -331,16 +331,16 @@ request_dropped .. signal:: request_dropped .. function:: request_dropped(request, spider) - Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be + Sent when a :class:`~scrapy.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object request_reached_downloader ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -348,15 +348,15 @@ request_reached_downloader .. signal:: request_reached_downloader .. function:: request_reached_downloader(request, spider) - Sent when a :class:`~scrapy.http.Request` reached downloader. + Sent when a :class:`~scrapy.Request` reached downloader. This signal does not support returning deferreds from its handlers. :param request: the request that reached downloader - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object request_left_downloader ~~~~~~~~~~~~~~~~~~~~~~~ @@ -366,16 +366,16 @@ request_left_downloader .. versionadded:: 2.0 - Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of + Sent when a :class:`~scrapy.Request` leaves the downloader, even in case of failure. This signal does not support returning deferreds from its handlers. :param request: the request that reached the downloader - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object bytes_received ~~~~~~~~~~~~~~ @@ -402,10 +402,10 @@ bytes_received :type data: :class:`bytes` object :param request: the request that generated the download - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider associated with the response - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object headers_received ~~~~~~~~~~~~~~~~ @@ -432,10 +432,10 @@ headers_received :type body_length: `int` :param request: the request that generated the download - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider associated with the response - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object Response signals ---------------- @@ -455,10 +455,10 @@ response_received :type response: :class:`~scrapy.http.Response` object :param request: the request that generated the response - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. note:: The ``request`` argument might not contain the original request that reached the downloader, if a :ref:`topics-downloader-middleware` modifies @@ -479,7 +479,7 @@ response_downloaded :type response: :class:`~scrapy.http.Response` object :param request: the request that generated the response - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 11bbbb58d..f0158dc41 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -93,7 +93,7 @@ object gives you access, for example, to the :ref:`settings `. :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_spider_output(response, result, spider) @@ -102,7 +102,7 @@ object gives you access, for example, to the :ref:`settings `. it has processed the response. :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.http.Request` objects and :ref:`item object + :class:`~scrapy.Request` objects and :ref:`item object `. :param response: the response which generated this output from the @@ -110,11 +110,11 @@ object gives you access, for example, to the :ref:`settings `. :type response: :class:`~scrapy.http.Response` object :param result: the result returned by the spider - :type result: an iterable of :class:`~scrapy.http.Request` objects and + :type result: an iterable of :class:`~scrapy.Request` objects and :ref:`item object ` :param spider: the spider whose result is being processed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_spider_exception(response, exception, spider) @@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Request` objects and :ref:`item object + iterable of :class:`~scrapy.Request` objects and :ref:`item object `. If it returns ``None``, Scrapy will continue processing this exception, @@ -142,7 +142,7 @@ object gives you access, for example, to the :ref:`settings `. :type exception: :exc:`Exception` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_start_requests(start_requests, spider) @@ -152,7 +152,7 @@ object gives you access, for example, to the :ref:`settings `. items). It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.http.Request` objects. + return another iterable of :class:`~scrapy.Request` objects. .. note:: When implementing this method in your spider middleware, you should always return an iterable (that follows the input one) and @@ -164,10 +164,10 @@ object gives you access, for example, to the :ref:`settings `. (like a time limit or item/page count). :param start_requests: the start requests - :type start_requests: an iterable of :class:`~scrapy.http.Request` + :type start_requests: an iterable of :class:`~scrapy.Request` :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: from_crawler(cls, crawler) @@ -251,7 +251,7 @@ this:: .. reqmeta:: handle_httpstatus_all The ``handle_httpstatus_list`` key of :attr:`Request.meta -` can also be used to specify which response codes to +` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` to ``True`` if you want to allow any response code for a request, and ``False`` to disable the effects of the ``handle_httpstatus_all`` key. @@ -295,7 +295,7 @@ OffsiteMiddleware Filters out Requests for URLs outside the domains covered by the spider. This middleware filters out every request whose host names aren't in the - spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute. + spider's :attr:`~scrapy.Spider.allowed_domains` attribute. All subdomains of any domain in the list are also allowed. E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` but not ``www2.example.com`` nor ``example.com``. @@ -313,10 +313,10 @@ OffsiteMiddleware will be printed (but only for the first request filtered). If the spider doesn't define an - :attr:`~scrapy.spiders.Spider.allowed_domains` attribute, or the + :attr:`~scrapy.Spider.allowed_domains` attribute, or the attribute is empty, the offsite middleware will allow all requests. - If the request has the :attr:`~scrapy.http.Request.dont_filter` attribute + If the request has the :attr:`~scrapy.Request.dont_filter` attribute set, the offsite middleware will allow the request even if its domain is not listed in allowed domains. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 903fbd383..67b9e2e0e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -17,15 +17,15 @@ For spiders, the scraping cycle goes through something like this: those requests. The first requests to perform are obtained by calling the - :meth:`~scrapy.spiders.Spider.start_requests` method which (by default) - generates :class:`~scrapy.http.Request` for the URLs specified in the - :attr:`~scrapy.spiders.Spider.start_urls` and the - :attr:`~scrapy.spiders.Spider.parse` method as callback function for the + :meth:`~scrapy.Spider.start_requests` method which (by default) + generates :class:`~scrapy.Request` for the URLs specified in the + :attr:`~scrapy.Spider.start_urls` and the + :attr:`~scrapy.Spider.parse` method as callback function for the Requests. 2. In the callback function, you parse the response (web page) and return :ref:`item objects `, - :class:`~scrapy.http.Request` objects, or an iterable of these objects. + :class:`~scrapy.Request` objects, or an iterable of these objects. Those Requests will also contain a callback (maybe the same) and will then be downloaded by Scrapy and then their response handled by the specified callback. @@ -50,7 +50,8 @@ We will talk about those types here. scrapy.Spider ============= -.. class:: Spider() +.. class:: scrapy.spiders.Spider() +.. class:: scrapy.Spider() This is the simplest spider, and the one from which every other spider must inherit (including spiders that come bundled with Scrapy, as well as spiders @@ -86,7 +87,7 @@ scrapy.Spider A list of URLs where the spider will begin to crawl from, when no particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data + listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data contained in the start URLs. .. attribute:: custom_settings @@ -179,7 +180,7 @@ scrapy.Spider the same requirements as the :class:`Spider` class. This method, as well as any other Request callback, must return an - iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects + iterable of :class:`~scrapy.Request` and/or :ref:`item objects `. :param response: the response to parse @@ -234,7 +235,7 @@ Return multiple Requests and items from a single callback:: yield scrapy.Request(response.urljoin(href), self.parse) Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; -to give data more structure you can use :class:`~scrapy.item.Item` objects:: +to give data more structure you can use :class:`~scrapy.Item` objects:: import scrapy from myproject.items import MyItem @@ -373,7 +374,7 @@ CrawlSpider This method is called for each response produced for the URLs in the spider's ``start_urls`` attribute. It allows to parse the initial responses and must return either an - :ref:`item object `, a :class:`~scrapy.http.Request` + :ref:`item object `, a :class:`~scrapy.Request` object, or an iterable containing any of them. Crawling rules @@ -383,7 +384,7 @@ Crawling rules ``link_extractor`` is a :ref:`Link Extractor ` object which defines how links will be extracted from each crawled page. Each produced link will - be used to generate a :class:`~scrapy.http.Request` object, which will contain the + be used to generate a :class:`~scrapy.Request` object, which will contain the link's text in its ``meta`` dictionary (under the ``link_text`` key). If omitted, a default link extractor created with no arguments will be used, resulting in all links being extracted. @@ -392,9 +393,9 @@ Crawling rules object with that name will be used) to be called for each link extracted with the specified link extractor. This callback receives a :class:`~scrapy.http.Response` as its first argument and must return either a single instance or an iterable of - :ref:`item objects ` and/or :class:`~scrapy.http.Request` objects + :ref:`item objects ` and/or :class:`~scrapy.Request` objects (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response` - object will contain the text of the link that produced the :class:`~scrapy.http.Request` + object will contain the text of the link that produced the :class:`~scrapy.Request` in its ``meta`` dictionary (under the ``link_text`` key) ``cb_kwargs`` is a dict containing the keyword arguments to be passed to the @@ -411,7 +412,7 @@ Crawling rules ``process_request`` is a callable (or a string, in which case a method from the spider object with that name will be used) which will be called for every - :class:`~scrapy.http.Request` extracted by this rule. This callable should + :class:`~scrapy.Request` extracted by this rule. This callable should take said request as first argument and the :class:`~scrapy.http.Response` from which the request originated as second argument. It must return a ``Request`` object or ``None`` (to filter out the request). @@ -470,7 +471,7 @@ Let's now take a look at an example CrawlSpider with rules:: This spider would start crawling example.com's home page, collecting category links, and item links, parsing the latter with the ``parse_item`` method. For each item response, some data will be extracted from the HTML using XPath, and -an :class:`~scrapy.item.Item` will be filled with it. +an :class:`~scrapy.Item` will be filled with it. XMLFeedSpider ------------- @@ -493,11 +494,11 @@ XMLFeedSpider - ``'iternodes'`` - a fast iterator based on regular expressions - - ``'html'`` - an iterator which uses :class:`~scrapy.selector.Selector`. + - ``'html'`` - an iterator which uses :class:`~scrapy.Selector`. Keep in mind this uses DOM parsing and must load all DOM in memory which could be a problem for big feeds - - ``'xml'`` - an iterator which uses :class:`~scrapy.selector.Selector`. + - ``'xml'`` - an iterator which uses :class:`~scrapy.Selector`. Keep in mind this uses DOM parsing and must load all DOM in memory which could be a problem for big feeds @@ -515,7 +516,7 @@ XMLFeedSpider available in that document that will be processed with this spider. The ``prefix`` and ``uri`` will be used to automatically register namespaces using the - :meth:`~scrapy.selector.Selector.register_namespace` method. + :meth:`~scrapy.Selector.register_namespace` method. You can then specify nodes with namespaces in the :attr:`itertag` attribute. @@ -542,10 +543,10 @@ XMLFeedSpider This method is called for the nodes matching the provided tag name (``itertag``). Receives the response and an - :class:`~scrapy.selector.Selector` for each node. Overriding this + :class:`~scrapy.Selector` for each node. Overriding this method is mandatory. Otherwise, you spider won't work. This method must return an :ref:`item object `, a - :class:`~scrapy.http.Request` object, or an iterable containing any of + :class:`~scrapy.Request` object, or an iterable containing any of them. .. method:: process_results(response, results) @@ -587,7 +588,7 @@ These spiders are pretty easy to use, let's have a look at one example:: Basically what we did up there was to create a spider that downloads a feed from the given ``start_urls``, and then iterates through each of its ``item`` tags, -prints them out, and stores some random data in an :class:`~scrapy.item.Item`. +prints them out, and stores some random data in an :class:`~scrapy.Item`. CSVFeedSpider ------------- From bcce0660573cf9309e3decf853488da2ef8bc576 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 14 Jul 2021 12:56:07 -0300 Subject: [PATCH 0341/2083] Update ItemFilter (#5203) --- scrapy/extensions/feedexport.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 84a79e32d..bd4808e2b 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,6 +11,7 @@ import sys import warnings from datetime import datetime from tempfile import NamedTemporaryFile +from typing import Any, Optional, Tuple from urllib.parse import unquote, urlparse from twisted.internet import defer, threads @@ -54,16 +55,19 @@ class ItemFilter: :param feed_options: feed specific options passed from FeedExporter :type feed_options: dict """ + feed_options: Optional[dict] + item_classes: Tuple - def __init__(self, feed_options): + def __init__(self, feed_options: Optional[dict]) -> None: self.feed_options = feed_options - self.item_classes = set() + if feed_options is not None: + self.item_classes = tuple( + load_object(item_class) for item_class in feed_options.get("item_classes") or () + ) + else: + self.item_classes = tuple() - if 'item_classes' in self.feed_options: - for item_class in self.feed_options['item_classes']: - self.item_classes.add(load_object(item_class)) - - def accepts(self, item): + def accepts(self, item: Any) -> bool: """ Return ``True`` if `item` should be exported or ``False`` otherwise. @@ -73,9 +77,8 @@ class ItemFilter: :rtype: bool """ if self.item_classes: - return isinstance(item, tuple(self.item_classes)) - - return True # accept all items if none declared in item_classes + return isinstance(item, self.item_classes) + return True # accept all items by default class IFeedStorage(Interface): From 89b654b82df763a59d6a37e92e796c1478c768ce Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 16 Jul 2021 15:18:14 +0500 Subject: [PATCH 0342/2083] Make the pylint test pass (#5207) Co-authored-by: Vostretsov Nikita --- .github/workflows/checks.yml | 4 ++++ .github/workflows/tests-ubuntu.yml | 4 ++++ pylintrc | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e7080db9a..6bdfcb5dc 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,6 +19,7 @@ jobs: - python-version: 3.8 env: TOXENV: pylint + TOX_PIP_VERSION: 20.3.3 - python-version: 3.9 env: TOXENV: typing @@ -37,5 +38,8 @@ jobs: - name: Run check env: ${{ matrix.env }} run: | + if [[ ! -z "$TOX_PIP_VERSION" ]]; then + pip install tox-pip-version + fi pip install -U tox tox diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index b42e8b127..521d7ae70 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -40,6 +40,7 @@ jobs: - python-version: 3.8 env: TOXENV: extra-deps + TOX_PIP_VERSION: 20.3.3 - python-version: 3.9 env: TOXENV: asyncio @@ -68,6 +69,9 @@ jobs: $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi + if [[ ! -z "$TOX_PIP_VERSION" ]]; then + pip install tox-pip-version + fi pip install -U tox tox diff --git a/pylintrc b/pylintrc index 972bf99de..a44712507 100644 --- a/pylintrc +++ b/pylintrc @@ -6,6 +6,7 @@ jobs=1 # >1 hides results disable=abstract-method, anomalous-backslash-in-string, arguments-differ, + arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, bad-continuation, @@ -21,6 +22,8 @@ disable=abstract-method, cell-var-from-loop, comparison-with-callable, consider-iterating-dictionary, + consider-using-dict-items, + consider-using-from-import, consider-using-in, consider-using-set-comprehension, consider-using-sys-exit, @@ -105,6 +108,7 @@ disable=abstract-method, unsubscriptable-object, unused-argument, unused-import, + unused-private-member, unused-variable, unused-wildcard-import, used-before-assignment, From ee2df97bbdf9120ccefc9c132bcbf0994479f948 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 16 Jul 2021 17:28:32 +0500 Subject: [PATCH 0343/2083] Pin the libxml2 version in CI as a newer one breaks lxml (#5208) --- .github/workflows/tests-ubuntu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 521d7ae70..57188bd63 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -57,7 +57,8 @@ jobs: if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update - sudo apt-get install libxml2-dev libxslt-dev + # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version + sudo apt-get install libxml2-dev/bionic-updates libxslt-dev - name: Run tests env: ${{ matrix.env }} From 7306a81188f81964ad85f4936ce29e3aa0084447 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 19 Jul 2021 20:09:11 +0500 Subject: [PATCH 0344/2083] Disable builtin middlewares in spider middleware tests. --- tests/test_spidermiddleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 0a6b96c0c..b0ca2f62e 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -19,7 +19,7 @@ class SpiderMiddlewareTestCase(TestCase): def setUp(self): self.request = Request('http://example.com/index.html') self.response = Response(self.request.url, request=self.request) - self.crawler = get_crawler(Spider) + self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}}) self.spider = self.crawler._create_spider('foo') self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) From 70dddfe2b293171db4c58511175edf55bbe1831c Mon Sep 17 00:00:00 2001 From: Pascal Corpet Date: Wed, 21 Jul 2021 17:10:10 +0200 Subject: [PATCH 0345/2083] Typing: switch to a newer version of MyPy to check types --- tests/CrawlerProcess/asyncio_deferred_signal.py | 2 ++ tox.ini | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index 46c2a12a4..bdd3c1fef 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -1,5 +1,6 @@ import asyncio import sys +from typing import Optional from scrapy import Spider from scrapy.crawler import CrawlerProcess @@ -31,6 +32,7 @@ class UrlSpider(Spider): if __name__ == "__main__": + ASYNCIO_EVENT_LOOP: Optional[str] try: ASYNCIO_EVENT_LOOP = sys.argv[1] except IndexError: diff --git a/tox.ini b/tox.ini index 8167aff96..4c4bbff6e 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,10 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==0.780 + lxml-stubs==0.2.0 + mypy==0.910 + types-pyOpenSSL==20.0.3 + types-setuptools==57.0.0 commands = mypy --show-error-codes {posargs: scrapy tests} From 209c1fce02a776b2917559c09d977827f858743a Mon Sep 17 00:00:00 2001 From: Aaron Tan <70739609+aaron-tan@users.noreply.github.com> Date: Sat, 24 Jul 2021 14:50:48 +1000 Subject: [PATCH 0346/2083] Reference MailSender in StatsMailer Added a reference to MailSender in the StatsMailer extension description and included a link to the document detailing how to instantiate MailSender and using Scrapy settings objects. --- docs/topics/extensions.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 9e86fd0fe..3cabcefdd 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -323,6 +323,15 @@ domain has finished scraping, including the Scrapy stats collected. The email will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS` setting. +Emails can be sent using the MailSender class + +.. module:: scrapy.mail + :synopsis: MailSender class + +.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) + +To see a full list of parameters, including examples on how to instantiate MailSender and using mail settings, see :ref:`topics-email` + .. module:: scrapy.extensions.debug :synopsis: Extensions for debugging Scrapy From b22a0043988a4f3c54709988de99f489db44f78d Mon Sep 17 00:00:00 2001 From: Rob Banagale Date: Mon, 26 Jul 2021 11:51:32 -0700 Subject: [PATCH 0347/2083] Document media pipeline file naming (#5152) --- docs/topics/media-pipeline.rst | 88 ++++++++++++++++++++++++++++------ 1 file changed, 74 insertions(+), 14 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 3438cb637..46bd2859b 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -111,25 +111,82 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting:: IMAGES_STORE = '/path/to/valid/dir' +.. _topics-file-naming: + +File Naming +=========== + +Default File Naming +------------------- + +By default, files are stored using an `SHA-1 hash`_ of their URLs for the file names. + +For example, the following image URL:: + + http://www.example.com/image.jpg + +Whose ``SHA-1 hash`` is:: + + 3afec3b4765f8f0a07b78f98c07b83f013567a0a + +Will be downloaded and stored using your chosen :ref:`storage method ` and the following file name:: + + 3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg + +Custom File Naming +------------------- + +You may wish to use a different calculated file name for saved files. +For example, classifying an image by including meta in the file name. + +Customize file names by overriding the ``file_path`` method of your +media pipeline. + +For example, an image pipeline with image URL:: + + http://www.example.com/product/images/large/front/0000000004166 + +Can be processed into a file name with a condensed hash and the perspective +``front``:: + + 00b08510e4_front.jpg + +By overriding ``file_path`` like this: + +.. code-block:: python + + import hashlib + from os.path import splitext + + def file_path(self, request, response=None, info=None, *, item=None): + image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5) + image_perspective = request.url.split('/')[-2] + image_filename = f'{image_url_hash}_{image_perspective}.jpg' + + return image_filename + +.. warning:: + If your custom file name scheme relies on meta data that can vary between + scrapes it may lead to unexpected re-downloading of existing media using + new file names. + + For example, if your custom file name scheme uses a product title and the + site changes an item's product title between scrapes, Scrapy will re-download + the same media using updated file names. + +For more information about the ``file_path`` method, see :ref:`topics-media-pipeline-override`. + +.. _topics-supported-storage: + Supported Storage ================= File system storage ------------------- -The files are stored using a `SHA1 hash`_ of their URLs for the file names. +File system storage will save files to the following path:: -For example, the following image URL:: - - http://www.example.com/image.jpg - -Whose ``SHA1 hash`` is:: - - 3afec3b4765f8f0a07b78f98c07b83f013567a0a - -Will be downloaded and stored in the following file:: - - /full/3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg + /full/ Where: @@ -139,6 +196,9 @@ Where: * ``full`` is a sub-directory to separate full images from thumbnails (if used). For more info see :ref:`topics-images-thumbnails`. +* ```` is the file name assigned to the file. For more info see :ref:`topics-file-naming`. + + .. _media-pipeline-ftp: FTP server storage @@ -353,9 +413,9 @@ Where: * ```` is the one specified in the :setting:`IMAGES_THUMBS` dictionary keys (``small``, ``big``, etc) -* ```` is the `SHA1 hash`_ of the image url +* ```` is the `SHA-1 hash`_ of the image url -.. _SHA1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions +.. _SHA-1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions Example of image files stored using ``small`` and ``big`` thumbnail names:: From abe0b37d307d40897ca6d7e61aa5c137c8e6a4c1 Mon Sep 17 00:00:00 2001 From: laggardkernel Date: Tue, 27 Jul 2021 17:11:32 +0800 Subject: [PATCH 0348/2083] Cleanup leftover boto2 code in S3DownloaderHandler (#5209) S3DownloaderHandler.conn is a leftover attribute from 5e99758. --- scrapy/core/downloader/handlers/s3.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 1966570d4..31f1be31a 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,5 +1,3 @@ -from urllib.parse import unquote - from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore_available @@ -59,7 +57,7 @@ class S3DownloadHandler: url = f'{scheme}://{bucket}.s3.amazonaws.com{path}' if self.anon: request = request.replace(url=url) - elif self._signer is not None: + else: import botocore.awsrequest awsrequest = botocore.awsrequest.AWSRequest( method=request.method, @@ -69,14 +67,4 @@ class S3DownloadHandler: self._signer.add_auth(awsrequest) request = request.replace( url=url, headers=awsrequest.headers.items()) - else: - signed_headers = self.conn.make_request( - method=request.method, - bucket=bucket, - key=unquote(p.path), - query_args=unquote(p.query), - headers=request.headers, - data=request.body, - ) - request = request.replace(url=url, headers=signed_headers) return self._download_http(request, spider) From 7e4321f201a795166d779f2aa0b36d38cb50106e Mon Sep 17 00:00:00 2001 From: laggardkernel Date: Mon, 19 Jul 2021 12:00:42 +0800 Subject: [PATCH 0349/2083] Add support for temporary security credential in AWS auth --- docs/topics/feed-exports.rst | 5 +++-- docs/topics/settings.rst | 14 ++++++++++++++ scrapy/core/downloader/handlers/s3.py | 5 ++++- scrapy/extensions/feedexport.py | 5 ++++- scrapy/pipelines/files.py | 3 +++ scrapy/pipelines/images.py | 1 + tests/test_feedexport.py | 12 ++++++++---- tox.ini | 1 + 8 files changed, 38 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 216a8bc52..af60de716 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -135,7 +135,7 @@ Here are some examples to illustrate: - ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` -.. note:: :ref:`Spider arguments ` become spider attributes, hence +.. note:: :ref:`Spider arguments ` become spider attributes, hence they can also be used as storage URI parameters. @@ -200,6 +200,7 @@ passed through the following settings: - :setting:`AWS_ACCESS_KEY_ID` - :setting:`AWS_SECRET_ACCESS_KEY` +- :setting:`AWS_SESSION_TOKEN` (Optional) You can also define a custom ACL and custom endpoint for exported feeds using this setting: @@ -357,7 +358,7 @@ For instance:: 'item_export_kwargs': { 'export_empty_fields': True, }, - }, + }, '/home/user/documents/items.xml': { 'format': 'xml', 'fields': ['name', 'price'], diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1290b4a5e..58daafa6f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -204,6 +204,20 @@ Default: ``None`` The AWS secret key used by code that requires access to `Amazon Web services`_, such as the :ref:`S3 feed storage backend `. +.. setting:: AWS_SESSION_TOKEN + +AWS_SESSION_TOKEN +----------------- + +Default: ``None`` (Optional) + +The AWS security token used by code that requires access to `Amazon Web services`_, +such as the :ref:`S3 feed storage backend `. + +The security token is only required by a *temporary security credentials*. +Using of temporary security credentials is discouraged cause the credentials +are short term. It may expires before the scraping is done. + .. setting:: AWS_ENDPOINT_URL AWS_ENDPOINT_URL diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 31f1be31a..51ca1ed5e 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -10,6 +10,7 @@ class S3DownloadHandler: def __init__(self, settings, *, crawler=None, aws_access_key_id=None, aws_secret_access_key=None, + aws_session_token=None, httpdownloadhandler=HTTPDownloadHandler, **kw): if not is_botocore_available(): raise NotConfigured('missing botocore library') @@ -18,6 +19,8 @@ class S3DownloadHandler: aws_access_key_id = settings['AWS_ACCESS_KEY_ID'] if not aws_secret_access_key: aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY'] + if not aws_session_token: + aws_session_token = settings['AWS_SESSION_TOKEN'] # If no credentials could be found anywhere, # consider this an anonymous connection request by default; @@ -36,7 +39,7 @@ class S3DownloadHandler: if not self.anon: SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] self._signer = SignerCls(botocore.credentials.Credentials( - aws_access_key_id, aws_secret_access_key)) + aws_access_key_id, aws_secret_access_key, aws_session_token)) _http_handler = create_instance( objcls=httpdownloadhandler, diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index bd4808e2b..564c736f2 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -154,13 +154,14 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *, - feed_options=None): + feed_options=None, session_token=None): if not is_botocore_available(): raise NotConfigured('missing botocore library') u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key self.secret_key = u.password or secret_key + self.session_token = session_token self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url @@ -169,6 +170,7 @@ class S3FeedStorage(BlockingFeedStorage): self.s3_client = session.create_client( 's3', aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, endpoint_url=self.endpoint_url) if feed_options and feed_options.get('overwrite', True) is False: logger.warning('S3 does not support appending to files. To ' @@ -182,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage): uri, access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], + session_token=crawler.settings['AWS_SESSION_TOKEN'], acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None, feed_options=feed_options, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 13ecd4e6c..8766ef66f 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -79,6 +79,7 @@ class FSFilesStore: class S3FilesStore: AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None + AWS_SESSION_TOKEN = None AWS_ENDPOINT_URL = None AWS_REGION_NAME = None AWS_USE_SSL = None @@ -98,6 +99,7 @@ class S3FilesStore: 's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID, aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, + aws_session_token=self.AWS_SESSION_TOKEN, endpoint_url=self.AWS_ENDPOINT_URL, region_name=self.AWS_REGION_NAME, use_ssl=self.AWS_USE_SSL, @@ -349,6 +351,7 @@ class FilesPipeline(MediaPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN'] s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e3ab23ea5..9c99dc69e 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -92,6 +92,7 @@ class ImagesPipeline(FilesPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN'] s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index da0b2c786..389808306 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -244,7 +244,8 @@ class S3FeedStorageTest(unittest.TestCase): def test_parse_credentials(self): skip_if_no_boto() aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', - 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} + 'AWS_SECRET_ACCESS_KEY': 'settings_secret', + 'AWS_SESSION_TOKEN': 'settings_token'} crawler = get_crawler(settings_dict=aws_credentials) # Instantiate with crawler storage = S3FeedStorage.from_crawler( @@ -253,12 +254,15 @@ class S3FeedStorageTest(unittest.TestCase): ) self.assertEqual(storage.access_key, 'settings_key') self.assertEqual(storage.secret_key, 'settings_secret') + self.assertEqual(storage.session_token, 'settings_token') # Instantiate directly storage = S3FeedStorage('s3://mybucket/export.csv', aws_credentials['AWS_ACCESS_KEY_ID'], - aws_credentials['AWS_SECRET_ACCESS_KEY']) + aws_credentials['AWS_SECRET_ACCESS_KEY'], + session_token=aws_credentials['AWS_SESSION_TOKEN']) self.assertEqual(storage.access_key, 'settings_key') self.assertEqual(storage.secret_key, 'settings_secret') + self.assertEqual(storage.session_token, 'settings_token') # URI priority > settings priority storage = S3FeedStorage('s3://uri_key:uri_secret@mybucket/export.csv', aws_credentials['AWS_ACCESS_KEY_ID'], @@ -1957,8 +1961,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): class S3FeedStorageWithoutFeedOptions(S3FeedStorage): - def __init__(self, uri, access_key, secret_key, acl, endpoint_url): - super().__init__(uri, access_key, secret_key, acl, endpoint_url) + def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs): + super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs) class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): diff --git a/tox.ini b/tox.ini index 4c4bbff6e..96050223b 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,7 @@ passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN GCS_TEST_FILE_URI GCS_PROJECT_ID #allow tox virtualenv to upgrade pip/wheel/setuptools From 8e7b96d8a2a6d712be29773b4c69e190d0c1ac7b Mon Sep 17 00:00:00 2001 From: laggardkernel Date: Tue, 27 Jul 2021 19:29:25 +0800 Subject: [PATCH 0350/2083] Tweak doc for setting AWS_SESSION_TOKEN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/feed-exports.rst | 4 +++- docs/topics/settings.rst | 9 ++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index af60de716..2b3217d62 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -200,7 +200,9 @@ passed through the following settings: - :setting:`AWS_ACCESS_KEY_ID` - :setting:`AWS_SECRET_ACCESS_KEY` -- :setting:`AWS_SESSION_TOKEN` (Optional) +- :setting:`AWS_SESSION_TOKEN` (only needed for `temporary security credentials`_) + +.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys You can also define a custom ACL and custom endpoint for exported feeds using this setting: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 58daafa6f..1a1a833df 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -209,14 +209,13 @@ such as the :ref:`S3 feed storage backend `. AWS_SESSION_TOKEN ----------------- -Default: ``None`` (Optional) +Default: ``None`` The AWS security token used by code that requires access to `Amazon Web services`_, -such as the :ref:`S3 feed storage backend `. +such as the :ref:`S3 feed storage backend `, when using +`temporary security credentials`_. -The security token is only required by a *temporary security credentials*. -Using of temporary security credentials is discouraged cause the credentials -are short term. It may expires before the scraping is done. +.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys .. setting:: AWS_ENDPOINT_URL From d55b6fcad6a792c16022324c6dd6402f8fde8641 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Wed, 28 Jul 2021 12:10:34 +1000 Subject: [PATCH 0351/2083] Fix for duplicate object description error --- docs/topics/extensions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 3cabcefdd..08272a25d 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -328,7 +328,7 @@ Emails can be sent using the MailSender class .. module:: scrapy.mail :synopsis: MailSender class -.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) +.. class:: MailSender To see a full list of parameters, including examples on how to instantiate MailSender and using mail settings, see :ref:`topics-email` From 494e0ad8ffc34e7d8079db6dd24fdc8265e81800 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 28 Jul 2021 14:29:50 -0300 Subject: [PATCH 0352/2083] Update docs/topics/dynamic-content.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/dynamic-content.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 56c8b6ae9..9706f43fe 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -272,10 +272,10 @@ The following is a simple snippet to illustrate its usage within Scrapy:: For this example to work, Scrapy needs to be running on top of the :ref:`asyncio reactor `. -Keep in mind that this is just a proof of concept, since it circumvents -most of the Scrapy components (middlewares, dupefilter, etc). - -The following is a list of 3rd party projects which provide better integration: +Using pypeteer_ directly circumvents most of the +Scrapy components (middlewares, dupefilter, etc). Use +one of the following Scrapy plugins for better integration +with Scrapy: * https://github.com/elacuesta/scrapy-pyppeteer * https://github.com/lopuhin/scrapy-pyppeteer From 0e3d50dd186666362ab8358c9f89036917242c81 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 28 Jul 2021 14:30:16 -0300 Subject: [PATCH 0353/2083] Update docs/topics/dynamic-content.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/dynamic-content.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 9706f43fe..e918bc006 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -252,6 +252,7 @@ Since version 2.0, it is possible to integrate libraries that use the ``async/await`` syntax. One such library is `pyppeteer`_ (an unnoficial Python port of `puppeteer`_), which uses headless Chrome to download and render pages. + The following is a simple snippet to illustrate its usage within Scrapy:: import pyppeteer From 4b62ac6c3ace093d16cf97e737689ac7942d52f9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 28 Jul 2021 15:00:24 -0300 Subject: [PATCH 0354/2083] Update headless browser docs to mention playwright --- docs/topics/dynamic-content.rst | 55 ++++++++++++++------------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index f96be0bbc..ea5d06210 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -246,55 +246,46 @@ Using a headless browser ======================== A `headless browser`_ is a special web browser that provides an API for -automation. +automation. By installing the :ref:`asyncio reactor `, +it is possible to integrate ``asyncio``-based libraries which handle headless browsers. -Since version 2.0, it is possible to integrate libraries that use the -``async/await`` syntax. One such library is `pyppeteer`_ (an unnoficial -Python port of `puppeteer`_), which uses headless Chrome to download and -render pages. +One such library is `playwright-python`_ (an official Python port of `playwright`_). +The following is a simple snippet to illustrate its usage within a Scrapy spider:: -The following is a simple snippet to illustrate its usage within Scrapy:: - - import pyppeteer import scrapy + from playwright.async_api import async_playwright - class PyppeteerSpider(scrapy.Spider): - name = "pyppeteer" - start_urls = ["data:,"] # avoid making an actual upstream request + class PlaywrightSpider(scrapy.Spider): + name = "playwright" + start_urls = ["data:,"] # avoid using the default Scrapy downloader async def parse(self, response): - browser = await pyppeteer.launch() - page = await browser.newPage() - await page.goto("https:/example.org") - title = await page.title() - await page.close() - yield {"title": title} + async with async_playwright() as pw: + browser = await pw.chromium.launch() + page = await browser.new_page() + await page.goto("https:/example.org") + title = await page.title() + return {"title": title} -For this example to work, Scrapy needs to be running on top of the -:ref:`asyncio reactor `. - -Using pypeteer_ directly circumvents most of the -Scrapy components (middlewares, dupefilter, etc). Use -one of the following Scrapy plugins for better integration -with Scrapy: - -* https://github.com/elacuesta/scrapy-pyppeteer -* https://github.com/lopuhin/scrapy-pyppeteer -* https://github.com/clemfromspace/scrapy-puppeteer +However, using `playwright-python`_ directly as in the above example +circumvents most of the Scrapy components (middlewares, dupefilter, etc). +We recommend using `scrapy-playwright`_ for a better integration. .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29 -.. _chompjs: https://github.com/Nykakin/chompjs .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets +.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript +.. _Splash: https://github.com/scrapinghub/splash +.. _chompjs: https://github.com/Nykakin/chompjs .. _curl: https://curl.haxx.se/ .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser -.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript .. _js2xml: https://github.com/scrapinghub/js2xml -.. _puppeteer: https://pptr.dev/ +.. _playwright-python: https://github.com/microsoft/playwright-python +.. _playwright: https://github.com/microsoft/playwright .. _pyppeteer: https://pyppeteer.github.io/pyppeteer/ .. _pytesseract: https://github.com/madmaze/pytesseract +.. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright .. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash -.. _Splash: https://github.com/scrapinghub/splash .. _tabula-py: https://github.com/chezou/tabula-py .. _wget: https://www.gnu.org/software/wget/ .. _wgrep: https://github.com/stav/wgrep From cc89f6be381d72a2528c8a672158671305019324 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 29 Jul 2021 17:12:44 -0300 Subject: [PATCH 0355/2083] Response.attributes (#5218) --- docs/topics/request-response.rst | 7 ++-- scrapy/http/response/__init__.py | 23 +++++++++---- scrapy/http/response/text.py | 8 ++--- tests/test_http_response.py | 59 ++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 15 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a6a3daf31..d3e08efd4 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -670,9 +670,6 @@ Response objects .. autoclass:: Response - A :class:`Response` object represents an HTTP response, which is usually - downloaded (by the Downloader) and fed to the Spiders for processing. - :param url: the URL of this response :type url: str @@ -829,6 +826,8 @@ Response objects handlers, i.e. for ``http(s)`` responses. For other handlers, :attr:`protocol` is always ``None``. + .. autoattribute:: Response.attributes + .. method:: Response.copy() Returns a new Response which is a copy of this Response. @@ -925,6 +924,8 @@ TextResponse objects A :class:`~scrapy.Selector` instance using the response as target. The selector is lazily instantiated on first access. + .. autoattribute:: TextResponse.attributes + :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 185a9bb67..4de6c9b5b 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,7 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ -from typing import Generator +from typing import Generator, Tuple from urllib.parse import urljoin from scrapy.exceptions import NotSupported @@ -16,6 +16,19 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): + """An object that represents an HTTP response, which is usually + downloaded (by the Downloader) and fed to the Spiders for processing. + """ + + attributes: Tuple[str, ...] = ( + "url", "status", "headers", "body", "flags", "request", "certificate", "ip_address", "protocol", + ) + """A tuple of :class:`str` objects containing the name of all public + attributes of the class that are also keyword parameters of the + ``__init__`` method. + + Currently used by :meth:`Response.replace`. + """ def __init__( self, @@ -97,12 +110,8 @@ class Response(object_ref): return self.replace() def replace(self, *args, **kwargs): - """Create a new Response with the same attributes except for those - given new values. - """ - for x in [ - "url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol", - ]: + """Create a new Response with the same attributes except for those given new values""" + for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index e36e14880..27bd55c07 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst import json import warnings from contextlib import suppress -from typing import Generator +from typing import Generator, Tuple from urllib.parse import urljoin import parsel @@ -30,6 +30,8 @@ class TextResponse(Response): _DEFAULT_ENCODING = 'ascii' _cached_decoded_json = _NONE + attributes: Tuple[str, ...] = Response.attributes + ("encoding",) + def __init__(self, *args, **kwargs): self._encoding = kwargs.pop('encoding', None) self._cached_benc = None @@ -53,10 +55,6 @@ class TextResponse(Response): else: super()._set_body(body) - def replace(self, *args, **kwargs): - kwargs.setdefault('encoding', self.encoding) - return Response.replace(self, *args, **kwargs) - @property def encoding(self): return self._declared_encoding() or self._body_inferred_encoding() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 04a594d03..cf34a9e5c 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -820,3 +820,62 @@ class XmlResponseTest(TextResponseTest): response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(), response.selector.xpath("//s2:elem/text()").getall(), ) + + +class CustomResponse(TextResponse): + attributes = TextResponse.attributes + ("foo", "bar") + + def __init__(self, *args, **kwargs) -> None: + self.foo = kwargs.pop("foo", None) + self.bar = kwargs.pop("bar", None) + self.lost = kwargs.pop("lost", None) + super().__init__(*args, **kwargs) + + +class CustomResponseTest(TextResponseTest): + response_class = CustomResponse + + def test_copy(self): + super().test_copy() + r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost") + r2 = r1.copy() + self.assertIsInstance(r2, self.response_class) + self.assertEqual(r1.foo, r2.foo) + self.assertEqual(r1.bar, r2.bar) + self.assertEqual(r1.lost, "lost") + self.assertIsNone(r2.lost) + + def test_replace(self): + super().test_replace() + r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost") + + r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") + self.assertIsInstance(r2, self.response_class) + self.assertEqual(r1.foo, "foo") + self.assertEqual(r1.bar, "bar") + self.assertEqual(r1.lost, "lost") + self.assertEqual(r2.foo, "new-foo") + self.assertEqual(r2.bar, "new-bar") + self.assertEqual(r2.lost, "new-lost") + + r3 = r1.replace(foo="new-foo", bar="new-bar") + self.assertIsInstance(r3, self.response_class) + self.assertEqual(r1.foo, "foo") + self.assertEqual(r1.bar, "bar") + self.assertEqual(r1.lost, "lost") + self.assertEqual(r3.foo, "new-foo") + self.assertEqual(r3.bar, "new-bar") + self.assertIsNone(r3.lost) + + r4 = r1.replace(foo="new-foo") + self.assertIsInstance(r4, self.response_class) + self.assertEqual(r1.foo, "foo") + self.assertEqual(r1.bar, "bar") + self.assertEqual(r1.lost, "lost") + self.assertEqual(r4.foo, "new-foo") + self.assertEqual(r4.bar, "bar") + self.assertIsNone(r4.lost) + + with self.assertRaises(TypeError) as ctx: + r1.replace(unknown="unknown") + self.assertEqual(str(ctx.exception), "__init__() got an unexpected keyword argument 'unknown'") From 880a4d9493338516aa6d12d602dadf9be60d3053 Mon Sep 17 00:00:00 2001 From: Aaron Tan <70739609+aaron-tan@users.noreply.github.com> Date: Sun, 1 Aug 2021 11:02:27 +1000 Subject: [PATCH 0356/2083] Update docs/topics/extensions.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/extensions.rst | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 08272a25d..297e1fdc5 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -323,14 +323,10 @@ domain has finished scraping, including the Scrapy stats collected. The email will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS` setting. -Emails can be sent using the MailSender class - -.. module:: scrapy.mail - :synopsis: MailSender class - -.. class:: MailSender - -To see a full list of parameters, including examples on how to instantiate MailSender and using mail settings, see :ref:`topics-email` +Emails can be sent using the :class:`~scrapy.mail.MailSender` class. To see a +full list of parameters, including examples on how to instantiate +:class:`~scrapy.mail.MailSender` and use mail settings, see +:ref:`topics-email`. .. module:: scrapy.extensions.debug :synopsis: Extensions for debugging Scrapy From 2bf2f9d6db89968bcb5df6bc7d093fdd53de5ba5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Aug 2021 19:44:11 +0500 Subject: [PATCH 0357/2083] Add Python 3.10b4 tests on Ubuntu. --- .github/workflows/tests-ubuntu.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 57188bd63..57c994158 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -45,6 +45,14 @@ jobs: env: TOXENV: asyncio + # 3.10-pre + - python-version: "3.10.0-beta.4" + env: + TOXENV: py + - python-version: "3.10.0-beta.4" + env: + TOXENV: asyncio + steps: - uses: actions/checkout@v2 @@ -54,7 +62,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4' run: | sudo apt-get update # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version From ef6fb933b568c497ab3745284bc2a02725bced1c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 20 Jul 2021 12:02:15 +0500 Subject: [PATCH 0358/2083] Fix a Python 3.10 logging issue. --- scrapy/utils/signal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 62808f3ce..fbafc9d45 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,5 +1,5 @@ """Helper functions for working with signals""" -import collections +import collections.abc import logging from twisted.internet.defer import DeferredList, Deferred @@ -21,7 +21,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): Failures instead of exceptions. """ dont_log = named.pop('dont_log', ()) - dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,) + dont_log = tuple(dont_log) if isinstance(dont_log, collections.abc.Sequence) else (dont_log,) dont_log += (StopDownload, ) spider = named.get('spider', None) responses = [] From 93bf1ae7e3966d56539411c59d172c2971ee61e0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Aug 2021 20:16:29 +0500 Subject: [PATCH 0359/2083] Fix tests for the 3.10 TypeError message change. --- tests/test_http_response.py | 2 +- tests/test_request_cb_kwargs.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index cf34a9e5c..c376a46cd 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -878,4 +878,4 @@ class CustomResponseTest(TextResponseTest): with self.assertRaises(TypeError) as ctx: r1.replace(unknown="unknown") - self.assertEqual(str(ctx.exception), "__init__() got an unexpected keyword argument 'unknown'") + self.assertTrue(str(ctx.exception).endswith("__init__() got an unexpected keyword argument 'unknown'")) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 145a4e9b2..b68184b87 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -158,12 +158,14 @@ class CallbackKeywordArgumentsTestCase(TestCase): if key in line.getMessage(): exceptions[key] = line self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError) - self.assertEqual( - str(exceptions['takes_less'].exc_info[1]), - "parse_takes_less() got an unexpected keyword argument 'number'" + self.assertTrue( + str(exceptions['takes_less'].exc_info[1]).endswith( + "parse_takes_less() got an unexpected keyword argument 'number'" + ) ) self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) - self.assertEqual( - str(exceptions['takes_more'].exc_info[1]), - "parse_takes_more() missing 1 required positional argument: 'other'" + self.assertTrue( + str(exceptions['takes_more'].exc_info[1]).endswith( + "parse_takes_more() missing 1 required positional argument: 'other'" + ) ) From 94baa4b27273e5a779bb977cea4c8eb5301fc3bf Mon Sep 17 00:00:00 2001 From: Mannan2812 <42071936+Mannan2812@users.noreply.github.com> Date: Fri, 6 Aug 2021 00:53:11 +0530 Subject: [PATCH 0360/2083] Fix FileFeedStoragePreFeedOptionsTest fails in CI/CD pipeline (#5198) --- tests/test_feedexport.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 389808306..53e6a2018 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1932,14 +1932,15 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): maxDiff = None def test_init(self): - settings_dict = { - 'FEED_URI': 'file:///tmp/foobar', - 'FEED_STORAGES': { - 'file': FileFeedStorageWithoutFeedOptions - }, - } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with tempfile.NamedTemporaryFile() as temp: + settings_dict = { + 'FEED_URI': f'file:///{temp.name}', + 'FEED_STORAGES': { + 'file': FileFeedStorageWithoutFeedOptions + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) spider = scrapy.Spider("default") with warnings.catch_warnings(record=True) as w: feed_exporter.open_spider(spider) From 8e7d2ef13312bfb4ec5e1800f00108806ddc12e8 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Sat, 7 Aug 2021 11:44:12 +1000 Subject: [PATCH 0361/2083] Document JOBDIR option issue #5173 Add JOBDIR setting to the settings page. Add default JOBDIR setting to global defaults in scrapy.settings.default_settings module. --- docs/topics/settings.rst | 11 +++++++++++ scrapy/settings/default_settings.py | 2 ++ 2 files changed, 13 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1a1a833df..5e820b0a9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -989,6 +989,17 @@ Default: ``{}`` A dict containing the pipelines enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. +.. setting:: JOBDIR + +JOBDIR +------ + +Default: ``''`` + +A string indicating the directory for storing the required data to keep the state of a single job to enable persistence support. This directory must not be shared by different spiders or jobs/runs of the same spider. + +For more info on this setting, see :ref:`topics-jobs` + .. setting:: LOG_ENABLED LOG_ENABLED diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 4ef330dd2..9137086c0 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -199,6 +199,8 @@ ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} ITEM_PIPELINES_BASE = {} +JOBDIR = '' + LOG_ENABLED = True LOG_ENCODING = 'utf-8' LOG_FORMATTER = 'scrapy.logformatter.LogFormatter' From 48eff4ee8f21535e98baf2bdff91749fee002c10 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Sun, 8 Aug 2021 20:52:14 +1000 Subject: [PATCH 0362/2083] Remove JOBDIR from default settings --- scrapy/settings/default_settings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9137086c0..4ef330dd2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -199,8 +199,6 @@ ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} ITEM_PIPELINES_BASE = {} -JOBDIR = '' - LOG_ENABLED = True LOG_ENCODING = 'utf-8' LOG_FORMATTER = 'scrapy.logformatter.LogFormatter' From 954f3035908f7f7f528ce4d3c5245f056645dc7f Mon Sep 17 00:00:00 2001 From: Aaron Tan <70739609+aaron-tan@users.noreply.github.com> Date: Mon, 9 Aug 2021 22:23:23 +1000 Subject: [PATCH 0363/2083] Update docs/topics/settings.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/settings.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5e820b0a9..2ab2020fa 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -996,9 +996,8 @@ JOBDIR Default: ``''`` -A string indicating the directory for storing the required data to keep the state of a single job to enable persistence support. This directory must not be shared by different spiders or jobs/runs of the same spider. - -For more info on this setting, see :ref:`topics-jobs` +A string indicating the directory for storing the state of a crawl when +:ref:`pausing and resuming crawls `. .. setting:: LOG_ENABLED From 1ba0f68483cfb8aa62759e21b3479ec9bea94beb Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 17:09:37 +0200 Subject: [PATCH 0364/2083] Allow comma-separated values in the rel tag Comma-separated `rel` values are often seen in the wild, because Google allows it (see https://developers.google.com/search/docs/advanced/guidelines/qualify-outbound-links). --- scrapy/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 5c986eedc..51cef1e91 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -138,7 +138,7 @@ def md5sum(file): def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" - return rel is not None and 'nofollow' in rel.split() + return rel is not None and 'nofollow' in rel.replace(',', ' ').split() def create_instance(objcls, settings, crawler, *args, **kwargs): From 18b6f30a7359d1a798c30888ddd10a1612d8e711 Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 21:13:50 +0200 Subject: [PATCH 0365/2083] Add test for rel_has_nofollow --- tests/test_utils_misc/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index e95a3a316..67367dbfb 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -4,7 +4,7 @@ import unittest from unittest import mock from scrapy.item import Item, Field -from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules +from scrapy.utils.misc import arg_to_iter, create_instance, load_object, rel_has_nofollow, set_environ, walk_modules __doctests__ = ['scrapy.utils.misc'] @@ -162,6 +162,12 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test_value' assert os.environ.get('some_test_environ') == 'test' + def test_rel_has_nofollow(self): + assert os.environ.get('some_test_environ') is None + asert rel_has_nofollow('ugc nofollow') == True + asert rel_has_nofollow('ugc,nofollow') == True + asert rel_has_nofollow('ugc') == False + if __name__ == "__main__": unittest.main() From 07d20a8ce45ab0cbf61d08214db4963302661257 Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 21:21:43 +0200 Subject: [PATCH 0366/2083] Fix test_rel_has_nofollow test --- tests/test_utils_misc/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 67367dbfb..b0d7acd12 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -163,10 +163,9 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test' def test_rel_has_nofollow(self): - assert os.environ.get('some_test_environ') is None - asert rel_has_nofollow('ugc nofollow') == True - asert rel_has_nofollow('ugc,nofollow') == True - asert rel_has_nofollow('ugc') == False + assert rel_has_nofollow('ugc nofollow') == True + assert rel_has_nofollow('ugc,nofollow') == True + assert rel_has_nofollow('ugc') == False if __name__ == "__main__": From 295f0e2bf5c352c6ddf27a188af10bf122d1c6b0 Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 21:38:29 +0200 Subject: [PATCH 0367/2083] Make flake8 happy --- tests/test_utils_misc/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index b0d7acd12..69f593ccd 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -163,9 +163,9 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test' def test_rel_has_nofollow(self): - assert rel_has_nofollow('ugc nofollow') == True - assert rel_has_nofollow('ugc,nofollow') == True - assert rel_has_nofollow('ugc') == False + assert rel_has_nofollow('ugc nofollow') is True + assert rel_has_nofollow('ugc,nofollow') is True + assert rel_has_nofollow('ugc') is False if __name__ == "__main__": From ce9d6c658b21a5d9d9605a2683b7a143f2077dfa Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Tue, 10 Aug 2021 22:21:51 +0200 Subject: [PATCH 0368/2083] Add more rel_has_nofollow tests --- tests/test_utils_misc/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 69f593ccd..47d73a2dd 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -166,6 +166,10 @@ class UtilsMiscTestCase(unittest.TestCase): assert rel_has_nofollow('ugc nofollow') is True assert rel_has_nofollow('ugc,nofollow') is True assert rel_has_nofollow('ugc') is False + assert rel_has_nofollow('nofollow') is True + assert rel_has_nofollow('nofollowfoo') is False + assert rel_has_nofollow('foonofollow') is False + assert rel_has_nofollow('ugc, , nofollow') is True if __name__ == "__main__": From 983b89ad4f72730c37b32c9240a697a4c1f24183 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 11 Aug 2021 10:39:23 +0500 Subject: [PATCH 0369/2083] Fix SpiderLoaderTest on Python 3.10. --- tests/test_spiderloader/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 4929f1e3e..8a35e9fd7 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -118,6 +118,11 @@ class SpiderLoaderTest(unittest.TestCase): settings = Settings({'SPIDER_MODULES': [module], 'SPIDER_LOADER_WARN_ONLY': True}) spider_loader = SpiderLoader.from_settings(settings) + if str(w[0].message).startswith("_SixMetaPathImporter"): + # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, + # at least until all six versions we can import (including botocore.vendored.six) + # are updated to 1.16.0+ + w.pop(0) self.assertIn("Could not load spiders from module", str(w[0].message)) spiders = spider_loader.list() From 74cee38a4e07c31a8dd2a8772ff18d1b9adf8a6b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 11 Aug 2021 14:19:08 +0500 Subject: [PATCH 0370/2083] Don't run the asyncio tests on 3.9. --- .github/workflows/tests-ubuntu.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 57c994158..81beda5da 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -41,9 +41,6 @@ jobs: env: TOXENV: extra-deps TOX_PIP_VERSION: 20.3.3 - - python-version: 3.9 - env: - TOXENV: asyncio # 3.10-pre - python-version: "3.10.0-beta.4" From b63369c148b9a1f87a974e21c5e307a62783d6fd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 11 Aug 2021 20:02:45 +0500 Subject: [PATCH 0371/2083] Rename tests/requirements-py3.txt to tests/requirements.txt. --- tests/{requirements-py3.txt => requirements.txt} | 0 tox.ini | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/{requirements-py3.txt => requirements.txt} (100%) diff --git a/tests/requirements-py3.txt b/tests/requirements.txt similarity index 100% rename from tests/requirements-py3.txt rename to tests/requirements.txt diff --git a/tox.ini b/tox.ini index 96050223b..e274fc8d2 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ minversion = 1.7.0 [testenv] deps = - -rtests/requirements-py3.txt + -rtests/requirements.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe @@ -82,7 +82,7 @@ deps = Twisted[http2]==17.9.0 w3lib==1.17.0 zope.interface==4.1.3 - -rtests/requirements-py3.txt + -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment From 2814e0e1972fa38151b6800c881d49f50edf9c6b Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 16 Aug 2021 16:22:01 +0500 Subject: [PATCH 0372/2083] Disable builtin middlewares in spider middleware tests. (#5229) --- tests/test_spidermiddleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 78e926adc..b39576996 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -15,7 +15,7 @@ class SpiderMiddlewareTestCase(TestCase): def setUp(self): self.request = Request('http://example.com/index.html') self.response = Response(self.request.url, request=self.request) - self.crawler = get_crawler(Spider) + self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}}) self.spider = self.crawler._create_spider('foo') self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) From 8bbaea9892003769672204a8ff4e989b5aab5ceb Mon Sep 17 00:00:00 2001 From: databender Date: Mon, 16 Aug 2021 16:57:43 +0530 Subject: [PATCH 0373/2083] updated documentation for python version for reppy --- docs/topics/downloader-middleware.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 80c6c2c37..222dda685 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1073,6 +1073,8 @@ In order to use this parser: * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` +* only works with python 3.8 and earlier + .. _rerp-parser: Robotexclusionrulesparser From 1a8b98843aee548a52faa36f5360a81a1624e208 Mon Sep 17 00:00:00 2001 From: databender Date: Mon, 16 Aug 2021 17:00:05 +0530 Subject: [PATCH 0374/2083] updated documentation for python version for reppy --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 222dda685..2c00ad45d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1073,7 +1073,7 @@ In order to use this parser: * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` -* only works with python 3.8 and earlier +* Only works with python 3.8 and earlier .. _rerp-parser: From cc1cb2de0c6e91393ceb6872c174aa2ac06c07ac Mon Sep 17 00:00:00 2001 From: databender Date: Mon, 16 Aug 2021 17:21:47 +0530 Subject: [PATCH 0375/2083] updated suggested changes --- docs/topics/downloader-middleware.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 2c00ad45d..fa211fb75 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1068,12 +1068,13 @@ Native implementation, provides better speed than Protego. In order to use this parser: +.. warning:: Does not support Python 3.9+ + * Install `Reppy `_ by running ``pip install reppy`` * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` -* Only works with python 3.8 and earlier .. _rerp-parser: From 013ac90f6129a9e8b862d66ca2ffa8f0f1fd674e Mon Sep 17 00:00:00 2001 From: umair ansari Date: Mon, 16 Aug 2021 18:00:06 +0530 Subject: [PATCH 0376/2083] Update docs/topics/downloader-middleware.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/downloader-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index fa211fb75..8323bc564 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1068,10 +1068,10 @@ Native implementation, provides better speed than Protego. In order to use this parser: -.. warning:: Does not support Python 3.9+ - * Install `Reppy `_ by running ``pip install reppy`` + .. warning:: Does not support Python 3.9+ + * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From ebddb77a331c6290e64e449ef9847e33b323a146 Mon Sep 17 00:00:00 2001 From: databender Date: Mon, 16 Aug 2021 18:08:26 +0530 Subject: [PATCH 0377/2083] updated suggested changes after review --- docs/topics/downloader-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index fa211fb75..4d7c87404 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1068,10 +1068,10 @@ Native implementation, provides better speed than Protego. In order to use this parser: -.. warning:: Does not support Python 3.9+ - * Install `Reppy `_ by running ``pip install reppy`` +.. warning:: Does not support Python 3.9+ + * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From bcf38a67194f25db66334af18bdf49b34c6a0c39 Mon Sep 17 00:00:00 2001 From: databender Date: Wed, 18 Aug 2021 14:48:47 +0530 Subject: [PATCH 0378/2083] added upstream issue for not supported python version --- docs/topics/downloader-middleware.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 4d7c87404..089d5683a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1070,7 +1070,9 @@ In order to use this parser: * Install `Reppy `_ by running ``pip install reppy`` -.. warning:: Does not support Python 3.9+ + .. warning:: `Upstream issue #122 + `_ prevents reppy usage in + Python 3.9+. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From d623ed15d1a79a91b55aa7aae2d942aac94abfdf Mon Sep 17 00:00:00 2001 From: databender Date: Wed, 18 Aug 2021 14:51:03 +0530 Subject: [PATCH 0379/2083] indentation updated --- docs/topics/downloader-middleware.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 089d5683a..928a59bf1 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1070,9 +1070,9 @@ In order to use this parser: * Install `Reppy `_ by running ``pip install reppy`` - .. warning:: `Upstream issue #122 - `_ prevents reppy usage in - Python 3.9+. +.. warning:: `Upstream issue #122 + `_ prevents reppy usage in + Python 3.9+. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From 2d2581c68f35799dc4372a257eaa8dbb5208481d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Aug 2021 12:46:42 +0200 Subject: [PATCH 0380/2083] Move documentation about avoiding bans into a topic of its own (#4039) --- docs/index.rst | 4 + docs/topics/avoiding-bans.rst | 340 ++++++++++++++++++++++++++++++++++ docs/topics/practices.rst | 34 ---- 3 files changed, 344 insertions(+), 34 deletions(-) create mode 100644 docs/topics/avoiding-bans.rst diff --git a/docs/index.rst b/docs/index.rst index 433798aa8..7647b3781 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -155,6 +155,7 @@ Solving specific problems topics/debug topics/contracts topics/practices + topics/avoiding-bans topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -179,6 +180,9 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. +:doc:`topics/avoiding-bans` + Avoid getting banned from websites. + :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. diff --git a/docs/topics/avoiding-bans.rst b/docs/topics/avoiding-bans.rst new file mode 100644 index 000000000..59f0da191 --- /dev/null +++ b/docs/topics/avoiding-bans.rst @@ -0,0 +1,340 @@ +.. _bans: + +============= +Avoiding bans +============= + +This topic covers some of the strategies that you can follow to avoid getting +different or bad responses from a website that you are crawling due to filters +such as regional filters, web browser filters, etc. + +.. _avoiding-crawls: + +Avoiding crawls +=============== + +The best way not to be banned from a website is not to send requests to it in +the first place. + +One way to avoid crawling a website is to find the desired dataset through +other means. For example, you can use Google’s `dataset search engine`_. + +If the target website is the only or best source of the desired information, +and you only need to extract the data on a monthly basis or a lower frequency, +you may be able to crawl a public snapshot of the target website instead. +`Common Crawl`_ is an open repository of web crawl data that you can access +freely. It contains monthly snapshots of a wide variety of websites and, if you +are lucky, your target website will be among them. + +.. _Common Crawl: https://commoncrawl.org/ +.. _dataset search engine: https://datasetsearch.research.google.com/ + + +.. _being-polite: + +Being polite +============ + +To avoid being banned, you should first avoid giving a website reasons to ban +you. + +.. _identifying-yourself: + +Identifying yourself +-------------------- + +If your crawling has a noticeable negative impact on a website or you crawl +content that should not be crawled, website administrators will need to do +something. + +Set :setting:`USER_AGENT` to a value that uniquely identifies your spider and +includes contact information, so that website administrators can contact you. + + +.. _following-robotstxt: + +Following robots.txt guidelines +------------------------------- + +Some websites provide a ``robots.txt`` file at their root path (e.g. +``http://example.com/robots.txt``) that describes the guidelines that they wish +bots to follow when crawling their website. + +Before you start writing a spider for a website, read their ``robots.txt`` +file and implement your spider following its guidelines. See the `robots.txt +standard draft`_ or the `robots.txt Google specification`_ for information on +how to read ``robots.txt`` files. + +To ensure that your spider does not crawl pages restricted by ``robots.txt`` +guidelines, set :setting:`ROBOTSTXT_OBEY` to ``True`` to enable the +:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` +middleware. When you do, if your spider attempts to crawl a restricted page, +this middleware aborts that request with the following message:: + + Forbidden by robots.txt + +Also set :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and +:setting:`DOWNLOAD_DELAY` to values that comply with the ``Crawl-Delay`` or +``Request-Rate`` directives from the ``robots.txt`` guidelines. + +You may also use the :ref:`AutoThrottle extension ` on top +of that, so that when the target website experiences a high load, your spider +automatically switches to higher download delays. + +.. _robots.txt Google specification: https://developers.google.com/search/reference/robots_txt +.. _robots.txt standard draft: https://tools.ietf.org/html/draft-koster-rep-00 + + +.. _choosing-crawl-speed: + +Finding the right guidelines on your own +---------------------------------------- + +If a website does not specify a desired download delay, or does not provide a +``robots.txt`` file, you should make an effort to find out the right values for +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY` that +will not have a noticeable negative impact on the target website. + +Use a service like `SimilarWeb`_ to find out the amount of monthly traffic that +the target website receives, and choose concurrency and delay values that will +not cause a noticeable traffic increase. + +.. _SimilarWeb: https://www.similarweb.com + + +.. _filters-and-challenges: + +Bypassing filters and solving challenges +======================================== + +Some websites implement filters and challenges that aim to deny access or alter +their content based on aspects of the visitor, such as the country where they +are or the web browsing tool they use. + +.. _regional-filter: + +Bypassing regional filters +-------------------------- + +Some websites send different or bad responses based on the region or country +associated to your `IP address`_. + +To bypass these filters, get access to a `proxy server`_ that has an outgoing +IP address from a region that gets the desired responses. + +Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` +middleware to configure your spider to use that proxy. + +.. _IP address: https://en.wikipedia.org/wiki/IP_address +.. _proxy server: https://en.wikipedia.org/wiki/Proxy_server + + +.. _web-browser-filter: + +Bypassing web browser filters +----------------------------- + +Some websites send different or bad responses if they detect that your request +does not come from a web browser. + +To bypass these filters, switch your :setting:`USER_AGENT` to a value copied +from those that popular web browsers use. In some rare cases, you may need a +user agent string from a specific web browser. + +There are multiple Scrapy plugins that can rotate your requests through popular +web browser user agent strings, such as scrapy-fake-useragent_, +scrapy-random-useragent_ or Scrapy-UserAgents_. + +For advanced web browser filters, +:ref:`pre-rendering JavaScript ` or +:ref:`using a headless browser ` may be necessary. +Use these options only as a last resort, however, because they cause a higher +load per request on the target website. + +.. _scrapy-fake-useragent: https://github.com/alecxe/scrapy-fake-useragent +.. _scrapy-random-useragent: https://github.com/cleocn/scrapy-random-useragent +.. _Scrapy-UserAgents: https://pypi.org/project/Scrapy-UserAgents/ + + +.. _request-delay-filter: + +Bypassing request delay filters +------------------------------- + +Some websites may ban your IP after they detect that your requests use a +constant download delay. + +To help bypassing these filters, the :setting:`RANDOMIZE_DOWNLOAD_DELAY` +setting is enabled by default. When that is not enough, an +:ref:`IP address rotation solution ` may be much more effective. + + +.. _isp-filter: + +Bypassing internet service provider filters +------------------------------------------- + +Some websites send different or bad responses if they detect that your request +comes from an IP address that belongs to a `data center`_, as opposed to a +residential IP address from an `internet service provider`_ or a mobile IP +address from a `mobile network`_. + +To bypass these filters, get access to a proxy server that has an outgoing IP +address that is either residential or mobile. Note that you may also get +different responses depending on whether your IP address is residential or +mobile. + +Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` +middleware to configure your spider to use that proxy. + +.. _data center: https://en.wikipedia.org/wiki/Data_center +.. _internet service provider: https://en.wikipedia.org/wiki/Internet_service_provider +.. _mobile network: https://en.wikipedia.org/wiki/Cellular_network + + +.. _captcha: + +Solving CAPTCHA challenges +-------------------------- + +Some websites require you to solve a `CAPTCHA challenge`_ to get the desired +response. + +To bypass these filters, several options exist: + +- You could have your spider present the CAPTCHA challenge to you and wait + for you to solve it manually. + +- Some CAPTCHA challenges can be solved using an `optical character + recognition`_ (OCR) solution such as pytesseract_. + +- Paid CAPTCHA solving services exist. + +Whichever solution you choose, implement it as a :ref:`downloader middleware +` that automatically detects CAPTCHA challenges +in responses and solves them, so that your spider code only receives successful +responses. + +.. _CAPTCHA challenge: https://en.wikipedia.org/wiki/CAPTCHA +.. _optical character recognition: https://en.wikipedia.org/wiki/Optical_character_recognition +.. _pytesseract: https://github.com/madmaze/pytesseract + + +.. _ip-rotation: + +IP address rotation solutions +============================= + +See below some of the different solutions there are to have your requests use +different outgoing IP addresses. + +When using this approach, remember to set :setting:`COOKIES_ENABLED` to +``False`` to disable global cookie handling. This prevents websites from +identifying two requests as coming from the same user agent even if they come +from different IP addresses and have different user-agent strings. You can +still include some cookies manually in your requests. Define them through the +``Cookies`` header of your requests. See +:class:`Request.headers `. + +.. _smart-proxy: + +Smart proxies +------------- + +An increasing number of websites use solutions that apply many of the above +filters and challenges at the same time. + +There are paid proxy services, like `Zyte Smart Proxy Manager`_, that +automatically bypass website filters and challenges, so that your spider only +gets successful responses. They also allow managing sessions to simulate user +behavior. + +For Zyte Smart Proxy Manager, installing scrapy-crawlera_ will offer advanced +integration with Scrapy. For other services, use the +:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` middleware +or implement your own :ref:`downloader middleware +`. + +.. _scrapy-crawlera: https://scrapy-crawlera.readthedocs.io/en/latest/ +.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ + + +.. _rotating-proxy: + +Rotating proxies +---------------- + +Rotating proxy services like ProxyMesh_ send different requests through +different proxies. This can decrease the likelihood of being affected by some +filters or challenges. + +.. _ProxyMesh: https://proxymesh.com/ + + +.. _free-proxies: + +Free proxies +------------ + +You can easily find lists of free proxies in the internet, and you can use +a solution like `scrapy-rotating-proxies`_ to configure multiple proxies in +your spider and have requests rotate through them automatically. + +This approach, however, has serious drawbacks: + +- Free proxies may stop working at any moment. You need to implement a way to + refresh your list of free proxies. + +- In addition to handling occasional bad responses from websites, you + need to handle all kinds of bad responses from proxies. You may even need + to inspect the response body to determine if a response comes from the + target website or from a misbehaving proxy. + +- Advanced antibot solutions may automatically detect and filter out traffic + from free proxies. + +.. _scrapy-rotating-proxies: https://github.com/TeamHG-Memex/scrapy-rotating-proxies + + +.. _custom-rotating-proxy: + +Custom rotating proxy +--------------------- + +If you have spare servers, you can set them up as proxies and use scrapoxy_ to +build a custom proxy that rotates traffic through them. However, the initial +setup can be complex, and your requests will be vulnerable to +:ref:`internet service provider filtering `. + +.. _scrapoxy: https://scrapoxy.io/ + + +.. _tor: + +The Tor network +--------------- + +It is possible to send requests through the `Tor network`_. + +The initial setup to have Scrapy working with Tor is not straightforward. +Use a search engine to find up-to-date documentation specific to using +Scrapy and Tor together. + +The main drawback of using the Tor network is that traffic can be extremely +slow. + +.. _Tor network: https://en.wikipedia.org/wiki/Tor_(anonymity_network) + + +.. _commercial-support: + +Seeking professional help +========================= + +Avoiding bans, filters and challenges can be difficult and tricky, and may +sometimes require special infrastructure. + +If you find yourself unable to prevent your spider from getting bad responses, +consider contacting `commercial support`_. + +.. _commercial support: https://scrapy.org/support/ diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 732eba587..a7a6fd129 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -226,39 +226,5 @@ crawl:: curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2 curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3 -.. _bans: -Avoiding getting banned -======================= - -Some websites implement certain measures to prevent bots from crawling them, -with varying degrees of sophistication. Getting around those measures can be -difficult and tricky, and may sometimes require special infrastructure. Please -consider contacting `commercial support`_ if in doubt. - -Here are some tips to keep in mind when dealing with these kinds of sites: - -* rotate your user agent from a pool of well-known ones from browsers (google - around to get a list of them) -* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use - cookies to spot bot behaviour -* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting. -* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites - directly -* use a pool of rotating IPs. For example, the free `Tor project`_ or paid - services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a - super proxy that you can attach your own proxies to. -* use a highly distributed downloader that circumvents bans internally, so you - can just focus on parsing clean pages. One example of such downloaders is - `Zyte Smart Proxy Manager`_ - -If you are still unable to prevent your bot getting banned, consider contacting -`commercial support`_. - -.. _Tor project: https://www.torproject.org/ -.. _commercial support: https://scrapy.org/support/ -.. _ProxyMesh: https://proxymesh.com/ -.. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders -.. _scrapoxy: https://scrapoxy.io/ -.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ From 572d347b3bc0149042f04ea83aff4a4f8fc7a831 Mon Sep 17 00:00:00 2001 From: databender Date: Wed, 18 Aug 2021 16:17:52 +0530 Subject: [PATCH 0381/2083] warning view updated --- docs/topics/downloader-middleware.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 928a59bf1..99d57bda9 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1070,9 +1070,8 @@ In order to use this parser: * Install `Reppy `_ by running ``pip install reppy`` -.. warning:: `Upstream issue #122 - `_ prevents reppy usage in - Python 3.9+. + .. warning:: `Upstream issue #122 + `_ prevents reppy usage in Python 3.9+. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` From bbeed6ae8fd9aed3651b104e4cc3e56495e1b1b9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 19 Aug 2021 14:09:30 -0300 Subject: [PATCH 0382/2083] CaseInsensitiveDict: preserve original keys (only lookups are key-insensitive) --- scrapy/utils/datatypes.py | 36 ++++++++++++++++++++++------------- tests/test_utils_datatypes.py | 13 +++++++++++-- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index ca6089e0f..1d56811f0 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -79,35 +79,45 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class CaseInsensitiveDict(collections.UserDict): +class CaseInsensitiveDict(collections.UserDict,): """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups. It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods. """ + def __init__(self, *args, **kwargs) -> None: + self._keys: dict = {} + super().__init__(*args, **kwargs) + def __getitem__(self, key: AnyStr) -> Any: - return super().__getitem__(self.normkey(key)) + normalized_key = self.normkey(key) + return super().__getitem__(self._keys[normalized_key.lower()]) def __setitem__(self, key: AnyStr, value: Any) -> None: - super().__setitem__(self.normkey(key), self.normvalue(value)) + normalized_key = self.normkey(key) + if normalized_key.lower() in self._keys: + del self[self._keys[normalized_key.lower()]] + super().__setitem__(normalized_key, self.normvalue(value)) + self._keys[normalized_key.lower()] = normalized_key def __delitem__(self, key: AnyStr) -> None: - super().__delitem__(self.normkey(key)) + normalized_key = self.normkey(key) + stored_key = self._keys.pop(normalized_key.lower()) + super().__delitem__(stored_key) def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] - return super().__contains__(self.normkey(key)) - - def normkey(self, key: AnyStr) -> AnyStr: - """Method to normalize dictionary key access""" - return key.lower() - - def normvalue(self, value: Any) -> Any: - """Method to normalize values prior to be set""" - return value + normalized_key = self.normkey(key) + return normalized_key.lower() in self._keys def __repr__(self) -> str: return f"<{self.__class__.__name__}: {super().__repr__()}>" + def normkey(self, key: AnyStr) -> AnyStr: + return key + + def normvalue(self, value: Any) -> Any: + return value + class LocalCache(collections.OrderedDict): """Dictionary with a finite number of keys. diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index c033cd537..5faaabe81 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,4 +1,5 @@ import copy +from typing import Iterator import unittest import warnings from collections.abc import Mapping, MutableMapping @@ -191,8 +192,16 @@ class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): dict_class = CaseInsensitiveDict def test_repr(self): - d = self.dict_class({"foo": "bar"}) - self.assertEqual(repr(d), "") + d1 = self.dict_class({"foo": "bar"}) + self.assertEqual(repr(d1), "") + d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) + self.assertEqual(repr(d2), "") + + def test_iter(self): + d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) + iterkeys = iter(d) + self.assertIsInstance(iterkeys, Iterator) + self.assertEqual(list(iterkeys), ["AsDf", "FoO"]) class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): From 10ebf6384ed58253c237224d523e602b1f3c2224 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 19 Aug 2021 14:12:55 -0300 Subject: [PATCH 0383/2083] Remove unnecessary comma --- scrapy/utils/datatypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 1d56811f0..6eeabe1ee 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -79,7 +79,7 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class CaseInsensitiveDict(collections.UserDict,): +class CaseInsensitiveDict(collections.UserDict): """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups. It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods. From cd17c829cf0d7a006ab5594d56fe182a0ffc71d6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Aug 2021 19:55:35 +0500 Subject: [PATCH 0384/2083] Revert "Move documentation about avoiding bans into a topic of its own (#4039)" This reverts commit 2d2581c68f35799dc4372a257eaa8dbb5208481d. --- docs/index.rst | 4 - docs/topics/avoiding-bans.rst | 340 ---------------------------------- docs/topics/practices.rst | 34 ++++ 3 files changed, 34 insertions(+), 344 deletions(-) delete mode 100644 docs/topics/avoiding-bans.rst diff --git a/docs/index.rst b/docs/index.rst index 7647b3781..433798aa8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -155,7 +155,6 @@ Solving specific problems topics/debug topics/contracts topics/practices - topics/avoiding-bans topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -180,9 +179,6 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. -:doc:`topics/avoiding-bans` - Avoid getting banned from websites. - :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. diff --git a/docs/topics/avoiding-bans.rst b/docs/topics/avoiding-bans.rst deleted file mode 100644 index 59f0da191..000000000 --- a/docs/topics/avoiding-bans.rst +++ /dev/null @@ -1,340 +0,0 @@ -.. _bans: - -============= -Avoiding bans -============= - -This topic covers some of the strategies that you can follow to avoid getting -different or bad responses from a website that you are crawling due to filters -such as regional filters, web browser filters, etc. - -.. _avoiding-crawls: - -Avoiding crawls -=============== - -The best way not to be banned from a website is not to send requests to it in -the first place. - -One way to avoid crawling a website is to find the desired dataset through -other means. For example, you can use Google’s `dataset search engine`_. - -If the target website is the only or best source of the desired information, -and you only need to extract the data on a monthly basis or a lower frequency, -you may be able to crawl a public snapshot of the target website instead. -`Common Crawl`_ is an open repository of web crawl data that you can access -freely. It contains monthly snapshots of a wide variety of websites and, if you -are lucky, your target website will be among them. - -.. _Common Crawl: https://commoncrawl.org/ -.. _dataset search engine: https://datasetsearch.research.google.com/ - - -.. _being-polite: - -Being polite -============ - -To avoid being banned, you should first avoid giving a website reasons to ban -you. - -.. _identifying-yourself: - -Identifying yourself --------------------- - -If your crawling has a noticeable negative impact on a website or you crawl -content that should not be crawled, website administrators will need to do -something. - -Set :setting:`USER_AGENT` to a value that uniquely identifies your spider and -includes contact information, so that website administrators can contact you. - - -.. _following-robotstxt: - -Following robots.txt guidelines -------------------------------- - -Some websites provide a ``robots.txt`` file at their root path (e.g. -``http://example.com/robots.txt``) that describes the guidelines that they wish -bots to follow when crawling their website. - -Before you start writing a spider for a website, read their ``robots.txt`` -file and implement your spider following its guidelines. See the `robots.txt -standard draft`_ or the `robots.txt Google specification`_ for information on -how to read ``robots.txt`` files. - -To ensure that your spider does not crawl pages restricted by ``robots.txt`` -guidelines, set :setting:`ROBOTSTXT_OBEY` to ``True`` to enable the -:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` -middleware. When you do, if your spider attempts to crawl a restricted page, -this middleware aborts that request with the following message:: - - Forbidden by robots.txt - -Also set :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and -:setting:`DOWNLOAD_DELAY` to values that comply with the ``Crawl-Delay`` or -``Request-Rate`` directives from the ``robots.txt`` guidelines. - -You may also use the :ref:`AutoThrottle extension ` on top -of that, so that when the target website experiences a high load, your spider -automatically switches to higher download delays. - -.. _robots.txt Google specification: https://developers.google.com/search/reference/robots_txt -.. _robots.txt standard draft: https://tools.ietf.org/html/draft-koster-rep-00 - - -.. _choosing-crawl-speed: - -Finding the right guidelines on your own ----------------------------------------- - -If a website does not specify a desired download delay, or does not provide a -``robots.txt`` file, you should make an effort to find out the right values for -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY` that -will not have a noticeable negative impact on the target website. - -Use a service like `SimilarWeb`_ to find out the amount of monthly traffic that -the target website receives, and choose concurrency and delay values that will -not cause a noticeable traffic increase. - -.. _SimilarWeb: https://www.similarweb.com - - -.. _filters-and-challenges: - -Bypassing filters and solving challenges -======================================== - -Some websites implement filters and challenges that aim to deny access or alter -their content based on aspects of the visitor, such as the country where they -are or the web browsing tool they use. - -.. _regional-filter: - -Bypassing regional filters --------------------------- - -Some websites send different or bad responses based on the region or country -associated to your `IP address`_. - -To bypass these filters, get access to a `proxy server`_ that has an outgoing -IP address from a region that gets the desired responses. - -Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` -middleware to configure your spider to use that proxy. - -.. _IP address: https://en.wikipedia.org/wiki/IP_address -.. _proxy server: https://en.wikipedia.org/wiki/Proxy_server - - -.. _web-browser-filter: - -Bypassing web browser filters ------------------------------ - -Some websites send different or bad responses if they detect that your request -does not come from a web browser. - -To bypass these filters, switch your :setting:`USER_AGENT` to a value copied -from those that popular web browsers use. In some rare cases, you may need a -user agent string from a specific web browser. - -There are multiple Scrapy plugins that can rotate your requests through popular -web browser user agent strings, such as scrapy-fake-useragent_, -scrapy-random-useragent_ or Scrapy-UserAgents_. - -For advanced web browser filters, -:ref:`pre-rendering JavaScript ` or -:ref:`using a headless browser ` may be necessary. -Use these options only as a last resort, however, because they cause a higher -load per request on the target website. - -.. _scrapy-fake-useragent: https://github.com/alecxe/scrapy-fake-useragent -.. _scrapy-random-useragent: https://github.com/cleocn/scrapy-random-useragent -.. _Scrapy-UserAgents: https://pypi.org/project/Scrapy-UserAgents/ - - -.. _request-delay-filter: - -Bypassing request delay filters -------------------------------- - -Some websites may ban your IP after they detect that your requests use a -constant download delay. - -To help bypassing these filters, the :setting:`RANDOMIZE_DOWNLOAD_DELAY` -setting is enabled by default. When that is not enough, an -:ref:`IP address rotation solution ` may be much more effective. - - -.. _isp-filter: - -Bypassing internet service provider filters -------------------------------------------- - -Some websites send different or bad responses if they detect that your request -comes from an IP address that belongs to a `data center`_, as opposed to a -residential IP address from an `internet service provider`_ or a mobile IP -address from a `mobile network`_. - -To bypass these filters, get access to a proxy server that has an outgoing IP -address that is either residential or mobile. Note that you may also get -different responses depending on whether your IP address is residential or -mobile. - -Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` -middleware to configure your spider to use that proxy. - -.. _data center: https://en.wikipedia.org/wiki/Data_center -.. _internet service provider: https://en.wikipedia.org/wiki/Internet_service_provider -.. _mobile network: https://en.wikipedia.org/wiki/Cellular_network - - -.. _captcha: - -Solving CAPTCHA challenges --------------------------- - -Some websites require you to solve a `CAPTCHA challenge`_ to get the desired -response. - -To bypass these filters, several options exist: - -- You could have your spider present the CAPTCHA challenge to you and wait - for you to solve it manually. - -- Some CAPTCHA challenges can be solved using an `optical character - recognition`_ (OCR) solution such as pytesseract_. - -- Paid CAPTCHA solving services exist. - -Whichever solution you choose, implement it as a :ref:`downloader middleware -` that automatically detects CAPTCHA challenges -in responses and solves them, so that your spider code only receives successful -responses. - -.. _CAPTCHA challenge: https://en.wikipedia.org/wiki/CAPTCHA -.. _optical character recognition: https://en.wikipedia.org/wiki/Optical_character_recognition -.. _pytesseract: https://github.com/madmaze/pytesseract - - -.. _ip-rotation: - -IP address rotation solutions -============================= - -See below some of the different solutions there are to have your requests use -different outgoing IP addresses. - -When using this approach, remember to set :setting:`COOKIES_ENABLED` to -``False`` to disable global cookie handling. This prevents websites from -identifying two requests as coming from the same user agent even if they come -from different IP addresses and have different user-agent strings. You can -still include some cookies manually in your requests. Define them through the -``Cookies`` header of your requests. See -:class:`Request.headers `. - -.. _smart-proxy: - -Smart proxies -------------- - -An increasing number of websites use solutions that apply many of the above -filters and challenges at the same time. - -There are paid proxy services, like `Zyte Smart Proxy Manager`_, that -automatically bypass website filters and challenges, so that your spider only -gets successful responses. They also allow managing sessions to simulate user -behavior. - -For Zyte Smart Proxy Manager, installing scrapy-crawlera_ will offer advanced -integration with Scrapy. For other services, use the -:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` middleware -or implement your own :ref:`downloader middleware -`. - -.. _scrapy-crawlera: https://scrapy-crawlera.readthedocs.io/en/latest/ -.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ - - -.. _rotating-proxy: - -Rotating proxies ----------------- - -Rotating proxy services like ProxyMesh_ send different requests through -different proxies. This can decrease the likelihood of being affected by some -filters or challenges. - -.. _ProxyMesh: https://proxymesh.com/ - - -.. _free-proxies: - -Free proxies ------------- - -You can easily find lists of free proxies in the internet, and you can use -a solution like `scrapy-rotating-proxies`_ to configure multiple proxies in -your spider and have requests rotate through them automatically. - -This approach, however, has serious drawbacks: - -- Free proxies may stop working at any moment. You need to implement a way to - refresh your list of free proxies. - -- In addition to handling occasional bad responses from websites, you - need to handle all kinds of bad responses from proxies. You may even need - to inspect the response body to determine if a response comes from the - target website or from a misbehaving proxy. - -- Advanced antibot solutions may automatically detect and filter out traffic - from free proxies. - -.. _scrapy-rotating-proxies: https://github.com/TeamHG-Memex/scrapy-rotating-proxies - - -.. _custom-rotating-proxy: - -Custom rotating proxy ---------------------- - -If you have spare servers, you can set them up as proxies and use scrapoxy_ to -build a custom proxy that rotates traffic through them. However, the initial -setup can be complex, and your requests will be vulnerable to -:ref:`internet service provider filtering `. - -.. _scrapoxy: https://scrapoxy.io/ - - -.. _tor: - -The Tor network ---------------- - -It is possible to send requests through the `Tor network`_. - -The initial setup to have Scrapy working with Tor is not straightforward. -Use a search engine to find up-to-date documentation specific to using -Scrapy and Tor together. - -The main drawback of using the Tor network is that traffic can be extremely -slow. - -.. _Tor network: https://en.wikipedia.org/wiki/Tor_(anonymity_network) - - -.. _commercial-support: - -Seeking professional help -========================= - -Avoiding bans, filters and challenges can be difficult and tricky, and may -sometimes require special infrastructure. - -If you find yourself unable to prevent your spider from getting bad responses, -consider contacting `commercial support`_. - -.. _commercial support: https://scrapy.org/support/ diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index a7a6fd129..732eba587 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -226,5 +226,39 @@ crawl:: curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2 curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3 +.. _bans: +Avoiding getting banned +======================= + +Some websites implement certain measures to prevent bots from crawling them, +with varying degrees of sophistication. Getting around those measures can be +difficult and tricky, and may sometimes require special infrastructure. Please +consider contacting `commercial support`_ if in doubt. + +Here are some tips to keep in mind when dealing with these kinds of sites: + +* rotate your user agent from a pool of well-known ones from browsers (google + around to get a list of them) +* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use + cookies to spot bot behaviour +* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting. +* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites + directly +* use a pool of rotating IPs. For example, the free `Tor project`_ or paid + services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a + super proxy that you can attach your own proxies to. +* use a highly distributed downloader that circumvents bans internally, so you + can just focus on parsing clean pages. One example of such downloaders is + `Zyte Smart Proxy Manager`_ + +If you are still unable to prevent your bot getting banned, consider contacting +`commercial support`_. + +.. _Tor project: https://www.torproject.org/ +.. _commercial support: https://scrapy.org/support/ +.. _ProxyMesh: https://proxymesh.com/ +.. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders +.. _scrapoxy: https://scrapoxy.io/ +.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ From 3f635eb683821667b7a46a99531a95a8c05b6e1b Mon Sep 17 00:00:00 2001 From: "Matsievskiy S.V" Date: Tue, 24 Aug 2021 12:05:50 +0300 Subject: [PATCH 0385/2083] Extract domain from genspider URL (#4439) --- scrapy/commands/genspider.py | 12 +++++++++++- tests/test_commands.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 5f44daa70..2082a4974 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,6 +4,7 @@ import string from importlib import import_module from os.path import join, dirname, abspath, exists, splitext +from urllib.parse import urlparse import scrapy from scrapy.commands import ScrapyCommand @@ -22,6 +23,14 @@ def sanitize_module_name(module_name): return module_name +def extract_domain(url): + """Extract domain name from URL string""" + o = urlparse(url) + if o.scheme == '' and o.netloc == '': + o = urlparse("//" + url.lstrip("/")) + return o.netloc + + class Command(ScrapyCommand): requires_project = False @@ -59,7 +68,8 @@ class Command(ScrapyCommand): if len(args) != 2: raise UsageError() - name, domain = args[0:2] + name, url = args[0:2] + domain = extract_domain(url) module = sanitize_module_name(name) if self.settings.get('BOT_NAME') == module: diff --git a/tests/test_commands.py b/tests/test_commands.py index 74b917d93..086286b3a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,6 +3,7 @@ import json import optparse import os import platform +import re import subprocess import sys import tempfile @@ -94,6 +95,15 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) + def find_in_file(self, filename, regex): + """Find first pattern occurrence in file""" + pattern = re.compile(regex) + with open(filename, "r") as f: + for line in f: + match = pattern.search(line) + if match is not None: + return match + class StartprojectTest(ProjectTest): @@ -482,6 +492,26 @@ class GenspiderCommandTest(CommandTest): def test_same_filename_as_existing_spider_force(self): self.test_same_filename_as_existing_spider(force=True) + def test_url(self, url='test.com', domain="test.com"): + self.assertEqual(0, self.call('genspider', '--force', 'test_name', url)) + self.assertEqual(domain, + self.find_in_file(join(self.proj_mod_path, + 'spiders', 'test_name.py'), + r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1)) + self.assertEqual('http://%s/' % domain, + self.find_in_file(join(self.proj_mod_path, + 'spiders', 'test_name.py'), + r'start_urls\s*=\s*\[\'(.+)\'\]').group(1)) + + def test_url_schema(self): + self.test_url('http://test.com', 'test.com') + + def test_url_path(self): + self.test_url('test.com/some/other/page', 'test.com') + + def test_url_schema_path(self): + self.test_url('https://test.com/some/other/page', 'test.com') + class GenspiderStandaloneCommandTest(ProjectTest): From 43ea21e8306bc66e8f07abc84cce680726abc7dc Mon Sep 17 00:00:00 2001 From: D R Siddhartha Date: Tue, 24 Aug 2021 15:18:01 +0530 Subject: [PATCH 0386/2083] Feed post-processing plugin support (#5190) --- .github/workflows/tests-ubuntu.yml | 2 +- docs/topics/feed-exports.rst | 67 +++- scrapy/extensions/feedexport.py | 4 + scrapy/extensions/postprocessing.py | 154 +++++++++ tests/test_feedexport.py | 496 ++++++++++++++++++++++++++++ 5 files changed, 721 insertions(+), 2 deletions(-) create mode 100644 scrapy/extensions/postprocessing.py diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 81beda5da..ef1c8362f 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -20,7 +20,7 @@ jobs: - python-version: pypy3 env: TOXENV: pypy3 - PYPY_VERSION: 3.6-v7.3.1 + PYPY_VERSION: 3.6-v7.3.3 # pinned deps - python-version: 3.6.12 diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2b3217d62..116967280 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -272,6 +272,7 @@ in multiple files, with the specified maximum item count per file. That way, as soon as a file reaches the maximum item count, that file is delivered to the feed URI, allowing item delivery to start way before the end of the crawl. + .. _item-filter: Item filtering @@ -312,6 +313,63 @@ ItemFilter :members: +.. _post-processing: + +Post-Processing +=============== + +.. versionadded:: VERSION + +Scrapy provides an option to activate plugins to post-process feeds before they are exported +to feed storages. In addition to using :ref:`builtin plugins `, you +can create your own :ref:`plugins `. + +These plugins can be activated through the ``postprocessing`` option of a feed. +The option must be passed a list of post-processing plugins in the order you want +the feed to be processed. These plugins can be declared either as an import string +or with the imported class of the plugin. Parameters to plugins can be passed +through the feed options. See :ref:`feed options ` for examples. + +.. _builtin-plugins: + +Built-in Plugins +---------------- + +.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin + +.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin + +.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin + +.. _custom-plugins: + +Custom Plugins +-------------- + +Each plugin is a class that must implement the following methods: + +.. method:: __init__(self, file, feed_options) + + Initialize the plugin. + + :param file: file-like object having at least the `write`, `tell` and `close` methods implemented + + :param feed_options: feed-specific :ref:`options ` + :type feed_options: :class:`dict` + +.. method:: write(self, data) + + Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file. + It must return number of bytes written. + +.. method:: close(self) + + Close the target file object. + +To pass a parameter to your plugin, use :ref:`feed options `. You +can then access those parameters from the ``__init__`` method of your plugin. + + Settings ======== @@ -368,10 +426,12 @@ For instance:: 'encoding': 'latin1', 'indent': 8, }, - pathlib.Path('items.csv'): { + pathlib.Path('items.csv.gz'): { 'format': 'csv', 'fields': ['price', 'name'], 'item_filter': 'myproject.filters.MyCustomFilter2', + 'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 5, }, } @@ -435,6 +495,11 @@ as a fallback value if that key is not provided for a specific feed definition: - ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. +- ``postprocessing``: list of :ref:`plugins ` to use for post-processing. + + The plugins will be used in the order of the list passed. + + .. versionadded:: VERSION .. setting:: FEED_EXPORT_ENCODING diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 564c736f2..0f5bf01d0 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -20,6 +20,7 @@ from zope.interface import implementer, Interface from scrapy import signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file @@ -396,6 +397,9 @@ class FeedExporter: """ storage = self._get_storage(uri, feed_options) file = storage.open(spider) + if "postprocessing" in feed_options: + file = PostProcessingManager(feed_options["postprocessing"], file, feed_options) + exporter = self._get_exporter( file=file, format=feed_options['format'], diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py new file mode 100644 index 000000000..413c2e55e --- /dev/null +++ b/scrapy/extensions/postprocessing.py @@ -0,0 +1,154 @@ +""" +Extension for processing data before they are exported to feeds. +""" +from bz2 import BZ2File +from gzip import GzipFile +from io import IOBase +from lzma import LZMAFile +from typing import Any, BinaryIO, Dict, List + +from scrapy.utils.misc import load_object + + +class GzipPlugin: + """ + Compresses received data using `gzip `_. + + Accepted ``feed_options`` parameters: + + - `gzip_compresslevel` + - `gzip_mtime` + - `gzip_filename` + + See :py:class:`gzip.GzipFile` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + compress_level = self.feed_options.get("gzip_compresslevel", 9) + mtime = self.feed_options.get("gzip_mtime") + filename = self.feed_options.get("gzip_filename") + self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level, + mtime=mtime, filename=filename) + + def write(self, data: bytes) -> int: + return self.gzipfile.write(data) + + def close(self) -> None: + self.gzipfile.close() + self.file.close() + + +class Bz2Plugin: + """ + Compresses received data using `bz2 `_. + + Accepted ``feed_options`` parameters: + + - `bz2_compresslevel` + + See :py:class:`bz2.BZ2File` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + compress_level = self.feed_options.get("bz2_compresslevel", 9) + self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level) + + def write(self, data: bytes) -> int: + return self.bz2file.write(data) + + def close(self) -> None: + self.bz2file.close() + self.file.close() + + +class LZMAPlugin: + """ + Compresses received data using `lzma `_. + + Accepted ``feed_options`` parameters: + + - `lzma_format` + - `lzma_check` + - `lzma_preset` + - `lzma_filters` + + .. note:: + ``lzma_filters`` cannot be used in pypy version 7.3.1 and older. + + See :py:class:`lzma.LZMAFile` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + + format = self.feed_options.get("lzma_format") + check = self.feed_options.get("lzma_check", -1) + preset = self.feed_options.get("lzma_preset") + filters = self.feed_options.get("lzma_filters") + self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format, + check=check, preset=preset, filters=filters) + + def write(self, data: bytes) -> int: + return self.lzmafile.write(data) + + def close(self) -> None: + self.lzmafile.close() + self.file.close() + + +# io.IOBase is subclassed here, so that exporters can use the PostProcessingManager +# instance as a file like writable object. This could be needed by some exporters +# such as CsvItemExporter which wraps the feed storage with io.TextIOWrapper. +class PostProcessingManager(IOBase): + """ + This will manage and use declared plugins to process data in a + pipeline-ish way. + :param plugins: all the declared plugins for the feed + :type plugins: list + :param file: final target file where the processed data will be written + :type file: file like object + """ + + def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.plugins = self._load_plugins(plugins) + self.file = file + self.feed_options = feed_options + self.head_plugin = self._get_head_plugin() + + def write(self, data: bytes) -> int: + """ + Uses all the declared plugins to process data first, then writes + the processed data to target file. + :param data: data passed to be written to target file + :type data: bytes + :return: returns number of bytes written + :rtype: int + """ + return self.head_plugin.write(data) + + def tell(self) -> int: + return self.file.tell() + + def close(self) -> None: + """ + Close the target file along with all the plugins. + """ + self.head_plugin.close() + + def writable(self) -> bool: + return True + + def _load_plugins(self, plugins: List[Any]) -> List[Any]: + plugins = [load_object(plugin) for plugin in plugins] + return plugins + + def _get_head_plugin(self) -> Any: + prev = self.file + for plugin in self.plugins[::-1]: + prev = plugin(prev, self.feed_options) + return prev diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 53e6a2018..253f3119c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,5 +1,8 @@ +import bz2 import csv +import gzip import json +import lzma import os import random import shutil @@ -1473,6 +1476,499 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(row['expected'], data[feed_options['format']]) +class FeedPostProcessedExportsTest(FeedExportTestBase): + __test__ = True + + items = [{'foo': 'bar'}] + expected = b'foo\r\nbar\r\n' + + class MyPlugin1: + def __init__(self, file, feed_options): + self.file = file + self.feed_options = feed_options + self.char = self.feed_options.get('plugin1_char', b'') + + def write(self, data): + written_count = self.file.write(data) + written_count += self.file.write(self.char) + return written_count + + def close(self): + self.file.close() + + def _named_tempfile(self, name): + return os.path.join(self.temp_dir, name) + + @defer.inlineCallbacks + def run_and_export(self, spider_cls, settings): + """ Run spider with specified settings; return exported data with filename. """ + + FEEDS = settings.get('FEEDS') or {} + settings['FEEDS'] = { + printf_escape(path_to_url(file_path)): feed_options + for file_path, feed_options in FEEDS.items() + } + + content = {} + try: + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + spider_cls.start_urls = [s.url('/')] + yield runner.crawl(spider_cls) + + for file_path, feed_options in FEEDS.items(): + if not os.path.exists(str(file_path)): + continue + + with open(str(file_path), 'rb') as f: + content[str(file_path)] = f.read() + + finally: + for file_path in FEEDS.keys(): + if not os.path.exists(str(file_path)): + continue + + os.remove(str(file_path)) + + return content + + def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=''): + data_stream = BytesIO() + gzipf = gzip.GzipFile(fileobj=data_stream, filename=filename, mtime=mtime, + compresslevel=compresslevel, mode="wb") + gzipf.write(data) + gzipf.close() + data_stream.seek(0) + return data_stream.read() + + @defer.inlineCallbacks + def test_gzip_plugin(self): + + filename = self._named_tempfile('gzip_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + gzip.decompress(data[filename]) + except OSError: + self.fail("Received invalid gzip data.") + + @defer.inlineCallbacks + def test_gzip_plugin_compresslevel(self): + + filename_to_compressed = { + self._named_tempfile('compresslevel_0'): self.get_gzip_compressed(self.expected, compresslevel=0), + self._named_tempfile('compresslevel_9'): self.get_gzip_compressed(self.expected, compresslevel=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('compresslevel_0'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 0, + 'gzip_mtime': 0, + 'gzip_filename': "", + }, + self._named_tempfile('compresslevel_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 9, + 'gzip_mtime': 0, + 'gzip_filename': "", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_gzip_plugin_mtime(self): + filename_to_compressed = { + self._named_tempfile('mtime_123'): self.get_gzip_compressed(self.expected, mtime=123), + self._named_tempfile('mtime_123456789'): self.get_gzip_compressed(self.expected, mtime=123456789), + } + + settings = { + 'FEEDS': { + self._named_tempfile('mtime_123'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 123, + 'gzip_filename': "", + }, + self._named_tempfile('mtime_123456789'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 123456789, + 'gzip_filename': "", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_gzip_plugin_filename(self): + filename_to_compressed = { + self._named_tempfile('filename_FILE1'): self.get_gzip_compressed(self.expected, filename="FILE1"), + self._named_tempfile('filename_FILE2'): self.get_gzip_compressed(self.expected, filename="FILE2"), + } + + settings = { + 'FEEDS': { + self._named_tempfile('filename_FILE1'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 0, + 'gzip_filename': "FILE1", + }, + self._named_tempfile('filename_FILE2'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 0, + 'gzip_filename': "FILE2", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin(self): + + filename = self._named_tempfile('lzma_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + lzma.decompress(data[filename]) + except lzma.LZMAError: + self.fail("Received invalid lzma data.") + + @defer.inlineCallbacks + def test_lzma_plugin_format(self): + + filename_to_compressed = { + self._named_tempfile('format_FORMAT_XZ'): lzma.compress(self.expected, format=lzma.FORMAT_XZ), + self._named_tempfile('format_FORMAT_ALONE'): lzma.compress(self.expected, format=lzma.FORMAT_ALONE), + } + + settings = { + 'FEEDS': { + self._named_tempfile('format_FORMAT_XZ'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_format': lzma.FORMAT_XZ, + }, + self._named_tempfile('format_FORMAT_ALONE'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_format': lzma.FORMAT_ALONE, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_check(self): + + filename_to_compressed = { + self._named_tempfile('check_CHECK_NONE'): lzma.compress(self.expected, check=lzma.CHECK_NONE), + self._named_tempfile('check_CHECK_CRC256'): lzma.compress(self.expected, check=lzma.CHECK_SHA256), + } + + settings = { + 'FEEDS': { + self._named_tempfile('check_CHECK_NONE'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_check': lzma.CHECK_NONE, + }, + self._named_tempfile('check_CHECK_CRC256'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_check': lzma.CHECK_SHA256, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_preset(self): + + filename_to_compressed = { + self._named_tempfile('preset_PRESET_0'): lzma.compress(self.expected, preset=0), + self._named_tempfile('preset_PRESET_9'): lzma.compress(self.expected, preset=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('preset_PRESET_0'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_preset': 0, + }, + self._named_tempfile('preset_PRESET_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_preset': 9, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_filters(self): + import sys + if "PyPy" in sys.version: + # https://foss.heptapod.net/pypy/pypy/-/issues/3527 + raise unittest.SkipTest("lzma filters doesn't work in PyPy") + + filters = [{'id': lzma.FILTER_LZMA2}] + compressed = lzma.compress(self.expected, filters=filters) + filename = self._named_tempfile('filters') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_filters': filters, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(compressed, data[filename]) + result = lzma.decompress(data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_bz2_plugin(self): + + filename = self._named_tempfile('bz2_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + bz2.decompress(data[filename]) + except OSError: + self.fail("Received invalid bz2 data.") + + @defer.inlineCallbacks + def test_bz2_plugin_compresslevel(self): + + filename_to_compressed = { + self._named_tempfile('compresslevel_1'): bz2.compress(self.expected, compresslevel=1), + self._named_tempfile('compresslevel_9'): bz2.compress(self.expected, compresslevel=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('compresslevel_1'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + 'bz2_compresslevel': 1, + }, + self._named_tempfile('compresslevel_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + 'bz2_compresslevel': 9, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = bz2.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_custom_plugin(self): + filename = self._named_tempfile('csv_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(self.expected, data[filename]) + + @defer.inlineCallbacks + def test_custom_plugin_with_parameter(self): + + expected = b'foo\r\n\nbar\r\n\n' + filename = self._named_tempfile('newline') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + 'plugin1_char': b'\n' + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(expected, data[filename]) + + @defer.inlineCallbacks + def test_custom_plugin_with_compression(self): + + expected = b'foo\r\n\nbar\r\n\n' + + filename_to_decompressor = { + self._named_tempfile('bz2'): bz2.decompress, + self._named_tempfile('lzma'): lzma.decompress, + self._named_tempfile('gzip'): gzip.decompress, + } + + settings = { + 'FEEDS': { + self._named_tempfile('bz2'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.Bz2Plugin'], + 'plugin1_char': b'\n', + }, + self._named_tempfile('lzma'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.LZMAPlugin'], + 'plugin1_char': b'\n', + }, + self._named_tempfile('gzip'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'], + 'plugin1_char': b'\n', + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, decompressor in filename_to_decompressor.items(): + result = decompressor(data[filename]) + self.assertEqual(expected, result) + + @defer.inlineCallbacks + def test_exports_compatibility_with_postproc(self): + import marshal + import pickle + filename_to_expected = { + self._named_tempfile('csv'): b'foo\r\nbar\r\n', + self._named_tempfile('json'): b'[\n{"foo": "bar"}\n]', + self._named_tempfile('jsonlines'): b'{"foo": "bar"}\n', + self._named_tempfile('xml'): b'\n' + b'\nbar\n', + } + + settings = { + 'FEEDS': { + self._named_tempfile('csv'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + # empty plugin to activate postprocessing.PostProcessingManager + }, + self._named_tempfile('json'): { + 'format': 'json', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('jsonlines'): { + 'format': 'jsonlines', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('xml'): { + 'format': 'xml', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('marshal'): { + 'format': 'marshal', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('pickle'): { + 'format': 'pickle', + 'postprocessing': [self.MyPlugin1], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, result in data.items(): + if 'pickle' in filename: + expected, result = self.items[0], pickle.loads(result) + elif 'marshal' in filename: + expected, result = self.items[0], marshal.loads(result) + else: + expected = filename_to_expected[filename] + self.assertEqual(expected, result) + + class BatchDeliveriesTest(FeedExportTestBase): __test__ = True _file_mark = '_%(batch_time)s_#%(batch_id)02d_' From 8284de5e7613c47e69d40d4f6a2a1dc846b50dd6 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 24 Aug 2021 15:15:29 +0500 Subject: [PATCH 0387/2083] Fix/silence the Pylint messages added in 2.10 (#5235) --- pylintrc | 1 + scrapy/utils/conf.py | 2 +- scrapy/utils/deprecate.py | 2 +- tests/keys/__init__.py | 8 ++++---- tests/test_downloader_handlers_http2.py | 2 +- tests/test_engine.py | 2 +- tests/test_exporters.py | 4 ++-- tests/test_http2_client_protocol.py | 2 +- tests/test_loader_deprecated.py | 2 +- tests/test_request_cb_kwargs.py | 2 +- tests/test_scheduler.py | 10 +++++----- tests/test_utils_conf.py | 4 ++-- tests/test_utils_misc/__init__.py | 2 +- 13 files changed, 22 insertions(+), 21 deletions(-) diff --git a/pylintrc b/pylintrc index a44712507..699686e16 100644 --- a/pylintrc +++ b/pylintrc @@ -105,6 +105,7 @@ disable=abstract-method, unnecessary-lambda, unnecessary-pass, unreachable, + unspecified-encoding, unsubscriptable-object, unused-argument, unused-import, diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index b904c4a03..24873f75d 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -121,7 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings): out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) - out.setdefault("item_export_kwargs", dict()) + out.setdefault("item_export_kwargs", {}) if settings["FEED_EXPORT_INDENT"] is None: out.setdefault("indent", None) else: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index f5b17416f..ae727464c 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -79,7 +79,7 @@ def create_deprecated_class( # for implementation details def __instancecheck__(cls, inst): return any(cls.__subclasscheck__(c) - for c in {type(inst), inst.__class__}) + for c in (type(inst), inst.__class__)) def __subclasscheck__(cls, sub): if cls is not DeprecatedClass.deprecated_class: diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index da202be4d..bb4a8e5af 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -40,9 +40,9 @@ def generate_keys(): subject = issuer = Name( [ - NameAttribute(NameOID.COUNTRY_NAME, u"IE"), - NameAttribute(NameOID.ORGANIZATION_NAME, u"Scrapy"), - NameAttribute(NameOID.COMMON_NAME, u"localhost"), + NameAttribute(NameOID.COUNTRY_NAME, "IE"), + NameAttribute(NameOID.ORGANIZATION_NAME, "Scrapy"), + NameAttribute(NameOID.COMMON_NAME, "localhost"), ] ) cert = ( @@ -54,7 +54,7 @@ def generate_keys(): .not_valid_before(datetime.utcnow()) .not_valid_after(datetime.utcnow() + timedelta(days=10)) .add_extension( - SubjectAlternativeName([DNSName(u"localhost")]), + SubjectAlternativeName([DNSName("localhost")]), critical=False, ) .sign(key, SHA256(), default_backend()) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 53bb4fe92..8c8c30597 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -219,7 +219,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): certfile = 'keys/localhost.crt' scheme = 'https' - host = u'127.0.0.1' + host = '127.0.0.1' expected_http_proxy_request_body = b'/' diff --git a/tests/test_engine.py b/tests/test_engine.py index 92bf45f25..fa7d0c8d4 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -152,7 +152,7 @@ class CrawlerRun: self.itemerror = [] self.itemresp = [] self.headers = {} - self.bytes = defaultdict(lambda: list()) + self.bytes = defaultdict(list) self.signals_caught = {} self.spider_class = spider_class diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 04bae31d3..b263b3475 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -362,14 +362,14 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_errors_default(self): with self.assertRaises(UnicodeEncodeError): self.assertExportResult( - item=dict(text=u'W\u0275\u200Brd'), + item=dict(text='W\u0275\u200Brd'), expected=None, encoding='windows-1251', ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item=dict(text=u'W\u0275\u200Brd'), + item=dict(text='W\u0275\u200Brd'), include_headers_line=False, expected='Wɵ​rd\r\n', encoding='windows-1251', diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 677ede92b..49c83132f 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -201,7 +201,7 @@ class Https2ClientProtocolTestCase(TestCase): self.site = Site(root, timeout=None) # Start server for testing - self.hostname = u'localhost' + self.hostname = 'localhost' context_factory = ssl_context_factory(self.key_file, self.certificate_file) server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 41afa2896..0fd52da5f 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -703,7 +703,7 @@ class DeprecatedUtilityFunctionsTestCase(unittest.TestCase): return None with warnings.catch_warnings(record=True) as w: - wrap_loader_context(function, context=dict()) + wrap_loader_context(function, context={}) assert len(w) == 1 assert issubclass(w[0].category, ScrapyDeprecationWarning) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index b68184b87..738502de8 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -57,7 +57,7 @@ class KeywordArgumentsSpider(MockServerSpider): }, } - checks = list() + checks = [] def start_requests(self): data = {'key': 'value', 'number': 123} diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 512a7460e..2d4bfa165 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -22,7 +22,7 @@ MockSlot = collections.namedtuple('MockSlot', ['active']) class MockDownloader: def __init__(self): - self.slots = dict() + self.slots = {} def _get_slot_key(self, request, spider): if Downloader.DOWNLOAD_SLOT in request.meta: @@ -31,7 +31,7 @@ class MockDownloader: return urlparse_cached(request).hostname or '' def increment(self, slot_key): - slot = self.slots.setdefault(slot_key, MockSlot(active=list())) + slot = self.slots.setdefault(slot_key, MockSlot(active=[])) slot.active.append(1) def decrement(self, slot_key): @@ -114,7 +114,7 @@ class BaseSchedulerInMemoryTester(SchedulerHandler): for url, priority in _PRIORITIES: self.scheduler.enqueue_request(Request(url, priority=priority)) - priorities = list() + priorities = [] while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) @@ -167,7 +167,7 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): self.close_scheduler() self.create_scheduler() - priorities = list() + priorities = [] while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) @@ -259,7 +259,7 @@ class DownloaderAwareSchedulerTestMixin: self.close_scheduler() self.create_scheduler() - dequeued_slots = list() + dequeued_slots = [] requests = [] downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index dc2560add..a92880626 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -176,7 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": (1, 2, 3, 4), "batch_item_count": 2, - "item_export_kwargs": dict(), + "item_export_kwargs": {}, }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -199,7 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": None, "batch_item_count": 2, - "item_export_kwargs": dict(), + "item_export_kwargs": {}, }) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 47d73a2dd..b83c1d6f0 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -27,7 +27,7 @@ class UtilsMiscTestCase(unittest.TestCase): def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') - self.assertRaises(TypeError, load_object, dict()) + self.assertRaises(TypeError, load_object, {}) def test_walk_modules(self): mods = walk_modules('tests.test_utils_misc.test_walk_modules') From ac9175964dda07da1e838ebb063fc8dd95925e0d Mon Sep 17 00:00:00 2001 From: maanijou <19888963+maanijou@users.noreply.github.com> Date: Sun, 12 Sep 2021 17:59:20 +0200 Subject: [PATCH 0388/2083] Improve documentation for spider middlewares --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f0158dc41..f3fb0d5d7 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` objects and :ref:`item object + iterable of :class:`~scrapy.Request` objects or :ref:`item object `. If it returns ``None``, Scrapy will continue processing this exception, From e5998fb8469dff1e2e965826d2b43f9bad0c17ad Mon Sep 17 00:00:00 2001 From: kamran890 Date: Wed, 22 Sep 2021 03:00:18 +0500 Subject: [PATCH 0389/2083] Document spider.state attribute (#5174) --- docs/topics/jobs.rst | 2 ++ docs/topics/spiders.rst | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index e49f37a2f..f16d306c7 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command:: scrapy crawl somespider -s JOBDIR=crawls/somespider-1 +.. _topics-keeping-persistent-state-between-batches: + Keeping persistent state between batches ======================================== diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 67b9e2e0e..4d3d32941 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -122,6 +122,11 @@ scrapy.Spider send log messages through it as described on :ref:`topics-logging-from-spiders`. + .. attribute:: state + + A dict you can use to persist some spider state between batches. + See :ref:`topics-keeping-persistent-state-between-batches` to know more about it. + .. method:: from_crawler(crawler, *args, **kwargs) This is the class method used by Scrapy to create your spiders. From 1829dd774ca0f056e97f7c4621575ef9126e4b57 Mon Sep 17 00:00:00 2001 From: "Reza (Milad) Maanijou" <19888963+maanijou@users.noreply.github.com> Date: Sat, 25 Sep 2021 20:22:09 +0330 Subject: [PATCH 0390/2083] Update spider-middleware.rst --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f3fb0d5d7..08be2b03c 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` objects or :ref:`item object + iterable of :class:`~scrapy.Request` objects or :ref:`item objects `. If it returns ``None``, Scrapy will continue processing this exception, From dfdb779756aa1df2059980de02d359e4e93bac55 Mon Sep 17 00:00:00 2001 From: "Reza (Milad) Maanijou" <19888963+maanijou@users.noreply.github.com> Date: Sun, 26 Sep 2021 12:45:44 +0330 Subject: [PATCH 0391/2083] Apply review comments --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 08be2b03c..f1373b9ee 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,7 +122,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` objects or :ref:`item objects + iterable of :class:`~scrapy.Request` or :ref:`item objects `. If it returns ``None``, Scrapy will continue processing this exception, From 3c57825b0f3a7bb250aec753b09f964f36500e2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 26 Sep 2021 13:41:26 +0200 Subject: [PATCH 0392/2083] Update docs/topics/spider-middleware.rst --- docs/topics/spider-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f1373b9ee..73bedf655 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,8 +122,8 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` or :ref:`item objects - `. + iterable of :class:`~scrapy.Request` or :ref:`item ` + objects. If it returns ``None``, Scrapy will continue processing this exception, executing any other :meth:`process_spider_exception` in the following From 74f146bbe0c41e2c21f431c00ff34d6c5d10cb63 Mon Sep 17 00:00:00 2001 From: Deepanshu <73387559+iDeepverma@users.noreply.github.com> Date: Fri, 1 Oct 2021 01:47:05 +0530 Subject: [PATCH 0393/2083] Document update URLLENGTH_LIMIT --- docs/topics/settings.rst | 8 ++++++-- docs/topics/spider-middleware.rst | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2ab2020fa..4d3ae20cc 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1645,8 +1645,12 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` -The maximum URL length to allow for crawled URLs. For more information about -the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 +The maximum URL length to allow for crawled URLs. You can set this to ``0`` +to disable :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` for working +with URLs longer than the default value. The default limit acts as a stopping condition in case of +URLs of increasing length, usually caused by a loop. +For more information about the default value +for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 .. setting:: USER_AGENT diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 73bedf655..a0a7b1fb6 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -440,4 +440,5 @@ UrlLengthMiddleware settings (see the settings documentation for more info): * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. + If ``0``, then :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is disabled. From 890f884de46602352de48fa844edd9959c62e473 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Fri, 1 Oct 2021 04:50:42 -0300 Subject: [PATCH 0394/2083] Allow 'callback' key in keyword arguments for request callbacks (#5251) --- scrapy/core/scraper.py | 2 +- tests/test_request_cb_kwargs.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index d6d6f64f9..f40bccbb3 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -156,7 +156,7 @@ class Scraper: callback = result.request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) dfd = defer_succeed(result) - dfd.addCallback(callback, **result.request.cb_kwargs) + dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs) else: # result is a Failure result.request = request warn_on_generator_with_return_value(spider, request.errback) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 738502de8..8b96fe1a1 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -60,7 +60,7 @@ class KeywordArgumentsSpider(MockServerSpider): checks = [] def start_requests(self): - data = {'key': 'value', 'number': 123} + data = {'key': 'value', 'number': 123, 'callback': 'some_callback'} yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data) yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data) yield Request(self.mockserver.url('/general_without'), self.parse_general) @@ -88,7 +88,8 @@ class KeywordArgumentsSpider(MockServerSpider): if response.url.endswith('/general_with'): self.checks.append(kwargs['key'] == 'value') self.checks.append(kwargs['number'] == 123) - self.crawler.stats.inc_value('boolean_checks', 2) + self.checks.append(kwargs['callback'] == 'some_callback') + self.crawler.stats.inc_value('boolean_checks', 3) elif response.url.endswith('/general_without'): self.checks.append(kwargs == {}) self.crawler.stats.inc_value('boolean_checks') @@ -110,7 +111,7 @@ class KeywordArgumentsSpider(MockServerSpider): TypeError: parse_takes_less() got an unexpected keyword argument 'number' """ - def parse_takes_more(self, response, key, number, other): + def parse_takes_more(self, response, key, number, callback, other): """ Should raise TypeError: parse_takes_more() missing 1 required positional argument: 'other' @@ -161,11 +162,13 @@ class CallbackKeywordArgumentsTestCase(TestCase): self.assertTrue( str(exceptions['takes_less'].exc_info[1]).endswith( "parse_takes_less() got an unexpected keyword argument 'number'" - ) + ), + msg="Exception message: " + str(exceptions['takes_less'].exc_info[1]), ) self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) self.assertTrue( str(exceptions['takes_more'].exc_info[1]).endswith( "parse_takes_more() missing 1 required positional argument: 'other'" - ) + ), + msg="Exception message: " + str(exceptions['takes_more'].exc_info[1]), ) From fbb1236fd6ee283414360f5209b3a569e112c8cb Mon Sep 17 00:00:00 2001 From: Deepanshu verma <73387559+iDeepverma@users.noreply.github.com> Date: Fri, 1 Oct 2021 18:46:11 +0530 Subject: [PATCH 0395/2083] Update docs/topics/settings.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit added suggestion Co-authored-by: Adrián Chaves --- docs/topics/settings.rst | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4d3ae20cc..ed8f1f105 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1645,12 +1645,19 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` -The maximum URL length to allow for crawled URLs. You can set this to ``0`` -to disable :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` for working -with URLs longer than the default value. The default limit acts as a stopping condition in case of -URLs of increasing length, usually caused by a loop. -For more information about the default value -for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 +The maximum URL length to allow for crawled URLs. + +This setting can act as a stopping condition in case of URLs of ever-increasing +length, which may be caused for example by a programming error either in the +target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and +:setting:`DEPTH_LIMIT`. + +Use ``0`` to allow URLs of any length. + +The default value is copied from the `Microsoft Internet Explorer maximum URL +length`_, even though this setting exists for different reasons. + +.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 .. setting:: USER_AGENT From d91d82b5064c512e741da106b5fb3a398027d5a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Oct 2021 16:31:29 +0200 Subject: [PATCH 0396/2083] Make Scrapy SFW again --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d3e08efd4..42ce22158 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -300,7 +300,7 @@ errors if needed:: "http://www.httpbin.org/status/404", # Not found error "http://www.httpbin.org/status/500", # server issue "http://www.httpbin.org:12345/", # non-responding host, timeout expected - "http://www.httphttpbinbin.org/", # DNS error expected + "https://example.invalid/", # DNS error expected ] def start_requests(self): From de2043f9c1661208de7b73305a8e3a2395d29583 Mon Sep 17 00:00:00 2001 From: Deepanshu <73387559+iDeepverma@users.noreply.github.com> Date: Fri, 1 Oct 2021 20:20:00 +0530 Subject: [PATCH 0397/2083] updated docs/topics/spider-middleware.rst --- docs/topics/spider-middleware.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a0a7b1fb6..f27bc79c0 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -440,5 +440,3 @@ UrlLengthMiddleware settings (see the settings documentation for more info): * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. - If ``0``, then :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is disabled. - From 47533985f4dc7e58895e1a34c3ea88502f83572a Mon Sep 17 00:00:00 2001 From: Peter Morrison Date: Fri, 1 Oct 2021 12:30:14 -0600 Subject: [PATCH 0398/2083] Document file expiration method in media-pipeline --- docs/topics/media-pipeline.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 46bd2859b..10d2ac990 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -383,6 +383,9 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key: and pipeline class MyPipeline will have expiration time set to 180. +The last modified time from the file is used to determine the age of the file in days, +which is then compared to the set expiration time to determine if the file is expired. + .. _topics-images-thumbnails: Thumbnail generation for images From de70b3c58b5b4b2f95468218c194b1e4b99a33c4 Mon Sep 17 00:00:00 2001 From: Deepanshu verma <73387559+iDeepverma@users.noreply.github.com> Date: Sat, 2 Oct 2021 12:12:58 +0530 Subject: [PATCH 0399/2083] Update spider-middleware.rst added empty line --- docs/topics/spider-middleware.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f27bc79c0..3545e760b 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -440,3 +440,4 @@ UrlLengthMiddleware settings (see the settings documentation for more info): * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. + From f10880022273c005a6cb6d9e6ff9cc9a370cc375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 2 Oct 2021 13:25:15 +0200 Subject: [PATCH 0400/2083] Update spider-middleware.rst --- docs/topics/spider-middleware.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 3545e760b..f27bc79c0 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -440,4 +440,3 @@ UrlLengthMiddleware settings (see the settings documentation for more info): * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. - From ef263042d75586e94d74290d1a41c432f7d87bec Mon Sep 17 00:00:00 2001 From: Raihan Nismara <31585789+raihan71@users.noreply.github.com> Date: Sun, 3 Oct 2021 13:26:20 +0700 Subject: [PATCH 0401/2083] Using Logo Scrapy in Readme.md Logo scrapy used in readme.md made looks nicer --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 5750e2c0f..05f10bb6c 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,6 @@ +.. image:: /artwork/scrapy-logo.jpg + :width: 400px + ====== Scrapy ====== From b9647b85d3bc9dcefc7d829ce8304bb28f7cc798 Mon Sep 17 00:00:00 2001 From: Ryan Whelchel Date: Sun, 3 Oct 2021 17:32:38 -0400 Subject: [PATCH 0402/2083] docs: restructed phrasing for clarity --- docs/intro/tutorial.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 438f3d6df..ba27b18bc 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -277,9 +277,9 @@ As an alternative, you could've written: >>> response.css('title::text')[0].get() 'Quotes to Scrape' -However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList` -instance avoids an ``IndexError`` and returns ``None`` when it doesn't -find any element matching the selection. +Directly accessing an index on a :class:`~scrapy.selector.SelectorList` +instance could potentially run into an ``IndexError``. It is recommended to use +``.get()`` directly instead as it avoids such index errors. There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail From 764cf0178bb7bc346f1c15ed7ab0adcbb75c43b0 Mon Sep 17 00:00:00 2001 From: Ryan Whelchel Date: Tue, 5 Oct 2021 10:22:57 -0400 Subject: [PATCH 0403/2083] Update docs/intro/tutorial.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/intro/tutorial.rst | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index ba27b18bc..fa321a770 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -277,9 +277,20 @@ As an alternative, you could've written: >>> response.css('title::text')[0].get() 'Quotes to Scrape' -Directly accessing an index on a :class:`~scrapy.selector.SelectorList` -instance could potentially run into an ``IndexError``. It is recommended to use -``.get()`` directly instead as it avoids such index errors. +Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will +raise an :exc:`IndexError` exception if there are no results:: + + >>> response.css('noelement')[0].get() + Traceback (most recent call last): + File "", line 1, in + ... + IndexError: list index out of range + +You might want to use ``.get()`` directly on the +:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` +if there are no results:: + +>>> response.css("noelement").get() There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail From b081f18a2f232a1bfd04c829ae6894cac93a89a1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 16 Aug 2019 14:53:42 +0500 Subject: [PATCH 0404/2083] Add http_auth_domain to HttpAuthMiddleware. --- docs/topics/downloader-middleware.rst | 18 ++++- scrapy/downloadermiddlewares/httpauth.py | 21 ++++- tests/test_downloadermiddleware_httpauth.py | 87 ++++++++++++++++++++- 3 files changed, 119 insertions(+), 7 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 99d57bda9..28b019c80 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -323,8 +323,21 @@ HttpAuthMiddleware This middleware authenticates all requests generated from certain spiders using `Basic access authentication`_ (aka. HTTP auth). - To enable HTTP authentication from certain spiders, set the ``http_user`` - and ``http_pass`` attributes of those spiders. + To enable HTTP authentication for a spider, set the ``http_user`` and + ``http_pass`` spider attributes to the authentication data and the + ``http_auth_domain`` spider attribute to the domain which requires this + authentication (its subdomains will be also handled in the same way). + You can set ``http_auth_domain`` to ``None`` to enable the + authentication for all requests but usually this is not needed. + + .. warning:: + In the previous Scrapy versions HttpAuthMiddleware sent the + authentication data with all requests, which is a security problem if + the spider makes requests to several different domains. Currently if + the ``http_auth_domain`` attribute is not set, the middleware will use + the domain of the first request, which will work for some spider but + not for others. In the future the middleware will produce an error + instead. Example:: @@ -334,6 +347,7 @@ HttpAuthMiddleware http_user = 'someuser' http_pass = 'somepass' + http_auth_domain = 'intranet.example.com' name = 'intranet.example.com' # .. rest of the spider code omitted ... diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 089bf0d85..1bee3e279 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -3,10 +3,14 @@ HTTP basic auth downloader middleware See documentation in docs/topics/downloader-middleware.rst """ +import warnings from w3lib.http import basic_auth_header from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.url import url_is_from_any_domain class HttpAuthMiddleware: @@ -24,8 +28,23 @@ class HttpAuthMiddleware: pwd = getattr(spider, 'http_pass', '') if usr or pwd: self.auth = basic_auth_header(usr, pwd) + if not hasattr(spider, 'http_auth_domain'): + warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security ' + 'problems if the spider makes requests to several different domains. http_auth_domain ' + 'will be set to the domain of the first request, please set it to the correct value ' + 'explicitly.', + category=ScrapyDeprecationWarning) + self.domain_unset = True + else: + self.domain = spider.http_auth_domain + self.domain_unset = False def process_request(self, request, spider): auth = getattr(self, 'auth', None) if auth and b'Authorization' not in request.headers: - request.headers[b'Authorization'] = auth + domain = urlparse_cached(request).hostname + if self.domain_unset: + self.domain = domain + self.domain_unset = False + if not self.domain or url_is_from_any_domain(request.url, [self.domain]): + request.headers[b'Authorization'] = auth diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 3381632b0..0362e2018 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,13 +1,60 @@ import unittest +from w3lib.http import basic_auth_header + from scrapy.http import Request from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.spiders import Spider +class TestSpiderLegacy(Spider): + http_user = 'foo' + http_pass = 'bar' + + class TestSpider(Spider): http_user = 'foo' http_pass = 'bar' + http_auth_domain = 'example.com' + + +class TestSpiderAny(Spider): + http_user = 'foo' + http_pass = 'bar' + http_auth_domain = None + + +class HttpAuthMiddlewareLegacyTest(unittest.TestCase): + + def setUp(self): + self.spider = TestSpiderLegacy('foo') + + def test_auth(self): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) + + # initial request, sets the domain and sends the header + req = Request('http://example.com/') + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + # subsequent request to the same domain, should send the header + req = Request('http://example.com/') + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + # subsequent request to a different domain, shouldn't send the header + req = Request('http://example-noauth.com/') + assert mw.process_request(req, self.spider) is None + self.assertNotIn('Authorization', req.headers) + + def test_auth_already_set(self): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) + req = Request('http://example.com/', + headers=dict(Authorization='Digest 123')) + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], b'Digest 123') class HttpAuthMiddlewareTest(unittest.TestCase): @@ -20,13 +67,45 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def tearDown(self): del self.mw - def test_auth(self): - req = Request('http://scrapytest.org/') + def test_no_auth(self): + req = Request('http://example-noauth.com/') assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') + self.assertNotIn('Authorization', req.headers) + + def test_auth_domain(self): + req = Request('http://example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + def test_auth_subdomain(self): + req = Request('http://foo.example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) def test_auth_already_set(self): - req = Request('http://scrapytest.org/', + req = Request('http://example.com/', + headers=dict(Authorization='Digest 123')) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], b'Digest 123') + + +class HttpAuthAnyMiddlewareTest(unittest.TestCase): + + def setUp(self): + self.mw = HttpAuthMiddleware() + self.spider = TestSpiderAny('foo') + self.mw.spider_opened(self.spider) + + def tearDown(self): + del self.mw + + def test_auth(self): + req = Request('http://example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + def test_auth_already_set(self): + req = Request('http://example.com/', headers=dict(Authorization='Digest 123')) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers['Authorization'], b'Digest 123') From 7ec5f299c42d07768c02970f0a11f018ed790188 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 22 Aug 2019 20:32:56 +0500 Subject: [PATCH 0405/2083] Small documentation fixes. --- docs/topics/downloader-middleware.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 28b019c80..caf44a903 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -328,16 +328,16 @@ HttpAuthMiddleware ``http_auth_domain`` spider attribute to the domain which requires this authentication (its subdomains will be also handled in the same way). You can set ``http_auth_domain`` to ``None`` to enable the - authentication for all requests but usually this is not needed. + authentication for all requests but you risk leaking your authentication + credentials to unrelated domains. .. warning:: - In the previous Scrapy versions HttpAuthMiddleware sent the - authentication data with all requests, which is a security problem if - the spider makes requests to several different domains. Currently if - the ``http_auth_domain`` attribute is not set, the middleware will use - the domain of the first request, which will work for some spider but - not for others. In the future the middleware will produce an error - instead. + In previous Scrapy versions HttpAuthMiddleware sent the authentication + data with all requests, which is a security problem if the spider + makes requests to several different domains. Currently if the + ``http_auth_domain`` attribute is not set, the middleware will use the + domain of the first request, which will work for some spiders but not + for others. In the future the middleware will produce an error instead. Example:: From f0105a882df200f71088603fecaeb9d40679c387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 5 Oct 2021 13:29:06 +0200 Subject: [PATCH 0406/2083] Cover 2.5.1 in the release notes --- docs/news.rst | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 0ea412e75..4b5cbb2da 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,44 @@ Release notes ============= +.. _release-2.5.1: + +Scrapy 2.5.1 (2021-10-05) +------------------------- + +* **Security bug fix:** + + If you use + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` + (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP + authentication, any request exposes your credentials to the request target. + + To prevent unintended exposure of authentication credentials to unintended + domains, you must now additionally set a new, additional spider attribute, + ``http_auth_domain``, and point it to the specific domain to which the + authentication credentials must be sent. + + If the ``http_auth_domain`` spider attribute is not set, the domain of the + first request will be considered the HTTP authentication target, and + authentication credentials will only be sent in requests targeting that + domain. + + If you need to send the same HTTP authentication credentials to multiple + domains, you can use :func:`w3lib.http.basic_auth_header` instead to + set the value of the ``Authorization`` header of your requests. + + If you *really* want your spider to send the same HTTP authentication + credentials to any domain, set the ``http_auth_domain`` spider attribute + to ``None``. + + Finally, if you are a user of `scrapy-splash`_, know that this version of + Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will + need to upgrade scrapy-splash to a greater version for it to continue to + work. + +.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash + + .. _release-2.5.0: Scrapy 2.5.0 (2021-04-06) From 735750c254e6e82af46b4ebbb35e28b8c0a52250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 5 Oct 2021 21:10:49 +0200 Subject: [PATCH 0407/2083] Cover 1.8.1 in the release notes --- docs/news.rst | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 4b5cbb2da..5e590f027 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1492,6 +1492,44 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.1: + +Scrapy 1.8.1 (2021-10-05) +------------------------- + +* **Security bug fix:** + + If you use + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` + (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP + authentication, any request exposes your credentials to the request target. + + To prevent unintended exposure of authentication credentials to unintended + domains, you must now additionally set a new, additional spider attribute, + ``http_auth_domain``, and point it to the specific domain to which the + authentication credentials must be sent. + + If the ``http_auth_domain`` spider attribute is not set, the domain of the + first request will be considered the HTTP authentication target, and + authentication credentials will only be sent in requests targeting that + domain. + + If you need to send the same HTTP authentication credentials to multiple + domains, you can use :func:`w3lib.http.basic_auth_header` instead to + set the value of the ``Authorization`` header of your requests. + + If you *really* want your spider to send the same HTTP authentication + credentials to any domain, set the ``http_auth_domain`` spider attribute + to ``None``. + + Finally, if you are a user of `scrapy-splash`_, know that this version of + Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will + need to upgrade scrapy-splash to a greater version for it to continue to + work. + +.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash + + .. _release-1.8.0: Scrapy 1.8.0 (2019-10-28) From 6c858cec91b013853a73e6215b74c90b609bc2da Mon Sep 17 00:00:00 2001 From: Laerte <5853172+Laerte@users.noreply.github.com> Date: Wed, 6 Oct 2021 12:32:04 -0300 Subject: [PATCH 0408/2083] Cookies: Cast primitive types to str (#5253) * cast primitive types to str * add tests --- scrapy/downloadermiddlewares/cookies.py | 4 ++-- tests/test_downloadermiddleware_cookies.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index d95ed3d38..0eee8d758 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -80,8 +80,8 @@ class CookiesMiddleware: logger.warning(msg.format(request, cookie, key)) return continue - if isinstance(cookie[key], str): - decoded[key] = cookie[key] + if isinstance(cookie[key], (bool, float, int, str)): + decoded[key] = str(cookie[key]) else: try: decoded[key] = cookie[key].decode("utf8") diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index aff8542e9..36021bfbf 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -347,3 +347,24 @@ class CookiesMiddlewareTest(TestCase): self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1') self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2') self.assertCookieValEqual(req3.headers['Cookie'], 'key=') + + def test_primitive_type_cookies(self): + # Boolean + req1 = Request('http://example.org', cookies={'a': True}) + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], b'a=True') + + # Float + req2 = Request('http://example.org', cookies={'a': 9.5}) + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], b'a=9.5') + + # Integer + req3 = Request('http://example.org', cookies={'a': 10}) + assert self.mw.process_request(req3, self.spider) is None + self.assertCookieValEqual(req3.headers['Cookie'], b'a=10') + + # String + req4 = Request('http://example.org', cookies={'a': 'b'}) + assert self.mw.process_request(req4, self.spider) is None + self.assertCookieValEqual(req4.headers['Cookie'], b'a=b') From b1cb007b3b8fef6f037cd1abd38fe9da7190ed26 Mon Sep 17 00:00:00 2001 From: MarvinPetzoldt <78762153+MarvinPetzoldt@users.noreply.github.com> Date: Wed, 6 Oct 2021 19:08:19 +0200 Subject: [PATCH 0409/2083] Fixed documentation example --- docs/topics/exporters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 8c30122b6..923336769 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -122,7 +122,7 @@ Example:: class ProductXmlExporter(XmlItemExporter): def serialize_field(self, field, name, value): - if field == 'price': + if name == 'price': return f'$ {str(value)}' return super().serialize_field(field, name, value) From 029cab72e8a9b20ebb6b9540e9e01747f0e83dba Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Wed, 6 Oct 2021 14:34:09 -0300 Subject: [PATCH 0410/2083] [CI] fix pypy test (#5264) --- tests/test_request_cb_kwargs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 8b96fe1a1..473a93e69 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -105,7 +105,7 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(default == 99) self.crawler.stats.inc_value('boolean_checks', 4) - def parse_takes_less(self, response, key): + def parse_takes_less(self, response, key, callback): """ Should raise TypeError: parse_takes_less() got an unexpected keyword argument 'number' From d3f1bf79e883fe3662df827ee47cfc93a372ff02 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 7 Oct 2021 17:27:20 +0300 Subject: [PATCH 0411/2083] Use f-strings where appropriate (#5246) --- scrapy/__init__.py | 2 +- scrapy/core/downloader/contextfactory.py | 12 +++++---- scrapy/core/downloader/tls.py | 8 +++--- scrapy/downloadermiddlewares/httpcache.py | 2 +- scrapy/extensions/feedexport.py | 33 ++++++++++------------- scrapy/extensions/httpcache.py | 4 +-- scrapy/utils/misc.py | 2 +- tests/spiders.py | 16 +++++------ tests/test_closespider.py | 2 +- tests/test_commands.py | 4 +-- tests/test_downloader_handlers_http2.py | 2 +- tests/test_http_request.py | 19 +++++++------ 12 files changed, 51 insertions(+), 55 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 8a8065bf2..396f98219 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -29,7 +29,7 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version if sys.version_info < (3, 6): - print("Scrapy %s requires Python 3.6+" % __version__) + print(f"Scrapy {__version__} requires Python 3.6+") sys.exit(1) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 073ef16bf..b5318c7bb 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -135,11 +135,13 @@ def load_context_factory_from_settings(settings, crawler): settings=settings, crawler=crawler, ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + msg = ( + f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept " + "a `method` argument (type OpenSSL.SSL method, e.g. " + "OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` " + "argument and/or a `tls_ciphers` argument. Please, upgrade your " + "context factory class to handle them or ignore them." + ) warnings.warn(msg) return context_factory diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 2b8990b75..19a56d9b6 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions): verifyHostname(connection, self._hostnameASCII) except (CertificateError, VerificationError) as e: logger.warning( - 'Remote certificate is not valid for hostname "{}"; {}'.format( - self._hostnameASCII, e)) + 'Remote certificate is not valid for hostname "%s"; %s', + self._hostnameASCII, e) except ValueError as e: logger.warning( 'Ignoring error while verifying certificate ' - 'from host "{}" (exception: {})'.format( - self._hostnameASCII, repr(e))) + 'from host "%s" (exception: %r)', + self._hostnameASCII, e) DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT') diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 62f1c3a29..80ed7ac75 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -70,7 +70,7 @@ class HttpCacheMiddleware: self.stats.inc_value('httpcache/miss', spider=spider) if self.ignore_missing: self.stats.inc_value('httpcache/ignore', spider=spider) - raise IgnoreRequest("Ignored request not in cache: %s" % request) + raise IgnoreRequest(f"Ignored request not in cache: {request}") return None # first time request # Return cached response only if not expired diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0f5bf01d0..370723368 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -38,11 +38,10 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): kwargs['feed_options'] = feed_options else: warnings.warn( - "{} does not support the 'feed_options' keyword argument. Add a " + f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a " "'feed_options' parameter to its signature to remove this " "warning. This parameter will become mandatory in a future " - "version of Scrapy." - .format(builder.__qualname__), + "version of Scrapy.", category=ScrapyDeprecationWarning ) return builder(*preargs, uri, *args, **kwargs) @@ -356,32 +355,28 @@ class FeedExporter: # properly closed. return defer.maybeDeferred(slot.storage.store, slot.file) slot.finish_exporting() - logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" - log_args = {'format': slot.format, - 'itemcount': slot.itemcount, - 'uri': slot.uri} + logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" d = defer.maybeDeferred(slot.storage.store, slot.file) - # Use `largs=log_args` to copy log_args into function's scope - # instead of using `log_args` from the outer scope d.addCallback( - self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__ + self._handle_store_success, logmsg, spider, type(slot.storage).__name__ ) d.addErrback( - self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__ + self._handle_store_error, logmsg, spider, type(slot.storage).__name__ ) return d - def _handle_store_error(self, f, largs, logfmt, spider, slot_type): + def _handle_store_error(self, f, logmsg, spider, slot_type): logger.error( - logfmt % "Error storing", largs, + "Error storing %s", logmsg, exc_info=failure_to_exc_info(f), extra={'spider': spider} ) self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") - def _handle_store_success(self, f, largs, logfmt, spider, slot_type): + def _handle_store_success(self, f, logmsg, spider, slot_type): logger.info( - logfmt % "Stored", largs, extra={'spider': spider} + "Stored %s", logmsg, + extra={'spider': spider} ) self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") @@ -474,10 +469,10 @@ class FeedExporter: for uri_template, values in self.feeds.items(): if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): logger.error( - '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT ' + '%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT ' 'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: ' - 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' - ''.format(uri_template) + 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count', + uri_template ) return False return True @@ -526,7 +521,7 @@ class FeedExporter: instance = build_instance(feedcls) method_name = '__new__' if instance is None: - raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name)) + raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None") return instance def _get_uri_params(self, spider, uri_params, slot=None): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index e0c04b2de..d0ae29b90 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -226,7 +226,7 @@ class DbmCacheStorage: dbpath = os.path.join(self.cachedir, f'{spider.name}.db') self.db = self.dbmodule.open(dbpath, 'c') - logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider}) def close_spider(self, spider): self.db.close() @@ -280,7 +280,7 @@ class FilesystemCacheStorage: self._open = gzip.open if self.use_gzip else open def open_spider(self, spider): - logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}, + logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir}, extra={'spider': spider}) def close_spider(self, spider): diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 51cef1e91..11c4206c2 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -50,7 +50,7 @@ def load_object(path): return path else: raise TypeError("Unexpected argument type, expected string " - "or object, got: %s" % type(path)) + f"or object, got: {type(path)}") try: dot = path.rindex('.') diff --git a/tests/spiders.py b/tests/spiders.py index 5b45f897e..67dbbbe0f 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -86,7 +86,7 @@ class SimpleSpider(MetaSpider): self.start_urls = [url] def parse(self, response): - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefSpider(SimpleSpider): @@ -95,7 +95,7 @@ class AsyncDefSpider(SimpleSpider): async def parse(self, response): await defer.succeed(42) - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioSpider(SimpleSpider): @@ -105,7 +105,7 @@ class AsyncDefAsyncioSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") class AsyncDefAsyncioReturnSpider(SimpleSpider): @@ -115,7 +115,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") return [{'id': 1}, {'id': 2}] @@ -126,7 +126,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.1) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") return {"foo": 42} @@ -138,7 +138,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): await asyncio.sleep(0.2) req_id = response.meta.get('req_id', 0) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d, req_id %d" % (status, req_id)) + self.logger.info(f"Got response {status}, req_id {req_id}") if req_id > 0: return reqs = [] @@ -155,7 +155,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) yield {'foo': 42} - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioGenLoopSpider(SimpleSpider): @@ -166,7 +166,7 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider): for i in range(10): await asyncio.sleep(0.1) yield {'foo': i} - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioGenComplexSpider(SimpleSpider): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 5ec5e2989..be8adadb3 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -41,7 +41,7 @@ class TestCloseSpider(TestCase): yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') - key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__) + key = f'spider_exceptions/{crawler.spider.exception_cls.__name__}' errorcount = crawler.stats.get_value(key) self.assertTrue(errorcount >= close_on) diff --git a/tests/test_commands.py b/tests/test_commands.py index 086286b3a..75098a77a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -498,7 +498,7 @@ class GenspiderCommandTest(CommandTest): self.find_in_file(join(self.proj_mod_path, 'spiders', 'test_name.py'), r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1)) - self.assertEqual('http://%s/' % domain, + self.assertEqual(f'http://{domain}/', self.find_in_file(join(self.proj_mod_path, 'spiders', 'test_name.py'), r'start_urls\s*=\s*\[\'(.+)\'\]').group(1)) @@ -708,7 +708,7 @@ class MySpider(scrapy.Spider): ]) import asyncio loop = asyncio.new_event_loop() - self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log) + self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log) def test_output(self): spider_code = """ diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 8c8c30597..3a9db3ee5 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -248,7 +248,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.body, b'/') - http_proxy = '%s?noconnect' % self.getURL('') + http_proxy = f"{self.getURL('')}?noconnect" request = Request('https://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex( Warning, diff --git a/tests/test_http_request.py b/tests/test_http_request.py index b610087bd..579ef9fa2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1217,18 +1217,17 @@ class FormRequestTest(RequestTest): response, formcss="input[name='abc']") def test_from_response_valid_form_methods(self): - body = """ - - """ + form_methods = [[method, method] for method in self.request_class.valid_form_methods] + form_methods.append(['UNKNOWN', 'GET']) - for method in self.request_class.valid_form_methods: - response = _buildresponse(body % method) + for method, expected in form_methods: + response = _buildresponse( + f'
' + '' + '
' + ) r = self.request_class.from_response(response) - self.assertEqual(r.method, method) - - response = _buildresponse(body % 'UNKNOWN') - r = self.request_class.from_response(response) - self.assertEqual(r.method, 'GET') + self.assertEqual(r.method, expected) def _buildresponse(body, **kwargs): From 65d60b9692dc3475b42d14c744d8a5ac0f2b38cf Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Sun, 10 Oct 2021 05:06:36 -0300 Subject: [PATCH 0412/2083] [docs] add missing parameter to headers_received signal (#5270) --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index a67cc1879..63ad3a9ad 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -413,7 +413,7 @@ headers_received .. versionadded:: 2.5 .. signal:: headers_received -.. function:: headers_received(headers, request, spider) +.. function:: headers_received(headers, body_length, request, spider) Sent by the HTTP 1.1 and S3 download handlers when the response headers are available for a given request, before downloading any additional content. From 6fbd6f941f00037ae4718805352e0fa86a781e41 Mon Sep 17 00:00:00 2001 From: ankur19 Date: Sat, 9 Oct 2021 19:09:51 -0400 Subject: [PATCH 0413/2083] Fix issue#5145 Fix condition for failing tests set Selector to None on AttributeError Add test and remove unused imports Fix imports --- scrapy/loader/__init__.py | 5 ++++- tests/test_loader.py | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 014951a8e..91337b949 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -83,6 +83,9 @@ class ItemLoader(itemloaders.ItemLoader): def __init__(self, item=None, selector=None, response=None, parent=None, **context): if selector is None and response is not None: - selector = self.default_selector_class(response) + try: + selector = self.default_selector_class(response) + except AttributeError: + selector = None context.update(response=response) super().__init__(item=item, selector=selector, parent=parent, **context) diff --git a/tests/test_loader.py b/tests/test_loader.py index b0bc82f4e..f7ab1f236 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -4,7 +4,7 @@ import attr from itemadapter import ItemAdapter from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst -from scrapy.http import HtmlResponse +from scrapy.http import HtmlResponse, Response from scrapy.item import Item, Field from scrapy.loader import ItemLoader from scrapy.selector import Selector @@ -304,6 +304,12 @@ class SelectortemLoaderTest(unittest.TestCase): l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), ['Marta']) + + def test_init_method_with_base_response(self): + """Selector should be None after initialization""" + response = Response("https://scrapy.org") + l = TestItemLoader(response=response) + self.assertIs(l.selector, None) def test_init_method_with_response(self): l = TestItemLoader(response=self.response) From 3a263280bad53a26490381f293a454be6c25ea30 Mon Sep 17 00:00:00 2001 From: "Kian-Meng, Ang" Date: Mon, 11 Oct 2021 22:32:42 +0800 Subject: [PATCH 0414/2083] Fix typos --- docs/news.rst | 10 +++++----- docs/topics/settings.rst | 4 ++-- docs/topics/shell.rst | 2 +- docs/topics/spiders.rst | 4 ++-- docs/versioning.rst | 2 +- extras/qpsclient.py | 2 +- scrapy/downloadermiddlewares/retry.py | 2 +- scrapy/exporters.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/pipelines/files.py | 8 ++++---- scrapy/spiders/feed.py | 4 ++-- scrapy/utils/console.py | 2 +- scrapy/utils/datatypes.py | 2 +- scrapy/utils/defer.py | 2 +- scrapy/utils/request.py | 4 ++-- sep/sep-001.rst | 2 +- sep/sep-005.rst | 2 +- sep/sep-014.rst | 2 +- sep/sep-021.rst | 2 +- tests/test_http_response.py | 4 ++-- tests/test_request_attribute_binding.py | 8 ++++---- tests/test_utils_defer.py | 4 ++-- tests/test_utils_template.py | 2 +- 23 files changed, 39 insertions(+), 39 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 5e590f027..509366c17 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1830,7 +1830,7 @@ New features * A new scheduler priority queue, ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant - scheduling improvement on crawls targetting multiple web domains, at the + scheduling improvement on crawls targeting multiple web domains, at the cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) * A new :attr:`Request.cb_kwargs ` attribute @@ -2868,7 +2868,7 @@ Bug fixes - Fix for selected callbacks when using ``CrawlSpider`` with :command:`scrapy parse ` (:issue:`2225`). - Fix for invalid JSON and XML files when spider yields no items (:issue:`872`). -- Implement ``flush()`` fpr ``StreamLogger`` avoiding a warning in logs (:issue:`2125`). +- Implement ``flush()`` for ``StreamLogger`` avoiding a warning in logs (:issue:`2125`). Refactoring ~~~~~~~~~~~ @@ -3731,7 +3731,7 @@ Scrapy 0.24.3 (2014-08-09) - adding some xpath tips to selectors docs (:commit:`2d103e0`) - fix tests to account for https://github.com/scrapy/w3lib/pull/23 (:commit:`f8d366a`) - get_func_args maximum recursion fix #728 (:commit:`81344ea`) -- Updated input/ouput processor example according to #560. (:commit:`f7c4ea8`) +- Updated input/output processor example according to #560. (:commit:`f7c4ea8`) - Fixed Python syntax in tutorial. (:commit:`db59ed9`) - Add test case for tunneling proxy (:commit:`f090260`) - Bugfix for leaking Proxy-Authorization header to remote host when using tunneling (:commit:`d8793af`) @@ -4393,7 +4393,7 @@ Scrapyd changes ~~~~~~~~~~~~~~~ - Scrapyd now uses one process per spider -- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default) +- It stores one log file per spider run, and rotate them keeping the latest 5 logs per spider (by default) - A minimal web ui was added, available at http://localhost:6800 by default - There is now a ``scrapy server`` command to start a Scrapyd server of the current project @@ -4429,7 +4429,7 @@ New features and improvements - Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195) - Support for overriding default request headers per spider (#181) - Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186) -- Splitted Debian package into two packages - the library and the service (#187) +- Split Debian package into two packages - the library and the service (#187) - Scrapy log refactoring (#188) - New extension for keeping persistent spider contexts among different runs (#203) - Added ``dont_redirect`` request.meta key for avoiding redirects (#233) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2ab2020fa..19a549a02 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1566,7 +1566,7 @@ If a reactor is already installed, :meth:`CrawlerRunner.__init__ ` raises :exc:`Exception` if the installed reactor does not match the -:setting:`TWISTED_REACTOR` setting; therfore, having top-level +:setting:`TWISTED_REACTOR` setting; therefore, having top-level :mod:`~twisted.internet.reactor` imports in project files and imported third-party libraries will make Scrapy raise :exc:`Exception` when it checks which reactor is installed. @@ -1658,7 +1658,7 @@ Default: ``"Scrapy/VERSION (+https://scrapy.org)"`` The default User-Agent to use when crawling, unless overridden. This user agent is also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and -there is no overridding User-Agent header specified for the request. +there is no overriding User-Agent header specified for the request. Settings documented elsewhere: diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 8c90a506c..007e9fc2f 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -99,7 +99,7 @@ Available Shortcuts shortcuts - ``fetch(url[, redirect=True])`` - fetch a new response from the given URL - and update all related objects accordingly. You can optionaly ask for HTTP + and update all related objects accordingly. You can optionally ask for HTTP 3xx redirections to not be followed by passing ``redirect=False`` - ``fetch(request)`` - fetch a new response from the given request and update diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 4d3d32941..99e74233a 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -372,7 +372,7 @@ CrawlSpider described below. If multiple rules match the same link, the first one will be used, according to the order they're defined in this attribute. - This spider also exposes an overrideable method: + This spider also exposes an overridable method: .. method:: parse_start_url(response, **kwargs) @@ -534,7 +534,7 @@ XMLFeedSpider itertag = 'n:url' # ... - Apart from these new attributes, this spider has the following overrideable + Apart from these new attributes, this spider has the following overridable methods too: .. method:: adapt_response(response) diff --git a/docs/versioning.rst b/docs/versioning.rst index 57643ea9a..9d02757b0 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -13,7 +13,7 @@ There are 3 numbers in a Scrapy version: *A.B.C* large changes. * *B* is the release number. This will include many changes including features and things that possibly break backward compatibility, although we strive to - keep theses cases at a minimum. + keep these cases at a minimum. * *C* is the bugfix release number. Backward-incompatibilities are explicitly mentioned in the :ref:`release notes `, diff --git a/extras/qpsclient.py b/extras/qpsclient.py index f9fb70342..28703650d 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -1,5 +1,5 @@ """ -A spider that generate light requests to meassure QPS troughput +A spider that generate light requests to meassure QPS throughput usage: diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index f1fdc3858..c6cc7c56d 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -2,7 +2,7 @@ An extension to retry failed requests that are potentially caused by temporary problems such as a connection timeout or HTTP 500 error. -You can change the behaviour of this middleware by modifing the scraping settings: +You can change the behaviour of this middleware by modifying the scraping settings: RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry diff --git a/scrapy/exporters.py b/scrapy/exporters.py index fb4b565cf..36cca2d05 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -30,7 +30,7 @@ class BaseItemExporter: self._configure(kwargs, dont_fail=dont_fail) def _configure(self, options, dont_fail=False): - """Configure the exporter by poping options from the ``options`` dict. + """Configure the exporter by popping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) """ diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index e941c4321..b5d2585a8 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -88,7 +88,7 @@ class LxmlParserLinkExtractor: def _process_links(self, links): """ Normalize and filter extracted links - The subclass should override it if neccessary + The subclass should override it if necessary """ return self._deduplicate_if_needed(links) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8766ef66f..5c52c6c28 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -85,7 +85,7 @@ class S3FilesStore: AWS_USE_SSL = None AWS_VERIFY = None - POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings + POLICY = 'private' # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings HEADERS = { 'Cache-Control': 'max-age=172800', } @@ -142,7 +142,7 @@ class S3FilesStore: **extra) def _headers_to_botocore_kwargs(self, headers): - """ Convert headers to botocore keyword agruments. + """ Convert headers to botocore keyword arguments. """ # This is required while we need to support both boto and botocore. mapping = CaselessDict({ @@ -190,7 +190,7 @@ class GCSFilesStore: CACHE_CONTROL = 'max-age=172800' # The bucket's default object ACL will be applied to the object. - # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. + # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. POLICY = None def __init__(self, uri): @@ -291,7 +291,7 @@ class FilesPipeline(MediaPipeline): """Abstract pipeline that implement the file downloading This pipeline tries to minimize network transfers and file processing, - doing stat of the files and determining if file is new, uptodate or + doing stat of the files and determining if file is new, up-to-date or expired. ``new`` files are those that pipeline never processed and needs to be diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 6ed17e4dd..bef2d6b24 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -43,7 +43,7 @@ class XMLFeedSpider(Spider): return response def parse_node(self, response, selector): - """This method must be overriden with your custom spider functionality""" + """This method must be overridden with your custom spider functionality""" if hasattr(self, 'parse_item'): # backward compatibility return self.parse_item(response, selector) raise NotImplementedError @@ -113,7 +113,7 @@ class CSVFeedSpider(Spider): return response def parse_row(self, response, row): - """This method must be overriden with your custom spider functionality""" + """This method must be overridden with your custom spider functionality""" raise NotImplementedError def parse_rows(self, response): diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 133261fd7..1bc0bd45f 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -14,7 +14,7 @@ def _embed_ipython_shell(namespace={}, banner=''): @wraps(_embed_ipython_shell) def wrapper(namespace=namespace, banner=''): config = load_default_config() - # Always use .instace() to ensure _instance propagation to all parents + # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports # and clear the instance to always have the fresh env # on repeated breaks like with inspect_response() diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index e31284a7f..47df8a717 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -41,7 +41,7 @@ class CaselessDict(dict): return key.lower() def normvalue(self, value): - """Method to normalize values prior to be setted""" + """Method to normalize values prior to be set""" return value def get(self, key, def_val=None): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index b317c12a3..b02bfdccb 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -34,7 +34,7 @@ def defer_succeed(result) -> Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop - It delays by 100ms so reactor has a chance to go trough readers and writers + It delays by 100ms so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 57dcc5f2c..70ef3ba2b 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -48,7 +48,7 @@ def request_fingerprint( the fingerprint. For this reason, request headers are ignored by default when calculating - the fingeprint. If you want to include specific headers use the + the fingerprint. If you want to include specific headers use the include_headers argument, which is a list of Request headers to include. Also, servers usually ignore fragments in urls when handling requests, @@ -78,7 +78,7 @@ def request_fingerprint( def request_authenticate(request: Request, username: str, password: str) -> None: - """Autenticate the given request (in place) using the HTTP basic access + """Authenticate the given request (in place) using the HTTP basic access authentication mechanism (RFC 2617) and the given username and password """ request.headers['Authorization'] = basic_auth_header(username, password) diff --git a/sep/sep-001.rst b/sep/sep-001.rst index 00226283f..f704e113f 100644 --- a/sep/sep-001.rst +++ b/sep/sep-001.rst @@ -260,7 +260,7 @@ ItemForm ia['width'] = x.x('//p[@class="width"]') ia['volume'] = x.x('//p[@class="volume"]') - # another example passing parametes on instance + # another example passing parameters on instance ia = NewsForm(response, encoding='utf-8') ia['name'] = x.x('//p[@class="name"]') diff --git a/sep/sep-005.rst b/sep/sep-005.rst index e795838e4..08ed367b3 100644 --- a/sep/sep-005.rst +++ b/sep/sep-005.rst @@ -107,7 +107,7 @@ gUsing default_builder This will use default_builder as the builder for every field in the item class. -As a reducer is not set reducers will be set based on Item Field classess. +As a reducer is not set reducers will be set based on Item Field classes. gReset default_builder for a field ================================== diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 8ca81824d..0859e3f7c 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -64,7 +64,7 @@ Request Processors takes requests objects and can perform any action to them, like filtering or modifying on the fly. The current ``LinkExtractor`` had integrated link processing, like -canonicalize. Request Processors can be reutilized and applied in serie. +canonicalize. Request Processors can be reutilized and applied in series. Request Generator ----------------- diff --git a/sep/sep-021.rst b/sep/sep-021.rst index 372429791..c1ec16f7f 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -22,7 +22,7 @@ Instead, the hooks are spread over: * Downloader handlers (DOWNLOADER_HANDLERS) * Item pipelines (ITEM_PIPELINES) * Feed exporters and storages (FEED_EXPORTERS, FEED_STORAGES) -* Overrideable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) +* Overridable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) * Generic extensions (EXTENSIONS) * CLI commands (COMMANDS_MODULE) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index c376a46cd..0ec5257e1 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -19,7 +19,7 @@ class BaseResponseTest(unittest.TestCase): response_class = Response def test_init(self): - # Response requires url in the consturctor + # Response requires url in the constructor self.assertRaises(Exception, self.response_class) self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class)) self.assertRaises(TypeError, self.response_class, b"http://example.com") @@ -392,7 +392,7 @@ class TextResponseTest(BaseResponseTest): def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" r = self.response_class("http://www.example.com", - headers={"Content-type": ["text/html; charset=UKNOWN"]}, + headers={"Content-type": ["text/html; charset=UNKNOWN"]}, body=b"\xc2\xa3") self.assertEqual(r._declared_encoding(), None) self._assert_response_values(r, 'utf-8', "\xa3") diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 00c532c41..25d9657d5 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -106,9 +106,9 @@ class CrawlTestCase(TestCase): """ Downloader middleware which returns a response with an specific 'request' attribute. - * The spider callback should receive the overriden response.request - * Handlers listening to the response_received signal should receive the overriden response.request - * The "crawled" log message should show the overriden response.request + * The spider callback should receive the overridden response.request + * Handlers listening to the response_received signal should receive the overridden response.request + * The "crawled" log message should show the overridden response.request """ signal_params = {} @@ -144,7 +144,7 @@ class CrawlTestCase(TestCase): An exception is raised but caught by the next middleware, which returns a Response with a specific 'request' attribute. - The spider callback should receive the overriden response.request + The spider callback should receive the overridden response.request """ url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 7a5f458c7..032dbc8c5 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -23,7 +23,7 @@ class MustbeDeferredTest(unittest.TestCase): dfd = mustbe_deferred(_append, 1) dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + steps.append(2) # add another value, that should be caught by assertEqual return dfd def test_unfired_deferred(self): @@ -37,7 +37,7 @@ class MustbeDeferredTest(unittest.TestCase): dfd = mustbe_deferred(_append, 1) dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + steps.append(2) # add another value, that should be caught by assertEqual return dfd diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 5ff2e41ef..1d5e63363 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -36,7 +36,7 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): self.assertEqual(result.read().decode('utf8'), rendered) os.remove(render_path) - assert not os.path.exists(render_path) # Failure of test iself + assert not os.path.exists(render_path) # Failure of test itself if '__main__' == __name__: From d08199f631814bdaadafc441bb47cbe71ced2d8d Mon Sep 17 00:00:00 2001 From: Jake Herbst Date: Tue, 12 Oct 2021 13:20:09 -0400 Subject: [PATCH 0415/2083] Removing unnecessary line from docs to prevent test failure (#5274) --- docs/intro/tutorial.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index fa321a770..ca5856881 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -282,7 +282,6 @@ raise an :exc:`IndexError` exception if there are no results:: >>> response.css('noelement')[0].get() Traceback (most recent call last): - File "", line 1, in ... IndexError: list index out of range From e5b057cfd4472e970b9b51757a1b823b2f585b09 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Oct 2021 19:06:51 +0500 Subject: [PATCH 0416/2083] Don't use a HTTP request in a case that will not be awaited. --- tests/test_utils_asyncgen.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 41993a934..d9e6bc2eb 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,3 +1,4 @@ +from twisted.internet.defer import Deferred from twisted.trial import unittest from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal @@ -31,7 +32,10 @@ async def process_iterable(iterable): async def process_iterable_awaiting(iterable): async for i in iterable: yield i * 2 - await get_web_client_agent_req('http://example.com') + d = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d.callback, 42) + await d class ProcessIterableUniversalTest(unittest.TestCase): From a642b73e1a1ba355bae9d5b9d1e87a9da0696293 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Oct 2021 19:21:49 +0500 Subject: [PATCH 0417/2083] Remove an unused import. --- tests/test_utils_asyncgen.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index d9e6bc2eb..7abe17c22 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -3,7 +3,6 @@ from twisted.trial import unittest from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal from scrapy.utils.defer import deferred_f_from_coro_f -from scrapy.utils.test import get_web_client_agent_req class AsyncgenUtilsTest(unittest.TestCase): From 3243aa2cd54c8789eb5667998aa8296f6adaa9a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=AD=E4=B9=9D=E9=BC=8E?= <109224573@qq.com> Date: Thu, 14 Oct 2021 10:18:26 +0800 Subject: [PATCH 0418/2083] docs: fix typo --- docs/topics/loaders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index c0f534493..0d63700c8 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -56,7 +56,7 @@ chapter `:: l.add_xpath('name', '//div[@class="product_name"]') l.add_xpath('name', '//div[@class="product_title"]') l.add_xpath('price', '//p[@id="price"]') - l.add_css('stock', 'p#stock]') + l.add_css('stock', 'p#stock') l.add_value('last_updated', 'today') # you can also use literal values return l.load_item() From ca320feb2afa5eae3990ed21df6a8df930e8cf9f Mon Sep 17 00:00:00 2001 From: Erik Kemperman Date: Fri, 15 Oct 2021 15:43:55 +0200 Subject: [PATCH 0419/2083] Add LOG_FILE_APPEND to settings --- docs/topics/logging.rst | 5 ++++- docs/topics/settings.rst | 10 ++++++++++ scrapy/settings/default_settings.py | 1 + scrapy/utils/log.py | 3 ++- tests/test_crawler.py | 29 ++++++++++++++++++++++++++++- 5 files changed, 45 insertions(+), 3 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index dda04dc4d..d593c74c6 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -143,6 +143,7 @@ Logging settings These settings can be used to configure the logging: * :setting:`LOG_FILE` +* :setting:`LOG_FILE_APPEND` * :setting:`LOG_ENABLED` * :setting:`LOG_ENCODING` * :setting:`LOG_LEVEL` @@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If :setting:`LOG_FILE` is set, messages sent through the root logger will be redirected to a file named :setting:`LOG_FILE` with encoding :setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log -messages will be displayed on the standard error. Lastly, if +messages will be displayed on the standard error. If :setting:`LOG_FILE` is set +and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten +(discarding the output from previous runs, if any). Lastly, if :setting:`LOG_ENABLED` is ``False``, there won't be any visible log output. :setting:`LOG_LEVEL` determines the minimum level of severity to display, those diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e63aca312..210c1def7 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1026,6 +1026,16 @@ Default: ``None`` File name to use for logging output. If ``None``, standard error will be used. +.. setting:: LOG_FILE_APPEND + +LOG_FILE_APPEND +--------------- + +Default: ``True`` + +If ``False``, the log file specified with :setting:`LOG_FILE` will be +overwritten (discarding the output from previous runs, if any). + .. setting:: LOG_FORMAT LOG_FORMAT diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 4ef330dd2..8389a70cb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -207,6 +207,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S' LOG_STDOUT = False LOG_LEVEL = 'DEBUG' LOG_FILE = None +LOG_FILE_APPEND = True LOG_SHORT_NAMES = False SCHEDULER_DEBUG = False diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 6c456ed60..0441c0358 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -124,8 +124,9 @@ def _get_handler(settings): """ Return a log handler object according to settings """ filename = settings.get('LOG_FILE') if filename: + mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w' encoding = settings.get('LOG_ENCODING') - handler = logging.FileHandler(filename, encoding=encoding) + handler = logging.FileHandler(filename, mode=mode, encoding=encoding) elif settings.getbool('LOG_ENABLED'): handler = logging.StreamHandler() else: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index dec517bb6..a80ad4388 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -98,6 +98,8 @@ class CrawlerLoggingTestCase(unittest.TestCase): def test_spider_custom_settings_log_level(self): log_file = self.mktemp() + with open(log_file, 'wb') as fo: + fo.write('previous message\n'.encode('utf-8')) class MySpider(scrapy.Spider): name = 'spider' @@ -119,8 +121,9 @@ class CrawlerLoggingTestCase(unittest.TestCase): logging.error('error message') with open(log_file, 'rb') as fo: - logged = fo.read().decode('utf8') + logged = fo.read().decode('utf-8') + self.assertIn('previous message', logged) self.assertNotIn('debug message', logged) self.assertIn('info message', logged) self.assertIn('warning message', logged) @@ -131,6 +134,30 @@ class CrawlerLoggingTestCase(unittest.TestCase): crawler.stats.get_value('log_count/INFO') - info_count, 1) self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) + def test_spider_custom_settings_log_append(self): + log_file = self.mktemp() + with open(log_file, 'wb') as fo: + fo.write('previous message\n'.encode('utf-8')) + + class MySpider(scrapy.Spider): + name = 'spider' + custom_settings = { + 'LOG_FILE': log_file, + 'LOG_FILE_APPEND': False, + # disable telnet if not available to avoid an extra warning + 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, + } + + configure_logging() + crawler = Crawler(MySpider, {}) + logging.debug('debug message') + + with open(log_file, 'rb') as fo: + logged = fo.read().decode('utf-8') + + self.assertNotIn('previous message', logged) + self.assertIn('debug message', logged) + class SpiderLoaderWithWrongInterface: From 98ee3ddffeeabdc896c3f2cffce8310ebfc97570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Oct 2021 17:18:46 +0200 Subject: [PATCH 0420/2083] Freeze flake8 --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index e274fc8d2..07552ba8d 100644 --- a/tox.ini +++ b/tox.ini @@ -57,6 +57,7 @@ deps = # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 pytest-flake8 + flake8==3.9.2 # https://github.com/tholo/pytest-flake8/issues/81 commands = py.test --flake8 {posargs:docs scrapy tests} From d774d6a9c46cb9697d8fe64a55da9ae321be7539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Oct 2021 17:25:22 +0200 Subject: [PATCH 0421/2083] Remove unused variable --- tests/test_crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index a80ad4388..be067155e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -149,7 +149,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): } configure_logging() - crawler = Crawler(MySpider, {}) + Crawler(MySpider, {}) logging.debug('debug message') with open(log_file, 'rb') as fo: From aec7146e2f870f5e8f0f58bd596b1ec40c64b3c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Raphael=20Tom=C3=A9=20Santana?= Date: Fri, 15 Oct 2021 20:38:53 -0300 Subject: [PATCH 0422/2083] Add how Scrapy is pronounced to the docs --- docs/intro/overview.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index dd80c7bd0..d75f7f636 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -4,7 +4,7 @@ Scrapy at a glance ================== -Scrapy is an application framework for crawling web sites and extracting +Scrapy (pronounced SKRAY-peye /ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting structured data which can be used for a wide range of useful applications, like data mining, information processing or historical archival. From 027ecd8686d7da74b73412e4aa38d8be36b5b9f1 Mon Sep 17 00:00:00 2001 From: raphaelts3 Date: Sat, 16 Oct 2021 10:52:54 -0300 Subject: [PATCH 0423/2083] Update docs/intro/overview.rst Co-authored-by: azzamsa <17734314+azzamsa@users.noreply.github.com> --- docs/intro/overview.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index d75f7f636..405bf845d 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -4,7 +4,7 @@ Scrapy at a glance ================== -Scrapy (pronounced SKRAY-peye /ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting +Scrapy (/ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting structured data which can be used for a wide range of useful applications, like data mining, information processing or historical archival. From cfff79cee6a97528185b7d24e2b660b99c07945f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 18 Oct 2021 17:09:17 -0300 Subject: [PATCH 0424/2083] Make Python 3.10 support official (#5265) --- .github/workflows/checks.yml | 8 ++++---- .github/workflows/publish.yml | 4 ++-- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 14 ++++++-------- .readthedocs.yml | 9 +++++---- setup.py | 1 + tox.ini | 1 + 7 files changed, 20 insertions(+), 19 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6bdfcb5dc..80df9469d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -8,10 +8,10 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.9 + - python-version: "3.10" env: TOXENV: security - - python-version: 3.9 + - python-version: "3.10" env: TOXENV: flake8 # Pylint requires installing reppy, which does not support Python 3.9 @@ -20,10 +20,10 @@ jobs: env: TOXENV: pylint TOX_PIP_VERSION: 20.3.3 - - python-version: 3.9 + - python-version: 3.6 env: TOXENV: typing - - python-version: 3.8 # Keep in sync with .readthedocs.yml + - python-version: "3.10" # Keep in sync with .readthedocs.yml env: TOXENV: docs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b48066ea4..44b682830 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,10 +9,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: "3.10" - name: Check Tag id: check-release-tag diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 095ca1013..3aaf688c7 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index ef1c8362f..5ea50e644 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -17,6 +17,12 @@ jobs: - python-version: 3.9 env: TOXENV: py + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio - python-version: pypy3 env: TOXENV: pypy3 @@ -42,14 +48,6 @@ jobs: TOXENV: extra-deps TOX_PIP_VERSION: 20.3.3 - # 3.10-pre - - python-version: "3.10.0-beta.4" - env: - TOXENV: py - - python-version: "3.10.0-beta.4" - env: - TOXENV: asyncio - steps: - uses: actions/checkout@v2 diff --git a/.readthedocs.yml b/.readthedocs.yml index 80a1cd036..390be3749 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,12 +5,13 @@ sphinx: fail_on_warning: true build: - image: latest + os: ubuntu-20.04 + tools: + # For available versions, see: + # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python + python: "3.10" # Keep in sync with .github/workflows/checks.yml python: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image - version: 3.8 # Keep in sync with .github/workflows/checks.yml install: - requirements: docs/requirements.txt - path: . diff --git a/setup.py b/setup.py index ed2b6e347..3a6ff2836 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,7 @@ setup( 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', diff --git a/tox.ini b/tox.ini index 07552ba8d..021dd9988 100644 --- a/tox.ini +++ b/tox.ini @@ -41,6 +41,7 @@ deps = types-pyOpenSSL==20.0.3 types-setuptools==57.0.0 commands = + pip install types-dataclasses # remove once py36 support is dropped mypy --show-error-codes {posargs: scrapy tests} [testenv:security] From 144d1eb8341c427fa1fae109db3e9487f255d3fe Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 22 Oct 2021 21:46:01 +0500 Subject: [PATCH 0425/2083] Add Deferred-to-Future helpers (#5288) --- conftest.py | 6 ++++ docs/topics/asyncio.rst | 17 ++++++++++ docs/topics/coroutines.rst | 13 +++++--- docs/topics/item-pipeline.rst | 4 ++- pytest.ini | 1 + scrapy/utils/defer.py | 63 +++++++++++++++++++++++++++++++++-- tests/test_pipelines.py | 30 ++++++++++++++++- 7 files changed, 126 insertions(+), 8 deletions(-) diff --git a/conftest.py b/conftest.py index 05b4ccdad..117087790 100644 --- a/conftest.py +++ b/conftest.py @@ -75,6 +75,12 @@ def only_asyncio(request, reactor_pytest): pytest.skip('This test is only run with --reactor=asyncio') +@pytest.fixture(autouse=True) +def only_not_asyncio(request, reactor_pytest): + if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio': + pytest.skip('This test is only run without --reactor=asyncio') + + def pytest_configure(config): if config.getoption("--reactor") == "asyncio": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 82c5f271f..28241ae24 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -39,5 +39,22 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to use it instead of the default asyncio event loop. +.. _asyncio-await-dfd: +Awaiting on Deferreds +===================== +When the asyncio reactor isn't installed, you can await on Deferreds in the +coroutines directly. When it is installed, this is not possible anymore, due to +specifics of the Scrapy coroutine integration (the coroutines are wrapped into +:class:`asyncio.Future` objects, not into +:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into +Futures. Scrapy provides two helpers for this: + +.. autofunction:: scrapy.utils.defer.deferred_to_future +.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future +.. tip:: If you need to use these functions in code that aims to be compatible + with lower versions of Scrapy that do not provide these functions, + down to Scrapy 2.0 (earlier versions do not support + :mod:`asyncio`), you can copy the implementation of these functions + into your own code. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 0904637b0..2aef755c7 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -75,23 +75,28 @@ coroutines, functions that return Deferreds and functions that return :term:`awaitable objects ` such as :class:`~asyncio.Future`. This means you can use many useful Python libraries providing such code:: - class MySpider(Spider): + class MySpiderDeferred(Spider): # ... - async def parse_with_deferred(self, response): + async def parse(self, response): additional_response = await treq.get('https://additional.url') additional_data = await treq.content(additional_response) # ... use response and additional_data to yield items and requests - async def parse_with_asyncio(self, response): + class MySpiderAsyncio(Spider): + # ... + async def parse(self, response): async with aiohttp.ClientSession() as session: async with session.get('https://additional.url') as additional_response: - additional_data = await r.text() + additional_data = await additional_response.text() # ... use response and additional_data to yield items and requests .. note:: Many libraries that use coroutines, such as `aio-libs`_, require the :mod:`asyncio` loop and to use them you need to :doc:`enable asyncio support in Scrapy`. +.. note:: If you want to ``await`` on Deferreds while using the asyncio reactor, + you need to :ref:`wrap them`. + Common use cases for asynchronous code include: * requesting data from websites, databases and other services (in callbacks, diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 5351a2293..391751364 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -190,6 +190,8 @@ item. import scrapy from itemadapter import ItemAdapter + from scrapy.utils.defer import maybe_deferred_to_future + class ScreenshotPipeline: """Pipeline that uses Splash to render screenshot of @@ -202,7 +204,7 @@ item. encoded_item_url = quote(adapter["url"]) screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url) - response = await spider.crawler.engine.download(request, spider) + response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider)) if response.status != 200: # Error happened, return item. diff --git a/pytest.ini b/pytest.ini index 6de08c78d..fa5d6b34f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,5 +20,6 @@ addopts = --ignore=docs/utils markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed + only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed filterwarnings= ignore::DeprecationWarning:twisted.web.test.test_webclient diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index b02bfdccb..d7adc0a77 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -3,9 +3,16 @@ Helper functions for dealing with Twisted deferreds """ import asyncio import inspect -from collections.abc import Coroutine +from asyncio import Future from functools import wraps -from typing import Any, Callable, Generator, Iterable +from typing import ( + Any, + Callable, + Coroutine, + Generator, + Iterable, + Union +) from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred @@ -171,3 +178,55 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: return defer.fail(result) else: return defer.succeed(result) + + +def deferred_to_future(d: Deferred) -> Future: + """ + .. versionadded:: VERSION + + Return an :class:`asyncio.Future` object that wraps *d*. + + When :ref:`using the asyncio reactor `, you cannot await + on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy + callables defined as coroutines `, you can only await on + ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects + allows you to wait on them:: + + class MySpider(Spider): + ... + async def parse(self, response): + d = treq.get('https://example.com/additional') + additional_response = await deferred_to_future(d) + """ + return d.asFuture(asyncio.get_event_loop()) + + +def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: + """ + .. versionadded:: VERSION + + Return *d* as an object that can be awaited from a :ref:`Scrapy callable + defined as a coroutine `. + + What you can await in Scrapy callables defined as coroutines depends on the + value of :setting:`TWISTED_REACTOR`: + + - When not using the asyncio reactor, you can only await on + :class:`~twisted.internet.defer.Deferred` objects. + + - When :ref:`using the asyncio reactor `, you can only + await on :class:`asyncio.Future` objects. + + If you want to write code that uses ``Deferred`` objects but works with any + reactor, use this function on all ``Deferred`` objects:: + + class MySpider(Spider): + ... + async def parse(self, response): + d = treq.get('https://example.com/additional') + extra_response = await maybe_deferred_to_future(d) + """ + if not is_asyncio_reactor_installed(): + return d + else: + return deferred_to_future(d) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index ff3af9a74..8e432b913 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -6,6 +6,7 @@ from twisted.internet.defer import Deferred from twisted.trial import unittest from scrapy import Spider, signals, Request +from scrapy.utils.defer import maybe_deferred_to_future, deferred_to_future from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver import MockServer @@ -31,18 +32,38 @@ class DeferredPipeline: class AsyncDefPipeline: async def process_item(self, item, spider): - await defer.succeed(42) + d = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d.callback, None) + await maybe_deferred_to_future(d) item['pipeline_passed'] = True return item class AsyncDefAsyncioPipeline: async def process_item(self, item, spider): + d = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d.callback, None) + await deferred_to_future(d) await asyncio.sleep(0.2) item['pipeline_passed'] = await get_from_asyncio_queue(True) return item +class AsyncDefNotAsyncioPipeline: + async def process_item(self, item, spider): + d1 = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d1.callback, None) + await d1 + d2 = Deferred() + reactor.callLater(0, d2.callback, None) + await maybe_deferred_to_future(d2) + item['pipeline_passed'] = True + return item + + class ItemSpider(Spider): name = 'itemspider' @@ -99,3 +120,10 @@ class PipelineTestCase(unittest.TestCase): crawler = self._create_crawler(AsyncDefAsyncioPipeline) yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(self.items), 1) + + @mark.only_not_asyncio() + @defer.inlineCallbacks + def test_asyncdef_not_asyncio_pipeline(self): + crawler = self._create_crawler(AsyncDefNotAsyncioPipeline) + yield crawler.crawl(mockserver=self.mockserver) + self.assertEqual(len(self.items), 1) From 51adf71b1b4ae703bb1cb561882c710eedac2359 Mon Sep 17 00:00:00 2001 From: azzamsa Date: Sun, 24 Oct 2021 10:52:56 +0700 Subject: [PATCH 0426/2083] refactor: use `pytest` command as the recommended entry point `pytest` is recommended command since pytest 3.0. There is a possibility for `py.test` to be deprecated or even removed. https://github.com/pytest-dev/pytest/issues/1629 --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 021dd9988..e4514f512 100644 --- a/tox.ini +++ b/tox.ini @@ -29,7 +29,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = pip install -U -ctests/upper-constraints.txt {opts} {packages} @@ -60,7 +60,7 @@ deps = pytest-flake8 flake8==3.9.2 # https://github.com/tholo/pytest-flake8/issues/81 commands = - py.test --flake8 {posargs:docs scrapy tests} + pytest --flake8 {posargs:docs scrapy tests} [testenv:pylint] basepython = python3 @@ -142,7 +142,7 @@ setenv = [testenv:pypy3] basepython = pypy3 commands = - py.test {posargs:--durations=10 docs scrapy tests} + pytest {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython} From 67994d1dddcba4c1fe53dd7bdf7b978d1733ae1b Mon Sep 17 00:00:00 2001 From: azzamsa Date: Wed, 27 Oct 2021 21:55:05 +0700 Subject: [PATCH 0427/2083] fix: `CodeBlockParser` has been renamed to `PythonCodeBlockParser` --- docs/conftest.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/conftest.py b/docs/conftest.py index 8c735e838..a0636f8ac 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -3,7 +3,11 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE from scrapy.http.response.html import HtmlResponse from sybil import Sybil -from sybil.parsers.codeblock import CodeBlockParser +try: + # >2.0.1 + from sybil.parsers.codeblock import PythonCodeBlockParser +except ImportError: + from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser from sybil.parsers.doctest import DocTestParser from sybil.parsers.skip import skip @@ -21,7 +25,7 @@ def setup(namespace): pytest_collect_file = Sybil( parsers=[ DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE), - CodeBlockParser(future_imports=['print_function']), + PythonCodeBlockParser(future_imports=['print_function']), skip, ], pattern='*.rst', From 55cce25a799ab0ed9d5edd60ff0f35988318e9b9 Mon Sep 17 00:00:00 2001 From: azzamsa Date: Mon, 25 Oct 2021 21:14:11 +0700 Subject: [PATCH 0428/2083] test: `test_format_engine_status` --- tests/test_crawl.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 84bac9b50..7bda3bef2 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -274,6 +274,28 @@ with multiples lines self.assertEqual(s['engine.spider.name'], crawler.spider.name) self.assertEqual(s['len(engine.scraper.slot.active)'], 1) + @defer.inlineCallbacks + def test_format_engine_status(self): + from scrapy.utils.engine import format_engine_status + est = [] + + def cb(response): + est.append(format_engine_status(crawler.engine)) + + crawler = self.runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) + self.assertEqual(len(est), 1, est) + est = est[0].split("\n")[2:-2] # remove header & footer + # convert to dict + est = [x.split(":") for x in est] + est = [x for sublist in est for x in sublist] # flatten + est = [x.lstrip().rstrip() for x in est] + it = iter(est) + s = dict(zip(it, it)) + + self.assertEqual(s['engine.spider.name'], crawler.spider.name) + self.assertEqual(s['len(engine.scraper.slot.active)'], '1') + @defer.inlineCallbacks def test_graceful_crawl_error_handling(self): """ From 28eba610e22c0d2a42e830b4e64746edf44598f9 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 15 Nov 2021 12:24:54 +0500 Subject: [PATCH 0429/2083] Re-enable Windows tests for Python 3.9 and 3.10. (#5316) --- .github/workflows/tests-windows.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 30fda33e8..6fabf5cde 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -17,10 +17,12 @@ jobs: - python-version: 3.8 env: TOXENV: py - # https://twistedmatrix.com/trac/ticket/9990 - #- python-version: 3.9 - #env: - #TOXENV: py + - python-version: 3.9 + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: py steps: - uses: actions/checkout@v2 From f2c800c5c9f88b4b583181e9cf49eb3cd8d538f0 Mon Sep 17 00:00:00 2001 From: Samuel Marchal Date: Mon, 15 Nov 2021 11:14:54 +0100 Subject: [PATCH 0430/2083] Improve open_in_browser base tag injection (#5319) --- scrapy/utils/response.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b3ef7b463..8b109dced 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,8 +3,9 @@ This module provides some useful functions for working with scrapy.http.Response objects """ import os -import webbrowser +import re import tempfile +import webbrowser from typing import Any, Callable, Iterable, Optional, Tuple, Union from weakref import WeakKeyDictionary @@ -80,8 +81,9 @@ def open_in_browser( body = response.body if isinstance(response, HtmlResponse): if b'' - body = body.replace(b'', to_bytes(repl)) + repl = fr'\1' + body = re.sub(b"", b"", body, flags=re.DOTALL) + body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) ext = '.html' elif isinstance(response, TextResponse): ext = '.txt' From 75ed765476a2ac66ea8f52e7b29186864f65535c Mon Sep 17 00:00:00 2001 From: Samuel Marchal Date: Mon, 15 Nov 2021 14:31:24 +0100 Subject: [PATCH 0431/2083] Test coverage for open_in_browser base tag injection (#5319) --- tests/test_utils_response.py | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index d6f4c0bb5..0a09f6109 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase): self.assertEqual(response_status_message(200), '200 OK') self.assertEqual(response_status_message(404), '404 Not Found') self.assertEqual(response_status_message(573), "573 Unknown Status") + + def test_inject_base_url(self): + url = "http://www.example.com" + + def check_base_url(burl): + path = urlparse(burl).path + if not os.path.exists(path): + path = burl.replace('file://', '') + with open(path, "rb") as f: + bbody = f.read() + self.assertEqual(bbody.count(b''), 1) + return True + + r1 = HtmlResponse(url, body=b""" + + Dummy +

Hello world.

+ """) + r2 = HtmlResponse(url, body=b""" + + Dummy + Hello world. + """) + r3 = HtmlResponse(url, body=b""" + + Dummy + +
Hello header
+

Hello world.

+ + """) + r4 = HtmlResponse(url, body=b""" + + + Dummy +

Hello world.

+ """) + r5 = HtmlResponse(url, body=b""" + + + + Standard head + +

Hello world.

+ """) + + assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url" + assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head" + assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag" + assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment" + assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment" From c316ca45a5b1b19622c96049c9378d8c45adba60 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 16 Nov 2021 01:20:56 -0800 Subject: [PATCH 0432/2083] Use augmented assignment statements (#5322) --- scrapy/core/downloader/handlers/http11.py | 2 +- scrapy/core/http2/stream.py | 4 ++-- tests/test_request_left.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 8a91d4c5e..38935667d 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -213,7 +213,7 @@ class TunnelingAgent(Agent): # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy - key = key + self._proxyConf + key += self._proxyConf return super()._requestWithEndpoint( key=key, endpoint=endpoint, diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index c2a4b702f..5c393c027 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -285,8 +285,8 @@ class Stream: self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False) - bytes_to_send_size = bytes_to_send_size - chunk_size - self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size + bytes_to_send_size -= chunk_size + self.metadata['remaining_content_length'] -= chunk_size self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length']) diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 373b2e49c..4d4483881 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -22,7 +22,7 @@ class SignalCatcherSpider(Spider): return spider def on_request_left(self, request, spider): - self.caught_times = self.caught_times + 1 + self.caught_times += 1 class TestCatching(TestCase): From eea56c4912ebabf94d366b4ead0d0d0a40a1d682 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 20:13:54 +0200 Subject: [PATCH 0433/2083] per slot settings: creation of Downloader.Slot objects from per slot settings added --- scrapy/core/downloader/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 4f7ab594f..f680b9082 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -83,6 +83,7 @@ class Downloader: self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) + self.per_slot_settings = self.settings.getdict('PER_SLOT_SETTINGS', {}) def fetch(self, request, spider): def _deactivate(response): @@ -99,9 +100,13 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + conc = self.per_slot_settings.get(key,{}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - self.slots[key] = Slot(conc, delay, self.randomize_delay) + delay = self.per_slot_settings.get(key,{}).get('delay', delay) + randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) + new_slot = Slot(conc, delay, randomize_delay) + self.slots[key] = new_slot + return key, self.slots[key] From d5acf88ca55988a6c92046da5ef2bf065b0b3e72 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 20:14:28 +0200 Subject: [PATCH 0434/2083] per slot settings: logging added (create Slot) --- scrapy/core/downloader/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index f680b9082..9aab80171 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,3 +1,4 @@ +import logging import random from time import time from datetime import datetime @@ -12,6 +13,7 @@ from scrapy import signals from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core.downloader.handlers import DownloadHandlers +logger = logging.getLogger(__name__) class Slot: """Downloader slot""" @@ -106,7 +108,7 @@ class Downloader: randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot - + logger.debug(f"Downloader slot created {'from per slot settings' if key in self.per_slot_settings.keys() else ''}: {new_slot}") return key, self.slots[key] From 1b7d7ecfcdb6067f3dccd69fc56e25dc1dbf498f Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 21:05:24 +0200 Subject: [PATCH 0435/2083] per slot settings: logging updated (create/close Slot) --- scrapy/core/downloader/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 9aab80171..461b211ab 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -108,7 +108,8 @@ class Downloader: randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot - logger.debug(f"Downloader slot created {'from per slot settings' if key in self.per_slot_settings.keys() else ''}: {new_slot}") + logger.debug( + f"Downloader slot '{key}' - created {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''}: {new_slot}") return key, self.slots[key] @@ -205,4 +206,7 @@ class Downloader: mintime = time() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: - self.slots.pop(key).close() + inactive_slot = self.slots.pop(key) + inactive_slot.close() + logger.debug( + f"Downloader slot '{key}' - closed {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''} : {inactive_slot}") From 8185aa5265c663326474897c24aa0691a2ee0e5c Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 21:06:08 +0200 Subject: [PATCH 0436/2083] per slot settings: codestyle fix --- scrapy/core/downloader/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 461b211ab..d5809ffa0 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -15,6 +15,7 @@ from scrapy.core.downloader.handlers import DownloadHandlers logger = logging.getLogger(__name__) + class Slot: """Downloader slot""" @@ -102,9 +103,9 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.per_slot_settings.get(key,{}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) + conc = self.per_slot_settings.get(key, {}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - delay = self.per_slot_settings.get(key,{}).get('delay', delay) + delay = self.per_slot_settings.get(key, {}).get('delay', delay) randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot From 6ec66c96fb962a276c2d285cd5ffa34d84925df1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Nov 2021 12:25:45 +0500 Subject: [PATCH 0437/2083] Fix and pin pylint. --- pylintrc | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pylintrc b/pylintrc index 699686e16..2cdd6321e 100644 --- a/pylintrc +++ b/pylintrc @@ -112,6 +112,7 @@ disable=abstract-method, unused-private-member, unused-variable, unused-wildcard-import, + use-implicit-booleaness-not-comparison, used-before-assignment, useless-object-inheritance, # Required for Python 2 support useless-return, diff --git a/tox.ini b/tox.ini index e4514f512..2031a2d92 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint + pylint==2.12.1 commands = pylint conftest.py docs extras scrapy setup.py tests From 4cc039628eb4861f98f7997e90125745e30f8687 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Nov 2021 19:52:03 +0500 Subject: [PATCH 0438/2083] Fix typing of middleware methods. --- scrapy/core/downloader/middleware.py | 5 ++++- scrapy/core/spidermw.py | 3 ++- scrapy/middleware.py | 16 ++++++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index a5619d8a4..289147466 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,7 +3,7 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from typing import Callable, Union +from typing import Callable, Union, cast from twisted.internet import defer from twisted.python.failure import Failure @@ -37,6 +37,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): @defer.inlineCallbacks def process_request(request: Request): for method in self.methods['process_request']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( @@ -55,6 +56,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response for method in self.methods['process_response']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( @@ -69,6 +71,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): def process_exception(failure: Failure): exception = failure.value for method in self.methods['process_exception']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 7e58521ac..7cdc28284 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ from itertools import islice -from typing import Any, Callable, Generator, Iterable, Union +from typing import Any, Callable, Generator, Iterable, Union, cast from twisted.internet.defer import Deferred from twisted.python.failure import Failure @@ -47,6 +47,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Any: for method in self.methods['process_spider_input']: + method = cast(Callable, method) try: result = method(response=response, spider=spider) if result is not None: diff --git a/scrapy/middleware.py b/scrapy/middleware.py index bbec38086..e8f60287a 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Deque, Dict +from typing import Callable, Deque, Dict, Optional, cast, Iterable from twisted.internet.defer import Deferred @@ -21,7 +21,8 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods: Dict[str, Deque[Callable]] = defaultdict(deque) + # Optional because process_spider_output and process_spider_exception can be None + self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -64,14 +65,17 @@ class MiddlewareManager: self.methods['close_spider'].appendleft(mw.close_spider) def _process_parallel(self, methodname: str, obj, *args) -> Deferred: - return process_parallel(self.methods[methodname], obj, *args) + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_parallel(methods, obj, *args) def _process_chain(self, methodname: str, obj, *args) -> Deferred: - return process_chain(self.methods[methodname], obj, *args) + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_chain(methods, obj, *args) def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: - return process_chain_both(self.methods[cb_methodname], - self.methods[eb_methodname], obj, *args) + cb_methods = cast(Iterable[Callable], self.methods[cb_methodname]) + eb_methods = cast(Iterable[Callable], self.methods[eb_methodname]) + return process_chain_both(cb_methods, eb_methods, obj, *args) def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) From eb62906c3e4c1e1f8e3e6c7965a04d5e65c61907 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Dec 2021 17:40:41 +0500 Subject: [PATCH 0439/2083] Extract utils.log.log_reactor_info(). --- scrapy/utils/log.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 0441c0358..9887ecc40 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -143,7 +143,7 @@ def _get_handler(settings): return handler -def log_scrapy_info(settings): +def log_scrapy_info(settings: Settings) -> None: logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) versions = [ @@ -152,6 +152,10 @@ def log_scrapy_info(settings): if name != "Scrapy" ] logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) + log_reactor_info() + + +def log_reactor_info() -> None: from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) from twisted.internet import asyncioreactor From 6483dfdbe17cd66c409435b95a05850a3c94b5ee Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Dec 2021 19:53:39 +0500 Subject: [PATCH 0440/2083] Move install_shutdown_handlers() from __init__() to start(). --- scrapy/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 578016536..357f14dc0 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -278,7 +278,6 @@ class CrawlerProcess(CrawlerRunner): def __init__(self, settings=None, install_root_handler=True): super().__init__(settings) - install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) @@ -318,6 +317,7 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) + install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() From d6a384b3cfdb36cd19b32942479487a9e47c244b Mon Sep 17 00:00:00 2001 From: yogender26 <95638485+yogender26@users.noreply.github.com> Date: Thu, 23 Dec 2021 04:09:05 +0530 Subject: [PATCH 0441/2083] corrrection of coma (#5347) --- scrapy/commands/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 6e77551c6..5f1dabd33 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -43,14 +43,14 @@ class ScrapyCommand: def long_desc(self): """A long description of the command. Return short description when not - available. It cannot contain newlines, since contents will be formatted + available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() def help(self): """An extensive help for the command. It will be shown when using the - "help" command. It can contain newlines, since no post-formatting will + "help" command. It can contain newlines since no post-formatting will be applied to its contents. """ return self.long_desc() From 46ef9cf771789f1db513bbf2f65243d3320ce695 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 22 Dec 2021 21:24:59 +0500 Subject: [PATCH 0442/2083] Don't install non-working shutdown handlers in `scrapy shell`. --- scrapy/commands/shell.py | 2 +- scrapy/crawler.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index d1944df3d..de81986d8 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -75,6 +75,6 @@ class Command(ScrapyCommand): def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, - kwargs={'stop_after_crawl': False}) + kwargs={'stop_after_crawl': False, 'install_signal_handlers': False}) t.daemon = True t.start() diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 357f14dc0..e54ad9750 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -297,7 +297,7 @@ class CrawlerProcess(CrawlerRunner): {'signame': signame}) reactor.callFromThread(self._stop_reactor) - def start(self, stop_after_crawl=True): + def start(self, stop_after_crawl=True, install_signal_handlers=True): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache @@ -308,6 +308,9 @@ class CrawlerProcess(CrawlerRunner): :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished + + :param bool install_signal_handlers: whether to install the shutdown + handlers (default: True) """ from twisted.internet import reactor if stop_after_crawl: @@ -317,7 +320,8 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) - install_shutdown_handlers(self._signal_shutdown) + if install_signal_handlers: + install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() From 60c8838554a79e70c22a7c6a57baedfcaf521444 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 16:07:18 +0500 Subject: [PATCH 0443/2083] Move installing the reactor from CrawlerProcess to Crawler. --- scrapy/crawler.py | 31 ++++++++++++++++++++----------- scrapy/utils/log.py | 1 - tests/test_crawler.py | 5 ++++- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e54ad9750..95cfb1bd1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -25,6 +25,7 @@ from scrapy.utils.log import ( configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, + log_reactor_info, log_scrapy_info, LogCounterHandler, ) @@ -38,7 +39,7 @@ logger = logging.getLogger(__name__) class Crawler: - def __init__(self, spidercls, settings=None): + def __init__(self, spidercls, settings=None, init_reactor: bool = False): if isinstance(spidercls, Spider): raise ValueError('The spidercls argument must be a class, not an object') @@ -69,6 +70,19 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) + + if init_reactor: + # this needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor + if self.settings.get("TWISTED_REACTOR"): + install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) + else: + from twisted.internet import default + default.install() + log_reactor_info() + if self.settings.get("TWISTED_REACTOR"): + verify_installed_reactor(self.settings["TWISTED_REACTOR"]) + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() @@ -153,7 +167,6 @@ class CrawlerRunner: self._crawlers = set() self._active = set() self.bootstrap_failed = False - self._handle_twisted_reactor() @property def spiders(self): @@ -247,10 +260,6 @@ class CrawlerRunner: while self._active: yield defer.DeferredList(self._active) - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - verify_installed_reactor(self.settings["TWISTED_REACTOR"]) - class CrawlerProcess(CrawlerRunner): """ @@ -297,6 +306,11 @@ class CrawlerProcess(CrawlerRunner): {'signame': signame}) reactor.callFromThread(self._stop_reactor) + def _create_crawler(self, spidercls): + if isinstance(spidercls, str): + spidercls = self.spider_loader.load(spidercls) + return Crawler(spidercls, self.settings, init_reactor=True) + def start(self, stop_after_crawl=True, install_signal_handlers=True): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool @@ -341,8 +355,3 @@ class CrawlerProcess(CrawlerRunner): reactor.stop() except RuntimeError: # raised if already stopped or in shutdown stage pass - - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) - super()._handle_twisted_reactor() diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 9887ecc40..78e302d19 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -152,7 +152,6 @@ def log_scrapy_info(settings: Settings) -> None: if name != "Scrapy" ] logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) - log_reactor_info() def log_reactor_info() -> None: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index be067155e..118cb631b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -271,6 +271,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertEqual(runner.bootstrap_failed, True) + @defer.inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ @@ -279,9 +280,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase): else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - CrawlerRunner(settings={ + runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) + yield runner.crawl(NoRequestsSpider) @defer.inlineCallbacks # https://twistedmatrix.com/trac/ticket/9766 @@ -301,6 +303,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): runner = CrawlerProcess(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) + yield runner.crawl(NoRequestsSpider) @defer.inlineCallbacks def test_crawler_process_asyncio_enabled_false(self): From 041699b54cfa6cde9f886a98ff300e3276e2eaad Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 16:14:47 +0500 Subject: [PATCH 0444/2083] Remove tests that want to modify the test process reactor. --- tests/test_crawler.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 118cb631b..f445c181e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -285,33 +285,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase): }) yield runner.crawl(NoRequestsSpider) - @defer.inlineCallbacks - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") - def test_crawler_process_asyncio_enabled_true(self): - with LogCapture(level=logging.DEBUG) as log: - if self.reactor_pytest == 'asyncio': - runner = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - yield runner.crawl(NoRequestsSpider) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) - else: - msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" - with self.assertRaisesRegex(Exception, msg): - runner = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - yield runner.crawl(NoRequestsSpider) - - @defer.inlineCallbacks - def test_crawler_process_asyncio_enabled_false(self): - runner = CrawlerProcess(settings={"TWISTED_REACTOR": None}) - with LogCapture(level=logging.DEBUG) as log: - yield runner.crawl(NoRequestsSpider) - self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) - class ScriptRunnerMixin: def run_script(self, script_name, *script_args): From ebcafdf4a9e0692bf301546b6d60465b3b2c4b06 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 16:35:26 +0500 Subject: [PATCH 0445/2083] Add tests for TWISTED_REACTOR in custom_settings. --- .../twisted_reactor_custom_settings.py | 14 +++++++++++ ...wisted_reactor_custom_settings_conflict.py | 22 +++++++++++++++++ .../twisted_reactor_custom_settings_same.py | 21 ++++++++++++++++ tests/test_crawler.py | 24 +++++++++++++++++++ 4 files changed, 81 insertions(+) create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings.py create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings_same.py diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.py new file mode 100644 index 000000000..56304bd23 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py @@ -0,0 +1,14 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py new file mode 100644 index 000000000..9a6c01d72 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class PollReactorSpider(scrapy.Spider): + name = 'poll_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor", + } + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(PollReactorSpider) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py new file mode 100644 index 000000000..1f5a44010 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py @@ -0,0 +1,21 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider1(scrapy.Spider): + name = 'asyncio_reactor1' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + +class AsyncioReactorSpider2(scrapy.Spider): + name = 'asyncio_reactor2' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider1) +process.crawl(AsyncioReactorSpider2) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f445c181e..6d6763aec 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -361,6 +361,30 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_reactor_asyncio_custom_settings(self): + log = self.run_script("twisted_reactor_custom_settings.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_reactor_asyncio_custom_settings_same(self): + log = self.run_script("twisted_reactor_custom_settings_same.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_reactor_asyncio_custom_settings_conflict(self): + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) + self.assertIn("(twisted.internet.pollreactor.PollReactor) does not match the requested one", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') From 002513438204eea5062b5a1d75fb4f261880da4f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 16:45:17 +0500 Subject: [PATCH 0446/2083] Completely skip WindowsRunSpiderCommandTest outside Windows. --- tests/test_commands.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 75098a77a..efe9b0531 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -765,6 +765,7 @@ class MySpider(scrapy.Spider): self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) +@skipIf(platform.system() != 'Windows', "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = 'myspider.pyw' @@ -777,35 +778,27 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_run_good_spider(self): super().test_run_good_spider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider(self): super().test_runspider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_dnscache_disabled(self): super().test_runspider_dnscache_disabled() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_level(self): super().test_runspider_log_level() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_short_names(self): super().test_runspider_log_short_names() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_no_spider_found(self): super().test_runspider_no_spider_found() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_output(self): super().test_output() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_overwrite_output(self): super().test_overwrite_output() From 9c4bfb48362f736fce81b71a6ca1fa0b3600231d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 17:17:36 +0500 Subject: [PATCH 0447/2083] Remove an unused import. --- tests/test_crawler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6d6763aec..d68c50026 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -7,7 +7,6 @@ import warnings from unittest import skipIf from pytest import raises, mark -from testfixtures import LogCapture from twisted import version as twisted_version from twisted.internet import defer from twisted.python.versions import Version From d4565318c7061c2ccd17fa5d5eabcacef8c34826 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 17:40:31 +0500 Subject: [PATCH 0448/2083] Fix a reactor test on Windows. --- .../twisted_reactor_custom_settings_conflict.py | 8 ++++---- tests/test_crawler.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py index 9a6c01d72..3f219098c 100644 --- a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -2,10 +2,10 @@ import scrapy from scrapy.crawler import CrawlerProcess -class PollReactorSpider(scrapy.Spider): - name = 'poll_reactor' +class SelectReactorSpider(scrapy.Spider): + name = 'select_reactor' custom_settings = { - "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor", + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", } @@ -17,6 +17,6 @@ class AsyncioReactorSpider(scrapy.Spider): process = CrawlerProcess() -process.crawl(PollReactorSpider) +process.crawl(SelectReactorSpider) process.crawl(AsyncioReactorSpider) process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index d68c50026..e7d5c8132 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -381,8 +381,8 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio_custom_settings_conflict(self): log = self.run_script("twisted_reactor_custom_settings_conflict.py") - self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - self.assertIn("(twisted.internet.pollreactor.PollReactor) does not match the requested one", log) + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log) @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') From 940cc0776ff86f726e79c2ab2018f4b83a833936 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 24 Dec 2021 17:12:50 +0500 Subject: [PATCH 0449/2083] Add docs about TWISTED_REACTOR and other per-process settings. --- docs/topics/practices.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 732eba587..bd0dd8ce0 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -102,6 +102,17 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished +.. note:: + .. versionchanged:: VERSION + + The Twisted reactor is now installed when + :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a + :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, + :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now + honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions + they are silently ignored when set there and you need to set these settings + in some other way. + .. seealso:: :doc:`twisted:core/howto/reactor-basics` .. _run-multiple-spiders: @@ -193,6 +204,25 @@ Same example but running the spiders sequentially by chaining the deferreds: crawl() reactor.run() # the script will block here until the last crawl call is finished +Different spiders can set different values for the same setting, but when they +run in the same process it may be impossible, by design or because of some +limitations, to use these different values. What happens in practice is +different for different settings: + +* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value + (:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the + default one) cannot be read from the per-spider settings. These are applied + when the :class:`~scrapy.crawler.CrawlerRunner` or + :class:`~scrapy.crawler.CrawlerProcess` object is created. +* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first + available value is used, and if a spider requests a different reactor an + exception will be raised. These are applied when the reactor is installed. +* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the + ones used by the resolver (:setting:`DNSCACHE_ENABLED`, + :setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy) + the first available value is used. These are applied when the reactor is + started. + .. seealso:: :ref:`run-from-script`. .. _distributed-crawls: From a986792def6df1b2bbdf1bc996308d3afd8528c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 24 Dec 2021 19:43:14 +0500 Subject: [PATCH 0450/2083] Add more docs for TWISTED_REACTOR. --- docs/topics/settings.rst | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 210c1def7..cff6d80cb 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1638,10 +1638,18 @@ which raises :exc:`Exception`, becomes:: The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will not attempt to install any specific reactor, and the -default reactor defined by Twisted for the current platform will be used. This -is to maintain backward compatibility and avoid possible problems caused by -using a non-default reactor. +means that Scrapy will install the default reactor defined by Twisted for the +current platform will be used. This is to maintain backward compatibility and +avoid possible problems caused by using a non-default reactor. + +.. note:: + .. versionchanged:: VERSION + + Previously this setting had no effect in a spider + :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but + if you :ref:`run several spiders in one process `, + they must not have different values for this setting, because they will use + a single reactor instance. For additional information, see :doc:`core/howto/choosing-reactor`. From a9dfd85ea6e983f255afc1b5b0f295fccddcbacb Mon Sep 17 00:00:00 2001 From: Burak Can Kahraman Date: Thu, 30 Dec 2021 15:48:53 +0300 Subject: [PATCH 0451/2083] Document coroutines for signals. --- docs/topics/coroutines.rst | 2 ++ docs/topics/signals.rst | 20 +++++++++----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 2aef755c7..549552bd1 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -1,3 +1,5 @@ +.. _topics-coroutines: + ========== Coroutines ========== diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 63ad3a9ad..328fb88d2 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -51,12 +51,12 @@ Deferred signal handlers ======================== Some signals support returning :class:`~twisted.internet.defer.Deferred` -objects from their handlers, allowing you to run asynchronous code that -does not block Scrapy. If a signal handler returns a -:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that -:class:`~twisted.internet.defer.Deferred` to fire. +or :term:`awaitable objects ` from their handlers, allowing +you to run asynchronous code that does not block Scrapy. If a signal +handler returns one of these objects, Scrapy waits for that asynchronous +operation to finish. -Let's take an example:: +Let's take an example using :ref:`coroutines `:: class SignalSpider(scrapy.Spider): name = 'signals' @@ -68,17 +68,15 @@ Let's take an example:: crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped) return spider - def item_scraped(self, item): + async def item_scraped(self, item): # Send the scraped item to the server - d = treq.post( + response = await treq.post( 'http://example.com/post', json.dumps(item).encode('ascii'), headers={b'Content-Type': [b'application/json']} ) - # The next item will be scraped only after - # deferred (d) is fired - return d + return response def parse(self, response): for quote in response.css('div.quote'): @@ -89,7 +87,7 @@ Let's take an example:: } See the :ref:`topics-signals-ref` below to know which signals support -:class:`~twisted.internet.defer.Deferred`. +:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects `. .. _topics-signals-ref: From 7380888cad2c255e6f4a7efb6fe4cfd1240fb42c Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 30 Dec 2021 18:55:16 +0500 Subject: [PATCH 0452/2083] Fix a warning message. (#5359) --- scrapy/utils/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 24873f75d..24a6187b9 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None, message = ( 'The -t command line option is deprecated in favor of ' 'specifying the output format within the output URI. See the ' - 'documentation of the -o and -O options for more information.', + 'documentation of the -o and -O options for more information.' ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} From 64261d9e389737621caa85f320cf81ef2aef1faa Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 Dec 2021 15:45:59 +0500 Subject: [PATCH 0453/2083] Slight refactoring. --- scrapy/crawler.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 95cfb1bd1..a638254f1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -71,17 +71,18 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) + reactor_class = self.settings.get("TWISTED_REACTOR") if init_reactor: # this needs to be done after the spider settings are merged, # but before something imports twisted.internet.reactor - if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) + if reactor_class: + install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) else: from twisted.internet import default default.install() log_reactor_info() - if self.settings.get("TWISTED_REACTOR"): - verify_installed_reactor(self.settings["TWISTED_REACTOR"]) + if reactor_class: + verify_installed_reactor(reactor_class) self.extensions = ExtensionManager.from_crawler(self) From b81938684b55fca29e48faa766f2b6f6e3ab5d6a Mon Sep 17 00:00:00 2001 From: Andrey Oskin Date: Fri, 31 Dec 2021 21:49:18 +1100 Subject: [PATCH 0454/2083] Docs: correct process repetition start step (#5356) The process repeats from step 3, the scheduler feeds request to the engine. Steps 1 and 2 are not parts of the loop as their incarnations steps 7 and 8 are parts of the loop. --- docs/topics/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 71d027c86..0c3a7ed88 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -67,7 +67,7 @@ this: the :ref:`Scheduler ` and asks for possible next Requests to crawl. -9. The process repeats (from step 1) until there are no more requests from the +9. The process repeats (from step 3) until there are no more requests from the :ref:`Scheduler `. Components From e4bdd1cb958b7d89b86ea66f0af1cec2d91a6d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Miech?= Date: Fri, 31 Dec 2021 11:57:12 +0100 Subject: [PATCH 0455/2083] downloader.webclient: make reactor import local (#5357) --- scrapy/core/downloader/webclient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 915cb5fe3..06cb96489 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -3,7 +3,7 @@ from time import time from urllib.parse import urlparse, urlunparse, urldefrag from twisted.web.http import HTTPClient -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.protocol import ClientFactory from scrapy.http import Headers @@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory): p.followRedirect = self.followRedirect p.afterFoundGet = self.afterFoundGet if self.timeout: + from twisted.internet import reactor timeoutCall = reactor.callLater(self.timeout, p.timeout) self.deferred.addBoth(self._cancelTimeout, timeoutCall) return p From 57dc58123b98e2026025cc87bdee474bf0656dcb Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 31 Dec 2021 17:15:08 +0500 Subject: [PATCH 0456/2083] Remove the experimental note about asyncio (#5332) --- docs/topics/asyncio.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 28241ae24..402352721 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -10,11 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet - recommended for production environments. Future Scrapy versions - may introduce related changes without a deprecation period or - warning. - .. _install-asyncio: Installing the asyncio reactor From a2763c608d33a9254034d650457b7efa7b434ec0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 Dec 2021 18:35:00 +0500 Subject: [PATCH 0457/2083] Remove unused MiddlewareManager._process_chain_both(). --- scrapy/middleware.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index e8f60287a..2eb1d8609 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -9,7 +9,7 @@ from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.settings import Settings from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.defer import process_parallel, process_chain, process_chain_both +from scrapy.utils.defer import process_parallel, process_chain logger = logging.getLogger(__name__) @@ -72,11 +72,6 @@ class MiddlewareManager: methods = cast(Iterable[Callable], self.methods[methodname]) return process_chain(methods, obj, *args) - def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: - cb_methods = cast(Iterable[Callable], self.methods[cb_methodname]) - eb_methods = cast(Iterable[Callable], self.methods[eb_methodname]) - return process_chain_both(cb_methods, eb_methods, obj, *args) - def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) From 6eaceec735d551f5b777bc641ff8d85dbb3ba98c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 Dec 2021 20:14:24 +0500 Subject: [PATCH 0458/2083] Implement docs suggestions. --- docs/news.rst | 22 ++++++++++++++++++++++ docs/topics/practices.rst | 11 ----------- docs/topics/settings.rst | 13 ++----------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 509366c17..2afe318f6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,3 +1,25 @@ +.. note:: + .. versionchanged:: VERSION + + The Twisted reactor is now installed when + :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a + :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, + :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now + honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions + they are silently ignored when set there and you need to set these settings + in some other way. + + +.. note:: + .. versionchanged:: VERSION + + Previously this setting had no effect in a spider + :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but + if you :ref:`run several spiders in one process `, + they must not have different values for this setting, because they will use + a single reactor instance. + + .. _news: Release notes diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index bd0dd8ce0..1a9d56143 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -102,17 +102,6 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished -.. note:: - .. versionchanged:: VERSION - - The Twisted reactor is now installed when - :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a - :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, - :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now - honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions - they are silently ignored when set there and you need to set these settings - in some other way. - .. seealso:: :doc:`twisted:core/howto/reactor-basics` .. _run-multiple-spiders: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index cff6d80cb..f6c95c502 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1639,17 +1639,8 @@ which raises :exc:`Exception`, becomes:: The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which means that Scrapy will install the default reactor defined by Twisted for the -current platform will be used. This is to maintain backward compatibility and -avoid possible problems caused by using a non-default reactor. - -.. note:: - .. versionchanged:: VERSION - - Previously this setting had no effect in a spider - :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but - if you :ref:`run several spiders in one process `, - they must not have different values for this setting, because they will use - a single reactor instance. +current platform. This is to maintain backward compatibility and avoid possible +problems caused by using a non-default reactor. For additional information, see :doc:`core/howto/choosing-reactor`. From c5ab58056c29c2c35b183572b0780acfbb15dfe8 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Sat, 1 Jan 2022 00:38:10 +0500 Subject: [PATCH 0459/2083] Set WindowsSelectorEventLoopPolicy on Windows (#5315) --- .github/workflows/tests-windows.yml | 3 ++ docs/topics/asyncio.rst | 28 +++++++++++++++++++ scrapy/utils/reactor.py | 5 ++++ .../CrawlerProcess/asyncio_enabled_reactor.py | 3 ++ tests/test_commands.py | 11 +++----- tests/test_crawler.py | 16 ----------- tests/test_downloader_handlers.py | 16 +++++++++++ tests/test_utils_asyncio.py | 7 +---- 8 files changed, 60 insertions(+), 29 deletions(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 6fabf5cde..ab7385118 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -23,6 +23,9 @@ jobs: - python-version: "3.10" env: TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio steps: - uses: actions/checkout@v2 diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 402352721..8712d4268 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -36,6 +36,34 @@ use it instead of the default asyncio event loop. .. _asyncio-await-dfd: +Windows-specific notes +====================== + +The Windows implementation of :mod:`asyncio` can use two event loop +implementations: :class:`~asyncio.SelectorEventLoop` (default before Python +3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop` +(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the +event loop class needs to be changed. Scrapy since VERSION does this +automatically when you change the :setting:`TWISTED_REACTOR` setting or call +:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor +by other means or use an older Scrapy version you need to call the following +code before installing the reactor:: + + import asyncio + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + +You can put this in the same function that installs the reactor, if you do that +yourself, or in some code that runs before the reactor is installed, e.g. +``settings.py``. + +.. note:: Other libraries you use may require + :class:`~asyncio.ProactorEventLoop`, e.g. because it supports + subprocesses (this is the case with `playwright`_), so you cannot use + them together with Scrapy on Windows (but you should be able to use + them on WSL or native Linux). + +.. _playwright: https://github.com/microsoft/playwright-python + Awaiting on Deferreds ===================== diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 6723d9b37..96395543c 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,4 +1,5 @@ import asyncio +import sys from contextlib import suppress from twisted.internet import asyncioreactor, error @@ -57,6 +58,10 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): + if sys.version_info >= (3, 8) and sys.platform == "win32": + policy = asyncio.get_event_loop_policy() + if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 8568bd8b8..f2a93074b 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,6 +1,9 @@ import asyncio +import sys from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) import scrapy diff --git a/tests/test_commands.py b/tests/test_commands.py index 75098a77a..81d1a1cab 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -674,9 +674,6 @@ class MySpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' @@ -699,15 +696,15 @@ class MySpider(scrapy.Spider): ]) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_custom_asyncio_loop_enabled_false(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' ]) import asyncio - loop = asyncio.new_event_loop() + if sys.platform != 'win32': + loop = asyncio.new_event_loop() + else: + loop = asyncio.SelectorEventLoop() self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log) def test_output(self): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index be067155e..7bc4fba40 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,7 +4,6 @@ import platform import subprocess import sys import warnings -from unittest import skipIf from pytest import raises, mark from testfixtures import LogCapture @@ -284,9 +283,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase): }) @defer.inlineCallbacks - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_crawler_process_asyncio_enabled_true(self): with LogCapture(level=logging.DEBUG) as log: if self.reactor_pytest == 'asyncio': @@ -328,17 +324,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -377,9 +367,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) @@ -404,9 +391,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 9c11820e5..a1ea4c679 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,6 +1,7 @@ import contextlib import os import shutil +import sys import tempfile from typing import Optional, Type from unittest import mock @@ -287,6 +288,12 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_nodata_rcvd(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + # client connects but no data is received spider = Spider('foo') meta = {'download_timeout': 0.5} @@ -296,6 +303,11 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_server_hangs(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) # client connects, server send headers and some body bytes but hangs spider = Spider('foo') meta = {'download_timeout': 0.5} @@ -1055,6 +1067,10 @@ class BaseFTPTestCase(unittest.TestCase): class FTPTestCase(BaseFTPTestCase): def test_invalid_credentials(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) from twisted.protocols.ftp import ConnectionLost meta = dict(self.req_meta) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a2114bd18..295323e4d 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,6 +1,4 @@ -import platform -import sys -from unittest import skipIf, TestCase +from unittest import TestCase from pytest import mark @@ -14,9 +12,6 @@ class AsyncioTest(TestCase): # the result should depend only on the pytest --reactor argument self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_install_asyncio_reactor(self): # this should do nothing install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") From f789547551ae0eb79f41c9de44525bf597a0ffa5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Jan 2022 18:28:32 +0500 Subject: [PATCH 0460/2083] Implement spider middleware iterable upgrade/downgrade. --- docs/topics/coroutines.rst | 104 ++++--- docs/topics/spider-middleware.rst | 9 + scrapy/core/scraper.py | 3 +- scrapy/core/spidermw.py | 135 +++++++-- scrapy/middleware.py | 7 +- scrapy/spidermiddlewares/depth.py | 71 ++--- scrapy/spidermiddlewares/offsite.py | 40 +-- scrapy/spidermiddlewares/referer.py | 22 +- scrapy/spidermiddlewares/urllength.py | 33 +-- scrapy/utils/asyncgen.py | 47 +-- scrapy/utils/python.py | 4 +- tests/test_spidermiddleware.py | 302 +++++++++++++++++--- tests/test_spidermiddleware_output_chain.py | 17 ++ tests/test_utils_asyncgen.py | 52 +--- 14 files changed, 553 insertions(+), 293 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index c514fc00a..073b6bd9a 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -35,11 +35,10 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` - method of :ref:`spider middlewares `. + method of :ref:`spider middlewares `. See + :ref:`async-spider-middlewares`. .. versionadded:: VERSION - .. note:: This method needs to be an async generator, not just a coroutine that - returns an iterable. Usage ===== @@ -106,8 +105,8 @@ Common use cases for asynchronous code include: * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the :signal:`spider_opened` handler); -* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see - :ref:`the screenshot pipeline example`). +* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download` + (see :ref:`the screenshot pipeline example`). .. _aio-libs: https://github.com/aio-libs @@ -119,59 +118,72 @@ Asynchronous spider middlewares .. versionadded:: VERSION .. note:: This currently applies to :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`. + In the future it will also apply to + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests`. Middleware methods discussed here can take and return async iterables. They can return the same type of iterable or they can take a normal one and return an async one. If such method needs to return an async iterable it must be an async generator, not just a coroutine that returns an iterable. -.. autofunction:: scrapy.utils.asyncgen.as_async_generator +As the result of a middleware method is passed to the same method of the next +middleware, it needs to be adapted if the second method expects a different +type. Scrapy will do this transparently: -In the simplest form that supports both sync and async input it can be written -like this:: +* A normal iterable is wrapped into an async one which shouldn't cause any side + effects. +* An async iterable is downgraded to a normal one by waiting until all results + are available and wrapping them in a normal iterable. This is problematic + because it pauses the normal middleware processing for this iterable and + because all results can be skipped if exceptions are raised during + processing. This case emits a warning and will be deprecated and then removed + in a later Scrapy version. +* Async iterables returned from + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception` + won't be downgraded, an exception will be raised if that is needed. - from scrapy.utils.asyncgen import as_async_generator +As downgrading is undesirable, here is the proposed way to avoid it. If all +middlewares, including 3rd-party ones, support async iterables as input, no +downgrading will happen. But removing normal iterable support (making the +method a coroutine) from a middleware published as a separate project or used +internally in projects for older Scrapy versions breaks backwards +compatibility. So, as an interim measure (it will be deprecated and then +removed in a later Scrapy version), a middleware can provide both sync and +async methods in the following form:: - class ProcessSpiderOutputAsyncGenMiddleware: - async def process_spider_output(self, response, result, spider): - async for r in as_async_generator(result): + class UniversalSpiderMiddleware: + def process_spider_output(self, response, result, spider): + for r in result: # ... do something with r yield r -If the middleware input (the callback result for ``process_spider_output``) is -an async iterable, all middlewares that process it must support it. The -built-in ones do, but the ones in your project and 3rd-party ones will need to -be updated to support it, as the code that expects a normal iterable will break -on an async one. If these middlewares receive an async iterable, they must -return one as well. On the other hand, if they receive a normal iterable, they -shouldn't break and ideally should return a normal iterable too. There can be -several possible implementations of this. + async def process_spider_output_async(self, response, result, spider): + async for r in result: + # ... do something with r + yield r -The simplest one, always converting normal iterables to async ones, is provided -above. Because a result of a middleware method is passed to the same method of -the next middleware, it's only possible to mix middlewares with synchronous and -asynchronous implementations of the same method if all synchronous ones are -called first (which isn't always possible). +In this case normal and async iterables will be passed to the respective +methods without any wrapping or downgrading, and in older versions of Scrapy +the coroutine method will just be ignored. When the backwards compatibility is +no longer needed the non-coroutine method can be dropped and the coroutine one +renamed to the normal name. It may be possible to extract common code from both +methods to reduce code duplication, as in the simplest case the only difference +between them will be ``for`` vs ``async for``. -Another option is to make separate methods for normal and async iterables and -choose one at run time:: +So, to recap: - from inspect import isasyncgen - - class ProcessSpiderOutputAsyncGenMiddleware: - def _normal_process_spider_output(self, response, result, spider): - # ... do something with normal result - - async def _async_process_spider_output(self, response, result, spider): - # ... do the same with async result - - def process_spider_output(self, response, result, spider): - if isasyncgen(result): - return self._async_process_spider_output(self, response, result, spider) - else: - return self._normal_process_spider_output(self, response, result, spider) - -If you are writing a middleware that you intend to publish or to use in many -projects, this is likely the best way to implement it. It may be possible to -extract common code from both methods to reduce code duplication, as in the -simplest case the only difference between them will be ``for`` vs ``async for``. +* If you don't intend to use async callbacks or middlewares containing async + code in your project, nothing should change for you yet. At some point in the + future some of the 3rd-party middlewares you use may drop backwards + compatibility, which shouldn't lead to immediate problems but may be a sign + to start converting your code to ``async def`` too. +* If you maintain a middleware that can be used with projects you can't control + (e.g. one you published for other people to use, or one that needs to support + some old project that can't be modernized), we recommend adding a + ``process_spider_output_async`` method so that the amount of unnecessary + iterable conversions is reduced but no compatibility is broken. +* If you use async callbacks, try to make sure all middlewares support them. + Note that you can modernize 3rd-party middlewares by subclassing them. +* If you want to write and publish a middleware that requires async code, you + should write in the docs that the minimum support Scrapy version is VERSION + (maybe even check this at the run time, using :attr:`scrapy.__version__`). diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f8d4a356f..edfc2e4bb 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -123,6 +123,15 @@ object gives you access, for example, to the :ref:`settings `. :param spider: the spider whose result is being processed :type spider: :class:`~scrapy.Spider` object + .. method:: process_spider_output_async(response, result, spider) + + .. versionadded:: VERSION + + If exists, this methid will be called instead of + :meth:`process_spider_output` when ``result`` is an async iterable. + If this method exists, it must be a coroutine while + :meth:`process_spider_output` must not be a coroutine. + .. method:: process_spider_exception(response, exception, spider) This method is called when a spider or :meth:`process_spider_output` diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index e64cfae0a..e1fdd8d13 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -192,7 +192,8 @@ class Scraper: spider=spider ) - def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred: + def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request, + response: Response, spider: Spider) -> Deferred: if not result: return defer_succeed(None) it: Union[Generator, AsyncGenerator] diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 009ece06d..6075670b0 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,22 +3,27 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +import logging +from inspect import isasyncgenfunction from itertools import islice -from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union, cast +from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager -from scrapy.utils.asyncgen import _process_iterable_universal +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list -from scrapy.utils.defer import mustbe_deferred +from scrapy.utils.defer import mustbe_deferred, deferred_from_coro, deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.python import MutableAsyncChain, MutableChain +logger = logging.getLogger(__name__) + + ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] @@ -30,6 +35,10 @@ class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' + def __init__(self, *middlewares): + super().__init__(*middlewares) + self.downgrade_warning_done = False + @classmethod def _get_mwlist_from_settings(cls, settings): return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) @@ -40,7 +49,7 @@ class SpiderMiddlewareManager(MiddlewareManager): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_start_requests'): self.methods['process_start_requests'].appendleft(mw.process_start_requests) - process_spider_output = getattr(mw, 'process_spider_output', None) + process_spider_output = self._get_async_method_pair(mw, 'process_spider_output') self.methods['process_spider_output'].appendleft(process_spider_output) process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) @@ -64,8 +73,19 @@ class SpiderMiddlewareManager(MiddlewareManager): def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable], exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain] ) -> Union[Generator, AsyncGenerator]: - @_process_iterable_universal - async def _evaluate_async_iterable(iterable): + + def process_sync(iterable: Iterable): + try: + for r in iterable: + yield r + except Exception as ex: + exception_result = self._process_spider_exception(response, spider, Failure(ex), + exception_processor_index) + if isinstance(exception_result, Failure): + raise + recover_to.extend(exception_result) + + async def process_async(iterable: AsyncIterable): try: async for r in iterable: yield r @@ -75,7 +95,10 @@ class SpiderMiddlewareManager(MiddlewareManager): if isinstance(exception_result, Failure): raise recover_to.extend(exception_result) - return _evaluate_async_iterable(iterable) + + if isinstance(iterable, AsyncIterable): + return process_async(iterable) + return process_sync(iterable) def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index: int = 0) -> Union[Failure, MutableChain]: @@ -87,11 +110,22 @@ class SpiderMiddlewareManager(MiddlewareManager): for method_index, method in enumerate(method_list, start=start_index): if method is None: continue + method = cast(Callable, method) result = method(response=response, exception=exception, spider=spider) if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - return self._process_spider_output(response, spider, result, method_index + 1) + dfd: Deferred = self._process_spider_output(response, spider, result, method_index + 1) + # _process_spider_output() returns a Deferred only because of downgrading so this can be + # simplified when downgrading is removed. + if dfd.called: + # the result is available immediately if _process_spider_output didn't do downgrading + return dfd.result + else: + # we forbid waiting here because otherwise we would need to return a deferred from + # _process_spider_exception too, which complicates the architecture + msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" + raise _InvalidOutput(msg) elif result is None: continue else: @@ -100,9 +134,13 @@ class SpiderMiddlewareManager(MiddlewareManager): raise _InvalidOutput(msg) return _failure + # This method cannot be made async def, as _process_spider_exception relies on the Deferred result + # being available immediately which doesn't work when it's a wrapped coroutine. + # It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed. + @inlineCallbacks def _process_spider_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable], start_index: int = 0 - ) -> Union[MutableChain, MutableAsyncChain]: + ) -> Deferred: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered: Union[MutableChain, MutableAsyncChain] @@ -112,11 +150,43 @@ class SpiderMiddlewareManager(MiddlewareManager): else: recovered = MutableChain() + # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async. + # 1. def foo. Sync iterables are passed as is, async ones are downgraded. + # 2. async def foo. Sync iterables are upgraded, async ones are passed as is. + # 3. def foo + async def foo_async. Iterables are passed to the respective method. + # Storing methods and method tuples in the same list is weird but we should be able to roll this back + # when we drop this compatibility feature. + method_list = islice(self.methods['process_spider_output'], start_index, None) - for method_index, method in enumerate(method_list, start=start_index): - if method is None: + for method_index, method_pair in enumerate(method_list, start=start_index): + if method_pair is None: continue + need_upgrade = need_downgrade = False + if isinstance(method_pair, tuple): + # This tuple handling is only needed until _async compatibility methods are removed. + method_sync, method_async = method_pair + method = method_async if last_result_is_async else method_sync + else: + method = method_pair + if not last_result_is_async and isasyncgenfunction(method): + need_upgrade = True + elif last_result_is_async and not isasyncgenfunction(method): + need_downgrade = True try: + if need_upgrade: + # Iterable -> AsyncIterable + result = as_async_generator(result) + elif need_downgrade: + if not self.downgrade_warning_done: + logger.warning(f"Async iterable passed to {method.__qualname__} " + f"was downgraded to a non-async one") + self.downgrade_warning_done = True + assert isinstance(result, AsyncIterable) + # AsyncIterable -> Iterable + result = yield deferred_from_coro(collect_asyncgen(result)) + if isinstance(recovered, AsyncIterable): + recovered_collected = yield deferred_from_coro(collect_asyncgen(recovered)) + recovered = MutableChain(recovered_collected) # might fail directly if the output value is not a generator result = method(response=response, result=result, spider=spider) except Exception as ex: @@ -130,8 +200,6 @@ class SpiderMiddlewareManager(MiddlewareManager): msg = (f"Middleware {method.__qualname__} must return an " f"iterable, got {type(result)}") raise _InvalidOutput(msg) - if last_result_is_async and isinstance(result, Iterable): - raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable") last_result_is_async = isinstance(result, AsyncIterable) if last_result_is_async: @@ -139,31 +207,58 @@ class SpiderMiddlewareManager(MiddlewareManager): else: return MutableChain(result, recovered) # type: ignore[arg-type] - def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] - ) -> Union[MutableChain, MutableAsyncChain]: + async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] + ) -> Union[MutableChain, MutableAsyncChain]: recovered: Union[MutableChain, MutableAsyncChain] if isinstance(result, AsyncIterable): recovered = MutableAsyncChain() else: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) - result = self._process_spider_output(response, spider, result) + result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result)) if isinstance(result, AsyncIterable): return MutableAsyncChain(result, recovered) else: + if isinstance(recovered, AsyncIterable): + recovered_collected = await collect_asyncgen(recovered) + recovered = MutableChain(recovered_collected) return MutableChain(result, recovered) # type: ignore[arg-type] def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Deferred: - def process_callback_output(result: Union[Iterable, AsyncIterable]) -> Union[MutableChain, MutableAsyncChain]: - return self._process_callback_output(response, spider, result) + async def process_callback_output(result: Union[Iterable, AsyncIterable] + ) -> Union[MutableChain, MutableAsyncChain]: + return await self._process_callback_output(response, spider, result) def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: return self._process_spider_exception(response, spider, _failure) dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) - dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) + dfd.addCallbacks(callback=deferred_f_from_coro_f(process_callback_output), errback=process_spider_exception) return dfd def process_start_requests(self, start_requests, spider: Spider) -> Deferred: return self._process_chain('process_start_requests', start_requests, spider) + + # This method is only needed until _async compatibility methods are removed. + @staticmethod + def _get_async_method_pair(mw: Any, methodname: str) -> Union[None, Callable, Tuple[Callable, Callable]]: + normal_method = getattr(mw, methodname, None) + methodname_async = methodname + "_async" + async_method = getattr(mw, methodname_async, None) + if not async_method: + return normal_method + if not normal_method: + logger.error(f"Middleware {mw.__qualname__} has {methodname_async} " + f"without {methodname}, skipping this method.") + return None + if not isasyncgenfunction(async_method): + logger.error(f"{async_method.__qualname__} is not " + f"an async generator function, skipping this method.") + return normal_method + if isasyncgenfunction(normal_method): + logger.error(f"{normal_method.__qualname__} is an async " + f"generator function while {methodname_async} exists, " + f"skipping both methods.") + return None + return normal_method, async_method diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 2eb1d8609..8d7e5a602 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Deque, Dict, Optional, cast, Iterable +from typing import Callable, Deque, Dict, Iterable, Tuple, Union, cast from twisted.internet.defer import Deferred @@ -21,8 +21,9 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - # Optional because process_spider_output and process_spider_exception can be None - self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque) + # Only process_spider_output and process_spider_exception can be None. + # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. + self.methods: Dict[str, Deque[Union[None, Callable, Tuple[Callable, Callable]]]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 973404b2b..29634c3ad 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -7,7 +7,6 @@ See documentation in docs/topics/spider-middleware.rst import logging from scrapy.http import Request -from scrapy.utils.asyncgen import _process_iterable_universal logger = logging.getLogger(__name__) @@ -29,36 +28,42 @@ class DepthMiddleware: return cls(maxdepth, crawler.stats, verbose, prio) def process_spider_output(self, response, result, spider): - def _filter(request): - if isinstance(request, Request): - depth = response.meta['depth'] + 1 - request.meta['depth'] = depth - if self.prio: - request.priority -= depth * self.prio - if self.maxdepth and depth > self.maxdepth: - logger.debug( - "Ignoring link (depth > %(maxdepth)d): %(requrl)s ", - {'maxdepth': self.maxdepth, 'requrl': request.url}, - extra={'spider': spider} - ) - return False - else: - if self.verbose_stats: - self.stats.inc_value(f'request_depth_count/{depth}', - spider=spider) - self.stats.max_value('request_depth_max', depth, - spider=spider) + # base case (depth=0) + if 'depth' not in response.meta: + response.meta['depth'] = 0 + if self.verbose_stats: + self.stats.inc_value('request_depth_count/0', spider=spider) + + return (r for r in result or () if self._filter(r, response, spider)) + + async def process_spider_output_async(self, response, result, spider): + # base case (depth=0) + if 'depth' not in response.meta: + response.meta['depth'] = 0 + if self.verbose_stats: + self.stats.inc_value('request_depth_count/0', spider=spider) + + async for r in result or (): + if self._filter(r, response, spider): + yield r + + def _filter(self, request, response, spider): + if not isinstance(request, Request): return True - - @_process_iterable_universal - async def process(result): - # base case (depth=0) - if 'depth' not in response.meta: - response.meta['depth'] = 0 - if self.verbose_stats: - self.stats.inc_value('request_depth_count/0', spider=spider) - - async for r in result or (): - if _filter(r): - yield r - return process(result) + depth = response.meta['depth'] + 1 + request.meta['depth'] = depth + if self.prio: + request.priority -= depth * self.prio + if self.maxdepth and depth > self.maxdepth: + logger.debug( + "Ignoring link (depth > %(maxdepth)d): %(requrl)s ", + {'maxdepth': self.maxdepth, 'requrl': request.url}, + extra={'spider': spider} + ) + return False + if self.verbose_stats: + self.stats.inc_value(f'request_depth_count/{depth}', + spider=spider) + self.stats.max_value('request_depth_max', depth, + spider=spider) + return True diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 074ec7a4e..448bc1367 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -9,7 +9,6 @@ import warnings from scrapy import signals from scrapy.http import Request -from scrapy.utils.asyncgen import _process_iterable_universal from scrapy.utils.httpobj import urlparse_cached logger = logging.getLogger(__name__) @@ -27,24 +26,27 @@ class OffsiteMiddleware: return o def process_spider_output(self, response, result, spider): - @_process_iterable_universal - async def process(result): - async for x in result: - if isinstance(x, Request): - if x.dont_filter or self.should_follow(x, spider): - yield x - else: - domain = urlparse_cached(x).hostname - if domain and domain not in self.domains_seen: - self.domains_seen.add(domain) - logger.debug( - "Filtered offsite request to %(domain)r: %(request)s", - {'domain': domain, 'request': x}, extra={'spider': spider}) - self.stats.inc_value('offsite/domains', spider=spider) - self.stats.inc_value('offsite/filtered', spider=spider) - else: - yield x - return process(result) + return (r for r in result or () if self._filter(r, spider)) + + async def process_spider_output_async(self, response, result, spider): + async for r in result or (): + if self._filter(r, spider): + yield r + + def _filter(self, request, spider) -> bool: + if not isinstance(request, Request): + return True + if request.dont_filter or self.should_follow(request, spider): + return True + domain = urlparse_cached(request).hostname + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {'domain': domain, 'request': request}, extra={'spider': spider}) + self.stats.inc_value('offsite/domains', spider=spider) + self.stats.inc_value('offsite/filtered', spider=spider) + return False def should_follow(self, request, spider): regex = self.host_regex diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index e0a22592f..8027beb92 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -11,7 +11,6 @@ from w3lib.url import safe_url_string from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.utils.asyncgen import _process_iterable_universal from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url @@ -334,19 +333,18 @@ class RefererMiddleware: return cls() if cls else self.default_policy() def process_spider_output(self, response, result, spider): - def _set_referer(r): - if isinstance(r, Request): - referrer = self.policy(response, r).referrer(response.url, r.url) - if referrer is not None: - r.headers.setdefault('Referer', referrer) - return r + return (self._set_referer(r, response) for r in result or ()) - @_process_iterable_universal - async def process(result): - async for r in result or (): - yield _set_referer(r) + async def process_spider_output_async(self, response, result, spider): + async for r in result or (): + yield self._set_referer(r, response) - return process(result) + def _set_referer(self, r, response): + if isinstance(r, Request): + referrer = self.policy(response, r).referrer(response.url, r.url) + if referrer is not None: + r.headers.setdefault('Referer', referrer) + return r def request_scheduled(self, request, spider): # check redirected request to patch "Referer" header if necessary diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 63b1d36bd..7ad64d2af 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -8,7 +8,6 @@ import logging from scrapy.http import Request from scrapy.exceptions import NotConfigured -from scrapy.utils.asyncgen import _process_iterable_universal logger = logging.getLogger(__name__) @@ -26,22 +25,20 @@ class UrlLengthMiddleware: return cls(maxlength) def process_spider_output(self, response, result, spider): - def _filter(request): - if isinstance(request, Request) and len(request.url) > self.maxlength: - logger.info( - "Ignoring link (url length > %(maxlength)d): %(url)s ", - {'maxlength': self.maxlength, 'url': request.url}, - extra={'spider': spider} - ) - spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider) - return False - else: - return True + return (r for r in result or () if self._filter(r, spider)) - @_process_iterable_universal - async def process(result): - async for r in result or (): - if _filter(r): - yield r + async def process_spider_output_async(self, response, result, spider): + async for r in result or (): + if self._filter(r, spider): + yield r - return process(result) + def _filter(self, request, spider): + if isinstance(request, Request) and len(request.url) > self.maxlength: + logger.info( + "Ignoring link (url length > %(maxlength)d): %(url)s ", + {'maxlength': self.maxlength, 'url': request.url}, + extra={'spider': spider} + ) + spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider) + return False + return True diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index ae9a79989..9f794de92 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,6 +1,4 @@ -import functools -import inspect -from typing import AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union +from typing import AsyncGenerator, AsyncIterable, Iterable, Union async def collect_asyncgen(result: AsyncIterable): @@ -18,46 +16,3 @@ async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerat else: for r in it: yield r - - -# https://stackoverflow.com/a/66170760/113586 -def _process_iterable_universal(process_async: Callable): - """ Takes a function that takes an async iterable, args and kwargs. Returns - a function that takes any iterable, args and kwargs. - - Requires that process_async only awaits on the iterable and synchronous functions, - so it's better to use this only in the Scrapy code itself. - """ - - # If this stops working, all internal uses can be just replaced with manually-written - # process_sync functions. - - def process_sync(iterable: Iterable, *args, **kwargs) -> Generator: - agen = process_async(as_async_generator(iterable), *args, **kwargs) - if not inspect.isasyncgen(agen): - raise ValueError(f"process_async returned wrong type {type(agen)}") - sent = None - while True: - try: - gen = agen.asend(sent) - gen.send(None) - except StopIteration as e: - sent = yield e.value - except StopAsyncIteration: - return - else: - gen.throw(RuntimeError, - f"Synchronously-called function '{process_async.__name__}' has blocked, " - f"you can't use {_process_iterable_universal.__name__} with it.") - - @functools.wraps(process_async) - def process(iterable: Union[Iterable, AsyncIterable], *args, **kwargs) -> Union[Generator, AsyncGenerator]: - if isinstance(iterable, AsyncIterable): - # call process_async directly - return process_async(iterable, *args, **kwargs) - if isinstance(iterable, Iterable): - # convert process_async to process_sync - return process_sync(iterable, *args, **kwargs) - raise TypeError(f"Wrong iterable type {type(iterable)}") - - return process diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index d086347bc..11c089ac2 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -10,7 +10,7 @@ import warnings import weakref from functools import partial, wraps from itertools import chain -from typing import AsyncIterable, Iterable, Union +from typing import AsyncGenerator, AsyncIterable, Iterable, Union from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator @@ -359,7 +359,7 @@ class MutableChain(Iterable): return self.__next__() -async def _async_chain(*iterables: Union[Iterable, AsyncIterable]): +async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator: for it in iterables: async for o in as_async_generator(it): yield o diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index b0ca2f62e..f9f2b6642 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,6 +1,8 @@ import collections.abc +from typing import Optional from unittest import mock +from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase from twisted.python.failure import Failure @@ -8,8 +10,8 @@ from twisted.python.failure import Failure from scrapy.spiders import Spider from scrapy.http import Request, Response from scrapy.exceptions import _InvalidOutput -from scrapy.utils.asyncgen import _process_iterable_universal, as_async_generator, collect_asyncgen -from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.asyncgen import collect_asyncgen +from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.test import get_crawler from scrapy.core.spidermw import SpiderMiddlewareManager @@ -115,33 +117,40 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects + @staticmethod + def _construct_mw_setting(*mw_classes, start_index: Optional[int] = None): + if start_index is None: + start_index = 10 + return {i: c for c, i in enumerate(mw_classes, start=start_index)} + @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes): - for mw_cls in mw_classes: - self.mwman._add_middleware(mw_cls()) + def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + setting = self._construct_mw_setting(*mw_classes, start_index=start_index) + self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting}) + self.spider = self.crawler._create_spider('foo') + self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider) return result @defer.inlineCallbacks - def _test_simple_base(self, *mw_classes): - result = yield self._get_middleware_result(*mw_classes) + def _test_simple_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None): + with LogCapture() as log: + result = yield self._get_middleware_result(*mw_classes, start_index=start_index) self.assertIsInstance(result, collections.abc.Iterable) result_list = list(result) self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) + self.assertEqual("downgraded to a non-async" in str(log), downgrade) @defer.inlineCallbacks - def _test_asyncgen_base(self, *mw_classes): - result = yield self._get_middleware_result(*mw_classes) + def _test_asyncgen_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None): + with LogCapture() as log: + result = yield self._get_middleware_result(*mw_classes, start_index=start_index) self.assertIsInstance(result, collections.abc.AsyncIterator) result_list = yield deferred_from_coro(collect_asyncgen(result)) self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) - - @defer.inlineCallbacks - def _test_asyncgen_fail(self, *mw_classes): - with self.assertRaisesRegex(TypeError, "Synchronous .+ called with an async iterable"): - yield self._get_middleware_result(*mw_classes) + self.assertEqual("downgraded to a non-async" in str(log), downgrade) class ProcessSpiderOutputSimpleMiddleware: @@ -152,17 +161,36 @@ class ProcessSpiderOutputSimpleMiddleware: class ProcessSpiderOutputAsyncGenMiddleware: async def process_spider_output(self, response, result, spider): - async for r in as_async_generator(result): + async for r in result: yield r class ProcessSpiderOutputUniversalMiddleware: def process_spider_output(self, response, result, spider): - @_process_iterable_universal - async def process(result): - async for r in result: - yield r - return process(result) + for r in result: + yield r + + async def process_spider_output_async(self, response, result, spider): + async for r in result: + yield r + + +class ProcessSpiderExceptionSimpleIterableMiddleware: + def process_spider_exception(self, response, exception, spider): + yield {'foo': 1} + yield {'foo': 2} + yield {'foo': 3} + + +class ProcessSpiderExceptionAsyncIterableMiddleware: + async def process_spider_exception(self, response, exception, spider): + yield {'foo': 1} + d = defer.Deferred() + from twisted.internet import reactor + reactor.callLater(0, d.callback, None) + await maybe_deferred_to_future(d) + yield {'foo': 2} + yield {'foo': 3} class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): @@ -183,18 +211,19 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): return self._test_simple_base(self.MW_SIMPLE) def test_asyncgen(self): - """ Asyncgen mw """ + """ Asyncgen mw; upgrade """ return self._test_asyncgen_base(self.MW_ASYNCGEN) def test_simple_asyncgen(self): - """ Simple mw -> asyncgen mw """ + """ Simple mw -> asyncgen mw; upgrade """ return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE) def test_asyncgen_simple(self): - """ Asyncgen mw -> simple mw; cannot work """ - return self._test_asyncgen_fail(self.MW_SIMPLE, - self.MW_ASYNCGEN) + """ Asyncgen mw -> simple mw; upgrade then downgrade """ + return self._test_simple_base(self.MW_SIMPLE, + self.MW_ASYNCGEN, + downgrade=True) def test_universal(self): """ Universal mw """ @@ -211,12 +240,12 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): self.MW_SIMPLE) def test_universal_asyncgen(self): - """ Universal mw -> asyncgen mw """ + """ Universal mw -> asyncgen mw; upgrade """ return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) def test_asyncgen_universal(self): - """ Asyncgen mw -> universal mw """ + """ Asyncgen mw -> universal mw; upgrade """ return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) @@ -229,27 +258,31 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): yield item def test_simple(self): - """ Simple mw; cannot work """ - return self._test_asyncgen_fail(self.MW_SIMPLE) + """ Simple mw; downgrade """ + return self._test_simple_base(self.MW_SIMPLE, + downgrade=True) def test_simple_asyncgen(self): - """ Simple mw -> asyncgen mw; cannot work """ - return self._test_asyncgen_fail(self.MW_ASYNCGEN, - self.MW_SIMPLE) + """ Simple mw -> asyncgen mw; downgrade then upgrade """ + return self._test_asyncgen_base(self.MW_ASYNCGEN, + self.MW_SIMPLE, + downgrade=True) def test_universal(self): """ Universal mw """ return self._test_asyncgen_base(self.MW_UNIVERSAL) def test_universal_simple(self): - """ Universal mw -> simple mw; cannot work """ - return self._test_asyncgen_fail(self.MW_SIMPLE, - self.MW_UNIVERSAL) + """ Universal mw -> simple mw; downgrade """ + return self._test_simple_base(self.MW_SIMPLE, + self.MW_UNIVERSAL, + downgrade=True) def test_simple_universal(self): - """ Simple mw -> universal mw; cannot work """ - return self._test_asyncgen_fail(self.MW_UNIVERSAL, - self.MW_SIMPLE) + """ Simple mw -> universal mw; downgrade """ + return self._test_simple_base(self.MW_UNIVERSAL, + self.MW_SIMPLE, + downgrade=True) class ProcessStartRequestsSimpleMiddleware: @@ -269,13 +302,198 @@ class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): yield Request(f'https://example.com/{i}', dont_filter=True) @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes): - for mw_cls in mw_classes: - self.mwman._add_middleware(mw_cls()) + def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + setting = self._construct_mw_setting(*mw_classes, start_index=start_index) + self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting}) + self.spider = self.crawler._create_spider('foo') + self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) start_requests = iter(self._start_requests()) results = yield self.mwman.process_start_requests(start_requests, self.spider) return results def test_simple(self): """ Simple mw """ - self._test_simple_base(self.MW_SIMPLE) + return self._test_simple_base(self.MW_SIMPLE) + + +class UniversalMiddlewareNoSync: + async def process_spider_output_async(self, response, result, spider): + yield + + +class UniversalMiddlewareBothSync: + def process_spider_output(self, response, result, spider): + yield + + def process_spider_output_async(self, response, result, spider): + yield + + +class UniversalMiddlewareBothAsync: + async def process_spider_output(self, response, result, spider): + yield + + async def process_spider_output_async(self, response, result, spider): + yield + + +class UniversalMiddlewareManagerTest(TestCase): + def setUp(self): + self.mwman = SpiderMiddlewareManager() + + def test_simple_mw(self): + mw = ProcessSpiderOutputSimpleMiddleware + self.mwman._add_middleware(mw) + self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output) + + def test_async_mw(self): + mw = ProcessSpiderOutputAsyncGenMiddleware + self.mwman._add_middleware(mw) + self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output) + + def test_universal_mw(self): + mw = ProcessSpiderOutputUniversalMiddleware + self.mwman._add_middleware(mw) + self.assertEqual(self.mwman.methods['process_spider_output'][0], + (mw.process_spider_output, mw.process_spider_output_async)) + + def test_universal_mw_no_sync(self): + with LogCapture() as log: + self.mwman._add_middleware(UniversalMiddlewareNoSync) + self.assertIn("UniversalMiddlewareNoSync has process_spider_output_async" + " without process_spider_output", str(log)) + self.assertEqual(self.mwman.methods['process_spider_output'][0], None) + + def test_universal_mw_both_sync(self): + mw = UniversalMiddlewareBothSync + with LogCapture() as log: + self.mwman._add_middleware(mw) + self.assertIn("UniversalMiddlewareBothSync.process_spider_output_async " + "is not an async generator function", str(log)) + self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output) + + def test_universal_mw_both_async(self): + with LogCapture() as log: + self.mwman._add_middleware(UniversalMiddlewareBothAsync) + self.assertIn("UniversalMiddlewareBothAsync.process_spider_output " + "is an async generator function while process_spider_output_async exists", + str(log)) + self.assertEqual(self.mwman.methods['process_spider_output'][0], None) + + +class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): + ITEM_TYPE = dict + MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware + MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware + MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware + + def _scrape_func(self, *args, **kwargs): + yield {'foo': 1} + yield {'foo': 2} + yield {'foo': 3} + + @defer.inlineCallbacks + def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + setting = self._construct_mw_setting(*mw_classes, start_index=start_index) + self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES': setting}) + self.spider = self.crawler._create_spider('foo') + self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) + result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider) + return result + + def test_just_builtin(self): + return self._test_simple_base() + + def test_builtin_simple(self): + return self._test_simple_base(self.MW_SIMPLE, start_index=1000) + + def test_builtin_async(self): + """ Upgrade """ + return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) + + def test_builtin_universal(self): + return self._test_simple_base(self.MW_UNIVERSAL, start_index=1000) + + def test_simple_builtin(self): + return self._test_simple_base(self.MW_SIMPLE) + + def test_async_builtin(self): + """ Upgrade """ + return self._test_asyncgen_base(self.MW_ASYNCGEN) + + def test_universal_builtin(self): + return self._test_simple_base(self.MW_UNIVERSAL) + + +class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): + async def _scrape_func(self, *args, **kwargs): + for item in super()._scrape_func(): + yield item + + def test_just_builtin(self): + return self._test_asyncgen_base() + + def test_builtin_simple(self): + """ Downgrade """ + return self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000) + + def test_builtin_async(self): + return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) + + def test_builtin_universal(self): + return self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) + + def test_simple_builtin(self): + """ Downgrade """ + return self._test_simple_base(self.MW_SIMPLE, downgrade=True) + + def test_async_builtin(self): + return self._test_asyncgen_base(self.MW_ASYNCGEN) + + def test_universal_builtin(self): + return self._test_asyncgen_base(self.MW_UNIVERSAL) + + +class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase): + ITEM_TYPE = dict + MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware + MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware + MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware + MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware + MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIterableMiddleware + + def _scrape_func(self, *args, **kwargs): + 1 / 0 + + @defer.inlineCallbacks + def _test_asyncgen_nodowngrade(self, *mw_classes): + with self.assertRaisesRegex(_InvalidOutput, "Async iterable returned from .+ cannot be downgraded"): + yield self._get_middleware_result(*mw_classes) + + def test_exc_simple(self): + """ Simple exc mw """ + return self._test_simple_base(self.MW_EXC_SIMPLE) + + def test_exc_async(self): + """ Async exc mw """ + return self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) + + def test_exc_simple_simple(self): + """ Simple exc mw -> simple output mw """ + return self._test_simple_base(self.MW_SIMPLE, + self.MW_EXC_SIMPLE) + + def test_exc_async_async(self): + """ Async exc mw -> async output mw """ + return self._test_asyncgen_base(self.MW_ASYNCGEN, + self.MW_EXC_ASYNCGEN) + + def test_exc_simple_async(self): + """ Simple exc mw -> async output mw; upgrade """ + return self._test_asyncgen_base(self.MW_ASYNCGEN, + self.MW_EXC_SIMPLE) + + def test_exc_async_simple(self): + """ Async exc mw -> simple output mw; cannot work as downgrading is not supported """ + return self._test_asyncgen_nodowngrade(self.MW_SIMPLE, + self.MW_EXC_ASYNCGEN) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 088c14ca8..dac246fb6 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -28,6 +28,7 @@ class RecoveryMiddleware: class RecoverySpider(Spider): name = 'RecoverySpider' custom_settings = { + 'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': { RecoveryMiddleware: 10, }, @@ -107,6 +108,13 @@ class GeneratorCallbackSpider(Spider): raise ImportError() +class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider): + async def parse(self, response): + yield {'test': 1} + yield {'test': 2} + raise ImportError() + + # ================================================================================ # (2.1) exceptions from a spider callback (generator, middleware right after callback) class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider): @@ -360,6 +368,15 @@ class TestSpiderMiddleware(TestCase): self.assertIn("Middleware: ImportError exception caught", str(log2)) self.assertIn("'item_scraped_count': 2", str(log2)) + @defer.inlineCallbacks + def test_async_generator_callback(self): + """ + Same as test_generator_callback but with an async callback. + """ + log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider) + self.assertIn("Middleware: ImportError exception caught", str(log2)) + self.assertIn("'item_scraped_count': 2", str(log2)) + @defer.inlineCallbacks def test_generator_callback_right_after_callback(self): """ diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 7abe17c22..9ae66c57c 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,7 +1,6 @@ -from twisted.internet.defer import Deferred from twisted.trial import unittest -from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import deferred_f_from_coro_f @@ -19,52 +18,3 @@ class AsyncgenUtilsTest(unittest.TestCase): ag = as_async_generator(range(42)) results = await collect_asyncgen(ag) self.assertEqual(results, list(range(42))) - - -@_process_iterable_universal -async def process_iterable(iterable): - async for i in iterable: - yield i * 2 - - -@_process_iterable_universal -async def process_iterable_awaiting(iterable): - async for i in iterable: - yield i * 2 - d = Deferred() - from twisted.internet import reactor - reactor.callLater(0, d.callback, 42) - await d - - -class ProcessIterableUniversalTest(unittest.TestCase): - - def test_normal(self): - iterable = iter([1, 2, 3]) - results = list(process_iterable(iterable)) - self.assertEqual(results, [2, 4, 6]) - - @deferred_f_from_coro_f - async def test_async(self): - iterable = as_async_generator([1, 2, 3]) - results = await collect_asyncgen(process_iterable(iterable)) - self.assertEqual(results, [2, 4, 6]) - - @deferred_f_from_coro_f - async def test_blocking(self): - iterable = [1, 2, 3] - with self.assertRaisesRegex(RuntimeError, "Synchronously-called function"): - list(process_iterable_awaiting(iterable)) - - def test_invalid_iterable(self): - with self.assertRaisesRegex(TypeError, "Wrong iterable type"): - process_iterable(None) - - @deferred_f_from_coro_f - async def test_invalid_process(self): - @_process_iterable_universal - def process_iterable_invalid(iterable): - pass - - with self.assertRaisesRegex(ValueError, "process_async returned wrong type"): - list(process_iterable_invalid([])) From e079bffdab11402d1936103ba453e43e97925079 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Jan 2022 19:21:07 +0500 Subject: [PATCH 0461/2083] Disable logging-fstring-interpolation in pylint. --- pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/pylintrc b/pylintrc index 2cdd6321e..0d29dc709 100644 --- a/pylintrc +++ b/pylintrc @@ -49,6 +49,7 @@ disable=abstract-method, keyword-arg-before-vararg, line-too-long, logging-format-interpolation, + logging-fstring-interpolation, logging-not-lazy, lost-exception, method-hidden, From 4bdaa54af4470582845206a9fa21779f80a4b123 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 28 Jan 2022 15:39:32 +0200 Subject: [PATCH 0462/2083] response_httprepr memory issue fixed (#4972) * response_httprepr replaced by response.body * unused import deleted * get_header_size function added * response size calculation updated * flake8 codestyle fix * added counting status code, line breaks to response size * get_status size: list to tuple, comments added * test added: comparing new response size counting method with old `len(response_httprepr)` * downloader stats : unreachable code deleted * `get_status_size` optimized * comment added * tests.test_downloadermiddleware_stats: statement formatting updated * scrapy.utils.response: `response_httprepr` -> deprecated * tests.test_downloadermiddleware_stats: flake8 fix --- scrapy/downloadermiddlewares/stats.py | 22 +++++++++++++++++++--- scrapy/utils/response.py | 2 ++ tests/test_downloadermiddleware_stats.py | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 5479cd0e2..25fb1ed9d 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,7 +1,22 @@ from scrapy.exceptions import NotConfigured +from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr -from scrapy.utils.response import response_httprepr -from scrapy.utils.python import global_object_name + +from twisted.web import http + + +def get_header_size(headers): + size = 0 + for key, value in headers.items(): + if isinstance(value, (list, tuple)): + for v in value: + size += len(b": ") + len(key) + len(v) + return size + len(b'\r\n') * (len(headers.keys()) - 1) + + +def get_status_size(response_status): + return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15 + # resp.status + b"\r\n" + b"HTTP/1.1 <100-599> " class DownloaderStats: @@ -24,7 +39,8 @@ class DownloaderStats: def process_response(self, request, response, spider): self.stats.inc_value('downloader/response_count', spider=spider) self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider) - reslen = len(response_httprepr(response)) + reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4 + # response.body + b"\r\n"+ response.header + b"\r\n" + response.status self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) return response diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 8b109dced..741dce350 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -14,6 +14,7 @@ from scrapy.http.response import Response from twisted.web import http from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.decorators import deprecated from w3lib import html @@ -51,6 +52,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str: return f'{status_int} {to_unicode(message)}' +@deprecated def response_httprepr(response: Response) -> bytes: """Return raw HTTP representation (as bytes) of the given response. This is provided only for reference, since it's not the exact stream of bytes diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 1f2616e35..9e75f0a50 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,8 +1,10 @@ +from itertools import product from unittest import TestCase from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.utils.response import response_httprepr from scrapy.utils.test import get_crawler @@ -37,6 +39,23 @@ class TestDownloaderStats(TestCase): self.mw.process_response(self.req, self.res, self.spider) self.assertStatsEqual('downloader/response_count', 1) + def test_response_len(self): + body = (b'', b'not_empty') # empty/notempty body + headers = ({}, {'lang': 'en'}, {'lang': 'en', 'User-Agent': 'scrapy'}) # 0 headers, 1h and 2h + test_responses = [ # form test responses with all combinations of body/headers + Response( + url='scrapytest.org', + status=200, + body=r[0], + headers=r[1] + ) + for r in product(body, headers) + ] + for test_response in test_responses: + self.crawler.stats.set_value('downloader/response_bytes', 0) + self.mw.process_response(self.req, test_response, self.spider) + self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response))) + def test_process_exception(self): self.mw.process_exception(self.req, MyException(), self.spider) self.assertStatsEqual('downloader/exception_count', 1) From 30d5779ea94ed1e9343a4590895a3f5e65e444b9 Mon Sep 17 00:00:00 2001 From: "Sixuan (Cherie) Wu" <73203695+inspurwusixuan@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:30:30 -0800 Subject: [PATCH 0463/2083] Fix FEED_URI_PARAMS: custom params throws KeyError (#4966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix FEED_URI_PARAMS: custom params throws KeyError closes #4962 * another try FEED_URI_PARAMS * add warning message and change default function * Add tests for FEED_URI_PARAMS * FEED_URI_PARAMS: warn if the params dict has been modified in-place * [Doc] FEED_URI_PARAMS: modifying params in-place is deprecated * Remove whileline * Rename parameters for lambda function * Type hints for FeedExporter._get_uri_params Co-authored-by: Adrián Chaves Co-authored-by: Eugenio Lacuesta --- docs/topics/feed-exports.rst | 7 +- scrapy/extensions/feedexport.py | 26 +++-- tests/test_feedexport.py | 163 ++++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 116967280..7994027d2 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -322,7 +322,7 @@ Post-Processing Scrapy provides an option to activate plugins to post-process feeds before they are exported to feed storages. In addition to using :ref:`builtin plugins `, you -can create your own :ref:`plugins `. +can create your own :ref:`plugins `. These plugins can be activated through the ``postprocessing`` option of a feed. The option must be passed a list of post-processing plugins in the order you want @@ -366,7 +366,7 @@ Each plugin is a class that must implement the following methods: Close the target file object. -To pass a parameter to your plugin, use :ref:`feed options `. You +To pass a parameter to your plugin, use :ref:`feed options `. You can then access those parameters from the ``__init__`` method of your plugin. @@ -744,6 +744,9 @@ The function signature should be as follows: :param spider: source spider of the feed items :type spider: scrapy.Spider + .. caution:: The function should return a new dictionary, modifying + the received ``params`` in-place is deprecated. + For example, to include the :attr:`name ` of the source spider in the feed URI: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 370723368..e7097b7a1 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,14 +11,14 @@ import sys import warnings from datetime import datetime from tempfile import NamedTemporaryFile -from typing import Any, Optional, Tuple +from typing import Any, Callable, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads from w3lib.url import file_uri_to_path from zope.interface import implementer, Interface -from scrapy import signals +from scrapy import signals, Spider from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available @@ -524,7 +524,12 @@ class FeedExporter: raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None") return instance - def _get_uri_params(self, spider, uri_params, slot=None): + def _get_uri_params( + self, + spider: Spider, + uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]], + slot: Optional[_FeedSlot] = None, + ) -> dict: params = {} for k in dir(spider): params[k] = getattr(spider, k) @@ -532,9 +537,18 @@ class FeedExporter: params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-') params['batch_time'] = utc_now.isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 - uripar_function = load_object(uri_params) if uri_params else lambda x, y: None - uripar_function(params, spider) - return params + original_params = params.copy() + uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params + new_params = uripar_function(params, spider) + if new_params is None or original_params != params: + warnings.warn( + 'Modifying the params dictionary in-place in the function defined in ' + 'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS ' + 'setting is deprecated. The function must return a new dictionary ' + 'instead.', + category=ScrapyDeprecationWarning + ) + return new_params if new_params is not None else params def _load_filter(self, feed_options): # load the item filter if declared else load the default filter class diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 253f3119c..f0acf1941 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2608,3 +2608,166 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): ), ) ) + + +class URIParamsTest: + + spider_name = "uri_params_spider" + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + raise NotImplementedError + + def test_default(self): + settings = self.build_settings( + uri='file:///tmp/%(name)s', + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_none(self): + def uri_params(params, spider): + pass + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual( + messages, + ( + ( + 'Modifying the params dictionary in-place in the ' + 'function defined in the FEED_URI_PARAMS setting or ' + 'in the uri_params key of the FEEDS setting is ' + 'deprecated. The function must return a new ' + 'dictionary instead.' + ), + ) + ) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_empty_dict(self): + def uri_params(params, spider): + return {} + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + with self.assertRaises(KeyError): + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + def test_params_as_is(self): + def uri_params(params, spider): + return params + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_custom_param(self): + def uri_params(params, spider): + return {**params, 'foo': self.spider_name} + + settings = self.build_settings( + uri='file:///tmp/%(foo)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + +class URIParamsSettingTest(URIParamsTest, unittest.TestCase): + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + extra_settings = {} + if uri_params: + extra_settings['FEED_URI_PARAMS'] = uri_params + return { + 'FEED_URI': uri, + **extra_settings, + } + + +class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase): + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + options = { + 'format': 'jl', + } + if uri_params: + options['uri_params'] = uri_params + return { + 'FEEDS': { + uri: options, + }, + } From fe43411bc4d0164a0f0ecc596c23b59c99d31f17 Mon Sep 17 00:00:00 2001 From: Laerte <5853172+Laerte@users.noreply.github.com> Date: Fri, 4 Feb 2022 05:57:57 -0300 Subject: [PATCH 0464/2083] Fix TypeError on using pathlib.Path as key on FEEDS settings (#5384) --- scrapy/settings/__init__.py | 6 +++++- tests/test_cmdline/__init__.py | 4 ++++ tests/test_cmdline/settings.py | 9 +++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 1fe1e6fd1..6b1ad0828 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -375,9 +375,13 @@ class BaseSettings(MutableMapping): return len(self.attributes) def _to_dict(self): - return {k: (v._to_dict() if isinstance(v, BaseSettings) else v) + return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items()} + def _get_key(self, key_value): + return (key_value if isinstance(key_value, (bool, float, int, str, type(None))) + else str(key_value)) + def copy_to_dict(self): """ Make a copy of current settings and convert to a dict. diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 591075a98..8233e0101 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -64,3 +64,7 @@ class CmdlineTest(unittest.TestCase): settingsdict = json.loads(settingsstr) self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) self.assertEqual(200, settingsdict[EXT_PATH]) + + def test_pathlib_path_as_feeds_key(self): + self.assertEqual(self._execute('settings', '--get', 'FEEDS'), + json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}})) diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py index 8a719ddf2..b0ac6e98b 100644 --- a/tests/test_cmdline/settings.py +++ b/tests/test_cmdline/settings.py @@ -1,5 +1,14 @@ +from pathlib import Path + EXTENSIONS = { 'tests.test_cmdline.extensions.TestExtension': 0, } TEST1 = 'default' + +FEEDS = { + Path('items.csv'): { + 'format': 'csv', + 'fields': ['price', 'name'], + }, +} From 9be878fc09cf71bb2cb98695f5042cb344bd2e25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 4 Feb 2022 12:27:39 +0100 Subject: [PATCH 0465/2083] CI: stop using tox-pip-version (#5389) --- .github/workflows/checks.yml | 4 ---- .github/workflows/tests-ubuntu.yml | 4 ---- 2 files changed, 8 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 80df9469d..98fa44c7f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,7 +19,6 @@ jobs: - python-version: 3.8 env: TOXENV: pylint - TOX_PIP_VERSION: 20.3.3 - python-version: 3.6 env: TOXENV: typing @@ -38,8 +37,5 @@ jobs: - name: Run check env: ${{ matrix.env }} run: | - if [[ ! -z "$TOX_PIP_VERSION" ]]; then - pip install tox-pip-version - fi pip install -U tox tox diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 5ea50e644..1fc8d914b 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -46,7 +46,6 @@ jobs: - python-version: 3.8 env: TOXENV: extra-deps - TOX_PIP_VERSION: 20.3.3 steps: - uses: actions/checkout@v2 @@ -73,9 +72,6 @@ jobs: $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - if [[ ! -z "$TOX_PIP_VERSION" ]]; then - pip install tox-pip-version - fi pip install -U tox tox From 55ae2109c95e497d4a730afeb9caf71aa78a7723 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 5 Feb 2022 13:02:02 -0300 Subject: [PATCH 0466/2083] Remove deprecated TextResponse.body_as_unicode --- scrapy/http/response/text.py | 9 --------- tests/test_http_response.py | 18 ------------------ 2 files changed, 27 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 27bd55c07..89516b9b6 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -6,7 +6,6 @@ See documentation in docs/topics/request-response.rst """ import json -import warnings from contextlib import suppress from typing import Generator, Tuple from urllib.parse import urljoin @@ -16,7 +15,6 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, http_content_type_encoding, resolve_encoding) from w3lib.html import strip_html5_whitespace -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode @@ -66,13 +64,6 @@ class TextResponse(Response): or self._body_declared_encoding() ) - def body_as_unicode(self): - """Return body as unicode""" - warnings.warn('Response.body_as_unicode() is deprecated, ' - 'please use Response.text instead.', - ScrapyDeprecationWarning, stacklevel=2) - return self.text - def json(self): """ .. versionadded:: 2.2 diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 0ec5257e1..2986f884f 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,10 +1,8 @@ import unittest from unittest import mock -from warnings import catch_warnings, filterwarnings from w3lib.encoding import resolve_encoding -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import (Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers) from scrapy.selector import Selector @@ -134,9 +132,6 @@ class BaseResponseTest(unittest.TestCase): assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - self.assertEqual(response.body_as_unicode(), body_unicode) self.assertEqual(response.text, body_unicode) def _assert_response_encoding(self, response, encoding): @@ -346,12 +341,6 @@ class TextResponseTest(BaseResponseTest): original_string = unicode_string.encode('cp1251') r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') - # check body_as_unicode - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - self.assertTrue(isinstance(r1.body_as_unicode(), str)) - self.assertEqual(r1.body_as_unicode(), unicode_string) - # check response.text self.assertTrue(isinstance(r1.text, str)) self.assertEqual(r1.text, unicode_string) @@ -683,13 +672,6 @@ class TextResponseTest(BaseResponseTest): with self.assertRaises(ValueError): response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]') - def test_body_as_unicode_deprecation_warning(self): - with catch_warnings(record=True) as warnings: - r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8') - self.assertEqual(r1.body_as_unicode(), 'Hello') - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - def test_json_response(self): json_body = b"""{"ip": "109.187.217.200"}""" json_response = self.response_class("http://www.example.com", body=json_body) From 38d2a154ec79767558f699ec663697ccc7f64ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Ferenc=20Gyarmati?= Date: Sun, 6 Feb 2022 18:52:15 +0100 Subject: [PATCH 0467/2083] docs: use https scheme for each quotes.toscrape.com url occurrence --- docs/intro/examples.rst | 2 +- docs/intro/overview.rst | 4 +-- docs/intro/tutorial.rst | 52 ++++++++++++++++----------------- docs/topics/developer-tools.rst | 20 ++++++------- docs/topics/logging.rst | 2 +- docs/topics/settings.rst | 4 +-- docs/topics/signals.rst | 2 +- 7 files changed, 43 insertions(+), 43 deletions(-) diff --git a/docs/intro/examples.rst b/docs/intro/examples.rst index 96363c7d5..edff894c6 100644 --- a/docs/intro/examples.rst +++ b/docs/intro/examples.rst @@ -7,7 +7,7 @@ Examples The best way to learn is with examples, and Scrapy is no exception. For this reason, there is an example Scrapy project named quotesbot_, that you can use to play and learn more about Scrapy. It contains two spiders for -http://quotes.toscrape.com, one using CSS selectors and another one using XPath +https://quotes.toscrape.com, one using CSS selectors and another one using XPath expressions. The quotesbot_ project is available at: https://github.com/scrapy/quotesbot. diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 405bf845d..f3d652621 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. Here's the code for a spider that scrapes famous quotes from website -http://quotes.toscrape.com, following the pagination:: +https://quotes.toscrape.com, following the pagination:: import scrapy @@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination:: class QuotesSpider(scrapy.Spider): name = 'quotes' start_urls = [ - 'http://quotes.toscrape.com/tag/humor/', + 'https://quotes.toscrape.com/tag/humor/', ] def parse(self, response): diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index ca5856881..5697b9608 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -7,7 +7,7 @@ Scrapy Tutorial In this tutorial, we'll assume that Scrapy is already installed on your system. If that's not the case, see :ref:`intro-install`. -We are going to scrape `quotes.toscrape.com `_, a website +We are going to scrape `quotes.toscrape.com `_, a website that lists quotes from famous authors. This tutorial will walk you through these tasks: @@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named def start_requests(self): urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -143,9 +143,9 @@ similar to this:: 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened 2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) 2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023 - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished) @@ -184,8 +184,8 @@ for your spider:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] def parse(self, response): @@ -207,7 +207,7 @@ Extracting data The best way to learn how to extract data with Scrapy is trying selectors using the :ref:`Scrapy shell `. Run:: - scrapy shell 'http://quotes.toscrape.com/page/1/' + scrapy shell 'https://quotes.toscrape.com/page/1/' .. note:: @@ -217,18 +217,18 @@ using the :ref:`Scrapy shell `. Run:: On Windows, use double quotes instead:: - scrapy shell "http://quotes.toscrape.com/page/1/" + scrapy shell "https://quotes.toscrape.com/page/1/" You will see something like:: [ ... Scrapy log here ... ] - 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [s] Available Scrapy objects: [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) [s] crawler [s] item {} - [s] request - [s] response <200 http://quotes.toscrape.com/page/1/> + [s] request + [s] response <200 https://quotes.toscrape.com/page/1/> [s] settings [s] spider [s] Useful shortcuts: @@ -241,7 +241,7 @@ object: .. invisible-code-block: python - response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html') + response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html') >>> response.css('title') [] @@ -355,7 +355,7 @@ Extracting quotes and authors Now that you know a bit about selection and extraction, let's complete our spider by writing the code to extract the quotes from the web page. -Each quote in http://quotes.toscrape.com is represented by HTML elements that look +Each quote in https://quotes.toscrape.com is represented by HTML elements that look like this: .. code-block:: html @@ -379,7 +379,7 @@ like this: Let's open up scrapy shell and play a bit to find out how to extract the data we want:: - $ scrapy shell 'http://quotes.toscrape.com' + $ scrapy shell 'https://quotes.toscrape.com' We get a list of selectors for the quote HTML elements with: @@ -444,8 +444,8 @@ in the callback, as you can see below:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] def parse(self, response): @@ -458,9 +458,9 @@ in the callback, as you can see below:: If you run this spider, it will output the extracted data with the log:: - 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.â€'} - 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.â€"} @@ -505,7 +505,7 @@ Following links =============== Let's say, instead of just scraping the stuff from the first two pages -from http://quotes.toscrape.com, you want quotes from all the pages in the website. +from https://quotes.toscrape.com, you want quotes from all the pages in the website. Now that you know how to extract data from pages, let's see how to follow links from them. @@ -549,7 +549,7 @@ page, extracting data from it:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/1/', ] def parse(self, response): @@ -600,7 +600,7 @@ As a shortcut for creating Request objects you can use class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/1/', ] def parse(self, response): @@ -654,7 +654,7 @@ this time for scraping author information:: class AuthorSpider(scrapy.Spider): name = 'author' - start_urls = ['http://quotes.toscrape.com/'] + start_urls = ['https://quotes.toscrape.com/'] def parse(self, response): author_page_links = response.css('.author + a') @@ -727,7 +727,7 @@ with a specific tag, building the URL based on the argument:: name = "quotes" def start_requests(self): - url = 'http://quotes.toscrape.com/' + url = 'https://quotes.toscrape.com/' tag = getattr(self, 'tag', None) if tag is not None: url = url + 'tag/' + tag @@ -747,7 +747,7 @@ with a specific tag, building the URL based on the argument:: If you pass the ``tag=humor`` argument to this spider, you'll notice that it will only visit URLs from the ``humor`` tag, such as -``http://quotes.toscrape.com/tag/humor``. +``https://quotes.toscrape.com/tag/humor``. You can :ref:`learn more about handling spider arguments here `. diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 057b1ec62..96475899f 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class= "text"`` we will see the quote-text we clicked on. The `Inspector` lets you copy XPaths to selected elements. Let's try it out. -First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal: +First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal: .. code-block:: none - $ scrapy shell "http://quotes.toscrape.com/" + $ scrapy shell "https://quotes.toscrape.com/" Then, back to your web browser, right-click on the ``span`` tag, select ``Copy > XPath`` and paste it in the Scrapy shell like so: .. invisible-code-block: python - response = load_response('http://quotes.toscrape.com/', 'quotes.html') + response = load_response('https://quotes.toscrape.com/', 'quotes.html') >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.â€'] @@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the type ``json``. If we click on this request, we see that the request URL is -``http://quotes.toscrape.com/api/quotes?page=1`` and the response +``https://quotes.toscrape.com/api/quotes?page=1`` and the response is a JSON-object that contains our quotes. We can also right-click on the request and open ``Open in new tab`` to get a better overview. @@ -247,7 +247,7 @@ also request each page to get every quote on the site:: name = 'quote' allowed_domains = ['quotes.toscrape.com'] page = 1 - start_urls = ['http://quotes.toscrape.com/api/quotes?page=1'] + start_urls = ['https://quotes.toscrape.com/api/quotes?page=1'] def parse(self, response): data = json.loads(response.text) @@ -255,7 +255,7 @@ also request each page to get every quote on the site:: yield {"quote": quote["text"]} if data["has_next"]: self.page += 1 - url = f"http://quotes.toscrape.com/api/quotes?page={self.page}" + url = f"https://quotes.toscrape.com/api/quotes?page={self.page}" yield scrapy.Request(url=url, callback=self.parse) This spider starts at the first page of the quotes-API. With each @@ -280,7 +280,7 @@ request:: from scrapy import Request request = Request.from_curl( - "curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil" + "curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil" "la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce" "pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X" "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM" @@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down to identifying the correct request and replicating it in your spider. .. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools -.. _quotes.toscrape.com: http://quotes.toscrape.com -.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll -.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10 +.. _quotes.toscrape.com: https://quotes.toscrape.com +.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll +.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index d593c74c6..3bf23d5f5 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -218,7 +218,7 @@ For example, let's say you're scraping a website which returns many HTTP 404 and 500 responses, and you want to hide all messages like this:: 2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring - response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code + response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code is not handled or not allowed The first thing to note is a logger name - it is in brackets: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f6c95c502..4e105642d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1597,7 +1597,7 @@ In order to use the reactor installed by Scrapy:: def start_requests(self): reactor.callLater(self.timeout, self.stop) - urls = ['http://quotes.toscrape.com/page/1'] + urls = ['https://quotes.toscrape.com/page/1'] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -1625,7 +1625,7 @@ which raises :exc:`Exception`, becomes:: from twisted.internet import reactor reactor.callLater(self.timeout, self.stop) - urls = ['http://quotes.toscrape.com/page/1'] + urls = ['https://quotes.toscrape.com/page/1'] for url in urls: yield scrapy.Request(url=url, callback=self.parse) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 63ad3a9ad..2fbd0b51c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -60,7 +60,7 @@ Let's take an example:: class SignalSpider(scrapy.Spider): name = 'signals' - start_urls = ['http://quotes.toscrape.com/page/1/'] + start_urls = ['https://quotes.toscrape.com/page/1/'] @classmethod def from_crawler(cls, crawler, *args, **kwargs): From bbfa185664cef79299b48cb0ae22065439bb07fc Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 6 Feb 2022 18:12:28 -0300 Subject: [PATCH 0468/2083] Remove deprecated BaseItem class --- scrapy/exporters.py | 4 +-- scrapy/item.py | 34 ++------------------ scrapy/utils/misc.py | 4 +-- tests/test_item.py | 75 +------------------------------------------- 4 files changed, 8 insertions(+), 109 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 36cca2d05..1c26e81db 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -13,7 +13,7 @@ from xml.sax.saxutils import XMLGenerator from itemadapter import is_item, ItemAdapter from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import _BaseItem +from scrapy.item import Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -315,7 +315,7 @@ class PythonItemExporter(BaseItemExporter): return serializer(value) def _serialize_value(self, value): - if isinstance(value, _BaseItem): + if isinstance(value, Item): return self.export_item(value) elif is_item(value): return dict(self._serialize_item(value)) diff --git a/scrapy/item.py b/scrapy/item.py index 2ccd7ad18..839bee3fa 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -15,39 +15,11 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref -class _BaseItem(object_ref): - """ - Temporary class used internally to avoid the deprecation - warning raised by isinstance checks using BaseItem. - """ - pass - - -class _BaseItemMeta(ABCMeta): - def __instancecheck__(cls, instance): - if cls is BaseItem: - warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__instancecheck__(instance) - - -class BaseItem(_BaseItem, metaclass=_BaseItemMeta): - """ - Deprecated, please use :class:`scrapy.item.Item` instead - """ - - def __new__(cls, *args, **kwargs): - if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)): - warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls, *args, **kwargs) - - class Field(dict): """Container of field metadata""" -class ItemMeta(_BaseItemMeta): +class ItemMeta(ABCMeta): """Metaclass_ of :class:`Item` that handles field definitions. .. _metaclass: https://realpython.com/python-metaclasses @@ -74,7 +46,7 @@ class ItemMeta(_BaseItemMeta): return super().__new__(mcs, class_name, bases, new_attrs) -class DictItem(MutableMapping, BaseItem): +class DictItem(MutableMapping, object_ref): fields: Dict[str, Field] = {} @@ -118,7 +90,7 @@ class DictItem(MutableMapping, BaseItem): def __iter__(self): return iter(self._values) - __hash__ = BaseItem.__hash__ + __hash__ = object_ref.__hash__ def keys(self): return self._values.keys() diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 11c4206c2..1221b39b2 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -14,11 +14,11 @@ from w3lib.html import replace_entities from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.python import flatten, to_unicode -from scrapy.item import _BaseItem +from scrapy.item import Item from scrapy.utils.deprecate import ScrapyDeprecationWarning -_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes +_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes def arg_to_iter(arg): diff --git a/tests/test_item.py b/tests/test_item.py index c94bb44af..7d82fbffe 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -3,7 +3,7 @@ from unittest import mock from warnings import catch_warnings, filterwarnings from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta +from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta class ItemTest(unittest.TestCase): @@ -318,79 +318,6 @@ class DictItemTest(unittest.TestCase): self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) -class BaseItemTest(unittest.TestCase): - - def test_isinstance_check(self): - - class SubclassedBaseItem(BaseItem): - pass - - class SubclassedItem(Item): - pass - - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem)) - self.assertTrue(isinstance(Item(), BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), BaseItem)) - - # make sure internal checks using private _BaseItem class succeed - self.assertTrue(isinstance(BaseItem(), _BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), _BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), _BaseItem)) - - def test_deprecation_warning(self): - """ - Make sure deprecation warnings are logged whenever BaseItem is used, - either instantiated or in an isinstance check - """ - with catch_warnings(record=True) as warnings: - BaseItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - - class SubclassedBaseItem(BaseItem): - pass - - SubclassedBaseItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - self.assertFalse(isinstance("foo", BaseItem)) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - -class ItemNoDeprecationWarningTest(unittest.TestCase): - def test_no_deprecation_warning(self): - """ - Make sure deprecation warnings are NOT logged whenever BaseItem subclasses are used. - """ - class SubclassedItem(Item): - pass - - with catch_warnings(record=True) as warnings: - Item() - SubclassedItem() - _BaseItem() - self.assertFalse(isinstance("foo", _BaseItem)) - self.assertFalse(isinstance("foo", Item)) - self.assertFalse(isinstance("foo", SubclassedItem)) - self.assertTrue(isinstance(_BaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), Item)) - self.assertTrue(isinstance(SubclassedItem(), SubclassedItem)) - self.assertEqual(len(warnings), 0) - if __name__ == "__main__": unittest.main() From c8c1edd43b04ce7a2d9b1da198af26ba271cb1d6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 6 Feb 2022 18:27:41 -0300 Subject: [PATCH 0469/2083] Flake8 adjustments --- tests/test_item.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_item.py b/tests/test_item.py index 7d82fbffe..a12e425e0 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,6 +1,6 @@ import unittest from unittest import mock -from warnings import catch_warnings, filterwarnings +from warnings import catch_warnings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta @@ -318,6 +318,5 @@ class DictItemTest(unittest.TestCase): self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - if __name__ == "__main__": unittest.main() From fca49cca929de035fb5d179d83f7f79da22fd205 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 6 Feb 2022 18:31:55 -0300 Subject: [PATCH 0470/2083] Remove deprecated DictItem class --- docs/conf.py | 1 - scrapy/item.py | 55 +++++++++++++++++++--------------------------- tests/test_item.py | 31 +------------------------- 3 files changed, 23 insertions(+), 64 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 406c4d94a..d5e139e66 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -272,7 +272,6 @@ coverage_ignore_pyobjects = [ r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions # Never documented before, and deprecated now. - r'^scrapy\.item\.DictItem$', r'^scrapy\.linkextractors\.FilteringLinkExtractor$', # Implementation detail of LxmlLinkExtractor diff --git a/scrapy/item.py b/scrapy/item.py index 839bee3fa..2521ac829 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -9,9 +9,7 @@ from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat from typing import Dict -from warnings import warn -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref @@ -46,15 +44,30 @@ class ItemMeta(ABCMeta): return super().__new__(mcs, class_name, bases, new_attrs) -class DictItem(MutableMapping, object_ref): +class Item(MutableMapping, object_ref, metaclass=ItemMeta): + """ + Base class for scraped items. - fields: Dict[str, Field] = {} + In Scrapy, an object is considered an ``item`` if it is an instance of either + :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a + spider callback is evaluated, only instances of :class:`Item` or + :class:`dict` are passed to :ref:`item pipelines `. - def __new__(cls, *args, **kwargs): - if issubclass(cls, DictItem) and not issubclass(cls, Item): - warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls, *args, **kwargs) + If you need instances of a custom class to be considered items by Scrapy, + you must inherit from either :class:`Item` or :class:`dict`. + + Items must declare :class:`Field` attributes, which are processed and stored + in the ``fields`` attribute. This restricts the set of allowed field names + and prevents typos, raising ``KeyError`` when referring to undefined fields. + Additionally, fields can be used to define metadata and control the way + data is processed internally. Please refer to the :ref:`documentation + about fields ` for additional information. + + Unlike instances of :class:`dict`, instances of :class:`Item` may be + :ref:`tracked ` to debug memory leaks. + """ + + fields: Dict[str, Field] def __init__(self, *args, **kwargs): self._values = {} @@ -105,27 +118,3 @@ class DictItem(MutableMapping, object_ref): """Return a :func:`~copy.deepcopy` of this item. """ return deepcopy(self) - - -class Item(DictItem, metaclass=ItemMeta): - """ - Base class for scraped items. - - In Scrapy, an object is considered an ``item`` if it is an instance of either - :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a - spider callback is evaluated, only instances of :class:`Item` or - :class:`dict` are passed to :ref:`item pipelines `. - - If you need instances of a custom class to be considered items by Scrapy, - you must inherit from either :class:`Item` or :class:`dict`. - - Items must declare :class:`Field` attributes, which are processed and stored - in the ``fields`` attribute. This restricts the set of allowed field names - and prevents typos, raising ``KeyError`` when referring to undefined fields. - Additionally, fields can be used to define metadata and control the way - data is processed internally. Please refer to the :ref:`documentation - about fields ` for additional information. - - Unlike instances of :class:`dict`, instances of :class:`Item` may be - :ref:`tracked ` to debug memory leaks. - """ diff --git a/tests/test_item.py b/tests/test_item.py index a12e425e0..25f2aea0a 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,9 +1,7 @@ import unittest from unittest import mock -from warnings import catch_warnings -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta +from scrapy.item import ABCMeta, Field, Item, ItemMeta class ItemTest(unittest.TestCase): @@ -254,18 +252,6 @@ class ItemTest(unittest.TestCase): item['tags'].append('tag2') assert item['tags'] != copied_item['tags'] - def test_dictitem_deprecation_warning(self): - """Make sure the DictItem deprecation warning is not issued for - Item""" - with catch_warnings(record=True) as warnings: - Item() - self.assertEqual(len(warnings), 0) - - class SubclassedItem(Item): - pass - SubclassedItem() - self.assertEqual(len(warnings), 0) - class ItemMetaTest(unittest.TestCase): @@ -303,20 +289,5 @@ class ItemMetaClassCellRegression(unittest.TestCase): super().__init__(*args, **kwargs) -class DictItemTest(unittest.TestCase): - - def test_deprecation_warning(self): - with catch_warnings(record=True) as warnings: - DictItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with catch_warnings(record=True) as warnings: - class SubclassedDictItem(DictItem): - pass - SubclassedDictItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - if __name__ == "__main__": unittest.main() From b282a7af012a4804eb91bdd850df3b86065b3fd6 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 8 Feb 2022 01:25:08 +0500 Subject: [PATCH 0471/2083] Temporarily pin Twisted to an older version in CI (#5401) --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 2031a2d92..cf7855cf9 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,8 @@ deps = mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 + # Temporary until the tests are updated + Twisted<22.1.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From 4bda0976b28917405f54c781afda5ea55b65b16b Mon Sep 17 00:00:00 2001 From: Laerte <5853172+Laerte@users.noreply.github.com> Date: Tue, 8 Feb 2022 10:57:19 -0300 Subject: [PATCH 0472/2083] Fix csviter call, add parse_rows test (#5394) --- scrapy/spiders/feed.py | 2 +- tests/test_spider.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index bef2d6b24..79e12e030 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -123,7 +123,7 @@ class CSVFeedSpider(Spider): process_results methods for pre and post-processing purposes. """ - for row in csviter(response, self.delimiter, self.headers, self.quotechar): + for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar): ret = iterate_spider_output(self.parse_row(response, row)) for result_item in self.process_results(response, ret): yield result_item diff --git a/tests/test_spider.py b/tests/test_spider.py index a7c3ee048..689349999 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -21,6 +21,7 @@ from scrapy.spiders import ( ) from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler +from tests import get_testdata class SpiderTest(unittest.TestCase): @@ -167,6 +168,23 @@ class CSVFeedSpiderTest(SpiderTest): spider_class = CSVFeedSpider + def test_parse_rows(self): + body = get_testdata('feeds', 'feed-sample6.csv') + response = Response("http://example.org/dummy.csv", body=body) + + class _CrawlSpider(self.spider_class): + name = "test" + delimiter = "," + quotechar = "'" + + def parse_row(self, response, row): + return row + + spider = _CrawlSpider() + rows = list(spider.parse_rows(response)) + assert rows[0] == {'id': '1', 'name': 'alpha', 'value': 'foobar'} + assert len(rows) == 4 + class CrawlSpiderTest(SpiderTest): From fd55f62207bbbb18d7758c8e2ef46fe9115eb2c5 Mon Sep 17 00:00:00 2001 From: Raihan Nismara <31585789+raihan71@users.noreply.github.com> Date: Tue, 8 Feb 2022 21:36:25 +0700 Subject: [PATCH 0473/2083] Update Logo in README.rst (#5258) --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 05f10bb6c..6b563d638 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,4 @@ -.. image:: /artwork/scrapy-logo.jpg - :width: 400px +.. image:: https://scrapy.org/img/scrapylogo.png ====== Scrapy From 1e1cfc26dbdde5fc3035169884c4d1218844d0de Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 8 Feb 2022 21:01:16 +0500 Subject: [PATCH 0474/2083] Copy resource classes from twisted.web.test.test_webclient. --- pytest.ini | 2 - tests/mockserver.py | 74 ++++++++++++++++++++++++++++--- tests/test_downloader_handlers.py | 12 +++-- tests/test_webclient.py | 18 ++++---- 4 files changed, 86 insertions(+), 20 deletions(-) diff --git a/pytest.ini b/pytest.ini index fa5d6b34f..ae2ed2029 100644 --- a/pytest.ini +++ b/pytest.ini @@ -21,5 +21,3 @@ addopts = markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed -filterwarnings= - ignore::DeprecationWarning:twisted.web.test.test_webclient diff --git a/tests/mockserver.py b/tests/mockserver.py index ab9aec6a6..72d7e0241 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -14,10 +14,9 @@ from twisted.internet import defer, reactor, ssl from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory -from twisted.web.resource import EncodingResourceWrapper, Resource +from twisted.web import resource, server from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site from twisted.web.static import File -from twisted.web.test.test_webclient import PayloadResource from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode @@ -35,7 +34,70 @@ def getarg(request, name, default=None, type=None): return default -class LeafResource(Resource): +# most of the following resources are copied from twisted.web.test.test_webclient +class ForeverTakingResource(resource.Resource): + """ + L{ForeverTakingResource} is a resource which never finishes responding + to requests. + """ + + def __init__(self, write=False): + resource.Resource.__init__(self) + self._write = write + + def render(self, request): + if self._write: + request.write(b"some bytes") + return server.NOT_DONE_YET + + +class ErrorResource(resource.Resource): + def render(self, request): + request.setResponseCode(401) + if request.args.get(b"showlength"): + request.setHeader(b"content-length", b"0") + return b"" + + +class NoLengthResource(resource.Resource): + def render(self, request): + return b"nolength" + + +class HostHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of the host header + from the request. + """ + + def render(self, request): + return request.requestHeaders.getRawHeaders(b"host")[0] + + +class PayloadResource(resource.Resource): + """ + A testing resource which renders itself as the contents of the request body + as long as the request body is 100 bytes long, otherwise which renders + itself as C{"ERROR"}. + """ + + def render(self, request): + data = request.content.read() + contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0] + if len(data) != 100 or int(contentLength) != 100: + return b"ERROR" + return data + + +class BrokenDownloadResource(resource.Resource): + def render(self, request): + # only sends 3 bytes even though it claims to send 5 + request.setHeader(b"content-length", b"5") + request.write(b"abc") + return b"" + + +class LeafResource(resource.Resource): isLeaf = True @@ -175,10 +237,10 @@ class ArbitraryLengthPayloadResource(LeafResource): return request.content.read() -class Root(Resource): +class Root(resource.Resource): def __init__(self): - Resource.__init__(self) + resource.Resource.__init__(self) self.putChild(b"status", Status()) self.putChild(b"follow", Follow()) self.putChild(b"delay", Delay()) @@ -187,7 +249,7 @@ class Root(Resource): self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) self.putChild(b"payload", PayloadResource()) - self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) + self.putChild(b"xpayload", resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) try: from tests import tests_datadir diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a1ea4c679..2bb53950d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -15,8 +15,6 @@ from twisted.trial import unittest from twisted.web import resource, server, static, util from twisted.web._newclient import ResponseFailed from twisted.web.http import _DataLoss -from twisted.web.test.test_webclient import (ForeverTakingResource, HostHeaderResource, - NoLengthResource, PayloadResource) from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers @@ -34,7 +32,15 @@ from scrapy.spiders import Spider from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto -from tests.mockserver import MockServer, ssl_context_factory, Echo +from tests.mockserver import ( + Echo, + ForeverTakingResource, + HostHeaderResource, + MockServer, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) from tests.spiders import SingleRequestSpider diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 6e4cb9b6e..a6d55cb38 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -21,14 +21,6 @@ except ImportError: from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from twisted.internet.defer import inlineCallbacks -from twisted.web.test.test_webclient import ( - ForeverTakingResource, - ErrorResource, - NoLengthResource, - HostHeaderResource, - PayloadResource, - BrokenDownloadResource, -) from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory @@ -36,7 +28,15 @@ from scrapy.http import Request, Headers from scrapy.settings import Settings from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes, to_unicode -from tests.mockserver import ssl_context_factory +from tests.mockserver import ( + BrokenDownloadResource, + ErrorResource, + ForeverTakingResource, + HostHeaderResource, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): From 77547a1ab554a5b9afa7d7a343d8f90ef1d4cfe8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 8 Feb 2022 21:06:02 +0500 Subject: [PATCH 0475/2083] Revert "Temporarily pin Twisted to an older version in CI (#5401)" This reverts commit b282a7af012a4804eb91bdd850df3b86065b3fd6. --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index cf7855cf9..2031a2d92 100644 --- a/tox.ini +++ b/tox.ini @@ -19,8 +19,6 @@ deps = mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 - # Temporary until the tests are updated - Twisted<22.1.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From e2e2ffd0d162cfed5a2e82e9fb9472dbf233c919 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 9 Feb 2022 11:52:07 -0800 Subject: [PATCH 0476/2083] Move from optparse to argparse (#5374) --- scrapy/cmdline.py | 14 +++---- scrapy/commands/__init__.py | 75 ++++++++++++++++++++++++------------ scrapy/commands/check.py | 8 ++-- scrapy/commands/fetch.py | 10 ++--- scrapy/commands/genspider.py | 20 +++++----- scrapy/commands/parse.py | 44 ++++++++++----------- scrapy/commands/settings.py | 20 +++++----- scrapy/commands/shell.py | 12 +++--- scrapy/commands/version.py | 4 +- scrapy/commands/view.py | 3 +- tests/test_command_parse.py | 19 +++++++++ tests/test_commands.py | 38 ++++++++++++++---- 12 files changed, 168 insertions(+), 99 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 91482ce01..491c4beab 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,13 +1,13 @@ import sys import os -import optparse +import argparse import cProfile import inspect import pkg_resources import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.commands import ScrapyCommand +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings @@ -123,8 +123,6 @@ def execute(argv=None, settings=None): inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) - parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), - conflict_handler='resolve') if not cmdname: _print_commands(settings, inproject) sys.exit(0) @@ -133,12 +131,14 @@ def execute(argv=None, settings=None): sys.exit(2) cmd = cmds[cmdname] - parser.usage = f"scrapy {cmdname} {cmd.syntax()}" - parser.description = cmd.long_desc() + parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, + usage=f"scrapy {cmdname} {cmd.syntax()}", + conflict_handler='resolve', + description=cmd.long_desc()) settings.setdict(cmd.default_settings, priority='command') cmd.settings = settings cmd.add_options(parser) - opts, args = parser.parse_args(args=argv[1:]) + opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) cmd.crawler_process = CrawlerProcess(settings) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 5f1dabd33..fb304b8c0 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -2,7 +2,7 @@ Base class for Scrapy commands """ import os -from optparse import OptionGroup +import argparse from typing import Any, Dict from twisted.python import failure @@ -59,22 +59,20 @@ class ScrapyCommand: """ Populate option parse with options available for this command """ - group = OptionGroup(parser, "Global Options") - group.add_option("--logfile", metavar="FILE", - help="log file. if omitted stderr will be used") - group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help=f"log level (default: {self.settings['LOG_LEVEL']})") - group.add_option("--nolog", action="store_true", - help="disable logging completely") - group.add_option("--profile", metavar="FILE", default=None, - help="write python cProfile stats to FILE") - group.add_option("--pidfile", metavar="FILE", - help="write process ID to FILE") - group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", - help="set/override setting (may be repeated)") - group.add_option("--pdb", action="store_true", help="enable pdb on failure") - - parser.add_option_group(group) + group = parser.add_argument_group(title='Global Options') + group.add_argument("--logfile", metavar="FILE", + help="log file. if omitted stderr will be used") + group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None, + help=f"log level (default: {self.settings['LOG_LEVEL']})") + group.add_argument("--nolog", action="store_true", + help="disable logging completely") + group.add_argument("--profile", metavar="FILE", default=None, + help="write python cProfile stats to FILE") + group.add_argument("--pidfile", metavar="FILE", + help="write process ID to FILE") + group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE", + help="set/override setting (may be repeated)") + group.add_argument("--pdb", action="store_true", help="enable pdb on failure") def process_options(self, args, opts): try: @@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand): """ def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="append scraped items to the end of FILE (use - for stdout)") - parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append", - help="dump scraped items into FILE, overwriting any existing file") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items") + parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_argument("-o", "--output", metavar="FILE", action="append", + help="append scraped items to the end of FILE (use - for stdout)") + parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append", + help="dump scraped items into FILE, overwriting any existing file") + parser.add_argument("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand): opts.overwrite_output, ) self.settings.set('FEEDS', feeds, priority='cmdline') + + +class ScrapyHelpFormatter(argparse.HelpFormatter): + """ + Help Formatter for scrapy command line help messages. + """ + def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): + super().__init__(prog, indent_increment=indent_increment, + max_help_position=max_help_position, width=width) + + def _join_parts(self, part_strings): + parts = self.format_part_strings(part_strings) + return super()._join_parts(parts) + + def format_part_strings(self, part_strings): + """ + Underline and title case command line help message headers. + """ + if part_strings and part_strings[0].startswith("usage: "): + part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):] + headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')] + for index in headings[::-1]: + char = '-' if "Global Options" in part_strings[index] else '=' + part_strings[index] = part_strings[index][:-2].title() + underline = ''.join(["\n", (char * len(part_strings[index])), "\n"]) + part_strings.insert(index + 1, underline) + return part_strings diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index ae21d86e6..a16f4beb7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -49,10 +49,10 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-l", "--list", dest="list", action="store_true", - help="only list contracts, without checking them") - parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true', - help="print contract tests for all spiders") + parser.add_argument("-l", "--list", dest="list", action="store_true", + help="only list contracts, without checking them") + parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true', + help="print contract tests for all spiders") def run(self, args, opts): # load contracts diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 95f87e8c3..9b2ebb37f 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -26,11 +26,11 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", help="use this spider") - parser.add_option("--headers", dest="headers", action="store_true", - help="print response HTTP headers instead of body") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, - help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_argument("--spider", dest="spider", help="use this spider") + parser.add_argument("--headers", dest="headers", action="store_true", + help="print response HTTP headers instead of body") + parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2082a4974..ed5f588e9 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -44,16 +44,16 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-l", "--list", dest="list", action="store_true", - help="List available templates") - parser.add_option("-e", "--edit", dest="edit", action="store_true", - help="Edit spider after creating it") - parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE", - help="Dump template to standard output") - parser.add_option("-t", "--template", dest="template", default="basic", - help="Uses a custom template.") - parser.add_option("--force", dest="force", action="store_true", - help="If the spider already exists, overwrite it with the template") + parser.add_argument("-l", "--list", dest="list", action="store_true", + help="List available templates") + parser.add_argument("-e", "--edit", dest="edit", action="store_true", + help="Edit spider after creating it") + parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE", + help="Dump template to standard output") + parser.add_argument("-t", "--template", dest="template", default="basic", + help="Uses a custom template.") + parser.add_argument("--force", dest="force", action="store_true", + help="If the spider already exists, overwrite it with the template") def run(self, args, opts): if opts.list: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 52118db1b..a3f6b96f4 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand): def add_options(self, parser): BaseRunSpiderCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", default=None, - help="use this spider without looking for one") - parser.add_option("--pipelines", action="store_true", - help="process items through pipelines") - parser.add_option("--nolinks", dest="nolinks", action="store_true", - help="don't show links to follow (extracted requests)") - parser.add_option("--noitems", dest="noitems", action="store_true", - help="don't show scraped items") - parser.add_option("--nocolour", dest="nocolour", action="store_true", - help="avoid using pygments to colorize the output") - parser.add_option("-r", "--rules", dest="rules", action="store_true", - help="use CrawlSpider rules to discover the callback") - parser.add_option("-c", "--callback", dest="callback", - help="use this callback for parsing, instead looking for a callback") - parser.add_option("-m", "--meta", dest="meta", - help="inject extra meta into the Request, it must be a valid raw json string") - parser.add_option("--cbkwargs", dest="cbkwargs", - help="inject extra callback kwargs into the Request, it must be a valid raw json string") - parser.add_option("-d", "--depth", dest="depth", type="int", default=1, - help="maximum depth for parsing requests [default: %default]") - parser.add_option("-v", "--verbose", dest="verbose", action="store_true", - help="print each depth level one by one") + parser.add_argument("--spider", dest="spider", default=None, + help="use this spider without looking for one") + parser.add_argument("--pipelines", action="store_true", + help="process items through pipelines") + parser.add_argument("--nolinks", dest="nolinks", action="store_true", + help="don't show links to follow (extracted requests)") + parser.add_argument("--noitems", dest="noitems", action="store_true", + help="don't show scraped items") + parser.add_argument("--nocolour", dest="nocolour", action="store_true", + help="avoid using pygments to colorize the output") + parser.add_argument("-r", "--rules", dest="rules", action="store_true", + help="use CrawlSpider rules to discover the callback") + parser.add_argument("-c", "--callback", dest="callback", + help="use this callback for parsing, instead looking for a callback") + parser.add_argument("-m", "--meta", dest="meta", + help="inject extra meta into the Request, it must be a valid raw json string") + parser.add_argument("--cbkwargs", dest="cbkwargs", + help="inject extra callback kwargs into the Request, it must be a valid raw json string") + parser.add_argument("-d", "--depth", dest="depth", type=int, default=1, + help="maximum depth for parsing requests [default: %default]") + parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", + help="print each depth level one by one") @property def max_level(self): diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 8d49e440f..1b2e2601e 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -18,16 +18,16 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--get", dest="get", metavar="SETTING", - help="print raw setting value") - parser.add_option("--getbool", dest="getbool", metavar="SETTING", - help="print setting value, interpreted as a boolean") - parser.add_option("--getint", dest="getint", metavar="SETTING", - help="print setting value, interpreted as an integer") - parser.add_option("--getfloat", dest="getfloat", metavar="SETTING", - help="print setting value, interpreted as a float") - parser.add_option("--getlist", dest="getlist", metavar="SETTING", - help="print setting value, interpreted as a list") + parser.add_argument("--get", dest="get", metavar="SETTING", + help="print raw setting value") + parser.add_argument("--getbool", dest="getbool", metavar="SETTING", + help="print setting value, interpreted as a boolean") + parser.add_argument("--getint", dest="getint", metavar="SETTING", + help="print setting value, interpreted as an integer") + parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING", + help="print setting value, interpreted as a float") + parser.add_argument("--getlist", dest="getlist", metavar="SETTING", + help="print setting value, interpreted as a list") def run(self, args, opts): settings = self.crawler_process.settings diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index de81986d8..f67a5886a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -33,12 +33,12 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-c", dest="code", - help="evaluate the code in the shell, print the result and exit") - parser.add_option("--spider", dest="spider", - help="use this spider") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, - help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_argument("-c", dest="code", + help="evaluate the code in the shell, print the result and exit") + parser.add_argument("--spider", dest="spider", + help="use this spider") + parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 1237610cb..c6a3c273a 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -16,8 +16,8 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--verbose", "-v", dest="verbose", action="store_true", - help="also display twisted/python/platform info (useful for bug reports)") + parser.add_argument("--verbose", "-v", dest="verbose", action="store_true", + help="also display twisted/python/platform info (useful for bug reports)") def run(self, args, opts): if opts.verbose: diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index c8f873334..b1f52abe2 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,3 +1,4 @@ +import argparse from scrapy.commands import fetch from scrapy.utils.response import open_in_browser @@ -12,7 +13,7 @@ class Command(fetch.Command): def add_options(self, parser): super().add_options(parser) - parser.remove_option("--headers") + parser.add_argument('--headers', help=argparse.SUPPRESS) def _print_response(self, response, opts): open_in_browser(response) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index ed3848d88..f21ee971d 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,6 +1,9 @@ import os +import argparse from os.path import join, abspath, isfile, exists from twisted.internet import defer +from scrapy.commands import parse +from scrapy.settings import Settings from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest from scrapy.utils.python import to_unicode @@ -239,3 +242,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} content = '[\n{},\n{"foo": "bar"}\n]' with open(file_path, 'r') as f: self.assertEqual(f.read(), content) + + def test_parse_add_options(self): + command = parse.Command() + command.settings = Settings() + parser = argparse.ArgumentParser( + prog='scrapy', formatter_class=argparse.HelpFormatter, + conflict_handler='resolve', prefix_chars='-' + ) + command.add_options(parser) + namespace = parser.parse_args( + ['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name] + ) + self.assertTrue(namespace.nolinks) + self.assertEqual(namespace.depth, 2) + self.assertEqual(namespace.spider, self.spider_name) + self.assertTrue(namespace.verbose) diff --git a/tests/test_commands.py b/tests/test_commands.py index 7473b53df..7cd19b29a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,6 +1,6 @@ import inspect import json -import optparse +import argparse import os import platform import re @@ -23,7 +23,7 @@ from twisted.python.versions import Version from twisted.trial import unittest import scrapy -from scrapy.commands import ScrapyCommand +from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -37,19 +37,28 @@ class CommandSettings(unittest.TestCase): def setUp(self): self.command = ScrapyCommand() self.command.settings = Settings() - self.parser = optparse.OptionParser( - formatter=optparse.TitledHelpFormatter(), - conflict_handler='resolve', - ) + self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, + conflict_handler='resolve') self.command.add_options(self.parser) def test_settings_json_string(self): feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' - opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) + opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) self.command.process_options(args, opts) self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) + def test_help_formatter(self): + formatter = ScrapyHelpFormatter(prog='scrapy') + part_strings = ['usage: scrapy genspider [options] \n\n', + '\n', 'optional arguments:\n', '\n', 'Global Options:\n'] + self.assertEqual( + formatter._join_parts(part_strings), + ('Usage\n=====\n scrapy genspider [options] \n\n\n' + 'Optional Arguments\n==================\n\n' + 'Global Options\n--------------\n') + ) + class ProjectTest(unittest.TestCase): project_name = 'testproject' @@ -812,6 +821,21 @@ class BenchCommandTest(CommandTest): self.assertNotIn('Unhandled Error', log) +class ViewCommandTest(CommandTest): + + def test_methods(self): + command = view.Command() + command.settings = Settings() + parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-', + formatter_class=ScrapyHelpFormatter, + conflict_handler='resolve') + command.add_options(parser) + self.assertEqual(command.short_desc(), + "Open URL in browser, as seen by Scrapy") + self.assertIn("URL using the Scrapy downloader and show its", + command.long_desc()) + + class CrawlCommandTest(CommandTest): def crawl(self, code, args=()): From 5d7c0a5f861327c1a51ffcb3a39e283eb999fae7 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 10 Feb 2022 14:50:12 +0500 Subject: [PATCH 0477/2083] Use toscrape.com instead of example.com in test_command_check. (#5407) --- tests/test_command_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 34f5e59dd..c3d705194 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -19,11 +19,11 @@ import scrapy class CheckSpider(scrapy.Spider): name = '{self.spider_name}' - start_urls = ['http://example.com'] + start_urls = ['http://toscrape.com'] def parse(self, response, **cb_kwargs): \"\"\" - @url http://example.com + @url http://toscrape.com {contracts} \"\"\" {parse_def} From 115d35270691176e128bd30f6a2a8bcf1289fe5c Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 11 Feb 2022 01:21:31 +0200 Subject: [PATCH 0478/2083] per slot settings: setting renamed from `PER_SLOT_SETTINGS` to `DOWNLOAD_SLOTS` --- scrapy/core/downloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d5809ffa0..0e32714a8 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -86,7 +86,7 @@ class Downloader: self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) - self.per_slot_settings = self.settings.getdict('PER_SLOT_SETTINGS', {}) + self.per_slot_settings = self.settings.getdict('DOWNLOAD_SLOTS', {}) def fetch(self, request, spider): def _deactivate(response): From be97402e46c5e4793e23b713728e1777fbe4d49c Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 11 Feb 2022 01:22:27 +0200 Subject: [PATCH 0479/2083] per slot settings: log notifications (debug) removed --- scrapy/core/downloader/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 0e32714a8..53dc546de 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -109,8 +109,6 @@ class Downloader: randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot - logger.debug( - f"Downloader slot '{key}' - created {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''}: {new_slot}") return key, self.slots[key] @@ -209,5 +207,3 @@ class Downloader: if not slot.active and slot.lastseen + slot.delay < mintime: inactive_slot = self.slots.pop(key) inactive_slot.close() - logger.debug( - f"Downloader slot '{key}' - closed {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''} : {inactive_slot}") From befb6df119058db8a6a340b8235ccb565a60f3ca Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Fri, 11 Feb 2022 06:19:27 -0300 Subject: [PATCH 0480/2083] Remove Python 2 code from WrappedRequest --- scrapy/http/cookies.py | 6 +----- tests/test_http_cookies.py | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index bf4ae7b45..b43c383fe 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -142,10 +142,6 @@ class WrappedRequest: """ return self.request.meta.get('is_unverifiable', False) - def get_origin_req_host(self): - return urlparse_cached(self.request).hostname - - # python3 uses attributes instead of methods @property def full_url(self): return self.get_full_url() @@ -164,7 +160,7 @@ class WrappedRequest: @property def origin_req_host(self): - return self.get_origin_req_host() + return urlparse_cached(self.request).hostname def has_header(self, name): return name in self.request.headers diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 540e27907..08420332c 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -34,7 +34,6 @@ class WrappedRequestTest(TestCase): self.assertTrue(self.wrapped.unverifiable) def test_get_origin_req_host(self): - self.assertEqual(self.wrapped.get_origin_req_host(), 'www.example.com') self.assertEqual(self.wrapped.origin_req_host, 'www.example.com') def test_has_header(self): From bbb693d046a1942965ea9579bf7bb8a20fc92ba3 Mon Sep 17 00:00:00 2001 From: Boris Zabolotskikh Date: Mon, 14 Feb 2022 12:07:45 +0300 Subject: [PATCH 0481/2083] Update downloader-middleware.rst Added a link to the method --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index caf44a903..44201d0d5 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -89,7 +89,7 @@ object gives you access, for example, to the :ref:`settings `. methods of installed middleware is always called on every response. If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling - process_request methods and reschedule the returned request. Once the newly returned + :meth:`process_request` methods and reschedule the returned request. Once the newly returned request is performed, the appropriate middleware chain will be called on the downloaded response. From 187b5c887602218dee2fb57ad1dba223c11ce84f Mon Sep 17 00:00:00 2001 From: Abhishek K M <67158080+Sync271@users.noreply.github.com> Date: Mon, 14 Feb 2022 23:46:53 +0530 Subject: [PATCH 0482/2083] Update the documentation link for robots.txt (#5415) --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index caf44a903..f9208d550 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1019,7 +1019,7 @@ Parsers vary in several aspects: (shorter) rule Performance comparison of different parsers is available at `the following link -`_. +`_. .. _protego-parser: From 3b42ccfebadd72d9b455f6526ab63835b72b1558 Mon Sep 17 00:00:00 2001 From: Gowtham Chowdary <42214663+GowthamChowdary@users.noreply.github.com> Date: Thu, 17 Feb 2022 02:03:56 +0530 Subject: [PATCH 0483/2083] Add a link to Discord (#5422) --- docs/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 433798aa8..69becd4a8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,12 +24,14 @@ Having trouble? We'd like to help! * Search for questions on the archives of the `scrapy-users mailing list`_. * Ask a question in the `#scrapy IRC channel`_, * Report bugs with Scrapy in our `issue tracker`_. +* Join the Discord community `Scrapy Discord`_. .. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users .. _Scrapy subreddit: https://www.reddit.com/r/scrapy/ .. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues +.. _Scrapy Discord : https://discord.gg/mv3yErfpvq First steps From 08557e09db4bcb109eb78e9058622ab5cef77415 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 23 Feb 2022 23:52:18 +0500 Subject: [PATCH 0484/2083] Pin old markupsafe when we pin old mitmproxy (#5427) --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index 2031a2d92..fcd3563b2 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,8 @@ deps = #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' + # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) + markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy' # Extras botocore>=1.4.87 passenv = @@ -127,6 +129,9 @@ deps = robotexclusionrulesparser Pillow>=4.0.0 Twisted[http2]>=17.9.0 + # Twisted[http2] currently forces old mitmproxy because of h2 version restrictions in their deps, + # so we need to pin old markupsafe here too + markupsafe < 2.1.0 [testenv:asyncio] commands = From aa0306a167ef34b23cc2ec407a48359a4b5a8d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Mar 2022 12:16:37 +0100 Subject: [PATCH 0485/2083] Cover 2.6.0 in the release notes (#5399) --- docs/conf.py | 4 + docs/index.rst | 4 +- docs/news.rst | 379 +++++++++++++++++++++++++++++-- docs/topics/asyncio.rst | 33 ++- docs/topics/commands.rst | 10 +- docs/topics/feed-exports.rst | 10 +- docs/topics/media-pipeline.rst | 2 + docs/topics/request-response.rst | 4 + docs/topics/spiders.rst | 5 +- scrapy/utils/defer.py | 4 +- 10 files changed, 407 insertions(+), 48 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d5e139e66..55aa72d5a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -303,10 +303,14 @@ intersphinx_mapping = { hoverxref_auto_ref = True hoverxref_role_types = { "class": "tooltip", + "command": "tooltip", "confval": "tooltip", "hoverxref": "tooltip", "mod": "tooltip", "ref": "tooltip", + "reqmeta": "tooltip", + "setting": "tooltip", + "signal": "tooltip", } hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal'] diff --git a/docs/index.rst b/docs/index.rst index 69becd4a8..75e08f537 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,6 +12,8 @@ testing. .. _web crawling: https://en.wikipedia.org/wiki/Web_crawler .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping +.. _getting-help: + Getting help ============ @@ -31,7 +33,7 @@ Having trouble? We'd like to help! .. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues -.. _Scrapy Discord : https://discord.gg/mv3yErfpvq +.. _Scrapy Discord: https://discord.gg/mv3yErfpvq First steps diff --git a/docs/news.rst b/docs/news.rst index 47a808693..2128f2f0e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,30 +1,360 @@ -.. note:: - .. versionchanged:: VERSION - - The Twisted reactor is now installed when - :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a - :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, - :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now - honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions - they are silently ignored when set there and you need to set these settings - in some other way. - - -.. note:: - .. versionchanged:: VERSION - - Previously this setting had no effect in a spider - :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but - if you :ref:`run several spiders in one process `, - they must not have different values for this setting, because they will use - a single reactor instance. - - .. _news: Release notes ============= +.. _release-2.6.0: + +Scrapy 2.6.0 (2022-02-??) +------------------------- + +Highlights: + +* Python 3.10 support + +* :ref:`asyncio support ` is no longer considered + experimental, and works out-of-the-box on Windows regardless of your Python + version + +* Feed exports now support :class:`pathlib.Path` output paths and per-feed + :ref:`item filtering ` and + :ref:`post-processing ` + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- The h2_ dependency is now optional, only needed to + :ref:`enable HTTP/2 support `. (:issue:`5113`) + + .. _h2: https://pypi.org/project/h2/ + + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The ``formdata`` parameter of :class:`~scrapy.FormRequest`, if specified + for a non-POST request, now overrides the URL query string, instead of + being appended to it. (:issue:`2919`, :issue:`3579`) + +- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, now + the return value of that function, and not the ``params`` input parameter, + will determine the feed URI parameters, unless that return value is + ``None``. (:issue:`4962`, :issue:`4966`) + +- In :class:`scrapy.core.engine.ExecutionEngine`, methods + :meth:`~scrapy.core.engine.ExecutionEngine.crawl`, + :meth:`~scrapy.core.engine.ExecutionEngine.download`, + :meth:`~scrapy.core.engine.ExecutionEngine.schedule`, + and :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle` + now raise :exc:`RuntimeError` if called before + :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`. (:issue:`5090`) + + These methods used to assume that + :attr:`ExecutionEngine.slot ` had + been defined by a prior call to + :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`, so they were + raising :exc:`AttributeError` instead. + +- If the API of the configured :ref:`scheduler ` does not + meet expectations, :exc:`TypeError` is now raised at startup time. Before, + other exceptions would be raised at run time. (:issue:`3559`) + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- ``scrapy.http.TextResponse.body_as_unicode``, deprecated in Scrapy 2.2, has + now been removed. (:issue:`5393`) + +- ``scrapy.item.BaseItem``, deprecated in Scrapy 2.2, has now been removed. + (:issue:`5398`) + +- ``scrapy.item.DictItem``, deprecated in Scrapy 1.8, has now been removed. + (:issue:`5398`) + +- ``scrapy.Spider.make_requests_from_url``, deprecated in Scrapy 1.4, has now + been removed. (:issue:`4178`, :issue:`4356`) + + +Deprecations +~~~~~~~~~~~~ + +- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, + returning ``None`` or modifying the ``params`` input parameter is now + deprecated. Return a new dictionary instead. (:issue:`4962`, :issue:`4966`) + +- :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`) + + - Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new + :meth:`Request.to_dict ` method. + + - Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new + :func:`scrapy.utils.request.request_from_dict` function. + +- In :mod:`scrapy.squeues`, the following queue classes are deprecated: + :class:`~scrapy.squeues.PickleFifoDiskQueueNonRequest`, + :class:`~scrapy.squeues.PickleLifoDiskQueueNonRequest`, + :class:`~scrapy.squeues.MarshalFifoDiskQueueNonRequest`, + and :class:`~scrapy.squeues.MarshalLifoDiskQueueNonRequest`. You should + instead use: + :class:`~scrapy.squeues.PickleFifoDiskQueue`, + :class:`~scrapy.squeues.PickleLifoDiskQueue`, + :class:`~scrapy.squeues.MarshalFifoDiskQueue`, + and :class:`~scrapy.squeues.MarshalLifoDiskQueue`. (:issue:`5117`) + +- Many aspects of :class:`scrapy.core.engine.ExecutionEngine` that come from + a time when this class could handle multiple :class:`~scrapy.Spider` + objects at a time have been deprecated. (:issue:`5090`) + + - The :meth:`~scrapy.core.engine.ExecutionEngine.has_capacity` method + is deprecated. + + - The :meth:`~scrapy.core.engine.ExecutionEngine.schedule` method is + deprecated, use :meth:`~scrapy.core.engine.ExecutionEngine.crawl` or + :meth:`~scrapy.core.engine.ExecutionEngine.download` instead. + + - The :attr:`~scrapy.core.engine.ExecutionEngine.open_spiders` attribute + is deprecated, use :attr:`~scrapy.core.engine.ExecutionEngine.spider` + instead. + + - The ``spider`` parameter is deprecated for the following methods: + + - :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle` + + - :meth:`~scrapy.core.engine.ExecutionEngine.crawl` + + - :meth:`~scrapy.core.engine.ExecutionEngine.download` + + Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider` + first to set the :class:`~scrapy.Spider` object. + + +New features +~~~~~~~~~~~~ + +- You can now use :ref:`item filtering ` to control which items + are exported to each output feed. (:issue:`4575`, :issue:`5178`, + :issue:`5161`, :issue:`5203`) + +- You can now apply :ref:`post-processing ` to feeds, and + :ref:`built-in post-processing plugins ` are provided for + output file compression. (:issue:`2174`, :issue:`5168`, :issue:`5190`) + +- The :setting:`FEEDS` setting now supports :class:`pathlib.Path` objects as + keys. (:issue:`5383`, :issue:`5384`) + +- Enabling :ref:`asyncio ` while using Windows and Python 3.8 + or later will automatically switch the asyncio event loop to one that + allows Scrapy to work. See :ref:`asyncio-windows`. (:issue:`4976`, + :issue:`5315`) + +- The :command:`genspider` command now supports a start URL instead of a + domain name. (:issue:`4439`) + +- :mod:`scrapy.utils.defer` gained 2 new functions, + :func:`~scrapy.utils.defer.deferred_to_future` and + :func:`~scrapy.utils.defer.maybe_deferred_to_future`, to help :ref:`await + on Deferreds when using the asyncio reactor `. + (:issue:`5288`) + +- :ref:`Amazon S3 feed export storage ` gained + support for `temporary security credentials`_ + (:setting:`AWS_SESSION_TOKEN`) and endpoint customization + (:setting:`AWS_ENDPOINT_URL`). (:issue:`4998`, :issue:`5210`) + + .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys + +- New :setting:`LOG_FILE_APPEND` setting to allow truncating the log file. + (:issue:`5279`) + +- :attr:`Request.cookies ` values that are + :class:`bool`, :class:`float` or :class:`int` are cast to :class:`str`. + (:issue:`5252`, :issue:`5253`) + +- You may now raise :exc:`~scrapy.exceptions.CloseSpider` from a handler of + the :signal:`spider_idle` signal to customize the reason why the spider is + stopping. (:issue:`5191`) + +- When using + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`, the + proxy URL for non-HTTPS HTTP/1.1 requests no longer needs to include a URL + scheme. (:issue:`4505`, :issue:`4649`) + +- All built-in queues now expose a ``peek`` method that returns the next + queue object (like ``pop``) but does not remove the returned object from + the queue. (:issue:`5112`) + + If the underlying queue does not support peeking (e.g. because you are not + using ``queuelib`` 1.6.1 or later), the ``peek`` method raises + :exc:`NotImplementedError`. + +- :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have + an ``attributes`` attribute that makes subclassing easier. For + :class:`~scrapy.http.Request`, it also allows subclasses to work with + :func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`, + :issue:`5130`, :issue:`5218`) + +- The :meth:`~scrapy.core.scheduler.BaseScheduler.open` and + :meth:`~scrapy.core.scheduler.BaseScheduler.close` methods of the + :ref:`scheduler ` are now optional. (:issue:`3559`) + +- HTTP/1.1 :exc:`~scrapy.core.downloader.handlers.http11.TunnelError` + exceptions now only truncate response bodies longer than 1000 characters, + instead of those longer than 32 characters, making it easier to debug such + errors. (:issue:`4881`, :issue:`5007`) + +- :class:`~scrapy.loader.ItemLoader` now supports non-text responses. + (:issue:`5145`, :issue:`5269`) + + +Bug fixes +~~~~~~~~~ + +- The :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` settings + are no longer ignored if defined in :attr:`~scrapy.Spider.custom_settings`. + (:issue:`4485`, :issue:`5352`) + +- Removed a module-level Twisted reactor import that could prevent + :ref:`using the asyncio reactor `. (:issue:`5357`) + +- The :command:`startproject` command works with existing folders again. + (:issue:`4665`, :issue:`4676`) + +- The :setting:`FEED_URI_PARAMS` setting now behaves as documented. + (:issue:`4962`, :issue:`4966`) + +- :attr:`Request.cb_kwargs ` once again allows the + ``callback`` keyword. (:issue:`5237`, :issue:`5251`, :issue:`5264`) + +- Made :func:`scrapy.utils.response.open_in_browser` support more complex + HTML. (:issue:`5319`, :issue:`5320`) + +- Fixed :attr:`CSVFeedSpider.quotechar + ` being interpreted as the CSV file + encoding. (:issue:`5391`, :issue:`5394`) + +- Added missing setuptools_ to the list of dependencies. (:issue:`5122`) + + .. _setuptools: https://pypi.org/project/setuptools/ + +- :class:`LinkExtractor ` + now also works as expected with links that have comma-separated ``rel`` + attribute values including ``nofollow``. (:issue:`5225`) + +- Fixed a :exc:`TypeError` that could be raised during :ref:`feed export + ` parameter parsing. (:issue:`5359`) + + +Documentation +~~~~~~~~~~~~~ + +- :ref:`asyncio support ` is no longer considered + experimental. (:issue:`5332`) + +- Included :ref:`Windows-specific help for asyncio usage `. + (:issue:`4976`, :issue:`5315`) + +- Rewrote :ref:`topics-headless-browsing` with up-to-date best practices. + (:issue:`4484`, :issue:`4613`) + +- Documented :ref:`local file naming in media pipelines + `. (:issue:`5069`, :issue:`5152`) + +- :ref:`faq` now covers spider file name collision issues. (:issue:`2680`, + :issue:`3669`) + +- Provided better context and instructions to disable the + :setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`) + +- Documented that :ref:`reppy-parser` does not support Python 3.9+. + (:issue:`5226`, :issue:`5231`) + +- Documented :ref:`the scheduler component `. + (:issue:`3537`, :issue:`3559`) + +- Documented the method used by :ref:`media pipelines + ` to :ref:`determine if a file has expired + `. (:issue:`5120`, :issue:`5254`) + +- :ref:`run-multiple-spiders` now features + :func:`scrapy.utils.project.get_project_settings` usage. (:issue:`5070`) + +- :ref:`run-multiple-spiders` now covers what happens when you define + different per-spider values for some settings that cannot differ at run + time. (:issue:`4485`, :issue:`5352`) + +- Extended the documentation of the + :class:`~scrapy.extensions.statsmailer.StatsMailer` extension. + (:issue:`5199`, :issue:`5217`) + +- Added :setting:`JOBDIR` to :ref:`topics-settings`. (:issue:`5173`, + :issue:`5224`) + +- Documented :attr:`Spider.attribute `. + (:issue:`5174`, :issue:`5244`) + +- Documented :attr:`TextResponse.urljoin `. + (:issue:`1582`) + +- Added the ``body_length`` parameter to the documented signature of the + :signal:`headers_received` signal. (:issue:`5270`) + +- Clarified :meth:`SelectorList.get ` usage + in the :ref:`tutorial `. (:issue:`5256`) + +- The documentation now features the shortest import path of classes with + multiple import paths. (:issue:`2733`, :issue:`5099`) + +- ``quotes.toscrape.com`` references now use HTTPS instead of HTTP. + (:issue:`5395`, :issue:`5396`) + +- Added a link to `our Discord server `_ + to :ref:`getting-help`. (:issue:`5421`, :issue:`5422`) + +- The pronunciation of the project name is now :ref:`officially + ` /ˈskreɪpaɪ/. (:issue:`5280`, :issue:`5281`) + +- Added the Scrapy logo to the README. (:issue:`5255`, :issue:`5258`) + +- Fixed issues and implemented minor improvements. (:issue:`3155`, + :issue:`4335`, :issue:`5074`, :issue:`5098`, :issue:`5134`, :issue:`5180`, + :issue:`5194`, :issue:`5239`, :issue:`5266`, :issue:`5271`, :issue:`5273`, + :issue:`5274`, :issue:`5276`, :issue:`5347`, :issue:`5356`, :issue:`5414`, + :issue:`5415`, :issue:`5416`, :issue:`5419`, :issue:`5420`) + + +Quality Assurance +~~~~~~~~~~~~~~~~~ + +- Added support for Python 3.10. (:issue:`5212`, :issue:`5221`, + :issue:`5265`) + +- Significantly reduced memory usage by + :func:`scrapy.utils.response.response_httprepr`, used by the + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` downloader + middleware, which is enabled by default. (:issue:`4964`, :issue:`4972`) + +- Removed uses of the deprecated :mod:`optparse` module. (:issue:`5366`, + :issue:`5374`) + +- Extended typing hints. (:issue:`5077`, :issue:`5090`, :issue:`5100`, + :issue:`5108`, :issue:`5171`, :issue:`5215`, :issue:`5334`) + +- Improved tests, fixed CI issues, removed unused code. (:issue:`5094`, + :issue:`5157`, :issue:`5162`, :issue:`5198`, :issue:`5207`, :issue:`5208`, + :issue:`5229`, :issue:`5298`, :issue:`5299`, :issue:`5310`, :issue:`5316`, + :issue:`5333`, :issue:`5388`, :issue:`5389`, :issue:`5400`, :issue:`5401`, + :issue:`5404`, :issue:`5405`, :issue:`5407`, :issue:`5410`, :issue:`5412`, + :issue:`5425`, :issue:`5427`) + +- Implemented improvements for contributors. (:issue:`5080`, :issue:`5082`, + :issue:`5177`, :issue:`5200`) + +- Implemented cleanups. (:issue:`5095`, :issue:`5106`, :issue:`5209`, + :issue:`5228`, :issue:`5235`, :issue:`5245`, :issue:`5246`, :issue:`5292`, + :issue:`5314`, :issue:`5322`) + + .. _release-2.5.1: Scrapy 2.5.1 (2021-10-05) @@ -1000,9 +1330,8 @@ Bug fixes * zope.interface 5.0.0 and later versions are now supported (:issue:`4447`, :issue:`4448`) -* :meth:`Spider.make_requests_from_url - `, deprecated in Scrapy - 1.4.0, now issues a warning when used (:issue:`4412`) +* ``Spider.make_requests_from_url``, deprecated in Scrapy 1.4.0, now issues a + warning when used (:issue:`4412`) Documentation diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 8712d4268..3a6941a2c 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -10,6 +10,7 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. + .. _install-asyncio: Installing the asyncio reactor @@ -25,6 +26,7 @@ reactor manually. You can do that using install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') + .. _using-custom-loops: Using custom asyncio loops @@ -34,20 +36,30 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to use it instead of the default asyncio event loop. -.. _asyncio-await-dfd: + +.. _asyncio-windows: Windows-specific notes ====================== The Windows implementation of :mod:`asyncio` can use two event loop -implementations: :class:`~asyncio.SelectorEventLoop` (default before Python -3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop` -(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the -event loop class needs to be changed. Scrapy since VERSION does this -automatically when you change the :setting:`TWISTED_REACTOR` setting or call -:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor -by other means or use an older Scrapy version you need to call the following -code before installing the reactor:: +implementations: + +- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required + when using Twisted. + +- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work + with Twisted. + +So on Python 3.8+ the event loop class needs to be changed. + +.. versionchanged:: 2.6.0 + The event loop class is changed automatically when you change the + :setting:`TWISTED_REACTOR` setting or call + :func:`~scrapy.utils.reactor.install_reactor`. + +To change the event loop class manually, call the following code before +installing the reactor:: import asyncio asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) @@ -64,6 +76,9 @@ yourself, or in some code that runs before the reactor is installed, e.g. .. _playwright: https://github.com/microsoft/playwright-python + +.. _asyncio-await-dfd: + Awaiting on Deferreds ===================== diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index eef6b36ff..8c0b8e55f 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -230,10 +230,16 @@ Usage example:: genspider --------- -* Syntax: ``scrapy genspider [-t template] `` +* Syntax: ``scrapy genspider [-t template] `` * Requires project: *no* -Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. +.. versionadded:: 2.6.0 + The ability to pass a URL instead of a domain. + +Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. + +.. note:: Even if an HTTPS URL is specified, the protocol used in + ``start_urls`` is always HTTP. This is a known issue: :issue:`3553`. Usage example:: diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7994027d2..9a13eb82f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -278,7 +278,7 @@ feed URI, allowing item delivery to start way before the end of the crawl. Item filtering ============== -.. versionadded:: VERSION +.. versionadded:: 2.6.0 You can filter items that you want to allow for a particular feed by using the ``item_classes`` option in :ref:`feeds options `. Only items of @@ -318,7 +318,7 @@ ItemFilter Post-Processing =============== -.. versionadded:: VERSION +.. versionadded:: 2.6.0 Scrapy provides an option to activate plugins to post-process feeds before they are exported to feed storages. In addition to using :ref:`builtin plugins `, you @@ -457,13 +457,13 @@ as a fallback value if that key is not provided for a specific feed definition: If undefined or empty, all items are exported. - .. versionadded:: VERSION + .. versionadded:: 2.6.0 - ``item_filter``: a :ref:`filter class ` to filter items to export. :class:`~scrapy.extensions.feedexport.ItemFilter` is used be default. - .. versionadded:: VERSION + .. versionadded:: 2.6.0 - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. @@ -499,7 +499,7 @@ as a fallback value if that key is not provided for a specific feed definition: The plugins will be used in the order of the list passed. - .. versionadded:: VERSION + .. versionadded:: 2.6.0 .. setting:: FEED_EXPORT_ENCODING diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 10d2ac990..7dff78390 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -356,6 +356,8 @@ setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used. Additional features =================== +.. _file-expiration: + File expiration --------------- diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index e0435e901..92a471faf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -110,6 +110,10 @@ Request objects :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. + .. versionadded:: 2.6.0 + Cookie values that are :class:`bool`, :class:`float` or :class:`int` + are casted to :class:`str`. + :type cookies: dict or list :param encoding: the encoding of this request (defaults to ``'utf-8'``). diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 99e74233a..ece02ae47 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -42,15 +42,12 @@ Even though this cycle applies (more or less) to any kind of spider, there are different kinds of default spiders bundled into Scrapy for different purposes. We will talk about those types here. -.. module:: scrapy.spiders - :synopsis: Spiders base class, spider manager and spider middleware - .. _topics-spiders-ref: scrapy.Spider ============= -.. class:: scrapy.spiders.Spider() +.. class:: scrapy.spiders.Spider .. class:: scrapy.Spider() This is the simplest spider, and the one from which every other spider diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d7adc0a77..7ecb8ea3f 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -182,7 +182,7 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: def deferred_to_future(d: Deferred) -> Future: """ - .. versionadded:: VERSION + .. versionadded:: 2.6.0 Return an :class:`asyncio.Future` object that wraps *d*. @@ -203,7 +203,7 @@ def deferred_to_future(d: Deferred) -> Future: def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: """ - .. versionadded:: VERSION + .. versionadded:: 2.6.0 Return *d* as an object that can be awaited from a :ref:`Scrapy callable defined as a coroutine `. From 8ce01b3b76d4634f55067d6cfdf632ec70ba304a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Mar 2022 12:26:05 +0100 Subject: [PATCH 0486/2083] Merge pull request from GHSA-cjvr-mfj7-j4j8 * Do not carry over cookies to a different domain on redirect * Cover the cookie-domain redirect fix in the release notes * Cover 1.8.2 in the release notes * Fix redirect Cookie handling when the cookie middleware is disabled * Update the 1.8.2 release date --- docs/news.rst | 67 ++++++++- scrapy/downloadermiddlewares/redirect.py | 31 ++++- tests/test_downloadermiddleware_cookies.py | 155 +++++++++++++++++++++ 3 files changed, 247 insertions(+), 6 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 2128f2f0e..aef12d9db 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,11 +5,13 @@ Release notes .. _release-2.6.0: -Scrapy 2.6.0 (2022-02-??) +Scrapy 2.6.0 (2022-03-01) ------------------------- Highlights: +* :ref:`Security fixes for cookie handling <2.6-security-fixes>` + * Python 3.10 support * :ref:`asyncio support ` is no longer considered @@ -20,6 +22,37 @@ Highlights: :ref:`item filtering ` and :ref:`post-processing ` +.. _2.6-security-fixes: + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- When a :class:`~scrapy.http.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.http.Request` object to be + scheduled, the cookies defined in the original + :class:`~scrapy.http.Request` object are no longer copied into the new + :class:`~scrapy.http.Request` object. + + If you manually set the ``Cookie`` header on a + :class:`~scrapy.http.Request` object and the domain name of the redirect + URL is not an exact match for the domain of the URL of the original + :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.http.Request` object. + + The old behavior could be exploited by an attacker to gain access to your + cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more + information. + + .. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8 + + .. note:: It is still possible to enable the sharing of cookies between + different domains with a shared domain suffix (e.g. + ``example.com`` and any subdomain) by defining the shared domain + suffix (e.g. ``example.com``) as the cookie domain when defining + your cookies. See the documentation of the + :class:`~scrapy.http.Request` class for more information. + + Modified requirements ~~~~~~~~~~~~~~~~~~~~~ @@ -1842,6 +1875,38 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.2: + +Scrapy 1.8.2 (2022-03-01) +------------------------- + +**Security bug fixes:** + +- When a :class:`~scrapy.http.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.http.Request` object to be + scheduled, the cookies defined in the original + :class:`~scrapy.http.Request` object are no longer copied into the new + :class:`~scrapy.http.Request` object. + + If you manually set the ``Cookie`` header on a + :class:`~scrapy.http.Request` object and the domain name of the redirect + URL is not an exact match for the domain of the URL of the original + :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.http.Request` object. + + The old behavior could be exploited by an attacker to gain access to your + cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more + information. + + .. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8 + + .. note:: It is still possible to enable the sharing of cookies between + different domains with a shared domain suffix (e.g. + ``example.com`` and any subdomain) by defining the shared domain + suffix (e.g. ``example.com``) as the cookie domain when defining + your cookies. See the documentation of the + :class:`~scrapy.http.Request` class for more information. + .. _release-1.8.1: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 4053fecc5..fcd6c298b 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -4,6 +4,7 @@ from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string from scrapy.http import HtmlResponse +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -11,6 +12,21 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured logger = logging.getLogger(__name__) +def _build_redirect_request(source_request, *, url, method=None, body=None): + redirect_request = source_request.replace( + url=url, + method=method, + body=body, + cookies=None, + ) + if 'Cookie' in redirect_request.headers: + source_request_netloc = urlparse_cached(source_request).netloc + redirect_request_netloc = urlparse_cached(redirect_request).netloc + if source_request_netloc != redirect_request_netloc: + del redirect_request.headers['Cookie'] + return redirect_request + + class BaseRedirectMiddleware: enabled_setting = 'REDIRECT_ENABLED' @@ -47,10 +63,15 @@ class BaseRedirectMiddleware: raise IgnoreRequest("max redirections reached") def _redirect_request_using_get(self, request, redirect_url): - redirected = request.replace(url=redirect_url, method='GET', body='') - redirected.headers.pop('Content-Type', None) - redirected.headers.pop('Content-Length', None) - return redirected + redirect_request = _build_redirect_request( + request, + url=redirect_url, + method='GET', + body='', + ) + redirect_request.headers.pop('Content-Type', None) + redirect_request.headers.pop('Content-Length', None) + return redirect_request class RedirectMiddleware(BaseRedirectMiddleware): @@ -80,7 +101,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): redirected_url = urljoin(request.url, location) if response.status in (301, 307, 308) or request.method == 'HEAD': - redirected = request.replace(url=redirected_url) + redirected = _build_redirect_request(request, url=redirected_url) return self._redirect(redirected, request, spider, response.status) redirected = self._redirect_request_using_get(request, redirected_url) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 36021bfbf..1747f3b94 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -6,8 +6,10 @@ import pytest from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request +from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler @@ -23,9 +25,11 @@ class CookiesMiddlewareTest(TestCase): def setUp(self): self.spider = Spider('foo') self.mw = CookiesMiddleware() + self.redirect_middleware = RedirectMiddleware(settings=Settings()) def tearDown(self): del self.mw + del self.redirect_middleware def test_basic(self): req = Request('http://scrapytest.org/') @@ -368,3 +372,154 @@ class CookiesMiddlewareTest(TestCase): req4 = Request('http://example.org', cookies={'a': 'b'}) assert self.mw.process_request(req4, self.spider) is None self.assertCookieValEqual(req4.headers['Cookie'], b'a=b') + + def _test_cookie_redirect( + self, + source, + target, + *, + cookies1, + cookies2, + ): + input_cookies = {'a': 'b'} + + if not isinstance(source, dict): + source = {'url': source} + if not isinstance(target, dict): + target = {'url': target} + target.setdefault('status', 301) + + request1 = Request(cookies=input_cookies, **source) + self.mw.process_request(request1, self.spider) + cookies = request1.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies1 else None) + + response = Response( + headers={ + 'Location': target['url'], + }, + **target, + ) + self.assertEqual( + self.mw.process_response(request1, response, self.spider), + response, + ) + + request2 = self.redirect_middleware.process_response( + request1, + response, + self.spider, + ) + self.assertIsInstance(request2, Request) + + self.mw.process_request(request2, self.spider) + cookies = request2.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies2 else None) + + def test_cookie_redirect_same_domain(self): + self._test_cookie_redirect( + 'https://toscrape.com', + 'https://toscrape.com', + cookies1=True, + cookies2=True, + ) + + def test_cookie_redirect_same_domain_forcing_get(self): + self._test_cookie_redirect( + 'https://toscrape.com', + {'url': 'https://toscrape.com', 'status': 302}, + cookies1=True, + cookies2=True, + ) + + def test_cookie_redirect_different_domain(self): + self._test_cookie_redirect( + 'https://toscrape.com', + 'https://example.com', + cookies1=True, + cookies2=False, + ) + + def test_cookie_redirect_different_domain_forcing_get(self): + self._test_cookie_redirect( + 'https://toscrape.com', + {'url': 'https://example.com', 'status': 302}, + cookies1=True, + cookies2=False, + ) + + def _test_cookie_header_redirect( + self, + source, + target, + *, + cookies2, + ): + """Test the handling of a user-defined Cookie header when building a + redirect follow-up request. + + We follow RFC 6265 for cookie handling. The Cookie header can only + contain a list of key-value pairs (i.e. no additional cookie + parameters like Domain or Path). Because of that, we follow the same + rules that we would follow for the handling of the Set-Cookie response + header when the Domain is not set: the cookies must be limited to the + target URL domain (not even subdomains can receive those cookies). + + .. note:: This method tests the scenario where the cookie middleware is + disabled. Because of known issue #1992, when the cookies + middleware is enabled we do not need to be concerned about + the Cookie header getting leaked to unintended domains, + because the middleware empties the header from every request. + """ + if not isinstance(source, dict): + source = {'url': source} + if not isinstance(target, dict): + target = {'url': target} + target.setdefault('status', 301) + + request1 = Request(headers={'Cookie': b'a=b'}, **source) + + response = Response( + headers={ + 'Location': target['url'], + }, + **target, + ) + + request2 = self.redirect_middleware.process_response( + request1, + response, + self.spider, + ) + self.assertIsInstance(request2, Request) + + cookies = request2.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies2 else None) + + def test_cookie_header_redirect_same_domain(self): + self._test_cookie_header_redirect( + 'https://toscrape.com', + 'https://toscrape.com', + cookies2=True, + ) + + def test_cookie_header_redirect_same_domain_forcing_get(self): + self._test_cookie_header_redirect( + 'https://toscrape.com', + {'url': 'https://toscrape.com', 'status': 302}, + cookies2=True, + ) + + def test_cookie_header_redirect_different_domain(self): + self._test_cookie_header_redirect( + 'https://toscrape.com', + 'https://example.com', + cookies2=False, + ) + + def test_cookie_header_redirect_different_domain_forcing_get(self): + self._test_cookie_header_redirect( + 'https://toscrape.com', + {'url': 'https://example.com', 'status': 302}, + cookies2=False, + ) From e865c4430e58a4faa0e0766b23830f8423d6167a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Mar 2022 12:38:19 +0100 Subject: [PATCH 0487/2083] Merge pull request from GHSA-mfjm-vh54-3f96 * Ignore cookies with a public suffix as domain unless it matches the request domain * Fix the merge of 1.8.2 release notes * Re-apply removal of tldextract restriction --- docs/news.rst | 24 +++ scrapy/downloadermiddlewares/cookies.py | 34 ++++- setup.py | 1 + tests/test_downloadermiddleware_cookies.py | 170 +++++++++++++++++++++ 4 files changed, 226 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index aef12d9db..9590fb1c4 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -52,6 +52,18 @@ Security bug fixes your cookies. See the documentation of the :class:`~scrapy.http.Request` class for more information. +- When the domain of a cookie, either received in the ``Set-Cookie`` header + of a response or defined in a :class:`~scrapy.http.Request` object, is set + to a `public suffix `_, the cookie is now + ignored unless the cookie domain is the same as the request domain. + + The old behavior could be exploited by an attacker to inject cookies from a + controlled domain into your cookiejar that could be sent to other domains + not controlled by the attacker. Please, see the `mfjm-vh54-3f96 security + advisory`_ for more information. + + .. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96 + Modified requirements ~~~~~~~~~~~~~~~~~~~~~ @@ -1875,6 +1887,7 @@ affect subclasses: (:issue:`3884`) + .. _release-1.8.2: Scrapy 1.8.2 (2022-03-01) @@ -1907,6 +1920,17 @@ Scrapy 1.8.2 (2022-03-01) your cookies. See the documentation of the :class:`~scrapy.http.Request` class for more information. +- When the domain of a cookie, either received in the ``Set-Cookie`` header + of a response or defined in a :class:`~scrapy.http.Request` object, is set + to a `public suffix `_, the cookie is now + ignored unless the cookie domain is the same as the request domain. + + The old behavior could be exploited by an attacker to inject cookies into + your requests to some other domains. Please, see the `mfjm-vh54-3f96 + security advisory`_ for more information. + + .. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96 + .. _release-1.8.1: diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 0eee8d758..3afa06077 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,15 +1,26 @@ import logging from collections import defaultdict +from tldextract import TLDExtract + from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) +_split_domain = TLDExtract(include_psl_private_domains=True) + + +def _is_public_domain(domain): + parts = _split_domain(domain) + return not parts.domain + + class CookiesMiddleware: """This middleware enables working with sites that need cookies""" @@ -23,14 +34,29 @@ class CookiesMiddleware: raise NotConfigured return cls(crawler.settings.getbool('COOKIES_DEBUG')) + def _process_cookies(self, cookies, *, jar, request): + for cookie in cookies: + cookie_domain = cookie.domain + if cookie_domain.startswith('.'): + cookie_domain = cookie_domain[1:] + + request_domain = urlparse_cached(request).hostname.lower() + + if cookie_domain and _is_public_domain(cookie_domain): + if cookie_domain != request_domain: + continue + cookie.domain = request_domain + + jar.set_cookie_if_ok(cookie, request) + def process_request(self, request, spider): if request.meta.get('dont_merge_cookies', False): return cookiejarkey = request.meta.get("cookiejar") jar = self.jars[cookiejarkey] - for cookie in self._get_request_cookies(jar, request): - jar.set_cookie_if_ok(cookie, request) + cookies = self._get_request_cookies(jar, request) + self._process_cookies(cookies, jar=jar, request=request) # set Cookie header request.headers.pop('Cookie', None) @@ -44,7 +70,9 @@ class CookiesMiddleware: # extract cookies from Set-Cookie and drop invalid/expired cookies cookiejarkey = request.meta.get("cookiejar") jar = self.jars[cookiejarkey] - jar.extract_cookies(response, request) + cookies = jar.make_cookies(response, request) + self._process_cookies(cookies, jar=jar, request=request) + self._debug_set_cookie(response, spider) return response diff --git a/setup.py b/setup.py index 3a6ff2836..d86c0f285 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'setuptools', + 'tldextract', ] extras_require = {} cpython_dependencies = [ diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 1747f3b94..ba7453255 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -15,6 +15,48 @@ from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler +def _cookie_to_set_cookie_value(cookie): + """Given a cookie defined as a dictionary with name and value keys, and + optional path and domain keys, return the equivalent string that can be + associated to a ``Set-Cookie`` header.""" + decoded = {} + for key in ("name", "value", "path", "domain"): + if cookie.get(key) is None: + if key in ("name", "value"): + return + continue + if isinstance(cookie[key], (bool, float, int, str)): + decoded[key] = str(cookie[key]) + else: + try: + decoded[key] = cookie[key].decode("utf8") + except UnicodeDecodeError: + decoded[key] = cookie[key].decode("latin1", errors="replace") + + cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" + for key, value in decoded.items(): # path, domain + cookie_str += f"; {key.capitalize()}={value}" + return cookie_str + + +def _cookies_to_set_cookie_list(cookies): + """Given a group of cookie defined either as a dictionary or as a list of + dictionaries (i.e. in a format supported by the cookies parameter of + Request), return the equivalen list of strings that can be associated to a + ``Set-Cookie`` header.""" + if not cookies: + return [] + if isinstance(cookies, dict): + cookies = ({"name": k, "value": v} for k, v in cookies.items()) + return filter( + None, + ( + _cookie_to_set_cookie_value(cookie) + for cookie in cookies + ) + ) + + class CookiesMiddlewareTest(TestCase): def assertCookieValEqual(self, first, second, msg=None): @@ -523,3 +565,131 @@ class CookiesMiddlewareTest(TestCase): {'url': 'https://example.com', 'status': 302}, cookies2=False, ) + + def _test_user_set_cookie_domain_followup( + self, + url1, + url2, + domain, + *, + cookies1, + cookies2, + ): + input_cookies = [ + { + 'name': 'a', + 'value': 'b', + 'domain': domain, + } + ] + + request1 = Request(url1, cookies=input_cookies) + self.mw.process_request(request1, self.spider) + cookies = request1.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies1 else None) + + request2 = Request(url2) + self.mw.process_request(request2, self.spider) + cookies = request2.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies2 else None) + + def test_user_set_cookie_domain_suffix_private(self): + self._test_user_set_cookie_domain_followup( + 'https://books.toscrape.com', + 'https://quotes.toscrape.com', + 'toscrape.com', + cookies1=True, + cookies2=True, + ) + + def test_user_set_cookie_domain_suffix_public_period(self): + self._test_user_set_cookie_domain_followup( + 'https://foo.co.uk', + 'https://bar.co.uk', + 'co.uk', + cookies1=False, + cookies2=False, + ) + + def test_user_set_cookie_domain_suffix_public_private(self): + self._test_user_set_cookie_domain_followup( + 'https://foo.blogspot.com', + 'https://bar.blogspot.com', + 'blogspot.com', + cookies1=False, + cookies2=False, + ) + + def test_user_set_cookie_domain_public_period(self): + self._test_user_set_cookie_domain_followup( + 'https://co.uk', + 'https://co.uk', + 'co.uk', + cookies1=True, + cookies2=True, + ) + + def _test_server_set_cookie_domain_followup( + self, + url1, + url2, + domain, + *, + cookies, + ): + request1 = Request(url1) + self.mw.process_request(request1, self.spider) + + input_cookies = [ + { + 'name': 'a', + 'value': 'b', + 'domain': domain, + } + ] + + headers = { + 'Set-Cookie': _cookies_to_set_cookie_list(input_cookies), + } + response = Response(url1, status=200, headers=headers) + self.assertEqual( + self.mw.process_response(request1, response, self.spider), + response, + ) + + request2 = Request(url2) + self.mw.process_request(request2, self.spider) + actual_cookies = request2.headers.get('Cookie') + self.assertEqual(actual_cookies, b"a=b" if cookies else None) + + def test_server_set_cookie_domain_suffix_private(self): + self._test_server_set_cookie_domain_followup( + 'https://books.toscrape.com', + 'https://quotes.toscrape.com', + 'toscrape.com', + cookies=True, + ) + + def test_server_set_cookie_domain_suffix_public_period(self): + self._test_server_set_cookie_domain_followup( + 'https://foo.co.uk', + 'https://bar.co.uk', + 'co.uk', + cookies=False, + ) + + def test_server_set_cookie_domain_suffix_public_private(self): + self._test_server_set_cookie_domain_followup( + 'https://foo.blogspot.com', + 'https://bar.blogspot.com', + 'blogspot.com', + cookies=False, + ) + + def test_server_set_cookie_domain_public_period(self): + self._test_server_set_cookie_domain_followup( + 'https://co.uk', + 'https://co.uk', + 'co.uk', + cookies=True, + ) From 6b63e7c14758fdc59f37cb6c2c9b88abebe8606f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Mar 2022 12:43:11 +0100 Subject: [PATCH 0488/2083] =?UTF-8?q?Bump=20version:=202.5.0=20=E2=86=92?= =?UTF-8?q?=202.6.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d9e4a2831..5a5b51a01 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.5.0 +current_version = 2.6.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 437459cd9..e70b4523a 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.5.0 +2.6.0 From 84853c4fa6eb30bdcba0f70b4426994d731509fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Mar 2022 13:01:20 +0100 Subject: [PATCH 0489/2083] bandit: allow-list B324 for the time being --- .bandit.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.bandit.yml b/.bandit.yml index 243379b0b..41f1bb597 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -8,6 +8,7 @@ skips: - B311 - B320 - B321 +- B324 - B402 # https://github.com/scrapy/scrapy/issues/4180 - B403 - B404 From d60636d0de94c5a08c25d1d6820faed0b45506b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Mar 2022 13:06:58 +0100 Subject: [PATCH 0490/2083] Fix redirect handling regression --- scrapy/downloadermiddlewares/redirect.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index fcd6c298b..c8c84ffb2 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -12,11 +12,10 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured logger = logging.getLogger(__name__) -def _build_redirect_request(source_request, *, url, method=None, body=None): +def _build_redirect_request(source_request, *, url, **kwargs): redirect_request = source_request.replace( url=url, - method=method, - body=body, + **kwargs, cookies=None, ) if 'Cookie' in redirect_request.headers: From fab3e907297abd89106fb040c1c0c6a24b9522a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Mar 2022 13:41:20 +0100 Subject: [PATCH 0491/2083] Cover 2.6.1 in the release notes --- docs/news.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 9590fb1c4..5d92067b5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,15 @@ Release notes ============= +.. _release-2.6.1: + +Scrapy 2.6.1 (2022-03-01) +------------------------- + +Fixes a regression introduced in 2.6.0 that would unset the request method when +following redirects. + + .. _release-2.6.0: Scrapy 2.6.0 (2022-03-01) From 23537a0f9580bfb28ac5d8b88f37df47e838f463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Mar 2022 13:48:40 +0100 Subject: [PATCH 0492/2083] =?UTF-8?q?Bump=20version:=202.6.0=20=E2=86=92?= =?UTF-8?q?=202.6.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 5a5b51a01..1d9b9c02f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.6.0 +current_version = 2.6.1 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index e70b4523a..6a6a3d8e3 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.6.0 +2.6.1 From 50c8becbe02e6e71a6e7a57af0b28bcf38d9a3c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 1 Mar 2022 17:29:08 +0100 Subject: [PATCH 0493/2083] Freeze and upgrade CI packages (#5429) --- tests/requirements.txt | 2 +- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index bd72c8c46..398d1d16d 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,7 +3,7 @@ attrs dataclasses; python_version == '3.6' pyftpdlib pytest -pytest-cov +pytest-cov==3.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures diff --git a/tox.ini b/tox.ini index fcd3563b2..db151f215 100644 --- a/tox.ini +++ b/tox.ini @@ -49,7 +49,7 @@ commands = [testenv:security] basepython = python3 deps = - bandit + bandit==1.7.3 commands = bandit -r -c .bandit.yml {posargs:scrapy} @@ -68,7 +68,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.12.1 + pylint==2.12.2 commands = pylint conftest.py docs extras scrapy setup.py tests From ccdbb795ff2fef0ff89c1fa478ad2d7c52ef64be Mon Sep 17 00:00:00 2001 From: Florentin Date: Tue, 1 Mar 2022 22:01:55 +0100 Subject: [PATCH 0494/2083] Recommend Common Crawl instead of Google Cache --- docs/topics/practices.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 1a9d56143..d0207fd18 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -262,7 +262,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use cookies to spot bot behaviour * use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting. -* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites +* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites directly * use a pool of rotating IPs. For example, the free `Tor project`_ or paid services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a @@ -277,7 +277,7 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _Tor project: https://www.torproject.org/ .. _commercial support: https://scrapy.org/support/ .. _ProxyMesh: https://proxymesh.com/ -.. _Google cache: http://www.googleguide.com/cached_pages.html +.. _Common Crawl: https://commoncrawl.org/ .. _testspiders: https://github.com/scrapinghub/testspiders .. _scrapoxy: https://scrapoxy.io/ .. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ From 3ecbea4b876ed084f30bb4063e1993dd9c3cdb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 2 Mar 2022 16:06:49 +0100 Subject: [PATCH 0495/2083] CrawlerProcess: initiate the reactor only once --- scrapy/crawler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index a638254f1..9939a19eb 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -290,6 +290,7 @@ class CrawlerProcess(CrawlerRunner): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) + self._initiated_reactor = False def _signal_shutdown(self, signum, _): from twisted.internet import reactor @@ -310,7 +311,9 @@ class CrawlerProcess(CrawlerRunner): def _create_crawler(self, spidercls): if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings, init_reactor=True) + init_reactor = not self._initiated_reactor + self._initiated_reactor = True + return Crawler(spidercls, self.settings, init_reactor=init_reactor) def start(self, stop_after_crawl=True, install_signal_handlers=True): """ From 96fc4dae15181695c58040389fa502857a2b0df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 2 Mar 2022 16:14:39 +0100 Subject: [PATCH 0496/2083] CrawlerProcess: test a multi-spider scenario --- tests/CrawlerProcess/multi.py | 16 ++++++++++++++++ tests/test_crawler.py | 6 ++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/CrawlerProcess/multi.py diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py new file mode 100644 index 000000000..aaa1af5c5 --- /dev/null +++ b/tests/CrawlerProcess/multi.py @@ -0,0 +1,16 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 8f6227109..957525382 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -302,6 +302,12 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + def test_multi(self): + log = self.run_script('multi.py') + self.assertIn('Spider closed (finished)', log) + self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertNotIn("ReactorAlreadyInstalledError", log) + def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) From 3bf6baeaa705ddc2d3417206f3db816dabadf5a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 2 Mar 2022 17:03:41 +0100 Subject: [PATCH 0497/2083] =?UTF-8?q?initiated=20=E2=86=92=20initialized?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/crawler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 9939a19eb..d669d93a8 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -290,7 +290,7 @@ class CrawlerProcess(CrawlerRunner): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) - self._initiated_reactor = False + self._initialized_reactor = False def _signal_shutdown(self, signum, _): from twisted.internet import reactor @@ -311,8 +311,8 @@ class CrawlerProcess(CrawlerRunner): def _create_crawler(self, spidercls): if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - init_reactor = not self._initiated_reactor - self._initiated_reactor = True + init_reactor = not self._initialized_reactor + self._initialized_reactor = True return Crawler(spidercls, self.settings, init_reactor=init_reactor) def start(self, stop_after_crawl=True, install_signal_handlers=True): From 62a00812669aa0906aa0d4e9a4c2e87be3b74975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 7 Mar 2022 12:00:44 +0100 Subject: [PATCH 0498/2083] engine: prevent slot method call after unsetting the slot --- scrapy/core/engine.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index f9de7ee23..6602f661d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -136,7 +136,9 @@ class ExecutionEngine: self.paused = False def _next_request(self) -> None: - assert self.slot is not None # typing + if self.slot is None: + return + assert self.spider is not None # typing if self.paused: @@ -184,7 +186,8 @@ class ExecutionEngine: d.addErrback(lambda f: logger.info('Error while removing request from slot', exc_info=failure_to_exc_info(f), extra={'spider': self.spider})) - d.addBoth(lambda _: self.slot.nextcall.schedule()) + slot = self.slot + d.addBoth(lambda _: slot.nextcall.schedule()) d.addErrback(lambda f: logger.info('Error while scheduling new request', exc_info=failure_to_exc_info(f), extra={'spider': self.spider})) From d469214fe73574d35521e8e87963629a6c12bcd8 Mon Sep 17 00:00:00 2001 From: Ali Rastegar <68519335+VolVox99@users.noreply.github.com> Date: Tue, 8 Mar 2022 01:29:22 -0800 Subject: [PATCH 0499/2083] Update tutorial.rst (#5442) Fixed typo --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 5697b9608..cde1b1ef4 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -488,7 +488,7 @@ The `JSON Lines`_ format is useful because it's stream-like, you can easily append new records to it. It doesn't have the same problem of JSON when you run twice. Also, as each record is a separate line, you can process big files without having to fit everything in memory, there are tools like `JQ`_ to help -doing that at the command-line. +do that at the command-line. In small projects (like the one in this tutorial), that should be enough. However, if you want to perform more complex things with the scraped items, you From e264cc30d1e73d53de2e4048d7d9b6cffd59f8f0 Mon Sep 17 00:00:00 2001 From: NaincyKumariKnoldus <87004609+NaincyKumariKnoldus@users.noreply.github.com> Date: Thu, 10 Mar 2022 19:24:33 +0530 Subject: [PATCH 0500/2083] removed the pywin32 docs section (#5370) --- docs/faq.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 8283cab11..8a9ba809b 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -94,15 +94,6 @@ How can I scrape an item with attributes in different pages? See :ref:`topics-request-response-ref-request-callback-arguments`. - -Scrapy crashes with: ImportError: No module named win32api ----------------------------------------------------------- - -You need to install `pywin32`_ because of `this Twisted bug`_. - -.. _pywin32: https://sourceforge.net/projects/pywin32/ -.. _this Twisted bug: https://twistedmatrix.com/trac/ticket/3707 - How can I simulate a user login in my spider? --------------------------------------------- From c1d4be8cb5ed610fef4a2af60f9b654faab7a243 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Tue, 15 Mar 2022 07:30:30 -0300 Subject: [PATCH 0501/2083] =?UTF-8?q?Restore=20=E2=80=98-o=20-=E2=80=99=20?= =?UTF-8?q?support=20(#5445)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/cmdline.py | 17 +++++++++++++---- tests/test_commands.py | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 491c4beab..5ee1f0f44 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -14,6 +14,15 @@ from scrapy.utils.project import inside_project, get_project_settings from scrapy.utils.python import garbage_collect +class ScrapyArgumentParser(argparse.ArgumentParser): + def _parse_optional(self, arg_string): + # if starts with -: it means that is a parameter not a argument + if arg_string[:2] == '-:': + return None + + return super()._parse_optional(arg_string) + + def _iter_command_classes(module_name): # TODO: add `name` attribute to commands and and merge this function with # scrapy.utils.spider.iter_spider_classes @@ -131,10 +140,10 @@ def execute(argv=None, settings=None): sys.exit(2) cmd = cmds[cmdname] - parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, - usage=f"scrapy {cmdname} {cmd.syntax()}", - conflict_handler='resolve', - description=cmd.long_desc()) + parser = ScrapyArgumentParser(formatter_class=ScrapyHelpFormatter, + usage=f"scrapy {cmdname} {cmd.syntax()}", + conflict_handler='resolve', + description=cmd.long_desc()) settings.setdict(cmd.default_settings, priority='command') cmd.settings = settings cmd.add_options(parser) diff --git a/tests/test_commands.py b/tests/test_commands.py index 7cd19b29a..b5e6c2b8b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -770,6 +770,21 @@ class MySpider(scrapy.Spider): log = self.get_log(spider_code, args=args) self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) + def test_output_stdout(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return [] +""" + args = ['-o', '-:json'] + log = self.get_log(spider_code, args=args) + self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log) + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): From b59a69be1790f138afe89df3dfed17ac48384d8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 15 Mar 2022 12:10:41 +0100 Subject: [PATCH 0502/2083] Test that a low CLOSEPIDER_TIMEOUT does not raise an exception --- tests/test_engine.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_engine.py b/tests/test_engine.py index fa7d0c8d4..b8fd341f6 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -12,9 +12,11 @@ module with the ``runserver`` argument:: import os import re +import subprocess import sys import warnings from collections import defaultdict +from threading import Timer from urllib.parse import urlparse import attr @@ -502,6 +504,37 @@ class EngineTest(unittest.TestCase): self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated") + def test_short_timeout(self): + args = ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'fetch', + '-s', + 'CLOSESPIDER_TIMEOUT=0.001', + '-s', + 'LOG_LEVEL=DEBUG', + 'http://toscrape.com', + ) + p = subprocess.Popen( + args, + stderr=subprocess.PIPE, + ) + + def kill_proc(): + p.kill() + p.communicate() + assert False, 'Command took too much time to complete' + + timer = Timer(15, kill_proc) + try: + timer.start() + _, stderr = p.communicate() + finally: + timer.cancel() + + self.assertNotIn(b'Traceback', stderr) + if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == 'runserver': From 78ba4b033b016be7fbb22bfa9e6d5d389380e6d4 Mon Sep 17 00:00:00 2001 From: Yann Defretin Date: Wed, 16 Mar 2022 15:14:24 +0100 Subject: [PATCH 0503/2083] fixed detection of extension like ".tar.gz" in URL --- scrapy/utils/url.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index a6a2a9e8b..bae5a9433 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -5,7 +5,6 @@ library. Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ -import posixpath import re from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse @@ -31,8 +30,8 @@ def url_is_from_spider(url, spider): def url_has_any_extension(url, extensions): - return posixpath.splitext(parse_url(url).path)[1].lower() in extensions - + """Return True if the url ends with one of the extensions provided""" + return any(parse_url(url).path.lower().endswith(ext) for ext in extensions) def parse_url(url, encoding=None): """Return urlparsed url from the given argument (which could be an already From fd08bb6cd99f16c9ce433583d4698801dc7e0ac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 16 Mar 2022 14:34:57 +0100 Subject: [PATCH 0504/2083] Refactor the asynchronous process_spider_output documentation --- docs/topics/coroutines.rst | 157 +++++++++++++++++------------- docs/topics/spider-middleware.rst | 22 +++-- 2 files changed, 102 insertions(+), 77 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 073b6bd9a..361cd5e60 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -17,9 +17,12 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.Request` callbacks. + If you are using any custom or third-party :ref:`spider middleware + `, see :ref:`sync-async-spider-middleware`. + .. versionchanged:: VERSION - Output of async callbacks is now processed asynchronously instead of collecting - all of it first. + Output of async callbacks is now processed asynchronously instead of + collecting all of it first. - The :meth:`process_item` method of :ref:`item pipelines `. @@ -34,18 +37,26 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. -- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` - method of :ref:`spider middlewares `. See - :ref:`async-spider-middlewares`. +- The + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` + method of :ref:`spider middlewares `. + + It must be defined as an :term:`asynchronous generator`. The input + ``result`` parameter is an :term:`asynchronous iterable`. + + See also :ref:`sync-async-spider-middleware` and + :ref:`universal-spider-middleware`. .. versionadded:: VERSION -Usage -===== +General usage +============= -There are several use cases for coroutines in Scrapy. Code that would -return Deferreds when written for previous Scrapy versions, such as downloader -middlewares and signal handlers, can be rewritten to be shorter and cleaner:: +There are several use cases for coroutines in Scrapy. + +Code that would return Deferreds when written for previous Scrapy versions, +such as downloader middlewares and signal handlers, can be rewritten to be +shorter and cleaner:: from itemadapter import ItemAdapter @@ -110,46 +121,73 @@ Common use cases for asynchronous code include: .. _aio-libs: https://github.com/aio-libs -.. _async-spider-middlewares: -Asynchronous spider middlewares -=============================== +.. _sync-async-spider-middleware: + +Mixing synchronous and asynchronous spider middlewares +====================================================== .. versionadded:: VERSION -.. note:: This currently applies to - :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`. - In the future it will also apply to - :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests`. -Middleware methods discussed here can take and return async iterables. They can -return the same type of iterable or they can take a normal one and return an -async one. If such method needs to return an async iterable it must be an async -generator, not just a coroutine that returns an iterable. +The output of a :class:`~scrapy.Request` callback is passed as the ``result`` +parameter to the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method +of the first :ref:`spider middleware ` from the +:ref:`list of active spider middlewares `. +Then the output of that ``process_spider_output`` method is passed to the +``process_spider_output`` method of the next spider middleware, and so on for +every active spider middleware. -As the result of a middleware method is passed to the same method of the next -middleware, it needs to be adapted if the second method expects a different -type. Scrapy will do this transparently: +Scrapy supports mixing :ref:`coroutine methods ` and synchronous methods +in this chain of calls. -* A normal iterable is wrapped into an async one which shouldn't cause any side - effects. -* An async iterable is downgraded to a normal one by waiting until all results - are available and wrapping them in a normal iterable. This is problematic - because it pauses the normal middleware processing for this iterable and - because all results can be skipped if exceptions are raised during - processing. This case emits a warning and will be deprecated and then removed - in a later Scrapy version. -* Async iterables returned from - :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception` - won't be downgraded, an exception will be raised if that is needed. +However, if any of the ``process_spider_output`` methods is defined as a +synchronous method, and the previous ``Request`` callback or +``process_spider_output`` method is a coroutine, there are some drawbacks to +the asynchronous-to-synchronous conversion that Scrapy does so that the +synchronous ``process_spider_output`` method gets a synchronous iterable as its +``result`` parameter: -As downgrading is undesirable, here is the proposed way to avoid it. If all -middlewares, including 3rd-party ones, support async iterables as input, no -downgrading will happen. But removing normal iterable support (making the -method a coroutine) from a middleware published as a separate project or used -internally in projects for older Scrapy versions breaks backwards -compatibility. So, as an interim measure (it will be deprecated and then -removed in a later Scrapy version), a middleware can provide both sync and -async methods in the following form:: +- The whole output of the previous ``Request`` callback or + ``process_spider_output`` method is awaited at this point. + +- If an exception raises while awaiting the output of the previous + ``Request`` callback or ``process_spider_output`` method, none of that + output will be processed. + +Asynchronous-to-synchronous conversions are supported for backward +compatibility, but they are deprecated and will stop working in a future +version of Scrapy. + +To avoid asynchronous-to-synchronous conversion, when defining ``Request`` +callbacks as coroutine methods or when using spider middlewares whose +``process_spider_output`` method is an :term:`asynchronous generator`, all +active spider middlewares must either have their ``process_spider_output`` +method defined as an asynchronous generator or :ref:`define a +process_spider_output_async method `. + +.. note:: When using third-party spider middlewares that only define a + synchronous ``process_spider_output`` method, consider + :ref:`making them universal ` through + :ref:`subclassing `. + + +.. _universal-spider-middleware: + +Universal spider middleware +=========================== + +.. versionadded:: VERSION + +To allow writing a spider middleware that supports asynchronous execution of +its ``process_spider_output`` method in Scrapy VERSION and later (avoiding +:ref:`asynchronous-to-synchronous conversions `) +while maintaining support for older Scrapy versions, you may define +``process_spider_output`` as a synchronous method and define an +:term:`asynchronous generator` version of that method with an alternative name: +``process_spider_output_async``. + +For example:: class UniversalSpiderMiddleware: def process_spider_output(self, response, result, spider): @@ -162,28 +200,13 @@ async methods in the following form:: # ... do something with r yield r -In this case normal and async iterables will be passed to the respective -methods without any wrapping or downgrading, and in older versions of Scrapy -the coroutine method will just be ignored. When the backwards compatibility is -no longer needed the non-coroutine method can be dropped and the coroutine one -renamed to the normal name. It may be possible to extract common code from both -methods to reduce code duplication, as in the simplest case the only difference -between them will be ``for`` vs ``async for``. +.. note:: This is an interim measure to allow, for a time, to write code that + works in Scrapy VERSION and later without requiring + asynchronous-to-synchronous conversions, and works in earlier Scrapy + versions as well. -So, to recap: - -* If you don't intend to use async callbacks or middlewares containing async - code in your project, nothing should change for you yet. At some point in the - future some of the 3rd-party middlewares you use may drop backwards - compatibility, which shouldn't lead to immediate problems but may be a sign - to start converting your code to ``async def`` too. -* If you maintain a middleware that can be used with projects you can't control - (e.g. one you published for other people to use, or one that needs to support - some old project that can't be modernized), we recommend adding a - ``process_spider_output_async`` method so that the amount of unnecessary - iterable conversions is reduced but no compatibility is broken. -* If you use async callbacks, try to make sure all middlewares support them. - Note that you can modernize 3rd-party middlewares by subclassing them. -* If you want to write and publish a middleware that requires async code, you - should write in the docs that the minimum support Scrapy version is VERSION - (maybe even check this at the run time, using :attr:`scrapy.__version__`). + In some future version of Scrapy, however, this feature will be + deprecated and, eventually, in a later version of Scrapy, this + feature will be removed, and all spider middlewares will be expected + to define their ``process_spider_output`` method as an asynchronous + generator. diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index edfc2e4bb..787545ed2 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -98,10 +98,6 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_spider_output(response, result, spider) - .. versionchanged:: VERSION - Since VERSION this can take and return an :term:`python:asynchronous - iterable`. - This method is called with the results returned from the Spider, after it has processed the response. @@ -109,8 +105,15 @@ object gives you access, for example, to the :ref:`settings `. :class:`~scrapy.Request` objects and :ref:`item objects `. - .. note:: When defined as a :ref:`coroutine `, this method needs - to be an async generator, not just return an iterable. + .. versionchanged:: VERSION + This method may be defined as an :term:`asynchronous generator`, in + which case ``result`` is an :term:`asynchronous iterable`. + + Consider defining this method as an :term:`asynchronous generator`, + which will be a requirement in a future version of Scrapy. However, if + you wish your spider middleware to work with Scrapy versions earlier + than Scrapy VERSION, :ref:`make your spider middleware universal + ` instead. :param response: the response which generated this output from the spider @@ -127,10 +130,9 @@ object gives you access, for example, to the :ref:`settings `. .. versionadded:: VERSION - If exists, this methid will be called instead of - :meth:`process_spider_output` when ``result`` is an async iterable. - If this method exists, it must be a coroutine while - :meth:`process_spider_output` must not be a coroutine. + If defined, this method must be an :term:`asynchronous generator`, + which will be called instead of :meth:`process_spider_output` if + ``result`` is an :term:`asynchronous iterable`. .. method:: process_spider_exception(response, exception, spider) From c961438d5d9998344460d930ea502fae40553043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 16 Mar 2022 18:45:56 +0100 Subject: [PATCH 0505/2083] tests: cover scenarios of bad results from process_spider_output --- scrapy/core/spidermw.py | 19 ++++++++---- tests/test_spidermiddleware.py | 54 +++++++++++++++++++++++++++------- 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 6075670b0..1aa02f29f 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ import logging -from inspect import isasyncgenfunction +from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast @@ -61,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - msg = (f"Middleware {method.__qualname__} must return None " + msg = (f"{method.__qualname__} must return None " f"or raise an exception, got {type(result)}") raise _InvalidOutput(msg) except _InvalidOutput: @@ -129,7 +129,7 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = (f"Middleware {method.__qualname__} must return None " + msg = (f"{method.__qualname__} must return None " f"or an iterable, got {type(result)}") raise _InvalidOutput(msg) return _failure @@ -197,8 +197,17 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) else: - msg = (f"Middleware {method.__qualname__} must return an " - f"iterable, got {type(result)}") + if iscoroutine(result): + result.close() # Silence warning about not awaiting + msg = ( + f"{method.__qualname__} must be an asynchronous " + f"generator (i.e. use yield)" + ) + else: + msg = ( + f"{method.__qualname__} must return an iterable, got " + f"{type(result)}" + ) raise _InvalidOutput(msg) last_result_is_async = isinstance(result, AsyncIterable) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index f9f2b6642..ed0912b82 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -123,6 +123,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): start_index = 10 return {i: c for c, i in enumerate(mw_classes, start=start_index)} + def _scrape_func(self, *args, **kwargs): + yield {'foo': 1} + yield {'foo': 2} + yield {'foo': 3} + @defer.inlineCallbacks def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) @@ -201,11 +206,6 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware - def _scrape_func(self, *args, **kwargs): - yield {'foo': 1} - yield {'foo': 2} - yield {'foo': 3} - def test_simple(self): """ Simple mw """ return self._test_simple_base(self.MW_SIMPLE) @@ -285,6 +285,45 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): downgrade=True) +class ProcessSpiderOutputNonIterableMiddleware: + def process_spider_output(self, response, result, spider): + return + + +class ProcessSpiderOutputCoroutineMiddleware: + async def process_spider_output(self, response, result, spider): + results = [] + for r in result: + results.append(r) + return results + + +class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): + + @defer.inlineCallbacks + def test_non_iterable(self): + with self.assertRaisesRegex( + _InvalidOutput, + ( + "\.process_spider_output must return an iterable, got " + ), + ): + yield self._get_middleware_result( + ProcessSpiderOutputNonIterableMiddleware, + ) + + @defer.inlineCallbacks + def test_coroutine(self): + with self.assertRaisesRegex( + _InvalidOutput, + "\.process_spider_output must be an asynchronous generator", + ): + yield self._get_middleware_result( + ProcessSpiderOutputCoroutineMiddleware, + ) + + class ProcessStartRequestsSimpleMiddleware: def process_start_requests(self, start_requests, spider): for r in start_requests: @@ -387,11 +426,6 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware - def _scrape_func(self, *args, **kwargs): - yield {'foo': 1} - yield {'foo': 2} - yield {'foo': 3} - @defer.inlineCallbacks def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) From b78e6915c6b259b31d3c37cae1529e85e797c964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 16 Mar 2022 20:17:25 +0100 Subject: [PATCH 0506/2083] Clarify that without async-to-sync conversions items yielded before an exception are processed --- docs/topics/coroutines.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 361cd5e60..efc4566a0 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -155,6 +155,9 @@ synchronous ``process_spider_output`` method gets a synchronous iterable as its ``Request`` callback or ``process_spider_output`` method, none of that output will be processed. + This contrasts with the regular behavior, where all items yielded before + an exception raises are processed. + Asynchronous-to-synchronous conversions are supported for backward compatibility, but they are deprecated and will stop working in a future version of Scrapy. From 5b4b8b6fb12874d4a0a11c261341639c9af95b10 Mon Sep 17 00:00:00 2001 From: Yann Defretin Date: Wed, 16 Mar 2022 22:32:05 +0100 Subject: [PATCH 0507/2083] added test for new url_has_any_extension function --- tests/test_utils_url.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 144c7bd76..58e2be622 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,6 +1,8 @@ import unittest +from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.spiders import Spider +from scrapy.utils.misc import arg_to_iter from scrapy.utils.url import ( add_http_if_no_scheme, guess_scheme, @@ -8,9 +10,9 @@ from scrapy.utils.url import ( strip_url, url_is_from_any_domain, url_is_from_spider, + url_has_any_extension, ) - __doctests__ = ['scrapy.utils.url'] @@ -81,6 +83,15 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider)) + def test_url_has_any_extension(self): + deny_extensions = {'.' + e for e in arg_to_iter(IGNORED_EXTENSIONS)} + self.assertTrue(url_has_any_extension("http://www.example.com/archive.tar.gz", deny_extensions)) + self.assertTrue(url_has_any_extension("http://www.example.com/page.doc", deny_extensions)) + self.assertTrue(url_has_any_extension("http://www.example.com/page.pdf", deny_extensions)) + self.assertFalse(url_has_any_extension("http://www.example.com/page.htm", deny_extensions)) + self.assertFalse(url_has_any_extension("http://www.example.com/", deny_extensions)) + self.assertFalse(url_has_any_extension("http://www.example.com/page.doc.html", deny_extensions)) + class AddHttpIfNoScheme(unittest.TestCase): From 9a28eb0bad1acf986d997905a410058f77911b7c Mon Sep 17 00:00:00 2001 From: Eugene Date: Thu, 17 Mar 2022 05:39:54 +0100 Subject: [PATCH 0508/2083] Suggest installing the brotli package instead of brotlipy (#4267) --- docs/topics/downloader-middleware.rst | 5 +++-- tests/requirements.txt | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index a15637ed6..912600428 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -704,14 +704,15 @@ HttpCompressionMiddleware sent/received from web sites. This middleware also supports decoding `brotli-compressed`_ as well as - `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is + `zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt -.. _brotlipy: https://pypi.org/project/brotlipy/ +.. _brotli: https://pypi.org/project/Brotli/ .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt .. _zstandard: https://pypi.org/project/zstandard/ + HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/requirements.txt b/tests/requirements.txt index 398d1d16d..d2a8aae1b 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -12,7 +12,7 @@ uvloop; platform_system != "Windows" and python_version > '3.6' # optional for shell wrapper tests bpython -brotlipy # optional for HTTP compress downloader middleware tests +brotli # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" From 0905d42e33871e976760d880316210d4953cd5df Mon Sep 17 00:00:00 2001 From: Yann Defretin Date: Thu, 17 Mar 2022 11:19:09 +0100 Subject: [PATCH 0509/2083] refactored url_has_any_extension function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/url.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index bae5a9433..4d5e9ae82 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -31,7 +31,8 @@ def url_is_from_spider(url, spider): def url_has_any_extension(url, extensions): """Return True if the url ends with one of the extensions provided""" - return any(parse_url(url).path.lower().endswith(ext) for ext in extensions) + lowercase_path = parse_url(url).path.lower() + return any(lowercase_path.endswith(ext) for ext in extensions) def parse_url(url, encoding=None): """Return urlparsed url from the given argument (which could be an already From 6a3f2ee6876145bd4bd9ee1ff89d94474a1e85a0 Mon Sep 17 00:00:00 2001 From: FJMonteroInformatica Date: Thu, 17 Mar 2022 20:09:56 +0100 Subject: [PATCH 0510/2083] HTML Conventions --- docs/_static/selectors-sample1.html | 31 +++++++------- .../link_extractor/linkextractor.html | 40 ++++++++++--------- .../link_extractor/linkextractor_latin1.html | 8 ++-- .../link_extractor/linkextractor_no_href.html | 3 +- .../link_extractor/linkextractor_noenc.html | 23 ++++++----- tests/sample_data/test_site/index.html | 31 +++++++------- tests/sample_data/test_site/item1.html | 27 ++++++------- tests/sample_data/test_site/item2.html | 29 ++++++-------- 8 files changed, 96 insertions(+), 96 deletions(-) diff --git a/docs/_static/selectors-sample1.html b/docs/_static/selectors-sample1.html index 8a79a3381..915718832 100644 --- a/docs/_static/selectors-sample1.html +++ b/docs/_static/selectors-sample1.html @@ -1,16 +1,17 @@ - - - - Example website - - - - - + + + + + Example website + + + + + \ No newline at end of file diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html index 2307ea865..e3a2a4145 100644 --- a/tests/sample_data/link_extractor/linkextractor.html +++ b/tests/sample_data/link_extractor/linkextractor.html @@ -1,20 +1,22 @@ + + - - -Sample page with links for testing LinkExtractor - - - - - + + + Sample page with links for testing LinkExtractor + + + + + \ No newline at end of file diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html index e7eee18de..1e05bf0f0 100644 --- a/tests/sample_data/link_extractor/linkextractor_latin1.html +++ b/tests/sample_data/link_extractor/linkextractor_latin1.html @@ -1,3 +1,5 @@ + + @@ -7,11 +9,11 @@ diff --git a/tests/sample_data/link_extractor/linkextractor_no_href.html b/tests/sample_data/link_extractor/linkextractor_no_href.html index 0b01cede8..2d67ec6ff 100644 --- a/tests/sample_data/link_extractor/linkextractor_no_href.html +++ b/tests/sample_data/link_extractor/linkextractor_no_href.html @@ -1,3 +1,5 @@ + + @@ -21,5 +23,4 @@ - \ No newline at end of file diff --git a/tests/sample_data/link_extractor/linkextractor_noenc.html b/tests/sample_data/link_extractor/linkextractor_noenc.html index f9166adbe..6fa137cd9 100644 --- a/tests/sample_data/link_extractor/linkextractor_noenc.html +++ b/tests/sample_data/link_extractor/linkextractor_noenc.html @@ -1,14 +1,17 @@ + + - - -Sample page without encoding for testing LinkExtractor - + + + Sample page without encoding for testing LinkExtractor + + - + diff --git a/tests/sample_data/test_site/index.html b/tests/sample_data/test_site/index.html index d268c846a..afe17d8e2 100644 --- a/tests/sample_data/test_site/index.html +++ b/tests/sample_data/test_site/index.html @@ -1,18 +1,15 @@ + + - - -Scrapy test site - - - - -

Scrapy test site

- - - - - + + Scrapy test site + + +

Scrapy test site

+ + + \ No newline at end of file diff --git a/tests/sample_data/test_site/item1.html b/tests/sample_data/test_site/item1.html index ceeb6dc87..ee39f16f3 100644 --- a/tests/sample_data/test_site/item1.html +++ b/tests/sample_data/test_site/item1.html @@ -1,17 +1,14 @@ + + - - -Item 1 - Scrapy test site - - - - -

Item 1 name

- -
    -
  • Price: $100
    • -
  • Stock: 12
    • -
- - + + Item 1 - Scrapy test site + + +

Item 1 name

+
    +
  • Price: $100
    • +
  • Stock: 12
    • +
+ diff --git a/tests/sample_data/test_site/item2.html b/tests/sample_data/test_site/item2.html index a64c92810..f40f70750 100644 --- a/tests/sample_data/test_site/item2.html +++ b/tests/sample_data/test_site/item2.html @@ -1,17 +1,14 @@ + + - - -Item 2 - Scrapy test site - - - - -

Item 2 name

- -
    -
  • Price: $200
    • -
  • Stock: 5
    • -
- - - + + Item 2 - Scrapy test site + + +

Item 2 name

+
    +
  • Price: $200
    • +
  • Stock: 5
    • +
+ + \ No newline at end of file From b95c634b861bacc2b2ee3beeb5fcf64ab8607eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 17 Mar 2022 22:23:25 +0100 Subject: [PATCH 0511/2083] Document how to enforce Scrapy versions on Scrapy components --- docs/index.rst | 13 +++-- docs/topics/asyncio.rst | 24 +++++++++ docs/topics/components.rst | 84 +++++++++++++++++++++++++++++++ docs/topics/coroutines.rst | 6 +-- docs/topics/spider-middleware.rst | 8 +-- 5 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 docs/topics/components.rst diff --git a/docs/index.rst b/docs/index.rst index 75e08f537..6e22db884 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -229,10 +229,11 @@ Extending Scrapy topics/downloader-middleware topics/spider-middleware topics/extensions - topics/api topics/signals topics/scheduler topics/exporters + topics/components + topics/api :doc:`topics/architecture` @@ -247,9 +248,6 @@ Extending Scrapy :doc:`topics/extensions` Extend Scrapy with your custom functionality -:doc:`topics/api` - Use it on extensions and middlewares to extend Scrapy functionality - :doc:`topics/signals` See all available signals and how to work with them. @@ -259,6 +257,13 @@ Extending Scrapy :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). +:doc:`topics/components` + Learn the common API and some good practices when building custom Scrapy + components. + +:doc:`topics/api` + Use it on extensions and middlewares to extend Scrapy functionality + All the rest ============ diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 3a6941a2c..dbee7146d 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -96,3 +96,27 @@ Futures. Scrapy provides two helpers for this: down to Scrapy 2.0 (earlier versions do not support :mod:`asyncio`), you can copy the implementation of these functions into your own code. + + +.. _enforce-asyncio-requirement: + +Enforcing asyncio as a requirement +================================== + +If you are writing a :ref:`component ` that requires asyncio +to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to +:ref:`enforce it as a requirement `. For +example:: + + from scrapy.utils.reactor import is_asyncio_reactor_installed + + class MyComponent: + + def __init__(self): + if not is_asyncio_reactor_installed(): + raise ValueError( + f"{MyComponent.__qualname__} requires the asyncio Twisted " + f"reactor. Make sure you have it configured in the " + f"TWISTED_REACTOR setting. See the asyncio documentation " + f"of Scrapy for more information." + ) diff --git a/docs/topics/components.rst b/docs/topics/components.rst new file mode 100644 index 000000000..1fff2d61a --- /dev/null +++ b/docs/topics/components.rst @@ -0,0 +1,84 @@ +.. _topics-components: + +========== +Components +========== + +A Scrapy component is any class whose objects are created using +:func:`scrapy.utils.misc.create_instance`. + +That includes the classes that you may assign to the following settings: + +- :setting:`DNS_RESOLVER` + +- :setting:`DOWNLOAD_HANDLERS` + +- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` + +- :setting:`DOWNLOADER_MIDDLEWARES` + +- :setting:`DUPEFILTER_CLASS` + +- :setting:`EXTENSIONS` + +- :setting:`FEED_EXPORTERS` + +- :setting:`FEED_STORAGES` + +- :setting:`ITEM_PIPELINES` + +- :setting:`SCHEDULER` + +- :setting:`SCHEDULER_DISK_QUEUE` + +- :setting:`SCHEDULER_MEMORY_QUEUE` + +- :setting:`SCHEDULER_PRIORITY_QUEUE` + +- :setting:`SPIDER_MIDDLEWARES` + +Third-party Scrapy components may also let you define additional Scrapy +components, usually configurable through :ref:`settings `, to +modify their behavior. + +.. _enforce-component-requirements: + +Enforcing component requirements +================================ + +Sometimes, your components may only be intended to work under certain +conditions. For example, the may require a minimum version of Scrapy to work as +intended, or they may require certain settings to have specific values. + +In addition to describing those conditions in the documentation of your +component, it is a good practice to raise an exception from the ``__init__`` +method of your component if those conditions are not met at run time. + +In the case of :ref:`downloader middlewares `, +:ref:`extensions `, :ref:`item pipelines +`, and :ref:`spider middlewares +`, you should raise +:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a +parameter to the exception so that it is printed in the logs, for the user to +see. For other components, feel free to raise whatever other exception feels +right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy +version mismatch, while :exc:`ValueError` may be better if the issue is the +value of a setting. + +If your requirement is a minimum Scrapy version, you may use +:attr:`scrapy.__version__` to enforce your requirement. For example:: + + from pkg_resources import parse_version + + import scrapy + + class MyComponent: + + def __init__(self): + if parse_version(scrapy.__version__) < parse_version('VERSION'): + raise RuntimeError( + f"{MyComponent.__qualname__} requires Scrapy VERSION or " + f"later, which allow defining the process_spider_output " + f"method of spider middlewares as an asynchronous " + f"generator." + ) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index efc4566a0..55d013c06 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -162,7 +162,7 @@ Asynchronous-to-synchronous conversions are supported for backward compatibility, but they are deprecated and will stop working in a future version of Scrapy. -To avoid asynchronous-to-synchronous conversion, when defining ``Request`` +To avoid asynchronous-to-synchronous conversions, when defining ``Request`` callbacks as coroutine methods or when using spider middlewares whose ``process_spider_output`` method is an :term:`asynchronous generator`, all active spider middlewares must either have their ``process_spider_output`` @@ -177,8 +177,8 @@ process_spider_output_async method `. .. _universal-spider-middleware: -Universal spider middleware -=========================== +Universal spider middlewares +============================ .. versionadded:: VERSION diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 787545ed2..816cb5e03 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -111,9 +111,11 @@ object gives you access, for example, to the :ref:`settings `. Consider defining this method as an :term:`asynchronous generator`, which will be a requirement in a future version of Scrapy. However, if - you wish your spider middleware to work with Scrapy versions earlier - than Scrapy VERSION, :ref:`make your spider middleware universal - ` instead. + you plan on sharing your spider middleware with other people, consider + either :ref:`enforcing Scrapy VERSION ` + as a minimum requirement of your spider middleware, or :ref:`making + your spider middleware universal ` so that + it works with Scrapy versions earlier than Scrapy VERSION. :param response: the response which generated this output from the spider From fcf3d8e0a0df447fd7cd81e98c846a16b8b42a73 Mon Sep 17 00:00:00 2001 From: D00399830 Date: Mon, 21 Mar 2022 14:09:31 -0600 Subject: [PATCH 0512/2083] Updated the documentation for developer tools to have JavaScript instead of Javascript, as JavaScript is the more correct way to write it --- docs/topics/developer-tools.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 96475899f..9bf97c628 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -19,14 +19,14 @@ Caveats with inspecting the live browser DOM Since Developer Tools operate on a live browser DOM, what you'll actually see when inspecting the page source is not the original HTML, but a modified one -after applying some browser clean up and executing Javascript code. Firefox, +after applying some browser clean up and executing JavaScript code. Firefox, in particular, is known for adding ```` elements to tables. Scrapy, on the other hand, does not modify the original page HTML, so you won't be able to extract any data if you use ```` in your XPath expressions. Therefore, you should keep in mind the following things: -* Disable Javascript while inspecting the DOM looking for XPaths to be +* Disable JavaScript while inspecting the DOM looking for XPaths to be used in Scrapy (in the Developer Tools settings click `Disable JavaScript`) * Never use full XPath paths, use relative and clever ones based on attributes From 2227be7af6d0504d52bd4c2e299efb1becbd992f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Ruiz?= Date: Tue, 22 Mar 2022 15:21:16 +0100 Subject: [PATCH 0513/2083] Fix a typo in the HTTP cache documentation (#5455) --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 912600428..29e350651 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -366,7 +366,7 @@ HttpCacheMiddleware This middleware provides low-level cache to all HTTP requests and responses. It has to be combined with a cache storage backend as well as a cache policy. - Scrapy ships with three HTTP cache storage backends: + Scrapy ships with the following HTTP cache storage backends: * :ref:`httpcache-storage-fs` * :ref:`httpcache-storage-dbm` From 0beed7055cdec3683bf67ec01b573ddf87df723f Mon Sep 17 00:00:00 2001 From: Silvio Pavanetto Date: Wed, 23 Mar 2022 17:28:55 +0100 Subject: [PATCH 0514/2083] fix: return unique_list only when link_extractor.unique is True --- scrapy/linkextractors/lxmlhtml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index b5d2585a8..caef504a0 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -161,4 +161,6 @@ class LxmlLinkExtractor(FilteringLinkExtractor): for doc in docs: links = self._extract_links(doc, response.url, response.encoding, base_url) all_links.extend(self._process_links(links)) - return unique_list(all_links) + if self.link_extractor.unique: + return unique_list(all_links) + return all_links From 4af22bf157a5d25703cf5142097ba1c277f0d0d4 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 8 Apr 2022 14:26:23 +0500 Subject: [PATCH 0515/2083] Pin mitmproxy to < 8 for now (#5459) --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index fcd3563b2..aba94d79d 100644 --- a/tox.ini +++ b/tox.ini @@ -12,10 +12,11 @@ deps = -rtests/requirements.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 - # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe + # Python 3.9+ requires mitmproxy >= 5.3.0 # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' - mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' + # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 + mitmproxy >= 4.0.4, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy' From bae3f8745589b09e0ea75fa463a5423546ca442c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Apr 2022 12:04:02 +0200 Subject: [PATCH 0516/2083] Cover a backward-incompatible Request serialization change in the 2.6 release notes --- docs/news.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 5d92067b5..e4e2bce3c 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -113,6 +113,9 @@ Backward-incompatible changes meet expectations, :exc:`TypeError` is now raised at startup time. Before, other exceptions would be raised at run time. (:issue:`3559`) +- The ``_encoding`` field of serialized :class:`~scrapy.http.Request` objects + is now named ``encoding``, in line with all other fields (:issue:`5130`) + Deprecation removals ~~~~~~~~~~~~~~~~~~~~ From 319e67f779163df3ad44327bfbc9733edcb34908 Mon Sep 17 00:00:00 2001 From: Yash <76577754+yash-fn@users.noreply.github.com> Date: Sat, 26 Mar 2022 18:17:03 -0500 Subject: [PATCH 0517/2083] documentation update for multiple spiders i noticed passing settings to configure logging function made weird output go away. checked documentation and it says first parameter is settings file. Is this correct? --- docs/topics/practices.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index d0207fd18..7313c9246 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -180,8 +180,8 @@ Same example but running the spiders sequentially by chaining the deferreds: # Your second spider definition ... - configure_logging() settings = get_project_settings() + configure_logging(settings) runner = CrawlerRunner(settings) @defer.inlineCallbacks From aead27bcbdf7c2a4d959dcb357c7f12cc8411739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Apr 2022 15:06:22 +0200 Subject: [PATCH 0518/2083] Add release notes for 2.6.2 (#5448) --- docs/news.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index e4e2bce3c..2e0b43455 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,23 @@ Release notes ============= +.. _release-2.6.2: + +Scrapy 2.6.2 (2022-03-15) +------------------------- + +Fixes additional regressions introduced in 2.6.0: + +- :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple + spiders (:issue:`5435`, :issue:`5436`) + +- Fixed an exception that was being logged after the spider finished under + certain conditions (:issue:`5437`, :issue:`5440`) + +- The ``--output``/``-o`` command-line parameter supports again a value + starting with a hyphen (:issue:`5444`, :issue:`5445`) + + .. _release-2.6.1: Scrapy 2.6.1 (2022-03-01) From 636127ec1ea2b8949438015c2167ab5d009ff1bf Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 17 Apr 2022 12:01:24 -0700 Subject: [PATCH 0519/2083] tests that all CLI help messages don't throw errors --- tests/test_commands.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index b5e6c2b8b..76d5f3935 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -930,3 +930,17 @@ class MySpider(scrapy.Spider): args = ['-o', 'example1.json', '-O', 'example2.json'] log = self.get_log(spider_code, args=args) self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) + + +class HelpMessageTest(CommandTest): + + def setUp(self): + super().setUp() + self.commands = ["parse", "startproject", "view", "crawl", "edit", + "list", "fetch", "settings", "shell", "runspider", + "version", "genspider", "check", "bench"] + + def test_help_messages(self): + for command in self.commands: + _, out, _ = self.proc(command, "-h") + self.assertIn("Usage", out) From b0f5503cb8d0590aab4d9c91b0ee660d98d1e4a6 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 16 Apr 2022 18:17:47 -0700 Subject: [PATCH 0520/2083] Fixes Issue #5481 --- scrapy/commands/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index a3f6b96f4..6365fbdd0 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -51,7 +51,7 @@ class Command(BaseRunSpiderCommand): parser.add_argument("--cbkwargs", dest="cbkwargs", help="inject extra callback kwargs into the Request, it must be a valid raw json string") parser.add_argument("-d", "--depth", dest="depth", type=int, default=1, - help="maximum depth for parsing requests [default: %default]") + help=f"maximum depth for parsing requests [default: {self.max_level}]") parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="print each depth level one by one") From 56c9098d6af2a57cc11927959d47253999cddd46 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 16 Apr 2022 18:33:12 -0700 Subject: [PATCH 0521/2083] changed default depth to 1 --- scrapy/commands/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 6365fbdd0..a798ef945 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -51,7 +51,7 @@ class Command(BaseRunSpiderCommand): parser.add_argument("--cbkwargs", dest="cbkwargs", help="inject extra callback kwargs into the Request, it must be a valid raw json string") parser.add_argument("-d", "--depth", dest="depth", type=int, default=1, - help=f"maximum depth for parsing requests [default: {self.max_level}]") + help="maximum depth for parsing requests [default: 1]") parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="print each depth level one by one") From 915c288205e2b9a0bdbbe18cc67cd23ba5bb4de3 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 17 Apr 2022 10:49:50 -0700 Subject: [PATCH 0522/2083] edit --- scrapy/commands/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index a798ef945..8e52d0d76 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -51,7 +51,7 @@ class Command(BaseRunSpiderCommand): parser.add_argument("--cbkwargs", dest="cbkwargs", help="inject extra callback kwargs into the Request, it must be a valid raw json string") parser.add_argument("-d", "--depth", dest="depth", type=int, default=1, - help="maximum depth for parsing requests [default: 1]") + help="maximum depth for parsing requests [default: %(default)s]") parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="print each depth level one by one") From 7de9ed5bd42052f6491c939513adde6f6243ce2f Mon Sep 17 00:00:00 2001 From: PluT00 <314lut00@gmail.com> Date: Sun, 1 May 2022 01:34:35 +0300 Subject: [PATCH 0523/2083] Deprecate scrapy.pipelines.images.NoimagesDrop --- scrapy/pipelines/images.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 9c99dc69e..df4575a41 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -15,10 +15,12 @@ from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings +from scrapy.utils.decorators import deprecated from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes +@deprecated() class NoimagesDrop(DropItem): """Product with no images exception""" From 9f659bd63c04656f1c0601d6338f9050d5049fe8 Mon Sep 17 00:00:00 2001 From: PluT00 <314lut00@gmail.com> Date: Sun, 1 May 2022 13:36:15 +0300 Subject: [PATCH 0524/2083] Fix deprecation of scrapy.pipelines.images.NoimagesDrop --- scrapy/pipelines/images.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index df4575a41..c7a04a0cf 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -5,25 +5,28 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib +import warnings from contextlib import suppress from io import BytesIO from itemadapter import ItemAdapter -from scrapy.exceptions import DropItem, NotConfigured +from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings -from scrapy.utils.decorators import deprecated from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes -@deprecated() class NoimagesDrop(DropItem): """Product with no images exception""" + def __init__(self, *args, **kwargs): + warnings.warn("The NoimagesDrop class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2) + super().__init__(*args, **kwargs) + class ImageException(FileException): """General image error exception""" From cc16af35af8b20e04b42ca84431820b2b092f379 Mon Sep 17 00:00:00 2001 From: PluT00 <314lut00@gmail.com> Date: Tue, 3 May 2022 11:29:21 +0300 Subject: [PATCH 0525/2083] Add deprecation warning test for scrapy.pipelines.images.NoimagesDrop --- tests/test_pipeline_images.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index c69cd0e4a..613190f9c 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -4,14 +4,16 @@ import random from shutil import rmtree from tempfile import mkdtemp from unittest import skipIf +from warnings import catch_warnings import attr from itemadapter import ItemAdapter from twisted.trial import unittest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.item import Field, Item -from scrapy.pipelines.images import ImagesPipeline +from scrapy.pipelines.images import ImagesPipeline, NoimagesDrop from scrapy.settings import Settings from scrapy.utils.python import to_bytes @@ -413,6 +415,22 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): expected_value) +class NoimagesDropTestCase(unittest.TestCase): + + def test_deprecation_warning(self): + arg = str() + with catch_warnings(record=True) as warnings: + NoimagesDrop(arg) + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + with catch_warnings(record=True) as warnings: + class SubclassedNoimagesDrop(NoimagesDrop): + pass + SubclassedNoimagesDrop(arg) + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + + def _create_image(format, *a, **kw): buf = io.BytesIO() Image.new(*a, **kw).save(buf, format) From b2afcbfe2bf090827540d072866bef0d1ab3a3e8 Mon Sep 17 00:00:00 2001 From: AngelikiBoura <73474686+AngelikiBoura@users.noreply.github.com> Date: Thu, 5 May 2022 16:49:52 +0300 Subject: [PATCH 0526/2083] Fix typos in three files for Flake8 check (#5487) * Fix typos in extensions files Made some fixes in files memusage.py and statsmailer.py in order to pass the flake8 check. * Fix typos in twisted_reactor_custom_settings_same.py A small change was needed in order for flake8 check to pass. --- scrapy/extensions/memusage.py | 10 +++++----- scrapy/extensions/statsmailer.py | 1 + .../twisted_reactor_custom_settings_same.py | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 9de119a10..f5081a7d7 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -33,8 +33,8 @@ class MemoryUsage: self.crawler = crawler self.warned = False self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL') - self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 - self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 + self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024 + self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024 self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS') self.mail = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) @@ -77,7 +77,7 @@ class MemoryUsage: def _check_limit(self): if self.get_virtual_size() > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) - mem = self.limit/1024/1024 + mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: @@ -94,11 +94,11 @@ class MemoryUsage: self.crawler.stop() def _check_warning(self): - if self.warned: # warn only once + if self.warned: # warn only once return if self.get_virtual_size() > self.warning: self.crawler.stats.set_value('memusage/warning_reached', 1) - mem = self.warning/1024/1024 + mem = self.warning / 1024 / 1024 logger.warning("Memory usage reached %(memusage)dM", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index bcdbaff24..739e6b958 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -8,6 +8,7 @@ from scrapy import signals from scrapy.mail import MailSender from scrapy.exceptions import NotConfigured + class StatsMailer: def __init__(self, stats, recipients, mail): diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py index 1f5a44010..72bb986bc 100644 --- a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py @@ -8,6 +8,7 @@ class AsyncioReactorSpider1(scrapy.Spider): "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } + class AsyncioReactorSpider2(scrapy.Spider): name = 'asyncio_reactor2' custom_settings = { From 83c1939281197242511931c6e9f356f2498eb623 Mon Sep 17 00:00:00 2001 From: Andreas Tziortziortziopoulos Date: Fri, 6 May 2022 03:59:30 +0300 Subject: [PATCH 0527/2083] Issue #3264, fix error handling when spider is not matched Changes Implementation: - Check whether Spider exists or is None, and if it's None skip execution of start_requests() with non existing Spider Testing: - Add a test case with invalid url inside test_command_parse Test proves that non-matched Spider does not throw an AttributeError --- scrapy/commands/parse.py | 3 ++- tests/test_command_parse.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index a3f6b96f4..99fc8f955 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -146,7 +146,8 @@ class Command(BaseRunSpiderCommand): def _start_requests(spider): yield self.prepare_request(spider, Request(url), opts) - self.spidercls.start_requests = _start_requests + if self.spidercls: + self.spidercls.start_requests = _start_requests def start_parsing(self, url, opts): self.crawler_process.crawl(self.spidercls, **opts.spargs) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index f21ee971d..0622074a3 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,6 +1,7 @@ import os import argparse from os.path import join, abspath, isfile, exists + from twisted.internet import defer from scrapy.commands import parse from scrapy.settings import Settings @@ -222,6 +223,10 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) + status, out, stderr = yield self.execute([self.url('/invalid_url')]) + self.assertEqual(status, 0) + self.assertIn("""""", _textmode(stderr)) + @defer.inlineCallbacks def test_output_flag(self): """Checks if a file was created successfully having From 2006060688976ca469d794cdd3b753a00bfb83c9 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Wed, 11 May 2022 10:29:53 +0300 Subject: [PATCH 0528/2083] per slot settings: codestyle(flake8) fix, code line length --- scrapy/core/downloader/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 53dc546de..511693830 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -103,7 +103,9 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.per_slot_settings.get(key, {}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) + conc = self.per_slot_settings.get(key, {}).get( + 'concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) delay = self.per_slot_settings.get(key, {}).get('delay', delay) randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) From 0ffc52a491e6e6c46196b3aa92767856f64a8ebc Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Wed, 11 May 2022 10:40:12 +0300 Subject: [PATCH 0529/2083] per slot settings: test added (delays for each download slots) --- tests/test_downloaderslotssettings.py | 75 +++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/test_downloaderslotssettings.py diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py new file mode 100644 index 000000000..6ea03fb78 --- /dev/null +++ b/tests/test_downloaderslotssettings.py @@ -0,0 +1,75 @@ +import time + +from scrapy.crawler import CrawlerRunner +from scrapy.http import Request + +from tests.mockserver import MockServer +from tests.spiders import MetaSpider + +from twisted.internet import defer +from twisted.trial.unittest import TestCase + + +class DownloaderSlotsSettingsTestSpider(MetaSpider): + + name = 'downloader_slots' + + custom_settings = { + "DOWNLOAD_DELAY": 1, + "RANDOMIZE_DOWNLOAD_DELAY": False, + "DOWNLOAD_SLOTS": { + 'quotes.toscrape.com': { + 'concurrency': 1, + 'delay': 1.5, + 'randomize_delay': False + }, + 'books.toscrape.com': { + 'delay': 2, + 'randomize_delay': False + } + } + } + + def start_requests(self): + self.times = {None: []} + + slots = list(self.custom_settings.get('DOWNLOAD_SLOTS', {}).keys()) + [None] + + for slot in slots: + url = self.mockserver.url(f"/?downloader_slot={slot}") + self.times[slot] = [] + yield Request(url, callback=self.parse, meta={'download_slot': slot}) + + def parse(self, response): + slot = response.meta.get('download_slot', None) + self.times[slot].append(time.time()) + url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") + yield Request(url, callback=self.not_parse, meta={'download_slot': slot}) + + def not_parse(self, response): + slot = response.meta.get('download_slot', None) + self.times[slot].append(time.time()) + + +class CrawlTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self.runner = CrawlerRunner() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_delay(self): + crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider) + yield crawler.crawl(mockserver=self.mockserver) + slots = crawler.engine.downloader.slots + times = crawler.spider.times + tolerance = 0.3 + + delays_real = {k: v[1] - v[0] for k, v in times.items()} + error_delta = {k: 1 - delays_real[k] / v.delay for k, v in slots.items()} + + self.assertTrue(max(list(error_delta.values())) < tolerance) From 1c031b8a8dd719e6011ee29889bc8181cdbc9a9b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 12 May 2022 13:10:08 -0300 Subject: [PATCH 0530/2083] Underscore CaseInsensitiveDict normkey/normvalue --- scrapy/utils/datatypes.py | 19 +++++++++---------- tests/test_utils_datatypes.py | 10 +++++++--- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index f45e1c9b8..807a95504 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -80,9 +80,8 @@ class CaselessDict(dict): class CaseInsensitiveDict(collections.UserDict): - """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups. - - It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods. + """A dict-like structure that accepts strings or bytes + as keys and allows case-insensitive lookups. """ def __init__(self, *args, **kwargs) -> None: @@ -90,32 +89,32 @@ class CaseInsensitiveDict(collections.UserDict): super().__init__(*args, **kwargs) def __getitem__(self, key: AnyStr) -> Any: - normalized_key = self.normkey(key) + normalized_key = self._normkey(key) return super().__getitem__(self._keys[normalized_key.lower()]) def __setitem__(self, key: AnyStr, value: Any) -> None: - normalized_key = self.normkey(key) + normalized_key = self._normkey(key) if normalized_key.lower() in self._keys: del self[self._keys[normalized_key.lower()]] - super().__setitem__(normalized_key, self.normvalue(value)) + super().__setitem__(normalized_key, self._normvalue(value)) self._keys[normalized_key.lower()] = normalized_key def __delitem__(self, key: AnyStr) -> None: - normalized_key = self.normkey(key) + normalized_key = self._normkey(key) stored_key = self._keys.pop(normalized_key.lower()) super().__delitem__(stored_key) def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] - normalized_key = self.normkey(key) + normalized_key = self._normkey(key) return normalized_key.lower() in self._keys def __repr__(self) -> str: return f"<{self.__class__.__name__}: {super().__repr__()}>" - def normkey(self, key: AnyStr) -> AnyStr: + def _normkey(self, key: AnyStr) -> AnyStr: return key - def normvalue(self, value: Any) -> Any: + def _normvalue(self, value: Any) -> Any: return value diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 5faaabe81..0a724f237 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,8 +1,8 @@ import copy -from typing import Iterator import unittest import warnings from collections.abc import Mapping, MutableMapping +from typing import Iterator from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request @@ -144,19 +144,23 @@ class CaseInsensitiveDictMixin: def test_normkey(self): class MyDict(self.dict_class): - def normkey(self, key): + def _normkey(self, key): return key.title() + normkey = _normkey # deprecated CaselessDict class + d = MyDict() d['key-one'] = 2 self.assertEqual(list(d.keys()), ['Key-One']) def test_normvalue(self): class MyDict(self.dict_class): - def normvalue(self, value): + def _normvalue(self, value): if value is not None: return value + 1 + normvalue = _normvalue # deprecated CaselessDict class + d = MyDict({'key': 1}) self.assertEqual(d['key'], 2) self.assertEqual(d.get('key'), 2) From 84c29a286f6b9bc94a5318ca68cd4fc21244443a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 20 May 2022 06:45:38 +0200 Subject: [PATCH 0531/2083] Unset the release date of still-unreleased 2.6.2 (#5503) --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 2e0b43455..ca9aeb783 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.6.2: -Scrapy 2.6.2 (2022-03-15) +Scrapy 2.6.2 (2022-0?-??) ------------------------- Fixes additional regressions introduced in 2.6.0: From 1c1cd5d8eae48eade88560426499846d217a555f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 20 May 2022 07:05:26 +0200 Subject: [PATCH 0532/2083] Update the 2.6.2 release notes --- docs/news.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index ca9aeb783..ffeb50390 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -19,6 +19,9 @@ Fixes additional regressions introduced in 2.6.0: - The ``--output``/``-o`` command-line parameter supports again a value starting with a hyphen (:issue:`5444`, :issue:`5445`) +- The ``scrapy parse -h`` command no longer throws an error (:issue:`5481`, + :issue:`5482`) + .. _release-2.6.1: From 965fde24a4798ee51f05ce8669b7a28958ad3238 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 8 Apr 2022 14:26:23 +0500 Subject: [PATCH 0533/2083] Pin mitmproxy to < 8 for now (#5459) --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index db151f215..d13bb7b38 100644 --- a/tox.ini +++ b/tox.ini @@ -12,10 +12,11 @@ deps = -rtests/requirements.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 - # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe + # Python 3.9+ requires mitmproxy >= 5.3.0 # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' - mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' + # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 + mitmproxy >= 4.0.4, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy' From 078622cfb0ee364acba5d91a20244f9c1ee87d30 Mon Sep 17 00:00:00 2001 From: Maxime Nannan <28675918+mnannan@users.noreply.github.com> Date: Fri, 20 May 2022 08:30:06 +0200 Subject: [PATCH 0534/2083] Fix file expiration issue with GCS (#5318) --- scrapy/pipelines/files.py | 10 +++++++--- tests/test_pipeline_files.py | 23 +++++++++++++++++++++++ tox.ini | 7 ++++--- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 5c52c6c28..906e7eb24 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -222,8 +222,8 @@ class GCSFilesStore: return {'checksum': checksum, 'last_modified': last_modified} else: return {} - - return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess) + blob_path = self._get_blob_path(path) + return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess) def _get_content_type(self, headers): if headers and 'Content-Type' in headers: @@ -231,8 +231,12 @@ class GCSFilesStore: else: return 'application/octet-stream' + def _get_blob_path(self, path): + return self.prefix + path + def persist_file(self, path, buf, info, meta=None, headers=None): - blob = self.bucket.blob(self.prefix + path) + blob_path = self._get_blob_path(path) + blob = self.bucket.blob(blob_path) blob.cache_control = self.CACHE_CONTROL blob.metadata = {k: str(v) for k, v in (meta or {}).items()} return threads.deferToThread( diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4e1b90787..0ff2045ed 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -525,6 +525,29 @@ class TestGCSFilesStore(unittest.TestCase): self.assertEqual(blob.content_type, 'application/octet-stream') self.assertIn(expected_policy, acl) + @defer.inlineCallbacks + def test_blob_path_consistency(self): + """Test to make sure that paths used to store files is the same as the one used to get + already uploaded files. + """ + assert_gcs_environ() + try: + import google.cloud.storage # noqa + except ModuleNotFoundError: + raise unittest.SkipTest("google-cloud-storage is not installed") + else: + with mock.patch('google.cloud.storage') as _: + with mock.patch('scrapy.pipelines.files.time') as _: + uri = 'gs://my_bucket/my_prefix/' + store = GCSFilesStore(uri) + store.bucket = mock.Mock() + path = 'full/my_data.txt' + yield store.persist_file(path, mock.Mock(), info=None, meta=None, headers=None) + yield store.stat_file(path, info=None) + expected_blob_path = store.prefix + path + store.bucket.blob.assert_called_with(expected_blob_path) + store.bucket.get_blob.assert_called_with(expected_blob_path) + class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks diff --git a/tox.ini b/tox.ini index d13bb7b38..6951b6d16 100644 --- a/tox.ini +++ b/tox.ini @@ -126,13 +126,14 @@ setenv = deps = {[testenv]deps} boto + google-cloud-storage + # Twisted[http2] currently forces old mitmproxy because of h2 version + # restrictions in their deps, so we need to pin old markupsafe here too. + markupsafe < 2.1.0 reppy robotexclusionrulesparser Pillow>=4.0.0 Twisted[http2]>=17.9.0 - # Twisted[http2] currently forces old mitmproxy because of h2 version restrictions in their deps, - # so we need to pin old markupsafe here too - markupsafe < 2.1.0 [testenv:asyncio] commands = From b5c15d87ff5770220bca31792c89e58804b923bb Mon Sep 17 00:00:00 2001 From: Andreas Tziortziortziopoulos Date: Sun, 22 May 2022 12:19:20 +0300 Subject: [PATCH 0535/2083] [issue3264] Separate test for not matched spider to a url --- tests/test_command_parse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 0622074a3..0d992be56 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -223,9 +223,10 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) + @defer.inlineCallbacks + def test_crawlspider_not_exists_with_not_matched_url(self): status, out, stderr = yield self.execute([self.url('/invalid_url')]) self.assertEqual(status, 0) - self.assertIn("""""", _textmode(stderr)) @defer.inlineCallbacks def test_output_flag(self): From 86331900125dc311223cbd1ebb0e10d09e7c592d Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Tue, 24 May 2022 14:47:00 +0430 Subject: [PATCH 0536/2083] pass on item to thumb_path function as additional argument resolves #5504 --- scrapy/pipelines/images.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 9c99dc69e..45ac03820 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -141,7 +141,7 @@ class ImagesPipeline(FilesPipeline): yield path, image, buf for thumb_id, size in self.thumbs.items(): - thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) + thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item) thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_path, thumb_image, thumb_buf @@ -179,6 +179,6 @@ class ImagesPipeline(FilesPipeline): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return f'full/{image_guid}.jpg' - def thumb_path(self, request, thumb_id, response=None, info=None): + def thumb_path(self, request, thumb_id, response=None, info=None, item=None): thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return f'thumbs/{thumb_id}/{thumb_guid}.jpg' From f39def4492f838b2414324e22a12f3b355f5c062 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Wed, 25 May 2022 23:57:38 +0430 Subject: [PATCH 0537/2083] add docs --- docs/topics/media-pipeline.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 7dff78390..2513faae2 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -656,6 +656,26 @@ See here the methods that you can override in your custom Images Pipeline: .. versionadded:: 2.4 The *item* parameter. + .. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None) + + This method is called for every item of :setting:`IMAGES_THUMBS` per downloaded item. It returns the + thumbnail download path of the image originating from the specified + :class:`response `. + + In addition to ``response``, this method receives the original + :class:`request `, + ``thumb_id``, + :class:`info ` and + :class:`item `. + + You can override this method to customize the thumbnail download path of each image. + You can use the ``item`` to determine the file path based on some item + property. + + By default the :meth:`thumb_path` method returns + ``thumbs//.``. + + .. method:: ImagesPipeline.get_media_requests(item, info) Works the same way as :meth:`FilesPipeline.get_media_requests` method, From 5c586d78f0e1c5b66358ed644bb6e528ad4b062b Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Wed, 25 May 2022 23:58:09 +0430 Subject: [PATCH 0538/2083] add tests --- tests/test_pipeline_images.py | 16 ++++++++++++++++ tests/test_pipeline_media.py | 28 +++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index c69cd0e4a..dd94d296b 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -93,6 +93,22 @@ class ImagesPipelineTestCase(unittest.TestCase): info=object()), 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') + def test_thumbnail_name_from_item(self): + """ + Custom thumbnail name based on item data, overriding default implementation + """ + + class CustomImagesPipeline(ImagesPipeline): + def thumb_path(self, request, thumb_id, response=None, info=None, item=None): + return f"thumb/{thumb_id}/{item.get('path')}" + + thumb_path = CustomImagesPipeline.from_settings(Settings( + {'IMAGES_STORE': self.tempdir} + )).thumb_path + item = dict(path='path-to-store-file') + request = Request("http://example.com") + self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file') + def test_convert_image(self): SIZE = (100, 100) # straigh forward case: RGB and JPEG diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 893d43052..a802c7cf1 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,4 +1,5 @@ from typing import Optional +import io from testfixtures import LogCapture from twisted.trial import unittest @@ -355,9 +356,12 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): def get_media_requests(self, item, info): item_url = item['image_urls'][0] + output_img = io.BytesIO() + img = Image.new('RGB', (60, 30), color='red') + img.save(output_img, format='JPEG') return Request( item_url, - meta={'response': Response(item_url, status=200, body=b'data')} + meta={'response': Response(item_url, status=200, body=output_img.getvalue())} ) def inc_stats(self, *args, **kwargs): @@ -379,9 +383,13 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): self._mockcalled.append('file_path') return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info) + def thumb_path(self, request, thumb_id, response=None, info=None): + self._mockcalled.append('thumb_path') + return super(MockedMediaPipelineDeprecatedMethods, self).thumb_path(request, thumb_id, response, info) + def get_images(self, response, request, info): self._mockcalled.append('get_images') - return [] + return super(MockedMediaPipelineDeprecatedMethods, self).get_images(response, request, info) def image_downloaded(self, response, request, info): self._mockcalled.append('image_downloaded') @@ -392,7 +400,11 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): skip = skip_pillow def setUp(self): - self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func) + self.pipe = MockedMediaPipelineDeprecatedMethods( + store_uri='store-uri', + download_func=_mocked_download_func, + settings=Settings({"IMAGES_THUMBS": {'small': (50, 50)}}) + ) self.pipe.open_spider(None) self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[]) @@ -444,6 +456,16 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): ) self._assert_method_called_with_warnings('file_path', message, warnings) + @inlineCallbacks + def test_thumb_path_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, ' + 'please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)' + ) + self._assert_method_called_with_warnings('thumb_path', message, warnings) + @inlineCallbacks def test_get_images_called(self): yield self.pipe.process_item(self.item, None) From 896f16f2def7c276350421352dddf6e7b0145519 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Wed, 25 May 2022 23:59:25 +0430 Subject: [PATCH 0539/2083] make thumb_path method backwards compatible --- scrapy/pipelines/images.py | 2 +- scrapy/pipelines/media.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 45ac03820..6b97190ee 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -179,6 +179,6 @@ class ImagesPipeline(FilesPipeline): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return f'full/{image_guid}.jpg' - def thumb_path(self, request, thumb_id, response=None, info=None, item=None): + def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None): thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return f'thumbs/{thumb_id}/{thumb_guid}.jpg' diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index d1bccf323..430c37227 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -121,7 +121,7 @@ class MediaPipeline: def _make_compatible(self): """Make overridable methods of MediaPipeline and subclasses backwards compatible""" methods = [ - "file_path", "media_to_download", "media_downloaded", + "file_path", "thumb_path", "media_to_download", "media_downloaded", "file_downloaded", "image_downloaded", "get_images" ] From 2c65066ad9e293630da2c594af06ad483abe800d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 27 May 2022 19:56:42 -0300 Subject: [PATCH 0540/2083] Avoid exceptions on copy --- scrapy/utils/datatypes.py | 7 +++++-- tests/test_utils_datatypes.py | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 807a95504..fd5ac3b08 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -94,8 +94,11 @@ class CaseInsensitiveDict(collections.UserDict): def __setitem__(self, key: AnyStr, value: Any) -> None: normalized_key = self._normkey(key) - if normalized_key.lower() in self._keys: - del self[self._keys[normalized_key.lower()]] + try: + lower_key = self._keys[normalized_key.lower()] + del self[lower_key] + except KeyError: + pass super().__setitem__(normalized_key, self._normvalue(value)) self._keys[normalized_key.lower()] = normalized_key diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 0a724f237..36df9006f 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -187,9 +187,15 @@ class CaseInsensitiveDictMixin: def test_copy(self): h1 = self.dict_class({'header1': 'value'}) h2 = copy.copy(h1) + assert isinstance(h2, self.dict_class) self.assertEqual(h1, h2) self.assertEqual(h1.get('header1'), h2.get('header1')) - assert isinstance(h2, self.dict_class) + self.assertEqual(h1.get('header1'), h2.get('HEADER1')) + h3 = h1.copy() + assert isinstance(h3, self.dict_class) + self.assertEqual(h1, h3) + self.assertEqual(h1.get('header1'), h3.get('header1')) + self.assertEqual(h1.get('header1'), h3.get('HEADER1')) class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): From c5627af15bcf413c04539aeb47dd07cf8b3e4092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 7 Jun 2022 18:44:54 +0200 Subject: [PATCH 0541/2083] Centralize request fingerprints (#4524) Co-authored-by: Mikhail Korobov --- docs/conf.py | 2 + docs/news.rst | 2 +- docs/topics/api.rst | 7 + docs/topics/item-pipeline.rst | 4 +- docs/topics/request-response.rst | 268 +++++++ docs/topics/settings.rst | 8 +- scrapy/crawler.py | 7 + scrapy/dupefilters.py | 49 +- scrapy/extensions/httpcache.py | 14 +- scrapy/pipelines/media.py | 4 +- scrapy/settings/default_settings.py | 3 + .../templates/project/module/settings.py.tmpl | 3 + scrapy/utils/request.py | 235 +++++- scrapy/utils/test.py | 9 +- tests/test_crawler.py | 3 +- tests/test_dupefilters.py | 80 +- tests/test_pipeline_files.py | 5 +- tests/test_pipeline_media.py | 53 +- tests/test_utils_request.py | 699 ++++++++++++++++-- 19 files changed, 1304 insertions(+), 151 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 55aa72d5a..378b01804 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -294,7 +294,9 @@ intersphinx_mapping = { 'tox': ('https://tox.readthedocs.io/en/latest', None), 'twisted': ('https://twistedmatrix.com/documents/current', None), 'twistedapi': ('https://twistedmatrix.com/documents/current/api', None), + 'w3lib': ('https://w3lib.readthedocs.io/en/latest', None), } +intersphinx_disabled_reftypes = [] # Options for sphinx-hoverxref options diff --git a/docs/news.rst b/docs/news.rst index 5d92067b5..2d0ab485e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1643,7 +1643,7 @@ New features :issue:`4370`) * A new ``keep_fragments`` parameter of - :func:`scrapy.utils.request.request_fingerprint` allows to generate + ``scrapy.utils.request.request_fingerprint`` allows to generate different fingerprints for requests with different fragments in their URL (:issue:`4104`) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 900b19c7a..60b5acd10 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares :class:`scrapy.Spider` subclass and a :class:`scrapy.settings.Settings` object. + .. attribute:: request_fingerprinter + + The request fingerprint builder of this crawler. + + This is used from extensions and middlewares to build short, unique + identifiers for requests. See :ref:`request-fingerprints`. + .. attribute:: settings The settings manager of this crawler. diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 391751364..882ff5661 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -60,9 +60,9 @@ Additionally, they may also implement the following methods: :param spider: the spider which was closed :type spider: :class:`~scrapy.Spider` object -.. method:: from_crawler(cls, crawler) +.. classmethod:: from_crawler(cls, crawler) - If present, this classmethod is called to create a pipeline instance + If present, this class method is called to create a pipeline instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance of the pipeline. Crawler object provides access to all Scrapy core components like settings and signals; it is a way for pipeline to diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 92a471faf..49cb69f67 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -339,6 +339,7 @@ errors if needed:: request = failure.request self.logger.error('TimeoutError on %s', request.url) + .. _errback-cb_kwargs: Accessing additional data in errback functions @@ -364,6 +365,273 @@ achieve this by using ``Failure.request.cb_kwargs``:: main_url=failure.request.cb_kwargs['main_url'], ) + +.. _request-fingerprints: + +Request fingerprints +-------------------- + +There are some aspects of scraping, such as filtering out duplicate requests +(see :setting:`DUPEFILTER_CLASS`) or caching responses (see +:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short, +unique identifier from a :class:`~scrapy.http.Request` object: a request +fingerprint. + +You often do not need to worry about request fingerprints, the default request +fingerprinter works for most projects. + +However, there is no universal way to generate a unique identifier from a +request, because different situations require comparing requests differently. +For example, sometimes you may need to compare URLs case-insensitively, include +URL fragments, exclude certain URL query parameters, include some or all +headers, etc. + +To change how request fingerprints are built for your requests, use the +:setting:`REQUEST_FINGERPRINTER_CLASS` setting. + +.. setting:: REQUEST_FINGERPRINTER_CLASS + +REQUEST_FINGERPRINTER_CLASS +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: VERSION + +Default: :class:`scrapy.utils.request.RequestFingerprinter` + +A :ref:`request fingerprinter class ` or its +import path. + +.. autoclass:: scrapy.utils.request.RequestFingerprinter + + +.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION + +REQUEST_FINGERPRINTER_IMPLEMENTATION +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: VERSION + +Default: ``'PREVIOUS_VERSION'`` + +Determines which request fingerprinting algorithm is used by the default +request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`). + +Possible values are: + +- ``'PREVIOUS_VERSION'`` (default) + + This implementation uses the same request fingerprinting algorithm as + Scrapy PREVIOUS_VERSION and earlier versions. + + Even though this is the default value for backward compatibility reasons, + it is a deprecated value. + +- ``'VERSION'`` + + This implementation was introduced in Scrapy VERSION to fix an issue of the + previous implementation. + + New projects should use this value. The :command:`startproject` command + sets this value in the generated ``settings.py`` file. + +If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are +using Scrapy components where changing the request fingerprinting algorithm +would cause undesired results, you need to carefully decide when to change the +value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS` +setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request +fingerprinting algorithm and does not log this warning ( +:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a +class). + +Scenarios where changing the request fingerprinting algorithm may cause +undesired results include, for example, using the HTTP cache middleware (see +:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`). +Changing the request fingerprinting algorithm would invalidade the current +cache, requiring you to redownload all requests again. + +Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in +your settings to switch already to the request fingerprinting implementation +that will be the only request fingerprinting implementation available in a +future version of Scrapy, and remove the deprecation warning triggered by using +the default value (``'PREVIOUS_VERSION'``). + + +.. _PREVIOUS_VERSION-request-fingerprinter: +.. _custom-request-fingerprinter: + +Writing your own request fingerprinter +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A request fingerprinter is a class that must implement the following method: + +.. method:: fingerprint(self, request) + + Return a :class:`bytes` object that uniquely identifies *request*. + + See also :ref:`request-fingerprint-restrictions`. + + :param request: request to fingerprint + :type request: scrapy.http.Request + +Additionally, it may also implement the following methods: + +.. classmethod:: from_crawler(cls, crawler) + + If present, this class method is called to create a request fingerprinter + instance from a :class:`~scrapy.crawler.Crawler` object. It must return a + new instance of the request fingerprinter. + + *crawler* provides access to all Scrapy core components like settings and + signals; it is a way for the request fingerprinter to access them and hook + its functionality into Scrapy. + + :param crawler: crawler that uses this request fingerprinter + :type crawler: :class:`~scrapy.crawler.Crawler` object + +.. classmethod:: from_settings(cls, settings) + + If present, and ``from_crawler`` is not defined, this class method is called + to create a request fingerprinter instance from a + :class:`~scrapy.settings.Settings` object. It must return a new instance of + the request fingerprinter. + +The ``fingerprint`` method of the default request fingerprinter, +:class:`scrapy.utils.request.RequestFingerprinter`, uses +:func:`scrapy.utils.request.fingerprint` with its default parameters. For some +common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well +in your ``fingerprint`` method implementation: + +.. autofunction:: scrapy.utils.request.fingerprint + +For example, to take the value of a request header named ``X-ID`` into +account:: + + # my_project/settings.py + REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter' + + # my_project/utils.py + from scrapy.utils.request import fingerprint + + class RequestFingerprinter: + + def fingerprint(self, request): + return fingerprint(request, include_headers=['X-ID']) + +You can also write your own fingerprinting logic from scratch. + +However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure +you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints: + +- Caching saves CPU by ensuring that fingerprints are calculated only once + per request, and not once per Scrapy component that needs the fingerprint + of a request. + +- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that + request objects do not stay in memory forever just because you have + references to them in your cache dictionary. + +For example, to take into account only the URL of a request, without any prior +URL canonicalization or taking the request method or body into account:: + + from hashlib import sha1 + from weakref import WeakKeyDictionary + + from scrapy.utils.python import to_bytes + + class RequestFingerprinter: + + cache = WeakKeyDictionary() + + def fingerprint(self, request): + if request not in self.cache: + fp = sha1() + fp.update(to_bytes(request.url)) + self.cache[request] = fp.digest() + return self.cache[request] + +If you need to be able to override the request fingerprinting for arbitrary +requests from your spider callbacks, you may implement a request fingerprinter +that reads fingerprints from :attr:`request.meta ` +when available, and then falls back to +:func:`~scrapy.utils.request.fingerprint`. For example:: + + from scrapy.utils.request import fingerprint + + class RequestFingerprinter: + + def fingerprint(self, request): + if 'fingerprint' in request.meta: + return request.meta['fingerprint'] + return fingerprint(request) + +If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION +without using the deprecated ``'PREVIOUS_VERSION'`` value of the +:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following +request fingerprinter:: + + from hashlib import sha1 + from weakref import WeakKeyDictionary + + from scrapy.utils.python import to_bytes + from w3lib.url import canonicalize_url + + class RequestFingerprinter: + + cache = WeakKeyDictionary() + + def fingerprint(self, request): + if request not in self.cache: + fp = sha1() + fp.update(to_bytes(request.method)) + fp.update(to_bytes(canonicalize_url(request.url))) + fp.update(request.body or b'') + self.cache[request] = fp.digest() + return self.cache[request] + + +.. _request-fingerprint-restrictions: + +Request fingerprint restrictions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Scrapy components that use request fingerprints may impose additional +restrictions on the format of the fingerprints that your :ref:`request +fingerprinter ` generates. + +The following built-in Scrapy components have such restrictions: + +- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default + value of :setting:`HTTPCACHE_STORAGE`) + + Request fingerprints must be at least 1 byte long. + + Path and filename length limits of the file system of + :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`, + the following directory structure is created: + + - :attr:`Spider.name ` + + - first byte of a request fingerprint as hexadecimal + + - fingerprint as hexadecimal + + - filenames up to 16 characters long + + For example, if a request fingerprint is made of 20 bytes (default), + :setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``, + and the name of your spider is ``'my_spider'`` your file system must + support a file path like:: + + /home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers + +- :class:`scrapy.extensions.httpcache.DbmCacheStorage` + + The underlying DBM implementation must support keys as long as twice + the number of bytes of a request fingerprint, plus 5. For example, + if a request fingerprint is made of 20 bytes (default), + 45-character-long keys must be supported. + + .. _topics-request-meta: Request.meta special keys diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4e105642d..2046c6446 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -825,12 +825,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'`` The class used to detect and filter duplicate requests. -The default (``RFPDupeFilter``) filters based on request fingerprint using -the ``scrapy.utils.request.request_fingerprint`` function. In order to change -the way duplicates are checked you could subclass ``RFPDupeFilter`` and -override its ``request_fingerprint`` method. This method should accept -scrapy :class:`~scrapy.Request` object and return its fingerprint -(a string). +The default (``RFPDupeFilter``) filters based on the +:setting:`REQUEST_FINGERPRINTER_CLASS` setting. You can disable filtering of duplicate requests by setting :setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index a638254f1..fdca7b335 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -51,6 +51,7 @@ class Crawler: self.spidercls.update_settings(self.settings) self.signals = SignalManager(self) + self.stats = load_object(self.settings['STATS_CLASS'])(self) handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL')) @@ -71,6 +72,12 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) + self.request_fingerprinter = create_instance( + load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']), + settings=self.settings, + crawler=self, + ) + reactor_class = self.settings.get("TWISTED_REACTOR") if init_reactor: # this needs to be done after the spider settings are merged, diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 292c68099..d1b0559ef 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,14 +1,16 @@ import logging import os from typing import Optional, Set, Type, TypeVar +from warnings import warn from twisted.internet.defer import Deferred from scrapy.http.request import Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.job import job_dir -from scrapy.utils.request import referer_str, request_fingerprint +from scrapy.utils.request import referer_str, RequestFingerprinter BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter") @@ -39,8 +41,15 @@ RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter") class RFPDupeFilter(BaseDupeFilter): """Request Fingerprint duplicates filter""" - def __init__(self, path: Optional[str] = None, debug: bool = False) -> None: + def __init__( + self, + path: Optional[str] = None, + debug: bool = False, + *, + fingerprinter=None, + ) -> None: self.file = None + self.fingerprinter = fingerprinter or RequestFingerprinter() self.fingerprints: Set[str] = set() self.logdupes = True self.debug = debug @@ -51,9 +60,39 @@ class RFPDupeFilter(BaseDupeFilter): self.fingerprints.update(x.rstrip() for x in self.file) @classmethod - def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV: + def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV: debug = settings.getbool('DUPEFILTER_DEBUG') - return cls(job_dir(settings), debug) + try: + return cls(job_dir(settings), debug, fingerprinter=fingerprinter) + except TypeError: + warn( + "RFPDupeFilter subclasses must either modify their '__init__' " + "method to support a 'fingerprinter' parameter or reimplement " + "the 'from_settings' class method.", + ScrapyDeprecationWarning, + ) + result = cls(job_dir(settings), debug) + result.fingerprinter = fingerprinter + return result + + @classmethod + def from_crawler(cls, crawler): + try: + return cls.from_settings( + crawler.settings, + fingerprinter=crawler.request_fingerprinter, + ) + except TypeError: + warn( + "RFPDupeFilter subclasses must either modify their overridden " + "'__init__' method and 'from_settings' class method to " + "support a 'fingerprinter' parameter, or reimplement the " + "'from_crawler' class method.", + ScrapyDeprecationWarning, + ) + result = cls.from_settings(crawler.settings) + result.fingerprinter = crawler.request_fingerprinter + return result def request_seen(self, request: Request) -> bool: fp = self.request_fingerprint(request) @@ -65,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter): return False def request_fingerprint(self, request: Request) -> str: - return request_fingerprint(request) + return self.fingerprinter.fingerprint(request).hex() def close(self, reason: str) -> None: if self.file: diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index d0ae29b90..c71484cfa 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.request import request_fingerprint logger = logging.getLogger(__name__) @@ -228,6 +227,8 @@ class DbmCacheStorage: logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider}) + self._fingerprinter = spider.crawler.request_fingerprinter + def close_spider(self, spider): self.db.close() @@ -244,7 +245,7 @@ class DbmCacheStorage: return response def store_response(self, spider, request, response): - key = self._request_key(request) + key = self._fingerprinter.fingerprint(request).hex() data = { 'status': response.status, 'url': response.url, @@ -255,7 +256,7 @@ class DbmCacheStorage: self.db[f'{key}_time'] = str(time()) def _read_data(self, spider, request): - key = self._request_key(request) + key = self._fingerprinter.fingerprint(request).hex() db = self.db tkey = f'{key}_time' if tkey not in db: @@ -267,9 +268,6 @@ class DbmCacheStorage: return pickle.loads(db[f'{key}_data']) - def _request_key(self, request): - return request_fingerprint(request) - class FilesystemCacheStorage: @@ -283,6 +281,8 @@ class FilesystemCacheStorage: logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir}, extra={'spider': spider}) + self._fingerprinter = spider.crawler.request_fingerprinter + def close_spider(self, spider): pass @@ -329,7 +329,7 @@ class FilesystemCacheStorage: f.write(request.body) def _get_request_path(self, spider, request): - key = request_fingerprint(request) + key = self._fingerprinter.fingerprint(request).hex() return os.path.join(self.cachedir, spider.name, key[0:2], key) def _read_meta(self, spider, request): diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 430c37227..5308a9793 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -11,7 +11,6 @@ from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter from scrapy.utils.log import failure_to_exc_info @@ -77,6 +76,7 @@ class MediaPipeline: except AttributeError: pipe = cls() pipe.crawler = crawler + pipe._fingerprinter = crawler.request_fingerprinter return pipe def open_spider(self, spider): @@ -90,7 +90,7 @@ class MediaPipeline: return dfd.addCallback(self.item_completed, item, info) def _process_request(self, request, info, item): - fp = request_fingerprint(request) + fp = self._fingerprinter.fingerprint(request) cb = request.callback or (lambda _: _) eb = request.errback request.callback = None diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8389a70cb..f5a3efe69 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -246,6 +246,9 @@ REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' +REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter' +REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION' + RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index a414b5fde..5e541e2c0 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -86,3 +86,6 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_DIR = 'httpcache' #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' + +# Set settings whose default value is deprecated to a future-proof value +REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION' diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 70ef3ba2b..cf33317ce 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -4,7 +4,9 @@ scrapy.http.Request objects """ import hashlib -from typing import Dict, Iterable, Optional, Tuple, Union +import json +import warnings +from typing import Dict, Iterable, List, Optional, Tuple, Union from urllib.parse import urlunparse from weakref import WeakKeyDictionary @@ -12,13 +14,22 @@ from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url from scrapy import Request, Spider +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode -_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" -_fingerprint_cache = WeakKeyDictionary() +_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" +_deprecated_fingerprint_cache = WeakKeyDictionary() + + +def _serialize_headers(headers, request): + for header in headers: + if header in request.headers: + yield header + for value in request.headers.getlist(header): + yield value def request_fingerprint( @@ -26,6 +37,123 @@ def request_fingerprint( include_headers: Optional[Iterable[Union[bytes, str]]] = None, keep_fragments: bool = False, ) -> str: + """ + Return the request fingerprint as an hexadecimal string. + + The request fingerprint is a hash that uniquely identifies the resource the + request points to. For example, take the following two urls: + + http://www.example.com/query?id=111&cat=222 + http://www.example.com/query?cat=222&id=111 + + Even though those are two different URLs both point to the same resource + and are equivalent (i.e. they should return the same response). + + Another example are cookies used to store session ids. Suppose the + following page is only accessible to authenticated users: + + http://www.example.com/members/offers.html + + Lots of sites use a cookie to store the session id, which adds a random + component to the HTTP Request and thus should be ignored when calculating + the fingerprint. + + For this reason, request headers are ignored by default when calculating + the fingerprint. If you want to include specific headers use the + include_headers argument, which is a list of Request headers to include. + + Also, servers usually ignore fragments in urls when handling requests, + so they are also ignored by default when calculating the fingerprint. + If you want to include them, set the keep_fragments argument to True + (for instance when handling requests with a headless browser). + """ + if include_headers or keep_fragments: + message = ( + 'Call to deprecated function ' + 'scrapy.utils.request.request_fingerprint().\n' + '\n' + 'If you are using this function in a Scrapy component because you ' + 'need a non-default fingerprinting algorithm, and you are OK ' + 'with that non-default fingerprinting algorithm being used by ' + 'all Scrapy components and not just the one calling this ' + 'function, use crawler.request_fingerprinter.fingerprint() ' + 'instead in your Scrapy component (you can get the crawler ' + 'object from the \'from_crawler\' class method), and use the ' + '\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your ' + 'non-default fingerprinting algorithm.\n' + '\n' + 'Otherwise, consider using the ' + 'scrapy.utils.request.fingerprint() function instead.\n' + '\n' + 'If you switch to \'fingerprint()\', or assign the ' + '\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses ' + '\'fingerprint()\', the generated fingerprints will not only be ' + 'bytes instead of a string, but they will also be different from ' + 'those generated by \'request_fingerprint()\'. Before you switch, ' + 'make sure that you understand the consequences of this (e.g. ' + 'cache invalidation) and are OK with them; otherwise, consider ' + 'implementing your own function which returns the same ' + 'fingerprints as the deprecated \'request_fingerprint()\' function.' + ) + else: + message = ( + 'Call to deprecated function ' + 'scrapy.utils.request.request_fingerprint().\n' + '\n' + 'If you are using this function in a Scrapy component, and you ' + 'are OK with users of your component changing the fingerprinting ' + 'algorithm through settings, use ' + 'crawler.request_fingerprinter.fingerprint() instead in your ' + 'Scrapy component (you can get the crawler object from the ' + '\'from_crawler\' class method).\n' + '\n' + 'Otherwise, consider using the ' + 'scrapy.utils.request.fingerprint() function instead.\n' + '\n' + 'Either way, the resulting fingerprints will be returned as ' + 'bytes, not as a string, and they will also be different from ' + 'those generated by \'request_fingerprint()\'. Before you switch, ' + 'make sure that you understand the consequences of this (e.g. ' + 'cache invalidation) and are OK with them; otherwise, consider ' + 'implementing your own function which returns the same ' + 'fingerprints as the deprecated \'request_fingerprint()\' function.' + ) + warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) + processed_include_headers: Optional[Tuple[bytes, ...]] = None + if include_headers: + processed_include_headers = tuple( + to_bytes(h.lower()) for h in sorted(include_headers) + ) + cache = _deprecated_fingerprint_cache.setdefault(request, {}) + cache_key = (processed_include_headers, keep_fragments) + if cache_key not in cache: + fp = hashlib.sha1() + fp.update(to_bytes(request.method)) + fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) + fp.update(request.body or b'') + if processed_include_headers: + for part in _serialize_headers(processed_include_headers, request): + fp.update(part) + cache[cache_key] = fp.hexdigest() + return cache[cache_key] + + +def _request_fingerprint_as_bytes(*args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return bytes.fromhex(request_fingerprint(*args, **kwargs)) + + +_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" +_fingerprint_cache = WeakKeyDictionary() + + +def fingerprint( + request: Request, + *, + include_headers: Optional[Iterable[Union[bytes, str]]] = None, + keep_fragments: bool = False, +) -> bytes: """ Return the request fingerprint. @@ -43,7 +171,7 @@ def request_fingerprint( http://www.example.com/members/offers.html - Lot of sites use a cookie to store the session id, which adds a random + Lots of sites use a cookie to store the session id, which adds a random component to the HTTP Request and thus should be ignored when calculating the fingerprint. @@ -55,29 +183,96 @@ def request_fingerprint( so they are also ignored by default when calculating the fingerprint. If you want to include them, set the keep_fragments argument to True (for instance when handling requests with a headless browser). - """ - headers: Optional[Tuple[bytes, ...]] = None + processed_include_headers: Optional[Tuple[bytes, ...]] = None if include_headers: - headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) + processed_include_headers = tuple( + to_bytes(h.lower()) for h in sorted(include_headers) + ) cache = _fingerprint_cache.setdefault(request, {}) - cache_key = (headers, keep_fragments) + cache_key = (processed_include_headers, keep_fragments) if cache_key not in cache: - fp = hashlib.sha1() - fp.update(to_bytes(request.method)) - fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) - fp.update(request.body or b'') - if headers: - for hdr in headers: - if hdr in request.headers: - fp.update(hdr) - for v in request.headers.getlist(hdr): - fp.update(v) - cache[cache_key] = fp.hexdigest() + # To decode bytes reliably (JSON does not support bytes), regardless of + # character encoding, we use bytes.hex() + headers: Dict[str, List[str]] = {} + if processed_include_headers: + for header in processed_include_headers: + if header in request.headers: + headers[header.hex()] = [ + header_value.hex() + for header_value in request.headers.getlist(header) + ] + fingerprint_data = { + 'method': to_unicode(request.method), + 'url': canonicalize_url(request.url, keep_fragments=keep_fragments), + 'body': (request.body or b'').hex(), + 'headers': headers, + } + fingerprint_json = json.dumps(fingerprint_data, sort_keys=True) + cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() return cache[cache_key] -def request_authenticate(request: Request, username: str, password: str) -> None: +class RequestFingerprinter: + """Default fingerprinter. + + It takes into account a canonical version + (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url + ` and the values of :attr:`request.method + ` and :attr:`request.body + `. It then generates an `SHA1 + `_ hash. + + .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`. + """ + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def __init__(self, crawler=None): + if crawler: + implementation = crawler.settings.get( + 'REQUEST_FINGERPRINTER_IMPLEMENTATION' + ) + else: + implementation = 'PREVIOUS_VERSION' + if implementation == 'PREVIOUS_VERSION': + message = ( + '\'PREVIOUS_VERSION\' is a deprecated value for the ' + '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n' + '\n' + 'It is also the default value. In other words, it is normal ' + 'to get this warning if you have not defined a value for the ' + '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so ' + 'for backward compatibility reasons, but it will change in a ' + 'future version of Scrapy.\n' + '\n' + 'See the documentation of the ' + '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for ' + 'information on how to handle this deprecation.' + ) + warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) + self._fingerprint = _request_fingerprint_as_bytes + elif implementation == 'VERSION': + self._fingerprint = fingerprint + else: + raise ValueError( + f'Got an invalid value on setting ' + f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': ' + f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) ' + f'and \'VERSION\'.' + ) + + def fingerprint(self, request): + return self._fingerprint(request) + + +def request_authenticate( + request: Request, + username: str, + password: str, +) -> None: """Authenticate the given request (in place) using the HTTP basic access authentication mechanism (RFC 2617) and the given username and password """ diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 24c38283a..b90ea5009 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -54,7 +54,7 @@ def get_ftp_content_and_delete( return "".join(ftp_data) -def get_crawler(spidercls=None, settings_dict=None): +def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. @@ -62,7 +62,12 @@ def get_crawler(spidercls=None, settings_dict=None): from scrapy.crawler import CrawlerRunner from scrapy.spiders import Spider - runner = CrawlerRunner(settings_dict) + # Set by default settings that prevent deprecation warnings. + settings = {} + if prevent_warnings: + settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION' + settings.update(settings_dict or {}) + runner = CrawlerRunner(settings) return runner.create_crawler(spidercls or Spider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 8f6227109..f7aa769e4 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -104,7 +104,8 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { 'LOG_LEVEL': 'INFO', 'LOG_FILE': log_file, - # disable telnet if not available to avoid an extra warning + # settings to avoid extra warnings + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, } diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 680bb6dc8..b7df2554a 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -15,6 +15,16 @@ from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider +def _get_dupefilter(*, crawler=None, settings=None, open=True): + if crawler is None: + crawler = get_crawler(settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + dupefilter = scheduler.df + if open: + dupefilter.open() + return dupefilter + + class FromCrawlerRFPDupeFilter(RFPDupeFilter): @classmethod @@ -64,9 +74,7 @@ class RFPDupeFilterTest(unittest.TestCase): self.assertEqual(scheduler.df.method, 'n/a') def test_filter(self): - dupefilter = RFPDupeFilter() - dupefilter.open() - + dupefilter = _get_dupefilter() r1 = Request('http://scrapytest.org/1') r2 = Request('http://scrapytest.org/2') r3 = Request('http://scrapytest.org/2') @@ -85,7 +93,7 @@ class RFPDupeFilterTest(unittest.TestCase): path = tempfile.mkdtemp() try: - df = RFPDupeFilter(path) + df = _get_dupefilter(settings={'JOBDIR': path}, open=False) try: df.open() assert not df.request_seen(r1) @@ -93,7 +101,8 @@ class RFPDupeFilterTest(unittest.TestCase): finally: df.close('finished') - df2 = RFPDupeFilter(path) + df2 = _get_dupefilter(settings={'JOBDIR': path}, open=False) + assert df != df2 try: df2.open() assert df2.request_seen(r1) @@ -109,26 +118,24 @@ class RFPDupeFilterTest(unittest.TestCase): output of request_seen. """ + dupefilter = _get_dupefilter() r1 = Request('http://scrapytest.org/index.html') r2 = Request('http://scrapytest.org/INDEX.html') - dupefilter = RFPDupeFilter() - dupefilter.open() - assert not dupefilter.request_seen(r1) assert not dupefilter.request_seen(r2) dupefilter.close('finished') - class CaseInsensitiveRFPDupeFilter(RFPDupeFilter): + class RequestFingerprinter: - def request_fingerprint(self, request): + def fingerprint(self, request): fp = hashlib.sha1() fp.update(to_bytes(request.url.lower())) - return fp.hexdigest() + return fp.digest() - case_insensitive_dupefilter = CaseInsensitiveRFPDupeFilter() - case_insensitive_dupefilter.open() + settings = {'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter} + case_insensitive_dupefilter = _get_dupefilter(settings=settings) assert not case_insensitive_dupefilter.request_seen(r1) assert case_insensitive_dupefilter.request_seen(r2) @@ -142,8 +149,10 @@ class RFPDupeFilterTest(unittest.TestCase): r1 = Request('http://scrapytest.org/1') path = tempfile.mkdtemp() + crawler = get_crawler(settings_dict={'JOBDIR': path}) try: - df = RFPDupeFilter(path) + scheduler = Scheduler.from_crawler(crawler) + df = scheduler.df df.open() df.request_seen(r1) df.close('finished') @@ -164,11 +173,8 @@ class RFPDupeFilterTest(unittest.TestCase): settings = {'DUPEFILTER_DEBUG': False, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(SimpleSpider, settings_dict=settings) - scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) - - dupefilter = scheduler.df - dupefilter.open() + dupefilter = _get_dupefilter(crawler=crawler) r1 = Request('http://scrapytest.org/index.html') r2 = Request('http://scrapytest.org/index.html') @@ -193,11 +199,41 @@ class RFPDupeFilterTest(unittest.TestCase): settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(SimpleSpider, settings_dict=settings) - scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) - - dupefilter = scheduler.df - dupefilter.open() + dupefilter = _get_dupefilter(crawler=crawler) + + r1 = Request('http://scrapytest.org/index.html') + r2 = Request('http://scrapytest.org/index.html', + headers={'Referer': 'http://scrapytest.org/INDEX.html'}) + + dupefilter.log(r1, spider) + dupefilter.log(r2, spider) + + assert crawler.stats.get_value('dupefilter/filtered') == 2 + log.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: (referer: None)' + ) + ) + log.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: ' + ' (referer: http://scrapytest.org/INDEX.html)' + ) + ) + + dupefilter.close('finished') + + def test_log_debug_default_dupefilter(self): + with LogCapture() as log: + settings = {'DUPEFILTER_DEBUG': True} + crawler = get_crawler(SimpleSpider, settings_dict=settings) + spider = SimpleSpider.from_crawler(crawler) + dupefilter = _get_dupefilter(crawler=crawler) r1 = Request('http://scrapytest.org/index.html') r2 = Request('http://scrapytest.org/index.html', diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 0ff2045ed..4228173ed 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -25,6 +25,7 @@ from scrapy.pipelines.files import ( from scrapy.settings import Settings from scrapy.utils.test import ( assert_gcs_environ, + get_crawler, get_ftp_content_and_delete, get_gcs_content_and_delete, skip_if_no_boto, @@ -47,7 +48,9 @@ class FilesPipelineTestCase(unittest.TestCase): def setUp(self): self.tempdir = mkdtemp() - self.pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})) + settings_dict = {'FILES_STORE': self.tempdir} + crawler = get_crawler(spidercls=None, settings_dict=settings_dict) + self.pipeline = FilesPipeline.from_crawler(crawler) self.pipeline.download_func = _mocked_download_func self.pipeline.open_spider(None) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index a802c7cf1..84e867660 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -7,17 +7,17 @@ from twisted.python.failure import Failure from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks +from scrapy import signals from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider -from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.request import request_fingerprint +from scrapy.pipelines.files import FileException from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline -from scrapy.pipelines.files import FileException +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all -from scrapy import signals +from scrapy.utils.test import get_crawler try: @@ -39,11 +39,14 @@ class BaseMediaPipelineTestCase(unittest.TestCase): settings = None def setUp(self): - self.spider = Spider('media.com') - self.pipe = self.pipeline_class(download_func=_mocked_download_func, - settings=Settings(self.settings)) + spider_cls = Spider + self.spider = spider_cls('media.com') + crawler = get_crawler(spider_cls, self.settings) + self.pipe = self.pipeline_class.from_crawler(crawler) + self.pipe.download_func = _mocked_download_func self.pipe.open_spider(self.spider) self.info = self.pipe.spiderinfo + self.fingerprint = crawler.request_fingerprinter.fingerprint def tearDown(self): for name, signal in vars(signals).items(): @@ -156,7 +159,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): self.assertEqual(failure.value.__context__, def_gen_return_exc) # Let's calculate the request fingerprint and fake some runtime data... - fp = request_fingerprint(request) + fp = self.fingerprint(request) info = self.pipe.spiderinfo info.downloading.add(fp) info.waiting[fp] = [] @@ -273,7 +276,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=req) # pass a single item new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item - assert request_fingerprint(req) in self.info.downloaded + self.assertIn(self.fingerprint(req), self.info.downloaded) # returns iterable of Requests req1 = Request('http://url1') @@ -281,8 +284,8 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=iter([req1, req2])) new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item - assert request_fingerprint(req1) in self.info.downloaded - assert request_fingerprint(req2) in self.info.downloaded + assert self.fingerprint(req1) in self.info.downloaded + assert self.fingerprint(req2) in self.info.downloaded @inlineCallbacks def test_results_are_cached_across_multiple_items(self): @@ -298,7 +301,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=req2) new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) - self.assertEqual(request_fingerprint(req1), request_fingerprint(req2)) + self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) self.assertEqual(new_item['results'], [(True, rsp1)]) @inlineCallbacks @@ -314,7 +317,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): @inlineCallbacks def test_wait_if_request_is_downloading(self): def _check_downloading(response): - fp = request_fingerprint(req1) + fp = self.fingerprint(req1) self.assertTrue(fp in self.info.downloading) self.assertTrue(fp in self.info.waiting) self.assertTrue(fp not in self.info.downloaded) @@ -351,7 +354,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): def __init__(self, *args, **kwargs): - super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._mockcalled = [] def get_media_requests(self, item, info): @@ -369,19 +372,19 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): def media_to_download(self, request, info): self._mockcalled.append('media_to_download') - return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info) + return super().media_to_download(request, info) def media_downloaded(self, response, request, info): self._mockcalled.append('media_downloaded') - return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info) + return super().media_downloaded(response, request, info) def file_downloaded(self, response, request, info): self._mockcalled.append('file_downloaded') - return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info) + return super().file_downloaded(response, request, info) def file_path(self, request, response=None, info=None): self._mockcalled.append('file_path') - return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info) + return super().file_path(request, response, info) def thumb_path(self, request, thumb_id, response=None, info=None): self._mockcalled.append('thumb_path') @@ -393,18 +396,20 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): def image_downloaded(self, response, request, info): self._mockcalled.append('image_downloaded') - return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info) + return super().image_downloaded(response, request, info) class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): skip = skip_pillow def setUp(self): - self.pipe = MockedMediaPipelineDeprecatedMethods( - store_uri='store-uri', - download_func=_mocked_download_func, - settings=Settings({"IMAGES_THUMBS": {'small': (50, 50)}}) - ) + settings_dict = { + 'IMAGES_STORE': 'store-uri', + 'IMAGES_THUMBS': {'small': (50, 50)}, + } + crawler = get_crawler(spidercls=None, settings_dict=settings_dict) + self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler) + self.pipe.download_func = _mocked_download_func self.pipe.open_spider(None) self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[]) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 7e0049b1d..e9edfee98 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,73 +1,29 @@ import unittest +import warnings +from hashlib import sha1 +from typing import Dict, Mapping, Optional, Tuple, Union +from weakref import WeakKeyDictionary + +import pytest +from w3lib.url import canonicalize_url + from scrapy.http import Request +from scrapy.utils.deprecate import ScrapyDeprecationWarning +from scrapy.utils.python import to_bytes from scrapy.utils.request import ( + _deprecated_fingerprint_cache, _fingerprint_cache, + _request_fingerprint_as_bytes, + fingerprint, request_authenticate, request_fingerprint, request_httprepr, ) +from scrapy.utils.test import get_crawler class UtilsRequestTest(unittest.TestCase): - def test_request_fingerprint(self): - r1 = Request("http://www.example.com/query?id=111&cat=222") - r2 = Request("http://www.example.com/query?cat=222&id=111") - self.assertEqual(request_fingerprint(r1), request_fingerprint(r1)) - self.assertEqual(request_fingerprint(r1), request_fingerprint(r2)) - - r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199') - r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199') - self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2)) - - # make sure caching is working - self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][(None, False)]) - - r1 = Request("http://www.example.com/members/offers.html") - r2 = Request("http://www.example.com/members/offers.html") - r2.headers['SESSIONID'] = b"somehash" - self.assertEqual(request_fingerprint(r1), request_fingerprint(r2)) - - r1 = Request("http://www.example.com/") - r2 = Request("http://www.example.com/") - r2.headers['Accept-Language'] = b'en' - r3 = Request("http://www.example.com/") - r3.headers['Accept-Language'] = b'en' - r3.headers['SESSIONID'] = b"somehash" - - self.assertEqual(request_fingerprint(r1), request_fingerprint(r2), request_fingerprint(r3)) - - self.assertEqual(request_fingerprint(r1), - request_fingerprint(r1, include_headers=['Accept-Language'])) - - self.assertNotEqual( - request_fingerprint(r1), - request_fingerprint(r2, include_headers=['Accept-Language'])) - - self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']), - request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language'])) - - r1 = Request("http://www.example.com/test.html") - r2 = Request("http://www.example.com/test.html#fragment") - self.assertEqual(request_fingerprint(r1), request_fingerprint(r2)) - self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, keep_fragments=True)) - self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r2, keep_fragments=True)) - self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2, keep_fragments=True)) - - r1 = Request("http://www.example.com") - r2 = Request("http://www.example.com", method='POST') - r3 = Request("http://www.example.com", method='POST', body=b'request body') - - self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2)) - self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r3)) - - # cached fingerprint must be cleared on request copy - r1 = Request("http://www.example.com") - fp1 = request_fingerprint(r1) - r2 = r1.replace(url="http://www.example.com/other") - fp2 = request_fingerprint(r2) - self.assertNotEqual(fp1, fp2) - def test_request_authenticate(self): r = Request("http://www.example.com") request_authenticate(r, 'someuser', 'somepass') @@ -93,5 +49,632 @@ class UtilsRequestTest(unittest.TestCase): request_httprepr(Request("ftp://localhost/tmp/foo.txt")) +class FingerprintTest(unittest.TestCase): + maxDiff = None + + function = staticmethod(fingerprint) + cache: Union[ + "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]", + "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]", + ] = _fingerprint_cache + default_cache_key = (None, False) + known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( + ( + Request("http://example.org"), + b'xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD', + {}, + ), + ( + Request("https://example.org"), + b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l', + {}, + ), + ( + Request("https://example.org?a"), + b'G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c', + {}, + ), + ( + Request("https://example.org?a=b"), + b'\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17', + {}, + ), + ( + Request("https://example.org?a=b&a"), + b't+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96', + {}, + ), + ( + Request("https://example.org?a=b&a=c"), + b'\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef', + {}, + ), + ( + Request("https://example.org", method='POST'), + b'\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d', + {}, + ), + ( + Request("https://example.org", body=b'a'), + b'\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_', + {}, + ), + ( + Request("https://example.org", method='POST', body=b'a'), + b'5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7', + {}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l', + {}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + b']\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.', + {'include_headers': ['A']}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + b'<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f', + {'keep_fragments': True}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + b'\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11', + {'include_headers': ['A'], 'keep_fragments': True}, + ), + ( + Request("https://example.org/ab"), + b'N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6', + {}, + ), + ( + Request("https://example.org/a", body=b'b'), + b'_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5', + {}, + ), + ) + + def test_query_string_key_order(self): + r1 = Request("http://www.example.com/query?id=111&cat=222") + r2 = Request("http://www.example.com/query?cat=222&id=111") + self.assertEqual(self.function(r1), self.function(r1)) + self.assertEqual(self.function(r1), self.function(r2)) + + def test_query_string_key_without_value(self): + r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199') + r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199') + self.assertNotEqual(self.function(r1), self.function(r2)) + + def test_caching(self): + r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199') + self.assertEqual( + self.function(r1), + self.cache[r1][self.default_cache_key] + ) + + def test_header(self): + r1 = Request("http://www.example.com/members/offers.html") + r2 = Request("http://www.example.com/members/offers.html") + r2.headers['SESSIONID'] = b"somehash" + self.assertEqual(self.function(r1), self.function(r2)) + + def test_headers(self): + r1 = Request("http://www.example.com/") + r2 = Request("http://www.example.com/") + r2.headers['Accept-Language'] = b'en' + r3 = Request("http://www.example.com/") + r3.headers['Accept-Language'] = b'en' + r3.headers['SESSIONID'] = b"somehash" + + self.assertEqual(self.function(r1), self.function(r2), self.function(r3)) + + self.assertEqual(self.function(r1), + self.function(r1, include_headers=['Accept-Language'])) + + self.assertNotEqual( + self.function(r1), + self.function(r2, include_headers=['Accept-Language'])) + + self.assertEqual(self.function(r3, include_headers=['accept-language', 'sessionid']), + self.function(r3, include_headers=['SESSIONID', 'Accept-Language'])) + + def test_fragment(self): + r1 = Request("http://www.example.com/test.html") + r2 = Request("http://www.example.com/test.html#fragment") + self.assertEqual(self.function(r1), self.function(r2)) + self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True)) + self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True)) + self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True)) + + def test_method_and_body(self): + r1 = Request("http://www.example.com") + r2 = Request("http://www.example.com", method='POST') + r3 = Request("http://www.example.com", method='POST', body=b'request body') + + self.assertNotEqual(self.function(r1), self.function(r2)) + self.assertNotEqual(self.function(r2), self.function(r3)) + + def test_request_replace(self): + # cached fingerprint must be cleared on request copy + r1 = Request("http://www.example.com") + fp1 = self.function(r1) + r2 = r1.replace(url="http://www.example.com/other") + fp2 = self.function(r2) + self.assertNotEqual(fp1, fp2) + + def test_part_separation(self): + # An old implementation used to serialize request data in a way that + # would put the body right after the URL. + r1 = Request("http://www.example.com/foo") + fp1 = self.function(r1) + r2 = Request("http://www.example.com/f", body=b'oo') + fp2 = self.function(r2) + self.assertNotEqual(fp1, fp2) + + def test_hashes(self): + """Test hardcoded hashes, to make sure future changes to not introduce + backward incompatibilities.""" + actual = [ + self.function(request, **kwargs) + for request, _, kwargs in self.known_hashes + ] + expected = [ + _fingerprint + for _, _fingerprint, _ in self.known_hashes + ] + self.assertEqual(actual, expected) + + +class RequestFingerprintTest(FingerprintTest): + function = staticmethod(request_fingerprint) + cache = _deprecated_fingerprint_cache + known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( + ( + Request("http://example.org"), + 'b2e5245ef826fd9576c93bd6e392fce3133fab62', + {}, + ), + ( + Request("https://example.org"), + 'bd10a0a89ea32cdee77917320f1309b0da87e892', + {}, + ), + ( + Request("https://example.org?a"), + '2fb7d48ae02f04b749f40caa969c0bc3c43204ce', + {}, + ), + ( + Request("https://example.org?a=b"), + '42e5fe149b147476e3f67ad0670c57b4cc57856a', + {}, + ), + ( + Request("https://example.org?a=b&a"), + 'd23a9787cb56c6375c2cae4453c5a8c634526942', + {}, + ), + ( + Request("https://example.org?a=b&a=c"), + '9a18a7a8552a9182b7f1e05d33876409e421e5c5', + {}, + ), + ( + Request("https://example.org", method='POST'), + 'ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6', + {}, + ), + ( + Request("https://example.org", body=b'a'), + '4bb136e54e715a4ea7a9dd1101831765d33f2d60', + {}, + ), + ( + Request("https://example.org", method='POST', body=b'a'), + '6c6595374a304b293be762f7b7be3f54e9947c65', + {}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + 'bd10a0a89ea32cdee77917320f1309b0da87e892', + {}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + '515b633cb3ca502a33a9d8c890e889ec1e425e65', + {'include_headers': ['A']}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + '505c96e7da675920dfef58725e8c957dfdb38f47', + {'keep_fragments': True}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + 'd6f673cdcb661b7970c2b9a00ee63e87d1e2e5da', + {'include_headers': ['A'], 'keep_fragments': True}, + ), + ( + Request("https://example.org/ab"), + '4e2870fee58582d6f81755e9b8fdefe3cba0c951', + {}, + ), + ( + Request("https://example.org/a", body=b'b'), + '4e2870fee58582d6f81755e9b8fdefe3cba0c951', + {}, + ), + ) + + @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True) + def test_part_separation(self): + super().test_part_separation() + + def test_deprecation_default_parameters(self): + with pytest.warns(ScrapyDeprecationWarning) as warnings: + self.function(Request("http://www.example.com")) + messages = [str(warning.message) for warning in warnings] + self.assertTrue( + any( + 'Call to deprecated function' in message + for message in messages + ) + ) + self.assertFalse(any('non-default' in message for message in messages)) + + def test_deprecation_non_default_parameters(self): + with pytest.warns(ScrapyDeprecationWarning) as warnings: + self.function(Request("http://www.example.com"), keep_fragments=True) + messages = [str(warning.message) for warning in warnings] + self.assertTrue( + any( + 'Call to deprecated function' in message + for message in messages + ) + ) + self.assertTrue(any('non-default' in message for message in messages)) + + +class RequestFingerprintAsBytesTest(FingerprintTest): + function = staticmethod(_request_fingerprint_as_bytes) + cache = _deprecated_fingerprint_cache + known_hashes = RequestFingerprintTest.known_hashes + + def test_caching(self): + r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199') + self.assertEqual( + self.function(r1), + bytes.fromhex(self.cache[r1][self.default_cache_key]) + ) + + @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True) + def test_part_separation(self): + super().test_part_separation() + + def test_hashes(self): + actual = [ + self.function(request, **kwargs) + for request, _, kwargs in self.known_hashes + ] + expected = [ + bytes.fromhex(_fingerprint) + for _, _fingerprint, _ in self.known_hashes + ] + self.assertEqual(actual, expected) + + +_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary() + + +def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False): + if include_headers: + include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) + cache = _fingerprint_cache_2_6.setdefault(request, {}) + cache_key = (include_headers, keep_fragments) + if cache_key not in cache: + fp = sha1() + fp.update(to_bytes(request.method)) + fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) + fp.update(request.body or b'') + if include_headers: + for hdr in include_headers: + if hdr in request.headers: + fp.update(hdr) + for v in request.headers.getlist(hdr): + fp.update(v) + cache[cache_key] = fp.hexdigest() + return cache[cache_key] + + +REQUEST_OBJECTS_TO_TEST = ( + Request("http://www.example.com/"), + Request("http://www.example.com/query?id=111&cat=222"), + Request("http://www.example.com/query?cat=222&id=111"), + Request('http://www.example.com/hnnoticiaj1.aspx?78132,199'), + Request('http://www.example.com/hnnoticiaj1.aspx?78160,199'), + Request("http://www.example.com/members/offers.html"), + Request( + "http://www.example.com/members/offers.html", + headers={'SESSIONID': b"somehash"}, + ), + Request( + "http://www.example.com/", + headers={'Accept-Language': b"en"}, + ), + Request( + "http://www.example.com/", + headers={ + 'Accept-Language': b"en", + 'SESSIONID': b"somehash", + }, + ), + Request("http://www.example.com/test.html"), + Request("http://www.example.com/test.html#fragment"), + Request("http://www.example.com", method='POST'), + Request("http://www.example.com", method='POST', body=b'request body'), +) + + +class BackwardCompatibilityTestCase(unittest.TestCase): + + def test_function_backward_compatibility(self): + include_headers_to_test = ( + None, + ['Accept-Language'], + ['accept-language', 'sessionid'], + ['SESSIONID', 'Accept-Language'], + ) + for request_object in REQUEST_OBJECTS_TO_TEST: + for include_headers in include_headers_to_test: + for keep_fragments in (False, True): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fp = request_fingerprint( + request_object, + include_headers=include_headers, + keep_fragments=keep_fragments, + ) + old_fp = request_fingerprint_2_6( + request_object, + include_headers=include_headers, + keep_fragments=keep_fragments, + ) + self.assertEqual(fp, old_fp) + + def test_component_backward_compatibility(self): + for request_object in REQUEST_OBJECTS_TO_TEST: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + crawler = get_crawler(prevent_warnings=False) + fp = crawler.request_fingerprinter.fingerprint(request_object) + old_fp = request_fingerprint_2_6(request_object) + self.assertEqual(fp.hex(), old_fp) + + def test_custom_component_backward_compatibility(self): + """Tests that the backward-compatible request fingerprinting class featured + in the documentation is indeed backward compatible and does not cause a + warning to be logged.""" + + class RequestFingerprinter: + + cache = WeakKeyDictionary() + + def fingerprint(self, request): + if request not in self.cache: + fp = sha1() + fp.update(to_bytes(request.method)) + fp.update(to_bytes(canonicalize_url(request.url))) + fp.update(request.body or b'') + self.cache[request] = fp.digest() + return self.cache[request] + + for request_object in REQUEST_OBJECTS_TO_TEST: + with warnings.catch_warnings() as logged_warnings: + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + } + crawler = get_crawler(settings_dict=settings) + fp = crawler.request_fingerprinter.fingerprint(request_object) + old_fp = request_fingerprint_2_6(request_object) + self.assertEqual(fp.hex(), old_fp) + self.assertFalse(logged_warnings) + + +class RequestFingerprinterTestCase(unittest.TestCase): + + def test_default_implementation(self): + with warnings.catch_warnings(record=True) as logged_warnings: + crawler = get_crawler(prevent_warnings=False) + request = Request('https://example.com') + self.assertEqual( + crawler.request_fingerprinter.fingerprint(request), + _request_fingerprint_as_bytes(request), + ) + self.assertTrue(logged_warnings) + + def test_deprecated_implementation(self): + settings = { + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION', + } + with warnings.catch_warnings(record=True) as logged_warnings: + crawler = get_crawler(settings_dict=settings) + request = Request('https://example.com') + self.assertEqual( + crawler.request_fingerprinter.fingerprint(request), + _request_fingerprint_as_bytes(request), + ) + self.assertTrue(logged_warnings) + + def test_recommended_implementation(self): + settings = { + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + } + with warnings.catch_warnings(record=True) as logged_warnings: + crawler = get_crawler(settings_dict=settings) + request = Request('https://example.com') + self.assertEqual( + crawler.request_fingerprinter.fingerprint(request), + fingerprint(request), + ) + self.assertFalse(logged_warnings) + + def test_unknown_implementation(self): + settings = { + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.5', + } + with self.assertRaises(ValueError): + get_crawler(settings_dict=settings) + + +class CustomRequestFingerprinterTestCase(unittest.TestCase): + + def test_include_headers(self): + + class RequestFingerprinter: + + def fingerprint(self, request): + return fingerprint(request, include_headers=['X-ID']) + + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + } + crawler = get_crawler(settings_dict=settings) + + r1 = Request("http://www.example.com", headers={'X-ID': '1'}) + fp1 = crawler.request_fingerprinter.fingerprint(r1) + r2 = Request("http://www.example.com", headers={'X-ID': '2'}) + fp2 = crawler.request_fingerprinter.fingerprint(r2) + self.assertNotEqual(fp1, fp2) + + def test_dont_canonicalize(self): + + class RequestFingerprinter: + cache = WeakKeyDictionary() + + def fingerprint(self, request): + if request not in self.cache: + fp = sha1() + fp.update(to_bytes(request.url)) + self.cache[request] = fp.digest() + return self.cache[request] + + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + } + crawler = get_crawler(settings_dict=settings) + + r1 = Request("http://www.example.com?a=1&a=2") + fp1 = crawler.request_fingerprinter.fingerprint(r1) + r2 = Request("http://www.example.com?a=2&a=1") + fp2 = crawler.request_fingerprinter.fingerprint(r2) + self.assertNotEqual(fp1, fp2) + + def test_meta(self): + + class RequestFingerprinter: + + def fingerprint(self, request): + if 'fingerprint' in request.meta: + return request.meta['fingerprint'] + return fingerprint(request) + + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + } + crawler = get_crawler(settings_dict=settings) + + r1 = Request("http://www.example.com") + fp1 = crawler.request_fingerprinter.fingerprint(r1) + r2 = Request("http://www.example.com", meta={'fingerprint': 'a'}) + fp2 = crawler.request_fingerprinter.fingerprint(r2) + r3 = Request("http://www.example.com", meta={'fingerprint': 'a'}) + fp3 = crawler.request_fingerprinter.fingerprint(r3) + r4 = Request("http://www.example.com", meta={'fingerprint': 'b'}) + fp4 = crawler.request_fingerprinter.fingerprint(r4) + self.assertNotEqual(fp1, fp2) + self.assertNotEqual(fp1, fp4) + self.assertNotEqual(fp2, fp4) + self.assertEqual(fp2, fp3) + + def test_from_crawler(self): + + class RequestFingerprinter: + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def __init__(self, crawler): + self._fingerprint = crawler.settings['FINGERPRINT'] + + def fingerprint(self, request): + return self._fingerprint + + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + 'FINGERPRINT': b'fingerprint', + } + crawler = get_crawler(settings_dict=settings) + + request = Request("http://www.example.com") + fingerprint = crawler.request_fingerprinter.fingerprint(request) + self.assertEqual(fingerprint, settings['FINGERPRINT']) + + def test_from_settings(self): + + class RequestFingerprinter: + + @classmethod + def from_settings(cls, settings): + return cls(settings) + + def __init__(self, settings): + self._fingerprint = settings['FINGERPRINT'] + + def fingerprint(self, request): + return self._fingerprint + + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + 'FINGERPRINT': b'fingerprint', + } + crawler = get_crawler(settings_dict=settings) + + request = Request("http://www.example.com") + fingerprint = crawler.request_fingerprinter.fingerprint(request) + self.assertEqual(fingerprint, settings['FINGERPRINT']) + + def test_from_crawler_and_settings(self): + + class RequestFingerprinter: + + # This method is ignored due to the presence of from_crawler + @classmethod + def from_settings(cls, settings): + return cls(settings) + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def __init__(self, crawler): + self._fingerprint = crawler.settings['FINGERPRINT'] + + def fingerprint(self, request): + return self._fingerprint + + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + 'FINGERPRINT': b'fingerprint', + } + crawler = get_crawler(settings_dict=settings) + + request = Request("http://www.example.com") + fingerprint = crawler.request_fingerprinter.fingerprint(request) + self.assertEqual(fingerprint, settings['FINGERPRINT']) + + if __name__ == "__main__": unittest.main() From 407562b38b6ab375ae650c8799bdd511025527f4 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 9 Jun 2022 00:25:03 -0300 Subject: [PATCH 0542/2083] Drop Python 3.6 support (#5514) * chore: Drop Python 3.6 support * Attend PR comments * Tweak versions * Update dependencies version * fix: Ubuntu workflow * fix windows workflow * chore: Remove comment * update `install_requires` dependencies versions * move lxml to main pinned requirements * Attend code-review comments * remove non-pinned 3.7 from windows workflow * simplify condition * lint * remove paragraph * refactor * remove leftover --- .github/workflows/checks.yml | 2 +- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 11 ++++------- .github/workflows/tests-windows.yml | 5 +---- README.rst | 2 +- docs/contributing.rst | 10 +++++----- docs/intro/install.rst | 4 ++-- docs/topics/items.rst | 5 ----- docs/topics/media-pipeline.rst | 2 +- scrapy/__init__.py | 4 ++-- scrapy/utils/py36.py | 11 ----------- setup.py | 19 ++++++------------- tests/requirements.txt | 4 +--- tests/test_utils_python.py | 8 +------- tox.ini | 23 ++++++++--------------- 15 files changed, 34 insertions(+), 78 deletions(-) delete mode 100644 scrapy/utils/py36.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 98fa44c7f..b26f344ff 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,7 +19,7 @@ jobs: - python-version: 3.8 env: TOXENV: pylint - - python-version: 3.6 + - python-version: 3.7 env: TOXENV: typing - python-version: "3.10" # Keep in sync with .readthedocs.yml diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 3aaf688c7..7819a4e12 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 1fc8d914b..be40c7c71 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -8,9 +8,6 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.7 - env: - TOXENV: py - python-version: 3.8 env: TOXENV: py @@ -26,19 +23,19 @@ jobs: - python-version: pypy3 env: TOXENV: pypy3 - PYPY_VERSION: 3.6-v7.3.3 + PYPY_VERSION: 3.9-v7.3.9 # pinned deps - - python-version: 3.6.12 + - python-version: 3.7.13 env: TOXENV: pinned - - python-version: 3.6.12 + - python-version: 3.7.13 env: TOXENV: asyncio-pinned - python-version: pypy3 env: TOXENV: pypy3-pinned - PYPY_VERSION: 3.6-v7.2.0 + PYPY_VERSION: 3.7-v7.3.5 # extras # extra-deps includes reppy, which does not support Python 3.9 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index ab7385118..955b9b449 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -8,12 +8,9 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.6 - env: - TOXENV: windows-pinned - python-version: 3.7 env: - TOXENV: py + TOXENV: windows-pinned - python-version: 3.8 env: TOXENV: py diff --git a/README.rst b/README.rst index 6b563d638..b543a30f4 100644 --- a/README.rst +++ b/README.rst @@ -57,7 +57,7 @@ including a list of features. Requirements ============ -* Python 3.6+ +* Python 3.7+ * Works on Linux, Windows, macOS, BSD Install diff --git a/docs/contributing.rst b/docs/contributing.rst index 4d2580a6c..946bdc23e 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use: To run the tests on a specific :doc:`tox ` environment, use ``-e `` with an environment name from ``tox.ini``. For example, to run -the tests with Python 3.6 use:: +the tests with Python 3.7 use:: - tox -e py36 + tox -e py37 You can also specify a comma-separated list of environments, and use :ref:`tox’s parallel mode ` to run the tests on multiple environments in parallel:: - tox -e py36,py38 -p auto + tox -e py37,py38 -p auto To pass command-line options to :doc:`pytest `, add them after ``--`` in your call to :doc:`tox `. Using ``--`` overrides the @@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well:: tox -- scrapy tests -x # stop after first failure You can also use the `pytest-xdist`_ plugin. For example, to run all tests on -the Python 3.6 :doc:`tox ` environment using all your CPU cores:: +the Python 3.7 :doc:`tox ` environment using all your CPU cores:: - tox -e py36 -- scrapy tests -n auto + tox -e py37 -- scrapy tests -n auto To see coverage report install :doc:`coverage ` (``pip install coverage``) and run: diff --git a/docs/intro/install.rst b/docs/intro/install.rst index b8d3a16bc..1f01c068d 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,8 +9,8 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.6+, either the CPython implementation (default) or -the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). +Scrapy requires Python 3.7+, either the CPython implementation (default) or +the PyPy 7.3.5+ implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 7cd482d07..167014381 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -102,11 +102,6 @@ Additionally, ``dataclass`` items also allow to: * define custom field metadata through :func:`dataclasses.field`, which can be used to :ref:`customize serialization `. -They work natively in Python 3.7 or later, or using the `dataclasses -backport`_ in Python 3.6. - -.. _dataclasses backport: https://pypi.org/project/dataclasses/ - Example:: from dataclasses import dataclass diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 2513faae2..0925e6bb5 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for +The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for thumbnailing and normalizing images to JPEG/RGB format. .. _Pillow: https://github.com/python-pillow/Pillow diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 396f98219..86e584396 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version -if sys.version_info < (3, 6): - print(f"Scrapy {__version__} requires Python 3.6+") +if sys.version_info < (3, 7): + print(f"Scrapy {__version__} requires Python 3.7+") sys.exit(1) diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py deleted file mode 100644 index 653e2bbbb..000000000 --- a/scrapy/utils/py36.py +++ /dev/null @@ -1,11 +0,0 @@ -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.asyncgen import collect_asyncgen # noqa: F401 - - -warnings.warn( - "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, -) diff --git a/setup.py b/setup.py index d86c0f285..ed197273f 100644 --- a/setup.py +++ b/setup.py @@ -19,35 +19,29 @@ def has_environment_marker_platform_impl_support(): install_requires = [ - 'Twisted>=17.9.0', - 'cryptography>=2.0', + 'Twisted>=18.9.0', + 'cryptography>=2.8', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'pyOpenSSL>=16.2.0', + 'pyOpenSSL>=19.1.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', 'w3lib>=1.17.0', - 'zope.interface>=4.1.3', + 'zope.interface>=5.1.0', 'protego>=0.1.15', 'itemadapter>=0.1.0', 'setuptools', 'tldextract', + 'lxml>=4.3.0', ] extras_require = {} cpython_dependencies = [ - 'lxml>=3.5.0', 'PyDispatcher>=2.0.5', ] if has_environment_marker_platform_impl_support(): extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies extras_require[':platform_python_implementation == "PyPy"'] = [ - # Earlier lxml versions are affected by - # https://foss.heptapod.net/pypy/pypy/-/issues/2498, - # which was fixed in Cython 0.26, released on 2017-06-19, and used to - # generate the C headers of lxml release tarballs published since then, the - # first of which was: - 'lxml>=4.0.0', 'PyPyDispatcher>=2.1.0', ] else: @@ -84,7 +78,6 @@ setup( 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', @@ -95,7 +88,7 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=3.6', + python_requires='>=3.7', install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/requirements.txt b/tests/requirements.txt index d2a8aae1b..d9373dfa8 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,14 +1,12 @@ # Tests requirements attrs -dataclasses; python_version == '3.6' pyftpdlib pytest pytest-cov==3.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures -uvloop < 0.15.0; platform_system != "Windows" and python_version == '3.6' -uvloop; platform_system != "Windows" and python_version > '3.6' +uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 4b3964154..7dec5624a 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -3,7 +3,6 @@ import gc import operator import platform import unittest -from datetime import datetime from itertools import count from warnings import catch_warnings, filterwarnings @@ -224,12 +223,7 @@ class UtilsPythonTestCase(unittest.TestCase): elif platform.python_implementation() == 'PyPy': self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj']) - - build_date = datetime.strptime(platform.python_build()[1], '%b %d %Y') - if build_date >= datetime(2020, 4, 7): # PyPy 3.6-v7.3.1 - self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable']) - else: - self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) + self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable']) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) diff --git a/tox.ini b/tox.ini index 6951b6d16..ab8a715c2 100644 --- a/tox.ini +++ b/tox.ini @@ -11,15 +11,13 @@ minversion = 1.7.0 deps = -rtests/requirements.txt # mitmproxy does not support PyPy - # mitmproxy does not support Windows when running Python < 3.7 # Python 3.9+ requires mitmproxy >= 5.3.0 # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 - mitmproxy >= 4.0.4, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' - mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' + mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) - markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy' + markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy' # Extras botocore>=1.4.87 passenv = @@ -44,7 +42,6 @@ deps = types-pyOpenSSL==20.0.3 types-setuptools==57.0.0 commands = - pip install types-dataclasses # remove once py36 support is dropped mypy --show-error-codes {posargs: scrapy tests} [testenv:security] @@ -75,18 +72,19 @@ commands = [pinned] deps = - cryptography==2.0 + cryptography==2.8 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - pyOpenSSL==16.2.0 + pyOpenSSL==19.1.0 queuelib==1.4.2 service_identity==16.0.0 - Twisted[http2]==17.9.0 + Twisted[http2]==18.9.0 w3lib==1.17.0 - zope.interface==4.1.3 + zope.interface==5.1.0 + lxml==4.3.0 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies @@ -95,7 +93,7 @@ deps = # Extras botocore==1.4.87 google-cloud-storage==1.29.0 - Pillow==4.0.0 + Pillow==7.1.0 setenv = _SCRAPY_PINNED=true install_command = @@ -104,7 +102,6 @@ install_command = [testenv:pinned] deps = {[pinned]deps} - lxml==3.5.0 PyDispatcher==2.0.5 install_command = {[pinned]install_command} setenv = @@ -114,9 +111,6 @@ setenv = basepython = python3 deps = {[pinned]deps} - # First lxml version that includes a Windows wheel for Python 3.6, so we do - # not need to build lxml from sources in a CI Windows job: - lxml==3.8.0 PyDispatcher==2.0.5 install_command = {[pinned]install_command} setenv = @@ -155,7 +149,6 @@ commands = basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} - lxml==4.0.0 PyPyDispatcher==2.1.0 commands = {[testenv:pypy3]commands} install_command = {[pinned]install_command} From 2e6721fd86e3bd00301f8cd3ceb4175b2f395017 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 9 Jun 2022 08:37:01 -0300 Subject: [PATCH 0543/2083] docs: Update minimal versions that Scrapy is tested against --- docs/intro/install.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 1f01c068d..23c3af74b 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -54,9 +54,9 @@ Scrapy is written in pure Python and depends on a few key Python packages (among The minimal versions which Scrapy is tested against are: -* Twisted 14.0 -* lxml 3.4 -* pyOpenSSL 0.14 +* Twisted 18.9.0 +* lxml 4.3.0 +* pyOpenSSL 19.1.0 Scrapy may work with older versions of these packages but it is not guaranteed it will continue working From 6770d1ec62012fcfe8a36fdebeeb89cb5157c2df Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 9 Jun 2022 09:08:09 -0300 Subject: [PATCH 0544/2083] chore(tests): Remove validations for unsupported modules versions --- tests/test_downloadermiddleware.py | 20 +------------------- tests/test_utils_signal.py | 20 -------------------- tests/test_webclient.py | 5 ----- 3 files changed, 1 insertion(+), 44 deletions(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index b538a0ed3..38be915f2 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,13 +1,11 @@ import asyncio -from unittest import mock, SkipTest +from unittest import mock from pytest import mark -from twisted import version as twisted_version from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial.unittest import TestCase from twisted.python.failure import Failure -from twisted.python.versions import Version from scrapy.http import Request, Response from scrapy.spiders import Spider @@ -218,16 +216,6 @@ class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): - if ( - self.reactor_pytest == 'asyncio' - and twisted_version < Version('twisted', 18, 4, 0) - ): - raise SkipTest( - 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' - 'hangs when using AsyncIO and Twisted versions lower than ' - '18.4.0' - ) - resp = Response('http://example.com/index.html') class CoroMiddleware: @@ -248,12 +236,6 @@ class MiddlewareUsingCoro(ManagerTestCase): @mark.only_asyncio() def test_asyncdef_asyncio(self): - if twisted_version < Version('twisted', 18, 4, 0): - raise SkipTest( - 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' - 'hangs when using Twisted versions lower than 18.4.0' - ) - resp = Response('http://example.com/index.html') class CoroMiddleware: diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index ad7394232..a36e7bc97 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,13 +1,10 @@ import asyncio -from unittest import SkipTest from pydispatch import dispatcher from pytest import mark from testfixtures import LogCapture -from twisted import version as twisted_version from twisted.internet import defer, reactor from twisted.python.failure import Failure -from twisted.python.versions import Version from twisted.trial import unittest from scrapy.utils.signal import send_catch_log, send_catch_log_deferred @@ -81,16 +78,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): return "OK" def test_send_catch_log(self): - if ( - self.reactor_pytest == 'asyncio' - and twisted_version < Version('twisted', 18, 4, 0) - ): - raise SkipTest( - 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' - 'fails due to a timeout when using AsyncIO and Twisted ' - 'versions lower than 18.4.0' - ) - return super().test_send_catch_log() @@ -104,13 +91,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): return await get_from_asyncio_queue("OK") def test_send_catch_log(self): - if twisted_version < Version('twisted', 18, 4, 0): - raise SkipTest( - 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' - 'fails due to a timeout when using Twisted versions lower ' - 'than 18.4.0' - ) - return super().test_send_catch_log() diff --git a/tests/test_webclient.py b/tests/test_webclient.py index a6d55cb38..0d5827339 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -4,10 +4,7 @@ Tests borrowed from the twisted.web.client tests. """ import os import shutil -import sys -from pkg_resources import parse_version -import cryptography import OpenSSL.SSL from twisted.trial import unittest from twisted.web import server, static, util, resource @@ -417,8 +414,6 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): ).addCallback(self.assertEqual, to_bytes(s)) def testPayloadDisabledCipher(self): - if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"): - self.skipTest("This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1") s = "0123456789" * 10 settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) From c4c5c9f25841a783aab2c682125f3200d6c6e446 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 9 Jun 2022 10:00:44 -0300 Subject: [PATCH 0545/2083] docs: Remove minimal versions paragraphs --- docs/intro/install.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 23c3af74b..c1fd6d522 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -52,12 +52,6 @@ Scrapy is written in pure Python and depends on a few key Python packages (among * `twisted`_, an asynchronous networking framework * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs -The minimal versions which Scrapy is tested against are: - -* Twisted 18.9.0 -* lxml 4.3.0 -* pyOpenSSL 19.1.0 - Scrapy may work with older versions of these packages but it is not guaranteed it will continue working because it’s not being tested against them. From 197aca2c94201f9944404f30fc4a002309cad99b Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 9 Jun 2022 10:11:49 -0300 Subject: [PATCH 0546/2083] docs: Remove leftover --- docs/intro/install.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index c1fd6d522..80a9c16d6 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -52,10 +52,6 @@ Scrapy is written in pure Python and depends on a few key Python packages (among * `twisted`_, an asynchronous networking framework * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs -Scrapy may work with older versions of these packages -but it is not guaranteed it will continue working -because it’s not being tested against them. - Some of these packages themselves depends on non-Python packages that might require additional installation steps depending on your platform. Please check :ref:`platform-specific guides below `. From ddfd192b704dddfefe2dd78345de239995a40159 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sat, 11 Jun 2022 23:51:34 +0430 Subject: [PATCH 0547/2083] add tests for multiple headers with same name --- tests/test_http_headers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 64ff7a73d..0c51fd701 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -38,6 +38,13 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2']) assert h.getlist('X-Forwarded-For') is not hlist + def test_multivalue_for_one_header(self): + h = Headers((("a", "b"), ("a", "c"))) + self.assertEqual(h["a"], b"c") + self.assertEqual(h.get("a"), b"c") + self.assertEqual(h.getlist("a"), [b"b", b"c"]) + assert h.getlist("a") is not ["b", "c"] + def test_encode_utf8(self): h = Headers({'key': '\xa3'}, encoding='utf-8') key, val = dict(h).popitem() From 6a0bcf97cc6016cb966b92170709bc7518cf62c2 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sat, 11 Jun 2022 23:52:21 +0430 Subject: [PATCH 0548/2083] Merge values of multiple headers with same name (#5515) --- scrapy/http/headers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 1a2b99b0a..a9471d721 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,6 +1,7 @@ from w3lib.http import headers_dict_to_raw from scrapy.utils.datatypes import CaselessDict from scrapy.utils.python import to_unicode +from collections.abc import Mapping class Headers(CaselessDict): @@ -10,6 +11,13 @@ class Headers(CaselessDict): self.encoding = encoding super().__init__(seq) + def update(self, seq): + seq = seq.items() if isinstance(seq, Mapping) else seq + iseq = {} + for k, v in seq: + iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) + super().update(iseq) + def normkey(self, key): """Normalize key to bytes""" return self._tobytes(key.title()) @@ -86,4 +94,5 @@ class Headers(CaselessDict): def __copy__(self): return self.__class__(self) + copy = __copy__ From a135d6caf050f4b7b5af28d4cccc5d5ef51dbaf6 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Mon, 13 Jun 2022 15:24:30 +0430 Subject: [PATCH 0549/2083] Move Mapping import line up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/http/headers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index a9471d721..9c03fe54f 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,7 +1,8 @@ +from collections.abc import Mapping + from w3lib.http import headers_dict_to_raw from scrapy.utils.datatypes import CaselessDict from scrapy.utils.python import to_unicode -from collections.abc import Mapping class Headers(CaselessDict): From 892c2a46554bdf80d49d3f28cc012c49cd1e19ca Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Mon, 13 Jun 2022 23:46:42 +0430 Subject: [PATCH 0550/2083] delete unnecessary test --- tests/test_http_headers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 0c51fd701..1ca936247 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -43,7 +43,6 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h["a"], b"c") self.assertEqual(h.get("a"), b"c") self.assertEqual(h.getlist("a"), [b"b", b"c"]) - assert h.getlist("a") is not ["b", "c"] def test_encode_utf8(self): h = Headers({'key': '\xa3'}, encoding='utf-8') From 9e265a2c1f6bccb551e8292785e09462594b8402 Mon Sep 17 00:00:00 2001 From: Kromitvs <74136201+Kromitvs@users.noreply.github.com> Date: Thu, 16 Jun 2022 19:52:19 +0100 Subject: [PATCH 0551/2083] Mind body to choose response class in cache, FTP and HTTP/1.0 (#4873) --- scrapy/core/downloader/handlers/ftp.py | 6 +-- scrapy/core/downloader/webclient.py | 2 +- scrapy/extensions/httpcache.py | 4 +- scrapy/responsetypes.py | 10 ++-- tests/test_downloader_handlers.py | 54 ++++++++++++++++++-- tests/test_downloadermiddleware_httpcache.py | 15 ++++++ tests/test_responsetypes.py | 2 + 7 files changed, 78 insertions(+), 15 deletions(-) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 3ef129587..a495874bd 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -102,11 +102,11 @@ class FTPDownloadHandler: def _build_response(self, result, request, protocol): self.result = result - respcls = responsetypes.from_args(url=request.url) protocol.close() - body = protocol.filename or protocol.body.read() headers = {"local filename": protocol.filename or '', "size": protocol.size} - return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers) + body = to_bytes(protocol.filename or protocol.body.read()) + respcls = responsetypes.from_args(url=request.url, body=body) + return respcls(url=request.url, status=200, body=body, headers=headers) def _failed(self, result, request): message = result.getErrorMessage() diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 06cb96489..7d048c1e4 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -112,7 +112,7 @@ class ScrapyHTTPClientFactory(ClientFactory): request.meta['download_latency'] = self.headers_time - self.start_time status = int(self.status) headers = Headers(self.response_headers) - respcls = responsetypes.from_args(headers=headers, url=self._url) + respcls = responsetypes.from_args(headers=headers, url=self._url, body=body) return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version)) def _set_connection_attributes(self, request): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index c71484cfa..843e14812 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -240,7 +240,7 @@ class DbmCacheStorage: status = data['status'] headers = Headers(data['headers']) body = data['body'] - respcls = responsetypes.from_args(headers=headers, url=url) + respcls = responsetypes.from_args(headers=headers, url=url, body=body) response = respcls(url=url, headers=headers, status=status, body=body) return response @@ -299,7 +299,7 @@ class FilesystemCacheStorage: url = metadata.get('response_url') status = metadata['status'] headers = Headers(headers_raw_to_dict(rawheaders)) - respcls = responsetypes.from_args(headers=headers, url=url) + respcls = responsetypes.from_args(headers=headers, url=url, body=body) response = respcls(url=url, headers=headers, status=status, body=body) return response diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 6ed9f8b8f..3efd4d2fd 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -95,12 +95,14 @@ class ResponseTypes: chunk = to_bytes(chunk) if not binary_is_text(chunk): return self.from_mimetype('application/octet-stream') - elif b"" in chunk.lower(): + lowercase_chunk = chunk.lower() + if b"" in lowercase_chunk: return self.from_mimetype('text/html') - elif b"' in lowercase_chunk: + return self.from_mimetype('text/html') + return self.from_mimetype('text') def from_args(self, headers=None, url=None, filename=None, body=None): """Guess the most appropriate Response class based on diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2bb53950d..72f52121e 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -25,7 +25,7 @@ from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Headers, Request +from scrapy.http import Headers, HtmlResponse, Request from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider @@ -389,6 +389,23 @@ class HttpTestCase(unittest.TestCase): d.addCallback(self.assertEqual, b'159') return d + def _test_response_class(self, filename, body, response_class): + def _test(response): + self.assertEqual(type(response), response_class) + + request = Request(self.getURL(filename), body=body) + return self.download_request(request, Spider('foo')).addCallback(_test) + + def test_response_class_from_url(self): + return self._test_response_class('foo.html', b'', HtmlResponse) + + def test_response_class_from_body(self): + return self._test_response_class( + 'foo', + b"\n.", + HtmlResponse, + ) + class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" @@ -971,6 +988,12 @@ class BaseFTPTestCase(unittest.TestCase): password = "passwd" req_meta = {"ftp_user": username, "ftp_password": password} + test_files = ( + ('file.txt', b"I have the power!"), + ('file with spaces.txt', b"Moooooooooo power!"), + ('html-file-without-extension', b"\n."), + ) + def setUp(self): from twisted.protocols.ftp import FTPRealm, FTPFactory from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler @@ -981,8 +1004,8 @@ class BaseFTPTestCase(unittest.TestCase): userdir = os.path.join(self.directory, self.username) os.mkdir(userdir) fp = FilePath(userdir) - fp.child('file.txt').setContent(b"I have the power!") - fp.child('file with spaces.txt').setContent(b"Moooooooooo power!") + for filename, content in self.test_files: + fp.child(filename).setContent(content) # setup server realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) @@ -1069,6 +1092,27 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) + def _test_response_class(self, filename, response_class): + f, local_fname = tempfile.mkstemp() + local_fname = to_bytes(local_fname) + os.close(f) + meta = {} + meta.update(self.req_meta) + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/{filename}", + meta=meta) + d = self.download_handler.download_request(request, None) + + def _test(r): + self.assertEqual(type(r), response_class) + os.remove(local_fname) + return self._add_test_callbacks(d, _test) + + def test_response_class_from_url(self): + return self._test_response_class('file.txt', TextResponse) + + def test_response_class_from_body(self): + return self._test_response_class('html-file-without-extension', HtmlResponse) + class FTPTestCase(BaseFTPTestCase): @@ -1104,8 +1148,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase): os.mkdir(self.directory) fp = FilePath(self.directory) - fp.child('file.txt').setContent(b"I have the power!") - fp.child('file with spaces.txt').setContent(b"Moooooooooo power!") + for filename, content in self.test_files: + fp.child(filename).setContent(content) # setup server for anonymous access realm = FTPRealm(anonymousRoot=self.directory) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 0c6dcf2aa..928c007f5 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -122,6 +122,21 @@ class DefaultStorageTest(_BaseTest): time.sleep(0.5) # give the chance to expire assert storage.retrieve_response(self.spider, self.request) + def test_storage_no_content_type_header(self): + """Test that the response body is used to get the right response class + even if there is no Content-Type header""" + with self._storage() as storage: + assert storage.retrieve_response(self.spider, self.request) is None + response = Response( + 'http://www.example.com', + body=b'\n.', + status=202, + ) + storage.store_response(self.spider, self.request, response) + cached_response = storage.retrieve_response(self.spider, self.request) + self.assertIsInstance(cached_response, HtmlResponse) + self.assertEqualResponse(response, cached_response) + class DbmStorageTest(DefaultStorageTest): diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index c07d3a99c..4b4095fb0 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -54,6 +54,8 @@ class ResponseTypesTest(unittest.TestCase): (b'\x03\x02\xdf\xdd\x23', Response), (b'Some plain text\ndata with tabs\t and null bytes\0', TextResponse), (b'Hello', HtmlResponse), + # https://codersblock.com/blog/the-smallest-valid-html5-page/ + (b'\n.', HtmlResponse), (b' Date: Thu, 16 Jun 2022 20:53:14 +0200 Subject: [PATCH 0552/2083] Update for Python 3.7+ --- docs/topics/exporters.rst | 10 +--------- scrapy/exporters.py | 2 +- scrapy/settings/__init__.py | 9 ++++----- tests/test_exporters.py | 5 ++--- tests/test_feedexport.py | 10 +++------- 5 files changed, 11 insertions(+), 25 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 7580011ac..3c36ef002 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -205,7 +205,7 @@ BaseItemExporter ['field1', 'field2'] - - A dict [3]_ where keys are fields and values are output names:: + - A dict where keys are fields and values are output names:: {'field1': 'Field 1', 'field2': 'Field 2'} @@ -214,14 +214,6 @@ BaseItemExporter all their possible fields, exporters that do not support exporting a different subset of fields per item will only export the fields found in the first item exported. - .. [3] Dicts preserve insertion order since `Python 3.7`_ - (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version - of Python, use an OrderedDict_ to enforce a specific field order. - - .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html - .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation - .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html - .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict .. attribute:: export_empty_fields diff --git a/scrapy/exporters.py b/scrapy/exporters.py index ad12f26d6..76cbe4d4b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -2,13 +2,13 @@ Item Exporters are used to export/serialize items into different formats. """ -from collections import Mapping import csv import io import marshal import pickle import pprint import warnings +from collections.abc import Mapping from xml.sax.saxutils import XMLGenerator from itemadapter import is_item, ItemAdapter diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 2bbe38481..6cacc63e1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,6 +1,5 @@ import json import copy -from collections import OrderedDict from collections.abc import MutableMapping from importlib import import_module from pprint import pformat @@ -199,7 +198,7 @@ class BaseSettings(MutableMapping): return dict(value) def getdictorlist(self, name, default=None): - """Get a setting value as either an ``OrderedDict`` or a list. + """Get a setting value as either a :class:`dict` or a :class:`list`. If the setting is already a dict or a list, a copy of it will be returned. @@ -209,7 +208,7 @@ class BaseSettings(MutableMapping): For example, settings populated from the command line will return: - - ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to + - ``{'key1': 'value1', 'key2': 'value2'}`` if set to ``'{"key1": "value1", "key2": "value2"}'`` - ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'`` @@ -222,10 +221,10 @@ class BaseSettings(MutableMapping): """ value = self.get(name, default) if value is None: - return OrderedDict() + return {} if isinstance(value, str): try: - return json.loads(value, object_pairs_hook=OrderedDict) + return json.loads(value) except ValueError: return value.split(',') return copy.deepcopy(value) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 6ba7428f6..096cd3116 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -4,7 +4,6 @@ import marshal import pickle import tempfile import unittest -from collections import OrderedDict from io import BytesIO from datetime import datetime from warnings import catch_warnings, filterwarnings @@ -114,11 +113,11 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(name, 'John\xa3') ie = self._get_exporter( - fields_to_export=OrderedDict([('name', u'å稱')]) + fields_to_export={'name': 'å稱'} ) self.assertEqual( list(ie._get_serialized_fields(self.i)), - [(u'å稱', u'John\xa3')] + [('å稱', 'John\xa3')] ) def test_field_custom_serializer(self): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 83aabbdc7..9098e035d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -11,7 +11,7 @@ import sys import tempfile import warnings from abc import ABC, abstractmethod -from collections import defaultdict, OrderedDict +from collections import defaultdict from contextlib import ExitStack from io import BytesIO from logging import getLogger @@ -998,9 +998,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_items_field_names(self): items = [{'foo': 'bar'}] - header = OrderedDict(( - ("foo", "Foo"), - )) + header = {'foo': 'Foo'} rows = [{'Foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': header} yield self.assertExported(items, list(header.values()), rows, @@ -1023,9 +1021,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_items_json_field_names(self): items = [{'foo': 'bar'}] - header = OrderedDict(( - ("foo", "Foo"), - )) + header = {'foo': 'Foo'} rows = [{'Foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} yield self.assertExported(items, list(header.values()), rows, From 1b9ed22becf03311ec014dc9b7e0c09ce87b612c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 08:27:17 +0200 Subject: [PATCH 0553/2083] Remove Python < 3.7 leftover --- tests/test_feedexport.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 9098e035d..946c94bd4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1004,8 +1004,6 @@ class FeedExportTest(FeedExportTestBase): yield self.assertExported(items, list(header.values()), rows, settings=settings) - @pytest.mark.skipif(sys.version_info < (3, 7), - reason='Only official in Python 3.7+') @defer.inlineCallbacks def test_export_items_dict_field_names(self): items = [{'foo': 'bar'}] From 24f382fa459434cccfa4c0a8884a48d09d75e243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 08:31:45 +0200 Subject: [PATCH 0554/2083] test_feedexport: remove ordered=False --- tests/test_feedexport.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 946c94bd4..4006b5957 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -657,8 +657,8 @@ class FeedExportTestBase(ABC, unittest.TestCase): return data @defer.inlineCallbacks - def assertExported(self, items, header, rows, settings=None, ordered=True): - yield self.assertExportedCsv(items, header, rows, settings, ordered) + def assertExported(self, items, header, rows, settings=None): + yield self.assertExportedCsv(items, header, rows, settings) yield self.assertExportedJsonLines(items, rows, settings) yield self.assertExportedXml(items, rows, settings) yield self.assertExportedPickle(items, rows, settings) @@ -719,7 +719,7 @@ class FeedExportTest(FeedExportTestBase): return content @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': { @@ -730,10 +730,7 @@ class FeedExportTest(FeedExportTestBase): reader = csv.DictReader(to_unicode(data['csv']).splitlines()) got_rows = list(reader) - if ordered: - self.assertEqual(reader.fieldnames, header) - else: - self.assertEqual(set(reader.fieldnames), set(header)) + self.assertEqual(reader.fieldnames, header) self.assertEqual(rows, got_rows) @@ -886,7 +883,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} ] header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, ordered=False) + yield self.assertExported(items, header, rows) @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): @@ -958,7 +955,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam4', 'foo': '', 'baz': ''}, ] rows_jl = [dict(row) for row in items] - yield self.assertExportedCsv(items, header, rows_csv, ordered=False) + yield self.assertExportedCsv(items, header, rows_csv) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -968,7 +965,7 @@ class FeedExportTest(FeedExportTestBase): header = ["foo"] rows = [{'foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': []} - yield self.assertExportedCsv(items, header, rows, ordered=False) + yield self.assertExportedCsv(items, header, rows) yield self.assertExportedJsonLines(items, rows, settings) @defer.inlineCallbacks @@ -1146,7 +1143,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam', 'foo': 'bar'} ] rows_jl = items - yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False) + yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -2065,7 +2062,7 @@ class BatchDeliveriesTest(FeedExportTestBase): self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': { From 3729c6d26698ae6b8a7ef297606a1c7630d82619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 08:33:34 +0200 Subject: [PATCH 0555/2083] Remove unused import and redundant import --- tests/test_feedexport.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 4006b5957..8ef221b70 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -21,7 +21,6 @@ from unittest import mock from urllib.parse import urljoin, quote from urllib.request import pathname2url -import pytest import lxml.etree from testfixtures import LogCapture from twisted.internet import defer @@ -731,7 +730,6 @@ class FeedExportTest(FeedExportTestBase): reader = csv.DictReader(to_unicode(data['csv']).splitlines()) got_rows = list(reader) self.assertEqual(reader.fieldnames, header) - self.assertEqual(rows, got_rows) @defer.inlineCallbacks @@ -1815,7 +1813,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_lzma_plugin_filters(self): - import sys if "PyPy" in sys.version: # https://foss.heptapod.net/pypy/pypy/-/issues/3527 raise unittest.SkipTest("lzma filters doesn't work in PyPy") From 6e878490e823a8105276b71ca5f6dc789465d330 Mon Sep 17 00:00:00 2001 From: Michel Ace Date: Fri, 17 Jun 2022 08:37:14 +0200 Subject: [PATCH 0556/2083] Support and prefer the .jsonl file extension (#4848) --- docs/intro/overview.rst | 4 ++-- docs/intro/tutorial.rst | 2 +- docs/topics/feed-exports.rst | 3 ++- docs/topics/item-pipeline.rst | 8 ++++---- scrapy/settings/default_settings.py | 1 + 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index f3d652621..cfa6bfa83 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -45,9 +45,9 @@ https://quotes.toscrape.com, following the pagination:: Put this in a text file, name it to something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: - scrapy runspider quotes_spider.py -o quotes.jl + scrapy runspider quotes_spider.py -o quotes.jsonl -When this finishes you will have in the ``quotes.jl`` file a list of the +When this finishes you will have in the ``quotes.jsonl`` file a list of the quotes in JSON Lines format, containing text and author, looking like this:: {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index cde1b1ef4..75928077e 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -482,7 +482,7 @@ to append new content to any existing file. However, appending to a JSON file makes the file contents invalid JSON. When appending to a file, consider using a different serialization format, such as `JSON Lines`_:: - scrapy crawl quotes -o quotes.jl + scrapy crawl quotes -o quotes.jsonl The `JSON Lines`_ format is useful because it's stream-like, you can easily append new records to it. It doesn't have the same problem of JSON when you run diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 9a13eb82f..398f80633 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -638,6 +638,7 @@ Default:: { 'json': 'scrapy.exporters.JsonItemExporter', 'jsonlines': 'scrapy.exporters.JsonLinesItemExporter', + 'jsonl': 'scrapy.exporters.JsonLinesItemExporter', 'jl': 'scrapy.exporters.JsonLinesItemExporter', 'csv': 'scrapy.exporters.CsvItemExporter', 'xml': 'scrapy.exporters.XmlItemExporter', @@ -763,7 +764,7 @@ source spider in the feed URI: #. Use ``%(spider_name)s`` in your feed URI:: - scrapy crawl -o "%(spider_name)s.jl" + scrapy crawl -o "%(spider_name)s.jsonl" .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 882ff5661..af294f52c 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -99,11 +99,11 @@ contain a price:: raise DropItem(f"Missing price in {item}") -Write items to a JSON file --------------------------- +Write items to a JSON lines file +-------------------------------- The following pipeline stores all scraped items (from all spiders) into a -single ``items.jl`` file, containing one item per line serialized in JSON +single ``items.jsonl`` file, containing one item per line serialized in JSON format:: import json @@ -113,7 +113,7 @@ format:: class JsonWriterPipeline: def open_spider(self, spider): - self.file = open('items.jl', 'w') + self.file = open('items.jsonl', 'w') def close_spider(self, spider): self.file.close() diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index f5a3efe69..ff86af125 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -154,6 +154,7 @@ FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', 'jsonlines': 'scrapy.exporters.JsonLinesItemExporter', + 'jsonl': 'scrapy.exporters.JsonLinesItemExporter', 'jl': 'scrapy.exporters.JsonLinesItemExporter', 'csv': 'scrapy.exporters.CsvItemExporter', 'xml': 'scrapy.exporters.XmlItemExporter', From 516e2d6ec0da77b8e0c01eb5188311b5fbeaa22e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 08:55:45 +0200 Subject: [PATCH 0557/2083] Revert "test_feedexport: remove ordered=False" This reverts commit 24f382fa459434cccfa4c0a8884a48d09d75e243. --- tests/test_feedexport.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8ef221b70..fe90501fb 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -656,8 +656,8 @@ class FeedExportTestBase(ABC, unittest.TestCase): return data @defer.inlineCallbacks - def assertExported(self, items, header, rows, settings=None): - yield self.assertExportedCsv(items, header, rows, settings) + def assertExported(self, items, header, rows, settings=None, ordered=True): + yield self.assertExportedCsv(items, header, rows, settings, ordered) yield self.assertExportedJsonLines(items, rows, settings) yield self.assertExportedXml(items, rows, settings) yield self.assertExportedPickle(items, rows, settings) @@ -718,7 +718,7 @@ class FeedExportTest(FeedExportTestBase): return content @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None): + def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} settings.update({ 'FEEDS': { @@ -729,7 +729,11 @@ class FeedExportTest(FeedExportTestBase): reader = csv.DictReader(to_unicode(data['csv']).splitlines()) got_rows = list(reader) - self.assertEqual(reader.fieldnames, header) + if ordered: + self.assertEqual(reader.fieldnames, header) + else: + self.assertEqual(set(reader.fieldnames), set(header)) + self.assertEqual(rows, got_rows) @defer.inlineCallbacks @@ -881,7 +885,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} ] header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows) + yield self.assertExported(items, header, rows, ordered=False) @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): @@ -953,7 +957,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam4', 'foo': '', 'baz': ''}, ] rows_jl = [dict(row) for row in items] - yield self.assertExportedCsv(items, header, rows_csv) + yield self.assertExportedCsv(items, header, rows_csv, ordered=False) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -963,7 +967,7 @@ class FeedExportTest(FeedExportTestBase): header = ["foo"] rows = [{'foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': []} - yield self.assertExportedCsv(items, header, rows) + yield self.assertExportedCsv(items, header, rows, ordered=False) yield self.assertExportedJsonLines(items, rows, settings) @defer.inlineCallbacks @@ -1141,7 +1145,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam', 'foo': 'bar'} ] rows_jl = items - yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv) + yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -2059,7 +2063,7 @@ class BatchDeliveriesTest(FeedExportTestBase): self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None): + def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} settings.update({ 'FEEDS': { From bc285f393ca8ff33ef715f98ef3367c973d23ab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 09:00:39 +0200 Subject: [PATCH 0558/2083] Revert "Revert "test_feedexport: remove ordered=False"" This reverts commit 516e2d6ec0da77b8e0c01eb5188311b5fbeaa22e. --- tests/test_feedexport.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index fe90501fb..8ef221b70 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -656,8 +656,8 @@ class FeedExportTestBase(ABC, unittest.TestCase): return data @defer.inlineCallbacks - def assertExported(self, items, header, rows, settings=None, ordered=True): - yield self.assertExportedCsv(items, header, rows, settings, ordered) + def assertExported(self, items, header, rows, settings=None): + yield self.assertExportedCsv(items, header, rows, settings) yield self.assertExportedJsonLines(items, rows, settings) yield self.assertExportedXml(items, rows, settings) yield self.assertExportedPickle(items, rows, settings) @@ -718,7 +718,7 @@ class FeedExportTest(FeedExportTestBase): return content @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': { @@ -729,11 +729,7 @@ class FeedExportTest(FeedExportTestBase): reader = csv.DictReader(to_unicode(data['csv']).splitlines()) got_rows = list(reader) - if ordered: - self.assertEqual(reader.fieldnames, header) - else: - self.assertEqual(set(reader.fieldnames), set(header)) - + self.assertEqual(reader.fieldnames, header) self.assertEqual(rows, got_rows) @defer.inlineCallbacks @@ -885,7 +881,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} ] header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, ordered=False) + yield self.assertExported(items, header, rows) @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): @@ -957,7 +953,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam4', 'foo': '', 'baz': ''}, ] rows_jl = [dict(row) for row in items] - yield self.assertExportedCsv(items, header, rows_csv, ordered=False) + yield self.assertExportedCsv(items, header, rows_csv) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -967,7 +963,7 @@ class FeedExportTest(FeedExportTestBase): header = ["foo"] rows = [{'foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': []} - yield self.assertExportedCsv(items, header, rows, ordered=False) + yield self.assertExportedCsv(items, header, rows) yield self.assertExportedJsonLines(items, rows, settings) @defer.inlineCallbacks @@ -1145,7 +1141,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam', 'foo': 'bar'} ] rows_jl = items - yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False) + yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -2063,7 +2059,7 @@ class BatchDeliveriesTest(FeedExportTestBase): self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': { From ec5cf3e9cea3c66aca4cf1aad576f33edca3ad1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 09:10:18 +0200 Subject: [PATCH 0559/2083] test_feedexport: solve ordered comparison issues --- tests/test_feedexport.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8ef221b70..ec48f8d4a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -726,11 +726,9 @@ class FeedExportTest(FeedExportTestBase): }, }) data = yield self.exported_data(items, settings) - reader = csv.DictReader(to_unicode(data['csv']).splitlines()) - got_rows = list(reader) - self.assertEqual(reader.fieldnames, header) - self.assertEqual(rows, got_rows) + self.assertEqual(reader.fieldnames, list(header)) + self.assertEqual(rows, list(reader)) @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): @@ -1141,7 +1139,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam', 'foo': 'bar'} ] rows_jl = items - yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv) + yield self.assertExportedCsv(items, ['foo', 'egg'], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks From 4ef71829b22b7362d08d6897090595138107852f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 10:37:27 +0200 Subject: [PATCH 0560/2083] If TWISTED_REACTOR is None, reuse any pre-installed reactor (#5528) --- docs/topics/settings.rst | 7 ++- scrapy/crawler.py | 3 +- scrapy/utils/reactor.py | 2 +- tests/CrawlerProcess/reactor_default.py | 17 +++++ .../reactor_default_twisted_reactor_select.py | 20 ++++++ tests/CrawlerProcess/reactor_select.py | 19 ++++++ ..._select_subclass_twisted_reactor_select.py | 31 +++++++++ .../reactor_select_twisted_reactor_select.py | 22 +++++++ tests/test_crawler.py | 63 +++++++++++++++++-- 9 files changed, 172 insertions(+), 12 deletions(-) create mode 100644 tests/CrawlerProcess/reactor_default.py create mode 100644 tests/CrawlerProcess/reactor_default_twisted_reactor_select.py create mode 100644 tests/CrawlerProcess/reactor_select.py create mode 100644 tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py create mode 100644 tests/CrawlerProcess/reactor_select_twisted_reactor_select.py diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4e105642d..f3b28c4c4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1638,9 +1638,10 @@ which raises :exc:`Exception`, becomes:: The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will install the default reactor defined by Twisted for the -current platform. This is to maintain backward compatibility and avoid possible -problems caused by using a non-default reactor. +means that Scrapy will use the existing reactor if one is already installed, or +install the default reactor defined by Twisted for the current platform. This +is to maintain backward compatibility and avoid possible problems caused by +using a non-default reactor. For additional information, see :doc:`core/howto/choosing-reactor`. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d669d93a8..dcf0c2146 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -78,8 +78,7 @@ class Crawler: if reactor_class: install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) else: - from twisted.internet import default - default.install() + from twisted.internet import reactor # noqa: F401 log_reactor_info() if reactor_class: verify_installed_reactor(reactor_class) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 96395543c..bc543b230 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -83,7 +83,7 @@ def verify_installed_reactor(reactor_path): path.""" from twisted.internet import reactor reactor_class = load_object(reactor_path) - if not isinstance(reactor, reactor_class): + if not reactor.__class__ == reactor_class: msg = ("The installed reactor " f"({reactor.__module__}.{reactor.__class__.__name__}) does not " f"match the requested one ({reactor_path})") diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py new file mode 100644 index 000000000..5a21a3717 --- /dev/null +++ b/tests/CrawlerProcess/reactor_default.py @@ -0,0 +1,17 @@ +import scrapy +from scrapy.crawler import CrawlerProcess +from twisted.internet import reactor + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.start() + diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py new file mode 100644 index 000000000..c476722ef --- /dev/null +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -0,0 +1,20 @@ +import scrapy +from scrapy.crawler import CrawlerProcess +from twisted.internet import reactor + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", +}) + +process.crawl(NoRequestsSpider) +process.start() + + diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py new file mode 100644 index 000000000..eac6e2f89 --- /dev/null +++ b/tests/CrawlerProcess/reactor_select.py @@ -0,0 +1,19 @@ +import scrapy +from scrapy.crawler import CrawlerProcess +from twisted.internet import selectreactor +selectreactor.install() + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.start() + + diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py new file mode 100644 index 000000000..47f480605 --- /dev/null +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -0,0 +1,31 @@ +import scrapy +from scrapy.crawler import CrawlerProcess +from twisted.internet.main import installReactor +from twisted.internet.selectreactor import SelectReactor + + +class SelectReactorSubclass(SelectReactor): + pass + + +reactor = SelectReactorSubclass() +installReactor(reactor) + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", +}) + +process.crawl(NoRequestsSpider) +process.start() + + + + diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py new file mode 100644 index 000000000..e0d2dab26 --- /dev/null +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import CrawlerProcess +from twisted.internet import selectreactor +selectreactor.install() + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", +}) + +process.crawl(NoRequestsSpider) +process.start() + + + diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 957525382..1ff2e8a67 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -308,6 +308,57 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) self.assertNotIn("ReactorAlreadyInstalledError", log) + def test_reactor_default(self): + log = self.run_script('reactor_default.py') + self.assertIn('Spider closed (finished)', log) + self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertNotIn("ReactorAlreadyInstalledError", log) + + def test_reactor_default_twisted_reactor_select(self): + log = self.run_script('reactor_default_twisted_reactor_select.py') + if platform.system() == 'Windows': + # The goal of this test function is to test that, when a reactor is + # installed (the default one here) and a different reactor is + # configured (select here), an error raises. + # + # In Windows the default reactor is the select reactor, so that + # error does not raise. + # + # If that ever becomes the case on more platforms (i.e. if Linux + # also starts using the select reactor by default in a future + # version of Twisted), then we will need to rethink this test. + self.assertIn('Spider closed (finished)', log) + else: + self.assertNotIn('Spider closed (finished)', log) + self.assertIn( + ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ), + log, + ) + + def test_reactor_select(self): + log = self.run_script('reactor_select.py') + self.assertIn('Spider closed (finished)', log) + self.assertNotIn("ReactorAlreadyInstalledError", log) + + def test_reactor_select_twisted_reactor_select(self): + log = self.run_script('reactor_select_twisted_reactor_select.py') + self.assertIn('Spider closed (finished)', log) + self.assertNotIn("ReactorAlreadyInstalledError", log) + + def test_reactor_select_subclass_twisted_reactor_select(self): + log = self.run_script('reactor_select_subclass_twisted_reactor_select.py') + self.assertNotIn('Spider closed (finished)', log) + self.assertIn( + ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ), + log, + ) + def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -340,33 +391,33 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertNotIn("TimeoutError", log) self.assertNotIn("twisted.internet.error.DNSLookupError", log) - def test_reactor_select(self): + def test_twisted_reactor_select(self): log = self.run_script("twisted_reactor_select.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) @mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows") - def test_reactor_poll(self): + def test_twisted_reactor_poll(self): log = self.run_script("twisted_reactor_poll.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - def test_reactor_asyncio(self): + def test_twisted_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - def test_reactor_asyncio_custom_settings(self): + def test_twisted_reactor_asyncio_custom_settings(self): log = self.run_script("twisted_reactor_custom_settings.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - def test_reactor_asyncio_custom_settings_same(self): + def test_twisted_reactor_asyncio_custom_settings_same(self): log = self.run_script("twisted_reactor_custom_settings_same.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - def test_reactor_asyncio_custom_settings_conflict(self): + def test_twisted_reactor_asyncio_custom_settings_conflict(self): log = self.run_script("twisted_reactor_custom_settings_conflict.py") self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log) From 54bfb9649bdec565f9798cc41643ed1bae25bd67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 11:51:02 +0200 Subject: [PATCH 0561/2083] Cover #5525 in the 2.6.2 release notes (#5535) --- docs/conf.py | 1 + docs/news.rst | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 55aa72d5a..9a0afe73e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -295,6 +295,7 @@ intersphinx_mapping = { 'twisted': ('https://twistedmatrix.com/documents/current', None), 'twistedapi': ('https://twistedmatrix.com/documents/current/api', None), } +intersphinx_disabled_reftypes = [] # Options for sphinx-hoverxref options diff --git a/docs/news.rst b/docs/news.rst index ffeb50390..7993b4b4f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,14 +5,19 @@ Release notes .. _release-2.6.2: -Scrapy 2.6.2 (2022-0?-??) -------------------------- +Scrapy 2.6.2 (to be determined) +------------------------------- Fixes additional regressions introduced in 2.6.0: - :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple spiders (:issue:`5435`, :issue:`5436`) +- Installing a Twisted reactor before Scrapy does (e.g. importing + :mod:`twisted.internet.reactor` somewhere at the module level) no longer + prevents Scrapy from starting, as long as a different reactor is not + specified in :setting:`TWISTED_REACTOR` (:issue:`5525`, :issue:`5528`) + - Fixed an exception that was being logged after the spider finished under certain conditions (:issue:`5437`, :issue:`5440`) From e3e69d1209407c72a6478936bdbfd32cc22e9432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 20 Jun 2022 11:46:13 +0200 Subject: [PATCH 0562/2083] Pin documentation requirements (#5536) --- docs/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index a0930ba1e..9f9aef711 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -Sphinx>=3.0 -sphinx-hoverxref>=0.2b1 -sphinx-notfound-page>=0.4 -sphinx-rtd-theme>=0.5.2 \ No newline at end of file +sphinx==5.0.2 +sphinx-hoverxref==1.1.1 +sphinx-notfound-page==0.8 +sphinx-rtd-theme==1.0.0 From d8223adfacc7e0ae684e5f9463474707bfcb008d Mon Sep 17 00:00:00 2001 From: Emanuele Date: Mon, 20 Jun 2022 11:54:05 +0200 Subject: [PATCH 0563/2083] =?UTF-8?q?Typo:=20cleanup=20(verb)=20=E2=86=92?= =?UTF-8?q?=20clean=20up=20(#5538)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.rst b/docs/README.rst index 0b7afa548..36dd5aea4 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -43,7 +43,7 @@ This command will fire up your default browser and open the main page of your Start over ---------- -To cleanup all generated documentation files and start from scratch run:: +To clean up all generated documentation files and start from scratch run:: make clean From 34e4ed72ea87601ce17c8d248d6a54d3e8b73194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 21 Jun 2022 12:46:54 +0200 Subject: [PATCH 0564/2083] Document how DOWNLOAD_DELAY affects per-domain concurrency --- docs/topics/settings.rst | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2046c6446..468975d6a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -633,25 +633,41 @@ DOWNLOAD_DELAY Default: ``0`` -The amount of time (in secs) that the downloader should wait before downloading -consecutive pages from the same website. This can be used to throttle the -crawling speed to avoid hitting servers too hard. Decimal numbers are -supported. Example:: +Minimum seconds to wait between 2 consecutive requests to the same domain. - DOWNLOAD_DELAY = 0.25 # 250 ms of delay +Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting +servers too hard. + +Decimal numbers are supported. For example, to send a maximum of 4 requests +every 10 seconds:: + + DOWNLOAD_DELAY = 2.5 This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY` -setting (which is enabled by default). By default, Scrapy doesn't wait a fixed -amount of time between requests, but uses a random interval between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`. +setting, which is enabled by default. When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced -per ip address instead of per domain. +per IP address instead of per domain. + +Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain +concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response +time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective +concurrency for that domain is 1. When testing throttling configurations, it +usually makes sense to lower :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` first, +and only increase :setting:`DOWNLOAD_DELAY` once +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is 1 but a higher throttling is +desired. .. _spider-download_delay-attribute: -You can also change this setting per spider by setting ``download_delay`` +You can change this setting per spider by setting the ``download_delay`` spider attribute. +It is also possible to change this setting per domain, although it requires +non-trivial code. See the implementation of the :ref:`AutoThrottle +` extension for an example. + + .. setting:: DOWNLOAD_HANDLERS DOWNLOAD_HANDLERS From 387326fad42c4709933851108f2370d3105b8dd1 Mon Sep 17 00:00:00 2001 From: Vardhaman <83634399+cyai@users.noreply.github.com> Date: Thu, 23 Jun 2022 14:40:49 +0530 Subject: [PATCH 0565/2083] MAINT: Updated f-string format Updated the code with the f-string method for better and cleaner understanding. --- scrapy/downloadermiddlewares/cookies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 3afa06077..c592acb57 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -104,8 +104,8 @@ class CookiesMiddleware: for key in ("name", "value", "path", "domain"): if cookie.get(key) is None: if key in ("name", "value"): - msg = "Invalid cookie found in request {}: {} ('{}' is missing)" - logger.warning(msg.format(request, cookie, key)) + msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" + logger.warning(msg) return continue if isinstance(cookie[key], (bool, float, int, str)): From c4c816624fc4fd5fbc1866c507b661e92704136a Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 30 Jun 2022 10:42:01 -0300 Subject: [PATCH 0566/2083] chore: Deprecate the `scrapy.downloadermiddlewares.decompression` module --- scrapy/downloadermiddlewares/decompression.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 0fcf8fb8c..98f18a836 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -9,10 +9,19 @@ import tarfile import zipfile from io import BytesIO from tempfile import mktemp +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes +warnings.warn( + 'scrapy.downloadermiddlewares.decompression is deprecated', + ScrapyDeprecationWarning, + stacklevel=2, +) + + logger = logging.getLogger(__name__) From fe08a119d965b2291e44801901e15b58a1f959ad Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 30 Jun 2022 10:46:00 -0300 Subject: [PATCH 0567/2083] chore: import only used function --- scrapy/downloadermiddlewares/decompression.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 98f18a836..e01e9cc76 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -9,13 +9,13 @@ import tarfile import zipfile from io import BytesIO from tempfile import mktemp -import warnings +from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes -warnings.warn( +warn( 'scrapy.downloadermiddlewares.decompression is deprecated', ScrapyDeprecationWarning, stacklevel=2, From 09c3a4ad082dd6fc431be65975292d2eba369ad6 Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Tue, 12 Jul 2022 12:41:46 +0200 Subject: [PATCH 0568/2083] Fix doc: `scrapy.exporter` to `scrapy.exporters` --- docs/topics/exporters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 3c36ef002..9360ecf37 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -117,7 +117,7 @@ after your custom code. Example:: - from scrapy.exporter import XmlItemExporter + from scrapy.exporters import XmlItemExporter class ProductXmlExporter(XmlItemExporter): From 1c7ed4f2e59a94651d6d9d136cab78821cd3e80e Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Thu, 6 Jan 2022 22:21:56 +0100 Subject: [PATCH 0569/2083] [doc] Remove incompatible web service project * Abandoned since 2017 * Not compatible with Python3 --- docs/index.rst | 4 ---- docs/topics/webservice.rst | 11 ----------- 2 files changed, 15 deletions(-) delete mode 100644 docs/topics/webservice.rst diff --git a/docs/index.rst b/docs/index.rst index 75e08f537..40c6cb485 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -130,7 +130,6 @@ Built-in services topics/stats topics/email topics/telnetconsole - topics/webservice :doc:`topics/logging` Learn how to use Python's builtin logging on Scrapy. @@ -144,9 +143,6 @@ Built-in services :doc:`topics/telnetconsole` Inspect a running crawler using a built-in Python console. -:doc:`topics/webservice` - Monitor and control a crawler using a web service. - Solving specific problems ========================= diff --git a/docs/topics/webservice.rst b/docs/topics/webservice.rst deleted file mode 100644 index 2c4052c04..000000000 --- a/docs/topics/webservice.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _topics-webservice: - -=========== -Web Service -=========== - -webservice has been moved into a separate project. - -It is hosted at: - - https://github.com/scrapy-plugins/scrapy-jsonrpc From 2f13f23d927900de0a89197ded0ee7aed387e351 Mon Sep 17 00:00:00 2001 From: Ikko Ashimine Date: Fri, 15 Jul 2022 18:16:23 +0900 Subject: [PATCH 0570/2083] Fix typo in sep-014.rst requets -> requests --- sep/sep-014.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 0859e3f7c..2521aa0e5 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -590,11 +590,11 @@ Request Generator def generate_requests(self, response): """ - Extract and process new requets from response + Extract and process new requests from response """ requests = [] for ext in self._request_extractors: - requets.extend(ext.extract_requests(response)) + requests.extend(ext.extract_requests(response)) for proc in self._request_processors: requests = proc(requests) From 9b33b82a8b802c3906c2f1eaf1b88efee9b2fb09 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sun, 17 Jul 2022 15:50:40 +0500 Subject: [PATCH 0571/2083] Fixed intersphinx references --- docs/conf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 378b01804..3241295af 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -291,9 +291,9 @@ intersphinx_mapping = { 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), - 'tox': ('https://tox.readthedocs.io/en/latest', None), - 'twisted': ('https://twistedmatrix.com/documents/current', None), - 'twistedapi': ('https://twistedmatrix.com/documents/current/api', None), + 'tox': ('https://tox.wiki/en/latest/', None), + 'twisted': ('https://docs.twisted.org/en/stable/', None), + 'twistedapi': ('https://docs.twisted.org/en/stable/api/', None), 'w3lib': ('https://w3lib.readthedocs.io/en/latest', None), } intersphinx_disabled_reftypes = [] From 26c70318cb14806a07ee09d0283e9d5d306490e9 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sun, 17 Jul 2022 16:47:20 +0500 Subject: [PATCH 0572/2083] make Scrapy testing suite more robust in environments where non-existing hosts are resolvable --- tests/__init__.py | 10 ++++++++++ tests/test_command_shell.py | 4 +++- tests/test_crawl.py | 4 ++++ tests/test_downloader_handlers.py | 5 ++++- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 12ce79fa9..bb62851dc 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,6 +5,7 @@ see https://docs.scrapy.org/en/latest/contributing.html#running-tests """ import os +import socket # ignore system-wide proxies for tests # which would send requests to a totally unsuspecting server @@ -25,6 +26,15 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data') +# In some environments accessing a non-existing host doesn't raise an +# error. In such cases we're going to skip tests which rely on it. +try: + socket.getaddrinfo('non-existing-host', 80) + NON_EXISTING_RESOLVABLE = True +except socket.gaierror: + NON_EXISTING_RESOLVABLE = False + + def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 16c9559b5..33189e9be 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -6,7 +6,7 @@ from twisted.internet import defer from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest -from tests import tests_datadir +from tests import tests_datadir, NON_EXISTING_RESOLVABLE class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @@ -109,6 +109,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_dns_failures(self): + if NON_EXISTING_RESOLVABLE: + raise unittest.SkipTest("Non-existing hosts are resolvable") url = 'www.somedomainthatdoesntexi.st' errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 7bda3bef2..f9ffcd6bb 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -3,6 +3,7 @@ import logging from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse +import unittest from pytest import mark from testfixtures import LogCapture @@ -17,6 +18,7 @@ from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import to_unicode +from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import MockServer from tests.spiders import ( AsyncDefAsyncioGenComplexSpider, @@ -137,6 +139,8 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_retry_dns_error(self): + if NON_EXISTING_RESOLVABLE: + raise unittest.SkipTest("Non-existing hosts are resolvable") crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as log: # try to fetch the homepage of a non-existent domain diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 72f52121e..883960084 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -4,7 +4,7 @@ import shutil import sys import tempfile from typing import Optional, Type -from unittest import mock +from unittest import mock, SkipTest from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal @@ -32,6 +32,7 @@ from scrapy.spiders import Spider from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto +from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import ( Echo, ForeverTakingResource, @@ -791,6 +792,8 @@ class Http11ProxyTestCase(HttpProxyTestCase): @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): """ Test TunnelingTCP4ClientEndpoint """ + if NON_EXISTING_RESOLVABLE: + raise SkipTest("Non-existing hosts are resolvable") http_proxy = self.getURL('') domain = 'https://no-such-domain.nosuch' request = Request( From e248360e6e3dbb36fab185caf131707195fa6a26 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 18 Jul 2022 23:49:08 +0500 Subject: [PATCH 0573/2083] remove compatibility code from tests for the case dataclasses module is not available It was Python 3.6 compat code, and Python 3.6 support is dropped. --- tests/test_exporters.py | 32 ++++++++++++++--------------- tests/test_loader.py | 23 +++++++-------------- tests/test_pipeline_files.py | 37 ++++++++++++---------------------- tests/test_pipeline_images.py | 38 ++++++++++++----------------------- tests/test_utils_serialize.py | 18 +++++++---------- 5 files changed, 55 insertions(+), 93 deletions(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 096cd3116..69ac928c3 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -4,6 +4,7 @@ import marshal import pickle import tempfile import unittest +import dataclasses from io import BytesIO from datetime import datetime from warnings import catch_warnings, filterwarnings @@ -21,31 +22,30 @@ from scrapy.exporters import ( ) +def custom_serializer(value): + return str(int(value) + 2) + + class TestItem(Item): name = Field() age = Field() -def custom_serializer(value): - return str(int(value) + 2) - - class CustomFieldItem(Item): name = Field() age = Field(serializer=custom_serializer) -try: - from dataclasses import make_dataclass, field -except ImportError: - TestDataClass = None - CustomFieldDataclass = None -else: - TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)]) - CustomFieldDataclass = make_dataclass( - "CustomFieldDataclass", - [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))] - ) +@dataclasses.dataclass +class TestDataClass: + name: str + age: int + + +@dataclasses.dataclass +class CustomFieldDataclass: + name: str + age: int = dataclasses.field(metadata={"serializer": custom_serializer}) class BaseItemExporterTest(unittest.TestCase): @@ -54,8 +54,6 @@ class BaseItemExporterTest(unittest.TestCase): custom_field_item_class = CustomFieldItem def setUp(self): - if self.item_class is None: - raise unittest.SkipTest("item class is None") self.i = self.item_class(name='John\xa3', age='22') self.output = BytesIO() self.ie = self._get_exporter() diff --git a/tests/test_loader.py b/tests/test_loader.py index f7ab1f236..c0937b349 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,4 +1,5 @@ import unittest +import dataclasses import attr from itemadapter import ItemAdapter @@ -10,13 +11,6 @@ from scrapy.loader import ItemLoader from scrapy.selector import Selector -try: - from dataclasses import make_dataclass, field as dataclass_field -except ImportError: - make_dataclass = None - dataclass_field = None - - # test items class NameItem(Item): name = Field() @@ -41,6 +35,11 @@ class AttrsNameItem: name = attr.ib(default="") +@dataclasses.dataclass +class TestDataClass: + name: list = dataclasses.field(default_factory=list) + + # test item loaders class NameItemLoader(ItemLoader): default_item_class = TestItem @@ -187,16 +186,8 @@ class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase item_class = AttrsNameItem -@unittest.skipIf(not make_dataclass, "dataclasses module is not available") class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if make_dataclass: - self.item_class = make_dataclass( - "TestDataClass", - [("name", list, dataclass_field(default_factory=list))], - ) + item_class = TestDataClass class BaseNoInputReprocessingLoader(ItemLoader): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4228173ed..5d381c018 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -7,6 +7,7 @@ from shutil import rmtree from tempfile import mkdtemp from unittest import mock, skipIf from urllib.parse import urlparse +import dataclasses import attr from itemadapter import ItemAdapter @@ -32,13 +33,6 @@ from scrapy.utils.test import ( ) -try: - from dataclasses import make_dataclass, field as dataclass_field -except ImportError: - make_dataclass = None - dataclass_field = None - - def _mocked_download_func(request, info): response = request.meta.get('response') return response() if callable(response) else response @@ -226,24 +220,19 @@ class FilesPipelineTestCaseFieldsItem(FilesPipelineTestCaseFieldsMixin, unittest item_class = FilesPipelineTestItem -@skipIf(not make_dataclass, "dataclasses module is not available") -class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): +@dataclasses.dataclass +class FilesPipelineTestDataClass: + name: str + # default fields + file_urls: list = dataclasses.field(default_factory=list) + files: list = dataclasses.field(default_factory=list) + # overridden fields + custom_file_urls: list = dataclasses.field(default_factory=list) + custom_files: list = dataclasses.field(default_factory=list) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if make_dataclass: - self.item_class = make_dataclass( - "FilesPipelineTestDataClass", - [ - ("name", str), - # default fields - ("file_urls", list, dataclass_field(default_factory=list)), - ("files", list, dataclass_field(default_factory=list)), - # overridden fields - ("custom_file_urls", list, dataclass_field(default_factory=list)), - ("custom_files", list, dataclass_field(default_factory=list)), - ], - ) + +class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = FilesPipelineTestDataClass @attr.s diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index dd94d296b..e6f5bea21 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -4,6 +4,7 @@ import random from shutil import rmtree from tempfile import mkdtemp from unittest import skipIf +import dataclasses import attr from itemadapter import ItemAdapter @@ -16,13 +17,6 @@ from scrapy.settings import Settings from scrapy.utils.python import to_bytes -try: - from dataclasses import make_dataclass, field as dataclass_field -except ImportError: - make_dataclass = None - dataclass_field = None - - try: from PIL import Image except ImportError: @@ -203,25 +197,19 @@ class ImagesPipelineTestCaseFieldsItem(ImagesPipelineTestCaseFieldsMixin, unitte item_class = ImagesPipelineTestItem -@skipIf(not make_dataclass, "dataclasses module is not available") -class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): - item_class = None +@dataclasses.dataclass +class ImagesPipelineTestDataClass: + name: str + # default fields + image_urls: list = dataclasses.field(default_factory=list) + images: list = dataclasses.field(default_factory=list) + # overridden fields + custom_image_urls: list = dataclasses.field(default_factory=list) + custom_images: list = dataclasses.field(default_factory=list) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if make_dataclass: - self.item_class = make_dataclass( - "FilesPipelineTestDataClass", - [ - ("name", str), - # default fields - ("image_urls", list, dataclass_field(default_factory=list)), - ("images", list, dataclass_field(default_factory=list)), - # overridden fields - ("custom_image_urls", list, dataclass_field(default_factory=list)), - ("custom_images", list, dataclass_field(default_factory=list)), - ], - ) + +class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = ImagesPipelineTestDataClass @attr.s diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index daf022aee..a51de1877 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -1,6 +1,7 @@ import datetime import json import unittest +import dataclasses from decimal import Decimal import attr @@ -10,12 +11,6 @@ from scrapy.http import Request, Response from scrapy.utils.serialize import ScrapyJSONEncoder -try: - from dataclasses import make_dataclass -except ImportError: - make_dataclass = None - - class JsonEncoderTestCase(unittest.TestCase): def setUp(self): @@ -56,12 +51,13 @@ class JsonEncoderTestCase(unittest.TestCase): self.assertIn(r.url, rs) self.assertIn(str(r.status), rs) - @unittest.skipIf(not make_dataclass, "No dataclass support") def test_encode_dataclass_item(self): - TestDataClass = make_dataclass( - "TestDataClass", - [("name", str), ("url", str), ("price", int)], - ) + @dataclasses.dataclass + class TestDataClass: + name: str + url: str + price: int + item = TestDataClass(name="Product", url="http://product.org", price=1) encoded = self.encoder.encode(item) self.assertEqual( From 105468959363ee50b597038ac30fd32d3ea1b1f2 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 18 Jul 2022 23:53:30 +0500 Subject: [PATCH 0574/2083] remove unused imports thanks flake8! --- tests/test_pipeline_files.py | 2 +- tests/test_pipeline_images.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 5d381c018..d641e7a43 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -5,7 +5,7 @@ from datetime import datetime from io import BytesIO from shutil import rmtree from tempfile import mkdtemp -from unittest import mock, skipIf +from unittest import mock from urllib.parse import urlparse import dataclasses diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index e6f5bea21..0082e7a4e 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -3,7 +3,6 @@ import io import random from shutil import rmtree from tempfile import mkdtemp -from unittest import skipIf import dataclasses import attr From b103664bf45b079e5488b13a0737866de1b7dc50 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 19 Jul 2022 20:39:26 +0500 Subject: [PATCH 0575/2083] Address 2/3 of warnings from tests (#5561) --- pytest.ini | 3 + tests/test_contracts.py | 4 +- tests/test_crawl.py | 100 +++---- tests/test_crawler.py | 36 ++- tests/test_downloadermiddleware_httpauth.py | 14 +- tests/test_downloadermiddleware_httpproxy.py | 12 +- tests/test_downloadermiddleware_stats.py | 7 +- tests/test_dupefilters.py | 27 +- tests/test_engine.py | 228 ++++++++-------- tests/test_engine_stop_download_bytes.py | 32 ++- tests/test_engine_stop_download_headers.py | 32 ++- tests/test_feedexport.py | 269 +++++++------------ tests/test_logformatter.py | 6 +- tests/test_pipeline_crawl.py | 12 +- tests/test_request_attribute_binding.py | 21 +- tests/test_request_cb_kwargs.py | 5 +- tests/test_scheduler.py | 3 +- tests/test_scheduler_base.py | 15 +- tests/test_spiderloader/__init__.py | 5 +- tests/test_utils_project.py | 27 +- tests/test_utils_request.py | 13 +- tests/test_utils_response.py | 21 +- 22 files changed, 424 insertions(+), 468 deletions(-) diff --git a/pytest.ini b/pytest.ini index ae2ed2029..af0f2fb6e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -21,3 +21,6 @@ addopts = markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed +filterwarnings = + ignore:scrapy.downloadermiddlewares.decompression is deprecated + ignore:Module scrapy.utils.reqser is deprecated diff --git a/tests/test_contracts.py b/tests/test_contracts.py index d0f4a68c2..136056f50 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -5,11 +5,11 @@ from twisted.python import failure from twisted.trial import unittest from scrapy import FormRequest -from scrapy.crawler import CrawlerRunner from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.http import Request from scrapy.item import Item, Field +from scrapy.utils.test import get_crawler from scrapy.contracts import ContractsManager, Contract from scrapy.contracts.default import ( UrlContract, @@ -398,7 +398,7 @@ class ContractsManagerTest(unittest.TestCase): TestSameUrlSpider.parse_first.__doc__ = contract_doc TestSameUrlSpider.parse_second.__doc__ = contract_doc - crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) + crawler = get_crawler(TestSameUrlSpider) yield crawler.crawl() self.assertEqual(crawler.spider.visited, 2) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f9ffcd6bb..59c271868 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -18,6 +18,7 @@ from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import to_unicode +from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import MockServer from tests.spiders import ( @@ -49,14 +50,13 @@ class CrawlTestCase(TestCase): def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() - self.runner = CrawlerRunner() def tearDown(self): self.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_follow_all(self): - crawler = self.runner.create_crawler(FollowAllSpider) + crawler = get_crawler(FollowAllSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url @@ -79,7 +79,7 @@ class CrawlTestCase(TestCase): settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} - crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider) + crawler = get_crawler(FollowAllSpider, settings) yield crawler.crawl(**crawl_kwargs) times = crawler.spider.times total_time = times[-1] - times[0] @@ -92,7 +92,7 @@ class CrawlTestCase(TestCase): # of ``total`` and ``delay`` values that are too small for the test # code above to have any meaning. settings["DOWNLOAD_DELAY"] = 0 - crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider) + crawler = get_crawler(FollowAllSpider, settings) yield crawler.crawl(**crawl_kwargs) times = crawler.spider.times total_time = times[-1] - times[0] @@ -102,7 +102,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_success(self): - crawler = self.runner.create_crawler(DelaySpider) + crawler = get_crawler(DelaySpider) yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 > 0) @@ -110,7 +110,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_failure(self): - crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider) + crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) @@ -125,14 +125,14 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_retry_503(self): - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver) self._assert_retried(log) @defer.inlineCallbacks def test_retry_conn_failed(self): - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) with LogCapture() as log: yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver) self._assert_retried(log) @@ -141,7 +141,7 @@ class CrawlTestCase(TestCase): def test_retry_dns_error(self): if NON_EXISTING_RESOLVABLE: raise unittest.SkipTest("Non-existing hosts are resolvable") - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) with LogCapture() as log: # try to fetch the homepage of a non-existent domain yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) @@ -150,7 +150,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): with LogCapture('scrapy', level=logging.ERROR) as log: - crawler = self.runner.create_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) self.assertEqual(len(log.records), 1) @@ -161,7 +161,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_yielding(self): with LogCapture('scrapy', level=logging.ERROR) as log: - crawler = self.runner.create_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) self.assertEqual(len(log.records), 1) @@ -172,7 +172,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_lazyness(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenStartRequestsSpider, settings) yield crawler.crawl(mockserver=self.mockserver) self.assertTrue( crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), @@ -181,7 +181,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider) + crawler = get_crawler(DuplicateStartRequestsSpider, settings) yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver) self.assertEqual(crawler.spider.visited, 6) @@ -210,7 +210,7 @@ Connection: close foo body with multiples lines '''}) - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver) self.assertEqual(str(log).count("Got response 200"), 1) @@ -218,7 +218,7 @@ with multiples lines @defer.inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver) self._assert_retried(log) @@ -226,7 +226,7 @@ with multiples lines @defer.inlineCallbacks def test_retry_conn_aborted(self): # connection lost before receiving data - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver) self._assert_retried(log) @@ -245,7 +245,7 @@ with multiples lines req0.meta['next'] = req1 req1.meta['next'] = req2 req2.meta['next'] = req3 - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=req0, mockserver=self.mockserver) # basic asserts in case of weird communication errors self.assertIn('responses', crawler.spider.meta) @@ -271,7 +271,7 @@ with multiples lines def cb(response): est.append(get_engine_status(crawler.engine)) - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) self.assertEqual(len(est), 1, est) s = dict(est[0]) @@ -286,7 +286,7 @@ with multiples lines def cb(response): est.append(format_engine_status(crawler.engine)) - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) self.assertEqual(len(est), 1, est) est = est[0].split("\n")[2:-2] # remove header & footer @@ -317,7 +317,7 @@ with multiples lines def start_requests(self): raise TestError - crawler = self.runner.create_crawler(FaultySpider) + crawler = get_crawler(FaultySpider) yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) self.assertFalse(crawler.crawling) @@ -328,26 +328,28 @@ with multiples lines "tests.pipelines.ZeroDivisionErrorPipeline": 300, } } - crawler = CrawlerRunner(settings).create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider, settings) yield self.assertFailure( - self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver), + crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver), ZeroDivisionError) self.assertFalse(crawler.crawling) @defer.inlineCallbacks def test_crawlerrunner_accepts_crawler(self): - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) + runner = CrawlerRunner() with LogCapture() as log: - yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + yield runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) self.assertIn("Got response 200", str(log)) @defer.inlineCallbacks def test_crawl_multiple(self): - self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) - self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) + runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}) + runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) with LogCapture() as log: - yield self.runner.join() + yield runner.join() self._assert_retried(log) self.assertIn("Got response 200", str(log)) @@ -358,7 +360,6 @@ class CrawlSpiderTestCase(TestCase): def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() - self.runner = CrawlerRunner() def tearDown(self): self.mockserver.__exit__(None, None, None) @@ -370,7 +371,7 @@ class CrawlSpiderTestCase(TestCase): def _on_item_scraped(item): items.append(item) - crawler = self.runner.create_crawler(spider_cls) + crawler = get_crawler(spider_cls) crawler.signals.connect(_on_item_scraped, signals.item_scraped) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) @@ -378,10 +379,9 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_crawlspider_with_parse(self): - self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver) - + crawler = get_crawler(CrawlSpiderWithParseMethod) with LogCapture() as log: - yield self.runner.join() + yield crawler.crawl(mockserver=self.mockserver) self.assertIn("[parse] status 200 (foo: None)", str(log)) self.assertIn("[parse] status 201 (foo: None)", str(log)) @@ -389,10 +389,9 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_crawlspider_with_errback(self): - self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver) - + crawler = get_crawler(CrawlSpiderWithErrback) with LogCapture() as log: - yield self.runner.join() + yield crawler.crawl(mockserver=self.mockserver) self.assertIn("[parse] status 200 (foo: None)", str(log)) self.assertIn("[parse] status 201 (foo: None)", str(log)) @@ -403,18 +402,19 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_async_def_parse(self): - self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + crawler = get_crawler(AsyncDefSpider) with LogCapture() as log: - yield self.runner.join() + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) self.assertIn("Got response 200", str(log)) @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse(self): - runner = CrawlerRunner({"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"}) - runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + crawler = get_crawler(AsyncDefAsyncioSpider, { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + }) with LogCapture() as log: - yield runner.join() + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) self.assertIn("Got response 200", str(log)) @mark.only_asyncio() @@ -433,7 +433,7 @@ class CrawlSpiderTestCase(TestCase): def _on_item_scraped(item): items.append(item) - crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider) + crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) @@ -479,14 +479,14 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_response_ssl_certificate_none(self): - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=False) yield crawler.crawl(seed=url, mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta['responses'][0].certificate) @defer.inlineCallbacks def test_response_ssl_certificate(self): - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=True) yield crawler.crawl(seed=url, mockserver=self.mockserver) cert = crawler.spider.meta['responses'][0].certificate @@ -497,7 +497,7 @@ class CrawlSpiderTestCase(TestCase): @mark.xfail(reason="Responses with no body return early and contain no certificate") @defer.inlineCallbacks def test_response_ssl_certificate_empty_response(self): - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/status?n=200", is_secure=True) yield crawler.crawl(seed=url, mockserver=self.mockserver) cert = crawler.spider.meta['responses'][0].certificate @@ -507,7 +507,7 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_dns_server_ip_address_none(self): - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url('/status?n=200') yield crawler.crawl(seed=url, mockserver=self.mockserver) ip_address = crawler.spider.meta['responses'][0].ip_address @@ -515,7 +515,7 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_dns_server_ip_address(self): - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url('/echo?body=test') expected_netloc, _ = urlparse(url).netloc.split(':') yield crawler.crawl(seed=url, mockserver=self.mockserver) @@ -525,7 +525,7 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_bytes_received_stop_download_callback(self): - crawler = self.runner.create_crawler(BytesReceivedCallbackSpider) + crawler = get_crawler(BytesReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("failure")) self.assertIsInstance(crawler.spider.meta["response"], Response) @@ -534,7 +534,7 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_bytes_received_stop_download_errback(self): - crawler = self.runner.create_crawler(BytesReceivedErrbackSpider) + crawler = get_crawler(BytesReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("response")) self.assertIsInstance(crawler.spider.meta["failure"], Failure) @@ -549,7 +549,7 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_headers_received_stop_download_callback(self): - crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider) + crawler = get_crawler(HeadersReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("failure")) self.assertIsInstance(crawler.spider.meta["response"], Response) @@ -557,7 +557,7 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_headers_received_stop_download_errback(self): - crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider) + crawler = get_crawler(HeadersReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("response")) self.assertIsInstance(crawler.spider.meta["failure"], Failure) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f7aa769e4..d67abed7c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -13,11 +13,13 @@ from twisted.trial import unittest import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object +from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv @@ -34,9 +36,6 @@ class BaseCrawlerTest(unittest.TestCase): class CrawlerTestCase(BaseCrawlerTest): - def setUp(self): - self.crawler = Crawler(DefaultSpider, Settings()) - def test_populate_spidercls_settings(self): spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'} project_settings = {'TEST1': 'project', 'TEST3': 'project'} @@ -46,7 +45,9 @@ class CrawlerTestCase(BaseCrawlerTest): settings = Settings() settings.setdict(project_settings, priority='project') - crawler = Crawler(CustomSettingsSpider, settings) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + crawler = Crawler(CustomSettingsSpider, settings) self.assertEqual(crawler.settings.get('TEST1'), 'spider') self.assertEqual(crawler.settings.get('TEST2'), 'spider') @@ -56,12 +57,14 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertTrue(crawler.settings.frozen) def test_crawler_accepts_dict(self): - crawler = Crawler(DefaultSpider, {'foo': 'bar'}) + crawler = get_crawler(DefaultSpider, {'foo': 'bar'}) self.assertEqual(crawler.settings['foo'], 'bar') self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED') def test_crawler_accepts_None(self): - crawler = Crawler(DefaultSpider) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + crawler = Crawler(DefaultSpider) self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED') def test_crawler_rejects_spider_objects(self): @@ -77,7 +80,7 @@ class SpiderSettingsTestCase(unittest.TestCase): 'AUTOTHROTTLE_ENABLED': True } - crawler = Crawler(MySpider, {}) + crawler = get_crawler(MySpider) enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] self.assertIn(AutoThrottle, enabled_exts) @@ -91,7 +94,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): class MySpider(scrapy.Spider): name = 'spider' - Crawler(MySpider, {}) + get_crawler(MySpider) assert get_scrapy_root_handler() is None def test_spider_custom_settings_log_level(self): @@ -111,7 +114,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): configure_logging() self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) - crawler = Crawler(MySpider, {}) + crawler = get_crawler(MySpider) self.assertEqual(get_scrapy_root_handler().level, logging.INFO) info_count = crawler.stats.get_value('log_count/INFO') logging.debug('debug message') @@ -148,7 +151,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): } configure_logging() - Crawler(MySpider, {}) + get_crawler(MySpider) logging.debug('debug message') with open(log_file, 'rb') as fo: @@ -229,22 +232,25 @@ class NoRequestsSpider(scrapy.Spider): @mark.usefixtures('reactor_pytest') class CrawlerRunnerHasSpider(unittest.TestCase): + def _runner(self): + return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}) + @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful(self): - runner = CrawlerRunner() + runner = self._runner() yield runner.crawl(NoRequestsSpider) self.assertEqual(runner.bootstrap_failed, False) @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): - runner = CrawlerRunner() + runner = self._runner() yield runner.crawl(NoRequestsSpider) yield runner.crawl(NoRequestsSpider) self.assertEqual(runner.bootstrap_failed, False) @defer.inlineCallbacks def test_crawler_runner_bootstrap_failed(self): - runner = CrawlerRunner() + runner = self._runner() try: yield runner.crawl(ExceptionSpider) @@ -257,7 +263,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): @defer.inlineCallbacks def test_crawler_runner_bootstrap_failed_for_several(self): - runner = CrawlerRunner() + runner = self._runner() try: yield runner.crawl(ExceptionSpider) @@ -275,12 +281,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION", }) else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION", }) yield runner.crawl(NoRequestsSpider) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 0362e2018..b9f3e24a4 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,7 +1,9 @@ import unittest +import pytest from w3lib.http import basic_auth_header +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.spiders import Spider @@ -30,8 +32,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): self.spider = TestSpiderLegacy('foo') def test_auth(self): - mw = HttpAuthMiddleware() - mw.spider_opened(self.spider) + with pytest.warns(ScrapyDeprecationWarning, + match="Using HttpAuthMiddleware without http_auth_domain is deprecated"): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) # initial request, sets the domain and sends the header req = Request('http://example.com/') @@ -49,8 +53,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): self.assertNotIn('Authorization', req.headers) def test_auth_already_set(self): - mw = HttpAuthMiddleware() - mw.spider_opened(self.spider) + with pytest.warns(ScrapyDeprecationWarning, + match="Using HttpAuthMiddleware without http_auth_domain is deprecated"): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) req = Request('http://example.com/', headers=dict(Authorization='Digest 123')) assert mw.process_request(req, self.spider) is None diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 7c97bf32a..4ac85c1ec 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -1,13 +1,13 @@ import os -from functools import partial + +import pytest from twisted.trial.unittest import TestCase from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.crawler import Crawler -from scrapy.settings import Settings +from scrapy.utils.test import get_crawler spider = Spider('foo') @@ -23,9 +23,9 @@ class TestHttpProxyMiddleware(TestCase): os.environ = self._oldenv def test_not_enabled(self): - settings = Settings({'HTTPPROXY_ENABLED': False}) - crawler = Crawler(Spider, settings) - self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler)) + crawler = get_crawler(Spider, {'HTTPPROXY_ENABLED': False}) + with pytest.raises(NotConfigured): + HttpProxyMiddleware.from_crawler(crawler) def test_no_environment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 9e75f0a50..7d88ba4d2 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,7 +1,9 @@ +import warnings from itertools import product from unittest import TestCase from scrapy.downloadermiddlewares.stats import DownloaderStats +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.response import response_httprepr @@ -54,7 +56,10 @@ class TestDownloaderStats(TestCase): for test_response in test_responses: self.crawler.stats.set_value('downloader/response_bytes', 0) self.mw.process_response(self.req, test_response, self.spider) - self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response))) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + resp_size = len(response_httprepr(test_response)) + self.assertStatsEqual('downloader/response_bytes', resp_size) def test_process_exception(self): self.mw.process_exception(self.req, MyException(), self.spider) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index b7df2554a..8a37a8ebe 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -10,7 +10,6 @@ from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request from scrapy.core.scheduler import Scheduler from scrapy.utils.python import to_bytes -from scrapy.utils.job import job_dir from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider @@ -29,8 +28,7 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter): @classmethod def from_crawler(cls, crawler): - debug = crawler.settings.getbool('DUPEFILTER_DEBUG') - df = cls(job_dir(crawler.settings), debug) + df = super().from_crawler(crawler) df.method = 'from_crawler' return df @@ -38,9 +36,8 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter): class FromSettingsRFPDupeFilter(RFPDupeFilter): @classmethod - def from_settings(cls, settings): - debug = settings.getbool('DUPEFILTER_DEBUG') - df = cls(job_dir(settings), debug) + def from_settings(cls, settings, *, fingerprinter=None): + df = super().from_settings(settings, fingerprinter=fingerprinter) df.method = 'from_settings' return df @@ -53,7 +50,8 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_crawler_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -61,14 +59,16 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_settings_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter} + 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter, + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'from_settings') def test_df_direct_scheduler(self): - settings = {'DUPEFILTER_CLASS': DirectDupeFilter} + settings = {'DUPEFILTER_CLASS': DirectDupeFilter, + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, 'n/a') @@ -171,7 +171,8 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, - 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) @@ -197,7 +198,8 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) @@ -230,7 +232,8 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug_default_dupefilter(self): with LogCapture() as log: - settings = {'DUPEFILTER_DEBUG': True} + settings = {'DUPEFILTER_DEBUG': True, + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) diff --git a/tests/test_engine.py b/tests/test_engine.py index fa7d0c8d4..1bd802bcf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -13,10 +13,11 @@ module with the ``runserver`` argument:: import os import re import sys -import warnings from collections import defaultdict from urllib.parse import urlparse +from dataclasses import dataclass +import pytest import attr from itemadapter import ItemAdapter from pydispatch import dispatcher @@ -50,6 +51,13 @@ class AttrsItem: price = attr.ib(default=0) +@dataclass +class DataClassItem: + name: str = "" + url: str = "" + price: int = 0 + + class TestSpider(Spider): name = "scrapytest.org" allowed_domains = ["scrapytest.org", "localhost"] @@ -92,17 +100,8 @@ class AttrsItemsSpider(TestSpider): item_cls = AttrsItem -try: - from dataclasses import make_dataclass -except ImportError: - DataClassItemsSpider = None -else: - TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) - - class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef] - def parse_item(self, response): - item = super().parse_item(response) - return TestDataClass(**item) +class DataClassItemsSpider(TestSpider): + item_cls = DataClassItem class ItemZeroDivisionErrorSpider(TestSpider): @@ -188,7 +187,7 @@ class CrawlerRun: return self.deferred def stop(self): - self.port.stopListening() + self.port.stopListening() # FIXME: wait for this Deferred for name, signal in vars(signals).items(): if not name.startswith('_'): disconnect_all(signal) @@ -239,79 +238,77 @@ class EngineTest(unittest.TestCase): def test_crawler(self): for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): - if spider is None: - continue - self.run = CrawlerRun(spider) - yield self.run.run() - self._assert_visited_urls() - self._assert_scheduled_requests(count=9) - self._assert_downloaded_responses(count=9) - self._assert_scraped_items() - self._assert_signals_caught() - self._assert_bytes_received() + run = CrawlerRun(spider) + yield run.run() + self._assert_visited_urls(run) + self._assert_scheduled_requests(run, count=9) + self._assert_downloaded_responses(run, count=9) + self._assert_scraped_items(run) + self._assert_signals_caught(run) + self._assert_bytes_received(run) @defer.inlineCallbacks def test_crawler_dupefilter(self): - self.run = CrawlerRun(TestDupeFilterSpider) - yield self.run.run() - self._assert_scheduled_requests(count=8) - self._assert_dropped_requests() + run = CrawlerRun(TestDupeFilterSpider) + yield run.run() + self._assert_scheduled_requests(run, count=8) + self._assert_dropped_requests(run) @defer.inlineCallbacks def test_crawler_itemerror(self): - self.run = CrawlerRun(ItemZeroDivisionErrorSpider) - yield self.run.run() - self._assert_items_error() + run = CrawlerRun(ItemZeroDivisionErrorSpider) + yield run.run() + self._assert_items_error(run) @defer.inlineCallbacks def test_crawler_change_close_reason_on_idle(self): - self.run = CrawlerRun(ChangeCloseReasonSpider) - yield self.run.run() - self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'}, - self.run.signals_caught[signals.spider_closed]) + run = CrawlerRun(ChangeCloseReasonSpider) + yield run.run() + self.assertEqual({'spider': run.spider, 'reason': 'custom_reason'}, + run.signals_caught[signals.spider_closed]) - def _assert_visited_urls(self): + def _assert_visited_urls(self, run: CrawlerRun): must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"] - urls_visited = {rp[0].url for rp in self.run.respplug} - urls_expected = {self.run.geturl(p) for p in must_be_visited} + urls_visited = {rp[0].url for rp in run.respplug} + urls_expected = {run.geturl(p) for p in must_be_visited} assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" - def _assert_scheduled_requests(self, count=None): - self.assertEqual(count, len(self.run.reqplug)) + def _assert_scheduled_requests(self, run: CrawlerRun, count=None): + self.assertEqual(count, len(run.reqplug)) paths_expected = ['/item999.html', '/item2.html', '/item1.html'] - urls_requested = {rq[0].url for rq in self.run.reqplug} - urls_expected = {self.run.geturl(p) for p in paths_expected} + urls_requested = {rq[0].url for rq in run.reqplug} + urls_expected = {run.geturl(p) for p in paths_expected} assert urls_expected <= urls_requested - scheduled_requests_count = len(self.run.reqplug) - dropped_requests_count = len(self.run.reqdropped) - responses_count = len(self.run.respplug) + scheduled_requests_count = len(run.reqplug) + dropped_requests_count = len(run.reqdropped) + responses_count = len(run.respplug) self.assertEqual(scheduled_requests_count, dropped_requests_count + responses_count) - self.assertEqual(len(self.run.reqreached), + self.assertEqual(len(run.reqreached), responses_count) - def _assert_dropped_requests(self): - self.assertEqual(len(self.run.reqdropped), 1) + def _assert_dropped_requests(self, run: CrawlerRun): + self.assertEqual(len(run.reqdropped), 1) - def _assert_downloaded_responses(self, count): + def _assert_downloaded_responses(self, run: CrawlerRun, count): # response tests - self.assertEqual(count, len(self.run.respplug)) - self.assertEqual(count, len(self.run.reqreached)) + self.assertEqual(count, len(run.respplug)) + self.assertEqual(count, len(run.reqreached)) - for response, _ in self.run.respplug: - if self.run.getpath(response.url) == '/item999.html': + for response, _ in run.respplug: + if run.getpath(response.url) == '/item999.html': self.assertEqual(404, response.status) - if self.run.getpath(response.url) == '/redirect': + if run.getpath(response.url) == '/redirect': self.assertEqual(302, response.status) - def _assert_items_error(self): - self.assertEqual(2, len(self.run.itemerror)) - for item, response, spider, failure in self.run.itemerror: + def _assert_items_error(self, run: CrawlerRun): + self.assertEqual(2, len(run.itemerror)) + for item, response, spider, failure in run.itemerror: self.assertEqual(failure.value.__class__, ZeroDivisionError) - self.assertEqual(spider, self.run.spider) + self.assertEqual(spider, run.spider) self.assertEqual(item['url'], response.url) if 'item1.html' in item['url']: @@ -321,9 +318,9 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) - def _assert_scraped_items(self): - self.assertEqual(2, len(self.run.itemresp)) - for item, response in self.run.itemresp: + def _assert_scraped_items(self, run: CrawlerRun): + self.assertEqual(2, len(run.itemresp)) + for item, response in run.itemresp: item = ItemAdapter(item) self.assertEqual(item['url'], response.url) if 'item1.html' in item['url']: @@ -333,26 +330,26 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) - def _assert_headers_received(self): - for headers in self.run.headers.values(): + def _assert_headers_received(self, run: CrawlerRun): + for headers in run.headers.values(): self.assertIn(b"Server", headers) self.assertIn(b"TwistedWeb", headers[b"Server"]) self.assertIn(b"Date", headers) self.assertIn(b"Content-Type", headers) - def _assert_bytes_received(self): - self.assertEqual(9, len(self.run.bytes)) - for request, data in self.run.bytes.items(): + def _assert_bytes_received(self, run: CrawlerRun): + self.assertEqual(9, len(run.bytes)) + for request, data in run.bytes.items(): joined_data = b"".join(data) - if self.run.getpath(request.url) == "/": + if run.getpath(request.url) == "/": self.assertEqual(joined_data, get_testdata("test_site", "index.html")) - elif self.run.getpath(request.url) == "/item1.html": + elif run.getpath(request.url) == "/item1.html": self.assertEqual(joined_data, get_testdata("test_site", "item1.html")) - elif self.run.getpath(request.url) == "/item2.html": + elif run.getpath(request.url) == "/item2.html": self.assertEqual(joined_data, get_testdata("test_site", "item2.html")) - elif self.run.getpath(request.url) == "/redirected": + elif run.getpath(request.url) == "/redirected": self.assertEqual(joined_data, b"Redirected here") - elif self.run.getpath(request.url) == '/redirect': + elif run.getpath(request.url) == '/redirect': self.assertEqual( joined_data, b"\n\n" @@ -364,7 +361,7 @@ class EngineTest(unittest.TestCase): b" \n" b"\n" ) - elif self.run.getpath(request.url) == "/tem999.html": + elif run.getpath(request.url) == "/tem999.html": self.assertEqual( joined_data, b"\n\n" @@ -375,27 +372,27 @@ class EngineTest(unittest.TestCase): b" \n" b"\n" ) - elif self.run.getpath(request.url) == "/numbers": + elif run.getpath(request.url) == "/numbers": # signal was fired multiple times self.assertTrue(len(data) > 1) # bytes were received in order numbers = [str(x).encode("utf8") for x in range(2**18)] self.assertEqual(joined_data, b"".join(numbers)) - def _assert_signals_caught(self): - assert signals.engine_started in self.run.signals_caught - assert signals.engine_stopped in self.run.signals_caught - assert signals.spider_opened in self.run.signals_caught - assert signals.spider_idle in self.run.signals_caught - assert signals.spider_closed in self.run.signals_caught - assert signals.headers_received in self.run.signals_caught + def _assert_signals_caught(self, run: CrawlerRun): + assert signals.engine_started in run.signals_caught + assert signals.engine_stopped in run.signals_caught + assert signals.spider_opened in run.signals_caught + assert signals.spider_idle in run.signals_caught + assert signals.spider_closed in run.signals_caught + assert signals.headers_received in run.signals_caught - self.assertEqual({'spider': self.run.spider}, - self.run.signals_caught[signals.spider_opened]) - self.assertEqual({'spider': self.run.spider}, - self.run.signals_caught[signals.spider_idle]) - self.assertEqual({'spider': self.run.spider, 'reason': 'finished'}, - self.run.signals_caught[signals.spider_closed]) + self.assertEqual({'spider': run.spider}, + run.signals_caught[signals.spider_opened]) + self.assertEqual({'spider': run.spider}, + run.signals_caught[signals.spider_idle]) + self.assertEqual({'spider': run.spider, 'reason': 'finished'}, + run.signals_caught[signals.spider_closed]) @defer.inlineCallbacks def test_close_downloader(self): @@ -407,28 +404,29 @@ class EngineTest(unittest.TestCase): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() - yield self.assertFailure(e.start(), RuntimeError).addBoth( - lambda exc: self.assertEqual(str(exc), "Engine already running") - ) - yield e.stop() + try: + yield self.assertFailure(e.start(), RuntimeError).addBoth( + lambda exc: self.assertEqual(str(exc), "Engine already running") + ) + finally: + yield e.stop() @defer.inlineCallbacks def test_close_spiders_downloader(self): - with warnings.catch_warnings(record=True) as warning_list: + with pytest.warns(ScrapyDeprecationWarning, + match="ExecutionEngine.open_spiders is deprecated, " + "please use ExecutionEngine.spider instead"): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) self.assertEqual(len(e.open_spiders), 1) yield e.close() self.assertEqual(len(e.open_spiders), 0) - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) - self.assertEqual( - str(warning_list[0].message), - "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", - ) @defer.inlineCallbacks def test_close_engine_spiders_downloader(self): - with warnings.catch_warnings(record=True) as warning_list: + with pytest.warns(ScrapyDeprecationWarning, + match="ExecutionEngine.open_spiders is deprecated, " + "please use ExecutionEngine.spider instead"): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() @@ -436,61 +434,47 @@ class EngineTest(unittest.TestCase): yield e.close() self.assertFalse(e.running) self.assertEqual(len(e.open_spiders), 0) - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) - self.assertEqual( - str(warning_list[0].message), - "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", - ) @defer.inlineCallbacks def test_crawl_deprecated_spider_arg(self): - with warnings.catch_warnings(record=True) as warning_list: + with pytest.warns(ScrapyDeprecationWarning, + match="Passing a 'spider' argument to " + "ExecutionEngine.crawl is deprecated"): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) spider = TestSpider() yield e.open_spider(spider, []) e.start() e.crawl(Request("data:,"), spider) yield e.close() - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) - self.assertEqual( - str(warning_list[0].message), - "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", - ) @defer.inlineCallbacks def test_download_deprecated_spider_arg(self): - with warnings.catch_warnings(record=True) as warning_list: + with pytest.warns(ScrapyDeprecationWarning, + match="Passing a 'spider' argument to " + "ExecutionEngine.download is deprecated"): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) spider = TestSpider() yield e.open_spider(spider, []) e.start() e.download(Request("data:,"), spider) yield e.close() - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) - self.assertEqual( - str(warning_list[0].message), - "Passing a 'spider' argument to ExecutionEngine.download is deprecated", - ) @defer.inlineCallbacks def test_deprecated_schedule(self): - with warnings.catch_warnings(record=True) as warning_list: + with pytest.warns(ScrapyDeprecationWarning, + match="ExecutionEngine.schedule is deprecated, please use " + "ExecutionEngine.crawl or ExecutionEngine.download instead"): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) spider = TestSpider() yield e.open_spider(spider, []) e.start() e.schedule(Request("data:,"), spider) yield e.close() - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) - self.assertEqual( - str(warning_list[0].message), - "ExecutionEngine.schedule is deprecated, please use " - "ExecutionEngine.crawl or ExecutionEngine.download instead", - ) @defer.inlineCallbacks def test_deprecated_has_capacity(self): - with warnings.catch_warnings(record=True) as warning_list: + with pytest.warns(ScrapyDeprecationWarning, + match="ExecutionEngine.has_capacity is deprecated"): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) self.assertTrue(e.has_capacity()) spider = TestSpider() @@ -499,8 +483,6 @@ class EngineTest(unittest.TestCase): e.start() yield e.close() self.assertTrue(e.has_capacity()) - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) - self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated") if __name__ == "__main__": diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 0ba69e096..933e4067d 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -23,36 +23,34 @@ class BytesReceivedEngineTest(EngineTest): @defer.inlineCallbacks def test_crawler(self): for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): - if spider is None: - continue - self.run = BytesReceivedCrawlerRun(spider) + run = BytesReceivedCrawlerRun(spider) with LogCapture() as log: - yield self.run.run() + yield run.run() log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - f"Download stopped for " + f"Download stopped for " "from signal handler BytesReceivedCrawlerRun.bytes_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - f"Download stopped for " + f"Download stopped for " "from signal handler BytesReceivedCrawlerRun.bytes_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - f"Download stopped for " + f"Download stopped for " "from signal handler BytesReceivedCrawlerRun.bytes_received")) - self._assert_visited_urls() - self._assert_scheduled_requests(count=9) - self._assert_downloaded_responses(count=9) - self._assert_signals_caught() - self._assert_headers_received() - self._assert_bytes_received() + self._assert_visited_urls(run) + self._assert_scheduled_requests(run, count=9) + self._assert_downloaded_responses(run, count=9) + self._assert_signals_caught(run) + self._assert_headers_received(run) + self._assert_bytes_received(run) - def _assert_bytes_received(self): - self.assertEqual(9, len(self.run.bytes)) - for request, data in self.run.bytes.items(): + def _assert_bytes_received(self, run: CrawlerRun): + self.assertEqual(9, len(run.bytes)) + for request, data in run.bytes.items(): joined_data = b"".join(data) self.assertTrue(len(data) == 1) # signal was fired only once - if self.run.getpath(request.url) == "/numbers": + if run.getpath(request.url) == "/numbers": # Received bytes are not the complete response. The exact amount depends # on the buffer size, which can vary, so we only check that the amount # of received bytes is strictly less than the full response. diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index fad6643ad..8975d0e3f 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -23,34 +23,32 @@ class HeadersReceivedEngineTest(EngineTest): @defer.inlineCallbacks def test_crawler(self): for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): - if spider is None: - continue - self.run = HeadersReceivedCrawlerRun(spider) + run = HeadersReceivedCrawlerRun(spider) with LogCapture() as log: - yield self.run.run() + yield run.run() log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - f"Download stopped for from" + f"Download stopped for from" " signal handler HeadersReceivedCrawlerRun.headers_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - f"Download stopped for from signal" + f"Download stopped for from signal" " handler HeadersReceivedCrawlerRun.headers_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - f"Download stopped for from" + f"Download stopped for from" " signal handler HeadersReceivedCrawlerRun.headers_received")) - self._assert_visited_urls() - self._assert_downloaded_responses(count=6) - self._assert_signals_caught() - self._assert_bytes_received() - self._assert_headers_received() + self._assert_visited_urls(run) + self._assert_downloaded_responses(run, count=6) + self._assert_signals_caught(run) + self._assert_bytes_received(run) + self._assert_headers_received(run) - def _assert_bytes_received(self): - self.assertEqual(0, len(self.run.bytes)) + def _assert_bytes_received(self, run: CrawlerRun): + self.assertEqual(0, len(run.bytes)) - def _assert_visited_urls(self): + def _assert_visited_urls(self, run: CrawlerRun): must_be_visited = ["/", "/redirect", "/redirected"] - urls_visited = {rp[0].url for rp in self.run.respplug} - urls_expected = {self.run.geturl(p) for p in must_be_visited} + urls_visited = {rp[0].url for rp in run.respplug} + urls_expected = {run.geturl(p) for p in must_be_visited} assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ec48f8d4a..ecd1b59d3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -22,6 +22,7 @@ from urllib.parse import urljoin, quote from urllib.request import pathname2url import lxml.etree +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial import unittest @@ -30,7 +31,6 @@ from zope.interface import implementer from zope.interface.verify import verifyObject import scrapy -from scrapy.crawler import CrawlerRunner from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( @@ -697,9 +697,9 @@ class FeedExportTest(FeedExportTestBase): content = {} try: with MockServer() as s: - runner = CrawlerRunner(Settings(settings)) spider_cls.start_urls = [s.url('/')] - yield runner.crawl(spider_cls) + crawler = get_crawler(spider_cls, settings) + yield crawler.crawl() for file_path, feed_options in FEEDS.items(): if not os.path.exists(str(file_path)): @@ -1554,9 +1554,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): content = {} try: with MockServer() as s: - runner = CrawlerRunner(Settings(settings)) spider_cls.start_urls = [s.url('/')] - yield runner.crawl(spider_cls) + crawler = get_crawler(spider_cls, settings) + yield crawler.crawl() for file_path, feed_options in FEEDS.items(): if not os.path.exists(str(file_path)): @@ -2026,9 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase): content = defaultdict(list) try: with MockServer() as s: - runner = CrawlerRunner(Settings(settings)) spider_cls.start_urls = [s.url('/')] - yield runner.crawl(spider_cls) + crawler = get_crawler(spider_cls, settings) + yield crawler.crawl() for path, feed in FEEDS.items(): dir_name = os.path.dirname(path) @@ -2048,7 +2048,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -2064,7 +2064,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -2080,7 +2080,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -2098,7 +2098,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -2123,7 +2123,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -2140,7 +2140,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -2166,7 +2166,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'FEED_EXPORT_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, settings=Settings(settings)) + yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """ @@ -2382,9 +2382,9 @@ class BatchDeliveriesTest(FeedExportTestBase): yield item with MockServer() as server: - runner = CrawlerRunner(Settings(settings)) TestSpider.start_urls = [server.url('/')] - yield runner.crawl(TestSpider) + crawler = get_crawler(TestSpider, settings) + yield crawler.crawl() self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1) for stub in CustomS3FeedStorage.stubs[:-1]: @@ -2434,25 +2434,16 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): 'file': StdoutFeedStorageWithoutFeedOptions }, } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") - with warnings.catch_warnings(record=True) as w: + with pytest.warns(ScrapyDeprecationWarning, + match="StdoutFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument."): feed_exporter.open_spider(spider) - messages = tuple(str(item.message) for item in w - if item.category is ScrapyDeprecationWarning) - self.assertEqual( - messages, - ( - ( - "StdoutFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove " - "this warning. This parameter will become mandatory " - "in a future version of Scrapy." - ), - ) - ) class FileFeedStorageWithoutFeedOptions(FileFeedStorage): @@ -2476,25 +2467,16 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): 'file': FileFeedStorageWithoutFeedOptions }, } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) spider = scrapy.Spider("default") - with warnings.catch_warnings(record=True) as w: + + with pytest.warns(ScrapyDeprecationWarning, + match="FileFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument."): feed_exporter.open_spider(spider) - messages = tuple(str(item.message) for item in w - if item.category is ScrapyDeprecationWarning) - self.assertEqual( - messages, - ( - ( - "FileFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove " - "this warning. This parameter will become mandatory " - "in a future version of Scrapy." - ), - ) - ) class S3FeedStorageWithoutFeedOptions(S3FeedStorage): @@ -2524,26 +2506,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): 'file': S3FeedStorageWithoutFeedOptions }, } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with pytest.warns(ScrapyDeprecationWarning, + match="S3FeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument."): feed_exporter.open_spider(spider) - messages = tuple(str(item.message) for item in w - if item.category is ScrapyDeprecationWarning) - self.assertEqual( - messages, - ( - ( - "S3FeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove " - "this warning. This parameter will become mandatory " - "in a future version of Scrapy." - ), - ) - ) def test_from_crawler(self): settings_dict = { @@ -2552,26 +2526,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): 'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler }, } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with pytest.warns(ScrapyDeprecationWarning, + match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " + "the 'feed_options' keyword argument."): feed_exporter.open_spider(spider) - messages = tuple(str(item.message) for item in w - if item.category is ScrapyDeprecationWarning) - self.assertEqual( - messages, - ( - ( - "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler " - "does not support the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove " - "this warning. This parameter will become mandatory " - "in a future version of Scrapy." - ), - ) - ) class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage): @@ -2601,26 +2567,18 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): 'file': FTPFeedStorageWithoutFeedOptions }, } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with pytest.warns(ScrapyDeprecationWarning, + match="FTPFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument."): feed_exporter.open_spider(spider) - messages = tuple(str(item.message) for item in w - if item.category is ScrapyDeprecationWarning) - self.assertEqual( - messages, - ( - ( - "FTPFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove " - "this warning. This parameter will become mandatory " - "in a future version of Scrapy." - ), - ) - ) def test_from_crawler(self): settings_dict = { @@ -2629,50 +2587,50 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): 'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler }, } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with pytest.warns(ScrapyDeprecationWarning, + match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " + "the 'feed_options' keyword argument."): feed_exporter.open_spider(spider) - messages = tuple(str(item.message) for item in w - if item.category is ScrapyDeprecationWarning) - self.assertEqual( - messages, - ( - ( - "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler " - "does not support the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove " - "this warning. This parameter will become mandatory " - "in a future version of Scrapy." - ), - ) - ) class URIParamsTest: spider_name = "uri_params_spider" + deprecated_options = False def build_settings(self, uri='file:///tmp/foobar', uri_params=None): raise NotImplementedError + def _crawler_feed_exporter(self, settings): + if self.deprecated_options: + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + else: + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + return crawler, feed_exporter + def test_default(self): settings = self.build_settings( uri='file:///tmp/%(name)s', ) - crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) + crawler, feed_exporter = self._crawler_feed_exporter(settings) spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - messages = tuple( - str(item.message) for item in w - if item.category is ScrapyDeprecationWarning - ) - self.assertEqual(messages, tuple()) self.assertEqual( feed_exporter.slots[0].uri, @@ -2687,28 +2645,13 @@ class URIParamsTest: uri='file:///tmp/%(name)s', uri_params=uri_params, ) - crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) + crawler, feed_exporter = self._crawler_feed_exporter(settings) spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with pytest.warns(ScrapyDeprecationWarning, + match="Modifying the params dictionary in-place"): feed_exporter.open_spider(spider) - messages = tuple( - str(item.message) for item in w - if item.category is ScrapyDeprecationWarning - ) - self.assertEqual( - messages, - ( - ( - 'Modifying the params dictionary in-place in the ' - 'function defined in the FEED_URI_PARAMS setting or ' - 'in the uri_params key of the FEEDS setting is ' - 'deprecated. The function must return a new ' - 'dictionary instead.' - ), - ) - ) self.assertEqual( feed_exporter.slots[0].uri, @@ -2723,18 +2666,14 @@ class URIParamsTest: uri='file:///tmp/%(name)s', uri_params=uri_params, ) - crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) + crawler, feed_exporter = self._crawler_feed_exporter(settings) spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) with self.assertRaises(KeyError): feed_exporter.open_spider(spider) - messages = tuple( - str(item.message) for item in w - if item.category is ScrapyDeprecationWarning - ) - self.assertEqual(messages, tuple()) def test_params_as_is(self): def uri_params(params, spider): @@ -2744,17 +2683,12 @@ class URIParamsTest: uri='file:///tmp/%(name)s', uri_params=uri_params, ) - crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) + crawler, feed_exporter = self._crawler_feed_exporter(settings) spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - messages = tuple( - str(item.message) for item in w - if item.category is ScrapyDeprecationWarning - ) - self.assertEqual(messages, tuple()) self.assertEqual( feed_exporter.slots[0].uri, @@ -2769,17 +2703,12 @@ class URIParamsTest: uri='file:///tmp/%(foo)s', uri_params=uri_params, ) - crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) + crawler, feed_exporter = self._crawler_feed_exporter(settings) spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - messages = tuple( - str(item.message) for item in w - if item.category is ScrapyDeprecationWarning - ) - self.assertEqual(messages, tuple()) self.assertEqual( feed_exporter.slots[0].uri, @@ -2788,6 +2717,7 @@ class URIParamsTest: class URIParamsSettingTest(URIParamsTest, unittest.TestCase): + deprecated_options = True def build_settings(self, uri='file:///tmp/foobar', uri_params=None): extra_settings = {} @@ -2800,6 +2730,7 @@ class URIParamsSettingTest(URIParamsTest, unittest.TestCase): class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase): + deprecated_options = False def build_settings(self, uri='file:///tmp/foobar', uri_params=None): options = { diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 6381f895b..f3bb23bda 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -5,8 +5,8 @@ from twisted.internet import defer from twisted.python.failure import Failure from twisted.trial.unittest import TestCase as TwistedTestCase -from scrapy.crawler import CrawlerRunner from scrapy.exceptions import DropItem +from scrapy.utils.test import get_crawler from scrapy.http import Request, Response from scrapy.item import Item, Field from scrapy.logformatter import LogFormatter @@ -202,7 +202,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): @defer.inlineCallbacks def test_show_messages(self): - crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider) + crawler = get_crawler(ItemSpider, self.base_settings) with LogCapture() as lc: yield crawler.crawl(mockserver=self.mockserver) self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc)) @@ -213,7 +213,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): def test_skip_messages(self): settings = self.base_settings.copy() settings['LOG_FORMATTER'] = SkipMessagesLogFormatter - crawler = CrawlerRunner(settings).create_crawler(ItemSpider) + crawler = get_crawler(ItemSpider, settings) with LogCapture() as lc: yield crawler.crawl(mockserver=self.mockserver) self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc)) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index f49fda701..e46532a1c 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -64,6 +64,7 @@ class FileDownloadCrawlTestCase(TestCase): self.tmpmediastore = self.mktemp() os.mkdir(self.tmpmediastore) self.settings = { + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', 'ITEM_PIPELINES': {self.pipeline_class: 1}, self.store_setting_key: self.tmpmediastore, } @@ -78,8 +79,10 @@ class FileDownloadCrawlTestCase(TestCase): def _on_item_scraped(self, item): self.items.append(item) - def _create_crawler(self, spider_class, **kwargs): - crawler = self.runner.create_crawler(spider_class, **kwargs) + def _create_crawler(self, spider_class, runner=None, **kwargs): + if runner is None: + runner = self.runner + crawler = runner.create_crawler(spider_class, **kwargs) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) return crawler @@ -167,9 +170,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_redirected_allowed(self): settings = dict(self.settings) settings.update({'MEDIA_ALLOW_REDIRECTS': True}) - self.runner = CrawlerRunner(settings) - - crawler = self._create_crawler(RedirectedMediaDownloadSpider) + runner = CrawlerRunner(settings) + crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner) with LogCapture() as log: yield crawler.crawl( self.mockserver.url("/files/images/"), diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 25d9657d5..0406d906f 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -2,8 +2,8 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy import Request, signals -from scrapy.crawler import CrawlerRunner from scrapy.http.response import Response +from scrapy.utils.test import get_crawler from testfixtures import LogCapture @@ -71,7 +71,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_response_200(self): url = self.mockserver.url("/status?n=200") - crawler = CrawlerRunner().create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] self.assertEqual(response.request.url, url) @@ -80,7 +80,7 @@ class CrawlTestCase(TestCase): def test_response_error(self): for status in ("404", "500"): url = self.mockserver.url(f"/status?n={status}") - crawler = CrawlerRunner().create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] response = failure.value.response @@ -90,12 +90,11 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_downloader_middleware_raise_exception(self): url = self.mockserver.url("/status?n=200") - runner = CrawlerRunner(settings={ + crawler = get_crawler(SingleRequestSpider, { "DOWNLOADER_MIDDLEWARES": { RaiseExceptionRequestMiddleware: 590, }, }) - crawler = runner.create_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] self.assertEqual(failure.request.url, url) @@ -117,12 +116,11 @@ class CrawlTestCase(TestCase): signal_params["request"] = request url = self.mockserver.url("/status?n=200") - runner = CrawlerRunner(settings={ + crawler = get_crawler(SingleRequestSpider, { "DOWNLOADER_MIDDLEWARES": { ProcessResponseMiddleware: 595, } }) - crawler = runner.create_crawler(SingleRequestSpider) crawler.signals.connect(signal_handler, signal=signals.response_received) with LogCapture() as log: @@ -147,13 +145,12 @@ class CrawlTestCase(TestCase): The spider callback should receive the overridden response.request """ url = self.mockserver.url("/status?n=200") - runner = CrawlerRunner(settings={ + crawler = get_crawler(SingleRequestSpider, { "DOWNLOADER_MIDDLEWARES": { RaiseExceptionRequestMiddleware: 590, CatchExceptionOverrideRequestMiddleware: 595, }, }) - crawler = runner.create_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] self.assertEqual(response.body, b"Caught ZeroDivisionError") @@ -168,13 +165,12 @@ class CrawlTestCase(TestCase): The spider callback should receive the original response.request """ url = self.mockserver.url("/status?n=200") - runner = CrawlerRunner(settings={ + crawler = get_crawler(SingleRequestSpider, { "DOWNLOADER_MIDDLEWARES": { RaiseExceptionRequestMiddleware: 590, CatchExceptionDoNotOverrideRequestMiddleware: 595, }, }) - crawler = runner.create_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] self.assertEqual(response.body, b"Caught ZeroDivisionError") @@ -186,12 +182,11 @@ class CrawlTestCase(TestCase): Downloader middleware which returns a response with a specific 'request' attribute, with an alternative callback """ - runner = CrawlerRunner(settings={ + crawler = get_crawler(AlternativeCallbacksSpider, { "DOWNLOADER_MIDDLEWARES": { AlternativeCallbacksMiddleware: 595, } }) - crawler = runner.create_crawler(AlternativeCallbacksSpider) with LogCapture() as log: url = self.mockserver.url("/status?n=200") diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 473a93e69..002a04358 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -3,7 +3,7 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.http import Request -from scrapy.crawler import CrawlerRunner +from scrapy.utils.test import get_crawler from tests.spiders import MockServerSpider from tests.mockserver import MockServer @@ -140,14 +140,13 @@ class CallbackKeywordArgumentsTestCase(TestCase): def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() - self.runner = CrawlerRunner() def tearDown(self): self.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_callback_kwargs(self): - crawler = self.runner.create_crawler(KeywordArgumentsSpider) + crawler = get_crawler(KeywordArgumentsSpider) with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) self.assertTrue(all(crawler.spider.checks)) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 2d4bfa165..ac66056ba 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -52,6 +52,7 @@ class MockCrawler(Crawler): SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, JOBDIR=jobdir, DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', + REQUEST_FINGERPRINTER_IMPLEMENTATION='VERSION', ) super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) @@ -334,7 +335,7 @@ class TestIncompatibility(unittest.TestCase): SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue', CONCURRENT_REQUESTS_PER_IP=1, ) - crawler = Crawler(Spider, settings) + crawler = get_crawler(Spider, settings) scheduler = Scheduler.from_crawler(crawler) spider = Spider(name='spider') scheduler.open(spider) diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index bf90b4320..fc234a83d 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -7,10 +7,10 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase as TwistedTestCase from scrapy.core.scheduler import BaseScheduler -from scrapy.crawler import CrawlerRunner from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.utils.request import request_fingerprint +from scrapy.utils.request import fingerprint +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer @@ -21,13 +21,13 @@ URLS = [urljoin("https://example.org", p) for p in PATHS] class MinimalScheduler: def __init__(self) -> None: - self.requests: Dict[str, Request] = {} + self.requests: Dict[bytes, Request] = {} def has_pending_requests(self) -> bool: return bool(self.requests) def enqueue_request(self, request: Request) -> bool: - fp = request_fingerprint(request) + fp = fingerprint(request) if fp not in self.requests: self.requests[fp] = request return True @@ -147,9 +147,12 @@ class MinimalSchedulerCrawlTest(TwistedTestCase): @defer.inlineCallbacks def test_crawl(self): with MockServer() as mockserver: - settings = {"SCHEDULER": self.scheduler_cls} + settings = { + "SCHEDULER": self.scheduler_cls, + } with LogCapture() as log: - yield CrawlerRunner(settings).crawl(TestSpider, mockserver) + crawler = get_crawler(TestSpider, settings) + yield crawler.crawl(mockserver) for path in PATHS: self.assertIn(f"{{'path': '{path}'}}", str(log)) self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log)) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 8a35e9fd7..3719c7c9f 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -96,7 +96,10 @@ class SpiderLoaderTest(unittest.TestCase): def test_crawler_runner_loading(self): module = 'tests.test_spiderloader.test_spiders.spider1' - runner = CrawlerRunner({'SPIDER_MODULES': [module]}) + runner = CrawlerRunner({ + 'SPIDER_MODULES': [module], + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + }) self.assertRaisesRegex(KeyError, 'Spider not found', runner.create_crawler, 'spider2') diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 1ef4eeb14..46452415a 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -3,6 +3,7 @@ import os import tempfile import shutil import contextlib +import warnings from pytest import warns @@ -68,20 +69,21 @@ class GetProjectSettingsTestCase(unittest.TestCase): envvars = { 'SCRAPY_SETTINGS_MODULE': value, } - with set_env(**envvars), warns(None) as warnings: - settings = get_project_settings() - assert not warnings + with warnings.catch_warnings(): + warnings.simplefilter("error") + with set_env(**envvars): + settings = get_project_settings() + assert settings.get('SETTINGS_MODULE') == value def test_invalid_envvar(self): envvars = { 'SCRAPY_FOO': 'bar', } - with set_env(**envvars), warns(None) as warnings: - get_project_settings() - assert len(warnings) == 1 - assert warnings[0].category == ScrapyDeprecationWarning - assert str(warnings[0].message).endswith(': FOO') + with warns(ScrapyDeprecationWarning, match=': FOO') as record: + with set_env(**envvars): + get_project_settings() + assert len(record) == 1 def test_valid_and_invalid_envvars(self): value = 'tests.test_cmdline.settings' @@ -89,9 +91,8 @@ class GetProjectSettingsTestCase(unittest.TestCase): 'SCRAPY_FOO': 'bar', 'SCRAPY_SETTINGS_MODULE': value, } - with set_env(**envvars), warns(None) as warnings: - settings = get_project_settings() - assert len(warnings) == 1 - assert warnings[0].category == ScrapyDeprecationWarning - assert str(warnings[0].message).endswith(': FOO') + with warns(ScrapyDeprecationWarning, match=': FOO') as record: + with set_env(**envvars): + settings = get_project_settings() + assert len(record) == 1 assert settings.get('SETTINGS_MODULE') == value diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index e9edfee98..5ee772c0b 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -308,13 +308,22 @@ class RequestFingerprintTest(FingerprintTest): ), ) + def setUp(self) -> None: + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + + def tearDown(self) -> None: + warnings.simplefilter("default", ScrapyDeprecationWarning) + @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True) def test_part_separation(self): super().test_part_separation() + +class RequestFingerprintDeprecationTest(unittest.TestCase): + def test_deprecation_default_parameters(self): with pytest.warns(ScrapyDeprecationWarning) as warnings: - self.function(Request("http://www.example.com")) + request_fingerprint(Request("http://www.example.com")) messages = [str(warning.message) for warning in warnings] self.assertTrue( any( @@ -326,7 +335,7 @@ class RequestFingerprintTest(FingerprintTest): def test_deprecation_non_default_parameters(self): with pytest.warns(ScrapyDeprecationWarning) as warnings: - self.function(Request("http://www.example.com"), keep_fragments=True) + request_fingerprint(Request("http://www.example.com"), keep_fragments=True) messages = [str(warning.message) for warning in warnings] self.assertTrue( any( diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 0a09f6109..d20852e62 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,7 +1,9 @@ import os import unittest +import warnings from urllib.parse import urlparse +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse, HtmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import (response_httprepr, open_in_browser, @@ -15,14 +17,21 @@ class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response') def test_response_httprepr(self): - r1 = Response("http://www.example.com") - self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ScrapyDeprecationWarning) - r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body=b"Some body") - self.assertEqual(response_httprepr(r1), b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body') + r1 = Response("http://www.example.com") + self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n') - r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body=b"Some body") - self.assertEqual(response_httprepr(r1), b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body') + r1 = Response("http://www.example.com", status=404, + headers={"Content-type": "text/html"}, body=b"Some body") + self.assertEqual(response_httprepr(r1), + b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body') + + r1 = Response("http://www.example.com", status=6666, + headers={"Content-type": "text/html"}, body=b"Some body") + self.assertEqual(response_httprepr(r1), + b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body') def test_open_in_browser(self): url = "http:///www.example.com/some/page.html" From f60c7ae768fa2f238ea6d7646098fe99ef8ad461 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 20 Jul 2022 14:01:22 +0500 Subject: [PATCH 0576/2083] Fixed heading levels in downloader middleware docs (#5567) --- docs/topics/downloader-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 29e350651..986da0476 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -955,7 +955,7 @@ default because HTTP specs say so. .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST ---------------------- +^^^^^^^^^^^^^^^^^^^^^ Default: ``-1`` @@ -1119,7 +1119,7 @@ In order to use this parser: .. _support-for-new-robots-parser: Implementing support for a new parser -------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can implement support for a new robots.txt_ parser by subclassing the abstract base class :class:`~scrapy.robotstxt.RobotParser` and From 42056090516bb0cc5d349e232298c711ec452bc5 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sun, 17 Jul 2022 15:50:40 +0500 Subject: [PATCH 0577/2083] Fixed intersphinx references --- docs/conf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9a0afe73e..3241295af 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -291,9 +291,10 @@ intersphinx_mapping = { 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), - 'tox': ('https://tox.readthedocs.io/en/latest', None), - 'twisted': ('https://twistedmatrix.com/documents/current', None), - 'twistedapi': ('https://twistedmatrix.com/documents/current/api', None), + 'tox': ('https://tox.wiki/en/latest/', None), + 'twisted': ('https://docs.twisted.org/en/stable/', None), + 'twistedapi': ('https://docs.twisted.org/en/stable/api/', None), + 'w3lib': ('https://w3lib.readthedocs.io/en/latest', None), } intersphinx_disabled_reftypes = [] From b21c16099ed0acb0589677afd98c0ae3b78cd17d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 22 Jul 2022 19:18:33 +0500 Subject: [PATCH 0578/2083] Fix flake8 issues. --- tests/test_spidermiddleware.py | 6 +++--- tests/test_utils_python.py | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index ed0912b82..edde6f682 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -305,8 +305,8 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): with self.assertRaisesRegex( _InvalidOutput, ( - "\.process_spider_output must return an iterable, got " + r"\.process_spider_output must return an iterable, got " ), ): yield self._get_middleware_result( @@ -317,7 +317,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): def test_coroutine(self): with self.assertRaisesRegex( _InvalidOutput, - "\.process_spider_output must be an asynchronous generator", + r"\.process_spider_output must be an asynchronous generator", ): yield self._get_middleware_result( ProcessSpiderOutputCoroutineMiddleware, diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 00b06b839..b1a8fdc04 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -2,8 +2,6 @@ import functools import gc import operator import platform -import unittest -from datetime import datetime from itertools import count from warnings import catch_warnings, filterwarnings From af7dd16d8ded3e6cb2946603688f4f4a5212e80f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 25 Jul 2022 13:15:17 +0200 Subject: [PATCH 0579/2083] Merge pull request from GHSA-9x8m-2xpf-crp3 * Enforce matching proxy request meta and Proxy-Authorization header * Cover proxy credential security fix in the release notes * Remove extra empty line * Reword the security issue description * Address scenario where Proxy-Authorization is unexpectedly removed by a prior middleware * Set the release date of Scrapy 2.6.2 and 1.8.3 --- docs/news.rst | 106 ++++++- scrapy/downloadermiddlewares/httpproxy.py | 56 ++-- tests/test_downloadermiddleware_httpproxy.py | 313 ++++++++++++++++++- 3 files changed, 439 insertions(+), 36 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 7993b4b4f..5bd9ca059 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,10 +5,57 @@ Release notes .. _release-2.6.2: -Scrapy 2.6.2 (to be determined) -------------------------------- +Scrapy 2.6.2 (2022-07-25) +------------------------- -Fixes additional regressions introduced in 2.6.0: +**Security bug fix:** + +- When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` + processes a request with :reqmeta:`proxy` metadata, and that + :reqmeta:`proxy` metadata includes proxy credentials, + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets + the ``Proxy-Authentication`` header, but only if that header is not already + set. + + There are third-party proxy-rotation downloader middlewares that set + different :reqmeta:`proxy` metadata every time they process a request. + + Because of request retries and redirects, the same request can be processed + by downloader middlewares more than once, including both + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and + any third-party proxy-rotation downloader middleware. + + These third-party proxy-rotation downloader middlewares could change the + :reqmeta:`proxy` metadata of a request to a new value, but fail to remove + the ``Proxy-Authentication`` header from the previous value of the + :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent + to a different proxy. + + To prevent the unintended leaking of proxy credentials, the behavior of + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now + as follows when processing a request: + + - If the request being processed defines :reqmeta:`proxy` metadata that + includes credentials, the ``Proxy-Authorization`` header is always + updated to feature those credentials. + + - If the request being processed defines :reqmeta:`proxy` metadata + without credentials, the ``Proxy-Authorization`` header is removed + *unless* it was originally defined for the same proxy URL. + + To remove proxy credentials while keeping the same proxy URL, remove + the ``Proxy-Authorization`` header. + + - If the request has no :reqmeta:`proxy` metadata, or that metadata is a + falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is + removed. + + It is no longer possible to set a proxy URL through the + :reqmeta:`proxy` metadata but set the credentials through the + ``Proxy-Authorization`` header. Set proxy credentials through the + :reqmeta:`proxy` metadata instead. + +Also fixes the following regressions introduced in 2.6.0: - :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple spiders (:issue:`5435`, :issue:`5436`) @@ -1925,6 +1972,59 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.3: + +Scrapy 1.8.3 (2022-07-25) +------------------------- + +**Security bug fix:** + +- When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` + processes a request with :reqmeta:`proxy` metadata, and that + :reqmeta:`proxy` metadata includes proxy credentials, + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets + the ``Proxy-Authentication`` header, but only if that header is not already + set. + + There are third-party proxy-rotation downloader middlewares that set + different :reqmeta:`proxy` metadata every time they process a request. + + Because of request retries and redirects, the same request can be processed + by downloader middlewares more than once, including both + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and + any third-party proxy-rotation downloader middleware. + + These third-party proxy-rotation downloader middlewares could change the + :reqmeta:`proxy` metadata of a request to a new value, but fail to remove + the ``Proxy-Authentication`` header from the previous value of the + :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent + to a different proxy. + + To prevent the unintended leaking of proxy credentials, the behavior of + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now + as follows when processing a request: + + - If the request being processed defines :reqmeta:`proxy` metadata that + includes credentials, the ``Proxy-Authorization`` header is always + updated to feature those credentials. + + - If the request being processed defines :reqmeta:`proxy` metadata + without credentials, the ``Proxy-Authorization`` header is removed + *unless* it was originally defined for the same proxy URL. + + To remove proxy credentials while keeping the same proxy URL, remove + the ``Proxy-Authorization`` header. + + - If the request has no :reqmeta:`proxy` metadata, or that metadata is a + falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is + removed. + + It is no longer possible to set a proxy URL through the + :reqmeta:`proxy` metadata but set the credentials through the + ``Proxy-Authorization`` header. Set proxy credentials through the + :reqmeta:`proxy` metadata instead. + + .. _release-1.8.2: Scrapy 1.8.2 (2022-03-01) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index d2665b655..1deda42bd 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -45,31 +45,37 @@ class HttpProxyMiddleware: return creds, proxy_url def process_request(self, request, spider): - # ignore if proxy is already set + creds, proxy_url = None, None if 'proxy' in request.meta: - if request.meta['proxy'] is None: - return - # extract credentials if present - creds, proxy_url = self._get_proxy(request.meta['proxy'], '') + if request.meta['proxy'] is not None: + creds, proxy_url = self._get_proxy(request.meta['proxy'], '') + elif self.proxies: + parsed = urlparse_cached(request) + scheme = parsed.scheme + if ( + ( + # 'no_proxy' is only supported by http schemes + scheme not in ('http', 'https') + or not proxy_bypass(parsed.hostname) + ) + and scheme in self.proxies + ): + creds, proxy_url = self.proxies[scheme] + + self._set_proxy_and_creds(request, proxy_url, creds) + + def _set_proxy_and_creds(self, request, proxy_url, creds): + if proxy_url: request.meta['proxy'] = proxy_url - if creds and not request.headers.get('Proxy-Authorization'): - request.headers['Proxy-Authorization'] = b'Basic ' + creds - return - elif not self.proxies: - return - - parsed = urlparse_cached(request) - scheme = parsed.scheme - - # 'no_proxy' is only supported by http schemes - if scheme in ('http', 'https') and proxy_bypass(parsed.hostname): - return - - if scheme in self.proxies: - self._set_proxy(request, scheme) - - def _set_proxy(self, request, scheme): - creds, proxy = self.proxies[scheme] - request.meta['proxy'] = proxy + elif request.meta.get('proxy') is not None: + request.meta['proxy'] = None if creds: - request.headers['Proxy-Authorization'] = b'Basic ' + creds + request.headers[b'Proxy-Authorization'] = b'Basic ' + creds + request.meta['_auth_proxy'] = proxy_url + elif '_auth_proxy' in request.meta: + if proxy_url != request.meta['_auth_proxy']: + if b'Proxy-Authorization' in request.headers: + del request.headers[b'Proxy-Authorization'] + del request.meta['_auth_proxy'] + elif b'Proxy-Authorization' in request.headers: + del request.headers[b'Proxy-Authorization'] diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 7c97bf32a..67134cf93 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -65,12 +65,12 @@ class TestHttpProxyMiddleware(TestCase): mw = HttpProxyMiddleware() req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') def test_proxy_auth_empty_passwd(self): @@ -78,12 +78,12 @@ class TestHttpProxyMiddleware(TestCase): mw = HttpProxyMiddleware() req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') def test_proxy_auth_encoding(self): @@ -92,26 +92,26 @@ class TestHttpProxyMiddleware(TestCase): mw = HttpProxyMiddleware(auth_encoding='utf-8') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') # default latin-1 encoding mw = HttpProxyMiddleware(auth_encoding='latin-1') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') # proxy from request.meta, latin-1 encoding req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') def test_proxy_already_seted(self): @@ -152,3 +152,300 @@ class TestHttpProxyMiddleware(TestCase): # '/var/run/docker.sock' may be used by the user for # no_proxy value but is not parseable and should be skipped assert 'no' not in mw.proxies + + def test_add_proxy_without_credentials(self): + middleware = HttpProxyMiddleware() + request = Request('https://example.com') + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = 'https://example.com' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_add_proxy_with_credentials(self): + middleware = HttpProxyMiddleware() + request = Request('https://example.com') + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = 'https://user1:password1@example.com' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + encoded_credentials = middleware._basic_auth_header( + 'user1', + 'password1', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + def test_remove_proxy_without_credentials(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://example.com'}, + ) + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = None + assert middleware.process_request(request, spider) is None + self.assertIsNone(request.meta['proxy']) + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_remove_proxy_with_credentials(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = None + assert middleware.process_request(request, spider) is None + self.assertIsNone(request.meta['proxy']) + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_add_credentials(self): + """If the proxy request meta switches to a proxy URL with the same + proxy and adds credentials (there were no credentials before), the new + credentials must be used.""" + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://example.com'}, + ) + assert middleware.process_request(request, spider) is None + + request.meta['proxy'] = 'https://user1:password1@example.com' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + encoded_credentials = middleware._basic_auth_header( + 'user1', + 'password1', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + def test_change_credentials(self): + """If the proxy request meta switches to a proxy URL with different + credentials, those new credentials must be used.""" + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = 'https://user2:password2@example.com' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + encoded_credentials = middleware._basic_auth_header( + 'user2', + 'password2', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + def test_remove_credentials(self): + """If the proxy request meta switches to a proxy URL with the same + proxy but no credentials, the original credentials must be still + used. + + To remove credentials while keeping the same proxy URL, users must + delete the Proxy-Authorization header. + """ + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + + request.meta['proxy'] = 'https://example.com' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + encoded_credentials = middleware._basic_auth_header( + 'user1', + 'password1', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + request.meta['proxy'] = 'https://example.com' + del request.headers[b'Proxy-Authorization'] + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_change_proxy_add_credentials(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://example.com'}, + ) + assert middleware.process_request(request, spider) is None + + request.meta['proxy'] = 'https://user1:password1@example.org' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.org') + encoded_credentials = middleware._basic_auth_header( + 'user1', + 'password1', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + def test_change_proxy_keep_credentials(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + + request.meta['proxy'] = 'https://user1:password1@example.org' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.org') + encoded_credentials = middleware._basic_auth_header( + 'user1', + 'password1', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + # Make sure, indirectly, that _auth_proxy is updated. + request.meta['proxy'] = 'https://example.com' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_change_proxy_change_credentials(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + + request.meta['proxy'] = 'https://user2:password2@example.org' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.org') + encoded_credentials = middleware._basic_auth_header( + 'user2', + 'password2', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + def test_change_proxy_remove_credentials(self): + """If the proxy request meta switches to a proxy URL with a different + proxy and no credentials, no credentials must be used.""" + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = 'https://example.org' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta, {'proxy': 'https://example.org'}) + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_change_proxy_remove_credentials_preremoved_header(self): + """Corner case of proxy switch with credentials removal where the + credentials have been removed beforehand. + + It ensures that our implementation does not assume that the credentials + header exists when trying to remove it. + """ + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = 'https://example.org' + del request.headers[b'Proxy-Authorization'] + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta, {'proxy': 'https://example.org'}) + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_proxy_authentication_header_undefined_proxy(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + headers={'Proxy-Authorization': 'Basic foo'}, + ) + assert middleware.process_request(request, spider) is None + self.assertNotIn('proxy', request.meta) + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_proxy_authentication_header_disabled_proxy(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + headers={'Proxy-Authorization': 'Basic foo'}, + meta={'proxy': None}, + ) + assert middleware.process_request(request, spider) is None + self.assertIsNone(request.meta['proxy']) + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_proxy_authentication_header_proxy_without_credentials(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + headers={'Proxy-Authorization': 'Basic foo'}, + meta={'proxy': 'https://example.com'}, + ) + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_proxy_authentication_header_proxy_with_same_credentials(self): + middleware = HttpProxyMiddleware() + encoded_credentials = middleware._basic_auth_header( + 'user1', + 'password1', + ) + request = Request( + 'https://example.com', + headers={'Proxy-Authorization': b'Basic ' + encoded_credentials}, + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + def test_proxy_authentication_header_proxy_with_different_credentials(self): + middleware = HttpProxyMiddleware() + encoded_credentials1 = middleware._basic_auth_header( + 'user1', + 'password1', + ) + request = Request( + 'https://example.com', + headers={'Proxy-Authorization': b'Basic ' + encoded_credentials1}, + meta={'proxy': 'https://user2:password2@example.com'}, + ) + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + encoded_credentials2 = middleware._basic_auth_header( + 'user2', + 'password2', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials2, + ) From aecbccbaa567b07694141a4503e9abf1bb2c919f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 25 Jul 2022 13:33:23 +0200 Subject: [PATCH 0580/2083] =?UTF-8?q?Bump=20version:=202.6.1=20=E2=86=92?= =?UTF-8?q?=202.6.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1d9b9c02f..2e2f7949a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.6.1 +current_version = 2.6.2 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 6a6a3d8e3..097a15a2a 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.6.1 +2.6.2 From 5862f6b8e1cd9e240c4b3e79f2eddd0b9f33ef8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 25 Jul 2022 17:25:10 +0200 Subject: [PATCH 0581/2083] Update frozen CI packages (#5574) --- pylintrc | 6 +----- tox.ini | 13 ++++++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pylintrc b/pylintrc index 2cdd6321e..d8e47dc11 100644 --- a/pylintrc +++ b/pylintrc @@ -9,11 +9,9 @@ disable=abstract-method, arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, - bad-continuation, bad-indentation, bad-mcs-classmethod-argument, bad-super-call, - bad-whitespace, bare-except, blacklisted-name, broad-except, @@ -52,7 +50,6 @@ disable=abstract-method, logging-not-lazy, lost-exception, method-hidden, - misplaced-comparison-constant, missing-docstring, missing-final-newline, multiple-imports, @@ -60,12 +57,10 @@ disable=abstract-method, no-else-continue, no-else-raise, no-else-return, - no-init, no-member, no-method-argument, no-name-in-module, no-self-argument, - no-self-use, no-value-for-parameter, not-an-iterable, not-callable, @@ -102,6 +97,7 @@ disable=abstract-method, ungrouped-imports, unidiomatic-typecheck, unnecessary-comprehension, + unnecessary-dunder-call, unnecessary-lambda, unnecessary-pass, unreachable, diff --git a/tox.ini b/tox.ini index ab8a715c2..4d1bb574d 100644 --- a/tox.ini +++ b/tox.ini @@ -47,7 +47,7 @@ commands = [testenv:security] basepython = python3 deps = - bandit==1.7.3 + bandit==1.7.4 commands = bandit -r -c .bandit.yml {posargs:scrapy} @@ -57,16 +57,17 @@ deps = {[testenv]deps} # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 - pytest-flake8 - flake8==3.9.2 # https://github.com/tholo/pytest-flake8/issues/81 + pytest-flake8==1.1.1 + flake8==4.0.1 commands = pytest --flake8 {posargs:docs scrapy tests} [testenv:pylint] -basepython = python3 +# reppy does not support Python 3.9+ +basepython = python3.8 deps = {[testenv:extra-deps]deps} - pylint==2.12.2 + pylint==2.14.5 commands = pylint conftest.py docs extras scrapy setup.py tests @@ -117,6 +118,8 @@ setenv = {[pinned]setenv} [testenv:extra-deps] +# reppy does not support Python 3.9+ +basepython = python3.8 deps = {[testenv]deps} boto From 56e2eeac1066cd2d3c710b76a3d6210b3ac257f5 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jul 2022 09:41:12 +0500 Subject: [PATCH 0582/2083] fix typing issues by upgrading mypy --- tests/test_utils_request.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 5ee772c0b..8bc7922b6 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -52,7 +52,7 @@ class UtilsRequestTest(unittest.TestCase): class FingerprintTest(unittest.TestCase): maxDiff = None - function = staticmethod(fingerprint) + function: staticmethod = staticmethod(fingerprint) cache: Union[ "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]", "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]", diff --git a/tox.ini b/tox.ini index 4d1bb574d..2110e1020 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,7 @@ install_command = basepython = python3 deps = lxml-stubs==0.2.0 - mypy==0.910 + mypy==0.971 types-pyOpenSSL==20.0.3 types-setuptools==57.0.0 commands = From 67011cd9576b02d696c7e0c0dc7f76074e30901b Mon Sep 17 00:00:00 2001 From: Aftab Alam <88653530+itsAftabAlam@users.noreply.github.com> Date: Wed, 27 Jul 2022 10:20:26 +0530 Subject: [PATCH 0583/2083] updated README.rst , added hyperlink to banner (#5284) Co-authored-by: Mikhail Korobov --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index b543a30f4..358302c76 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,5 @@ .. image:: https://scrapy.org/img/scrapylogo.png + :target: https://scrapy.org/ ====== Scrapy From 83ecdf1bcab310be5f2c59e5c005b8968d72a72c Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 27 Jul 2022 23:12:31 +0500 Subject: [PATCH 0584/2083] Update docs/topics/components.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/components.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 1fff2d61a..c44f3def2 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -47,7 +47,7 @@ Enforcing component requirements ================================ Sometimes, your components may only be intended to work under certain -conditions. For example, the may require a minimum version of Scrapy to work as +conditions. For example, they may require a minimum version of Scrapy to work as intended, or they may require certain settings to have specific values. In addition to describing those conditions in the documentation of your From 0f1112f3e22d91e505f736fc78fae94eda07c72d Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 27 Jul 2022 23:12:43 +0500 Subject: [PATCH 0585/2083] Update docs/index.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index ea4950e4c..5404969e0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -258,7 +258,7 @@ Extending Scrapy components. :doc:`topics/api` - Use it on extensions and middlewares to extend Scrapy functionality + Use it on extensions and middlewares to extend Scrapy functionality. All the rest From c7b90c6e1e3de20256d8bf6fc5d18da44d562b0d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 28 Jul 2022 13:44:36 +0500 Subject: [PATCH 0586/2083] Extract more common code. --- scrapy/spidermiddlewares/depth.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 29634c3ad..4c923b1b3 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -28,25 +28,22 @@ class DepthMiddleware: return cls(maxdepth, crawler.stats, verbose, prio) def process_spider_output(self, response, result, spider): - # base case (depth=0) - if 'depth' not in response.meta: - response.meta['depth'] = 0 - if self.verbose_stats: - self.stats.inc_value('request_depth_count/0', spider=spider) - + self._init_depth(response, spider) return (r for r in result or () if self._filter(r, response, spider)) async def process_spider_output_async(self, response, result, spider): + self._init_depth(response, spider) + async for r in result or (): + if self._filter(r, response, spider): + yield r + + def _init_depth(self, response, spider): # base case (depth=0) if 'depth' not in response.meta: response.meta['depth'] = 0 if self.verbose_stats: self.stats.inc_value('request_depth_count/0', spider=spider) - async for r in result or (): - if self._filter(r, response, spider): - yield r - def _filter(self, request, response, spider): if not isinstance(request, Request): return True From 5735e93541d26ad3aef5e4fc45fa3d1c4dc0fa2a Mon Sep 17 00:00:00 2001 From: felipeboffnunes Date: Thu, 28 Jul 2022 18:37:21 -0300 Subject: [PATCH 0587/2083] fix_post_processing_feed_export --- scrapy/extensions/feedexport.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e7097b7a1..3e98a5a0b 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -350,13 +350,19 @@ class FeedExporter: return defer.DeferredList(deferred_list) if deferred_list else None def _close_slot(self, slot, spider): + + def get_file(slot_): + if isinstance(slot_.file, PostProcessingManager): + return slot_.file.file + return slot_.file + if not slot.itemcount and not slot.store_empty: # We need to call slot.storage.store nonetheless to get the file # properly closed. - return defer.maybeDeferred(slot.storage.store, slot.file) + return defer.maybeDeferred(slot.storage.store, get_file(slot)) slot.finish_exporting() logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d = defer.maybeDeferred(slot.storage.store, slot.file) + d = defer.maybeDeferred(slot.storage.store, get_file(slot)) d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ From 4be9c969fd4bdc3206bdabfb5c22e524d9e355ac Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 29 Jul 2022 12:15:39 +0300 Subject: [PATCH 0588/2083] per slot settings: logger deleted as not used --- scrapy/core/downloader/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 511693830..fa8ac01e5 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,4 +1,3 @@ -import logging import random from time import time from datetime import datetime @@ -13,8 +12,6 @@ from scrapy import signals from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core.downloader.handlers import DownloadHandlers -logger = logging.getLogger(__name__) - class Slot: """Downloader slot""" From d599fff2b97ff96c3face707b980926b6eb48a2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 29 Jul 2022 19:10:28 +0200 Subject: [PATCH 0589/2083] Test the life cycle of a storage file --- tests/test_feedexport.py | 55 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ecd1b59d3..a5fd1e467 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1517,6 +1517,61 @@ class FeedExportTest(FeedExportTestBase): data = yield self.exported_data(items, settings) self.assertEqual(row['expected'], data[feed_options['format']]) + @defer.inlineCallbacks + def test_storage_file_no_postprocessing(self): + + @implementer(IFeedStorage) + class Storage: + + def __init__(self, uri, *, feed_options=None): + pass + + def open(self, spider): + Storage.open_file = tempfile.NamedTemporaryFile(prefix='feed-') + return Storage.open_file + + def store(self, file): + Storage.store_file = file + file.close() + + settings = { + 'FEEDS': {self._random_temp_filename(): {'format': 'jsonlines'}}, + 'FEED_STORAGES': {'file': Storage}, + } + yield self.exported_no_data(settings) + self.assertIs(Storage.open_file, Storage.store_file) + + @defer.inlineCallbacks + def test_storage_file_postprocessing(self): + + @implementer(IFeedStorage) + class Storage: + + def __init__(self, uri, *, feed_options=None): + pass + + def open(self, spider): + Storage.open_file = tempfile.NamedTemporaryFile(prefix='feed-') + return Storage.open_file + + def store(self, file): + Storage.store_file = file + file.close() + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'jsonlines', + 'postprocessing': [ + 'scrapy.extensions.postprocessing.GzipPlugin', + ], + }, + }, + 'FEED_STORAGES': {'file': Storage}, + } + yield self.exported_no_data(settings) + self.assertIs(Storage.open_file, Storage.store_file) + class FeedPostProcessedExportsTest(FeedExportTestBase): __test__ = True From aabdd0b657e1b4398a5a1aad79044401e9dbf909 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 12 Aug 2022 15:34:48 +0300 Subject: [PATCH 0590/2083] per slot settings: logger deleted as not used (step 2) --- scrapy/core/downloader/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index fa8ac01e5..d908f4d7e 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -204,5 +204,4 @@ class Downloader: mintime = time() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: - inactive_slot = self.slots.pop(key) - inactive_slot.close() + self.slots.pop(key).close() From 13c5ad7e688271ae47f2a489bf665cb15387b3a3 Mon Sep 17 00:00:00 2001 From: "Alexandr N. Zamaraev" Date: Tue, 16 Aug 2022 11:03:37 +0700 Subject: [PATCH 0591/2083] Partial in is_generator_with_return_value See path in #5592 --- scrapy/utils/misc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 1221b39b2..2e25f6421 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -9,6 +9,7 @@ from collections import deque from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules +from functools import partial from w3lib.html import replace_entities @@ -226,7 +227,11 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - code = re.sub(r"^[\t ]+", "", inspect.getsource(callable)) + func = callable + while isinstance(func, partial): + func = func.func + + code = re.sub(r"^[\t ]+", "", inspect.getsource(func)) tree = ast.parse(code) for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): From c49b5aaf77a668e44bffb54c52310084e4068162 Mon Sep 17 00:00:00 2001 From: zaid-ismail031 Date: Thu, 25 Aug 2022 02:17:03 +0200 Subject: [PATCH 0592/2083] Changed incorrect information regarding the return type of parse/request callback method. --- docs/topics/spiders.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index ece02ae47..ffe41cf3e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -181,9 +181,10 @@ scrapy.Spider scraped data and/or more URLs to follow. Other Requests callbacks have the same requirements as the :class:`Spider` class. - This method, as well as any other Request callback, must return an - iterable of :class:`~scrapy.Request` and/or :ref:`item objects - `. + This method, as well as any other Request callback, must return a + :class:`~scrapy.Request` object, an :ref:`item object `, an + iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects + `, or ``None``. :param response: the response to parse :type response: :class:`~scrapy.http.Response` From e411ea94eb853eedeee111469c6442d275864a09 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sun, 28 Aug 2022 20:28:13 +0430 Subject: [PATCH 0593/2083] BOM should take precedence over Content-Type header when detecting the encoding closes #5601 --- scrapy/http/response/text.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 89516b9b6..bfcde878d 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -11,8 +11,13 @@ from typing import Generator, Tuple from urllib.parse import urljoin import parsel -from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, - http_content_type_encoding, resolve_encoding) +from w3lib.encoding import ( + html_body_declared_encoding, + html_to_unicode, + http_content_type_encoding, + resolve_encoding, + read_bom, +) from w3lib.html import strip_html5_whitespace from scrapy.http import Request @@ -60,6 +65,7 @@ class TextResponse(Response): def _declared_encoding(self): return ( self._encoding + or self._bom_encoding() or self._headers_encoding() or self._body_declared_encoding() ) @@ -117,6 +123,10 @@ class TextResponse(Response): def _body_declared_encoding(self): return html_body_declared_encoding(self.body) + @memoizemethod_noargs + def _bom_encoding(self): + return read_bom(self.body)[0] + @property def selector(self): from scrapy.selector import Selector From a988c4b78b9ca104b74cff60c46c1842e3f25652 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Mon, 29 Aug 2022 17:08:30 +0430 Subject: [PATCH 0594/2083] add test --- tests/test_http_response.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 2986f884f..5d67a5e74 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,3 +1,4 @@ +import codecs import unittest from unittest import mock @@ -358,6 +359,8 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=gb2312"]}) r7 = self.response_class("http://www.example.com", body=b"\xa8D", headers={"Content-type": ["text/html; charset=gbk"]}) + r8 = self.response_class("http://www.example.com", body=codecs.BOM_UTF8 + b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=cp1251"]}) self.assertEqual(r1._headers_encoding(), "utf-8") self.assertEqual(r2._headers_encoding(), None) @@ -367,7 +370,10 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r3._declared_encoding(), "cp1252") self.assertEqual(r4._headers_encoding(), None) self.assertEqual(r5._headers_encoding(), None) + self.assertEqual(r8._headers_encoding(), "cp1251") + self.assertEqual(r8._declared_encoding(), "utf-8") self._assert_response_encoding(r5, "utf-8") + self._assert_response_encoding(r8, "utf-8") assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' self._assert_response_values(r1, 'utf-8', "\xa3") self._assert_response_values(r2, 'utf-8', "\xa3") From f4bcc3e67de0896999be48d1b0fcede0be0d69e4 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 6 Sep 2022 01:15:41 -0300 Subject: [PATCH 0595/2083] fix: failed tests --- tests/test_crawler.py | 13 +++++++++---- tests/test_spider.py | 12 ++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 4e9ef7740..4e599d69c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -23,6 +23,8 @@ from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version from tests.mockserver import MockServer @@ -381,10 +383,13 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) - self.assertIn( - "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", - log) + if parse_version(w3lib_version) < parse_version("2.0.0"): + self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) + self.assertIn( + "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", + log) + else: + self.assertIn("ValueError: invalid hostname:", log) def test_caching_hostname_resolver_ipv6(self): log = self.run_script("caching_hostname_resolver_ipv6.py") diff --git a/tests/test_spider.py b/tests/test_spider.py index 689349999..7b36304b1 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -22,6 +22,8 @@ from scrapy.spiders import ( from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler from tests import get_testdata +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version class SpiderTest(unittest.TestCase): @@ -360,10 +362,12 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) + urls = ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', + 'http://EXAMPLE.ORG/ABOUT.HTML', + 'http://EXAMPLE.ORG/NOFOLLOW.HTML'] + if parse_version(w3lib_version) >= parse_version('2.0.0'): + urls = list(map(lambda u: u.replace("EXAMPLE.ORG", "example.org"), urls)) + self.assertEqual([r.url for r in output], urls) def test_process_request_instance_method_with_response(self): From 1289422284991c09cbee8cea17bef837b3efc31b Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 6 Sep 2022 08:17:58 -0300 Subject: [PATCH 0596/2083] chore: Skip `test_ipv6_default_name_resolver` test if w3lib version >= 2.0.0 --- tests/test_crawler.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 4e599d69c..e2a14be55 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -380,16 +380,15 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(parse_version(w3lib_version) >= parse_version("2.0.0"), + reason='w3lib 2.0.0 and later do not allow invalid domains.') def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') self.assertIn('Spider closed (finished)', log) - if parse_version(w3lib_version) < parse_version("2.0.0"): - self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) - self.assertIn( - "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", - log) - else: - self.assertIn("ValueError: invalid hostname:", log) + self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) + self.assertIn( + "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", + log) def test_caching_hostname_resolver_ipv6(self): log = self.run_script("caching_hostname_resolver_ipv6.py") From 582a6bf6dbcb01da40cbec6b51269add2db39cf1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 6 Sep 2022 10:03:18 -0300 Subject: [PATCH 0597/2083] refactor: Use `safe_url_string` to standardize url output --- tests/test_spider.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 7b36304b1..e1527620f 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -22,8 +22,7 @@ from scrapy.spiders import ( from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler from tests import get_testdata -from pkg_resources import parse_version -from w3lib import __version__ as w3lib_version +from w3lib.url import safe_url_string class SpiderTest(unittest.TestCase): @@ -362,12 +361,10 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - urls = ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML'] - if parse_version(w3lib_version) >= parse_version('2.0.0'): - urls = list(map(lambda u: u.replace("EXAMPLE.ORG", "example.org"), urls)) - self.assertEqual([r.url for r in output], urls) + self.assertEqual([r.url for r in output], + [safe_url_string('http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML'), + safe_url_string('http://EXAMPLE.ORG/ABOUT.HTML'), + safe_url_string('http://EXAMPLE.ORG/NOFOLLOW.HTML')]) def test_process_request_instance_method_with_response(self): From ce0ca51485545aed8eb33a285bc3d2fbf8a8e407 Mon Sep 17 00:00:00 2001 From: "Magsen (CD)" Date: Tue, 13 Sep 2022 12:07:58 +0200 Subject: [PATCH 0598/2083] fix: typo in tutorial fix: typo in tutorial --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 75928077e..092123d1d 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -379,7 +379,7 @@ like this: Let's open up scrapy shell and play a bit to find out how to extract the data we want:: - $ scrapy shell 'https://quotes.toscrape.com' + scrapy shell 'https://quotes.toscrape.com' We get a list of selectors for the quote HTML elements with: From 77c055ee28a767b2c8222274cc7556ab9cc56edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 14 Sep 2022 14:47:14 +0200 Subject: [PATCH 0599/2083] Relax Proxy-Authorization restrictions --- scrapy/downloadermiddlewares/httpproxy.py | 5 ++++- tests/test_downloadermiddleware_httpproxy.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 1deda42bd..dd8a7e797 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -78,4 +78,7 @@ class HttpProxyMiddleware: del request.headers[b'Proxy-Authorization'] del request.meta['_auth_proxy'] elif b'Proxy-Authorization' in request.headers: - del request.headers[b'Proxy-Authorization'] + if proxy_url: + request.meta['_auth_proxy'] = proxy_url + else: + del request.headers[b'Proxy-Authorization'] diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 70eb94d77..44434f90e 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -400,6 +400,9 @@ class TestHttpProxyMiddleware(TestCase): self.assertNotIn(b'Proxy-Authorization', request.headers) def test_proxy_authentication_header_proxy_without_credentials(self): + """As long as the proxy URL in request metadata remains the same, the + Proxy-Authorization header is used and kept, and may even be + changed.""" middleware = HttpProxyMiddleware() request = Request( 'https://example.com', @@ -408,7 +411,16 @@ class TestHttpProxyMiddleware(TestCase): ) assert middleware.process_request(request, spider) is None self.assertEqual(request.meta['proxy'], 'https://example.com') - self.assertNotIn(b'Proxy-Authorization', request.headers) + self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo') + + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo') + + request.headers['Proxy-Authorization'] = b'Basic bar' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual(request.headers['Proxy-Authorization'], b'Basic bar') def test_proxy_authentication_header_proxy_with_same_credentials(self): middleware = HttpProxyMiddleware() From 1429aa011ce2a7a43691fc359c88bf40e512176c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Tue, 20 Sep 2022 12:47:20 -0400 Subject: [PATCH 0600/2083] Update test-standard link in contributing docs (#5631) --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 946bdc23e..9cfe10012 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -214,7 +214,7 @@ Tests ===== Tests are implemented using the :doc:`Twisted unit-testing framework -`. Running tests requires +`. Running tests requires :doc:`tox `. .. _running-tests: From 5f194202114fd38530c78299d51b6966b4802f59 Mon Sep 17 00:00:00 2001 From: Tim B Date: Wed, 21 Sep 2022 07:27:27 +0100 Subject: [PATCH 0601/2083] Documented how settings must be picklable (#5629) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves Co-authored-by: Laerte Pereira <5853172+Laerte@users.noreply.github.com> --- docs/topics/settings.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 6722ce9ed..90f13f3ef 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -98,6 +98,10 @@ class. The global defaults are located in the ``scrapy.settings.default_settings`` module and documented in the :ref:`topics-settings-ref` section. +Compatibility with pickle +========================= + +Setting values must be :ref:`picklable `. Import paths and classes ======================== From 385acd5598fb06a126ac485087b58a4581096876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sat, 24 Sep 2022 14:58:14 -0400 Subject: [PATCH 0602/2083] Match pyOpenSSL and service_identity to Twisted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- setup.py | 6 +++--- tests/test_crawler.py | 2 +- tox.ini | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index ed197273f..fcc902c62 100644 --- a/setup.py +++ b/setup.py @@ -20,13 +20,13 @@ def has_environment_marker_platform_impl_support(): install_requires = [ 'Twisted>=18.9.0', - 'cryptography>=2.8', + 'cryptography>=3.3', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'pyOpenSSL>=19.1.0', + 'pyOpenSSL>=21.0.0', 'queuelib>=1.4.2', - 'service_identity>=16.0.0', + 'service_identity>=18.1.0', 'w3lib>=1.17.0', 'zope.interface>=5.1.0', 'protego>=0.1.15', diff --git a/tests/test_crawler.py b/tests/test_crawler.py index e2a14be55..cf15ba9b9 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -327,7 +327,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_reactor_default_twisted_reactor_select(self): log = self.run_script('reactor_default_twisted_reactor_select.py') - if platform.system() == 'Windows': + if platform.system() in ['Windows', 'Darwin']: # The goal of this test function is to test that, when a reactor is # installed (the default one here) and a different reactor is # configured (select here), an error raises. diff --git a/tox.ini b/tox.ini index 2110e1020..2bf9454d0 100644 --- a/tox.ini +++ b/tox.ini @@ -73,15 +73,15 @@ commands = [pinned] deps = - cryptography==2.8 + cryptography==3.3 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - pyOpenSSL==19.1.0 + pyOpenSSL==21.0.0 queuelib==1.4.2 - service_identity==16.0.0 + service_identity==18.1.0 Twisted[http2]==18.9.0 w3lib==1.17.0 zope.interface==5.1.0 From 79a4bc3da02bfe2b07c596f6b677760b2c04f96f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sun, 25 Sep 2022 14:17:57 -0400 Subject: [PATCH 0603/2083] Cleanup METHOD_SSLv3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- docs/topics/settings.rst | 1 - scrapy/core/downloader/contextfactory.py | 2 +- scrapy/core/downloader/tls.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 90f13f3ef..a711fd197 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -564,7 +564,6 @@ This setting must be one of these string values: set this if you want the behavior of Scrapy<1.1 - ``'TLSv1.1'``: forces TLS version 1.1 - ``'TLSv1.2'``: forces TLS version 1.2 -- ``'SSLv3'``: forces SSL version 3 (**not recommended**) .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index b5318c7bb..4abde2238 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,7 +21,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): which allows TLS protocol negotiation 'A TLS/SSL connection established with [this method] may - understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.' + understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.' """ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs): diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 19a56d9b6..698a1c85c 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -11,7 +11,6 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info logger = logging.getLogger(__name__) -METHOD_SSLv3 = 'SSLv3' METHOD_TLS = 'TLS' METHOD_TLSv10 = 'TLSv1.0' METHOD_TLSv11 = 'TLSv1.1' @@ -20,7 +19,6 @@ METHOD_TLSv12 = 'TLSv1.2' openssl_methods = { METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended) METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only From 1d79994dccf396caca93f53cd50646281fee1def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 27 Sep 2022 17:01:33 +0200 Subject: [PATCH 0604/2083] Copy 2.6.3 release notes from the 2.6 branch --- docs/news.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index d27c105a5..9469d0fe5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,32 @@ Release notes ============= +.. _release-2.6.3: + +Scrapy 2.6.3 (2022-09-27) +------------------------- + +- Added support for pyOpenSSL_ 22.1.0, removing support for SSLv3 + (:issue:`5634`, :issue:`5635`, :issue:`5636`). + +- Upgraded the minimum versions of the following dependencies: + + - cryptography_: 2.0 → 3.3 + + - pyOpenSSL_: 16.2.0 → 21.0.0 + + - service_identity_: 16.0.0 → 18.1.0 + + - Twisted_: 17.9.0 → 18.9.0 + + - zope.interface_: 4.1.3 → 5.0.0 + + (:issue:`5621`, :issue:`5632`) + +- Fixes test and documentation issues (:issue:`5612`, :issue:`5617`, + :issue:`5631`). + + .. _release-2.6.2: Scrapy 2.6.2 (2022-07-25) From 3ca7877781fdae5415c57a9a0f348b1797795209 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 29 Sep 2022 11:51:11 -0300 Subject: [PATCH 0605/2083] chore: Skip `batch_path_differ` test on Windows --- tests/test_feedexport.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ecd1b59d3..ad2383018 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2285,6 +2285,7 @@ class BatchDeliveriesTest(FeedExportTestBase): for expected_batch, got_batch in zip(expected, data[fmt]): self.assertEqual(expected_batch, got_batch) + @pytest.mark.skipif(sys.platform == 'win32', reason='Odd behaviour on file creation/output') @defer.inlineCallbacks def test_batch_path_differ(self): """ @@ -2305,7 +2306,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) - self.assertEqual(len(items) + 1, len(data['json'])) + self.assertEqual(len(items), len([_ for _ in data['json'] if _])) @defer.inlineCallbacks def test_stats_batch_file_success(self): From 9f006e3aa595acc6499f08a90696d7cd0dc0bca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sun, 25 Sep 2022 13:48:51 -0400 Subject: [PATCH 0606/2083] Match pyOpenSSL and types-pyOpenSSL versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2bf9454d0..0a47a1c0b 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ basepython = python3 deps = lxml-stubs==0.2.0 mypy==0.971 - types-pyOpenSSL==20.0.3 + types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 commands = mypy --show-error-codes {posargs: scrapy tests} From 116d9a97481f5ee4028a1f3f8e72ca34b35a0be9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Fri, 30 Sep 2022 22:58:14 -0400 Subject: [PATCH 0607/2083] Correct distutils deprecation warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- scrapy/utils/display.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index f4d17224b..d28df40c7 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -5,7 +5,7 @@ pprint and pformat wrappers with colorization support import ctypes import platform import sys -from distutils.version import LooseVersion as parse_version +from packaging.version import Version as parse_version from pprint import pformat as pformat_ From c3f35d2ad79d19e7e000b8ba0df7d1d9f10658b2 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sat, 1 Oct 2022 19:37:51 +0200 Subject: [PATCH 0608/2083] ci(tests-windows): upgrade actions/setup-python to v4 --- .github/workflows/tests-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 955b9b449..0d85a8e09 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 9fcbf3bcbc8b8d09bb7d8b246dae9156d6848a5b Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sat, 1 Oct 2022 19:38:16 +0200 Subject: [PATCH 0609/2083] ci(tests-windows): upgrade actions/checkout to v3 --- .github/workflows/tests-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 955b9b449..2d8c140af 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -25,7 +25,7 @@ jobs: TOXENV: asyncio steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 From 1a2cb61e22506c47e78bc800739d9cb2a8bf0144 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sat, 1 Oct 2022 20:27:10 +0200 Subject: [PATCH 0610/2083] ci(test-macos): upgrade actions/checkout to v3 --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 7819a4e12..8c9ef5c2f 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -10,7 +10,7 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 From af95331296a91444947095af19f8244d8bc915e1 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sat, 1 Oct 2022 20:27:23 +0200 Subject: [PATCH 0611/2083] ci(test-macos): upgrade actions/setup-python to v4 --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 7819a4e12..d49272fbf 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 759ad5dee4120dc2197ea6fa1559c92f77472abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sun, 2 Oct 2022 09:09:04 -0400 Subject: [PATCH 0612/2083] Require packaging --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index fcc902c62..8b9c43738 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'setuptools', + 'packaging', 'tldextract', 'lxml>=4.3.0', ] From c7d800ab229df50c2767ad67460e90f6213fc293 Mon Sep 17 00:00:00 2001 From: Abdul Rauf Date: Sun, 2 Oct 2022 19:12:48 +0500 Subject: [PATCH 0613/2083] CI: add Twine check in check workflow --- .github/workflows/checks.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b26f344ff..a708474ef 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -39,3 +39,8 @@ jobs: run: | pip install -U tox tox + - name: Twine check + run: | + pip install twine + python setup.py sdist + twine check dist/* From 69bf5c662555db64979a90c5c284b1faa4f24992 Mon Sep 17 00:00:00 2001 From: Abdul Rauf Date: Sun, 2 Oct 2022 20:27:24 +0500 Subject: [PATCH 0614/2083] CI: move twinecheck to tox env --- .github/workflows/checks.yml | 8 +++----- tox.ini | 8 ++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a708474ef..e515959ad 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -25,6 +25,9 @@ jobs: - python-version: "3.10" # Keep in sync with .readthedocs.yml env: TOXENV: docs + - python-version: "3.10" + env: + TOXENV: twinecheck steps: - uses: actions/checkout@v2 @@ -39,8 +42,3 @@ jobs: run: | pip install -U tox tox - - name: Twine check - run: | - pip install twine - python setup.py sdist - twine check dist/* diff --git a/tox.ini b/tox.ini index 2bf9454d0..2d94ba78f 100644 --- a/tox.ini +++ b/tox.ini @@ -71,6 +71,14 @@ deps = commands = pylint conftest.py docs extras scrapy setup.py tests +[testenv:twinecheck] +basepython = python3 +deps = + twine==4.0.1 +commands = + python setup.py sdist + twine check dist/* + [pinned] deps = cryptography==3.3 From 80194f1c0374b2aa429de4c7ea70a683c946db95 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 2 Oct 2022 15:22:06 -0300 Subject: [PATCH 0615/2083] CrawlSpider: add support for async def callbacks --- scrapy/spiders/crawl.py | 6 ++++-- tests/spiders.py | 12 ++++++++++++ tests/test_crawl.py | 11 +++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 1dcf2e6ab..d860ae0b4 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,7 +6,7 @@ See documentation in docs/topics/spiders.rst """ import copy -from typing import Sequence +from typing import Awaitable, Sequence from scrapy.http import Request, HtmlResponse from scrapy.linkextractors import LinkExtractor @@ -109,9 +109,11 @@ class CrawlSpider(Spider): rule = self._rules[failure.request.meta['rule']] return self._handle_failure(failure, rule.errback) - def _parse_response(self, response, callback, cb_kwargs, follow=True): + async def _parse_response(self, response, callback, cb_kwargs, follow=True): if callback: cb_res = callback(response, **cb_kwargs) or () + if isinstance(cb_res, Awaitable): + cb_res = await cb_res cb_res = self.process_results(response, cb_res) for request_or_item in iterate_spider_output(cb_res): yield request_or_item diff --git a/tests/spiders.py b/tests/spiders.py index 3b69aa7ae..2b78e1f7c 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -369,6 +369,18 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"}) +class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod): + """A CrawlSpider with an async def callback""" + name = 'crawl_spider_with_async_callback' + rules = ( + Rule(LinkExtractor(), callback='parse_async', follow=True), + ) + + async def parse_async(self, response, foo=None): + self.logger.info('[parse_async] status %i (foo: %s)', response.status, foo) + return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"}) + + class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = 'crawl_spider_with_errback' rules = ( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index c11871745..d14021319 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -37,6 +37,7 @@ from tests.spiders import ( BrokenStartRequestsSpider, BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, + CrawlSpiderWithAsyncCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, DelaySpider, @@ -391,6 +392,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[parse] status 201 (foo: None)", str(log)) self.assertIn("[parse] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks + def test_crawlspider_with_async_callback(self): + crawler = get_crawler(CrawlSpiderWithAsyncCallback) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse_async] status 200 (foo: None)", str(log)) + self.assertIn("[parse_async] status 201 (foo: None)", str(log)) + self.assertIn("[parse_async] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback) From da8f915091a769b3c3aca0f60d2370c2240213a1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 2 Oct 2022 17:37:10 -0300 Subject: [PATCH 0616/2083] Adapt for asyng generator callbacks --- scrapy/spiders/crawl.py | 11 +++++++---- scrapy/utils/asyncgen.py | 2 +- tests/spiders.py | 12 ++++++++++++ tests/test_crawl.py | 11 +++++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d860ae0b4..edac082d0 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,11 +6,12 @@ See documentation in docs/topics/spiders.rst """ import copy -from typing import Awaitable, Sequence +from typing import AsyncIterable, Awaitable, Sequence -from scrapy.http import Request, HtmlResponse +from scrapy.http import Request, Response, HtmlResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider +from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.spider import iterate_spider_output @@ -78,7 +79,7 @@ class CrawlSpider(Spider): def parse_start_url(self, response, **kwargs): return [] - def process_results(self, response, results): + def process_results(self, response: Response, results: list): return results def _build_request(self, rule_index, link): @@ -112,7 +113,9 @@ class CrawlSpider(Spider): async def _parse_response(self, response, callback, cb_kwargs, follow=True): if callback: cb_res = callback(response, **cb_kwargs) or () - if isinstance(cb_res, Awaitable): + if isinstance(cb_res, AsyncIterable): + cb_res = await collect_asyncgen(cb_res) + elif isinstance(cb_res, Awaitable): cb_res = await cb_res cb_res = self.process_results(response, cb_res) for request_or_item in iterate_spider_output(cb_res): diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 9f794de92..c84b51e8c 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,7 +1,7 @@ from typing import AsyncGenerator, AsyncIterable, Iterable, Union -async def collect_asyncgen(result: AsyncIterable): +async def collect_asyncgen(result: AsyncIterable) -> list: results = [] async for x in result: results.append(x) diff --git a/tests/spiders.py b/tests/spiders.py index 2b78e1f7c..5ea8a4a21 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -381,6 +381,18 @@ class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod): return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"}) +class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod): + """A CrawlSpider with an async generator callback""" + name = 'crawl_spider_with_async_generator_callback' + rules = ( + Rule(LinkExtractor(), callback='parse_async_gen', follow=True), + ) + + async def parse_async_gen(self, response, foo=None): + self.logger.info('[parse_async_gen] status %i (foo: %s)', response.status, foo) + yield Request(self.mockserver.url("/status?n=202"), self.parse_async_gen, cb_kwargs={"foo": "bar"}) + + class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = 'crawl_spider_with_errback' rules = ( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d14021319..8be4b6fe1 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -38,6 +38,7 @@ from tests.spiders import ( BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, + CrawlSpiderWithAsyncGeneratorCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, DelaySpider, @@ -402,6 +403,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[parse_async] status 201 (foo: None)", str(log)) self.assertIn("[parse_async] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks + def test_crawlspider_with_async_generator_callback(self): + crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log)) + self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log)) + self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback) From 41041ae740260c4bddda06ab7f4d8e4351e6e0e4 Mon Sep 17 00:00:00 2001 From: Felipe A Date: Mon, 3 Oct 2022 00:48:12 -0300 Subject: [PATCH 0617/2083] refact: add Osx DS_Store file to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index d77d24624..6c5c50e08 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ test-output.* # Windows Thumbs.db + +# OSX miscellaneous +.DS_Store \ No newline at end of file From 82d10f09145186fbd3a2c445287a7bf3e1e54263 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 6 Oct 2022 20:27:06 +0600 Subject: [PATCH 0618/2083] Add Ubuntu tests for Python 3.11rc2. --- .github/workflows/tests-ubuntu.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index be40c7c71..9e62b8e23 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -20,6 +20,12 @@ jobs: - python-version: "3.10" env: TOXENV: asyncio + - python-version: "3.11.0-rc.2" + env: + TOXENV: py + - python-version: "3.11.0-rc.2" + env: + TOXENV: asyncio - python-version: pypy3 env: TOXENV: pypy3 @@ -53,7 +59,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4' + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' run: | sudo apt-get update # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version From fa58ab21e4670db1b11b6097d809cd18dd2ca667 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 6 Oct 2022 20:59:06 +0600 Subject: [PATCH 0619/2083] Replace _getargspec_py23() with inspect.getfullargspec(). --- scrapy/utils/python.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 11c089ac2..8ce030d9d 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -180,23 +180,6 @@ def binary_is_text(data): return all(c not in _BINARYCHARS for c in data) -def _getargspec_py23(func): - """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords, - defaults) - - Was identical to inspect.getargspec() in python2, but uses - inspect.getfullargspec() for python3 behind the scenes to avoid - DeprecationWarning. - - >>> def f(a, b=2, *ar, **kw): - ... pass - - >>> _getargspec_py23(f) - ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,)) - """ - return inspect.ArgSpec(*inspect.getfullargspec(func)[:4]) - - def get_func_args(func, stripself=False): """Return the argument name list of a callable""" if inspect.isfunction(func): @@ -248,9 +231,9 @@ def get_spec(func): """ if inspect.isfunction(func) or inspect.ismethod(func): - spec = _getargspec_py23(func) + spec = inspect.getfullargspec(func) elif hasattr(func, '__call__'): - spec = _getargspec_py23(func.__call__) + spec = inspect.getfullargspec(func.__call__) else: raise TypeError(f'{type(func)} is not callable') From e60e8224a23ddd4fd98c54318d128f6744f40a6a Mon Sep 17 00:00:00 2001 From: Abinash Satapathy Date: Thu, 6 Oct 2022 19:58:48 +0200 Subject: [PATCH 0620/2083] Update and rename INSTALL to INSTALL.md --- INSTALL | 4 ---- INSTALL.md | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 INSTALL create mode 100644 INSTALL.md diff --git a/INSTALL b/INSTALL deleted file mode 100644 index 06e812936..000000000 --- a/INSTALL +++ /dev/null @@ -1,4 +0,0 @@ -For information about installing Scrapy see: - -* docs/intro/install.rst (local file) -* https://docs.scrapy.org/en/latest/intro/install.html (online version) diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 000000000..495413f97 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,4 @@ +For information about installing Scrapy see: + +* [Local docs](docs/intro/install.rst) +* [Online docs](https://docs.scrapy.org/en/latest/intro/install.html) From 300c42bfdf0afe0809b03cf571ebe5da7352301e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 14:53:19 +0600 Subject: [PATCH 0621/2083] Install PyPy using actions/setup-python. --- .github/workflows/tests-ubuntu.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 9e62b8e23..633e01c95 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -26,10 +26,9 @@ jobs: - python-version: "3.11.0-rc.2" env: TOXENV: asyncio - - python-version: pypy3 + - python-version: pypy3.9 env: TOXENV: pypy3 - PYPY_VERSION: 3.9-v7.3.9 # pinned deps - python-version: 3.7.13 @@ -38,10 +37,9 @@ jobs: - python-version: 3.7.13 env: TOXENV: asyncio-pinned - - python-version: pypy3 + - python-version: pypy3.7 env: TOXENV: pypy3-pinned - PYPY_VERSION: 3.7-v7.3.5 # extras # extra-deps includes reppy, which does not support Python 3.9 @@ -59,7 +57,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' run: | sudo apt-get update # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version @@ -68,13 +66,6 @@ jobs: - name: Run tests env: ${{ matrix.env }} run: | - if [[ ! -z "$PYPY_VERSION" ]]; then - export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi pip install -U tox tox From 4aea925714f7c9beb68d3d478ad528f00d488942 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 15:01:50 +0600 Subject: [PATCH 0622/2083] Update action versions. --- .github/workflows/tests-ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 633e01c95..9cd0df468 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -49,10 +49,10 @@ jobs: TOXENV: extra-deps steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 424849b27582d3a74fab0c43d842576722b48e25 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 15:17:55 +0600 Subject: [PATCH 0623/2083] Update mypy. --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2804ebe95..12a4516ce 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,8 @@ install_command = basepython = python3 deps = lxml-stubs==0.2.0 - mypy==0.971 + mypy==0.982 + types-attrs==19.1.0 types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 commands = From ccb6a8c098501c0f4322faeac17f9b97dad0c5e3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 15:27:14 +0600 Subject: [PATCH 0624/2083] Add a note about flake8. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 12a4516ce..7002c2f18 100644 --- a/tox.ini +++ b/tox.ini @@ -59,6 +59,7 @@ deps = # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 pytest-flake8==1.1.1 + # newer ones don't work: https://github.com/tholo/pytest-flake8/issues/87 flake8==4.0.1 commands = pytest --flake8 {posargs:docs scrapy tests} From d1515cc0755ae1d587ffbe4171544442fa10ef8d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 15:30:44 +0600 Subject: [PATCH 0625/2083] Update pylint. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 7002c2f18..463828e7c 100644 --- a/tox.ini +++ b/tox.ini @@ -69,7 +69,7 @@ commands = basepython = python3.8 deps = {[testenv:extra-deps]deps} - pylint==2.14.5 + pylint==2.15.3 commands = pylint conftest.py docs extras scrapy setup.py tests From ed9bc84d551c7302cd075aa52f30d38465749f05 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 8 Oct 2022 19:11:16 +0600 Subject: [PATCH 0626/2083] Remove a pin on libxml2-dev. --- .github/workflows/tests-ubuntu.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 9cd0df468..7915a9aab 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -60,8 +60,7 @@ jobs: if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' run: | sudo apt-get update - # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version - sudo apt-get install libxml2-dev/bionic-updates libxslt-dev + sudo apt-get install libxml2-dev libxslt-dev - name: Run tests env: ${{ matrix.env }} From 92f2d75ed39e4d75f6cca23b8d3aaf934677f4f3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 8 Oct 2022 19:12:32 +0600 Subject: [PATCH 0627/2083] Add a classifier for Python 3.11. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8b9c43738..a43cf08c8 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ setup( 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', From 5fa613b419f0d94b7dca8ae4ba2782ca268a9d7c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Oct 2022 11:02:57 +0600 Subject: [PATCH 0628/2083] Run flake8 directly. --- conftest.py | 10 ---------- tox.ini | 3 +-- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/conftest.py b/conftest.py index 117087790..d7fe80321 100644 --- a/conftest.py +++ b/conftest.py @@ -42,16 +42,6 @@ def chdir(tmpdir): tmpdir.chdir() -def pytest_collection_modifyitems(session, config, items): - # Avoid executing tests when executing `--flake8` flag (pytest-flake8) - try: - from pytest_flake8 import Flake8Item - if config.getoption('--flake8'): - items[:] = [item for item in items if isinstance(item, Flake8Item)] - except ImportError: - pass - - def pytest_addoption(parser): parser.addoption( "--reactor", diff --git a/tox.ini b/tox.ini index 463828e7c..822e96fde 100644 --- a/tox.ini +++ b/tox.ini @@ -58,11 +58,10 @@ deps = {[testenv]deps} # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 - pytest-flake8==1.1.1 # newer ones don't work: https://github.com/tholo/pytest-flake8/issues/87 flake8==4.0.1 commands = - pytest --flake8 {posargs:docs scrapy tests} + flake8 {posargs:docs scrapy tests} [testenv:pylint] # reppy does not support Python 3.9+ From eeb199adda2d7b56c78879df79c7294148ca15f1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Oct 2022 11:10:59 +0600 Subject: [PATCH 0629/2083] Fix flake8 issues in previously ignored files. --- .flake8 | 2 ++ docs/_ext/scrapydocs.py | 24 +++++++++---------- scrapy/utils/testsite.py | 2 +- .../CrawlerProcess/asyncio_deferred_signal.py | 1 - .../CrawlerProcess/asyncio_enabled_reactor.py | 4 ++-- tests/CrawlerProcess/reactor_default.py | 3 +-- .../reactor_default_twisted_reactor_select.py | 4 +--- tests/CrawlerProcess/reactor_select.py | 2 -- ..._select_subclass_twisted_reactor_select.py | 4 ---- .../reactor_select_twisted_reactor_select.py | 3 --- 10 files changed, 19 insertions(+), 30 deletions(-) diff --git a/.flake8 b/.flake8 index 1c503fb0b..d7aebc24b 100644 --- a/.flake8 +++ b/.flake8 @@ -4,6 +4,8 @@ max-line-length = 119 ignore = W503 exclude = + docs/conf.py + # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file # F401: Module imported but unused diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 640660943..d02a2e17b 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -80,24 +80,24 @@ def replace_settingslist_nodes(app, doctree, fromdocname): def setup(app): app.add_crossref_type( - directivename = "setting", - rolename = "setting", - indextemplate = "pair: %s; setting", + directivename="setting", + rolename="setting", + indextemplate="pair: %s; setting", ) app.add_crossref_type( - directivename = "signal", - rolename = "signal", - indextemplate = "pair: %s; signal", + directivename="signal", + rolename="signal", + indextemplate="pair: %s; signal", ) app.add_crossref_type( - directivename = "command", - rolename = "command", - indextemplate = "pair: %s; command", + directivename="command", + rolename="command", + indextemplate="pair: %s; command", ) app.add_crossref_type( - directivename = "reqmeta", - rolename = "reqmeta", - indextemplate = "pair: %s; reqmeta", + directivename="reqmeta", + rolename="reqmeta", + indextemplate="pair: %s; reqmeta", ) app.add_role('source', source_role) app.add_role('commit', commit_role) diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index fce77be32..5d3710391 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -23,7 +23,7 @@ class NoMetaRefreshRedirect(util.Redirect): def render(self, request): content = util.Redirect.render(self, request) return content.replace(b'http-equiv=\"refresh\"', - b'http-no-equiv=\"do-not-refresh-me\"') + b'http-no-equiv=\"do-not-refresh-me\"') def test_site(): diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index bdd3c1fef..b83f6a585 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -5,7 +5,6 @@ from typing import Optional from scrapy import Spider from scrapy.crawler import CrawlerProcess from scrapy.utils.defer import deferred_from_coro -from twisted.internet.defer import Deferred class UppercasePipeline: diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index f2a93074b..e561d63c7 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -6,8 +6,8 @@ if sys.version_info >= (3, 8) and sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) -import scrapy -from scrapy.crawler import CrawlerProcess +import scrapy # noqa: E402 +from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 5a21a3717..2c867df61 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,6 +1,6 @@ import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import reactor +from twisted.internet import reactor # noqa: F401 class NoRequestsSpider(scrapy.Spider): @@ -14,4 +14,3 @@ process = CrawlerProcess(settings={}) process.crawl(NoRequestsSpider) process.start() - diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index c476722ef..c2b30b044 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -1,6 +1,6 @@ import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import reactor +from twisted.internet import reactor # noqa: F401 class NoRequestsSpider(scrapy.Spider): @@ -16,5 +16,3 @@ process = CrawlerProcess(settings={ process.crawl(NoRequestsSpider) process.start() - - diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index eac6e2f89..ca70c06a0 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -15,5 +15,3 @@ process = CrawlerProcess(settings={}) process.crawl(NoRequestsSpider) process.start() - - diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 47f480605..0035daf1e 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -25,7 +25,3 @@ process = CrawlerProcess(settings={ process.crawl(NoRequestsSpider) process.start() - - - - diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index e0d2dab26..4f8394edb 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -17,6 +17,3 @@ process = CrawlerProcess(settings={ process.crawl(NoRequestsSpider) process.start() - - - From 5bf42606792c69268cc34da287293e9406e25883 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Oct 2022 11:14:20 +0600 Subject: [PATCH 0630/2083] Update flake8. --- tests/test_http_response.py | 6 ++++-- tox.ini | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 5d67a5e74..b42c95045 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -705,7 +705,8 @@ class HtmlResponseTest(TextResponseTest): def test_html_encoding(self): - body = b"""Some page + body = b"""Some page + Price: \xa3100' """ r1 = self.response_class("http://www.example.com", body=body) @@ -719,7 +720,8 @@ class HtmlResponseTest(TextResponseTest): self._assert_response_values(r2, 'iso-8859-1', body) # for conflicting declarations headers must take precedence - body = b"""Some page + body = b"""Some page + Price: \xa3100' """ r3 = self.response_class("http://www.example.com", body=body, diff --git a/tox.ini b/tox.ini index 822e96fde..eee99cb2d 100644 --- a/tox.ini +++ b/tox.ini @@ -58,8 +58,7 @@ deps = {[testenv]deps} # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 - # newer ones don't work: https://github.com/tholo/pytest-flake8/issues/87 - flake8==4.0.1 + flake8==5.0.4 commands = flake8 {posargs:docs scrapy tests} From b792632046093504b1df8bcb0b92d4cb52ebf436 Mon Sep 17 00:00:00 2001 From: Nirjas Jakilim Date: Mon, 10 Oct 2022 11:47:02 +0600 Subject: [PATCH 0631/2083] updated setup-python and checkout workflow --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e515959ad..cc8f20f44 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -30,10 +30,10 @@ jobs: TOXENV: twinecheck steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 44b682830..8e307189d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,10 +7,10 @@ jobs: if: startsWith(github.event.ref, 'refs/tags/') steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: "3.10" From d12fcc555b57dbd62c90212bd1d918d39e1ba45d Mon Sep 17 00:00:00 2001 From: Derek Date: Tue, 11 Oct 2022 10:32:45 -0700 Subject: [PATCH 0632/2083] Link to the Code of Conduct (#5659) --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 358302c76..d416ced3c 100644 --- a/README.rst +++ b/README.rst @@ -95,8 +95,7 @@ See https://docs.scrapy.org/en/master/contributing.html for details. Code of Conduct --------------- -Please note that this project is released with a Contributor Code of Conduct -(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). +Please note that this project is released with a Contributor `Code of Conduct `_. By participating in this project you agree to abide by its terms. Please report unacceptable behavior to opensource@zyte.com. From 96fb663ae1a95286b0634bc51bc17b42eb29bd0f Mon Sep 17 00:00:00 2001 From: Abdul Rauf Date: Tue, 11 Oct 2022 10:34:18 -0700 Subject: [PATCH 0633/2083] README: set Bash highlighting for pip install (#5648) --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d416ced3c..970bf2c35 100644 --- a/README.rst +++ b/README.rst @@ -64,7 +64,9 @@ Requirements Install ======= -The quick way:: +The quick way: + +.. code:: bash pip install scrapy From da9a2f8a946d6341be1cdd8cad0616a18bba6dbe Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 12 Oct 2022 11:10:39 -0300 Subject: [PATCH 0634/2083] Remove mention of minimum PyPy versions from the documentation (#5678) --- docs/intro/install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 80a9c16d6..f28f5216a 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -10,7 +10,7 @@ Supported Python versions ========================= Scrapy requires Python 3.7+, either the CPython implementation (default) or -the PyPy 7.3.5+ implementation (see :ref:`python:implementations`). +the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: @@ -219,7 +219,7 @@ After any of these workarounds you should be able to install Scrapy:: PyPy ---- -We recommend using the latest PyPy version. The version tested is 5.9.0. +We recommend using the latest PyPy version. For PyPy3, only Linux installation was tested. Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. From 715c05d504d22e87935ae42cee55ee35b12c2ebd Mon Sep 17 00:00:00 2001 From: gabrielztk <38334108+gabrielztk@users.noreply.github.com> Date: Thu, 13 Oct 2022 07:22:10 -0300 Subject: [PATCH 0635/2083] =?UTF-8?q?transport.producer.loseConnection()?= =?UTF-8?q?=20=E2=86=92=20transport.loseConnection()=20(#4995)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/core/downloader/handlers/http11.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 38935667d..6b8a18f1a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -384,8 +384,7 @@ class ScrapyAgent: logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": request, "handler": handler.__qualname__}) txresponse._transport.stopProducing() - with suppress(AttributeError): - txresponse._transport._producer.loseConnection() + txresponse._transport.loseConnection() return { "txresponse": txresponse, "body": b"", @@ -417,7 +416,7 @@ class ScrapyAgent: logger.warning(warning_msg, warning_args) - txresponse._transport._producer.loseConnection() + txresponse._transport.loseConnection() raise defer.CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: @@ -543,7 +542,7 @@ class _ResponseReader(protocol.Protocol): logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": self._request, "handler": handler.__qualname__}) self.transport.stopProducing() - self.transport._producer.loseConnection() + self.transport.loseConnection() failure = result if result.value.fail else None self._finish_response(flags=["download_stopped"], failure=failure) From 62cc26e209b66ce5941f15736f669c75dcac9a59 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Oct 2022 22:03:54 +0600 Subject: [PATCH 0636/2083] Change TWISTED_REACTOR in the default template. --- docs/topics/settings.rst | 5 +++++ scrapy/templates/project/module/settings.py.tmpl | 1 + 2 files changed, 6 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a711fd197..0b1ef71cf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1642,6 +1642,11 @@ install the default reactor defined by Twisted for the current platform. This is to maintain backward compatibility and avoid possible problems caused by using a non-default reactor. +.. versionchanged:: VERSION + The :command:`startproject` command now sets this setting to + ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated + ``settings.py`` file. + For additional information, see :doc:`core/howto/choosing-reactor`. diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 5e541e2c0..c0c34e986 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -89,3 +89,4 @@ ROBOTSTXT_OBEY = True # Set settings whose default value is deprecated to a future-proof value REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION' +TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' From 22a59d0005c03d866ce98d352024974a9e48e7d1 Mon Sep 17 00:00:00 2001 From: Nirjas Jakilim Date: Fri, 14 Oct 2022 23:41:50 +0600 Subject: [PATCH 0637/2083] CI: use the latest version of Ubuntu (#5675) --- .github/workflows/checks.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index cc8f20f44..439dfee51 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: checks: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8e307189d..f6b098b80 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,7 +3,7 @@ on: [push] jobs: publish: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest if: startsWith(github.event.ref, 'refs/tags/') steps: diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 7915a9aab..d2bfe4a5f 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: tests: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: From 043575123c57db9f055c3668a3d80485989df0b2 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sat, 15 Oct 2022 11:41:05 +0330 Subject: [PATCH 0638/2083] Add async callback support to the parse command (#5577) --- scrapy/commands/parse.py | 55 ++++++++++++++++++++++--------------- tests/test_command_parse.py | 15 ++++++++++ 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 9b4fb0ed6..d93ab2ac5 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -5,6 +5,8 @@ from typing import Dict from itemadapter import is_item, ItemAdapter from w3lib.url import is_url +from twisted.internet.defer import maybeDeferred + from scrapy.commands import BaseRunSpiderCommand from scrapy.http import Request from scrapy.utils import display @@ -110,16 +112,19 @@ class Command(BaseRunSpiderCommand): if not opts.nolinks: self.print_requests(colour=colour) - def run_callback(self, response, callback, cb_kwargs=None): - cb_kwargs = cb_kwargs or {} + def _get_items_and_requests(self, spider_output, opts, depth, spider, callback): items, requests = [], [] - - for x in iterate_spider_output(callback(response, **cb_kwargs)): + for x in spider_output: if is_item(x): items.append(x) elif isinstance(x, Request): requests.append(x) - return items, requests + return items, requests, opts, depth, spider, callback + + def run_callback(self, response, callback, cb_kwargs=None): + cb_kwargs = cb_kwargs or {} + d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs)) + return d def get_callback_from_rules(self, spider, response): if getattr(spider, 'rules', None): @@ -158,6 +163,25 @@ class Command(BaseRunSpiderCommand): logger.error('No response downloaded for: %(url)s', {'url': url}) + def scraped_data(self, args): + items, requests, opts, depth, spider, callback = args + if opts.pipelines: + itemproc = self.pcrawler.engine.scraper.itemproc + for item in items: + itemproc.process_item(item, spider) + self.add_items(depth, items) + self.add_requests(depth, requests) + + scraped_data = items if opts.output else [] + if depth < opts.depth: + for req in requests: + req.meta['_depth'] = depth + 1 + req.meta['_callback'] = req.callback + req.callback = callback + scraped_data += requests + + return scraped_data + def prepare_request(self, spider, request, opts): def callback(response, **cb_kwargs): # memorize first request @@ -191,23 +215,10 @@ class Command(BaseRunSpiderCommand): # parse items and requests depth = response.meta['_depth'] - items, requests = self.run_callback(response, cb, cb_kwargs) - if opts.pipelines: - itemproc = self.pcrawler.engine.scraper.itemproc - for item in items: - itemproc.process_item(item, spider) - self.add_items(depth, items) - self.add_requests(depth, requests) - - scraped_data = items if opts.output else [] - if depth < opts.depth: - for req in requests: - req.meta['_depth'] = depth + 1 - req.meta['_callback'] = req.callback - req.callback = callback - scraped_data += requests - - return scraped_data + d = self.run_callback(response, cb, cb_kwargs) + d.addCallback(self._get_items_and_requests, opts, depth, spider, callback) + d.addCallback(self.scraped_data) + return d # update request meta if any extra meta was passed through the --meta/-m opts. if opts.meta: diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 0d992be56..8368356e2 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -29,7 +29,15 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule +from scrapy.utils.test import get_from_asyncio_queue +class AsyncDefAsyncioSpider(scrapy.Spider): + + name = 'asyncdef{self.spider_name}' + + async def parse(self, response): + status = await get_from_asyncio_queue(response.status) + return [scrapy.Item(), dict(foo='bar')] class MySpider(scrapy.Spider): name = '{self.spider_name}' @@ -160,6 +168,13 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url('/html')]) self.assertIn("INFO: It Works!", _textmode(stderr)) + @defer.inlineCallbacks + def test_asyncio_parse_items(self): + status, out, stderr = yield self.execute( + ['--spider', 'asyncdef' + self.spider_name, '-c', 'parse', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + @defer.inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( From 75bb516edbc39f9a657e71e75f05f0fd6a33d60d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 15 Oct 2022 10:26:38 +0200 Subject: [PATCH 0639/2083] Adapt tests to the new value of TWISTED_REACTOR for new projects --- tests/test_commands.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 76d5f3935..eaca41102 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -689,8 +689,15 @@ class MySpider(scrapy.Spider): ]) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - def test_asyncio_enabled_false(self): + def test_asyncio_enabled_default(self): log = self.get_log(self.debug_log_spider, args=[]) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + def test_asyncio_enabled_false(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', 'TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor' + ]) + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') From 960a7f68f6939744b39d528e28f8f925a2e12ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 15 Oct 2022 11:27:00 +0200 Subject: [PATCH 0640/2083] Verify that the installed asyncio event loop matches ASYNCIO_EVENT_LOOP (#5529) Co-authored-by: Laerte Pereira <5853172+Laerte@users.noreply.github.com> --- scrapy/crawler.py | 14 ++++++++-- scrapy/utils/reactor.py | 18 ++++++++++++ .../asyncio_enabled_reactor_different_loop.py | 25 +++++++++++++++++ .../asyncio_enabled_reactor_same_loop.py | 28 +++++++++++++++++++ tests/test_crawler.py | 23 +++++++++++++++ 5 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py create mode 100644 tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e768bca12..65174d846 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -31,7 +31,12 @@ from scrapy.utils.log import ( ) from scrapy.utils.misc import create_instance, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names -from scrapy.utils.reactor import install_reactor, verify_installed_reactor +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + verify_installed_asyncio_event_loop, + verify_installed_reactor, +) logger = logging.getLogger(__name__) @@ -78,17 +83,20 @@ class Crawler: crawler=self, ) - reactor_class = self.settings.get("TWISTED_REACTOR") + reactor_class = self.settings["TWISTED_REACTOR"] + event_loop = self.settings["ASYNCIO_EVENT_LOOP"] if init_reactor: # this needs to be done after the spider settings are merged, # but before something imports twisted.internet.reactor if reactor_class: - install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) + install_reactor(reactor_class, event_loop) else: from twisted.internet import reactor # noqa: F401 log_reactor_info() if reactor_class: verify_installed_reactor(reactor_class) + if is_asyncio_reactor_installed() and event_loop: + verify_installed_asyncio_event_loop(event_loop) self.extensions = ExtensionManager.from_crawler(self) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index bc543b230..652733ce8 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -90,6 +90,24 @@ def verify_installed_reactor(reactor_path): raise Exception(msg) +def verify_installed_asyncio_event_loop(loop_path): + from twisted.internet import reactor + loop_class = load_object(loop_path) + if isinstance(reactor._asyncioEventloop, loop_class): + return + installed = ( + f"{reactor._asyncioEventloop.__class__.__module__}" + f".{reactor._asyncioEventloop.__class__.__qualname__}" + ) + specified = f"{loop_class.__module__}.{loop_class.__qualname__}" + raise Exception( + "Scrapy found an asyncio Twisted reactor already " + f"installed, and its event loop class ({installed}) does " + "not match the one specified in the ASYNCIO_EVENT_LOOP " + f"setting ({specified})" + ) + + def is_asyncio_reactor_installed(): from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py new file mode 100644 index 000000000..ea8242f67 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -0,0 +1,25 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncioreactor.install(asyncio.get_event_loop()) + +import scrapy # noqa: E402 +from scrapy.crawler import CrawlerProcess # noqa: E402 + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py new file mode 100644 index 000000000..d24bf3031 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -0,0 +1,28 @@ +import asyncio +import sys + +from uvloop import Loop + +from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncio.set_event_loop(Loop()) +asyncioreactor.install(asyncio.get_event_loop()) + +import scrapy # noqa: E402 +from scrapy.crawler import CrawlerProcess # noqa: E402 + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index cf15ba9b9..c61d461f7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -454,6 +454,29 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + def test_asyncio_enabled_reactor_same_loop(self): + log = self.run_script("asyncio_enabled_reactor_same_loop.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + def test_asyncio_enabled_reactor_different_loop(self): + log = self.run_script("asyncio_enabled_reactor_different_loop.py") + self.assertNotIn("Spider closed (finished)", log) + self.assertIn( + ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ), + log, + ) + def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) From c49764ffd7111d8a465a0d077d0df38bd40449fa Mon Sep 17 00:00:00 2001 From: mattkohl-flex Date: Mon, 17 Oct 2022 11:15:17 +0100 Subject: [PATCH 0641/2083] typo fixes --- docs/intro/install.rst | 2 +- docs/topics/contracts.rst | 2 +- docs/topics/extensions.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/request-response.rst | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index f28f5216a..9ab479edd 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -187,7 +187,7 @@ solutions: * Install `homebrew`_ following the instructions in https://brew.sh/ * Update your ``PATH`` variable to state that homebrew packages should be - used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly + used before system packages (Change ``.bashrc`` to ``.zshrc`` accordingly if you're using `zsh`_ as default shell):: echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index ef296dc9e..c29a3a410 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -102,7 +102,7 @@ override three methods: .. method:: Contract.post_process(output) This allows processing the output of the callback. Iterators are - converted listified before being passed to this hook. + converted to lists before being passed to this hook. Raise :class:`~scrapy.exceptions.ContractFail` from :class:`~scrapy.contracts.Contract.pre_process` or diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 297e1fdc5..130657b0b 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -17,7 +17,7 @@ settings, just like any other Scrapy code. It is customary for extensions to prefix their settings with their own name, to avoid collision with existing (and future) extensions. For example, a -hypothetic extension to handle `Google Sitemaps`_ would use settings like +hypothetical extension to handle `Google Sitemaps`_ would use settings like ``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on. .. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 477652704..33441838a 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -154,7 +154,7 @@ Too many spiders? If your project has too many spiders executed in parallel, the output of :func:`prefs()` can be difficult to read. For this reason, that function has a ``ignore`` argument which can be used to -ignore a particular class (and all its subclases). For +ignore a particular class (and all its subclasses). For example, this won't show any live references to spiders: >>> from scrapy.spiders import Spider diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 49cb69f67..7eb6942ac 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -446,7 +446,7 @@ class). Scenarios where changing the request fingerprinting algorithm may cause undesired results include, for example, using the HTTP cache middleware (see :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`). -Changing the request fingerprinting algorithm would invalidade the current +Changing the request fingerprinting algorithm would invalidate the current cache, requiring you to redownload all requests again. Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in From 06c8f673afe9af08784e62d67930a8cbc9887ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 17 Oct 2022 15:04:29 +0200 Subject: [PATCH 0642/2083] 2.7 release notes (#5680) * Fix the display name of documented fingerprinter class methods * Initial draft for the Scrapy 2.7 release notes * Update VERSION and PREVIOUS_VERSION references * Clarify the restrictions lifted for item field output names * Fix the description of the BOM bug fix * Fix the note about changes in MIME sniffing * Fix typo * Extend highlights * Fyx typo --- docs/_ext/scrapydocs.py | 2 +- docs/news.rst | 193 +++++++++++++++++- docs/topics/components.rst | 4 +- docs/topics/coroutines.rst | 12 +- docs/topics/request-response.rst | 45 ++-- docs/topics/settings.rst | 10 +- docs/topics/spider-middleware.rst | 8 +- scrapy/settings/default_settings.py | 2 +- .../templates/project/module/settings.py.tmpl | 2 +- scrapy/utils/request.py | 12 +- scrapy/utils/test.py | 2 +- tests/test_crawl.py | 2 +- tests/test_crawler.py | 8 +- tests/test_dupefilters.py | 12 +- tests/test_pipeline_crawl.py | 2 +- tests/test_scheduler.py | 2 +- tests/test_spiderloader/__init__.py | 2 +- tests/test_utils_request.py | 4 +- 18 files changed, 259 insertions(+), 65 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index d02a2e17b..f0f382da3 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -15,7 +15,7 @@ class SettingsListDirective(Directive): def is_setting_index(node): - if node.tagname == 'index': + if node.tagname == 'index' and node['entries']: # index entries for setting directives look like: # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] entry_type, info, refid = node['entries'][0][:3] diff --git a/docs/news.rst b/docs/news.rst index 9469d0fe5..d8b9fcd1e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,195 @@ Release notes ============= +.. _release-2.7.0: + +Scrapy 2.7.0 (to be released) +----------------------------- + +Highlights: + +- Added Python 3.11 support, dropped Python 3.6 support +- Improved support for :ref:`asynchronous callbacks ` +- :ref:`Asyncio support ` is enabled by default on new + projects +- Output names of item fields can now be arbitrary strings +- Centralized :ref:`request fingerprinting ` + configuration is now possible + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +Python 3.7 or greater is now required; support for Python 3.6 has been dropped. +Support for the upcoming Python 3.11 has been added. + +The minimum required version of some dependencies has changed as well: + +- lxml_: 3.5.0 → 4.3.0 + +- Pillow_ (:ref:`images pipeline `): 4.0.0 → 7.1.0 + +- zope.interface_: 5.0.0 → 5.1.0 + +(:issue:`5512`, :issue:`5514`, :issue:`5524`, :issue:`5563`, :issue:`5664`, +:issue:`5670`, :issue:`5678`) + + +Deprecations +~~~~~~~~~~~~ + +- :meth:`ImagesPipeline.thumb_path + ` must now accept an + ``item`` parameter (:issue:`5504`, :issue:`5508`). + +- The ``scrapy.downloadermiddlewares.decompression`` module is now + deprecated (:issue:`5546`, :issue:`5547`). + + +New features +~~~~~~~~~~~~ + +- The + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` + method of :ref:`spider middlewares ` can now be + defined as an :term:`asynchronous generator` (:issue:`4978`). + +- The output of :class:`~scrapy.Request` callbacks defined as + :ref:`coroutines ` is now processed asynchronously + (:issue:`4978`). + +- :class:`~scrapy.spiders.crawl.CrawlSpider` now supports :ref:`asynchronous + callbacks ` (:issue:`5657`). + +- New projects created with the :command:`startproject` command have + :ref:`asyncio support ` enabled by default (:issue:`5590`, + :issue:`5679`). + +- The :setting:`FEED_EXPORT_FIELDS` setting can now be defined as a + dictionary to customize the output name of item fields, lifting the + restriction that required output names to be valid Python identifiers, e.g. + preventing them to have whitespace (:issue:`1008`, :issue:`3266`, + :issue:`3696`). + +- You can now customize :ref:`request fingerprinting ` + through the new :setting:`REQUEST_FINGERPRINTER_CLASS` setting, instead of + having to change it on every Scrapy component that relies on request + fingerprinting (:issue:`900`, :issue:`3420`, :issue:`4113`, :issue:`4762`, + :issue:`4524`). + +- ``jsonl`` is now supported and encouraged as a file extension for `JSON + Lines`_ files (:issue:`4848`). + + .. _JSON Lines: https://jsonlines.org/ + +- :meth:`ImagesPipeline.thumb_path + ` now receives the + source :ref:`item ` (:issue:`5504`, :issue:`5508`). + + +Bug fixes +~~~~~~~~~ + +- When using Google Cloud Storage with a :ref:`media pipeline + `, :setting:`FILES_EXPIRES` now also works when + :setting:`FILES_STORE` does not point at the root of your Google Cloud + Storage bucket (:issue:`5317`, :issue:`5318`). + +- The :command:`parse` command now supports :ref:`asynchronous callbacks + ` (:issue:`5424`, :issue:`5577`). + +- When using the :command:`parse` command with a URL for which there is no + available spider, an exception is no longer raised (:issue:`3264`, + :issue:`3265`, :issue:`5375`, :issue:`5376`, :issue:`5497`). + +- :class:`~scrapy.http.TextResponse` now gives higher priority to the `byte + order mark`_ when determining the text encoding of the response body, + following the `HTML living standard`_ (:issue:`5601`, :issue:`5611`). + + .. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark + .. _HTML living standard: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding + +- MIME sniffing takes the response body into account in FTP and HTTP/1.0 + requests, as well as in cached requests (:issue:`4873`). + +- MIME sniffing now detects valid HTML 5 documents even if the ``html`` tag + is missing (:issue:`4873`). + +- An exception is now raised if :setting:`ASYNCIO_EVENT_LOOP` has a value + that does not match the asyncio event loop actually installed + (:issue:`5529`). + +- Fixed :meth:`Headers.getlist ` + returning only the last header (:issue:`5515`, :issue:`5526`). + +- Fixed :class:`LinkExtractor + ` not ignoring the + ``tar.gz`` file extension by default (:issue:`1837`, :issue:`2067`, + :issue:`4066`) + + +Documentation +~~~~~~~~~~~~~ + +- Clarified the return type of :meth:`Spider.parse ` + (:issue:`5602`, :issue:`5608`). + +- To enable + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + to do `brotli compression`_, installing brotli_ is now recommended instead + of installing brotlipy_, as the former provides a more recent version of + brotli. + + .. _brotli: https://github.com/google/brotli + .. _brotli compression: https://www.ietf.org/rfc/rfc7932.txt + +- :ref:`Signal documentation ` now mentions :ref:`coroutine + support ` and uses it in code examples (:issue:`4852`, + :issue:`5358`). + +- :ref:`bans` now recommends `Common Crawl`_ instead of `Google cache`_ + (:issue:`3582`, :issue:`5432`). + + .. _Common Crawl: https://commoncrawl.org/ + .. _Google cache: http://www.googleguide.com/cached_pages.html + +- The new :ref:`topics-components` topic covers enforcing requirements on + Scrapy components, like :ref:`downloader middlewares + `, :ref:`extensions `, + :ref:`item pipelines `, :ref:`spider middlewares + `, and more; :ref:`enforce-asyncio-requirement` + has also been added (:issue:`4978`). + +- :ref:`topics-settings` now indicates that setting values must be + :ref:`picklable ` (:issue:`5607`, :issue:`5629`). + +- Removed outdated documentation (:issue:`5446`, :issue:`5373`, + :issue:`5369`, :issue:`5370`, :issue:`5554`). + +- Fixed typos (:issue:`5442`, :issue:`5455`, :issue:`5457`, :issue:`5461`, + :issue:`5538`, :issue:`5553`, :issue:`5558`, :issue:`5624`, :issue:`5631`). + +- Fixed other issues (:issue:`5283`, :issue:`5284`, :issue:`5559`, + :issue:`5567`, :issue:`5648`, :issue:`5659`, :issue:`5665`). + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added a continuous integration job to run `twine check`_ (:issue:`5655`, + :issue:`5656`). + + .. _twine check: https://twine.readthedocs.io/en/stable/#twine-check + +- Addressed test issues and warnings (:issue:`5560`, :issue:`5561`, + :issue:`5612`, :issue:`5617`, :issue:`5639`, :issue:`5645`, :issue:`5662`, + :issue:`5671`, :issue:`5675`). + +- Cleaned up code (:issue:`4991`, :issue:`4995`, :issue:`5451`, + :issue:`5487`, :issue:`5542`, :issue:`5667`, :issue:`5668`, :issue:`5672`). + +- Applied minor code improvements (:issue:`5661`). + + .. _release-2.6.3: Scrapy 2.6.3 (2022-09-27) @@ -3139,7 +3328,7 @@ New Features ~~~~~~~~~~~~ - Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`) -- Support `brotli`_-compressed content; requires optional `brotlipy`_ +- Support `brotli-compressed`_ content; requires optional `brotlipy`_ (:issue:`2535`) - New :ref:`response.follow ` shortcut for creating requests (:issue:`1940`) @@ -3176,7 +3365,7 @@ New Features - ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command (:issue:`2740`) -.. _brotli: https://github.com/google/brotli +.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://github.com/python-hyper/brotlipy/ Bug fixes diff --git a/docs/topics/components.rst b/docs/topics/components.rst index c44f3def2..ca301b827 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -75,9 +75,9 @@ If your requirement is a minimum Scrapy version, you may use class MyComponent: def __init__(self): - if parse_version(scrapy.__version__) < parse_version('VERSION'): + if parse_version(scrapy.__version__) < parse_version('2.7'): raise RuntimeError( - f"{MyComponent.__qualname__} requires Scrapy VERSION or " + f"{MyComponent.__qualname__} requires Scrapy 2.7 or " f"later, which allow defining the process_spider_output " f"method of spider middlewares as an asynchronous " f"generator." diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 750263385..a1ba4ba5c 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -22,7 +22,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): If you are using any custom or third-party :ref:`spider middleware `, see :ref:`sync-async-spider-middleware`. - .. versionchanged:: VERSION + .. versionchanged:: 2.7 Output of async callbacks is now processed asynchronously instead of collecting all of it first. @@ -49,7 +49,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): See also :ref:`sync-async-spider-middleware` and :ref:`universal-spider-middleware`. - .. versionadded:: VERSION + .. versionadded:: 2.7 General usage ============= @@ -129,7 +129,7 @@ Common use cases for asynchronous code include: Mixing synchronous and asynchronous spider middlewares ====================================================== -.. versionadded:: VERSION +.. versionadded:: 2.7 The output of a :class:`~scrapy.Request` callback is passed as the ``result`` parameter to the @@ -182,10 +182,10 @@ process_spider_output_async method `. Universal spider middlewares ============================ -.. versionadded:: VERSION +.. versionadded:: 2.7 To allow writing a spider middleware that supports asynchronous execution of -its ``process_spider_output`` method in Scrapy VERSION and later (avoiding +its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding :ref:`asynchronous-to-synchronous conversions `) while maintaining support for older Scrapy versions, you may define ``process_spider_output`` as a synchronous method and define an @@ -206,7 +206,7 @@ For example:: yield r .. note:: This is an interim measure to allow, for a time, to write code that - works in Scrapy VERSION and later without requiring + works in Scrapy 2.7 and later without requiring asynchronous-to-synchronous conversions, and works in earlier Scrapy versions as well. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 49cb69f67..4393e1c68 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -394,7 +394,7 @@ To change how request fingerprints are built for your requests, use the REQUEST_FINGERPRINTER_CLASS ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: VERSION +.. versionadded:: 2.7 Default: :class:`scrapy.utils.request.RequestFingerprinter` @@ -409,38 +409,38 @@ import path. REQUEST_FINGERPRINTER_IMPLEMENTATION ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: VERSION +.. versionadded:: 2.7 -Default: ``'PREVIOUS_VERSION'`` +Default: ``'2.6'`` Determines which request fingerprinting algorithm is used by the default request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`). Possible values are: -- ``'PREVIOUS_VERSION'`` (default) +- ``'2.6'`` (default) This implementation uses the same request fingerprinting algorithm as - Scrapy PREVIOUS_VERSION and earlier versions. + Scrapy 2.6 and earlier versions. Even though this is the default value for backward compatibility reasons, it is a deprecated value. -- ``'VERSION'`` +- ``'2.7'`` - This implementation was introduced in Scrapy VERSION to fix an issue of the + This implementation was introduced in Scrapy 2.7 to fix an issue of the previous implementation. New projects should use this value. The :command:`startproject` command sets this value in the generated ``settings.py`` file. -If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are +If you are using the default value (``'2.6'``) for this setting, and you are using Scrapy components where changing the request fingerprinting algorithm would cause undesired results, you need to carefully decide when to change the value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS` -setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request +setting to a custom request fingerprinter class that implements the 2.6 request fingerprinting algorithm and does not log this warning ( -:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a +:ref:`2.6-request-fingerprinter` includes an example implementation of such a class). Scenarios where changing the request fingerprinting algorithm may cause @@ -449,14 +449,14 @@ undesired results include, for example, using the HTTP cache middleware (see Changing the request fingerprinting algorithm would invalidade the current cache, requiring you to redownload all requests again. -Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in +Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in your settings to switch already to the request fingerprinting implementation that will be the only request fingerprinting implementation available in a future version of Scrapy, and remove the deprecation warning triggered by using -the default value (``'PREVIOUS_VERSION'``). +the default value (``'2.6'``). -.. _PREVIOUS_VERSION-request-fingerprinter: +.. _2.6-request-fingerprinter: .. _custom-request-fingerprinter: Writing your own request fingerprinter @@ -464,6 +464,8 @@ Writing your own request fingerprinter A request fingerprinter is a class that must implement the following method: +.. currentmodule:: None + .. method:: fingerprint(self, request) Return a :class:`bytes` object that uniquely identifies *request*. @@ -476,6 +478,7 @@ A request fingerprinter is a class that must implement the following method: Additionally, it may also implement the following methods: .. classmethod:: from_crawler(cls, crawler) + :noindex: If present, this class method is called to create a request fingerprinter instance from a :class:`~scrapy.crawler.Crawler` object. It must return a @@ -495,11 +498,13 @@ Additionally, it may also implement the following methods: :class:`~scrapy.settings.Settings` object. It must return a new instance of the request fingerprinter. -The ``fingerprint`` method of the default request fingerprinter, +.. currentmodule:: scrapy.http + +The :meth:`fingerprint` method of the default request fingerprinter, :class:`scrapy.utils.request.RequestFingerprinter`, uses :func:`scrapy.utils.request.fingerprint` with its default parameters. For some -common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well -in your ``fingerprint`` method implementation: +common use cases you can use :func:`scrapy.utils.request.fingerprint` as well +in your :meth:`fingerprint` method implementation: .. autofunction:: scrapy.utils.request.fingerprint @@ -519,7 +524,7 @@ account:: You can also write your own fingerprinting logic from scratch. -However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure +However, if you do not use :func:`scrapy.utils.request.fingerprint`, make sure you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints: - Caching saves CPU by ensuring that fingerprints are calculated only once @@ -553,7 +558,7 @@ If you need to be able to override the request fingerprinting for arbitrary requests from your spider callbacks, you may implement a request fingerprinter that reads fingerprints from :attr:`request.meta ` when available, and then falls back to -:func:`~scrapy.utils.request.fingerprint`. For example:: +:func:`scrapy.utils.request.fingerprint`. For example:: from scrapy.utils.request import fingerprint @@ -564,8 +569,8 @@ when available, and then falls back to return request.meta['fingerprint'] return fingerprint(request) -If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION -without using the deprecated ``'PREVIOUS_VERSION'`` value of the +If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6 +without using the deprecated ``'2.6'`` value of the :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following request fingerprinter:: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0b1ef71cf..40bcda288 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1642,7 +1642,7 @@ install the default reactor defined by Twisted for the current platform. This is to maintain backward compatibility and avoid possible problems caused by using a non-default reactor. -.. versionchanged:: VERSION +.. versionchanged:: 2.7 The :command:`startproject` command now sets this setting to ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated ``settings.py`` file. @@ -1661,14 +1661,14 @@ Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. -This setting can act as a stopping condition in case of URLs of ever-increasing -length, which may be caused for example by a programming error either in the -target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and +This setting can act as a stopping condition in case of URLs of ever-increasing +length, which may be caused for example by a programming error either in the +target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and :setting:`DEPTH_LIMIT`. Use ``0`` to allow URLs of any length. -The default value is copied from the `Microsoft Internet Explorer maximum URL +The default value is copied from the `Microsoft Internet Explorer maximum URL length`_, even though this setting exists for different reasons. .. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 816cb5e03..303401a3c 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -105,17 +105,17 @@ object gives you access, for example, to the :ref:`settings `. :class:`~scrapy.Request` objects and :ref:`item objects `. - .. versionchanged:: VERSION + .. versionchanged:: 2.7 This method may be defined as an :term:`asynchronous generator`, in which case ``result`` is an :term:`asynchronous iterable`. Consider defining this method as an :term:`asynchronous generator`, which will be a requirement in a future version of Scrapy. However, if you plan on sharing your spider middleware with other people, consider - either :ref:`enforcing Scrapy VERSION ` + either :ref:`enforcing Scrapy 2.7 ` as a minimum requirement of your spider middleware, or :ref:`making your spider middleware universal ` so that - it works with Scrapy versions earlier than Scrapy VERSION. + it works with Scrapy versions earlier than Scrapy 2.7. :param response: the response which generated this output from the spider @@ -130,7 +130,7 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_spider_output_async(response, result, spider) - .. versionadded:: VERSION + .. versionadded:: 2.7 If defined, this method must be an :term:`asynchronous generator`, which will be called instead of :meth:`process_spider_output` if diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ff86af125..29ff028be 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -248,7 +248,7 @@ REFERER_ENABLED = True REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter' -REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION' +REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.6' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index c0c34e986..bbf60982c 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -88,5 +88,5 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' # Set settings whose default value is deprecated to a future-proof value -REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION' +REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7' TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index cf33317ce..fbddc41fb 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -236,10 +236,10 @@ class RequestFingerprinter: 'REQUEST_FINGERPRINTER_IMPLEMENTATION' ) else: - implementation = 'PREVIOUS_VERSION' - if implementation == 'PREVIOUS_VERSION': + implementation = '2.6' + if implementation == '2.6': message = ( - '\'PREVIOUS_VERSION\' is a deprecated value for the ' + '\'2.6\' is a deprecated value for the ' '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n' '\n' 'It is also the default value. In other words, it is normal ' @@ -254,14 +254,14 @@ class RequestFingerprinter: ) warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) self._fingerprint = _request_fingerprint_as_bytes - elif implementation == 'VERSION': + elif implementation == '2.7': self._fingerprint = fingerprint else: raise ValueError( f'Got an invalid value on setting ' f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': ' - f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) ' - f'and \'VERSION\'.' + f'{implementation!r}. Valid values are \'2.6\' (deprecated) ' + f'and \'2.7\'.' ) def fingerprint(self, request): diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 0b828f7c0..445cd2e3a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -65,7 +65,7 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): # Set by default settings that prevent deprecation warnings. settings = {} if prevent_warnings: - settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION' + settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = '2.7' settings.update(settings_dict or {}) runner = CrawlerRunner(settings) return runner.create_crawler(spidercls or Spider) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 8be4b6fe1..5383ec652 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -350,7 +350,7 @@ with multiples lines @defer.inlineCallbacks def test_crawl_multiple(self): - runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}) + runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}) runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c61d461f7..da6024c2b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -110,7 +110,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): 'LOG_LEVEL': 'INFO', 'LOG_FILE': log_file, # settings to avoid extra warnings - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, } @@ -235,7 +235,7 @@ class NoRequestsSpider(scrapy.Spider): class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): - return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}) + return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}) @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful(self): @@ -283,14 +283,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", }) else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", }) yield runner.crawl(NoRequestsSpider) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 8a37a8ebe..6ebb716b0 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -51,7 +51,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_crawler_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -60,7 +60,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_settings_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -68,7 +68,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_direct_scheduler(self): settings = {'DUPEFILTER_CLASS': DirectDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, 'n/a') @@ -172,7 +172,7 @@ class RFPDupeFilterTest(unittest.TestCase): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) @@ -199,7 +199,7 @@ class RFPDupeFilterTest(unittest.TestCase): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) @@ -233,7 +233,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug_default_dupefilter(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index e46532a1c..0e174cd34 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -64,7 +64,7 @@ class FileDownloadCrawlTestCase(TestCase): self.tmpmediastore = self.mktemp() os.mkdir(self.tmpmediastore) self.settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', 'ITEM_PIPELINES': {self.pipeline_class: 1}, self.store_setting_key: self.tmpmediastore, } diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index ac66056ba..50a7755c1 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -52,7 +52,7 @@ class MockCrawler(Crawler): SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, JOBDIR=jobdir, DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', - REQUEST_FINGERPRINTER_IMPLEMENTATION='VERSION', + REQUEST_FINGERPRINTER_IMPLEMENTATION='2.7', ) super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 3719c7c9f..7a590f96c 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -98,7 +98,7 @@ class SpiderLoaderTest(unittest.TestCase): module = 'tests.test_spiderloader.test_spiders.spider1' runner = CrawlerRunner({ 'SPIDER_MODULES': [module], - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', }) self.assertRaisesRegex(KeyError, 'Spider not found', diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 8bc7922b6..a92d9a0ac 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -505,7 +505,7 @@ class RequestFingerprinterTestCase(unittest.TestCase): def test_deprecated_implementation(self): settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.6', } with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) @@ -518,7 +518,7 @@ class RequestFingerprinterTestCase(unittest.TestCase): def test_recommended_implementation(self): settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', } with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) From 20b79a0f2e47800bf4648c7f890c8170fc8f5ede Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Oct 2022 19:09:22 +0600 Subject: [PATCH 0643/2083] =?UTF-8?q?Bump=20version:=202.6.2=20=E2=86=92?= =?UTF-8?q?=202.7.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2e2f7949a..f88071685 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.6.2 +current_version = 2.7.0 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index d8b9fcd1e..1ec183a1d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.7.0: -Scrapy 2.7.0 (to be released) +Scrapy 2.7.0 (2022-10-17) ----------------------------- Highlights: diff --git a/scrapy/VERSION b/scrapy/VERSION index 097a15a2a..24ba9a38d 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.6.2 +2.7.0 From 40d9ca3bdd6fd50438295f91cff275fcac57e0fa Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Mon, 17 Oct 2022 17:40:10 -0400 Subject: [PATCH 0644/2083] use pathlib --- conftest.py | 2 +- docs/Makefile | 4 +- docs/conf.py | 9 +- docs/conftest.py | 9 +- docs/intro/tutorial.rst | 10 +- docs/topics/item-pipeline.rst | 4 +- docs/topics/media-pipeline.rst | 9 +- docs/utils/linkfix.py | 9 +- scrapy/commands/__init__.py | 4 +- scrapy/commands/genspider.py | 48 +++-- scrapy/commands/runspider.py | 21 +- scrapy/commands/startproject.py | 44 ++-- scrapy/core/downloader/handlers/file.py | 5 +- scrapy/core/scheduler.py | 19 +- scrapy/crawler.py | 8 +- scrapy/dupefilters.py | 4 +- scrapy/extensions/feedexport.py | 16 +- scrapy/extensions/httpcache.py | 48 +++-- scrapy/extensions/spiderstate.py | 12 +- scrapy/http/response/__init__.py | 4 +- scrapy/pipelines/files.py | 41 ++-- scrapy/settings/default_settings.py | 4 +- scrapy/spiders/__init__.py | 9 +- scrapy/squeues.py | 11 +- scrapy/utils/conf.py | 32 +-- scrapy/utils/job.py | 6 +- scrapy/utils/project.py | 29 +-- scrapy/utils/request.py | 2 +- scrapy/utils/template.py | 12 +- scrapy/utils/test.py | 5 +- setup.py | 5 +- tests/__init__.py | 17 +- tests/keys/__init__.py | 22 +- tests/mockserver.py | 7 +- tests/test_cmdline/__init__.py | 14 +- .../__init__.py | 4 +- tests/test_command_check.py | 9 +- tests/test_command_parse.py | 22 +- tests/test_command_shell.py | 6 +- tests/test_commands.py | 199 +++++++++--------- tests/test_crawler.py | 32 ++- tests/test_dependencies.py | 7 +- tests/test_downloader_handlers.py | 69 +++--- ...st_downloadermiddleware_httpcompression.py | 7 +- tests/test_dupefilters.py | 4 +- tests/test_engine.py | 6 +- tests/test_feedexport.py | 155 +++++++------- tests/test_http2_client_protocol.py | 15 +- tests/test_pipeline_crawl.py | 14 +- tests/test_pipeline_files.py | 3 +- tests/test_proxy_connect.py | 6 +- tests/test_spiderloader/__init__.py | 40 ++-- tests/test_spiderstate.py | 4 +- tests/test_utils_gz.py | 44 ++-- tests/test_utils_iterators.py | 7 - tests/test_utils_misc/__init__.py | 3 +- tests/test_utils_project.py | 17 +- tests/test_utils_response.py | 12 +- tests/test_utils_template.py | 22 +- tests/test_webclient.py | 19 +- 60 files changed, 595 insertions(+), 636 deletions(-) diff --git a/conftest.py b/conftest.py index d7fe80321..7c1da3556 100644 --- a/conftest.py +++ b/conftest.py @@ -21,7 +21,7 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with open('tests/ignores.txt') as reader: +with Path('tests/ignores.txt').open() as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != '#': diff --git a/docs/Makefile b/docs/Makefile index 87d5d3047..596cb6cef 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -86,8 +86,8 @@ coverage: BUILDER = coverage coverage: build htmlview: html - $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \ - os.path.realpath('build/html/index.html'))" + $(PYTHON) -c "import webbrowser; from pathlib import Path; \ + webbrowser.open('file://' + Path('build/html/index.html').resolve())" clean: -rm -rf build/* diff --git a/docs/conf.py b/docs/conf.py index 3241295af..d2a77003e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,13 +11,12 @@ import sys from datetime import datetime -from os import path +from pathlib import Path # If your extensions are in another directory, add it here. If the directory -# is relative to the documentation root, use os.path.abspath to make it -# absolute, like shown here. -sys.path.append(path.join(path.dirname(__file__), "_ext")) -sys.path.insert(0, path.dirname(path.dirname(__file__))) +# is relative to the documentation root, use Path.absolute to make it absolute. +sys.path.append(str(Path(__file__).parent / "_ext")) +sys.path.insert(0, str(Path(__file__).parent.parent)) # General configuration diff --git a/docs/conftest.py b/docs/conftest.py index a0636f8ac..24a72a4b6 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -1,5 +1,5 @@ -import os from doctest import ELLIPSIS, NORMALIZE_WHITESPACE +from pathlib import Path from scrapy.http.response.html import HtmlResponse from sybil import Sybil @@ -12,10 +12,9 @@ from sybil.parsers.doctest import DocTestParser from sybil.parsers.skip import skip -def load_response(url, filename): - input_path = os.path.join(os.path.dirname(__file__), '_tests', filename) - with open(input_path, 'rb') as input_file: - return HtmlResponse(url, body=input_file.read()) +def load_response(url: str, filename: str) -> HtmlResponse: + input_path = Path(__file__).parent / '_tests' / filename + return HtmlResponse(url, body=input_path.read_bytes()) def setup(namespace): diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 092123d1d..901a170b4 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -85,6 +85,8 @@ page content to extract data. This is the code for our first Spider. Save it in a file named ``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:: + from pathlib import Path + import scrapy @@ -102,8 +104,7 @@ This is the code for our first Spider. Save it in a file named def parse(self, response): page = response.url.split("/")[-2] filename = f'quotes-{page}.html' - with open(filename, 'wb') as f: - f.write(response.body) + Path(filename).write_bytes(response.body) self.log(f'Saved file {filename}') @@ -178,6 +179,8 @@ with a list of URLs. This list will then be used by the default implementation of :meth:`~scrapy.Spider.start_requests` to create the initial requests for your spider:: + from pathlib import Path + import scrapy @@ -191,8 +194,7 @@ for your spider:: def parse(self, response): page = response.url.split("/")[-2] filename = f'quotes-{page}.html' - with open(filename, 'wb') as f: - f.write(response.body) + Path(filename).write_bytes(response.body) The :meth:`~scrapy.Spider.parse` method will be called to handle each of the requests for those URLs, even though we haven't explicitly told Scrapy diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index af294f52c..1672ccbcc 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -186,6 +186,7 @@ item. :: import hashlib + from pathlib import Path from urllib.parse import quote import scrapy @@ -214,8 +215,7 @@ item. url = adapter["url"] url_hash = hashlib.md5(url.encode("utf8")).hexdigest() filename = f"{url_hash}.png" - with open(filename, "wb") as f: - f.write(response.body) + Path(filename).write_bytes(response.body) # Store filename in item. adapter["screenshot_filename"] = filename diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 0925e6bb5..a528746b0 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -156,7 +156,6 @@ By overriding ``file_path`` like this: .. code-block:: python import hashlib - from os.path import splitext def file_path(self, request, response=None, info=None, *, item=None): image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5) @@ -498,7 +497,7 @@ See here the methods that you can override in your custom Files Pipeline: approach to download all files into the ``files`` folder with their original filenames (e.g. ``files/foo.png``):: - import os + from pathlib import PurePosixPath from urllib.parse import urlparse from scrapy.pipelines.files import FilesPipeline @@ -506,7 +505,7 @@ See here the methods that you can override in your custom Files Pipeline: class MyFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return 'files/' + os.path.basename(urlparse(request.url).path) + return 'files/' + PurePosixPath(urlparse(request.url).path).name Similarly, you can use the ``item`` to determine the file path based on some item property. @@ -637,7 +636,7 @@ See here the methods that you can override in your custom Images Pipeline: approach to download all files into the ``files`` folder with their original filenames (e.g. ``files/foo.png``):: - import os + from pathlib import PurePosixPath from urllib.parse import urlparse from scrapy.pipelines.images import ImagesPipeline @@ -645,7 +644,7 @@ See here the methods that you can override in your custom Images Pipeline: class MyImagesPipeline(ImagesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return 'files/' + os.path.basename(urlparse(request.url).path) + return 'files/' + PurePosixPath(urlparse(request.url).path).name Similarly, you can use the ``item`` to determine the file path based on some item property. diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 95a3f17d5..7a0c5288c 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -13,6 +13,7 @@ Author: dufferzafar """ import re +from pathlib import Path def main(): @@ -27,7 +28,7 @@ def main(): # Read lines from the linkcheck output file try: - with open("build/linkcheck/output.txt") as out: + with Path("build/linkcheck/output.txt").open() as out: output_lines = out.readlines() except IOError: print("linkcheck output not found; please run linkcheck first.") @@ -51,14 +52,12 @@ def main(): # Update the previous file if _filename: - with open(_filename, "w") as _file: - _file.write(_contents) + Path(_filename).write_text(_contents) _filename = newfilename # Read the new file to memory - with open(_filename) as _file: - _contents = _file.read() + _contents = Path(_filename).read_text() _contents = _contents.replace(match.group(3), match.group(4)) else: diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index fb304b8c0..8570d90bd 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -3,6 +3,7 @@ Base class for Scrapy commands """ import os import argparse +from pathlib import Path from typing import Any, Dict from twisted.python import failure @@ -93,8 +94,7 @@ class ScrapyCommand: self.settings.set('LOG_ENABLED', False, priority='cmdline') if opts.pidfile: - with open(opts.pidfile, "w") as f: - f.write(str(os.getpid()) + os.linesep) + Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep) if opts.pdb: failure.startDebugMode() diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index ed5f588e9..01b4a0dbd 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -2,8 +2,9 @@ import os import shutil import string +from pathlib import Path from importlib import import_module -from os.path import join, dirname, abspath, exists, splitext +from typing import Optional from urllib.parse import urlparse import scrapy @@ -62,8 +63,7 @@ class Command(ScrapyCommand): if opts.dump: template_file = self._find_template(opts.dump) if template_file: - with open(template_file, "r") as f: - print(f.read()) + print(template_file.read_text()) return if len(args) != 2: raise UsageError() @@ -98,11 +98,11 @@ class Command(ScrapyCommand): } if self.settings.get('NEWSPIDER_MODULE'): spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) - spiders_dir = abspath(dirname(spiders_module.__file__)) + spiders_dir = Path(spiders_module.__file__).parent.resolve() else: spiders_module = None - spiders_dir = "." - spider_file = f"{join(spiders_dir, module)}.py" + spiders_dir = Path(".") + spider_file = f"{spiders_dir / module}.py" shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) print(f"Created spider {name!r} using template {template_name!r} ", @@ -110,24 +110,25 @@ class Command(ScrapyCommand): if spiders_module: print(f"in module:\n {spiders_module.__name__}.{module}") - def _find_template(self, template): - template_file = join(self.templates_dir, f'{template}.tmpl') - if exists(template_file): + def _find_template(self, template: str) -> Optional[Path]: + template_file = Path(self.templates_dir, f'{template}.tmpl') + if template_file.exists(): return template_file print(f"Unable to find template: {template}\n") print('Use "scrapy genspider --list" to see all available templates.') def _list_templates(self): print("Available templates:") - for filename in sorted(os.listdir(self.templates_dir)): - if filename.endswith('.tmpl'): - print(f" {splitext(filename)[0]}") + for file in sorted(Path(self.templates_dir).iterdir()): + if file.suffix == '.tmpl': + print(f" {file.stem}") - def _spider_exists(self, name): + def _spider_exists(self, name: str) -> bool: if not self.settings.get('NEWSPIDER_MODULE'): # if run as a standalone command and file with same filename already exists - if exists(name + ".py"): - print(f"{abspath(name + '.py')} already exists") + path = Path(name + ".py") + if path.exists(): + print(f"{path.resolve()} already exists") return True return False @@ -143,17 +144,18 @@ class Command(ScrapyCommand): # a file with the same name exists in the target directory spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) - spiders_dir = dirname(spiders_module.__file__) - spiders_dir_abs = abspath(spiders_dir) - if exists(join(spiders_dir_abs, name + ".py")): - print(f"{join(spiders_dir_abs, (name + '.py'))} already exists") + spiders_dir = Path(spiders_module.__file__).parent + spiders_dir_abs = spiders_dir.resolve() + path = spiders_dir_abs / (name + ".py") + if path.exists(): + print(f"{path} already exists") return True return False @property - def templates_dir(self): - return join( - self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'), + def templates_dir(self) -> str: + return str(Path( + self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'), 'spiders' - ) + )) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index b957c29fb..c41135508 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,22 +1,23 @@ import sys -import os +from os import PathLike +from pathlib import Path from importlib import import_module +from types import ModuleType from scrapy.utils.spider import iter_spider_classes from scrapy.exceptions import UsageError from scrapy.commands import BaseRunSpiderCommand -def _import_file(filepath): - abspath = os.path.abspath(filepath) - dirname, file = os.path.split(abspath) - fname, fext = os.path.splitext(file) - if fext not in ('.py', '.pyw'): +def _import_file(filepath: str | PathLike[str]) -> ModuleType: + abspath = Path(filepath).resolve() + dirname = str(abspath.parent) + if abspath.suffix not in ('.py', '.pyw'): raise ValueError(f"Not a Python source file: {abspath}") if dirname: sys.path = [dirname] + sys.path try: - module = import_module(fname) + module = import_module(abspath.stem) finally: if dirname: sys.path.pop(0) @@ -40,13 +41,13 @@ class Command(BaseRunSpiderCommand): def run(self, args, opts): if len(args) != 1: raise UsageError() - filename = args[0] - if not os.path.exists(filename): + filename = Path(args[0]) + if not filename.exists(): raise UsageError(f"File not found: {filename}\n") try: module = _import_file(filename) except (ImportError, ValueError) as e: - raise UsageError(f"Unable to load {filename!r}: {e}\n") + raise UsageError(f"Unable to load {str(filename)!r}: {e}\n") spclasses = list(iter_spider_classes(module)) if not spclasses: raise UsageError(f"No spider found in file: {filename}\n") diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 1b6374c39..4323cdb53 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -2,7 +2,7 @@ import re import os import string from importlib.util import find_spec -from os.path import join, exists, abspath +from pathlib import Path from shutil import ignore_patterns, move, copy2, copystat from stat import S_IWUSR as OWNER_WRITE_PERMISSION @@ -54,7 +54,7 @@ class Command(ScrapyCommand): return True return False - def _copytree(self, src, dst): + def _copytree(self, src: Path, dst: Path): """ Since the original function always creates the directory, to resolve the issue a new function had to be created. It's a simple copy and @@ -64,19 +64,19 @@ class Command(ScrapyCommand): https://github.com/scrapy/scrapy/pull/2005 """ ignore = IGNORE - names = os.listdir(src) + names = [x.name for x in src.iterdir()] ignored_names = ignore(src, names) - if not os.path.exists(dst): - os.makedirs(dst) + if not dst.exists(): + dst.mkdir(parents=True) for name in names: if name in ignored_names: continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - if os.path.isdir(srcname): + srcname = src / name + dstname = dst / name + if srcname.is_dir(): self._copytree(srcname, dstname) else: copy2(srcname, dstname) @@ -90,36 +90,36 @@ class Command(ScrapyCommand): raise UsageError() project_name = args[0] - project_dir = args[0] if len(args) == 2: - project_dir = args[1] + project_dir = Path(args[1]) + else: + project_dir = Path(args[0]) - if exists(join(project_dir, 'scrapy.cfg')): + if (project_dir / 'scrapy.cfg').exists(): self.exitcode = 1 - print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}') + print(f'Error: scrapy.cfg already exists in {project_dir.resolve()}') return if not self._is_valid_name(project_name): self.exitcode = 1 return - self._copytree(self.templates_dir, abspath(project_dir)) - move(join(project_dir, 'module'), join(project_dir, project_name)) + self._copytree(Path(self.templates_dir), project_dir.resolve()) + move(project_dir / 'module', project_dir / project_name) for paths in TEMPLATES_TO_RENDER: - path = join(*paths) - tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name)) - render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) + tplfile = Path(project_dir, *(string.Template(s).substitute(project_name=project_name) for s in paths)) + render_templatefile(str(tplfile), project_name=project_name, ProjectName=string_camelcase(project_name)) print(f"New Scrapy project '{project_name}', using template directory " f"'{self.templates_dir}', created in:") - print(f" {abspath(project_dir)}\n") + print(f" {project_dir.resolve()}\n") print("You can start your first spider with:") print(f" cd {project_dir}") print(" scrapy genspider example example.com") @property - def templates_dir(self): - return join( - self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'), + def templates_dir(self) -> str: + return str(Path( + self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'), 'project' - ) + )) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 0d94e3df0..4824167da 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -1,3 +1,5 @@ +from pathlib import Path + from w3lib.url import file_uri_to_path from scrapy.responsetypes import responsetypes @@ -10,7 +12,6 @@ class FileDownloadHandler: @defers def download_request(self, request, spider): filepath = file_uri_to_path(request.url) - with open(filepath, 'rb') as fo: - body = fo.read() + body = Path(filepath).read_bytes() respcls = responsetypes.from_args(filename=filepath, body=body) return respcls(url=request.url, body=body) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 5ba0fb63b..366449f51 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,8 +1,7 @@ import json import logging -import os from abc import abstractmethod -from os.path import exists, join +from pathlib import Path from typing import Optional, Type, TypeVar from twisted.internet.defer import Deferred @@ -324,19 +323,19 @@ class Scheduler(BaseScheduler): def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: """ Return a folder name to keep disk queue state at """ if jobdir is not None: - dqdir = join(jobdir, 'requests.queue') - if not exists(dqdir): - os.makedirs(dqdir) - return dqdir + dqdir = Path(jobdir, 'requests.queue') + if not dqdir.exists(): + dqdir.mkdir(parents=True) + return str(dqdir) return None def _read_dqs_state(self, dqdir: str) -> list: - path = join(dqdir, 'active.json') - if not exists(path): + path = Path(dqdir, 'active.json') + if not path.exists(): return [] - with open(path) as f: + with path.open() as f: return json.load(f) def _write_dqs_state(self, dqdir: str, state: list) -> None: - with open(join(dqdir, 'active.json'), 'w') as f: + with Path(dqdir, 'active.json').open('w') as f: json.dump(state, f) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e768bca12..b7108cdcc 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,7 +1,10 @@ +from __future__ import annotations + import logging import pprint import signal import warnings +from typing import TYPE_CHECKING from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement @@ -33,6 +36,9 @@ from scrapy.utils.misc import create_instance, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.reactor import install_reactor, verify_installed_reactor +if TYPE_CHECKING: + from scrapy.utils.request import RequestFingerprinter + logger = logging.getLogger(__name__) @@ -72,7 +78,7 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) - self.request_fingerprinter = create_instance( + self.request_fingerprinter: RequestFingerprinter = create_instance( load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']), settings=self.settings, crawler=self, diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index d1b0559ef..2b8b09614 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,5 +1,5 @@ import logging -import os +from pathlib import Path from typing import Optional, Set, Type, TypeVar from warnings import warn @@ -55,7 +55,7 @@ class RFPDupeFilter(BaseDupeFilter): self.debug = debug self.logger = logging.getLogger(__name__) if path: - self.file = open(os.path.join(path, 'requests.seen'), 'a+') + self.file = Path(path, 'requests.seen').open('a+') self.file.seek(0) self.fingerprints.update(x.rstrip() for x in self.file) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e7097b7a1..0aa27e417 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -5,13 +5,13 @@ See documentation in docs/topics/feed-exports.rst """ import logging -import os import re import sys import warnings from datetime import datetime +from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any, Callable, Optional, Tuple, Union +from typing import IO, Any, Callable, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads @@ -101,7 +101,7 @@ class BlockingFeedStorage: def open(self, spider): path = spider.crawler.settings['FEED_TEMPDIR'] - if path and not os.path.isdir(path): + if path and not Path(path).is_dir(): raise OSError('Not a Directory: ' + str(path)) return NamedTemporaryFile(prefix='feed-', dir=path) @@ -141,11 +141,11 @@ class FileFeedStorage: feed_options = feed_options or {} self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab' - def open(self, spider): - dirname = os.path.dirname(self.path) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname) - return open(self.path, self.write_mode) + def open(self, spider) -> IO[Any]: + dirname = Path(self.path).parent + if dirname and not dirname.exists(): + dirname.mkdir(parents=True) + return Path(self.path).open(self.write_mode) def store(self, file): file.close() diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 843e14812..3057bf157 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,16 +1,18 @@ import gzip import logging -import os import pickle from email.utils import mktime_tz, parsedate_tz from importlib import import_module +from pathlib import Path from time import time from weakref import WeakKeyDictionary from w3lib.http import headers_raw_to_dict, headers_dict_to_raw from scrapy.http import Headers, Response +from scrapy.http.request import Request from scrapy.responsetypes import responsetypes +from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode @@ -221,9 +223,9 @@ class DbmCacheStorage: self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.db = None - def open_spider(self, spider): - dbpath = os.path.join(self.cachedir, f'{spider.name}.db') - self.db = self.dbmodule.open(dbpath, 'c') + def open_spider(self, spider: Spider): + dbpath = Path(self.cachedir, f'{spider.name}.db') + self.db = self.dbmodule.open(str(dbpath), 'c') logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider}) @@ -277,7 +279,7 @@ class FilesystemCacheStorage: self.use_gzip = settings.getbool('HTTPCACHE_GZIP') self._open = gzip.open if self.use_gzip else open - def open_spider(self, spider): + def open_spider(self, spider: Spider): logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir}, extra={'spider': spider}) @@ -286,15 +288,15 @@ class FilesystemCacheStorage: def close_spider(self, spider): pass - def retrieve_response(self, spider, request): + def retrieve_response(self, spider: Spider, request: Request): """Return response if present in cache, or None otherwise.""" metadata = self._read_meta(spider, request) if metadata is None: return # not cached rpath = self._get_request_path(spider, request) - with self._open(os.path.join(rpath, 'response_body'), 'rb') as f: + with self._open(rpath / 'response_body', 'rb') as f: body = f.read() - with self._open(os.path.join(rpath, 'response_headers'), 'rb') as f: + with self._open(rpath / 'response_headers', 'rb') as f: rawheaders = f.read() url = metadata.get('response_url') status = metadata['status'] @@ -303,11 +305,11 @@ class FilesystemCacheStorage: response = respcls(url=url, headers=headers, status=status, body=body) return response - def store_response(self, spider, request, response): + def store_response(self, spider: Spider, request: Request, response): """Store the given response in the cache.""" rpath = self._get_request_path(spider, request) - if not os.path.exists(rpath): - os.makedirs(rpath) + if not rpath.exists(): + rpath.mkdir(parents=True) metadata = { 'url': request.url, 'method': request.method, @@ -315,29 +317,29 @@ class FilesystemCacheStorage: 'response_url': response.url, 'timestamp': time(), } - with self._open(os.path.join(rpath, 'meta'), 'wb') as f: + with self._open(rpath / 'meta', 'wb') as f: f.write(to_bytes(repr(metadata))) - with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f: + with self._open(rpath / 'pickled_meta', 'wb') as f: pickle.dump(metadata, f, protocol=4) - with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f: + with self._open(rpath / 'response_headers', 'wb') as f: f.write(headers_dict_to_raw(response.headers)) - with self._open(os.path.join(rpath, 'response_body'), 'wb') as f: + with self._open(rpath / 'response_body', 'wb') as f: f.write(response.body) - with self._open(os.path.join(rpath, 'request_headers'), 'wb') as f: + with self._open(rpath / 'request_headers', 'wb') as f: f.write(headers_dict_to_raw(request.headers)) - with self._open(os.path.join(rpath, 'request_body'), 'wb') as f: + with self._open(rpath / 'request_body', 'wb') as f: f.write(request.body) - def _get_request_path(self, spider, request): + def _get_request_path(self, spider: Spider, request: Request) -> Path: key = self._fingerprinter.fingerprint(request).hex() - return os.path.join(self.cachedir, spider.name, key[0:2], key) + return Path(self.cachedir, spider.name, key[0:2], key) - def _read_meta(self, spider, request): + def _read_meta(self, spider: Spider, request: Request): rpath = self._get_request_path(spider, request) - metapath = os.path.join(rpath, 'pickled_meta') - if not os.path.exists(metapath): + metapath = rpath / 'pickled_meta' + if not metapath.exists(): return # not found - mtime = os.stat(metapath).st_mtime + mtime = metapath.stat().st_mtime if 0 < self.expiration_secs < time() - mtime: return # expired with self._open(metapath, 'rb') as f: diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index bea00596e..e9c8b1d6a 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,5 +1,5 @@ -import os import pickle +from pathlib import Path from scrapy import signals from scrapy.exceptions import NotConfigured @@ -25,16 +25,16 @@ class SpiderState: def spider_closed(self, spider): if self.jobdir: - with open(self.statefn, 'wb') as f: + with Path(self.statefn).open('wb') as f: pickle.dump(spider.state, f, protocol=4) def spider_opened(self, spider): - if self.jobdir and os.path.exists(self.statefn): - with open(self.statefn, 'rb') as f: + if self.jobdir and Path(self.statefn).exists(): + with Path(self.statefn).open('rb') as f: spider.state = pickle.load(f) else: spider.state = {} @property - def statefn(self): - return os.path.join(self.jobdir, 'spider.state') + def statefn(self) -> str: + return str(Path(self.jobdir, 'spider.state')) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 4de6c9b5b..9359cc7c8 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -32,7 +32,7 @@ class Response(object_ref): def __init__( self, - url, + url: str, status=200, headers=None, body=b"", @@ -75,7 +75,7 @@ class Response(object_ref): def _get_url(self): return self._url - def _set_url(self, url): + def _set_url(self, url: str): if isinstance(url, str): self._url = url else: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 906e7eb24..ffb12d910 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -13,6 +13,8 @@ from collections import defaultdict from contextlib import suppress from ftplib import FTP from io import BytesIO +from pathlib import Path +from typing import Optional from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -39,41 +41,40 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir): + def __init__(self, basedir: str): if '://' in basedir: basedir = basedir.split('://', 1)[1] self.basedir = basedir - self._mkdir(self.basedir) - self.created_directories = defaultdict(set) + self._mkdir(Path(self.basedir)) + self.created_directories: defaultdict[str, set[str]] = defaultdict(set) - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file(self, path: str, buf, info, meta=None, headers=None): absolute_path = self._get_filesystem_path(path) - self._mkdir(os.path.dirname(absolute_path), info) - with open(absolute_path, 'wb') as f: - f.write(buf.getvalue()) + self._mkdir(absolute_path.parent, info) + absolute_path.write_bytes(buf.getvalue()) - def stat_file(self, path, info): + def stat_file(self, path: str, info): absolute_path = self._get_filesystem_path(path) try: - last_modified = os.path.getmtime(absolute_path) + last_modified = absolute_path.stat().st_mtime except os.error: return {} - with open(absolute_path, 'rb') as f: + with absolute_path.open('rb') as f: checksum = md5sum(f) return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path): + def _get_filesystem_path(self, path: str) -> Path: path_comps = path.split('/') - return os.path.join(self.basedir, *path_comps) + return Path(self.basedir, *path_comps) - def _mkdir(self, dirname, domain=None): + def _mkdir(self, dirname: Path, domain: Optional[str] = None): seen = self.created_directories[domain] if domain else set() - if dirname not in seen: - if not os.path.exists(dirname): - os.makedirs(dirname) - seen.add(dirname) + if str(dirname) not in seen: + if not dirname.exists(): + dirname.mkdir(parents=True) + seen.add(str(dirname)) class S3FilesStore: @@ -374,8 +375,8 @@ class FilesPipeline(MediaPipeline): store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) - def _get_store(self, uri): - if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir + def _get_store(self, uri: str): + if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir scheme = 'file' else: scheme = urlparse(uri).scheme @@ -510,7 +511,7 @@ class FilesPipeline(MediaPipeline): def file_path(self, request, response=None, info=None, *, item=None): media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - media_ext = os.path.splitext(request.url)[1] + media_ext = Path(request.url).suffix # Handles empty and wild extensions by trying to guess the # mime type then extension or default to empty string otherwise if media_ext not in mimetypes.types_map: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ff86af125..84e0a94d2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -15,7 +15,7 @@ Scrapy developers, if you add a setting here remember to: import sys from importlib import import_module -from os.path import join, abspath, dirname +from pathlib import Path AJAXCRAWL_ENABLED = False @@ -288,7 +288,7 @@ STATS_DUMP = True STATSMAILER_RCPTS = [] -TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) +TEMPLATES_DIR = str((Path(__file__).parent / '..' / 'templates').resolve()) URLLENGTH_LIMIT = 2083 diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index d8248c606..9a97e7801 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -3,14 +3,19 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ +from __future__ import annotations + import logging -from typing import Optional +from typing import TYPE_CHECKING, Optional from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider +if TYPE_CHECKING: + from scrapy.crawler import Crawler + class Spider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this @@ -49,7 +54,7 @@ class Spider(object_ref): spider._set_crawler(crawler) return spider - def _set_crawler(self, crawler): + def _set_crawler(self, crawler: Crawler): self.crawler = crawler self.settings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index dff9b1350..1f2dee55f 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -3,8 +3,9 @@ Scheduler queues """ import marshal -import os import pickle +from os import PathLike +from pathlib import Path from queuelib import queue @@ -16,10 +17,10 @@ def _with_mkdir(queue_class): class DirectoriesCreated(queue_class): - def __init__(self, path, *args, **kwargs): - dirname = os.path.dirname(path) - if not os.path.exists(dirname): - os.makedirs(dirname, exist_ok=True) + def __init__(self, path: str | PathLike[str], *args, **kwargs): + dirname = Path(path).parent + if not dirname.exists(): + dirname.mkdir(parents=True, exist_ok=True) super().__init__(path, *args, **kwargs) return DirectoriesCreated diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 00cc53725..e247f5999 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -4,6 +4,8 @@ import sys import warnings from configparser import ConfigParser from operator import itemgetter +from pathlib import Path +from typing import Optional from scrapy.exceptions import ScrapyDeprecationWarning, UsageError @@ -65,17 +67,17 @@ def arglist_to_dict(arglist): return dict(x.split('=', 1) for x in arglist) -def closest_scrapy_cfg(path='.', prevpath=None): +def closest_scrapy_cfg(path: str | os.PathLike[str] = '.', prevpath: Optional[str | os.PathLike] = None) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents """ - if path == prevpath: + if prevpath is not None and str(path) == str(prevpath): return '' - path = os.path.abspath(path) - cfgfile = os.path.join(path, 'scrapy.cfg') - if os.path.exists(cfgfile): - return cfgfile - return closest_scrapy_cfg(os.path.dirname(path), path) + path = Path(path).resolve() + cfgfile = path / 'scrapy.cfg' + if cfgfile.exists(): + return str(cfgfile) + return closest_scrapy_cfg(path.parent, path) def init_env(project='default', set_syspath=True): @@ -88,7 +90,7 @@ def init_env(project='default', set_syspath=True): os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project) closest = closest_scrapy_cfg() if closest: - projdir = os.path.dirname(closest) + projdir = str(Path(closest).parent) if set_syspath and projdir not in sys.path: sys.path.append(projdir) @@ -101,13 +103,13 @@ def get_config(use_closest=True): return cfg -def get_sources(use_closest=True): - xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config') +def get_sources(use_closest=True) -> list[str]: + xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or Path('~/.config').expanduser() sources = [ '/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg', - xdg_config_home + '/scrapy.cfg', - os.path.expanduser('~/.scrapy.cfg'), + str(Path(xdg_config_home) / 'scrapy.cfg'), + str(Path('~/.scrapy.cfg').expanduser()), ] if use_closest: sources.append(closest_scrapy_cfg()) @@ -129,8 +131,8 @@ def feed_complete_default_values_from_settings(feed, settings): return out -def feed_process_params_from_cli(settings, output, output_format=None, - overwrite_output=None): +def feed_process_params_from_cli(settings, output: list[str], output_format=None, + overwrite_output: Optional[list[str]] = None): """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary @@ -180,7 +182,7 @@ def feed_process_params_from_cli(settings, output, output_format=None, feed_uri, feed_format = element.rsplit(':', 1) except ValueError: feed_uri = element - feed_format = os.path.splitext(element)[1].replace('.', '') + feed_format = Path(element).suffix.replace('.', '') else: if feed_uri == '-': feed_uri = 'stdout:' diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index c92ef36f5..a65f92e95 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path from typing import Optional from scrapy.settings import BaseSettings @@ -6,6 +6,6 @@ from scrapy.settings import BaseSettings def job_dir(settings: BaseSettings) -> Optional[str]: path = settings['JOBDIR'] - if path and not os.path.exists(path): - os.makedirs(path) + if path and not Path(path).exists(): + Path(path).mkdir(parents=True) return path diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index c66af497e..e54b71d45 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -2,7 +2,7 @@ import os import warnings from importlib import import_module -from os.path import join, dirname, abspath, isabs, exists +from pathlib import Path from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env from scrapy.settings import Settings @@ -25,36 +25,37 @@ def inside_project(): return bool(closest_scrapy_cfg()) -def project_data_dir(project='default'): +def project_data_dir(project='default') -> str: """Return the current project data dir, creating it if it doesn't exist""" if not inside_project(): raise NotConfigured("Not inside a project") cfg = get_config() if cfg.has_option(DATADIR_CFG_SECTION, project): - d = cfg.get(DATADIR_CFG_SECTION, project) + d = Path(cfg.get(DATADIR_CFG_SECTION, project)) else: scrapy_cfg = closest_scrapy_cfg() if not scrapy_cfg: raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir") - d = abspath(join(dirname(scrapy_cfg), '.scrapy')) - if not exists(d): - os.makedirs(d) - return d + d = (Path(scrapy_cfg).parent / '.scrapy').resolve() + if not d.exists(): + d.mkdir(parents=True) + return str(d) -def data_path(path, createdir=False): +def data_path(path: str, createdir=False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. """ - if not isabs(path): + path_obj = Path(path) + if not path_obj.is_absolute(): if inside_project(): - path = join(project_data_dir(), path) + path_obj = Path(project_data_dir(), path) else: - path = join('.scrapy', path) - if createdir and not exists(path): - os.makedirs(path) - return path + path_obj = Path('.scrapy', path) + if createdir and not path_obj.exists(): + path_obj.mkdir(parents=True) + return str(path_obj) def get_project_settings(): diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index cf33317ce..545d489be 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -264,7 +264,7 @@ class RequestFingerprinter: f'and \'VERSION\'.' ) - def fingerprint(self, request): + def fingerprint(self, request: Request): return self._fingerprint(request) diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index f068be737..8075902b3 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,23 +1,21 @@ """Helper functions for working with templates""" -import os import re import string +from pathlib import Path -def render_templatefile(path, **kwargs): - with open(path, 'rb') as fp: - raw = fp.read().decode('utf8') +def render_templatefile(path: str, **kwargs): + raw = Path(path).read_text('utf8') content = string.Template(raw).substitute(**kwargs) render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path if path.endswith('.tmpl'): - os.rename(path, render_path) + Path(path).rename(render_path) - with open(render_path, 'wb') as fp: - fp.write(content.encode('utf8')) + Path(render_path).write_text(content, 'utf8') CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 0b828f7c0..4d01f1ef1 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -4,6 +4,7 @@ This module contains some assorted functions used in tests import asyncio import os +from pathlib import Path from posixpath import split from unittest import mock @@ -71,11 +72,11 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): return runner.create_crawler(spidercls or Spider) -def get_pythonpath(): +def get_pythonpath() -> str: """Return a PYTHONPATH suitable to use in processes so that they find this installation of Scrapy""" scrapy_path = import_module('scrapy').__path__[0] - return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '') + return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get('PYTHONPATH', '') def get_testenv(): diff --git a/setup.py b/setup.py index a43cf08c8..e413ea6e4 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,9 @@ -from os.path import dirname, join +from pathlib import Path from pkg_resources import parse_version from setuptools import setup, find_packages, __version__ as setuptools_version -with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: - version = f.read().decode('ascii').strip() +version = (Path(__file__).parent / 'scrapy/VERSION').read_text('ascii').strip() def has_environment_marker_platform_impl_support(): diff --git a/tests/__init__.py b/tests/__init__.py index bb62851dc..be263fa16 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,6 +6,7 @@ see https://docs.scrapy.org/en/latest/contributing.html#running-tests import os import socket +from pathlib import Path # ignore system-wide proxies for tests # which would send requests to a totally unsuspecting server @@ -16,14 +17,12 @@ os.environ['ftp_proxy'] = '' # Absolutize paths to coverage config and output file because tests that # spawn subprocesses also changes current working directory. -_sourceroot = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_sourceroot = Path(__file__).resolve().parent.parent if 'COV_CORE_CONFIG' in os.environ: - os.environ['COVERAGE_FILE'] = os.path.join(_sourceroot, '.coverage') - os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot, - os.environ['COV_CORE_CONFIG']) + os.environ['COVERAGE_FILE'] = str(_sourceroot / '.coverage') + os.environ['COV_CORE_CONFIG'] = str(_sourceroot / os.environ['COV_CORE_CONFIG']) -tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'sample_data') +tests_datadir = str(Path(__file__).parent.resolve() / 'sample_data') # In some environments accessing a non-existing host doesn't raise an @@ -35,8 +34,6 @@ except socket.gaierror: NON_EXISTING_RESOLVABLE = False -def get_testdata(*paths): +def get_testdata(*paths: str) -> bytes: """Return test data""" - path = os.path.join(tests_datadir, *paths) - with open(path, 'rb') as f: - return f.read() + return Path(tests_datadir, *paths).read_bytes() diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index bb4a8e5af..3a41b3a3e 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -1,5 +1,5 @@ -import os from datetime import datetime, timedelta +from pathlib import Path from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa @@ -22,21 +22,20 @@ from cryptography.x509.oid import NameOID # https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate def generate_keys(): - folder = os.path.dirname(__file__) + folder = Path(__file__).parent key = rsa.generate_private_key( public_exponent=65537, key_size=2048, backend=default_backend(), ) - with open(os.path.join(folder, 'localhost.key'), "wb") as f: - f.write( - key.private_bytes( - encoding=Encoding.PEM, - format=PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=NoEncryption(), - ) - ) + (folder / 'localhost.key').write_bytes( + key.private_bytes( + encoding=Encoding.PEM, + format=PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=NoEncryption(), + ), + ) subject = issuer = Name( [ @@ -59,5 +58,4 @@ def generate_keys(): ) .sign(key, SHA256(), default_backend()) ) - with open(os.path.join(folder, 'localhost.crt'), "wb") as f: - f.write(cert.public_bytes(Encoding.PEM)) + (folder / 'localhost.crt').write_bytes(cert.public_bytes(Encoding.PEM)) diff --git a/tests/mockserver.py b/tests/mockserver.py index 72d7e0241..7916798f7 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,6 +1,5 @@ import argparse import json -import os import random import sys from pathlib import Path @@ -253,7 +252,7 @@ class Root(resource.Resource): self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) try: from tests import tests_datadir - self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/'))) + self.putChild(b"files", File(str(Path(tests_datadir, 'test_site/files/')))) except Exception: pass self.putChild(b"redirect-to", RedirectTo()) @@ -346,8 +345,8 @@ class MockFTPServer: def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( - os.path.join(os.path.dirname(__file__), keyfile), - os.path.join(os.path.dirname(__file__), certfile), + str(Path(__file__).parent / keyfile), + str(Path(__file__).parent / certfile), ) if cipher_string: ctx = factory.getContext() diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 8233e0101..da73a4c45 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,11 +1,11 @@ import json -import os import pstats import shutil import sys import tempfile import unittest from io import StringIO +from pathlib import Path from subprocess import Popen, PIPE from scrapy.utils.test import get_testenv @@ -36,17 +36,17 @@ class CmdlineTest(unittest.TestCase): self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override') def test_profiling(self): - path = tempfile.mkdtemp() - filename = os.path.join(path, 'res.prof') + path = Path(tempfile.mkdtemp()) + filename = path / 'res.prof' try: - self._execute('version', '--profile', filename) - self.assertTrue(os.path.exists(filename)) + self._execute('version', '--profile', str(filename)) + self.assertTrue(filename.exists()) out = StringIO() - stats = pstats.Stats(filename, stream=out) + stats = pstats.Stats(str(filename), stream=out) stats.print_stats() out.seek(0) stats = out.read() - self.assertIn(os.path.join('scrapy', 'commands', 'version.py'), + self.assertIn(str(Path('scrapy', 'commands', 'version.py')), stats) self.assertIn('tottime', stats) finally: diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index d341888d3..fcafcef68 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -1,6 +1,6 @@ -import os import sys import unittest +from pathlib import Path from subprocess import Popen, PIPE @@ -8,7 +8,7 @@ class CmdlineCrawlPipelineTest(unittest.TestCase): def _execute(self, spname): args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname) - cwd = os.path.dirname(os.path.abspath(__file__)) + cwd = Path(__file__).resolve().parent proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd) proc.communicate() return proc.returncode diff --git a/tests/test_command_check.py b/tests/test_command_check.py index c3d705194..4077a9bce 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -1,5 +1,3 @@ -from os.path import join, abspath - from tests.test_commands import CommandTest @@ -10,11 +8,10 @@ class CheckCommandTest(CommandTest): def setUp(self): super(CheckCommandTest, self).setUp() self.spider_name = 'check_spider' - self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py')) + self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve() def _write_contract(self, contracts, parse_def): - with open(self.spider, 'w') as file: - file.write(f""" + self.spider.write_text(f""" import scrapy class CheckSpider(scrapy.Spider): @@ -27,7 +24,7 @@ class CheckSpider(scrapy.Spider): {contracts} \"\"\" {parse_def} - """) + """) def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 0d992be56..154287d74 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,6 +1,6 @@ import os import argparse -from os.path import join, abspath, isfile, exists +from pathlib import Path from twisted.internet import defer from scrapy.commands import parse @@ -23,9 +23,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): def setUp(self): super().setUp() self.spider_name = 'parse_spider' - fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) - with open(fname, 'w') as f: - f.write(f""" + (self.proj_mod_path / 'spiders' / 'myspider.py').write_text(f""" import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule @@ -88,9 +86,7 @@ class MyBadCrawlSpider(CrawlSpider): return [scrapy.Item(), dict(foo='bar')] """) - fname = abspath(join(self.proj_mod_path, 'pipelines.py')) - with open(fname, 'w') as f: - f.write(""" + (self.proj_mod_path / 'pipelines.py').write_text(""" import logging class MyPipeline: @@ -101,8 +97,7 @@ class MyPipeline: return item """) - fname = abspath(join(self.proj_mod_path, 'settings.py')) - with open(fname, 'a') as f: + with (self.proj_mod_path / 'settings.py').open("a") as f: f.write(f""" ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """) @@ -234,7 +229,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} correct format containing correct data in it. """ file_name = 'data.json' - file_path = join(self.proj_path, file_name) + file_path = Path(self.proj_path, file_name) yield self.execute([ '--spider', self.spider_name, '-c', 'parse', @@ -242,12 +237,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url('/html') ]) - self.assertTrue(exists(file_path)) - self.assertTrue(isfile(file_path)) + self.assertTrue(file_path.exists()) + self.assertTrue(file_path.is_file()) content = '[\n{},\n{"foo": "bar"}\n]' - with open(file_path, 'r') as f: - self.assertEqual(f.read(), content) + self.assertEqual(file_path.read_text(), content) def test_parse_add_options(self): command = parse.Command() diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 33189e9be..33c98ad69 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,4 +1,4 @@ -from os.path import join +from pathlib import Path from twisted.trial import unittest from twisted.internet import defer @@ -96,8 +96,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_local_file(self): - filepath = join(tests_datadir, 'test_site', 'index.html') - _, out, _ = yield self.execute([filepath, '-c', 'item']) + filepath = Path(tests_datadir, 'test_site', 'index.html') + _, out, _ = yield self.execute([str(filepath), '-c', 'item']) assert b'{}' in out @defer.inlineCallbacks diff --git a/tests/test_commands.py b/tests/test_commands.py index 76d5f3935..39f718cce 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -9,12 +9,12 @@ import sys import tempfile from contextlib import contextmanager from itertools import chain -from os.path import exists, join, abspath, getmtime from pathlib import Path from shutil import rmtree, copytree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import mkdtemp from threading import Timer +from typing import Generator, Optional from unittest import skipIf from pytest import mark @@ -66,8 +66,8 @@ class ProjectTest(unittest.TestCase): def setUp(self): self.temp_path = mkdtemp() self.cwd = self.temp_path - self.proj_path = join(self.temp_path, self.project_name) - self.proj_mod_path = join(self.proj_path, self.project_name) + self.proj_path = Path(self.temp_path, self.project_name) + self.proj_mod_path = self.proj_path / self.project_name self.env = get_testenv() def tearDown(self): @@ -104,10 +104,10 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) - def find_in_file(self, filename, regex): + def find_in_file(self, filename: str | os.PathLike[str], regex) -> Optional[re.Match]: """Find first pattern occurrence in file""" pattern = re.compile(regex) - with open(filename, "r") as f: + with Path(filename).open("r") as f: for line in f: match = pattern.search(line) if match is not None: @@ -122,13 +122,13 @@ class StartprojectTest(ProjectTest): print(err, file=sys.stderr) self.assertEqual(p.returncode, 0) - assert exists(join(self.proj_path, 'scrapy.cfg')) - assert exists(join(self.proj_path, 'testproject')) - assert exists(join(self.proj_mod_path, '__init__.py')) - assert exists(join(self.proj_mod_path, 'items.py')) - assert exists(join(self.proj_mod_path, 'pipelines.py')) - assert exists(join(self.proj_mod_path, 'settings.py')) - assert exists(join(self.proj_mod_path, 'spiders', '__init__.py')) + assert Path(self.proj_path, 'scrapy.cfg').exists() + assert Path(self.proj_path, 'testproject').exists() + assert Path(self.proj_mod_path, '__init__.py').exists() + assert Path(self.proj_mod_path, 'items.py').exists() + assert Path(self.proj_mod_path, 'pipelines.py').exists() + assert Path(self.proj_mod_path, 'settings.py').exists() + assert Path(self.proj_mod_path, 'spiders', '__init__.py').exists() self.assertEqual(1, self.call('startproject', self.project_name)) self.assertEqual(1, self.call('startproject', 'wrong---project---name')) @@ -138,13 +138,13 @@ class StartprojectTest(ProjectTest): project_dir = mkdtemp() self.assertEqual(0, self.call('startproject', self.project_name, project_dir)) - assert exists(join(abspath(project_dir), 'scrapy.cfg')) - assert exists(join(abspath(project_dir), 'testproject')) - assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'items.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py')) + assert Path(project_dir, 'scrapy.cfg').exists() + assert Path(project_dir, 'testproject').exists() + assert Path(project_dir, self.project_name, '__init__.py').exists() + assert Path(project_dir, self.project_name, 'items.py').exists() + assert Path(project_dir, self.project_name, 'pipelines.py').exists() + assert Path(project_dir, self.project_name, 'settings.py').exists() + assert Path(project_dir, self.project_name, 'spiders', '__init__.py').exists() self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2')) @@ -158,40 +158,42 @@ class StartprojectTest(ProjectTest): def test_existing_project_dir(self): project_dir = mkdtemp() project_name = self.project_name + '_existing' - project_path = os.path.join(project_dir, project_name) - os.mkdir(project_path) + project_path = Path(project_dir, project_name) + project_path.mkdir() p, out, err = self.proc('startproject', project_name, cwd=project_dir) print(out) print(err, file=sys.stderr) self.assertEqual(p.returncode, 0) - assert exists(join(abspath(project_path), 'scrapy.cfg')) - assert exists(join(abspath(project_path), project_name)) - assert exists(join(join(abspath(project_path), project_name), '__init__.py')) - assert exists(join(join(abspath(project_path), project_name), 'items.py')) - assert exists(join(join(abspath(project_path), project_name), 'pipelines.py')) - assert exists(join(join(abspath(project_path), project_name), 'settings.py')) - assert exists(join(join(abspath(project_path), project_name), 'spiders', '__init__.py')) + assert Path(project_path, 'scrapy.cfg').exists() + assert Path(project_path, project_name).exists() + assert Path(project_path, project_name, '__init__.py').exists() + assert Path(project_path, project_name, 'items.py').exists() + assert Path(project_path, project_name, 'pipelines.py').exists() + assert Path(project_path, project_name, 'settings.py').exists() + assert Path(project_path, project_name, 'spiders', '__init__.py').exists() -def get_permissions_dict(path, renamings=None, ignore=None): +def get_permissions_dict(path: str | os.PathLike[str], renamings=None, ignore=None) -> dict[str, str]: - def get_permissions(path): - return oct(os.stat(path).st_mode) + def get_permissions(path: Path) -> str: + return oct(path.stat().st_mode) + + path_obj = Path(path) renamings = renamings or tuple() permissions_dict = { - '.': get_permissions(path), + '.': get_permissions(path_obj), } - for root, dirs, files in os.walk(path): + for root, dirs, files in os.walk(path_obj): nodes = list(chain(dirs, files)) if ignore: ignored_names = ignore(root, nodes) nodes = [node for node in nodes if node not in ignored_names] for node in nodes: - absolute_path = os.path.join(root, node) - relative_path = os.path.relpath(absolute_path, path) + absolute_path = Path(root, node) + relative_path = str(absolute_path.relative_to(path)) for search_string, replacement in renamings: relative_path = relative_path.replace( search_string, @@ -208,28 +210,27 @@ class StartprojectTemplatesTest(ProjectTest): def setUp(self): super().setUp() - self.tmpl = join(self.temp_path, 'templates') - self.tmpl_proj = join(self.tmpl, 'project') + self.tmpl = str(Path(self.temp_path, 'templates')) + self.tmpl_proj = str(Path(self.tmpl, 'project')) def test_startproject_template_override(self): - copytree(join(scrapy.__path__[0], 'templates'), self.tmpl) - with open(join(self.tmpl_proj, 'root_template'), 'w'): - pass - assert exists(join(self.tmpl_proj, 'root_template')) + copytree(Path(scrapy.__path__[0], 'templates'), self.tmpl) + Path(self.tmpl_proj, 'root_template').write_bytes(b"") + assert Path(self.tmpl_proj, 'root_template').exists() args = ['--set', f'TEMPLATES_DIR={self.tmpl}'] p, out, err = self.proc('startproject', self.project_name, *args) self.assertIn(f"New Scrapy project '{self.project_name}', " "using template directory", out) self.assertIn(self.tmpl_proj, out) - assert exists(join(self.proj_path, 'root_template')) + assert Path(self.proj_path, 'root_template').exists() def test_startproject_permissions_from_writable(self): """Check that generated files have the right permissions when the template folder has the same permissions as in the project, i.e. everything is writable.""" scrapy_path = scrapy.__path__[0] - project_template = os.path.join(scrapy_path, 'templates', 'project') + project_template = Path(scrapy_path, 'templates', 'project') project_name = 'startproject1' renamings = ( ('module', project_name), @@ -255,7 +256,7 @@ class StartprojectTemplatesTest(ProjectTest): ) process.wait() - project_dir = os.path.join(destination, project_name) + project_dir = Path(destination, project_name) actual_permissions = get_permissions_dict(project_dir) self.assertEqual(actual_permissions, expected_permissions) @@ -268,8 +269,8 @@ class StartprojectTemplatesTest(ProjectTest): See https://github.com/scrapy/scrapy/pull/4604 """ scrapy_path = scrapy.__path__[0] - templates_dir = os.path.join(scrapy_path, 'templates') - project_template = os.path.join(templates_dir, 'project') + templates_dir = Path(scrapy_path, 'templates') + project_template = Path(templates_dir, 'project') project_name = 'startproject2' renamings = ( ('module', project_name), @@ -281,16 +282,16 @@ class StartprojectTemplatesTest(ProjectTest): IGNORE, ) - def _make_read_only(path): - current_permissions = os.stat(path).st_mode - os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION) + def _make_read_only(path: Path): + current_permissions = path.stat().st_mode + path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION) read_only_templates_dir = str(Path(mkdtemp()) / 'templates') copytree(templates_dir, read_only_templates_dir) for root, dirs, files in os.walk(read_only_templates_dir): for node in chain(dirs, files): - _make_read_only(os.path.join(root, node)) + _make_read_only(Path(root, node)) destination = mkdtemp() process = subprocess.Popen( @@ -308,7 +309,7 @@ class StartprojectTemplatesTest(ProjectTest): ) process.wait() - project_dir = os.path.join(destination, project_name) + project_dir = Path(destination, project_name) actual_permissions = get_permissions_dict(project_dir) self.assertEqual(actual_permissions, expected_permissions) @@ -317,7 +318,7 @@ class StartprojectTemplatesTest(ProjectTest): """Check that pre-existing folders and files in the destination folder do not see their permissions modified.""" scrapy_path = scrapy.__path__[0] - project_template = os.path.join(scrapy_path, 'templates', 'project') + project_template = Path(scrapy_path, 'templates', 'project') project_name = 'startproject3' renamings = ( ('module', project_name), @@ -330,7 +331,7 @@ class StartprojectTemplatesTest(ProjectTest): ) destination = mkdtemp() - project_dir = os.path.join(destination, project_name) + project_dir = Path(destination, project_name) existing_nodes = { oct(permissions)[2:] + extension: permissions @@ -339,10 +340,9 @@ class StartprojectTemplatesTest(ProjectTest): 0o444, 0o555, 0o644, 0o666, 0o755, 0o777, ) } - os.mkdir(project_dir) - project_dir_path = Path(project_dir) + project_dir.mkdir() for node, permissions in existing_nodes.items(): - path = project_dir_path / node + path = project_dir / node if node.endswith('.d'): path.mkdir(mode=permissions) else: @@ -378,7 +378,7 @@ class StartprojectTemplatesTest(ProjectTest): os.umask(cur_mask) scrapy_path = scrapy.__path__[0] - project_template = os.path.join( + project_template = Path( scrapy_path, 'templates', 'project' @@ -409,7 +409,7 @@ class StartprojectTemplatesTest(ProjectTest): ) process.wait() - project_dir = os.path.join(destination, project_name) + project_dir = Path(destination, project_name) actual_permissions = get_permissions_dict(project_dir) self.assertEqual(actual_permissions, expected_permissions) @@ -420,7 +420,7 @@ class CommandTest(ProjectTest): def setUp(self): super().setUp() self.call('startproject', self.project_name) - self.cwd = join(self.temp_path, self.project_name) + self.cwd = Path(self.temp_path, self.project_name) self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings' @@ -429,10 +429,10 @@ class GenspiderCommandTest(CommandTest): def test_arguments(self): # only pass one argument. spider script shouldn't be created self.assertEqual(2, self.call('genspider', 'test_name')) - assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) + assert not Path(self.proj_mod_path, 'spiders', 'test_name.py').exists() # pass two arguments . spider script should be created self.assertEqual(0, self.call('genspider', 'test_name', 'test.com')) - assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) + assert Path(self.proj_mod_path, 'spiders', 'test_name.py').exists() def test_template(self, tplname='crawl'): args = [f'--template={tplname}'] if tplname else [] @@ -440,11 +440,11 @@ class GenspiderCommandTest(CommandTest): spmodule = f"{self.project_name}.spiders.{spname}" p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out) - self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) - modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) + self.assertTrue(Path(self.proj_mod_path, 'spiders', 'test_spider.py').exists()) + modify_time_before = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn(f"Spider {spname!r} already exists in module", out) - modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) + modify_time_after = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) def test_template_basic(self): @@ -465,37 +465,37 @@ class GenspiderCommandTest(CommandTest): def test_same_name_as_project(self): self.assertEqual(2, self.call('genspider', self.project_name)) - assert not exists(join(self.proj_mod_path, 'spiders', f'{self.project_name}.py')) + assert not Path(self.proj_mod_path, 'spiders', f'{self.project_name}.py').exists() def test_same_filename_as_existing_spider(self, force=False): file_name = 'example' - file_path = join(self.proj_mod_path, 'spiders', f'{file_name}.py') + file_path = Path(self.proj_mod_path, 'spiders', f'{file_name}.py') self.assertEqual(0, self.call('genspider', file_name, 'example.com')) - assert exists(file_path) + assert file_path.exists() # change name of spider but not its file name - with open(file_path, 'r+') as spider_file: + with file_path.open('r+') as spider_file: file_data = spider_file.read() file_data = file_data.replace("name = \'example\'", "name = \'renamed\'") spider_file.seek(0) spider_file.write(file_data) spider_file.truncate() - modify_time_before = getmtime(file_path) + modify_time_before = file_path.stat().st_mtime file_contents_before = file_data if force: p, out, err = self.proc('genspider', '--force', file_name, 'example.com') self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out) - modify_time_after = getmtime(file_path) + modify_time_after = file_path.stat().st_mtime self.assertNotEqual(modify_time_after, modify_time_before) - file_contents_after = open(file_path, 'r').read() + file_contents_after = file_path.read_text() self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') self.assertIn(f"{file_path} already exists", out) - modify_time_after = getmtime(file_path) + modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) - file_contents_after = open(file_path, 'r').read() + file_contents_after = file_path.read_text() self.assertEqual(file_contents_after, file_contents_before) def test_same_filename_as_existing_spider_force(self): @@ -504,11 +504,11 @@ class GenspiderCommandTest(CommandTest): def test_url(self, url='test.com', domain="test.com"): self.assertEqual(0, self.call('genspider', '--force', 'test_name', url)) self.assertEqual(domain, - self.find_in_file(join(self.proj_mod_path, + self.find_in_file(Path(self.proj_mod_path, 'spiders', 'test_name.py'), r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1)) self.assertEqual(f'http://{domain}/', - self.find_in_file(join(self.proj_mod_path, + self.find_in_file(Path(self.proj_mod_path, 'spiders', 'test_name.py'), r'start_urls\s*=\s*\[\'(.+)\'\]').group(1)) @@ -526,31 +526,31 @@ class GenspiderStandaloneCommandTest(ProjectTest): def test_generate_standalone_spider(self): self.call('genspider', 'example', 'example.com') - assert exists(join(self.temp_path, 'example.py')) + assert Path(self.temp_path, 'example.py').exists() def test_same_name_as_existing_file(self, force=False): file_name = 'example' - file_path = join(self.temp_path, file_name + '.py') + file_path = Path(self.temp_path, file_name + '.py') p, out, err = self.proc('genspider', file_name, 'example.com') self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out) - assert exists(file_path) - modify_time_before = getmtime(file_path) - file_contents_before = open(file_path, 'r').read() + assert file_path.exists() + modify_time_before = file_path.stat().st_mtime + file_contents_before = file_path.read_text() if force: # use different template to ensure contents were changed p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com') self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out) - modify_time_after = getmtime(file_path) + modify_time_after = file_path.stat().st_mtime self.assertNotEqual(modify_time_after, modify_time_before) - file_contents_after = open(file_path, 'r').read() + file_contents_after = file_path.read_text() self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn(f"{join(self.temp_path, file_name + '.py')} already exists", out) - modify_time_after = getmtime(file_path) + self.assertIn(f"{Path(self.temp_path, file_name + '.py')} already exists", out) + modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) - file_contents_after = open(file_path, 'r').read() + file_contents_after = file_path.read_text() self.assertEqual(file_contents_after, file_contents_before) def test_same_name_as_existing_file_force(self): @@ -588,17 +588,16 @@ class BadSpider(scrapy.Spider): """ @contextmanager - def _create_file(self, content, name=None): - tmpdir = self.mktemp() - os.mkdir(tmpdir) + def _create_file(self, content, name=None) -> Generator[str, None, None]: + tmpdir = Path(self.mktemp()) + tmpdir.mkdir() if name: - fname = abspath(join(tmpdir, name)) + fname = (tmpdir / name).resolve() else: - fname = abspath(join(tmpdir, self.spider_filename)) - with open(fname, 'w') as f: - f.write(content) + fname = (tmpdir / self.spider_filename).resolve() + fname.write_text(content) try: - yield fname + yield str(fname) finally: rmtree(tmpdir) @@ -747,12 +746,11 @@ class MySpider(scrapy.Spider): ) return [] """ - with open(os.path.join(self.cwd, "example.json"), "w") as f1: - f1.write("not empty") + Path(self.cwd, "example.json").write_text("not empty") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) - with open(os.path.join(self.cwd, "example.json")) as f2: + with Path(self.cwd, "example.json").open() as f2: first_line = f2.readline() self.assertNotEqual(first_line, "not empty") @@ -854,9 +852,7 @@ class ViewCommandTest(CommandTest): class CrawlCommandTest(CommandTest): def crawl(self, code, args=()): - fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) - with open(fname, 'w') as f: - f.write(code) + Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code) return self.proc('crawl', 'myspider', *args) def get_log(self, code, args=()): @@ -908,12 +904,11 @@ class MySpider(scrapy.Spider): ) return [] """ - with open(os.path.join(self.cwd, "example.json"), "w") as f1: - f1.write("not empty") + Path(self.cwd, "example.json").write_text("not empty") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) - with open(os.path.join(self.cwd, "example.json")) as f2: + with Path(self.cwd, "example.json").open() as f2: first_line = f2.readline() self.assertNotEqual(first_line, "not empty") diff --git a/tests/test_crawler.py b/tests/test_crawler.py index cf15ba9b9..19f4229a3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,9 +1,9 @@ import logging -import os import platform import subprocess import sys import warnings +from pathlib import Path from pytest import raises, mark from twisted import version as twisted_version @@ -100,15 +100,14 @@ class CrawlerLoggingTestCase(unittest.TestCase): assert get_scrapy_root_handler() is None def test_spider_custom_settings_log_level(self): - log_file = self.mktemp() - with open(log_file, 'wb') as fo: - fo.write('previous message\n'.encode('utf-8')) + log_file = Path(self.mktemp()) + log_file.write_text('previous message\n', encoding='utf-8') class MySpider(scrapy.Spider): name = 'spider' custom_settings = { 'LOG_LEVEL': 'INFO', - 'LOG_FILE': log_file, + 'LOG_FILE': str(log_file), # settings to avoid extra warnings 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, @@ -124,8 +123,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): logging.warning('warning message') logging.error('error message') - with open(log_file, 'rb') as fo: - logged = fo.read().decode('utf-8') + logged = log_file.read_text(encoding='utf-8') self.assertIn('previous message', logged) self.assertNotIn('debug message', logged) @@ -139,14 +137,13 @@ class CrawlerLoggingTestCase(unittest.TestCase): self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) def test_spider_custom_settings_log_append(self): - log_file = self.mktemp() - with open(log_file, 'wb') as fo: - fo.write('previous message\n'.encode('utf-8')) + log_file = Path(self.mktemp()) + log_file.write_text('previous message\n', encoding='utf-8') class MySpider(scrapy.Spider): name = 'spider' custom_settings = { - 'LOG_FILE': log_file, + 'LOG_FILE': str(log_file), 'LOG_FILE_APPEND': False, # disable telnet if not available to avoid an extra warning 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, @@ -156,8 +153,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): get_crawler(MySpider) logging.debug('debug message') - with open(log_file, 'rb') as fo: - logged = fo.read().decode('utf-8') + logged = log_file.read_text(encoding='utf-8') self.assertNotIn('previous message', logged) self.assertIn('debug message', logged) @@ -296,9 +292,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: - def run_script(self, script_name, *script_args): - script_path = os.path.join(self.script_dir, script_name) - args = [sys.executable, script_path] + list(script_args) + def run_script(self, script_name: str, *script_args): + script_path = self.script_dir / script_name + args = [sys.executable, str(script_path)] + list(script_args) p = subprocess.Popen(args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() @@ -306,7 +302,7 @@ class ScriptRunnerMixin: class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): - script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') + script_dir = Path(__file__).parent.resolve() / 'CrawlerProcess' def test_simple(self): log = self.run_script('simple.py') @@ -463,7 +459,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): - script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') + script_dir = Path(__file__).parent.resolve() / 'CrawlerRunner' def test_response_ip_address(self): log = self.run_script("ip_address.py") diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 5e63ebffb..2558e4f91 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -2,6 +2,7 @@ import os import re from configparser import ConfigParser from importlib import import_module +from pathlib import Path from twisted import version as twisted_version from twisted.trial import unittest @@ -29,11 +30,7 @@ class ScrapyUtilsTest(unittest.TestCase): if not os.environ.get('_SCRAPY_PINNED', None): self.skipTest('Not in a pinned environment') - tox_config_file_path = os.path.join( - os.path.dirname(__file__), - '..', - 'tox.ini', - ) + tox_config_file_path = Path(__file__) / '..' / 'tox.ini' config_parser = ConfigParser() config_parser.read(tox_config_file_path) pattern = r'Twisted\[http2\]==([\d.]+)' diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 883960084..29ff8c2dc 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -3,6 +3,7 @@ import os import shutil import sys import tempfile +from pathlib import Path from typing import Optional, Type from unittest import mock, SkipTest @@ -10,7 +11,6 @@ from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal from twisted.internet import defer, error, reactor from twisted.protocols.policies import WrappingFactory -from twisted.python.filepath import FilePath from twisted.trial import unittest from twisted.web import resource, server, static, util from twisted.web._newclient import ResponseFailed @@ -108,14 +108,13 @@ class LoadTestCase(unittest.TestCase): class FileTestCase(unittest.TestCase): def setUp(self): - self.tmpname = self.mktemp() - with open(self.tmpname + '^', 'w') as f: - f.write('0123456789') + self.tmpname = Path(self.mktemp() + '^') + Path(self.tmpname).write_text('0123456789') handler = create_instance(FileDownloadHandler, None, get_crawler()) self.download_request = handler.download_request def tearDown(self): - os.unlink(self.tmpname + '^') + self.tmpname.unlink() def test_download(self): def _test(response): @@ -124,7 +123,7 @@ class FileTestCase(unittest.TestCase): self.assertEqual(response.body, b'0123456789') self.assertEqual(response.protocol, None) - request = Request(path_to_file_uri(self.tmpname + '^')) + request = Request(path_to_file_uri(str(self.tmpname))) assert request.url.upper().endswith('%5E') return self.download_request(request, Spider('foo')).addCallback(_test) @@ -223,10 +222,10 @@ class HttpTestCase(unittest.TestCase): certfile = 'keys/localhost.crt' def setUp(self): - self.tmpname = self.mktemp() - os.mkdir(self.tmpname) - FilePath(self.tmpname).child("file").setContent(b"0123456789") - r = static.File(self.tmpname) + self.tmpname = Path(self.mktemp()) + self.tmpname.mkdir() + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) @@ -626,10 +625,10 @@ class Https11CustomCiphers(unittest.TestCase): certfile = 'keys/localhost.crt' def setUp(self): - self.tmpname = self.mktemp() - os.mkdir(self.tmpname) - FilePath(self.tmpname).child("file").setContent(b"0123456789") - r = static.File(self.tmpname) + self.tmpname = Path(self.mktemp()) + self.tmpname.mkdir() + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) self.site = server.Site(r, timeout=None) self.host = 'localhost' self.port = reactor.listenSSL( @@ -1002,16 +1001,15 @@ class BaseFTPTestCase(unittest.TestCase): from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dirs and test file - self.directory = self.mktemp() - os.mkdir(self.directory) - userdir = os.path.join(self.directory, self.username) - os.mkdir(userdir) - fp = FilePath(userdir) + self.directory = Path(self.mktemp()) + self.directory.mkdir() + userdir = self.directory / self.username + userdir.mkdir() for filename, content in self.test_files: - fp.child(filename).setContent(content) + (userdir / filename).write_bytes(content) # setup server - realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) + realm = FTPRealm(anonymousRoot=str(self.directory), userHome=str(self.directory)) p = portal.Portal(realm) users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() users_checker.addUser(self.username, self.password) @@ -1076,28 +1074,28 @@ class BaseFTPTestCase(unittest.TestCase): def test_ftp_local_filename(self): f, local_fname = tempfile.mkstemp() - local_fname = to_bytes(local_fname) + fname_bytes = to_bytes(local_fname) + local_fname = Path(local_fname) os.close(f) - meta = {"ftp_local_filename": local_fname} + meta = {"ftp_local_filename": fname_bytes} meta.update(self.req_meta) request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(r.body, local_fname) - self.assertEqual(r.headers, {b'Local Filename': [local_fname], + self.assertEqual(r.body, fname_bytes) + self.assertEqual(r.headers, {b'Local Filename': [fname_bytes], b'Size': [b'17']}) - self.assertTrue(os.path.exists(local_fname)) - with open(local_fname, "rb") as f: - self.assertEqual(f.read(), b"I have the power!") - os.remove(local_fname) + self.assertTrue(local_fname.exists()) + self.assertEqual(local_fname.read_bytes(), b"I have the power!") + local_fname.unlink() return self._add_test_callbacks(d, _test) def _test_response_class(self, filename, response_class): f, local_fname = tempfile.mkstemp() - local_fname = to_bytes(local_fname) + local_fname = Path(local_fname) os.close(f) meta = {} meta.update(self.req_meta) @@ -1107,7 +1105,7 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(type(r), response_class) - os.remove(local_fname) + local_fname.unlink() return self._add_test_callbacks(d, _test) def test_response_class_from_url(self): @@ -1147,15 +1145,14 @@ class AnonymousFTPTestCase(BaseFTPTestCase): from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dir and test file - self.directory = self.mktemp() - os.mkdir(self.directory) + self.directory = Path(self.mktemp()) + self.directory.mkdir() - fp = FilePath(self.directory) for filename, content in self.test_files: - fp.child(filename).setContent(content) + (self.directory / filename).write_bytes(content) # setup server for anonymous access - realm = FTPRealm(anonymousRoot=self.directory) + realm = FTPRealm(anonymousRoot=str(self.directory)) p = portal.Portal(realm) p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 40e9f3a96..6f4e217e6 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,6 +1,6 @@ from gzip import GzipFile from io import BytesIO -from os.path import join +from pathlib import Path from unittest import TestCase, SkipTest from warnings import catch_warnings @@ -15,7 +15,7 @@ from tests import tests_datadir from w3lib.encoding import resolve_encoding -SAMPLEDIR = join(tests_datadir, 'compressed') +SAMPLEDIR = Path(tests_datadir, 'compressed') FORMAT = { 'gzip': ('html-gzip.bin', 'gzip'), @@ -46,8 +46,7 @@ class HttpCompressionTest(TestCase): samplefile, contentencoding = FORMAT[coding] - with open(join(SAMPLEDIR, samplefile), 'rb') as sample: - body = sample.read() + body = (SAMPLEDIR / samplefile).read_bytes() headers = { 'Server': 'Yaws/1.49 Yet Another Web Server', diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 8a37a8ebe..911d23069 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -2,8 +2,8 @@ import hashlib import tempfile import unittest import shutil -import os import sys +from pathlib import Path from testfixtures import LogCapture from scrapy.dupefilters import RFPDupeFilter @@ -157,7 +157,7 @@ class RFPDupeFilterTest(unittest.TestCase): df.request_seen(r1) df.close('finished') - with open(os.path.join(path, 'requests.seen'), 'rb') as seen_file: + with Path(path, 'requests.seen').open('rb') as seen_file: line = next(seen_file).decode() assert not line.endswith('\r\r\n') if sys.platform == 'win32': diff --git a/tests/test_engine.py b/tests/test_engine.py index 5677052f6..aa3313659 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -10,11 +10,11 @@ module with the ``runserver`` argument:: python test_engine.py runserver """ -import os import re import subprocess import sys from collections import defaultdict +from pathlib import Path from threading import Timer from urllib.parse import urlparse from dataclasses import dataclass @@ -127,8 +127,8 @@ class ChangeCloseReasonSpider(TestSpider): def start_test_site(debug=False): - root_dir = os.path.join(tests_datadir, "test_site") - r = static.File(root_dir) + root_dir = Path(tests_datadir, "test_site") + r = static.File(str(root_dir)) r.putChild(b"redirect", util.Redirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) numbers = [str(x).encode("utf8") for x in range(2**18)] diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ad2383018..98905d2c0 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -3,7 +3,6 @@ import csv import gzip import json import lzma -import os import random import shutil import string @@ -15,6 +14,7 @@ from collections import defaultdict from contextlib import ExitStack from io import BytesIO from logging import getLogger +from os import PathLike from pathlib import Path from string import ascii_letters, digits from unittest import mock @@ -63,41 +63,41 @@ def printf_escape(string): return string.replace('%', '%%') -def build_url(path): - if path[0] != '/': - path = '/' + path - return urljoin('file:', path) +def build_url(path: str | PathLike[str]) -> str: + path_str = str(path) + if path_str[0] != '/': + path_str = '/' + path_str + return urljoin('file:', path_str) class FileFeedStorageTest(unittest.TestCase): def test_store_file_uri(self): - path = os.path.abspath(self.mktemp()) - uri = path_to_file_uri(path) + path = Path(self.mktemp()).resolve() + uri = path_to_file_uri(str(path)) return self._assert_stores(FileFeedStorage(uri), path) def test_store_file_uri_makedirs(self): - path = os.path.abspath(self.mktemp()) - path = os.path.join(path, 'more', 'paths', 'file.txt') - uri = path_to_file_uri(path) + path = Path(self.mktemp()).resolve() / 'more' / 'paths' / 'file.txt' + uri = path_to_file_uri(str(path)) return self._assert_stores(FileFeedStorage(uri), path) def test_store_direct_path(self): - path = os.path.abspath(self.mktemp()) - return self._assert_stores(FileFeedStorage(path), path) + path = Path(self.mktemp()).resolve() + return self._assert_stores(FileFeedStorage(str(path)), path) def test_store_direct_path_relative(self): - path = self.mktemp() - return self._assert_stores(FileFeedStorage(path), path) + path = Path(self.mktemp()) + return self._assert_stores(FileFeedStorage(str(path)), path) def test_interface(self): path = self.mktemp() st = FileFeedStorage(path) verifyObject(IFeedStorage, st) - def _store(self, feed_options=None): - path = os.path.abspath(self.mktemp()) - storage = FileFeedStorage(path, feed_options=feed_options) + def _store(self, feed_options=None) -> Path: + path = Path(self.mktemp()).resolve() + storage = FileFeedStorage(str(path), feed_options=feed_options) spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") @@ -106,27 +106,26 @@ class FileFeedStorageTest(unittest.TestCase): def test_append(self): path = self._store() - return self._assert_stores(FileFeedStorage(path), path, b"contentcontent") + return self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") def test_overwrite(self): path = self._store({"overwrite": True}) return self._assert_stores( - FileFeedStorage(path, feed_options={"overwrite": True}), + FileFeedStorage(str(path), feed_options={"overwrite": True}), path ) @defer.inlineCallbacks - def _assert_stores(self, storage, path, expected_content=b"content"): + def _assert_stores(self, storage, path: Path, expected_content=b"content"): spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") yield storage.store(file) - self.assertTrue(os.path.exists(path)) + self.assertTrue(path.exists()) try: - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), expected_content) + self.assertEqual(path.read_bytes(), expected_content) finally: - os.unlink(path) + path.unlink() class FTPFeedStorageTest(unittest.TestCase): @@ -152,13 +151,12 @@ class FTPFeedStorageTest(unittest.TestCase): file.write(content) return storage.store(file) - def _assert_stored(self, path, content): + def _assert_stored(self, path: Path, content): self.assertTrue(path.exists()) try: - with path.open('rb') as fp: - self.assertEqual(fp.read(), content) + self.assertEqual(path.read_bytes(), content) finally: - os.unlink(str(path)) + path.unlink() @defer.inlineCallbacks def test_append(self): @@ -221,24 +219,24 @@ class BlockingFeedStorageTest(unittest.TestCase): b = BlockingFeedStorage() tmp = b.open(self.get_test_spider()) - tmp_path = os.path.dirname(tmp.name) - self.assertEqual(tmp_path, tempfile.gettempdir()) + tmp_path = Path(tmp.name).parent + self.assertEqual(str(tmp_path), tempfile.gettempdir()) def test_temp_file(self): b = BlockingFeedStorage() - tests_path = os.path.dirname(os.path.abspath(__file__)) - spider = self.get_test_spider({'FEED_TEMPDIR': tests_path}) + tests_path = Path(__file__).resolve().parent + spider = self.get_test_spider({'FEED_TEMPDIR': str(tests_path)}) tmp = b.open(spider) - tmp_path = os.path.dirname(tmp.name) + tmp_path = Path(tmp.name).parent self.assertEqual(tmp_path, tests_path) def test_invalid_folder(self): b = BlockingFeedStorage() - tests_path = os.path.dirname(os.path.abspath(__file__)) - invalid_path = os.path.join(tests_path, 'invalid_path') - spider = self.get_test_spider({'FEED_TEMPDIR': invalid_path}) + tests_path = Path(__file__).resolve().parent + invalid_path = tests_path / 'invalid_path' + spider = self.get_test_spider({'FEED_TEMPDIR': str(invalid_path)}) self.assertRaises(OSError, b.open, spider=spider) @@ -564,13 +562,13 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): class DummyBlockingFeedStorage(BlockingFeedStorage): def __init__(self, uri, *args, feed_options=None): - self.path = file_uri_to_path(uri) + self.path = Path(file_uri_to_path(uri)) def _store_in_thread(self, file): - dirname = os.path.dirname(self.path) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname) - with open(self.path, 'ab') as output_file: + dirname = self.path.parent + if dirname and not dirname.exists(): + dirname.mkdir(parents=True) + with self.path.open('ab') as output_file: output_file.write(file.read()) file.close() @@ -613,10 +611,10 @@ class FeedExportTestBase(ABC, unittest.TestCase): foo = scrapy.Field() hello = scrapy.Field() - def _random_temp_filename(self, inter_dir=''): + def _random_temp_filename(self, inter_dir='') -> Path: chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) - return os.path.join(self.temp_dir, inter_dir, filename) + return Path(self.temp_dir, inter_dir, filename) def setUp(self): self.temp_dir = tempfile.mkdtemp() @@ -702,18 +700,17 @@ class FeedExportTest(FeedExportTestBase): yield crawler.crawl() for file_path, feed_options in FEEDS.items(): - if not os.path.exists(str(file_path)): + if not Path(file_path).exists(): continue - with open(str(file_path), 'rb') as f: - content[feed_options['format']] = f.read() + content[feed_options['format']] = Path(file_path).read_bytes() finally: for file_path in FEEDS.keys(): - if not os.path.exists(str(file_path)): + if not Path(file_path).exists(): continue - os.remove(str(file_path)) + Path(file_path).unlink() return content @@ -808,7 +805,7 @@ class FeedExportTest(FeedExportTestBase): def test_stats_file_success(self): settings = { "FEEDS": { - printf_escape(path_to_url(self._random_temp_filename())): { + printf_escape(path_to_url(str(self._random_temp_filename()))): { "format": "json", } }, @@ -823,7 +820,7 @@ class FeedExportTest(FeedExportTestBase): def test_stats_file_failed(self): settings = { "FEEDS": { - printf_escape(path_to_url(self._random_temp_filename())): { + printf_escape(path_to_url(str(self._random_temp_filename()))): { "format": "json", } }, @@ -846,7 +843,7 @@ class FeedExportTest(FeedExportTestBase): 'AWS_ACCESS_KEY_ID': 'access_key', 'AWS_SECRET_ACCESS_KEY': 'secret_key', "FEEDS": { - printf_escape(path_to_url(self._random_temp_filename())): { + printf_escape(path_to_url(str(self._random_temp_filename()))): { "format": "json", }, "s3://bucket/key/foo.csv": { @@ -1427,12 +1424,11 @@ class FeedExportTest(FeedExportTestBase): self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler) @defer.inlineCallbacks - def test_pathlib_uri(self): - feed_path = Path(self._random_temp_filename()) + def test_str_uri(self): settings = { 'FEED_STORE_EMPTY': True, 'FEEDS': { - feed_path: {'format': 'csv'} + str(self._random_temp_filename()): {'format': 'csv'} }, } data = yield self.exported_no_data(settings) @@ -1538,8 +1534,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): def close(self): self.file.close() - def _named_tempfile(self, name): - return os.path.join(self.temp_dir, name) + def _named_tempfile(self, name) -> str: + return str(Path(self.temp_dir, name)) @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1559,18 +1555,17 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): yield crawler.crawl() for file_path, feed_options in FEEDS.items(): - if not os.path.exists(str(file_path)): + if not Path(file_path).exists(): continue - with open(str(file_path), 'rb') as f: - content[str(file_path)] = f.read() + content[str(file_path)] = Path(file_path).read_bytes() finally: for file_path in FEEDS.keys(): - if not os.path.exists(str(file_path)): + if not Path(file_path).exists(): continue - os.remove(str(file_path)) + Path(file_path).unlink() return content @@ -2031,11 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase): yield crawler.crawl() for path, feed in FEEDS.items(): - dir_name = os.path.dirname(path) - for file in sorted(os.listdir(dir_name)): - with open(os.path.join(dir_name, file), 'rb') as f: - data = f.read() - content[feed['format']].append(data) + dir_name = Path(path).parent + for file in sorted(dir_name.iterdir()): + content[feed['format']].append(file.read_bytes()) finally: self.tearDown() defer.returnValue(content) @@ -2045,7 +2038,7 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = settings or {} settings.update({ 'FEEDS': { - os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, + self._random_temp_filename() / 'jl' / self._file_mark: {'format': 'jl'}, }, }) batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') @@ -2061,7 +2054,7 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = settings or {} settings.update({ 'FEEDS': { - os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, + self._random_temp_filename() / 'csv' / self._file_mark: {'format': 'csv'}, }, }) batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') @@ -2077,7 +2070,7 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = settings or {} settings.update({ 'FEEDS': { - os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, + self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'}, }, }) batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') @@ -2094,8 +2087,8 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = settings or {} settings.update({ 'FEEDS': { - os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, - os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, + self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'}, + self._random_temp_filename() / 'json' / self._file_mark: {'format': 'json'}, }, }) batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') @@ -2120,7 +2113,7 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = settings or {} settings.update({ 'FEEDS': { - os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, + self._random_temp_filename() / 'pickle' / self._file_mark: {'format': 'pickle'}, }, }) batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') @@ -2137,7 +2130,7 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = settings or {} settings.update({ 'FEEDS': { - os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, + self._random_temp_filename() / 'marshal' / self._file_mark: {'format': 'marshal'}, }, }) batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') @@ -2184,7 +2177,7 @@ class BatchDeliveriesTest(FeedExportTestBase): for fmt in ('json', 'jsonlines', 'xml', 'csv'): settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, + self._random_temp_filename() / fmt / self._file_mark: {'format': fmt}, }, 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 } @@ -2204,7 +2197,7 @@ class BatchDeliveriesTest(FeedExportTestBase): for fmt, expctd in formats: settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, + self._random_temp_filename() / fmt / self._file_mark: {'format': fmt}, }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, @@ -2237,19 +2230,19 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + self._random_temp_filename() / 'json' / self._file_mark: { 'format': 'json', 'indent': 0, 'fields': ['bar'], 'encoding': 'utf-8', }, - os.path.join(self._random_temp_filename(), 'xml', self._file_mark): { + self._random_temp_filename() / 'xml' / self._file_mark: { 'format': 'xml', 'indent': 2, 'fields': ['foo'], 'encoding': 'latin-1', }, - os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { + self._random_temp_filename() / 'csv' / self._file_mark: { 'format': 'csv', 'indent': None, 'fields': ['foo', 'bar'], @@ -2272,7 +2265,7 @@ class BatchDeliveriesTest(FeedExportTestBase): } settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + self._random_temp_filename() / 'json' / self._file_mark: { 'format': 'json', 'indent': None, 'encoding': 'utf-8', @@ -2299,7 +2292,7 @@ class BatchDeliveriesTest(FeedExportTestBase): ] settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), '%(batch_time)s'): { + self._random_temp_filename() / '%(batch_time)s': { 'format': 'json', }, }, @@ -2312,7 +2305,7 @@ class BatchDeliveriesTest(FeedExportTestBase): def test_stats_batch_file_success(self): settings = { "FEEDS": { - build_url(os.path.join(self._random_temp_filename(), "json", self._file_mark)): { + build_url(str(self._random_temp_filename() / "json" / self._file_mark)): { "format": "json", } }, diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 49c83132f..402348cf9 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,10 +1,10 @@ import json -import os import random import re import shutil import string from ipaddress import IPv4Address +from pathlib import Path from unittest import mock, skipIf from urllib.parse import urlencode @@ -163,9 +163,8 @@ class RequestHeaders(LeafResource): return bytes(json.dumps(headers), 'utf-8') -def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: - with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: - pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) +def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate: + pem = key_file.read_text() + certificate_file.read_text() return PrivateCertificate.loadPEM(pem) @@ -173,12 +172,12 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2ClientProtocolTestCase(TestCase): scheme = 'https' - key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key') - certificate_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.crt') + key_file = Path(__file__).parent / 'keys' / 'localhost.key' + certificate_file = Path(__file__).parent / 'keys' / 'localhost.crt' def _init_resource(self): self.temp_directory = self.mktemp() - os.mkdir(self.temp_directory) + Path(self.temp_directory).mkdir() r = File(self.temp_directory) r.putChild(b'get-data-html-small', GetDataHtmlSmall()) r.putChild(b'get-data-html-large', GetDataHtmlLarge()) @@ -202,7 +201,7 @@ class Https2ClientProtocolTestCase(TestCase): # Start server for testing self.hostname = 'localhost' - context_factory = ssl_context_factory(self.key_file, self.certificate_file) + context_factory = ssl_context_factory(str(self.key_file), str(self.certificate_file)) server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) self.server = yield server_endpoint.listen(self.site) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index e46532a1c..5d6547279 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,5 +1,5 @@ -import os import shutil +from pathlib import Path from testfixtures import LogCapture from twisted.internet import defer @@ -61,12 +61,12 @@ class FileDownloadCrawlTestCase(TestCase): self.mockserver.__enter__() # prepare a directory for storing files - self.tmpmediastore = self.mktemp() - os.mkdir(self.tmpmediastore) + self.tmpmediastore = Path(self.mktemp()) + self.tmpmediastore.mkdir() self.settings = { 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', 'ITEM_PIPELINES': {self.pipeline_class: 1}, - self.store_setting_key: self.tmpmediastore, + self.store_setting_key: str(self.tmpmediastore), } self.runner = CrawlerRunner(self.settings) self.items = [] @@ -111,9 +111,7 @@ class FileDownloadCrawlTestCase(TestCase): # check that the image files where actually written to the media store for item in items: for i in item[self.media_key]: - self.assertTrue( - os.path.exists( - os.path.join(self.tmpmediastore, i['path']))) + self.assertTrue((self.tmpmediastore / i['path']).exists()) def _assert_files_download_failure(self, crawler, items, code, logs): @@ -133,7 +131,7 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(logs.count(file_dl_failure), 3) # check that no files were written to the media store - self.assertEqual(os.listdir(self.tmpmediastore), []) + self.assertEqual([x for x in self.tmpmediastore.iterdir()], []) @defer.inlineCallbacks def test_download_media(self): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index d641e7a43..4acd29bf7 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -3,6 +3,7 @@ import random import time from datetime import datetime from io import BytesIO +from pathlib import Path from shutil import rmtree from tempfile import mkdtemp from unittest import mock @@ -89,7 +90,7 @@ class FilesPipelineTestCase(unittest.TestCase): self.assertEqual(self.pipeline.store.basedir, self.tempdir) path = 'some/image/key.jpg' - fullpath = os.path.join(self.tempdir, 'some', 'image', 'key.jpg') + fullpath = Path(self.tempdir, 'some', 'image', 'key.jpg') self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath) @defer.inlineCallbacks diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index afdfb2578..ea7701b5d 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -2,6 +2,7 @@ import json import os import re import sys +from pathlib import Path from subprocess import Popen, PIPE from urllib.parse import urlsplit, urlunsplit from testfixtures import LogCapture @@ -27,14 +28,13 @@ from mitmproxy.tools.main import mitmdump sys.argv[0] = "mitmdump" sys.exit(mitmdump()) """ - cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'keys', 'mitmproxy-ca.pem') + cert_path = Path(__file__).parent.resolve() / 'keys' / 'mitmproxy-ca.pem' self.proc = Popen([sys.executable, '-c', script, '--listen-host', '127.0.0.1', '--listen-port', '0', '--proxyauth', f'{self.auth_user}:{self.auth_pass}', - '--certs', cert_path, + '--certs', str(cert_path), '--ssl-insecure', ], stdout=PIPE, env=get_testenv()) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 3719c7c9f..b9025cc14 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,7 +1,7 @@ import sys -import os import shutil import warnings +from pathlib import Path from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -17,10 +17,10 @@ from scrapy.settings import Settings from scrapy.http import Request from scrapy.crawler import CrawlerRunner -module_dir = os.path.dirname(os.path.abspath(__file__)) +module_dir = Path(__file__).resolve().parent -def _copytree(source, target): +def _copytree(source: Path, target: Path): try: shutil.copytree(source, target) except shutil.Error: @@ -30,18 +30,18 @@ def _copytree(source, target): class SpiderLoaderTest(unittest.TestCase): def setUp(self): - orig_spiders_dir = os.path.join(module_dir, 'test_spiders') - self.tmpdir = tempfile.mkdtemp() - self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') + orig_spiders_dir = module_dir / 'test_spiders' + self.tmpdir = Path(tempfile.mkdtemp()) + self.spiders_dir = self.tmpdir / 'test_spiders_xxx' _copytree(orig_spiders_dir, self.spiders_dir) - sys.path.append(self.tmpdir) + sys.path.append(str(self.tmpdir)) settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) self.spider_loader = SpiderLoader.from_settings(settings) def tearDown(self): del self.spider_loader del sys.modules['test_spiders_xxx'] - sys.path.remove(self.tmpdir) + sys.path.remove(str(self.tmpdir)) def test_interface(self): verifyObject(ISpiderLoader, self.spider_loader) @@ -135,22 +135,22 @@ class SpiderLoaderTest(unittest.TestCase): class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): - orig_spiders_dir = os.path.join(module_dir, 'test_spiders') - self.tmpdir = self.mktemp() - os.mkdir(self.tmpdir) - self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') + orig_spiders_dir = module_dir / 'test_spiders' + self.tmpdir = Path(self.mktemp()) + self.tmpdir.mkdir() + self.spiders_dir = self.tmpdir / 'test_spiders_xxx' _copytree(orig_spiders_dir, self.spiders_dir) - sys.path.append(self.tmpdir) + sys.path.append(str(self.tmpdir)) self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) def tearDown(self): del sys.modules['test_spiders_xxx'] - sys.path.remove(self.tmpdir) + sys.path.remove(str(self.tmpdir)) def test_dupename_warning(self): # copy 1 spider module so as to have duplicate spider name - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py')) + shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider3.py', + self.tmpdir / 'test_spiders_xxx' / 'spider3dupe.py') with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) @@ -171,10 +171,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name # This should issue 2 warning, 1 for each duplicate spider name - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py')) - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py')) + shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider1.py', + self.tmpdir / 'test_spiders_xxx' / 'spider1dupe.py') + shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider2.py', + self.tmpdir / 'test_spiders_xxx' / 'spider2dupe.py') with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 383fadfeb..ab215576e 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,5 +1,5 @@ -import os from datetime import datetime +from pathlib import Path import shutil from twisted.trial import unittest @@ -13,7 +13,7 @@ class SpiderStateTest(unittest.TestCase): def test_store_load(self): jobdir = self.mktemp() - os.mkdir(jobdir) + Path(jobdir).mkdir() try: spider = Spider(name='default') dt = datetime.now() diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 4943731cb..ca98bff21 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,5 +1,5 @@ import unittest -from os.path import join +from pathlib import Path from w3lib.encoding import html_to_unicode @@ -8,46 +8,40 @@ from scrapy.http import Response from tests import tests_datadir -SAMPLEDIR = join(tests_datadir, 'compressed') +SAMPLEDIR = Path(tests_datadir, 'compressed') class GunzipTest(unittest.TestCase): def test_gunzip_basic(self): - with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f: - r1 = Response("http://www.example.com", body=f.read()) - self.assertTrue(gzip_magic_number(r1)) + r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'feed-sample1.xml.gz').read_bytes()) + self.assertTrue(gzip_magic_number(r1)) - r2 = Response("http://www.example.com", body=gunzip(r1.body)) - self.assertFalse(gzip_magic_number(r2)) - self.assertEqual(len(r2.body), 9950) + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + self.assertFalse(gzip_magic_number(r2)) + self.assertEqual(len(r2.body), 9950) def test_gunzip_truncated(self): - with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f: - text = gunzip(f.read()) - assert text.endswith(b'') - self.assertFalse(gzip_magic_number(r2)) + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert r2.body.endswith(b'') + self.assertFalse(gzip_magic_number(r2)) def test_is_gzipped_empty(self): r1 = Response("http://www.example.com") self.assertFalse(gzip_magic_number(r1)) def test_gunzip_illegal_eof(self): - with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f: - text = html_to_unicode('charset=cp1252', gunzip(f.read()))[1] - with open(join(SAMPLEDIR, 'unexpected-eof-output.txt'), 'rb') as o: - expected_text = o.read().decode("utf-8") - self.assertEqual(len(text), len(expected_text)) - self.assertEqual(text, expected_text) + text = html_to_unicode('charset=cp1252', gunzip((SAMPLEDIR / 'unexpected-eof.gz').read_bytes()))[1] + expected_text = (SAMPLEDIR / 'unexpected-eof-output.txt').read_text(encoding="utf-8") + self.assertEqual(len(text), len(expected_text)) + self.assertEqual(text, expected_text) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index f84cb2956..ba3136b96 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,5 +1,3 @@ -import os - from pytest import mark from twisted.trial import unittest @@ -303,11 +301,6 @@ class LxmlXmliterTestCase(XmliterTestCase): class UtilsCsvTestCase(unittest.TestCase): - sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds') - sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv') - sample_feed2_path = os.path.join(sample_feeds_dir, 'feed-sample4.csv') - sample_feed3_path = os.path.join(sample_feeds_dir, 'feed-sample5.csv') - def test_csviter_defaults(self): body = get_testdata('feeds', 'feed-sample3.csv') response = TextResponse(url="http://example.com/", body=body) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index b83c1d6f0..dc5b9e123 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -1,6 +1,7 @@ import sys import os import unittest +from pathlib import Path from unittest import mock from scrapy.item import Item, Field @@ -55,7 +56,7 @@ class UtilsMiscTestCase(unittest.TestCase): self.assertRaises(ImportError, walk_modules, 'nomodule999') def test_walk_modules_egg(self): - egg = os.path.join(os.path.dirname(__file__), 'test.egg') + egg = str(Path(__file__).parent / 'test.egg') sys.path.append(egg) try: mods = walk_modules('testegg') diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 46452415a..f35f039a9 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -4,6 +4,7 @@ import tempfile import shutil import contextlib import warnings +from pathlib import Path from pytest import warns @@ -18,9 +19,7 @@ def inside_a_project(): try: os.chdir(project_dir) - with open('scrapy.cfg', 'w') as f: - # create an empty scrapy.cfg - f.close() + Path('scrapy.cfg').touch() yield project_dir finally: @@ -31,20 +30,20 @@ def inside_a_project(): class ProjectUtilsTest(unittest.TestCase): def test_data_path_outside_project(self): self.assertEqual( - os.path.join('.scrapy', 'somepath'), + str(Path('.scrapy', 'somepath')), data_path('somepath') ) - abspath = os.path.join(os.path.sep, 'absolute', 'path') + abspath = str(Path(os.path.sep, 'absolute', 'path')) self.assertEqual(abspath, data_path(abspath)) def test_data_path_inside_project(self): with inside_a_project() as proj_path: - expected = os.path.join(proj_path, '.scrapy', 'somepath') + expected = Path(proj_path, '.scrapy', 'somepath') self.assertEqual( - os.path.realpath(expected), - os.path.realpath(data_path('somepath')) + expected.resolve(), + Path(data_path('somepath')).resolve() ) - abspath = os.path.join(os.path.sep, 'absolute', 'path') + abspath = str(Path(os.path.sep, 'absolute', 'path').resolve()) self.assertEqual(abspath, data_path(abspath)) diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index d20852e62..cdf972933 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,6 +1,6 @@ -import os import unittest import warnings +from pathlib import Path from urllib.parse import urlparse from scrapy.exceptions import ScrapyDeprecationWarning @@ -39,10 +39,9 @@ class ResponseUtilsTest(unittest.TestCase): def browser_open(burl): path = urlparse(burl).path - if not os.path.exists(path): + if not path or not Path(path).exists(): path = burl.replace('file://', '') - with open(path, "rb") as f: - bbody = f.read() + bbody = Path(path).read_bytes() self.assertIn(b'', bbody) return True response = HtmlResponse(url, body=body) @@ -98,10 +97,9 @@ class ResponseUtilsTest(unittest.TestCase): def check_base_url(burl): path = urlparse(burl).path - if not os.path.exists(path): + if not path or not Path(path).exists(): path = burl.replace('file://', '') - with open(path, "rb") as f: - bbody = f.read() + bbody = Path(path).read_bytes() self.assertEqual(bbody.count(b''), 1) return True diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 1d5e63363..b1aca5ed3 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path from shutil import rmtree from tempfile import mkdtemp import unittest @@ -22,21 +22,19 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): template = 'from ${project_name}.spiders.${name} import ${classname}' rendered = 'from proj.spiders.spi import TheSpider' - template_path = os.path.join(self.tmp_path, 'templ.py.tmpl') - render_path = os.path.join(self.tmp_path, 'templ.py') + template_path = Path(self.tmp_path, 'templ.py.tmpl') + render_path = Path(self.tmp_path, 'templ.py') - with open(template_path, 'wb') as tmpl_file: - tmpl_file.write(template.encode('utf8')) - assert os.path.isfile(template_path) # Failure of test itself + template_path.write_text(template, encoding='utf8') + assert template_path.is_file() # Failure of test itself - render_templatefile(template_path, **context) + render_templatefile(str(template_path), **context) - self.assertFalse(os.path.exists(template_path)) - with open(render_path, 'rb') as result: - self.assertEqual(result.read().decode('utf8'), rendered) + self.assertFalse(template_path.exists()) + self.assertEqual(render_path.read_text(encoding='utf8'), rendered) - os.remove(render_path) - assert not os.path.exists(render_path) # Failure of test itself + render_path.unlink() + assert not render_path.exists() # Failure of test itself if '__main__' == __name__: diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 0d5827339..69d9a9e3a 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -2,8 +2,8 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ -import os import shutil +from pathlib import Path import OpenSSL.SSL from twisted.trial import unittest @@ -15,7 +15,6 @@ except ImportError: # deprecated in Twisted 19.7.0 # (remove once we bump our requirement past that version) from twisted.test.proto_helpers import StringTransport -from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from twisted.internet.defer import inlineCallbacks @@ -230,10 +229,10 @@ class WebClientTestCase(unittest.TestCase): return reactor.listenTCP(0, site, interface="127.0.0.1") def setUp(self): - self.tmpname = self.mktemp() - os.mkdir(self.tmpname) - FilePath(self.tmpname).child("file").setContent(b"0123456789") - r = static.File(self.tmpname) + self.tmpname = Path(self.mktemp()) + self.tmpname.mkdir() + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"error", ErrorResource()) @@ -379,10 +378,10 @@ class WebClientSSLTestCase(unittest.TestCase): return f"https://127.0.0.1:{self.portno}/{path}" def setUp(self): - self.tmpname = self.mktemp() - os.mkdir(self.tmpname) - FilePath(self.tmpname).child("file").setContent(b"0123456789") - r = static.File(self.tmpname) + self.tmpname = Path(self.mktemp()) + self.tmpname.mkdir() + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) r.putChild(b"payload", PayloadResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) From 7daf735f45ab9f3d8760d7a59e5f72c277d0687d Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Tue, 18 Oct 2022 20:07:29 -0400 Subject: [PATCH 0645/2083] downgrade type hint syntax --- scrapy/commands/runspider.py | 3 ++- scrapy/squeues.py | 3 ++- scrapy/utils/conf.py | 10 +++++----- tests/test_commands.py | 6 +++--- tests/test_feedexport.py | 3 ++- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index c41135508..a8db1cd86 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -3,13 +3,14 @@ from os import PathLike from pathlib import Path from importlib import import_module from types import ModuleType +from typing import Union from scrapy.utils.spider import iter_spider_classes from scrapy.exceptions import UsageError from scrapy.commands import BaseRunSpiderCommand -def _import_file(filepath: str | PathLike[str]) -> ModuleType: +def _import_file(filepath: Union[str, PathLike]) -> ModuleType: abspath = Path(filepath).resolve() dirname = str(abspath.parent) if abspath.suffix not in ('.py', '.pyw'): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 1f2dee55f..2fa84fc00 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -6,6 +6,7 @@ import marshal import pickle from os import PathLike from pathlib import Path +from typing import Union from queuelib import queue @@ -17,7 +18,7 @@ def _with_mkdir(queue_class): class DirectoriesCreated(queue_class): - def __init__(self, path: str | PathLike[str], *args, **kwargs): + def __init__(self, path: Union[str, PathLike], *args, **kwargs): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index e247f5999..0dfa714e8 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -5,7 +5,7 @@ import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import Optional +from typing import List, Optional, Union from scrapy.exceptions import ScrapyDeprecationWarning, UsageError @@ -67,7 +67,7 @@ def arglist_to_dict(arglist): return dict(x.split('=', 1) for x in arglist) -def closest_scrapy_cfg(path: str | os.PathLike[str] = '.', prevpath: Optional[str | os.PathLike] = None) -> str: +def closest_scrapy_cfg(path: Union[str, os.PathLike] = '.', prevpath: Optional[Union[str, os.PathLike]] = None) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents """ @@ -103,7 +103,7 @@ def get_config(use_closest=True): return cfg -def get_sources(use_closest=True) -> list[str]: +def get_sources(use_closest=True) -> List[str]: xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or Path('~/.config').expanduser() sources = [ '/etc/scrapy.cfg', @@ -131,8 +131,8 @@ def feed_complete_default_values_from_settings(feed, settings): return out -def feed_process_params_from_cli(settings, output: list[str], output_format=None, - overwrite_output: Optional[list[str]] = None): +def feed_process_params_from_cli(settings, output: List[str], output_format=None, + overwrite_output: Optional[List[str]] = None): """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary diff --git a/tests/test_commands.py b/tests/test_commands.py index 39f718cce..71ae85094 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -14,7 +14,7 @@ from shutil import rmtree, copytree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import mkdtemp from threading import Timer -from typing import Generator, Optional +from typing import Dict, Generator, Optional, Union from unittest import skipIf from pytest import mark @@ -104,7 +104,7 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) - def find_in_file(self, filename: str | os.PathLike[str], regex) -> Optional[re.Match]: + def find_in_file(self, filename: Union[str, os.PathLike], regex) -> Optional[re.Match]: """Find first pattern occurrence in file""" pattern = re.compile(regex) with Path(filename).open("r") as f: @@ -175,7 +175,7 @@ class StartprojectTest(ProjectTest): assert Path(project_path, project_name, 'spiders', '__init__.py').exists() -def get_permissions_dict(path: str | os.PathLike[str], renamings=None, ignore=None) -> dict[str, str]: +def get_permissions_dict(path: Union[str, os.PathLike], renamings=None, ignore=None) -> Dict[str, str]: def get_permissions(path: Path) -> str: return oct(path.stat().st_mode) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 98905d2c0..97c3a74b3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -17,6 +17,7 @@ from logging import getLogger from os import PathLike from pathlib import Path from string import ascii_letters, digits +from typing import Union from unittest import mock from urllib.parse import urljoin, quote from urllib.request import pathname2url @@ -63,7 +64,7 @@ def printf_escape(string): return string.replace('%', '%%') -def build_url(path: str | PathLike[str]) -> str: +def build_url(path: Union[str, PathLike]) -> str: path_str = str(path) if path_str[0] != '/': path_str = '/' + path_str From fb26e6b650005e63c7017b55ff7b9755938a6c11 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Tue, 18 Oct 2022 20:21:11 -0400 Subject: [PATCH 0646/2083] resolve path to search in error --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 71ae85094..c7284b6c3 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -547,7 +547,7 @@ class GenspiderStandaloneCommandTest(ProjectTest): self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn(f"{Path(self.temp_path, file_name + '.py')} already exists", out) + self.assertIn(f"{Path(self.temp_path, file_name + '.py').resolve()} already exists", out) modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) file_contents_after = file_path.read_text() From 93d82648e59a1a211c114d0b8be2d76e0d79b582 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Wed, 19 Oct 2022 08:08:21 -0400 Subject: [PATCH 0647/2083] resolve this path too --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index c7284b6c3..b61f314ec 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -492,7 +492,7 @@ class GenspiderCommandTest(CommandTest): self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn(f"{file_path} already exists", out) + self.assertIn(f"{file_path.resolve()} already exists", out) modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) file_contents_after = file_path.read_text() From 065db7b56688ce5cbdc1508cfda9603d33b17a87 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Wed, 19 Oct 2022 08:28:26 -0400 Subject: [PATCH 0648/2083] fix some mypy issues --- scrapy/commands/__init__.py | 4 ++-- scrapy/commands/genspider.py | 5 +++-- scrapy/spiders/__init__.py | 2 +- scrapy/utils/conf.py | 4 ++-- tests/test_commands.py | 1 + tests/test_crawler.py | 2 ++ 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 8570d90bd..2c205a712 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -15,7 +15,7 @@ from scrapy.exceptions import UsageError class ScrapyCommand: requires_project = False - crawler_process = None + crawler_process: Any = None # default settings to be used for this command instead of global defaults default_settings: Dict[str, Any] = {} @@ -23,7 +23,7 @@ class ScrapyCommand: exitcode = 0 def __init__(self): - self.settings = None # set in scrapy.cmdline + self.settings: Any = None # set in scrapy.cmdline def set_crawler(self, crawler): if hasattr(self, '_crawler'): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 01b4a0dbd..facb593da 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,7 +4,7 @@ import string from pathlib import Path from importlib import import_module -from typing import Optional +from typing import Optional, cast from urllib.parse import urlparse import scrapy @@ -116,6 +116,7 @@ class Command(ScrapyCommand): return template_file print(f"Unable to find template: {template}\n") print('Use "scrapy genspider --list" to see all available templates.') + return None def _list_templates(self): print("Available templates:") @@ -144,7 +145,7 @@ class Command(ScrapyCommand): # a file with the same name exists in the target directory spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) - spiders_dir = Path(spiders_module.__file__).parent + spiders_dir = Path(cast(str, spiders_module.__file__)).parent spiders_dir_abs = spiders_dir.resolve() path = spiders_dir_abs / (name + ".py") if path.exists(): diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 9a97e7801..0a9b124b9 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -22,7 +22,7 @@ class Spider(object_ref): class. """ - name: Optional[str] = None + name: str custom_settings: Optional[dict] = None def __init__(self, name=None, **kwargs): diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 0dfa714e8..82defa033 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -5,7 +5,7 @@ import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import List, Optional, Union +from typing import Any, Dict, List, Optional, Union from scrapy.exceptions import ScrapyDeprecationWarning, UsageError @@ -176,7 +176,7 @@ def feed_process_params_from_cli(settings, output: List[str], output_format=None 'URIs are specified' ) - result = {} + result: Dict[str, Dict[str, Any]] = {} for element in output: try: feed_uri, feed_format = element.rsplit(':', 1) diff --git a/tests/test_commands.py b/tests/test_commands.py index b61f314ec..f8081f450 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -112,6 +112,7 @@ class ProjectTest(unittest.TestCase): match = pattern.search(line) if match is not None: return match + return None class StartprojectTest(ProjectTest): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 19f4229a3..a0703ad47 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -292,6 +292,8 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: + script_dir: Path + def run_script(self, script_name: str, *script_args): script_path = self.script_dir / script_name args = [sys.executable, str(script_path)] + list(script_args) From c85de908193a534c60140492d5f793ab32ef3b48 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Thu, 20 Oct 2022 19:44:35 -0400 Subject: [PATCH 0649/2083] fix path to tox.ini --- tests/test_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 2558e4f91..74fdd966b 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -30,7 +30,7 @@ class ScrapyUtilsTest(unittest.TestCase): if not os.environ.get('_SCRAPY_PINNED', None): self.skipTest('Not in a pinned environment') - tox_config_file_path = Path(__file__) / '..' / 'tox.ini' + tox_config_file_path = Path(__file__).parent / '..' / 'tox.ini' config_parser = ConfigParser() config_parser.read(tox_config_file_path) pattern = r'Twisted\[http2\]==([\d.]+)' From 24d6ac1f529e1e2ea5b04cff30cf9015311ef420 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Thu, 20 Oct 2022 20:03:38 -0400 Subject: [PATCH 0650/2083] downgrade the last 3.9 type hints --- scrapy/pipelines/files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index ffb12d910..4e0211d1b 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -14,7 +14,7 @@ from contextlib import suppress from ftplib import FTP from io import BytesIO from pathlib import Path -from typing import Optional +from typing import DefaultDict, Optional, Set from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -46,7 +46,7 @@ class FSFilesStore: basedir = basedir.split('://', 1)[1] self.basedir = basedir self._mkdir(Path(self.basedir)) - self.created_directories: defaultdict[str, set[str]] = defaultdict(set) + self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set) def persist_file(self, path: str, buf, info, meta=None, headers=None): absolute_path = self._get_filesystem_path(path) From 12a26755ae601c9b3244a4a6b55d0e880fc21864 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Thu, 20 Oct 2022 20:14:43 -0400 Subject: [PATCH 0651/2083] improve render_templatefile --- scrapy/commands/startproject.py | 2 +- scrapy/utils/template.py | 15 +++++++++------ tests/test_utils_template.py | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 4323cdb53..9e1e95ab1 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -109,7 +109,7 @@ class Command(ScrapyCommand): move(project_dir / 'module', project_dir / project_name) for paths in TEMPLATES_TO_RENDER: tplfile = Path(project_dir, *(string.Template(s).substitute(project_name=project_name) for s in paths)) - render_templatefile(str(tplfile), project_name=project_name, ProjectName=string_camelcase(project_name)) + render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) print(f"New Scrapy project '{project_name}', using template directory " f"'{self.templates_dir}', created in:") print(f" {project_dir.resolve()}\n") diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 8075902b3..2177817d9 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,21 +1,24 @@ """Helper functions for working with templates""" +from os import PathLike import re import string from pathlib import Path +from typing import Union -def render_templatefile(path: str, **kwargs): - raw = Path(path).read_text('utf8') +def render_templatefile(path: Union[str, PathLike], **kwargs): + path_obj = Path(path) + raw = path_obj.read_text('utf8') content = string.Template(raw).substitute(**kwargs) - render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path + render_path = path_obj.with_suffix('') if path_obj.suffix == '.tmpl' else path_obj - if path.endswith('.tmpl'): - Path(path).rename(render_path) + if path_obj.suffix == '.tmpl': + path_obj.rename(render_path) - Path(render_path).write_text(content, 'utf8') + render_path.write_text(content, 'utf8') CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index b1aca5ed3..4f1e8772b 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -28,7 +28,7 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): template_path.write_text(template, encoding='utf8') assert template_path.is_file() # Failure of test itself - render_templatefile(str(template_path), **context) + render_templatefile(template_path, **context) self.assertFalse(template_path.exists()) self.assertEqual(render_path.read_text(encoding='utf8'), rendered) From 607eece72a91beb0e9a598a987e51a56bcffb434 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Thu, 20 Oct 2022 20:18:06 -0400 Subject: [PATCH 0652/2083] move to after check --- scrapy/commands/runspider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index a8db1cd86..ed16c3fb6 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -12,9 +12,9 @@ from scrapy.commands import BaseRunSpiderCommand def _import_file(filepath: Union[str, PathLike]) -> ModuleType: abspath = Path(filepath).resolve() - dirname = str(abspath.parent) if abspath.suffix not in ('.py', '.pyw'): raise ValueError(f"Not a Python source file: {abspath}") + dirname = str(abspath.parent) if dirname: sys.path = [dirname] + sys.path try: From b33244e2f0d877b8911f949308222db0b076d665 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 21 Oct 2022 19:17:04 +0500 Subject: [PATCH 0653/2083] Fix the flake8 per-file ignore syntax (#5688) --- .flake8 | 17 +++++++++-------- scrapy/utils/url.py | 1 + tests/test_loader.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.flake8 b/.flake8 index d7aebc24b..0c64d009e 100644 --- a/.flake8 +++ b/.flake8 @@ -6,16 +6,17 @@ ignore = W503 exclude = docs/conf.py +per-file-ignores = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file # F401: Module imported but unused - scrapy/__init__.py E402 - scrapy/core/downloader/handlers/http.py F401 - scrapy/http/__init__.py F401 - scrapy/linkextractors/__init__.py E402 F401 - scrapy/selector/__init__.py F401 - scrapy/spiders/__init__.py E402 F401 + scrapy/__init__.py:E402 + scrapy/core/downloader/handlers/http.py:F401 + scrapy/http/__init__.py:F401 + scrapy/linkextractors/__init__.py:E402,F401 + scrapy/selector/__init__.py:F401 + scrapy/spiders/__init__.py:E402,F401 # Issues pending a review: - scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 + scrapy/utils/url.py:F403,F405 + tests/test_loader.py:E741 diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 4d5e9ae82..21201ace5 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -34,6 +34,7 @@ def url_has_any_extension(url, extensions): lowercase_path = parse_url(url).path.lower() return any(lowercase_path.endswith(ext) for ext in extensions) + def parse_url(url, encoding=None): """Return urlparsed url from the given argument (which could be an already parsed url) diff --git a/tests/test_loader.py b/tests/test_loader.py index c0937b349..b3e44d36b 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -295,7 +295,7 @@ class SelectortemLoaderTest(unittest.TestCase): l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), ['Marta']) - + def test_init_method_with_base_response(self): """Selector should be None after initialization""" response = Response("https://scrapy.org") From 69d1b8fc0832f1cf00727c73eff6fbde11824c20 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Fri, 21 Oct 2022 20:21:08 -0400 Subject: [PATCH 0654/2083] dirname cannot be falsey --- scrapy/commands/runspider.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index ed16c3fb6..22fa6a53d 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -15,13 +15,11 @@ def _import_file(filepath: Union[str, PathLike]) -> ModuleType: if abspath.suffix not in ('.py', '.pyw'): raise ValueError(f"Not a Python source file: {abspath}") dirname = str(abspath.parent) - if dirname: - sys.path = [dirname] + sys.path + sys.path = [dirname] + sys.path try: module = import_module(abspath.stem) finally: - if dirname: - sys.path.pop(0) + sys.path.pop(0) return module From 28396c34977535ca59ed5d8481b50dd1e8171e49 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Fri, 21 Oct 2022 20:23:07 -0400 Subject: [PATCH 0655/2083] convert _get_request_path back to str --- scrapy/extensions/httpcache.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 3057bf157..58c4e644e 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -293,7 +293,7 @@ class FilesystemCacheStorage: metadata = self._read_meta(spider, request) if metadata is None: return # not cached - rpath = self._get_request_path(spider, request) + rpath = Path(self._get_request_path(spider, request)) with self._open(rpath / 'response_body', 'rb') as f: body = f.read() with self._open(rpath / 'response_headers', 'rb') as f: @@ -307,7 +307,7 @@ class FilesystemCacheStorage: def store_response(self, spider: Spider, request: Request, response): """Store the given response in the cache.""" - rpath = self._get_request_path(spider, request) + rpath = Path(self._get_request_path(spider, request)) if not rpath.exists(): rpath.mkdir(parents=True) metadata = { @@ -330,12 +330,12 @@ class FilesystemCacheStorage: with self._open(rpath / 'request_body', 'wb') as f: f.write(request.body) - def _get_request_path(self, spider: Spider, request: Request) -> Path: + def _get_request_path(self, spider: Spider, request: Request) -> str: key = self._fingerprinter.fingerprint(request).hex() - return Path(self.cachedir, spider.name, key[0:2], key) + return str(Path(self.cachedir, spider.name, key[0:2], key)) def _read_meta(self, spider: Spider, request: Request): - rpath = self._get_request_path(spider, request) + rpath = Path(self._get_request_path(spider, request)) metapath = rpath / 'pickled_meta' if not metapath.exists(): return # not found From 726680c7125ab3a6622b12e25d45dbfedc5a39b3 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Fri, 21 Oct 2022 20:32:13 -0400 Subject: [PATCH 0656/2083] change crawler_process to optional --- scrapy/commands/__init__.py | 5 +++-- scrapy/commands/genspider.py | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 2c205a712..1f088041c 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -4,9 +4,10 @@ Base class for Scrapy commands import os import argparse from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, Optional from twisted.python import failure +from scrapy.crawler import CrawlerProcess from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli from scrapy.exceptions import UsageError @@ -15,7 +16,7 @@ from scrapy.exceptions import UsageError class ScrapyCommand: requires_project = False - crawler_process: Any = None + crawler_process: Optional[CrawlerProcess] = None # default settings to be used for this command instead of global defaults default_settings: Dict[str, Any] = {} diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index facb593da..ed9660252 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -133,6 +133,10 @@ class Command(ScrapyCommand): return True return False + assert ( + self.crawler_process is not None + ), "crawler_process must be set before calling run" + try: spidercls = self.crawler_process.spider_loader.load(name) except KeyError: From b61b71c6f015d45f6e98a4280bf4993517180045 Mon Sep 17 00:00:00 2001 From: Godson-Gnanaraj Date: Tue, 25 Oct 2022 08:44:43 +0530 Subject: [PATCH 0657/2083] Replace indentation of source before parsing with ast. closes #5323 --- scrapy/utils/misc.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 1221b39b2..c0258c8d9 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -226,7 +226,14 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - code = re.sub(r"^[\t ]+", "", inspect.getsource(callable)) + pattern = r"(^[\t ]+)" + src = inspect.getsource(callable) + match = re.match(pattern, src) # Find indentation + code = re.sub(pattern, "", src) + if match: + # Remove indentation + code = re.sub(f"\n{match.group(0)}", "\n", code) + tree = ast.parse(code) for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): From f4e2a10ed6a44300738bd3701ea62b432c9c1a06 Mon Sep 17 00:00:00 2001 From: Kaushal Sharma Date: Tue, 25 Oct 2022 02:45:46 -0700 Subject: [PATCH 0658/2083] =?UTF-8?q?Image.ANTIALIAS=20=E2=86=92=20Image.R?= =?UTF-8?q?esampling.LANCZOS=20(#5692)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/pipelines/images.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 6b97190ee..67b3224b3 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -160,7 +160,14 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() - image.thumbnail(size, self._Image.ANTIALIAS) + try: + # Image.Resampling.LANCZOS was added in Pillow 9.1.0 + # remove this try except block, + # when updating the minimum requirements for Pillow. + resampling_filter = self._Image.Resampling.LANCZOS + except AttributeError: + resampling_filter = self._Image.ANTIALIAS + image.thumbnail(size, resampling_filter) buf = BytesIO() image.save(buf, 'JPEG') From 830e1c5dd85618a27749bbe41d35b11fb2bdd348 Mon Sep 17 00:00:00 2001 From: Godson-Gnanaraj Date: Wed, 26 Oct 2022 01:26:54 +0530 Subject: [PATCH 0659/2083] Add test for parsing decorated methods --- ...t_return_with_argument_inside_generator.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 1c85ca353..72277d701 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -165,6 +165,89 @@ https://example.org warn_on_generator_with_return_value(None, l2) self.assertEqual(len(w), 0) + def test_generators_return_none_with_decorator(self): + def decorator(func): + def inner_func(): + func() + return inner_func + + @decorator + def f3(): + yield 1 + return None + + @decorator + def g3(): + yield 1 + return + + @decorator + def h3(): + yield 1 + + @decorator + def i3(): + yield 1 + yield from generator_that_returns_stuff() + + @decorator + def j3(): + yield 1 + + def helper(): + return 0 + + yield helper() + + @decorator + def k3(): + """ +docstring + """ + url = """ +https://example.org + """ + yield url + return + + @decorator + def l3(): + return + + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f3) + assert not is_generator_with_return_value(g3) + assert not is_generator_with_return_value(h3) + assert not is_generator_with_return_value(i3) + assert not is_generator_with_return_value(j3) # not recursive + assert not is_generator_with_return_value(k3) # not recursive + assert not is_generator_with_return_value(l3) + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_none) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, f3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, g3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, h3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, i3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, j3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, k3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, l3) + self.assertEqual(len(w), 0) + @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error) def test_indentation_error(self): with warnings.catch_warnings(record=True) as w: From b0ddffc47b9cee5e6146497b42de3787da76d2ad Mon Sep 17 00:00:00 2001 From: Godson-Gnanaraj Date: Wed, 26 Oct 2022 06:53:43 +0530 Subject: [PATCH 0660/2083] Misc. changes: - compile regex - readability improvements --- scrapy/utils/misc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index c0258c8d9..4d4fb9600 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -226,13 +226,13 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - pattern = r"(^[\t ]+)" src = inspect.getsource(callable) - match = re.match(pattern, src) # Find indentation - code = re.sub(pattern, "", src) + pattern = re.compile(r"(^[\t ]+)") + code = pattern.sub("", src) + + match = pattern.match(src) # finds indentation if match: - # Remove indentation - code = re.sub(f"\n{match.group(0)}", "\n", code) + code = re.sub(f"\n{match.group(0)}", "\n", code) # remove indentation tree = ast.parse(code) for node in walk_callable(tree): From 2464939b7ee8381f904f7b0625aee6de0989b6c8 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 26 Oct 2022 15:58:20 -0300 Subject: [PATCH 0661/2083] Fixed deprecation warning in scrapy.core.engine (#5589) * Change `download` function logic * Fix CI error in 3.7 checks * Make `spider` parameter optional in `_download` function, assign spider value from self if `None` --- scrapy/core/engine.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 6602f661d..1228e78da 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -257,9 +257,7 @@ class ExecutionEngine: def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" - if spider is None: - spider = self.spider - else: + if spider is not None: warnings.warn( "Passing a 'spider' argument to ExecutionEngine.download is deprecated", category=ScrapyDeprecationWarning, @@ -267,7 +265,7 @@ class ExecutionEngine: ) if spider is not self.spider: logger.warning("The spider '%s' does not match the open spider", spider.name) - if spider is None: + if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") return self._download(request, spider).addBoth(self._downloaded, request, spider) @@ -278,11 +276,14 @@ class ExecutionEngine: self.slot.remove_request(request) return self.download(result, spider) if isinstance(result, Request) else result - def _download(self, request: Request, spider: Spider) -> Deferred: + def _download(self, request: Request, spider: Optional[Spider]) -> Deferred: assert self.slot is not None # typing self.slot.add_request(request) + if spider is None: + spider = self.spider + def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: if not isinstance(result, (Response, Request)): raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}") From b394f2165acd662e55762d524b916f146291e422 Mon Sep 17 00:00:00 2001 From: Andrei Andrukhovich Date: Wed, 26 Oct 2022 23:11:28 +0300 Subject: [PATCH 0662/2083] =?UTF-8?q?Fix=20typo:=20[they]=20depends=20?= =?UTF-8?q?=E2=86=92=20depend=20(#5694)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/intro/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 9ab479edd..2c2079f68 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -52,7 +52,7 @@ Scrapy is written in pure Python and depends on a few key Python packages (among * `twisted`_, an asynchronous networking framework * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs -Some of these packages themselves depends on non-Python packages +Some of these packages themselves depend on non-Python packages that might require additional installation steps depending on your platform. Please check :ref:`platform-specific guides below `. From a214147359b8840abbb67bbe2a4a4066b271d2a5 Mon Sep 17 00:00:00 2001 From: Johanan Idicula Date: Wed, 26 Oct 2022 21:43:35 -0400 Subject: [PATCH 0663/2083] ci: Update macos runner The GitHub Actions macos-10.15 runner image is now deprecated, and GitHub Actions has begun to temporarily fail jobs referencing it during brownout periods. The image will be fully unsupported by 2022-12-01, which is just about a month away. This change updates the macOS runner image to the latest generally-available version, to help reduce spurious CI failures during the brownout periods, and to stay abreast of the sunsetting of the macos-10.15 image. See also: actions/runner-images#5583 --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index fdb9f4980..61f1857f8 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: tests: - runs-on: macos-10.15 + runs-on: macos-11 strategy: fail-fast: false matrix: From ca50af645390e38299082d9ef4682c4be482ae70 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 27 Oct 2022 17:12:26 +0600 Subject: [PATCH 0664/2083] Remove an unused import. --- tests/test_pipeline_images.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index c4ce2736f..c189d08bf 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -4,7 +4,6 @@ import io import random from shutil import rmtree from tempfile import mkdtemp -from unittest import skipIf from warnings import catch_warnings import attr From b6541830849a6edd41da6b539d272790a4661d7d Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 27 Oct 2022 17:00:36 +0500 Subject: [PATCH 0665/2083] Use Python 3.11 as the default in CI (#5696) --- .github/workflows/checks.yml | 8 ++++---- .github/workflows/publish.yml | 2 +- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 9 +++------ .github/workflows/tests-windows.yml | 7 +++++++ .readthedocs.yml | 2 +- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 439dfee51..8c1ae4bd3 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -8,10 +8,10 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.10" + - python-version: "3.11" env: TOXENV: security - - python-version: "3.10" + - python-version: "3.11" env: TOXENV: flake8 # Pylint requires installing reppy, which does not support Python 3.9 @@ -22,10 +22,10 @@ jobs: - python-version: 3.7 env: TOXENV: typing - - python-version: "3.10" # Keep in sync with .readthedocs.yml + - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs - - python-version: "3.10" + - python-version: "3.11" env: TOXENV: twinecheck diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f6b098b80..991b0b6e8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Check Tag id: check-release-tag diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 61f1857f8..174d245ca 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index d2bfe4a5f..9c3ce8115 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -17,13 +17,10 @@ jobs: - python-version: "3.10" env: TOXENV: py - - python-version: "3.10" - env: - TOXENV: asyncio - - python-version: "3.11.0-rc.2" + - python-version: "3.11" env: TOXENV: py - - python-version: "3.11.0-rc.2" + - python-version: "3.11" env: TOXENV: asyncio - python-version: pypy3.9 @@ -57,7 +54,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 14683fd53..f60c48841 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -23,6 +23,13 @@ jobs: - python-version: "3.10" env: TOXENV: asyncio +# no binary package for lxml for 3.11 yet +# - python-version: "3.11" +# env: +# TOXENV: py +# - python-version: "3.11" +# env: +# TOXENV: asyncio steps: - uses: actions/checkout@v3 diff --git a/.readthedocs.yml b/.readthedocs.yml index 390be3749..e71d34f3a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.10" # Keep in sync with .github/workflows/checks.yml + python: "3.11" # Keep in sync with .github/workflows/checks.yml python: install: From 3a34fa839938d7640e56f3ce12ba60aed31a6ecf Mon Sep 17 00:00:00 2001 From: Godson <30664729+Godson-Gnanaraj@users.noreply.github.com> Date: Thu, 27 Oct 2022 17:32:12 +0530 Subject: [PATCH 0666/2083] Get the event loop from event_loop_policy to avoid a deprecation warning (#5689) --- scrapy/utils/defer.py | 8 +++++--- scrapy/utils/reactor.py | 21 ++++++++++++++++----- tests/test_utils_asyncio.py | 6 ++++-- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 8fcf31cab..38aefd6d0 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -26,7 +26,7 @@ from twisted.python import failure from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest -from scrapy.utils.reactor import is_asyncio_reactor_installed +from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy def defer_fail(_failure: Failure) -> Deferred: @@ -269,7 +269,8 @@ def deferred_from_coro(o) -> Any: return ensureDeferred(o) else: # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - return Deferred.fromFuture(asyncio.ensure_future(o)) + event_loop = get_asyncio_event_loop_policy().get_event_loop() + return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) return o @@ -320,7 +321,8 @@ def deferred_to_future(d: Deferred) -> Future: d = treq.get('https://example.com/additional') additional_response = await deferred_to_future(d) """ - return d.asFuture(asyncio.get_event_loop()) + policy = get_asyncio_event_loop_policy() + return d.asFuture(policy.get_event_loop()) def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 652733ce8..ddf354d88 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -51,6 +51,19 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) +def get_asyncio_event_loop_policy(): + policy = asyncio.get_event_loop_policy() + if ( + sys.version_info >= (3, 8) + and sys.platform == "win32" + and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy) + ): + policy = asyncio.WindowsSelectorEventLoopPolicy() + asyncio.set_event_loop_policy(policy) + + return policy + + def install_reactor(reactor_path, event_loop_path=None): """Installs the :mod:`~twisted.internet.reactor` with the specified import path. Also installs the asyncio event loop with the specified import @@ -58,16 +71,14 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - if sys.version_info >= (3, 8) and sys.platform == "win32": - policy = asyncio.get_event_loop_policy() - if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + policy = get_asyncio_event_loop_policy() if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() asyncio.set_event_loop(event_loop) else: - event_loop = asyncio.get_event_loop() + event_loop = policy.get_event_loop() + asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 295323e4d..741c6a505 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,3 +1,4 @@ +import warnings from unittest import TestCase from pytest import mark @@ -13,5 +14,6 @@ class AsyncioTest(TestCase): self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') def test_install_asyncio_reactor(self): - # this should do nothing - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + with warnings.catch_warnings(record=True) as w: + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + self.assertEqual(len(w), 0) From b71d0292d5ff85d652e2943ee1a727a128b55594 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 27 Oct 2022 18:13:47 +0600 Subject: [PATCH 0667/2083] Add a test for processing partial callbacks. --- .../test_return_with_argument_inside_generator.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 72277d701..562f72fee 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,5 +1,6 @@ import unittest import warnings +from functools import partial from unittest import mock from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value @@ -254,3 +255,10 @@ https://example.org warn_on_generator_with_return_value(None, top_level_return_none) self.assertEqual(len(w), 1) self.assertIn('Unable to determine', str(w[0].message)) + + def test_partial(self): + def cb(arg1, arg2): + yield {} + + partial_cb = partial(cb, arg1=42) + assert not is_generator_with_return_value(partial_cb) From fd692f309105d917f5f46bd00a88c550d6cc7da3 Mon Sep 17 00:00:00 2001 From: Magnus Offermanns Date: Thu, 27 Oct 2022 14:43:31 +0200 Subject: [PATCH 0668/2083] Prevent running the -O and -t command-line options together (#5605) Co-authored-by: Andrey Rakhmatullin --- docs/topics/commands.rst | 20 ++++++++++++++++++++ scrapy/commands/__init__.py | 6 ++++-- scrapy/utils/conf.py | 19 ++++++++++++++++--- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8c0b8e55f..362190116 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -271,11 +271,31 @@ crawl Start crawling using a spider. +Supported options: + +* ``-h, --help``: show a help message and exit + +* ``-a NAME=VALUE``: set a spider argument (may be repeated) + +* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) + +* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) + +* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O`` + Usage examples:: $ scrapy crawl myspider [ ... myspider starts crawling ... ] + $ scrapy -o myfile:csv myspider + [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] + + $ scrapy -O myfile:json myspider + [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ] + + $ scrapy -o myfile -t csv myspider + [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] .. command:: check diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index fb304b8c0..49c4e2f42 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand): parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_argument("-o", "--output", metavar="FILE", action="append", - help="append scraped items to the end of FILE (use - for stdout)") + help="append scraped items to the end of FILE (use - for stdout)," + " to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)") parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append", - help="dump scraped items into FILE, overwriting any existing file") + help="dump scraped items into FILE, overwriting any existing file," + " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)") parser.add_argument("-t", "--output-format", metavar="FORMAT", help="format to use for dumping items") diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 00cc53725..6404edda6 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -155,6 +155,15 @@ def feed_process_params_from_cli(settings, output, output_format=None, raise UsageError( "Please use only one of -o/--output and -O/--overwrite-output" ) + if output_format: + raise UsageError( + "-t/--output-format is a deprecated command line option" + " and does not work in combination with -O/--overwrite-output." + " To specify a format please specify it after a colon at the end of the" + " output URI (i.e. -O :)." + " Example working in the tutorial: " + "scrapy crawl quotes -O quotes.json:json" + ) output = overwrite_output overwrite = True @@ -162,9 +171,13 @@ def feed_process_params_from_cli(settings, output, output_format=None, if len(output) == 1: check_valid_format(output_format) message = ( - 'The -t command line option is deprecated in favor of ' - 'specifying the output format within the output URI. See the ' - 'documentation of the -o and -O options for more information.' + "The -t/--output-format command line option is deprecated in favor of " + "specifying the output format within the output URI using the -o/--output or the" + " -O/--overwrite-output option (i.e. -o/-O :). See the documentation" + " of the -o or -O option or the following examples for more information. " + "Examples working in the tutorial: " + "scrapy crawl quotes -o quotes.csv:csv or " + "scrapy crawl quotes -O quotes.json:json" ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} From bd9e482c2f0db92065708c8291be6e8bc1f05218 Mon Sep 17 00:00:00 2001 From: iamkaushal Date: Thu, 27 Oct 2022 23:21:55 +0530 Subject: [PATCH 0669/2083] added typing.io and typing.re in pytest warning filter to ignore --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest.ini b/pytest.ini index af0f2fb6e..f5fbf2529 100644 --- a/pytest.ini +++ b/pytest.ini @@ -24,3 +24,5 @@ markers = filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated + ignore:typing.re is deprecated + ignore:typing.io is deprecated From 9f45be439de8a3b9a6d201c33e98b408a73c02bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=B4=8E=E0=B4=A4=E0=B4=BF=E0=B4=B0=E0=B4=BE=E0=B4=B3?= =?UTF-8?q?=E0=B4=BF=E0=B4=95=E0=B5=8D=E0=B4=95=E0=B5=8A=E0=B4=B0=E0=B5=81?= =?UTF-8?q?=20=E0=B4=AA=E0=B5=8B=E0=B4=B0=E0=B4=BE=E0=B4=B3=E0=B4=BF?= <108031802+pankali@users.noreply.github.com> Date: Fri, 28 Oct 2022 02:13:37 +0200 Subject: [PATCH 0670/2083] Update Code of Conduct to Contributor Covenant v2.1 --- CODE_OF_CONDUCT.md | 152 +++++++++++++++++++++++++++++++-------------- 1 file changed, 104 insertions(+), 48 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 902cd523e..3c8e4d1b5 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,77 +1,133 @@ + # Contributor Covenant Code of Conduct ## Our Pledge -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to make participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of experience, -nationality, personal appearance, race, religion, or sexual identity and -orientation. +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. ## Our Standards -Examples of behavior that contributes to creating a positive environment -include: +Examples of behavior that contributes to a positive environment for our +community include: -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community -Examples of unacceptable behavior by participants include: +Examples of unacceptable behavior include: -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission +* Publishing others' private information, such as a physical or email address, + without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting -## Our Responsibilities +## Enforcement Responsibilities -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. ## Scope -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at opensource@zyte.com. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. +reported to the community leaders responsible for enforcement at +opensource@zyte.com. +All complaints will be reviewed and investigated promptly and fairly. -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at [http://contributor-covenant.org/version/1/4][version]. +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations From 3259a4252566f4130f14ce2acd03531a735dae90 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 30 Oct 2022 13:17:37 -0300 Subject: [PATCH 0671/2083] CrawlSpider: pass cb_kwargs from process_request --- scrapy/spiders/crawl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index edac082d0..2d9328633 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -102,9 +102,9 @@ class CrawlSpider(Spider): request = self._build_request(rule_index, link) yield rule.process_request(request, response) - def _callback(self, response): + def _callback(self, response, **cb_kwargs): rule = self._rules[response.meta['rule']] - return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow) + return self._parse_response(response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow) def _errback(self, failure): rule = self._rules[failure.request.meta['rule']] From b18560315bda057610ecda165694f9c0f7445c1f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 30 Oct 2022 18:28:16 -0300 Subject: [PATCH 0672/2083] Add tests --- tests/spiders.py | 11 +++++++++++ tests/test_crawl.py | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/spiders.py b/tests/spiders.py index 5ea8a4a21..7952e3d47 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -419,6 +419,17 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): self.logger.info('[errback] status %i', failure.value.response.status) +class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod): + name = 'crawl_spider_with_process_request_cb_kwargs' + rules = ( + Rule(LinkExtractor(), callback='parse', follow=True, process_request="process_request"), + ) + + def process_request(self, request, response): + request.cb_kwargs["foo"] = "process_request" + return request + + class BytesReceivedCallbackSpider(MetaSpider): full_response_length = 2**18 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 5383ec652..5ec96e4a7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -41,6 +41,7 @@ from tests.spiders import ( CrawlSpiderWithAsyncGeneratorCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, + CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, DuplicateStartRequestsSpider, FollowAllSpider, @@ -426,6 +427,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[errback] status 500", str(log)) self.assertIn("[errback] status 501", str(log)) + @defer.inlineCallbacks + def test_crawlspider_process_request_cb_kwargs(self): + crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse] status 200 (foo: process_request)", str(log)) + self.assertIn("[parse] status 201 (foo: process_request)", str(log)) + self.assertIn("[parse] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_async_def_parse(self): crawler = get_crawler(AsyncDefSpider) From 940a73863bf7dcb16b3f2d9f5efb83efe4599712 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Nov 2022 19:00:33 +0600 Subject: [PATCH 0673/2083] Release notes for 2.7.1. --- docs/news.rst | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 1ec183a1d..b7c8c85b5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,57 @@ Release notes ============= +.. _release-2.7.1: + +Scrapy 2.7.1 (Not relased yet) +------------------------------ + +New features +~~~~~~~~~~~~ + +- Relaxed the restriction introduced in 2.6.2 so that the + ``Proxy-Authentication`` header can again be set explicitly, as long as the + proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and + for as long as that proxy URL remains the same (:issue:`5626`). + +Bug fixes +~~~~~~~~~ + +- Using ``-O``/``--overwrite-output`` and ``-t``/``--output-format`` options + together now produces an error instead of ignoring the former option + (:issue:`5516`, :issue:`5605`). + +- Replaced deprecated :mod:`asyncio` APIs that implicitly use the current + event loop with code that explicitly requests a loop from the event loop + policy (:issue:`5685`, :issue:`5689`). + +- Fixed uses of deprecated Scrapy APIs in the Scrapy itself (:issue:`5588`, + :issue:`5589`). + +- Fixed uses of a deprecated Pillow API (:issue:`5684`, :issue:`5692`). + +- Improved code that checks if generators return values, so that it no longer + fails on decorated methods and partial methods (:issue:`5323`, + :issue:`5592`, :issue:`5599`, :issue:`5691`). + +Documentation +~~~~~~~~~~~~~ + +- Upgraded the Code of Conduct to Contributor Covenant v2.1 (:issue:`5698`). + +- Fixed typos (:issue:`5681`, :issue:`5694`). + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Re-enabled some erroneously disabled flake8 checks (:issue:`5688`). + +- Ignored harmless deprecation warnings from :mod:`typing` in tests + (:issue:`5686`, :issue:`5697`). + +- Modernized CI configuration (:issue:`5695`, :issue:`5696`). + + .. _release-2.7.0: Scrapy 2.7.0 (2022-10-17) From 5ec175b8bb08f93c431d7d64d2389b90ec7a1f37 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Nov 2022 13:54:00 +0600 Subject: [PATCH 0674/2083] Small relnotes fixes. --- docs/news.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index b7c8c85b5..257626526 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -14,7 +14,8 @@ New features - Relaxed the restriction introduced in 2.6.2 so that the ``Proxy-Authentication`` header can again be set explicitly, as long as the proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and - for as long as that proxy URL remains the same (:issue:`5626`). + for as long as that proxy URL remains the same; this restores compatibility + with scrapy-zyte-smartproxy 2.1.0 and older (:issue:`5626`). Bug fixes ~~~~~~~~~ @@ -27,7 +28,7 @@ Bug fixes event loop with code that explicitly requests a loop from the event loop policy (:issue:`5685`, :issue:`5689`). -- Fixed uses of deprecated Scrapy APIs in the Scrapy itself (:issue:`5588`, +- Fixed uses of deprecated Scrapy APIs in Scrapy itself (:issue:`5588`, :issue:`5589`). - Fixed uses of a deprecated Pillow API (:issue:`5684`, :issue:`5692`). @@ -51,7 +52,7 @@ Quality assurance - Ignored harmless deprecation warnings from :mod:`typing` in tests (:issue:`5686`, :issue:`5697`). -- Modernized CI configuration (:issue:`5695`, :issue:`5696`). +- Modernized our CI configuration (:issue:`5695`, :issue:`5696`). .. _release-2.7.0: From 6ded3cf4cd134b615239babe28bb28c3ff524b05 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Nov 2022 17:00:47 +0600 Subject: [PATCH 0675/2083] =?UTF-8?q?Bump=20version:=202.7.0=20=E2=86=92?= =?UTF-8?q?=202.7.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 4 ++-- scrapy/VERSION | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f88071685..b949d81c4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.7.0 +current_version = 2.7.1 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index 257626526..e5fc2971a 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,8 +5,8 @@ Release notes .. _release-2.7.1: -Scrapy 2.7.1 (Not relased yet) ------------------------------- +Scrapy 2.7.1 (2022-11-02) +------------------------- New features ~~~~~~~~~~~~ diff --git a/scrapy/VERSION b/scrapy/VERSION index 24ba9a38d..860487ca1 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.7.0 +2.7.1 From a34b929a40c12933f75db4665b71348444a5d603 Mon Sep 17 00:00:00 2001 From: srki24 Date: Fri, 4 Nov 2022 18:00:17 +0100 Subject: [PATCH 0676/2083] issues/5043 Detaching the stream --- scrapy/exporters.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 76cbe4d4b..243ec4fe1 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -247,6 +247,12 @@ class CsvItemExporter(BaseItemExporter): values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) + def finish_exporting(self): + # Detaching stream in order to avoid file closing. + # The file will be closed with slot.storage.store + # https://github.com/scrapy/scrapy/issues/5043 + self.stream.detach() + def _build_row(self, values): for s in values: try: From 6c0890ff54a8d49237415e5b7d7dfbf216e88577 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 7 Nov 2022 16:36:54 +0500 Subject: [PATCH 0677/2083] Simplify the changes after the merge --- tests/test_pipeline_images.py | 37 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 81c9a027a..0c9a5733f 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -7,7 +7,6 @@ from shutil import rmtree from tempfile import mkdtemp from unittest import skipIf from unittest.mock import patch -from warnings import catch_warnings import attr from itemadapter import ItemAdapter @@ -91,6 +90,22 @@ class ImagesPipelineTestCase(unittest.TestCase): info=object()), 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') + def test_thumbnail_name_from_item(self): + """ + Custom thumbnail name based on item data, overriding default implementation + """ + + class CustomImagesPipeline(ImagesPipeline): + def thumb_path(self, request, thumb_id, response=None, info=None, item=None): + return f"thumb/{thumb_id}/{item.get('path')}" + + thumb_path = CustomImagesPipeline.from_settings(Settings( + {'IMAGES_STORE': self.tempdir} + )).thumb_path + item = dict(path='path-to-store-file') + request = Request("http://example.com") + self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file') + def test_get_images_exception(self): self.pipeline.min_width = 100 self.pipeline.min_height = 100 @@ -231,22 +246,6 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - def test_thumbnail_name_from_item(self): - """ - Custom thumbnail name based on item data, overriding default implementation - """ - - class CustomImagesPipeline(ImagesPipeline): - def thumb_path(self, request, thumb_id, response=None, info=None, item=None): - return f"thumb/{thumb_id}/{item.get('path')}" - - thumb_path = CustomImagesPipeline.from_settings(Settings( - {'IMAGES_STORE': self.tempdir} - )).thumb_path - item = dict(path='path-to-store-file') - request = Request("http://example.com") - self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file') - class DeprecatedImagesPipeline(ImagesPipeline): def file_key(self, url): @@ -536,11 +535,11 @@ class NoimagesDropTestCase(unittest.TestCase): def test_deprecation_warning(self): arg = str() - with catch_warnings(record=True) as warnings: + with warnings.catch_warnings(record=True) as warnings: NoimagesDrop(arg) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with catch_warnings(record=True) as warnings: + with warnings.catch_warnings(record=True) as warnings: class SubclassedNoimagesDrop(NoimagesDrop): pass SubclassedNoimagesDrop(arg) From bbe24d79a5ee6a2afc8cd50bff4ac0e6df26886c Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 7 Nov 2022 17:08:54 +0500 Subject: [PATCH 0678/2083] Fix test issues --- tests/test_pipeline_images.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 0c9a5733f..f98d40fda 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -5,7 +5,6 @@ import random import warnings from shutil import rmtree from tempfile import mkdtemp -from unittest import skipIf from unittest.mock import patch import attr @@ -535,16 +534,16 @@ class NoimagesDropTestCase(unittest.TestCase): def test_deprecation_warning(self): arg = str() - with warnings.catch_warnings(record=True) as warnings: + with warnings.catch_warnings(record=True) as w: NoimagesDrop(arg) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with warnings.catch_warnings(record=True) as warnings: + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, ScrapyDeprecationWarning) + with warnings.catch_warnings(record=True) as w: class SubclassedNoimagesDrop(NoimagesDrop): pass SubclassedNoimagesDrop(arg) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, ScrapyDeprecationWarning) def _create_image(format, *a, **kw): From ae3fd0172972f672c3cf3291bcf9f28073e8d4d0 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 10 Nov 2022 11:38:46 -0300 Subject: [PATCH 0679/2083] =?UTF-8?q?Fix=20stray=20=E2=80=9Ccommands?= =?UTF-8?q?=E2=80=9D=20(#5712)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/cmdline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 5ee1f0f44..68267fb74 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -7,7 +7,7 @@ import pkg_resources import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings @@ -32,7 +32,7 @@ def _iter_command_classes(module_name): inspect.isclass(obj) and issubclass(obj, ScrapyCommand) and obj.__module__ == module.__name__ - and not obj == ScrapyCommand + and obj not in (ScrapyCommand, BaseRunSpiderCommand) ): yield obj From 29bf7f5a6c8460e030e465351d2e6d38acf22f3d Mon Sep 17 00:00:00 2001 From: Hassan Shoayb <79839316+Hassan-Shoayb@users.noreply.github.com> Date: Mon, 14 Nov 2022 14:15:00 +0530 Subject: [PATCH 0680/2083] broad-crawls.rst: fix a typo (#5714) --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 63b60312e..0927ac2d2 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -68,7 +68,7 @@ IP (:setting:`CONCURRENT_REQUESTS_PER_IP`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much -to increase it will depend on how much CPU and memory you crawler will have +to increase it will depend on how much CPU and memory your crawler will have available. A good starting point is ``100``:: From 1200a545439677942085f392d7477ee37b62691e Mon Sep 17 00:00:00 2001 From: islem-esi Date: Tue, 15 Nov 2022 16:28:45 +0100 Subject: [PATCH 0681/2083] minor fix for readability --- scrapy/cmdline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 68267fb74..8218a51c8 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -78,7 +78,8 @@ def _pop_command_name(argv): def _print_header(settings, inproject): version = scrapy.__version__ if inproject: - print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n") + print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n") + else: print(f"Scrapy {version} - no active project\n") From 12d52a4f089798f266cbc6f86df3cc9c1cd58257 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 18 Nov 2022 14:16:18 +0200 Subject: [PATCH 0682/2083] per slot settings: code optimized --- scrapy/core/downloader/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d908f4d7e..f16afe99b 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -100,12 +100,13 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.per_slot_settings.get(key, {}).get( + slot_settings = self.per_slot_settings.get(key, {}) + conc = slot_settings.get( 'concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - delay = self.per_slot_settings.get(key, {}).get('delay', delay) - randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) + delay = slot_settings.get('delay', delay) + randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot From 042012f6bdaf4fbd5b978ff7cc9b6796286ac7c4 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 18 Nov 2022 14:56:29 +0200 Subject: [PATCH 0683/2083] per slot settings: error calculation metho updated --- tests/test_downloaderslotssettings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 6ea03fb78..a092d01bf 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -70,6 +70,6 @@ class CrawlTestCase(TestCase): tolerance = 0.3 delays_real = {k: v[1] - v[0] for k, v in times.items()} - error_delta = {k: 1 - delays_real[k] / v.delay for k, v in slots.items()} + error_delta = {k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) for k, v in slots.items()} self.assertTrue(max(list(error_delta.values())) < tolerance) From d5b6c236a90aac37b7942793e4943347ebaa65b8 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 21 Nov 2022 05:32:26 -0300 Subject: [PATCH 0684/2083] Remove deprecated code (#5719) --- scrapy/utils/boto.py | 23 ----------------------- scrapy/utils/gz.py | 11 ----------- scrapy/utils/python.py | 38 -------------------------------------- tests/test_utils_python.py | 34 ++-------------------------------- 4 files changed, 2 insertions(+), 104 deletions(-) diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 3374c57c7..39a681001 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,27 +1,4 @@ """Boto/botocore helpers""" -import warnings - -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning - - -def is_botocore(): - """ Returns True if botocore is available, otherwise raises NotConfigured. Never returns False. - - Previously, when boto was supported in addition to botocore, this returned False if boto was available - but botocore wasn't. - """ - message = ( - 'is_botocore() is deprecated and always returns True or raises an Exception, ' - 'so it cannot be used for checking if boto is available instead of botocore. ' - 'You can use scrapy.utils.boto.is_botocore_available() to check if botocore ' - 'is available.' - ) - warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) - try: - import botocore # noqa: F401 - return True - except ImportError: - raise NotConfigured('missing botocore library') def is_botocore_available(): diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 76156a4b8..0810e1f1d 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -2,17 +2,6 @@ import struct from gzip import GzipFile from io import BytesIO -from scrapy.utils.decorators import deprecated - - -# - GzipFile's read() has issues returning leftover uncompressed data when -# input is corrupted -# - read1(), which fetches data before raising EOFError on next call -# works here -@deprecated('GzipFile.read1') -def read1(gzf, size=-1): - return gzf.read1(size) - def gunzip(data): """Gunzip the given data and return as much data as possible. diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 8ce030d9d..0d9fdbf23 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,20 +1,16 @@ """ This module contains essential stuff that should've come with Python itself ;) """ -import errno import gc import inspect import re import sys -import warnings import weakref from functools import partial, wraps from itertools import chain from typing import AsyncGenerator, AsyncIterable, Iterable, Union -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator -from scrapy.utils.decorators import deprecated def flatten(x): @@ -112,12 +108,6 @@ def to_bytes(text, encoding=None, errors='strict'): return text.encode(encoding, errors) -@deprecated('to_unicode') -def to_native_str(text, encoding=None, errors='strict'): - """ Return str representation of ``text``. """ - return to_unicode(text, encoding, errors) - - def re_rsearch(pattern, text, chunk_size=1024): """ This function does a reverse search in a text using a regular expression @@ -263,30 +253,6 @@ def equal_attributes(obj1, obj2, attributes): return True -class WeakKeyCache: - - def __init__(self, default_factory): - warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2) - self.default_factory = default_factory - self._weakdict = weakref.WeakKeyDictionary() - - def __getitem__(self, key): - if key not in self._weakdict: - self._weakdict[key] = self.default_factory(key) - return self._weakdict[key] - - -@deprecated -def retry_on_eintr(function, *args, **kw): - """Run a function and retry it while getting EINTR errors""" - while True: - try: - return function(*args, **kw) - except IOError as e: - if e.errno != errno.EINTR: - raise - - def without_none_values(iterable): """Return a copy of ``iterable`` with all ``None`` entries removed. @@ -337,10 +303,6 @@ class MutableChain(Iterable): def __next__(self): return next(self.data) - @deprecated("scrapy.utils.python.MutableChain.__next__") - def next(self): - return self.__next__() - async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator: for it in iterables: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index b1a8fdc04..403e4f8fe 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,18 +1,14 @@ import functools -import gc import operator import platform -from itertools import count -from warnings import catch_warnings, filterwarnings from twisted.trial import unittest -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - WeakKeyCache, get_func_args, to_bytes, to_unicode, + get_func_args, to_bytes, to_unicode, without_none_values, MutableChain, MutableAsyncChain) @@ -27,12 +23,7 @@ class MutableChainTest(unittest.TestCase): m.extend([9, 10], (11, 12)) self.assertEqual(next(m), 0) self.assertEqual(m.__next__(), 1) - with catch_warnings(record=True) as warnings: - self.assertEqual(m.next(), 2) - self.assertEqual(len(warnings), 1) - self.assertIn('scrapy.utils.python.MutableChain.__next__', - str(warnings[0].message)) - self.assertEqual(list(m), list(range(3, 13))) + self.assertEqual(list(m), list(range(2, 13))) class MutableAsyncChainTest(unittest.TestCase): @@ -209,27 +200,6 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) - def test_weakkeycache(self): - class _Weakme: - pass - - _values = count() - - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - wk = WeakKeyCache(lambda k: next(_values)) - - k = _Weakme() - v = wk[k] - self.assertEqual(v, wk[k]) - self.assertNotEqual(v, wk[_Weakme()]) - self.assertEqual(v, wk[k]) - del k - for _ in range(100): - if wk._weakdict: - gc.collect() - self.assertFalse(len(wk._weakdict)) - def test_get_func_args(self): def f1(a, b, c): pass From 1a6408c3faadbd2b8b7622b8aee230b112620fad Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Mon, 21 Nov 2022 07:30:20 -0300 Subject: [PATCH 0685/2083] Remove `FilteringLinkExtractor` --- scrapy/linkextractors/__init__.py | 89 ------------------------------- scrapy/linkextractors/lxmlhtml.py | 83 ++++++++++++++++++++++------ tests/test_linkextractors.py | 32 ----------- 3 files changed, 66 insertions(+), 138 deletions(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 08a6ca1e8..b3b1eea55 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -6,18 +6,6 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ import re -from urllib.parse import urlparse -from warnings import warn - -from parsel.csstranslator import HTMLTranslator -from w3lib.url import canonicalize_url - -from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.misc import arg_to_iter -from scrapy.utils.url import ( - url_is_from_any_domain, url_has_any_extension, -) - # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ @@ -55,82 +43,5 @@ def _is_valid_url(url): return url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'} -class FilteringLinkExtractor: - - _csstranslator = HTMLTranslator() - - def __new__(cls, *args, **kwargs): - from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor - if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor): - warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' - 'please use scrapy.linkextractors.LinkExtractor instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls) - - def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, - restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): - - self.link_extractor = link_extractor - - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(deny)] - - self.allow_domains = set(arg_to_iter(allow_domains)) - self.deny_domains = set(arg_to_iter(deny_domains)) - - self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) - self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath, - arg_to_iter(restrict_css))) - - self.canonicalize = canonicalize - if deny_extensions is None: - deny_extensions = IGNORED_EXTENSIONS - self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} - self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(restrict_text)] - - def _link_allowed(self, link): - if not _is_valid_url(link.url): - return False - if self.allow_res and not _matches(link.url, self.allow_res): - return False - if self.deny_res and _matches(link.url, self.deny_res): - return False - parsed_url = urlparse(link.url) - if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains): - return False - if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains): - return False - if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): - return False - if self.restrict_text and not _matches(link.text, self.restrict_text): - return False - return True - - def matches(self, url): - - if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): - return False - if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): - return False - - allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True] - denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] - return any(allowed) and not any(denied) - - def _process_links(self, links): - links = [x for x in links if self._link_allowed(x)] - if self.canonicalize: - for link in links: - link.url = canonicalize_url(link.url) - links = self.link_extractor._process_links(links) - return links - - def _extract_links(self, *args, **kwargs): - return self.link_extractor._extract_links(*args, **kwargs) - - # Top-level imports from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index b5d2585a8..55639f504 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -3,18 +3,20 @@ Link extractor based on lxml.html """ import operator from functools import partial -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse import lxml.etree as etree +from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link -from scrapy.linkextractors import FilteringLinkExtractor +from scrapy.linkextractors import (IGNORED_EXTENSIONS, _is_valid_url, _matches, + _re_type, re) from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url - +from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain # from lxml/src/lxml/html/__init__.py XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" @@ -98,7 +100,8 @@ class LxmlParserLinkExtractor: return links -class LxmlLinkExtractor(FilteringLinkExtractor): +class LxmlLinkExtractor: + _csstranslator = HTMLTranslator() def __init__( self, @@ -118,7 +121,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor): restrict_text=None, ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) - lx = LxmlParserLinkExtractor( + self.link_extractor = LxmlParserLinkExtractor( tag=partial(operator.contains, tags), attr=partial(operator.contains, attrs), unique=unique, @@ -126,18 +129,64 @@ class LxmlLinkExtractor(FilteringLinkExtractor): strip=strip, canonicalized=canonicalize ) - super().__init__( - link_extractor=lx, - allow=allow, - deny=deny, - allow_domains=allow_domains, - deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, - restrict_css=restrict_css, - canonicalize=canonicalize, - deny_extensions=deny_extensions, - restrict_text=restrict_text, - ) + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(allow)] + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(deny)] + + self.allow_domains = set(arg_to_iter(allow_domains)) + self.deny_domains = set(arg_to_iter(deny_domains)) + + self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath, + arg_to_iter(restrict_css))) + + if deny_extensions is None: + deny_extensions = IGNORED_EXTENSIONS + self.canonicalize = canonicalize + self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} + self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(restrict_text)] + + def _link_allowed(self, link): + if not _is_valid_url(link.url): + return False + if self.allow_res and not _matches(link.url, self.allow_res): + return False + if self.deny_res and _matches(link.url, self.deny_res): + return False + parsed_url = urlparse(link.url) + if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains): + return False + if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains): + return False + if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): + return False + if self.restrict_text and not _matches(link.text, self.restrict_text): + return False + return True + + def matches(self, url): + + if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): + return False + if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): + return False + + allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True] + denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] + return any(allowed) and not any(denied) + + def _process_links(self, links): + links = [x for x in links if self._link_allowed(x)] + if self.canonicalize: + for link in links: + link.url = canonicalize_url(link.url) + links = self.link_extractor._process_links(links) + return links + + def _extract_links(self, *args, **kwargs): + return self.link_extractor._extract_links(*args, **kwargs) def extract_links(self, response): """Returns a list of :class:`~scrapy.link.Link` objects from the diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 6f133d77a..e28dc9bdb 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,12 +1,9 @@ import pickle import re import unittest -from warnings import catch_warnings -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link -from scrapy.linkextractors import FilteringLinkExtractor from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor from tests import get_testdata @@ -517,32 +514,3 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): def test_restrict_xpaths_with_html_entities(self): super().test_restrict_xpaths_with_html_entities() - - def test_filteringlinkextractor_deprecation_warning(self): - """Make sure the FilteringLinkExtractor deprecation warning is not - issued for LxmlLinkExtractor""" - with catch_warnings(record=True) as warnings: - LxmlLinkExtractor() - self.assertEqual(len(warnings), 0) - - class SubclassedLxmlLinkExtractor(LxmlLinkExtractor): - pass - - SubclassedLxmlLinkExtractor() - self.assertEqual(len(warnings), 0) - - -class FilteringLinkExtractorTest(unittest.TestCase): - - def test_deprecation_warning(self): - args = [None] * 10 - with catch_warnings(record=True) as warnings: - FilteringLinkExtractor(*args) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with catch_warnings(record=True) as warnings: - class SubclassedFilteringLinkExtractor(FilteringLinkExtractor): - pass - SubclassedFilteringLinkExtractor(*args) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) From c04ccbceb91b99976b674f6a57e15ed5ad5b7565 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 21 Nov 2022 15:49:33 +0100 Subject: [PATCH 0686/2083] doc: add instructions to debug Scrapy spiders in Visual Studio Code --- docs/topics/debug.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 4d452b4df..edbcaf432 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -150,3 +150,31 @@ available in all future runs should they be necessary again:: For more information, check the :ref:`topics-logging` section. .. _base tag: https://www.w3schools.com/tags/tag_base.asp + +Visual Studio Code +================== + +.. highlight:: json + +To debug spiders with Visual Studio Code you can use the following ``launch.json``:: + + { + "version": "0.1.0", + "configurations": [ + { + "name": "Python: Launch Scrapy Spider", + "type": "python", + "request": "launch", + "module": "scrapy", + "args": [ + "runspider", + "${file}" + ], + "console": "integratedTerminal" + } + ] + } + + +Also, make sure you enable "User Uncaught Exceptions", to catch exceptions in +your Scrapy spider. From 24a18e9af13a482c1dd6036046226cf57318477b Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Mon, 21 Nov 2022 17:41:06 -0500 Subject: [PATCH 0687/2083] Adds virtualsize property to _check_limit error log. --- scrapy/extensions/memusage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index f5081a7d7..aba0c8d7e 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -79,7 +79,7 @@ class MemoryUsage: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", - {'memusage': mem}, extra={'crawler': self.crawler}) + {'memusage': mem, 'virtualsize': self.get_virtual_size()}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " From 8f2adad7a7f2d858acd082afd6e9a6835bfddcd7 Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 00:48:24 -0500 Subject: [PATCH 0688/2083] Log self.get_virtual_size() on every call of _check_limit --- scrapy/extensions/memusage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index aba0c8d7e..c94899e0f 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -79,7 +79,7 @@ class MemoryUsage: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", - {'memusage': mem, 'virtualsize': self.get_virtual_size()}, extra={'crawler': self.crawler}) + {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " @@ -92,6 +92,8 @@ class MemoryUsage: self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded') else: self.crawler.stop() + else: + logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': self.get_virtual_size()}) def _check_warning(self): if self.warned: # warn only once From eb159c78f10f546e38e62a9230608269ac13acdd Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 03:36:00 -0500 Subject: [PATCH 0689/2083] Use variable and convert to megabytes. --- scrapy/extensions/memusage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index c94899e0f..bf2ee4e6d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -75,7 +75,8 @@ class MemoryUsage: self.crawler.stats.max_value('memusage/max', self.get_virtual_size()) def _check_limit(self): - if self.get_virtual_size() > self.limit: + current_mem_usage = self.get_virtual_size() + if current_mem_usage > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", @@ -93,7 +94,7 @@ class MemoryUsage: else: self.crawler.stop() else: - logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': self.get_virtual_size()}) + logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': current_mem_usage / 1024 / 1024}) def _check_warning(self): if self.warned: # warn only once From 8e0025f53dc724d986855b572b6237d0a96fd821 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 22 Nov 2022 09:38:45 -0300 Subject: [PATCH 0690/2083] Remove support for override settings with `SCRAPY_` environment variables --- scrapy/utils/project.py | 17 +++++------------ tests/test_utils_project.py | 18 +++++++----------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index c66af497e..fce198db4 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -6,7 +6,7 @@ from os.path import join, dirname, abspath, isabs, exists from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env from scrapy.settings import Settings -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured ENVVAR = 'SCRAPY_SETTINGS_MODULE' @@ -67,23 +67,16 @@ def get_project_settings(): if settings_module_path: settings.setmodule(settings_module_path, priority='project') - scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if - k.startswith('SCRAPY_')} valid_envvars = { 'CHECK', 'PROJECT', 'PYTHON_SHELL', 'SETTINGS_MODULE', } - setting_envvars = {k for k in scrapy_envvars if k not in valid_envvars} - if setting_envvars: - setting_envvar_list = ', '.join(sorted(setting_envvars)) - warnings.warn( - 'Use of environment variables prefixed with SCRAPY_ to override ' - 'settings is deprecated. The following environment variables are ' - f'currently defined: {setting_envvar_list}', - ScrapyDeprecationWarning - ) + + scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if + k.startswith('SCRAPY_') and k.replace('SCRAPY_', '') in valid_envvars} + settings.setdict(scrapy_envvars, priority='project') return settings diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 46452415a..e77ffa18b 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -5,9 +5,6 @@ import shutil import contextlib import warnings -from pytest import warns - -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.project import data_path, get_project_settings @@ -80,10 +77,10 @@ class GetProjectSettingsTestCase(unittest.TestCase): envvars = { 'SCRAPY_FOO': 'bar', } - with warns(ScrapyDeprecationWarning, match=': FOO') as record: - with set_env(**envvars): - get_project_settings() - assert len(record) == 1 + with set_env(**envvars): + settings = get_project_settings() + + assert settings.get("SCRAPY_FOO") is None def test_valid_and_invalid_envvars(self): value = 'tests.test_cmdline.settings' @@ -91,8 +88,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): 'SCRAPY_FOO': 'bar', 'SCRAPY_SETTINGS_MODULE': value, } - with warns(ScrapyDeprecationWarning, match=': FOO') as record: - with set_env(**envvars): - settings = get_project_settings() - assert len(record) == 1 + with set_env(**envvars): + settings = get_project_settings() assert settings.get('SETTINGS_MODULE') == value + assert settings.get('SCRAPY_FOO') is None From 1506479672ee54adc2d7d1ecffc0b224f2fcf7aa Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 22 Nov 2022 10:07:32 -0300 Subject: [PATCH 0691/2083] Remove deprecated test --- tests/test_cmdline/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 8233e0101..802f5c198 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -31,10 +31,6 @@ class CmdlineTest(unittest.TestCase): self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s', 'TEST1=override'), 'override') - def test_override_settings_using_envvar(self): - self.env['SCRAPY_TEST1'] = 'override' - self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override') - def test_profiling(self): path = tempfile.mkdtemp() filename = os.path.join(path, 'res.prof') From fc8968672a5cc699f0103aeb2da1ebea7ed3c235 Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 11:49:28 -0500 Subject: [PATCH 0692/2083] renamed variables to clarify that we are using peak memory and not current memory utilization. --- scrapy/extensions/memusage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index bf2ee4e6d..7bc6564e7 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -75,8 +75,8 @@ class MemoryUsage: self.crawler.stats.max_value('memusage/max', self.get_virtual_size()) def _check_limit(self): - current_mem_usage = self.get_virtual_size() - if current_mem_usage > self.limit: + peak_mem_usage = self.get_virtual_size() + if peak_mem_usage > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", @@ -94,7 +94,7 @@ class MemoryUsage: else: self.crawler.stop() else: - logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': current_mem_usage / 1024 / 1024}) + logger.info("Peak memory usage is %(virtualsize)dM", {'virtualsize': peak_mem_usage / 1024 / 1024}) def _check_warning(self): if self.warned: # warn only once From bdc0bca5b1aab15873b82f93a4bdf8fb3ce82824 Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 12:10:49 -0500 Subject: [PATCH 0693/2083] Replace M occurrences with MiB for accuracy. --- scrapy/extensions/memusage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 7bc6564e7..4fdf86479 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -79,12 +79,12 @@ class MemoryUsage: if peak_mem_usage > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 - logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", + logger.error("Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " - f"memory usage exceeded {mem}M at {socket.gethostname()}" + f"memory usage exceeded {mem}MiB at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) @@ -94,7 +94,7 @@ class MemoryUsage: else: self.crawler.stop() else: - logger.info("Peak memory usage is %(virtualsize)dM", {'virtualsize': peak_mem_usage / 1024 / 1024}) + logger.info("Peak memory usage is %(virtualsize)dMiB", {'virtualsize': peak_mem_usage / 1024 / 1024}) def _check_warning(self): if self.warned: # warn only once @@ -102,12 +102,12 @@ class MemoryUsage: if self.get_virtual_size() > self.warning: self.crawler.stats.set_value('memusage/warning_reached', 1) mem = self.warning / 1024 / 1024 - logger.warning("Memory usage reached %(memusage)dM", + logger.warning("Memory usage reached %(memusage)dMiB", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} warning: " - f"memory usage reached {mem}M at {socket.gethostname()}" + f"memory usage reached {mem}MiB at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/warning_notified', 1) From 1fdd0a70a0d56bc7829aa3028c191c062cd9f935 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 23 Nov 2022 12:16:48 +0500 Subject: [PATCH 0694/2083] Restore lost typing --- scrapy/core/http2/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 780191505..14bf4c5fe 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -151,7 +151,7 @@ class Stream: self._deferred_response = Deferred(_cancel) - def __repr__(self): + def __repr__(self) -> str: return f'Stream(id={self.stream_id!r})' @property From c3b1700774bd16623622963e98fd3ec759b8a88f Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 23 Nov 2022 12:17:30 +0500 Subject: [PATCH 0695/2083] Restore lost typing --- scrapy/http/request/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 305eef918..1ececaf1d 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -121,7 +121,7 @@ class Request(object_ref): def encoding(self) -> str: return self._encoding - def __repr__(self): + def __repr__(self) -> str: return f"<{self.method} {self.url}>" def copy(self) -> "Request": From e769532644e1176c7984dd513249285c7f64c7d0 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 23 Nov 2022 08:30:11 -0300 Subject: [PATCH 0696/2083] Remove `noconnect` deprecate code --- scrapy/core/downloader/handlers/http11.py | 17 ++--------------- scrapy/core/downloader/handlers/http2.py | 14 +------------- tests/test_downloader_handlers.py | 14 +------------- tests/test_downloader_handlers_http2.py | 15 --------------- 4 files changed, 4 insertions(+), 56 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 6b8a18f1a..f07f0780e 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -3,7 +3,6 @@ import ipaddress import logging import re -import warnings from contextlib import suppress from io import BytesIO from time import time @@ -22,7 +21,7 @@ from zope.interface import implementer from scrapy import signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse -from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload +from scrapy.exceptions import StopDownload from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.python import to_bytes, to_unicode @@ -279,17 +278,7 @@ class ScrapyAgent: proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) - omitConnectTunnel = b'noconnect' in proxyParams - if omitConnectTunnel: - warnings.warn( - "Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Zyte Smart Proxy Manager, it doesn't require " - "this mode anymore, so you should update scrapy-crawlera " - "to scrapy-zyte-smartproxy and remove '?noconnect' " - "from the Zyte Smart Proxy Manager URL.", - ScrapyDeprecationWarning, - ) - if scheme == b'https' and not omitConnectTunnel: + if scheme == b'https': proxyAuth = request.headers.get(b'Proxy-Authorization', None) proxyConf = (proxyHost, proxyPort, proxyAuth) return self._TunnelingAgent( @@ -302,8 +291,6 @@ class ScrapyAgent: ) else: proxyScheme = proxyScheme or b'http' - proxyHost = to_bytes(proxyHost, encoding='ascii') - proxyPort = to_bytes(str(proxyPort), encoding='ascii') proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) return self._ProxyAgent( reactor=reactor, diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 7bb88a193..3f1b36e92 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,4 +1,3 @@ -import warnings from time import time from typing import Optional, Type, TypeVar from urllib.parse import urldefrag @@ -69,19 +68,8 @@ class ScrapyH2Agent: if proxy: _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) scheme = _parse(request.url)[0] - proxy_host = proxy_host.decode() - omit_connect_tunnel = b'noconnect' in proxy_params - if omit_connect_tunnel: - warnings.warn( - "Using HTTPS proxies in the noconnect mode is not " - "supported by the downloader handler. If you use Zyte " - "Smart Proxy Manager, it doesn't require this mode " - "anymore, so you should update scrapy-crawlera to " - "scrapy-zyte-smartproxy and remove '?noconnect' from the " - "Zyte Smart Proxy Manager URL." - ) - if scheme == b'https' and not omit_connect_tunnel: + if scheme == b'https': # ToDo raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported') return self._ProxyAgent( diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 883960084..c69bd3da1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -24,7 +24,7 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured from scrapy.http import Headers, HtmlResponse, Request from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes @@ -757,18 +757,6 @@ class HttpProxyTestCase(unittest.TestCase): request = Request('http://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) - def test_download_with_proxy_https_noconnect(self): - def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'https://example.com') - - http_proxy = f'{self.getURL("")}?noconnect' - request = Request('https://example.com', meta={'proxy': http_proxy}) - with self.assertWarnsRegex(ScrapyDeprecationWarning, - r'Using HTTPS proxies in the noconnect mode is deprecated'): - return self.download_request(request, Spider('foo')).addCallback(_test) - def test_download_without_proxy(self): def _test(response): self.assertEqual(response.status, 200) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 3a9db3ee5..079267535 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -242,21 +242,6 @@ class Https2ProxyTestCase(Http11ProxyTestCase): def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" - def test_download_with_proxy_https_noconnect(self): - def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'/') - - http_proxy = f"{self.getURL('')}?noconnect" - request = Request('https://example.com', meta={'proxy': http_proxy}) - with self.assertWarnsRegex( - Warning, - r'Using HTTPS proxies in the noconnect mode is not supported by the ' - r'downloader handler.' - ): - return self.download_request(request, Spider('foo')).addCallback(_test) - @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): with self.assertRaises(NotImplementedError): From f6e9e6592a28a11517295847b177b14f41cb8a26 Mon Sep 17 00:00:00 2001 From: Hanzallah Burney Date: Wed, 23 Nov 2022 19:48:34 +0500 Subject: [PATCH 0697/2083] Cleanup settings._DictProxy and scrapy.telnet (#5730) --- scrapy/settings/__init__.py | 24 ------------------------ scrapy/utils/deprecate.py | 5 ++--- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b4e12ffdc..43ee433d1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -435,30 +435,6 @@ class BaseSettings(MutableMapping): p.text(pformat(self.copy_to_dict())) -class _DictProxy(MutableMapping): - - def __init__(self, settings, priority): - self.o = {} - self.settings = settings - self.priority = priority - - def __len__(self): - return len(self.o) - - def __getitem__(self, k): - return self.o[k] - - def __setitem__(self, k, v): - self.settings.set(k, v, priority=self.priority) - self.o[k] = v - - def __delitem__(self, k): - del self.o[k] - - def __iter__(self, k, v): - return iter(self.o) - - class Settings(BaseSettings): """ This object stores Scrapy settings for the configuration of internal diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index ae727464c..a0c83f9f1 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -2,6 +2,7 @@ import warnings import inspect +from typing import List, Tuple from scrapy.exceptions import ScrapyDeprecationWarning @@ -126,9 +127,7 @@ def _clspath(cls, forced=None): return f'{cls.__module__}.{cls.__name__}' -DEPRECATION_RULES = [ - ('scrapy.telnet.', 'scrapy.extensions.telnet.'), -] +DEPRECATION_RULES: List[Tuple[str, str]] = [] def update_classpath(path): From b6e98ce6b6c766ee735f45f25f78d957327f54e9 Mon Sep 17 00:00:00 2001 From: Hanzallah Burney Date: Thu, 24 Nov 2022 15:01:15 +0100 Subject: [PATCH 0698/2083] Remove unnecessary backwards compatibility comments (#5732) --- scrapy/utils/conf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 6404edda6..7a5f8f065 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -44,14 +44,12 @@ def build_component_list(compdict, custom=None, convert=update_classpath): raise ValueError(f'Invalid value {value} for component {name}, ' 'please provide a real number or None instead') - # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): _check_components(custom) return type(custom)(convert(c) for c in custom) if custom is not None: compdict.update(custom) - # END Backward compatibility _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) From f85c3f3d68b03b12c0af3b3aa09ab5faad19bc37 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 25 Nov 2022 18:46:14 +0600 Subject: [PATCH 0699/2083] Add a comment about `tmpname + '^'`. --- tests/test_downloader_handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c69bd3da1..8835267fe 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -109,6 +109,7 @@ class FileTestCase(unittest.TestCase): def setUp(self): self.tmpname = self.mktemp() + # add a special char to check that they are handled correctly with open(self.tmpname + '^', 'w') as f: f.write('0123456789') handler = create_instance(FileDownloadHandler, None, get_crawler()) From f9a29f03d9a0eb9173a91f225177b7bee7d382c9 Mon Sep 17 00:00:00 2001 From: Mark Mayo Date: Sun, 27 Nov 2022 23:00:13 +1300 Subject: [PATCH 0700/2083] Address some issues reported by Pylint (#5677) Co-authored-by: Felipe Boff Nunes <51033921+felipeboffnunes@users.noreply.github.com> Co-authored-by: Andrey Rahmatullin --- conftest.py | 2 +- docs/_ext/scrapydocs.py | 2 +- docs/conftest.py | 8 +++-- docs/utils/linkfix.py | 9 +++--- pylintrc | 27 +---------------- scrapy/commands/__init__.py | 2 +- scrapy/commands/genspider.py | 2 +- scrapy/commands/parse.py | 2 -- scrapy/core/downloader/handlers/http10.py | 3 +- scrapy/core/downloader/handlers/http11.py | 22 +++++++------- scrapy/core/downloader/tls.py | 1 - scrapy/core/downloader/webclient.py | 4 +-- scrapy/core/http2/agent.py | 2 +- scrapy/core/http2/protocol.py | 2 +- scrapy/core/scheduler.py | 4 +-- scrapy/core/scraper.py | 30 +++++++++---------- scrapy/core/spidermw.py | 21 ++++++------- scrapy/downloadermiddlewares/cookies.py | 3 +- .../downloadermiddlewares/httpcompression.py | 1 - scrapy/downloadermiddlewares/redirect.py | 8 ++--- scrapy/downloadermiddlewares/retry.py | 17 +++++------ scrapy/downloadermiddlewares/robotstxt.py | 3 +- scrapy/downloadermiddlewares/stats.py | 4 +-- scrapy/dupefilters.py | 2 +- scrapy/exporters.py | 4 +-- scrapy/extensions/httpcache.py | 14 ++++----- scrapy/extensions/telnet.py | 2 +- scrapy/extensions/throttle.py | 2 +- scrapy/http/headers.py | 7 ++--- scrapy/http/request/__init__.py | 2 +- scrapy/http/request/form.py | 4 +-- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/pipelines/__init__.py | 2 +- scrapy/pipelines/files.py | 6 ++-- scrapy/responsetypes.py | 10 +++---- scrapy/robotstxt.py | 1 - scrapy/settings/__init__.py | 6 ++-- scrapy/spidermiddlewares/offsite.py | 2 +- scrapy/spidermiddlewares/referer.py | 5 ++-- scrapy/spiders/crawl.py | 2 +- scrapy/spiders/sitemap.py | 4 +-- scrapy/utils/conf.py | 14 ++++----- scrapy/utils/defer.py | 22 ++++++-------- scrapy/utils/display.py | 2 +- scrapy/utils/ftp.py | 1 - scrapy/utils/iterators.py | 13 ++++---- scrapy/utils/misc.py | 15 ++++------ scrapy/utils/python.py | 5 ++-- scrapy/utils/response.py | 6 ++-- scrapy/utils/serialize.py | 19 ++++++------ scrapy/utils/spider.py | 5 ++-- setup.py | 2 +- tests/CrawlerProcess/reactor_default.py | 3 +- .../reactor_default_twisted_reactor_select.py | 3 +- tests/CrawlerProcess/reactor_select.py | 4 ++- ..._select_subclass_twisted_reactor_select.py | 4 +-- .../reactor_select_twisted_reactor_select.py | 4 ++- tests/CrawlerRunner/ip_address.py | 8 ++--- tests/mockserver.py | 3 +- tests/test_command_check.py | 4 +-- tests/test_command_parse.py | 8 ++--- tests/test_commands.py | 28 ++++++++--------- tests/test_crawler.py | 5 ++-- tests/test_downloader_handlers.py | 5 ++-- tests/test_downloader_handlers_http2.py | 6 ++-- tests/test_downloadermiddleware_cookies.py | 2 +- ...test_downloadermiddleware_decompression.py | 2 +- tests/test_downloadermiddleware_httpcache.py | 7 ++--- ...st_downloadermiddleware_httpcompression.py | 3 +- tests/test_exporters.py | 6 ++-- tests/test_http2_client_protocol.py | 2 +- tests/test_http_request.py | 2 +- tests/test_http_response.py | 2 +- tests/test_loader_deprecated.py | 5 ++-- tests/test_pipeline_media.py | 4 +-- tests/test_request_attribute_binding.py | 4 +-- tests/test_request_cb_kwargs.py | 2 +- tests/test_scheduler_base.py | 2 -- tests/test_spider.py | 2 +- tests/test_spiderloader/__init__.py | 2 +- tests/test_spidermiddleware_httperror.py | 2 +- tests/test_spidermiddleware_output_chain.py | 1 - tests/test_spidermiddleware_referer.py | 4 +-- tests/test_squeues_request.py | 1 - tests/test_utils_defer.py | 5 ++-- tests/test_utils_deprecate.py | 1 + tests/test_webclient.py | 2 +- 87 files changed, 219 insertions(+), 289 deletions(-) mode change 100755 => 100644 docs/utils/linkfix.py diff --git a/conftest.py b/conftest.py index 7c1da3556..2a5d55083 100644 --- a/conftest.py +++ b/conftest.py @@ -21,7 +21,7 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with Path('tests/ignores.txt').open() as reader: +with Path('tests/ignores.txt').open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != '#': diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index f0f382da3..337604cf1 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,8 +1,8 @@ +from operator import itemgetter from docutils.parsers.rst.roles import set_classes from docutils import nodes from docutils.parsers.rst import Directive from sphinx.util.nodes import make_refnode -from operator import itemgetter class settingslist_node(nodes.General, nodes.Element): diff --git a/docs/conftest.py b/docs/conftest.py index 24a72a4b6..a6dacd265 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -1,15 +1,17 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE from pathlib import Path -from scrapy.http.response.html import HtmlResponse from sybil import Sybil +from sybil.parsers.doctest import DocTestParser +from sybil.parsers.skip import skip + try: # >2.0.1 from sybil.parsers.codeblock import PythonCodeBlockParser except ImportError: from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser -from sybil.parsers.doctest import DocTestParser -from sybil.parsers.skip import skip + +from scrapy.http.response.html import HtmlResponse def load_response(url: str, filename: str) -> HtmlResponse: diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py old mode 100755 new mode 100644 index 7a0c5288c..a0d9a1cbd --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -13,6 +13,7 @@ Author: dufferzafar """ import re +import sys from pathlib import Path @@ -28,11 +29,11 @@ def main(): # Read lines from the linkcheck output file try: - with Path("build/linkcheck/output.txt").open() as out: + with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out: output_lines = out.readlines() except IOError: print("linkcheck output not found; please run linkcheck first.") - exit(1) + sys.exit(1) # For every line, fix the respective file for line in output_lines: @@ -52,12 +53,12 @@ def main(): # Update the previous file if _filename: - Path(_filename).write_text(_contents) + Path(_filename).write_text(_contents, encoding="utf-8") _filename = newfilename # Read the new file to memory - _contents = Path(_filename).read_text() + _contents = Path(_filename).read_text(encoding="utf-8") _contents = _contents.replace(match.group(3), match.group(4)) else: diff --git a/pylintrc b/pylintrc index 18819feba..0a2276fb8 100644 --- a/pylintrc +++ b/pylintrc @@ -9,27 +9,19 @@ disable=abstract-method, arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, - bad-indentation, bad-mcs-classmethod-argument, - bad-super-call, bare-except, - blacklisted-name, broad-except, c-extension-no-member, catching-non-exception, cell-var-from-loop, comparison-with-callable, - consider-iterating-dictionary, consider-using-dict-items, - consider-using-from-import, consider-using-in, - consider-using-set-comprehension, - consider-using-sys-exit, consider-using-with, cyclic-import, dangerous-default-value, - deprecated-method, - deprecated-module, + disallowed-name, duplicate-code, # https://github.com/PyCQA/pylint/issues/214 eval-used, expression-not-assigned, @@ -52,18 +44,12 @@ disable=abstract-method, lost-exception, method-hidden, missing-docstring, - missing-final-newline, - multiple-imports, - multiple-statements, - no-else-continue, no-else-raise, - no-else-return, no-member, no-method-argument, no-name-in-module, no-self-argument, no-value-for-parameter, - not-an-iterable, not-callable, pointless-statement, pointless-string-statement, @@ -74,10 +60,7 @@ disable=abstract-method, redefined-outer-name, reimported, signature-differs, - singleton-comparison, super-init-not-called, - super-with-arguments, - superfluous-parens, too-few-public-methods, too-many-ancestors, too-many-arguments, @@ -89,31 +72,23 @@ disable=abstract-method, too-many-locals, too-many-public-methods, too-many-return-statements, - trailing-newlines, - trailing-whitespace, unbalanced-tuple-unpacking, undefined-variable, undefined-loop-variable, unexpected-special-method-signature, - ungrouped-imports, - unidiomatic-typecheck, unnecessary-comprehension, unnecessary-dunder-call, - unnecessary-lambda, unnecessary-pass, unreachable, - unspecified-encoding, unsubscriptable-object, unused-argument, unused-import, unused-private-member, unused-variable, unused-wildcard-import, - use-implicit-booleaness-not-comparison, used-before-assignment, useless-object-inheritance, # Required for Python 2 support useless-return, useless-super-delegation, wildcard-import, - wrong-import-order, wrong-import-position diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index c4d4bebb2..d0fb4efd8 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -95,7 +95,7 @@ class ScrapyCommand: self.settings.set('LOG_ENABLED', False, priority='cmdline') if opts.pidfile: - Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep) + Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep, encoding="utf-8") if opts.pdb: failure.startDebugMode() diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index ed9660252..b7c2f85fb 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -63,7 +63,7 @@ class Command(ScrapyCommand): if opts.dump: template_file = self._find_template(opts.dump) if template_file: - print(template_file.read_text()) + print(template_file.read_text(encoding="utf-8")) return if len(args) != 2: raise UsageError() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index d93ab2ac5..c2df7f8a5 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -6,14 +6,12 @@ from itemadapter import is_item, ItemAdapter from w3lib.url import is_url from twisted.internet.defer import maybeDeferred - from scrapy.commands import BaseRunSpiderCommand from scrapy.http import Request from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output, spidercls_for_request from scrapy.exceptions import UsageError - logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index c0146a0a6..a75532d2a 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -33,5 +33,4 @@ class HTTP10DownloadHandler: crawler=self._crawler, ) return reactor.connectSSL(host, port, factory, client_context_factory) - else: - return reactor.connectTCP(host, port, factory) + return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index f07f0780e..1c98e60e1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -26,7 +26,6 @@ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.python import to_bytes, to_unicode - logger = logging.getLogger(__name__) @@ -289,16 +288,15 @@ class ScrapyAgent: bindAddress=bindaddress, pool=self._pool, ) - else: - proxyScheme = proxyScheme or b'http' - proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) - return self._ProxyAgent( - reactor=reactor, - proxyURI=to_bytes(proxyURI, encoding='ascii'), - connectTimeout=timeout, - bindAddress=bindaddress, - pool=self._pool, - ) + proxyScheme = proxyScheme or b'http' + proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) + return self._ProxyAgent( + reactor=reactor, + proxyURI=to_bytes(proxyURI, encoding='ascii'), + connectTimeout=timeout, + bindAddress=bindaddress, + pool=self._pool, + ) return self._Agent( reactor=reactor, @@ -567,7 +565,7 @@ class _ResponseReader(protocol.Protocol): self._finish_response(flags=["dataloss"]) return - elif not self._fail_on_dataloss_warned: + if not self._fail_on_dataloss_warned: logger.warning("Got data loss in %s. If you want to process broken " "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" " -- This message won't be shown in further requests", diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 698a1c85c..7d67a426f 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -7,7 +7,6 @@ from twisted.internet.ssl import AcceptableCiphers from scrapy.utils.ssl import x509name_to_string, get_temp_key_info - logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 7d048c1e4..255ca62e6 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,8 +1,8 @@ import re from time import time from urllib.parse import urlparse, urlunparse, urldefrag - from twisted.web.http import HTTPClient + from twisted.internet import defer from twisted.internet.protocol import ClientFactory @@ -185,7 +185,7 @@ class ScrapyHTTPClientFactory(ClientFactory): @param version: The HTTP version. @type version: L{bytes} @param status: The HTTP status code, an integer represented as a - bytestring. + bytestring. @type status: L{bytes} @param message: The HTTP status message. @type message: L{bytes} diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index f7b0c3f99..c6b357be3 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -140,7 +140,7 @@ class ScrapyProxyH2Agent(H2Agent): connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None, ) -> None: - super(ScrapyProxyH2Agent, self).__init__( + super().__init__( reactor=reactor, pool=pool, context_factory=context_factory, diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 1d150b7ce..6047f9ca8 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -42,7 +42,7 @@ class InvalidNegotiatedProtocol(H2Error): self.negotiated_protocol = negotiated_protocol def __str__(self) -> str: - return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}") + return f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}" class RemoteTerminatedConnection(H2Error): diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 366449f51..6dd5d51a8 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -333,9 +333,9 @@ class Scheduler(BaseScheduler): path = Path(dqdir, 'active.json') if not path.exists(): return [] - with path.open() as f: + with path.open(encoding="utf-8") as f: return json.load(f) def _write_dqs_state(self, dqdir: str, state: list) -> None: - with Path(dqdir, 'active.json').open('w') as f: + with Path(dqdir, 'active.json').open('w', encoding="utf-8") as f: json.dump(state, f) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index e1fdd8d13..7225e0743 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -152,9 +152,9 @@ class Scraper: """ if isinstance(result, Response): return self.spidermw.scrape_response(self.call_spider, result, request, spider) - else: # result is a Failure - dfd = self.call_spider(result, request, spider) - return dfd.addErrback(self._log_download_errors, result, request, spider) + # else result is a Failure + dfd = self.call_spider(result, request, spider) + return dfd.addErrback(self._log_download_errors, result, request, spider) def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: if isinstance(result, Response): @@ -276,17 +276,15 @@ class Scraper: return self.signals.send_catch_log_deferred( signal=signals.item_dropped, item=item, response=response, spider=spider, exception=output.value) - else: - logkws = self.logformatter.item_error(item, ex, response, spider) - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}, - exc_info=failure_to_exc_info(output)) - return self.signals.send_catch_log_deferred( - signal=signals.item_error, item=item, response=response, - spider=spider, failure=output) - else: - logkws = self.logformatter.scraped(output, response, spider) - if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + logkws = self.logformatter.item_error(item, ex, response, spider) + logger.log(*logformatter_adapter(logkws), extra={'spider': spider}, + exc_info=failure_to_exc_info(output)) return self.signals.send_catch_log_deferred( - signal=signals.item_scraped, item=output, response=response, - spider=spider) + signal=signals.item_error, item=item, response=response, + spider=spider, failure=output) + logkws = self.logformatter.scraped(output, response, spider) + if logkws is not None: + logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + return self.signals.send_catch_log_deferred( + signal=signals.item_scraped, item=output, response=response, + spider=spider) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1aa02f29f..91e4b9cb4 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -121,11 +121,10 @@ class SpiderMiddlewareManager(MiddlewareManager): if dfd.called: # the result is available immediately if _process_spider_output didn't do downgrading return dfd.result - else: - # we forbid waiting here because otherwise we would need to return a deferred from - # _process_spider_exception too, which complicates the architecture - msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" - raise _InvalidOutput(msg) + # we forbid waiting here because otherwise we would need to return a deferred from + # _process_spider_exception too, which complicates the architecture + msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" + raise _InvalidOutput(msg) elif result is None: continue else: @@ -213,8 +212,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if last_result_is_async: return MutableAsyncChain(result, recovered) - else: - return MutableChain(result, recovered) # type: ignore[arg-type] + return MutableChain(result, recovered) # type: ignore[arg-type] async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] ) -> Union[MutableChain, MutableAsyncChain]: @@ -227,11 +225,10 @@ class SpiderMiddlewareManager(MiddlewareManager): result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result)) if isinstance(result, AsyncIterable): return MutableAsyncChain(result, recovered) - else: - if isinstance(recovered, AsyncIterable): - recovered_collected = await collect_asyncgen(recovered) - recovered = MutableChain(recovered_collected) - return MutableChain(result, recovered) # type: ignore[arg-type] + if isinstance(recovered, AsyncIterable): + recovered_collected = await collect_asyncgen(recovered) + recovered = MutableChain(recovered_collected) + return MutableChain(result, recovered) # type: ignore[arg-type] def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Deferred: diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index c592acb57..3cba269f1 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -9,7 +9,6 @@ from scrapy.http.cookies import CookieJar from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode - logger = logging.getLogger(__name__) @@ -129,7 +128,7 @@ class CookiesMiddleware: """ if not request.cookies: return [] - elif isinstance(request.cookies, dict): + if isinstance(request.cookies, dict): cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) else: cookies = request.cookies diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 4e7feeeaf..e57f6fcf8 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -8,7 +8,6 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip - ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] try: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index c8c84ffb2..fba825947 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -8,7 +8,6 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured - logger = logging.getLogger(__name__) @@ -56,10 +55,9 @@ class BaseRedirectMiddleware: {'reason': reason, 'redirected': redirected, 'request': request}, extra={'spider': spider}) return redirected - else: - logger.debug("Discarding %(request)s: max redirections reached", - {'request': request}, extra={'spider': spider}) - raise IgnoreRequest("max redirections reached") + logger.debug("Discarding %(request)s: max redirections reached", + {'request': request}, extra={'spider': spider}) + raise IgnoreRequest("max redirections reached") def _redirect_request_using_get(self, request, redirect_url): redirect_request = _build_redirect_request( diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index c6cc7c56d..2de610e86 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -113,15 +113,14 @@ def get_retry_request( stats.inc_value(f'{stats_base_key}/count') stats.inc_value(f'{stats_base_key}/reason_count/{reason}') return new_request - else: - stats.inc_value(f'{stats_base_key}/max_reached') - logger.error( - "Gave up retrying %(request)s (failed %(retry_times)d times): " - "%(reason)s", - {'request': request, 'retry_times': retry_times, 'reason': reason}, - extra={'spider': spider}, - ) - return None + stats.inc_value(f'{stats_base_key}/max_reached') + logger.error( + "Gave up retrying %(request)s (failed %(retry_times)d times): " + "%(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider}, + ) + return None class RetryMiddleware: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index e66bf177e..7bd39aa43 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -81,8 +81,7 @@ class RobotsTxtMiddleware: return result self._parsers[netloc].addCallback(cb) return d - else: - return self._parsers[netloc] + return self._parsers[netloc] def _logerror(self, failure, request, spider): if failure.type is not IgnoreRequest: diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 25fb1ed9d..2bb022c16 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,9 +1,9 @@ +from twisted.web import http + from scrapy.exceptions import NotConfigured from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr -from twisted.web import http - def get_header_size(headers): size = 0 diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 2b8b09614..6dca11d31 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -55,7 +55,7 @@ class RFPDupeFilter(BaseDupeFilter): self.debug = debug self.logger = logging.getLogger(__name__) if path: - self.file = Path(path, 'requests.seen').open('a+') + self.file = Path(path, 'requests.seen').open('a+', encoding="utf-8") self.file.seek(0) self.fingerprints.update(x.rstrip() for x in self.file) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 76cbe4d4b..2135c3bbc 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -334,9 +334,9 @@ class PythonItemExporter(BaseItemExporter): def _serialize_value(self, value): if isinstance(value, Item): return self.export_item(value) - elif is_item(value): + if is_item(value): return dict(self._serialize_item(value)) - elif is_listlike(value): + if is_listlike(value): return [self._serialize_value(v) for v in value] encode_func = to_bytes if self.binary else to_unicode if isinstance(value, (str, bytes)): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 58c4e644e..4e76fe5e5 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -17,7 +17,6 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode - logger = logging.getLogger(__name__) @@ -81,25 +80,24 @@ class RFC2616Policy: if b'no-store' in cc: return False # Never cache 304 (Not Modified) responses - elif response.status == 304: + if response.status == 304: return False # Cache unconditionally if configured to do so - elif self.always_store: + if self.always_store: return True # Any hint on response expiration is good - elif b'max-age' in cc or b'Expires' in response.headers: + if b'max-age' in cc or b'Expires' in response.headers: return True # Firefox fallbacks this statuses to one year expiration if none is set - elif response.status in (300, 301, 308): + if response.status in (300, 301, 308): return True # Other statuses without expiration requires at least one validator - elif response.status in (200, 203, 401): + if response.status in (200, 203, 401): return b'Last-Modified' in response.headers or b'ETag' in response.headers # Any other is probably not eligible for caching # Makes no sense to cache responses that does not contain expiration # info and can not be revalidated - else: - return False + return False def is_cached_response_fresh(self, cachedresponse, request): cc = self._parse_cachecontrol(cachedresponse) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 1663604e7..054350e4a 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -11,6 +11,7 @@ import binascii import os from twisted.internet import protocol + try: from twisted.conch import manhole, telnet from twisted.conch.insults import insults @@ -26,7 +27,6 @@ from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp from scrapy.utils.decorators import defers - logger = logging.getLogger(__name__) # signal to update telnet variables diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 56e5ad2d2..eb21e426e 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -56,7 +56,7 @@ class AutoThrottle: { 'slot': key, 'concurrency': conc, 'delay': slot.delay * 1000, 'delaydiff': diff * 1000, - 'latency': latency * 1000, 'size': size + 'latency': latency * 1000, 'size': size, }, extra={'spider': spider} ) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 9c03fe54f..2e0020890 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -37,12 +37,11 @@ class Headers(CaselessDict): def _tobytes(self, x): if isinstance(x, bytes): return x - elif isinstance(x, str): + if isinstance(x, str): return x.encode(self.encoding) - elif isinstance(x, int): + if isinstance(x, int): return str(x).encode(self.encoding) - else: - raise TypeError(f'Unsupported value type: {type(x)}') + raise TypeError(f'Unsupported value type: {type(x)}') def __getitem__(self, key): try: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 1ececaf1d..a1001fc4a 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -185,7 +185,7 @@ class Request(object_ref): } for attr in self.attributes: d.setdefault(attr, getattr(self, attr)) - if type(self) is not Request: + if type(self) is not Request: # pylint: disable=unidiomatic-typecheck d["_class"] = self.__module__ + '.' + self.__class__.__name__ return d diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 0c947565a..8e0a7fae2 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -190,7 +190,7 @@ def _select_value(ele: SelectElement, n: str, v: str): # And for select tags without options o = ele.value_options return (n, o[0]) if o else (None, None) - elif v is not None and multiple: + if v is not None and multiple: # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') @@ -236,7 +236,7 @@ def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tup el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') - elif len(el) > 1: + if len(el) > 1: raise ValueError(f"Multiple elements found ({el!r}) matching the " f"criteria in clickdata: {clickdata!r}") else: diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 55639f504..1e6ab984a 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -5,7 +5,7 @@ import operator from functools import partial from urllib.parse import urljoin, urlparse -import lxml.etree as etree +from lxml import etree from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index b5725a8ee..c42dd423e 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -18,7 +18,7 @@ class ItemPipelineManager(MiddlewareManager): return build_component_list(settings.getwithbase('ITEM_PIPELINES')) def _add_middleware(self, pipe): - super(ItemPipelineManager, self)._add_middleware(pipe) + super()._add_middleware(pipe) if hasattr(pipe, 'process_item'): self.methods['process_item'].append(deferred_f_from_coro_f(pipe.process_item)) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4e0211d1b..51aedafe8 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -221,16 +221,14 @@ class GCSFilesStore: checksum = blob.md5_hash last_modified = time.mktime(blob.updated.timetuple()) return {'checksum': checksum, 'last_modified': last_modified} - else: - return {} + return {} blob_path = self._get_blob_path(path) return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess) def _get_content_type(self, headers): if headers and 'Content-Type' in headers: return headers['Content-Type'] - else: - return 'application/octet-stream' + return 'application/octet-stream' def _get_blob_path(self, path): return self.prefix + path diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 3efd4d2fd..a34d7d25f 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -42,11 +42,10 @@ class ResponseTypes: """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response - elif mimetype in self.classes: + if mimetype in self.classes: return self.classes[mimetype] - else: - basetype = f"{mimetype.split('/')[0]}/*" - return self.classes.get(basetype, Response) + basetype = f"{mimetype.split('/')[0]}/*" + return self.classes.get(basetype, Response) def from_content_type(self, content_type, content_encoding=None): """Return the most appropriate Response class from an HTTP Content-Type @@ -83,8 +82,7 @@ class ResponseTypes: mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) - else: - return Response + return Response def from_body(self, body): """Try to guess the appropriate response based on the body content. diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index f8649e56b..1d40f0484 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -4,7 +4,6 @@ from abc import ABCMeta, abstractmethod from scrapy.utils.python import to_unicode - logger = logging.getLogger(__name__) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 43ee433d1..b540e6182 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -24,8 +24,7 @@ def get_settings_priority(priority): """ if isinstance(priority, str): return SETTINGS_PRIORITIES[priority] - else: - return priority + return priority class SettingsAttribute: @@ -260,8 +259,7 @@ class BaseSettings(MutableMapping): """ if len(self) > 0: return max(self.getpriority(name) for name in self) - else: - return get_settings_priority('default') + return get_settings_priority('default') def __setitem__(self, name, value): self.set(name, value) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 448bc1367..98a186510 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -65,7 +65,7 @@ class OffsiteMiddleware: for domain in allowed_domains: if domain is None: continue - elif url_pattern.match(domain): + if url_pattern.match(domain): message = ("allowed_domains accepts only domains, not URLs. " f"Ignoring URL entry {domain} in allowed_domains.") warnings.warn(message, URLWarning) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 8027beb92..4a6c4de5e 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -189,8 +189,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) - else: - return origin + return origin class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -216,7 +215,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) - elif ( + if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) or not self.tls_protected(response_url) ): diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 2d9328633..b514ed9ce 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -26,7 +26,7 @@ def _identity_process_request(request, response): def _get_method(method, spider): if callable(method): return method - elif isinstance(method, str): + if isinstance(method, str): return getattr(spider, method, None) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 1f72e76b7..43d138753 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -69,7 +69,7 @@ class SitemapSpider(Spider): """ if isinstance(response, XmlResponse): return response.body - elif gzip_magic_number(response): + if gzip_magic_number(response): return gunzip(response.body) # actual gzipped sitemap files are decompressed above ; # if we are here (response body is not gzipped) @@ -80,7 +80,7 @@ class SitemapSpider(Spider): # without actually being a .xml.gz file in the first place, # merely XML gzip-compressed on the fly, # in other word, here, we have plain XML - elif response.url.endswith('.xml') or response.url.endswith('.xml.gz'): + if response.url.endswith('.xml') or response.url.endswith('.xml.gz'): return response.body diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 126ead020..8ea42ce75 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -35,9 +35,8 @@ def build_component_list(compdict, custom=None, convert=update_classpath): else: compbs.set(convert(k), v, priority=prio) return compbs - else: - _check_components(compdict) - return {convert(k): v for k, v in compdict.items()} + _check_components(compdict) + return {convert(k): v for k, v in compdict.items()} def _validate_values(compdict): """Fail if a value in the components dict is not a real number or None.""" @@ -181,11 +180,10 @@ def feed_process_params_from_cli(settings, output: List[str], output_format=None ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} - else: - raise UsageError( - 'The -t command-line option cannot be used if multiple output ' - 'URIs are specified' - ) + raise UsageError( + 'The -t command-line option cannot be used if multiple output ' + 'URIs are specified' + ) result: Dict[str, Dict[str, Any]] = {} for element in output: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 38aefd6d0..ddacfaa49 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -58,10 +58,9 @@ def defer_succeed(result) -> Deferred: def defer_result(result) -> Deferred: if isinstance(result, Deferred): return result - elif isinstance(result, failure.Failure): + if isinstance(result, failure.Failure): return defer_fail(result) - else: - return defer_succeed(result) + return defer_succeed(result) def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: @@ -267,10 +266,9 @@ def deferred_from_coro(o) -> Any: # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" return ensureDeferred(o) - else: - # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - event_loop = get_asyncio_event_loop_policy().get_event_loop() - return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) + # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor + event_loop = get_asyncio_event_loop_policy().get_event_loop() + return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) return o @@ -295,12 +293,11 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: if isinstance(result, Deferred): return result - elif asyncio.isfuture(result) or inspect.isawaitable(result): + if asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) - elif isinstance(result, failure.Failure): + if isinstance(result, failure.Failure): return defer.fail(result) - else: - return defer.succeed(result) + return defer.succeed(result) def deferred_to_future(d: Deferred) -> Future: @@ -352,5 +349,4 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: """ if not is_asyncio_reactor_installed(): return d - else: - return deferred_to_future(d) + return deferred_to_future(d) diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index d28df40c7..64969cb13 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -5,8 +5,8 @@ pprint and pformat wrappers with colorization support import ctypes import platform import sys -from packaging.version import Version as parse_version from pprint import pformat as pformat_ +from packaging.version import Version as parse_version def _enable_windows_terminal_processing(): diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 6cace4f07..775b25ea8 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,5 +1,4 @@ import posixpath - from ftplib import error_perm, FTP from posixpath import dirname diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 3b504e56a..da77ca46f 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -138,8 +138,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): {'csvlnum': csv_r.line_num, 'csvrow': len(row), 'csvheader': len(headers)}) continue - else: - yield dict(zip(headers, row)) + yield dict(zip(headers, row)) def _body_or_str(obj, unicode=True): @@ -152,11 +151,9 @@ def _body_or_str(obj, unicode=True): if isinstance(obj, Response): if not unicode: return obj.body - elif isinstance(obj, TextResponse): + if isinstance(obj, TextResponse): return obj.text - else: - return obj.body.decode('utf-8') - elif isinstance(obj, str): + return obj.body.decode('utf-8') + if isinstance(obj, str): return obj if unicode else obj.encode('utf-8') - else: - return obj.decode('utf-8') if unicode else obj + return obj.decode('utf-8') if unicode else obj diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index e0f7ca9e5..1e0342ace 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -30,10 +30,9 @@ def arg_to_iter(arg): """ if arg is None: return [] - elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'): + if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'): return arg - else: - return [arg] + return [arg] def load_object(path): @@ -49,9 +48,8 @@ def load_object(path): if not isinstance(path, str): if callable(path): return path - else: - raise TypeError("Unexpected argument type, expected string " - f"or object, got: {type(path)}") + raise TypeError("Unexpected argument type, expected string " + f"or object, got: {type(path)}") try: dot = path.rindex('.') @@ -115,9 +113,8 @@ def extract_regex(regex, text, encoding='utf-8'): if isinstance(text, str): return [replace_entities(s, keep=['lt', 'amp']) for s in strings] - else: - return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp']) - for s in strings] + return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp']) + for s in strings] def md5sum(file): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 0d9fdbf23..9df1c91de 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -187,10 +187,9 @@ def get_func_args(func, stripself=False): elif hasattr(func, '__call__'): if inspect.isroutine(func): return [] - elif getattr(func, '__name__', None) == '__call__': + if getattr(func, '__name__', None) == '__call__': return [] - else: - return get_func_args(func.__call__, True) + return get_func_args(func.__call__, True) else: raise TypeError(f'{type(func)} is not callable') if stripself: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 741dce350..23bd2da65 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -9,14 +9,14 @@ import webbrowser from typing import Any, Callable, Iterable, Optional, Tuple, Union from weakref import WeakKeyDictionary +from twisted.web import http +from w3lib import html import scrapy from scrapy.http.response import Response -from twisted.web import http + from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.decorators import deprecated -from w3lib import html - _baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index a73cf03c5..3602043f3 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -16,24 +16,23 @@ class ScrapyJSONEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, set): return list(o) - elif isinstance(o, datetime.datetime): + if isinstance(o, datetime.datetime): return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}") - elif isinstance(o, datetime.date): + if isinstance(o, datetime.date): return o.strftime(self.DATE_FORMAT) - elif isinstance(o, datetime.time): + if isinstance(o, datetime.time): return o.strftime(self.TIME_FORMAT) - elif isinstance(o, decimal.Decimal): + if isinstance(o, decimal.Decimal): return str(o) - elif isinstance(o, defer.Deferred): + if isinstance(o, defer.Deferred): return str(o) - elif is_item(o): + if is_item(o): return ItemAdapter(o).asdict() - elif isinstance(o, Request): + if isinstance(o, Request): return f"<{type(o).__name__} {o.method} {o.url}>" - elif isinstance(o, Response): + if isinstance(o, Response): return f"<{type(o).__name__} {o.status} {o.url}>" - else: - return super().default(o) + return super().default(o) class ScrapyJSONDecoder(json.JSONDecoder): diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index d0fd1757d..b2da69404 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -12,12 +12,11 @@ logger = logging.getLogger(__name__) def iterate_spider_output(result): if inspect.isasyncgen(result): return result - elif inspect.iscoroutine(result): + if inspect.iscoroutine(result): d = deferred_from_coro(result) d.addCallback(iterate_spider_output) return d - else: - return arg_to_iter(deferred_from_coro(result)) + return arg_to_iter(deferred_from_coro(result)) def iter_spider_classes(module): diff --git a/setup.py b/setup.py index e413ea6e4..82ac86cdd 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ setup( 'Tracker': 'https://github.com/scrapy/scrapy/issues', }, description='A high-level Web Crawling and Web Scraping framework', - long_description=open('README.rst').read(), + long_description=open('README.rst', encoding="utf-8").read(), author='Scrapy developers', maintainer='Pablo Hoffman', maintainer_email='pablo@pablohoffman.com', diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 2c867df61..06b849de3 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,6 +1,7 @@ +from twisted.internet import reactor # noqa: F401 + import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import reactor # noqa: F401 class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index c2b30b044..a6dff0e05 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -1,6 +1,7 @@ +from twisted.internet import reactor # noqa: F401 + import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import reactor # noqa: F401 class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index ca70c06a0..907f50784 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -1,6 +1,8 @@ +from twisted.internet import selectreactor + import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import selectreactor + selectreactor.install() diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 0035daf1e..6f48da691 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -1,7 +1,7 @@ -import scrapy -from scrapy.crawler import CrawlerProcess from twisted.internet.main import installReactor from twisted.internet.selectreactor import SelectReactor +import scrapy +from scrapy.crawler import CrawlerProcess class SelectReactorSubclass(SelectReactor): diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index 4f8394edb..c422b13ff 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -1,6 +1,8 @@ +from twisted.internet import selectreactor + import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import selectreactor + selectreactor.install() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index f545de39f..417294447 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -8,7 +8,6 @@ from twisted.python.runtime import platform from scrapy import Spider, Request from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging - from tests.mockserver import MockServer, MockDNSServer @@ -30,9 +29,10 @@ class LocalhostSpider(Spider): def parse(self, response): netloc = urlparse(response.url).netloc - self.logger.info("Host: %s" % netloc.split(":")[0]) - self.logger.info("Type: %s" % type(response.ip_address)) - self.logger.info("IP address: %s" % response.ip_address) + host = netloc.split(":")[0] + self.logger.info(f"Host: {host}") + self.logger.info(f"Type: {type(response.ip_address)}") + self.logger.info(f"IP address: {response.ip_address}") if __name__ == "__main__": diff --git a/tests/mockserver.py b/tests/mockserver.py index 7916798f7..6d2d95692 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -29,8 +29,7 @@ def getarg(request, name, default=None, type=None): if type is not None: value = type(value) return value - else: - return default + return default # most of the following resources are copied from twisted.web.test.test_webclient diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 4077a9bce..049076e5c 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -6,7 +6,7 @@ class CheckCommandTest(CommandTest): command = 'check' def setUp(self): - super(CheckCommandTest, self).setUp() + super().setUp() self.spider_name = 'check_spider' self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve() @@ -24,7 +24,7 @@ class CheckSpider(scrapy.Spider): {contracts} \"\"\" {parse_def} - """) + """, encoding="utf-8") def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 4053c95d5..368b80513 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -92,7 +92,7 @@ class MyBadCrawlSpider(CrawlSpider): def parse(self, response): return [scrapy.Item(), dict(foo='bar')] -""") +""", encoding="utf-8") (self.proj_mod_path / 'pipelines.py').write_text(""" import logging @@ -103,9 +103,9 @@ class MyPipeline: def process_item(self, item, spider): logging.info('It Works!') return item -""") +""", encoding="utf-8") - with (self.proj_mod_path / 'settings.py').open("a") as f: + with (self.proj_mod_path / 'settings.py').open("a", encoding="utf-8") as f: f.write(f""" ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """) @@ -256,7 +256,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.assertTrue(file_path.is_file()) content = '[\n{},\n{"foo": "bar"}\n]' - self.assertEqual(file_path.read_text(), content) + self.assertEqual(file_path.read_text(encoding="utf-8"), content) def test_parse_add_options(self): command = parse.Command() diff --git a/tests/test_commands.py b/tests/test_commands.py index ea4cde363..91476abf8 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -107,7 +107,7 @@ class ProjectTest(unittest.TestCase): def find_in_file(self, filename: Union[str, os.PathLike], regex) -> Optional[re.Match]: """Find first pattern occurrence in file""" pattern = re.compile(regex) - with Path(filename).open("r") as f: + with Path(filename).open("r", encoding="utf-8") as f: for line in f: match = pattern.search(line) if match is not None: @@ -475,7 +475,7 @@ class GenspiderCommandTest(CommandTest): assert file_path.exists() # change name of spider but not its file name - with file_path.open('r+') as spider_file: + with file_path.open('r+', encoding="utf-8") as spider_file: file_data = spider_file.read() file_data = file_data.replace("name = \'example\'", "name = \'renamed\'") spider_file.seek(0) @@ -489,14 +489,14 @@ class GenspiderCommandTest(CommandTest): self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out) modify_time_after = file_path.stat().st_mtime self.assertNotEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text() + file_contents_after = file_path.read_text(encoding="utf-8") self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') self.assertIn(f"{file_path.resolve()} already exists", out) modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text() + file_contents_after = file_path.read_text(encoding="utf-8") self.assertEqual(file_contents_after, file_contents_before) def test_same_filename_as_existing_spider_force(self): @@ -536,7 +536,7 @@ class GenspiderStandaloneCommandTest(ProjectTest): self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out) assert file_path.exists() modify_time_before = file_path.stat().st_mtime - file_contents_before = file_path.read_text() + file_contents_before = file_path.read_text(encoding="utf-8") if force: # use different template to ensure contents were changed @@ -544,14 +544,14 @@ class GenspiderStandaloneCommandTest(ProjectTest): self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out) modify_time_after = file_path.stat().st_mtime self.assertNotEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text() + file_contents_after = file_path.read_text(encoding="utf-8") self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') self.assertIn(f"{Path(self.temp_path, file_name + '.py').resolve()} already exists", out) modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text() + file_contents_after = file_path.read_text(encoding="utf-8") self.assertEqual(file_contents_after, file_contents_before) def test_same_name_as_existing_file_force(self): @@ -596,7 +596,7 @@ class BadSpider(scrapy.Spider): fname = (tmpdir / name).resolve() else: fname = (tmpdir / self.spider_filename).resolve() - fname.write_text(content) + fname.write_text(content, encoding="utf-8") try: yield str(fname) finally: @@ -754,11 +754,11 @@ class MySpider(scrapy.Spider): ) return [] """ - Path(self.cwd, "example.json").write_text("not empty") + Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) - with Path(self.cwd, "example.json").open() as f2: + with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: first_line = f2.readline() self.assertNotEqual(first_line, "not empty") @@ -798,7 +798,7 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = 'myspider.pyw' def setUp(self): - super(WindowsRunSpiderCommandTest, self).setUp() + super().setUp() def test_start_requests_errors(self): log = self.get_log(self.badspider, name='badspider.pyw') @@ -860,7 +860,7 @@ class ViewCommandTest(CommandTest): class CrawlCommandTest(CommandTest): def crawl(self, code, args=()): - Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code) + Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code, encoding="utf-8") return self.proc('crawl', 'myspider', *args) def get_log(self, code, args=()): @@ -912,11 +912,11 @@ class MySpider(scrapy.Spider): ) return [] """ - Path(self.cwd, "example.json").write_text("not empty") + Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) - with Path(self.cwd, "example.json").open() as f2: + with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: first_line = f2.readline() self.assertNotEqual(first_line, "not empty") diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c09f1a6f2..e0902fdbe 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -11,6 +11,9 @@ from twisted.internet import defer from twisted.python.versions import Version from twisted.trial import unittest +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version + import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess from scrapy.exceptions import ScrapyDeprecationWarning @@ -23,8 +26,6 @@ from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv -from pkg_resources import parse_version -from w3lib import __version__ as w3lib_version from tests.mockserver import MockServer diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0334be743..3dc2745a0 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -110,7 +110,7 @@ class FileTestCase(unittest.TestCase): def setUp(self): # add a special char to check that they are handled correctly self.tmpname = Path(self.mktemp() + '^') - Path(self.tmpname).write_text('0123456789') + Path(self.tmpname).write_text("0123456789", encoding="utf-8") handler = create_instance(FileDownloadHandler, None, get_crawler()) self.download_request = handler.download_request @@ -722,8 +722,7 @@ class UriResource(resource.Resource): # ToDo: implement proper HTTPS proxy tests, not faking them. if request.method != b'CONNECT': return request.uri - else: - return b'' + return b'' class HttpProxyTestCase(unittest.TestCase): diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 079267535..c2fa3ec57 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -171,7 +171,7 @@ class Https2InvalidDNSId(Https2TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" def setUp(self): - super(Https2InvalidDNSId, self).setUp() + super().setUp() self.host = '127.0.0.1' @@ -190,7 +190,7 @@ class Https2InvalidDNSPattern(Https2TestCase): 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' ) - super(Https2InvalidDNSPattern, self).setUp() + super().setUp() @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") @@ -245,4 +245,4 @@ class Https2ProxyTestCase(Http11ProxyTestCase): @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): with self.assertRaises(NotImplementedError): - yield super(Https2ProxyTestCase, self).test_download_with_proxy_https_timeout() + yield super().test_download_with_proxy_https_timeout() diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index ba7453255..dd835b9c9 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,6 +1,6 @@ import logging -from testfixtures import LogCapture from unittest import TestCase +from testfixtures import LogCapture import pytest diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index b2b5ce77d..1c4cae6d1 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -2,8 +2,8 @@ from unittest import TestCase, main from scrapy.http import Response, XmlResponse from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware from scrapy.spiders import Spider -from tests import get_testdata from scrapy.utils.test import assert_samelines +from tests import get_testdata def _test_data(formats): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 928c007f5..b3d8264ba 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -272,10 +272,9 @@ class RFC2616PolicyTest(DefaultStorageTest): if result: assert isinstance(result, (Request, Response)) return result - else: - result = mw.process_response(request, response, self.spider) - assert isinstance(result, Response) - return result + result = mw.process_response(request, response, self.spider) + assert isinstance(result, Response) + return result except Exception: print('Request', request) print('Response', response) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 6f4e217e6..b0272143d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -4,6 +4,7 @@ from pathlib import Path from unittest import TestCase, SkipTest from warnings import catch_warnings +from w3lib.encoding import resolve_encoding from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS @@ -12,8 +13,6 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir -from w3lib.encoding import resolve_encoding - SAMPLEDIR = Path(tests_datadir, 'compressed') diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 69ac928c3..86c5d8b5e 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -400,8 +400,7 @@ class XmlItemExporterTest(BaseItemExporterTest): children = list(elem.iterchildren()) if children: return [(child.tag, sorted(xmltuple(child))) for child in children] - else: - return [(elem.tag, [(elem.text, ())])] + return [(elem.tag, [(elem.text, ())])] def xmlsplit(xmlcontent): doc = lxml.etree.fromstring(xmlcontent) @@ -621,8 +620,7 @@ class CustomExporterItemTest(unittest.TestCase): def serialize_field(self, field, name, value): if name == 'age': return str(int(value) + 1) - else: - return super().serialize_field(field, name, value) + return super().serialize_field(field, name, value) i = self.item_class(name='John', age='22') a = ItemAdapter(i) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 402348cf9..e461bcf2c 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -164,7 +164,7 @@ class RequestHeaders(LeafResource): def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate: - pem = key_file.read_text() + certificate_file.read_text() + pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text(encoding="utf-8") return PrivateCertificate.loadPEM(pem) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 579ef9fa2..9f7f1854f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -223,7 +223,7 @@ class RequestTest(unittest.TestCase): r1 = CustomRequest('http://www.example.com') r2 = r1.copy() - assert type(r2) is CustomRequest + assert isinstance(r2, CustomRequest) def test_replace(self): """Test Request.replace() method""" diff --git a/tests/test_http_response.py b/tests/test_http_response.py index b42c95045..f51f3d988 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -102,7 +102,7 @@ class BaseResponseTest(unittest.TestCase): r1 = CustomResponse('http://www.example.com') r2 = r1.copy() - assert type(r2) is CustomResponse + assert isinstance(r2, CustomResponse) def test_replace(self): """Test Response.replace() method""" diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 0fd52da5f..14b3b5568 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -391,10 +391,9 @@ class BasicItemLoaderTest(unittest.TestCase): def join(values, sep=None, loader_context=None, ignored=None): if sep is not None: return sep.join(values) - elif loader_context and 'sep' in loader_context: + if loader_context and 'sep' in loader_context: return loader_context['sep'].join(values) - else: - return ''.join(values) + return ''.join(values) class TestItemLoader(NameItemLoader): name_out = Compose(partial(join, sep='+')) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 84e867660..0a94ae699 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -388,11 +388,11 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): def thumb_path(self, request, thumb_id, response=None, info=None): self._mockcalled.append('thumb_path') - return super(MockedMediaPipelineDeprecatedMethods, self).thumb_path(request, thumb_id, response, info) + return super().thumb_path(request, thumb_id, response, info) def get_images(self, response, request, info): self._mockcalled.append('get_images') - return super(MockedMediaPipelineDeprecatedMethods, self).get_images(response, request, info) + return super().get_images(response, request, info) def image_downloaded(self, response, request, info): self._mockcalled.append('image_downloaded') diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 0406d906f..15e400327 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -1,12 +1,12 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase +from testfixtures import LogCapture + from scrapy import Request, signals from scrapy.http.response import Response from scrapy.utils.test import get_crawler -from testfixtures import LogCapture - from tests.mockserver import MockServer from tests.spiders import SingleRequestSpider diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 002a04358..63e3aac00 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -91,7 +91,7 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(kwargs['callback'] == 'some_callback') self.crawler.stats.inc_value('boolean_checks', 3) elif response.url.endswith('/general_without'): - self.checks.append(kwargs == {}) + self.checks.append(kwargs == {}) # pylint: disable=use-implicit-booleaness-not-comparison self.crawler.stats.inc_value('boolean_checks') def parse_no_kwargs(self, response): diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index fc234a83d..6602b626d 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -11,10 +11,8 @@ from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.request import fingerprint from scrapy.utils.test import get_crawler - from tests.mockserver import MockServer - PATHS = ["/a", "/b", "/c"] URLS = [urljoin("https://example.org", p) for p in PATHS] diff --git a/tests/test_spider.py b/tests/test_spider.py index e1527620f..cb66066b0 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -7,6 +7,7 @@ from unittest import mock from testfixtures import LogCapture from twisted.trial import unittest +from w3lib.url import safe_url_string from scrapy import signals from scrapy.settings import Settings from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse @@ -22,7 +23,6 @@ from scrapy.spiders import ( from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler from tests import get_testdata -from w3lib.url import safe_url_string class SpiderTest(unittest.TestCase): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 697105967..0b6b51a5b 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -3,6 +3,7 @@ import shutil import warnings from pathlib import Path +import tempfile from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -10,7 +11,6 @@ from twisted.trial import unittest # ugly hack to avoid cyclic imports of scrapy.spiders when running this test # alone import scrapy -import tempfile from scrapy.interfaces import ISpiderLoader from scrapy.spiderloader import SpiderLoader from scrapy.settings import Settings diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 46f74ae52..ee11ee492 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -6,11 +6,11 @@ from twisted.trial.unittest import TestCase as TrialTestCase from twisted.internet import defer from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError from scrapy.settings import Settings +from tests.mockserver import MockServer from tests.spiders import MockServerSpider diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index dac246fb6..8dd1def17 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -4,7 +4,6 @@ from twisted.trial.unittest import TestCase from scrapy import Request, Spider from scrapy.utils.test import get_crawler - from tests.mockserver import MockServer diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 9456b01d4..63daf0b8a 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,8 +1,8 @@ from urllib.parse import urlparse from unittest import TestCase import warnings - from scrapy.http import Response, Request + from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.downloadermiddlewares.redirect import RedirectMiddleware @@ -380,7 +380,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): scheme = urlparse(request).scheme if scheme == 'https': return b'https://python.org/' - elif scheme == 'http': + if scheme == 'http': return b'http://python.org/' diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index c5fcc1853..22dafb2d2 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -16,7 +16,6 @@ from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler - """ Queues that handle requests """ diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index d39de7430..97c1c60d1 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -193,9 +193,8 @@ class AsyncCooperatorTest(unittest.TestCase): delay = random.random() / 8 reactor.callLater(delay, dfd.callback, None) return dfd - else: - # simulate trivial sync processing - results.append(o) + # simulate trivial sync processing + results.append(o) @staticmethod def get_async_iterable(length): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index e47afa266..50c63dfab 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -2,6 +2,7 @@ import inspect import unittest from unittest import mock import warnings + from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import create_deprecated_class, update_classpath diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 69d9a9e3a..3dcaf71dd 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -47,7 +47,7 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): from twisted.web.client import _makeGetterFactory return _makeGetterFactory( - to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs + to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs, ).deferred From 5bd27191a2fd9ec50936c05a377f76bd14aa8266 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 28 Nov 2022 23:37:57 +0500 Subject: [PATCH 0701/2083] Bump mypy, flake8, and pylint (#5738) --- scrapy/commands/__init__.py | 2 +- scrapy/core/scraper.py | 24 ++++++++++++++++++++++-- scrapy/crawler.py | 4 ++-- scrapy/http/response/__init__.py | 7 +++---- scrapy/http/response/text.py | 6 ++---- scrapy/middleware.py | 4 ++-- tox.ini | 6 +++--- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index d0fb4efd8..b9ba3335e 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -23,7 +23,7 @@ class ScrapyCommand: exitcode = 0 - def __init__(self): + def __init__(self) -> None: self.settings: Any = None # set in scrapy.cmdline def set_crawler(self, crawler): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7225e0743..69ac1cdaf 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,8 +1,22 @@ """This module implements the Scraper component which parses responses and extracts information from them""" +from __future__ import annotations + import logging from collections import deque -from typing import Any, AsyncGenerator, AsyncIterable, Deque, Generator, Iterable, Optional, Set, Tuple, Union +from typing import ( + Any, + AsyncGenerator, + AsyncIterable, + Deque, + Generator, + Iterable, + Optional, + Set, + TYPE_CHECKING, + Tuple, + Union, +) from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks @@ -26,6 +40,10 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from scrapy.crawler import Crawler + + QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] @@ -75,7 +93,7 @@ class Slot: class Scraper: - def __init__(self, crawler): + def __init__(self, crawler: Crawler) -> None: self.slot: Optional[Slot] = None self.spidermw = SpiderMiddlewareManager.from_crawler(crawler) itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR']) @@ -174,6 +192,7 @@ class Scraper: def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None: exc = _failure.value if isinstance(exc, CloseSpider): + assert self.crawler.engine is not None # typing self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') return logkws = self.logformatter.spider_error(_failure, request, response, spider) @@ -214,6 +233,7 @@ class Scraper: """ assert self.slot is not None # typing if isinstance(output, Request): + assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) elif is_item(output): self.slot.itemproc_size += 1 diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 04c6891e3..4700a30ab 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,7 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement @@ -109,7 +109,7 @@ class Crawler: self.settings.freeze() self.crawling = False self.spider = None - self.engine = None + self.engine: Optional[ExecutionEngine] = None @defer.inlineCallbacks def crawl(self, *args, **kwargs): diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index fb2d0f165..7626946ec 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -140,8 +140,7 @@ class Response(object_ref): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, cb_kwargs=None, flags=None): - # type: (...) -> Request + dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. It accepts the same arguments as ``Request.__init__`` method, @@ -179,8 +178,8 @@ class Response(object_ref): def follow_all(self, urls, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, cb_kwargs=None, flags=None): - # type: (...) -> Generator[Request, None, None] + dont_filter=False, errback=None, cb_kwargs=None, + flags=None) -> Generator[Request, None, None]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index bfcde878d..da81d0a4a 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -142,8 +142,7 @@ class TextResponse(Response): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None, cb_kwargs=None, flags=None): - # type: (...) -> Request + dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. It accepts the same arguments as ``Request.__init__`` method, @@ -184,8 +183,7 @@ class TextResponse(Response): def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, dont_filter=False, errback=None, cb_kwargs=None, flags=None, - css=None, xpath=None): - # type: (...) -> Generator[Request, None, None] + css=None, xpath=None) -> Generator[Request, None, None]: """ A generator that produces :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 8d7e5a602..431bd76dc 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Deque, Dict, Iterable, Tuple, Union, cast +from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, cast from twisted.internet.defer import Deferred @@ -19,7 +19,7 @@ class MiddlewareManager: component_name = 'foo middleware' - def __init__(self, *middlewares): + def __init__(self, *middlewares: Any) -> None: self.middlewares = middlewares # Only process_spider_output and process_spider_exception can be None. # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. diff --git a/tox.ini b/tox.ini index eee99cb2d..4d0f0291b 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,7 @@ install_command = basepython = python3 deps = lxml-stubs==0.2.0 - mypy==0.982 + mypy==0.991 types-attrs==19.1.0 types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 @@ -58,7 +58,7 @@ deps = {[testenv]deps} # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 - flake8==5.0.4 + flake8==6.0.0 commands = flake8 {posargs:docs scrapy tests} @@ -67,7 +67,7 @@ commands = basepython = python3.8 deps = {[testenv:extra-deps]deps} - pylint==2.15.3 + pylint==2.15.6 commands = pylint conftest.py docs extras scrapy setup.py tests From e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d Mon Sep 17 00:00:00 2001 From: Emmanuel Rondan Date: Tue, 29 Nov 2022 11:30:46 -0300 Subject: [PATCH 0702/2083] adding black formatter to all the code --- conftest.py | 21 +- docs/_ext/scrapydocs.py | 76 +- docs/conf.py | 175 +- docs/conftest.py | 8 +- docs/utils/linkfix.py | 4 +- extras/qps-bench-server.py | 15 +- extras/qpsclient.py | 10 +- scrapy/__init__.py | 15 +- scrapy/__main__.py | 2 +- scrapy/cmdline.py | 32 +- scrapy/commands/__init__.py | 123 +- scrapy/commands/bench.py | 22 +- scrapy/commands/check.py | 27 +- scrapy/commands/crawl.py | 11 +- scrapy/commands/edit.py | 12 +- scrapy/commands/fetch.py | 39 +- scrapy/commands/genspider.py | 94 +- scrapy/commands/list.py | 2 +- scrapy/commands/parse.py | 153 +- scrapy/commands/runspider.py | 4 +- scrapy/commands/settings.py | 40 +- scrapy/commands/shell.py | 42 +- scrapy/commands/startproject.py | 61 +- scrapy/commands/version.py | 12 +- scrapy/commands/view.py | 7 +- scrapy/contracts/__init__.py | 35 +- scrapy/contracts/default.py | 60 +- scrapy/core/downloader/__init__.py | 58 +- scrapy/core/downloader/contextfactory.py | 49 +- scrapy/core/downloader/handlers/__init__.py | 23 +- scrapy/core/downloader/handlers/datauri.py | 7 +- scrapy/core/downloader/handlers/ftp.py | 22 +- scrapy/core/downloader/handlers/http10.py | 9 +- scrapy/core/downloader/handlers/http11.py | 259 ++- scrapy/core/downloader/handlers/http2.py | 38 +- scrapy/core/downloader/handlers/s3.py | 60 +- scrapy/core/downloader/middleware.py | 43 +- scrapy/core/downloader/tls.py | 56 +- scrapy/core/downloader/webclient.py | 56 +- scrapy/core/engine.py | 154 +- scrapy/core/http2/agent.py | 14 +- scrapy/core/http2/protocol.py | 118 +- scrapy/core/http2/stream.py | 234 +- scrapy/core/scheduler.py | 96 +- scrapy/core/scraper.py | 180 +- scrapy/core/spidermw.py | 189 +- scrapy/crawler.py | 75 +- scrapy/downloadermiddlewares/ajaxcrawl.py | 32 +- scrapy/downloadermiddlewares/cookies.py | 35 +- scrapy/downloadermiddlewares/decompression.py | 21 +- .../downloadermiddlewares/defaultheaders.py | 3 +- .../downloadermiddlewares/downloadtimeout.py | 7 +- scrapy/downloadermiddlewares/httpauth.py | 24 +- scrapy/downloadermiddlewares/httpcache.py | 82 +- .../downloadermiddlewares/httpcompression.py | 40 +- scrapy/downloadermiddlewares/httpproxy.py | 56 +- scrapy/downloadermiddlewares/redirect.py | 82 +- scrapy/downloadermiddlewares/retry.py | 69 +- scrapy/downloadermiddlewares/robotstxt.py | 46 +- scrapy/downloadermiddlewares/stats.py | 36 +- scrapy/downloadermiddlewares/useragent.py | 8 +- scrapy/dupefilters.py | 30 +- scrapy/exceptions.py | 11 +- scrapy/exporters.py | 94 +- scrapy/extension.py | 4 +- scrapy/extensions/closespider.py | 47 +- scrapy/extensions/corestats.py | 19 +- scrapy/extensions/debug.py | 22 +- scrapy/extensions/feedexport.py | 295 +-- scrapy/extensions/httpcache.py | 157 +- scrapy/extensions/logstats.py | 22 +- scrapy/extensions/memdebug.py | 11 +- scrapy/extensions/memusage.py | 56 +- scrapy/extensions/postprocessing.py | 27 +- scrapy/extensions/spiderstate.py | 6 +- scrapy/extensions/statsmailer.py | 1 - scrapy/extensions/telnet.py | 69 +- scrapy/extensions/throttle.py | 34 +- scrapy/http/common.py | 1 + scrapy/http/cookies.py | 27 +- scrapy/http/headers.py | 17 +- scrapy/http/request/__init__.py | 53 +- scrapy/http/request/form.py | 101 +- scrapy/http/request/json_request.py | 30 +- scrapy/http/request/rpc.py | 13 +- scrapy/http/response/__init__.py | 67 +- scrapy/http/response/text.py | 84 +- scrapy/interfaces.py | 1 - scrapy/item.py | 19 +- scrapy/link.py | 12 +- scrapy/linkextractors/__init__.py | 86 +- scrapy/linkextractors/lxmlhtml.py | 85 +- scrapy/loader/common.py | 2 +- scrapy/loader/processors.py | 12 +- scrapy/logformatter.py | 80 +- scrapy/mail.py | 123 +- scrapy/middleware.py | 40 +- scrapy/pipelines/__init__.py | 12 +- scrapy/pipelines/files.py | 321 +-- scrapy/pipelines/images.py | 150 +- scrapy/pipelines/media.py | 69 +- scrapy/pqueues.py | 37 +- scrapy/resolver.py | 24 +- scrapy/responsetypes.py | 65 +- scrapy/robotstxt.py | 16 +- scrapy/selector/unified.py | 19 +- scrapy/settings/__init__.py | 57 +- scrapy/settings/default_settings.py | 220 +- scrapy/shell.py | 63 +- scrapy/signalmanager.py | 11 +- scrapy/spiderloader.py | 19 +- scrapy/spidermiddlewares/depth.py | 27 +- scrapy/spidermiddlewares/httperror.py | 24 +- scrapy/spidermiddlewares/offsite.py | 29 +- scrapy/spidermiddlewares/referer.py | 92 +- scrapy/spidermiddlewares/urllength.py | 11 +- scrapy/spiders/__init__.py | 19 +- scrapy/spiders/crawl.py | 20 +- scrapy/spiders/feed.py | 46 +- scrapy/spiders/init.py | 2 +- scrapy/spiders/sitemap.py | 25 +- scrapy/squeues.py | 31 +- scrapy/statscollectors.py | 11 +- scrapy/utils/asyncgen.py | 2 +- scrapy/utils/benchserver.py | 17 +- scrapy/utils/boto.py | 1 + scrapy/utils/conf.py | 82 +- scrapy/utils/console.py | 41 +- scrapy/utils/curl.py | 50 +- scrapy/utils/datatypes.py | 2 + scrapy/utils/decorators.py | 5 + scrapy/utils/defer.py | 53 +- scrapy/utils/deprecate.py | 36 +- scrapy/utils/display.py | 3 +- scrapy/utils/ftp.py | 8 +- scrapy/utils/gz.py | 10 +- scrapy/utils/httpobj.py | 4 +- scrapy/utils/iterators.py | 67 +- scrapy/utils/job.py | 2 +- scrapy/utils/log.py | 91 +- scrapy/utils/misc.py | 54 +- scrapy/utils/ossignal.py | 5 +- scrapy/utils/project.py | 41 +- scrapy/utils/python.py | 43 +- scrapy/utils/reactor.py | 15 +- scrapy/utils/reqser.py | 14 +- scrapy/utils/request.py | 156 +- scrapy/utils/response.py | 37 +- scrapy/utils/signal.py | 59 +- scrapy/utils/sitemap.py | 22 +- scrapy/utils/spider.py | 20 +- scrapy/utils/ssl.py | 32 +- scrapy/utils/template.py | 14 +- scrapy/utils/test.py | 27 +- scrapy/utils/testproc.py | 10 +- scrapy/utils/testsite.py | 23 +- scrapy/utils/trackref.py | 3 +- scrapy/utils/url.py | 60 +- setup.py | 110 +- tests/CrawlerProcess/asyncio_custom_loop.py | 12 +- .../CrawlerProcess/asyncio_deferred_signal.py | 10 +- .../asyncio_enabled_no_reactor.py | 10 +- .../CrawlerProcess/asyncio_enabled_reactor.py | 11 +- .../asyncio_enabled_reactor_different_loop.py | 13 +- .../asyncio_enabled_reactor_same_loop.py | 13 +- .../caching_hostname_resolver.py | 15 +- .../caching_hostname_resolver_ipv6.py | 11 +- tests/CrawlerProcess/default_name_resolver.py | 1 + tests/CrawlerProcess/multi.py | 2 +- tests/CrawlerProcess/reactor_default.py | 2 +- .../reactor_default_twisted_reactor_select.py | 10 +- tests/CrawlerProcess/reactor_select.py | 2 +- ..._select_subclass_twisted_reactor_select.py | 10 +- .../reactor_select_twisted_reactor_select.py | 10 +- tests/CrawlerProcess/simple.py | 2 +- .../CrawlerProcess/twisted_reactor_asyncio.py | 10 +- .../twisted_reactor_custom_settings.py | 2 +- ...wisted_reactor_custom_settings_conflict.py | 4 +- .../twisted_reactor_custom_settings_same.py | 4 +- tests/CrawlerProcess/twisted_reactor_poll.py | 10 +- .../CrawlerProcess/twisted_reactor_select.py | 10 +- tests/CrawlerRunner/ip_address.py | 2 +- tests/__init__.py | 16 +- tests/ftpserver.py | 8 +- tests/keys/__init__.py | 4 +- tests/mocks/dummydbm.py | 3 +- tests/mockserver.py | 90 +- tests/pipelines.py | 2 - tests/spiders.py | 175 +- tests/test_closespider.py | 33 +- tests/test_cmdline/__init__.py | 39 +- tests/test_cmdline/extensions.py | 3 +- tests/test_cmdline/settings.py | 10 +- .../__init__.py | 7 +- .../test_spider/pipelines.py | 4 +- .../test_spider/settings.py | 4 +- .../test_spider/spiders/exception.py | 6 +- .../test_spider/spiders/normal.py | 6 +- tests/test_command_check.py | 21 +- tests/test_command_fetch.py | 26 +- tests/test_command_parse.py | 179 +- tests/test_command_shell.py | 91 +- tests/test_command_version.py | 28 +- tests/test_commands.py | 594 +++--- tests/test_contracts.py | 122 +- tests/test_core_downloader.py | 5 +- tests/test_crawl.py | 236 ++- tests/test_crawler.py | 346 +-- tests/test_dependencies.py | 22 +- tests/test_downloader_handlers.py | 659 +++--- tests/test_downloader_handlers_http2.py | 111 +- tests/test_downloadermiddleware.py | 82 +- ...test_downloadermiddleware_ajaxcrawlable.py | 32 +- tests/test_downloadermiddleware_cookies.py | 475 +++-- ...test_downloadermiddleware_decompression.py | 18 +- ...est_downloadermiddleware_defaultheaders.py | 13 +- ...st_downloadermiddleware_downloadtimeout.py | 17 +- tests/test_downloadermiddleware_httpauth.py | 78 +- tests/test_downloadermiddleware_httpcache.py | 358 ++-- ...st_downloadermiddleware_httpcompression.py | 279 +-- tests/test_downloadermiddleware_httpproxy.py | 383 ++-- tests/test_downloadermiddleware_redirect.py | 324 +-- tests/test_downloadermiddleware_retry.py | 194 +- tests/test_downloadermiddleware_robotstxt.py | 153 +- tests/test_downloadermiddleware_stats.py | 40 +- tests/test_downloadermiddleware_useragent.py | 40 +- tests/test_dupefilters.py | 178 +- tests/test_engine.py | 192 +- tests/test_engine_stop_download_bytes.py | 43 +- tests/test_engine_stop_download_headers.py | 47 +- tests/test_exporters.py | 243 ++- tests/test_extension_telnet.py | 11 +- tests/test_feedexport.py | 1884 +++++++++-------- tests/test_http2_client_protocol.py | 360 ++-- tests/test_http_cookies.py | 33 +- tests/test_http_headers.py | 185 +- tests/test_http_request.py | 1056 +++++---- tests/test_http_response.py | 649 +++--- tests/test_item.py | 193 +- tests/test_link.py | 21 +- tests/test_linkextractors.py | 763 +++++-- tests/test_loader.py | 400 ++-- tests/test_loader_deprecated.py | 497 +++-- tests/test_logformatter.py | 109 +- tests/test_mail.py | 144 +- tests/test_middleware.py | 32 +- tests/test_pipeline_crawl.py | 90 +- tests/test_pipeline_files.py | 353 +-- tests/test_pipeline_images.py | 320 ++- tests/test_pipeline_media.py | 304 +-- tests/test_pipelines.py | 23 +- tests/test_pqueues.py | 26 +- tests/test_proxy_connect.py | 59 +- tests/test_request_attribute_binding.py | 76 +- tests/test_request_cb_kwargs.py | 134 +- tests/test_request_dict.py | 85 +- tests/test_request_left.py | 11 +- tests/test_responsetypes.py | 97 +- tests/test_robotstxt_interface.py | 98 +- tests/test_scheduler.py | 105 +- tests/test_scheduler_base.py | 6 +- tests/test_selector.py | 95 +- tests/test_settings/__init__.py | 477 +++-- tests/test_settings/default_settings.py | 5 +- tests/test_signals.py | 11 +- tests/test_spider.py | 436 ++-- tests/test_spiderloader/__init__.py | 120 +- .../test_spiders/nested/spider4.py | 4 +- .../test_spiderloader/test_spiders/spider3.py | 4 +- tests/test_spidermiddleware.py | 231 +- tests/test_spidermiddleware_depth.py | 17 +- tests/test_spidermiddleware_httperror.py | 112 +- tests/test_spidermiddleware_offsite.py | 69 +- tests/test_spidermiddleware_output_chain.py | 217 +- tests/test_spidermiddleware_referer.py | 1161 ++++++---- tests/test_spidermiddleware_urllength.py | 21 +- tests/test_spiderstate.py | 13 +- tests/test_squeues.py | 44 +- tests/test_squeues_request.py | 23 +- tests/test_stats.py | 94 +- tests/test_toplevel.py | 5 +- tests/test_urlparse_monkeypatches.py | 11 +- tests/test_utils_asyncio.py | 7 +- tests/test_utils_conf.py | 242 ++- tests/test_utils_console.py | 20 +- tests/test_utils_curl.py | 53 +- tests/test_utils_datatypes.py | 161 +- tests/test_utils_defer.py | 24 +- tests/test_utils_deprecate.py | 132 +- tests/test_utils_display.py | 37 +- tests/test_utils_gz.py | 31 +- tests/test_utils_httpobj.py | 1 - tests/test_utils_iterators.py | 339 +-- tests/test_utils_log.py | 70 +- tests/test_utils_misc/__init__.py | 145 +- ...t_return_with_argument_inside_generator.py | 28 +- tests/test_utils_project.py | 37 +- tests/test_utils_python.py | 105 +- tests/test_utils_request.py | 311 ++- tests/test_utils_response.py | 140 +- tests/test_utils_serialize.py | 31 +- tests/test_utils_signal.py | 26 +- tests/test_utils_sitemap.py | 170 +- tests/test_utils_spider.py | 8 +- tests/test_utils_template.py | 19 +- tests/test_utils_trackref.py | 34 +- tests/test_utils_url.py | 642 +++--- tests/test_webclient.py | 306 +-- 308 files changed, 16487 insertions(+), 11921 deletions(-) diff --git a/conftest.py b/conftest.py index 2a5d55083..585356a3e 100644 --- a/conftest.py +++ b/conftest.py @@ -9,7 +9,7 @@ from tests.keys import generate_keys def _py_files(folder): - return (str(p) for p in Path(folder).rglob('*.py')) + return (str(p) for p in Path(folder).rglob("*.py")) collect_ignore = [ @@ -21,16 +21,16 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with Path('tests/ignores.txt').open(encoding="utf-8") as reader: +with Path("tests/ignores.txt").open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() - if file_path and file_path[0] != '#': + if file_path and file_path[0] != "#": collect_ignore.append(file_path) if not H2_ENABLED: collect_ignore.extend( ( - 'scrapy/core/downloader/handlers/http2.py', + "scrapy/core/downloader/handlers/http2.py", *_py_files("scrapy/core/http2"), ) ) @@ -50,7 +50,7 @@ def pytest_addoption(parser): ) -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def reactor_pytest(request): if not request.cls: # doctests @@ -61,14 +61,17 @@ def reactor_pytest(request): @pytest.fixture(autouse=True) def only_asyncio(request, reactor_pytest): - if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio': - pytest.skip('This test is only run with --reactor=asyncio') + if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio": + pytest.skip("This test is only run with --reactor=asyncio") @pytest.fixture(autouse=True) def only_not_asyncio(request, reactor_pytest): - if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio': - pytest.skip('This test is only run without --reactor=asyncio') + if ( + request.node.get_closest_marker("only_not_asyncio") + and reactor_pytest == "asyncio" + ): + pytest.skip("This test is only run without --reactor=asyncio") def pytest_configure(config): diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 337604cf1..1419792fc 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -11,15 +11,15 @@ class settingslist_node(nodes.General, nodes.Element): class SettingsListDirective(Directive): def run(self): - return [settingslist_node('')] + return [settingslist_node("")] def is_setting_index(node): - if node.tagname == 'index' and node['entries']: + if node.tagname == "index" and node["entries"]: # index entries for setting directives look like: # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] - entry_type, info, refid = node['entries'][0][:3] - return entry_type == 'pair' and info.endswith('; setting') + entry_type, info, refid = node["entries"][0][:3] + return entry_type == "pair" and info.endswith("; setting") return False @@ -30,14 +30,14 @@ def get_setting_target(node): def get_setting_name_and_refid(node): """Extract setting name from directive index node""" - entry_type, info, refid = node['entries'][0][:3] - return info.replace('; setting', ''), refid + entry_type, info, refid = node["entries"][0][:3] + return info.replace("; setting", ""), refid def collect_scrapy_settings_refs(app, doctree): env = app.builder.env - if not hasattr(env, 'scrapy_all_settings'): + if not hasattr(env, "scrapy_all_settings"): env.scrapy_all_settings = [] for node in doctree.traverse(is_setting_index): @@ -46,18 +46,23 @@ def collect_scrapy_settings_refs(app, doctree): setting_name, refid = get_setting_name_and_refid(node) - env.scrapy_all_settings.append({ - 'docname': env.docname, - 'setting_name': setting_name, - 'refid': refid, - }) + env.scrapy_all_settings.append( + { + "docname": env.docname, + "setting_name": setting_name, + "refid": refid, + } + ) def make_setting_element(setting_data, app, fromdocname): - refnode = make_refnode(app.builder, fromdocname, - todocname=setting_data['docname'], - targetid=setting_data['refid'], - child=nodes.Text(setting_data['setting_name'])) + refnode = make_refnode( + app.builder, + fromdocname, + todocname=setting_data["docname"], + targetid=setting_data["refid"], + child=nodes.Text(setting_data["setting_name"]), + ) p = nodes.paragraph() p += refnode @@ -71,10 +76,13 @@ def replace_settingslist_nodes(app, doctree, fromdocname): for node in doctree.traverse(settingslist_node): settings_list = nodes.bullet_list() - settings_list.extend([make_setting_element(d, app, fromdocname) - for d in sorted(env.scrapy_all_settings, - key=itemgetter('setting_name')) - if fromdocname != d['docname']]) + settings_list.extend( + [ + make_setting_element(d, app, fromdocname) + for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) + if fromdocname != d["docname"] + ] + ) node.replace_self(settings_list) @@ -99,41 +107,41 @@ def setup(app): rolename="reqmeta", indextemplate="pair: %s; reqmeta", ) - app.add_role('source', source_role) - app.add_role('commit', commit_role) - app.add_role('issue', issue_role) - app.add_role('rev', rev_role) + app.add_role("source", source_role) + app.add_role("commit", commit_role) + app.add_role("issue", issue_role) + app.add_role("rev", rev_role) app.add_node(settingslist_node) - app.add_directive('settingslist', SettingsListDirective) + app.add_directive("settingslist", SettingsListDirective) - app.connect('doctree-read', collect_scrapy_settings_refs) - app.connect('doctree-resolved', replace_settingslist_nodes) + app.connect("doctree-read", collect_scrapy_settings_refs) + app.connect("doctree-resolved", replace_settingslist_nodes) def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = 'https://github.com/scrapy/scrapy/blob/master/' + text + ref = "https://github.com/scrapy/scrapy/blob/master/" + text set_classes(options) node = nodes.reference(rawtext, text, refuri=ref, **options) return [node], [] def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = 'https://github.com/scrapy/scrapy/issues/' + text + ref = "https://github.com/scrapy/scrapy/issues/" + text set_classes(options) - node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options) + node = nodes.reference(rawtext, "issue " + text, refuri=ref, **options) return [node], [] def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = 'https://github.com/scrapy/scrapy/commit/' + text + ref = "https://github.com/scrapy/scrapy/commit/" + text set_classes(options) - node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options) + node = nodes.reference(rawtext, "commit " + text, refuri=ref, **options) return [node], [] def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = 'http://hg.scrapy.org/scrapy/changeset/' + text + ref = "http://hg.scrapy.org/scrapy/changeset/" + text set_classes(options) - node = nodes.reference(rawtext, 'r' + text, refuri=ref, **options) + node = nodes.reference(rawtext, "r" + text, refuri=ref, **options) return [node], [] diff --git a/docs/conf.py b/docs/conf.py index d2a77003e..38ca81932 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,30 +25,30 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - 'hoverxref.extension', - 'notfound.extension', - 'scrapydocs', - 'sphinx.ext.autodoc', - 'sphinx.ext.coverage', - 'sphinx.ext.intersphinx', - 'sphinx.ext.viewcode', + "hoverxref.extension", + "notfound.extension", + "scrapydocs", + "sphinx.ext.autodoc", + "sphinx.ext.coverage", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8' +# source_encoding = 'utf-8' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'Scrapy' -copyright = f'2008–{datetime.now().year}, Scrapy developers' +project = "Scrapy" +copyright = f"2008–{datetime.now().year}, Scrapy developers" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -57,50 +57,51 @@ copyright = f'2008–{datetime.now().year}, Scrapy developers' # The short X.Y version. try: import scrapy - version = '.'.join(map(str, scrapy.version_info[:2])) + + version = ".".join(map(str, scrapy.version_info[:2])) release = scrapy.__version__ except ImportError: - version = '' - release = '' + version = "" + release = "" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -language = 'en' +language = "en" # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. -#unused_docs = [] +# unused_docs = [] -exclude_patterns = ['build'] +exclude_patterns = ["build"] # List of directories, relative to source directory, that shouldn't be searched # for source files. -exclude_trees = ['.build'] +exclude_trees = [".build"] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # List of Sphinx warnings that will not be raised -suppress_warnings = ['epub.unknown_project_files'] +suppress_warnings = ["epub.unknown_project_files"] # Options for HTML output @@ -108,17 +109,18 @@ suppress_warnings = ['epub.unknown_project_files'] # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # Add path to the RTD explicitly to robustify builds (otherwise might # fail in a clean Debian build env) import sphinx_rtd_theme + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The style sheet to use for HTML and HTML Help pages. A file of that name @@ -128,44 +130,44 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = "%b %d, %Y" # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_use_modindex = True +# html_use_modindex = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, the reST sources are included in the HTML build as _sources/. html_copy_source = True @@ -173,16 +175,16 @@ html_copy_source = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = '' +# html_file_suffix = '' # Output file base name for HTML help builder. -htmlhelp_basename = 'Scrapydoc' +htmlhelp_basename = "Scrapydoc" html_css_files = [ - 'custom.css', + "custom.css", ] @@ -190,34 +192,33 @@ html_css_files = [ # ------------------------ # The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' +# latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' +# latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ - ('index', 'Scrapy.tex', 'Scrapy Documentation', - 'Scrapy developers', 'manual'), + ("index", "Scrapy.tex", "Scrapy Documentation", "Scrapy developers", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # Additional stuff for the LaTeX preamble. -#latex_preamble = '' +# latex_preamble = '' # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_use_modindex = True +# latex_use_modindex = True # Options for the linkcheck builder @@ -226,8 +227,9 @@ latex_documents = [ # A list of regular expressions that match URIs that should not be checked when # doing a linkcheck build. linkcheck_ignore = [ - 'http://localhost:\d+', 'http://hg.scrapy.org', - 'http://directory.google.com/' + "http://localhost:\d+", + "http://hg.scrapy.org", + "http://directory.google.com/", ] @@ -237,44 +239,35 @@ coverage_ignore_pyobjects = [ # Contract’s add_pre_hook and add_post_hook are not documented because # they should be transparent to contract developers, for whom pre_hook and # post_hook should be the actual concern. - r'\bContract\.add_(pre|post)_hook$', - + r"\bContract\.add_(pre|post)_hook$", # ContractsManager is an internal class, developers are not expected to # interact with it directly in any way. - r'\bContractsManager\b$', - + r"\bContractsManager\b$", # For default contracts we only want to document their general purpose in # their __init__ method, the methods they reimplement to achieve that purpose # should be irrelevant to developers using those contracts. - r'\w+Contract\.(adjust_request_args|(pre|post)_process)$', - + r"\w+Contract\.(adjust_request_args|(pre|post)_process)$", # Methods of downloader middlewares are not documented, only the classes # themselves, since downloader middlewares are controlled through Scrapy # settings. - r'^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.', - + r"^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.", # Base classes of downloader middlewares are implementation details that # are not meant for users. - r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware', - + r"^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware", # Private exception used by the command-line interface implementation. - r'^scrapy\.exceptions\.UsageError', - + r"^scrapy\.exceptions\.UsageError", # Methods of BaseItemExporter subclasses are only documented in # BaseItemExporter. - r'^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.', - + r"^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.", # Extension behavior is only modified through settings. Methods of # extension classes, as well as helper functions, are implementation # details that are not documented. - r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.', # methods - r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions - + r"^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.", # methods + r"^scrapy\.extensions\.[a-z]\w*?\.[a-z]", # helper functions # Never documented before, and deprecated now. - r'^scrapy\.linkextractors\.FilteringLinkExtractor$', - + r"^scrapy\.linkextractors\.FilteringLinkExtractor$", # Implementation detail of LxmlLinkExtractor - r'^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor', + r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor", ] @@ -282,18 +275,18 @@ coverage_ignore_pyobjects = [ # ------------------------------------- intersphinx_mapping = { - 'attrs': ('https://www.attrs.org/en/stable/', None), - 'coverage': ('https://coverage.readthedocs.io/en/stable', None), - 'cryptography' : ('https://cryptography.io/en/latest/', None), - 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), - 'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None), - 'pytest': ('https://docs.pytest.org/en/latest', None), - 'python': ('https://docs.python.org/3', None), - 'sphinx': ('https://www.sphinx-doc.org/en/master', None), - 'tox': ('https://tox.wiki/en/latest/', None), - 'twisted': ('https://docs.twisted.org/en/stable/', None), - 'twistedapi': ('https://docs.twisted.org/en/stable/api/', None), - 'w3lib': ('https://w3lib.readthedocs.io/en/latest', None), + "attrs": ("https://www.attrs.org/en/stable/", None), + "coverage": ("https://coverage.readthedocs.io/en/stable", None), + "cryptography": ("https://cryptography.io/en/latest/", None), + "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), + "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), + "pytest": ("https://docs.pytest.org/en/latest", None), + "python": ("https://docs.python.org/3", None), + "sphinx": ("https://www.sphinx-doc.org/en/master", None), + "tox": ("https://tox.wiki/en/latest/", None), + "twisted": ("https://docs.twisted.org/en/stable/", None), + "twistedapi": ("https://docs.twisted.org/en/stable/api/", None), + "w3lib": ("https://w3lib.readthedocs.io/en/latest", None), } intersphinx_disabled_reftypes = [] @@ -313,16 +306,16 @@ hoverxref_role_types = { "setting": "tooltip", "signal": "tooltip", } -hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal'] +hoverxref_roles = ["command", "reqmeta", "setting", "signal"] def setup(app): - app.connect('autodoc-skip-member', maybe_skip_member) + app.connect("autodoc-skip-member", maybe_skip_member) def maybe_skip_member(app, what, name, obj, skip, options): if not skip: # autodocs was generating a text "alias of" for the following members # https://github.com/sphinx-doc/sphinx/issues/4422 - return name in {'default_item_class', 'default_selector_class'} + return name in {"default_item_class", "default_selector_class"} return skip diff --git a/docs/conftest.py b/docs/conftest.py index a6dacd265..32f849a36 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -15,20 +15,20 @@ from scrapy.http.response.html import HtmlResponse def load_response(url: str, filename: str) -> HtmlResponse: - input_path = Path(__file__).parent / '_tests' / filename + input_path = Path(__file__).parent / "_tests" / filename return HtmlResponse(url, body=input_path.read_bytes()) def setup(namespace): - namespace['load_response'] = load_response + namespace["load_response"] = load_response pytest_collect_file = Sybil( parsers=[ DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE), - PythonCodeBlockParser(future_imports=['print_function']), + PythonCodeBlockParser(future_imports=["print_function"]), skip, ], - pattern='*.rst', + pattern="*.rst", setup=setup, ).pytest() diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index a0d9a1cbd..efb4c202e 100644 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -25,7 +25,7 @@ def main(): _contents = None # A regex that matches standard linkcheck output lines - line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') + line_re = re.compile(r"(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))") # Read lines from the linkcheck output file try: @@ -66,5 +66,5 @@ def main(): print("Not Understood: " + line) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index a6472b1ba..622164c75 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -7,7 +7,6 @@ from twisted.internet import reactor class Root(Resource): - def __init__(self): Resource.__init__(self) self.concurrent = 0 @@ -26,9 +25,9 @@ class Root(Resource): delta = now - self.lasttime # reset stats on high iter-request times caused by client restarts - if delta > 3: # seconds + if delta > 3: # seconds self._reset_stats() - return '' + return "" self.tail.appendleft(delta) self.lasttime = now @@ -37,15 +36,17 @@ class Root(Resource): if now - self.lastmark >= 3: self.lastmark = now qps = len(self.tail) / sum(self.tail) - print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}') + print( + f"samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}" + ) - if 'latency' in request.args: - latency = float(request.args['latency'][0]) + if "latency" in request.args: + latency = float(request.args["latency"][0]) reactor.callLater(latency, self._finish, request) return NOT_DONE_YET self.concurrent -= 1 - return '' + return "" def _finish(self, request): self.concurrent -= 1 diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 28703650d..83bb08561 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -13,13 +13,13 @@ from scrapy.http import Request class QPSSpider(Spider): - name = 'qps' - benchurl = 'http://localhost:8880/' + name = "qps" + benchurl = "http://localhost:8880/" # Max concurrency is limited by global CONCURRENT_REQUESTS setting max_concurrent_requests = 8 # Requests per second goal - qps = None # same as: 1 / download_delay + qps = None # same as: 1 / download_delay download_delay = None # time in seconds to delay server responses latency = None @@ -37,11 +37,11 @@ class QPSSpider(Spider): def start_requests(self): url = self.benchurl if self.latency is not None: - url += f'?latency={self.latency}' + url += f"?latency={self.latency}" slots = int(self.slots) if slots > 1: - urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)] + urls = [url.replace("localhost", f"127.0.0.{x + 1}") for x in range(slots)] else: urls = [url] diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 86e584396..f0d85198d 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -16,14 +16,21 @@ from scrapy.item import Item, Field __all__ = [ - '__version__', 'version_info', 'twisted_version', 'Spider', - 'Request', 'FormRequest', 'Selector', 'Item', 'Field', + "__version__", + "version_info", + "twisted_version", + "Spider", + "Request", + "FormRequest", + "Selector", + "Item", + "Field", ] # Scrapy and Twisted versions __version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip() -version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.')) +version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split(".")) twisted_version = (_txv.major, _txv.minor, _txv.micro) @@ -34,7 +41,7 @@ if sys.version_info < (3, 7): # Ignore noisy twisted deprecation warnings -warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted') +warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted") del pkgutil diff --git a/scrapy/__main__.py b/scrapy/__main__.py index e467e057f..697b9b1e9 100644 --- a/scrapy/__main__.py +++ b/scrapy/__main__.py @@ -1,4 +1,4 @@ from scrapy.cmdline import execute -if __name__ == '__main__': +if __name__ == "__main__": execute() diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 8218a51c8..1d74923a9 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -17,7 +17,7 @@ from scrapy.utils.python import garbage_collect class ScrapyArgumentParser(argparse.ArgumentParser): def _parse_optional(self, arg_string): # if starts with -: it means that is a parameter not a argument - if arg_string[:2] == '-:': + if arg_string[:2] == "-:": return None return super()._parse_optional(arg_string) @@ -41,12 +41,12 @@ def _get_commands_from_module(module, inproject): d = {} for cmd in _iter_command_classes(module): if inproject or not cmd.requires_project: - cmdname = cmd.__module__.split('.')[-1] + cmdname = cmd.__module__.split(".")[-1] d[cmdname] = cmd() return d -def _get_commands_from_entry_points(inproject, group='scrapy.commands'): +def _get_commands_from_entry_points(inproject, group="scrapy.commands"): cmds = {} for entry_point in pkg_resources.iter_entry_points(group): obj = entry_point.load() @@ -58,9 +58,9 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'): def _get_commands_dict(settings, inproject): - cmds = _get_commands_from_module('scrapy.commands', inproject) + cmds = _get_commands_from_module("scrapy.commands", inproject) cmds.update(_get_commands_from_entry_points(inproject)) - cmds_module = settings['COMMANDS_MODULE'] + cmds_module = settings["COMMANDS_MODULE"] if cmds_module: cmds.update(_get_commands_from_module(cmds_module, inproject)) return cmds @@ -69,7 +69,7 @@ def _get_commands_dict(settings, inproject): def _pop_command_name(argv): i = 0 for arg in argv[1:]: - if not arg.startswith('-'): + if not arg.startswith("-"): del argv[i] return arg i += 1 @@ -124,11 +124,11 @@ def execute(argv=None, settings=None): settings = get_project_settings() # set EDITOR from environment if available try: - editor = os.environ['EDITOR'] + editor = os.environ["EDITOR"] except KeyError: pass else: - settings['EDITOR'] = editor + settings["EDITOR"] = editor inproject = inside_project() cmds = _get_commands_dict(settings, inproject) @@ -141,11 +141,13 @@ def execute(argv=None, settings=None): sys.exit(2) cmd = cmds[cmdname] - parser = ScrapyArgumentParser(formatter_class=ScrapyHelpFormatter, - usage=f"scrapy {cmdname} {cmd.syntax()}", - conflict_handler='resolve', - description=cmd.long_desc()) - settings.setdict(cmd.default_settings, priority='command') + parser = ScrapyArgumentParser( + formatter_class=ScrapyHelpFormatter, + usage=f"scrapy {cmdname} {cmd.syntax()}", + conflict_handler="resolve", + description=cmd.long_desc(), + ) + settings.setdict(cmd.default_settings, priority="command") cmd.settings = settings cmd.add_options(parser) opts, args = parser.parse_known_args(args=argv[1:]) @@ -168,12 +170,12 @@ def _run_command_profiled(cmd, args, opts): sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() p = cProfile.Profile() - p.runctx('cmd.run(args, opts)', globals(), loc) + p.runctx("cmd.run(args, opts)", globals(), loc) if opts.profile: p.dump_stats(opts.profile) -if __name__ == '__main__': +if __name__ == "__main__": try: execute() finally: diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index b9ba3335e..f37d61321 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -27,7 +27,7 @@ class ScrapyCommand: self.settings: Any = None # set in scrapy.cmdline def set_crawler(self, crawler): - if hasattr(self, '_crawler'): + if hasattr(self, "_crawler"): raise RuntimeError("crawler already set") self._crawler = crawler @@ -61,41 +61,58 @@ class ScrapyCommand: """ Populate option parse with options available for this command """ - group = parser.add_argument_group(title='Global Options') - group.add_argument("--logfile", metavar="FILE", - help="log file. if omitted stderr will be used") - group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None, - help=f"log level (default: {self.settings['LOG_LEVEL']})") - group.add_argument("--nolog", action="store_true", - help="disable logging completely") - group.add_argument("--profile", metavar="FILE", default=None, - help="write python cProfile stats to FILE") - group.add_argument("--pidfile", metavar="FILE", - help="write process ID to FILE") - group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE", - help="set/override setting (may be repeated)") + group = parser.add_argument_group(title="Global Options") + group.add_argument( + "--logfile", metavar="FILE", help="log file. if omitted stderr will be used" + ) + group.add_argument( + "-L", + "--loglevel", + metavar="LEVEL", + default=None, + help=f"log level (default: {self.settings['LOG_LEVEL']})", + ) + group.add_argument( + "--nolog", action="store_true", help="disable logging completely" + ) + group.add_argument( + "--profile", + metavar="FILE", + default=None, + help="write python cProfile stats to FILE", + ) + group.add_argument("--pidfile", metavar="FILE", help="write process ID to FILE") + group.add_argument( + "-s", + "--set", + action="append", + default=[], + metavar="NAME=VALUE", + help="set/override setting (may be repeated)", + ) group.add_argument("--pdb", action="store_true", help="enable pdb on failure") def process_options(self, args, opts): try: - self.settings.setdict(arglist_to_dict(opts.set), - priority='cmdline') + self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False) if opts.logfile: - self.settings.set('LOG_ENABLED', True, priority='cmdline') - self.settings.set('LOG_FILE', opts.logfile, priority='cmdline') + self.settings.set("LOG_ENABLED", True, priority="cmdline") + self.settings.set("LOG_FILE", opts.logfile, priority="cmdline") if opts.loglevel: - self.settings.set('LOG_ENABLED', True, priority='cmdline') - self.settings.set('LOG_LEVEL', opts.loglevel, priority='cmdline') + self.settings.set("LOG_ENABLED", True, priority="cmdline") + self.settings.set("LOG_LEVEL", opts.loglevel, priority="cmdline") if opts.nolog: - self.settings.set('LOG_ENABLED', False, priority='cmdline') + self.settings.set("LOG_ENABLED", False, priority="cmdline") if opts.pidfile: - Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep, encoding="utf-8") + Path(opts.pidfile).write_text( + str(os.getpid()) + os.linesep, encoding="utf-8" + ) if opts.pdb: failure.startDebugMode() @@ -111,18 +128,39 @@ class BaseRunSpiderCommand(ScrapyCommand): """ Common class used to share functionality between the crawl, parse and runspider commands """ + def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_argument("-o", "--output", metavar="FILE", action="append", - help="append scraped items to the end of FILE (use - for stdout)," - " to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)") - parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append", - help="dump scraped items into FILE, overwriting any existing file," - " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)") - parser.add_argument("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items") + parser.add_argument( + "-a", + dest="spargs", + action="append", + default=[], + metavar="NAME=VALUE", + help="set spider argument (may be repeated)", + ) + parser.add_argument( + "-o", + "--output", + metavar="FILE", + action="append", + help="append scraped items to the end of FILE (use - for stdout)," + " to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)", + ) + parser.add_argument( + "-O", + "--overwrite-output", + metavar="FILE", + action="append", + help="dump scraped items into FILE, overwriting any existing file," + " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)", + ) + parser.add_argument( + "-t", + "--output-format", + metavar="FORMAT", + help="format to use for dumping items", + ) def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -137,16 +175,21 @@ class BaseRunSpiderCommand(ScrapyCommand): opts.output_format, opts.overwrite_output, ) - self.settings.set('FEEDS', feeds, priority='cmdline') + self.settings.set("FEEDS", feeds, priority="cmdline") class ScrapyHelpFormatter(argparse.HelpFormatter): """ Help Formatter for scrapy command line help messages. """ + def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): - super().__init__(prog, indent_increment=indent_increment, - max_help_position=max_help_position, width=width) + super().__init__( + prog, + indent_increment=indent_increment, + max_help_position=max_help_position, + width=width, + ) def _join_parts(self, part_strings): parts = self.format_part_strings(part_strings) @@ -157,11 +200,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): Underline and title case command line help message headers. """ if part_strings and part_strings[0].startswith("usage: "): - part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):] - headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')] + part_strings[0] = "Usage\n=====\n " + part_strings[0][len("usage: ") :] + headings = [ + i for i in range(len(part_strings)) if part_strings[i].endswith(":\n") + ] for index in headings[::-1]: - char = '-' if "Global Options" in part_strings[index] else '=' + char = "-" if "Global Options" in part_strings[index] else "=" part_strings[index] = part_strings[index][:-2].title() - underline = ''.join(["\n", (char * len(part_strings[index])), "\n"]) + underline = "".join(["\n", (char * len(part_strings[index])), "\n"]) part_strings.insert(index + 1, underline) return part_strings diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 6bdf9eae0..2e2a21f00 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -11,9 +11,9 @@ from scrapy.linkextractors import LinkExtractor class Command(ScrapyCommand): default_settings = { - 'LOG_LEVEL': 'INFO', - 'LOGSTATS_INTERVAL': 1, - 'CLOSESPIDER_TIMEOUT': 10, + "LOG_LEVEL": "INFO", + "LOGSTATS_INTERVAL": 1, + "CLOSESPIDER_TIMEOUT": 10, } def short_desc(self): @@ -26,12 +26,11 @@ class Command(ScrapyCommand): class _BenchServer: - def __enter__(self): from scrapy.utils.test import get_testenv - pargs = [sys.executable, '-u', '-m', 'scrapy.utils.benchserver'] - self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, - env=get_testenv()) + + pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] + self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv()) self.proc.stdout.readline() def __exit__(self, exc_type, exc_value, traceback): @@ -42,15 +41,16 @@ class _BenchServer: class _BenchSpider(scrapy.Spider): """A spider that follows all links""" - name = 'follow' + + name = "follow" total = 10000 show = 20 - baseurl = 'http://localhost:8998' + baseurl = "http://localhost:8998" link_extractor = LinkExtractor() def start_requests(self): - qargs = {'total': self.total, 'show': self.show} - url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}' + qargs = {"total": self.total, "show": self.show} + url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" return [scrapy.Request(url, dont_filter=True)] def parse(self, response): diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index a16f4beb7..efc7a46ed 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -39,7 +39,7 @@ class TextTestResult(_TextTestResult): class Command(ScrapyCommand): requires_project = True - default_settings = {'LOG_ENABLED': False} + default_settings = {"LOG_ENABLED": False} def syntax(self): return "[options] " @@ -49,14 +49,25 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_argument("-l", "--list", dest="list", action="store_true", - help="only list contracts, without checking them") - parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true', - help="print contract tests for all spiders") + parser.add_argument( + "-l", + "--list", + dest="list", + action="store_true", + help="only list contracts, without checking them", + ) + parser.add_argument( + "-v", + "--verbose", + dest="verbose", + default=False, + action="store_true", + help="print contract tests for all spiders", + ) def run(self, args, opts): # load contracts - contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS')) + contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) conman = ContractsManager(load_object(c) for c in contracts) runner = TextTestRunner(verbosity=2 if opts.verbose else 1) result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity) @@ -66,7 +77,7 @@ class Command(ScrapyCommand): spider_loader = self.crawler_process.spider_loader - with set_environ(SCRAPY_CHECK='true'): + with set_environ(SCRAPY_CHECK="true"): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) spidercls.start_requests = lambda s: conman.from_spider(s, result) @@ -85,7 +96,7 @@ class Command(ScrapyCommand): continue print(spider) for method in sorted(methods): - print(f' * {method}') + print(f" * {method}") else: start = time.time() self.crawler_process.start() diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 0f2a21b85..df8006f36 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -16,18 +16,23 @@ class Command(BaseRunSpiderCommand): if len(args) < 1: raise UsageError() elif len(args) > 1: - raise UsageError("running 'scrapy crawl' with more than one spider is not supported") + raise UsageError( + "running 'scrapy crawl' with more than one spider is not supported" + ) spname = args[0] crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) - if getattr(crawl_defer, 'result', None) is not None and issubclass(crawl_defer.result.type, Exception): + if getattr(crawl_defer, "result", None) is not None and issubclass( + crawl_defer.result.type, Exception + ): self.exitcode = 1 else: self.crawler_process.start() if ( self.crawler_process.bootstrap_failed - or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception + or hasattr(self.crawler_process, "has_exception") + and self.crawler_process.has_exception ): self.exitcode = 1 diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 177b20143..537b2013c 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -8,7 +8,7 @@ from scrapy.exceptions import UsageError class Command(ScrapyCommand): requires_project = True - default_settings = {'LOG_ENABLED': False} + default_settings = {"LOG_ENABLED": False} def syntax(self): return "" @@ -17,8 +17,10 @@ class Command(ScrapyCommand): return "Edit spider" def long_desc(self): - return ("Edit a spider using the editor defined in the EDITOR environment" - " variable or else the EDITOR setting") + return ( + "Edit a spider using the editor defined in the EDITOR environment" + " variable or else the EDITOR setting" + ) def _err(self, msg): sys.stderr.write(msg + os.linesep) @@ -28,12 +30,12 @@ class Command(ScrapyCommand): if len(args) != 1: raise UsageError() - editor = self.settings['EDITOR'] + editor = self.settings["EDITOR"] try: spidercls = self.crawler_process.spider_loader.load(args[0]) except KeyError: return self._err(f"Spider not found: {args[0]}") sfile = sys.modules[spidercls.__module__].__file__ - sfile = sfile.replace('.pyc', '.py') + sfile = sfile.replace(".pyc", ".py") self.exitcode = os.system(f'{editor} "{sfile}"') diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 9b2ebb37f..918db55c6 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -27,38 +27,51 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_argument("--spider", dest="spider", help="use this spider") - parser.add_argument("--headers", dest="headers", action="store_true", - help="print response HTTP headers instead of body") - parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False, - help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_argument( + "--headers", + dest="headers", + action="store_true", + help="print response HTTP headers instead of body", + ) + parser.add_argument( + "--no-redirect", + dest="no_redirect", + action="store_true", + default=False, + help="do not handle HTTP 3xx status codes and print response as-is", + ) def _print_headers(self, headers, prefix): for key, values in headers.items(): for value in values: - self._print_bytes(prefix + b' ' + key + b': ' + value) + self._print_bytes(prefix + b" " + key + b": " + value) def _print_response(self, response, opts): if opts.headers: - self._print_headers(response.request.headers, b'>') - print('>') - self._print_headers(response.headers, b'<') + self._print_headers(response.request.headers, b">") + print(">") + self._print_headers(response.headers, b"<") else: self._print_bytes(response.body) def _print_bytes(self, bytes_): - sys.stdout.buffer.write(bytes_ + b'\n') + sys.stdout.buffer.write(bytes_ + b"\n") def run(self, args, opts): if len(args) != 1 or not is_url(args[0]): raise UsageError() - request = Request(args[0], callback=self._print_response, - cb_kwargs={"opts": opts}, dont_filter=True) + request = Request( + args[0], + callback=self._print_response, + cb_kwargs={"opts": opts}, + dont_filter=True, + ) # by default, let the framework handle redirects, # i.e. command handles all codes expect 3xx if not opts.no_redirect: - request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400)) + request.meta["handle_httpstatus_list"] = SequenceExclude(range(300, 400)) else: - request.meta['handle_httpstatus_all'] = True + request.meta["handle_httpstatus_all"] = True spidercls = DefaultSpider spider_loader = self.crawler_process.spider_loader diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index b7c2f85fb..e880e44a9 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -18,7 +18,7 @@ def sanitize_module_name(module_name): with underscores and prefixing it with a letter if it doesn't start with one """ - module_name = module_name.replace('-', '_').replace('.', '_') + module_name = module_name.replace("-", "_").replace(".", "_") if module_name[0] not in string.ascii_letters: module_name = "a" + module_name return module_name @@ -27,7 +27,7 @@ def sanitize_module_name(module_name): def extract_domain(url): """Extract domain name from URL string""" o = urlparse(url) - if o.scheme == '' and o.netloc == '': + if o.scheme == "" and o.netloc == "": o = urlparse("//" + url.lstrip("/")) return o.netloc @@ -35,7 +35,7 @@ def extract_domain(url): class Command(ScrapyCommand): requires_project = False - default_settings = {'LOG_ENABLED': False} + default_settings = {"LOG_ENABLED": False} def syntax(self): return "[options] " @@ -45,16 +45,40 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_argument("-l", "--list", dest="list", action="store_true", - help="List available templates") - parser.add_argument("-e", "--edit", dest="edit", action="store_true", - help="Edit spider after creating it") - parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE", - help="Dump template to standard output") - parser.add_argument("-t", "--template", dest="template", default="basic", - help="Uses a custom template.") - parser.add_argument("--force", dest="force", action="store_true", - help="If the spider already exists, overwrite it with the template") + parser.add_argument( + "-l", + "--list", + dest="list", + action="store_true", + help="List available templates", + ) + parser.add_argument( + "-e", + "--edit", + dest="edit", + action="store_true", + help="Edit spider after creating it", + ) + parser.add_argument( + "-d", + "--dump", + dest="dump", + metavar="TEMPLATE", + help="Dump template to standard output", + ) + parser.add_argument( + "-t", + "--template", + dest="template", + default="basic", + help="Uses a custom template.", + ) + parser.add_argument( + "--force", + dest="force", + action="store_true", + help="If the spider already exists, overwrite it with the template", + ) def run(self, args, opts): if opts.list: @@ -72,7 +96,7 @@ class Command(ScrapyCommand): domain = extract_domain(url) module = sanitize_module_name(name) - if self.settings.get('BOT_NAME') == module: + if self.settings.get("BOT_NAME") == module: print("Cannot create a spider with the same name as your project") return @@ -87,17 +111,17 @@ class Command(ScrapyCommand): def _genspider(self, module, name, domain, template_name, template_file): """Generate the spider module, based on the given template""" - capitalized_module = ''.join(s.capitalize() for s in module.split('_')) + capitalized_module = "".join(s.capitalize() for s in module.split("_")) tvars = { - 'project_name': self.settings.get('BOT_NAME'), - 'ProjectName': string_camelcase(self.settings.get('BOT_NAME')), - 'module': module, - 'name': name, - 'domain': domain, - 'classname': f'{capitalized_module}Spider' + "project_name": self.settings.get("BOT_NAME"), + "ProjectName": string_camelcase(self.settings.get("BOT_NAME")), + "module": module, + "name": name, + "domain": domain, + "classname": f"{capitalized_module}Spider", } - if self.settings.get('NEWSPIDER_MODULE'): - spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) + if self.settings.get("NEWSPIDER_MODULE"): + spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) spiders_dir = Path(spiders_module.__file__).parent.resolve() else: spiders_module = None @@ -105,13 +129,15 @@ class Command(ScrapyCommand): spider_file = f"{spiders_dir / module}.py" shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - print(f"Created spider {name!r} using template {template_name!r} ", - end=('' if spiders_module else '\n')) + print( + f"Created spider {name!r} using template {template_name!r} ", + end=("" if spiders_module else "\n"), + ) if spiders_module: print(f"in module:\n {spiders_module.__name__}.{module}") def _find_template(self, template: str) -> Optional[Path]: - template_file = Path(self.templates_dir, f'{template}.tmpl') + template_file = Path(self.templates_dir, f"{template}.tmpl") if template_file.exists(): return template_file print(f"Unable to find template: {template}\n") @@ -121,11 +147,11 @@ class Command(ScrapyCommand): def _list_templates(self): print("Available templates:") for file in sorted(Path(self.templates_dir).iterdir()): - if file.suffix == '.tmpl': + if file.suffix == ".tmpl": print(f" {file.stem}") def _spider_exists(self, name: str) -> bool: - if not self.settings.get('NEWSPIDER_MODULE'): + if not self.settings.get("NEWSPIDER_MODULE"): # if run as a standalone command and file with same filename already exists path = Path(name + ".py") if path.exists(): @@ -148,7 +174,7 @@ class Command(ScrapyCommand): return True # a file with the same name exists in the target directory - spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) + spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) spiders_dir = Path(cast(str, spiders_module.__file__)).parent spiders_dir_abs = spiders_dir.resolve() path = spiders_dir_abs / (name + ".py") @@ -160,7 +186,9 @@ class Command(ScrapyCommand): @property def templates_dir(self) -> str: - return str(Path( - self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'), - 'spiders' - )) + return str( + Path( + self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"), + "spiders", + ) + ) diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 54d7bb228..2a81bd182 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -4,7 +4,7 @@ from scrapy.commands import ScrapyCommand class Command(ScrapyCommand): requires_project = True - default_settings = {'LOG_ENABLED': False} + default_settings = {"LOG_ENABLED": False} def short_desc(self): return "List available spiders" diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index c2df7f8a5..ac97b6193 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -32,28 +32,72 @@ class Command(BaseRunSpiderCommand): def add_options(self, parser): BaseRunSpiderCommand.add_options(self, parser) - parser.add_argument("--spider", dest="spider", default=None, - help="use this spider without looking for one") - parser.add_argument("--pipelines", action="store_true", - help="process items through pipelines") - parser.add_argument("--nolinks", dest="nolinks", action="store_true", - help="don't show links to follow (extracted requests)") - parser.add_argument("--noitems", dest="noitems", action="store_true", - help="don't show scraped items") - parser.add_argument("--nocolour", dest="nocolour", action="store_true", - help="avoid using pygments to colorize the output") - parser.add_argument("-r", "--rules", dest="rules", action="store_true", - help="use CrawlSpider rules to discover the callback") - parser.add_argument("-c", "--callback", dest="callback", - help="use this callback for parsing, instead looking for a callback") - parser.add_argument("-m", "--meta", dest="meta", - help="inject extra meta into the Request, it must be a valid raw json string") - parser.add_argument("--cbkwargs", dest="cbkwargs", - help="inject extra callback kwargs into the Request, it must be a valid raw json string") - parser.add_argument("-d", "--depth", dest="depth", type=int, default=1, - help="maximum depth for parsing requests [default: %(default)s]") - parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", - help="print each depth level one by one") + parser.add_argument( + "--spider", + dest="spider", + default=None, + help="use this spider without looking for one", + ) + parser.add_argument( + "--pipelines", action="store_true", help="process items through pipelines" + ) + parser.add_argument( + "--nolinks", + dest="nolinks", + action="store_true", + help="don't show links to follow (extracted requests)", + ) + parser.add_argument( + "--noitems", + dest="noitems", + action="store_true", + help="don't show scraped items", + ) + parser.add_argument( + "--nocolour", + dest="nocolour", + action="store_true", + help="avoid using pygments to colorize the output", + ) + parser.add_argument( + "-r", + "--rules", + dest="rules", + action="store_true", + help="use CrawlSpider rules to discover the callback", + ) + parser.add_argument( + "-c", + "--callback", + dest="callback", + help="use this callback for parsing, instead looking for a callback", + ) + parser.add_argument( + "-m", + "--meta", + dest="meta", + help="inject extra meta into the Request, it must be a valid raw json string", + ) + parser.add_argument( + "--cbkwargs", + dest="cbkwargs", + help="inject extra callback kwargs into the Request, it must be a valid raw json string", + ) + parser.add_argument( + "-d", + "--depth", + dest="depth", + type=int, + default=1, + help="maximum depth for parsing requests [default: %(default)s]", + ) + parser.add_argument( + "-v", + "--verbose", + dest="verbose", + action="store_true", + help="print each depth level one by one", + ) @property def max_level(self): @@ -98,13 +142,13 @@ class Command(BaseRunSpiderCommand): if opts.verbose: for level in range(1, self.max_level + 1): - print(f'\n>>> DEPTH LEVEL: {level} <<<') + print(f"\n>>> DEPTH LEVEL: {level} <<<") if not opts.noitems: self.print_items(level, colour) if not opts.nolinks: self.print_requests(level, colour) else: - print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<') + print(f"\n>>> STATUS DEPTH LEVEL {self.max_level} <<<") if not opts.noitems: self.print_items(colour=colour) if not opts.nolinks: @@ -125,14 +169,16 @@ class Command(BaseRunSpiderCommand): return d def get_callback_from_rules(self, spider, response): - if getattr(spider, 'rules', None): + if getattr(spider, "rules", None): for rule in spider.rules: if rule.link_extractor.matches(response.url): return rule.callback or "parse" else: - logger.error('No CrawlSpider rules found in spider %(spider)r, ' - 'please specify a callback to use for parsing', - {'spider': spider.name}) + logger.error( + "No CrawlSpider rules found in spider %(spider)r, " + "please specify a callback to use for parsing", + {"spider": spider.name}, + ) def set_spidercls(self, url, opts): spider_loader = self.crawler_process.spider_loader @@ -140,15 +186,17 @@ class Command(BaseRunSpiderCommand): try: self.spidercls = spider_loader.load(opts.spider) except KeyError: - logger.error('Unable to find spider: %(spider)s', - {'spider': opts.spider}) + logger.error( + "Unable to find spider: %(spider)s", {"spider": opts.spider} + ) else: self.spidercls = spidercls_for_request(spider_loader, Request(url)) if not self.spidercls: - logger.error('Unable to find spider for: %(url)s', {'url': url}) + logger.error("Unable to find spider for: %(url)s", {"url": url}) def _start_requests(spider): yield self.prepare_request(spider, Request(url), opts) + if self.spidercls: self.spidercls.start_requests = _start_requests @@ -158,8 +206,7 @@ class Command(BaseRunSpiderCommand): self.crawler_process.start() if not self.first_response: - logger.error('No response downloaded for: %(url)s', - {'url': url}) + logger.error("No response downloaded for: %(url)s", {"url": url}) def scraped_data(self, args): items, requests, opts, depth, spider, callback = args @@ -173,8 +220,8 @@ class Command(BaseRunSpiderCommand): scraped_data = items if opts.output else [] if depth < opts.depth: for req in requests: - req.meta['_depth'] = depth + 1 - req.meta['_callback'] = req.callback + req.meta["_depth"] = depth + 1 + req.meta["_callback"] = req.callback req.callback = callback scraped_data += requests @@ -187,7 +234,7 @@ class Command(BaseRunSpiderCommand): self.first_response = response # determine real callback - cb = response.meta['_callback'] + cb = response.meta["_callback"] if not cb: if opts.callback: cb = opts.callback @@ -195,23 +242,27 @@ class Command(BaseRunSpiderCommand): cb = self.get_callback_from_rules(spider, response) if not cb: - logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s', - {'url': response.url, 'spider': spider.name}) + logger.error( + "Cannot find a rule that matches %(url)r in spider: %(spider)s", + {"url": response.url, "spider": spider.name}, + ) return else: - cb = 'parse' + cb = "parse" if not callable(cb): cb_method = getattr(spider, cb, None) if callable(cb_method): cb = cb_method else: - logger.error('Cannot find callback %(callback)r in spider: %(spider)s', - {'callback': cb, 'spider': spider.name}) + logger.error( + "Cannot find callback %(callback)r in spider: %(spider)s", + {"callback": cb, "spider": spider.name}, + ) return # parse items and requests - depth = response.meta['_depth'] + depth = response.meta["_depth"] d = self.run_callback(response, cb, cb_kwargs) d.addCallback(self._get_items_and_requests, opts, depth, spider, callback) @@ -226,8 +277,8 @@ class Command(BaseRunSpiderCommand): if opts.cbkwargs: request.cb_kwargs.update(opts.cbkwargs) - request.meta['_depth'] = 1 - request.meta['_callback'] = request.callback + request.meta["_depth"] = 1 + request.meta["_callback"] = request.callback request.callback = callback return request @@ -242,16 +293,22 @@ class Command(BaseRunSpiderCommand): try: opts.meta = json.loads(opts.meta) except ValueError: - raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " - "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False) + raise UsageError( + "Invalid -m/--meta value, pass a valid json string to -m or --meta. " + 'Example: --meta=\'{"foo" : "bar"}\'', + print_help=False, + ) def process_request_cb_kwargs(self, opts): if opts.cbkwargs: try: opts.cbkwargs = json.loads(opts.cbkwargs) except ValueError: - raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. " - "Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False) + raise UsageError( + "Invalid --cbkwargs value, pass a valid json string to --cbkwargs. " + 'Example: --cbkwargs=\'{"foo" : "bar"}\'', + print_help=False, + ) def run(self, args, opts): # parse arguments diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 22fa6a53d..9751c6c30 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -12,7 +12,7 @@ from scrapy.commands import BaseRunSpiderCommand def _import_file(filepath: Union[str, PathLike]) -> ModuleType: abspath = Path(filepath).resolve() - if abspath.suffix not in ('.py', '.pyw'): + if abspath.suffix not in (".py", ".pyw"): raise ValueError(f"Not a Python source file: {abspath}") dirname = str(abspath.parent) sys.path = [dirname] + sys.path @@ -26,7 +26,7 @@ def _import_file(filepath: Union[str, PathLike]) -> ModuleType: class Command(BaseRunSpiderCommand): requires_project = False - default_settings = {'SPIDER_LOADER_WARN_ONLY': True} + default_settings = {"SPIDER_LOADER_WARN_ONLY": True} def syntax(self): return "[options] " diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 1b2e2601e..c43298ac7 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -7,8 +7,7 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): requires_project = False - default_settings = {'LOG_ENABLED': False, - 'SPIDER_LOADER_WARN_ONLY': True} + default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} def syntax(self): return "[options]" @@ -18,16 +17,33 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_argument("--get", dest="get", metavar="SETTING", - help="print raw setting value") - parser.add_argument("--getbool", dest="getbool", metavar="SETTING", - help="print setting value, interpreted as a boolean") - parser.add_argument("--getint", dest="getint", metavar="SETTING", - help="print setting value, interpreted as an integer") - parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING", - help="print setting value, interpreted as a float") - parser.add_argument("--getlist", dest="getlist", metavar="SETTING", - help="print setting value, interpreted as a list") + parser.add_argument( + "--get", dest="get", metavar="SETTING", help="print raw setting value" + ) + parser.add_argument( + "--getbool", + dest="getbool", + metavar="SETTING", + help="print setting value, interpreted as a boolean", + ) + parser.add_argument( + "--getint", + dest="getint", + metavar="SETTING", + help="print setting value, interpreted as an integer", + ) + parser.add_argument( + "--getfloat", + dest="getfloat", + metavar="SETTING", + help="print setting value, interpreted as a float", + ) + parser.add_argument( + "--getlist", + dest="getlist", + metavar="SETTING", + help="print setting value, interpreted as a list", + ) def run(self, args, opts): settings = self.crawler_process.settings diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index f67a5886a..1fad8f328 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -16,9 +16,9 @@ class Command(ScrapyCommand): requires_project = False default_settings = { - 'KEEP_ALIVE': True, - 'LOGSTATS_INTERVAL': 0, - 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter', + "KEEP_ALIVE": True, + "LOGSTATS_INTERVAL": 0, + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", } def syntax(self): @@ -28,17 +28,26 @@ class Command(ScrapyCommand): return "Interactive scraping console" def long_desc(self): - return ("Interactive console for scraping the given url or file. " - "Use ./file.html syntax or full path for local file.") + return ( + "Interactive console for scraping the given url or file. " + "Use ./file.html syntax or full path for local file." + ) def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_argument("-c", dest="code", - help="evaluate the code in the shell, print the result and exit") - parser.add_argument("--spider", dest="spider", - help="use this spider") - parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False, - help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_argument( + "-c", + dest="code", + help="evaluate the code in the shell, print the result and exit", + ) + parser.add_argument("--spider", dest="spider", help="use this spider") + parser.add_argument( + "--no-redirect", + dest="no_redirect", + action="store_true", + default=False, + help="do not handle HTTP 3xx status codes and print response as-is", + ) def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -58,8 +67,9 @@ class Command(ScrapyCommand): if opts.spider: spidercls = spider_loader.load(opts.spider) elif url: - spidercls = spidercls_for_request(spider_loader, Request(url), - spidercls, log_multiple=True) + spidercls = spidercls_for_request( + spider_loader, Request(url), spidercls, log_multiple=True + ) # The crawler is created this way since the Shell manually handles the # crawling engine, so the set up in the crawl method won't work @@ -74,7 +84,9 @@ class Command(ScrapyCommand): shell.start(url=url, redirect=not opts.no_redirect) def _start_crawler_thread(self): - t = Thread(target=self.crawler_process.start, - kwargs={'stop_after_crawl': False, 'install_signal_handlers': False}) + t = Thread( + target=self.crawler_process.start, + kwargs={"stop_after_crawl": False, "install_signal_handlers": False}, + ) t.daemon = True t.start() diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 9e1e95ab1..3ed1f5dbc 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -13,14 +13,14 @@ from scrapy.exceptions import UsageError TEMPLATES_TO_RENDER = ( - ('scrapy.cfg',), - ('${project_name}', 'settings.py.tmpl'), - ('${project_name}', 'items.py.tmpl'), - ('${project_name}', 'pipelines.py.tmpl'), - ('${project_name}', 'middlewares.py.tmpl'), + ("scrapy.cfg",), + ("${project_name}", "settings.py.tmpl"), + ("${project_name}", "items.py.tmpl"), + ("${project_name}", "pipelines.py.tmpl"), + ("${project_name}", "middlewares.py.tmpl"), ) -IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn') +IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") def _make_writable(path): @@ -31,8 +31,7 @@ def _make_writable(path): class Command(ScrapyCommand): requires_project = False - default_settings = {'LOG_ENABLED': False, - 'SPIDER_LOADER_WARN_ONLY': True} + default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} def syntax(self): return " [project_dir]" @@ -45,11 +44,13 @@ class Command(ScrapyCommand): spec = find_spec(module_name) return spec is not None and spec.loader is not None - if not re.search(r'^[_a-zA-Z]\w*$', project_name): - print('Error: Project names must begin with a letter and contain' - ' only\nletters, numbers and underscores') + if not re.search(r"^[_a-zA-Z]\w*$", project_name): + print( + "Error: Project names must begin with a letter and contain" + " only\nletters, numbers and underscores" + ) elif _module_exists(project_name): - print(f'Error: Module {project_name!r} already exists') + print(f"Error: Module {project_name!r} already exists") else: return True return False @@ -96,9 +97,9 @@ class Command(ScrapyCommand): else: project_dir = Path(args[0]) - if (project_dir / 'scrapy.cfg').exists(): + if (project_dir / "scrapy.cfg").exists(): self.exitcode = 1 - print(f'Error: scrapy.cfg already exists in {project_dir.resolve()}') + print(f"Error: scrapy.cfg already exists in {project_dir.resolve()}") return if not self._is_valid_name(project_name): @@ -106,12 +107,24 @@ class Command(ScrapyCommand): return self._copytree(Path(self.templates_dir), project_dir.resolve()) - move(project_dir / 'module', project_dir / project_name) + move(project_dir / "module", project_dir / project_name) for paths in TEMPLATES_TO_RENDER: - tplfile = Path(project_dir, *(string.Template(s).substitute(project_name=project_name) for s in paths)) - render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) - print(f"New Scrapy project '{project_name}', using template directory " - f"'{self.templates_dir}', created in:") + tplfile = Path( + project_dir, + *( + string.Template(s).substitute(project_name=project_name) + for s in paths + ), + ) + render_templatefile( + tplfile, + project_name=project_name, + ProjectName=string_camelcase(project_name), + ) + print( + f"New Scrapy project '{project_name}', using template directory " + f"'{self.templates_dir}', created in:" + ) print(f" {project_dir.resolve()}\n") print("You can start your first spider with:") print(f" cd {project_dir}") @@ -119,7 +132,9 @@ class Command(ScrapyCommand): @property def templates_dir(self) -> str: - return str(Path( - self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'), - 'project' - )) + return str( + Path( + self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"), + "project", + ) + ) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index c6a3c273a..faa0e242c 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -5,8 +5,7 @@ from scrapy.utils.versions import scrapy_components_versions class Command(ScrapyCommand): - default_settings = {'LOG_ENABLED': False, - 'SPIDER_LOADER_WARN_ONLY': True} + default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} def syntax(self): return "[-v]" @@ -16,8 +15,13 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_argument("--verbose", "-v", dest="verbose", action="store_true", - help="also display twisted/python/platform info (useful for bug reports)") + parser.add_argument( + "--verbose", + "-v", + dest="verbose", + action="store_true", + help="also display twisted/python/platform info (useful for bug reports)", + ) def run(self, args, opts): if opts.verbose: diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index b1f52abe2..a81af7565 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -4,16 +4,17 @@ from scrapy.utils.response import open_in_browser class Command(fetch.Command): - def short_desc(self): return "Open URL in browser, as seen by Scrapy" def long_desc(self): - return "Fetch a URL using the Scrapy downloader and show its contents in a browser" + return ( + "Fetch a URL using the Scrapy downloader and show its contents in a browser" + ) def add_options(self, parser): super().add_options(parser) - parser.add_argument('--headers', help=argparse.SUPPRESS) + parser.add_argument("--headers", help=argparse.SUPPRESS) def _print_response(self, response, opts): open_in_browser(response) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index b47e55092..86098edca 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -11,16 +11,17 @@ from scrapy.utils.spider import iterate_spider_output class Contract: - """ Abstract class for contracts """ + """Abstract class for contracts""" + request_cls = None def __init__(self, method, *args): - self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') - self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') + self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") + self.testcase_post = _create_testcase(method, f"@{self.name} post-hook") self.args = args def add_pre_hook(self, request, results): - if hasattr(self, 'pre_process'): + if hasattr(self, "pre_process"): cb = request.callback @wraps(cb) @@ -43,7 +44,7 @@ class Contract: return request def add_post_hook(self, request, results): - if hasattr(self, 'post_process'): + if hasattr(self, "post_process"): cb = request.callback @wraps(cb) @@ -88,12 +89,12 @@ class ContractsManager: def extract_contracts(self, method): contracts = [] - for line in method.__doc__.split('\n'): + for line in method.__doc__.split("\n"): line = line.strip() - if line.startswith('@'): - name, args = re.match(r'@(\w+)\s*(.*)', line).groups() - args = re.split(r'\s+', args) + if line.startswith("@"): + name, args = re.match(r"@(\w+)\s*(.*)", line).groups() + args = re.split(r"\s+", args) contracts.append(self.contracts[name](method, *args)) @@ -106,7 +107,7 @@ class ContractsManager: try: requests.append(self.from_method(bound_method, results)) except Exception: - case = _create_testcase(bound_method, 'contract') + case = _create_testcase(bound_method, "contract") results.addError(case, sys.exc_info()) return requests @@ -124,13 +125,13 @@ class ContractsManager: # Don't filter requests to allow # testing different callbacks on the same URL. - kwargs['dont_filter'] = True - kwargs['callback'] = method + kwargs["dont_filter"] = True + kwargs["callback"] = method for contract in contracts: kwargs = contract.adjust_request_args(kwargs) - args.remove('self') + args.remove("self") # check if all positional arguments are defined in kwargs if set(args).issubset(set(kwargs)): @@ -146,7 +147,7 @@ class ContractsManager: return request def _clean_req(self, request, method, results): - """ stop the request from returning objects and records any errors """ + """stop the request from returning objects and records any errors""" cb = request.callback @@ -156,11 +157,11 @@ class ContractsManager: output = cb(response, **cb_kwargs) output = list(iterate_spider_output(output)) except Exception: - case = _create_testcase(method, 'callback') + case = _create_testcase(method, "callback") results.addError(case, sys.exc_info()) def eb_wrapper(failure): - case = _create_testcase(method, 'errback') + case = _create_testcase(method, "errback") exc_info = failure.type, failure.value, failure.getTracebackObject() results.addError(case, exc_info) @@ -175,6 +176,6 @@ def _create_testcase(method, desc): def __str__(_self): return f"[{spider}] {method.__name__} ({desc})" - name = f'{spider}_{method.__name__}' + name = f"{spider}_{method.__name__}" setattr(ContractTestCase, name, lambda x: x) return ContractTestCase(name) diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 9704f5253..e41d83960 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -9,50 +9,50 @@ from scrapy.http import Request # contracts class UrlContract(Contract): - """ Contract to set the url of the request (mandatory) - @url http://scrapy.org + """Contract to set the url of the request (mandatory) + @url http://scrapy.org """ - name = 'url' + name = "url" def adjust_request_args(self, args): - args['url'] = self.args[0] + args["url"] = self.args[0] return args class CallbackKeywordArgumentsContract(Contract): - """ Contract to set the keyword arguments for the request. - The value should be a JSON-encoded dictionary, e.g.: + """Contract to set the keyword arguments for the request. + The value should be a JSON-encoded dictionary, e.g.: - @cb_kwargs {"arg1": "some value"} + @cb_kwargs {"arg1": "some value"} """ - name = 'cb_kwargs' + name = "cb_kwargs" def adjust_request_args(self, args): - args['cb_kwargs'] = json.loads(' '.join(self.args)) + args["cb_kwargs"] = json.loads(" ".join(self.args)) return args class ReturnsContract(Contract): - """ Contract to check the output of a callback + """Contract to check the output of a callback - general form: - @returns request(s)/item(s) [min=1 [max]] + general form: + @returns request(s)/item(s) [min=1 [max]] - e.g.: - @returns request - @returns request 2 - @returns request 2 10 - @returns request 0 10 + e.g.: + @returns request + @returns request 2 + @returns request 2 10 + @returns request 0 10 """ - name = 'returns' + name = "returns" object_type_verifiers = { - 'request': lambda x: isinstance(x, Request), - 'requests': lambda x: isinstance(x, Request), - 'item': is_item, - 'items': is_item, + "request": lambda x: isinstance(x, Request), + "requests": lambda x: isinstance(x, Request), + "item": is_item, + "items": is_item, } def __init__(self, *args, **kwargs): @@ -73,7 +73,7 @@ class ReturnsContract(Contract): try: self.max_bound = int(self.args[2]) except IndexError: - self.max_bound = float('inf') + self.max_bound = float("inf") def post_process(self, output): occurrences = 0 @@ -81,23 +81,25 @@ class ReturnsContract(Contract): if self.obj_type_verifier(x): occurrences += 1 - assertion = (self.min_bound <= occurrences <= self.max_bound) + assertion = self.min_bound <= occurrences <= self.max_bound if not assertion: if self.min_bound == self.max_bound: expected = self.min_bound else: - expected = f'{self.min_bound}..{self.max_bound}' + expected = f"{self.min_bound}..{self.max_bound}" - raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}") + raise ContractFail( + f"Returned {occurrences} {self.obj_name}, expected {expected}" + ) class ScrapesContract(Contract): - """ Contract to check presence of fields in scraped items - @scrapes page_name page_body + """Contract to check presence of fields in scraped items + @scrapes page_name page_body """ - name = 'scrapes' + name = "scrapes" def post_process(self, output): for x in output: diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 4f7ab594f..3a7de8072 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -41,9 +41,11 @@ class Slot: def __repr__(self): cls_name = self.__class__.__name__ - return (f"{cls_name}(concurrency={self.concurrency!r}, " - f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r})") + return ( + f"{cls_name}(concurrency={self.concurrency!r}, " + f"delay={self.delay:.2f}, " + f"randomize_delay={self.randomize_delay!r})" + ) def __str__(self): return ( @@ -56,11 +58,11 @@ class Slot: def _get_concurrency_delay(concurrency, spider, settings): - delay = settings.getfloat('DOWNLOAD_DELAY') - if hasattr(spider, 'download_delay'): + delay = settings.getfloat("DOWNLOAD_DELAY") + if hasattr(spider, "download_delay"): delay = spider.download_delay - if hasattr(spider, 'max_concurrent_requests'): + if hasattr(spider, "max_concurrent_requests"): concurrency = spider.max_concurrent_requests return concurrency, delay @@ -68,7 +70,7 @@ def _get_concurrency_delay(concurrency, spider, settings): class Downloader: - DOWNLOAD_SLOT = 'download_slot' + DOWNLOAD_SLOT = "download_slot" def __init__(self, crawler): self.settings = crawler.settings @@ -76,10 +78,10 @@ class Downloader: self.slots = {} self.active = set() self.handlers = DownloadHandlers(crawler) - self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS') - self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') - self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP') - self.randomize_delay = self.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY') + self.total_concurrency = self.settings.getint("CONCURRENT_REQUESTS") + self.domain_concurrency = self.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") + self.ip_concurrency = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") + self.randomize_delay = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) @@ -99,7 +101,9 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + conc = ( + self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) self.slots[key] = Slot(conc, delay, self.randomize_delay) @@ -109,7 +113,7 @@ class Downloader: if self.DOWNLOAD_SLOT in request.meta: return request.meta[self.DOWNLOAD_SLOT] - key = urlparse_cached(request).hostname or '' + key = urlparse_cached(request).hostname or "" if self.ip_concurrency: key = dnscache.get(key, key) @@ -124,9 +128,9 @@ class Downloader: return response slot.active.add(request) - self.signals.send_catch_log(signal=signals.request_reached_downloader, - request=request, - spider=spider) + self.signals.send_catch_log( + signal=signals.request_reached_downloader, request=request, spider=spider + ) deferred = defer.Deferred().addBoth(_deactivate) slot.queue.append((request, deferred)) self._process_queue(spider, slot) @@ -134,6 +138,7 @@ class Downloader: def _process_queue(self, spider, slot): from twisted.internet import reactor + if slot.latercall and slot.latercall.active(): return @@ -143,7 +148,9 @@ class Downloader: if delay: penalty = delay - now + slot.lastseen if penalty > 0: - slot.latercall = reactor.callLater(penalty, self._process_queue, spider, slot) + slot.latercall = reactor.callLater( + penalty, self._process_queue, spider, slot + ) return # Process enqueued requests if there are free slots to transfer for this slot @@ -166,11 +173,14 @@ class Downloader: # 2. Notify response_downloaded listeners about the recent download # before querying queue for next request def _downloaded(response): - self.signals.send_catch_log(signal=signals.response_downloaded, - response=response, - request=request, - spider=spider) + self.signals.send_catch_log( + signal=signals.response_downloaded, + response=response, + request=request, + spider=spider, + ) return response + dfd.addCallback(_downloaded) # 3. After response arrives, remove the request from transferring @@ -182,9 +192,9 @@ class Downloader: def finish_transferring(_): slot.transferring.remove(request) self._process_queue(spider, slot) - self.signals.send_catch_log(signal=signals.request_left_downloader, - request=request, - spider=spider) + self.signals.send_catch_log( + signal=signals.request_left_downloader, request=request, spider=spider + ) return _ return dfd.addBoth(finish_transferring) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 4abde2238..9dd2df638 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -2,13 +2,22 @@ import warnings from OpenSSL import SSL from twisted.internet._sslverify import _setAcceptableProtocols -from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers +from twisted.internet.ssl import ( + optionsForClientTLS, + CertificateOptions, + platformTrust, + AcceptableCiphers, +) from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer from zope.interface.verify import verifyObject -from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions +from scrapy.core.downloader.tls import ( + DEFAULT_CIPHERS, + openssl_methods, + ScrapyClientTLSOptions, +) from scrapy.utils.misc import create_instance, load_object @@ -24,7 +33,14 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.' """ - def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs): + def __init__( + self, + method=SSL.SSLv23_METHOD, + tls_verbose_logging=False, + tls_ciphers=None, + *args, + **kwargs, + ): super().__init__(*args, **kwargs) self._ssl_method = method self.tls_verbose_logging = tls_verbose_logging @@ -35,9 +51,15 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): @classmethod def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs): - tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING') - tls_ciphers = settings['DOWNLOADER_CLIENT_TLS_CIPHERS'] - return cls(method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, *args, **kwargs) + tls_verbose_logging = settings.getbool("DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING") + tls_ciphers = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + return cls( + method=method, + tls_verbose_logging=tls_verbose_logging, + tls_ciphers=tls_ciphers, + *args, + **kwargs, + ) def getCertificateOptions(self): # setting verify=True will require you to provide CAs @@ -53,7 +75,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # not calling super().__init__ return CertificateOptions( verify=False, - method=getattr(self, 'method', getattr(self, '_ssl_method', None)), + method=getattr(self, "method", getattr(self, "_ssl_method", None)), fixBrokenPeers=True, acceptableCiphers=self.tls_ciphers, ) @@ -64,8 +86,11 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): return self.getCertificateOptions().getContext() def creatorForNetloc(self, hostname, port): - return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(), - verbose_logging=self.tls_verbose_logging) + return ScrapyClientTLSOptions( + hostname.decode("ascii"), + self.getContext(), + verbose_logging=self.tls_verbose_logging, + ) @implementer(IPolicyForHTTPS) @@ -95,7 +120,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): return optionsForClientTLS( hostname=hostname.decode("ascii"), trustRoot=platformTrust(), - extraCertificateOptions={'method': self._ssl_method}, + extraCertificateOptions={"method": self._ssl_method}, ) @@ -118,8 +143,8 @@ class AcceptableProtocolsContextFactory: def load_context_factory_from_settings(settings, crawler): - ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] - context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + ssl_method = openssl_methods[settings.get("DOWNLOADER_CLIENT_TLS_METHOD")] + context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]) # try method-aware context factory try: context_factory = create_instance( diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 73aeb2352..bb2141d28 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -15,14 +15,14 @@ logger = logging.getLogger(__name__) class DownloadHandlers: - def __init__(self, crawler): self._crawler = crawler self._schemes = {} # stores acceptable schemes on instancing self._handlers = {} # stores instanced handlers for schemes self._notconfigured = {} # remembers failed handlers handlers = without_none_values( - crawler.settings.getwithbase('DOWNLOAD_HANDLERS')) + crawler.settings.getwithbase("DOWNLOAD_HANDLERS") + ) for scheme, clspath in handlers.items(): self._schemes[scheme] = clspath self._load_handler(scheme, skip_lazy=True) @@ -38,7 +38,7 @@ class DownloadHandlers: if scheme in self._notconfigured: return None if scheme not in self._schemes: - self._notconfigured[scheme] = 'no handler available for that scheme' + self._notconfigured[scheme] = "no handler available for that scheme" return None return self._load_handler(scheme) @@ -47,7 +47,7 @@ class DownloadHandlers: path = self._schemes[scheme] try: dhcls = load_object(path) - if skip_lazy and getattr(dhcls, 'lazy', True): + if skip_lazy and getattr(dhcls, "lazy", True): return None dh = create_instance( objcls=dhcls, @@ -58,9 +58,12 @@ class DownloadHandlers: self._notconfigured[scheme] = str(ex) return None except Exception as ex: - logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"', - {"clspath": path, "scheme": scheme}, - exc_info=True, extra={'crawler': self._crawler}) + logger.error( + 'Loading "%(clspath)s" for scheme "%(scheme)s"', + {"clspath": path, "scheme": scheme}, + exc_info=True, + extra={"crawler": self._crawler}, + ) self._notconfigured[scheme] = str(ex) return None else: @@ -71,11 +74,13 @@ class DownloadHandlers: scheme = urlparse_cached(request).scheme handler = self._get_handler(scheme) if not handler: - raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}") + raise NotSupported( + f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}" + ) return handler.download_request(request, spider) @defer.inlineCallbacks def _close(self, *_a, **_kw): for dh in self._handlers.values(): - if hasattr(dh, 'close'): + if hasattr(dh, "close"): yield dh.close() diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index a45b4ff3c..8b78c53c1 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -14,9 +14,8 @@ class DataURIDownloadHandler: respcls = responsetypes.from_mimetype(uri.media_type) resp_kwargs = {} - if (issubclass(respcls, TextResponse) - and uri.media_type.split('/')[0] == 'text'): - charset = uri.media_type_parameters.get('charset') - resp_kwargs['encoding'] = charset + if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text": + charset = uri.media_type_parameters.get("charset") + resp_kwargs["encoding"] = charset return respcls(url=request.url, body=uri.data, **resp_kwargs) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index a495874bd..4081545ce 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -71,9 +71,9 @@ class FTPDownloadHandler: } def __init__(self, settings): - self.default_user = settings['FTP_USER'] - self.default_password = settings['FTP_PASSWORD'] - self.passive_mode = settings['FTP_PASSIVE_MODE'] + self.default_user = settings["FTP_USER"] + self.default_password = settings["FTP_PASSWORD"] + self.passive_mode = settings["FTP_PASSIVE_MODE"] @classmethod def from_crawler(cls, crawler): @@ -81,12 +81,16 @@ class FTPDownloadHandler: def download_request(self, request, spider): from twisted.internet import reactor + parsed_url = urlparse_cached(request) user = request.meta.get("ftp_user", self.default_user) password = request.meta.get("ftp_password", self.default_password) - passive_mode = 1 if bool(request.meta.get("ftp_passive", - self.passive_mode)) else 0 - creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode) + passive_mode = ( + 1 if bool(request.meta.get("ftp_passive", self.passive_mode)) else 0 + ) + creator = ClientCreator( + reactor, FTPClient, user, password, passive=passive_mode + ) dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path)) @@ -103,7 +107,7 @@ class FTPDownloadHandler: def _build_response(self, result, request, protocol): self.result = result protocol.close() - headers = {"local filename": protocol.filename or '', "size": protocol.size} + headers = {"local filename": protocol.filename or "", "size": protocol.size} body = to_bytes(protocol.filename or protocol.body.read()) respcls = responsetypes.from_args(url=request.url, body=body) return respcls(url=request.url, status=200, body=body, headers=headers) @@ -115,5 +119,7 @@ class FTPDownloadHandler: if m: ftpcode = m.group() httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"]) - return Response(url=request.url, status=httpcode, body=to_bytes(message)) + return Response( + url=request.url, status=httpcode, body=to_bytes(message) + ) raise result.type(result.value) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index a75532d2a..6c1dac4a5 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -8,8 +8,10 @@ class HTTP10DownloadHandler: lazy = False def __init__(self, settings, crawler=None): - self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) - self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"]) + self.ClientContextFactory = load_object( + settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] + ) self._settings = settings self._crawler = crawler @@ -25,8 +27,9 @@ class HTTP10DownloadHandler: def _connect(self, factory): from twisted.internet import reactor + host, port = to_unicode(factory.host), factory.port - if factory.scheme == b'https': + if factory.scheme == b"https": client_context_factory = create_instance( objcls=self.ClientContextFactory, settings=self._settings, diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1c98e60e1..201c84ff8 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -12,7 +12,13 @@ from twisted.internet import defer, protocol, ssl from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError from twisted.python.failure import Failure -from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI +from twisted.web.client import ( + Agent, + HTTPConnectionPool, + ResponseDone, + ResponseFailed, + URI, +) from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH @@ -36,14 +42,17 @@ class HTTP11DownloadHandler: self._crawler = crawler from twisted.internet import reactor + self._pool = HTTPConnectionPool(reactor, persistent=True) - self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') + self._pool.maxPersistentPerHost = settings.getint( + "CONCURRENT_REQUESTS_PER_DOMAIN" + ) self._pool._factory.noisy = False self._contextFactory = load_context_factory_from_settings(settings, crawler) - self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') - self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') - self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') + self._default_maxsize = settings.getint("DOWNLOAD_MAXSIZE") + self._default_warnsize = settings.getint("DOWNLOAD_WARNSIZE") + self._fail_on_dataloss = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS") self._disconnect_timeout = 1 @classmethod @@ -55,8 +64,8 @@ class HTTP11DownloadHandler: agent = ScrapyAgent( contextFactory=self._contextFactory, pool=self._pool, - maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), - warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), + maxsize=getattr(spider, "download_maxsize", self._default_maxsize), + warnsize=getattr(spider, "download_warnsize", self._default_warnsize), fail_on_dataloss=self._fail_on_dataloss, crawler=self._crawler, ) @@ -64,6 +73,7 @@ class HTTP11DownloadHandler: def close(self): from twisted.internet import reactor + d = self._pool.closeCachedConnections() # closeCachedConnections will hang on network or server issues, so # we'll manually timeout the deferred. @@ -96,11 +106,23 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): with this endpoint comes from the pool and a CONNECT has already been issued for it. """ + _truncatedLength = 1000 - _responseAnswer = r'HTTP/1\.. (?P\d{3})(?P.{,' + str(_truncatedLength) + r'})' + _responseAnswer = ( + r"HTTP/1\.. (?P\d{3})(?P.{," + str(_truncatedLength) + r"})" + ) _responseMatcher = re.compile(_responseAnswer.encode()) - def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): + def __init__( + self, + reactor, + host, + port, + proxyConf, + contextFactory, + timeout=30, + bindAddress=None, + ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) self._tunnelReadyDeferred = defer.Deferred() @@ -111,7 +133,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def requestTunnel(self, protocol): """Asks the proxy to open a tunnel.""" - tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort, self._proxyAuthHeader) + tunnelReq = tunnel_request_data( + self._tunneledHost, self._tunneledPort, self._proxyAuthHeader + ) protocol.transport.write(tunnelReq) self._protocolDataReceived = protocol.dataReceived protocol.dataReceived = self.processProxyResponse @@ -129,24 +153,30 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): # from the proxy so that we don't send those bytes to the TLS layer # # see https://github.com/scrapy/scrapy/issues/2491 - if b'\r\n\r\n' not in self._connectBuffer: + if b"\r\n\r\n" not in self._connectBuffer: return self._protocol.dataReceived = self._protocolDataReceived respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) - if respm and int(respm.group('status')) == 200: + if respm and int(respm.group("status")) == 200: # set proper Server Name Indication extension - sslOptions = self._contextFactory.creatorForNetloc(self._tunneledHost, self._tunneledPort) + sslOptions = self._contextFactory.creatorForNetloc( + self._tunneledHost, self._tunneledPort + ) self._protocol.transport.startTLS(sslOptions, self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) else: if respm: - extra = {'status': int(respm.group('status')), - 'reason': respm.group('reason').strip()} + extra = { + "status": int(respm.group("status")), + "reason": respm.group("reason").strip(), + } else: - extra = rcvd_bytes[:self._truncatedLength] + extra = rcvd_bytes[: self._truncatedLength] self._tunnelReadyDeferred.errback( - TunnelError('Could not open CONNECT tunnel with proxy ' - f'{self._host}:{self._port} [{extra!r}]') + TunnelError( + "Could not open CONNECT tunnel with proxy " + f"{self._host}:{self._port} [{extra!r}]" + ) ) def connectFailed(self, reason): @@ -173,12 +203,12 @@ def tunnel_request_data(host, port, proxy_auth_header=None): >>> s(tunnel_request_data(b"example.com", "8090")) 'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n' """ - host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port)) - tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n' - tunnel_req += b'Host: ' + host_value + b'\r\n' + host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port)) + tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n" + tunnel_req += b"Host: " + host_value + b"\r\n" if proxy_auth_header: - tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n' - tunnel_req += b'\r\n' + tunnel_req += b"Proxy-Authorization: " + proxy_auth_header + b"\r\n" + tunnel_req += b"\r\n" return tunnel_req @@ -190,8 +220,15 @@ class TunnelingAgent(Agent): proxy involved. """ - def __init__(self, reactor, proxyConf, contextFactory=None, - connectTimeout=None, bindAddress=None, pool=None): + def __init__( + self, + reactor, + proxyConf, + contextFactory=None, + connectTimeout=None, + bindAddress=None, + pool=None, + ): super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) self._proxyConf = proxyConf self._contextFactory = contextFactory @@ -207,7 +244,9 @@ class TunnelingAgent(Agent): bindAddress=self._endpointFactory._bindAddress, ) - def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath): + def _requestWithEndpoint( + self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath + ): # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -224,8 +263,9 @@ class TunnelingAgent(Agent): class ScrapyProxyAgent(Agent): - - def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None): + def __init__( + self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None + ): super().__init__( reactor=reactor, connectTimeout=connectTimeout, @@ -257,8 +297,17 @@ class ScrapyAgent: _ProxyAgent = ScrapyProxyAgent _TunnelingAgent = TunnelingAgent - def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, - maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None): + def __init__( + self, + contextFactory=None, + connectTimeout=10, + bindAddress=None, + pool=None, + maxsize=0, + warnsize=0, + fail_on_dataloss=True, + crawler=None, + ): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress @@ -271,14 +320,15 @@ class ScrapyAgent: def _get_agent(self, request, timeout): from twisted.internet import reactor - bindaddress = request.meta.get('bindaddress') or self._bindAddress - proxy = request.meta.get('proxy') + + bindaddress = request.meta.get("bindaddress") or self._bindAddress + proxy = request.meta.get("proxy") if proxy: proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) - if scheme == b'https': - proxyAuth = request.headers.get(b'Proxy-Authorization', None) + if scheme == b"https": + proxyAuth = request.headers.get(b"Proxy-Authorization", None) proxyConf = (proxyHost, proxyPort, proxyAuth) return self._TunnelingAgent( reactor=reactor, @@ -288,11 +338,11 @@ class ScrapyAgent: bindAddress=bindaddress, pool=self._pool, ) - proxyScheme = proxyScheme or b'http' - proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) + proxyScheme = proxyScheme or b"http" + proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, "", "", "")) return self._ProxyAgent( reactor=reactor, - proxyURI=to_bytes(proxyURI, encoding='ascii'), + proxyURI=to_bytes(proxyURI, encoding="ascii"), connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool, @@ -308,7 +358,8 @@ class ScrapyAgent: def download_request(self, request): from twisted.internet import reactor - timeout = request.meta.get('download_timeout') or self._connectTimeout + + timeout = request.meta.get("download_timeout") or self._connectTimeout agent = self._get_agent(request, timeout) # request details @@ -316,13 +367,15 @@ class ScrapyAgent: method = to_bytes(request.method) headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): - headers.removeHeader(b'Proxy-Authorization') + headers.removeHeader(b"Proxy-Authorization") if request.body: bodyproducer = _RequestBodyProducer(request.body) else: bodyproducer = None start_time = time() - d = agent.request(method, to_bytes(url, encoding='ascii'), headers, bodyproducer) + d = agent.request( + method, to_bytes(url, encoding="ascii"), headers, bodyproducer + ) # set download latency d.addCallback(self._cb_latency, request, start_time) # response body is ready to be consumed @@ -345,14 +398,14 @@ class ScrapyAgent: raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") def _cb_latency(self, result, request, start_time): - request.meta['download_latency'] = time() - start_time + request.meta["download_latency"] = time() - start_time return result @staticmethod def _headers_from_twisted_response(response): headers = Headers() if response.length != UNKNOWN_LENGTH: - headers[b'Content-Length'] = str(response.length).encode() + headers[b"Content-Length"] = str(response.length).encode() headers.update(response.headers.getAllRawHeaders()) return headers @@ -366,8 +419,10 @@ class ScrapyAgent: ) for handler, result in headers_received_result: if isinstance(result, Failure) and isinstance(result.value, StopDownload): - logger.debug("Download stopped for %(request)s from signal handler %(handler)s", - {"request": request, "handler": handler.__qualname__}) + logger.debug( + "Download stopped for %(request)s from signal handler %(handler)s", + {"request": request, "handler": handler.__qualname__}, + ) txresponse._transport.stopProducing() txresponse._transport.loseConnection() return { @@ -389,15 +444,23 @@ class ScrapyAgent: "ip_address": None, } - maxsize = request.meta.get('download_maxsize', self._maxsize) - warnsize = request.meta.get('download_warnsize', self._warnsize) + maxsize = request.meta.get("download_maxsize", self._maxsize) + warnsize = request.meta.get("download_warnsize", self._warnsize) expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 - fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss) + fail_on_dataloss = request.meta.get( + "download_fail_on_dataloss", self._fail_on_dataloss + ) if maxsize and expected_size > maxsize: - warning_msg = ("Cancelling download of %(url)s: expected response " - "size (%(size)s) larger than download max size (%(maxsize)s).") - warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} + warning_msg = ( + "Cancelling download of %(url)s: expected response " + "size (%(size)s) larger than download max size (%(maxsize)s)." + ) + warning_args = { + "url": request.url, + "size": expected_size, + "maxsize": maxsize, + } logger.warning(warning_msg, warning_args) @@ -405,9 +468,11 @@ class ScrapyAgent: raise defer.CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: - logger.warning("Expected response size (%(size)s) larger than " - "download warn size (%(warnsize)s) in request %(request)s.", - {'size': expected_size, 'warnsize': warnsize, 'request': request}) + logger.warning( + "Expected response size (%(size)s) larger than " + "download warn size (%(warnsize)s) in request %(request)s.", + {"size": expected_size, "warnsize": warnsize, "request": request}, + ) def _cancel(_): # Abort connection immediately. @@ -457,7 +522,6 @@ class ScrapyAgent: @implementer(IBodyProducer) class _RequestBodyProducer: - def __init__(self, body): self.body = body self.length = len(body) @@ -474,8 +538,16 @@ class _RequestBodyProducer: class _ResponseReader(protocol.Protocol): - - def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler): + def __init__( + self, + finished, + txresponse, + request, + maxsize, + warnsize, + fail_on_dataloss, + crawler, + ): self._finished = finished self._txresponse = txresponse self._request = request @@ -491,22 +563,28 @@ class _ResponseReader(protocol.Protocol): self._crawler = crawler def _finish_response(self, flags=None, failure=None): - self._finished.callback({ - "txresponse": self._txresponse, - "body": self._bodybuf.getvalue(), - "flags": flags, - "certificate": self._certificate, - "ip_address": self._ip_address, - "failure": failure, - }) + self._finished.callback( + { + "txresponse": self._txresponse, + "body": self._bodybuf.getvalue(), + "flags": flags, + "certificate": self._certificate, + "ip_address": self._ip_address, + "failure": failure, + } + ) def connectionMade(self): if self._certificate is None: with suppress(AttributeError): - self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate()) + self._certificate = ssl.Certificate( + self.transport._producer.getPeerCertificate() + ) if self._ip_address is None: - self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host) + self._ip_address = ipaddress.ip_address( + self.transport._producer.getPeer().host + ) def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. @@ -524,29 +602,40 @@ class _ResponseReader(protocol.Protocol): ) for handler, result in bytes_received_result: if isinstance(result, Failure) and isinstance(result.value, StopDownload): - logger.debug("Download stopped for %(request)s from signal handler %(handler)s", - {"request": self._request, "handler": handler.__qualname__}) + logger.debug( + "Download stopped for %(request)s from signal handler %(handler)s", + {"request": self._request, "handler": handler.__qualname__}, + ) self.transport.stopProducing() self.transport.loseConnection() failure = result if result.value.fail else None self._finish_response(flags=["download_stopped"], failure=failure) if self._maxsize and self._bytes_received > self._maxsize: - logger.warning("Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s) in request %(request)s.", - {'bytes': self._bytes_received, - 'maxsize': self._maxsize, - 'request': self._request}) + logger.warning( + "Received (%(bytes)s) bytes larger than download " + "max size (%(maxsize)s) in request %(request)s.", + { + "bytes": self._bytes_received, + "maxsize": self._maxsize, + "request": self._request, + }, + ) # Clear buffer earlier to avoid keeping data in memory for a long time. self._bodybuf.truncate(0) self._finished.cancel() - if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize: + if ( + self._warnsize + and self._bytes_received > self._warnsize + and not self._reached_warnsize + ): self._reached_warnsize = True - logger.warning("Received more bytes than download " - "warn size (%(warnsize)s) in request %(request)s.", - {'warnsize': self._warnsize, - 'request': self._request}) + logger.warning( + "Received more bytes than download " + "warn size (%(warnsize)s) in request %(request)s.", + {"warnsize": self._warnsize, "request": self._request}, + ) def connectionLost(self, reason): if self._finished.called: @@ -560,16 +649,20 @@ class _ResponseReader(protocol.Protocol): self._finish_response(flags=["partial"]) return - if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): + if reason.check(ResponseFailed) and any( + r.check(_DataLoss) for r in reason.value.reasons + ): if not self._fail_on_dataloss: self._finish_response(flags=["dataloss"]) return if not self._fail_on_dataloss_warned: - logger.warning("Got data loss in %s. If you want to process broken " - "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" - " -- This message won't be shown in further requests", - self._txresponse.request.absoluteURI.decode()) + logger.warning( + "Got data loss in %s. If you want to process broken " + "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" + " -- This message won't be shown in further requests", + self._txresponse.request.absoluteURI.decode(), + ) self._fail_on_dataloss_warned = True self._finished.errback(reason) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 3f1b36e92..25ac0307b 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -17,7 +17,9 @@ from scrapy.spiders import Spider from scrapy.utils.python import to_bytes -H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler") +H2DownloadHandlerOrSubclass = TypeVar( + "H2DownloadHandlerOrSubclass", bound="H2DownloadHandler" +) class H2DownloadHandler: @@ -25,11 +27,14 @@ class H2DownloadHandler: self._crawler = crawler from twisted.internet import reactor + self._pool = H2ConnectionPool(reactor, settings) self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod - def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass: + def from_crawler( + cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler + ) -> H2DownloadHandlerOrSubclass: return cls(crawler.settings, crawler) def download_request(self, request: Request, spider: Spider) -> Deferred: @@ -49,7 +54,8 @@ class ScrapyH2Agent: _ProxyAgent = ScrapyProxyH2Agent def __init__( - self, context_factory, + self, + context_factory, pool: H2ConnectionPool, connect_timeout: int = 10, bind_address: Optional[bytes] = None, @@ -63,19 +69,22 @@ class ScrapyH2Agent: def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent: from twisted.internet import reactor - bind_address = request.meta.get('bindaddress') or self._bind_address - proxy = request.meta.get('proxy') + + bind_address = request.meta.get("bindaddress") or self._bind_address + proxy = request.meta.get("proxy") if proxy: _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) scheme = _parse(request.url)[0] - if scheme == b'https': + if scheme == b"https": # ToDo - raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported') + raise NotImplementedError( + "Tunneling via CONNECT method using HTTP/2.0 is not yet supported" + ) return self._ProxyAgent( reactor=reactor, context_factory=self._context_factory, - proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')), + proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")), connect_timeout=timeout, bind_address=bind_address, pool=self._pool, @@ -91,7 +100,8 @@ class ScrapyH2Agent: def download_request(self, request: Request, spider: Spider) -> Deferred: from twisted.internet import reactor - timeout = request.meta.get('download_timeout') or self._connect_timeout + + timeout = request.meta.get("download_timeout") or self._connect_timeout agent = self._get_agent(request, timeout) start_time = time() @@ -103,12 +113,16 @@ class ScrapyH2Agent: return d @staticmethod - def _cb_latency(response: Response, request: Request, start_time: float) -> Response: - request.meta['download_latency'] = time() - start_time + def _cb_latency( + response: Response, request: Request, start_time: float + ) -> Response: + request.meta["download_latency"] = time() - start_time return response @staticmethod - def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response: + def _cb_timeout( + response: Response, request: Request, timeout: float, timeout_cl: DelayedCall + ) -> Response: if timeout_cl.active(): timeout_cl.cancel() return response diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 51ca1ed5e..81d8e8115 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -6,40 +6,49 @@ from scrapy.utils.misc import create_instance class S3DownloadHandler: - - def __init__(self, settings, *, - crawler=None, - aws_access_key_id=None, aws_secret_access_key=None, - aws_session_token=None, - httpdownloadhandler=HTTPDownloadHandler, **kw): + def __init__( + self, + settings, + *, + crawler=None, + aws_access_key_id=None, + aws_secret_access_key=None, + aws_session_token=None, + httpdownloadhandler=HTTPDownloadHandler, + **kw, + ): if not is_botocore_available(): - raise NotConfigured('missing botocore library') + raise NotConfigured("missing botocore library") if not aws_access_key_id: - aws_access_key_id = settings['AWS_ACCESS_KEY_ID'] + aws_access_key_id = settings["AWS_ACCESS_KEY_ID"] if not aws_secret_access_key: - aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY'] + aws_secret_access_key = settings["AWS_SECRET_ACCESS_KEY"] if not aws_session_token: - aws_session_token = settings['AWS_SESSION_TOKEN'] + aws_session_token = settings["AWS_SESSION_TOKEN"] # If no credentials could be found anywhere, # consider this an anonymous connection request by default; # unless 'anon' was set explicitly (True/False). - anon = kw.get('anon') + anon = kw.get("anon") if anon is None and not aws_access_key_id and not aws_secret_access_key: - kw['anon'] = True - self.anon = kw.get('anon') + kw["anon"] = True + self.anon = kw.get("anon") self._signer = None import botocore.auth import botocore.credentials - kw.pop('anon', None) + + kw.pop("anon", None) if kw: - raise TypeError(f'Unexpected keyword arguments: {kw}') + raise TypeError(f"Unexpected keyword arguments: {kw}") if not self.anon: - SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] - self._signer = SignerCls(botocore.credentials.Credentials( - aws_access_key_id, aws_secret_access_key, aws_session_token)) + SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"] + self._signer = SignerCls( + botocore.credentials.Credentials( + aws_access_key_id, aws_secret_access_key, aws_session_token + ) + ) _http_handler = create_instance( objcls=httpdownloadhandler, @@ -54,20 +63,21 @@ class S3DownloadHandler: def download_request(self, request, spider): p = urlparse_cached(request) - scheme = 'https' if request.meta.get('is_secure') else 'http' + scheme = "https" if request.meta.get("is_secure") else "http" bucket = p.hostname - path = p.path + '?' + p.query if p.query else p.path - url = f'{scheme}://{bucket}.s3.amazonaws.com{path}' + path = p.path + "?" + p.query if p.query else p.path + url = f"{scheme}://{bucket}.s3.amazonaws.com{path}" if self.anon: request = request.replace(url=url) else: import botocore.awsrequest + awsrequest = botocore.awsrequest.AWSRequest( method=request.method, - url=f'{scheme}://s3.amazonaws.com/{bucket}{path}', + url=f"{scheme}://s3.amazonaws.com/{bucket}{path}", headers=request.headers.to_unicode_dict(), - data=request.body) + data=request.body, + ) self._signer.add_auth(awsrequest) - request = request.replace( - url=url, headers=awsrequest.headers.items()) + request = request.replace(url=url, headers=awsrequest.headers.items()) return self._download_http(request, spider) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 289147466..3410b4255 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -18,28 +18,31 @@ from scrapy.utils.conf import build_component_list class DownloaderMiddlewareManager(MiddlewareManager): - component_name = 'downloader middleware' + component_name = "downloader middleware" @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list( - settings.getwithbase('DOWNLOADER_MIDDLEWARES')) + return build_component_list(settings.getwithbase("DOWNLOADER_MIDDLEWARES")) def _add_middleware(self, mw): - if hasattr(mw, 'process_request'): - self.methods['process_request'].append(mw.process_request) - if hasattr(mw, 'process_response'): - self.methods['process_response'].appendleft(mw.process_response) - if hasattr(mw, 'process_exception'): - self.methods['process_exception'].appendleft(mw.process_exception) + if hasattr(mw, "process_request"): + self.methods["process_request"].append(mw.process_request) + if hasattr(mw, "process_response"): + self.methods["process_response"].appendleft(mw.process_response) + if hasattr(mw, "process_exception"): + self.methods["process_exception"].appendleft(mw.process_exception) def download(self, download_func: Callable, request: Request, spider: Spider): @defer.inlineCallbacks def process_request(request: Request): - for method in self.methods['process_request']: + for method in self.methods["process_request"]: method = cast(Callable, method) - response = yield deferred_from_coro(method(request=request, spider=spider)) - if response is not None and not isinstance(response, (Response, Request)): + response = yield deferred_from_coro( + method(request=request, spider=spider) + ) + if response is not None and not isinstance( + response, (Response, Request) + ): raise _InvalidOutput( f"Middleware {method.__qualname__} must return None, Response or " f"Request, got {response.__class__.__name__}" @@ -55,9 +58,11 @@ class DownloaderMiddlewareManager(MiddlewareManager): elif isinstance(response, Request): return response - for method in self.methods['process_response']: + for method in self.methods["process_response"]: method = cast(Callable, method) - response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) + response = yield deferred_from_coro( + method(request=request, response=response, spider=spider) + ) if not isinstance(response, (Response, Request)): raise _InvalidOutput( f"Middleware {method.__qualname__} must return Response or Request, " @@ -70,10 +75,14 @@ class DownloaderMiddlewareManager(MiddlewareManager): @defer.inlineCallbacks def process_exception(failure: Failure): exception = failure.value - for method in self.methods['process_exception']: + for method in self.methods["process_exception"]: method = cast(Callable, method) - response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) - if response is not None and not isinstance(response, (Response, Request)): + response = yield deferred_from_coro( + method(request=request, exception=exception, spider=spider) + ) + if response is not None and not isinstance( + response, (Response, Request) + ): raise _InvalidOutput( f"Middleware {method.__qualname__} must return None, Response or " f"Request, got {type(response)}" diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 7d67a426f..db2639a75 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -2,7 +2,11 @@ import logging from OpenSSL import SSL from service_identity.exceptions import CertificateError -from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError +from twisted.internet._sslverify import ( + ClientTLSOptions, + verifyHostname, + VerificationError, +) from twisted.internet.ssl import AcceptableCiphers from scrapy.utils.ssl import x509name_to_string, get_temp_key_info @@ -10,17 +14,17 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info logger = logging.getLogger(__name__) -METHOD_TLS = 'TLS' -METHOD_TLSv10 = 'TLSv1.0' -METHOD_TLSv11 = 'TLSv1.1' -METHOD_TLSv12 = 'TLSv1.2' +METHOD_TLS = "TLS" +METHOD_TLSv10 = "TLSv1.0" +METHOD_TLSv11 = "TLSv1.1" +METHOD_TLSv12 = "TLSv1.2" openssl_methods = { - METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only - METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only - METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only + METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) + METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only + METHOD_TLSv11: getattr(SSL, "TLSv1_1_METHOD", 5), # TLS 1.1 only + METHOD_TLSv12: getattr(SSL, "TLSv1_2_METHOD", 6), # TLS 1.2 only } @@ -44,32 +48,38 @@ class ScrapyClientTLSOptions(ClientTLSOptions): connection.set_tlsext_host_name(self._hostnameBytes) elif where & SSL.SSL_CB_HANDSHAKE_DONE: if self.verbose_logging: - logger.debug('SSL connection to %s using protocol %s, cipher %s', - self._hostnameASCII, - connection.get_protocol_version_name(), - connection.get_cipher_name(), - ) + logger.debug( + "SSL connection to %s using protocol %s, cipher %s", + self._hostnameASCII, + connection.get_protocol_version_name(), + connection.get_cipher_name(), + ) server_cert = connection.get_peer_certificate() - logger.debug('SSL connection certificate: issuer "%s", subject "%s"', - x509name_to_string(server_cert.get_issuer()), - x509name_to_string(server_cert.get_subject()), - ) + logger.debug( + 'SSL connection certificate: issuer "%s", subject "%s"', + x509name_to_string(server_cert.get_issuer()), + x509name_to_string(server_cert.get_subject()), + ) key_info = get_temp_key_info(connection._ssl) if key_info: - logger.debug('SSL temp key: %s', key_info) + logger.debug("SSL temp key: %s", key_info) try: verifyHostname(connection, self._hostnameASCII) except (CertificateError, VerificationError) as e: logger.warning( 'Remote certificate is not valid for hostname "%s"; %s', - self._hostnameASCII, e) + self._hostnameASCII, + e, + ) except ValueError as e: logger.warning( - 'Ignoring error while verifying certificate ' + "Ignoring error while verifying certificate " 'from host "%s" (exception: %r)', - self._hostnameASCII, e) + self._hostnameASCII, + e, + ) -DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT') +DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString("DEFAULT") diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 255ca62e6..a00afb7af 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -15,33 +15,33 @@ from scrapy.responsetypes import responsetypes def _parsed_url_args(parsed): # Assume parsed is urlparse-d from Request.url, # which was passed via safe_url_string and is ascii-only. - path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) + path = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, "")) path = to_bytes(path, encoding="ascii") host = to_bytes(parsed.hostname, encoding="ascii") port = parsed.port scheme = to_bytes(parsed.scheme, encoding="ascii") netloc = to_bytes(parsed.netloc, encoding="ascii") if port is None: - port = 443 if scheme == b'https' else 80 + port = 443 if scheme == b"https" else 80 return scheme, netloc, host, port, path def _parse(url): - """ Return tuple of (scheme, netloc, host, port, path), + """Return tuple of (scheme, netloc, host, port, path), all in bytes except for port which is int. Assume url is from Request.url, which was passed via safe_url_string and is ascii-only. """ url = url.strip() - if not re.match(r'^\w+://', url): - url = '//' + url + if not re.match(r"^\w+://", url): + url = "//" + url parsed = urlparse(url) return _parsed_url_args(parsed) class ScrapyHTTPPageGetter(HTTPClient): - delimiter = b'\n' + delimiter = b"\n" def connectionMade(self): self.headers = Headers() # bucket for response headers @@ -75,8 +75,8 @@ class ScrapyHTTPPageGetter(HTTPClient): self.factory.noPage(reason) def handleResponse(self, response): - if self.factory.method.upper() == b'HEAD': - self.factory.page(b'') + if self.factory.method.upper() == b"HEAD": + self.factory.page(b"") elif self.length is not None and self.length > 0: self.factory.noPage(self._connection_lost_reason) else: @@ -87,12 +87,15 @@ class ScrapyHTTPPageGetter(HTTPClient): self.transport.loseConnection() # transport cleanup needed for HTTPS connections - if self.factory.url.startswith(b'https'): + if self.factory.url.startswith(b"https"): self.transport.stopProducing() self.factory.noPage( - defer.TimeoutError(f"Getting {self.factory.url} took longer " - f"than {self.factory.timeout} seconds.")) + defer.TimeoutError( + f"Getting {self.factory.url} took longer " + f"than {self.factory.timeout} seconds." + ) + ) # This class used to inherit from Twisted’s @@ -109,16 +112,24 @@ class ScrapyHTTPClientFactory(ClientFactory): afterFoundGet = False def _build_response(self, body, request): - request.meta['download_latency'] = self.headers_time - self.start_time + request.meta["download_latency"] = self.headers_time - self.start_time status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url, body=body) - return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version)) + return respcls( + url=self._url, + status=status, + headers=headers, + body=body, + protocol=to_unicode(self.version), + ) def _set_connection_attributes(self, request): parsed = urlparse_cached(request) - self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed) - proxy = request.meta.get('proxy') + self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args( + parsed + ) + proxy = request.meta.get("proxy") if proxy: self.scheme, _, self.host, self.port, _ = _parse(proxy) self.path = self.url @@ -126,12 +137,12 @@ class ScrapyHTTPClientFactory(ClientFactory): def __init__(self, request, timeout=180): self._url = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface - self.url = to_bytes(self._url, encoding='ascii') - self.method = to_bytes(request.method, encoding='ascii') + self.url = to_bytes(self._url, encoding="ascii") + self.method = to_bytes(request.method, encoding="ascii") self.body = request.body or None self.headers = Headers(request.headers) self.response_headers = None - self.timeout = request.meta.get('download_timeout') or timeout + self.timeout = request.meta.get("download_timeout") or timeout self.start_time = time() self.deferred = defer.Deferred().addCallback(self._build_response, request) @@ -146,16 +157,16 @@ class ScrapyHTTPClientFactory(ClientFactory): self._set_connection_attributes(request) # set Host header based on url - self.headers.setdefault('Host', self.netloc) + self.headers.setdefault("Host", self.netloc) # set Content-Length based len of body if self.body is not None: - self.headers['Content-Length'] = len(self.body) + self.headers["Content-Length"] = len(self.body) # just in case a broken http/1.1 decides to keep connection alive self.headers.setdefault("Connection", "close") # Content-Length must be specified in POST method even with no body - elif self.method == b'POST': - self.headers['Content-Length'] = 0 + elif self.method == b"POST": + self.headers["Content-Length"] = 0 def __repr__(self): return f"<{self.__class__.__name__}: {self.url}>" @@ -171,6 +182,7 @@ class ScrapyHTTPClientFactory(ClientFactory): p.afterFoundGet = self.afterFoundGet if self.timeout: from twisted.internet import reactor + timeoutCall = reactor.callLater(self.timeout, p.timeout) self.deferred.addBoth(self._cancelTimeout, timeoutCall) return p diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1228e78da..19696415b 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -79,13 +79,14 @@ class ExecutionEngine: self.running = False self.paused = False self.scheduler_cls = self._get_scheduler_class(crawler.settings) - downloader_cls = load_object(self.settings['DOWNLOADER']) + downloader_cls = load_object(self.settings["DOWNLOADER"]) self.downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback = spider_closed_callback def _get_scheduler_class(self, settings: BaseSettings) -> type: from scrapy.core.scheduler import BaseScheduler + scheduler_cls = load_object(settings["SCHEDULER"]) if not issubclass(scheduler_cls, BaseScheduler): raise TypeError( @@ -106,6 +107,7 @@ class ExecutionEngine: def stop(self) -> Deferred: """Gracefully stop the execution engine""" + @inlineCallbacks def _finish_stopping_engine(_) -> Deferred: yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) @@ -115,7 +117,11 @@ class ExecutionEngine: raise RuntimeError("Engine not running") self.running = False - dfd = self.close_spider(self.spider, reason="shutdown") if self.spider is not None else succeed(None) + dfd = ( + self.close_spider(self.spider, reason="shutdown") + if self.spider is not None + else succeed(None) + ) return dfd.addBoth(_finish_stopping_engine) def close(self) -> Deferred: @@ -126,7 +132,9 @@ class ExecutionEngine: if self.running: return self.stop() # will also close spider and downloader if self.spider is not None: - return self.close_spider(self.spider, reason="shutdown") # will also close downloader + return self.close_spider( + self.spider, reason="shutdown" + ) # will also close downloader return succeed(self.downloader.close()) def pause(self) -> None: @@ -144,7 +152,10 @@ class ExecutionEngine: if self.paused: return None - while not self._needs_backout() and self._next_request_from_scheduler() is not None: + while ( + not self._needs_backout() + and self._next_request_from_scheduler() is not None + ): pass if self.slot.start_requests is not None and not self._needs_backout(): @@ -154,7 +165,11 @@ class ExecutionEngine: self.slot.start_requests = None except Exception: self.slot.start_requests = None - logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': self.spider}) + logger.error( + "Error while obtaining start requests", + exc_info=True, + extra={"spider": self.spider}, + ) else: self.crawl(request) @@ -179,18 +194,30 @@ class ExecutionEngine: d = self._download(request, self.spider) d.addBoth(self._handle_downloader_output, request) - d.addErrback(lambda f: logger.info('Error while handling downloader output', - exc_info=failure_to_exc_info(f), - extra={'spider': self.spider})) + d.addErrback( + lambda f: logger.info( + "Error while handling downloader output", + exc_info=failure_to_exc_info(f), + extra={"spider": self.spider}, + ) + ) d.addBoth(lambda _: self.slot.remove_request(request)) - d.addErrback(lambda f: logger.info('Error while removing request from slot', - exc_info=failure_to_exc_info(f), - extra={'spider': self.spider})) + d.addErrback( + lambda f: logger.info( + "Error while removing request from slot", + exc_info=failure_to_exc_info(f), + extra={"spider": self.spider}, + ) + ) slot = self.slot d.addBoth(lambda _: slot.nextcall.schedule()) - d.addErrback(lambda f: logger.info('Error while scheduling new request', - exc_info=failure_to_exc_info(f), - extra={'spider': self.spider})) + d.addErrback( + lambda f: logger.info( + "Error while scheduling new request", + exc_info=failure_to_exc_info(f), + extra={"spider": self.spider}, + ) + ) return d def _handle_downloader_output( @@ -199,7 +226,9 @@ class ExecutionEngine: assert self.spider is not None # typing if not isinstance(result, (Request, Response, Failure)): - raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}") + raise TypeError( + f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}" + ) # downloader middleware can return requests (for example, redirects) if isinstance(result, Request): @@ -211,7 +240,7 @@ class ExecutionEngine: lambda f: logger.error( "Error while enqueuing downloader output", exc_info=failure_to_exc_info(f), - extra={'spider': self.spider}, + extra={"spider": self.spider}, ) ) return d @@ -244,16 +273,22 @@ class ExecutionEngine: stacklevel=2, ) if spider is not self.spider: - raise RuntimeError(f"The spider {spider.name!r} does not match the open spider") + raise RuntimeError( + f"The spider {spider.name!r} does not match the open spider" + ) if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") self._schedule_request(request, self.spider) self.slot.nextcall.schedule() # type: ignore[union-attr] def _schedule_request(self, request: Request, spider: Spider) -> None: - self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider) + self.signals.send_catch_log( + signals.request_scheduled, request=request, spider=spider + ) if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] - self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider) + self.signals.send_catch_log( + signals.request_dropped, request=request, spider=spider + ) def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" @@ -264,10 +299,14 @@ class ExecutionEngine: stacklevel=2, ) if spider is not self.spider: - logger.warning("The spider '%s' does not match the open spider", spider.name) + logger.warning( + "The spider '%s' does not match the open spider", spider.name + ) if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - return self._download(request, spider).addBoth(self._downloaded, request, spider) + return self._download(request, spider).addBoth( + self._downloaded, request, spider + ) def _downloaded( self, result: Union[Response, Request], request: Request, spider: Spider @@ -286,7 +325,9 @@ class ExecutionEngine: def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: if not isinstance(result, (Response, Request)): - raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}") + raise TypeError( + f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}" + ) if isinstance(result, Response): if result.request is None: result.request = request @@ -311,13 +352,19 @@ class ExecutionEngine: return dwld @inlineCallbacks - def open_spider(self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True): + def open_spider( + self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True + ): if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") - logger.info("Spider opened", extra={'spider': spider}) + logger.info("Spider opened", extra={"spider": spider}) nextcall = CallLaterOnce(self._next_request) - scheduler = create_instance(self.scheduler_cls, settings=None, crawler=self.crawler) - start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider) + scheduler = create_instance( + self.scheduler_cls, settings=None, crawler=self.crawler + ) + start_requests = yield self.scraper.spidermw.process_start_requests( + start_requests, spider + ) self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider if hasattr(scheduler, "open"): @@ -337,7 +384,9 @@ class ExecutionEngine: """ assert self.spider is not None # typing expected_ex = (DontCloseSpider, CloseSpider) - res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex) + res = self.signals.send_catch_log( + signals.spider_idle, spider=self.spider, dont_log=expected_ex + ) detected_ex = { ex: x.value for _, x in res @@ -347,7 +396,7 @@ class ExecutionEngine: if DontCloseSpider in detected_ex: return None if self.spider_is_idle(): - ex = detected_ex.get(CloseSpider, CloseSpider(reason='finished')) + ex = detected_ex.get(CloseSpider, CloseSpider(reason="finished")) assert isinstance(ex, CloseSpider) # typing self.close_spider(self.spider, reason=ex.reason) @@ -359,40 +408,55 @@ class ExecutionEngine: if self.slot.closing is not None: return self.slot.closing - logger.info("Closing spider (%(reason)s)", {'reason': reason}, extra={'spider': spider}) + logger.info( + "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider} + ) dfd = self.slot.close() def log_failure(msg: str) -> Callable: def errback(failure: Failure) -> None: - logger.error(msg, exc_info=failure_to_exc_info(failure), extra={'spider': spider}) + logger.error( + msg, exc_info=failure_to_exc_info(failure), extra={"spider": spider} + ) + return errback dfd.addBoth(lambda _: self.downloader.close()) - dfd.addErrback(log_failure('Downloader close failure')) + dfd.addErrback(log_failure("Downloader close failure")) dfd.addBoth(lambda _: self.scraper.close_spider(spider)) - dfd.addErrback(log_failure('Scraper close failure')) + dfd.addErrback(log_failure("Scraper close failure")) if hasattr(self.slot.scheduler, "close"): dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) dfd.addErrback(log_failure("Scheduler close failure")) - dfd.addBoth(lambda _: self.signals.send_catch_log_deferred( - signal=signals.spider_closed, spider=spider, reason=reason, - )) - dfd.addErrback(log_failure('Error while sending spider_close signal')) + dfd.addBoth( + lambda _: self.signals.send_catch_log_deferred( + signal=signals.spider_closed, + spider=spider, + reason=reason, + ) + ) + dfd.addErrback(log_failure("Error while sending spider_close signal")) dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason)) - dfd.addErrback(log_failure('Stats close failure')) + dfd.addErrback(log_failure("Stats close failure")) - dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", {'reason': reason}, extra={'spider': spider})) + dfd.addBoth( + lambda _: logger.info( + "Spider closed (%(reason)s)", + {"reason": reason}, + extra={"spider": spider}, + ) + ) - dfd.addBoth(lambda _: setattr(self, 'slot', None)) - dfd.addErrback(log_failure('Error while unassigning slot')) + dfd.addBoth(lambda _: setattr(self, "slot", None)) + dfd.addErrback(log_failure("Error while unassigning slot")) - dfd.addBoth(lambda _: setattr(self, 'spider', None)) - dfd.addErrback(log_failure('Error while unassigning spider')) + dfd.addBoth(lambda _: setattr(self, "spider", None)) + dfd.addErrback(log_failure("Error while unassigning spider")) dfd.addBoth(lambda _: self._spider_closed_callback(spider)) @@ -408,7 +472,11 @@ class ExecutionEngine: return [self.spider] if self.spider is not None else [] def has_capacity(self) -> bool: - warnings.warn("ExecutionEngine.has_capacity is deprecated", ScrapyDeprecationWarning, stacklevel=2) + warnings.warn( + "ExecutionEngine.has_capacity is deprecated", + ScrapyDeprecationWarning, + stacklevel=2, + ) return not bool(self.slot) def schedule(self, request: Request, spider: Spider) -> None: diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index c6b357be3..119443c80 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -28,7 +28,9 @@ class H2ConnectionPool: # Save all requests that arrive before the connection is established self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} - def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred: + def get_connection( + self, key: Tuple, uri: URI, endpoint: HostnameEndpoint + ) -> Deferred: if key in self._pending_requests: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol @@ -46,7 +48,9 @@ class H2ConnectionPool: # No connection is established for the given URI return self._new_connection(key, uri, endpoint) - def _new_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred: + def _new_connection( + self, key: Tuple, uri: URI, endpoint: HostnameEndpoint + ) -> Deferred: self._pending_requests[key] = deque() conn_lost_deferred = Deferred() @@ -102,7 +106,9 @@ class H2Agent: ) -> None: self._reactor = reactor self._pool = pool - self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2']) + self._context_factory = AcceptableProtocolsContextFactory( + context_factory, acceptable_protocols=[b"h2"] + ) self.endpoint_factory = _StandardEndpointFactory( self._reactor, self._context_factory, connect_timeout, bind_address ) @@ -118,7 +124,7 @@ class H2Agent: return uri.scheme, uri.host, uri.port def request(self, request: Request, spider: Spider) -> Deferred: - uri = URI.fromBytes(bytes(request.url, encoding='utf-8')) + uri = URI.fromBytes(bytes(request.url, encoding="utf-8")) try: endpoint = self.get_endpoint(uri) except SchemeNotSupported: diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 6047f9ca8..214deeed0 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -9,9 +9,15 @@ from h2.config import H2Configuration from h2.connection import H2Connection from h2.errors import ErrorCodes from h2.events import ( - Event, ConnectionTerminated, DataReceived, ResponseReceived, - SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived, - WindowUpdated + Event, + ConnectionTerminated, + DataReceived, + ResponseReceived, + SettingsAcknowledged, + StreamEnded, + StreamReset, + UnknownFrameReceived, + WindowUpdated, ) from h2.exceptions import FrameTooLargeError, H2Error from twisted.internet.defer import Deferred @@ -37,7 +43,6 @@ PROTOCOL_NAME = b"h2" class InvalidNegotiatedProtocol(H2Error): - def __init__(self, negotiated_protocol: bytes) -> None: self.negotiated_protocol = negotiated_protocol @@ -55,11 +60,13 @@ class RemoteTerminatedConnection(H2Error): self.terminate_event = event def __str__(self) -> str: - return f'Received GOAWAY frame from {self.remote_ip_address!r}' + return f"Received GOAWAY frame from {self.remote_ip_address!r}" class MethodNotAllowed405(H2Error): - def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None: + def __init__( + self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]] + ) -> None: self.remote_ip_address = remote_ip_address def __str__(self) -> str: @@ -70,7 +77,9 @@ class MethodNotAllowed405(H2Error): class H2ClientProtocol(Protocol, TimeoutMixin): IDLE_TIMEOUT = 240 - def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None: + def __init__( + self, uri: URI, settings: Settings, conn_lost_deferred: Deferred + ) -> None: """ Arguments: uri -- URI of the base url to which HTTP/2 Connection will be made. @@ -82,7 +91,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """ self._conn_lost_deferred = conn_lost_deferred - config = H2Configuration(client_side=True, header_encoding='utf-8') + config = H2Configuration(client_side=True, header_encoding="utf-8") self.conn = H2Connection(config=config) # ID of the next request stream @@ -105,31 +114,25 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # initialized when connection is successfully made self.metadata: Dict = { # Peer certificate instance - 'certificate': None, - + "certificate": None, # Address of the server we are connected to which # is updated when HTTP/2 connection is made successfully - 'ip_address': None, - + "ip_address": None, # URI of the peer HTTP/2 connection is made - 'uri': uri, - + "uri": uri, # Both ip_address and uri are used by the Stream before # initiating the request to verify that the base address - # Variables taken from Project Settings - 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'), - 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'), - + "default_download_maxsize": settings.getint("DOWNLOAD_MAXSIZE"), + "default_download_warnsize": settings.getint("DOWNLOAD_WARNSIZE"), # Counter to keep track of opened streams. This counter # is used to make sure that not more than MAX_CONCURRENT_STREAMS # streams are opened which leads to ProtocolError # We use simple FIFO policy to handle pending requests - 'active_streams': 0, - + "active_streams": 0, # Flag to keep track if settings were acknowledged by the remote # This ensures that we have established a HTTP/2 connection - 'settings_acknowledged': False, + "settings_acknowledged": False, } @property @@ -138,7 +141,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection """ - return bool(self.transport.connected) and self.metadata['settings_acknowledged'] + return bool(self.transport.connected) and self.metadata["settings_acknowledged"] @property def allowed_max_concurrent_streams(self) -> int: @@ -149,7 +152,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """ return min( self.conn.local_settings.max_concurrent_streams, - self.conn.remote_settings.max_concurrent_streams + self.conn.remote_settings.max_concurrent_streams, ) def _send_pending_requests(self) -> None: @@ -159,37 +162,39 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """ while ( self._pending_request_stream_pool - and self.metadata['active_streams'] < self.allowed_max_concurrent_streams + and self.metadata["active_streams"] < self.allowed_max_concurrent_streams and self.h2_connected ): - self.metadata['active_streams'] += 1 + self.metadata["active_streams"] += 1 stream = self._pending_request_stream_pool.popleft() stream.initiate_request() self._write_to_transport() def pop_stream(self, stream_id: int) -> Stream: - """Perform cleanup when a stream is closed - """ + """Perform cleanup when a stream is closed""" stream = self.streams.pop(stream_id) - self.metadata['active_streams'] -= 1 + self.metadata["active_streams"] -= 1 self._send_pending_requests() return stream def _new_stream(self, request: Request, spider: Spider) -> Stream: - """Instantiates a new Stream object - """ + """Instantiates a new Stream object""" stream = Stream( stream_id=next(self._stream_id_generator), request=request, protocol=self, - download_maxsize=getattr(spider, 'download_maxsize', self.metadata['default_download_maxsize']), - download_warnsize=getattr(spider, 'download_warnsize', self.metadata['default_download_warnsize']), + download_maxsize=getattr( + spider, "download_maxsize", self.metadata["default_download_maxsize"] + ), + download_warnsize=getattr( + spider, "download_warnsize", self.metadata["default_download_warnsize"] + ), ) self.streams[stream.stream_id] = stream return stream def _write_to_transport(self) -> None: - """ Write data to the underlying transport connection + """Write data to the underlying transport connection from the HTTP2 connection instance if any """ # Reset the idle timeout as connection is still actively sending data @@ -200,7 +205,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def request(self, request: Request, spider: Spider) -> Deferred: if not isinstance(request, Request): - raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}') + raise TypeError( + f"Expected scrapy.http.Request, received {request.__class__.__qualname__}" + ) stream = self._new_stream(request, spider) d = stream.get_response() @@ -221,7 +228,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.setTimeout(self.IDLE_TIMEOUT) destination = self.transport.getPeer() - self.metadata['ip_address'] = ipaddress.ip_address(destination.host) + self.metadata["ip_address"] = ipaddress.ip_address(destination.host) # Initiate H2 Connection self.conn.initiate_connection() @@ -237,9 +244,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """ Close the connection if it's not made via the expected protocol """ - if self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME: + if ( + self.transport.negotiatedProtocol is not None + and self.transport.negotiatedProtocol != PROTOCOL_NAME + ): # we have not initiated the connection yet, no need to send a GOAWAY frame to the remote peer - self._lose_connection_with_error([InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)]) + self._lose_connection_with_error( + [InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)] + ) def _check_received_data(self, data: bytes) -> None: """Checks for edge cases where the connection to remote fails @@ -248,8 +260,8 @@ class H2ClientProtocol(Protocol, TimeoutMixin): Arguments: data -- Data received from the remote """ - if data.startswith(b'HTTP/2.0 405 Method Not Allowed'): - raise MethodNotAllowed405(self.metadata['ip_address']) + if data.startswith(b"HTTP/2.0 405 Method Not Allowed"): + raise MethodNotAllowed405(self.metadata["ip_address"]) def dataReceived(self, data: bytes) -> None: # Reset the idle timeout as connection is still actively receiving data @@ -284,7 +296,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): if ( self.conn.open_outbound_streams > 0 or self.conn.open_inbound_streams > 0 - or self.metadata['active_streams'] > 0 + or self.metadata["active_streams"] > 0 ): error_code = ErrorCodes.PROTOCOL_ERROR else: @@ -292,9 +304,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.conn.close_connection(error_code=error_code) self._write_to_transport() - self._lose_connection_with_error([ - TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s") - ]) + self._lose_connection_with_error( + [TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")] + ) def connectionLost(self, reason: Failure = connectionDone) -> None: """Called by Twisted when the transport connection is lost. @@ -311,13 +323,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._conn_lost_deferred.callback(self._conn_lost_errors) for stream in self.streams.values(): - if stream.metadata['request_sent']: + if stream.metadata["request_sent"]: close_reason = StreamCloseReason.CONNECTION_LOST else: close_reason = StreamCloseReason.INACTIVE stream.close(close_reason, self._conn_lost_errors, from_protocol=True) - self.metadata['active_streams'] -= len(self.streams) + self.metadata["active_streams"] -= len(self.streams) self.streams.clear() self._pending_request_stream_pool.clear() self.conn.close_connection() @@ -345,13 +357,13 @@ class H2ClientProtocol(Protocol, TimeoutMixin): elif isinstance(event, SettingsAcknowledged): self.settings_acknowledged(event) elif isinstance(event, UnknownFrameReceived): - logger.warning('Unknown frame received: %s', event.frame) + logger.warning("Unknown frame received: %s", event.frame) # Event handler functions starts here def connection_terminated(self, event: ConnectionTerminated) -> None: - self._lose_connection_with_error([ - RemoteTerminatedConnection(self.metadata['ip_address'], event) - ]) + self._lose_connection_with_error( + [RemoteTerminatedConnection(self.metadata["ip_address"], event)] + ) def data_received(self, event: DataReceived) -> None: try: @@ -370,14 +382,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin): stream.receive_headers(event.headers) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: - self.metadata['settings_acknowledged'] = True + self.metadata["settings_acknowledged"] = True # Send off all the pending requests as now we have # established a proper HTTP/2 connection self._send_pending_requests() # Update certificate when our HTTP/2 connection is established - self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) + self.metadata["certificate"] = Certificate(self.transport.getPeerCertificate()) def stream_ended(self, event: StreamEnded) -> None: try: @@ -406,7 +418,9 @@ class H2ClientProtocol(Protocol, TimeoutMixin): @implementer(IProtocolNegotiationFactory) class H2ClientFactory(Factory): - def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None: + def __init__( + self, uri: URI, settings: Settings, conn_lost_deferred: Deferred + ) -> None: self.uri = uri self.settings = settings self.conn_lost_deferred = conn_lost_deferred diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index e197790f7..1b185cd8c 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -32,18 +32,19 @@ class InactiveStreamClosed(ConnectionClosed): self.request = request def __str__(self) -> str: - return f'InactiveStreamClosed: Connection was closed without sending the request {self.request!r}' + return f"InactiveStreamClosed: Connection was closed without sending the request {self.request!r}" class InvalidHostname(H2Error): - - def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None: + def __init__( + self, request: Request, expected_hostname: str, expected_netloc: str + ) -> None: self.request = request self.expected_hostname = expected_hostname self.expected_netloc = expected_netloc def __str__(self) -> str: - return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}' + return f"InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}" class StreamCloseReason(Enum): @@ -100,28 +101,31 @@ class Stream: self._request: Request = request self._protocol: "H2ClientProtocol" = protocol - self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) - self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) + self._download_maxsize = self._request.meta.get( + "download_maxsize", download_maxsize + ) + self._download_warnsize = self._request.meta.get( + "download_warnsize", download_warnsize + ) # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated self.metadata: Dict = { - 'request_content_length': 0 if self._request.body is None else len(self._request.body), - + "request_content_length": 0 + if self._request.body is None + else len(self._request.body), # Flag to keep track whether the stream has initiated the request - 'request_sent': False, - + "request_sent": False, # Flag to track whether we have logged about exceeding download warnsize - 'reached_warnsize': False, - + "reached_warnsize": False, # Each time we send a data frame, we will decrease value by the amount send. - 'remaining_content_length': 0 if self._request.body is None else len(self._request.body), - + "remaining_content_length": 0 + if self._request.body is None + else len(self._request.body), # Flag to keep track whether client (self) have closed this stream - 'stream_closed_local': False, - + "stream_closed_local": False, # Flag to keep track whether the server has closed the stream - 'stream_closed_server': False, + "stream_closed_server": False, } # Private variable used to build the response @@ -130,21 +134,19 @@ class Stream: self._response: Dict = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. - 'body': BytesIO(), - + "body": BytesIO(), # The amount of data received that counts against the # flow control window - 'flow_controlled_size': 0, - + "flow_controlled_size": 0, # Headers received after sending the request - 'headers': Headers({}), + "headers": Headers({}), } def _cancel(_) -> None: # Close this stream as gracefully as possible # If the associated request is initiated we reset this stream # else we directly call close() method - if self.metadata['request_sent']: + if self.metadata["request_sent"]: self.reset_stream(StreamCloseReason.CANCELLED) else: self.close(StreamCloseReason.CANCELLED) @@ -152,7 +154,7 @@ class Stream: self._deferred_response = Deferred(_cancel) def __repr__(self) -> str: - return f'Stream(id={self.stream_id!r})' + return f"Stream(id={self.stream_id!r})" @property def _log_warnsize(self) -> bool: @@ -163,14 +165,16 @@ class Stream: True if both the above conditions hold true False if any of the conditions is false """ - content_length_header = int(self._response['headers'].get(b'Content-Length', -1)) + content_length_header = int( + self._response["headers"].get(b"Content-Length", -1) + ) return ( self._download_warnsize and ( - self._response['flow_controlled_size'] > self._download_warnsize + self._response["flow_controlled_size"] > self._download_warnsize or content_length_header > self._download_warnsize ) - and not self.metadata['reached_warnsize'] + and not self.metadata["reached_warnsize"] ) def get_response(self) -> Deferred: @@ -183,9 +187,10 @@ class Stream: # Make sure that we are sending the request to the correct URL url = urlparse(self._request.url) return ( - url.netloc == str(self._protocol.metadata['uri'].host, 'utf-8') - or url.netloc == str(self._protocol.metadata['uri'].netloc, 'utf-8') - or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' + url.netloc == str(self._protocol.metadata["uri"].host, "utf-8") + or url.netloc == str(self._protocol.metadata["uri"].netloc, "utf-8") + or url.netloc + == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' ) def _get_request_headers(self) -> List[Tuple[str, str]]: @@ -193,7 +198,7 @@ class Stream: path = url.path if url.query: - path += '?' + url.query + path += "?" + url.query # This pseudo-header field MUST NOT be empty for "http" or "https" # URIs; "http" or "https" URIs that do not contain a path component @@ -202,40 +207,40 @@ class Stream: # a path component; these MUST include a ":path" pseudo-header field # with a value of '*' (refer RFC 7540 - Section 8.1.2.3) if not path: - path = '*' if self._request.method == 'OPTIONS' else '/' + path = "*" if self._request.method == "OPTIONS" else "/" # Make sure pseudo-headers comes before all the other headers headers = [ - (':method', self._request.method), - (':authority', url.netloc), + (":method", self._request.method), + (":authority", url.netloc), ] # The ":scheme" and ":path" pseudo-header fields MUST # be omitted for CONNECT method (refer RFC 7540 - Section 8.3) - if self._request.method != 'CONNECT': + if self._request.method != "CONNECT": headers += [ - (':scheme', self._protocol.metadata['uri'].scheme), - (':path', path), + (":scheme", self._protocol.metadata["uri"].scheme), + (":path", path), ] content_length = str(len(self._request.body)) - headers.append(('Content-Length', content_length)) + headers.append(("Content-Length", content_length)) - content_length_name = self._request.headers.normkey(b'Content-Length') + content_length_name = self._request.headers.normkey(b"Content-Length") for name, values in self._request.headers.items(): for value in values: - value = str(value, 'utf-8') + value = str(value, "utf-8") if name == content_length_name: if value != content_length: logger.warning( - 'Ignoring bad Content-Length header %r of request %r, ' - 'sending %r instead', + "Ignoring bad Content-Length header %r of request %r, " + "sending %r instead", value, self._request, content_length, ) continue - headers.append((str(name, 'utf-8'), value)) + headers.append((str(name, "utf-8"), value)) return headers @@ -243,7 +248,7 @@ class Stream: if self.check_request_url(): headers = self._get_request_headers() self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False) - self.metadata['request_sent'] = True + self.metadata["request_sent"] = True self.send_data() else: # Close this stream calling the response errback @@ -252,44 +257,53 @@ class Stream: def send_data(self) -> None: """Called immediately after the headers are sent. Here we send all the - data as part of the request. + data as part of the request. - If the content length is 0 initially then we end the stream immediately and - wait for response data. + If the content length is 0 initially then we end the stream immediately and + wait for response data. - Warning: Only call this method when stream not closed from client side - and has initiated request already by sending HEADER frame. If not then - stream will raise ProtocolError (raise by h2 state machine). - """ - if self.metadata['stream_closed_local']: + Warning: Only call this method when stream not closed from client side + and has initiated request already by sending HEADER frame. If not then + stream will raise ProtocolError (raise by h2 state machine). + """ + if self.metadata["stream_closed_local"]: raise StreamClosedError(self.stream_id) # Firstly, check what the flow control window is for current stream. - window_size = self._protocol.conn.local_flow_control_window(stream_id=self.stream_id) + window_size = self._protocol.conn.local_flow_control_window( + stream_id=self.stream_id + ) # Next, check what the maximum frame size is. max_frame_size = self._protocol.conn.max_outbound_frame_size # We will send no more than the window size or the remaining file size # of data in this call, whichever is smaller. - bytes_to_send_size = min(window_size, self.metadata['remaining_content_length']) + bytes_to_send_size = min(window_size, self.metadata["remaining_content_length"]) # We now need to send a number of data frames. while bytes_to_send_size > 0: chunk_size = min(bytes_to_send_size, max_frame_size) - data_chunk_start_id = self.metadata['request_content_length'] - self.metadata['remaining_content_length'] - data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size] + data_chunk_start_id = ( + self.metadata["request_content_length"] + - self.metadata["remaining_content_length"] + ) + data_chunk = self._request.body[ + data_chunk_start_id : data_chunk_start_id + chunk_size + ] self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False) bytes_to_send_size -= chunk_size - self.metadata['remaining_content_length'] -= chunk_size + self.metadata["remaining_content_length"] -= chunk_size - self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length']) + self.metadata["remaining_content_length"] = max( + 0, self.metadata["remaining_content_length"] + ) # End the stream if no more data needs to be send - if self.metadata['remaining_content_length'] == 0: + if self.metadata["remaining_content_length"] == 0: self._protocol.conn.end_stream(self.stream_id) # Q. What about the rest of the data? @@ -301,62 +315,64 @@ class Stream: blocked behind the flow control. """ if ( - self.metadata['remaining_content_length'] - and not self.metadata['stream_closed_server'] - and self.metadata['request_sent'] + self.metadata["remaining_content_length"] + and not self.metadata["stream_closed_server"] + and self.metadata["request_sent"] ): self.send_data() def receive_data(self, data: bytes, flow_controlled_length: int) -> None: - self._response['body'].write(data) - self._response['flow_controlled_size'] += flow_controlled_length + self._response["body"].write(data) + self._response["flow_controlled_size"] += flow_controlled_length # We check maxsize here in case the Content-Length header was not received - if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize: + if ( + self._download_maxsize + and self._response["flow_controlled_size"] > self._download_maxsize + ): self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) return if self._log_warnsize: - self.metadata['reached_warnsize'] = True + self.metadata["reached_warnsize"] = True warning_msg = ( f'Received more ({self._response["flow_controlled_size"]}) bytes than download ' - f'warn size ({self._download_warnsize}) in request {self._request}' + f"warn size ({self._download_warnsize}) in request {self._request}" ) logger.warning(warning_msg) # Acknowledge the data received self._protocol.conn.acknowledge_received_data( - self._response['flow_controlled_size'], - self.stream_id + self._response["flow_controlled_size"], self.stream_id ) def receive_headers(self, headers: List[HeaderTuple]) -> None: for name, value in headers: - self._response['headers'][name] = value + self._response["headers"][name] = value # Check if we exceed the allowed max data size which can be received - expected_size = int(self._response['headers'].get(b'Content-Length', -1)) + expected_size = int(self._response["headers"].get(b"Content-Length", -1)) if self._download_maxsize and expected_size > self._download_maxsize: self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) return if self._log_warnsize: - self.metadata['reached_warnsize'] = True + self.metadata["reached_warnsize"] = True warning_msg = ( - f'Expected response size ({expected_size}) larger than ' - f'download warn size ({self._download_warnsize}) in request {self._request}' + f"Expected response size ({expected_size}) larger than " + f"download warn size ({self._download_warnsize}) in request {self._request}" ) logger.warning(warning_msg) def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None: """Close this stream by sending a RST_FRAME to the remote peer""" - if self.metadata['stream_closed_local']: + if self.metadata["stream_closed_local"]: raise StreamClosedError(self.stream_id) # Clear buffer earlier to avoid keeping data in memory for a long time - self._response['body'].truncate(0) + self._response["body"].truncate(0) - self.metadata['stream_closed_local'] = True + self.metadata["stream_closed_local"] = True self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) self.close(reason) @@ -366,13 +382,14 @@ class Stream: errors: Optional[List[BaseException]] = None, from_protocol: bool = False, ) -> None: - """Based on the reason sent we will handle each case. - """ - if self.metadata['stream_closed_server']: + """Based on the reason sent we will handle each case.""" + if self.metadata["stream_closed_server"]: raise StreamClosedError(self.stream_id) if not isinstance(reason, StreamCloseReason): - raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}') + raise TypeError( + f"Expected StreamCloseReason, received {reason.__class__.__qualname__}" + ) # Have default value of errors as an empty list as # some cases can add a list of exceptions @@ -381,7 +398,7 @@ class Stream: if not from_protocol: self._protocol.pop_stream(self.stream_id) - self.metadata['stream_closed_server'] = True + self.metadata["stream_closed_server"] = True # We do not check for Content-Length or Transfer-Encoding in response headers # and add `partial` flag as in HTTP/1.1 as 'A request or response that includes @@ -392,13 +409,14 @@ class Stream: # receiving DATA_FRAME's when we have received the headers (not # having Content-Length) if reason is StreamCloseReason.MAXSIZE_EXCEEDED: - expected_size = int(self._response['headers'].get( - b'Content-Length', - self._response['flow_controlled_size']) + expected_size = int( + self._response["headers"].get( + b"Content-Length", self._response["flow_controlled_size"] + ) ) error_msg = ( - f'Cancelling download of {self._request.url}: received response ' - f'size ({expected_size}) larger than download max size ({self._download_maxsize})' + f"Cancelling download of {self._request.url}: received response " + f"size ({expected_size}) larger than download max size ({self._download_maxsize})" ) logger.error(error_msg) self._deferred_response.errback(CancelledError(error_msg)) @@ -416,16 +434,20 @@ class Stream: # There maybe no :status in headers, we make # HTTP Status Code: 499 - Client Closed Request - self._response['headers'][':status'] = '499' + self._response["headers"][":status"] = "499" self._fire_response_deferred() elif reason is StreamCloseReason.RESET: - self._deferred_response.errback(ResponseFailed([ - Failure( - f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM', - ProtocolError + self._deferred_response.errback( + ResponseFailed( + [ + Failure( + f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM', + ProtocolError, + ) + ] ) - ])) + ) elif reason is StreamCloseReason.CONNECTION_LOST: self._deferred_response.errback(ResponseFailed(errors)) @@ -436,33 +458,35 @@ class Stream: else: assert reason is StreamCloseReason.INVALID_HOSTNAME - self._deferred_response.errback(InvalidHostname( - self._request, - str(self._protocol.metadata['uri'].host, 'utf-8'), - f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' - )) + self._deferred_response.errback( + InvalidHostname( + self._request, + str(self._protocol.metadata["uri"].host, "utf-8"), + f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}', + ) + ) def _fire_response_deferred(self) -> None: """Builds response from the self._response dict and fires the response deferred callback with the generated response instance""" - body = self._response['body'].getvalue() + body = self._response["body"].getvalue() response_cls = responsetypes.from_args( - headers=self._response['headers'], + headers=self._response["headers"], url=self._request.url, body=body, ) response = response_cls( url=self._request.url, - status=int(self._response['headers'][':status']), - headers=self._response['headers'], + status=int(self._response["headers"][":status"]), + headers=self._response["headers"], body=body, request=self._request, - certificate=self._protocol.metadata['certificate'], - ip_address=self._protocol.metadata['ip_address'], - protocol='h2', + certificate=self._protocol.metadata["certificate"], + ip_address=self._protocol.metadata["ip_address"], + protocol="h2", ) self._deferred_response.callback(response) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 6dd5d51a8..1e6fc69e1 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -20,14 +20,18 @@ class BaseSchedulerMeta(type): """ Metaclass to check scheduler classes against the necessary interface """ + def __instancecheck__(cls, instance): return cls.__subclasscheck__(type(instance)) def __subclasscheck__(cls, subclass): return ( - hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests) - and hasattr(subclass, "enqueue_request") and callable(subclass.enqueue_request) - and hasattr(subclass, "next_request") and callable(subclass.next_request) + hasattr(subclass, "has_pending_requests") + and callable(subclass.has_pending_requests) + and hasattr(subclass, "enqueue_request") + and callable(subclass.enqueue_request) + and hasattr(subclass, "next_request") + and callable(subclass.next_request) ) @@ -162,6 +166,7 @@ class Scheduler(BaseScheduler): :param crawler: The crawler object corresponding to the current crawl. :type crawler: :class:`scrapy.crawler.Crawler` """ + def __init__( self, dupefilter, @@ -187,15 +192,15 @@ class Scheduler(BaseScheduler): """ Factory method, initializes the scheduler with arguments taken from the crawl settings """ - dupefilter_cls = load_object(crawler.settings['DUPEFILTER_CLASS']) + dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"]) return cls( dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler), jobdir=job_dir(crawler.settings), - dqclass=load_object(crawler.settings['SCHEDULER_DISK_QUEUE']), - mqclass=load_object(crawler.settings['SCHEDULER_MEMORY_QUEUE']), - logunser=crawler.settings.getbool('SCHEDULER_DEBUG'), + dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]), + mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]), + logunser=crawler.settings.getbool("SCHEDULER_DEBUG"), stats=crawler.stats, - pqclass=load_object(crawler.settings['SCHEDULER_PRIORITY_QUEUE']), + pqclass=load_object(crawler.settings["SCHEDULER_PRIORITY_QUEUE"]), crawler=crawler, ) @@ -239,11 +244,11 @@ class Scheduler(BaseScheduler): return False dqok = self._dqpush(request) if dqok: - self.stats.inc_value('scheduler/enqueued/disk', spider=self.spider) + self.stats.inc_value("scheduler/enqueued/disk", spider=self.spider) else: self._mqpush(request) - self.stats.inc_value('scheduler/enqueued/memory', spider=self.spider) - self.stats.inc_value('scheduler/enqueued', spider=self.spider) + self.stats.inc_value("scheduler/enqueued/memory", spider=self.spider) + self.stats.inc_value("scheduler/enqueued", spider=self.spider) return True def next_request(self) -> Optional[Request]: @@ -257,13 +262,13 @@ class Scheduler(BaseScheduler): """ request = self.mqs.pop() if request is not None: - self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider) + self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider) else: request = self._dqpop() if request is not None: - self.stats.inc_value('scheduler/dequeued/disk', spider=self.spider) + self.stats.inc_value("scheduler/dequeued/disk", spider=self.spider) if request is not None: - self.stats.inc_value('scheduler/dequeued', spider=self.spider) + self.stats.inc_value("scheduler/dequeued", spider=self.spider) return request def __len__(self) -> int: @@ -279,13 +284,19 @@ class Scheduler(BaseScheduler): self.dqs.push(request) except ValueError as e: # non serializable request if self.logunser: - msg = ("Unable to serialize request: %(request)s - reason:" - " %(reason)s - no more unserializable requests will be" - " logged (stats being collected)") - logger.warning(msg, {'request': request, 'reason': e}, - exc_info=True, extra={'spider': self.spider}) + msg = ( + "Unable to serialize request: %(request)s - reason:" + " %(reason)s - no more unserializable requests will be" + " logged (stats being collected)" + ) + logger.warning( + msg, + {"request": request, "reason": e}, + exc_info=True, + extra={"spider": self.spider}, + ) self.logunser = False - self.stats.inc_value('scheduler/unserializable', spider=self.spider) + self.stats.inc_value("scheduler/unserializable", spider=self.spider) return False else: return True @@ -299,43 +310,50 @@ class Scheduler(BaseScheduler): return None def _mq(self): - """ Create a new priority queue instance, with in-memory storage """ - return create_instance(self.pqclass, - settings=None, - crawler=self.crawler, - downstream_queue_cls=self.mqclass, - key='') + """Create a new priority queue instance, with in-memory storage""" + return create_instance( + self.pqclass, + settings=None, + crawler=self.crawler, + downstream_queue_cls=self.mqclass, + key="", + ) def _dq(self): - """ Create a new priority queue instance, with disk storage """ + """Create a new priority queue instance, with disk storage""" state = self._read_dqs_state(self.dqdir) - q = create_instance(self.pqclass, - settings=None, - crawler=self.crawler, - downstream_queue_cls=self.dqclass, - key=self.dqdir, - startprios=state) + q = create_instance( + self.pqclass, + settings=None, + crawler=self.crawler, + downstream_queue_cls=self.dqclass, + key=self.dqdir, + startprios=state, + ) if q: - logger.info("Resuming crawl (%(queuesize)d requests scheduled)", - {'queuesize': len(q)}, extra={'spider': self.spider}) + logger.info( + "Resuming crawl (%(queuesize)d requests scheduled)", + {"queuesize": len(q)}, + extra={"spider": self.spider}, + ) return q def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: - """ Return a folder name to keep disk queue state at """ + """Return a folder name to keep disk queue state at""" if jobdir is not None: - dqdir = Path(jobdir, 'requests.queue') + dqdir = Path(jobdir, "requests.queue") if not dqdir.exists(): dqdir.mkdir(parents=True) return str(dqdir) return None def _read_dqs_state(self, dqdir: str) -> list: - path = Path(dqdir, 'active.json') + path = Path(dqdir, "active.json") if not path.exists(): return [] with path.open(encoding="utf-8") as f: return json.load(f) def _write_dqs_state(self, dqdir: str, state: list) -> None: - with Path(dqdir, 'active.json').open('w', encoding="utf-8") as f: + with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: json.dump(state, f) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 69ac1cdaf..7c2eefbe6 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -63,7 +63,9 @@ class Slot: self.itemproc_size: int = 0 self.closing: Optional[Deferred] = None - def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred: + def add_response_request( + self, result: Union[Response, Failure], request: Request + ) -> Deferred: deferred = Deferred() self.queue.append((result, request, deferred)) if isinstance(result, Response): @@ -77,7 +79,9 @@ class Slot: self.active.add(request) return response, request, deferred - def finish_response(self, result: Union[Response, Failure], request: Request) -> None: + def finish_response( + self, result: Union[Response, Failure], request: Request + ) -> None: self.active.remove(request) if isinstance(result, Response): self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -92,13 +96,12 @@ class Slot: class Scraper: - def __init__(self, crawler: Crawler) -> None: self.slot: Optional[Slot] = None self.spidermw = SpiderMiddlewareManager.from_crawler(crawler) - itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR']) + itemproc_cls = load_object(crawler.settings["ITEM_PROCESSOR"]) self.itemproc = itemproc_cls.from_crawler(crawler) - self.concurrent_items = crawler.settings.getint('CONCURRENT_ITEMS') + self.concurrent_items = crawler.settings.getint("CONCURRENT_ITEMS") self.crawler = crawler self.signals = crawler.signals self.logformatter = crawler.logformatter @@ -106,7 +109,7 @@ class Scraper: @inlineCallbacks def open_spider(self, spider: Spider): """Open the given spider for scraping and allocate resources for it""" - self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE')) + self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) yield self.itemproc.open_spider(spider) def close_spider(self, spider: Spider) -> Deferred: @@ -127,7 +130,9 @@ class Scraper: if self.slot.closing and self.slot.is_idle(): self.slot.closing.callback(spider) - def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: + def enqueue_scrape( + self, result: Union[Response, Failure], request: Request, spider: Spider + ) -> Deferred: if self.slot is None: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) @@ -140,10 +145,13 @@ class Scraper: dfd.addBoth(finish_scraping) dfd.addErrback( - lambda f: logger.error('Scraper bug processing %(request)s', - {'request': request}, - exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + lambda f: logger.error( + "Scraper bug processing %(request)s", + {"request": request}, + exc_info=failure_to_exc_info(f), + extra={"spider": spider}, + ) + ) self._scrape_next(spider) return dfd @@ -153,35 +161,49 @@ class Scraper: response, request, deferred = self.slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: + def _scrape( + self, result: Union[Response, Failure], request: Request, spider: Spider + ) -> Deferred: """ Handle the downloaded response or failure through the spider callback/errback """ if not isinstance(result, (Response, Failure)): - raise TypeError(f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}") - dfd = self._scrape2(result, request, spider) # returns spider's processed output + raise TypeError( + f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" + ) + dfd = self._scrape2( + result, request, spider + ) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: + def _scrape2( + self, result: Union[Response, Failure], request: Request, spider: Spider + ) -> Deferred: """ Handle the different cases of request's result been a Response or a Failure """ if isinstance(result, Response): - return self.spidermw.scrape_response(self.call_spider, result, request, spider) + return self.spidermw.scrape_response( + self.call_spider, result, request, spider + ) # else result is a Failure dfd = self.call_spider(result, request, spider) return dfd.addErrback(self._log_download_errors, result, request, spider) - def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: + def call_spider( + self, result: Union[Response, Failure], request: Request, spider: Spider + ) -> Deferred: if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request callback = result.request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) dfd = defer_succeed(result) - dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs) + dfd.addCallbacks( + callback=callback, callbackKeywords=result.request.cb_kwargs + ) else: # result is a Failure result.request = request warn_on_generator_with_return_value(spider, request.errback) @@ -189,45 +211,69 @@ class Scraper: dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) - def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None: + def handle_spider_error( + self, _failure: Failure, request: Request, response: Response, spider: Spider + ) -> None: exc = _failure.value if isinstance(exc, CloseSpider): assert self.crawler.engine is not None # typing - self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') + self.crawler.engine.close_spider(spider, exc.reason or "cancelled") return logkws = self.logformatter.spider_error(_failure, request, response, spider) logger.log( *logformatter_adapter(logkws), exc_info=failure_to_exc_info(_failure), - extra={'spider': spider} + extra={"spider": spider}, ) self.signals.send_catch_log( signal=signals.spider_error, - failure=_failure, response=response, - spider=spider + failure=_failure, + response=response, + spider=spider, ) self.crawler.stats.inc_value( - f"spider_exceptions/{_failure.value.__class__.__name__}", - spider=spider + f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider ) - def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request, - response: Response, spider: Spider) -> Deferred: + def handle_spider_output( + self, + result: Union[Iterable, AsyncIterable], + request: Request, + response: Response, + spider: Spider, + ) -> Deferred: if not result: return defer_succeed(None) it: Union[Generator, AsyncGenerator] if isinstance(result, AsyncIterable): - it = aiter_errback(result, self.handle_spider_error, request, response, spider) - dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output, - request, response, spider) + it = aiter_errback( + result, self.handle_spider_error, request, response, spider + ) + dfd = parallel_async( + it, + self.concurrent_items, + self._process_spidermw_output, + request, + response, + spider, + ) else: - it = iter_errback(result, self.handle_spider_error, request, response, spider) - dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, - request, response, spider) + it = iter_errback( + result, self.handle_spider_error, request, response, spider + ) + dfd = parallel( + it, + self.concurrent_items, + self._process_spidermw_output, + request, + response, + spider, + ) return dfd - def _process_spidermw_output(self, output: Any, request: Request, response: Response, - spider: Spider) -> Optional[Deferred]: + def _process_spidermw_output( + self, output: Any, request: Request, response: Response, spider: Spider + ) -> Optional[Deferred]: """Process each Request/Item (given in the output parameter) returned from the given spider """ @@ -245,14 +291,19 @@ class Scraper: else: typename = type(output).__name__ logger.error( - 'Spider must return request, item, or None, got %(typename)r in %(request)s', - {'request': request, 'typename': typename}, - extra={'spider': spider}, + "Spider must return request, item, or None, got %(typename)r in %(request)s", + {"request": request, "typename": typename}, + extra={"spider": spider}, ) return None - def _log_download_errors(self, spider_failure: Failure, download_failure: Failure, request: Request, - spider: Spider) -> Union[Failure, None]: + def _log_download_errors( + self, + spider_failure: Failure, + download_failure: Failure, + request: Request, + spider: Spider, + ) -> Union[Failure, None]: """Log and silence errors that come from the engine (typically download errors that got propagated thru here). @@ -262,29 +313,33 @@ class Scraper: """ if not download_failure.check(IgnoreRequest): if download_failure.frames: - logkws = self.logformatter.download_error(download_failure, request, spider) + logkws = self.logformatter.download_error( + download_failure, request, spider + ) logger.log( *logformatter_adapter(logkws), - extra={'spider': spider}, + extra={"spider": spider}, exc_info=failure_to_exc_info(download_failure), ) else: errmsg = download_failure.getErrorMessage() if errmsg: logkws = self.logformatter.download_error( - download_failure, request, spider, errmsg) + download_failure, request, spider, errmsg + ) logger.log( *logformatter_adapter(logkws), - extra={'spider': spider}, + extra={"spider": spider}, ) if spider_failure is not download_failure: return spider_failure return None - def _itemproc_finished(self, output: Any, item: Any, response: Response, spider: Spider) -> None: - """ItemProcessor finished for the given ``item`` and returned ``output`` - """ + def _itemproc_finished( + self, output: Any, item: Any, response: Response, spider: Spider + ) -> None: + """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing self.slot.itemproc_size -= 1 if isinstance(output, Failure): @@ -292,19 +347,30 @@ class Scraper: if isinstance(ex, DropItem): logkws = self.logformatter.dropped(item, ex, response, spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) return self.signals.send_catch_log_deferred( - signal=signals.item_dropped, item=item, response=response, - spider=spider, exception=output.value) + signal=signals.item_dropped, + item=item, + response=response, + spider=spider, + exception=output.value, + ) logkws = self.logformatter.item_error(item, ex, response, spider) - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}, - exc_info=failure_to_exc_info(output)) + logger.log( + *logformatter_adapter(logkws), + extra={"spider": spider}, + exc_info=failure_to_exc_info(output), + ) return self.signals.send_catch_log_deferred( - signal=signals.item_error, item=item, response=response, - spider=spider, failure=output) + signal=signals.item_error, + item=item, + response=response, + spider=spider, + failure=output, + ) logkws = self.logformatter.scraped(output, response, spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) return self.signals.send_catch_log_deferred( - signal=signals.item_scraped, item=output, response=response, - spider=spider) + signal=signals.item_scraped, item=output, response=response, spider=spider + ) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 91e4b9cb4..1aaed5865 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -6,7 +6,17 @@ See documentation in docs/topics/spider-middleware.rst import logging from inspect import isasyncgenfunction, iscoroutine from itertools import islice -from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast +from typing import ( + Any, + AsyncGenerator, + AsyncIterable, + Callable, + Generator, + Iterable, + Tuple, + Union, + cast, +) from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -17,7 +27,12 @@ from scrapy.http import Response from scrapy.middleware import MiddlewareManager from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list -from scrapy.utils.defer import mustbe_deferred, deferred_from_coro, deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import ( + mustbe_deferred, + deferred_from_coro, + deferred_f_from_coro_f, + maybe_deferred_to_future, +) from scrapy.utils.python import MutableAsyncChain, MutableChain @@ -33,7 +48,7 @@ def _isiterable(o) -> bool: class SpiderMiddlewareManager(MiddlewareManager): - component_name = 'spider middleware' + component_name = "spider middleware" def __init__(self, *middlewares): super().__init__(*middlewares) @@ -41,28 +56,35 @@ class SpiderMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) + return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) def _add_middleware(self, mw): super()._add_middleware(mw) - if hasattr(mw, 'process_spider_input'): - self.methods['process_spider_input'].append(mw.process_spider_input) - if hasattr(mw, 'process_start_requests'): - self.methods['process_start_requests'].appendleft(mw.process_start_requests) - process_spider_output = self._get_async_method_pair(mw, 'process_spider_output') - self.methods['process_spider_output'].appendleft(process_spider_output) - process_spider_exception = getattr(mw, 'process_spider_exception', None) - self.methods['process_spider_exception'].appendleft(process_spider_exception) + if hasattr(mw, "process_spider_input"): + self.methods["process_spider_input"].append(mw.process_spider_input) + if hasattr(mw, "process_start_requests"): + self.methods["process_start_requests"].appendleft(mw.process_start_requests) + process_spider_output = self._get_async_method_pair(mw, "process_spider_output") + self.methods["process_spider_output"].appendleft(process_spider_output) + process_spider_exception = getattr(mw, "process_spider_exception", None) + self.methods["process_spider_exception"].appendleft(process_spider_exception) - def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, - spider: Spider) -> Any: - for method in self.methods['process_spider_input']: + def _process_spider_input( + self, + scrape_func: ScrapeFunc, + response: Response, + request: Request, + spider: Spider, + ) -> Any: + for method in self.methods["process_spider_input"]: method = cast(Callable, method) try: result = method(response=response, spider=spider) if result is not None: - msg = (f"{method.__qualname__} must return None " - f"or raise an exception, got {type(result)}") + msg = ( + f"{method.__qualname__} must return None " + f"or raise an exception, got {type(result)}" + ) raise _InvalidOutput(msg) except _InvalidOutput: raise @@ -70,17 +92,22 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable], - exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain] - ) -> Union[Generator, AsyncGenerator]: - + def _evaluate_iterable( + self, + response: Response, + spider: Spider, + iterable: Union[Iterable, AsyncIterable], + exception_processor_index: int, + recover_to: Union[MutableChain, MutableAsyncChain], + ) -> Union[Generator, AsyncGenerator]: def process_sync(iterable: Iterable): try: for r in iterable: yield r except Exception as ex: - exception_result = self._process_spider_exception(response, spider, Failure(ex), - exception_processor_index) + exception_result = self._process_spider_exception( + response, spider, Failure(ex), exception_processor_index + ) if isinstance(exception_result, Failure): raise recover_to.extend(exception_result) @@ -90,8 +117,9 @@ class SpiderMiddlewareManager(MiddlewareManager): async for r in iterable: yield r except Exception as ex: - exception_result = self._process_spider_exception(response, spider, Failure(ex), - exception_processor_index) + exception_result = self._process_spider_exception( + response, spider, Failure(ex), exception_processor_index + ) if isinstance(exception_result, Failure): raise recover_to.extend(exception_result) @@ -100,13 +128,20 @@ class SpiderMiddlewareManager(MiddlewareManager): return process_async(iterable) return process_sync(iterable) - def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, - start_index: int = 0) -> Union[Failure, MutableChain]: + def _process_spider_exception( + self, + response: Response, + spider: Spider, + _failure: Failure, + start_index: int = 0, + ) -> Union[Failure, MutableChain]: exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - method_list = islice(self.methods['process_spider_exception'], start_index, None) + method_list = islice( + self.methods["process_spider_exception"], start_index, None + ) for method_index, method in enumerate(method_list, start=start_index): if method is None: continue @@ -115,7 +150,9 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - dfd: Deferred = self._process_spider_output(response, spider, result, method_index + 1) + dfd: Deferred = self._process_spider_output( + response, spider, result, method_index + 1 + ) # _process_spider_output() returns a Deferred only because of downgrading so this can be # simplified when downgrading is removed. if dfd.called: @@ -128,8 +165,10 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = (f"{method.__qualname__} must return None " - f"or an iterable, got {type(result)}") + msg = ( + f"{method.__qualname__} must return None " + f"or an iterable, got {type(result)}" + ) raise _InvalidOutput(msg) return _failure @@ -137,9 +176,13 @@ class SpiderMiddlewareManager(MiddlewareManager): # being available immediately which doesn't work when it's a wrapped coroutine. # It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed. @inlineCallbacks - def _process_spider_output(self, response: Response, spider: Spider, - result: Union[Iterable, AsyncIterable], start_index: int = 0 - ) -> Deferred: + def _process_spider_output( + self, + response: Response, + spider: Spider, + result: Union[Iterable, AsyncIterable], + start_index: int = 0, + ) -> Deferred: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered: Union[MutableChain, MutableAsyncChain] @@ -156,7 +199,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # Storing methods and method tuples in the same list is weird but we should be able to roll this back # when we drop this compatibility feature. - method_list = islice(self.methods['process_spider_output'], start_index, None) + method_list = islice(self.methods["process_spider_output"], start_index, None) for method_index, method_pair in enumerate(method_list, start=start_index): if method_pair is None: continue @@ -177,24 +220,32 @@ class SpiderMiddlewareManager(MiddlewareManager): result = as_async_generator(result) elif need_downgrade: if not self.downgrade_warning_done: - logger.warning(f"Async iterable passed to {method.__qualname__} " - f"was downgraded to a non-async one") + logger.warning( + f"Async iterable passed to {method.__qualname__} " + f"was downgraded to a non-async one" + ) self.downgrade_warning_done = True assert isinstance(result, AsyncIterable) # AsyncIterable -> Iterable result = yield deferred_from_coro(collect_asyncgen(result)) if isinstance(recovered, AsyncIterable): - recovered_collected = yield deferred_from_coro(collect_asyncgen(recovered)) + recovered_collected = yield deferred_from_coro( + collect_asyncgen(recovered) + ) recovered = MutableChain(recovered_collected) # might fail directly if the output value is not a generator result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result = self._process_spider_exception(response, spider, Failure(ex), method_index + 1) + exception_result = self._process_spider_exception( + response, spider, Failure(ex), method_index + 1 + ) if isinstance(exception_result, Failure): raise return exception_result if _isiterable(result): - result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) + result = self._evaluate_iterable( + response, spider, result, method_index + 1, recovered + ) else: if iscoroutine(result): result.close() # Silence warning about not awaiting @@ -214,15 +265,18 @@ class SpiderMiddlewareManager(MiddlewareManager): return MutableAsyncChain(result, recovered) return MutableChain(result, recovered) # type: ignore[arg-type] - async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] - ) -> Union[MutableChain, MutableAsyncChain]: + async def _process_callback_output( + self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] + ) -> Union[MutableChain, MutableAsyncChain]: recovered: Union[MutableChain, MutableAsyncChain] if isinstance(result, AsyncIterable): recovered = MutableAsyncChain() else: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) - result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result)) + result = await maybe_deferred_to_future( + self._process_spider_output(response, spider, result) + ) if isinstance(result, AsyncIterable): return MutableAsyncChain(result, recovered) if isinstance(recovered, AsyncIterable): @@ -230,41 +284,60 @@ class SpiderMiddlewareManager(MiddlewareManager): recovered = MutableChain(recovered_collected) return MutableChain(result, recovered) # type: ignore[arg-type] - def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, - spider: Spider) -> Deferred: - async def process_callback_output(result: Union[Iterable, AsyncIterable] - ) -> Union[MutableChain, MutableAsyncChain]: + def scrape_response( + self, + scrape_func: ScrapeFunc, + response: Response, + request: Request, + spider: Spider, + ) -> Deferred: + async def process_callback_output( + result: Union[Iterable, AsyncIterable] + ) -> Union[MutableChain, MutableAsyncChain]: return await self._process_callback_output(response, spider, result) def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: return self._process_spider_exception(response, spider, _failure) - dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) - dfd.addCallbacks(callback=deferred_f_from_coro_f(process_callback_output), errback=process_spider_exception) + dfd = mustbe_deferred( + self._process_spider_input, scrape_func, response, request, spider + ) + dfd.addCallbacks( + callback=deferred_f_from_coro_f(process_callback_output), + errback=process_spider_exception, + ) return dfd def process_start_requests(self, start_requests, spider: Spider) -> Deferred: - return self._process_chain('process_start_requests', start_requests, spider) + return self._process_chain("process_start_requests", start_requests, spider) # This method is only needed until _async compatibility methods are removed. @staticmethod - def _get_async_method_pair(mw: Any, methodname: str) -> Union[None, Callable, Tuple[Callable, Callable]]: + def _get_async_method_pair( + mw: Any, methodname: str + ) -> Union[None, Callable, Tuple[Callable, Callable]]: normal_method = getattr(mw, methodname, None) methodname_async = methodname + "_async" async_method = getattr(mw, methodname_async, None) if not async_method: return normal_method if not normal_method: - logger.error(f"Middleware {mw.__qualname__} has {methodname_async} " - f"without {methodname}, skipping this method.") + logger.error( + f"Middleware {mw.__qualname__} has {methodname_async} " + f"without {methodname}, skipping this method." + ) return None if not isasyncgenfunction(async_method): - logger.error(f"{async_method.__qualname__} is not " - f"an async generator function, skipping this method.") + logger.error( + f"{async_method.__qualname__} is not " + f"an async generator function, skipping this method." + ) return normal_method if isasyncgenfunction(normal_method): - logger.error(f"{normal_method.__qualname__} is an async " - f"generator function while {methodname_async} exists, " - f"skipping both methods.") + logger.error( + f"{normal_method.__qualname__} is an async " + f"generator function while {methodname_async} exists, " + f"skipping both methods." + ) return None return normal_method, async_method diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 4700a30ab..f58cd73d3 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -49,10 +49,9 @@ logger = logging.getLogger(__name__) class Crawler: - def __init__(self, spidercls, settings=None, init_reactor: bool = False): if isinstance(spidercls, Spider): - raise ValueError('The spidercls argument must be a class, not an object') + raise ValueError("The spidercls argument must be a class, not an object") if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -63,14 +62,15 @@ class Crawler: self.signals = SignalManager(self) - self.stats = load_object(self.settings['STATS_CLASS'])(self) + self.stats = load_object(self.settings["STATS_CLASS"])(self) - handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL')) + handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) logging.root.addHandler(handler) d = dict(overridden_settings(self.settings)) - logger.info("Overridden settings:\n%(settings)s", - {'settings': pprint.pformat(d)}) + logger.info( + "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} + ) if get_scrapy_root_handler() is not None: # scrapy root handler already installed: update it with new settings @@ -80,11 +80,11 @@ class Crawler: self.__remove_handler = lambda: logging.root.removeHandler(handler) self.signals.connect(self.__remove_handler, signals.engine_stopped) - lf_cls = load_object(self.settings['LOG_FORMATTER']) + lf_cls = load_object(self.settings["LOG_FORMATTER"]) self.logformatter = lf_cls.from_crawler(self) self.request_fingerprinter: RequestFingerprinter = create_instance( - load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']), + load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), settings=self.settings, crawler=self, ) @@ -160,23 +160,26 @@ class CrawlerRunner: crawlers = property( lambda self: self._crawlers, doc="Set of :class:`crawlers ` started by " - ":meth:`crawl` and managed by this class." + ":meth:`crawl` and managed by this class.", ) @staticmethod def _get_spider_loader(settings): - """ Get SpiderLoader instance from settings """ - cls_path = settings.get('SPIDER_LOADER_CLASS') + """Get SpiderLoader instance from settings""" + cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) - excs = (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement + excs = ( + (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement + ) try: verifyClass(ISpiderLoader, loader_cls) except excs: warnings.warn( - 'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does ' - 'not fully implement scrapy.interfaces.ISpiderLoader interface. ' - 'Please add all missing methods to avoid unexpected runtime errors.', - category=ScrapyDeprecationWarning, stacklevel=2 + "SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does " + "not fully implement scrapy.interfaces.ISpiderLoader interface. " + "Please add all missing methods to avoid unexpected runtime errors.", + category=ScrapyDeprecationWarning, + stacklevel=2, ) return loader_cls.from_settings(settings.frozencopy()) @@ -191,9 +194,12 @@ class CrawlerRunner: @property def spiders(self): - warnings.warn("CrawlerRunner.spiders attribute is renamed to " - "CrawlerRunner.spider_loader.", - category=ScrapyDeprecationWarning, stacklevel=2) + warnings.warn( + "CrawlerRunner.spiders attribute is renamed to " + "CrawlerRunner.spider_loader.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) return self.spider_loader def crawl(self, crawler_or_spidercls, *args, **kwargs): @@ -220,8 +226,9 @@ class CrawlerRunner: """ if isinstance(crawler_or_spidercls, Spider): raise ValueError( - 'The crawler_or_spidercls argument cannot be a spider object, ' - 'it must be a spider class (or a Crawler object)') + "The crawler_or_spidercls argument cannot be a spider object, " + "it must be a spider class (or a Crawler object)" + ) crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) @@ -233,7 +240,7 @@ class CrawlerRunner: def _done(result): self.crawlers.discard(crawler) self._active.discard(d) - self.bootstrap_failed |= not getattr(crawler, 'spider', None) + self.bootstrap_failed |= not getattr(crawler, "spider", None) return result return d.addBoth(_done) @@ -251,8 +258,9 @@ class CrawlerRunner: """ if isinstance(crawler_or_spidercls, Spider): raise ValueError( - 'The crawler_or_spidercls argument cannot be a spider object, ' - 'it must be a spider class (or a Crawler object)') + "The crawler_or_spidercls argument cannot be a spider object, " + "it must be a spider class (or a Crawler object)" + ) if isinstance(crawler_or_spidercls, Crawler): return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) @@ -314,18 +322,23 @@ class CrawlerProcess(CrawlerRunner): def _signal_shutdown(self, signum, _): from twisted.internet import reactor + install_shutdown_handlers(self._signal_kill) signame = signal_names[signum] - logger.info("Received %(signame)s, shutting down gracefully. Send again to force ", - {'signame': signame}) + logger.info( + "Received %(signame)s, shutting down gracefully. Send again to force ", + {"signame": signame}, + ) reactor.callFromThread(self._graceful_stop_reactor) def _signal_kill(self, signum, _): from twisted.internet import reactor + install_shutdown_handlers(signal.SIG_IGN) signame = signal_names[signum] - logger.info('Received %(signame)s twice, forcing unclean shutdown', - {'signame': signame}) + logger.info( + "Received %(signame)s twice, forcing unclean shutdown", {"signame": signame} + ) reactor.callFromThread(self._stop_reactor) def _create_crawler(self, spidercls): @@ -351,6 +364,7 @@ class CrawlerProcess(CrawlerRunner): handlers (default: True) """ from twisted.internet import reactor + if stop_after_crawl: d = self.join() # Don't start the reactor if the deferreds are already fired @@ -364,8 +378,8 @@ class CrawlerProcess(CrawlerRunner): resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() tp = reactor.getThreadPool() - tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE')) - reactor.addSystemEventTrigger('before', 'shutdown', self.stop) + tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) + reactor.addSystemEventTrigger("before", "shutdown", self.stop) reactor.run(installSignalHandlers=False) # blocking call def _graceful_stop_reactor(self): @@ -375,6 +389,7 @@ class CrawlerProcess(CrawlerRunner): def _stop_reactor(self, _=None): from twisted.internet import reactor + try: reactor.stop() except RuntimeError: # raised if already stopped or in shutdown stage diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 4e12a5044..86ff7b9fe 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -17,14 +17,14 @@ class AjaxCrawlMiddleware: """ def __init__(self, settings): - if not settings.getbool('AJAXCRAWL_ENABLED'): + if not settings.getbool("AJAXCRAWL_ENABLED"): raise NotConfigured # XXX: Google parses at least first 100k bytes; scrapy's redirect # middleware parses first 4k. 4k turns out to be insufficient # for this middleware, and parsing 100k could be slow. # We use something in between (32K) by default. - self.lookup_bytes = settings.getint('AJAXCRAWL_MAXSIZE', 32768) + self.lookup_bytes = settings.getint("AJAXCRAWL_MAXSIZE", 32768) @classmethod def from_crawler(cls, crawler): @@ -35,23 +35,25 @@ class AjaxCrawlMiddleware: if not isinstance(response, HtmlResponse) or response.status != 200: return response - if request.method != 'GET': + if request.method != "GET": # other HTTP methods are either not safe or don't have a body return response - if 'ajax_crawlable' in request.meta: # prevent loops + if "ajax_crawlable" in request.meta: # prevent loops return response if not self._has_ajax_crawlable_variant(response): return response # scrapy already handles #! links properly - ajax_crawl_request = request.replace(url=request.url + '#!') - logger.debug("Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s", - {'ajax_crawl_request': ajax_crawl_request, 'request': request}, - extra={'spider': spider}) + ajax_crawl_request = request.replace(url=request.url + "#!") + logger.debug( + "Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s", + {"ajax_crawl_request": ajax_crawl_request, "request": request}, + extra={"spider": spider}, + ) - ajax_crawl_request.meta['ajax_crawlable'] = True + ajax_crawl_request.meta["ajax_crawlable"] = True return ajax_crawl_request def _has_ajax_crawlable_variant(self, response): @@ -59,12 +61,14 @@ class AjaxCrawlMiddleware: Return True if a page without hash fragment could be "AJAX crawlable" according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. """ - body = response.text[:self.lookup_bytes] + body = response.text[: self.lookup_bytes] return _has_ajaxcrawlable_meta(body) # XXX: move it to w3lib? -_ajax_crawlable_re = re.compile(r'') +_ajax_crawlable_re = re.compile( + r'' +) def _has_ajaxcrawlable_meta(text): @@ -82,12 +86,12 @@ def _has_ajaxcrawlable_meta(text): # Stripping scripts and comments is slow (about 20x slower than # just checking if a string is in text); this is a quick fail-fast # path that should work for most pages. - if 'fragment' not in text: + if "fragment" not in text: return False - if 'content' not in text: + if "content" not in text: return False - text = html.remove_tags_with_content(text, ('script', 'noscript')) + text = html.remove_tags_with_content(text, ("script", "noscript")) text = html.replace_entities(text) text = html.remove_comments(text) return _ajax_crawlable_re.search(text) is not None diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 3cba269f1..6495157d7 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -29,14 +29,14 @@ class CookiesMiddleware: @classmethod def from_crawler(cls, crawler): - if not crawler.settings.getbool('COOKIES_ENABLED'): + if not crawler.settings.getbool("COOKIES_ENABLED"): raise NotConfigured - return cls(crawler.settings.getbool('COOKIES_DEBUG')) + return cls(crawler.settings.getbool("COOKIES_DEBUG")) def _process_cookies(self, cookies, *, jar, request): for cookie in cookies: cookie_domain = cookie.domain - if cookie_domain.startswith('.'): + if cookie_domain.startswith("."): cookie_domain = cookie_domain[1:] request_domain = urlparse_cached(request).hostname.lower() @@ -49,7 +49,7 @@ class CookiesMiddleware: jar.set_cookie_if_ok(cookie, request) def process_request(self, request, spider): - if request.meta.get('dont_merge_cookies', False): + if request.meta.get("dont_merge_cookies", False): return cookiejarkey = request.meta.get("cookiejar") @@ -58,12 +58,12 @@ class CookiesMiddleware: self._process_cookies(cookies, jar=jar, request=request) # set Cookie header - request.headers.pop('Cookie', None) + request.headers.pop("Cookie", None) jar.add_cookie_header(request) self._debug_cookie(request, spider) def process_response(self, request, response, spider): - if request.meta.get('dont_merge_cookies', False): + if request.meta.get("dont_merge_cookies", False): return response # extract cookies from Set-Cookie and drop invalid/expired cookies @@ -78,21 +78,25 @@ class CookiesMiddleware: def _debug_cookie(self, request, spider): if self.debug: - cl = [to_unicode(c, errors='replace') - for c in request.headers.getlist('Cookie')] + cl = [ + to_unicode(c, errors="replace") + for c in request.headers.getlist("Cookie") + ] if cl: cookies = "\n".join(f"Cookie: {c}\n" for c in cl) msg = f"Sending cookies to: {request}\n{cookies}" - logger.debug(msg, extra={'spider': spider}) + logger.debug(msg, extra={"spider": spider}) def _debug_set_cookie(self, response, spider): if self.debug: - cl = [to_unicode(c, errors='replace') - for c in response.headers.getlist('Set-Cookie')] + cl = [ + to_unicode(c, errors="replace") + for c in response.headers.getlist("Set-Cookie") + ] if cl: cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl) msg = f"Received cookies from: {response}\n{cookies}" - logger.debug(msg, extra={'spider': spider}) + logger.debug(msg, extra={"spider": spider}) def _format_cookie(self, cookie, request): """ @@ -113,8 +117,11 @@ class CookiesMiddleware: try: decoded[key] = cookie[key].decode("utf8") except UnicodeDecodeError: - logger.warning("Non UTF-8 encoded cookie found in request %s: %s", - request, cookie) + logger.warning( + "Non UTF-8 encoded cookie found in request %s: %s", + request, + cookie, + ) decoded[key] = cookie[key].decode("latin1", errors="replace") cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index e01e9cc76..410015281 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -16,7 +16,7 @@ from scrapy.responsetypes import responsetypes warn( - 'scrapy.downloadermiddlewares.decompression is deprecated', + "scrapy.downloadermiddlewares.decompression is deprecated", ScrapyDeprecationWarning, stacklevel=2, ) @@ -26,15 +26,15 @@ logger = logging.getLogger(__name__) class DecompressionMiddleware: - """ This middleware tries to recognise and extract the possibly compressed - responses that may arrive. """ + """This middleware tries to recognise and extract the possibly compressed + responses that may arrive.""" def __init__(self): self._formats = { - 'tar': self._is_tar, - 'zip': self._is_zip, - 'gz': self._is_gzip, - 'bz2': self._is_bzip2 + "tar": self._is_tar, + "zip": self._is_zip, + "gz": self._is_gzip, + "bz2": self._is_bzip2, } def _is_tar(self, response): @@ -86,7 +86,10 @@ class DecompressionMiddleware: for fmt, func in self._formats.items(): new_response = func(response) if new_response: - logger.debug('Decompressed response with format: %(responsefmt)s', - {'responsefmt': fmt}, extra={'spider': spider}) + logger.debug( + "Decompressed response with format: %(responsefmt)s", + {"responsefmt": fmt}, + extra={"spider": spider}, + ) return new_response return response diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index f67961881..cdacc7368 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -8,13 +8,12 @@ from scrapy.utils.python import without_none_values class DefaultHeadersMiddleware: - def __init__(self, headers): self._headers = headers @classmethod def from_crawler(cls, crawler): - headers = without_none_values(crawler.settings['DEFAULT_REQUEST_HEADERS']) + headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"]) return cls(headers.items()) def process_request(self, request, spider): diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index d373a22df..a926ecf56 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -8,19 +8,18 @@ from scrapy import signals class DownloadTimeoutMiddleware: - def __init__(self, timeout=180): self._timeout = timeout @classmethod def from_crawler(cls, crawler): - o = cls(crawler.settings.getfloat('DOWNLOAD_TIMEOUT')) + o = cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT")) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o def spider_opened(self, spider): - self._timeout = getattr(spider, 'download_timeout', self._timeout) + self._timeout = getattr(spider, "download_timeout", self._timeout) def process_request(self, request, spider): if self._timeout: - request.meta.setdefault('download_timeout', self._timeout) + request.meta.setdefault("download_timeout", self._timeout) diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 1bee3e279..de5a81388 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -24,27 +24,29 @@ class HttpAuthMiddleware: return o def spider_opened(self, spider): - usr = getattr(spider, 'http_user', '') - pwd = getattr(spider, 'http_pass', '') + usr = getattr(spider, "http_user", "") + pwd = getattr(spider, "http_pass", "") if usr or pwd: self.auth = basic_auth_header(usr, pwd) - if not hasattr(spider, 'http_auth_domain'): - warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security ' - 'problems if the spider makes requests to several different domains. http_auth_domain ' - 'will be set to the domain of the first request, please set it to the correct value ' - 'explicitly.', - category=ScrapyDeprecationWarning) + if not hasattr(spider, "http_auth_domain"): + warnings.warn( + "Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security " + "problems if the spider makes requests to several different domains. http_auth_domain " + "will be set to the domain of the first request, please set it to the correct value " + "explicitly.", + category=ScrapyDeprecationWarning, + ) self.domain_unset = True else: self.domain = spider.http_auth_domain self.domain_unset = False def process_request(self, request, spider): - auth = getattr(self, 'auth', None) - if auth and b'Authorization' not in request.headers: + auth = getattr(self, "auth", None) + if auth and b"Authorization" not in request.headers: domain = urlparse_cached(request).hostname if self.domain_unset: self.domain = domain self.domain_unset = False if not self.domain or url_is_from_any_domain(request.url, [self.domain]): - request.headers[b'Authorization'] = auth + request.headers[b"Authorization"] = auth diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 80ed7ac75..eb2754f1d 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -29,21 +29,31 @@ HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddlew class HttpCacheMiddleware: - DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, - ConnectionRefusedError, ConnectionDone, ConnectError, - ConnectionLost, TCPTimedOutError, ResponseFailed, - IOError) + DOWNLOAD_EXCEPTIONS = ( + defer.TimeoutError, + TimeoutError, + DNSLookupError, + ConnectionRefusedError, + ConnectionDone, + ConnectError, + ConnectionLost, + TCPTimedOutError, + ResponseFailed, + IOError, + ) def __init__(self, settings: Settings, stats: StatsCollector) -> None: - if not settings.getbool('HTTPCACHE_ENABLED'): + if not settings.getbool("HTTPCACHE_ENABLED"): raise NotConfigured - self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings) - self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) - self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') + self.policy = load_object(settings["HTTPCACHE_POLICY"])(settings) + self.storage = load_object(settings["HTTPCACHE_STORAGE"])(settings) + self.ignore_missing = settings.getbool("HTTPCACHE_IGNORE_MISSING") self.stats = stats @classmethod - def from_crawler(cls: Type[HttpCacheMiddlewareTV], crawler: Crawler) -> HttpCacheMiddlewareTV: + def from_crawler( + cls: Type[HttpCacheMiddlewareTV], crawler: Crawler + ) -> HttpCacheMiddlewareTV: o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) @@ -56,78 +66,86 @@ class HttpCacheMiddleware: self.storage.close_spider(spider) def process_request(self, request: Request, spider: Spider) -> Optional[Response]: - if request.meta.get('dont_cache', False): + if request.meta.get("dont_cache", False): return None # Skip uncacheable requests if not self.policy.should_cache_request(request): - request.meta['_dont_cache'] = True # flag as uncacheable + request.meta["_dont_cache"] = True # flag as uncacheable return None # Look for cached response and check if expired cachedresponse = self.storage.retrieve_response(spider, request) if cachedresponse is None: - self.stats.inc_value('httpcache/miss', spider=spider) + self.stats.inc_value("httpcache/miss", spider=spider) if self.ignore_missing: - self.stats.inc_value('httpcache/ignore', spider=spider) + self.stats.inc_value("httpcache/ignore", spider=spider) raise IgnoreRequest(f"Ignored request not in cache: {request}") return None # first time request # Return cached response only if not expired - cachedresponse.flags.append('cached') + cachedresponse.flags.append("cached") if self.policy.is_cached_response_fresh(cachedresponse, request): - self.stats.inc_value('httpcache/hit', spider=spider) + self.stats.inc_value("httpcache/hit", spider=spider) return cachedresponse # Keep a reference to cached response to avoid a second cache lookup on # process_response hook - request.meta['cached_response'] = cachedresponse + request.meta["cached_response"] = cachedresponse return None - def process_response(self, request: Request, response: Response, spider: Spider) -> Response: - if request.meta.get('dont_cache', False): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Response: + if request.meta.get("dont_cache", False): return response # Skip cached responses and uncacheable requests - if 'cached' in response.flags or '_dont_cache' in request.meta: - request.meta.pop('_dont_cache', None) + if "cached" in response.flags or "_dont_cache" in request.meta: + request.meta.pop("_dont_cache", None) return response # RFC2616 requires origin server to set Date header, # https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 - if 'Date' not in response.headers: - response.headers['Date'] = formatdate(usegmt=True) + if "Date" not in response.headers: + response.headers["Date"] = formatdate(usegmt=True) # Do not validate first-hand responses - cachedresponse = request.meta.pop('cached_response', None) + cachedresponse = request.meta.pop("cached_response", None) if cachedresponse is None: - self.stats.inc_value('httpcache/firsthand', spider=spider) + self.stats.inc_value("httpcache/firsthand", spider=spider) self._cache_response(spider, response, request, cachedresponse) return response if self.policy.is_cached_response_valid(cachedresponse, response, request): - self.stats.inc_value('httpcache/revalidate', spider=spider) + self.stats.inc_value("httpcache/revalidate", spider=spider) return cachedresponse - self.stats.inc_value('httpcache/invalidate', spider=spider) + self.stats.inc_value("httpcache/invalidate", spider=spider) self._cache_response(spider, response, request, cachedresponse) return response def process_exception( self, request: Request, exception: Exception, spider: Spider ) -> Optional[Response]: - cachedresponse = request.meta.pop('cached_response', None) - if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS): - self.stats.inc_value('httpcache/errorrecovery', spider=spider) + cachedresponse = request.meta.pop("cached_response", None) + if cachedresponse is not None and isinstance( + exception, self.DOWNLOAD_EXCEPTIONS + ): + self.stats.inc_value("httpcache/errorrecovery", spider=spider) return cachedresponse return None def _cache_response( - self, spider: Spider, response: Response, request: Request, cachedresponse: Optional[Response] + self, + spider: Spider, + response: Response, + request: Request, + cachedresponse: Optional[Response], ) -> None: if self.policy.should_cache_response(response, request): - self.stats.inc_value('httpcache/store', spider=spider) + self.stats.inc_value("httpcache/store", spider=spider) self.storage.store_response(spider, request, response) else: - self.stats.inc_value('httpcache/uncacheable', spider=spider) + self.stats.inc_value("httpcache/uncacheable", spider=spider) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index e57f6fcf8..7b5f4824a 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -8,17 +8,19 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip -ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] +ACCEPTED_ENCODINGS = [b"gzip", b"deflate"] try: import brotli - ACCEPTED_ENCODINGS.append(b'br') + + ACCEPTED_ENCODINGS.append(b"br") except ImportError: pass try: import zstandard - ACCEPTED_ENCODINGS.append(b'zstd') + + ACCEPTED_ENCODINGS.append(b"zstd") except ImportError: pass @@ -26,12 +28,13 @@ except ImportError: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" + def __init__(self, stats=None): self.stats = stats @classmethod def from_crawler(cls, crawler): - if not crawler.settings.getbool('COMPRESSION_ENABLED'): + if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured try: return cls(stats=crawler.stats) @@ -47,21 +50,26 @@ class HttpCompressionMiddleware: return result def process_request(self, request, spider): - request.headers.setdefault('Accept-Encoding', - b", ".join(ACCEPTED_ENCODINGS)) + request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) def process_response(self, request, response, spider): - if request.method == 'HEAD': + if request.method == "HEAD": return response if isinstance(response, Response): - content_encoding = response.headers.getlist('Content-Encoding') + content_encoding = response.headers.getlist("Content-Encoding") if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) if self.stats: - self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) - self.stats.inc_value('httpcompression/response_count', spider=spider) + self.stats.inc_value( + "httpcompression/response_bytes", + len(decoded_body), + spider=spider, + ) + self.stats.inc_value( + "httpcompression/response_count", spider=spider + ) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) @@ -69,18 +77,18 @@ class HttpCompressionMiddleware: if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable - kwargs['encoding'] = None + kwargs["encoding"] = None response = response.replace(**kwargs) if not content_encoding: - del response.headers['Content-Encoding'] + del response.headers["Content-Encoding"] return response def _decode(self, body, encoding): - if encoding == b'gzip' or encoding == b'x-gzip': + if encoding == b"gzip" or encoding == b"x-gzip": body = gunzip(body) - if encoding == b'deflate': + if encoding == b"deflate": try: body = zlib.decompress(body) except zlib.error: @@ -90,9 +98,9 @@ class HttpCompressionMiddleware: # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx # http://www.gzip.org/zlib/zlib_faq.html#faq38 body = zlib.decompress(body, -15) - if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: + if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: body = brotli.decompress(body) - if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS: + if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: # Using its streaming API since its simple API could handle only cases # where there is content size data embedded in the frame reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body)) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index dd8a7e797..489867918 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -8,8 +8,7 @@ from scrapy.utils.python import to_bytes class HttpProxyMiddleware: - - def __init__(self, auth_encoding='latin-1'): + def __init__(self, auth_encoding="latin-1"): self.auth_encoding = auth_encoding self.proxies = {} for type_, url in getproxies().items(): @@ -22,20 +21,20 @@ class HttpProxyMiddleware: @classmethod def from_crawler(cls, crawler): - if not crawler.settings.getbool('HTTPPROXY_ENABLED'): + if not crawler.settings.getbool("HTTPPROXY_ENABLED"): raise NotConfigured - auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING') + auth_encoding = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") return cls(auth_encoding) def _basic_auth_header(self, username, password): user_pass = to_bytes( - f'{unquote(username)}:{unquote(password)}', - encoding=self.auth_encoding) + f"{unquote(username)}:{unquote(password)}", encoding=self.auth_encoding + ) return base64.b64encode(user_pass) def _get_proxy(self, url, orig_type): proxy_type, user, password, hostport = _parse_proxy(url) - proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) + proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) if user: creds = self._basic_auth_header(user, password) @@ -46,39 +45,36 @@ class HttpProxyMiddleware: def process_request(self, request, spider): creds, proxy_url = None, None - if 'proxy' in request.meta: - if request.meta['proxy'] is not None: - creds, proxy_url = self._get_proxy(request.meta['proxy'], '') + if "proxy" in request.meta: + if request.meta["proxy"] is not None: + creds, proxy_url = self._get_proxy(request.meta["proxy"], "") elif self.proxies: parsed = urlparse_cached(request) scheme = parsed.scheme if ( - ( - # 'no_proxy' is only supported by http schemes - scheme not in ('http', 'https') - or not proxy_bypass(parsed.hostname) - ) - and scheme in self.proxies - ): + # 'no_proxy' is only supported by http schemes + scheme not in ("http", "https") + or not proxy_bypass(parsed.hostname) + ) and scheme in self.proxies: creds, proxy_url = self.proxies[scheme] self._set_proxy_and_creds(request, proxy_url, creds) def _set_proxy_and_creds(self, request, proxy_url, creds): if proxy_url: - request.meta['proxy'] = proxy_url - elif request.meta.get('proxy') is not None: - request.meta['proxy'] = None + request.meta["proxy"] = proxy_url + elif request.meta.get("proxy") is not None: + request.meta["proxy"] = None if creds: - request.headers[b'Proxy-Authorization'] = b'Basic ' + creds - request.meta['_auth_proxy'] = proxy_url - elif '_auth_proxy' in request.meta: - if proxy_url != request.meta['_auth_proxy']: - if b'Proxy-Authorization' in request.headers: - del request.headers[b'Proxy-Authorization'] - del request.meta['_auth_proxy'] - elif b'Proxy-Authorization' in request.headers: + request.headers[b"Proxy-Authorization"] = b"Basic " + creds + request.meta["_auth_proxy"] = proxy_url + elif "_auth_proxy" in request.meta: + if proxy_url != request.meta["_auth_proxy"]: + if b"Proxy-Authorization" in request.headers: + del request.headers[b"Proxy-Authorization"] + del request.meta["_auth_proxy"] + elif b"Proxy-Authorization" in request.headers: if proxy_url: - request.meta['_auth_proxy'] = proxy_url + request.meta["_auth_proxy"] = proxy_url else: - del request.headers[b'Proxy-Authorization'] + del request.headers[b"Proxy-Authorization"] diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index fba825947..003c59fc4 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -17,57 +17,66 @@ def _build_redirect_request(source_request, *, url, **kwargs): **kwargs, cookies=None, ) - if 'Cookie' in redirect_request.headers: + if "Cookie" in redirect_request.headers: source_request_netloc = urlparse_cached(source_request).netloc redirect_request_netloc = urlparse_cached(redirect_request).netloc if source_request_netloc != redirect_request_netloc: - del redirect_request.headers['Cookie'] + del redirect_request.headers["Cookie"] return redirect_request class BaseRedirectMiddleware: - enabled_setting = 'REDIRECT_ENABLED' + enabled_setting = "REDIRECT_ENABLED" def __init__(self, settings): if not settings.getbool(self.enabled_setting): raise NotConfigured - self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES') - self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST') + self.max_redirect_times = settings.getint("REDIRECT_MAX_TIMES") + self.priority_adjust = settings.getint("REDIRECT_PRIORITY_ADJUST") @classmethod def from_crawler(cls, crawler): return cls(crawler.settings) def _redirect(self, redirected, request, spider, reason): - ttl = request.meta.setdefault('redirect_ttl', self.max_redirect_times) - redirects = request.meta.get('redirect_times', 0) + 1 + ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times) + redirects = request.meta.get("redirect_times", 0) + 1 if ttl and redirects <= self.max_redirect_times: - redirected.meta['redirect_times'] = redirects - redirected.meta['redirect_ttl'] = ttl - 1 - redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + [request.url] - redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + [reason] + redirected.meta["redirect_times"] = redirects + redirected.meta["redirect_ttl"] = ttl - 1 + redirected.meta["redirect_urls"] = request.meta.get("redirect_urls", []) + [ + request.url + ] + redirected.meta["redirect_reasons"] = request.meta.get( + "redirect_reasons", [] + ) + [reason] redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust - logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s", - {'reason': reason, 'redirected': redirected, 'request': request}, - extra={'spider': spider}) + logger.debug( + "Redirecting (%(reason)s) to %(redirected)s from %(request)s", + {"reason": reason, "redirected": redirected, "request": request}, + extra={"spider": spider}, + ) return redirected - logger.debug("Discarding %(request)s: max redirections reached", - {'request': request}, extra={'spider': spider}) + logger.debug( + "Discarding %(request)s: max redirections reached", + {"request": request}, + extra={"spider": spider}, + ) raise IgnoreRequest("max redirections reached") def _redirect_request_using_get(self, request, redirect_url): redirect_request = _build_redirect_request( request, url=redirect_url, - method='GET', - body='', + method="GET", + body="", ) - redirect_request.headers.pop('Content-Type', None) - redirect_request.headers.pop('Content-Length', None) + redirect_request.headers.pop("Content-Type", None) + redirect_request.headers.pop("Content-Length", None) return redirect_request @@ -79,25 +88,25 @@ class RedirectMiddleware(BaseRedirectMiddleware): def process_response(self, request, response, spider): if ( - request.meta.get('dont_redirect', False) - or response.status in getattr(spider, 'handle_httpstatus_list', []) - or response.status in request.meta.get('handle_httpstatus_list', []) - or request.meta.get('handle_httpstatus_all', False) + request.meta.get("dont_redirect", False) + or response.status in getattr(spider, "handle_httpstatus_list", []) + or response.status in request.meta.get("handle_httpstatus_list", []) + or request.meta.get("handle_httpstatus_all", False) ): return response allowed_status = (301, 302, 303, 307, 308) - if 'Location' not in response.headers or response.status not in allowed_status: + if "Location" not in response.headers or response.status not in allowed_status: return response - location = safe_url_string(response.headers['Location']) - if response.headers['Location'].startswith(b'//'): + location = safe_url_string(response.headers["Location"]) + if response.headers["Location"].startswith(b"//"): request_scheme = urlparse(request.url).scheme - location = request_scheme + '://' + location.lstrip('/') + location = request_scheme + "://" + location.lstrip("/") redirected_url = urljoin(request.url, location) - if response.status in (301, 307, 308) or request.method == 'HEAD': + if response.status in (301, 307, 308) or request.method == "HEAD": redirected = _build_redirect_request(request, url=redirected_url) return self._redirect(redirected, request, spider, response.status) @@ -107,25 +116,24 @@ class RedirectMiddleware(BaseRedirectMiddleware): class MetaRefreshMiddleware(BaseRedirectMiddleware): - enabled_setting = 'METAREFRESH_ENABLED' + enabled_setting = "METAREFRESH_ENABLED" def __init__(self, settings): super().__init__(settings) - self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS') - self._maxdelay = settings.getint('METAREFRESH_MAXDELAY') + self._ignore_tags = settings.getlist("METAREFRESH_IGNORE_TAGS") + self._maxdelay = settings.getint("METAREFRESH_MAXDELAY") def process_response(self, request, response, spider): if ( - request.meta.get('dont_redirect', False) - or request.method == 'HEAD' + request.meta.get("dont_redirect", False) + or request.method == "HEAD" or not isinstance(response, HtmlResponse) ): return response - interval, url = get_meta_refresh(response, - ignore_tags=self._ignore_tags) + interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags) if url and interval < self._maxdelay: redirected = self._redirect_request_using_get(request, url) - return self._redirect(redirected, request, spider, 'meta refresh') + return self._redirect(redirected, request, spider, "meta refresh") return response diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 2de610e86..8a8f15f9a 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -39,11 +39,11 @@ def get_retry_request( request: Request, *, spider: Spider, - reason: Union[str, Exception] = 'unspecified', + reason: Union[str, Exception] = "unspecified", max_retry_times: Optional[int] = None, priority_adjust: Optional[int] = None, logger: Logger = retry_logger, - stats_base_key: str = 'retry', + stats_base_key: str = "retry", ): """ Returns a new :class:`~scrapy.Request` object to retry the specified @@ -87,22 +87,22 @@ def get_retry_request( """ settings = spider.crawler.settings stats = spider.crawler.stats - retry_times = request.meta.get('retry_times', 0) + 1 + retry_times = request.meta.get("retry_times", 0) + 1 if max_retry_times is None: - max_retry_times = request.meta.get('max_retry_times') + max_retry_times = request.meta.get("max_retry_times") if max_retry_times is None: - max_retry_times = settings.getint('RETRY_TIMES') + max_retry_times = settings.getint("RETRY_TIMES") if retry_times <= max_retry_times: logger.debug( "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", - {'request': request, 'retry_times': retry_times, 'reason': reason}, - extra={'spider': spider} + {"request": request, "retry_times": retry_times, "reason": reason}, + extra={"spider": spider}, ) new_request: Request = request.copy() - new_request.meta['retry_times'] = retry_times + new_request.meta["retry_times"] = retry_times new_request.dont_filter = True if priority_adjust is None: - priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') + priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") new_request.priority = request.priority + priority_adjust if callable(reason): @@ -110,15 +110,14 @@ def get_retry_request( if isinstance(reason, Exception): reason = global_object_name(reason.__class__) - stats.inc_value(f'{stats_base_key}/count') - stats.inc_value(f'{stats_base_key}/reason_count/{reason}') + stats.inc_value(f"{stats_base_key}/count") + stats.inc_value(f"{stats_base_key}/reason_count/{reason}") return new_request - stats.inc_value(f'{stats_base_key}/max_reached') + stats.inc_value(f"{stats_base_key}/max_reached") logger.error( - "Gave up retrying %(request)s (failed %(retry_times)d times): " - "%(reason)s", - {'request': request, 'retry_times': retry_times, 'reason': reason}, - extra={'spider': spider}, + "Gave up retrying %(request)s (failed %(retry_times)d times): " "%(reason)s", + {"request": request, "retry_times": retry_times, "reason": reason}, + extra={"spider": spider}, ) return None @@ -127,24 +126,35 @@ class RetryMiddleware: # IOError is raised by the HttpCompression middleware when trying to # decompress an empty response - EXCEPTIONS_TO_RETRY = (defer.TimeoutError, TimeoutError, DNSLookupError, - ConnectionRefusedError, ConnectionDone, ConnectError, - ConnectionLost, TCPTimedOutError, ResponseFailed, - IOError, TunnelError) + EXCEPTIONS_TO_RETRY = ( + defer.TimeoutError, + TimeoutError, + DNSLookupError, + ConnectionRefusedError, + ConnectionDone, + ConnectError, + ConnectionLost, + TCPTimedOutError, + ResponseFailed, + IOError, + TunnelError, + ) def __init__(self, settings): - if not settings.getbool('RETRY_ENABLED'): + if not settings.getbool("RETRY_ENABLED"): raise NotConfigured - self.max_retry_times = settings.getint('RETRY_TIMES') - self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES')) - self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') + self.max_retry_times = settings.getint("RETRY_TIMES") + self.retry_http_codes = set( + int(x) for x in settings.getlist("RETRY_HTTP_CODES") + ) + self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") @classmethod def from_crawler(cls, crawler): return cls(crawler.settings) def process_response(self, request, response, spider): - if request.meta.get('dont_retry', False): + if request.meta.get("dont_retry", False): return response if response.status in self.retry_http_codes: reason = response_status_message(response.status) @@ -152,15 +162,14 @@ class RetryMiddleware: return response def process_exception(self, request, exception, spider): - if ( - isinstance(exception, self.EXCEPTIONS_TO_RETRY) - and not request.meta.get('dont_retry', False) + if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get( + "dont_retry", False ): return self._retry(request, exception, spider) def _retry(self, request, reason, spider): - max_retry_times = request.meta.get('max_retry_times', self.max_retry_times) - priority_adjust = request.meta.get('priority_adjust', self.priority_adjust) + max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) + priority_adjust = request.meta.get("priority_adjust", self.priority_adjust) return get_retry_request( request, reason=reason, diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7bd39aa43..326c35290 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -20,23 +20,23 @@ class RobotsTxtMiddleware: DOWNLOAD_PRIORITY = 1000 def __init__(self, crawler): - if not crawler.settings.getbool('ROBOTSTXT_OBEY'): + if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured - self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy') - self._robotstxt_useragent = crawler.settings.get('ROBOTSTXT_USER_AGENT', None) + self._default_useragent = crawler.settings.get("USER_AGENT", "Scrapy") + self._robotstxt_useragent = crawler.settings.get("ROBOTSTXT_USER_AGENT", None) self.crawler = crawler self._parsers = {} - self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER')) + self._parserimpl = load_object(crawler.settings.get("ROBOTSTXT_PARSER")) # check if parser dependencies are met, this should throw an error otherwise. - self._parserimpl.from_crawler(self.crawler, b'') + self._parserimpl.from_crawler(self.crawler, b"") @classmethod def from_crawler(cls, crawler): return cls(crawler) def process_request(self, request, spider): - if request.meta.get('dont_obey_robotstxt'): + if request.meta.get("dont_obey_robotstxt"): return d = maybeDeferred(self.robot_parser, request, spider) d.addCallback(self.process_request_2, request, spider) @@ -48,11 +48,14 @@ class RobotsTxtMiddleware: useragent = self._robotstxt_useragent if not useragent: - useragent = request.headers.get(b'User-Agent', self._default_useragent) + useragent = request.headers.get(b"User-Agent", self._default_useragent) if not rp.allowed(request.url, useragent): - logger.debug("Forbidden by robots.txt: %(request)s", - {'request': request}, extra={'spider': spider}) - self.crawler.stats.inc_value('robotstxt/forbidden') + logger.debug( + "Forbidden by robots.txt: %(request)s", + {"request": request}, + extra={"spider": spider}, + ) + self.crawler.stats.inc_value("robotstxt/forbidden") raise IgnoreRequest("Forbidden by robots.txt") def robot_parser(self, request, spider): @@ -65,13 +68,13 @@ class RobotsTxtMiddleware: robotsreq = Request( robotsurl, priority=self.DOWNLOAD_PRIORITY, - meta={'dont_obey_robotstxt': True} + meta={"dont_obey_robotstxt": True}, ) dfd = self.crawler.engine.download(robotsreq) dfd.addCallback(self._parse_robots, netloc, spider) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) - self.crawler.stats.inc_value('robotstxt/request_count') + self.crawler.stats.inc_value("robotstxt/request_count") if isinstance(self._parsers[netloc], Deferred): d = Deferred() @@ -79,21 +82,26 @@ class RobotsTxtMiddleware: def cb(result): d.callback(result) return result + self._parsers[netloc].addCallback(cb) return d return self._parsers[netloc] def _logerror(self, failure, request, spider): if failure.type is not IgnoreRequest: - logger.error("Error downloading %(request)s: %(f_exception)s", - {'request': request, 'f_exception': failure.value}, - exc_info=failure_to_exc_info(failure), - extra={'spider': spider}) + logger.error( + "Error downloading %(request)s: %(f_exception)s", + {"request": request, "f_exception": failure.value}, + exc_info=failure_to_exc_info(failure), + extra={"spider": spider}, + ) return failure def _parse_robots(self, response, netloc, spider): - self.crawler.stats.inc_value('robotstxt/response_count') - self.crawler.stats.inc_value(f'robotstxt/response_status_count/{response.status}') + self.crawler.stats.inc_value("robotstxt/response_count") + self.crawler.stats.inc_value( + f"robotstxt/response_status_count/{response.status}" + ) rp = self._parserimpl.from_crawler(self.crawler, response.body) rp_dfd = self._parsers[netloc] self._parsers[netloc] = rp @@ -101,7 +109,7 @@ class RobotsTxtMiddleware: def _robots_error(self, failure, netloc): if failure.type is not IgnoreRequest: - key = f'robotstxt/exception_count/{failure.type}' + key = f"robotstxt/exception_count/{failure.type}" self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] self._parsers[netloc] = None diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 2bb022c16..a0f62e262 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -11,40 +11,50 @@ def get_header_size(headers): if isinstance(value, (list, tuple)): for v in value: size += len(b": ") + len(key) + len(v) - return size + len(b'\r\n') * (len(headers.keys()) - 1) + return size + len(b"\r\n") * (len(headers.keys()) - 1) def get_status_size(response_status): - return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15 + return len(to_bytes(http.RESPONSES.get(response_status, b""))) + 15 # resp.status + b"\r\n" + b"HTTP/1.1 <100-599> " class DownloaderStats: - def __init__(self, stats): self.stats = stats @classmethod def from_crawler(cls, crawler): - if not crawler.settings.getbool('DOWNLOADER_STATS'): + if not crawler.settings.getbool("DOWNLOADER_STATS"): raise NotConfigured return cls(crawler.stats) def process_request(self, request, spider): - self.stats.inc_value('downloader/request_count', spider=spider) - self.stats.inc_value(f'downloader/request_method_count/{request.method}', spider=spider) + self.stats.inc_value("downloader/request_count", spider=spider) + self.stats.inc_value( + f"downloader/request_method_count/{request.method}", spider=spider + ) reqlen = len(request_httprepr(request)) - self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider) + self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider) def process_response(self, request, response, spider): - self.stats.inc_value('downloader/response_count', spider=spider) - self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider) - reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4 + self.stats.inc_value("downloader/response_count", spider=spider) + self.stats.inc_value( + f"downloader/response_status_count/{response.status}", spider=spider + ) + reslen = ( + len(response.body) + + get_header_size(response.headers) + + get_status_size(response.status) + + 4 + ) # response.body + b"\r\n"+ response.header + b"\r\n" + response.status - self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) + self.stats.inc_value("downloader/response_bytes", reslen, spider=spider) return response def process_exception(self, request, exception, spider): ex_class = global_object_name(exception.__class__) - self.stats.inc_value('downloader/exception_count', spider=spider) - self.stats.inc_value(f'downloader/exception_type_count/{ex_class}', spider=spider) + self.stats.inc_value("downloader/exception_count", spider=spider) + self.stats.inc_value( + f"downloader/exception_type_count/{ex_class}", spider=spider + ) diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 3ee7bd129..856a275ab 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -6,18 +6,18 @@ from scrapy import signals class UserAgentMiddleware: """This middleware allows spiders to override the user_agent""" - def __init__(self, user_agent='Scrapy'): + def __init__(self, user_agent="Scrapy"): self.user_agent = user_agent @classmethod def from_crawler(cls, crawler): - o = cls(crawler.settings['USER_AGENT']) + o = cls(crawler.settings["USER_AGENT"]) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o def spider_opened(self, spider): - self.user_agent = getattr(spider, 'user_agent', self.user_agent) + self.user_agent = getattr(spider, "user_agent", self.user_agent) def process_request(self, request, spider): if self.user_agent: - request.headers.setdefault(b'User-Agent', self.user_agent) + request.headers.setdefault(b"User-Agent", self.user_agent) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 6dca11d31..fa0f8f846 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -18,7 +18,9 @@ BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter") class BaseDupeFilter: @classmethod - def from_settings(cls: Type[BaseDupeFilterTV], settings: BaseSettings) -> BaseDupeFilterTV: + def from_settings( + cls: Type[BaseDupeFilterTV], settings: BaseSettings + ) -> BaseDupeFilterTV: return cls() def request_seen(self, request: Request) -> bool: @@ -55,13 +57,15 @@ class RFPDupeFilter(BaseDupeFilter): self.debug = debug self.logger = logging.getLogger(__name__) if path: - self.file = Path(path, 'requests.seen').open('a+', encoding="utf-8") + self.file = Path(path, "requests.seen").open("a+", encoding="utf-8") self.file.seek(0) self.fingerprints.update(x.rstrip() for x in self.file) @classmethod - def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV: - debug = settings.getbool('DUPEFILTER_DEBUG') + def from_settings( + cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None + ) -> RFPDupeFilterTV: + debug = settings.getbool("DUPEFILTER_DEBUG") try: return cls(job_dir(settings), debug, fingerprinter=fingerprinter) except TypeError: @@ -100,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter): return True self.fingerprints.add(fp) if self.file: - self.file.write(fp + '\n') + self.file.write(fp + "\n") return False def request_fingerprint(self, request: Request) -> str: @@ -113,13 +117,15 @@ class RFPDupeFilter(BaseDupeFilter): def log(self, request: Request, spider: Spider) -> None: if self.debug: msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)" - args = {'request': request, 'referer': referer_str(request)} - self.logger.debug(msg, args, extra={'spider': spider}) + args = {"request": request, "referer": referer_str(request)} + self.logger.debug(msg, args, extra={"spider": spider}) elif self.logdupes: - msg = ("Filtered duplicate request: %(request)s" - " - no more duplicates will be shown" - " (see DUPEFILTER_DEBUG to show all duplicates)") - self.logger.debug(msg, {'request': request}, extra={'spider': spider}) + msg = ( + "Filtered duplicate request: %(request)s" + " - no more duplicates will be shown" + " (see DUPEFILTER_DEBUG to show all duplicates)" + ) + self.logger.debug(msg, {"request": request}, extra={"spider": spider}) self.logdupes = False - spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider) + spider.crawler.stats.inc_value("dupefilter/filtered", spider=spider) diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 0c410f035..7f202b8b8 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -10,6 +10,7 @@ new exceptions here without documenting them there. class NotConfigured(Exception): """Indicates a missing configuration situation""" + pass @@ -18,6 +19,7 @@ class _InvalidOutput(TypeError): Indicates an invalid value has been returned by a middleware's processing method. Internal and undocumented, it should not be raised or caught by user code. """ + pass @@ -30,13 +32,14 @@ class IgnoreRequest(Exception): class DontCloseSpider(Exception): """Request the spider not to be closed yet""" + pass class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" - def __init__(self, reason='cancelled'): + def __init__(self, reason="cancelled"): super().__init__() self.reason = reason @@ -58,11 +61,13 @@ class StopDownload(Exception): class DropItem(Exception): """Drop item from the item pipeline""" + pass class NotSupported(Exception): """Indicates a feature or method is not supported""" + pass @@ -73,7 +78,7 @@ class UsageError(Exception): """To indicate a command-line usage error""" def __init__(self, *a, **kw): - self.print_help = kw.pop('print_help', True) + self.print_help = kw.pop("print_help", True) super().__init__(*a, **kw) @@ -81,9 +86,11 @@ class ScrapyDeprecationWarning(Warning): """Warning category for deprecated features, since the default DeprecationWarning is silenced on Python 2.7+ """ + pass class ContractFail(AssertionError): """Error raised in case of a failing contract""" + pass diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 2135c3bbc..7d9a9b6ff 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -19,13 +19,19 @@ from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder -__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', - 'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', - 'JsonItemExporter', 'MarshalItemExporter'] +__all__ = [ + "BaseItemExporter", + "PprintItemExporter", + "PickleItemExporter", + "CsvItemExporter", + "XmlItemExporter", + "JsonLinesItemExporter", + "JsonItemExporter", + "MarshalItemExporter", +] class BaseItemExporter: - def __init__(self, *, dont_fail=False, **kwargs): self._kwargs = kwargs self._configure(kwargs, dont_fail=dont_fail) @@ -35,10 +41,10 @@ class BaseItemExporter: If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) """ - self.encoding = options.pop('encoding', None) - self.fields_to_export = options.pop('fields_to_export', None) - self.export_empty_fields = options.pop('export_empty_fields', False) - self.indent = options.pop('indent', None) + self.encoding = options.pop("encoding", None) + self.fields_to_export = options.pop("fields_to_export", None) + self.export_empty_fields = options.pop("export_empty_fields", False) + self.indent = options.pop("indent", None) if not dont_fail and options: raise TypeError(f"Unexpected options: {', '.join(options.keys())}") @@ -46,7 +52,7 @@ class BaseItemExporter: raise NotImplementedError def serialize_field(self, field, name, value): - serializer = field.get('serializer', lambda x: x) + serializer = field.get("serializer", lambda x: x) return serializer(value) def start_exporting(self): @@ -74,8 +80,7 @@ class BaseItemExporter: field_iter = self.fields_to_export.items() else: field_iter = ( - (x, y) for x, y in self.fields_to_export.items() - if x in item + (x, y) for x, y in self.fields_to_export.items() if x in item ) else: if include_empty: @@ -98,36 +103,36 @@ class BaseItemExporter: class JsonLinesItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): super().__init__(dont_fail=True, **kwargs) self.file = file - self._kwargs.setdefault('ensure_ascii', not self.encoding) + self._kwargs.setdefault("ensure_ascii", not self.encoding) self.encoder = ScrapyJSONEncoder(**self._kwargs) def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) - data = self.encoder.encode(itemdict) + '\n' + data = self.encoder.encode(itemdict) + "\n" self.file.write(to_bytes(data, self.encoding)) class JsonItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): super().__init__(dont_fail=True, **kwargs) self.file = file # there is a small difference between the behaviour or JsonItemExporter.indent # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent # the addition of newlines everywhere - json_indent = self.indent if self.indent is not None and self.indent > 0 else None - self._kwargs.setdefault('indent', json_indent) - self._kwargs.setdefault('ensure_ascii', not self.encoding) + json_indent = ( + self.indent if self.indent is not None and self.indent > 0 else None + ) + self._kwargs.setdefault("indent", json_indent) + self._kwargs.setdefault("ensure_ascii", not self.encoding) self.encoder = ScrapyJSONEncoder(**self._kwargs) self.first_item = True def _beautify_newline(self): if self.indent is not None: - self.file.write(b'\n') + self.file.write(b"\n") def start_exporting(self): self.file.write(b"[") @@ -141,7 +146,7 @@ class JsonItemExporter(BaseItemExporter): if self.first_item: self.first_item = False else: - self.file.write(b',') + self.file.write(b",") self._beautify_newline() itemdict = dict(self._get_serialized_fields(item)) data = self.encoder.encode(itemdict) @@ -149,22 +154,21 @@ class JsonItemExporter(BaseItemExporter): class XmlItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): - self.item_element = kwargs.pop('item_element', 'item') - self.root_element = kwargs.pop('root_element', 'items') + self.item_element = kwargs.pop("item_element", "item") + self.root_element = kwargs.pop("root_element", "items") super().__init__(**kwargs) if not self.encoding: - self.encoding = 'utf-8' + self.encoding = "utf-8" self.xg = XMLGenerator(file, encoding=self.encoding) def _beautify_newline(self, new_item=False): if self.indent is not None and (self.indent > 0 or new_item): - self.xg.characters('\n') + self.xg.characters("\n") def _beautify_indent(self, depth=1): if self.indent: - self.xg.characters(' ' * self.indent * depth) + self.xg.characters(" " * self.indent * depth) def start_exporting(self): self.xg.startDocument() @@ -175,7 +179,7 @@ class XmlItemExporter(BaseItemExporter): self._beautify_indent(depth=1) self.xg.startElement(self.item_element, {}) self._beautify_newline() - for name, value in self._get_serialized_fields(item, default_value=''): + for name, value in self._get_serialized_fields(item, default_value=""): self._export_xml_field(name, value, depth=2) self._beautify_indent(depth=1) self.xg.endElement(self.item_element) @@ -188,7 +192,7 @@ class XmlItemExporter(BaseItemExporter): def _export_xml_field(self, name, serialized_value, depth): self._beautify_indent(depth=depth) self.xg.startElement(name, {}) - if hasattr(serialized_value, 'items'): + if hasattr(serialized_value, "items"): self._beautify_newline() for subname, value in serialized_value.items(): self._export_xml_field(subname, value, depth=depth + 1) @@ -196,7 +200,7 @@ class XmlItemExporter(BaseItemExporter): elif is_listlike(serialized_value): self._beautify_newline() for value in serialized_value: - self._export_xml_field('value', value, depth=depth + 1) + self._export_xml_field("value", value, depth=depth + 1) self._beautify_indent(depth=depth) elif isinstance(serialized_value, str): self.xg.characters(serialized_value) @@ -207,18 +211,24 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): - - def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs): + def __init__( + self, + file, + include_headers_line=True, + join_multivalued=",", + errors=None, + **kwargs, + ): super().__init__(dont_fail=True, **kwargs) if not self.encoding: - self.encoding = 'utf-8' + self.encoding = "utf-8" self.include_headers_line = include_headers_line self.stream = io.TextIOWrapper( file, line_buffering=False, write_through=True, encoding=self.encoding, - newline='', # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + newline="", # Windows needs this https://github.com/scrapy/scrapy/issues/3034 errors=errors, ) self.csv_writer = csv.writer(self.stream, **self._kwargs) @@ -226,7 +236,7 @@ class CsvItemExporter(BaseItemExporter): self._join_multivalued = join_multivalued def serialize_field(self, field, name, value): - serializer = field.get('serializer', self._join_if_needed) + serializer = field.get("serializer", self._join_if_needed) return serializer(value) def _join_if_needed(self, value): @@ -242,8 +252,7 @@ class CsvItemExporter(BaseItemExporter): self._headers_not_written = False self._write_headers_and_set_fields_to_export(item) - fields = self._get_serialized_fields(item, default_value='', - include_empty=True) + fields = self._get_serialized_fields(item, default_value="", include_empty=True) values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) @@ -268,7 +277,6 @@ class CsvItemExporter(BaseItemExporter): class PickleItemExporter(BaseItemExporter): - def __init__(self, file, protocol=4, **kwargs): super().__init__(**kwargs) self.file = file @@ -297,14 +305,13 @@ class MarshalItemExporter(BaseItemExporter): class PprintItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): super().__init__(**kwargs) self.file = file def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) - self.file.write(to_bytes(pprint.pformat(itemdict) + '\n')) + self.file.write(to_bytes(pprint.pformat(itemdict) + "\n")) class PythonItemExporter(BaseItemExporter): @@ -318,17 +325,18 @@ class PythonItemExporter(BaseItemExporter): """ def _configure(self, options, dont_fail=False): - self.binary = options.pop('binary', True) + self.binary = options.pop("binary", True) super()._configure(options, dont_fail) if self.binary: warnings.warn( "PythonItemExporter will drop support for binary export in the future", - ScrapyDeprecationWarning) + ScrapyDeprecationWarning, + ) if not self.encoding: - self.encoding = 'utf-8' + self.encoding = "utf-8" def serialize_field(self, field, name, value): - serializer = field.get('serializer', self._serialize_value) + serializer = field.get("serializer", self._serialize_value) return serializer(value) def _serialize_value(self, value): diff --git a/scrapy/extension.py b/scrapy/extension.py index 050b87e5f..89f6a694d 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -9,8 +9,8 @@ from scrapy.utils.conf import build_component_list class ExtensionManager(MiddlewareManager): - component_name = 'extension' + component_name = "extension" @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings.getwithbase('EXTENSIONS')) + return build_component_list(settings.getwithbase("EXTENSIONS")) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 812844c0a..bb6f832f2 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -11,15 +11,14 @@ from scrapy.exceptions import NotConfigured class CloseSpider: - def __init__(self, crawler): self.crawler = crawler self.close_on = { - 'timeout': crawler.settings.getfloat('CLOSESPIDER_TIMEOUT'), - 'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'), - 'pagecount': crawler.settings.getint('CLOSESPIDER_PAGECOUNT'), - 'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'), + "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"), + "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), + "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), + "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"), } if not any(self.close_on.values()): @@ -27,13 +26,13 @@ class CloseSpider: self.counter = defaultdict(int) - if self.close_on.get('errorcount'): + if self.close_on.get("errorcount"): crawler.signals.connect(self.error_count, signal=signals.spider_error) - if self.close_on.get('pagecount'): + if self.close_on.get("pagecount"): crawler.signals.connect(self.page_count, signal=signals.response_received) - if self.close_on.get('timeout'): + if self.close_on.get("timeout"): crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) - if self.close_on.get('itemcount'): + if self.close_on.get("itemcount"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @@ -42,27 +41,31 @@ class CloseSpider: return cls(crawler) def error_count(self, failure, response, spider): - self.counter['errorcount'] += 1 - if self.counter['errorcount'] == self.close_on['errorcount']: - self.crawler.engine.close_spider(spider, 'closespider_errorcount') + self.counter["errorcount"] += 1 + if self.counter["errorcount"] == self.close_on["errorcount"]: + self.crawler.engine.close_spider(spider, "closespider_errorcount") def page_count(self, response, request, spider): - self.counter['pagecount'] += 1 - if self.counter['pagecount'] == self.close_on['pagecount']: - self.crawler.engine.close_spider(spider, 'closespider_pagecount') + self.counter["pagecount"] += 1 + if self.counter["pagecount"] == self.close_on["pagecount"]: + self.crawler.engine.close_spider(spider, "closespider_pagecount") def spider_opened(self, spider): from twisted.internet import reactor - self.task = reactor.callLater(self.close_on['timeout'], - self.crawler.engine.close_spider, spider, - reason='closespider_timeout') + + self.task = reactor.callLater( + self.close_on["timeout"], + self.crawler.engine.close_spider, + spider, + reason="closespider_timeout", + ) def item_scraped(self, item, spider): - self.counter['itemcount'] += 1 - if self.counter['itemcount'] == self.close_on['itemcount']: - self.crawler.engine.close_spider(spider, 'closespider_itemcount') + self.counter["itemcount"] += 1 + if self.counter["itemcount"] == self.close_on["itemcount"]: + self.crawler.engine.close_spider(spider, "closespider_itemcount") def spider_closed(self, spider): - task = getattr(self, 'task', False) + task = getattr(self, "task", False) if task and task.active(): task.cancel() diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 675f8276f..30c987253 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -7,7 +7,6 @@ from scrapy import signals class CoreStats: - def __init__(self, stats): self.stats = stats self.start_time = None @@ -24,23 +23,25 @@ class CoreStats: def spider_opened(self, spider): self.start_time = datetime.utcnow() - self.stats.set_value('start_time', self.start_time, spider=spider) + self.stats.set_value("start_time", self.start_time, spider=spider) def spider_closed(self, spider, reason): finish_time = datetime.utcnow() elapsed_time = finish_time - self.start_time elapsed_time_seconds = elapsed_time.total_seconds() - self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider) - self.stats.set_value('finish_time', finish_time, spider=spider) - self.stats.set_value('finish_reason', reason, spider=spider) + self.stats.set_value( + "elapsed_time_seconds", elapsed_time_seconds, spider=spider + ) + self.stats.set_value("finish_time", finish_time, spider=spider) + self.stats.set_value("finish_reason", reason, spider=spider) def item_scraped(self, item, spider): - self.stats.inc_value('item_scraped_count', spider=spider) + self.stats.inc_value("item_scraped_count", spider=spider) def response_received(self, spider): - self.stats.inc_value('response_received_count', spider=spider) + self.stats.inc_value("response_received_count", spider=spider) def item_dropped(self, item, spider, exception): reason = exception.__class__.__name__ - self.stats.inc_value('item_dropped_count', spider=spider) - self.stats.inc_value(f'item_dropped_reasons_count/{reason}', spider=spider) + self.stats.inc_value("item_dropped_count", spider=spider) + self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider) diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index fd2a02d8d..8628b4a1e 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -18,7 +18,6 @@ logger = logging.getLogger(__name__) class StackTraceDump: - def __init__(self, crawler=None): self.crawler = crawler try: @@ -34,20 +33,23 @@ class StackTraceDump: def dump_stacktrace(self, signum, frame): log_args = { - 'stackdumps': self._thread_stacks(), - 'enginestatus': format_engine_status(self.crawler.engine), - 'liverefs': format_live_refs(), + "stackdumps": self._thread_stacks(), + "enginestatus": format_engine_status(self.crawler.engine), + "liverefs": format_live_refs(), } - logger.info("Dumping stack trace and engine status\n" - "%(enginestatus)s\n%(liverefs)s\n%(stackdumps)s", - log_args, extra={'crawler': self.crawler}) + logger.info( + "Dumping stack trace and engine status\n" + "%(enginestatus)s\n%(liverefs)s\n%(stackdumps)s", + log_args, + extra={"crawler": self.crawler}, + ) def _thread_stacks(self): id2name = dict((th.ident, th.name) for th in threading.enumerate()) - dumps = '' + dumps = "" for id_, frame in sys._current_frames().items(): - name = id2name.get(id_, '') - dump = ''.join(traceback.format_stack(frame)) + name = id2name.get(id_, "") + dump = "".join(traceback.format_stack(frame)) dumps += f"# Thread: {name}({id_})\n{dump}\n" return dumps diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0aa27e417..ca4d77a35 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -34,15 +34,15 @@ logger = logging.getLogger(__name__) def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): argument_names = get_func_args(builder) - if 'feed_options' in argument_names: - kwargs['feed_options'] = feed_options + if "feed_options" in argument_names: + kwargs["feed_options"] = feed_options else: warnings.warn( f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a " "'feed_options' parameter to its signature to remove this " "warning. This parameter will become mandatory in a future " "version of Scrapy.", - category=ScrapyDeprecationWarning + category=ScrapyDeprecationWarning, ) return builder(*preargs, uri, *args, **kwargs) @@ -55,6 +55,7 @@ class ItemFilter: :param feed_options: feed specific options passed from FeedExporter :type feed_options: dict """ + feed_options: Optional[dict] item_classes: Tuple @@ -62,7 +63,8 @@ class ItemFilter: self.feed_options = feed_options if feed_options is not None: self.item_classes = tuple( - load_object(item_class) for item_class in feed_options.get("item_classes") or () + load_object(item_class) + for item_class in feed_options.get("item_classes") or () ) else: self.item_classes = tuple() @@ -98,13 +100,12 @@ class IFeedStorage(Interface): @implementer(IFeedStorage) class BlockingFeedStorage: - def open(self, spider): - path = spider.crawler.settings['FEED_TEMPDIR'] + path = spider.crawler.settings["FEED_TEMPDIR"] if path and not Path(path).is_dir(): - raise OSError('Not a Directory: ' + str(path)) + raise OSError("Not a Directory: " + str(path)) - return NamedTemporaryFile(prefix='feed-', dir=path) + return NamedTemporaryFile(prefix="feed-", dir=path) def store(self, file): return threads.deferToThread(self._store_in_thread, file) @@ -115,16 +116,17 @@ class BlockingFeedStorage: @implementer(IFeedStorage) class StdoutFeedStorage: - def __init__(self, uri, _stdout=None, *, feed_options=None): if not _stdout: _stdout = sys.stdout.buffer self._stdout = _stdout - if feed_options and feed_options.get('overwrite', False) is True: - logger.warning('Standard output (stdout) storage does not support ' - 'overwriting. To suppress this warning, remove the ' - 'overwrite option from your FEEDS setting, or set ' - 'it to False.') + if feed_options and feed_options.get("overwrite", False) is True: + logger.warning( + "Standard output (stdout) storage does not support " + "overwriting. To suppress this warning, remove the " + "overwrite option from your FEEDS setting, or set " + "it to False." + ) def open(self, spider): return self._stdout @@ -135,11 +137,10 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri, *, feed_options=None): self.path = file_uri_to_path(uri) feed_options = feed_options or {} - self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab' + self.write_mode = "wb" if feed_options.get("overwrite", False) else "ab" def open(self, spider) -> IO[Any]: dirname = Path(self.path).parent @@ -152,11 +153,19 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): - - def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *, - feed_options=None, session_token=None): + def __init__( + self, + uri, + access_key=None, + secret_key=None, + acl=None, + endpoint_url=None, + *, + feed_options=None, + session_token=None, + ): if not is_botocore_available(): - raise NotConfigured('missing botocore library') + raise NotConfigured("missing botocore library") u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key @@ -166,41 +175,45 @@ class S3FeedStorage(BlockingFeedStorage): self.acl = acl self.endpoint_url = endpoint_url import botocore.session + session = botocore.session.get_session() self.s3_client = session.create_client( - 's3', aws_access_key_id=self.access_key, + "s3", + aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, aws_session_token=self.session_token, - endpoint_url=self.endpoint_url) - if feed_options and feed_options.get('overwrite', True) is False: - logger.warning('S3 does not support appending to files. To ' - 'suppress this warning, remove the overwrite ' - 'option from your FEEDS setting or set it to True.') + endpoint_url=self.endpoint_url, + ) + if feed_options and feed_options.get("overwrite", True) is False: + logger.warning( + "S3 does not support appending to files. To " + "suppress this warning, remove the overwrite " + "option from your FEEDS setting or set it to True." + ) @classmethod def from_crawler(cls, crawler, uri, *, feed_options=None): return build_storage( cls, uri, - access_key=crawler.settings['AWS_ACCESS_KEY_ID'], - secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], - session_token=crawler.settings['AWS_SESSION_TOKEN'], - acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, - endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None, + access_key=crawler.settings["AWS_ACCESS_KEY_ID"], + secret_key=crawler.settings["AWS_SECRET_ACCESS_KEY"], + session_token=crawler.settings["AWS_SESSION_TOKEN"], + acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None, + endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None, feed_options=feed_options, ) def _store_in_thread(self, file): file.seek(0) - kwargs = {'ACL': self.acl} if self.acl else {} + kwargs = {"ACL": self.acl} if self.acl else {} self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file, - **kwargs) + Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs + ) file.close() class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri, project_id, acl): self.project_id = project_id self.acl = acl @@ -212,13 +225,14 @@ class GCSFeedStorage(BlockingFeedStorage): def from_crawler(cls, crawler, uri): return cls( uri, - crawler.settings['GCS_PROJECT_ID'], - crawler.settings['FEED_STORAGE_GCS_ACL'] or None + crawler.settings["GCS_PROJECT_ID"], + crawler.settings["FEED_STORAGE_GCS_ACL"] or None, ) def _store_in_thread(self, file): file.seek(0) from google.cloud.storage import Client + client = Client(project=self.project_id) bucket = client.get_bucket(self.bucket_name) blob = bucket.blob(self.blob_name) @@ -226,37 +240,51 @@ class GCSFeedStorage(BlockingFeedStorage): class FTPFeedStorage(BlockingFeedStorage): - def __init__(self, uri, use_active_mode=False, *, feed_options=None): u = urlparse(uri) self.host = u.hostname - self.port = int(u.port or '21') + self.port = int(u.port or "21") self.username = u.username - self.password = unquote(u.password or '') + self.password = unquote(u.password or "") self.path = u.path self.use_active_mode = use_active_mode - self.overwrite = not feed_options or feed_options.get('overwrite', True) + self.overwrite = not feed_options or feed_options.get("overwrite", True) @classmethod def from_crawler(cls, crawler, uri, *, feed_options=None): return build_storage( cls, uri, - crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'), + crawler.settings.getbool("FEED_STORAGE_FTP_ACTIVE"), feed_options=feed_options, ) def _store_in_thread(self, file): ftp_store_file( - path=self.path, file=file, host=self.host, - port=self.port, username=self.username, - password=self.password, use_active_mode=self.use_active_mode, + path=self.path, + file=file, + host=self.host, + port=self.port, + username=self.username, + password=self.password, + use_active_mode=self.use_active_mode, overwrite=self.overwrite, ) class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template, filter): + def __init__( + self, + file, + exporter, + storage, + uri, + format, + store_empty, + batch_id, + uri_template, + filter, + ): self.file = file self.exporter = exporter self.storage = storage @@ -283,7 +311,6 @@ class _FeedSlot: class FeedExporter: - @classmethod def from_crawler(cls, crawler): exporter = cls(crawler) @@ -299,48 +326,55 @@ class FeedExporter: self.slots = [] self.filters = {} - if not self.settings['FEEDS'] and not self.settings['FEED_URI']: + if not self.settings["FEEDS"] and not self.settings["FEED_URI"]: raise NotConfigured # Begin: Backward compatibility for FEED_URI and FEED_FORMAT settings - if self.settings['FEED_URI']: + if self.settings["FEED_URI"]: warnings.warn( - 'The `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of ' - 'the `FEEDS` setting. Please see the `FEEDS` setting docs for more details', - category=ScrapyDeprecationWarning, stacklevel=2, + "The `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of " + "the `FEEDS` setting. Please see the `FEEDS` setting docs for more details", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + uri = str(self.settings["FEED_URI"]) # handle pathlib.Path objects + feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")} + self.feeds[uri] = feed_complete_default_values_from_settings( + feed_options, self.settings ) - uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects - feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} - self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) self.filters[uri] = self._load_filter(feed_options) # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' - for uri, feed_options in self.settings.getdict('FEEDS').items(): + for uri, feed_options in self.settings.getdict("FEEDS").items(): uri = str(uri) # handle pathlib.Path objects - self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) + self.feeds[uri] = feed_complete_default_values_from_settings( + feed_options, self.settings + ) self.filters[uri] = self._load_filter(feed_options) - self.storages = self._load_components('FEED_STORAGES') - self.exporters = self._load_components('FEED_EXPORTERS') + self.storages = self._load_components("FEED_STORAGES") + self.exporters = self._load_components("FEED_EXPORTERS") for uri, feed_options in self.feeds.items(): if not self._storage_supported(uri, feed_options): raise NotConfigured if not self._settings_are_valid(): raise NotConfigured - if not self._exporter_supported(feed_options['format']): + if not self._exporter_supported(feed_options["format"]): raise NotConfigured def open_spider(self, spider): for uri, feed_options in self.feeds.items(): - uri_params = self._get_uri_params(spider, feed_options['uri_params']) - self.slots.append(self._start_new_batch( - batch_id=1, - uri=uri % uri_params, - feed_options=feed_options, - spider=spider, - uri_template=uri, - )) + uri_params = self._get_uri_params(spider, feed_options["uri_params"]) + self.slots.append( + self._start_new_batch( + batch_id=1, + uri=uri % uri_params, + feed_options=feed_options, + spider=spider, + uri_template=uri, + ) + ) def close_spider(self, spider): deferred_list = [] @@ -368,16 +402,15 @@ class FeedExporter: def _handle_store_error(self, f, logmsg, spider, slot_type): logger.error( - "Error storing %s", logmsg, - exc_info=failure_to_exc_info(f), extra={'spider': spider} + "Error storing %s", + logmsg, + exc_info=failure_to_exc_info(f), + extra={"spider": spider}, ) self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") def _handle_store_success(self, f, logmsg, spider, slot_type): - logger.info( - "Stored %s", logmsg, - extra={'spider': spider} - ) + logger.info("Stored %s", logmsg, extra={"spider": spider}) self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): @@ -393,26 +426,28 @@ class FeedExporter: storage = self._get_storage(uri, feed_options) file = storage.open(spider) if "postprocessing" in feed_options: - file = PostProcessingManager(feed_options["postprocessing"], file, feed_options) + file = PostProcessingManager( + feed_options["postprocessing"], file, feed_options + ) exporter = self._get_exporter( file=file, - format=feed_options['format'], - fields_to_export=feed_options['fields'], - encoding=feed_options['encoding'], - indent=feed_options['indent'], - **feed_options['item_export_kwargs'], + format=feed_options["format"], + fields_to_export=feed_options["fields"], + encoding=feed_options["encoding"], + indent=feed_options["indent"], + **feed_options["item_export_kwargs"], ) slot = _FeedSlot( file=file, exporter=exporter, storage=storage, uri=uri, - format=feed_options['format'], - store_empty=feed_options['store_empty'], + format=feed_options["format"], + store_empty=feed_options["store_empty"], batch_id=batch_id, uri_template=uri_template, - filter=self.filters[uri_template] + filter=self.filters[uri_template], ) if slot.store_empty: slot.start_exporting() @@ -422,7 +457,9 @@ class FeedExporter: slots = [] for slot in self.slots: if not slot.filter.accepts(item): - slots.append(slot) # if slot doesn't accept item, continue with next slot + slots.append( + slot + ) # if slot doesn't accept item, continue with next slot continue slot.start_exporting() @@ -430,18 +467,22 @@ class FeedExporter: slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one if ( - self.feeds[slot.uri_template]['batch_item_count'] - and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] + self.feeds[slot.uri_template]["batch_item_count"] + and slot.itemcount >= self.feeds[slot.uri_template]["batch_item_count"] ): - uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) + uri_params = self._get_uri_params( + spider, self.feeds[slot.uri_template]["uri_params"], slot + ) self._close_slot(slot, spider) - slots.append(self._start_new_batch( - batch_id=slot.batch_id + 1, - uri=slot.uri_template % uri_params, - feed_options=self.feeds[slot.uri_template], - spider=spider, - uri_template=slot.uri_template, - )) + slots.append( + self._start_new_batch( + batch_id=slot.batch_id + 1, + uri=slot.uri_template % uri_params, + feed_options=self.feeds[slot.uri_template], + spider=spider, + uri_template=slot.uri_template, + ) + ) else: slots.append(slot) self.slots = slots @@ -459,7 +500,7 @@ class FeedExporter: def _exporter_supported(self, format): if format in self.exporters: return True - logger.error("Unknown feed format: %(format)s", {'format': format}) + logger.error("Unknown feed format: %(format)s", {"format": format}) def _settings_are_valid(self): """ @@ -467,12 +508,14 @@ class FeedExporter: %(batch_time)s or %(batch_id)d to distinguish different files of partial output """ for uri_template, values in self.feeds.items(): - if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): + if values["batch_item_count"] and not re.search( + r"%\(batch_time\)s|%\(batch_id\)", uri_template + ): logger.error( - '%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT ' - 'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: ' - 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count', - uri_template + "%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT " + "setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: " + "https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count", + uri_template, ) return False return True @@ -484,17 +527,17 @@ class FeedExporter: self._get_storage(uri, feed_options) return True except NotConfigured as e: - logger.error("Disabled feed storage scheme: %(scheme)s. " - "Reason: %(reason)s", - {'scheme': scheme, 'reason': str(e)}) + logger.error( + "Disabled feed storage scheme: %(scheme)s. " "Reason: %(reason)s", + {"scheme": scheme, "reason": str(e)}, + ) else: - logger.error("Unknown feed storage scheme: %(scheme)s", - {'scheme': scheme}) + logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme}) def _get_instance(self, objcls, *args, **kwargs): return create_instance( - objcls, self.settings, getattr(self, 'crawler', None), - *args, **kwargs) + objcls, self.settings, getattr(self, "crawler", None), *args, **kwargs + ) def _get_exporter(self, file, format, *args, **kwargs): return self._get_instance(self.exporters[format], file, *args, **kwargs) @@ -506,20 +549,22 @@ class FeedExporter: do not support it, and issuing a deprecation warning instead. """ feedcls = self.storages[urlparse(uri).scheme] - crawler = getattr(self, 'crawler', None) + crawler = getattr(self, "crawler", None) def build_instance(builder, *preargs): - return build_storage(builder, uri, feed_options=feed_options, preargs=preargs) + return build_storage( + builder, uri, feed_options=feed_options, preargs=preargs + ) - if crawler and hasattr(feedcls, 'from_crawler'): + if crawler and hasattr(feedcls, "from_crawler"): instance = build_instance(feedcls.from_crawler, crawler) - method_name = 'from_crawler' - elif hasattr(feedcls, 'from_settings'): + method_name = "from_crawler" + elif hasattr(feedcls, "from_settings"): instance = build_instance(feedcls.from_settings, self.settings) - method_name = 'from_settings' + method_name = "from_settings" else: instance = build_instance(feedcls) - method_name = '__new__' + method_name = "__new__" if instance is None: raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None") return instance @@ -534,19 +579,23 @@ class FeedExporter: for k in dir(spider): params[k] = getattr(spider, k) utc_now = datetime.utcnow() - params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-') - params['batch_time'] = utc_now.isoformat().replace(':', '-') - params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 + params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") + params["batch_time"] = utc_now.isoformat().replace(":", "-") + params["batch_id"] = slot.batch_id + 1 if slot is not None else 1 original_params = params.copy() - uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params + uripar_function = ( + load_object(uri_params_function) + if uri_params_function + else lambda params, _: params + ) new_params = uripar_function(params, spider) if new_params is None or original_params != params: warnings.warn( - 'Modifying the params dictionary in-place in the function defined in ' - 'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS ' - 'setting is deprecated. The function must return a new dictionary ' - 'instead.', - category=ScrapyDeprecationWarning + "Modifying the params dictionary in-place in the function defined in " + "the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS " + "setting is deprecated. The function must return a new dictionary " + "instead.", + category=ScrapyDeprecationWarning, ) return new_params if new_params is not None else params diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 4e76fe5e5..3bfbddc54 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -21,10 +21,11 @@ logger = logging.getLogger(__name__) class DummyPolicy: - def __init__(self, settings): - self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self.ignore_http_codes = [int(x) for x in settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')] + self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self.ignore_http_codes = [ + int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES") + ] def should_cache_request(self, request): return urlparse_cached(request).scheme not in self.ignore_schemes @@ -44,16 +45,17 @@ class RFC2616Policy: MAXAGE = 3600 * 24 * 365 # one year def __init__(self, settings): - self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE') - self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') + self.always_store = settings.getbool("HTTPCACHE_ALWAYS_STORE") + self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") self._cc_parsed = WeakKeyDictionary() self.ignore_response_cache_controls = [ - to_bytes(cc) for cc in settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS') + to_bytes(cc) + for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS") ] def _parse_cachecontrol(self, r): if r not in self._cc_parsed: - cch = r.headers.get(b'Cache-Control', b'') + cch = r.headers.get(b"Cache-Control", b"") parsed = parse_cachecontrol(cch) if isinstance(r, Response): for key in self.ignore_response_cache_controls: @@ -66,7 +68,7 @@ class RFC2616Policy: return False cc = self._parse_cachecontrol(request) # obey user-agent directive "Cache-Control: no-store" - if b'no-store' in cc: + if b"no-store" in cc: return False # Any other is eligible for caching return True @@ -77,7 +79,7 @@ class RFC2616Policy: # Status code 206 is not included because cache can not deal with partial contents cc = self._parse_cachecontrol(response) # obey directive "Cache-Control: no-store" - if b'no-store' in cc: + if b"no-store" in cc: return False # Never cache 304 (Not Modified) responses if response.status == 304: @@ -86,14 +88,14 @@ class RFC2616Policy: if self.always_store: return True # Any hint on response expiration is good - if b'max-age' in cc or b'Expires' in response.headers: + if b"max-age" in cc or b"Expires" in response.headers: return True # Firefox fallbacks this statuses to one year expiration if none is set if response.status in (300, 301, 308): return True # Other statuses without expiration requires at least one validator if response.status in (200, 203, 401): - return b'Last-Modified' in response.headers or b'ETag' in response.headers + return b"Last-Modified" in response.headers or b"ETag" in response.headers # Any other is probably not eligible for caching # Makes no sense to cache responses that does not contain expiration # info and can not be revalidated @@ -102,11 +104,13 @@ class RFC2616Policy: def is_cached_response_fresh(self, cachedresponse, request): cc = self._parse_cachecontrol(cachedresponse) ccreq = self._parse_cachecontrol(request) - if b'no-cache' in cc or b'no-cache' in ccreq: + if b"no-cache" in cc or b"no-cache" in ccreq: return False now = time() - freshnesslifetime = self._compute_freshness_lifetime(cachedresponse, request, now) + freshnesslifetime = self._compute_freshness_lifetime( + cachedresponse, request, now + ) currentage = self._compute_current_age(cachedresponse, request, now) reqmaxage = self._get_max_age(ccreq) @@ -116,7 +120,7 @@ class RFC2616Policy: if currentage < freshnesslifetime: return True - if b'max-stale' in ccreq and b'must-revalidate' not in cc: + if b"max-stale" in ccreq and b"must-revalidate" not in cc: # From RFC2616: "Indicates that the client is willing to # accept a response that has exceeded its expiration time. # If max-stale is assigned a value, then the client is @@ -124,7 +128,7 @@ class RFC2616Policy: # expiration time by no more than the specified number of # seconds. If no value is assigned to max-stale, then the # client is willing to accept a stale response of any age." - staleage = ccreq[b'max-stale'] + staleage = ccreq[b"max-stale"] if staleage is None: return True @@ -143,22 +147,24 @@ class RFC2616Policy: # as long as the old response didn't specify must-revalidate. if response.status >= 500: cc = self._parse_cachecontrol(cachedresponse) - if b'must-revalidate' not in cc: + if b"must-revalidate" not in cc: return True # Use the cached response if the server says it hasn't changed. return response.status == 304 def _set_conditional_validators(self, request, cachedresponse): - if b'Last-Modified' in cachedresponse.headers: - request.headers[b'If-Modified-Since'] = cachedresponse.headers[b'Last-Modified'] + if b"Last-Modified" in cachedresponse.headers: + request.headers[b"If-Modified-Since"] = cachedresponse.headers[ + b"Last-Modified" + ] - if b'ETag' in cachedresponse.headers: - request.headers[b'If-None-Match'] = cachedresponse.headers[b'ETag'] + if b"ETag" in cachedresponse.headers: + request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"] def _get_max_age(self, cc): try: - return max(0, int(cc[b'max-age'])) + return max(0, int(cc[b"max-age"])) except (KeyError, ValueError): return None @@ -171,18 +177,18 @@ class RFC2616Policy: return maxage # Parse date header or synthesize it if none exists - date = rfc1123_to_epoch(response.headers.get(b'Date')) or now + date = rfc1123_to_epoch(response.headers.get(b"Date")) or now # Try HTTP/1.0 Expires header - if b'Expires' in response.headers: - expires = rfc1123_to_epoch(response.headers[b'Expires']) + if b"Expires" in response.headers: + expires = rfc1123_to_epoch(response.headers[b"Expires"]) # When parsing Expires header fails RFC 2616 section 14.21 says we # should treat this as an expiration time in the past. return max(0, expires - date) if expires else 0 # Fallback to heuristic using last-modified header # This is not in RFC but on Firefox caching implementation - lastmodified = rfc1123_to_epoch(response.headers.get(b'Last-Modified')) + lastmodified = rfc1123_to_epoch(response.headers.get(b"Last-Modified")) if lastmodified and lastmodified <= date: return (date - lastmodified) / 10 @@ -199,13 +205,13 @@ class RFC2616Policy: currentage = 0 # If Date header is not set we assume it is a fast connection, and # clock is in sync with the server - date = rfc1123_to_epoch(response.headers.get(b'Date')) or now + date = rfc1123_to_epoch(response.headers.get(b"Date")) or now if now > date: currentage = now - date - if b'Age' in response.headers: + if b"Age" in response.headers: try: - age = int(response.headers[b'Age']) + age = int(response.headers[b"Age"]) currentage = max(currentage, age) except ValueError: pass @@ -214,18 +220,21 @@ class RFC2616Policy: class DbmCacheStorage: - def __init__(self, settings): - self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True) - self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') - self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) + self.cachedir = data_path(settings["HTTPCACHE_DIR"], createdir=True) + self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS") + self.dbmodule = import_module(settings["HTTPCACHE_DBM_MODULE"]) self.db = None def open_spider(self, spider: Spider): - dbpath = Path(self.cachedir, f'{spider.name}.db') - self.db = self.dbmodule.open(str(dbpath), 'c') + dbpath = Path(self.cachedir, f"{spider.name}.db") + self.db = self.dbmodule.open(str(dbpath), "c") - logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider}) + logger.debug( + "Using DBM cache storage in %(cachepath)s", + {"cachepath": dbpath}, + extra={"spider": spider}, + ) self._fingerprinter = spider.crawler.request_fingerprinter @@ -236,10 +245,10 @@ class DbmCacheStorage: data = self._read_data(spider, request) if data is None: return # not cached - url = data['url'] - status = data['status'] - headers = Headers(data['headers']) - body = data['body'] + url = data["url"] + status = data["status"] + headers = Headers(data["headers"]) + body = data["body"] respcls = responsetypes.from_args(headers=headers, url=url, body=body) response = respcls(url=url, headers=headers, status=status, body=body) return response @@ -247,18 +256,18 @@ class DbmCacheStorage: def store_response(self, spider, request, response): key = self._fingerprinter.fingerprint(request).hex() data = { - 'status': response.status, - 'url': response.url, - 'headers': dict(response.headers), - 'body': response.body, + "status": response.status, + "url": response.url, + "headers": dict(response.headers), + "body": response.body, } - self.db[f'{key}_data'] = pickle.dumps(data, protocol=4) - self.db[f'{key}_time'] = str(time()) + self.db[f"{key}_data"] = pickle.dumps(data, protocol=4) + self.db[f"{key}_time"] = str(time()) def _read_data(self, spider, request): key = self._fingerprinter.fingerprint(request).hex() db = self.db - tkey = f'{key}_time' + tkey = f"{key}_time" if tkey not in db: return # not found @@ -266,20 +275,22 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return # expired - return pickle.loads(db[f'{key}_data']) + return pickle.loads(db[f"{key}_data"]) class FilesystemCacheStorage: - def __init__(self, settings): - self.cachedir = data_path(settings['HTTPCACHE_DIR']) - self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') - self.use_gzip = settings.getbool('HTTPCACHE_GZIP') + self.cachedir = data_path(settings["HTTPCACHE_DIR"]) + self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS") + self.use_gzip = settings.getbool("HTTPCACHE_GZIP") self._open = gzip.open if self.use_gzip else open def open_spider(self, spider: Spider): - logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir}, - extra={'spider': spider}) + logger.debug( + "Using filesystem cache storage in %(cachedir)s", + {"cachedir": self.cachedir}, + extra={"spider": spider}, + ) self._fingerprinter = spider.crawler.request_fingerprinter @@ -292,12 +303,12 @@ class FilesystemCacheStorage: if metadata is None: return # not cached rpath = Path(self._get_request_path(spider, request)) - with self._open(rpath / 'response_body', 'rb') as f: + with self._open(rpath / "response_body", "rb") as f: body = f.read() - with self._open(rpath / 'response_headers', 'rb') as f: + with self._open(rpath / "response_headers", "rb") as f: rawheaders = f.read() - url = metadata.get('response_url') - status = metadata['status'] + url = metadata.get("response_url") + status = metadata["status"] headers = Headers(headers_raw_to_dict(rawheaders)) respcls = responsetypes.from_args(headers=headers, url=url, body=body) response = respcls(url=url, headers=headers, status=status, body=body) @@ -309,23 +320,23 @@ class FilesystemCacheStorage: if not rpath.exists(): rpath.mkdir(parents=True) metadata = { - 'url': request.url, - 'method': request.method, - 'status': response.status, - 'response_url': response.url, - 'timestamp': time(), + "url": request.url, + "method": request.method, + "status": response.status, + "response_url": response.url, + "timestamp": time(), } - with self._open(rpath / 'meta', 'wb') as f: + with self._open(rpath / "meta", "wb") as f: f.write(to_bytes(repr(metadata))) - with self._open(rpath / 'pickled_meta', 'wb') as f: + with self._open(rpath / "pickled_meta", "wb") as f: pickle.dump(metadata, f, protocol=4) - with self._open(rpath / 'response_headers', 'wb') as f: + with self._open(rpath / "response_headers", "wb") as f: f.write(headers_dict_to_raw(response.headers)) - with self._open(rpath / 'response_body', 'wb') as f: + with self._open(rpath / "response_body", "wb") as f: f.write(response.body) - with self._open(rpath / 'request_headers', 'wb') as f: + with self._open(rpath / "request_headers", "wb") as f: f.write(headers_dict_to_raw(request.headers)) - with self._open(rpath / 'request_body', 'wb') as f: + with self._open(rpath / "request_body", "wb") as f: f.write(request.body) def _get_request_path(self, spider: Spider, request: Request) -> str: @@ -334,13 +345,13 @@ class FilesystemCacheStorage: def _read_meta(self, spider: Spider, request: Request): rpath = Path(self._get_request_path(spider, request)) - metapath = rpath / 'pickled_meta' + metapath = rpath / "pickled_meta" if not metapath.exists(): return # not found mtime = metapath.stat().st_mtime if 0 < self.expiration_secs < time() - mtime: return # expired - with self._open(metapath, 'rb') as f: + with self._open(metapath, "rb") as f: return pickle.load(f) @@ -357,8 +368,8 @@ def parse_cachecontrol(header): """ directives = {} - for directive in header.split(b','): - key, sep, val = directive.strip().partition(b'=') + for directive in header.split(b","): + key, sep, val = directive.strip().partition(b"=") if key: directives[key.lower()] = val if sep else None return directives @@ -366,7 +377,7 @@ def parse_cachecontrol(header): def rfc1123_to_epoch(date_str): try: - date_str = to_unicode(date_str, encoding='ascii') + date_str = to_unicode(date_str, encoding="ascii") return mktime_tz(parsedate_tz(date_str)) except Exception: return None diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 0be2831a1..6295dcdb7 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -19,7 +19,7 @@ class LogStats: @classmethod def from_crawler(cls, crawler): - interval = crawler.settings.getfloat('LOGSTATS_INTERVAL') + interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured o = cls(crawler.stats, interval) @@ -35,17 +35,23 @@ class LogStats: self.task.start(self.interval) def log(self, spider): - items = self.stats.get_value('item_scraped_count', 0) - pages = self.stats.get_value('response_received_count', 0) + items = self.stats.get_value("item_scraped_count", 0) + pages = self.stats.get_value("response_received_count", 0) irate = (items - self.itemsprev) * self.multiplier prate = (pages - self.pagesprev) * self.multiplier self.pagesprev, self.itemsprev = pages, items - msg = ("Crawled %(pages)d pages (at %(pagerate)d pages/min), " - "scraped %(items)d items (at %(itemrate)d items/min)") - log_args = {'pages': pages, 'pagerate': prate, - 'items': items, 'itemrate': irate} - logger.info(msg, log_args, extra={'spider': spider}) + msg = ( + "Crawled %(pages)d pages (at %(pagerate)d pages/min), " + "scraped %(items)d items (at %(itemrate)d items/min)" + ) + log_args = { + "pages": pages, + "pagerate": prate, + "items": items, + "itemrate": irate, + } + logger.info(msg, log_args, extra={"spider": spider}) def spider_closed(self, spider, reason): if self.task and self.task.running: diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index cee44ea62..03ede0681 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -12,13 +12,12 @@ from scrapy.utils.trackref import live_refs class MemoryDebugger: - def __init__(self, stats): self.stats = stats @classmethod def from_crawler(cls, crawler): - if not crawler.settings.getbool('MEMDEBUG_ENABLED'): + if not crawler.settings.getbool("MEMDEBUG_ENABLED"): raise NotConfigured o = cls(crawler.stats) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) @@ -26,8 +25,12 @@ class MemoryDebugger: def spider_closed(self, spider, reason): gc.collect() - self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider) + self.stats.set_value( + "memdebug/gc_garbage_count", len(gc.garbage), spider=spider + ) for cls, wdict in live_refs.items(): if not wdict: continue - self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider) + self.stats.set_value( + f"memdebug/live_refs/{cls.__name__}", len(wdict), spider=spider + ) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 4fdf86479..2bba71972 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -20,22 +20,23 @@ logger = logging.getLogger(__name__) class MemoryUsage: - def __init__(self, crawler): - if not crawler.settings.getbool('MEMUSAGE_ENABLED'): + if not crawler.settings.getbool("MEMUSAGE_ENABLED"): raise NotConfigured try: # stdlib's resource module is only available on unix platforms. - self.resource = import_module('resource') + self.resource = import_module("resource") except ImportError: raise NotConfigured self.crawler = crawler self.warned = False - self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL') - self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024 - self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024 - self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS') + self.notify_mails = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") + self.limit = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 + self.warning = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 + self.check_interval = crawler.settings.getfloat( + "MEMUSAGE_CHECK_INTERVAL_SECONDS" + ) self.mail = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @@ -46,13 +47,13 @@ class MemoryUsage: def get_virtual_size(self): size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss - if sys.platform != 'darwin': + if sys.platform != "darwin": # on macOS ru_maxrss is in bytes, on Linux it is in KB size *= 1024 return size def engine_started(self): - self.crawler.stats.set_value('memusage/startup', self.get_virtual_size()) + self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) self.tasks = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) @@ -72,45 +73,56 @@ class MemoryUsage: tsk.stop() def update(self): - self.crawler.stats.max_value('memusage/max', self.get_virtual_size()) + self.crawler.stats.max_value("memusage/max", self.get_virtual_size()) def _check_limit(self): peak_mem_usage = self.get_virtual_size() if peak_mem_usage > self.limit: - self.crawler.stats.set_value('memusage/limit_reached', 1) + self.crawler.stats.set_value("memusage/limit_reached", 1) mem = self.limit / 1024 / 1024 - logger.error("Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...", - {'memusage': mem}, extra={'crawler': self.crawler}) + logger.error( + "Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...", + {"memusage": mem}, + extra={"crawler": self.crawler}, + ) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " f"memory usage exceeded {mem}MiB at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) - self.crawler.stats.set_value('memusage/limit_notified', 1) + self.crawler.stats.set_value("memusage/limit_notified", 1) if self.crawler.engine.spider is not None: - self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded') + self.crawler.engine.close_spider( + self.crawler.engine.spider, "memusage_exceeded" + ) else: self.crawler.stop() else: - logger.info("Peak memory usage is %(virtualsize)dMiB", {'virtualsize': peak_mem_usage / 1024 / 1024}) + logger.info( + "Peak memory usage is %(virtualsize)dMiB", + {"virtualsize": peak_mem_usage / 1024 / 1024}, + ) def _check_warning(self): if self.warned: # warn only once return if self.get_virtual_size() > self.warning: - self.crawler.stats.set_value('memusage/warning_reached', 1) + self.crawler.stats.set_value("memusage/warning_reached", 1) mem = self.warning / 1024 / 1024 - logger.warning("Memory usage reached %(memusage)dMiB", - {'memusage': mem}, extra={'crawler': self.crawler}) + logger.warning( + "Memory usage reached %(memusage)dMiB", + {"memusage": mem}, + extra={"crawler": self.crawler}, + ) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} warning: " f"memory usage reached {mem}MiB at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) - self.crawler.stats.set_value('memusage/warning_notified', 1) + self.crawler.stats.set_value("memusage/warning_notified", 1) self.warned = True def _send_report(self, rcpts, subject): @@ -120,7 +132,9 @@ class MemoryUsage: s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n" s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n" - s += "ENGINE STATUS ------------------------------------------------------- \r\n" + s += ( + "ENGINE STATUS ------------------------------------------------------- \r\n" + ) s += "\r\n" s += pformat(get_engine_status(self.crawler.engine)) s += "\r\n" diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 413c2e55e..79e3b1656 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -29,8 +29,13 @@ class GzipPlugin: compress_level = self.feed_options.get("gzip_compresslevel", 9) mtime = self.feed_options.get("gzip_mtime") filename = self.feed_options.get("gzip_filename") - self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level, - mtime=mtime, filename=filename) + self.gzipfile = GzipFile( + fileobj=self.file, + mode="wb", + compresslevel=compress_level, + mtime=mtime, + filename=filename, + ) def write(self, data: bytes) -> int: return self.gzipfile.write(data) @@ -55,7 +60,9 @@ class Bz2Plugin: self.file = file self.feed_options = feed_options compress_level = self.feed_options.get("bz2_compresslevel", 9) - self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level) + self.bz2file = BZ2File( + filename=self.file, mode="wb", compresslevel=compress_level + ) def write(self, data: bytes) -> int: return self.bz2file.write(data) @@ -90,8 +97,14 @@ class LZMAPlugin: check = self.feed_options.get("lzma_check", -1) preset = self.feed_options.get("lzma_preset") filters = self.feed_options.get("lzma_filters") - self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format, - check=check, preset=preset, filters=filters) + self.lzmafile = LZMAFile( + filename=self.file, + mode="wb", + format=format, + check=check, + preset=preset, + filters=filters, + ) def write(self, data: bytes) -> int: return self.lzmafile.write(data) @@ -114,7 +127,9 @@ class PostProcessingManager(IOBase): :type file: file like object """ - def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__( + self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any] + ) -> None: self.plugins = self._load_plugins(plugins) self.file = file self.feed_options = feed_options diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index e9c8b1d6a..929a3be70 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -25,16 +25,16 @@ class SpiderState: def spider_closed(self, spider): if self.jobdir: - with Path(self.statefn).open('wb') as f: + with Path(self.statefn).open("wb") as f: pickle.dump(spider.state, f, protocol=4) def spider_opened(self, spider): if self.jobdir and Path(self.statefn).exists(): - with Path(self.statefn).open('rb') as f: + with Path(self.statefn).open("rb") as f: spider.state = pickle.load(f) else: spider.state = {} @property def statefn(self) -> str: - return str(Path(self.jobdir, 'spider.state')) + return str(Path(self.jobdir, "spider.state")) diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 739e6b958..8733ad22b 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -10,7 +10,6 @@ from scrapy.exceptions import NotConfigured class StatsMailer: - def __init__(self, stats, recipients, mail): self.stats = stats self.recipients = recipients diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 054350e4a..271f22428 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -15,6 +15,7 @@ from twisted.internet import protocol try: from twisted.conch import manhole, telnet from twisted.conch.insults import insults + TWISTED_CONCH_AVAILABLE = True except (ImportError, SyntaxError): _TWISTED_CONCH_TRACEBACK = traceback.format_exc() @@ -35,24 +36,26 @@ update_telnet_vars = object() class TelnetConsole(protocol.ServerFactory): - def __init__(self, crawler): - if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'): + if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: raise NotConfigured( - 'TELNETCONSOLE_ENABLED setting is True but required twisted ' - 'modules failed to import:\n' + _TWISTED_CONCH_TRACEBACK) + "TELNETCONSOLE_ENABLED setting is True but required twisted " + "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK + ) self.crawler = crawler self.noisy = False - self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] - self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.username = crawler.settings['TELNETCONSOLE_USERNAME'] - self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] + self.portrange = [ + int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT") + ] + self.host = crawler.settings["TELNETCONSOLE_HOST"] + self.username = crawler.settings["TELNETCONSOLE_USERNAME"] + self.password = crawler.settings["TELNETCONSOLE_PASSWORD"] if not self.password: - self.password = binascii.hexlify(os.urandom(8)).decode('utf8') - logger.info('Telnet Password: %s', self.password) + self.password = binascii.hexlify(os.urandom(8)).decode("utf8") + logger.info("Telnet Password: %s", self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @@ -64,9 +67,11 @@ class TelnetConsole(protocol.ServerFactory): def start_listening(self): self.port = listen_tcp(self.portrange, self.host, self) h = self.port.getHost() - logger.info("Telnet console listening on %(host)s:%(port)d", - {'host': h.host, 'port': h.port}, - extra={'crawler': self.crawler}) + logger.info( + "Telnet console listening on %(host)s:%(port)d", + {"host": h.host, "port": h.port}, + extra={"crawler": self.crawler}, + ) def stop_listening(self): self.port.stopListening() @@ -74,41 +79,37 @@ class TelnetConsole(protocol.ServerFactory): def protocol(self): class Portal: """An implementation of IPortal""" + @defers def login(self_, credentials, mind, *interfaces): if not ( - credentials.username == self.username.encode('utf8') - and credentials.checkPassword(self.password.encode('utf8')) + credentials.username == self.username.encode("utf8") + and credentials.checkPassword(self.password.encode("utf8")) ): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( - insults.ServerProtocol, - manhole.Manhole, - self._get_telnet_vars() + insults.ServerProtocol, manhole.Manhole, self._get_telnet_vars() ) return (interfaces[0], protocol, lambda: None) - return telnet.TelnetTransport( - telnet.AuthenticatingTelnetProtocol, - Portal() - ) + return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal()) def _get_telnet_vars(self): # Note: if you add entries here also update topics/telnetconsole.rst telnet_vars = { - 'engine': self.crawler.engine, - 'spider': self.crawler.engine.spider, - 'slot': self.crawler.engine.slot, - 'crawler': self.crawler, - 'extensions': self.crawler.extensions, - 'stats': self.crawler.stats, - 'settings': self.crawler.settings, - 'est': lambda: print_engine_status(self.crawler.engine), - 'p': pprint.pprint, - 'prefs': print_live_refs, - 'help': "This is Scrapy telnet console. For more info see: " - "https://docs.scrapy.org/en/latest/topics/telnetconsole.html", + "engine": self.crawler.engine, + "spider": self.crawler.engine.spider, + "slot": self.crawler.engine.slot, + "crawler": self.crawler, + "extensions": self.crawler.extensions, + "stats": self.crawler.stats, + "settings": self.crawler.settings, + "est": lambda: print_engine_status(self.crawler.engine), + "p": pprint.pprint, + "prefs": print_live_refs, + "help": "This is Scrapy telnet console. For more info see: " + "https://docs.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index eb21e426e..79e20de2a 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -7,16 +7,19 @@ logger = logging.getLogger(__name__) class AutoThrottle: - def __init__(self, crawler): self.crawler = crawler - if not crawler.settings.getbool('AUTOTHROTTLE_ENABLED'): + if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"): raise NotConfigured self.debug = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") - self.target_concurrency = crawler.settings.getfloat("AUTOTHROTTLE_TARGET_CONCURRENCY") + self.target_concurrency = crawler.settings.getfloat( + "AUTOTHROTTLE_TARGET_CONCURRENCY" + ) crawler.signals.connect(self._spider_opened, signal=signals.spider_opened) - crawler.signals.connect(self._response_downloaded, signal=signals.response_downloaded) + crawler.signals.connect( + self._response_downloaded, signal=signals.response_downloaded + ) @classmethod def from_crawler(cls, crawler): @@ -29,17 +32,19 @@ class AutoThrottle: def _min_delay(self, spider): s = self.crawler.settings - return getattr(spider, 'download_delay', s.getfloat('DOWNLOAD_DELAY')) + return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY")) def _max_delay(self, spider): - return self.crawler.settings.getfloat('AUTOTHROTTLE_MAX_DELAY') + return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") def _start_delay(self, spider): - return max(self.mindelay, self.crawler.settings.getfloat('AUTOTHROTTLE_START_DELAY')) + return max( + self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") + ) def _response_downloaded(self, response, request, spider): key, slot = self._get_slot(request, spider) - latency = request.meta.get('download_latency') + latency = request.meta.get("download_latency") if latency is None or slot is None: return @@ -54,15 +59,18 @@ class AutoThrottle: "delay:%(delay)5d ms (%(delaydiff)+d) | " "latency:%(latency)5d ms | size:%(size)6d bytes", { - 'slot': key, 'concurrency': conc, - 'delay': slot.delay * 1000, 'delaydiff': diff * 1000, - 'latency': latency * 1000, 'size': size, + "slot": key, + "concurrency": conc, + "delay": slot.delay * 1000, + "delaydiff": diff * 1000, + "latency": latency * 1000, + "size": size, }, - extra={'spider': spider} + extra={"spider": spider}, ) def _get_slot(self, request, spider): - key = request.meta.get('download_slot') + key = request.meta.get("download_slot") return key, self.crawler.engine.downloader.slots.get(key) def _adjust_delay(self, slot, latency, response): diff --git a/scrapy/http/common.py b/scrapy/http/common.py index 98699d7fd..bc8861574 100644 --- a/scrapy/http/common.py +++ b/scrapy/http/common.py @@ -3,4 +3,5 @@ def obsolete_setter(setter, attrname): c = self.__class__.__name__ msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead" raise AttributeError(msg) + return newsetter diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index b43c383fe..94afedb08 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -36,7 +36,7 @@ class CookieJar: if not IPV4_RE.search(req_host): hosts = potential_domain_matches(req_host) - if '.' not in req_host: + if "." not in req_host: hosts += [req_host + ".local"] else: hosts = [req_host] @@ -96,14 +96,14 @@ def potential_domain_matches(domain): """ matches = [domain] try: - start = domain.index('.') + 1 - end = domain.rindex('.') + start = domain.index(".") + 1 + end = domain.rindex(".") while start < end: matches.append(domain[start:]) - start = domain.index('.', start) + 1 + start = domain.index(".", start) + 1 except ValueError: pass - return matches + ['.' + d for d in matches] + return matches + ["." + d for d in matches] class _DummyLock: @@ -140,7 +140,7 @@ class WrappedRequest: HTML document, and the user had no option to approve the automatic fetching of the image, this should be true. """ - return self.request.meta.get('is_unverifiable', False) + return self.request.meta.get("is_unverifiable", False) @property def full_url(self): @@ -166,13 +166,14 @@ class WrappedRequest: return name in self.request.headers def get_header(self, name, default=None): - return to_unicode(self.request.headers.get(name, default), - errors='replace') + return to_unicode(self.request.headers.get(name, default), errors="replace") def header_items(self): return [ - (to_unicode(k, errors='replace'), - [to_unicode(x, errors='replace') for x in v]) + ( + to_unicode(k, errors="replace"), + [to_unicode(x, errors="replace") for x in v], + ) for k, v in self.request.headers.items() ] @@ -181,7 +182,6 @@ class WrappedRequest: class WrappedResponse: - def __init__(self, response): self.response = response @@ -189,5 +189,6 @@ class WrappedResponse: return self def get_all(self, name, default=None): - return [to_unicode(v, errors='replace') - for v in self.response.headers.getlist(name)] + return [ + to_unicode(v, errors="replace") for v in self.response.headers.getlist(name) + ] diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 2e0020890..a5db30d6f 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -8,7 +8,7 @@ from scrapy.utils.python import to_unicode class Headers(CaselessDict): """Case insensitive http headers dictionary""" - def __init__(self, seq=None, encoding='utf-8'): + def __init__(self, seq=None, encoding="utf-8"): self.encoding = encoding super().__init__(seq) @@ -29,7 +29,7 @@ class Headers(CaselessDict): value = [] elif isinstance(value, (str, bytes)): value = [value] - elif not hasattr(value, '__iter__'): + elif not hasattr(value, "__iter__"): value = [value] return [self._tobytes(x) for x in value] @@ -41,7 +41,7 @@ class Headers(CaselessDict): return x.encode(self.encoding) if isinstance(x, int): return str(x).encode(self.encoding) - raise TypeError(f'Unsupported value type: {type(x)}') + raise TypeError(f"Unsupported value type: {type(x)}") def __getitem__(self, key): try: @@ -84,13 +84,16 @@ class Headers(CaselessDict): return headers_dict_to_raw(self) def to_unicode_dict(self): - """ Return headers as a CaselessDict with unicode keys + """Return headers as a CaselessDict with unicode keys and unicode values. Multiple values are joined with ','. """ return CaselessDict( - (to_unicode(key, encoding=self.encoding), - to_unicode(b','.join(value), encoding=self.encoding)) - for key, value in self.items()) + ( + to_unicode(key, encoding=self.encoding), + to_unicode(b",".join(value), encoding=self.encoding), + ) + for key, value in self.items() + ) def __copy__(self): return self.__class__(self) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a1001fc4a..e290f2143 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -27,9 +27,19 @@ class Request(object_ref): """ attributes: Tuple[str, ...] = ( - "url", "callback", "method", "headers", "body", - "cookies", "meta", "encoding", "priority", - "dont_filter", "errback", "flags", "cb_kwargs", + "url", + "callback", + "method", + "headers", + "body", + "cookies", + "meta", + "encoding", + "priority", + "dont_filter", + "errback", + "flags", + "cb_kwargs", ) """A tuple of :class:`str` objects containing the name of all public attributes of the class that are also keyword parameters of the @@ -64,9 +74,11 @@ class Request(object_ref): self.priority = priority if callback is not None and not callable(callback): - raise TypeError(f'callback must be a callable, got {type(callback).__name__}') + raise TypeError( + f"callback must be a callable, got {type(callback).__name__}" + ) if errback is not None and not callable(errback): - raise TypeError(f'errback must be a callable, got {type(errback).__name__}') + raise TypeError(f"errback must be a callable, got {type(errback).__name__}") self.callback = callback self.errback = errback @@ -101,13 +113,13 @@ class Request(object_ref): self._url = escape_ajax(s) if ( - '://' not in self._url - and not self._url.startswith('about:') - and not self._url.startswith('data:') + "://" not in self._url + and not self._url.startswith("about:") + and not self._url.startswith("data:") ): - raise ValueError(f'Missing scheme in request url: {self._url}') + raise ValueError(f"Missing scheme in request url: {self._url}") - url = property(_get_url, obsolete_setter(_set_url, 'url')) + url = property(_get_url, obsolete_setter(_set_url, "url")) def _get_body(self) -> bytes: return self._body @@ -115,7 +127,7 @@ class Request(object_ref): def _set_body(self, body: Optional[Union[str, bytes]]) -> None: self._body = b"" if body is None else to_bytes(body, self.encoding) - body = property(_get_body, obsolete_setter(_set_body, 'body')) + body = property(_get_body, obsolete_setter(_set_body, "body")) @property def encoding(self) -> str: @@ -131,12 +143,15 @@ class Request(object_ref): """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) - cls = kwargs.pop('cls', self.__class__) + cls = kwargs.pop("cls", self.__class__) return cls(*args, **kwargs) @classmethod def from_curl( - cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, **kwargs + cls: Type[RequestTypeVar], + curl_command: str, + ignore_unknown_options: bool = True, + **kwargs, ) -> RequestTypeVar: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the @@ -179,21 +194,25 @@ class Request(object_ref): """ d = { "url": self.url, # urls are safe (safe_string_url) - "callback": _find_method(spider, self.callback) if callable(self.callback) else self.callback, - "errback": _find_method(spider, self.errback) if callable(self.errback) else self.errback, + "callback": _find_method(spider, self.callback) + if callable(self.callback) + else self.callback, + "errback": _find_method(spider, self.errback) + if callable(self.errback) + else self.errback, "headers": dict(self.headers), } for attr in self.attributes: d.setdefault(attr, getattr(self, attr)) if type(self) is not Request: # pylint: disable=unidiomatic-typecheck - d["_class"] = self.__module__ + '.' + self.__class__.__name__ + d["_class"] = self.__module__ + "." + self.__class__.__name__ return d def _find_method(obj, func): """Helper function for Request.to_dict""" # Only instance methods contain ``__func__`` - if obj and hasattr(func, '__func__'): + if obj and hasattr(func, "__func__"): members = inspect.getmembers(obj, predicate=inspect.ismethod) for name, obj_func in members: # We need to use __func__ to access the original function object because instance diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 8e0a7fae2..ea519fb19 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -24,22 +24,26 @@ FormdataType = Optional[Union[dict, List[Tuple[str, str]]]] class FormRequest(Request): - valid_form_methods = ['GET', 'POST'] + valid_form_methods = ["GET", "POST"] def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None: - if formdata and kwargs.get('method') is None: - kwargs['method'] = 'POST' + if formdata and kwargs.get("method") is None: + kwargs["method"] = "POST" super().__init__(*args, **kwargs) if formdata: items = formdata.items() if isinstance(formdata, dict) else formdata form_query_str = _urlencode(items, self.encoding) - if self.method == 'POST': - self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') + if self.method == "POST": + self.headers.setdefault( + b"Content-Type", b"application/x-www-form-urlencoded" + ) self._set_body(form_query_str) else: - self._set_url(urlunsplit(urlsplit(self.url)._replace(query=form_query_str))) + self._set_url( + urlunsplit(urlsplit(self.url)._replace(query=form_query_str)) + ) @classmethod def from_response( @@ -55,28 +59,29 @@ class FormRequest(Request): formcss: Optional[str] = None, **kwargs, ) -> FormRequestTypeVar: - kwargs.setdefault('encoding', response.encoding) + kwargs.setdefault("encoding", response.encoding) if formcss is not None: from parsel.csstranslator import HTMLTranslator + formxpath = HTMLTranslator().css_to_xpath(formcss) form = _get_form(response, formname, formid, formnumber, formxpath) formdata = _get_inputs(form, formdata, dont_click, clickdata) - url = _get_form_url(form, kwargs.pop('url', None)) + url = _get_form_url(form, kwargs.pop("url", None)) - method = kwargs.pop('method', form.method) + method = kwargs.pop("method", form.method) if method is not None: method = method.upper() if method not in cls.valid_form_methods: - method = 'GET' + method = "GET" return cls(url=url, method=method, formdata=formdata, **kwargs) def _get_form_url(form: FormElement, url: Optional[str]) -> str: if url is None: - action = form.get('action') + action = form.get("action") if action is None: return form.base_url return urljoin(form.base_url, strip_html5_whitespace(action)) @@ -84,9 +89,11 @@ def _get_form_url(form: FormElement, url: Optional[str]) -> str: def _urlencode(seq: Iterable, enc: str) -> str: - values = [(to_bytes(k, enc), to_bytes(v, enc)) - for k, vs in seq - for v in (vs if is_listlike(vs) else [vs])] + values = [ + (to_bytes(k, enc), to_bytes(v, enc)) + for k, vs in seq + for v in (vs if is_listlike(vs) else [vs]) + ] return urlencode(values, doseq=True) @@ -99,7 +106,7 @@ def _get_form( ) -> FormElement: """Find the wanted form element within the given response.""" root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response)) - forms = root.xpath('//form') + forms = root.xpath("//form") if not forms: raise ValueError(f"No
element found in {response}") @@ -119,12 +126,12 @@ def _get_form( if nodes: el = nodes[0] while True: - if el.tag == 'form': + if el.tag == "form": return el el = el.getparent() if el is None: break - raise ValueError(f'No element found with {formxpath}') + raise ValueError(f"No element found with {formxpath}") # If we get here, it means that either formname was None or invalid if formnumber is not None: @@ -146,19 +153,21 @@ def _get_inputs( try: formdata_keys = dict(formdata or ()).keys() except (ValueError, TypeError): - raise ValueError('formdata should be a dict or iterable of tuples') + raise ValueError("formdata should be a dict or iterable of tuples") if not formdata: formdata = [] - inputs = form.xpath('descendant::textarea' - '|descendant::select' - '|descendant::input[not(@type) or @type[' - ' not(re:test(., "^(?:submit|image|reset)$", "i"))' - ' and (../@checked or' - ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', - namespaces={"re": "http://exslt.org/regular-expressions"}) + inputs = form.xpath( + "descendant::textarea" + "|descendant::select" + "|descendant::input[not(@type) or @type[" + ' not(re:test(., "^(?:submit|image|reset)$", "i"))' + " and (../@checked or" + ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', + namespaces={"re": "http://exslt.org/regular-expressions"}, + ) values = [ - (k, '' if v is None else v) + (k, "" if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys ] @@ -178,7 +187,7 @@ def _get_inputs( def _value(ele: HtmlElement): n = ele.name v = ele.value - if ele.tag == 'select': + if ele.tag == "select": return _select_value(ele, n, v) return n, v @@ -193,51 +202,57 @@ def _select_value(ele: SelectElement, n: str, v: str): if v is not None and multiple: # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 - selected_options = ele.xpath('.//option[@selected]') - values = [(o.get('value') or o.text or '').strip() for o in selected_options] + selected_options = ele.xpath(".//option[@selected]") + values = [(o.get("value") or o.text or "").strip() for o in selected_options] return n, values return n, v -def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tuple[str, str]]: +def _get_clickable( + clickdata: Optional[dict], form: FormElement +) -> Optional[Tuple[str, str]]: """ Returns the clickable element specified in clickdata, if the latter is given. If not, it returns the first clickable element found """ - clickables = list(form.xpath( - 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' - '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', - namespaces={"re": "http://exslt.org/regular-expressions"} - )) + clickables = list( + form.xpath( + 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', + namespaces={"re": "http://exslt.org/regular-expressions"}, + ) + ) if not clickables: return None # If we don't have clickdata, we just use the first clickable element if clickdata is None: el = clickables[0] - return (el.get('name'), el.get('value') or '') + return (el.get("name"), el.get("value") or "") # If clickdata is given, we compare it to the clickable elements to find a # match. We first look to see if the number is specified in clickdata, # because that uniquely identifies the element - nr = clickdata.get('nr', None) + nr = clickdata.get("nr", None) if nr is not None: try: el = list(form.inputs)[nr] except IndexError: pass else: - return (el.get('name'), el.get('value') or '') + return (el.get("name"), el.get("value") or "") # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = './/*' + ''.join(f'[@{k}="{v}"]' for k, v in clickdata.items()) + xpath = ".//*" + "".join(f'[@{k}="{v}"]' for k, v in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: - return (el[0].get('name'), el[0].get('value') or '') + return (el[0].get("name"), el[0].get("value") or "") if len(el) > 1: - raise ValueError(f"Multiple elements found ({el!r}) matching the " - f"criteria in clickdata: {clickdata!r}") + raise ValueError( + f"Multiple elements found ({el!r}) matching the " + f"criteria in clickdata: {clickdata!r}" + ) else: - raise ValueError(f'No clickable element matching clickdata: {clickdata!r}') + raise ValueError(f"No clickable element matching clickdata: {clickdata!r}") diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 728a2a104..35e2808c2 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -20,42 +20,44 @@ class JsonRequest(Request): def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} - dumps_kwargs.setdefault('sort_keys', True) + dumps_kwargs.setdefault("sort_keys", True) self._dumps_kwargs = dumps_kwargs - body_passed = kwargs.get('body', None) is not None - data = kwargs.pop('data', None) + body_passed = kwargs.get("body", None) is not None + data = kwargs.pop("data", None) data_passed = data is not None if body_passed and data_passed: - warnings.warn('Both body and data passed. data will be ignored') + warnings.warn("Both body and data passed. data will be ignored") elif not body_passed and data_passed: - kwargs['body'] = self._dumps(data) - if 'method' not in kwargs: - kwargs['method'] = 'POST' + kwargs["body"] = self._dumps(data) + if "method" not in kwargs: + kwargs["method"] = "POST" super().__init__(*args, **kwargs) - self.headers.setdefault('Content-Type', 'application/json') - self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') + self.headers.setdefault("Content-Type", "application/json") + self.headers.setdefault( + "Accept", "application/json, text/javascript, */*; q=0.01" + ) @property def dumps_kwargs(self) -> dict: return self._dumps_kwargs def replace(self, *args, **kwargs) -> Request: - body_passed = kwargs.get('body', None) is not None - data = kwargs.pop('data', None) + body_passed = kwargs.get("body", None) is not None + data = kwargs.pop("data", None) data_passed = data is not None if body_passed and data_passed: - warnings.warn('Both body and data passed. data will be ignored') + warnings.warn("Both body and data passed. data will be ignored") elif not body_passed and data_passed: - kwargs['body'] = self._dumps(data) + kwargs["body"] = self._dumps(data) return super().replace(*args, **kwargs) def _dumps(self, data: dict) -> str: - """Convert to JSON """ + """Convert to JSON""" return json.dumps(data, **self._dumps_kwargs) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 06d98cea5..c0a6e86c1 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -15,21 +15,20 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - def __init__(self, *args, encoding: Optional[str] = None, **kwargs): - if 'body' not in kwargs and 'params' in kwargs: + if "body" not in kwargs and "params" in kwargs: kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) - kwargs['body'] = xmlrpclib.dumps(**kw) + kwargs["body"] = xmlrpclib.dumps(**kw) # spec defines that requests must use POST method - kwargs.setdefault('method', 'POST') + kwargs.setdefault("method", "POST") # xmlrpc query multiples times over the same url - kwargs.setdefault('dont_filter', True) + kwargs.setdefault("dont_filter", True) # restore encoding if encoding is not None: - kwargs['encoding'] = encoding + kwargs["encoding"] = encoding super().__init__(*args, **kwargs) - self.headers.setdefault('Content-Type', 'text/xml') + self.headers.setdefault("Content-Type", "text/xml") diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 7626946ec..4213d491d 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -21,7 +21,15 @@ class Response(object_ref): """ attributes: Tuple[str, ...] = ( - "url", "status", "headers", "body", "flags", "request", "certificate", "ip_address", "protocol", + "url", + "status", + "headers", + "body", + "flags", + "request", + "certificate", + "ip_address", + "protocol", ) """A tuple of :class:`str` objects containing the name of all public attributes of the class that are also keyword parameters of the @@ -79,26 +87,28 @@ class Response(object_ref): if isinstance(url, str): self._url = url else: - raise TypeError(f'{type(self).__name__} url must be str, ' - f'got {type(url).__name__}') + raise TypeError( + f"{type(self).__name__} url must be str, " f"got {type(url).__name__}" + ) - url = property(_get_url, obsolete_setter(_set_url, 'url')) + url = property(_get_url, obsolete_setter(_set_url, "url")) def _get_body(self): return self._body def _set_body(self, body): if body is None: - self._body = b'' + self._body = b"" elif not isinstance(body, bytes): raise TypeError( "Response body must be bytes. " "If you want to pass unicode body use TextResponse " - "or HtmlResponse.") + "or HtmlResponse." + ) else: self._body = body - body = property(_get_body, obsolete_setter(_set_body, 'body')) + body = property(_get_body, obsolete_setter(_set_body, "body")) def __repr__(self): return f"<{self.status} {self.url}>" @@ -111,7 +121,7 @@ class Response(object_ref): """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) - cls = kwargs.pop('cls', self.__class__) + cls = kwargs.pop("cls", self.__class__) return cls(*args, **kwargs) def urljoin(self, url): @@ -138,9 +148,22 @@ class Response(object_ref): """ raise NotSupported("Response content isn't text") - def follow(self, url, callback=None, method='GET', headers=None, body=None, - cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request: + def follow( + self, + url, + callback=None, + method="GET", + headers=None, + body=None, + cookies=None, + meta=None, + encoding="utf-8", + priority=0, + dont_filter=False, + errback=None, + cb_kwargs=None, + flags=None, + ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. It accepts the same arguments as ``Request.__init__`` method, @@ -176,10 +199,22 @@ class Response(object_ref): flags=flags, ) - def follow_all(self, urls, callback=None, method='GET', headers=None, body=None, - cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, cb_kwargs=None, - flags=None) -> Generator[Request, None, None]: + def follow_all( + self, + urls, + callback=None, + method="GET", + headers=None, + body=None, + cookies=None, + meta=None, + encoding="utf-8", + priority=0, + dont_filter=False, + errback=None, + cb_kwargs=None, + flags=None, + ) -> Generator[Request, None, None]: """ .. versionadded:: 2.0 @@ -192,7 +227,7 @@ class Response(object_ref): method which supports selectors in addition to absolute/relative URLs and Link objects. """ - if not hasattr(urls, '__iter__'): + if not hasattr(urls, "__iter__"): raise TypeError("'urls' argument must be an iterable") return ( self.follow( diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index da81d0a4a..f9df4e1b0 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -30,13 +30,13 @@ _NONE = object() class TextResponse(Response): - _DEFAULT_ENCODING = 'ascii' + _DEFAULT_ENCODING = "ascii" _cached_decoded_json = _NONE attributes: Tuple[str, ...] = Response.attributes + ("encoding",) def __init__(self, *args, **kwargs): - self._encoding = kwargs.pop('encoding', None) + self._encoding = kwargs.pop("encoding", None) self._cached_benc = None self._cached_ubody = None self._cached_selector = None @@ -49,11 +49,13 @@ class TextResponse(Response): super()._set_url(url) def _set_body(self, body): - self._body = b'' # used by encoding detection + self._body = b"" # used by encoding detection if isinstance(body, str): if self._encoding is None: - raise TypeError('Cannot convert unicode body - ' - f'{type(self).__name__} has no encoding') + raise TypeError( + "Cannot convert unicode body - " + f"{type(self).__name__} has no encoding" + ) self._body = body.encode(self._encoding) else: super()._set_body(body) @@ -82,12 +84,12 @@ class TextResponse(Response): @property def text(self): - """ Body as unicode """ + """Body as unicode""" # access self.encoding before _cached_ubody to make sure # _body_inferred_encoding is called benc = self.encoding if self._cached_ubody is None: - charset = f'charset={benc}' + charset = f"charset={benc}" self._cached_ubody = html_to_unicode(charset, self.body)[1] return self._cached_ubody @@ -98,21 +100,24 @@ class TextResponse(Response): @memoizemethod_noargs def _headers_encoding(self): - content_type = self.headers.get(b'Content-Type', b'') + content_type = self.headers.get(b"Content-Type", b"") return http_content_type_encoding(to_unicode(content_type)) def _body_inferred_encoding(self): if self._cached_benc is None: - content_type = to_unicode(self.headers.get(b'Content-Type', b'')) - benc, ubody = html_to_unicode(content_type, self.body, - auto_detect_fun=self._auto_detect_fun, - default_encoding=self._DEFAULT_ENCODING) + content_type = to_unicode(self.headers.get(b"Content-Type", b"")) + benc, ubody = html_to_unicode( + content_type, + self.body, + auto_detect_fun=self._auto_detect_fun, + default_encoding=self._DEFAULT_ENCODING, + ) self._cached_benc = benc self._cached_ubody = ubody return self._cached_benc def _auto_detect_fun(self, text): - for enc in (self._DEFAULT_ENCODING, 'utf-8', 'cp1252'): + for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"): try: text.decode(enc) except UnicodeError: @@ -130,6 +135,7 @@ class TextResponse(Response): @property def selector(self): from scrapy.selector import Selector + if self._cached_selector is None: self._cached_selector = Selector(self) return self._cached_selector @@ -140,9 +146,22 @@ class TextResponse(Response): def css(self, query): return self.selector.css(query) - def follow(self, url, callback=None, method='GET', headers=None, body=None, - cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request: + def follow( + self, + url, + callback=None, + method="GET", + headers=None, + body=None, + cookies=None, + meta=None, + encoding=None, + priority=0, + dont_filter=False, + errback=None, + cb_kwargs=None, + flags=None, + ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. It accepts the same arguments as ``Request.__init__`` method, @@ -180,10 +199,24 @@ class TextResponse(Response): flags=flags, ) - def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None, - cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None, cb_kwargs=None, flags=None, - css=None, xpath=None) -> Generator[Request, None, None]: + def follow_all( + self, + urls=None, + callback=None, + method="GET", + headers=None, + body=None, + cookies=None, + meta=None, + encoding=None, + priority=0, + dont_filter=False, + errback=None, + cb_kwargs=None, + flags=None, + css=None, + xpath=None, + ) -> Generator[Request, None, None]: """ A generator that produces :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s @@ -251,12 +284,13 @@ def _url_from_selector(sel): if isinstance(sel.root, str): # e.g. ::attr(href) result return strip_html5_whitespace(sel.root) - if not hasattr(sel.root, 'tag'): + if not hasattr(sel.root, "tag"): raise _InvalidSelector(f"Unsupported selector: {sel}") - if sel.root.tag not in ('a', 'link'): - raise _InvalidSelector("Only and elements are supported; " - f"got <{sel.root.tag}>") - href = sel.root.get('href') + if sel.root.tag not in ("a", "link"): + raise _InvalidSelector( + "Only and elements are supported; " f"got <{sel.root.tag}>" + ) + href = sel.root.get("href") if href is None: raise _InvalidSelector(f"<{sel.root.tag}> element has no href attribute: {sel}") return strip_html5_whitespace(href) diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index 1896ec31e..9a2c5f170 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -2,7 +2,6 @@ from zope.interface import Interface class ISpiderLoader(Interface): - def from_settings(settings): """Return an instance of the class for the given settings""" diff --git a/scrapy/item.py b/scrapy/item.py index 2521ac829..d3eb90b7b 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -24,11 +24,11 @@ class ItemMeta(ABCMeta): """ def __new__(mcs, class_name, bases, attrs): - classcell = attrs.pop('__classcell__', None) - new_bases = tuple(base._class for base in bases if hasattr(base, '_class')) - _class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs) + classcell = attrs.pop("__classcell__", None) + new_bases = tuple(base._class for base in bases if hasattr(base, "_class")) + _class = super().__new__(mcs, "x_" + class_name, new_bases, attrs) - fields = getattr(_class, 'fields', {}) + fields = getattr(_class, "fields", {}) new_attrs = {} for n in dir(_class): v = getattr(_class, n) @@ -37,10 +37,10 @@ class ItemMeta(ABCMeta): elif n in attrs: new_attrs[n] = attrs[n] - new_attrs['fields'] = fields - new_attrs['_class'] = _class + new_attrs["fields"] = fields + new_attrs["_class"] = _class if classcell is not None: - new_attrs['__classcell__'] = classcell + new_attrs["__classcell__"] = classcell return super().__new__(mcs, class_name, bases, new_attrs) @@ -93,7 +93,7 @@ class Item(MutableMapping, object_ref, metaclass=ItemMeta): raise AttributeError(name) def __setattr__(self, name, value): - if not name.startswith('_'): + if not name.startswith("_"): raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value") super().__setattr__(name, value) @@ -115,6 +115,5 @@ class Item(MutableMapping, object_ref, metaclass=ItemMeta): return self.__class__(self) def deepcopy(self): - """Return a :func:`~copy.deepcopy` of this item. - """ + """Return a :func:`~copy.deepcopy` of this item.""" return deepcopy(self) diff --git a/scrapy/link.py b/scrapy/link.py index e70667361..704649731 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -24,9 +24,9 @@ class Link: of the anchor tag. """ - __slots__ = ['url', 'text', 'fragment', 'nofollow'] + __slots__ = ["url", "text", "fragment", "nofollow"] - def __init__(self, url, text='', fragment='', nofollow=False): + def __init__(self, url, text="", fragment="", nofollow=False): if not isinstance(url, str): got = url.__class__.__name__ raise TypeError(f"Link urls must be str objects, got {got}") @@ -44,10 +44,12 @@ class Link: ) def __hash__(self): - return hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow) + return ( + hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow) + ) def __repr__(self): return ( - f'Link(url={self.url!r}, text={self.text!r}, ' - f'fragment={self.fragment!r}, nofollow={self.nofollow!r})' + f"Link(url={self.url!r}, text={self.text!r}, " + f"fragment={self.fragment!r}, nofollow={self.nofollow!r})" ) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index b3b1eea55..ae2948d73 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -10,25 +10,81 @@ import re # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ # archives - '7z', '7zip', 'bz2', 'rar', 'tar', 'tar.gz', 'xz', 'zip', - + "7z", + "7zip", + "bz2", + "rar", + "tar", + "tar.gz", + "xz", + "zip", # images - 'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif', - 'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', 'cdr', 'ico', - + "mng", + "pct", + "bmp", + "gif", + "jpg", + "jpeg", + "png", + "pst", + "psp", + "tif", + "tiff", + "ai", + "drw", + "dxf", + "eps", + "ps", + "svg", + "cdr", + "ico", # audio - 'mp3', 'wma', 'ogg', 'wav', 'ra', 'aac', 'mid', 'au', 'aiff', - + "mp3", + "wma", + "ogg", + "wav", + "ra", + "aac", + "mid", + "au", + "aiff", # video - '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv', - 'm4a', 'm4v', 'flv', 'webm', - + "3gp", + "asf", + "asx", + "avi", + "mov", + "mp4", + "mpg", + "qt", + "rm", + "swf", + "wmv", + "m4a", + "m4v", + "flv", + "webm", # office suites - 'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg', - 'odp', - + "xls", + "xlsx", + "ppt", + "pptx", + "pps", + "doc", + "docx", + "odt", + "ods", + "odg", + "odp", # other - 'css', 'pdf', 'exe', 'bin', 'rss', 'dmg', 'iso', 'apk' + "css", + "pdf", + "exe", + "bin", + "rss", + "dmg", + "iso", + "apk", ] @@ -40,7 +96,7 @@ def _matches(url, regexs): def _is_valid_url(url): - return url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'} + return url.split("://", 1)[0] in {"http", "https", "file", "ftp"} # Top-level imports diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 1e6ab984a..0bd28a882 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -11,8 +11,13 @@ from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link -from scrapy.linkextractors import (IGNORED_EXTENSIONS, _is_valid_url, _matches, - _re_type, re) +from scrapy.linkextractors import ( + IGNORED_EXTENSIONS, + _is_valid_url, + _matches, + _re_type, + re, +) from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url @@ -26,8 +31,8 @@ _collect_string_content = etree.XPath("string()") def _nons(tag): if isinstance(tag, str): - if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE: - return tag.split('}')[-1] + if tag[0] == "{" and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE: + return tag.split("}")[-1] return tag @@ -41,14 +46,22 @@ def _canonicalize_link_url(link): class LxmlParserLinkExtractor: def __init__( - self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False + self, + tag="a", + attr="href", + process=None, + unique=False, + strip=True, + canonicalized=False, ): self.scan_tag = tag if callable(tag) else partial(operator.eq, tag) self.scan_attr = attr if callable(attr) else partial(operator.eq, attr) self.process_attr = process if callable(process) else _identity self.unique = unique self.strip = strip - self.link_key = operator.attrgetter("url") if canonicalized else _canonicalize_link_url + self.link_key = ( + operator.attrgetter("url") if canonicalized else _canonicalize_link_url + ) def _iter_links(self, document): for el in document.iter(etree.Element): @@ -78,17 +91,22 @@ class LxmlParserLinkExtractor: url = safe_url_string(url, encoding=response_encoding) # to fix relative links after process_value url = urljoin(response_url, url) - link = Link(url, _collect_string_content(el) or '', - nofollow=rel_has_nofollow(el.get('rel'))) + link = Link( + url, + _collect_string_content(el) or "", + nofollow=rel_has_nofollow(el.get("rel")), + ) links.append(link) return self._deduplicate_if_needed(links) def extract_links(self, response): base_url = get_base_url(response) - return self._extract_links(response.selector, response.url, response.encoding, base_url) + return self._extract_links( + response.selector, response.url, response.encoding, base_url + ) def _process_links(self, links): - """ Normalize and filter extracted links + """Normalize and filter extracted links The subclass should override it if necessary """ @@ -110,8 +128,8 @@ class LxmlLinkExtractor: allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), - attrs=('href',), + tags=("a", "area"), + attrs=("href",), canonicalize=False, unique=True, process_value=None, @@ -127,26 +145,31 @@ class LxmlLinkExtractor: unique=unique, process=process_value, strip=strip, - canonicalized=canonicalize + canonicalized=canonicalize, ) - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(deny)] + self.allow_res = [ + x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow) + ] + self.deny_res = [ + x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny) + ] self.allow_domains = set(arg_to_iter(allow_domains)) self.deny_domains = set(arg_to_iter(deny_domains)) self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) - self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath, - arg_to_iter(restrict_css))) + self.restrict_xpaths += tuple( + map(self._csstranslator.css_to_xpath, arg_to_iter(restrict_css)) + ) if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS self.canonicalize = canonicalize - self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} - self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(restrict_text)] + self.deny_extensions = {"." + e for e in arg_to_iter(deny_extensions)} + self.restrict_text = [ + x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(restrict_text) + ] def _link_allowed(self, link): if not _is_valid_url(link.url): @@ -156,11 +179,15 @@ class LxmlLinkExtractor: if self.deny_res and _matches(link.url, self.deny_res): return False parsed_url = urlparse(link.url) - if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains): + if self.allow_domains and not url_is_from_any_domain( + parsed_url, self.allow_domains + ): return False if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains): return False - if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): + if self.deny_extensions and url_has_any_extension( + parsed_url, self.deny_extensions + ): return False if self.restrict_text and not _matches(link.text, self.restrict_text): return False @@ -173,7 +200,11 @@ class LxmlLinkExtractor: if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): return False - allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True] + allowed = ( + (regex.search(url) for regex in self.allow_res) + if self.allow_res + else [True] + ) denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] return any(allowed) and not any(denied) @@ -200,9 +231,7 @@ class LxmlLinkExtractor: base_url = get_base_url(response) if self.restrict_xpaths: docs = [ - subdoc - for x in self.restrict_xpaths - for subdoc in response.xpath(x) + subdoc for x in self.restrict_xpaths for subdoc in response.xpath(x) ] else: docs = [response.selector] diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py index 3b8a6ee94..3e8644e0c 100644 --- a/scrapy/loader/common.py +++ b/scrapy/loader/common.py @@ -15,7 +15,7 @@ def wrap_loader_context(function, context): "scrapy.loader.common.wrap_loader_context has moved to a new library." "Please update your reference to itemloaders.common.wrap_loader_context", ScrapyDeprecationWarning, - stacklevel=2 + stacklevel=2, ) return common.wrap_loader_context(function, context) diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py index 51fbd19eb..f27a669d6 100644 --- a/scrapy/loader/processors.py +++ b/scrapy/loader/processors.py @@ -8,14 +8,14 @@ from itemloaders import processors from scrapy.utils.deprecate import create_deprecated_class -MapCompose = create_deprecated_class('MapCompose', processors.MapCompose) +MapCompose = create_deprecated_class("MapCompose", processors.MapCompose) -Compose = create_deprecated_class('Compose', processors.Compose) +Compose = create_deprecated_class("Compose", processors.Compose) -TakeFirst = create_deprecated_class('TakeFirst', processors.TakeFirst) +TakeFirst = create_deprecated_class("TakeFirst", processors.TakeFirst) -Identity = create_deprecated_class('Identity', processors.Identity) +Identity = create_deprecated_class("Identity", processors.Identity) -SelectJmes = create_deprecated_class('SelectJmes', processors.SelectJmes) +SelectJmes = create_deprecated_class("SelectJmes", processors.SelectJmes) -Join = create_deprecated_class('Join', processors.Join) +Join = create_deprecated_class("Join", processors.Join) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 87568b2d1..e0b93d812 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -54,20 +54,20 @@ class LogFormatter: def crawled(self, request, response, spider): """Logs a message when the crawler finds a webpage.""" - request_flags = f' {str(request.flags)}' if request.flags else '' - response_flags = f' {str(response.flags)}' if response.flags else '' + request_flags = f" {str(request.flags)}" if request.flags else "" + response_flags = f" {str(response.flags)}" if response.flags else "" return { - 'level': logging.DEBUG, - 'msg': CRAWLEDMSG, - 'args': { - 'status': response.status, - 'request': request, - 'request_flags': request_flags, - 'referer': referer_str(request), - 'response_flags': response_flags, + "level": logging.DEBUG, + "msg": CRAWLEDMSG, + "args": { + "status": response.status, + "request": request, + "request_flags": request_flags, + "referer": referer_str(request), + "response_flags": response_flags, # backward compatibility with Scrapy logformatter below 1.4 version - 'flags': response_flags - } + "flags": response_flags, + }, } def scraped(self, item, response, spider): @@ -77,23 +77,23 @@ class LogFormatter: else: src = response return { - 'level': logging.DEBUG, - 'msg': SCRAPEDMSG, - 'args': { - 'src': src, - 'item': item, - } + "level": logging.DEBUG, + "msg": SCRAPEDMSG, + "args": { + "src": src, + "item": item, + }, } def dropped(self, item, exception, response, spider): """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { - 'level': logging.WARNING, - 'msg': DROPPEDMSG, - 'args': { - 'exception': exception, - 'item': item, - } + "level": logging.WARNING, + "msg": DROPPEDMSG, + "args": { + "exception": exception, + "item": item, + }, } def item_error(self, item, exception, response, spider): @@ -103,11 +103,11 @@ class LogFormatter: .. versionadded:: 2.0 """ return { - 'level': logging.ERROR, - 'msg': ITEMERRORMSG, - 'args': { - 'item': item, - } + "level": logging.ERROR, + "msg": ITEMERRORMSG, + "args": { + "item": item, + }, } def spider_error(self, failure, request, response, spider): @@ -116,12 +116,12 @@ class LogFormatter: .. versionadded:: 2.0 """ return { - 'level': logging.ERROR, - 'msg': SPIDERERRORMSG, - 'args': { - 'request': request, - 'referer': referer_str(request), - } + "level": logging.ERROR, + "msg": SPIDERERRORMSG, + "args": { + "request": request, + "referer": referer_str(request), + }, } def download_error(self, failure, request, spider, errmsg=None): @@ -130,16 +130,16 @@ class LogFormatter: .. versionadded:: 2.0 """ - args = {'request': request} + args = {"request": request} if errmsg: msg = DOWNLOADERRORMSG_LONG - args['errmsg'] = errmsg + args["errmsg"] = errmsg else: msg = DOWNLOADERRORMSG_SHORT return { - 'level': logging.ERROR, - 'msg': msg, - 'args': args, + "level": logging.ERROR, + "msg": msg, + "args": args, } @classmethod diff --git a/scrapy/mail.py b/scrapy/mail.py index 2a25ccd44..fbde9c547 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -34,8 +34,15 @@ def _to_bytes_or_none(text): class MailSender: def __init__( - self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None, - smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False + self, + smtphost="localhost", + mailfrom="scrapy@localhost", + smtpuser=None, + smtppass=None, + smtpport=25, + smtptls=False, + smtpssl=False, + debug=False, ): self.smtphost = smtphost self.smtpport = smtpport @@ -49,44 +56,57 @@ class MailSender: @classmethod def from_settings(cls, settings): return cls( - smtphost=settings['MAIL_HOST'], - mailfrom=settings['MAIL_FROM'], - smtpuser=settings['MAIL_USER'], - smtppass=settings['MAIL_PASS'], - smtpport=settings.getint('MAIL_PORT'), - smtptls=settings.getbool('MAIL_TLS'), - smtpssl=settings.getbool('MAIL_SSL'), + smtphost=settings["MAIL_HOST"], + mailfrom=settings["MAIL_FROM"], + smtpuser=settings["MAIL_USER"], + smtppass=settings["MAIL_PASS"], + smtpport=settings.getint("MAIL_PORT"), + smtptls=settings.getbool("MAIL_TLS"), + smtpssl=settings.getbool("MAIL_SSL"), ) - def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None): + def send( + self, + to, + subject, + body, + cc=None, + attachs=(), + mimetype="text/plain", + charset=None, + _callback=None, + ): from twisted.internet import reactor + if attachs: msg = MIMEMultipart() else: - msg = MIMENonMultipart(*mimetype.split('/', 1)) + msg = MIMENonMultipart(*mimetype.split("/", 1)) to = list(arg_to_iter(to)) cc = list(arg_to_iter(cc)) - msg['From'] = self.mailfrom - msg['To'] = COMMASPACE.join(to) - msg['Date'] = formatdate(localtime=True) - msg['Subject'] = subject + msg["From"] = self.mailfrom + msg["To"] = COMMASPACE.join(to) + msg["Date"] = formatdate(localtime=True) + msg["Subject"] = subject rcpts = to[:] if cc: rcpts.extend(cc) - msg['Cc'] = COMMASPACE.join(cc) + msg["Cc"] = COMMASPACE.join(cc) if charset: msg.set_charset(charset) if attachs: - msg.attach(MIMEText(body, 'plain', charset or 'us-ascii')) + msg.attach(MIMEText(body, "plain", charset or "us-ascii")) for attach_name, mimetype, f in attachs: - part = MIMEBase(*mimetype.split('/')) + part = MIMEBase(*mimetype.split("/")) part.set_payload(f.read()) Encoders.encode_base64(part) - part.add_header('Content-Disposition', 'attachment', filename=attach_name) + part.add_header( + "Content-Disposition", "attachment", filename=attach_name + ) msg.attach(part) else: msg.set_payload(body) @@ -95,50 +115,79 @@ class MailSender: _callback(to=to, subject=subject, body=body, cc=cc, attach=attachs, msg=msg) if self.debug: - logger.debug('Debug mail sent OK: To=%(mailto)s Cc=%(mailcc)s ' - 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d', - {'mailto': to, 'mailcc': cc, 'mailsubject': subject, - 'mailattachs': len(attachs)}) + logger.debug( + "Debug mail sent OK: To=%(mailto)s Cc=%(mailcc)s " + 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d', + { + "mailto": to, + "mailcc": cc, + "mailsubject": subject, + "mailattachs": len(attachs), + }, + ) return - dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8')) + dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8")) dfd.addCallbacks( callback=self._sent_ok, errback=self._sent_failed, callbackArgs=[to, cc, subject, len(attachs)], errbackArgs=[to, cc, subject, len(attachs)], ) - reactor.addSystemEventTrigger('before', 'shutdown', lambda: dfd) + reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd) return dfd def _sent_ok(self, result, to, cc, subject, nattachs): - logger.info('Mail sent OK: To=%(mailto)s Cc=%(mailcc)s ' - 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d', - {'mailto': to, 'mailcc': cc, 'mailsubject': subject, - 'mailattachs': nattachs}) + logger.info( + "Mail sent OK: To=%(mailto)s Cc=%(mailcc)s " + 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d', + { + "mailto": to, + "mailcc": cc, + "mailsubject": subject, + "mailattachs": nattachs, + }, + ) def _sent_failed(self, failure, to, cc, subject, nattachs): errstr = str(failure.value) - logger.error('Unable to send mail: To=%(mailto)s Cc=%(mailcc)s ' - 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d' - '- %(mailerr)s', - {'mailto': to, 'mailcc': cc, 'mailsubject': subject, - 'mailattachs': nattachs, 'mailerr': errstr}) + logger.error( + "Unable to send mail: To=%(mailto)s Cc=%(mailcc)s " + 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d' + "- %(mailerr)s", + { + "mailto": to, + "mailcc": cc, + "mailsubject": subject, + "mailattachs": nattachs, + "mailerr": errstr, + }, + ) def _sendmail(self, to_addrs, msg): # Import twisted.mail here because it is not available in python3 from twisted.internet import reactor from twisted.mail.smtp import ESMTPSenderFactory + msg = BytesIO(msg) d = defer.Deferred() factory = ESMTPSenderFactory( - self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, - heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls, + self.smtpuser, + self.smtppass, + self.mailfrom, + to_addrs, + msg, + d, + heloFallback=True, + requireAuthentication=False, + requireTransportSecurity=self.smtptls, ) factory.noisy = False if self.smtpssl: - reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory()) + reactor.connectSSL( + self.smtphost, self.smtpport, factory, ssl.ClientContextFactory() + ) else: reactor.connectTCP(self.smtphost, self.smtpport, factory) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 431bd76dc..15f5b23e0 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -17,13 +17,15 @@ logger = logging.getLogger(__name__) class MiddlewareManager: """Base class for implementing middleware managers""" - component_name = 'foo middleware' + component_name = "foo middleware" def __init__(self, *middlewares: Any) -> None: self.middlewares = middlewares # Only process_spider_output and process_spider_exception can be None. # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. - self.methods: Dict[str, Deque[Union[None, Callable, Tuple[Callable, Callable]]]] = defaultdict(deque) + self.methods: Dict[ + str, Deque[Union[None, Callable, Tuple[Callable, Callable]]] + ] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -44,15 +46,21 @@ class MiddlewareManager: enabled.append(clspath) except NotConfigured as e: if e.args: - clsname = clspath.split('.')[-1] - logger.warning("Disabled %(clsname)s: %(eargs)s", - {'clsname': clsname, 'eargs': e.args[0]}, - extra={'crawler': crawler}) + clsname = clspath.split(".")[-1] + logger.warning( + "Disabled %(clsname)s: %(eargs)s", + {"clsname": clsname, "eargs": e.args[0]}, + extra={"crawler": crawler}, + ) - logger.info("Enabled %(componentname)ss:\n%(enabledlist)s", - {'componentname': cls.component_name, - 'enabledlist': pprint.pformat(enabled)}, - extra={'crawler': crawler}) + logger.info( + "Enabled %(componentname)ss:\n%(enabledlist)s", + { + "componentname": cls.component_name, + "enabledlist": pprint.pformat(enabled), + }, + extra={"crawler": crawler}, + ) return cls(*middlewares) @classmethod @@ -60,10 +68,10 @@ class MiddlewareManager: return cls.from_settings(crawler.settings, crawler) def _add_middleware(self, mw) -> None: - if hasattr(mw, 'open_spider'): - self.methods['open_spider'].append(mw.open_spider) - if hasattr(mw, 'close_spider'): - self.methods['close_spider'].appendleft(mw.close_spider) + if hasattr(mw, "open_spider"): + self.methods["open_spider"].append(mw.open_spider) + if hasattr(mw, "close_spider"): + self.methods["close_spider"].appendleft(mw.close_spider) def _process_parallel(self, methodname: str, obj, *args) -> Deferred: methods = cast(Iterable[Callable], self.methods[methodname]) @@ -74,7 +82,7 @@ class MiddlewareManager: return process_chain(methods, obj, *args) def open_spider(self, spider: Spider) -> Deferred: - return self._process_parallel('open_spider', spider) + return self._process_parallel("open_spider", spider) def close_spider(self, spider: Spider) -> Deferred: - return self._process_parallel('close_spider', spider) + return self._process_parallel("close_spider", spider) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index c42dd423e..536341fc6 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -11,16 +11,18 @@ from scrapy.utils.defer import deferred_f_from_coro_f class ItemPipelineManager(MiddlewareManager): - component_name = 'item pipeline' + component_name = "item pipeline" @classmethod def _get_mwlist_from_settings(cls, settings): - return build_component_list(settings.getwithbase('ITEM_PIPELINES')) + return build_component_list(settings.getwithbase("ITEM_PIPELINES")) def _add_middleware(self, pipe): super()._add_middleware(pipe) - if hasattr(pipe, 'process_item'): - self.methods['process_item'].append(deferred_f_from_coro_f(pipe.process_item)) + if hasattr(pipe, "process_item"): + self.methods["process_item"].append( + deferred_f_from_coro_f(pipe.process_item) + ) def process_item(self, item, spider): - return self._process_chain('process_item', item, spider) + return self._process_chain("process_item", item, spider) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 51aedafe8..01a9c41fe 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -42,8 +42,8 @@ class FileException(Exception): class FSFilesStore: def __init__(self, basedir: str): - if '://' in basedir: - basedir = basedir.split('://', 1)[1] + if "://" in basedir: + basedir = basedir.split("://", 1)[1] self.basedir = basedir self._mkdir(Path(self.basedir)) self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set) @@ -60,13 +60,13 @@ class FSFilesStore: except os.error: return {} - with absolute_path.open('rb') as f: + with absolute_path.open("rb") as f: checksum = md5sum(f) - return {'last_modified': last_modified, 'checksum': checksum} + return {"last_modified": last_modified, "checksum": checksum} def _get_filesystem_path(self, path: str) -> Path: - path_comps = path.split('/') + path_comps = path.split("/") return Path(self.basedir, *path_comps) def _mkdir(self, dirname: Path, domain: Optional[str] = None): @@ -86,49 +86,49 @@ class S3FilesStore: AWS_USE_SSL = None AWS_VERIFY = None - POLICY = 'private' # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings + POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings HEADERS = { - 'Cache-Control': 'max-age=172800', + "Cache-Control": "max-age=172800", } def __init__(self, uri): if not is_botocore_available(): - raise NotConfigured('missing botocore library') + raise NotConfigured("missing botocore library") import botocore.session + session = botocore.session.get_session() self.s3_client = session.create_client( - 's3', + "s3", aws_access_key_id=self.AWS_ACCESS_KEY_ID, aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, aws_session_token=self.AWS_SESSION_TOKEN, endpoint_url=self.AWS_ENDPOINT_URL, region_name=self.AWS_REGION_NAME, use_ssl=self.AWS_USE_SSL, - verify=self.AWS_VERIFY + verify=self.AWS_VERIFY, ) if not uri.startswith("s3://"): raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") - self.bucket, self.prefix = uri[5:].split('/', 1) + self.bucket, self.prefix = uri[5:].split("/", 1) def stat_file(self, path, info): def _onsuccess(boto_key): - checksum = boto_key['ETag'].strip('"') - last_modified = boto_key['LastModified'] + checksum = boto_key["ETag"].strip('"') + last_modified = boto_key["LastModified"] modified_stamp = time.mktime(last_modified.timetuple()) - return {'checksum': checksum, 'last_modified': modified_stamp} + return {"checksum": checksum, "last_modified": modified_stamp} return self._get_boto_key(path).addCallback(_onsuccess) def _get_boto_key(self, path): - key_name = f'{self.prefix}{path}' + key_name = f"{self.prefix}{path}" return threads.deferToThread( - self.s3_client.head_object, - Bucket=self.bucket, - Key=key_name) + self.s3_client.head_object, Bucket=self.bucket, Key=key_name + ) def persist_file(self, path, buf, info, meta=None, headers=None): """Upload file to S3 storage""" - key_name = f'{self.prefix}{path}' + key_name = f"{self.prefix}{path}" buf.seek(0) extra = self._headers_to_botocore_kwargs(self.HEADERS) if headers: @@ -140,39 +140,41 @@ class S3FilesStore: Body=buf, Metadata={k: str(v) for k, v in (meta or {}).items()}, ACL=self.POLICY, - **extra) + **extra, + ) def _headers_to_botocore_kwargs(self, headers): - """ Convert headers to botocore keyword arguments. - """ + """Convert headers to botocore keyword arguments.""" # This is required while we need to support both boto and botocore. - mapping = CaselessDict({ - 'Content-Type': 'ContentType', - 'Cache-Control': 'CacheControl', - 'Content-Disposition': 'ContentDisposition', - 'Content-Encoding': 'ContentEncoding', - 'Content-Language': 'ContentLanguage', - 'Content-Length': 'ContentLength', - 'Content-MD5': 'ContentMD5', - 'Expires': 'Expires', - 'X-Amz-Grant-Full-Control': 'GrantFullControl', - 'X-Amz-Grant-Read': 'GrantRead', - 'X-Amz-Grant-Read-ACP': 'GrantReadACP', - 'X-Amz-Grant-Write-ACP': 'GrantWriteACP', - 'X-Amz-Object-Lock-Legal-Hold': 'ObjectLockLegalHoldStatus', - 'X-Amz-Object-Lock-Mode': 'ObjectLockMode', - 'X-Amz-Object-Lock-Retain-Until-Date': 'ObjectLockRetainUntilDate', - 'X-Amz-Request-Payer': 'RequestPayer', - 'X-Amz-Server-Side-Encryption': 'ServerSideEncryption', - 'X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id': 'SSEKMSKeyId', - 'X-Amz-Server-Side-Encryption-Context': 'SSEKMSEncryptionContext', - 'X-Amz-Server-Side-Encryption-Customer-Algorithm': 'SSECustomerAlgorithm', - 'X-Amz-Server-Side-Encryption-Customer-Key': 'SSECustomerKey', - 'X-Amz-Server-Side-Encryption-Customer-Key-Md5': 'SSECustomerKeyMD5', - 'X-Amz-Storage-Class': 'StorageClass', - 'X-Amz-Tagging': 'Tagging', - 'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation', - }) + mapping = CaselessDict( + { + "Content-Type": "ContentType", + "Cache-Control": "CacheControl", + "Content-Disposition": "ContentDisposition", + "Content-Encoding": "ContentEncoding", + "Content-Language": "ContentLanguage", + "Content-Length": "ContentLength", + "Content-MD5": "ContentMD5", + "Expires": "Expires", + "X-Amz-Grant-Full-Control": "GrantFullControl", + "X-Amz-Grant-Read": "GrantRead", + "X-Amz-Grant-Read-ACP": "GrantReadACP", + "X-Amz-Grant-Write-ACP": "GrantWriteACP", + "X-Amz-Object-Lock-Legal-Hold": "ObjectLockLegalHoldStatus", + "X-Amz-Object-Lock-Mode": "ObjectLockMode", + "X-Amz-Object-Lock-Retain-Until-Date": "ObjectLockRetainUntilDate", + "X-Amz-Request-Payer": "RequestPayer", + "X-Amz-Server-Side-Encryption": "ServerSideEncryption", + "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": "SSEKMSKeyId", + "X-Amz-Server-Side-Encryption-Context": "SSEKMSEncryptionContext", + "X-Amz-Server-Side-Encryption-Customer-Algorithm": "SSECustomerAlgorithm", + "X-Amz-Server-Side-Encryption-Customer-Key": "SSECustomerKey", + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": "SSECustomerKeyMD5", + "X-Amz-Storage-Class": "StorageClass", + "X-Amz-Tagging": "Tagging", + "X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation", + } + ) extra = {} for key, value in headers.items(): try: @@ -188,7 +190,7 @@ class GCSFilesStore: GCS_PROJECT_ID = None - CACHE_CONTROL = 'max-age=172800' + CACHE_CONTROL = "max-age=172800" # The bucket's default object ACL will be applied to the object. # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. @@ -196,23 +198,24 @@ class GCSFilesStore: def __init__(self, uri): from google.cloud import storage + client = storage.Client(project=self.GCS_PROJECT_ID) - bucket, prefix = uri[5:].split('/', 1) + bucket, prefix = uri[5:].split("/", 1) self.bucket = client.bucket(bucket) self.prefix = prefix permissions = self.bucket.test_iam_permissions( - ['storage.objects.get', 'storage.objects.create'] + ["storage.objects.get", "storage.objects.create"] ) - if 'storage.objects.get' not in permissions: + if "storage.objects.get" not in permissions: logger.warning( "No 'storage.objects.get' permission for GSC bucket %(bucket)s. " "Checking if files are up to date will be impossible. Files will be downloaded every time.", - {'bucket': bucket} + {"bucket": bucket}, ) - if 'storage.objects.create' not in permissions: + if "storage.objects.create" not in permissions: logger.error( "No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!", - {'bucket': bucket} + {"bucket": bucket}, ) def stat_file(self, path, info): @@ -220,15 +223,18 @@ class GCSFilesStore: if blob: checksum = blob.md5_hash last_modified = time.mktime(blob.updated.timetuple()) - return {'checksum': checksum, 'last_modified': last_modified} + return {"checksum": checksum, "last_modified": last_modified} return {} + blob_path = self._get_blob_path(path) - return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess) + return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback( + _onsuccess + ) def _get_content_type(self, headers): - if headers and 'Content-Type' in headers: - return headers['Content-Type'] - return 'application/octet-stream' + if headers and "Content-Type" in headers: + return headers["Content-Type"] + return "application/octet-stream" def _get_blob_path(self, path): return self.prefix + path @@ -242,7 +248,7 @@ class GCSFilesStore: blob.upload_from_string, data=buf.getvalue(), content_type=self._get_content_type(headers), - predefined_acl=self.POLICY + predefined_acl=self.POLICY, ) @@ -261,14 +267,19 @@ class FTPFilesStore: self.port = int(u.port or 21) self.username = u.username or self.FTP_USERNAME self.password = u.password or self.FTP_PASSWORD - self.basedir = u.path.rstrip('/') + self.basedir = u.path.rstrip("/") def persist_file(self, path, buf, info, meta=None, headers=None): - path = f'{self.basedir}/{path}' + path = f"{self.basedir}/{path}" return threads.deferToThread( - ftp_store_file, path=path, file=buf, - host=self.host, port=self.port, username=self.username, - password=self.password, use_active_mode=self.USE_ACTIVE_MODE + ftp_store_file, + path=path, + file=buf, + host=self.host, + port=self.port, + username=self.username, + password=self.password, + use_active_mode=self.USE_ACTIVE_MODE, ) def stat_file(self, path, info): @@ -282,11 +293,12 @@ class FTPFilesStore: file_path = f"{self.basedir}/{path}" last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) m = hashlib.md5() - ftp.retrbinary(f'RETR {file_path}', m.update) - return {'last_modified': last_modified, 'checksum': m.hexdigest()} + ftp.retrbinary(f"RETR {file_path}", m.update) + return {"last_modified": last_modified, "checksum": m.hexdigest()} # The file doesn't exist except Exception: return {} + return threads.deferToThread(_stat_file, path) @@ -312,14 +324,14 @@ class FilesPipeline(MediaPipeline): MEDIA_NAME = "file" EXPIRES = 90 STORE_SCHEMES = { - '': FSFilesStore, - 'file': FSFilesStore, - 's3': S3FilesStore, - 'gs': GCSFilesStore, - 'ftp': FTPFilesStore + "": FSFilesStore, + "file": FSFilesStore, + "s3": S3FilesStore, + "gs": GCSFilesStore, + "ftp": FTPFilesStore, } - DEFAULT_FILES_URLS_FIELD = 'file_urls' - DEFAULT_FILES_RESULT_FIELD = 'files' + DEFAULT_FILES_URLS_FIELD = "file_urls" + DEFAULT_FILES_RESULT_FIELD = "files" def __init__(self, store_uri, download_func=None, settings=None): if not store_uri: @@ -330,52 +342,50 @@ class FilesPipeline(MediaPipeline): cls_name = "FilesPipeline" self.store = self._get_store(store_uri) - resolve = functools.partial(self._key_for_pipe, - base_class_name=cls_name, - settings=settings) - self.expires = settings.getint( - resolve('FILES_EXPIRES'), self.EXPIRES + resolve = functools.partial( + self._key_for_pipe, base_class_name=cls_name, settings=settings ) + self.expires = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES) if not hasattr(self, "FILES_URLS_FIELD"): self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD if not hasattr(self, "FILES_RESULT_FIELD"): self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD self.files_urls_field = settings.get( - resolve('FILES_URLS_FIELD'), self.FILES_URLS_FIELD + resolve("FILES_URLS_FIELD"), self.FILES_URLS_FIELD ) self.files_result_field = settings.get( - resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD + resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD ) super().__init__(download_func=download_func, settings=settings) @classmethod def from_settings(cls, settings): - s3store = cls.STORE_SCHEMES['s3'] - s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] - s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] - s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN'] - s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] - s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] - s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] - s3store.AWS_VERIFY = settings['AWS_VERIFY'] - s3store.POLICY = settings['FILES_STORE_S3_ACL'] + s3store = cls.STORE_SCHEMES["s3"] + s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] + s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] + s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] + s3store.AWS_ENDPOINT_URL = settings["AWS_ENDPOINT_URL"] + s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"] + s3store.AWS_USE_SSL = settings["AWS_USE_SSL"] + s3store.AWS_VERIFY = settings["AWS_VERIFY"] + s3store.POLICY = settings["FILES_STORE_S3_ACL"] - gcs_store = cls.STORE_SCHEMES['gs'] - gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] - gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None + gcs_store = cls.STORE_SCHEMES["gs"] + gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] + gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None - ftp_store = cls.STORE_SCHEMES['ftp'] - ftp_store.FTP_USERNAME = settings['FTP_USER'] - ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD'] - ftp_store.USE_ACTIVE_MODE = settings.getbool('FEED_STORAGE_FTP_ACTIVE') + ftp_store = cls.STORE_SCHEMES["ftp"] + ftp_store.FTP_USERNAME = settings["FTP_USER"] + ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] + ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") - store_uri = settings['FILES_STORE'] + store_uri = settings["FILES_STORE"] return cls(store_uri, settings=settings) def _get_store(self, uri: str): if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir - scheme = 'file' + scheme = "file" else: scheme = urlparse(uri).scheme store_cls = self.STORE_SCHEMES[scheme] @@ -386,7 +396,7 @@ class FilesPipeline(MediaPipeline): if not result: return # returning None force download - last_modified = result.get('last_modified', None) + last_modified = result.get("last_modified", None) if not last_modified: return # returning None force download @@ -397,25 +407,30 @@ class FilesPipeline(MediaPipeline): referer = referer_str(request) logger.debug( - 'File (uptodate): Downloaded %(medianame)s from %(request)s ' - 'referred in <%(referer)s>', - {'medianame': self.MEDIA_NAME, 'request': request, - 'referer': referer}, - extra={'spider': info.spider} + "File (uptodate): Downloaded %(medianame)s from %(request)s " + "referred in <%(referer)s>", + {"medianame": self.MEDIA_NAME, "request": request, "referer": referer}, + extra={"spider": info.spider}, ) - self.inc_stats(info.spider, 'uptodate') + self.inc_stats(info.spider, "uptodate") - checksum = result.get('checksum', None) - return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'} + checksum = result.get("checksum", None) + return { + "url": request.url, + "path": path, + "checksum": checksum, + "status": "uptodate", + } path = self.file_path(request, info=info, item=item) dfd = defer.maybeDeferred(self.store.stat_file, path, info) dfd.addCallbacks(_onsuccess, lambda _: None) dfd.addErrback( - lambda f: - logger.error(self.__class__.__name__ + '.store.stat_file', - exc_info=failure_to_exc_info(f), - extra={'spider': info.spider}) + lambda f: logger.error( + self.__class__.__name__ + ".store.stat_file", + exc_info=failure_to_exc_info(f), + extra={"spider": info.spider}, + ) ) return dfd @@ -423,11 +438,15 @@ class FilesPipeline(MediaPipeline): if not isinstance(failure.value, IgnoreRequest): referer = referer_str(request) logger.warning( - 'File (unknown-error): Error downloading %(medianame)s from ' - '%(request)s referred in <%(referer)s>: %(exception)s', - {'medianame': self.MEDIA_NAME, 'request': request, - 'referer': referer, 'exception': failure.value}, - extra={'spider': info.spider} + "File (unknown-error): Error downloading %(medianame)s from " + "%(request)s referred in <%(referer)s>: %(exception)s", + { + "medianame": self.MEDIA_NAME, + "request": request, + "referer": referer, + "exception": failure.value, + }, + extra={"spider": info.spider}, ) raise FileException @@ -437,29 +456,28 @@ class FilesPipeline(MediaPipeline): if response.status != 200: logger.warning( - 'File (code: %(status)s): Error downloading file from ' - '%(request)s referred in <%(referer)s>', - {'status': response.status, - 'request': request, 'referer': referer}, - extra={'spider': info.spider} + "File (code: %(status)s): Error downloading file from " + "%(request)s referred in <%(referer)s>", + {"status": response.status, "request": request, "referer": referer}, + extra={"spider": info.spider}, ) - raise FileException('download-error') + raise FileException("download-error") if not response.body: logger.warning( - 'File (empty-content): Empty file from %(request)s referred ' - 'in <%(referer)s>: no-content', - {'request': request, 'referer': referer}, - extra={'spider': info.spider} + "File (empty-content): Empty file from %(request)s referred " + "in <%(referer)s>: no-content", + {"request": request, "referer": referer}, + extra={"spider": info.spider}, ) - raise FileException('empty-content') + raise FileException("empty-content") - status = 'cached' if 'cached' in response.flags else 'downloaded' + status = "cached" if "cached" in response.flags else "downloaded" logger.debug( - 'File (%(status)s): Downloaded file from %(request)s referred in ' - '<%(referer)s>', - {'status': status, 'request': request, 'referer': referer}, - extra={'spider': info.spider} + "File (%(status)s): Downloaded file from %(request)s referred in " + "<%(referer)s>", + {"status": status, "request": request, "referer": referer}, + extra={"spider": info.spider}, ) self.inc_stats(info.spider, status) @@ -468,26 +486,33 @@ class FilesPipeline(MediaPipeline): checksum = self.file_downloaded(response, request, info, item=item) except FileException as exc: logger.warning( - 'File (error): Error processing file from %(request)s ' - 'referred in <%(referer)s>: %(errormsg)s', - {'request': request, 'referer': referer, 'errormsg': str(exc)}, - extra={'spider': info.spider}, exc_info=True + "File (error): Error processing file from %(request)s " + "referred in <%(referer)s>: %(errormsg)s", + {"request": request, "referer": referer, "errormsg": str(exc)}, + extra={"spider": info.spider}, + exc_info=True, ) raise except Exception as exc: logger.error( - 'File (unknown-error): Error processing file from %(request)s ' - 'referred in <%(referer)s>', - {'request': request, 'referer': referer}, - exc_info=True, extra={'spider': info.spider} + "File (unknown-error): Error processing file from %(request)s " + "referred in <%(referer)s>", + {"request": request, "referer": referer}, + exc_info=True, + extra={"spider": info.spider}, ) raise FileException(str(exc)) - return {'url': request.url, 'path': path, 'checksum': checksum, 'status': status} + return { + "url": request.url, + "path": path, + "checksum": checksum, + "status": status, + } def inc_stats(self, spider, status): - spider.crawler.stats.inc_value('file_count', spider=spider) - spider.crawler.stats.inc_value(f'file_status_count/{status}', spider=spider) + spider.crawler.stats.inc_value("file_count", spider=spider) + spider.crawler.stats.inc_value(f"file_status_count/{status}", spider=spider) # Overridable Interface def get_media_requests(self, item, info): @@ -513,8 +538,8 @@ class FilesPipeline(MediaPipeline): # Handles empty and wild extensions by trying to guess the # mime type then extension or default to empty string otherwise if media_ext not in mimetypes.types_map: - media_ext = '' + media_ext = "" media_type = mimetypes.guess_type(request.url)[0] if media_type: media_ext = mimetypes.guess_extension(media_type) - return f'full/{media_guid}{media_ext}' + return f"full/{media_guid}{media_ext}" diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 6a28a3b87..b8724a1cd 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -14,6 +14,7 @@ from itemadapter import ItemAdapter from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline + # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.misc import md5sum @@ -24,7 +25,11 @@ class NoimagesDrop(DropItem): """Product with no images exception""" def __init__(self, *args, **kwargs): - warnings.warn("The NoimagesDrop class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2) + warnings.warn( + "The NoimagesDrop class is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) super().__init__(*args, **kwargs) @@ -33,11 +38,9 @@ class ImageException(FileException): class ImagesPipeline(FilesPipeline): - """Abstract pipeline that implement the image thumbnail generation logic + """Abstract pipeline that implement the image thumbnail generation logic""" - """ - - MEDIA_NAME = 'image' + MEDIA_NAME = "image" # Uppercase attributes kept for backward compatibility with code that subclasses # ImagesPipeline. They may be overridden by settings. @@ -45,16 +48,17 @@ class ImagesPipeline(FilesPipeline): MIN_HEIGHT = 0 EXPIRES = 90 THUMBS = {} - DEFAULT_IMAGES_URLS_FIELD = 'image_urls' - DEFAULT_IMAGES_RESULT_FIELD = 'images' + DEFAULT_IMAGES_URLS_FIELD = "image_urls" + DEFAULT_IMAGES_RESULT_FIELD = "images" def __init__(self, store_uri, download_func=None, settings=None): try: from PIL import Image + self._Image = Image except ImportError: raise NotConfigured( - 'ImagesPipeline requires installing Pillow 4.0.0 or later' + "ImagesPipeline requires installing Pillow 4.0.0 or later" ) super().__init__(store_uri, settings=settings, download_func=download_func) @@ -62,12 +66,10 @@ class ImagesPipeline(FilesPipeline): if isinstance(settings, dict) or settings is None: settings = Settings(settings) - resolve = functools.partial(self._key_for_pipe, - base_class_name="ImagesPipeline", - settings=settings) - self.expires = settings.getint( - resolve("IMAGES_EXPIRES"), self.EXPIRES + resolve = functools.partial( + self._key_for_pipe, base_class_name="ImagesPipeline", settings=settings ) + self.expires = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES) if not hasattr(self, "IMAGES_RESULT_FIELD"): self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELD @@ -75,47 +77,39 @@ class ImagesPipeline(FilesPipeline): self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD self.images_urls_field = settings.get( - resolve('IMAGES_URLS_FIELD'), - self.IMAGES_URLS_FIELD + resolve("IMAGES_URLS_FIELD"), self.IMAGES_URLS_FIELD ) self.images_result_field = settings.get( - resolve('IMAGES_RESULT_FIELD'), - self.IMAGES_RESULT_FIELD - ) - self.min_width = settings.getint( - resolve('IMAGES_MIN_WIDTH'), self.MIN_WIDTH - ) - self.min_height = settings.getint( - resolve('IMAGES_MIN_HEIGHT'), self.MIN_HEIGHT - ) - self.thumbs = settings.get( - resolve('IMAGES_THUMBS'), self.THUMBS + resolve("IMAGES_RESULT_FIELD"), self.IMAGES_RESULT_FIELD ) + self.min_width = settings.getint(resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH) + self.min_height = settings.getint(resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT) + self.thumbs = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS) self._deprecated_convert_image = None @classmethod def from_settings(cls, settings): - s3store = cls.STORE_SCHEMES['s3'] - s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] - s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] - s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN'] - s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] - s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] - s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] - s3store.AWS_VERIFY = settings['AWS_VERIFY'] - s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] + s3store = cls.STORE_SCHEMES["s3"] + s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] + s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] + s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] + s3store.AWS_ENDPOINT_URL = settings["AWS_ENDPOINT_URL"] + s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"] + s3store.AWS_USE_SSL = settings["AWS_USE_SSL"] + s3store.AWS_VERIFY = settings["AWS_VERIFY"] + s3store.POLICY = settings["IMAGES_STORE_S3_ACL"] - gcs_store = cls.STORE_SCHEMES['gs'] - gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] - gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] or None + gcs_store = cls.STORE_SCHEMES["gs"] + gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] + gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None - ftp_store = cls.STORE_SCHEMES['ftp'] - ftp_store.FTP_USERNAME = settings['FTP_USER'] - ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD'] - ftp_store.USE_ACTIVE_MODE = settings.getbool('FEED_STORAGE_FTP_ACTIVE') + ftp_store = cls.STORE_SCHEMES["ftp"] + ftp_store.FTP_USERNAME = settings["FTP_USER"] + ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] + ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") - store_uri = settings['IMAGES_STORE'] + store_uri = settings["IMAGES_STORE"] return cls(store_uri, settings=settings) def file_downloaded(self, response, request, info, *, item=None): @@ -129,9 +123,12 @@ class ImagesPipeline(FilesPipeline): checksum = md5sum(buf) width, height = image.size self.store.persist_file( - path, buf, info, - meta={'width': width, 'height': height}, - headers={'Content-Type': 'image/jpeg'}) + path, + buf, + info, + meta={"width": width, "height": height}, + headers={"Content-Type": "image/jpeg"}, + ) return checksum def get_images(self, response, request, info, *, item=None): @@ -140,25 +137,35 @@ class ImagesPipeline(FilesPipeline): width, height = orig_image.size if width < self.min_width or height < self.min_height: - raise ImageException("Image too small " - f"({width}x{height} < " - f"{self.min_width}x{self.min_height})") + raise ImageException( + "Image too small " + f"({width}x{height} < " + f"{self.min_width}x{self.min_height})" + ) if self._deprecated_convert_image is None: - self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image) + self._deprecated_convert_image = "response_body" not in get_func_args( + self.convert_image + ) if self._deprecated_convert_image: - warnings.warn(f'{self.__class__.__name__}.convert_image() method overriden in a deprecated way, ' - 'overriden method does not accept response_body argument.', - category=ScrapyDeprecationWarning) + warnings.warn( + f"{self.__class__.__name__}.convert_image() method overriden in a deprecated way, " + "overriden method does not accept response_body argument.", + category=ScrapyDeprecationWarning, + ) if self._deprecated_convert_image: image, buf = self.convert_image(orig_image) else: - image, buf = self.convert_image(orig_image, response_body=BytesIO(response.body)) + image, buf = self.convert_image( + orig_image, response_body=BytesIO(response.body) + ) yield path, image, buf for thumb_id, size in self.thumbs.items(): - thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item) + thumb_path = self.thumb_path( + request, thumb_id, response=response, info=info, item=item + ) if self._deprecated_convert_image: thumb_image, thumb_buf = self.convert_image(image, size) else: @@ -167,21 +174,24 @@ class ImagesPipeline(FilesPipeline): def convert_image(self, image, size=None, response_body=None): if response_body is None: - warnings.warn(f'{self.__class__.__name__}.convert_image() method called in a deprecated way, ' - 'method called without response_body argument.', - category=ScrapyDeprecationWarning, stacklevel=2) + warnings.warn( + f"{self.__class__.__name__}.convert_image() method called in a deprecated way, " + "method called without response_body argument.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) - if image.format == 'PNG' and image.mode == 'RGBA': - background = self._Image.new('RGBA', image.size, (255, 255, 255)) + if image.format == "PNG" and image.mode == "RGBA": + background = self._Image.new("RGBA", image.size, (255, 255, 255)) background.paste(image, image) - image = background.convert('RGB') - elif image.mode == 'P': + image = background.convert("RGB") + elif image.mode == "P": image = image.convert("RGBA") - background = self._Image.new('RGBA', image.size, (255, 255, 255)) + background = self._Image.new("RGBA", image.size, (255, 255, 255)) background.paste(image, image) - image = background.convert('RGB') - elif image.mode != 'RGB': - image = image.convert('RGB') + image = background.convert("RGB") + elif image.mode != "RGB": + image = image.convert("RGB") if size: image = image.copy() @@ -193,11 +203,11 @@ class ImagesPipeline(FilesPipeline): except AttributeError: resampling_filter = self._Image.ANTIALIAS image.thumbnail(size, resampling_filter) - elif response_body is not None and image.format == 'JPEG': + elif response_body is not None and image.format == "JPEG": return image, response_body buf = BytesIO() - image.save(buf, 'JPEG') + image.save(buf, "JPEG") return image, buf def get_media_requests(self, item, info): @@ -211,8 +221,8 @@ class ImagesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - return f'full/{image_guid}.jpg' + return f"full/{image_guid}.jpg" def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None): thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - return f'thumbs/{thumb_id}/{thumb_guid}.jpg' + return f"thumbs/{thumb_id}/{thumb_guid}.jpg" diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 5308a9793..af23b4cc8 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -34,12 +34,10 @@ class MediaPipeline: if isinstance(settings, dict) or settings is None: settings = Settings(settings) - resolve = functools.partial(self._key_for_pipe, - base_class_name="MediaPipeline", - settings=settings) - self.allow_redirects = settings.getbool( - resolve('MEDIA_ALLOW_REDIRECTS'), False + resolve = functools.partial( + self._key_for_pipe, base_class_name="MediaPipeline", settings=settings ) + self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False) self._handle_statuses(self.allow_redirects) # Check if deprecated methods are being used and make them compatible @@ -64,7 +62,8 @@ class MediaPipeline: if ( not base_class_name or class_name == base_class_name - or settings and not settings.get(formatted_key) + or settings + and not settings.get(formatted_key) ): return key return formatted_key @@ -113,16 +112,23 @@ class MediaPipeline: dfd = mustbe_deferred(self.media_to_download, request, info, item=item) dfd.addCallback(self._check_media_to_download, request, info, item=item) dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) - dfd.addErrback(lambda f: logger.error( - f.value, exc_info=failure_to_exc_info(f), extra={'spider': info.spider}) + dfd.addErrback( + lambda f: logger.error( + f.value, exc_info=failure_to_exc_info(f), extra={"spider": info.spider} + ) ) return dfd.addBoth(lambda _: wad) # it must return wad at last def _make_compatible(self): """Make overridable methods of MediaPipeline and subclasses backwards compatible""" methods = [ - "file_path", "thumb_path", "media_to_download", "media_downloaded", - "file_downloaded", "image_downloaded", "get_images" + "file_path", + "thumb_path", + "media_to_download", + "media_downloaded", + "file_downloaded", + "image_downloaded", + "get_images", ] for method_name in methods: @@ -139,7 +145,7 @@ class MediaPipeline: if self._expects_item[func.__name__]: return func(*args, **kwargs) - kwargs.pop('item', None) + kwargs.pop("item", None) return func(*args, **kwargs) return wrapper @@ -148,19 +154,22 @@ class MediaPipeline: sig = signature(func) self._expects_item[func.__name__] = True - if 'item' not in sig.parameters: + if "item" not in sig.parameters: old_params = str(sig)[1:-1] new_params = old_params + ", *, item=None" - warn(f'{func.__name__}(self, {old_params}) is deprecated, ' - f'please use {func.__name__}(self, {new_params})', - ScrapyDeprecationWarning, stacklevel=2) + warn( + f"{func.__name__}(self, {old_params}) is deprecated, " + f"please use {func.__name__}(self, {new_params})", + ScrapyDeprecationWarning, + stacklevel=2, + ) self._expects_item[func.__name__] = False def _modify_media_request(self, request): if self.handle_httpstatus_list: - request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list + request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list else: - request.meta['handle_httpstatus_all'] = True + request.meta["handle_httpstatus_all"] = True def _check_media_to_download(self, result, request, info, item): if result is not None: @@ -169,14 +178,22 @@ class MediaPipeline: # this ugly code was left only to support tests. TODO: remove dfd = mustbe_deferred(self.download_func, request, info.spider) dfd.addCallbacks( - callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, - errback=self.media_failed, errbackArgs=(request, info)) + callback=self.media_downloaded, + callbackArgs=(request, info), + callbackKeywords={"item": item}, + errback=self.media_failed, + errbackArgs=(request, info), + ) else: self._modify_media_request(request) dfd = self.crawler.engine.download(request) dfd.addCallbacks( - callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, - errback=self.media_failed, errbackArgs=(request, info)) + callback=self.media_downloaded, + callbackArgs=(request, info), + callbackKeywords={"item": item}, + errback=self.media_failed, + errbackArgs=(request, info), + ) return dfd def _cache_result_and_execute_waiters(self, result, fp, info): @@ -207,9 +224,9 @@ class MediaPipeline: # # This problem does not occur in Python 2.7 since we don't have # Exception Chaining (https://www.python.org/dev/peps/pep-3134/). - context = getattr(result.value, '__context__', None) + context = getattr(result.value, "__context__", None) if isinstance(context, StopIteration): - setattr(result.value, '__context__', None) + setattr(result.value, "__context__", None) info.downloading.remove(fp) info.downloaded[fp] = result # cache result @@ -239,10 +256,10 @@ class MediaPipeline: for ok, value in results: if not ok: logger.error( - '%(class)s found errors processing %(item)s', - {'class': self.__class__.__name__, 'item': item}, + "%(class)s found errors processing %(item)s", + {"class": self.__class__.__name__, "item": item}, exc_info=failure_to_exc_info(value), - extra={'spider': info.spider} + extra={"spider": info.spider}, ) return item diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index b4b63e7c7..6f65184e5 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -18,11 +18,11 @@ def _path_safe(text): >>> _path_safe('some@symbol?').startswith('some_symbol_') True """ - pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in text]) + pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part - unique_slot = hashlib.md5(text.encode('utf8')).hexdigest() - return '-'.join([pathable_slot, unique_slot]) + unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() + return "-".join([pathable_slot, unique_slot]) class ScrapyPriorityQueue: @@ -77,7 +77,7 @@ class ScrapyPriorityQueue: self.downstream_queue_cls, None, self.crawler, - self.key + '/' + str(key), + self.key + "/" + str(key), ) def priority(self, request): @@ -128,7 +128,6 @@ class ScrapyPriorityQueue: class DownloaderInterface: - def __init__(self, crawler): self.downloader = crawler.engine.downloader @@ -139,14 +138,14 @@ class DownloaderInterface: return self.downloader._get_slot_key(request, None) def _active_downloads(self, slot): - """ Return a number of requests in a Downloader for a given slot """ + """Return a number of requests in a Downloader for a given slot""" if slot not in self.downloader.slots: return 0 return len(self.downloader.slots[slot].active) class DownloaderAwarePriorityQueue: - """ PriorityQueue which takes Downloader activity into account: + """PriorityQueue which takes Downloader activity into account: domains (slots) with the least amount of active downloads are dequeued first. """ @@ -156,17 +155,21 @@ class DownloaderAwarePriorityQueue: return cls(crawler, downstream_queue_cls, key, startprios) def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): - if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0: - raise ValueError(f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP') + if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: + raise ValueError( + f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' + ) if slot_startprios and not isinstance(slot_startprios, dict): - raise ValueError("DownloaderAwarePriorityQueue accepts " - "``slot_startprios`` as a dict; " - f"{slot_startprios.__class__!r} instance " - "is passed. Most likely, it means the state is" - "created by an incompatible priority queue. " - "Only a crawl started with the same priority " - "queue class can be resumed.") + raise ValueError( + "DownloaderAwarePriorityQueue accepts " + "``slot_startprios`` as a dict; " + f"{slot_startprios.__class__!r} instance " + "is passed. Most likely, it means the state is" + "created by an incompatible priority queue. " + "Only a crawl started with the same priority " + "queue class can be resumed." + ) self._downloader_interface = DownloaderInterface(crawler) self.downstream_queue_cls = downstream_queue_cls @@ -181,7 +184,7 @@ class DownloaderAwarePriorityQueue: return ScrapyPriorityQueue( self.crawler, self.downstream_queue_cls, - self.key + '/' + _path_safe(slot), + self.key + "/" + _path_safe(slot), startprios, ) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 0bef555a6..f5d2b8e05 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,6 +1,11 @@ from twisted.internet import defer from twisted.internet.base import ThreadedResolver -from twisted.internet.interfaces import IHostResolution, IHostnameResolver, IResolutionReceiver, IResolverSimple +from twisted.internet.interfaces import ( + IHostResolution, + IHostnameResolver, + IResolutionReceiver, + IResolverSimple, +) from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache @@ -23,11 +28,11 @@ class CachingThreadedResolver(ThreadedResolver): @classmethod def from_crawler(cls, crawler, reactor): - if crawler.settings.getbool('DNSCACHE_ENABLED'): - cache_size = crawler.settings.getint('DNSCACHE_SIZE') + if crawler.settings.getbool("DNSCACHE_ENABLED"): + cache_size = crawler.settings.getint("DNSCACHE_SIZE") else: cache_size = 0 - return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT')) + return cls(reactor, cache_size, crawler.settings.getfloat("DNS_TIMEOUT")) def install_on_reactor(self): self.reactor.installResolver(self) @@ -94,8 +99,8 @@ class CachingHostnameResolver: @classmethod def from_crawler(cls, crawler, reactor): - if crawler.settings.getbool('DNSCACHE_ENABLED'): - cache_size = crawler.settings.getint('DNSCACHE_SIZE') + if crawler.settings.getbool("DNSCACHE_ENABLED"): + cache_size = crawler.settings.getint("DNSCACHE_SIZE") else: cache_size = 0 return cls(reactor, cache_size) @@ -104,7 +109,12 @@ class CachingHostnameResolver: self.reactor.installNameResolver(self) def resolveHostName( - self, resolutionReceiver, hostName, portNumber=0, addressTypes=None, transportSemantics="TCP" + self, + resolutionReceiver, + hostName, + portNumber=0, + addressTypes=None, + transportSemantics="TCP", ): try: addresses = dnscache[hostName] diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index a34d7d25f..6b489bd8b 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -14,26 +14,26 @@ from scrapy.utils.python import binary_is_text, to_bytes, to_unicode class ResponseTypes: CLASSES = { - 'text/html': 'scrapy.http.HtmlResponse', - 'application/atom+xml': 'scrapy.http.XmlResponse', - 'application/rdf+xml': 'scrapy.http.XmlResponse', - 'application/rss+xml': 'scrapy.http.XmlResponse', - 'application/xhtml+xml': 'scrapy.http.HtmlResponse', - 'application/vnd.wap.xhtml+xml': 'scrapy.http.HtmlResponse', - 'application/xml': 'scrapy.http.XmlResponse', - 'application/json': 'scrapy.http.TextResponse', - 'application/x-json': 'scrapy.http.TextResponse', - 'application/json-amazonui-streaming': 'scrapy.http.TextResponse', - 'application/javascript': 'scrapy.http.TextResponse', - 'application/x-javascript': 'scrapy.http.TextResponse', - 'text/xml': 'scrapy.http.XmlResponse', - 'text/*': 'scrapy.http.TextResponse', + "text/html": "scrapy.http.HtmlResponse", + "application/atom+xml": "scrapy.http.XmlResponse", + "application/rdf+xml": "scrapy.http.XmlResponse", + "application/rss+xml": "scrapy.http.XmlResponse", + "application/xhtml+xml": "scrapy.http.HtmlResponse", + "application/vnd.wap.xhtml+xml": "scrapy.http.HtmlResponse", + "application/xml": "scrapy.http.XmlResponse", + "application/json": "scrapy.http.TextResponse", + "application/x-json": "scrapy.http.TextResponse", + "application/json-amazonui-streaming": "scrapy.http.TextResponse", + "application/javascript": "scrapy.http.TextResponse", + "application/x-javascript": "scrapy.http.TextResponse", + "text/xml": "scrapy.http.XmlResponse", + "text/*": "scrapy.http.TextResponse", } def __init__(self): self.classes = {} self.mimetypes = MimeTypes() - mimedata = get_data('scrapy', 'mime.types').decode('utf8') + mimedata = get_data("scrapy", "mime.types").decode("utf8") self.mimetypes.readfp(StringIO(mimedata)) for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) @@ -49,17 +49,20 @@ class ResponseTypes: def from_content_type(self, content_type, content_encoding=None): """Return the most appropriate Response class from an HTTP Content-Type - header """ + header""" if content_encoding: return Response - mimetype = to_unicode(content_type).split(';')[0].strip().lower() + mimetype = to_unicode(content_type).split(";")[0].strip().lower() return self.from_mimetype(mimetype) def from_content_disposition(self, content_disposition): try: - filename = to_unicode( - content_disposition, encoding='latin-1', errors='replace' - ).split(';')[1].split('=')[1].strip('"\'') + filename = ( + to_unicode(content_disposition, encoding="latin-1", errors="replace") + .split(";")[1] + .split("=")[1] + .strip("\"'") + ) return self.from_filename(filename) except IndexError: return Response @@ -68,13 +71,13 @@ class ResponseTypes: """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response - if b'Content-Type' in headers: + if b"Content-Type" in headers: cls = self.from_content_type( - content_type=headers[b'Content-Type'], - content_encoding=headers.get(b'Content-Encoding') + content_type=headers[b"Content-Type"], + content_encoding=headers.get(b"Content-Encoding"), ) - if cls is Response and b'Content-Disposition' in headers: - cls = self.from_content_disposition(headers[b'Content-Disposition']) + if cls is Response and b"Content-Disposition" in headers: + cls = self.from_content_disposition(headers[b"Content-Disposition"]) return cls def from_filename(self, filename): @@ -92,15 +95,15 @@ class ResponseTypes: chunk = body[:5000] chunk = to_bytes(chunk) if not binary_is_text(chunk): - return self.from_mimetype('application/octet-stream') + return self.from_mimetype("application/octet-stream") lowercase_chunk = chunk.lower() if b"" in lowercase_chunk: - return self.from_mimetype('text/html') + return self.from_mimetype("text/html") if b"' in lowercase_chunk: - return self.from_mimetype('text/html') - return self.from_mimetype('text') + return self.from_mimetype("text/xml") + if b"" in lowercase_chunk: + return self.from_mimetype("text/html") + return self.from_mimetype("text") def from_args(self, headers=None, url=None, filename=None, body=None): """Guess the most appropriate Response class based on diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 1d40f0484..0dadeef92 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -12,7 +12,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): if to_native_str_type: robotstxt_body = to_unicode(robotstxt_body) else: - robotstxt_body = robotstxt_body.decode('utf-8') + robotstxt_body = robotstxt_body.decode("utf-8") except UnicodeDecodeError: # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. # Switch to 'allow all' state. @@ -20,9 +20,9 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): "Failure while parsing robots.txt. File either contains garbage or " "is in an encoding other than UTF-8, treating it as an empty file.", exc_info=sys.exc_info(), - extra={'spider': spider}, + extra={"spider": spider}, ) - robotstxt_body = '' + robotstxt_body = "" return robotstxt_body @@ -57,8 +57,11 @@ class RobotParser(metaclass=ABCMeta): class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body, spider): from urllib.robotparser import RobotFileParser + self.spider = spider - robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) + robotstxt_body = decode_robotstxt( + robotstxt_body, spider, to_native_str_type=True + ) self.rp = RobotFileParser() self.rp.parse(robotstxt_body.splitlines()) @@ -77,8 +80,9 @@ class PythonRobotParser(RobotParser): class ReppyRobotParser(RobotParser): def __init__(self, robotstxt_body, spider): from reppy.robots import Robots + self.spider = spider - self.rp = Robots.parse('', robotstxt_body) + self.rp = Robots.parse("", robotstxt_body) @classmethod def from_crawler(cls, crawler, robotstxt_body): @@ -93,6 +97,7 @@ class ReppyRobotParser(RobotParser): class RerpRobotParser(RobotParser): def __init__(self, robotstxt_body, spider): from robotexclusionrulesparser import RobotExclusionRulesParser + self.spider = spider self.rp = RobotExclusionRulesParser() robotstxt_body = decode_robotstxt(robotstxt_body, spider) @@ -113,6 +118,7 @@ class RerpRobotParser(RobotParser): class ProtegoRobotParser(RobotParser): def __init__(self, robotstxt_body, spider): from protego import Protego + self.spider = spider robotstxt_body = decode_robotstxt(robotstxt_body, spider) self.rp = Protego.parse(robotstxt_body) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 08f08e8d7..6ba87428e 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -8,19 +8,18 @@ from scrapy.utils.python import to_bytes from scrapy.http import HtmlResponse, XmlResponse -__all__ = ['Selector', 'SelectorList'] +__all__ = ["Selector", "SelectorList"] def _st(response, st): if st is None: - return 'xml' if isinstance(response, XmlResponse) else 'html' + return "xml" if isinstance(response, XmlResponse) else "html" return st def _response_from_text(text, st): - rt = XmlResponse if st == 'xml' else HtmlResponse - return rt(url='about:blank', encoding='utf-8', - body=to_bytes(text, 'utf-8')) + rt = XmlResponse if st == "xml" else HtmlResponse + return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) class SelectorList(_ParselSelector.selectorlist_cls, object_ref): @@ -61,13 +60,15 @@ class Selector(_ParselSelector, object_ref): detection will occur. """ - __slots__ = ['response'] + __slots__ = ["response"] selectorlist_cls = SelectorList def __init__(self, response=None, text=None, type=None, root=None, **kwargs): if response is not None and text is not None: - raise ValueError(f'{self.__class__.__name__}.__init__() received ' - 'both response and text') + raise ValueError( + f"{self.__class__.__name__}.__init__() received " + "both response and text" + ) st = _st(response, type) @@ -76,7 +77,7 @@ class Selector(_ParselSelector, object_ref): if response is not None: text = response.text - kwargs.setdefault('base_url', response.url) + kwargs.setdefault("base_url", response.url) self.response = response super().__init__(text=text, type=st, root=root, **kwargs) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b540e6182..c0d0741c5 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -8,11 +8,11 @@ from scrapy.settings import default_settings SETTINGS_PRIORITIES = { - 'default': 0, - 'command': 10, - 'project': 20, - 'spider': 30, - 'cmdline': 40, + "default": 0, + "command": 10, + "project": 20, + "spider": 30, + "cmdline": 40, } @@ -76,7 +76,7 @@ class BaseSettings(MutableMapping): highest priority will be retrieved. """ - def __init__(self, values=None, priority='project'): + def __init__(self, values=None, priority="project"): self.frozen = False self.attributes = {} if values: @@ -126,9 +126,11 @@ class BaseSettings(MutableMapping): return True if got in ("False", "false"): return False - raise ValueError("Supported values for boolean settings " - "are 0/1, True/False, '0'/'1', " - "'True'/'False' and 'true'/'false'") + raise ValueError( + "Supported values for boolean settings " + "are 0/1, True/False, '0'/'1', " + "'True'/'False' and 'true'/'false'" + ) def getint(self, name, default=0): """ @@ -170,7 +172,7 @@ class BaseSettings(MutableMapping): """ value = self.get(name, default or []) if isinstance(value, str): - value = value.split(',') + value = value.split(",") return list(value) def getdict(self, name, default=None): @@ -223,7 +225,7 @@ class BaseSettings(MutableMapping): try: return json.loads(value) except ValueError: - return value.split(',') + return value.split(",") return copy.deepcopy(value) def getwithbase(self, name): @@ -234,7 +236,7 @@ class BaseSettings(MutableMapping): :type name: str """ compbs = BaseSettings() - compbs.update(self[name + '_BASE']) + compbs.update(self[name + "_BASE"]) compbs.update(self[name]) return compbs @@ -259,12 +261,12 @@ class BaseSettings(MutableMapping): """ if len(self) > 0: return max(self.getpriority(name) for name in self) - return get_settings_priority('default') + return get_settings_priority("default") def __setitem__(self, name, value): self.set(name, value) - def set(self, name, value, priority='project'): + def set(self, name, value, priority="project"): """ Store a key/value attribute with a given priority. @@ -292,10 +294,10 @@ class BaseSettings(MutableMapping): else: self.attributes[name].set(value, priority) - def setdict(self, values, priority='project'): + def setdict(self, values, priority="project"): self.update(values, priority) - def setmodule(self, module, priority='project'): + def setmodule(self, module, priority="project"): """ Store settings from a module with a given priority. @@ -317,7 +319,7 @@ class BaseSettings(MutableMapping): if key.isupper(): self.set(key, getattr(module, key), priority) - def update(self, values, priority='project'): + def update(self, values, priority="project"): """ Store key/value pairs with a given priority. @@ -350,7 +352,7 @@ class BaseSettings(MutableMapping): for name, value in values.items(): self.set(name, value, priority) - def delete(self, name, priority='project'): + def delete(self, name, priority="project"): self._assert_mutability() priority = get_settings_priority(priority) if priority >= self.getpriority(name): @@ -403,12 +405,17 @@ class BaseSettings(MutableMapping): return len(self.attributes) def _to_dict(self): - return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) - for k, v in self.items()} + return { + self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) + for k, v in self.items() + } def _get_key(self, key_value): - return (key_value if isinstance(key_value, (bool, float, int, str, type(None))) - else str(key_value)) + return ( + key_value + if isinstance(key_value, (bool, float, int, str, type(None))) + else str(key_value) + ) def copy_to_dict(self): """ @@ -444,17 +451,17 @@ class Settings(BaseSettings): described on :ref:`topics-settings-ref` already populated. """ - def __init__(self, values=None, priority='project'): + def __init__(self, values=None, priority="project"): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user super().__init__() - self.setmodule(default_settings, 'default') + self.setmodule(default_settings, "default") # Promote default dictionaries to BaseSettings instances for per-key # priorities for name, val in self.items(): if isinstance(val, dict): - self.set(name, BaseSettings(val, 'default'), 'default') + self.set(name, BaseSettings(val, "default"), "default") self.update(values, priority) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5e7379b89..260ec1701 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -27,14 +27,14 @@ AUTOTHROTTLE_MAX_DELAY = 60.0 AUTOTHROTTLE_START_DELAY = 5.0 AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 -BOT_NAME = 'scrapybot' +BOT_NAME = "scrapybot" CLOSESPIDER_TIMEOUT = 0 CLOSESPIDER_PAGECOUNT = 0 CLOSESPIDER_ITEMCOUNT = 0 CLOSESPIDER_ERRORCOUNT = 0 -COMMANDS_MODULE = '' +COMMANDS_MODULE = "" COMPRESSION_ENABLED = True @@ -47,11 +47,11 @@ CONCURRENT_REQUESTS_PER_IP = 0 COOKIES_ENABLED = True COOKIES_DEBUG = False -DEFAULT_ITEM_CLASS = 'scrapy.item.Item' +DEFAULT_ITEM_CLASS = "scrapy.item.Item" DEFAULT_REQUEST_HEADERS = { - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en', + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en", } DEPTH_LIMIT = 0 @@ -60,78 +60,82 @@ DEPTH_PRIORITY = 0 DNSCACHE_ENABLED = True DNSCACHE_SIZE = 10000 -DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver' +DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver" DNS_TIMEOUT = 60 DOWNLOAD_DELAY = 0 DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { - 'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler', - 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', - 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', - 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', - 's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler', - 'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler', + "data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler", + "file": "scrapy.core.downloader.handlers.file.FileDownloadHandler", + "http": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + "https": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + "s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler", + "ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler", } -DOWNLOAD_TIMEOUT = 180 # 3mins +DOWNLOAD_TIMEOUT = 180 # 3mins -DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m -DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m +DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m +DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m DOWNLOAD_FAIL_ON_DATALOSS = True -DOWNLOADER = 'scrapy.core.downloader.Downloader' +DOWNLOADER = "scrapy.core.downloader.Downloader" -DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' -DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory' -DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT' +DOWNLOADER_HTTPCLIENTFACTORY = ( + "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory" +) +DOWNLOADER_CLIENTCONTEXTFACTORY = ( + "scrapy.core.downloader.contextfactory.ScrapyClientContextFactory" +) +DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT" # Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation: -DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' +DOWNLOADER_CLIENT_TLS_METHOD = "TLS" DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side - 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, - 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, - 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, - 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550, - 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560, - 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580, - 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590, - 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600, - 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700, - 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750, - 'scrapy.downloadermiddlewares.stats.DownloaderStats': 850, - 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, + "scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100, + "scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300, + "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350, + "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400, + "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500, + "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550, + "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware": 560, + "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, + "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, + "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, + "scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700, + "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750, + "scrapy.downloadermiddlewares.stats.DownloaderStats": 850, + "scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware": 900, # Downloader side } DOWNLOADER_STATS = True -DUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter' +DUPEFILTER_CLASS = "scrapy.dupefilters.RFPDupeFilter" -EDITOR = 'vi' -if sys.platform == 'win32': - EDITOR = '%s -m idlelib.idle' +EDITOR = "vi" +if sys.platform == "win32": + EDITOR = "%s -m idlelib.idle" EXTENSIONS = {} EXTENSIONS_BASE = { - 'scrapy.extensions.corestats.CoreStats': 0, - 'scrapy.extensions.telnet.TelnetConsole': 0, - 'scrapy.extensions.memusage.MemoryUsage': 0, - 'scrapy.extensions.memdebug.MemoryDebugger': 0, - 'scrapy.extensions.closespider.CloseSpider': 0, - 'scrapy.extensions.feedexport.FeedExporter': 0, - 'scrapy.extensions.logstats.LogStats': 0, - 'scrapy.extensions.spiderstate.SpiderState': 0, - 'scrapy.extensions.throttle.AutoThrottle': 0, + "scrapy.extensions.corestats.CoreStats": 0, + "scrapy.extensions.telnet.TelnetConsole": 0, + "scrapy.extensions.memusage.MemoryUsage": 0, + "scrapy.extensions.memdebug.MemoryDebugger": 0, + "scrapy.extensions.closespider.CloseSpider": 0, + "scrapy.extensions.feedexport.FeedExporter": 0, + "scrapy.extensions.logstats.LogStats": 0, + "scrapy.extensions.spiderstate.SpiderState": 0, + "scrapy.extensions.throttle.AutoThrottle": 0, } FEED_TEMPDIR = None @@ -142,71 +146,71 @@ FEED_EXPORT_ENCODING = None FEED_EXPORT_FIELDS = None FEED_STORAGES = {} FEED_STORAGES_BASE = { - '': 'scrapy.extensions.feedexport.FileFeedStorage', - 'file': 'scrapy.extensions.feedexport.FileFeedStorage', - 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', - 'gs': 'scrapy.extensions.feedexport.GCSFeedStorage', - 's3': 'scrapy.extensions.feedexport.S3FeedStorage', - 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', + "": "scrapy.extensions.feedexport.FileFeedStorage", + "file": "scrapy.extensions.feedexport.FileFeedStorage", + "ftp": "scrapy.extensions.feedexport.FTPFeedStorage", + "gs": "scrapy.extensions.feedexport.GCSFeedStorage", + "s3": "scrapy.extensions.feedexport.S3FeedStorage", + "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage", } FEED_EXPORT_BATCH_ITEM_COUNT = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { - 'json': 'scrapy.exporters.JsonItemExporter', - 'jsonlines': 'scrapy.exporters.JsonLinesItemExporter', - 'jsonl': 'scrapy.exporters.JsonLinesItemExporter', - 'jl': 'scrapy.exporters.JsonLinesItemExporter', - 'csv': 'scrapy.exporters.CsvItemExporter', - 'xml': 'scrapy.exporters.XmlItemExporter', - 'marshal': 'scrapy.exporters.MarshalItemExporter', - 'pickle': 'scrapy.exporters.PickleItemExporter', + "json": "scrapy.exporters.JsonItemExporter", + "jsonlines": "scrapy.exporters.JsonLinesItemExporter", + "jsonl": "scrapy.exporters.JsonLinesItemExporter", + "jl": "scrapy.exporters.JsonLinesItemExporter", + "csv": "scrapy.exporters.CsvItemExporter", + "xml": "scrapy.exporters.XmlItemExporter", + "marshal": "scrapy.exporters.MarshalItemExporter", + "pickle": "scrapy.exporters.PickleItemExporter", } FEED_EXPORT_INDENT = 0 FEED_STORAGE_FTP_ACTIVE = False -FEED_STORAGE_GCS_ACL = '' -FEED_STORAGE_S3_ACL = '' +FEED_STORAGE_GCS_ACL = "" +FEED_STORAGE_S3_ACL = "" -FILES_STORE_S3_ACL = 'private' -FILES_STORE_GCS_ACL = '' +FILES_STORE_S3_ACL = "private" +FILES_STORE_GCS_ACL = "" -FTP_USER = 'anonymous' -FTP_PASSWORD = 'guest' +FTP_USER = "anonymous" +FTP_PASSWORD = "guest" FTP_PASSIVE_MODE = True GCS_PROJECT_ID = None HTTPCACHE_ENABLED = False -HTTPCACHE_DIR = 'httpcache' +HTTPCACHE_DIR = "httpcache" HTTPCACHE_IGNORE_MISSING = False -HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' +HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_ALWAYS_STORE = False HTTPCACHE_IGNORE_HTTP_CODES = [] -HTTPCACHE_IGNORE_SCHEMES = ['file'] +HTTPCACHE_IGNORE_SCHEMES = ["file"] HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = [] -HTTPCACHE_DBM_MODULE = 'dbm' -HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' +HTTPCACHE_DBM_MODULE = "dbm" +HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy" HTTPCACHE_GZIP = False HTTPPROXY_ENABLED = True -HTTPPROXY_AUTH_ENCODING = 'latin-1' +HTTPPROXY_AUTH_ENCODING = "latin-1" -IMAGES_STORE_S3_ACL = 'private' -IMAGES_STORE_GCS_ACL = '' +IMAGES_STORE_S3_ACL = "private" +IMAGES_STORE_GCS_ACL = "" -ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' +ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager" ITEM_PIPELINES = {} ITEM_PIPELINES_BASE = {} LOG_ENABLED = True -LOG_ENCODING = 'utf-8' -LOG_FORMATTER = 'scrapy.logformatter.LogFormatter' -LOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s' -LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S' +LOG_ENCODING = "utf-8" +LOG_FORMATTER = "scrapy.logformatter.LogFormatter" +LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s" +LOG_DATEFORMAT = "%Y-%m-%d %H:%M:%S" LOG_STDOUT = False -LOG_LEVEL = 'DEBUG' +LOG_LEVEL = "DEBUG" LOG_FILE = None LOG_FILE_APPEND = True LOG_SHORT_NAMES = False @@ -215,14 +219,14 @@ SCHEDULER_DEBUG = False LOGSTATS_INTERVAL = 60.0 -MAIL_HOST = 'localhost' +MAIL_HOST = "localhost" MAIL_PORT = 25 -MAIL_FROM = 'scrapy@localhost' +MAIL_FROM = "scrapy@localhost" MAIL_PASS = None MAIL_USER = None -MEMDEBUG_ENABLED = False # enable memory debugging -MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown +MEMDEBUG_ENABLED = False # enable memory debugging +MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 MEMUSAGE_ENABLED = True @@ -234,7 +238,7 @@ METAREFRESH_ENABLED = True METAREFRESH_IGNORE_TAGS = [] METAREFRESH_MAXDELAY = 100 -NEWSPIDER_MODULE = '' +NEWSPIDER_MODULE = "" RANDOMIZE_DOWNLOAD_DELAY = True @@ -245,10 +249,10 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True -REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' +REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" -REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter' -REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.6' +REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" +REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.6" RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests @@ -256,39 +260,39 @@ RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] RETRY_PRIORITY_ADJUST = -1 ROBOTSTXT_OBEY = False -ROBOTSTXT_PARSER = 'scrapy.robotstxt.ProtegoRobotParser' +ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser" ROBOTSTXT_USER_AGENT = None -SCHEDULER = 'scrapy.core.scheduler.Scheduler' -SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue' -SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue' -SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue' +SCHEDULER = "scrapy.core.scheduler.Scheduler" +SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue" +SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue" +SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 -SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' +SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader" SPIDER_LOADER_WARN_ONLY = False SPIDER_MIDDLEWARES = {} SPIDER_MIDDLEWARES_BASE = { # Engine side - 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500, - 'scrapy.spidermiddlewares.referer.RefererMiddleware': 700, - 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800, - 'scrapy.spidermiddlewares.depth.DepthMiddleware': 900, + "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, + "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500, + "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, + "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, + "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, # Spider side } SPIDER_MODULES = [] -STATS_CLASS = 'scrapy.statscollectors.MemoryStatsCollector' +STATS_CLASS = "scrapy.statscollectors.MemoryStatsCollector" STATS_DUMP = True STATSMAILER_RCPTS = [] -TEMPLATES_DIR = str((Path(__file__).parent / '..' / 'templates').resolve()) +TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) URLLENGTH_LIMIT = 2083 @@ -296,16 +300,16 @@ USER_AGENT = f'Scrapy/{import_module("scrapy").__version__} (+https://scrapy.org TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] -TELNETCONSOLE_HOST = '127.0.0.1' -TELNETCONSOLE_USERNAME = 'scrapy' +TELNETCONSOLE_HOST = "127.0.0.1" +TELNETCONSOLE_USERNAME = "scrapy" TELNETCONSOLE_PASSWORD = None TWISTED_REACTOR = None SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { - 'scrapy.contracts.default.UrlContract': 1, - 'scrapy.contracts.default.CallbackKeywordArgumentsContract': 1, - 'scrapy.contracts.default.ReturnsContract': 2, - 'scrapy.contracts.default.ScrapesContract': 3, + "scrapy.contracts.default.UrlContract": 1, + "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1, + "scrapy.contracts.default.ReturnsContract": 2, + "scrapy.contracts.default.ScrapesContract": 3, } diff --git a/scrapy/shell.py b/scrapy/shell.py index f2dff2ae3..18be383c1 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -30,7 +30,7 @@ class Shell: def __init__(self, crawler, update_vars=None, code=None): self.crawler = crawler self.update_vars = update_vars or (lambda x: None) - self.item_class = load_object(crawler.settings['DEFAULT_ITEM_CLASS']) + self.item_class = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) self.spider = None self.inthread = not threadable.isInIOThread() self.code = code @@ -61,19 +61,20 @@ class Shell: shell = python """ cfg = get_config() - section, option = 'settings', 'shell' - env = os.environ.get('SCRAPY_PYTHON_SHELL') + section, option = "settings", "shell" + env = os.environ.get("SCRAPY_PYTHON_SHELL") shells = [] if env: - shells += env.strip().lower().split(',') + shells += env.strip().lower().split(",") elif cfg.has_option(section, option): shells += [cfg.get(section, option).strip().lower()] else: # try all by default shells += DEFAULT_PYTHON_SHELLS.keys() # always add standard shell as fallback - shells += ['python'] - start_python_console(self.vars, shells=shells, - banner=self.vars.pop('banner', '')) + shells += ["python"] + start_python_console( + self.vars, shells=shells, banner=self.vars.pop("banner", "") + ) def _schedule(self, request, spider): spider = self._open_spider(request, spider) @@ -96,19 +97,23 @@ class Shell: def fetch(self, request_or_url, spider=None, redirect=True, **kwargs): from twisted.internet import reactor + if isinstance(request_or_url, Request): request = request_or_url else: url = any_to_uri(request_or_url) request = Request(url, dont_filter=True, **kwargs) if redirect: - request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400)) + request.meta["handle_httpstatus_list"] = SequenceExclude( + range(300, 400) + ) else: - request.meta['handle_httpstatus_all'] = True + request.meta["handle_httpstatus_all"] = True response = None try: response, spider = threads.blockingCallFromThread( - reactor, self._schedule, request, spider) + reactor, self._schedule, request, spider + ) except IgnoreRequest: pass self.populate_vars(response, request, spider) @@ -116,20 +121,20 @@ class Shell: def populate_vars(self, response=None, request=None, spider=None): import scrapy - self.vars['scrapy'] = scrapy - self.vars['crawler'] = self.crawler - self.vars['item'] = self.item_class() - self.vars['settings'] = self.crawler.settings - self.vars['spider'] = spider - self.vars['request'] = request - self.vars['response'] = response + self.vars["scrapy"] = scrapy + self.vars["crawler"] = self.crawler + self.vars["item"] = self.item_class() + self.vars["settings"] = self.crawler.settings + self.vars["spider"] = spider + self.vars["request"] = request + self.vars["response"] = response if self.inthread: - self.vars['fetch'] = self.fetch - self.vars['view'] = open_in_browser - self.vars['shelp'] = self.print_help + self.vars["fetch"] = self.fetch + self.vars["view"] = open_in_browser + self.vars["shelp"] = self.print_help self.update_vars(self.vars) if not self.code: - self.vars['banner'] = self.get_help() + self.vars["banner"] = self.get_help() def print_help(self): print(self.get_help()) @@ -137,16 +142,22 @@ class Shell: def get_help(self): b = [] b.append("Available Scrapy objects:") - b.append(" scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)") + b.append( + " scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)" + ) for k, v in sorted(self.vars.items()): if self._is_relevant(v): b.append(f" {k:<10} {v}") b.append("Useful shortcuts:") if self.inthread: - b.append(" fetch(url[, redirect=True]) " - "Fetch URL and update local objects (by default, redirects are followed)") - b.append(" fetch(req) " - "Fetch a scrapy.Request and update local objects ") + b.append( + " fetch(url[, redirect=True]) " + "Fetch URL and update local objects (by default, redirects are followed)" + ) + b.append( + " fetch(req) " + "Fetch a scrapy.Request and update local objects " + ) b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index ac4044c64..f00447a55 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -3,7 +3,6 @@ from scrapy.utils import signal as _signal class SignalManager: - def __init__(self, sender=dispatcher.Anonymous): self.sender = sender @@ -21,7 +20,7 @@ class SignalManager: :param signal: the signal to connect to :type signal: object """ - kwargs.setdefault('sender', self.sender) + kwargs.setdefault("sender", self.sender) return dispatcher.connect(receiver, signal, **kwargs) def disconnect(self, receiver, signal, **kwargs): @@ -30,7 +29,7 @@ class SignalManager: opposite effect of the :meth:`connect` method, and the arguments are the same. """ - kwargs.setdefault('sender', self.sender) + kwargs.setdefault("sender", self.sender) return dispatcher.disconnect(receiver, signal, **kwargs) def send_catch_log(self, signal, **kwargs): @@ -40,7 +39,7 @@ class SignalManager: The keyword arguments are passed to the signal handlers (connected through the :meth:`connect` method). """ - kwargs.setdefault('sender', self.sender) + kwargs.setdefault("sender", self.sender) return _signal.send_catch_log(signal, **kwargs) def send_catch_log_deferred(self, signal, **kwargs): @@ -54,7 +53,7 @@ class SignalManager: The keyword arguments are passed to the signal handlers (connected through the :meth:`connect` method). """ - kwargs.setdefault('sender', self.sender) + kwargs.setdefault("sender", self.sender) return _signal.send_catch_log_deferred(signal, **kwargs) def disconnect_all(self, signal, **kwargs): @@ -64,5 +63,5 @@ class SignalManager: :param signal: the signal to disconnect from :type signal: object """ - kwargs.setdefault('sender', self.sender) + kwargs.setdefault("sender", self.sender) return _signal.disconnect_all(signal, **kwargs) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 04fda311f..02a451a2b 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -17,8 +17,8 @@ class SpiderLoader: """ def __init__(self, settings): - self.spider_modules = settings.getlist('SPIDER_MODULES') - self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY') + self.spider_modules = settings.getlist("SPIDER_MODULES") + self.warn_only = settings.getbool("SPIDER_LOADER_WARN_ONLY") self._spiders = {} self._found = defaultdict(list) self._load_all_spiders() @@ -26,11 +26,13 @@ class SpiderLoader: def _check_name_duplicates(self): dupes = [] for name, locations in self._found.items(): - dupes.extend([ - f" {cls} named {name!r} (in {mod})" - for mod, cls in locations - if len(locations) > 1 - ]) + dupes.extend( + [ + f" {cls} named {name!r} (in {mod})" + for mod, cls in locations + if len(locations) > 1 + ] + ) if dupes: dupes_string = "\n\n".join(dupes) @@ -81,8 +83,7 @@ class SpiderLoader: Return the list of spider names that can handle the given request. """ return [ - name for name, cls in self._spiders.items() - if cls.handles_request(request) + name for name, cls in self._spiders.items() if cls.handles_request(request) ] def list(self): diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 4c923b1b3..eadc7c6ab 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -12,7 +12,6 @@ logger = logging.getLogger(__name__) class DepthMiddleware: - def __init__(self, maxdepth, stats, verbose_stats=False, prio=1): self.maxdepth = maxdepth self.stats = stats @@ -22,9 +21,9 @@ class DepthMiddleware: @classmethod def from_crawler(cls, crawler): settings = crawler.settings - maxdepth = settings.getint('DEPTH_LIMIT') - verbose = settings.getbool('DEPTH_STATS_VERBOSE') - prio = settings.getint('DEPTH_PRIORITY') + maxdepth = settings.getint("DEPTH_LIMIT") + verbose = settings.getbool("DEPTH_STATS_VERBOSE") + prio = settings.getint("DEPTH_PRIORITY") return cls(maxdepth, crawler.stats, verbose, prio) def process_spider_output(self, response, result, spider): @@ -39,28 +38,26 @@ class DepthMiddleware: def _init_depth(self, response, spider): # base case (depth=0) - if 'depth' not in response.meta: - response.meta['depth'] = 0 + if "depth" not in response.meta: + response.meta["depth"] = 0 if self.verbose_stats: - self.stats.inc_value('request_depth_count/0', spider=spider) + self.stats.inc_value("request_depth_count/0", spider=spider) def _filter(self, request, response, spider): if not isinstance(request, Request): return True - depth = response.meta['depth'] + 1 - request.meta['depth'] = depth + depth = response.meta["depth"] + 1 + request.meta["depth"] = depth if self.prio: request.priority -= depth * self.prio if self.maxdepth and depth > self.maxdepth: logger.debug( "Ignoring link (depth > %(maxdepth)d): %(requrl)s ", - {'maxdepth': self.maxdepth, 'requrl': request.url}, - extra={'spider': spider} + {"maxdepth": self.maxdepth, "requrl": request.url}, + extra={"spider": spider}, ) return False if self.verbose_stats: - self.stats.inc_value(f'request_depth_count/{depth}', - spider=spider) - self.stats.max_value('request_depth_max', depth, - spider=spider) + self.stats.inc_value(f"request_depth_count/{depth}", spider=spider) + self.stats.max_value("request_depth_max", depth, spider=spider) return True diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 9861456de..0d3e5fe0b 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -19,39 +19,41 @@ class HttpError(IgnoreRequest): class HttpErrorMiddleware: - @classmethod def from_crawler(cls, crawler): return cls(crawler.settings) def __init__(self, settings): - self.handle_httpstatus_all = settings.getbool('HTTPERROR_ALLOW_ALL') - self.handle_httpstatus_list = settings.getlist('HTTPERROR_ALLOWED_CODES') + self.handle_httpstatus_all = settings.getbool("HTTPERROR_ALLOW_ALL") + self.handle_httpstatus_list = settings.getlist("HTTPERROR_ALLOWED_CODES") def process_spider_input(self, response, spider): if 200 <= response.status < 300: # common case return meta = response.meta - if meta.get('handle_httpstatus_all', False): + if meta.get("handle_httpstatus_all", False): return - if 'handle_httpstatus_list' in meta: - allowed_statuses = meta['handle_httpstatus_list'] + if "handle_httpstatus_list" in meta: + allowed_statuses = meta["handle_httpstatus_list"] elif self.handle_httpstatus_all: return else: - allowed_statuses = getattr(spider, 'handle_httpstatus_list', self.handle_httpstatus_list) + allowed_statuses = getattr( + spider, "handle_httpstatus_list", self.handle_httpstatus_list + ) if response.status in allowed_statuses: return - raise HttpError(response, 'Ignoring non-200 response') + raise HttpError(response, "Ignoring non-200 response") def process_spider_exception(self, response, exception, spider): if isinstance(exception, HttpError): - spider.crawler.stats.inc_value('httperror/response_ignored_count') + spider.crawler.stats.inc_value("httperror/response_ignored_count") spider.crawler.stats.inc_value( - f'httperror/response_ignored_status_count/{response.status}' + f"httperror/response_ignored_status_count/{response.status}" ) logger.info( "Ignoring response %(response)r: HTTP status code is not handled or not allowed", - {'response': response}, extra={'spider': spider}, + {"response": response}, + extra={"spider": spider}, ) return [] diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 98a186510..c57ec8d48 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -15,7 +15,6 @@ logger = logging.getLogger(__name__) class OffsiteMiddleware: - def __init__(self, stats): self.stats = stats @@ -43,22 +42,24 @@ class OffsiteMiddleware: self.domains_seen.add(domain) logger.debug( "Filtered offsite request to %(domain)r: %(request)s", - {'domain': domain, 'request': request}, extra={'spider': spider}) - self.stats.inc_value('offsite/domains', spider=spider) - self.stats.inc_value('offsite/filtered', spider=spider) + {"domain": domain, "request": request}, + extra={"spider": spider}, + ) + self.stats.inc_value("offsite/domains", spider=spider) + self.stats.inc_value("offsite/filtered", spider=spider) return False def should_follow(self, request, spider): regex = self.host_regex # hostname can be None for wrong urls (like javascript links) - host = urlparse_cached(request).hostname or '' + host = urlparse_cached(request).hostname or "" return bool(regex.search(host)) def get_host_regex(self, spider): """Override this method to implement a different offsite policy""" - allowed_domains = getattr(spider, 'allowed_domains', None) + allowed_domains = getattr(spider, "allowed_domains", None) if not allowed_domains: - return re.compile('') # allow all by default + return re.compile("") # allow all by default url_pattern = re.compile(r"^https?://.*$") port_pattern = re.compile(r":\d+$") domains = [] @@ -66,16 +67,20 @@ class OffsiteMiddleware: if domain is None: continue if url_pattern.match(domain): - message = ("allowed_domains accepts only domains, not URLs. " - f"Ignoring URL entry {domain} in allowed_domains.") + message = ( + "allowed_domains accepts only domains, not URLs. " + f"Ignoring URL entry {domain} in allowed_domains." + ) warnings.warn(message, URLWarning) elif port_pattern.search(domain): - message = ("allowed_domains accepts only domains without ports. " - f"Ignoring entry {domain} in allowed_domains.") + message = ( + "allowed_domains accepts only domains without ports. " + f"Ignoring entry {domain} in allowed_domains." + ) warnings.warn(message, PortWarning) else: domains.append(re.escape(domain)) - regex = fr'^(.*\.)?({"|".join(domains)})$' + regex = rf'^(.*\.)?({"|".join(domains)})$' return re.compile(regex) def spider_opened(self, spider): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 4a6c4de5e..a99b6315b 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -16,7 +16,12 @@ from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url -LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) +LOCAL_SCHEMES = ( + "about", + "blob", + "data", + "filesystem", +) POLICY_NO_REFERRER = "no-referrer" POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade" @@ -61,11 +66,13 @@ class ReferrerPolicy: """ if not url: return None - return strip_url(url, - strip_credentials=True, - strip_fragment=True, - strip_default_port=True, - origin_only=origin_only) + return strip_url( + url, + strip_credentials=True, + strip_fragment=True, + strip_default_port=True, + origin_only=origin_only, + ) def origin(self, url): """Return serialized origin (scheme, host, path) for a request or response URL.""" @@ -74,12 +81,12 @@ class ReferrerPolicy: def potentially_trustworthy(self, url): # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy parsed_url = urlparse(url) - if parsed_url.scheme in ('data',): + if parsed_url.scheme in ("data",): return False return self.tls_protected(url) def tls_protected(self, url): - return urlparse(url).scheme in ('https', 'ftps') + return urlparse(url).scheme in ("https", "ftps") class NoReferrerPolicy(ReferrerPolicy): @@ -90,6 +97,7 @@ class NoReferrerPolicy(ReferrerPolicy): is to be sent along with requests made from a particular request client to any origin. The header will be omitted entirely. """ + name: str = POLICY_NO_REFERRER def referrer(self, response_url, request_url): @@ -110,6 +118,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): This is a user agent's default behavior, if no policy is otherwise specified. """ + name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response_url, request_url): @@ -127,6 +136,7 @@ class SameOriginPolicy(ReferrerPolicy): Cross-origin requests, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ + name: str = POLICY_SAME_ORIGIN def referrer(self, response_url, request_url): @@ -143,6 +153,7 @@ class OriginPolicy(ReferrerPolicy): when making both same-origin requests and cross-origin requests from a particular request client. """ + name: str = POLICY_ORIGIN def referrer(self, response_url, request_url): @@ -162,11 +173,13 @@ class StrictOriginPolicy(ReferrerPolicy): on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ + name: str = POLICY_STRICT_ORIGIN def referrer(self, response_url, request_url): if ( - self.tls_protected(response_url) and self.potentially_trustworthy(request_url) + self.tls_protected(response_url) + and self.potentially_trustworthy(request_url) or not self.tls_protected(response_url) ): return self.origin_referrer(response_url) @@ -183,6 +196,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): is sent as referrer information when making cross-origin requests from a particular request client. """ + name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response_url, request_url): @@ -209,6 +223,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ + name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response_url, request_url): @@ -216,7 +231,8 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): if origin == self.origin(request_url): return self.stripped_referrer(response_url) if ( - self.tls_protected(response_url) and self.potentially_trustworthy(request_url) + self.tls_protected(response_url) + and self.potentially_trustworthy(request_url) or not self.tls_protected(response_url) ): return self.origin_referrer(response_url) @@ -235,6 +251,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): to insecure origins. Carefully consider the impact of setting such a policy for potentially sensitive documents. """ + name: str = POLICY_UNSAFE_URL def referrer(self, response_url, request_url): @@ -247,24 +264,28 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): with the addition that "Referer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ('file', 's3') + + NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3") name: str = POLICY_SCRAPY_DEFAULT -_policy_classes = {p.name: p for p in ( - NoReferrerPolicy, - NoReferrerWhenDowngradePolicy, - SameOriginPolicy, - OriginPolicy, - StrictOriginPolicy, - OriginWhenCrossOriginPolicy, - StrictOriginWhenCrossOriginPolicy, - UnsafeUrlPolicy, - DefaultReferrerPolicy, -)} +_policy_classes = { + p.name: p + for p in ( + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + SameOriginPolicy, + OriginPolicy, + StrictOriginPolicy, + OriginWhenCrossOriginPolicy, + StrictOriginWhenCrossOriginPolicy, + UnsafeUrlPolicy, + DefaultReferrerPolicy, + ) +} # Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string -_policy_classes[''] = NoReferrerWhenDowngradePolicy +_policy_classes[""] = NoReferrerWhenDowngradePolicy def _load_policy_class(policy, warning_only=False): @@ -288,16 +309,14 @@ def _load_policy_class(policy, warning_only=False): class RefererMiddleware: - def __init__(self, settings=None): self.default_policy = DefaultReferrerPolicy if settings is not None: - self.default_policy = _load_policy_class( - settings.get('REFERRER_POLICY')) + self.default_policy = _load_policy_class(settings.get("REFERRER_POLICY")) @classmethod def from_crawler(cls, crawler): - if not crawler.settings.getbool('REFERER_ENABLED'): + if not crawler.settings.getbool("REFERER_ENABLED"): raise NotConfigured mw = cls(crawler.settings) @@ -319,12 +338,12 @@ class RefererMiddleware: it is used if valid - otherwise, the policy from settings is used. """ - policy_name = request.meta.get('referrer_policy') + policy_name = request.meta.get("referrer_policy") if policy_name is None: if isinstance(resp_or_url, Response): - policy_header = resp_or_url.headers.get('Referrer-Policy') + policy_header = resp_or_url.headers.get("Referrer-Policy") if policy_header is not None: - policy_name = to_unicode(policy_header.decode('latin1')) + policy_name = to_unicode(policy_header.decode("latin1")) if policy_name is None: return self.default_policy() @@ -342,14 +361,14 @@ class RefererMiddleware: if isinstance(r, Request): referrer = self.policy(response, r).referrer(response.url, r.url) if referrer is not None: - r.headers.setdefault('Referer', referrer) + r.headers.setdefault("Referer", referrer) return r def request_scheduled(self, request, spider): # check redirected request to patch "Referer" header if necessary - redirected_urls = request.meta.get('redirect_urls', []) + redirected_urls = request.meta.get("redirect_urls", []) if redirected_urls: - request_referrer = request.headers.get('Referer') + request_referrer = request.headers.get("Referer") # we don't patch the referrer value if there is none if request_referrer is not None: # the request's referrer header value acts as a surrogate @@ -359,9 +378,10 @@ class RefererMiddleware: # the information is not available using this hook parent_url = safe_url_string(request_referrer) policy_referrer = self.policy(parent_url, request).referrer( - parent_url, request.url) + parent_url, request.url + ) if policy_referrer != request_referrer: if policy_referrer is None: - request.headers.pop('Referer') + request.headers.pop("Referer") else: - request.headers['Referer'] = policy_referrer + request.headers["Referer"] = policy_referrer diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 7ad64d2af..9a21379f9 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -13,13 +13,12 @@ logger = logging.getLogger(__name__) class UrlLengthMiddleware: - def __init__(self, maxlength): self.maxlength = maxlength @classmethod def from_settings(cls, settings): - maxlength = settings.getint('URLLENGTH_LIMIT') + maxlength = settings.getint("URLLENGTH_LIMIT") if not maxlength: raise NotConfigured return cls(maxlength) @@ -36,9 +35,11 @@ class UrlLengthMiddleware: if isinstance(request, Request) and len(request.url) > self.maxlength: logger.info( "Ignoring link (url length > %(maxlength)d): %(url)s ", - {'maxlength': self.maxlength, 'url': request.url}, - extra={'spider': spider} + {"maxlength": self.maxlength, "url": request.url}, + extra={"spider": spider}, + ) + spider.crawler.stats.inc_value( + "urllength/request_ignored_count", spider=spider ) - spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider) return False return True diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index a37af97ce..f8cac5458 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -28,16 +28,16 @@ class Spider(object_ref): def __init__(self, name=None, **kwargs): if name is not None: self.name = name - elif not getattr(self, 'name', None): + elif not getattr(self, "name", None): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) - if not hasattr(self, 'start_urls'): + if not hasattr(self, "start_urls"): self.start_urls = [] @property def logger(self): logger = logging.getLogger(self.name) - return logging.LoggerAdapter(logger, {'spider': self}) + return logging.LoggerAdapter(logger, {"spider": self}) def log(self, message, level=logging.DEBUG, **kw): """Log the given message at the given log level @@ -60,11 +60,12 @@ class Spider(object_ref): crawler.signals.connect(self.close, signals.spider_closed) def start_requests(self): - if not self.start_urls and hasattr(self, 'start_url'): + if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " "or empty (but found 'start_url' attribute instead, " - "did you miss an 's'?)") + "did you miss an 's'?)" + ) for url in self.start_urls: yield Request(url, dont_filter=True) @@ -72,11 +73,13 @@ class Spider(object_ref): return self.parse(response, **kwargs) def parse(self, response, **kwargs): - raise NotImplementedError(f'{self.__class__.__name__}.parse callback is not defined') + raise NotImplementedError( + f"{self.__class__.__name__}.parse callback is not defined" + ) @classmethod def update_settings(cls, settings): - settings.setdict(cls.custom_settings or {}, priority='spider') + settings.setdict(cls.custom_settings or {}, priority="spider") @classmethod def handles_request(cls, request): @@ -84,7 +87,7 @@ class Spider(object_ref): @staticmethod def close(spider, reason): - closed = getattr(spider, 'closed', None) + closed = getattr(spider, "closed", None) if callable(closed): return closed(reason) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index b514ed9ce..d75b455ae 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -34,7 +34,6 @@ _default_link_extractor = LinkExtractor() class Rule: - def __init__( self, link_extractor=None, @@ -95,19 +94,24 @@ class CrawlSpider(Spider): return seen = set() for rule_index, rule in enumerate(self._rules): - links = [lnk for lnk in rule.link_extractor.extract_links(response) - if lnk not in seen] + links = [ + lnk + for lnk in rule.link_extractor.extract_links(response) + if lnk not in seen + ] for link in rule.process_links(links): seen.add(link) request = self._build_request(rule_index, link) yield rule.process_request(request, response) def _callback(self, response, **cb_kwargs): - rule = self._rules[response.meta['rule']] - return self._parse_response(response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow) + rule = self._rules[response.meta["rule"]] + return self._parse_response( + response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow + ) def _errback(self, failure): - rule = self._rules[failure.request.meta['rule']] + rule = self._rules[failure.request.meta["rule"]] return self._handle_failure(failure, rule.errback) async def _parse_response(self, response, callback, cb_kwargs, follow=True): @@ -140,5 +144,7 @@ class CrawlSpider(Spider): @classmethod def from_crawler(cls, crawler, *args, **kwargs): spider = super().from_crawler(crawler, *args, **kwargs) - spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True) + spider._follow_links = crawler.settings.getbool( + "CRAWLSPIDER_FOLLOW_LINKS", True + ) return spider diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 79e12e030..b3c5ff01e 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -21,8 +21,8 @@ class XMLFeedSpider(Spider): use iternodes, since it's a faster and cleaner. """ - iterator = 'iternodes' - itertag = 'item' + iterator = "iternodes" + itertag = "item" namespaces = () def process_results(self, response, results): @@ -44,7 +44,7 @@ class XMLFeedSpider(Spider): def parse_node(self, response, selector): """This method must be overridden with your custom spider functionality""" - if hasattr(self, 'parse_item'): # backward compatibility + if hasattr(self, "parse_item"): # backward compatibility return self.parse_item(response, selector) raise NotImplementedError @@ -62,22 +62,24 @@ class XMLFeedSpider(Spider): yield result_item def _parse(self, response, **kwargs): - if not hasattr(self, 'parse_node'): - raise NotConfigured('You must define parse_node method in order to scrape this XML feed') + if not hasattr(self, "parse_node"): + raise NotConfigured( + "You must define parse_node method in order to scrape this XML feed" + ) response = self.adapt_response(response) - if self.iterator == 'iternodes': + if self.iterator == "iternodes": nodes = self._iternodes(response) - elif self.iterator == 'xml': - selector = Selector(response, type='xml') + elif self.iterator == "xml": + selector = Selector(response, type="xml") self._register_namespaces(selector) - nodes = selector.xpath(f'//{self.itertag}') - elif self.iterator == 'html': - selector = Selector(response, type='html') + nodes = selector.xpath(f"//{self.itertag}") + elif self.iterator == "html": + selector = Selector(response, type="html") self._register_namespaces(selector) - nodes = selector.xpath(f'//{self.itertag}') + nodes = selector.xpath(f"//{self.itertag}") else: - raise NotSupported('Unsupported node iterator') + raise NotSupported("Unsupported node iterator") return self.parse_nodes(response, nodes) @@ -100,8 +102,12 @@ class CSVFeedSpider(Spider): and the file's headers. """ - delimiter = None # When this is None, python's csv module's default delimiter is used - quotechar = None # When this is None, python's csv module's default quotechar is used + delimiter = ( + None # When this is None, python's csv module's default delimiter is used + ) + quotechar = ( + None # When this is None, python's csv module's default quotechar is used + ) headers = None def process_results(self, response, results): @@ -123,13 +129,17 @@ class CSVFeedSpider(Spider): process_results methods for pre and post-processing purposes. """ - for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar): + for row in csviter( + response, self.delimiter, self.headers, quotechar=self.quotechar + ): ret = iterate_spider_output(self.parse_row(response, row)) for result_item in self.process_results(response, ret): yield result_item def _parse(self, response, **kwargs): - if not hasattr(self, 'parse_row'): - raise NotConfigured('You must define parse_row method in order to scrape this CSV feed') + if not hasattr(self, "parse_row"): + raise NotConfigured( + "You must define parse_row method in order to scrape this CSV feed" + ) response = self.adapt_response(response) return self.parse_rows(response) diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index fe8c94e78..3cb215b0f 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -13,7 +13,7 @@ class InitSpider(Spider): """This method must be set as the callback of your last initialization request. See self.init_request() docstring for more info. """ - return self.__dict__.pop('_postinit_reqs') + return self.__dict__.pop("_postinit_reqs") def init_request(self): """This function should return one initialization request, with the diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 43d138753..a1734a3b1 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -13,8 +13,8 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): sitemap_urls = () - sitemap_rules = [('', 'parse')] - sitemap_follow = [''] + sitemap_rules = [("", "parse")] + sitemap_follow = [""] sitemap_alternate_links = False def __init__(self, *a, **kw): @@ -39,24 +39,27 @@ class SitemapSpider(Spider): yield entry def _parse_sitemap(self, response): - if response.url.endswith('/robots.txt'): + if response.url.endswith("/robots.txt"): for url in sitemap_urls_from_robots(response.text, base_url=response.url): yield Request(url, callback=self._parse_sitemap) else: body = self._get_sitemap_body(response) if body is None: - logger.warning("Ignoring invalid sitemap: %(response)s", - {'response': response}, extra={'spider': self}) + logger.warning( + "Ignoring invalid sitemap: %(response)s", + {"response": response}, + extra={"spider": self}, + ) return s = Sitemap(body) it = self.sitemap_filter(s) - if s.type == 'sitemapindex': + if s.type == "sitemapindex": for loc in iterloc(it, self.sitemap_alternate_links): if any(x.search(loc) for x in self._follow): yield Request(loc, callback=self._parse_sitemap) - elif s.type == 'urlset': + elif s.type == "urlset": for loc in iterloc(it, self.sitemap_alternate_links): for r, c in self._cbs: if r.search(loc): @@ -80,7 +83,7 @@ class SitemapSpider(Spider): # without actually being a .xml.gz file in the first place, # merely XML gzip-compressed on the fly, # in other word, here, we have plain XML - if response.url.endswith('.xml') or response.url.endswith('.xml.gz'): + if response.url.endswith(".xml") or response.url.endswith(".xml.gz"): return response.body @@ -92,8 +95,8 @@ def regex(x): def iterloc(it, alt=False): for d in it: - yield d['loc'] + yield d["loc"] # Also consider alternate URLs (xhtml:link rel="alternate") - if alt and 'alternate' in d: - yield from d['alternate'] + if alt and "alternate" in d: + yield from d["alternate"] diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 2fa84fc00..6afe0d636 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -15,9 +15,7 @@ from scrapy.utils.request import request_from_dict def _with_mkdir(queue_class): - class DirectoriesCreated(queue_class): - def __init__(self, path: Union[str, PathLike], *args, **kwargs): dirname = Path(path).parent if not dirname.exists(): @@ -28,9 +26,7 @@ def _with_mkdir(queue_class): def _serializable_queue(queue_class, serialize, deserialize): - class SerializableQueue(queue_class): - def push(self, obj): s = serialize(obj) super().push(s) @@ -50,7 +46,9 @@ def _serializable_queue(queue_class, serialize, deserialize): try: s = super().peek() except AttributeError as ex: - raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex + raise NotImplementedError( + "The underlying queue class does not implement 'peek'" + ) from ex if s: return deserialize(s) @@ -58,9 +56,7 @@ def _serializable_queue(queue_class, serialize, deserialize): def _scrapy_serialization_queue(queue_class): - class ScrapyRequestQueue(queue_class): - def __init__(self, crawler, key): self.spider = crawler.spider super().__init__(key) @@ -95,7 +91,6 @@ def _scrapy_serialization_queue(queue_class): def _scrapy_non_serialization_queue(queue_class): - class ScrapyRequestQueue(queue_class): @classmethod def from_crawler(cls, crawler, *args, **kwargs): @@ -111,7 +106,9 @@ def _scrapy_non_serialization_queue(queue_class): try: s = super().peek() except AttributeError as ex: - raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex + raise NotImplementedError( + "The underlying queue class does not implement 'peek'" + ) from ex return s return ScrapyRequestQueue @@ -127,24 +124,16 @@ def _pickle_serialize(obj): _PickleFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), - _pickle_serialize, - pickle.loads + _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads ) _PickleLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), - _pickle_serialize, - pickle.loads + _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads ) _MarshalFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), - marshal.dumps, - marshal.loads + _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads ) _MarshalLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), - marshal.dumps, - marshal.loads + _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads ) # public queue classes diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index ba7d1a6bf..4181c7a2f 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -8,9 +8,8 @@ logger = logging.getLogger(__name__) class StatsCollector: - def __init__(self, crawler): - self._dump = crawler.settings.getbool('STATS_DUMP') + self._dump = crawler.settings.getbool("STATS_DUMP") self._stats = {} def get_value(self, key, default=None, spider=None): @@ -43,8 +42,10 @@ class StatsCollector: def close_spider(self, spider, reason): if self._dump: - logger.info("Dumping Scrapy stats:\n" + pprint.pformat(self._stats), - extra={'spider': spider}) + logger.info( + "Dumping Scrapy stats:\n" + pprint.pformat(self._stats), + extra={"spider": spider}, + ) self._persist_stats(self._stats, spider) def _persist_stats(self, stats, spider): @@ -52,7 +53,6 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): - def __init__(self, crawler): super().__init__(crawler) self.spider_stats = {} @@ -62,7 +62,6 @@ class MemoryStatsCollector(StatsCollector): class DummyStatsCollector(StatsCollector): - def get_value(self, key, default=None, spider=None): return default diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index c84b51e8c..0505db343 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -9,7 +9,7 @@ async def collect_asyncgen(result: AsyncIterable) -> list: async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator: - """ Wraps an iterable (sync or async) into an async generator. """ + """Wraps an iterable (sync or async) into an async generator.""" if isinstance(it, AsyncIterable): async for r in it: yield r diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 86238c4cd..32bc2e38c 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -13,26 +13,28 @@ class Root(Resource): return self def render(self, request): - total = _getarg(request, b'total', 100, int) - show = _getarg(request, b'show', 10, int) + total = _getarg(request, b"total", 100, int) + show = _getarg(request, b"show", 10, int) nlist = [random.randint(1, total) for _ in range(show)] request.write(b"") args = request.args.copy() for nl in nlist: - args['n'] = nl + args["n"] = nl argstr = urlencode(args, doseq=True) - request.write(f"follow {nl}
" - .encode('utf8')) + request.write( + f"follow {nl}
".encode("utf8") + ) request.write(b"") - return b'' + return b"" def _getarg(request, name, default=None, type=str): return type(request.args[name][0]) if name in request.args else default -if __name__ == '__main__': +if __name__ == "__main__": from twisted.internet import reactor + root = Root() factory = Site(root) httpPort = reactor.listenTCP(8998, Site(root)) @@ -40,5 +42,6 @@ if __name__ == '__main__': def _print_listening(): httpHost = httpPort.getHost() print(f"Bench server at http://{httpHost.host}:{httpHost.port}") + reactor.callWhenRunning(_print_listening) reactor.run() diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 39a681001..085ee7d25 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -4,6 +4,7 @@ def is_botocore_available(): try: import botocore # noqa: F401 + return True except ImportError: return False diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 8ea42ce75..2f1569ab6 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -19,8 +19,10 @@ def build_component_list(compdict, custom=None, convert=update_classpath): def _check_components(complist): if len({convert(c) for c in complist}) != len(complist): - raise ValueError(f'Some paths in {complist!r} convert to the same object, ' - 'please update your settings') + raise ValueError( + f"Some paths in {complist!r} convert to the same object, " + "please update your settings" + ) def _map_keys(compdict): if isinstance(compdict, BaseSettings): @@ -28,10 +30,11 @@ def build_component_list(compdict, custom=None, convert=update_classpath): for k, v in compdict.items(): prio = compdict.getpriority(k) if compbs.getpriority(convert(k)) == prio: - raise ValueError(f'Some paths in {list(compdict.keys())!r} ' - 'convert to the same ' - 'object, please update your settings' - ) + raise ValueError( + f"Some paths in {list(compdict.keys())!r} " + "convert to the same " + "object, please update your settings" + ) else: compbs.set(convert(k), v, priority=prio) return compbs @@ -42,8 +45,10 @@ def build_component_list(compdict, custom=None, convert=update_classpath): """Fail if a value in the components dict is not a real number or None.""" for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): - raise ValueError(f'Invalid value {value} for component {name}, ' - 'please provide a real number or None instead') + raise ValueError( + f"Invalid value {value} for component {name}, " + "please provide a real number or None instead" + ) if isinstance(custom, (list, tuple)): _check_components(custom) @@ -61,30 +66,33 @@ def arglist_to_dict(arglist): """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a dict """ - return dict(x.split('=', 1) for x in arglist) + return dict(x.split("=", 1) for x in arglist) -def closest_scrapy_cfg(path: Union[str, os.PathLike] = '.', prevpath: Optional[Union[str, os.PathLike]] = None) -> str: +def closest_scrapy_cfg( + path: Union[str, os.PathLike] = ".", + prevpath: Optional[Union[str, os.PathLike]] = None, +) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents """ if prevpath is not None and str(path) == str(prevpath): - return '' + return "" path = Path(path).resolve() - cfgfile = path / 'scrapy.cfg' + cfgfile = path / "scrapy.cfg" if cfgfile.exists(): return str(cfgfile) return closest_scrapy_cfg(path.parent, path) -def init_env(project='default', set_syspath=True): +def init_env(project="default", set_syspath=True): """Initialize environment to use command-line tool from inside a project dir. This sets the Scrapy settings module and modifies the Python path to be able to locate the project module. """ cfg = get_config() - if cfg.has_option('settings', project): - os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project) + if cfg.has_option("settings", project): + os.environ["SCRAPY_SETTINGS_MODULE"] = cfg.get("settings", project) closest = closest_scrapy_cfg() if closest: projdir = str(Path(closest).parent) @@ -101,12 +109,14 @@ def get_config(use_closest=True): def get_sources(use_closest=True) -> List[str]: - xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or Path('~/.config').expanduser() + xdg_config_home = ( + os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser() + ) sources = [ - '/etc/scrapy.cfg', - r'c:\scrapy\scrapy.cfg', - str(Path(xdg_config_home) / 'scrapy.cfg'), - str(Path('~/.scrapy.cfg').expanduser()), + "/etc/scrapy.cfg", + r"c:\scrapy\scrapy.cfg", + str(Path(xdg_config_home) / "scrapy.cfg"), + str(Path("~/.scrapy.cfg").expanduser()), ] if use_closest: sources.append(closest_scrapy_cfg()) @@ -115,7 +125,7 @@ def get_sources(use_closest=True) -> List[str]: def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() - out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')) + out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT")) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) @@ -128,15 +138,19 @@ def feed_complete_default_values_from_settings(feed, settings): return out -def feed_process_params_from_cli(settings, output: List[str], output_format=None, - overwrite_output: Optional[List[str]] = None): +def feed_process_params_from_cli( + settings, + output: List[str], + output_format=None, + overwrite_output: Optional[List[str]] = None, +): """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary suitable to be used as the FEEDS setting. """ valid_output_formats = without_none_values( - settings.getwithbase('FEED_EXPORTERS') + settings.getwithbase("FEED_EXPORTERS") ).keys() def check_valid_format(output_format): @@ -179,28 +193,28 @@ def feed_process_params_from_cli(settings, output: List[str], output_format=None "scrapy crawl quotes -O quotes.json:json" ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) - return {output[0]: {'format': output_format}} + return {output[0]: {"format": output_format}} raise UsageError( - 'The -t command-line option cannot be used if multiple output ' - 'URIs are specified' + "The -t command-line option cannot be used if multiple output " + "URIs are specified" ) result: Dict[str, Dict[str, Any]] = {} for element in output: try: - feed_uri, feed_format = element.rsplit(':', 1) + feed_uri, feed_format = element.rsplit(":", 1) except ValueError: feed_uri = element - feed_format = Path(element).suffix.replace('.', '') + feed_format = Path(element).suffix.replace(".", "") else: - if feed_uri == '-': - feed_uri = 'stdout:' + if feed_uri == "-": + feed_uri = "stdout:" check_valid_format(feed_format) - result[feed_uri] = {'format': feed_format} + result[feed_uri] = {"format": feed_format} if overwrite: - result[feed_uri]['overwrite'] = True + result[feed_uri]["overwrite"] = True # FEEDS setting should take precedence over the matching CLI options - result.update(settings.getdict('FEEDS')) + result.update(settings.getdict("FEEDS")) return result diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 1bc0bd45f..b64098d7b 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -2,7 +2,7 @@ from functools import wraps from collections import OrderedDict -def _embed_ipython_shell(namespace={}, banner=''): +def _embed_ipython_shell(namespace={}, banner=""): """Start an IPython Shell""" try: from IPython.terminal.embed import InteractiveShellEmbed @@ -12,7 +12,7 @@ def _embed_ipython_shell(namespace={}, banner=''): from IPython.frontend.terminal.ipapp import load_default_config @wraps(_embed_ipython_shell) - def wrapper(namespace=namespace, banner=''): + def wrapper(namespace=namespace, banner=""): config = load_default_config() # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports @@ -20,55 +20,64 @@ def _embed_ipython_shell(namespace={}, banner=''): # on repeated breaks like with inspect_response() InteractiveShellEmbed.clear_instance() shell = InteractiveShellEmbed.instance( - banner1=banner, user_ns=namespace, config=config) + banner1=banner, user_ns=namespace, config=config + ) shell() + return wrapper -def _embed_bpython_shell(namespace={}, banner=''): +def _embed_bpython_shell(namespace={}, banner=""): """Start a bpython shell""" import bpython @wraps(_embed_bpython_shell) - def wrapper(namespace=namespace, banner=''): + def wrapper(namespace=namespace, banner=""): bpython.embed(locals_=namespace, banner=banner) + return wrapper -def _embed_ptpython_shell(namespace={}, banner=''): +def _embed_ptpython_shell(namespace={}, banner=""): """Start a ptpython shell""" import ptpython.repl @wraps(_embed_ptpython_shell) - def wrapper(namespace=namespace, banner=''): + def wrapper(namespace=namespace, banner=""): print(banner) ptpython.repl.embed(locals=namespace) + return wrapper -def _embed_standard_shell(namespace={}, banner=''): +def _embed_standard_shell(namespace={}, banner=""): """Start a standard python shell""" import code + try: # readline module is only available on unix systems import readline except ImportError: pass else: import rlcompleter # noqa: F401 + readline.parse_and_bind("tab:complete") @wraps(_embed_standard_shell) - def wrapper(namespace=namespace, banner=''): + def wrapper(namespace=namespace, banner=""): code.interact(banner=banner, local=namespace) + return wrapper -DEFAULT_PYTHON_SHELLS = OrderedDict([ - ('ptpython', _embed_ptpython_shell), - ('ipython', _embed_ipython_shell), - ('bpython', _embed_bpython_shell), - ('python', _embed_standard_shell), -]) +DEFAULT_PYTHON_SHELLS = OrderedDict( + [ + ("ptpython", _embed_ptpython_shell), + ("ipython", _embed_ipython_shell), + ("bpython", _embed_bpython_shell), + ("python", _embed_standard_shell), + ] +) def get_shell_embed_func(shells=None, known_shells=None): @@ -89,7 +98,7 @@ def get_shell_embed_func(shells=None, known_shells=None): continue -def start_python_console(namespace=None, banner='', shells=None): +def start_python_console(namespace=None, banner="", shells=None): """Start Python console bound to the given namespace. Readline support and tab completion will be used on Unix, if available. """ diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 74f82ad75..3175e5fdc 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -9,52 +9,54 @@ from w3lib.http import basic_auth_header class CurlParser(argparse.ArgumentParser): def error(self, message): - error_msg = f'There was an error parsing the curl command: {message}' + error_msg = f"There was an error parsing the curl command: {message}" raise ValueError(error_msg) curl_parser = CurlParser() -curl_parser.add_argument('url') -curl_parser.add_argument('-H', '--header', dest='headers', action='append') -curl_parser.add_argument('-X', '--request', dest='method') -curl_parser.add_argument('-d', '--data', '--data-raw', dest='data') -curl_parser.add_argument('-u', '--user', dest='auth') +curl_parser.add_argument("url") +curl_parser.add_argument("-H", "--header", dest="headers", action="append") +curl_parser.add_argument("-X", "--request", dest="method") +curl_parser.add_argument("-d", "--data", "--data-raw", dest="data") +curl_parser.add_argument("-u", "--user", dest="auth") safe_to_ignore_arguments = [ - ['--compressed'], + ["--compressed"], # `--compressed` argument is not safe to ignore, but it's included here # because the `HttpCompressionMiddleware` is enabled by default - ['-s', '--silent'], - ['-v', '--verbose'], - ['-#', '--progress-bar'] + ["-s", "--silent"], + ["-v", "--verbose"], + ["-#", "--progress-bar"], ] for argument in safe_to_ignore_arguments: - curl_parser.add_argument(*argument, action='store_true') + curl_parser.add_argument(*argument, action="store_true") def _parse_headers_and_cookies(parsed_args): headers = [] cookies = {} for header in parsed_args.headers or (): - name, val = header.split(':', 1) + name, val = header.split(":", 1) name = name.strip() val = val.strip() - if name.title() == 'Cookie': + if name.title() == "Cookie": for name, morsel in SimpleCookie(val).items(): cookies[name] = morsel.value else: headers.append((name, val)) if parsed_args.auth: - user, password = parsed_args.auth.split(':', 1) - headers.append(('Authorization', basic_auth_header(user, password))) + user, password = parsed_args.auth.split(":", 1) + headers.append(("Authorization", basic_auth_header(user, password))) return headers, cookies -def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = True) -> dict: +def curl_to_request_kwargs( + curl_command: str, ignore_unknown_options: bool = True +) -> dict: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command @@ -66,7 +68,7 @@ def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = Tru curl_args = split(curl_command) - if curl_args[0] != 'curl': + if curl_args[0] != "curl": raise ValueError('A curl command must start with "curl"') parsed_args, argv = curl_parser.parse_known_args(curl_args[1:]) @@ -84,23 +86,23 @@ def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = Tru # needs the scheme to work parsed_url = urlparse(url) if not parsed_url.scheme: - url = 'http://' + url + url = "http://" + url - method = parsed_args.method or 'GET' + method = parsed_args.method or "GET" - result = {'method': method.upper(), 'url': url} + result = {"method": method.upper(), "url": url} headers, cookies = _parse_headers_and_cookies(parsed_args) if headers: - result['headers'] = headers + result["headers"] = headers if cookies: - result['cookies'] = cookies + result["cookies"] = cookies if parsed_args.data: - result['body'] = parsed_args.data + result["body"] = parsed_args.data if not parsed_args.method: # if the "data" is specified but the "method" is not specified, # the default method is 'POST' - result['method'] = 'POST' + result["method"] = "POST" return result diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 47df8a717..10497e9ed 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -30,10 +30,12 @@ class CaselessDict(dict): def __contains__(self, key): return dict.__contains__(self, self.normkey(key)) + has_key = __contains__ def __copy__(self): return self.__class__(self) + copy = __copy__ def normkey(self, key): diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index fef3882cb..4e684645b 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -19,6 +19,7 @@ def deprecated(use_instead=None): message += f" Use {use_instead} instead." warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) return func(*args, **kwargs) + return wrapped if callable(use_instead): @@ -29,9 +30,11 @@ def deprecated(use_instead=None): def defers(func): """Decorator to make sure a function always returns a deferred""" + @wraps(func) def wrapped(*a, **kw): return defer.maybeDeferred(func, *a, **kw) + return wrapped @@ -39,7 +42,9 @@ def inthread(func): """Decorator to call a function in a thread and return a deferred with the result """ + @wraps(func) def wrapped(*a, **kw): return threads.deferToThread(func, *a, **kw) + return wrapped diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index ddacfaa49..4af2a647a 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -16,7 +16,7 @@ from typing import ( Iterator, List, Optional, - Union + Union, ) from twisted.internet import defer @@ -26,7 +26,10 @@ from twisted.python import failure from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest -from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy +from scrapy.utils.reactor import ( + is_asyncio_reactor_installed, + get_asyncio_event_loop_policy, +) def defer_fail(_failure: Failure) -> Deferred: @@ -37,6 +40,7 @@ def defer_fail(_failure: Failure) -> Deferred: before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor + d = Deferred() reactor.callLater(0.1, d.errback, _failure) return d @@ -50,6 +54,7 @@ def defer_succeed(result) -> Deferred: before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor + d = Deferred() reactor.callLater(0.1, d.callback, result) return d @@ -80,7 +85,9 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: return defer_result(result) -def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> DeferredList: +def parallel( + iterable: Iterable, count: int, callable: Callable, *args, **named +) -> DeferredList: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -92,7 +99,7 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) class _AsyncCooperatorAdapter(Iterator): - """ A class that wraps an async iterable into a normal iterator suitable + """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more generic interface. @@ -136,7 +143,14 @@ class _AsyncCooperatorAdapter(Iterator): Cooperator/CooperativeTask and use it instead of this adapter to achieve the same goal. """ - def __init__(self, aiterable: AsyncIterable, callable: Callable, *callable_args, **callable_kwargs): + + def __init__( + self, + aiterable: AsyncIterable, + callable: Callable, + *callable_args, + **callable_kwargs + ): self.aiterator = aiterable.__aiter__() self.callable = callable self.callable_args = callable_args @@ -186,8 +200,10 @@ class _AsyncCooperatorAdapter(Iterator): return d -def parallel_async(async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named) -> DeferredList: - """ Like parallel but for async iterators """ +def parallel_async( + async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named +) -> DeferredList: + """Like parallel but for async iterators""" coop = Cooperator() work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) dl = DeferredList([coop.coiterate(work) for _ in range(count)]) @@ -203,14 +219,19 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: return d -def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> Deferred: +def process_chain_both( + callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw +) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" d = Deferred() for cb, eb in zip(callbacks, errbacks): d.addCallbacks( - callback=cb, errback=eb, - callbackArgs=a, callbackKeywords=kw, - errbackArgs=a, errbackKeywords=kw, + callback=cb, + errback=eb, + callbackArgs=a, + callbackKeywords=kw, + errbackArgs=a, + errbackKeywords=kw, ) if isinstance(input, failure.Failure): d.errback(input) @@ -243,7 +264,9 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: errback(failure.Failure(), *a, **kw) -async def aiter_errback(aiterable: AsyncIterable, errback: Callable, *a, **kw) -> AsyncGenerator: +async def aiter_errback( + aiterable: AsyncIterable, errback: Callable, *a, **kw +) -> AsyncGenerator: """Wraps an async iterable calling an errback if an error is caught while iterating it. Similar to scrapy.utils.defer.iter_errback() """ @@ -273,19 +296,21 @@ def deferred_from_coro(o) -> Any: def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: - """ Converts a coroutine function into a function that returns a Deferred. + """Converts a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. This is useful for callback chains, as callback functions are called with the previous callback result. """ + @wraps(coro_f) def f(*coro_args, **coro_kwargs): return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) + return f def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: - """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ + """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) except: # noqa: E722 diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index a0c83f9f1..4757fef0a 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -6,13 +6,14 @@ from typing import List, Tuple from scrapy.exceptions import ScrapyDeprecationWarning -def attribute(obj, oldattr, newattr, version='0.12'): +def attribute(obj, oldattr, newattr, version="0.12"): cname = obj.__class__.__name__ warnings.warn( f"{cname}.{oldattr} attribute is deprecated and will be no longer supported " f"in Scrapy {version}, use {cname}.{newattr} attribute instead", ScrapyDeprecationWarning, - stacklevel=3) + stacklevel=3, + ) def create_deprecated_class( @@ -24,7 +25,7 @@ def create_deprecated_class( old_class_path=None, new_class_path=None, subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.", - instance_warn_message="{cls} is deprecated, instantiate {new} instead." + instance_warn_message="{cls} is deprecated, instantiate {new} instead.", ): """ Return a "deprecated" class that causes its subclasses to issue a warning. @@ -67,11 +68,13 @@ def create_deprecated_class( old = meta.deprecated_class if old in bases and not (warn_once and meta.warned_on_subclass): meta.warned_on_subclass = True - msg = subclass_warn_message.format(cls=_clspath(cls), - old=_clspath(old, old_class_path), - new=_clspath(new_class, new_class_path)) + msg = subclass_warn_message.format( + cls=_clspath(cls), + old=_clspath(old, old_class_path), + new=_clspath(new_class, new_class_path), + ) if warn_once: - msg += ' (warning only on first subclass, there may be others)' + msg += " (warning only on first subclass, there may be others)" warnings.warn(msg, warn_category, stacklevel=2) super().__init__(name, bases, clsdict_) @@ -79,8 +82,7 @@ def create_deprecated_class( # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks # for implementation details def __instancecheck__(cls, inst): - return any(cls.__subclasscheck__(c) - for c in (type(inst), inst.__class__)) + return any(cls.__subclasscheck__(c) for c in (type(inst), inst.__class__)) def __subclasscheck__(cls, sub): if cls is not DeprecatedClass.deprecated_class: @@ -93,14 +95,16 @@ def create_deprecated_class( if not inspect.isclass(sub): raise TypeError("issubclass() arg 1 must be a class") - mro = getattr(sub, '__mro__', ()) + mro = getattr(sub, "__mro__", ()) return any(c in {cls, new_class} for c in mro) def __call__(cls, *args, **kwargs): old = DeprecatedClass.deprecated_class if cls is old: - msg = instance_warn_message.format(cls=_clspath(cls, old_class_path), - new=_clspath(new_class, new_class_path)) + msg = instance_warn_message.format( + cls=_clspath(cls, old_class_path), + new=_clspath(new_class, new_class_path), + ) warnings.warn(msg, warn_category, stacklevel=2) return super().__call__(*args, **kwargs) @@ -124,7 +128,7 @@ def create_deprecated_class( def _clspath(cls, forced=None): if forced is not None: return forced - return f'{cls.__module__}.{cls.__name__}' + return f"{cls.__module__}.{cls.__name__}" DEPRECATION_RULES: List[Tuple[str, str]] = [] @@ -135,8 +139,10 @@ def update_classpath(path): for prefix, replacement in DEPRECATION_RULES: if isinstance(path, str) and path.startswith(prefix): new_path = path.replace(prefix, replacement, 1) - warnings.warn(f"`{path}` class is deprecated, use `{new_path}` instead", - ScrapyDeprecationWarning) + warnings.warn( + f"`{path}` class is deprecated, use `{new_path}` instead", + ScrapyDeprecationWarning, + ) return new_path return path diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 64969cb13..f6dceb87f 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -37,11 +37,12 @@ def _colorize(text, colorize=True): else: from pygments.formatters import TerminalFormatter from pygments.lexers import PythonLexer + return highlight(text, PythonLexer(), TerminalFormatter()) def pformat(obj, *args, **kwargs): - return _colorize(pformat_(obj), kwargs.pop('colorize', True)) + return _colorize(pformat_(obj), kwargs.pop("colorize", True)) def pprint(obj, *args, **kwargs): diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 775b25ea8..9dbb4180f 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -18,8 +18,8 @@ def ftp_makedirs_cwd(ftp, path, first_call=True): def ftp_store_file( - *, path, file, host, port, - username, password, use_active_mode=False, overwrite=True): + *, path, file, host, port, username, password, use_active_mode=False, overwrite=True +): """Opens a FTP connection with passed credentials,sets current directory to the directory extracted from given path, then uploads the file to server """ @@ -31,6 +31,6 @@ def ftp_store_file( file.seek(0) dirname, filename = posixpath.split(path) ftp_makedirs_cwd(ftp, dirname) - command = 'STOR' if overwrite else 'APPE' - ftp.storbinary(f'{command} {filename}', file) + command = "STOR" if overwrite else "APPE" + ftp.storbinary(f"{command} {filename}", file) file.close() diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 0810e1f1d..e5df34d2e 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -10,7 +10,7 @@ def gunzip(data): """ f = GzipFile(fileobj=BytesIO(data)) output_list = [] - chunk = b'.' + chunk = b"." while chunk: try: chunk = f.read1(8196) @@ -20,15 +20,15 @@ def gunzip(data): # see issue 87 about catching struct.error # some pages are quite small so output_list is empty and f.extrabuf # contains the whole page content - if output_list or getattr(f, 'extrabuf', None): + if output_list or getattr(f, "extrabuf", None): try: - output_list.append(f.extrabuf[-f.extrasize:]) + output_list.append(f.extrabuf[-f.extrasize :]) finally: break else: raise - return b''.join(output_list) + return b"".join(output_list) def gzip_magic_number(response): - return response.body[:3] == b'\x1f\x8b\x08' + return response.body[:3] == b"\x1f\x8b\x08" diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index a90f1d278..540035ca9 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -7,7 +7,9 @@ from weakref import WeakKeyDictionary from scrapy.http import Request, Response -_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary() +_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = ( + WeakKeyDictionary() +) def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult: diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index da77ca46f..7d52d35c9 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -22,60 +22,62 @@ def xmliter(obj, nodename): """ nodename_patt = re.escape(nodename) - DOCUMENT_HEADER_RE = re.compile(r'<\?xml[^>]+>\s*', re.S) - HEADER_END_RE = re.compile(fr'<\s*/{nodename_patt}\s*>', re.S) - END_TAG_RE = re.compile(r'<\s*/([^\s>]+)\s*>', re.S) - NAMESPACE_RE = re.compile(r'((xmlns[:A-Za-z]*)=[^>\s]+)', re.S) + DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) + HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S) + END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S) + NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) text = _body_or_str(obj) document_header = re.search(DOCUMENT_HEADER_RE, text) - document_header = document_header.group().strip() if document_header else '' + document_header = document_header.group().strip() if document_header else "" header_end_idx = re_rsearch(HEADER_END_RE, text) - header_end = text[header_end_idx[1]:].strip() if header_end_idx else '' + header_end = text[header_end_idx[1] :].strip() if header_end_idx else "" namespaces = {} if header_end: for tagname in reversed(re.findall(END_TAG_RE, header_end)): - tag = re.search(fr'<\s*{tagname}.*?xmlns[:=][^>]*>', text[:header_end_idx[1]], re.S) + tag = re.search( + rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S + ) if tag: - namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group())) + namespaces.update( + reversed(x) for x in re.findall(NAMESPACE_RE, tag.group()) + ) - r = re.compile(fr'<{nodename_patt}[\s>].*?', re.DOTALL) + r = re.compile(rf"<{nodename_patt}[\s>].*?", re.DOTALL) for match in r.finditer(text): nodetext = ( document_header + match.group().replace( - nodename, - f'{nodename} {" ".join(namespaces.values())}', - 1 + nodename, f'{nodename} {" ".join(namespaces.values())}', 1 ) + header_end ) - yield Selector(text=nodetext, type='xml') + yield Selector(text=nodetext, type="xml") -def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): +def xmliter_lxml(obj, nodename, namespace=None, prefix="x"): from lxml import etree + reader = _StreamReader(obj) - tag = f'{{{namespace}}}{nodename}' if namespace else nodename + tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) - selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename) + selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: - nodetext = etree.tostring(node, encoding='unicode') + nodetext = etree.tostring(node, encoding="unicode") node.clear() - xs = Selector(text=nodetext, type='xml') + xs = Selector(text=nodetext, type="xml") if namespace: xs.register_namespace(prefix, namespace) yield xs.xpath(selxpath)[0] class _StreamReader: - def __init__(self, obj): self._ptr = 0 if isinstance(obj, Response): self._text, self.encoding = obj.body, obj.encoding else: - self._text, self.encoding = obj, 'utf-8' + self._text, self.encoding = obj, "utf-8" self._is_unicode = isinstance(self._text, str) def read(self, n=65535): @@ -90,11 +92,11 @@ class _StreamReader: def _read_unicode(self, n=65535): s, e = self._ptr, self._ptr + n self._ptr = e - return self._text[s:e].encode('utf-8') + return self._text[s:e].encode("utf-8") def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): - """ Returns an iterator of dictionaries from the given csv object + """Returns an iterator of dictionaries from the given csv object obj can be: - a Response object @@ -109,7 +111,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): quotechar is the character used to enclosure fields on the given obj. """ - encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8' + encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8" def row_to_unicode(row_): return [to_unicode(field, encoding) for field in row_] @@ -133,10 +135,15 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): for row in csv_r: row = row_to_unicode(row) if len(row) != len(headers): - logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, " - "should be: %(csvheader)d)", - {'csvlnum': csv_r.line_num, 'csvrow': len(row), - 'csvheader': len(headers)}) + logger.warning( + "ignoring row %(csvlnum)d (length: %(csvrow)d, " + "should be: %(csvheader)d)", + { + "csvlnum": csv_r.line_num, + "csvrow": len(row), + "csvheader": len(headers), + }, + ) continue yield dict(zip(headers, row)) @@ -153,7 +160,7 @@ def _body_or_str(obj, unicode=True): return obj.body if isinstance(obj, TextResponse): return obj.text - return obj.body.decode('utf-8') + return obj.body.decode("utf-8") if isinstance(obj, str): - return obj if unicode else obj.encode('utf-8') - return obj.decode('utf-8') if unicode else obj + return obj if unicode else obj.encode("utf-8") + return obj.decode("utf-8") if unicode else obj diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index a65f92e95..858affc03 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -5,7 +5,7 @@ from scrapy.settings import BaseSettings def job_dir(settings: BaseSettings) -> Optional[str]: - path = settings['JOBDIR'] + path = settings["JOBDIR"] if path and not Path(path).exists(): Path(path).mkdir(parents=True) return path diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 78e302d19..fea4deb4b 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -37,25 +37,25 @@ class TopLevelFormatter(logging.Filter): self.loggers = loggers or [] def filter(self, record): - if any(record.name.startswith(logger + '.') for logger in self.loggers): - record.name = record.name.split('.', 1)[0] + if any(record.name.startswith(logger + ".") for logger in self.loggers): + record.name = record.name.split(".", 1)[0] return True DEFAULT_LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'loggers': { - 'hpack': { - 'level': 'ERROR', + "version": 1, + "disable_existing_loggers": False, + "loggers": { + "hpack": { + "level": "ERROR", }, - 'scrapy': { - 'level': 'DEBUG', + "scrapy": { + "level": "DEBUG", }, - 'twisted': { - 'level': 'ERROR', + "twisted": { + "level": "ERROR", }, - } + }, } @@ -87,7 +87,7 @@ def configure_logging(settings=None, install_root_handler=True): # Route warnings through python logging logging.captureWarnings(True) - observer = twisted_log.PythonLoggingObserver('twisted') + observer = twisted_log.PythonLoggingObserver("twisted") observer.start() dictConfig(DEFAULT_LOGGING) @@ -95,8 +95,8 @@ def configure_logging(settings=None, install_root_handler=True): if isinstance(settings, dict) or settings is None: settings = Settings(settings) - if settings.getbool('LOG_STDOUT'): - sys.stdout = StreamLogger(logging.getLogger('stdout')) + if settings.getbool("LOG_STDOUT"): + sys.stdout = StreamLogger(logging.getLogger("stdout")) if install_root_handler: install_scrapy_root_handler(settings) @@ -105,8 +105,10 @@ def configure_logging(settings=None, install_root_handler=True): def install_scrapy_root_handler(settings): global _scrapy_root_handler - if (_scrapy_root_handler is not None - and _scrapy_root_handler in logging.root.handlers): + if ( + _scrapy_root_handler is not None + and _scrapy_root_handler in logging.root.handlers + ): logging.root.removeHandler(_scrapy_root_handler) logging.root.setLevel(logging.NOTSET) _scrapy_root_handler = _get_handler(settings) @@ -121,43 +123,46 @@ _scrapy_root_handler = None def _get_handler(settings): - """ Return a log handler object according to settings """ - filename = settings.get('LOG_FILE') + """Return a log handler object according to settings""" + filename = settings.get("LOG_FILE") if filename: - mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w' - encoding = settings.get('LOG_ENCODING') + mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w" + encoding = settings.get("LOG_ENCODING") handler = logging.FileHandler(filename, mode=mode, encoding=encoding) - elif settings.getbool('LOG_ENABLED'): + elif settings.getbool("LOG_ENABLED"): handler = logging.StreamHandler() else: handler = logging.NullHandler() formatter = logging.Formatter( - fmt=settings.get('LOG_FORMAT'), - datefmt=settings.get('LOG_DATEFORMAT') + fmt=settings.get("LOG_FORMAT"), datefmt=settings.get("LOG_DATEFORMAT") ) handler.setFormatter(formatter) - handler.setLevel(settings.get('LOG_LEVEL')) - if settings.getbool('LOG_SHORT_NAMES'): - handler.addFilter(TopLevelFormatter(['scrapy'])) + handler.setLevel(settings.get("LOG_LEVEL")) + if settings.getbool("LOG_SHORT_NAMES"): + handler.addFilter(TopLevelFormatter(["scrapy"])) return handler def log_scrapy_info(settings: Settings) -> None: - logger.info("Scrapy %(version)s started (bot: %(bot)s)", - {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) + logger.info( + "Scrapy %(version)s started (bot: %(bot)s)", + {"version": scrapy.__version__, "bot": settings["BOT_NAME"]}, + ) versions = [ f"{name} {version}" for name, version in scrapy_components_versions() if name != "Scrapy" ] - logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) + logger.info("Versions: %(versions)s", {"versions": ", ".join(versions)}) def log_reactor_info() -> None: from twisted.internet import reactor + logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) from twisted.internet import asyncioreactor + if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor): logger.debug( "Using asyncio event loop: %s.%s", @@ -172,10 +177,11 @@ class StreamLogger: Taken from: https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ """ + def __init__(self, logger, log_level=logging.INFO): self.logger = logger self.log_level = log_level - self.linebuf = '' + self.linebuf = "" def write(self, buf): for line in buf.rstrip().splitlines(): @@ -194,7 +200,7 @@ class LogCounterHandler(logging.Handler): self.crawler = crawler def emit(self, record): - sname = f'log_count/{record.levelname}' + sname = f"log_count/{record.levelname}" self.crawler.stats.inc_value(sname) @@ -204,19 +210,20 @@ def logformatter_adapter(logkws): and adapts it into a tuple of positional arguments for logger.log calls, handling backward compatibility as well. """ - if not {'level', 'msg', 'args'} <= set(logkws): - warnings.warn('Missing keys in LogFormatter method', - ScrapyDeprecationWarning) + if not {"level", "msg", "args"} <= set(logkws): + warnings.warn("Missing keys in LogFormatter method", ScrapyDeprecationWarning) - if 'format' in logkws: - warnings.warn('`format` key in LogFormatter methods has been ' - 'deprecated, use `msg` instead', - ScrapyDeprecationWarning) + if "format" in logkws: + warnings.warn( + "`format` key in LogFormatter methods has been " + "deprecated, use `msg` instead", + ScrapyDeprecationWarning, + ) - level = logkws.get('level', logging.INFO) - message = logkws.get('format', logkws.get('msg')) + level = logkws.get("level", logging.INFO) + message = logkws.get("format", logkws.get("msg")) # NOTE: This also handles 'args' being an empty dict, that case doesn't # play well in logger.log calls - args = logkws if not logkws.get('args') else logkws['args'] + args = logkws if not logkws.get("args") else logkws["args"] return (level, message, args) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 1e0342ace..dfd2f767c 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -30,7 +30,7 @@ def arg_to_iter(arg): """ if arg is None: return [] - if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'): + if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"): return arg return [arg] @@ -48,15 +48,16 @@ def load_object(path): if not isinstance(path, str): if callable(path): return path - raise TypeError("Unexpected argument type, expected string " - f"or object, got: {type(path)}") + raise TypeError( + "Unexpected argument type, expected string " f"or object, got: {type(path)}" + ) try: - dot = path.rindex('.') + dot = path.rindex(".") except ValueError: raise ValueError(f"Error loading object '{path}': not a full path") - module, name = path[:dot], path[dot + 1:] + module, name = path[:dot], path[dot + 1 :] mod = import_module(module) try: @@ -78,9 +79,9 @@ def walk_modules(path): mods = [] mod = import_module(path) mods.append(mod) - if hasattr(mod, '__path__'): + if hasattr(mod, "__path__"): for _, subpath, ispkg in iter_modules(mod.__path__): - fullpath = path + '.' + subpath + fullpath = path + "." + subpath if ispkg: mods += walk_modules(fullpath) else: @@ -89,7 +90,7 @@ def walk_modules(path): return mods -def extract_regex(regex, text, encoding='utf-8'): +def extract_regex(regex, text, encoding="utf-8"): """Extract a list of unicode strings from the given text/encoding using the following policies: * if the regex contains a named group called "extract" that will be returned @@ -99,22 +100,23 @@ def extract_regex(regex, text, encoding='utf-8'): warnings.warn( "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.", ScrapyDeprecationWarning, - stacklevel=2 + stacklevel=2, ) if isinstance(regex, str): regex = re.compile(regex, re.UNICODE) try: - strings = [regex.search(text).group('extract')] # named group + strings = [regex.search(text).group("extract")] # named group except Exception: - strings = regex.findall(text) # full regex or numbered groups + strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) if isinstance(text, str): - return [replace_entities(s, keep=['lt', 'amp']) for s in strings] - return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp']) - for s in strings] + return [replace_entities(s, keep=["lt", "amp"]) for s in strings] + return [ + replace_entities(to_unicode(s, encoding), keep=["lt", "amp"]) for s in strings + ] def md5sum(file): @@ -136,7 +138,7 @@ def md5sum(file): def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" - return rel is not None and 'nofollow' in rel.replace(',', ' ').split() + return rel is not None and "nofollow" in rel.replace(",", " ").split() def create_instance(objcls, settings, crawler, *args, **kwargs): @@ -160,15 +162,15 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): if crawler is None: raise ValueError("Specify at least one of settings and crawler.") settings = crawler.settings - if crawler and hasattr(objcls, 'from_crawler'): + if crawler and hasattr(objcls, "from_crawler"): instance = objcls.from_crawler(crawler, *args, **kwargs) - method_name = 'from_crawler' - elif hasattr(objcls, 'from_settings'): + method_name = "from_crawler" + elif hasattr(objcls, "from_settings"): instance = objcls.from_settings(settings, *args, **kwargs) - method_name = 'from_settings' + method_name = "from_settings" else: instance = objcls(*args, **kwargs) - method_name = '__new__' + method_name = "__new__" if instance is None: raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") return instance @@ -221,7 +223,9 @@ def is_generator_with_return_value(callable): def returns_none(return_node): value = return_node.value - return value is None or isinstance(value, ast.NameConstant) and value.value is None + return ( + value is None or isinstance(value, ast.NameConstant) and value.value is None + ) if inspect.isgeneratorfunction(callable): func = callable @@ -256,8 +260,8 @@ def warn_on_generator_with_return_value(spider, callable): warnings.warn( f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' 'a generator and includes a "return" statement with a value ' - 'different than None. This could lead to unexpected behaviour. Please see ' - 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' + "different than None. This could lead to unexpected behaviour. Please see " + "https://docs.python.org/3/reference/simple_stmts.html#the-return-statement " 'for details about the semantics of the "return" statement within generators', stacklevel=2, ) @@ -265,9 +269,9 @@ def warn_on_generator_with_return_value(spider, callable): callable_name = spider.__class__.__name__ + "." + callable.__name__ warnings.warn( f'Unable to determine whether or not "{callable_name}" is a generator with a return value. ' - 'This will not prevent your code from working, but it prevents Scrapy from detecting ' + "This will not prevent your code from working, but it prevents Scrapy from detecting " f'potential issues in your implementation of "{callable_name}". Please, report this in the ' - 'Scrapy issue tracker (https://github.com/scrapy/scrapy/issues), ' + "Scrapy issue tracker (https://github.com/scrapy/scrapy/issues), " f'including the code of "{callable_name}"', stacklevel=2, ) diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index cf867f3f8..18d856927 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -3,7 +3,7 @@ import signal signal_names = {} for signame in dir(signal): - if signame.startswith('SIG') and not signame.startswith('SIG_'): + if signame.startswith("SIG") and not signame.startswith("SIG_"): signum = getattr(signal, signame) if isinstance(signum, int): signal_names[signum] = signame @@ -16,10 +16,11 @@ def install_shutdown_handlers(function, override_sigint=True): (e.g. Pdb) """ from twisted.internet import reactor + reactor._handleSignals() signal.signal(signal.SIGTERM, function) if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows - if hasattr(signal, 'SIGBREAK'): + if hasattr(signal, "SIGBREAK"): signal.signal(signal.SIGBREAK, function) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index d21a144ad..4fbb6bcaf 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -9,23 +9,25 @@ from scrapy.settings import Settings from scrapy.exceptions import NotConfigured -ENVVAR = 'SCRAPY_SETTINGS_MODULE' -DATADIR_CFG_SECTION = 'datadir' +ENVVAR = "SCRAPY_SETTINGS_MODULE" +DATADIR_CFG_SECTION = "datadir" def inside_project(): - scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE') + scrapy_module = os.environ.get("SCRAPY_SETTINGS_MODULE") if scrapy_module is not None: try: import_module(scrapy_module) except ImportError as exc: - warnings.warn(f"Cannot import scrapy settings module {scrapy_module}: {exc}") + warnings.warn( + f"Cannot import scrapy settings module {scrapy_module}: {exc}" + ) else: return True return bool(closest_scrapy_cfg()) -def project_data_dir(project='default') -> str: +def project_data_dir(project="default") -> str: """Return the current project data dir, creating it if it doesn't exist""" if not inside_project(): raise NotConfigured("Not inside a project") @@ -35,8 +37,10 @@ def project_data_dir(project='default') -> str: else: scrapy_cfg = closest_scrapy_cfg() if not scrapy_cfg: - raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir") - d = (Path(scrapy_cfg).parent / '.scrapy').resolve() + raise NotConfigured( + "Unable to find scrapy.cfg file to infer project data dir" + ) + d = (Path(scrapy_cfg).parent / ".scrapy").resolve() if not d.exists(): d.mkdir(parents=True) return str(d) @@ -52,7 +56,7 @@ def data_path(path: str, createdir=False) -> str: if inside_project(): path_obj = Path(project_data_dir(), path) else: - path_obj = Path('.scrapy', path) + path_obj = Path(".scrapy", path) if createdir and not path_obj.exists(): path_obj.mkdir(parents=True) return str(path_obj) @@ -60,24 +64,27 @@ def data_path(path: str, createdir=False) -> str: def get_project_settings(): if ENVVAR not in os.environ: - project = os.environ.get('SCRAPY_PROJECT', 'default') + project = os.environ.get("SCRAPY_PROJECT", "default") init_env(project) settings = Settings() settings_module_path = os.environ.get(ENVVAR) if settings_module_path: - settings.setmodule(settings_module_path, priority='project') + settings.setmodule(settings_module_path, priority="project") valid_envvars = { - 'CHECK', - 'PROJECT', - 'PYTHON_SHELL', - 'SETTINGS_MODULE', + "CHECK", + "PROJECT", + "PYTHON_SHELL", + "SETTINGS_MODULE", } - scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if - k.startswith('SCRAPY_') and k.replace('SCRAPY_', '') in valid_envvars} + scrapy_envvars = { + k[7:]: v + for k, v in os.environ.items() + if k.startswith("SCRAPY_") and k.replace("SCRAPY_", "") in valid_envvars + } - settings.setdict(scrapy_envvars, priority='project') + settings.setdict(scrapy_envvars, priority="project") return settings diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 9df1c91de..1771e5550 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -82,29 +82,32 @@ def unique(list_, key=lambda x: x): return result -def to_unicode(text, encoding=None, errors='strict'): +def to_unicode(text, encoding=None, errors="strict"): """Return the unicode representation of a bytes object ``text``. If ``text`` is already an unicode object, return it as-is.""" if isinstance(text, str): return text if not isinstance(text, (bytes, str)): - raise TypeError('to_unicode must receive a bytes or str ' - f'object, got {type(text).__name__}') + raise TypeError( + "to_unicode must receive a bytes or str " + f"object, got {type(text).__name__}" + ) if encoding is None: - encoding = 'utf-8' + encoding = "utf-8" return text.decode(encoding, errors) -def to_bytes(text, encoding=None, errors='strict'): +def to_bytes(text, encoding=None, errors="strict"): """Return the binary representation of ``text``. If ``text`` is already a bytes object, return it as-is.""" if isinstance(text, bytes): return text if not isinstance(text, str): - raise TypeError('to_bytes must receive a str or bytes ' - f'object, got {type(text).__name__}') + raise TypeError( + "to_bytes must receive a str or bytes " f"object, got {type(text).__name__}" + ) if encoding is None: - encoding = 'utf-8' + encoding = "utf-8" return text.encode(encoding, errors) @@ -125,7 +128,7 @@ def re_rsearch(pattern, text, chunk_size=1024): def _chunk_iter(): offset = len(text) while True: - offset -= (chunk_size * 1024) + offset -= chunk_size * 1024 if offset <= 0: break yield (text[offset:], offset) @@ -162,7 +165,7 @@ _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} def binary_is_text(data): - """ Returns ``True`` if the given ``data`` argument (a ``bytes`` object) + """Returns ``True`` if the given ``data`` argument (a ``bytes`` object) does not contain unprintable control characters. """ if not isinstance(data, bytes): @@ -182,16 +185,19 @@ def get_func_args(func, stripself=False): elif inspect.ismethoddescriptor(func): return [] elif isinstance(func, partial): - return [x for x in get_func_args(func.func)[len(func.args):] - if not (func.keywords and x in func.keywords)] - elif hasattr(func, '__call__'): + return [ + x + for x in get_func_args(func.func)[len(func.args) :] + if not (func.keywords and x in func.keywords) + ] + elif hasattr(func, "__call__"): if inspect.isroutine(func): return [] - if getattr(func, '__name__', None) == '__call__': + if getattr(func, "__name__", None) == "__call__": return [] return get_func_args(func.__call__, True) else: - raise TypeError(f'{type(func)} is not callable') + raise TypeError(f"{type(func)} is not callable") if stripself: func_args.pop(0) return func_args @@ -221,10 +227,10 @@ def get_spec(func): if inspect.isfunction(func) or inspect.ismethod(func): spec = inspect.getfullargspec(func) - elif hasattr(func, '__call__'): + elif hasattr(func, "__call__"): spec = inspect.getfullargspec(func.__call__) else: - raise TypeError(f'{type(func)} is not callable') + raise TypeError(f"{type(func)} is not callable") defaults = spec.defaults or [] @@ -276,11 +282,14 @@ def global_object_name(obj): if hasattr(sys, "pypy_version_info"): + def garbage_collect(): # Collecting weakreferences can take two collections on PyPy. gc.collect() gc.collect() + else: + def garbage_collect(): gc.collect() diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ddf354d88..46d83059f 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -10,11 +10,12 @@ from scrapy.utils.misc import load_object def listen_tcp(portrange, host, factory): """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor + if len(portrange) > 2: raise ValueError(f"invalid portrange: {portrange}") if not portrange: return reactor.listenTCP(0, factory, interface=host) - if not hasattr(portrange, '__iter__'): + if not hasattr(portrange, "__iter__"): return reactor.listenTCP(portrange, factory, interface=host) if len(portrange) == 1: return reactor.listenTCP(portrange[0], factory, interface=host) @@ -39,6 +40,7 @@ class CallLaterOnce: def schedule(self, delay=0): from twisted.internet import reactor + if self._call is None: self._call = reactor.callLater(delay, self) @@ -93,16 +95,20 @@ def verify_installed_reactor(reactor_path): :mod:`~twisted.internet.reactor` does not match the specified import path.""" from twisted.internet import reactor + reactor_class = load_object(reactor_path) if not reactor.__class__ == reactor_class: - msg = ("The installed reactor " - f"({reactor.__module__}.{reactor.__class__.__name__}) does not " - f"match the requested one ({reactor_path})") + msg = ( + "The installed reactor " + f"({reactor.__module__}.{reactor.__class__.__name__}) does not " + f"match the requested one ({reactor_path})" + ) raise Exception(msg) def verify_installed_asyncio_event_loop(loop_path): from twisted.internet import reactor + loop_class = load_object(loop_path) if isinstance(reactor._asyncioEventloop, loop_class): return @@ -121,4 +127,5 @@ def verify_installed_asyncio_event_loop(loop_path): def is_asyncio_reactor_installed(): from twisted.internet import reactor + return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index c254b9f82..c818c8700 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -7,16 +7,22 @@ from scrapy.utils.request import request_from_dict as _from_dict warnings.warn( - ("Module scrapy.utils.reqser is deprecated, please use request.to_dict method" - " and/or scrapy.utils.request.request_from_dict instead"), + ( + "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" + " and/or scrapy.utils.request.request_from_dict instead" + ), category=ScrapyDeprecationWarning, stacklevel=2, ) -def request_to_dict(request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None) -> dict: +def request_to_dict( + request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None +) -> dict: return request.to_dict(spider=spider) -def request_from_dict(d: dict, spider: Optional["scrapy.Spider"] = None) -> "scrapy.Request": +def request_from_dict( + d: dict, spider: Optional["scrapy.Spider"] = None +) -> "scrapy.Request": return _from_dict(d, spider=spider) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index d1df8d335..3e29a9c57 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -69,54 +69,54 @@ def request_fingerprint( """ if include_headers or keep_fragments: message = ( - 'Call to deprecated function ' - 'scrapy.utils.request.request_fingerprint().\n' - '\n' - 'If you are using this function in a Scrapy component because you ' - 'need a non-default fingerprinting algorithm, and you are OK ' - 'with that non-default fingerprinting algorithm being used by ' - 'all Scrapy components and not just the one calling this ' - 'function, use crawler.request_fingerprinter.fingerprint() ' - 'instead in your Scrapy component (you can get the crawler ' - 'object from the \'from_crawler\' class method), and use the ' - '\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your ' - 'non-default fingerprinting algorithm.\n' - '\n' - 'Otherwise, consider using the ' - 'scrapy.utils.request.fingerprint() function instead.\n' - '\n' - 'If you switch to \'fingerprint()\', or assign the ' - '\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses ' - '\'fingerprint()\', the generated fingerprints will not only be ' - 'bytes instead of a string, but they will also be different from ' - 'those generated by \'request_fingerprint()\'. Before you switch, ' - 'make sure that you understand the consequences of this (e.g. ' - 'cache invalidation) and are OK with them; otherwise, consider ' - 'implementing your own function which returns the same ' - 'fingerprints as the deprecated \'request_fingerprint()\' function.' + "Call to deprecated function " + "scrapy.utils.request.request_fingerprint().\n" + "\n" + "If you are using this function in a Scrapy component because you " + "need a non-default fingerprinting algorithm, and you are OK " + "with that non-default fingerprinting algorithm being used by " + "all Scrapy components and not just the one calling this " + "function, use crawler.request_fingerprinter.fingerprint() " + "instead in your Scrapy component (you can get the crawler " + "object from the 'from_crawler' class method), and use the " + "'REQUEST_FINGERPRINTER_CLASS' setting to configure your " + "non-default fingerprinting algorithm.\n" + "\n" + "Otherwise, consider using the " + "scrapy.utils.request.fingerprint() function instead.\n" + "\n" + "If you switch to 'fingerprint()', or assign the " + "'REQUEST_FINGERPRINTER_CLASS' setting a class that uses " + "'fingerprint()', the generated fingerprints will not only be " + "bytes instead of a string, but they will also be different from " + "those generated by 'request_fingerprint()'. Before you switch, " + "make sure that you understand the consequences of this (e.g. " + "cache invalidation) and are OK with them; otherwise, consider " + "implementing your own function which returns the same " + "fingerprints as the deprecated 'request_fingerprint()' function." ) else: message = ( - 'Call to deprecated function ' - 'scrapy.utils.request.request_fingerprint().\n' - '\n' - 'If you are using this function in a Scrapy component, and you ' - 'are OK with users of your component changing the fingerprinting ' - 'algorithm through settings, use ' - 'crawler.request_fingerprinter.fingerprint() instead in your ' - 'Scrapy component (you can get the crawler object from the ' - '\'from_crawler\' class method).\n' - '\n' - 'Otherwise, consider using the ' - 'scrapy.utils.request.fingerprint() function instead.\n' - '\n' - 'Either way, the resulting fingerprints will be returned as ' - 'bytes, not as a string, and they will also be different from ' - 'those generated by \'request_fingerprint()\'. Before you switch, ' - 'make sure that you understand the consequences of this (e.g. ' - 'cache invalidation) and are OK with them; otherwise, consider ' - 'implementing your own function which returns the same ' - 'fingerprints as the deprecated \'request_fingerprint()\' function.' + "Call to deprecated function " + "scrapy.utils.request.request_fingerprint().\n" + "\n" + "If you are using this function in a Scrapy component, and you " + "are OK with users of your component changing the fingerprinting " + "algorithm through settings, use " + "crawler.request_fingerprinter.fingerprint() instead in your " + "Scrapy component (you can get the crawler object from the " + "'from_crawler' class method).\n" + "\n" + "Otherwise, consider using the " + "scrapy.utils.request.fingerprint() function instead.\n" + "\n" + "Either way, the resulting fingerprints will be returned as " + "bytes, not as a string, and they will also be different from " + "those generated by 'request_fingerprint()'. Before you switch, " + "make sure that you understand the consequences of this (e.g. " + "cache invalidation) and are OK with them; otherwise, consider " + "implementing your own function which returns the same " + "fingerprints as the deprecated 'request_fingerprint()' function." ) warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) processed_include_headers: Optional[Tuple[bytes, ...]] = None @@ -129,8 +129,10 @@ def request_fingerprint( if cache_key not in cache: fp = hashlib.sha1() fp.update(to_bytes(request.method)) - fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) - fp.update(request.body or b'') + fp.update( + to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)) + ) + fp.update(request.body or b"") if processed_include_headers: for part in _serialize_headers(processed_include_headers, request): fp.update(part) @@ -203,10 +205,10 @@ def fingerprint( for header_value in request.headers.getlist(header) ] fingerprint_data = { - 'method': to_unicode(request.method), - 'url': canonicalize_url(request.url, keep_fragments=keep_fragments), - 'body': (request.body or b'').hex(), - 'headers': headers, + "method": to_unicode(request.method), + "url": canonicalize_url(request.url, keep_fragments=keep_fragments), + "body": (request.body or b"").hex(), + "headers": headers, } fingerprint_json = json.dumps(fingerprint_data, sort_keys=True) cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() @@ -233,35 +235,35 @@ class RequestFingerprinter: def __init__(self, crawler=None): if crawler: implementation = crawler.settings.get( - 'REQUEST_FINGERPRINTER_IMPLEMENTATION' + "REQUEST_FINGERPRINTER_IMPLEMENTATION" ) else: - implementation = '2.6' - if implementation == '2.6': + implementation = "2.6" + if implementation == "2.6": message = ( - '\'2.6\' is a deprecated value for the ' - '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n' - '\n' - 'It is also the default value. In other words, it is normal ' - 'to get this warning if you have not defined a value for the ' - '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so ' - 'for backward compatibility reasons, but it will change in a ' - 'future version of Scrapy.\n' - '\n' - 'See the documentation of the ' - '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for ' - 'information on how to handle this deprecation.' + "'2.6' is a deprecated value for the " + "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n" + "\n" + "It is also the default value. In other words, it is normal " + "to get this warning if you have not defined a value for the " + "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so " + "for backward compatibility reasons, but it will change in a " + "future version of Scrapy.\n" + "\n" + "See the documentation of the " + "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for " + "information on how to handle this deprecation." ) warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) self._fingerprint = _request_fingerprint_as_bytes - elif implementation == '2.7': + elif implementation == "2.7": self._fingerprint = fingerprint else: raise ValueError( - f'Got an invalid value on setting ' - f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': ' - f'{implementation!r}. Valid values are \'2.6\' (deprecated) ' - f'and \'2.7\'.' + f"Got an invalid value on setting " + f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': " + f"{implementation!r}. Valid values are '2.6' (deprecated) " + f"and '2.7'." ) def fingerprint(self, request: Request): @@ -276,7 +278,7 @@ def request_authenticate( """Authenticate the given request (in place) using the HTTP basic access authentication mechanism (RFC 2617) and the given username and password """ - request.headers['Authorization'] = basic_auth_header(username, password) + request.headers["Authorization"] = basic_auth_header(username, password) def request_httprepr(request: Request) -> bytes: @@ -286,9 +288,9 @@ def request_httprepr(request: Request) -> bytes: by Twisted). """ parsed = urlparse_cached(request) - path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) + path = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, "")) s = to_bytes(request.method) + b" " + to_bytes(path) + b" HTTP/1.1\r\n" - s += b"Host: " + to_bytes(parsed.hostname or b'') + b"\r\n" + s += b"Host: " + to_bytes(parsed.hostname or b"") + b"\r\n" if request.headers: s += request.headers.to_string() + b"\r\n" s += b"\r\n" @@ -297,11 +299,11 @@ def request_httprepr(request: Request) -> bytes: def referer_str(request: Request) -> Optional[str]: - """ Return Referer HTTP header suitable for logging. """ - referrer = request.headers.get('Referer') + """Return Referer HTTP header suitable for logging.""" + referrer = request.headers.get("Referer") if referrer is None: return referrer - return to_unicode(referrer, errors='replace') + return to_unicode(referrer, errors="replace") def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 23bd2da65..3693177e4 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -25,31 +25,35 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] - _baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding) + _baseurl_cache[response] = html.get_base_url( + text, response.url, response.encoding + ) return _baseurl_cache[response] -_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary() +_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = ( + WeakKeyDictionary() +) def get_meta_refresh( response: "scrapy.http.response.text.TextResponse", - ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'), + ignore_tags: Optional[Iterable[str]] = ("script", "noscript"), ) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] _metaref_cache[response] = html.get_meta_refresh( - text, response.url, response.encoding, ignore_tags=ignore_tags) + text, response.url, response.encoding, ignore_tags=ignore_tags + ) return _metaref_cache[response] def response_status_message(status: Union[bytes, float, int, str]) -> str: - """Return status code plus status text descriptive message - """ + """Return status code plus status text descriptive message""" status_int = int(status) message = http.RESPONSES.get(status_int, "Unknown Status") - return f'{status_int} {to_unicode(message)}' + return f"{status_int} {to_unicode(message)}" @deprecated @@ -62,7 +66,7 @@ def response_httprepr(response: Response) -> bytes: b"HTTP/1.1 ", to_bytes(str(response.status)), b" ", - to_bytes(http.RESPONSES.get(response.status, b'')), + to_bytes(http.RESPONSES.get(response.status, b"")), b"\r\n", ] if response.headers: @@ -72,26 +76,29 @@ def response_httprepr(response: Response) -> bytes: def open_in_browser( - response: Union["scrapy.http.response.html.HtmlResponse", "scrapy.http.response.text.TextResponse"], + response: Union[ + "scrapy.http.response.html.HtmlResponse", + "scrapy.http.response.text.TextResponse", + ], _openfunc: Callable[[str], Any] = webbrowser.open, ) -> Any: """Open the given response in a local web browser, populating the tag for external links to work """ from scrapy.http import HtmlResponse, TextResponse + # XXX: this implementation is a bit dirty and could be improved body = response.body if isinstance(response, HtmlResponse): - if b'' + if b"' body = re.sub(b"", b"", body, flags=re.DOTALL) body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) - ext = '.html' + ext = ".html" elif isinstance(response, TextResponse): - ext = '.txt' + ext = ".txt" else: - raise TypeError("Unsupported response type: " - f"{response.__class__.__name__}") + raise TypeError("Unsupported response type: " f"{response.__class__.__name__}") fd, fname = tempfile.mkstemp(ext) os.write(fd, body) os.close(fd) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index fbafc9d45..b7c284174 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -5,7 +5,13 @@ import logging from twisted.internet.defer import DeferredList, Deferred from twisted.python.failure import Failure -from pydispatch.dispatcher import Anonymous, Any, disconnect, getAllReceivers, liveReceivers +from pydispatch.dispatcher import ( + Anonymous, + Any, + disconnect, + getAllReceivers, + liveReceivers, +) from pydispatch.robustapply import robustApply from scrapy.exceptions import StopDownload @@ -20,24 +26,36 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. """ - dont_log = named.pop('dont_log', ()) - dont_log = tuple(dont_log) if isinstance(dont_log, collections.abc.Sequence) else (dont_log,) - dont_log += (StopDownload, ) - spider = named.get('spider', None) + dont_log = named.pop("dont_log", ()) + dont_log = ( + tuple(dont_log) + if isinstance(dont_log, collections.abc.Sequence) + else (dont_log,) + ) + dont_log += (StopDownload,) + spider = named.get("spider", None) responses = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): try: - response = robustApply(receiver, signal=signal, sender=sender, *arguments, **named) + response = robustApply( + receiver, signal=signal, sender=sender, *arguments, **named + ) if isinstance(response, Deferred): - logger.error("Cannot return deferreds from signal handler: %(receiver)s", - {'receiver': receiver}, extra={'spider': spider}) + logger.error( + "Cannot return deferreds from signal handler: %(receiver)s", + {"receiver": receiver}, + extra={"spider": spider}, + ) except dont_log: result = Failure() except Exception: result = Failure() - logger.error("Error caught on signal handler: %(receiver)s", - {'receiver': receiver}, - exc_info=True, extra={'spider': spider}) + logger.error( + "Error caught on signal handler: %(receiver)s", + {"receiver": receiver}, + exc_info=True, + extra={"spider": spider}, + ) else: result = response responses.append((receiver, result)) @@ -49,19 +67,24 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): Returns a deferred that gets fired once all signal handlers deferreds were fired. """ + def logerror(failure, recv): if dont_log is None or not isinstance(failure.value, dont_log): - logger.error("Error caught on signal handler: %(receiver)s", - {'receiver': recv}, - exc_info=failure_to_exc_info(failure), - extra={'spider': spider}) + logger.error( + "Error caught on signal handler: %(receiver)s", + {"receiver": recv}, + exc_info=failure_to_exc_info(failure), + extra={"spider": spider}, + ) return failure - dont_log = named.pop('dont_log', None) - spider = named.get('spider', None) + dont_log = named.pop("dont_log", None) + spider = named.get("spider", None) dfds = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named) + d = maybeDeferred_coro( + robustApply, receiver, signal=signal, sender=sender, *arguments, **named + ) d.addErrback(logerror, receiver) d.addBoth(lambda result: (receiver, result)) dfds.append(d) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index a57a0c291..2622c2775 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -15,25 +15,27 @@ class Sitemap: (type=sitemapindex) files""" def __init__(self, xmltext): - xmlp = lxml.etree.XMLParser(recover=True, remove_comments=True, resolve_entities=False) + xmlp = lxml.etree.XMLParser( + recover=True, remove_comments=True, resolve_entities=False + ) self._root = lxml.etree.fromstring(xmltext, parser=xmlp) rt = self._root.tag - self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt + self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt def __iter__(self): for elem in self._root.getchildren(): d = {} for el in elem.getchildren(): tag = el.tag - name = tag.split('}', 1)[1] if '}' in tag else tag + name = tag.split("}", 1)[1] if "}" in tag else tag - if name == 'link': - if 'href' in el.attrib: - d.setdefault('alternate', []).append(el.get('href')) + if name == "link": + if "href" in el.attrib: + d.setdefault("alternate", []).append(el.get("href")) else: - d[name] = el.text.strip() if el.text else '' + d[name] = el.text.strip() if el.text else "" - if 'loc' in d: + if "loc" in d: yield d @@ -42,6 +44,6 @@ def sitemap_urls_from_robots(robots_text, base_url=None): robots.txt file """ for line in robots_text.splitlines(): - if line.lstrip().lower().startswith('sitemap:'): - url = line.split(':', 1)[1].strip() + if line.lstrip().lower().startswith("sitemap:"): + url = line.split(":", 1)[1].strip() yield urljoin(base_url, url) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index b2da69404..f829bceb8 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -32,13 +32,14 @@ def iter_spider_classes(module): inspect.isclass(obj) and issubclass(obj, Spider) and obj.__module__ == module.__name__ - and getattr(obj, 'name', None) + and getattr(obj, "name", None) ): yield obj -def spidercls_for_request(spider_loader, request, default_spidercls=None, - log_none=False, log_multiple=False): +def spidercls_for_request( + spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False +): """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using @@ -54,15 +55,18 @@ def spidercls_for_request(spider_loader, request, default_spidercls=None, return spider_loader.load(snames[0]) if len(snames) > 1 and log_multiple: - logger.error('More than one spider can handle: %(request)s - %(snames)s', - {'request': request, 'snames': ', '.join(snames)}) + logger.error( + "More than one spider can handle: %(request)s - %(snames)s", + {"request": request, "snames": ", ".join(snames)}, + ) if len(snames) == 0 and log_none: - logger.error('Unable to find spider that handles: %(request)s', - {'request': request}) + logger.error( + "Unable to find spider that handles: %(request)s", {"request": request} + ) return default_spidercls class DefaultSpider(Spider): - name = 'default' + name = "default" diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index ea4dde882..b73261723 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -6,7 +6,7 @@ from scrapy.utils.python import to_unicode # The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL. # Using the binding directly, as this code does, requires cryptography 2.4. -SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0) +SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, "SSL_OP_NO_TLSv1_3", 0) def ffi_buf_to_string(buf): @@ -16,13 +16,17 @@ def ffi_buf_to_string(buf): def x509name_to_string(x509name): # from OpenSSL.crypto.X509Name.__repr__ result_buffer = pyOpenSSLutil.ffi.new("char[]", 512) - pyOpenSSLutil.lib.X509_NAME_oneline(x509name._name, result_buffer, len(result_buffer)) + pyOpenSSLutil.lib.X509_NAME_oneline( + x509name._name, result_buffer, len(result_buffer) + ) return ffi_buf_to_string(result_buffer) def get_temp_key_info(ssl_object): - if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'): # requires OpenSSL 1.0.2 + if not hasattr( + pyOpenSSLutil.lib, "SSL_get_server_tmp_key" + ): # requires OpenSSL 1.0.2 return None # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() @@ -36,26 +40,28 @@ def get_temp_key_info(ssl_object): key_info = [] key_type = pyOpenSSLutil.lib.EVP_PKEY_id(temp_key) if key_type == pyOpenSSLutil.lib.EVP_PKEY_RSA: - key_info.append('RSA') + key_info.append("RSA") elif key_type == pyOpenSSLutil.lib.EVP_PKEY_DH: - key_info.append('DH') + key_info.append("DH") elif key_type == pyOpenSSLutil.lib.EVP_PKEY_EC: - key_info.append('ECDH') + key_info.append("ECDH") ec_key = pyOpenSSLutil.lib.EVP_PKEY_get1_EC_KEY(temp_key) ec_key = pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free) - nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name(pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key)) + nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name( + pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key) + ) cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid) if cname == pyOpenSSLutil.ffi.NULL: cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid) key_info.append(ffi_buf_to_string(cname)) else: key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) - key_info.append(f'{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits') - return ', '.join(key_info) + key_info.append(f"{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits") + return ", ".join(key_info) def get_openssl_version(): - system_openssl = OpenSSL.SSL.SSLeay_version( - OpenSSL.SSL.SSLEAY_VERSION - ).decode('ascii', errors='replace') - return f'{OpenSSL.version.__version__} ({system_openssl})' + system_openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION).decode( + "ascii", errors="replace" + ) + return f"{OpenSSL.version.__version__} ({system_openssl})" diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 2177817d9..89bedfc69 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -9,23 +9,23 @@ from typing import Union def render_templatefile(path: Union[str, PathLike], **kwargs): path_obj = Path(path) - raw = path_obj.read_text('utf8') + raw = path_obj.read_text("utf8") content = string.Template(raw).substitute(**kwargs) - render_path = path_obj.with_suffix('') if path_obj.suffix == '.tmpl' else path_obj + render_path = path_obj.with_suffix("") if path_obj.suffix == ".tmpl" else path_obj - if path_obj.suffix == '.tmpl': + if path_obj.suffix == ".tmpl": path_obj.rename(render_path) - render_path.write_text(content, 'utf8') + render_path.write_text(content, "utf8") -CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') +CAMELCASE_INVALID_CHARS = re.compile(r"[^a-zA-Z\d]") def string_camelcase(string): - """ Convert a word to its CamelCase version and remove invalid chars + """Convert a word to its CamelCase version and remove invalid chars >>> string_camelcase('lost-pound') 'LostPound' @@ -34,4 +34,4 @@ def string_camelcase(string): 'MissingImages' """ - return CAMELCASE_INVALID_CHARS.sub('', string.title()) + return CAMELCASE_INVALID_CHARS.sub("", string.title()) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 9171d6224..d21065706 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -15,18 +15,19 @@ from scrapy.utils.boto import is_botocore_available def assert_gcs_environ(): - if 'GCS_PROJECT_ID' not in os.environ: + if "GCS_PROJECT_ID" not in os.environ: raise SkipTest("GCS_PROJECT_ID not found") def skip_if_no_boto(): if not is_botocore_available(): - raise SkipTest('missing botocore library') + raise SkipTest("missing botocore library") def get_gcs_content_and_delete(bucket, path): from google.cloud import storage - client = storage.Client(project=os.environ.get('GCS_PROJECT_ID')) + + client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) bucket = client.get_bucket(bucket) blob = bucket.get_blob(path) content = blob.download_as_string() @@ -36,9 +37,10 @@ def get_gcs_content_and_delete(bucket, path): def get_ftp_content_and_delete( - path, host, port, username, - password, use_active_mode=False): + path, host, port, username, password, use_active_mode=False +): from ftplib import FTP + ftp = FTP() ftp.connect(host, port) ftp.login(username, password) @@ -48,7 +50,8 @@ def get_ftp_content_and_delete( def buffer_data(data): ftp_data.append(data) - ftp.retrbinary(f'RETR {path}', buffer_data) + + ftp.retrbinary(f"RETR {path}", buffer_data) dirname, filename = split(path) ftp.cwd(dirname) ftp.delete(filename) @@ -66,7 +69,7 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): # Set by default settings that prevent deprecation warnings. settings = {} if prevent_warnings: - settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = '2.7' + settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7" settings.update(settings_dict or {}) runner = CrawlerRunner(settings) return runner.create_crawler(spidercls or Spider) @@ -75,8 +78,8 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): def get_pythonpath() -> str: """Return a PYTHONPATH suitable to use in processes so that they find this installation of Scrapy""" - scrapy_path = import_module('scrapy').__path__[0] - return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get('PYTHONPATH', '') + scrapy_path = import_module("scrapy").__path__[0] + return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get("PYTHONPATH", "") def get_testenv(): @@ -84,7 +87,7 @@ def get_testenv(): this installation of Scrapy, instead of a system installed one. """ env = os.environ.copy() - env['PYTHONPATH'] = get_pythonpath() + env["PYTHONPATH"] = get_pythonpath() return env @@ -107,6 +110,7 @@ def mock_google_cloud_storage(): classes and set their proper return values. """ from google.cloud.storage import Client, Bucket, Blob + client_mock = mock.create_autospec(Client) bucket_mock = mock.create_autospec(Bucket) @@ -121,5 +125,6 @@ def mock_google_cloud_storage(): def get_web_client_agent_req(url): from twisted.internet import reactor from twisted.web.client import Agent # imports twisted.internet.reactor + agent = Agent(reactor) - return agent.request(b'GET', url.encode('utf-8')) + return agent.request(b"GET", url.encode("utf-8")) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index a54c7db95..fe5c8d74c 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -7,14 +7,15 @@ from twisted.internet import defer, protocol class ProcessTest: command = None - prefix = [sys.executable, '-m', 'scrapy.cmdline'] + prefix = [sys.executable, "-m", "scrapy.cmdline"] cwd = os.getcwd() # trial chdirs to temp dir def execute(self, args, check_code=True, settings=None): from twisted.internet import reactor + env = os.environ.copy() if settings is not None: - env['SCRAPY_SETTINGS_MODULE'] = settings + env["SCRAPY_SETTINGS_MODULE"] = settings cmd = self.prefix + [self.command] + list(args) pp = TestProcessProtocol() pp.deferred.addBoth(self._process_finished, cmd, check_code) @@ -32,11 +33,10 @@ class ProcessTest: class TestProcessProtocol(protocol.ProcessProtocol): - def __init__(self): self.deferred = defer.Deferred() - self.out = b'' - self.err = b'' + self.out = b"" + self.err = b"" self.exitcode = None def outReceived(self, data): diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 5d3710391..a47756c4b 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -4,9 +4,9 @@ from twisted.web import server, resource, static, util class SiteTest: - def setUp(self): from twisted.internet import reactor + super().setUp() self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") self.baseurl = f"http://localhost:{self.site.getHost().port}/" @@ -22,23 +22,34 @@ class SiteTest: class NoMetaRefreshRedirect(util.Redirect): def render(self, request): content = util.Redirect.render(self, request) - return content.replace(b'http-equiv=\"refresh\"', - b'http-no-equiv=\"do-not-refresh-me\"') + return content.replace( + b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' + ) def test_site(): r = resource.Resource() r.putChild(b"text", static.Data(b"Works", "text/plain")) - r.putChild(b"html", static.Data(b"

Works

World

", "text/html")) - r.putChild(b"enc-gb18030", static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030")) + r.putChild( + b"html", + static.Data( + b"

Works

World

", + "text/html", + ), + ) + r.putChild( + b"enc-gb18030", + static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), + ) r.putChild(b"redirect", util.Redirect(b"/redirected")) r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) return server.Site(r) -if __name__ == '__main__': +if __name__ == "__main__": from twisted.internet import reactor + port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") print(f"http://localhost:{port.getHost().port}/") reactor.run() diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index b0c6a2424..9aa775a1b 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -35,8 +35,7 @@ def format_live_refs(ignore=NoneType): """Return a tabular representation of tracked objects""" s = "Live References\n\n" now = time() - for cls, wdict in sorted(live_refs.items(), - key=lambda x: x[0].__name__): + for cls, wdict in sorted(live_refs.items(), key=lambda x: x[0].__name__): if not wdict: continue if issubclass(cls, ignore): diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 21201ace5..ad3bff4f0 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -21,12 +21,14 @@ def url_is_from_any_domain(url, domains): if not host: return False domains = [d.lower() for d in domains] - return any((host == d) or (host.endswith(f'.{d}')) for d in domains) + return any((host == d) or (host.endswith(f".{d}")) for d in domains) def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - return url_is_from_any_domain(url, [spider.name] + list(getattr(spider, 'allowed_domains', []))) + return url_is_from_any_domain( + url, [spider.name] + list(getattr(spider, "allowed_domains", [])) + ) def url_has_any_extension(url, extensions): @@ -68,9 +70,9 @@ def escape_ajax(url): 'www.example.com/ajax.html' """ defrag, frag = urldefrag(url) - if not frag.startswith('!'): + if not frag.startswith("!"): return url - return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:]) + return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:]) def add_http_if_no_scheme(url): @@ -87,7 +89,7 @@ def add_http_if_no_scheme(url): def _is_posix_path(string): return bool( re.match( - r''' + r""" ^ # start with... ( \. # ...a single dot, @@ -99,7 +101,7 @@ def _is_posix_path(string): )? # optional match of ".", ".." or ".blabla" / # at least one "/" for a file path, . # and something after the "/" - ''', + """, string, flags=re.VERBOSE, ) @@ -109,13 +111,13 @@ def _is_posix_path(string): def _is_windows_path(string): return bool( re.match( - r''' + r""" ^ ( [a-z]:\\ | \\\\ ) - ''', + """, string, flags=re.IGNORECASE | re.VERBOSE, ) @@ -134,7 +136,13 @@ def guess_scheme(url): return add_http_if_no_scheme(url) -def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True): +def strip_url( + url, + strip_credentials=True, + strip_default_port=True, + origin_only=False, + strip_fragment=True, +): """Strip URL string from some of its components: @@ -148,18 +156,24 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only= parsed_url = urlparse(url) netloc = parsed_url.netloc - if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password): - netloc = netloc.split('@')[-1] + if (strip_credentials or origin_only) and ( + parsed_url.username or parsed_url.password + ): + netloc = netloc.split("@")[-1] if strip_default_port and parsed_url.port: - if (parsed_url.scheme, parsed_url.port) in (('http', 80), - ('https', 443), - ('ftp', 21)): - netloc = netloc.replace(f':{parsed_url.port}', '') - return urlunparse(( - parsed_url.scheme, - netloc, - '/' if origin_only else parsed_url.path, - '' if origin_only else parsed_url.params, - '' if origin_only else parsed_url.query, - '' if strip_fragment else parsed_url.fragment - )) + if (parsed_url.scheme, parsed_url.port) in ( + ("http", 80), + ("https", 443), + ("ftp", 21), + ): + netloc = netloc.replace(f":{parsed_url.port}", "") + return urlunparse( + ( + parsed_url.scheme, + netloc, + "/" if origin_only else parsed_url.path, + "" if origin_only else parsed_url.params, + "" if origin_only else parsed_url.query, + "" if strip_fragment else parsed_url.fragment, + ) + ) diff --git a/setup.py b/setup.py index 82ac86cdd..27581f81d 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from pkg_resources import parse_version from setuptools import setup, find_packages, __version__ as setuptools_version -version = (Path(__file__).parent / 'scrapy/VERSION').read_text('ascii').strip() +version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() def has_environment_marker_platform_impl_support(): @@ -14,82 +14,82 @@ def has_environment_marker_platform_impl_support(): it is 18.5, see: https://setuptools.readthedocs.io/en/latest/history.html#id235 """ - return parse_version(setuptools_version) >= parse_version('18.5') + return parse_version(setuptools_version) >= parse_version("18.5") install_requires = [ - 'Twisted>=18.9.0', - 'cryptography>=3.3', - 'cssselect>=0.9.1', - 'itemloaders>=1.0.1', - 'parsel>=1.5.0', - 'pyOpenSSL>=21.0.0', - 'queuelib>=1.4.2', - 'service_identity>=18.1.0', - 'w3lib>=1.17.0', - 'zope.interface>=5.1.0', - 'protego>=0.1.15', - 'itemadapter>=0.1.0', - 'setuptools', - 'packaging', - 'tldextract', - 'lxml>=4.3.0', + "Twisted>=18.9.0", + "cryptography>=3.3", + "cssselect>=0.9.1", + "itemloaders>=1.0.1", + "parsel>=1.5.0", + "pyOpenSSL>=21.0.0", + "queuelib>=1.4.2", + "service_identity>=18.1.0", + "w3lib>=1.17.0", + "zope.interface>=5.1.0", + "protego>=0.1.15", + "itemadapter>=0.1.0", + "setuptools", + "packaging", + "tldextract", + "lxml>=4.3.0", ] extras_require = {} cpython_dependencies = [ - 'PyDispatcher>=2.0.5', + "PyDispatcher>=2.0.5", ] if has_environment_marker_platform_impl_support(): - extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies + extras_require[ + ':platform_python_implementation == "CPython"' + ] = cpython_dependencies extras_require[':platform_python_implementation == "PyPy"'] = [ - 'PyPyDispatcher>=2.1.0', + "PyPyDispatcher>=2.1.0", ] else: install_requires.extend(cpython_dependencies) setup( - name='Scrapy', + name="Scrapy", version=version, - url='https://scrapy.org', + url="https://scrapy.org", project_urls={ - 'Documentation': 'https://docs.scrapy.org/', - 'Source': 'https://github.com/scrapy/scrapy', - 'Tracker': 'https://github.com/scrapy/scrapy/issues', + "Documentation": "https://docs.scrapy.org/", + "Source": "https://github.com/scrapy/scrapy", + "Tracker": "https://github.com/scrapy/scrapy/issues", }, - description='A high-level Web Crawling and Web Scraping framework', - long_description=open('README.rst', encoding="utf-8").read(), - author='Scrapy developers', - maintainer='Pablo Hoffman', - maintainer_email='pablo@pablohoffman.com', - license='BSD', - packages=find_packages(exclude=('tests', 'tests.*')), + description="A high-level Web Crawling and Web Scraping framework", + long_description=open("README.rst", encoding="utf-8").read(), + author="Scrapy developers", + maintainer="Pablo Hoffman", + maintainer_email="pablo@pablohoffman.com", + license="BSD", + packages=find_packages(exclude=("tests", "tests.*")), include_package_data=True, zip_safe=False, - entry_points={ - 'console_scripts': ['scrapy = scrapy.cmdline:execute'] - }, + entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]}, classifiers=[ - 'Framework :: Scrapy', - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Software Development :: Libraries :: Application Frameworks', - 'Topic :: Software Development :: Libraries :: Python Modules', + "Framework :: Scrapy", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries :: Application Frameworks", + "Topic :: Software Development :: Libraries :: Python Modules", ], - python_requires='>=3.7', + python_requires=">=3.7", install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py index 1e4ada722..5e72aa6d4 100644 --- a/tests/CrawlerProcess/asyncio_custom_loop.py +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -3,15 +3,17 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] -process = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "ASYNCIO_EVENT_LOOP": "uvloop.Loop" -}) +process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index b83f6a585..1afef4d24 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -37,9 +37,11 @@ if __name__ == "__main__": except IndexError: ASYNCIO_EVENT_LOOP = None - process = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP, - }) + process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP, + } + ) process.crawl(UrlSpider) process.start() diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index d1e4a7bb5..6df6d76fa 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -3,14 +3,16 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] -process = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", -}) +process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } +) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index e561d63c7..01d23c963 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -2,6 +2,7 @@ import asyncio import sys from twisted.internet import asyncioreactor + if sys.version_info >= (3, 8) and sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) @@ -11,14 +12,16 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] -process = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", -}) +process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } +) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index ea8242f67..34ef00143 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -2,6 +2,7 @@ import asyncio import sys from twisted.internet import asyncioreactor + if sys.version_info >= (3, 8) and sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) @@ -11,15 +12,17 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] -process = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "ASYNCIO_EVENT_LOOP": "uvloop.Loop", -}) +process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index d24bf3031..79dd77bb2 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -4,6 +4,7 @@ import sys from uvloop import Loop from twisted.internet import asyncioreactor + if sys.version_info >= (3, 8) and sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.set_event_loop(Loop()) @@ -14,15 +15,17 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] -process = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "ASYNCIO_EVENT_LOOP": "uvloop.Loop", -}) +process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py index f9eab3543..7b0497bde 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver.py +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -8,6 +8,7 @@ class CachingHostnameResolverSpider(scrapy.Spider): """ Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution) """ + name = "caching_hostname_resolver_spider" def start_requests(self): @@ -15,16 +16,20 @@ class CachingHostnameResolverSpider(scrapy.Spider): def parse(self, response): for _ in range(10): - yield scrapy.Request(response.url, dont_filter=True, callback=self.ignore_response) + yield scrapy.Request( + response.url, dont_filter=True, callback=self.ignore_response + ) def ignore_response(self, response): self.logger.info(repr(response.ip_address)) if __name__ == "__main__": - process = CrawlerProcess(settings={ - "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", - }) + process = CrawlerProcess( + settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + } + ) process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) process.start() diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py index 3340d2f84..d5d19e27e 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py +++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py @@ -6,14 +6,17 @@ class CachingHostnameResolverSpider(scrapy.Spider): """ Finishes without a twisted.internet.error.DNSLookupError exception """ + name = "caching_hostname_resolver_spider" start_urls = ["http://[::1]"] if __name__ == "__main__": - process = CrawlerProcess(settings={ - "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", - }) + process = CrawlerProcess( + settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + } + ) process.crawl(CachingHostnameResolverSpider) process.start() diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py index 05a98fbec..cfc4b38b7 100644 --- a/tests/CrawlerProcess/default_name_resolver.py +++ b/tests/CrawlerProcess/default_name_resolver.py @@ -7,6 +7,7 @@ class IPv6Spider(scrapy.Spider): Raises a twisted.internet.error.DNSLookupError: the default name resolver does not handle IPv6 addresses. """ + name = "ipv6_spider" start_urls = ["http://[::1]"] diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py index aaa1af5c5..9f7eaf2ae 100644 --- a/tests/CrawlerProcess/multi.py +++ b/tests/CrawlerProcess/multi.py @@ -3,7 +3,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 06b849de3..078cb72cb 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -5,7 +5,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index a6dff0e05..744b4ecb5 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -5,15 +5,17 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] -process = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", -}) +process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", + } +) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index 907f50784..814a2a46d 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -7,7 +7,7 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 6f48da691..37626c081 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -13,15 +13,17 @@ installReactor(reactor) class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] -process = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", -}) +process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", + } +) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index c422b13ff..b397608ec 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -7,15 +7,17 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] -process = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", -}) +process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", + } +) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py index 5f6f1ae30..2d8769501 100644 --- a/tests/CrawlerProcess/simple.py +++ b/tests/CrawlerProcess/simple.py @@ -3,7 +3,7 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] diff --git a/tests/CrawlerProcess/twisted_reactor_asyncio.py b/tests/CrawlerProcess/twisted_reactor_asyncio.py index c6cbf949b..057d0efff 100644 --- a/tests/CrawlerProcess/twisted_reactor_asyncio.py +++ b/tests/CrawlerProcess/twisted_reactor_asyncio.py @@ -3,11 +3,13 @@ from scrapy.crawler import CrawlerProcess class AsyncioReactorSpider(scrapy.Spider): - name = 'asyncio_reactor' + name = "asyncio_reactor" -process = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", -}) +process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } +) process.crawl(AsyncioReactorSpider) process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.py index 56304bd23..4b6a33f66 100644 --- a/tests/CrawlerProcess/twisted_reactor_custom_settings.py +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py @@ -3,7 +3,7 @@ from scrapy.crawler import CrawlerProcess class AsyncioReactorSpider(scrapy.Spider): - name = 'asyncio_reactor' + name = "asyncio_reactor" custom_settings = { "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py index 3f219098c..19cc08be6 100644 --- a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -3,14 +3,14 @@ from scrapy.crawler import CrawlerProcess class SelectReactorSpider(scrapy.Spider): - name = 'select_reactor' + name = "select_reactor" custom_settings = { "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", } class AsyncioReactorSpider(scrapy.Spider): - name = 'asyncio_reactor' + name = "asyncio_reactor" custom_settings = { "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py index 72bb986bc..5c9b737c6 100644 --- a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py @@ -3,14 +3,14 @@ from scrapy.crawler import CrawlerProcess class AsyncioReactorSpider1(scrapy.Spider): - name = 'asyncio_reactor1' + name = "asyncio_reactor1" custom_settings = { "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } class AsyncioReactorSpider2(scrapy.Spider): - name = 'asyncio_reactor2' + name = "asyncio_reactor2" custom_settings = { "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } diff --git a/tests/CrawlerProcess/twisted_reactor_poll.py b/tests/CrawlerProcess/twisted_reactor_poll.py index 27063260b..b2ca04672 100644 --- a/tests/CrawlerProcess/twisted_reactor_poll.py +++ b/tests/CrawlerProcess/twisted_reactor_poll.py @@ -3,11 +3,13 @@ from scrapy.crawler import CrawlerProcess class PollReactorSpider(scrapy.Spider): - name = 'poll_reactor' + name = "poll_reactor" -process = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor", -}) +process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor", + } +) process.crawl(PollReactorSpider) process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_select.py b/tests/CrawlerProcess/twisted_reactor_select.py index 9af8ceb4d..5048081f7 100644 --- a/tests/CrawlerProcess/twisted_reactor_select.py +++ b/tests/CrawlerProcess/twisted_reactor_select.py @@ -3,11 +3,13 @@ from scrapy.crawler import CrawlerProcess class SelectReactorSpider(scrapy.Spider): - name = 'epoll_reactor' + name = "epoll_reactor" -process = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", -}) +process = CrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", + } +) process.crawl(SelectReactorSpider) process.start() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 417294447..26db16dd6 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -14,7 +14,7 @@ from tests.mockserver import MockServer, MockDNSServer # https://stackoverflow.com/a/32784190 def createResolver(servers=None, resolvconf=None, hosts=None): if hosts is None: - hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts' + hosts = b"/etc/hosts" if platform.getType() == "posix" else r"c:\windows\hosts" theResolver = Resolver(resolvconf, servers) hostResolver = hostsModule.Resolver(hosts) chain = [hostResolver, cache.CacheResolver(), theResolver] diff --git a/tests/__init__.py b/tests/__init__.py index be263fa16..5f0c0f7ad 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -11,24 +11,24 @@ from pathlib import Path # ignore system-wide proxies for tests # which would send requests to a totally unsuspecting server # (e.g. because urllib does not fully understand the proxy spec) -os.environ['http_proxy'] = '' -os.environ['https_proxy'] = '' -os.environ['ftp_proxy'] = '' +os.environ["http_proxy"] = "" +os.environ["https_proxy"] = "" +os.environ["ftp_proxy"] = "" # Absolutize paths to coverage config and output file because tests that # spawn subprocesses also changes current working directory. _sourceroot = Path(__file__).resolve().parent.parent -if 'COV_CORE_CONFIG' in os.environ: - os.environ['COVERAGE_FILE'] = str(_sourceroot / '.coverage') - os.environ['COV_CORE_CONFIG'] = str(_sourceroot / os.environ['COV_CORE_CONFIG']) +if "COV_CORE_CONFIG" in os.environ: + os.environ["COVERAGE_FILE"] = str(_sourceroot / ".coverage") + os.environ["COV_CORE_CONFIG"] = str(_sourceroot / os.environ["COV_CORE_CONFIG"]) -tests_datadir = str(Path(__file__).parent.resolve() / 'sample_data') +tests_datadir = str(Path(__file__).parent.resolve() / "sample_data") # In some environments accessing a non-existing host doesn't raise an # error. In such cases we're going to skip tests which rely on it. try: - socket.getaddrinfo('non-existing-host', 80) + socket.getaddrinfo("non-existing-host", 80) NON_EXISTING_RESOLVABLE = True except socket.gaierror: NON_EXISTING_RESOLVABLE = False diff --git a/tests/ftpserver.py b/tests/ftpserver.py index 6f0289e08..0c6ca3621 100644 --- a/tests/ftpserver.py +++ b/tests/ftpserver.py @@ -7,18 +7,18 @@ from pyftpdlib.servers import FTPServer def main(): parser = ArgumentParser() - parser.add_argument('-d', '--directory') + parser.add_argument("-d", "--directory") args = parser.parse_args() authorizer = DummyAuthorizer() - full_permissions = 'elradfmwMT' + full_permissions = "elradfmwMT" authorizer.add_anonymous(args.directory, perm=full_permissions) handler = FTPHandler handler.authorizer = authorizer - address = ('127.0.0.1', 2121) + address = ("127.0.0.1", 2121) server = FTPServer(address, handler) server.serve_forever() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index 3a41b3a3e..b306437db 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -29,7 +29,7 @@ def generate_keys(): key_size=2048, backend=default_backend(), ) - (folder / 'localhost.key').write_bytes( + (folder / "localhost.key").write_bytes( key.private_bytes( encoding=Encoding.PEM, format=PrivateFormat.TraditionalOpenSSL, @@ -58,4 +58,4 @@ def generate_keys(): ) .sign(key, SHA256(), default_backend()) ) - (folder / 'localhost.crt').write_bytes(cert.public_bytes(Encoding.PEM)) + (folder / "localhost.crt").write_bytes(cert.public_bytes(Encoding.PEM)) diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 75c74daf5..e77f53666 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -4,6 +4,7 @@ import collections class DummyDB(dict): """Provide dummy DBM-like interface.""" + def close(self): pass @@ -14,7 +15,7 @@ error = KeyError _DATABASES = collections.defaultdict(DummyDB) -def open(file, flag='r', mode=0o666): +def open(file, flag="r", mode=0o666): """Open or create a dummy database compatible. Arguments ``flag`` and ``mode`` are ignored. diff --git a/tests/mockserver.py b/tests/mockserver.py index 6d2d95692..9f1a707c9 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -111,7 +111,6 @@ class LeafResource(resource.Resource): class Follow(LeafResource): - def render(self, request): total = getarg(request, b"total", 100, type=int) show = getarg(request, b"show", 1, type=int) @@ -140,13 +139,12 @@ class Follow(LeafResource): class Delay(LeafResource): - def render_GET(self, request): n = getarg(request, b"n", 1, type=float) b = getarg(request, b"b", 1, type=int) if b: # send headers now and delay body - request.write('') + request.write("") self.deferRequest(request, n, self._delayedRender, request, n) return NOT_DONE_YET @@ -156,7 +154,6 @@ class Delay(LeafResource): class Status(LeafResource): - def render_GET(self, request): n = getarg(request, b"n", 200, type=int) request.setResponseCode(n) @@ -164,15 +161,15 @@ class Status(LeafResource): class Raw(LeafResource): - def render_GET(self, request): request.startedWriting = 1 self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET + render_POST = render_GET def _delayedRender(self, request): - raw = getarg(request, b'raw', b'HTTP 1.1 200 OK\n') + raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n") request.startedWriting = 1 request.write(raw) request.channel.transport.loseConnection() @@ -180,30 +177,29 @@ class Raw(LeafResource): class Echo(LeafResource): - def render_GET(self, request): output = { - 'headers': dict( + "headers": dict( (to_unicode(k), [to_unicode(v) for v in vs]) - for k, vs in request.requestHeaders.getAllRawHeaders()), - 'body': to_unicode(request.content.read()), + for k, vs in request.requestHeaders.getAllRawHeaders() + ), + "body": to_unicode(request.content.read()), } return to_bytes(json.dumps(output)) + render_POST = render_GET class RedirectTo(LeafResource): - def render(self, request): - goto = getarg(request, b'goto', b'/') + goto = getarg(request, b"goto", b"/") # we force the body content, otherwise Twisted redirectTo() # returns HTML with 0: return reqs = [] for i in range(1, 3): - req = Request(self.start_urls[0], dont_filter=True, meta={'req_id': i}) + req = Request(self.start_urls[0], dont_filter=True, meta={"req_id": i}) reqs.append(req) return reqs class AsyncDefAsyncioGenExcSpider(SimpleSpider): - name = 'asyncdef_asyncio_gen_exc' + name = "asyncdef_asyncio_gen_exc" async def parse(self, response): for i in range(10): await asyncio.sleep(0.1) - yield {'foo': i} + yield {"foo": i} if i > 5: raise ValueError("Stopping the processing") class AsyncDefDeferredDirectSpider(SimpleSpider): - name = 'asyncdef_deferred_direct' + name = "asyncdef_deferred_direct" async def parse(self, response): resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200")) - yield {'code': resp.code} + yield {"code": resp.code} class AsyncDefDeferredWrappedSpider(SimpleSpider): - name = 'asyncdef_deferred_wrapped' + name = "asyncdef_deferred_wrapped" async def parse(self, response): - resp = await deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200"))) - yield {'code': resp.code} + resp = await deferred_to_future( + get_web_client_agent_req(self.mockserver.url("/status?n=200")) + ) + yield {"code": resp.code} class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): - name = 'asyncdef_deferred_wrapped' + name = "asyncdef_deferred_wrapped" async def parse(self, response): - resp = await maybe_deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200"))) - yield {'code': resp.code} + resp = await maybe_deferred_to_future( + get_web_client_agent_req(self.mockserver.url("/status?n=200")) + ) + yield {"code": resp.code} class AsyncDefAsyncioGenSpider(SimpleSpider): - name = 'asyncdef_asyncio_gen' + name = "asyncdef_asyncio_gen" async def parse(self, response): await asyncio.sleep(0.2) - yield {'foo': 42} + yield {"foo": 42} self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioGenLoopSpider(SimpleSpider): - name = 'asyncdef_asyncio_gen_loop' + name = "asyncdef_asyncio_gen_loop" async def parse(self, response): for i in range(10): await asyncio.sleep(0.1) - yield {'foo': i} + yield {"foo": i} self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioGenComplexSpider(SimpleSpider): - name = 'asyncdef_asyncio_gen_complex' + name = "asyncdef_asyncio_gen_complex" initial_reqs = 4 following_reqs = 3 depth = 2 def _get_req(self, index, cb=None): - return Request(self.mockserver.url(f"/status?n=200&request={index}"), - meta={'index': index}, - dont_filter=True, - callback=cb) + return Request( + self.mockserver.url(f"/status?n=200&request={index}"), + meta={"index": index}, + dont_filter=True, + callback=cb, + ) def start_requests(self): for i in range(1, self.initial_reqs + 1): yield self._get_req(i) async def parse(self, response): - index = response.meta['index'] - yield {'index': index} - if index < 10 ** self.depth: + index = response.meta["index"] + yield {"index": index} + if index < 10**self.depth: for new_index in range(10 * index, 10 * index + self.following_reqs): yield self._get_req(new_index) yield self._get_req(index, cb=self.parse2) await asyncio.sleep(0.1) - yield {'index': index + 5} + yield {"index": index + 5} async def parse2(self, response): await asyncio.sleep(0.1) - yield {'index2': response.meta['index']} + yield {"index2": response.meta["index"]} class ItemSpider(FollowAllSpider): - name = 'item' + name = "item" def parse(self, response): for request in super().parse(response): @@ -254,11 +262,11 @@ class DefaultError(Exception): class ErrorSpider(FollowAllSpider): - name = 'error' + name = "error" exception_cls = DefaultError def raise_exception(self): - raise self.exception_cls('Expected exception') + raise self.exception_cls("Expected exception") def parse(self, response): for request in super().parse(response): @@ -280,16 +288,18 @@ class BrokenStartRequestsSpider(FollowAllSpider): 1 / 0 for s in range(100): - qargs = {'total': 10, 'seed': s} + qargs = {"total": 10, "seed": s} url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") - yield Request(url, meta={'seed': s}) + yield Request(url, meta={"seed": s}) if self.fail_yielding: 2 / 0 - assert self.seedsseen, 'All start requests consumed before any download happened' + assert ( + self.seedsseen + ), "All start requests consumed before any download happened" def parse(self, response): - self.seedsseen.append(response.meta.get('seed')) + self.seedsseen.append(response.meta.get("seed")) for req in super().parse(response): yield req @@ -307,21 +317,21 @@ class SingleRequestSpider(MetaSpider): yield Request(self.seed, callback=self.parse, errback=self.on_error) def parse(self, response): - self.meta.setdefault('responses', []).append(response) + self.meta.setdefault("responses", []).append(response) if callable(self.callback_func): return self.callback_func(response) - if 'next' in response.meta: - return response.meta['next'] + if "next" in response.meta: + return response.meta["next"] def on_error(self, failure): - self.meta['failure'] = failure + self.meta["failure"] = failure if callable(self.errback_func): return self.errback_func(failure) class DuplicateStartRequestsSpider(MockServerSpider): dont_filter = True - name = 'duplicatestartrequests' + name = "duplicatestartrequests" distinct_urls = 2 dupe_factor = 3 @@ -343,13 +353,12 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): """ A CrawlSpider which overrides the 'parse' method """ - name = 'crawl_spider_with_parse_method' + + name = "crawl_spider_with_parse_method" custom_settings: dict = { - 'RETRY_HTTP_CODES': [], # no need to retry + "RETRY_HTTP_CODES": [], # no need to retry } - rules = ( - Rule(LinkExtractor(), callback='parse', follow=True), - ) + rules = (Rule(LinkExtractor(), callback="parse", follow=True),) def start_requests(self): test_body = b""" @@ -365,39 +374,45 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): yield Request(url, method="POST", body=test_body) def parse(self, response, foo=None): - self.logger.info('[parse] status %i (foo: %s)', response.status, foo) - yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"}) + self.logger.info("[parse] status %i (foo: %s)", response.status, foo) + yield Request( + self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"} + ) class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod): """A CrawlSpider with an async def callback""" - name = 'crawl_spider_with_async_callback' - rules = ( - Rule(LinkExtractor(), callback='parse_async', follow=True), - ) + + name = "crawl_spider_with_async_callback" + rules = (Rule(LinkExtractor(), callback="parse_async", follow=True),) async def parse_async(self, response, foo=None): - self.logger.info('[parse_async] status %i (foo: %s)', response.status, foo) - return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"}) + self.logger.info("[parse_async] status %i (foo: %s)", response.status, foo) + return Request( + self.mockserver.url("/status?n=202"), + self.parse_async, + cb_kwargs={"foo": "bar"}, + ) class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod): """A CrawlSpider with an async generator callback""" - name = 'crawl_spider_with_async_generator_callback' - rules = ( - Rule(LinkExtractor(), callback='parse_async_gen', follow=True), - ) + + name = "crawl_spider_with_async_generator_callback" + rules = (Rule(LinkExtractor(), callback="parse_async_gen", follow=True),) async def parse_async_gen(self, response, foo=None): - self.logger.info('[parse_async_gen] status %i (foo: %s)', response.status, foo) - yield Request(self.mockserver.url("/status?n=202"), self.parse_async_gen, cb_kwargs={"foo": "bar"}) + self.logger.info("[parse_async_gen] status %i (foo: %s)", response.status, foo) + yield Request( + self.mockserver.url("/status?n=202"), + self.parse_async_gen, + cb_kwargs={"foo": "bar"}, + ) class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): - name = 'crawl_spider_with_errback' - rules = ( - Rule(LinkExtractor(), callback='parse', errback='errback', follow=True), - ) + name = "crawl_spider_with_errback" + rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),) def start_requests(self): test_body = b""" @@ -416,13 +431,18 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): yield Request(url, method="POST", body=test_body) def errback(self, failure): - self.logger.info('[errback] status %i', failure.value.response.status) + self.logger.info("[errback] status %i", failure.value.response.status) class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod): - name = 'crawl_spider_with_process_request_cb_kwargs' + name = "crawl_spider_with_process_request_cb_kwargs" rules = ( - Rule(LinkExtractor(), callback='parse', follow=True, process_request="process_request"), + Rule( + LinkExtractor(), + callback="parse", + follow=True, + process_request="process_request", + ), ) def process_request(self, request, response): @@ -457,14 +477,12 @@ class BytesReceivedCallbackSpider(MetaSpider): class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider): - def bytes_received(self, data, request, spider): self.meta["bytes_received"] = data raise StopDownload(fail=True) class HeadersReceivedCallbackSpider(MetaSpider): - @classmethod def from_crawler(cls, crawler, *args, **kwargs): spider = super().from_crawler(crawler, *args, **kwargs) @@ -486,7 +504,6 @@ class HeadersReceivedCallbackSpider(MetaSpider): class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider): - def headers_received(self, headers, body_length, request, spider): self.meta["headers_received"] = headers raise StopDownload(fail=True) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index be8adadb3..c497450f7 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -6,7 +6,6 @@ from tests.mockserver import MockServer class TestCloseSpider(TestCase): - def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() @@ -17,40 +16,40 @@ class TestCloseSpider(TestCase): @defer.inlineCallbacks def test_closespider_itemcount(self): close_on = 5 - crawler = get_crawler(ItemSpider, {'CLOSESPIDER_ITEMCOUNT': close_on}) + crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) - reason = crawler.spider.meta['close_reason'] - self.assertEqual(reason, 'closespider_itemcount') - itemcount = crawler.stats.get_value('item_scraped_count') + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_itemcount") + itemcount = crawler.stats.get_value("item_scraped_count") self.assertTrue(itemcount >= close_on) @defer.inlineCallbacks def test_closespider_pagecount(self): close_on = 5 - crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_PAGECOUNT': close_on}) + crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) - reason = crawler.spider.meta['close_reason'] - self.assertEqual(reason, 'closespider_pagecount') - pagecount = crawler.stats.get_value('response_received_count') + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_pagecount") + pagecount = crawler.stats.get_value("response_received_count") self.assertTrue(pagecount >= close_on) @defer.inlineCallbacks def test_closespider_errorcount(self): close_on = 5 - crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on}) + crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) - reason = crawler.spider.meta['close_reason'] - self.assertEqual(reason, 'closespider_errorcount') - key = f'spider_exceptions/{crawler.spider.exception_cls.__name__}' + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_errorcount") + key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}" errorcount = crawler.stats.get_value(key) self.assertTrue(errorcount >= close_on) @defer.inlineCallbacks def test_closespider_timeout(self): close_on = 0.1 - crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_TIMEOUT': close_on}) + crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) - reason = crawler.spider.meta['close_reason'] - self.assertEqual(reason, 'closespider_timeout') - total_seconds = crawler.stats.get_value('elapsed_time_seconds') + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_timeout") + total_seconds = crawler.stats.get_value("elapsed_time_seconds") self.assertTrue(total_seconds >= close_on) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 87ab1ee81..5aa35a6d9 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -12,47 +12,52 @@ from scrapy.utils.test import get_testenv class CmdlineTest(unittest.TestCase): - def setUp(self): self.env = get_testenv() - self.env['SCRAPY_SETTINGS_MODULE'] = 'tests.test_cmdline.settings' + self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings" def _execute(self, *new_args, **kwargs): - encoding = getattr(sys.stdout, 'encoding') or 'utf-8' - args = (sys.executable, '-m', 'scrapy.cmdline') + new_args + encoding = getattr(sys.stdout, "encoding") or "utf-8" + args = (sys.executable, "-m", "scrapy.cmdline") + new_args proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs) comm = proc.communicate()[0].strip() return comm.decode(encoding) def test_default_settings(self): - self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'default') + self.assertEqual(self._execute("settings", "--get", "TEST1"), "default") def test_override_settings_using_set_arg(self): - self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s', - 'TEST1=override'), 'override') + self.assertEqual( + self._execute("settings", "--get", "TEST1", "-s", "TEST1=override"), + "override", + ) def test_profiling(self): path = Path(tempfile.mkdtemp()) - filename = path / 'res.prof' + filename = path / "res.prof" try: - self._execute('version', '--profile', str(filename)) + self._execute("version", "--profile", str(filename)) self.assertTrue(filename.exists()) out = StringIO() stats = pstats.Stats(str(filename), stream=out) stats.print_stats() out.seek(0) stats = out.read() - self.assertIn(str(Path('scrapy', 'commands', 'version.py')), - stats) - self.assertIn('tottime', stats) + self.assertIn(str(Path("scrapy", "commands", "version.py")), stats) + self.assertIn("tottime", stats) finally: shutil.rmtree(path) def test_override_dict_settings(self): EXT_PATH = "tests.test_cmdline.extensions.DummyExtension" EXTENSIONS = {EXT_PATH: 200} - settingsstr = self._execute('settings', '--get', 'EXTENSIONS', '-s', - 'EXTENSIONS=' + json.dumps(EXTENSIONS)) + settingsstr = self._execute( + "settings", + "--get", + "EXTENSIONS", + "-s", + "EXTENSIONS=" + json.dumps(EXTENSIONS), + ) # XXX: There's gotta be a smarter way to do this... self.assertNotIn("...", settingsstr) for char in ("'", "<", ">"): @@ -62,5 +67,7 @@ class CmdlineTest(unittest.TestCase): self.assertEqual(200, settingsdict[EXT_PATH]) def test_pathlib_path_as_feeds_key(self): - self.assertEqual(self._execute('settings', '--get', 'FEEDS'), - json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}})) + self.assertEqual( + self._execute("settings", "--get", "FEEDS"), + json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}}), + ) diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index 005e45214..11c821f8d 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -2,9 +2,8 @@ class TestExtension: - def __init__(self, settings): - settings.set('TEST1', f"{settings['TEST1']} + started") + settings.set("TEST1", f"{settings['TEST1']} + started") @classmethod def from_crawler(cls, crawler): diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py index b0ac6e98b..32b15e191 100644 --- a/tests/test_cmdline/settings.py +++ b/tests/test_cmdline/settings.py @@ -1,14 +1,14 @@ from pathlib import Path EXTENSIONS = { - 'tests.test_cmdline.extensions.TestExtension': 0, + "tests.test_cmdline.extensions.TestExtension": 0, } -TEST1 = 'default' +TEST1 = "default" FEEDS = { - Path('items.csv'): { - 'format': 'csv', - 'fields': ['price', 'name'], + Path("items.csv"): { + "format": "csv", + "fields": ["price", "name"], }, } diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index fcafcef68..d5088e817 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -5,16 +5,15 @@ from subprocess import Popen, PIPE class CmdlineCrawlPipelineTest(unittest.TestCase): - def _execute(self, spname): - args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname) + args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname) cwd = Path(__file__).resolve().parent proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd) proc.communicate() return proc.returncode def test_open_spider_normally_in_pipeline(self): - self.assertEqual(self._execute('normal'), 0) + self.assertEqual(self._execute("normal"), 0) def test_exception_at_open_spider_in_pipeline(self): - self.assertEqual(self._execute('exception'), 1) + self.assertEqual(self._execute("exception"), 1) diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py index bd1f9cd8c..af15cac68 100644 --- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py @@ -1,5 +1,4 @@ class TestSpiderPipeline: - def open_spider(self, spider): pass @@ -8,9 +7,8 @@ class TestSpiderPipeline: class TestSpiderExceptionPipeline: - def open_spider(self, spider): - raise Exception('exception') + raise Exception("exception") def process_item(self, item, spider): return item diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py index ae782c0d8..45c5f2eb9 100644 --- a/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py @@ -1,2 +1,2 @@ -BOT_NAME = 'test_spider' -SPIDER_MODULES = ['test_spider.spiders'] +BOT_NAME = "test_spider" +SPIDER_MODULES = ["test_spider.spiders"] diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py index 300f45ebf..ebd4082c6 100644 --- a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py @@ -2,12 +2,10 @@ import scrapy class ExceptionSpider(scrapy.Spider): - name = 'exception' + name = "exception" custom_settings = { - 'ITEM_PIPELINES': { - 'test_spider.pipelines.TestSpiderExceptionPipeline': 300 - } + "ITEM_PIPELINES": {"test_spider.pipelines.TestSpiderExceptionPipeline": 300} } def parse(self, response): diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py index 87a40fdcb..9a258c80d 100644 --- a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py @@ -2,12 +2,10 @@ import scrapy class NormalSpider(scrapy.Spider): - name = 'normal' + name = "normal" custom_settings = { - 'ITEM_PIPELINES': { - 'test_spider.pipelines.TestSpiderPipeline': 300 - } + "ITEM_PIPELINES": {"test_spider.pipelines.TestSpiderPipeline": 300} } def parse(self, response): diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 049076e5c..1f299587f 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -3,15 +3,16 @@ from tests.test_commands import CommandTest class CheckCommandTest(CommandTest): - command = 'check' + command = "check" def setUp(self): super().setUp() - self.spider_name = 'check_spider' - self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve() + self.spider_name = "check_spider" + self.spider = (self.proj_mod_path / "spiders" / "checkspider.py").resolve() def _write_contract(self, contracts, parse_def): - self.spider.write_text(f""" + self.spider.write_text( + f""" import scrapy class CheckSpider(scrapy.Spider): @@ -24,13 +25,15 @@ class CheckSpider(scrapy.Spider): {contracts} \"\"\" {parse_def} - """, encoding="utf-8") + """, + encoding="utf-8", + ) - def _test_contract(self, contracts='', parse_def='pass'): + def _test_contract(self, contracts="", parse_def="pass"): self._write_contract(contracts, parse_def) - p, out, err = self.proc('check') - self.assertNotIn('F', out) - self.assertIn('OK', err) + p, out, err = self.proc("check") + self.assertNotIn("F", out) + self.assertIn("OK", err) self.assertEqual(p.returncode, 0) def test_check_returns_requests_contract(self): diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 9d3c8fe73..bd44fa76e 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -7,28 +7,30 @@ from scrapy.utils.testproc import ProcessTest class FetchTest(ProcessTest, SiteTest, unittest.TestCase): - command = 'fetch' + command = "fetch" @defer.inlineCallbacks def test_output(self): - _, out, _ = yield self.execute([self.url('/text')]) - self.assertEqual(out.strip(), b'Works') + _, out, _ = yield self.execute([self.url("/text")]) + self.assertEqual(out.strip(), b"Works") @defer.inlineCallbacks def test_redirect_default(self): - _, out, _ = yield self.execute([self.url('/redirect')]) - self.assertEqual(out.strip(), b'Redirected here') + _, out, _ = yield self.execute([self.url("/redirect")]) + self.assertEqual(out.strip(), b"Redirected here") @defer.inlineCallbacks def test_redirect_disabled(self): - _, out, err = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh')]) + _, out, err = yield self.execute( + ["--no-redirect", self.url("/redirect-no-meta-refresh")] + ) err = err.strip() - self.assertIn(b'downloader/response_status_count/302', err, err) - self.assertNotIn(b'downloader/response_status_count/200', err, err) + self.assertIn(b"downloader/response_status_count/302", err, err) + self.assertNotIn(b"downloader/response_status_count/200", err, err) @defer.inlineCallbacks def test_headers(self): - _, out, _ = yield self.execute([self.url('/text'), '--headers']) - out = out.replace(b'\r', b'') # required on win32 - assert b'Server: TwistedWeb' in out, out - assert b'Content-Type: text/plain' in out + _, out, _ = yield self.execute([self.url("/text"), "--headers"]) + out = out.replace(b"\r", b"") # required on win32 + assert b"Server: TwistedWeb" in out, out + assert b"Content-Type: text/plain" in out diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 368b80513..1ee1bf5a7 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -14,16 +14,17 @@ from tests.test_commands import CommandTest def _textmode(bstr): """Normalize input the same as writing to a file and reading from it in text mode""" - return to_unicode(bstr).replace(os.linesep, '\n') + return to_unicode(bstr).replace(os.linesep, "\n") class ParseCommandTest(ProcessTest, SiteTest, CommandTest): - command = 'parse' + command = "parse" def setUp(self): super().setUp() - self.spider_name = 'parse_spider' - (self.proj_mod_path / 'spiders' / 'myspider.py').write_text(f""" + self.spider_name = "parse_spider" + (self.proj_mod_path / "spiders" / "myspider.py").write_text( + f""" import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule @@ -92,9 +93,12 @@ class MyBadCrawlSpider(CrawlSpider): def parse(self, response): return [scrapy.Item(), dict(foo='bar')] -""", encoding="utf-8") +""", + encoding="utf-8", + ) - (self.proj_mod_path / 'pipelines.py').write_text(""" + (self.proj_mod_path / "pipelines.py").write_text( + """ import logging class MyPipeline: @@ -103,91 +107,141 @@ class MyPipeline: def process_item(self, item, spider): logging.info('It Works!') return item -""", encoding="utf-8") +""", + encoding="utf-8", + ) - with (self.proj_mod_path / 'settings.py').open("a", encoding="utf-8") as f: - f.write(f""" + with (self.proj_mod_path / "settings.py").open("a", encoding="utf-8") as f: + f.write( + f""" ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} -""") +""" + ) @defer.inlineCallbacks def test_spider_arguments(self): - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '-a', 'test_arg=1', - '-c', 'parse', - '--verbose', - self.url('/html')]) + _, _, stderr = yield self.execute( + [ + "--spider", + self.spider_name, + "-a", + "test_arg=1", + "-c", + "parse", + "--verbose", + self.url("/html"), + ] + ) self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_request_with_meta(self): raw_json_string = '{"foo" : "baz"}' - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '--meta', raw_json_string, - '-c', 'parse_request_with_meta', - '--verbose', - self.url('/html')]) + _, _, stderr = yield self.execute( + [ + "--spider", + self.spider_name, + "--meta", + raw_json_string, + "-c", + "parse_request_with_meta", + "--verbose", + self.url("/html"), + ] + ) self.assertIn("DEBUG: It Works!", _textmode(stderr)) - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '-m', raw_json_string, - '-c', 'parse_request_with_meta', - '--verbose', - self.url('/html')]) + _, _, stderr = yield self.execute( + [ + "--spider", + self.spider_name, + "-m", + raw_json_string, + "-c", + "parse_request_with_meta", + "--verbose", + self.url("/html"), + ] + ) self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_request_with_cb_kwargs(self): raw_json_string = '{"foo" : "bar", "key": "value"}' - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '--cbkwargs', raw_json_string, - '-c', 'parse_request_with_cb_kwargs', - '--verbose', - self.url('/html')]) + _, _, stderr = yield self.execute( + [ + "--spider", + self.spider_name, + "--cbkwargs", + raw_json_string, + "-c", + "parse_request_with_cb_kwargs", + "--verbose", + self.url("/html"), + ] + ) self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_request_without_meta(self): - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '-c', 'parse_request_without_meta', - '--nolinks', - self.url('/html')]) + _, _, stderr = yield self.execute( + [ + "--spider", + self.spider_name, + "-c", + "parse_request_without_meta", + "--nolinks", + self.url("/html"), + ] + ) self.assertIn("DEBUG: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_pipelines(self): - _, _, stderr = yield self.execute(['--spider', self.spider_name, - '--pipelines', - '-c', 'parse', - '--verbose', - self.url('/html')]) + _, _, stderr = yield self.execute( + [ + "--spider", + self.spider_name, + "--pipelines", + "-c", + "parse", + "--verbose", + self.url("/html"), + ] + ) self.assertIn("INFO: It Works!", _textmode(stderr)) @defer.inlineCallbacks def test_asyncio_parse_items(self): status, out, stderr = yield self.execute( - ['--spider', 'asyncdef' + self.spider_name, '-c', 'parse', self.url('/html')] + [ + "--spider", + "asyncdef" + self.spider_name, + "-c", + "parse", + self.url("/html"), + ] ) self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @defer.inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( - ['--spider', self.spider_name, '-c', 'parse', self.url('/html')] + ["--spider", self.spider_name, "-c", "parse", self.url("/html")] ) self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @defer.inlineCallbacks def test_parse_items_no_callback_passed(self): status, out, stderr = yield self.execute( - ['--spider', self.spider_name, self.url('/html')] + ["--spider", self.spider_name, self.url("/html")] ) self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @defer.inlineCallbacks def test_wrong_callback_passed(self): status, out, stderr = yield self.execute( - ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')] + ["--spider", self.spider_name, "-c", "dummy", self.url("/html")] ) self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find callback""", _textmode(stderr)) @@ -196,7 +250,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} def test_crawlspider_matching_rule_callback_set(self): """If a rule matches the URL, use it's defined callback.""" status, out, stderr = yield self.execute( - ['--spider', 'goodcrawl' + self.spider_name, '-r', self.url('/html')] + ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/html")] ) self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @@ -204,7 +258,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} def test_crawlspider_matching_rule_default_callback(self): """If a rule match but it has no callback set, use the 'parse' callback.""" status, out, stderr = yield self.execute( - ['--spider', 'goodcrawl' + self.spider_name, '-r', self.url('/text')] + ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/text")] ) self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out)) @@ -212,7 +266,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} def test_spider_with_no_rules_attribute(self): """Using -r with a spider with no rule should not produce items.""" status, out, stderr = yield self.execute( - ['--spider', self.spider_name, '-r', self.url('/html')] + ["--spider", self.spider_name, "-r", self.url("/html")] ) self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""No CrawlSpider rules found""", _textmode(stderr)) @@ -220,7 +274,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} @defer.inlineCallbacks def test_crawlspider_missing_callback(self): status, out, stderr = yield self.execute( - ['--spider', 'badcrawl' + self.spider_name, '-r', self.url('/html')] + ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/html")] ) self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") @@ -228,14 +282,14 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} def test_crawlspider_no_matching_rule(self): """The requested URL has no matching rule, so no items should be scraped""" status, out, stderr = yield self.execute( - ['--spider', 'badcrawl' + self.spider_name, '-r', self.url('/enc-gb18030')] + ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/enc-gb18030")] ) self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) @defer.inlineCallbacks def test_crawlspider_not_exists_with_not_matched_url(self): - status, out, stderr = yield self.execute([self.url('/invalid_url')]) + status, out, stderr = yield self.execute([self.url("/invalid_url")]) self.assertEqual(status, 0) @defer.inlineCallbacks @@ -243,14 +297,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """Checks if a file was created successfully having correct format containing correct data in it. """ - file_name = 'data.json' + file_name = "data.json" file_path = Path(self.proj_path, file_name) - yield self.execute([ - '--spider', self.spider_name, - '-c', 'parse', - '-o', file_name, - self.url('/html') - ]) + yield self.execute( + [ + "--spider", + self.spider_name, + "-c", + "parse", + "-o", + file_name, + self.url("/html"), + ] + ) self.assertTrue(file_path.exists()) self.assertTrue(file_path.is_file()) @@ -262,12 +321,14 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} command = parse.Command() command.settings = Settings() parser = argparse.ArgumentParser( - prog='scrapy', formatter_class=argparse.HelpFormatter, - conflict_handler='resolve', prefix_chars='-' + prog="scrapy", + formatter_class=argparse.HelpFormatter, + conflict_handler="resolve", + prefix_chars="-", ) command.add_options(parser) namespace = parser.parse_args( - ['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name] + ["--verbose", "--nolinks", "-d", "2", "--spider", self.spider_name] ) self.assertTrue(namespace.nolinks) self.assertEqual(namespace.depth, 2) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 33c98ad69..4c4242a1b 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -11,107 +11,118 @@ from tests import tests_datadir, NON_EXISTING_RESOLVABLE class ShellTest(ProcessTest, SiteTest, unittest.TestCase): - command = 'shell' + command = "shell" @defer.inlineCallbacks def test_empty(self): - _, out, _ = yield self.execute(['-c', 'item']) - assert b'{}' in out + _, out, _ = yield self.execute(["-c", "item"]) + assert b"{}" in out @defer.inlineCallbacks def test_response_body(self): - _, out, _ = yield self.execute([self.url('/text'), '-c', 'response.body']) - assert b'Works' in out + _, out, _ = yield self.execute([self.url("/text"), "-c", "response.body"]) + assert b"Works" in out @defer.inlineCallbacks def test_response_type_text(self): - _, out, _ = yield self.execute([self.url('/text'), '-c', 'type(response)']) - assert b'TextResponse' in out + _, out, _ = yield self.execute([self.url("/text"), "-c", "type(response)"]) + assert b"TextResponse" in out @defer.inlineCallbacks def test_response_type_html(self): - _, out, _ = yield self.execute([self.url('/html'), '-c', 'type(response)']) - assert b'HtmlResponse' in out + _, out, _ = yield self.execute([self.url("/html"), "-c", "type(response)"]) + assert b"HtmlResponse" in out @defer.inlineCallbacks def test_response_selector_html(self): - xpath = 'response.xpath("//p[@class=\'one\']/text()").get()' - _, out, _ = yield self.execute([self.url('/html'), '-c', xpath]) - self.assertEqual(out.strip(), b'Works') + xpath = "response.xpath(\"//p[@class='one']/text()\").get()" + _, out, _ = yield self.execute([self.url("/html"), "-c", xpath]) + self.assertEqual(out.strip(), b"Works") @defer.inlineCallbacks def test_response_encoding_gb18030(self): - _, out, _ = yield self.execute([self.url('/enc-gb18030'), '-c', 'response.encoding']) - self.assertEqual(out.strip(), b'gb18030') + _, out, _ = yield self.execute( + [self.url("/enc-gb18030"), "-c", "response.encoding"] + ) + self.assertEqual(out.strip(), b"gb18030") @defer.inlineCallbacks def test_redirect(self): - _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url']) - assert out.strip().endswith(b'/redirected') + _, out, _ = yield self.execute([self.url("/redirect"), "-c", "response.url"]) + assert out.strip().endswith(b"/redirected") @defer.inlineCallbacks def test_redirect_follow_302(self): - _, out, _ = yield self.execute([self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) - assert out.strip().endswith(b'200') + _, out, _ = yield self.execute( + [self.url("/redirect-no-meta-refresh"), "-c", "response.status"] + ) + assert out.strip().endswith(b"200") @defer.inlineCallbacks def test_redirect_not_follow_302(self): _, out, _ = yield self.execute( - ['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status'] + [ + "--no-redirect", + self.url("/redirect-no-meta-refresh"), + "-c", + "response.status", + ] ) - assert out.strip().endswith(b'302') + assert out.strip().endswith(b"302") @defer.inlineCallbacks def test_fetch_redirect_follow_302(self): """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" - url = self.url('/redirect-no-meta-refresh') + url = self.url("/redirect-no-meta-refresh") code = f"fetch('{url}')" - errcode, out, errout = yield self.execute(['-c', code]) + errcode, out, errout = yield self.execute(["-c", code]) self.assertEqual(errcode, 0, out) - assert b'Redirecting (302)' in errout - assert b'Crawled (200)' in errout + assert b"Redirecting (302)" in errout + assert b"Crawled (200)" in errout @defer.inlineCallbacks def test_fetch_redirect_not_follow_302(self): """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" - url = self.url('/redirect-no-meta-refresh') + url = self.url("/redirect-no-meta-refresh") code = f"fetch('{url}', redirect=False)" - errcode, out, errout = yield self.execute(['-c', code]) + errcode, out, errout = yield self.execute(["-c", code]) self.assertEqual(errcode, 0, out) - assert b'Crawled (302)' in errout + assert b"Crawled (302)" in errout @defer.inlineCallbacks def test_request_replace(self): - url = self.url('/text') + url = self.url("/text") code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" - errcode, out, _ = yield self.execute(['-c', code]) + errcode, out, _ = yield self.execute(["-c", code]) self.assertEqual(errcode, 0, out) @defer.inlineCallbacks def test_scrapy_import(self): - url = self.url('/text') + url = self.url("/text") code = f"fetch(scrapy.Request('{url}'))" - errcode, out, _ = yield self.execute(['-c', code]) + errcode, out, _ = yield self.execute(["-c", code]) self.assertEqual(errcode, 0, out) @defer.inlineCallbacks def test_local_file(self): - filepath = Path(tests_datadir, 'test_site', 'index.html') - _, out, _ = yield self.execute([str(filepath), '-c', 'item']) - assert b'{}' in out + filepath = Path(tests_datadir, "test_site", "index.html") + _, out, _ = yield self.execute([str(filepath), "-c", "item"]) + assert b"{}" in out @defer.inlineCallbacks def test_local_nofile(self): - filepath = 'file:///tests/sample_data/test_site/nothinghere.html' - errcode, out, err = yield self.execute([filepath, '-c', 'item'], check_code=False) + filepath = "file:///tests/sample_data/test_site/nothinghere.html" + errcode, out, err = yield self.execute( + [filepath, "-c", "item"], check_code=False + ) self.assertEqual(errcode, 1, out or err) - self.assertIn(b'No such file or directory', err) + self.assertIn(b"No such file or directory", err) @defer.inlineCallbacks def test_dns_failures(self): if NON_EXISTING_RESOLVABLE: raise unittest.SkipTest("Non-existing hosts are resolvable") - url = 'www.somedomainthatdoesntexi.st' - errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False) + url = "www.somedomainthatdoesntexi.st" + errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False) self.assertEqual(errcode, 1, out or err) - self.assertIn(b'DNS lookup failed', err) + self.assertIn(b"DNS lookup failed", err) diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 00d998388..f97a088a8 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -8,11 +8,11 @@ from scrapy.utils.testproc import ProcessTest class VersionTest(ProcessTest, unittest.TestCase): - command = 'version' + command = "version" @defer.inlineCallbacks def test_output(self): - encoding = getattr(sys.stdout, 'encoding') or 'utf-8' + encoding = getattr(sys.stdout, "encoding") or "utf-8" _, out, _ = yield self.execute([]) self.assertEqual( out.strip().decode(encoding), @@ -21,13 +21,25 @@ class VersionTest(ProcessTest, unittest.TestCase): @defer.inlineCallbacks def test_verbose_output(self): - encoding = getattr(sys.stdout, 'encoding') or 'utf-8' - _, out, _ = yield self.execute(['-v']) + encoding = getattr(sys.stdout, "encoding") or "utf-8" + _, out, _ = yield self.execute(["-v"]) headers = [ line.partition(":")[0].strip() for line in out.strip().decode(encoding).splitlines() ] - self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', - 'cssselect', 'parsel', 'w3lib', - 'Twisted', 'Python', 'pyOpenSSL', - 'cryptography', 'Platform']) + self.assertEqual( + headers, + [ + "Scrapy", + "lxml", + "libxml2", + "cssselect", + "parsel", + "w3lib", + "Twisted", + "Python", + "pyOpenSSL", + "cryptography", + "Platform", + ], + ) diff --git a/tests/test_commands.py b/tests/test_commands.py index 91476abf8..ed0eb85f5 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -33,35 +33,46 @@ from tests.test_crawler import ExceptionSpider, NoRequestsSpider class CommandSettings(unittest.TestCase): - def setUp(self): self.command = ScrapyCommand() self.command.settings = Settings() - self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, - conflict_handler='resolve') + self.parser = argparse.ArgumentParser( + formatter_class=ScrapyHelpFormatter, conflict_handler="resolve" + ) self.command.add_options(self.parser) def test_settings_json_string(self): feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' - opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) + opts, args = self.parser.parse_known_args( + args=["-s", f"FEEDS={feeds_json}", "spider.py"] + ) self.command.process_options(args, opts) - self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) - self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) + self.assertIsInstance( + self.command.settings["FEEDS"], scrapy.settings.BaseSettings + ) + self.assertEqual(dict(self.command.settings["FEEDS"]), json.loads(feeds_json)) def test_help_formatter(self): - formatter = ScrapyHelpFormatter(prog='scrapy') - part_strings = ['usage: scrapy genspider [options] \n\n', - '\n', 'optional arguments:\n', '\n', 'Global Options:\n'] + formatter = ScrapyHelpFormatter(prog="scrapy") + part_strings = [ + "usage: scrapy genspider [options] \n\n", + "\n", + "optional arguments:\n", + "\n", + "Global Options:\n", + ] self.assertEqual( formatter._join_parts(part_strings), - ('Usage\n=====\n scrapy genspider [options] \n\n\n' - 'Optional Arguments\n==================\n\n' - 'Global Options\n--------------\n') + ( + "Usage\n=====\n scrapy genspider [options] \n\n\n" + "Optional Arguments\n==================\n\n" + "Global Options\n--------------\n" + ), ) class ProjectTest(unittest.TestCase): - project_name = 'testproject' + project_name = "testproject" def setUp(self): self.temp_path = mkdtemp() @@ -75,15 +86,16 @@ class ProjectTest(unittest.TestCase): def call(self, *new_args, **kwargs): with tempfile.TemporaryFile() as out: - args = (sys.executable, '-m', 'scrapy.cmdline') + new_args - return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd, - env=self.env, **kwargs) + args = (sys.executable, "-m", "scrapy.cmdline") + new_args + return subprocess.call( + args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs + ) def proc(self, *new_args, **popen_kwargs): - args = (sys.executable, '-m', 'scrapy.cmdline') + new_args + args = (sys.executable, "-m", "scrapy.cmdline") + new_args p = subprocess.Popen( args, - cwd=popen_kwargs.pop('cwd', self.cwd), + cwd=popen_kwargs.pop("cwd", self.cwd), env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -93,7 +105,7 @@ class ProjectTest(unittest.TestCase): def kill_proc(): p.kill() p.communicate() - assert False, 'Command took too much time to complete' + assert False, "Command took too much time to complete" timer = Timer(15, kill_proc) try: @@ -104,7 +116,9 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) - def find_in_file(self, filename: Union[str, os.PathLike], regex) -> Optional[re.Match]: + def find_in_file( + self, filename: Union[str, os.PathLike], regex + ) -> Optional[re.Match]: """Find first pattern occurrence in file""" pattern = re.compile(regex) with Path(filename).open("r", encoding="utf-8") as f: @@ -116,68 +130,75 @@ class ProjectTest(unittest.TestCase): class StartprojectTest(ProjectTest): - def test_startproject(self): - p, out, err = self.proc('startproject', self.project_name) + p, out, err = self.proc("startproject", self.project_name) print(out) print(err, file=sys.stderr) self.assertEqual(p.returncode, 0) - assert Path(self.proj_path, 'scrapy.cfg').exists() - assert Path(self.proj_path, 'testproject').exists() - assert Path(self.proj_mod_path, '__init__.py').exists() - assert Path(self.proj_mod_path, 'items.py').exists() - assert Path(self.proj_mod_path, 'pipelines.py').exists() - assert Path(self.proj_mod_path, 'settings.py').exists() - assert Path(self.proj_mod_path, 'spiders', '__init__.py').exists() + assert Path(self.proj_path, "scrapy.cfg").exists() + assert Path(self.proj_path, "testproject").exists() + assert Path(self.proj_mod_path, "__init__.py").exists() + assert Path(self.proj_mod_path, "items.py").exists() + assert Path(self.proj_mod_path, "pipelines.py").exists() + assert Path(self.proj_mod_path, "settings.py").exists() + assert Path(self.proj_mod_path, "spiders", "__init__.py").exists() - self.assertEqual(1, self.call('startproject', self.project_name)) - self.assertEqual(1, self.call('startproject', 'wrong---project---name')) - self.assertEqual(1, self.call('startproject', 'sys')) + self.assertEqual(1, self.call("startproject", self.project_name)) + self.assertEqual(1, self.call("startproject", "wrong---project---name")) + self.assertEqual(1, self.call("startproject", "sys")) def test_startproject_with_project_dir(self): project_dir = mkdtemp() - self.assertEqual(0, self.call('startproject', self.project_name, project_dir)) + self.assertEqual(0, self.call("startproject", self.project_name, project_dir)) - assert Path(project_dir, 'scrapy.cfg').exists() - assert Path(project_dir, 'testproject').exists() - assert Path(project_dir, self.project_name, '__init__.py').exists() - assert Path(project_dir, self.project_name, 'items.py').exists() - assert Path(project_dir, self.project_name, 'pipelines.py').exists() - assert Path(project_dir, self.project_name, 'settings.py').exists() - assert Path(project_dir, self.project_name, 'spiders', '__init__.py').exists() + assert Path(project_dir, "scrapy.cfg").exists() + assert Path(project_dir, "testproject").exists() + assert Path(project_dir, self.project_name, "__init__.py").exists() + assert Path(project_dir, self.project_name, "items.py").exists() + assert Path(project_dir, self.project_name, "pipelines.py").exists() + assert Path(project_dir, self.project_name, "settings.py").exists() + assert Path(project_dir, self.project_name, "spiders", "__init__.py").exists() - self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2')) + self.assertEqual( + 0, self.call("startproject", self.project_name, project_dir + "2") + ) - self.assertEqual(1, self.call('startproject', self.project_name, project_dir)) - self.assertEqual(1, self.call('startproject', self.project_name + '2', project_dir)) - self.assertEqual(1, self.call('startproject', 'wrong---project---name')) - self.assertEqual(1, self.call('startproject', 'sys')) - self.assertEqual(2, self.call('startproject')) - self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params')) + self.assertEqual(1, self.call("startproject", self.project_name, project_dir)) + self.assertEqual( + 1, self.call("startproject", self.project_name + "2", project_dir) + ) + self.assertEqual(1, self.call("startproject", "wrong---project---name")) + self.assertEqual(1, self.call("startproject", "sys")) + self.assertEqual(2, self.call("startproject")) + self.assertEqual( + 2, + self.call("startproject", self.project_name, project_dir, "another_params"), + ) def test_existing_project_dir(self): project_dir = mkdtemp() - project_name = self.project_name + '_existing' + project_name = self.project_name + "_existing" project_path = Path(project_dir, project_name) project_path.mkdir() - p, out, err = self.proc('startproject', project_name, cwd=project_dir) + p, out, err = self.proc("startproject", project_name, cwd=project_dir) print(out) print(err, file=sys.stderr) self.assertEqual(p.returncode, 0) - assert Path(project_path, 'scrapy.cfg').exists() + assert Path(project_path, "scrapy.cfg").exists() assert Path(project_path, project_name).exists() - assert Path(project_path, project_name, '__init__.py').exists() - assert Path(project_path, project_name, 'items.py').exists() - assert Path(project_path, project_name, 'pipelines.py').exists() - assert Path(project_path, project_name, 'settings.py').exists() - assert Path(project_path, project_name, 'spiders', '__init__.py').exists() + assert Path(project_path, project_name, "__init__.py").exists() + assert Path(project_path, project_name, "items.py").exists() + assert Path(project_path, project_name, "pipelines.py").exists() + assert Path(project_path, project_name, "settings.py").exists() + assert Path(project_path, project_name, "spiders", "__init__.py").exists() -def get_permissions_dict(path: Union[str, os.PathLike], renamings=None, ignore=None) -> Dict[str, str]: - +def get_permissions_dict( + path: Union[str, os.PathLike], renamings=None, ignore=None +) -> Dict[str, str]: def get_permissions(path: Path) -> str: return oct(path.stat().st_mode) @@ -185,7 +206,7 @@ def get_permissions_dict(path: Union[str, os.PathLike], renamings=None, ignore=N renamings = renamings or tuple() permissions_dict = { - '.': get_permissions(path_obj), + ".": get_permissions(path_obj), } for root, dirs, files in os.walk(path_obj): nodes = list(chain(dirs, files)) @@ -196,10 +217,7 @@ def get_permissions_dict(path: Union[str, os.PathLike], renamings=None, ignore=N absolute_path = Path(root, node) relative_path = str(absolute_path.relative_to(path)) for search_string, replacement in renamings: - relative_path = relative_path.replace( - search_string, - replacement - ) + relative_path = relative_path.replace(search_string, replacement) permissions = get_permissions(absolute_path) permissions_dict[relative_path] = permissions return permissions_dict @@ -211,31 +229,33 @@ class StartprojectTemplatesTest(ProjectTest): def setUp(self): super().setUp() - self.tmpl = str(Path(self.temp_path, 'templates')) - self.tmpl_proj = str(Path(self.tmpl, 'project')) + self.tmpl = str(Path(self.temp_path, "templates")) + self.tmpl_proj = str(Path(self.tmpl, "project")) def test_startproject_template_override(self): - copytree(Path(scrapy.__path__[0], 'templates'), self.tmpl) - Path(self.tmpl_proj, 'root_template').write_bytes(b"") - assert Path(self.tmpl_proj, 'root_template').exists() + copytree(Path(scrapy.__path__[0], "templates"), self.tmpl) + Path(self.tmpl_proj, "root_template").write_bytes(b"") + assert Path(self.tmpl_proj, "root_template").exists() - args = ['--set', f'TEMPLATES_DIR={self.tmpl}'] - p, out, err = self.proc('startproject', self.project_name, *args) - self.assertIn(f"New Scrapy project '{self.project_name}', " - "using template directory", out) + args = ["--set", f"TEMPLATES_DIR={self.tmpl}"] + p, out, err = self.proc("startproject", self.project_name, *args) + self.assertIn( + f"New Scrapy project '{self.project_name}', " "using template directory", + out, + ) self.assertIn(self.tmpl_proj, out) - assert Path(self.proj_path, 'root_template').exists() + assert Path(self.proj_path, "root_template").exists() def test_startproject_permissions_from_writable(self): """Check that generated files have the right permissions when the template folder has the same permissions as in the project, i.e. everything is writable.""" scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, 'templates', 'project') - project_name = 'startproject1' + project_template = Path(scrapy_path, "templates", "project") + project_name = "startproject1" renamings = ( - ('module', project_name), - ('.tmpl', ''), + ("module", project_name), + (".tmpl", ""), ) expected_permissions = get_permissions_dict( project_template, @@ -247,9 +267,9 @@ class StartprojectTemplatesTest(ProjectTest): process = subprocess.Popen( ( sys.executable, - '-m', - 'scrapy.cmdline', - 'startproject', + "-m", + "scrapy.cmdline", + "startproject", project_name, ), cwd=destination, @@ -270,12 +290,12 @@ class StartprojectTemplatesTest(ProjectTest): See https://github.com/scrapy/scrapy/pull/4604 """ scrapy_path = scrapy.__path__[0] - templates_dir = Path(scrapy_path, 'templates') - project_template = Path(templates_dir, 'project') - project_name = 'startproject2' + templates_dir = Path(scrapy_path, "templates") + project_template = Path(templates_dir, "project") + project_name = "startproject2" renamings = ( - ('module', project_name), - ('.tmpl', ''), + ("module", project_name), + (".tmpl", ""), ) expected_permissions = get_permissions_dict( project_template, @@ -287,7 +307,7 @@ class StartprojectTemplatesTest(ProjectTest): current_permissions = path.stat().st_mode path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION) - read_only_templates_dir = str(Path(mkdtemp()) / 'templates') + read_only_templates_dir = str(Path(mkdtemp()) / "templates") copytree(templates_dir, read_only_templates_dir) for root, dirs, files in os.walk(read_only_templates_dir): @@ -298,12 +318,12 @@ class StartprojectTemplatesTest(ProjectTest): process = subprocess.Popen( ( sys.executable, - '-m', - 'scrapy.cmdline', - 'startproject', + "-m", + "scrapy.cmdline", + "startproject", project_name, - '--set', - f'TEMPLATES_DIR={read_only_templates_dir}', + "--set", + f"TEMPLATES_DIR={read_only_templates_dir}", ), cwd=destination, env=self.env, @@ -319,11 +339,11 @@ class StartprojectTemplatesTest(ProjectTest): """Check that pre-existing folders and files in the destination folder do not see their permissions modified.""" scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, 'templates', 'project') - project_name = 'startproject3' + project_template = Path(scrapy_path, "templates", "project") + project_name = "startproject3" renamings = ( - ('module', project_name), - ('.tmpl', ''), + ("module", project_name), + (".tmpl", ""), ) expected_permissions = get_permissions_dict( project_template, @@ -336,15 +356,20 @@ class StartprojectTemplatesTest(ProjectTest): existing_nodes = { oct(permissions)[2:] + extension: permissions - for extension in ('', '.d') + for extension in ("", ".d") for permissions in ( - 0o444, 0o555, 0o644, 0o666, 0o755, 0o777, + 0o444, + 0o555, + 0o644, + 0o666, + 0o755, + 0o777, ) } project_dir.mkdir() for node, permissions in existing_nodes.items(): path = project_dir / node - if node.endswith('.d'): + if node.endswith(".d"): path.mkdir(mode=permissions) else: path.touch(mode=permissions) @@ -353,11 +378,11 @@ class StartprojectTemplatesTest(ProjectTest): process = subprocess.Popen( ( sys.executable, - '-m', - 'scrapy.cmdline', - 'startproject', + "-m", + "scrapy.cmdline", + "startproject", project_name, - '.', + ".", ), cwd=project_dir, env=self.env, @@ -372,6 +397,7 @@ class StartprojectTemplatesTest(ProjectTest): """Check that generated files have the right permissions when the system uses a umask value that causes new files to have different permissions than those from the template folder.""" + @contextmanager def umask(new_mask): cur_mask = os.umask(new_mask) @@ -379,15 +405,11 @@ class StartprojectTemplatesTest(ProjectTest): os.umask(cur_mask) scrapy_path = scrapy.__path__[0] - project_template = Path( - scrapy_path, - 'templates', - 'project' - ) - project_name = 'umaskproject' + project_template = Path(scrapy_path, "templates", "project") + project_name = "umaskproject" renamings = ( - ('module', project_name), - ('.tmpl', ''), + ("module", project_name), + (".tmpl", ""), ) expected_permissions = get_permissions_dict( project_template, @@ -400,9 +422,9 @@ class StartprojectTemplatesTest(ProjectTest): process = subprocess.Popen( ( sys.executable, - '-m', - 'scrapy.cmdline', - 'startproject', + "-m", + "scrapy.cmdline", + "startproject", project_name, ), cwd=destination, @@ -417,67 +439,74 @@ class StartprojectTemplatesTest(ProjectTest): class CommandTest(ProjectTest): - def setUp(self): super().setUp() - self.call('startproject', self.project_name) + self.call("startproject", self.project_name) self.cwd = Path(self.temp_path, self.project_name) - self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings' + self.env["SCRAPY_SETTINGS_MODULE"] = f"{self.project_name}.settings" class GenspiderCommandTest(CommandTest): - def test_arguments(self): # only pass one argument. spider script shouldn't be created - self.assertEqual(2, self.call('genspider', 'test_name')) - assert not Path(self.proj_mod_path, 'spiders', 'test_name.py').exists() + self.assertEqual(2, self.call("genspider", "test_name")) + assert not Path(self.proj_mod_path, "spiders", "test_name.py").exists() # pass two arguments . spider script should be created - self.assertEqual(0, self.call('genspider', 'test_name', 'test.com')) - assert Path(self.proj_mod_path, 'spiders', 'test_name.py').exists() + self.assertEqual(0, self.call("genspider", "test_name", "test.com")) + assert Path(self.proj_mod_path, "spiders", "test_name.py").exists() - def test_template(self, tplname='crawl'): - args = [f'--template={tplname}'] if tplname else [] - spname = 'test_spider' + def test_template(self, tplname="crawl"): + args = [f"--template={tplname}"] if tplname else [] + spname = "test_spider" spmodule = f"{self.project_name}.spiders.{spname}" - p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out) - self.assertTrue(Path(self.proj_mod_path, 'spiders', 'test_spider.py').exists()) - modify_time_before = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime - p, out, err = self.proc('genspider', spname, 'test.com', *args) + p, out, err = self.proc("genspider", spname, "test.com", *args) + self.assertIn( + f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", + out, + ) + self.assertTrue(Path(self.proj_mod_path, "spiders", "test_spider.py").exists()) + modify_time_before = ( + Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime + ) + p, out, err = self.proc("genspider", spname, "test.com", *args) self.assertIn(f"Spider {spname!r} already exists in module", out) - modify_time_after = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime + modify_time_after = ( + Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime + ) self.assertEqual(modify_time_after, modify_time_before) def test_template_basic(self): - self.test_template('basic') + self.test_template("basic") def test_template_csvfeed(self): - self.test_template('csvfeed') + self.test_template("csvfeed") def test_template_xmlfeed(self): - self.test_template('xmlfeed') + self.test_template("xmlfeed") def test_list(self): - self.assertEqual(0, self.call('genspider', '--list')) + self.assertEqual(0, self.call("genspider", "--list")) def test_dump(self): - self.assertEqual(0, self.call('genspider', '--dump=basic')) - self.assertEqual(0, self.call('genspider', '-d', 'basic')) + self.assertEqual(0, self.call("genspider", "--dump=basic")) + self.assertEqual(0, self.call("genspider", "-d", "basic")) def test_same_name_as_project(self): - self.assertEqual(2, self.call('genspider', self.project_name)) - assert not Path(self.proj_mod_path, 'spiders', f'{self.project_name}.py').exists() + self.assertEqual(2, self.call("genspider", self.project_name)) + assert not Path( + self.proj_mod_path, "spiders", f"{self.project_name}.py" + ).exists() def test_same_filename_as_existing_spider(self, force=False): - file_name = 'example' - file_path = Path(self.proj_mod_path, 'spiders', f'{file_name}.py') - self.assertEqual(0, self.call('genspider', file_name, 'example.com')) + file_name = "example" + file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py") + self.assertEqual(0, self.call("genspider", file_name, "example.com")) assert file_path.exists() # change name of spider but not its file name - with file_path.open('r+', encoding="utf-8") as spider_file: + with file_path.open("r+", encoding="utf-8") as spider_file: file_data = spider_file.read() - file_data = file_data.replace("name = \'example\'", "name = \'renamed\'") + file_data = file_data.replace("name = 'example'", "name = 'renamed'") spider_file.seek(0) spider_file.write(file_data) spider_file.truncate() @@ -485,14 +514,16 @@ class GenspiderCommandTest(CommandTest): file_contents_before = file_data if force: - p, out, err = self.proc('genspider', '--force', file_name, 'example.com') - self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out) + p, out, err = self.proc("genspider", "--force", file_name, "example.com") + self.assertIn( + f"Created spider {file_name!r} using template 'basic' in module", out + ) modify_time_after = file_path.stat().st_mtime self.assertNotEqual(modify_time_after, modify_time_before) file_contents_after = file_path.read_text(encoding="utf-8") self.assertNotEqual(file_contents_after, file_contents_before) else: - p, out, err = self.proc('genspider', file_name, 'example.com') + p, out, err = self.proc("genspider", file_name, "example.com") self.assertIn(f"{file_path.resolve()} already exists", out) modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) @@ -502,53 +533,63 @@ class GenspiderCommandTest(CommandTest): def test_same_filename_as_existing_spider_force(self): self.test_same_filename_as_existing_spider(force=True) - def test_url(self, url='test.com', domain="test.com"): - self.assertEqual(0, self.call('genspider', '--force', 'test_name', url)) - self.assertEqual(domain, - self.find_in_file(Path(self.proj_mod_path, - 'spiders', 'test_name.py'), - r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1)) - self.assertEqual(f'http://{domain}/', - self.find_in_file(Path(self.proj_mod_path, - 'spiders', 'test_name.py'), - r'start_urls\s*=\s*\[\'(.+)\'\]').group(1)) + def test_url(self, url="test.com", domain="test.com"): + self.assertEqual(0, self.call("genspider", "--force", "test_name", url)) + self.assertEqual( + domain, + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"allowed_domains\s*=\s*\[\'(.+)\'\]", + ).group(1), + ) + self.assertEqual( + f"http://{domain}/", + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"start_urls\s*=\s*\[\'(.+)\'\]", + ).group(1), + ) def test_url_schema(self): - self.test_url('http://test.com', 'test.com') + self.test_url("http://test.com", "test.com") def test_url_path(self): - self.test_url('test.com/some/other/page', 'test.com') + self.test_url("test.com/some/other/page", "test.com") def test_url_schema_path(self): - self.test_url('https://test.com/some/other/page', 'test.com') + self.test_url("https://test.com/some/other/page", "test.com") class GenspiderStandaloneCommandTest(ProjectTest): - def test_generate_standalone_spider(self): - self.call('genspider', 'example', 'example.com') - assert Path(self.temp_path, 'example.py').exists() + self.call("genspider", "example", "example.com") + assert Path(self.temp_path, "example.py").exists() def test_same_name_as_existing_file(self, force=False): - file_name = 'example' - file_path = Path(self.temp_path, file_name + '.py') - p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out) + file_name = "example" + file_path = Path(self.temp_path, file_name + ".py") + p, out, err = self.proc("genspider", file_name, "example.com") + self.assertIn(f"Created spider {file_name!r} using template 'basic' ", out) assert file_path.exists() modify_time_before = file_path.stat().st_mtime file_contents_before = file_path.read_text(encoding="utf-8") if force: # use different template to ensure contents were changed - p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com') - self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out) + p, out, err = self.proc( + "genspider", "--force", "-t", "crawl", file_name, "example.com" + ) + self.assertIn(f"Created spider {file_name!r} using template 'crawl' ", out) modify_time_after = file_path.stat().st_mtime self.assertNotEqual(modify_time_after, modify_time_before) file_contents_after = file_path.read_text(encoding="utf-8") self.assertNotEqual(file_contents_after, file_contents_before) else: - p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn(f"{Path(self.temp_path, file_name + '.py').resolve()} already exists", out) + p, out, err = self.proc("genspider", file_name, "example.com") + self.assertIn( + f"{Path(self.temp_path, file_name + '.py').resolve()} already exists", + out, + ) modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) file_contents_after = file_path.read_text(encoding="utf-8") @@ -559,14 +600,13 @@ class GenspiderStandaloneCommandTest(ProjectTest): class MiscCommandsTest(CommandTest): - def test_list(self): - self.assertEqual(0, self.call('list')) + self.assertEqual(0, self.call("list")) class RunSpiderCommandTest(CommandTest): - spider_filename = 'myspider.py' + spider_filename = "myspider.py" debug_log_spider = """ import scrapy @@ -604,7 +644,7 @@ class BadSpider(scrapy.Spider): def runspider(self, code, name=None, args=()): with self._create_file(code, name) as fname: - return self.proc('runspider', fname, *args) + return self.proc("runspider", fname, *args) def get_log(self, code, name=None, args=()): p, stdout, stderr = self.runspider(code, name, args=args) @@ -618,18 +658,21 @@ class BadSpider(scrapy.Spider): self.assertIn("INFO: Spider closed (finished)", log) def test_run_fail_spider(self): - proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider)) + proc, _, _ = self.runspider( + "import scrapy\n" + inspect.getsource(ExceptionSpider) + ) ret = proc.returncode self.assertNotEqual(ret, 0) def test_run_good_spider(self): - proc, _, _ = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider)) + proc, _, _ = self.runspider( + "import scrapy\n" + inspect.getsource(NoRequestsSpider) + ) ret = proc.returncode self.assertEqual(ret, 0) def test_runspider_log_level(self): - log = self.get_log(self.debug_log_spider, - args=('-s', 'LOG_LEVEL=INFO')) + log = self.get_log(self.debug_log_spider, args=("-s", "LOG_LEVEL=INFO")) self.assertNotIn("DEBUG: It Works!", log) self.assertIn("INFO: Spider opened", log) @@ -649,19 +692,17 @@ class MySpider(scrapy.Spider): def parse(self, response): return {'test': 'value'} """ - log = self.get_log(dnscache_spider, args=('-s', 'DNSCACHE_ENABLED=False')) + log = self.get_log(dnscache_spider, args=("-s", "DNSCACHE_ENABLED=False")) self.assertNotIn("DNSLookupError", log) self.assertIn("INFO: Spider opened", log) def test_runspider_log_short_names(self): - log1 = self.get_log(self.debug_log_spider, - args=('-s', 'LOG_SHORT_NAMES=1')) + log1 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=1")) self.assertIn("[myspider] DEBUG: It Works!", log1) self.assertIn("[scrapy]", log1) self.assertNotIn("[scrapy.core.engine]", log1) - log2 = self.get_log(self.debug_log_spider, - args=('-s', 'LOG_SHORT_NAMES=0')) + log2 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=0")) self.assertIn("[myspider] DEBUG: It Works!", log2) self.assertNotIn("[scrapy]", log2) self.assertIn("[scrapy.core.engine]", log2) @@ -671,57 +712,89 @@ class MySpider(scrapy.Spider): self.assertIn("No spider found in file", log) def test_runspider_file_not_found(self): - _, _, log = self.proc('runspider', 'some_non_existent_file') + _, _, log = self.proc("runspider", "some_non_existent_file") self.assertIn("File not found: some_non_existent_file", log) def test_runspider_unable_to_load(self): - log = self.get_log('', name='myspider.txt') - self.assertIn('Unable to load', log) + log = self.get_log("", name="myspider.txt") + self.assertIn("Unable to load", log) def test_start_requests_errors(self): - log = self.get_log(self.badspider, name='badspider.py') + log = self.get_log(self.badspider, name="badspider.py") self.assertIn("start_requests", log) self.assertIn("badspider.py", log) def test_asyncio_enabled_true(self): - log = self.get_log(self.debug_log_spider, args=[ - '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' - ]) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + log = self.get_log( + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + ], + ) + self.assertIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) def test_asyncio_enabled_default(self): log = self.get_log(self.debug_log_spider, args=[]) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) def test_asyncio_enabled_false(self): - log = self.get_log(self.debug_log_spider, args=[ - '-s', 'TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor' - ]) - self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) - self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + log = self.get_log( + self.debug_log_spider, + args=["-s", "TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor"], + ) + self.assertIn( + "Using reactor: twisted.internet.selectreactor.SelectReactor", log + ) + self.assertNotIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) - @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') - @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') - @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + @mark.skipif( + sys.implementation.name == "pypy", + reason="uvloop does not support pypy properly", + ) + @mark.skipif( + platform.system() == "Windows", reason="uvloop does not support Windows" + ) + @mark.skipif( + twisted_version == Version("twisted", 21, 2, 0), + reason="https://twistedmatrix.com/trac/ticket/10106", + ) def test_custom_asyncio_loop_enabled_true(self): - log = self.get_log(self.debug_log_spider, args=[ - '-s', - 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor', - '-s', - 'ASYNCIO_EVENT_LOOP=uvloop.Loop', - ]) + log = self.get_log( + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "-s", + "ASYNCIO_EVENT_LOOP=uvloop.Loop", + ], + ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) def test_custom_asyncio_loop_enabled_false(self): - log = self.get_log(self.debug_log_spider, args=[ - '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' - ]) + log = self.get_log( + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + ], + ) import asyncio - if sys.platform != 'win32': + + if sys.platform != "win32": loop = asyncio.new_event_loop() else: loop = asyncio.SelectorEventLoop() - self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log) + self.assertIn( + f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", + log, + ) def test_output(self): spider_code = """ @@ -734,9 +807,11 @@ class MySpider(scrapy.Spider): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) return [] """ - args = ['-o', 'example.json'] + args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) - self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log) + self.assertIn( + "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log + ) def test_overwrite_output(self): spider_code = """ @@ -755,9 +830,12 @@ class MySpider(scrapy.Spider): return [] """ Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") - args = ['-O', 'example.json'] + args = ["-O", "example.json"] log = self.get_log(spider_code, args=args) - self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + self.assertIn( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', + log, + ) with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: first_line = f2.readline() self.assertNotEqual(first_line, "not empty") @@ -772,9 +850,11 @@ class MySpider(scrapy.Spider): def start_requests(self): return [] """ - args = ['-o', 'example1.json', '-O', 'example2.json'] + args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) - self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) + self.assertIn( + "error: Please use only one of -o/--output and -O/--overwrite-output", log + ) def test_output_stdout(self): spider_code = """ @@ -787,21 +867,21 @@ class MySpider(scrapy.Spider): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) return [] """ - args = ['-o', '-:json'] + args = ["-o", "-:json"] log = self.get_log(spider_code, args=args) self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log) -@skipIf(platform.system() != 'Windows', "Windows required for .pyw files") +@skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): - spider_filename = 'myspider.pyw' + spider_filename = "myspider.pyw" def setUp(self): super().setUp() def test_start_requests_errors(self): - log = self.get_log(self.badspider, name='badspider.pyw') + log = self.get_log(self.badspider, name="badspider.pyw") self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) @@ -834,34 +914,37 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): class BenchCommandTest(CommandTest): - def test_run(self): - _, _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', - '-s', 'CLOSESPIDER_TIMEOUT=0.01') - self.assertIn('INFO: Crawled', log) - self.assertNotIn('Unhandled Error', log) + _, _, log = self.proc( + "bench", "-s", "LOGSTATS_INTERVAL=0.001", "-s", "CLOSESPIDER_TIMEOUT=0.01" + ) + self.assertIn("INFO: Crawled", log) + self.assertNotIn("Unhandled Error", log) class ViewCommandTest(CommandTest): - def test_methods(self): command = view.Command() command.settings = Settings() - parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-', - formatter_class=ScrapyHelpFormatter, - conflict_handler='resolve') + parser = argparse.ArgumentParser( + prog="scrapy", + prefix_chars="-", + formatter_class=ScrapyHelpFormatter, + conflict_handler="resolve", + ) command.add_options(parser) - self.assertEqual(command.short_desc(), - "Open URL in browser, as seen by Scrapy") - self.assertIn("URL using the Scrapy downloader and show its", - command.long_desc()) + self.assertEqual(command.short_desc(), "Open URL in browser, as seen by Scrapy") + self.assertIn( + "URL using the Scrapy downloader and show its", command.long_desc() + ) class CrawlCommandTest(CommandTest): - def crawl(self, code, args=()): - Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code, encoding="utf-8") - return self.proc('crawl', 'myspider', *args) + Path(self.proj_mod_path, "spiders", "myspider.py").write_text( + code, encoding="utf-8" + ) + return self.proc("crawl", "myspider", *args) def get_log(self, code, args=()): _, _, stderr = self.crawl(code, args=args) @@ -892,9 +975,11 @@ class MySpider(scrapy.Spider): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) return [] """ - args = ['-o', 'example.json'] + args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) - self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log) + self.assertIn( + "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log + ) def test_overwrite_output(self): spider_code = """ @@ -913,9 +998,12 @@ class MySpider(scrapy.Spider): return [] """ Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") - args = ['-O', 'example.json'] + args = ["-O", "example.json"] log = self.get_log(spider_code, args=args) - self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + self.assertIn( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', + log, + ) with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: first_line = f2.readline() self.assertNotEqual(first_line, "not empty") @@ -930,18 +1018,32 @@ class MySpider(scrapy.Spider): def start_requests(self): return [] """ - args = ['-o', 'example1.json', '-O', 'example2.json'] + args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) - self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) + self.assertIn( + "error: Please use only one of -o/--output and -O/--overwrite-output", log + ) class HelpMessageTest(CommandTest): - def setUp(self): super().setUp() - self.commands = ["parse", "startproject", "view", "crawl", "edit", - "list", "fetch", "settings", "shell", "runspider", - "version", "genspider", "check", "bench"] + self.commands = [ + "parse", + "startproject", + "view", + "crawl", + "edit", + "list", + "fetch", + "settings", + "shell", + "runspider", + "version", + "genspider", + "check", + "bench", + ] def test_help_messages(self): for command in self.commands: diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 136056f50..7b104f618 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -26,52 +26,52 @@ class TestItem(Item): class ResponseMock: - url = 'http://scrapy.org' + url = "http://scrapy.org" class CustomSuccessContract(Contract): - name = 'custom_success_contract' + name = "custom_success_contract" def adjust_request_args(self, args): - args['url'] = 'http://scrapy.org' + args["url"] = "http://scrapy.org" return args class CustomFailContract(Contract): - name = 'custom_fail_contract' + name = "custom_fail_contract" def adjust_request_args(self, args): - raise TypeError('Error in adjust_request_args') + raise TypeError("Error in adjust_request_args") class CustomFormContract(Contract): - name = 'custom_form' + name = "custom_form" request_cls = FormRequest def adjust_request_args(self, args): - args['formdata'] = {'name': 'scrapy'} + args["formdata"] = {"name": "scrapy"} return args class TestSpider(Spider): - name = 'demo_spider' + name = "demo_spider" def returns_request(self, response): - """ method which returns request + """method which returns request @url http://scrapy.org @returns requests 1 """ - return Request('http://scrapy.org', callback=self.returns_item) + return Request("http://scrapy.org", callback=self.returns_item) def returns_item(self, response): - """ method which returns item + """method which returns item @url http://scrapy.org @returns items 1 1 """ return TestItem(url=response.url) def returns_request_cb_kwargs(self, response, url): - """ method which returns request + """method which returns request @url https://example.org @cb_kwargs {"url": "http://scrapy.org"} @returns requests 1 @@ -79,7 +79,7 @@ class TestSpider(Spider): return Request(url, callback=self.returns_item_cb_kwargs) def returns_item_cb_kwargs(self, response, name): - """ method which returns item + """method which returns item @url http://scrapy.org @cb_kwargs {"name": "Scrapy"} @returns items 1 1 @@ -87,7 +87,7 @@ class TestSpider(Spider): return TestItem(name=name, url=response.url) def returns_item_cb_kwargs_error_unexpected_keyword(self, response): - """ method which returns item + """method which returns item @url http://scrapy.org @cb_kwargs {"arg": "value"} @returns items 1 1 @@ -95,51 +95,51 @@ class TestSpider(Spider): return TestItem(url=response.url) def returns_item_cb_kwargs_error_missing_argument(self, response, arg): - """ method which returns item + """method which returns item @url http://scrapy.org @returns items 1 1 """ return TestItem(url=response.url) def returns_dict_item(self, response): - """ method which returns item + """method which returns item @url http://scrapy.org @returns items 1 1 """ return {"url": response.url} def returns_fail(self, response): - """ method which returns item + """method which returns item @url http://scrapy.org @returns items 0 0 """ return TestItem(url=response.url) def returns_dict_fail(self, response): - """ method which returns item + """method which returns item @url http://scrapy.org @returns items 0 0 """ - return {'url': response.url} + return {"url": response.url} def scrapes_item_ok(self, response): - """ returns item with name and url + """returns item with name and url @url http://scrapy.org @returns items 1 1 @scrapes name url """ - return TestItem(name='test', url=response.url) + return TestItem(name="test", url=response.url) def scrapes_dict_item_ok(self, response): - """ returns item with name and url + """returns item with name and url @url http://scrapy.org @returns items 1 1 @scrapes name url """ - return {'name': 'test', 'url': response.url} + return {"name": "test", "url": response.url} def scrapes_item_fail(self, response): - """ returns item with no name + """returns item with no name @url http://scrapy.org @returns items 1 1 @scrapes name url @@ -147,15 +147,15 @@ class TestSpider(Spider): return TestItem(url=response.url) def scrapes_dict_item_fail(self, response): - """ returns item with no name + """returns item with no name @url http://scrapy.org @returns items 1 1 @scrapes name url """ - return {'url': response.url} + return {"url": response.url} def scrapes_multiple_missing_fields(self, response): - """ returns item with no name + """returns item with no name @url http://scrapy.org @returns items 1 1 @scrapes name url @@ -163,7 +163,7 @@ class TestSpider(Spider): return {} def parse_no_url(self, response): - """ method with no url + """method with no url @returns items 1 1 """ pass @@ -177,7 +177,7 @@ class TestSpider(Spider): class CustomContractSuccessSpider(Spider): - name = 'custom_contract_success_spider' + name = "custom_contract_success_spider" def parse(self, response): """ @@ -187,7 +187,7 @@ class CustomContractSuccessSpider(Spider): class CustomContractFailSpider(Spider): - name = 'custom_contract_fail_spider' + name = "custom_contract_fail_spider" def parse(self, response): """ @@ -197,7 +197,7 @@ class CustomContractFailSpider(Spider): class InheritsTestSpider(TestSpider): - name = 'inherits_demo_spider' + name = "inherits_demo_spider" class ContractsManagerTest(unittest.TestCase): @@ -234,7 +234,8 @@ class ContractsManagerTest(unittest.TestCase): self.assertEqual(len(contracts), 2) self.assertEqual( frozenset(type(x) for x in contracts), - frozenset([UrlContract, ReturnsContract])) + frozenset([UrlContract, ReturnsContract]), + ) # returns request for valid method request = self.conman.from_method(spider.returns_request, self.results) @@ -251,26 +252,40 @@ class ContractsManagerTest(unittest.TestCase): # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request_cb_kwargs) self.assertEqual(len(contracts), 3) - self.assertEqual(frozenset(type(x) for x in contracts), - frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract])) + self.assertEqual( + frozenset(type(x) for x in contracts), + frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]), + ) contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs) self.assertEqual(len(contracts), 3) - self.assertEqual(frozenset(type(x) for x in contracts), - frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract])) + self.assertEqual( + frozenset(type(x) for x in contracts), + frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]), + ) - contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_unexpected_keyword) + contracts = self.conman.extract_contracts( + spider.returns_item_cb_kwargs_error_unexpected_keyword + ) self.assertEqual(len(contracts), 3) - self.assertEqual(frozenset(type(x) for x in contracts), - frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract])) + self.assertEqual( + frozenset(type(x) for x in contracts), + frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]), + ) - contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_missing_argument) + contracts = self.conman.extract_contracts( + spider.returns_item_cb_kwargs_error_missing_argument + ) self.assertEqual(len(contracts), 2) - self.assertEqual(frozenset(type(x) for x in contracts), - frozenset([UrlContract, ReturnsContract])) + self.assertEqual( + frozenset(type(x) for x in contracts), + frozenset([UrlContract, ReturnsContract]), + ) # returns_request - request = self.conman.from_method(spider.returns_request_cb_kwargs, self.results) + request = self.conman.from_method( + spider.returns_request_cb_kwargs, self.results + ) request.callback(response, **request.cb_kwargs) self.should_succeed() @@ -280,12 +295,16 @@ class ContractsManagerTest(unittest.TestCase): self.should_succeed() # returns_item (error, callback doesn't take keyword arguments) - request = self.conman.from_method(spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results) + request = self.conman.from_method( + spider.returns_item_cb_kwargs_error_unexpected_keyword, self.results + ) request.callback(response, **request.cb_kwargs) self.should_error() # returns_item (error, contract doesn't provide keyword arguments) - request = self.conman.from_method(spider.returns_item_cb_kwargs_error_missing_argument, self.results) + request = self.conman.from_method( + spider.returns_item_cb_kwargs_error_missing_argument, self.results + ) request.callback(response, **request.cb_kwargs) self.should_error() @@ -343,10 +362,12 @@ class ContractsManagerTest(unittest.TestCase): self.should_fail() # scrapes_multiple_missing_fields - request = self.conman.from_method(spider.scrapes_multiple_missing_fields, self.results) + request = self.conman.from_method( + spider.scrapes_multiple_missing_fields, self.results + ) request.callback(response) self.should_fail() - message = 'ContractFail: Missing fields: name, url' + message = "ContractFail: Missing fields: name, url" assert message in self.results.failures[-1][-1] def test_custom_contracts(self): @@ -361,7 +382,7 @@ class ContractsManagerTest(unittest.TestCase): response = ResponseMock() try: - raise HttpError(response, 'Ignoring non-200 response') + raise HttpError(response, "Ignoring non-200 response") except HttpError: failure_mock = failure.Failure() @@ -373,9 +394,8 @@ class ContractsManagerTest(unittest.TestCase): @defer.inlineCallbacks def test_same_url(self): - class TestSameUrlSpider(Spider): - name = 'test_same_url' + name = "test_same_url" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -406,7 +426,7 @@ class ContractsManagerTest(unittest.TestCase): def test_form_contract(self): spider = TestSpider() request = self.conman.from_method(spider.custom_form, self.results) - self.assertEqual(request.method, 'POST') + self.assertEqual(request.method, "POST") self.assertIsInstance(request, FormRequest) def test_inherited_contracts(self): diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 113ea8f19..9a6e9e4ff 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -4,7 +4,8 @@ from scrapy.core.downloader import Slot class SlotTest(unittest.TestCase): - def test_repr(self): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) - self.assertEqual(repr(slot), 'Slot(concurrency=8, delay=0.10, randomize_delay=True)') + self.assertEqual( + repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)" + ) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 5ec96e4a7..3f19bef22 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -53,7 +53,6 @@ from tests.spiders import ( class CrawlTestCase(TestCase): - def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() @@ -82,17 +81,17 @@ class CrawlTestCase(TestCase): mockserver=self.mockserver, total=total, ) - tolerance = (1 - (0.6 if randomize else 0.2)) + tolerance = 1 - (0.6 if randomize else 0.2) - settings = {"DOWNLOAD_DELAY": delay, - 'RANDOMIZE_DOWNLOAD_DELAY': randomize} + settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize} crawler = get_crawler(FollowAllSpider, settings) yield crawler.crawl(**crawl_kwargs) times = crawler.spider.times total_time = times[-1] - times[0] average = total_time / (len(times) - 1) - self.assertTrue(average > delay * tolerance, - f"download delay too small: {average}") + self.assertTrue( + average > delay * tolerance, f"download delay too small: {average}" + ) # Ensure that the same test parameters would cause a failure if no # download delay is set. Otherwise, it means we are using a combination @@ -104,8 +103,9 @@ class CrawlTestCase(TestCase): times = crawler.spider.times total_time = times[-1] - times[0] average = total_time / (len(times) - 1) - self.assertFalse(average > delay / tolerance, - "test total or delay values are too small") + self.assertFalse( + average > delay / tolerance, "test total or delay values are too small" + ) @defer.inlineCallbacks def test_timeout_success(self): @@ -134,14 +134,18 @@ class CrawlTestCase(TestCase): def test_retry_503(self): crawler = get_crawler(SimpleSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver) + yield crawler.crawl( + self.mockserver.url("/status?n=503"), mockserver=self.mockserver + ) self._assert_retried(log) @defer.inlineCallbacks def test_retry_conn_failed(self): crawler = get_crawler(SimpleSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver) + yield crawler.crawl( + "http://localhost:65432/status?n=503", mockserver=self.mockserver + ) self._assert_retried(log) @defer.inlineCallbacks @@ -151,12 +155,14 @@ class CrawlTestCase(TestCase): crawler = get_crawler(SimpleSpider) with LogCapture() as log: # try to fetch the homepage of a non-existent domain - yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) + yield crawler.crawl( + "http://dns.resolution.invalid./", mockserver=self.mockserver + ) self._assert_retried(log) @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): - with LogCapture('scrapy', level=logging.ERROR) as log: + with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) @@ -167,7 +173,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_yielding(self): - with LogCapture('scrapy', level=logging.ERROR) as log: + with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) @@ -183,16 +189,24 @@ class CrawlTestCase(TestCase): yield crawler.crawl(mockserver=self.mockserver) self.assertTrue( crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), - crawler.spider.seedsseen) + crawler.spider.seedsseen, + ) @defer.inlineCallbacks def test_start_requests_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(DuplicateStartRequestsSpider, settings) - yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver) + yield crawler.crawl( + dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver + ) self.assertEqual(crawler.spider.visited, 6) - yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4, mockserver=self.mockserver) + yield crawler.crawl( + dont_filter=False, + distinct_urls=3, + dupe_factor=4, + mockserver=self.mockserver, + ) self.assertEqual(crawler.spider.visited, 3) @defer.inlineCallbacks @@ -200,7 +214,10 @@ class CrawlTestCase(TestCase): # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" from urllib.parse import urlencode - query = urlencode({'raw': '''\ + + query = urlencode( + { + "raw": """\ HTTP/1.1 200 OK Server: Apache-Coyote/1.1 X-Powered-By: Servlet 2.4; JBoss-4.2.3.GA (build: SVNTag=JBoss_4_2_3_GA date=200807181417)/JBossWeb-2.0 @@ -216,10 +233,14 @@ Connection: close foo body with multiples lines -'''}) +""" + } + ) crawler = get_crawler(SimpleSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver) + yield crawler.crawl( + self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver + ) self.assertEqual(str(log).count("Got response 200"), 1) @defer.inlineCallbacks @@ -227,7 +248,9 @@ with multiples lines # connection lost after receiving data crawler = get_crawler(SimpleSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver) + yield crawler.crawl( + self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver + ) self._assert_retried(log) @defer.inlineCallbacks @@ -235,7 +258,9 @@ with multiples lines # connection lost before receiving data crawler = get_crawler(SimpleSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver) + yield crawler.crawl( + self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver + ) self._assert_retried(log) def _assert_retried(self, log): @@ -245,56 +270,62 @@ with multiples lines @defer.inlineCallbacks def test_referer_header(self): """Referer header is set by RefererMiddleware unless it is already set""" - req0 = Request(self.mockserver.url('/echo?headers=1&body=0'), dont_filter=1) + req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1) req1 = req0.replace() - req2 = req0.replace(headers={'Referer': None}) - req3 = req0.replace(headers={'Referer': 'http://example.com'}) - req0.meta['next'] = req1 - req1.meta['next'] = req2 - req2.meta['next'] = req3 + req2 = req0.replace(headers={"Referer": None}) + req3 = req0.replace(headers={"Referer": "http://example.com"}) + req0.meta["next"] = req1 + req1.meta["next"] = req2 + req2.meta["next"] = req3 crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=req0, mockserver=self.mockserver) # basic asserts in case of weird communication errors - self.assertIn('responses', crawler.spider.meta) - self.assertNotIn('failures', crawler.spider.meta) + self.assertIn("responses", crawler.spider.meta) + self.assertNotIn("failures", crawler.spider.meta) # start requests doesn't set Referer header - echo0 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body)) - self.assertNotIn('Referer', echo0['headers']) + echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) + self.assertNotIn("Referer", echo0["headers"]) # following request sets Referer to start request url - echo1 = json.loads(to_unicode(crawler.spider.meta['responses'][1].body)) - self.assertEqual(echo1['headers'].get('Referer'), [req0.url]) + echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) + self.assertEqual(echo1["headers"].get("Referer"), [req0.url]) # next request avoids Referer header - echo2 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body)) - self.assertNotIn('Referer', echo2['headers']) + echo2 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) + self.assertNotIn("Referer", echo2["headers"]) # last request explicitly sets a Referer header - echo3 = json.loads(to_unicode(crawler.spider.meta['responses'][3].body)) - self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com']) + echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body)) + self.assertEqual(echo3["headers"].get("Referer"), ["http://example.com"]) @defer.inlineCallbacks def test_engine_status(self): from scrapy.utils.engine import get_engine_status + est = [] def cb(response): est.append(get_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) + yield crawler.crawl( + seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver + ) self.assertEqual(len(est), 1, est) s = dict(est[0]) - self.assertEqual(s['engine.spider.name'], crawler.spider.name) - self.assertEqual(s['len(engine.scraper.slot.active)'], 1) + self.assertEqual(s["engine.spider.name"], crawler.spider.name) + self.assertEqual(s["len(engine.scraper.slot.active)"], 1) @defer.inlineCallbacks def test_format_engine_status(self): from scrapy.utils.engine import format_engine_status + est = [] def cb(response): est.append(format_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) + yield crawler.crawl( + seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver + ) self.assertEqual(len(est), 1, est) est = est[0].split("\n")[2:-2] # remove header & footer # convert to dict @@ -304,8 +335,8 @@ with multiples lines it = iter(est) s = dict(zip(it, it)) - self.assertEqual(s['engine.spider.name'], crawler.spider.name) - self.assertEqual(s['len(engine.scraper.slot.active)'], '1') + self.assertEqual(s["engine.spider.name"], crawler.spider.name) + self.assertEqual(s["len(engine.scraper.slot.active)"], "1") @defer.inlineCallbacks def test_graceful_crawl_error_handling(self): @@ -337,8 +368,11 @@ with multiples lines } crawler = get_crawler(SimpleSpider, settings) yield self.assertFailure( - crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver), - ZeroDivisionError) + crawler.crawl( + self.mockserver.url("/status?n=200"), mockserver=self.mockserver + ), + ZeroDivisionError, + ) self.assertFalse(crawler.crawling) @defer.inlineCallbacks @@ -346,14 +380,26 @@ with multiples lines crawler = get_crawler(SimpleSpider) runner = CrawlerRunner() with LogCapture() as log: - yield runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + yield runner.crawl( + crawler, + self.mockserver.url("/status?n=200"), + mockserver=self.mockserver, + ) self.assertIn("Got response 200", str(log)) @defer.inlineCallbacks def test_crawl_multiple(self): - runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}) - runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) - runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) + runner = CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"}) + runner.crawl( + SimpleSpider, + self.mockserver.url("/status?n=200"), + mockserver=self.mockserver, + ) + runner.crawl( + SimpleSpider, + self.mockserver.url("/status?n=503"), + mockserver=self.mockserver, + ) with LogCapture() as log: yield runner.join() @@ -363,7 +409,6 @@ with multiples lines class CrawlSpiderTestCase(TestCase): - def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() @@ -381,7 +426,9 @@ class CrawlSpiderTestCase(TestCase): crawler = get_crawler(spider_cls) crawler.signals.connect(_on_item_scraped, signals.item_scraped) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + yield crawler.crawl( + self.mockserver.url("/status?n=200"), mockserver=self.mockserver + ) return log, items, crawler.stats @defer.inlineCallbacks @@ -441,17 +488,24 @@ class CrawlSpiderTestCase(TestCase): def test_async_def_parse(self): crawler = get_crawler(AsyncDefSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + yield crawler.crawl( + self.mockserver.url("/status?n=200"), mockserver=self.mockserver + ) self.assertIn("Got response 200", str(log)) @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse(self): - crawler = get_crawler(AsyncDefAsyncioSpider, { - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor" - }) + crawler = get_crawler( + AsyncDefAsyncioSpider, + { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + }, + ) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + yield crawler.crawl( + self.mockserver.url("/status?n=200"), mockserver=self.mockserver + ) self.assertIn("Got response 200", str(log)) @mark.only_asyncio() @@ -459,8 +513,8 @@ class CrawlSpiderTestCase(TestCase): def test_async_def_asyncio_parse_items_list(self): log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) self.assertIn("Got response 200", str(log)) - self.assertIn({'id': 1}, items) - self.assertIn({'id': 2}, items) + self.assertIn({"id": 1}, items) + self.assertIn({"id": 2}, items) @mark.only_asyncio() @defer.inlineCallbacks @@ -473,7 +527,9 @@ class CrawlSpiderTestCase(TestCase): crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + yield crawler.crawl( + self.mockserver.url("/status?n=200"), mockserver=self.mockserver + ) self.assertIn("Got response 200", str(log)) self.assertIn({"foo": 42}, items) @@ -482,7 +538,7 @@ class CrawlSpiderTestCase(TestCase): def test_async_def_asyncgen_parse(self): log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) self.assertIn("Got response 200", str(log)) - itemcount = stats.get_value('item_scraped_count') + itemcount = stats.get_value("item_scraped_count") self.assertEqual(itemcount, 1) @mark.only_asyncio() @@ -490,10 +546,10 @@ class CrawlSpiderTestCase(TestCase): def test_async_def_asyncgen_parse_loop(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) self.assertIn("Got response 200", str(log)) - itemcount = stats.get_value('item_scraped_count') + itemcount = stats.get_value("item_scraped_count") self.assertEqual(itemcount, 10) for i in range(10): - self.assertIn({'foo': i}, items) + self.assertIn({"foo": i}, items) @mark.only_asyncio() @defer.inlineCallbacks @@ -502,22 +558,22 @@ class CrawlSpiderTestCase(TestCase): log = str(log) self.assertIn("Spider error processing", log) self.assertIn("ValueError", log) - itemcount = stats.get_value('item_scraped_count') + itemcount = stats.get_value("item_scraped_count") self.assertEqual(itemcount, 7) for i in range(7): - self.assertIn({'foo': i}, items) + self.assertIn({"foo": i}, items) @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) - itemcount = stats.get_value('item_scraped_count') + itemcount = stats.get_value("item_scraped_count") self.assertEqual(itemcount, 156) # some random items for i in [1, 4, 21, 22, 207, 311]: - self.assertIn({'index': i}, items) + self.assertIn({"index": i}, items) for i in [10, 30, 122]: - self.assertIn({'index2': i}, items) + self.assertIn({"index2": i}, items) @mark.only_asyncio() @defer.inlineCallbacks @@ -530,32 +586,32 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_async_def_deferred_direct(self): _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) - self.assertEqual(items, [{'code': 200}]) + self.assertEqual(items, [{"code": 200}]) @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_deferred_wrapped(self): log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) - self.assertEqual(items, [{'code': 200}]) + self.assertEqual(items, [{"code": 200}]) @defer.inlineCallbacks def test_async_def_deferred_maybe_wrapped(self): _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider) - self.assertEqual(items, [{'code': 200}]) + self.assertEqual(items, [{"code": 200}]) @defer.inlineCallbacks def test_response_ssl_certificate_none(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=False) yield crawler.crawl(seed=url, mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta['responses'][0].certificate) + self.assertIsNone(crawler.spider.meta["responses"][0].certificate) @defer.inlineCallbacks def test_response_ssl_certificate(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=True) yield crawler.crawl(seed=url, mockserver=self.mockserver) - cert = crawler.spider.meta['responses'][0].certificate + cert = crawler.spider.meta["responses"][0].certificate self.assertIsInstance(cert, Certificate) self.assertEqual(cert.getSubject().commonName, b"localhost") self.assertEqual(cert.getIssuer().commonName, b"localhost") @@ -566,7 +622,7 @@ class CrawlSpiderTestCase(TestCase): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/status?n=200", is_secure=True) yield crawler.crawl(seed=url, mockserver=self.mockserver) - cert = crawler.spider.meta['responses'][0].certificate + cert = crawler.spider.meta["responses"][0].certificate self.assertIsInstance(cert, Certificate) self.assertEqual(cert.getSubject().commonName, b"localhost") self.assertEqual(cert.getIssuer().commonName, b"localhost") @@ -574,18 +630,18 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_dns_server_ip_address_none(self): crawler = get_crawler(SingleRequestSpider) - url = self.mockserver.url('/status?n=200') + url = self.mockserver.url("/status?n=200") yield crawler.crawl(seed=url, mockserver=self.mockserver) - ip_address = crawler.spider.meta['responses'][0].ip_address + ip_address = crawler.spider.meta["responses"][0].ip_address self.assertIsNone(ip_address) @defer.inlineCallbacks def test_dns_server_ip_address(self): crawler = get_crawler(SingleRequestSpider) - url = self.mockserver.url('/echo?body=test') - expected_netloc, _ = urlparse(url).netloc.split(':') + url = self.mockserver.url("/echo?body=test") + expected_netloc, _ = urlparse(url).netloc.split(":") yield crawler.crawl(seed=url, mockserver=self.mockserver) - ip_address = crawler.spider.meta['responses'][0].ip_address + ip_address = crawler.spider.meta["responses"][0].ip_address self.assertIsInstance(ip_address, IPv4Address) self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) @@ -595,8 +651,14 @@ class CrawlSpiderTestCase(TestCase): yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("failure")) self.assertIsInstance(crawler.spider.meta["response"], Response) - self.assertEqual(crawler.spider.meta["response"].body, crawler.spider.meta.get("bytes_received")) - self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length) + self.assertEqual( + crawler.spider.meta["response"].body, + crawler.spider.meta.get("bytes_received"), + ) + self.assertLess( + len(crawler.spider.meta["response"].body), + crawler.spider.full_response_length, + ) @defer.inlineCallbacks def test_bytes_received_stop_download_errback(self): @@ -608,10 +670,12 @@ class CrawlSpiderTestCase(TestCase): self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) self.assertEqual( crawler.spider.meta["failure"].value.response.body, - crawler.spider.meta.get("bytes_received")) + crawler.spider.meta.get("bytes_received"), + ) self.assertLess( len(crawler.spider.meta["failure"].value.response.body), - crawler.spider.full_response_length) + crawler.spider.full_response_length, + ) @defer.inlineCallbacks def test_headers_received_stop_download_callback(self): @@ -619,7 +683,10 @@ class CrawlSpiderTestCase(TestCase): yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("failure")) self.assertIsInstance(crawler.spider.meta["response"], Response) - self.assertEqual(crawler.spider.meta["response"].headers, crawler.spider.meta.get("headers_received")) + self.assertEqual( + crawler.spider.meta["response"].headers, + crawler.spider.meta.get("headers_received"), + ) @defer.inlineCallbacks def test_headers_received_stop_download_errback(self): @@ -631,4 +698,5 @@ class CrawlSpiderTestCase(TestCase): self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) self.assertEqual( crawler.spider.meta["failure"].value.response.headers, - crawler.spider.meta.get("headers_received")) + crawler.spider.meta.get("headers_received"), + ) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index e0902fdbe..c6b93599e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -31,44 +31,42 @@ from tests.mockserver import MockServer class BaseCrawlerTest(unittest.TestCase): - def assertOptionIsDefault(self, settings, key): self.assertIsInstance(settings, Settings) self.assertEqual(settings[key], getattr(default_settings, key)) class CrawlerTestCase(BaseCrawlerTest): - def test_populate_spidercls_settings(self): - spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'} - project_settings = {'TEST1': 'project', 'TEST3': 'project'} + spider_settings = {"TEST1": "spider", "TEST2": "spider"} + project_settings = {"TEST1": "project", "TEST3": "project"} class CustomSettingsSpider(DefaultSpider): custom_settings = spider_settings settings = Settings() - settings.setdict(project_settings, priority='project') + settings.setdict(project_settings, priority="project") with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) crawler = Crawler(CustomSettingsSpider, settings) - self.assertEqual(crawler.settings.get('TEST1'), 'spider') - self.assertEqual(crawler.settings.get('TEST2'), 'spider') - self.assertEqual(crawler.settings.get('TEST3'), 'project') + self.assertEqual(crawler.settings.get("TEST1"), "spider") + self.assertEqual(crawler.settings.get("TEST2"), "spider") + self.assertEqual(crawler.settings.get("TEST3"), "project") self.assertFalse(settings.frozen) self.assertTrue(crawler.settings.frozen) def test_crawler_accepts_dict(self): - crawler = get_crawler(DefaultSpider, {'foo': 'bar'}) - self.assertEqual(crawler.settings['foo'], 'bar') - self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED') + crawler = get_crawler(DefaultSpider, {"foo": "bar"}) + self.assertEqual(crawler.settings["foo"], "bar") + self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") def test_crawler_accepts_None(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) crawler = Crawler(DefaultSpider) - self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED') + self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") def test_crawler_rejects_spider_objects(self): with raises(ValueError): @@ -78,10 +76,8 @@ class CrawlerTestCase(BaseCrawlerTest): class SpiderSettingsTestCase(unittest.TestCase): def test_spider_custom_settings(self): class MySpider(scrapy.Spider): - name = 'spider' - custom_settings = { - 'AUTOTHROTTLE_ENABLED': True - } + name = "spider" + custom_settings = {"AUTOTHROTTLE_ENABLED": True} crawler = get_crawler(MySpider) enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] @@ -95,73 +91,71 @@ class CrawlerLoggingTestCase(unittest.TestCase): logging.root.removeHandler(handler) class MySpider(scrapy.Spider): - name = 'spider' + name = "spider" get_crawler(MySpider) assert get_scrapy_root_handler() is None def test_spider_custom_settings_log_level(self): log_file = Path(self.mktemp()) - log_file.write_text('previous message\n', encoding='utf-8') + log_file.write_text("previous message\n", encoding="utf-8") class MySpider(scrapy.Spider): - name = 'spider' + name = "spider" custom_settings = { - 'LOG_LEVEL': 'INFO', - 'LOG_FILE': str(log_file), + "LOG_LEVEL": "INFO", + "LOG_FILE": str(log_file), # settings to avoid extra warnings - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', - 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } configure_logging() self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) crawler = get_crawler(MySpider) self.assertEqual(get_scrapy_root_handler().level, logging.INFO) - info_count = crawler.stats.get_value('log_count/INFO') - logging.debug('debug message') - logging.info('info message') - logging.warning('warning message') - logging.error('error message') + info_count = crawler.stats.get_value("log_count/INFO") + logging.debug("debug message") + logging.info("info message") + logging.warning("warning message") + logging.error("error message") - logged = log_file.read_text(encoding='utf-8') + logged = log_file.read_text(encoding="utf-8") - self.assertIn('previous message', logged) - self.assertNotIn('debug message', logged) - self.assertIn('info message', logged) - self.assertIn('warning message', logged) - self.assertIn('error message', logged) - self.assertEqual(crawler.stats.get_value('log_count/ERROR'), 1) - self.assertEqual(crawler.stats.get_value('log_count/WARNING'), 1) - self.assertEqual( - crawler.stats.get_value('log_count/INFO') - info_count, 1) - self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) + self.assertIn("previous message", logged) + self.assertNotIn("debug message", logged) + self.assertIn("info message", logged) + self.assertIn("warning message", logged) + self.assertIn("error message", logged) + self.assertEqual(crawler.stats.get_value("log_count/ERROR"), 1) + self.assertEqual(crawler.stats.get_value("log_count/WARNING"), 1) + self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1) + self.assertEqual(crawler.stats.get_value("log_count/DEBUG", 0), 0) def test_spider_custom_settings_log_append(self): log_file = Path(self.mktemp()) - log_file.write_text('previous message\n', encoding='utf-8') + log_file.write_text("previous message\n", encoding="utf-8") class MySpider(scrapy.Spider): - name = 'spider' + name = "spider" custom_settings = { - 'LOG_FILE': str(log_file), - 'LOG_FILE_APPEND': False, + "LOG_FILE": str(log_file), + "LOG_FILE_APPEND": False, # disable telnet if not available to avoid an extra warning - 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, + "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } configure_logging() get_crawler(MySpider) - logging.debug('debug message') + logging.debug("debug message") - logged = log_file.read_text(encoding='utf-8') + logged = log_file.read_text(encoding="utf-8") - self.assertNotIn('previous message', logged) - self.assertIn('debug message', logged) + self.assertNotIn("previous message", logged) + self.assertIn("debug message", logged) class SpiderLoaderWithWrongInterface: - def unneeded_method(self): pass @@ -171,11 +165,12 @@ class CustomSpiderLoader(SpiderLoader): class CrawlerRunnerTestCase(BaseCrawlerTest): - def test_spider_manager_verify_interface(self): - settings = Settings({ - 'SPIDER_LOADER_CLASS': SpiderLoaderWithWrongInterface, - }) + settings = Settings( + { + "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, + } + ) with warnings.catch_warnings(record=True) as w: self.assertRaises(AttributeError, CrawlerRunner, settings) self.assertEqual(len(w), 1) @@ -183,13 +178,13 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message)) def test_crawler_runner_accepts_dict(self): - runner = CrawlerRunner({'foo': 'bar'}) - self.assertEqual(runner.settings['foo'], 'bar') - self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED') + runner = CrawlerRunner({"foo": "bar"}) + self.assertEqual(runner.settings["foo"], "bar") + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") def test_crawler_runner_accepts_None(self): runner = CrawlerRunner() - self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED') + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") def test_deprecated_attribute_spiders(self): with warnings.catch_warnings(record=True) as w: @@ -198,41 +193,40 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): self.assertEqual(len(w), 1) self.assertIn("CrawlerRunner.spiders", str(w[0].message)) self.assertIn("CrawlerRunner.spider_loader", str(w[0].message)) - sl_cls = load_object(runner.settings['SPIDER_LOADER_CLASS']) + sl_cls = load_object(runner.settings["SPIDER_LOADER_CLASS"]) self.assertIsInstance(spiders, sl_cls) class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_dict(self): - runner = CrawlerProcess({'foo': 'bar'}) - self.assertEqual(runner.settings['foo'], 'bar') - self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED') + runner = CrawlerProcess({"foo": "bar"}) + self.assertEqual(runner.settings["foo"], "bar") + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") def test_crawler_process_accepts_None(self): runner = CrawlerProcess() - self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED') + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") class ExceptionSpider(scrapy.Spider): - name = 'exception' + name = "exception" @classmethod def from_crawler(cls, crawler, *args, **kwargs): - raise ValueError('Exception in from_crawler method') + raise ValueError("Exception in from_crawler method") class NoRequestsSpider(scrapy.Spider): - name = 'no_request' + name = "no_request" def start_requests(self): return [] -@mark.usefixtures('reactor_pytest') +@mark.usefixtures("reactor_pytest") class CrawlerRunnerHasSpider(unittest.TestCase): - def _runner(self): - return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}) + return CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"}) @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful(self): @@ -256,7 +250,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): except ValueError: pass else: - self.fail('Exception should be raised from spider') + self.fail("Exception should be raised from spider") self.assertEqual(runner.bootstrap_failed, True) @@ -269,7 +263,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): except ValueError: pass else: - self.fail('Exception should be raised from spider') + self.fail("Exception should be raised from spider") yield runner.crawl(NoRequestsSpider) @@ -277,18 +271,22 @@ class CrawlerRunnerHasSpider(unittest.TestCase): @defer.inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): - if self.reactor_pytest == 'asyncio': - CrawlerRunner(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - }) + if self.reactor_pytest == "asyncio": + CrawlerRunner( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } + ) else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - runner = CrawlerRunner(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - }) + runner = CrawlerRunner( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } + ) yield runner.crawl(NoRequestsSpider) @@ -298,35 +296,42 @@ class ScriptRunnerMixin: def run_script(self, script_name: str, *script_args): script_path = self.script_dir / script_name args = [sys.executable, str(script_path)] + list(script_args) - p = subprocess.Popen(args, env=get_testenv(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen( + args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) stdout, stderr = p.communicate() - return stderr.decode('utf-8') + return stderr.decode("utf-8") class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): - script_dir = Path(__file__).parent.resolve() / 'CrawlerProcess' + script_dir = Path(__file__).parent.resolve() / "CrawlerProcess" def test_simple(self): - log = self.run_script('simple.py') - self.assertIn('Spider closed (finished)', log) - self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + log = self.run_script("simple.py") + self.assertIn("Spider closed (finished)", log) + self.assertNotIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) def test_multi(self): - log = self.run_script('multi.py') - self.assertIn('Spider closed (finished)', log) - self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + log = self.run_script("multi.py") + self.assertIn("Spider closed (finished)", log) + self.assertNotIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) self.assertNotIn("ReactorAlreadyInstalledError", log) def test_reactor_default(self): - log = self.run_script('reactor_default.py') - self.assertIn('Spider closed (finished)', log) - self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + log = self.run_script("reactor_default.py") + self.assertIn("Spider closed (finished)", log) + self.assertNotIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) self.assertNotIn("ReactorAlreadyInstalledError", log) def test_reactor_default_twisted_reactor_select(self): - log = self.run_script('reactor_default_twisted_reactor_select.py') - if platform.system() in ['Windows', 'Darwin']: + log = self.run_script("reactor_default_twisted_reactor_select.py") + if platform.system() in ["Windows", "Darwin"]: # The goal of this test function is to test that, when a reactor is # installed (the default one here) and a different reactor is # configured (select here), an error raises. @@ -337,9 +342,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): # If that ever becomes the case on more platforms (i.e. if Linux # also starts using the select reactor by default in a future # version of Twisted), then we will need to rethink this test. - self.assertIn('Spider closed (finished)', log) + self.assertIn("Spider closed (finished)", log) else: - self.assertNotIn('Spider closed (finished)', log) + self.assertNotIn("Spider closed (finished)", log) self.assertIn( ( "does not match the requested one " @@ -349,18 +354,18 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) def test_reactor_select(self): - log = self.run_script('reactor_select.py') - self.assertIn('Spider closed (finished)', log) + log = self.run_script("reactor_select.py") + self.assertIn("Spider closed (finished)", log) self.assertNotIn("ReactorAlreadyInstalledError", log) def test_reactor_select_twisted_reactor_select(self): - log = self.run_script('reactor_select_twisted_reactor_select.py') - self.assertIn('Spider closed (finished)', log) + log = self.run_script("reactor_select_twisted_reactor_select.py") + self.assertIn("Spider closed (finished)", log) self.assertNotIn("ReactorAlreadyInstalledError", log) def test_reactor_select_subclass_twisted_reactor_select(self): - log = self.run_script('reactor_select_subclass_twisted_reactor_select.py') - self.assertNotIn('Spider closed (finished)', log) + log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") + self.assertNotIn("Spider closed (finished)", log) self.assertIn( ( "does not match the requested one " @@ -370,24 +375,34 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) def test_asyncio_enabled_no_reactor(self): - log = self.run_script('asyncio_enabled_no_reactor.py') - self.assertIn('Spider closed (finished)', log) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + log = self.run_script("asyncio_enabled_no_reactor.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) def test_asyncio_enabled_reactor(self): - log = self.run_script('asyncio_enabled_reactor.py') - self.assertIn('Spider closed (finished)', log) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + log = self.run_script("asyncio_enabled_reactor.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) - @mark.skipif(parse_version(w3lib_version) >= parse_version("2.0.0"), - reason='w3lib 2.0.0 and later do not allow invalid domains.') + @mark.skipif( + parse_version(w3lib_version) >= parse_version("2.0.0"), + reason="w3lib 2.0.0 and later do not allow invalid domains.", + ) def test_ipv6_default_name_resolver(self): - log = self.run_script('default_name_resolver.py') - self.assertIn('Spider closed (finished)', log) - self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) + log = self.run_script("default_name_resolver.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn( + "'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", + log, + ) self.assertIn( "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", - log) + log, + ) def test_caching_hostname_resolver_ipv6(self): log = self.run_script("caching_hostname_resolver_ipv6.py") @@ -406,9 +421,13 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_twisted_reactor_select(self): log = self.run_script("twisted_reactor_select.py") self.assertIn("Spider closed (finished)", log) - self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + self.assertIn( + "Using reactor: twisted.internet.selectreactor.SelectReactor", log + ) - @mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows") + @mark.skipif( + platform.system() == "Windows", reason="PollReactor is not supported on Windows" + ) def test_twisted_reactor_poll(self): log = self.run_script("twisted_reactor_poll.py") self.assertIn("Spider closed (finished)", log) @@ -417,54 +436,103 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_twisted_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) def test_twisted_reactor_asyncio_custom_settings(self): log = self.run_script("twisted_reactor_custom_settings.py") self.assertIn("Spider closed (finished)", log) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) def test_twisted_reactor_asyncio_custom_settings_same(self): log = self.run_script("twisted_reactor_custom_settings_same.py") self.assertIn("Spider closed (finished)", log) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) def test_twisted_reactor_asyncio_custom_settings_conflict(self): log = self.run_script("twisted_reactor_custom_settings_conflict.py") - self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) - self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log) + self.assertIn( + "Using reactor: twisted.internet.selectreactor.SelectReactor", log + ) + self.assertIn( + "(twisted.internet.selectreactor.SelectReactor) does not match the requested one", + log, + ) - @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') - @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') - @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + @mark.skipif( + sys.implementation.name == "pypy", + reason="uvloop does not support pypy properly", + ) + @mark.skipif( + platform.system() == "Windows", reason="uvloop does not support Windows" + ) + @mark.skipif( + twisted_version == Version("twisted", 21, 2, 0), + reason="https://twistedmatrix.com/trac/ticket/10106", + ) def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") self.assertIn("Spider closed (finished)", log) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly") - @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows") - @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + @mark.skipif( + sys.implementation.name == "pypy", + reason="uvloop does not support pypy properly", + ) + @mark.skipif( + platform.system() == "Windows", reason="uvloop does not support Windows" + ) + @mark.skipif( + twisted_version == Version("twisted", 21, 2, 0), + reason="https://twistedmatrix.com/trac/ticket/10106", + ) def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") self.assertIn("Spider closed (finished)", log) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') - @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') - @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + @mark.skipif( + sys.implementation.name == "pypy", + reason="uvloop does not support pypy properly", + ) + @mark.skipif( + platform.system() == "Windows", reason="uvloop does not support Windows" + ) + @mark.skipif( + twisted_version == Version("twisted", 21, 2, 0), + reason="https://twistedmatrix.com/trac/ticket/10106", + ) def test_asyncio_enabled_reactor_same_loop(self): log = self.run_script("asyncio_enabled_reactor_same_loop.py") self.assertIn("Spider closed (finished)", log) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') - @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') - @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + @mark.skipif( + sys.implementation.name == "pypy", + reason="uvloop does not support pypy properly", + ) + @mark.skipif( + platform.system() == "Windows", reason="uvloop does not support Windows" + ) + @mark.skipif( + twisted_version == Version("twisted", 21, 2, 0), + reason="https://twistedmatrix.com/trac/ticket/10106", + ) def test_asyncio_enabled_reactor_different_loop(self): log = self.run_script("asyncio_enabled_reactor_different_loop.py") self.assertNotIn("Spider closed (finished)", log) @@ -479,13 +547,15 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + ) self.assertNotIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): - script_dir = Path(__file__).parent.resolve() / 'CrawlerRunner' + script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" def test_response_ip_address(self): log = self.run_script("ip_address.py") diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 74fdd966b..f9b2d853c 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -9,15 +9,14 @@ from twisted.trial import unittest class ScrapyUtilsTest(unittest.TestCase): - def test_required_openssl_version(self): try: - module = import_module('OpenSSL') + module = import_module("OpenSSL") except ImportError: raise unittest.SkipTest("OpenSSL is not available") - if hasattr(module, '__version__'): - installed_version = [int(x) for x in module.__version__.split('.')[:2]] + if hasattr(module, "__version__"): + installed_version = [int(x) for x in module.__version__.split(".")[:2]] assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" def test_pinned_twisted_version(self): @@ -27,20 +26,17 @@ class ScrapyUtilsTest(unittest.TestCase): See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 """ - if not os.environ.get('_SCRAPY_PINNED', None): - self.skipTest('Not in a pinned environment') + if not os.environ.get("_SCRAPY_PINNED", None): + self.skipTest("Not in a pinned environment") - tox_config_file_path = Path(__file__).parent / '..' / 'tox.ini' + tox_config_file_path = Path(__file__).parent / ".." / "tox.ini" config_parser = ConfigParser() config_parser.read(tox_config_file_path) - pattern = r'Twisted\[http2\]==([\d.]+)' - match = re.search(pattern, config_parser['pinned']['deps']) + pattern = r"Twisted\[http2\]==([\d.]+)" + match = re.search(pattern, config_parser["pinned"]["deps"]) pinned_twisted_version_string = match[1] - self.assertEqual( - twisted_version.short(), - pinned_twisted_version_string - ) + self.assertEqual(twisted_version.short(), pinned_twisted_version_string) if __name__ == "__main__": diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 3dc2745a0..536d09f18 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -66,50 +66,48 @@ class OffDH: class LoadTestCase(unittest.TestCase): - def test_enabled_handler(self): - handlers = {'scheme': DummyDH} - crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + handlers = {"scheme": DummyDH} + crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertIn('scheme', dh._schemes) - self.assertIn('scheme', dh._handlers) - self.assertNotIn('scheme', dh._notconfigured) + self.assertIn("scheme", dh._schemes) + self.assertIn("scheme", dh._handlers) + self.assertNotIn("scheme", dh._notconfigured) def test_not_configured_handler(self): - handlers = {'scheme': OffDH} - crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + handlers = {"scheme": OffDH} + crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertIn('scheme', dh._schemes) - self.assertNotIn('scheme', dh._handlers) - self.assertIn('scheme', dh._notconfigured) + self.assertIn("scheme", dh._schemes) + self.assertNotIn("scheme", dh._handlers) + self.assertIn("scheme", dh._notconfigured) def test_disabled_handler(self): - handlers = {'scheme': None} - crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + handlers = {"scheme": None} + crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertNotIn('scheme', dh._schemes) + self.assertNotIn("scheme", dh._schemes) for scheme in handlers: # force load handlers dh._get_handler(scheme) - self.assertNotIn('scheme', dh._handlers) - self.assertIn('scheme', dh._notconfigured) + self.assertNotIn("scheme", dh._handlers) + self.assertIn("scheme", dh._notconfigured) def test_lazy_handlers(self): - handlers = {'scheme': DummyLazyDH} - crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + handlers = {"scheme": DummyLazyDH} + crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertIn('scheme', dh._schemes) - self.assertNotIn('scheme', dh._handlers) + self.assertIn("scheme", dh._schemes) + self.assertNotIn("scheme", dh._handlers) for scheme in handlers: # force load lazy handler dh._get_handler(scheme) - self.assertIn('scheme', dh._handlers) - self.assertNotIn('scheme', dh._notconfigured) + self.assertIn("scheme", dh._handlers) + self.assertNotIn("scheme", dh._notconfigured) class FileTestCase(unittest.TestCase): - def setUp(self): # add a special char to check that they are handled correctly - self.tmpname = Path(self.mktemp() + '^') + self.tmpname = Path(self.mktemp() + "^") Path(self.tmpname).write_text("0123456789", encoding="utf-8") handler = create_instance(FileDownloadHandler, None, get_crawler()) self.download_request = handler.download_request @@ -121,16 +119,16 @@ class FileTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.url, request.url) self.assertEqual(response.status, 200) - self.assertEqual(response.body, b'0123456789') + self.assertEqual(response.body, b"0123456789") self.assertEqual(response.protocol, None) request = Request(path_to_file_uri(str(self.tmpname))) - assert request.url.upper().endswith('%5E') - return self.download_request(request, Spider('foo')).addCallback(_test) + assert request.url.upper().endswith("%5E") + return self.download_request(request, Spider("foo")).addCallback(_test) def test_non_existent(self): - request = Request(f'file://{self.mktemp()}') - d = self.download_request(request, Spider('foo')) + request = Request(f"file://{self.mktemp()}") + d = self.download_request(request, Spider("foo")) return self.assertFailure(d, IOError) @@ -145,7 +143,6 @@ class ContentLengthHeaderResource(resource.Resource): class ChunkedResource(resource.Resource): - def render(self, request): def response(): request.write(b"chunked ") @@ -157,7 +154,6 @@ class ChunkedResource(resource.Resource): class BrokenChunkedResource(resource.Resource): - def render(self, request): def response(): request.write(b"chunked ") @@ -171,7 +167,6 @@ class BrokenChunkedResource(resource.Resource): class BrokenDownloadResource(resource.Resource): - def render(self, request): def response(): request.setHeader(b"Content-Length", b"20") @@ -185,7 +180,7 @@ class BrokenDownloadResource(resource.Resource): def closeConnection(request): # We have to force a disconnection for HTTP/1.1 clients. Otherwise # client keeps the connection open waiting for more data. - if hasattr(request.channel, 'loseConnection'): # twisted >=16.3.0 + if hasattr(request.channel, "loseConnection"): # twisted >=16.3.0 request.channel.loseConnection() else: request.channel.transport.loseConnection() @@ -215,12 +210,12 @@ class LargeChunkedFileResource(resource.Resource): class HttpTestCase(unittest.TestCase): - scheme = 'http' + scheme = "http" download_handler_cls: Type = HTTPDownloadHandler # only used for HTTPS tests - keyfile = 'keys/localhost.key' - certfile = 'keys/localhost.crt' + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" def setUp(self): self.tmpname = Path(self.mktemp()) @@ -242,23 +237,28 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) - self.host = 'localhost' - if self.scheme == 'https': + self.host = "localhost" + if self.scheme == "https": # Using WrappingFactory do not enable HTTP/2 failing all the # tests with H2DownloadHandler self.port = reactor.listenSSL( - 0, self.site, ssl_context_factory(self.keyfile, self.certfile), - interface=self.host) + 0, + self.site, + ssl_context_factory(self.keyfile, self.certfile), + interface=self.host, + ) else: self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) self.portno = self.port.getHost().port - self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) + self.download_handler = create_instance( + self.download_handler_cls, None, get_crawler() + ) self.download_request = self.download_handler.download_request @defer.inlineCallbacks def tearDown(self): yield self.port.stopListening() - if hasattr(self.download_handler, 'close'): + if hasattr(self.download_handler, "close"): yield self.download_handler.close() shutil.rmtree(self.tmpname) @@ -266,29 +266,29 @@ class HttpTestCase(unittest.TestCase): return f"{self.scheme}://{self.host}:{self.portno}/{path}" def test_download(self): - request = Request(self.getURL('file')) - d = self.download_request(request, Spider('foo')) + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.body) d.addCallback(self.assertEqual, b"0123456789") return d def test_download_head(self): - request = Request(self.getURL('file'), method='HEAD') - d = self.download_request(request, Spider('foo')) + request = Request(self.getURL("file"), method="HEAD") + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b'') + d.addCallback(self.assertEqual, b"") return d def test_redirect_status(self): - request = Request(self.getURL('redirect')) - d = self.download_request(request, Spider('foo')) + request = Request(self.getURL("redirect")) + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.status) d.addCallback(self.assertEqual, 302) return d def test_redirect_status_head(self): - request = Request(self.getURL('redirect'), method='HEAD') - d = self.download_request(request, Spider('foo')) + request = Request(self.getURL("redirect"), method="HEAD") + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.status) d.addCallback(self.assertEqual, 302) return d @@ -302,9 +302,9 @@ class HttpTestCase(unittest.TestCase): ) # client connects but no data is received - spider = Spider('foo') - meta = {'download_timeout': 0.5} - request = Request(self.getURL('wait'), meta=meta) + spider = Spider("foo") + meta = {"download_timeout": 0.5} + request = Request(self.getURL("wait"), meta=meta) d = self.download_request(request, spider) yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) @@ -316,34 +316,33 @@ class HttpTestCase(unittest.TestCase): "This test produces DirtyReactorAggregateError on Windows with asyncio" ) # client connects, server send headers and some body bytes but hangs - spider = Spider('foo') - meta = {'download_timeout': 0.5} - request = Request(self.getURL('hang-after-headers'), meta=meta) + spider = Spider("foo") + meta = {"download_timeout": 0.5} + request = Request(self.getURL("hang-after-headers"), meta=meta) d = self.download_request(request, spider) yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) def test_host_header_not_in_request_headers(self): def _test(response): - self.assertEqual( - response.body, to_bytes(f'{self.host}:{self.portno}')) + self.assertEqual(response.body, to_bytes(f"{self.host}:{self.portno}")) self.assertEqual(request.headers, {}) - request = Request(self.getURL('host')) - return self.download_request(request, Spider('foo')).addCallback(_test) + request = Request(self.getURL("host")) + return self.download_request(request, Spider("foo")).addCallback(_test) def test_host_header_seted_in_request_headers(self): - host = self.host + ':' + str(self.portno) + host = self.host + ":" + str(self.portno) def _test(response): self.assertEqual(response.body, host.encode()) - self.assertEqual(request.headers.get('Host'), host.encode()) + self.assertEqual(request.headers.get("Host"), host.encode()) - request = Request(self.getURL('host'), headers={'Host': host}) - return self.download_request(request, Spider('foo')).addCallback(_test) + request = Request(self.getURL("host"), headers={"Host": host}) + return self.download_request(request, Spider("foo")).addCallback(_test) - d = self.download_request(request, Spider('foo')) + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b'localhost') + d.addCallback(self.assertEqual, b"localhost") return d def test_content_length_zero_bodyless_post_request_headers(self): @@ -359,26 +358,27 @@ class HttpTestCase(unittest.TestCase): """ def _test(response): - self.assertEqual(response.body, b'0') + self.assertEqual(response.body, b"0") - request = Request(self.getURL('contentlength'), method='POST') - return self.download_request(request, Spider('foo')).addCallback(_test) + request = Request(self.getURL("contentlength"), method="POST") + return self.download_request(request, Spider("foo")).addCallback(_test) def test_content_length_zero_bodyless_post_only_one(self): def _test(response): import json - headers = Headers(json.loads(response.text)['headers']) - contentlengths = headers.getlist('Content-Length') + + headers = Headers(json.loads(response.text)["headers"]) + contentlengths = headers.getlist("Content-Length") self.assertEqual(len(contentlengths), 1) self.assertEqual(contentlengths, [b"0"]) - request = Request(self.getURL('echo'), method='POST') - return self.download_request(request, Spider('foo')).addCallback(_test) + request = Request(self.getURL("echo"), method="POST") + return self.download_request(request, Spider("foo")).addCallback(_test) def test_payload(self): - body = b'1' * 100 # PayloadResource requires body length to be 100 - request = Request(self.getURL('payload'), method='POST', body=body) - d = self.download_request(request, Spider('foo')) + body = b"1" * 100 # PayloadResource requires body length to be 100 + request = Request(self.getURL("payload"), method="POST", body=body) + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.body) d.addCallback(self.assertEqual, body) return d @@ -386,8 +386,8 @@ class HttpTestCase(unittest.TestCase): def test_response_header_content_length(self): request = Request(self.getURL("file"), method=b"GET") d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.headers[b'content-length']) - d.addCallback(self.assertEqual, b'159') + d.addCallback(lambda r: r.headers[b"content-length"]) + d.addCallback(self.assertEqual, b"159") return d def _test_response_class(self, filename, body, response_class): @@ -395,14 +395,14 @@ class HttpTestCase(unittest.TestCase): self.assertEqual(type(response), response_class) request = Request(self.getURL(filename), body=body) - return self.download_request(request, Spider('foo')).addCallback(_test) + return self.download_request(request, Spider("foo")).addCallback(_test) def test_response_class_from_url(self): - return self._test_response_class('foo.html', b'', HtmlResponse) + return self._test_response_class("foo.html", b"", HtmlResponse) def test_response_class_from_body(self): return self._test_response_class( - 'foo', + "foo", b"\n.", HtmlResponse, ) @@ -410,6 +410,7 @@ class HttpTestCase(unittest.TestCase): class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" + download_handler_cls: Type = HTTP10DownloadHandler def test_protocol(self): @@ -421,57 +422,58 @@ class Http10TestCase(HttpTestCase): class Https10TestCase(Http10TestCase): - scheme = 'https' + scheme = "https" class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" + download_handler_cls: Type = HTTP11DownloadHandler def test_download_without_maxsize_limit(self): - request = Request(self.getURL('file')) - d = self.download_request(request, Spider('foo')) + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.body) d.addCallback(self.assertEqual, b"0123456789") return d def test_response_class_choosing_request(self): """Tests choosing of correct response type - in case of Content-Type is empty but body contains text. + in case of Content-Type is empty but body contains text. """ - body = b'Some plain text\ndata with tabs\t and null bytes\0' + body = b"Some plain text\ndata with tabs\t and null bytes\0" def _test_type(response): self.assertEqual(type(response), TextResponse) - request = Request(self.getURL('nocontenttype'), body=body) - d = self.download_request(request, Spider('foo')) + request = Request(self.getURL("nocontenttype"), body=body) + d = self.download_request(request, Spider("foo")) d.addCallback(_test_type) return d @defer.inlineCallbacks def test_download_with_maxsize(self): - request = Request(self.getURL('file')) + request = Request(self.getURL("file")) # 10 is minimal size for this request and the limit is only counted on # response body. (regardless of headers) - d = self.download_request(request, Spider('foo', download_maxsize=10)) + d = self.download_request(request, Spider("foo", download_maxsize=10)) d.addCallback(lambda r: r.body) d.addCallback(self.assertEqual, b"0123456789") yield d - d = self.download_request(request, Spider('foo', download_maxsize=9)) + d = self.download_request(request, Spider("foo", download_maxsize=9)) yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) @defer.inlineCallbacks def test_download_with_maxsize_very_large_file(self): - with mock.patch('scrapy.core.downloader.handlers.http11.logger') as logger: - request = Request(self.getURL('largechunkedfile')) + with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger: + request = Request(self.getURL("largechunkedfile")) def check(logger): logger.warning.assert_called_once_with(mock.ANY, mock.ANY) - d = self.download_request(request, Spider('foo', download_maxsize=1500)) + d = self.download_request(request, Spider("foo", download_maxsize=1500)) yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) # As the error message is logged in the dataReceived callback, we @@ -479,39 +481,39 @@ class Http11TestCase(HttpTestCase): # after closing the connection. d = defer.Deferred() d.addCallback(check) - reactor.callLater(.1, d.callback, logger) + reactor.callLater(0.1, d.callback, logger) yield d @defer.inlineCallbacks def test_download_with_maxsize_per_req(self): - meta = {'download_maxsize': 2} - request = Request(self.getURL('file'), meta=meta) - d = self.download_request(request, Spider('foo')) + meta = {"download_maxsize": 2} + request = Request(self.getURL("file"), meta=meta) + d = self.download_request(request, Spider("foo")) yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) @defer.inlineCallbacks def test_download_with_small_maxsize_per_spider(self): - request = Request(self.getURL('file')) - d = self.download_request(request, Spider('foo', download_maxsize=2)) + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo", download_maxsize=2)) yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) def test_download_with_large_maxsize_per_spider(self): - request = Request(self.getURL('file')) - d = self.download_request(request, Spider('foo', download_maxsize=100)) + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo", download_maxsize=100)) d.addCallback(lambda r: r.body) d.addCallback(self.assertEqual, b"0123456789") return d def test_download_chunked_content(self): - request = Request(self.getURL('chunked')) - d = self.download_request(request, Spider('foo')) + request = Request(self.getURL("chunked")) + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.body) d.addCallback(self.assertEqual, b"chunked content\n") return d - def test_download_broken_content_cause_data_loss(self, url='broken'): + def test_download_broken_content_cause_data_loss(self, url="broken"): request = Request(self.getURL(url)) - d = self.download_request(request, Spider('foo')) + d = self.download_request(request, Spider("foo")) def checkDataLoss(failure): if failure.check(ResponseFailed): @@ -524,29 +526,31 @@ class Http11TestCase(HttpTestCase): return d def test_download_broken_chunked_content_cause_data_loss(self): - return self.test_download_broken_content_cause_data_loss('broken-chunked') + return self.test_download_broken_content_cause_data_loss("broken-chunked") - def test_download_broken_content_allow_data_loss(self, url='broken'): - request = Request(self.getURL(url), meta={'download_fail_on_dataloss': False}) - d = self.download_request(request, Spider('foo')) + def test_download_broken_content_allow_data_loss(self, url="broken"): + request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False}) + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ['dataloss']) + d.addCallback(self.assertEqual, ["dataloss"]) return d def test_download_broken_chunked_content_allow_data_loss(self): - return self.test_download_broken_content_allow_data_loss('broken-chunked') + return self.test_download_broken_content_allow_data_loss("broken-chunked") - def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): - crawler = get_crawler(settings_dict={'DOWNLOAD_FAIL_ON_DATALOSS': False}) + def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): + crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) download_handler = create_instance(self.download_handler_cls, None, crawler) request = Request(self.getURL(url)) - d = download_handler.download_request(request, Spider('foo')) + d = download_handler.download_request(request, Spider("foo")) d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ['dataloss']) + d.addCallback(self.assertEqual, ["dataloss"]) return d def test_download_broken_chunked_content_allow_data_loss_via_setting(self): - return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked') + return self.test_download_broken_content_allow_data_loss_via_setting( + "broken-chunked" + ) def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -557,7 +561,7 @@ class Http11TestCase(HttpTestCase): class Https11TestCase(Http11TestCase): - scheme = 'https' + scheme = "https" tls_log_message = ( 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' @@ -566,30 +570,34 @@ class Https11TestCase(Http11TestCase): @defer.inlineCallbacks def test_tls_logging(self): - crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True}) + crawler = get_crawler( + settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} + ) download_handler = create_instance(self.download_handler_cls, None, crawler) try: with LogCapture() as log_capture: - request = Request(self.getURL('file')) - d = download_handler.download_request(request, Spider('foo')) + request = Request(self.getURL("file")) + d = download_handler.download_request(request, Spider("foo")) d.addCallback(lambda r: r.body) d.addCallback(self.assertEqual, b"0123456789") yield d - log_capture.check_present(('scrapy.core.downloader.tls', 'DEBUG', self.tls_log_message)) + log_capture.check_present( + ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message) + ) finally: yield download_handler.close() class Https11WrongHostnameTestCase(Http11TestCase): - scheme = 'https' + scheme = "https" # above tests use a server certificate for "localhost", # client connection to "localhost" too. # here we test that even if the server certificate is for another domain, # "www.example.com" in this case, # the tests still pass - keyfile = 'keys/example-com.key.pem' - certfile = 'keys/example-com.cert.pem' + keyfile = "keys/example-com.key.pem" + certfile = "keys/example-com.cert.pem" class Https11InvalidDNSId(Https11TestCase): @@ -597,14 +605,14 @@ class Https11InvalidDNSId(Https11TestCase): def setUp(self): super().setUp() - self.host = '127.0.0.1' + self.host = "127.0.0.1" class Https11InvalidDNSPattern(Https11TestCase): """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - keyfile = 'keys/localhost.ip.key' - certfile = 'keys/localhost.ip.crt' + keyfile = "keys/localhost.ip.key" + certfile = "keys/localhost.ip.crt" def setUp(self): try: @@ -619,11 +627,11 @@ class Https11InvalidDNSPattern(Https11TestCase): class Https11CustomCiphers(unittest.TestCase): - scheme = 'https' + scheme = "https" download_handler_cls: Type = HTTP11DownloadHandler - keyfile = 'keys/localhost.key' - certfile = 'keys/localhost.crt' + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" def setUp(self): self.tmpname = Path(self.mktemp()) @@ -631,19 +639,28 @@ class Https11CustomCiphers(unittest.TestCase): (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) self.site = server.Site(r, timeout=None) - self.host = 'localhost' + self.host = "localhost" self.port = reactor.listenSSL( - 0, self.site, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'), - interface=self.host) + 0, + self.site, + ssl_context_factory( + self.keyfile, self.certfile, cipher_string="CAMELLIA256-SHA" + ), + interface=self.host, + ) self.portno = self.port.getHost().port - crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}) - self.download_handler = create_instance(self.download_handler_cls, None, crawler) + crawler = get_crawler( + settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"} + ) + self.download_handler = create_instance( + self.download_handler_cls, None, crawler + ) self.download_request = self.download_handler.download_request @defer.inlineCallbacks def tearDown(self): yield self.port.stopListening() - if hasattr(self.download_handler, 'close'): + if hasattr(self.download_handler, "close"): yield self.download_handler.close() shutil.rmtree(self.tmpname) @@ -651,8 +668,8 @@ class Https11CustomCiphers(unittest.TestCase): return f"{self.scheme}://{self.host}:{self.portno}/{path}" def test_download(self): - request = Request(self.getURL('file')) - d = self.download_request(request, Spider('foo')) + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.body) d.addCallback(self.assertEqual, b"0123456789") return d @@ -660,6 +677,7 @@ class Https11CustomCiphers(unittest.TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" + settings_dict: Optional[dict] = None def setUp(self): @@ -674,40 +692,48 @@ class Http11MockServerTestCase(unittest.TestCase): crawler = get_crawler(SingleRequestSpider, self.settings_dict) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it - yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000})) - failure = crawler.spider.meta['failure'] + yield crawler.crawl( + seed=Request( + url=self.mockserver.url("/partial"), meta={"download_maxsize": 1000} + ) + ) + failure = crawler.spider.meta["failure"] self.assertIsInstance(failure.value, defer.CancelledError) @defer.inlineCallbacks def test_download(self): crawler = get_crawler(SingleRequestSpider, self.settings_dict) - yield crawler.crawl(seed=Request(url=self.mockserver.url(''))) - failure = crawler.spider.meta.get('failure') + yield crawler.crawl(seed=Request(url=self.mockserver.url(""))) + failure = crawler.spider.meta.get("failure") self.assertTrue(failure is None) - reason = crawler.spider.meta['close_reason'] - self.assertTrue(reason, 'finished') + reason = crawler.spider.meta["close_reason"] + self.assertTrue(reason, "finished") @defer.inlineCallbacks def test_download_gzip_response(self): crawler = get_crawler(SingleRequestSpider, self.settings_dict) - body = b'1' * 100 # PayloadResource requires body length to be 100 - request = Request(self.mockserver.url('/payload'), method='POST', - body=body, meta={'download_maxsize': 50}) + body = b"1" * 100 # PayloadResource requires body length to be 100 + request = Request( + self.mockserver.url("/payload"), + method="POST", + body=body, + meta={"download_maxsize": 50}, + ) yield crawler.crawl(seed=request) - failure = crawler.spider.meta['failure'] + failure = crawler.spider.meta["failure"] # download_maxsize < 100, hence the CancelledError self.assertIsInstance(failure.value, defer.CancelledError) # See issue https://twistedmatrix.com/trac/ticket/8175 raise unittest.SkipTest("xpayload fails on PY3") - request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url=self.mockserver.url('/xpayload')) + request.headers.setdefault(b"Accept-Encoding", b"gzip,deflate") + request = request.replace(url=self.mockserver.url("/xpayload")) yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response - failure = crawler.spider.meta.get('failure') + failure = crawler.spider.meta.get("failure") self.assertTrue(failure is None) - reason = crawler.spider.meta['close_reason'] - self.assertTrue(reason, 'finished') + reason = crawler.spider.meta["close_reason"] + self.assertTrue(reason, "finished") class UriResource(resource.Resource): @@ -720,27 +746,29 @@ class UriResource(resource.Resource): # Note: this is an ugly hack for CONNECT request timeout test. # Returning some data here fail SSL/TLS handshake # ToDo: implement proper HTTPS proxy tests, not faking them. - if request.method != b'CONNECT': + if request.method != b"CONNECT": return request.uri - return b'' + return b"" class HttpProxyTestCase(unittest.TestCase): download_handler_cls: Type = HTTPDownloadHandler - expected_http_proxy_request_body = b'http://example.com' + expected_http_proxy_request_body = b"http://example.com" def setUp(self): site = server.Site(UriResource(), timeout=None) wrapper = WrappingFactory(site) - self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1') + self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") self.portno = self.port.getHost().port - self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) + self.download_handler = create_instance( + self.download_handler_cls, None, get_crawler() + ) self.download_request = self.download_handler.download_request @defer.inlineCallbacks def tearDown(self): yield self.port.stopListening() - if hasattr(self.download_handler, 'close'): + if hasattr(self.download_handler, "close"): yield self.download_handler.close() def getURL(self, path): @@ -752,25 +780,25 @@ class HttpProxyTestCase(unittest.TestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.body, self.expected_http_proxy_request_body) - http_proxy = self.getURL('') - request = Request('http://example.com', meta={'proxy': http_proxy}) - return self.download_request(request, Spider('foo')).addCallback(_test) + http_proxy = self.getURL("") + request = Request("http://example.com", meta={"proxy": http_proxy}) + return self.download_request(request, Spider("foo")).addCallback(_test) def test_download_without_proxy(self): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'/path/to/resource') + self.assertEqual(response.body, b"/path/to/resource") - request = Request(self.getURL('path/to/resource')) - return self.download_request(request, Spider('foo')).addCallback(_test) + request = Request(self.getURL("path/to/resource")) + return self.download_request(request, Spider("foo")).addCallback(_test) class Http10ProxyTestCase(HttpProxyTestCase): download_handler_cls: Type = HTTP10DownloadHandler def test_download_with_proxy_https_noconnect(self): - raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler') + raise unittest.SkipTest("noconnect is not supported in HTTP10DownloadHandler") class Http11ProxyTestCase(HttpProxyTestCase): @@ -778,14 +806,13 @@ class Http11ProxyTestCase(HttpProxyTestCase): @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): - """ Test TunnelingTCP4ClientEndpoint """ + """Test TunnelingTCP4ClientEndpoint""" if NON_EXISTING_RESOLVABLE: raise SkipTest("Non-existing hosts are resolvable") - http_proxy = self.getURL('') - domain = 'https://no-such-domain.nosuch' - request = Request( - domain, meta={'proxy': http_proxy, 'download_timeout': 0.2}) - d = self.download_request(request, Spider('foo')) + http_proxy = self.getURL("") + domain = "https://no-such-domain.nosuch" + request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) + d = self.download_request(request, Spider("foo")) timeout = yield self.assertFailure(d, error.TimeoutError) self.assertIn(domain, timeout.osError) @@ -795,13 +822,12 @@ class Http11ProxyTestCase(HttpProxyTestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.body, self.expected_http_proxy_request_body) - http_proxy = self.getURL('').replace('http://', '') - request = Request('http://example.com', meta={'proxy': http_proxy}) - return self.download_request(request, Spider('foo')).addCallback(_test) + http_proxy = self.getURL("").replace("http://", "") + request = Request("http://example.com", meta={"proxy": http_proxy}) + return self.download_request(request, Spider("foo")).addCallback(_test) class HttpDownloadHandlerMock: - def __init__(self, *args, **kwargs): pass @@ -810,7 +836,6 @@ class HttpDownloadHandlerMock: class S3AnonTestCase(unittest.TestCase): - def setUp(self): skip_if_no_boto() crawler = get_crawler() @@ -822,15 +847,14 @@ class S3AnonTestCase(unittest.TestCase): # anon=True, # implicit ) self.download_request = self.s3reqh.download_request - self.spider = Spider('foo') + self.spider = Spider("foo") def test_anon_request(self): - req = Request('s3://aws-publicdatasets/') + req = Request("s3://aws-publicdatasets/") httpreq = self.download_request(req, self.spider) - self.assertEqual(hasattr(self.s3reqh, 'anon'), True) + self.assertEqual(hasattr(self.s3reqh, "anon"), True) self.assertEqual(self.s3reqh.anon, True) - self.assertEqual( - httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/') + self.assertEqual(httpreq.url, "http://aws-publicdatasets.s3.amazonaws.com/") class S3TestCase(unittest.TestCase): @@ -840,8 +864,8 @@ class S3TestCase(unittest.TestCase): # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf # and the tests described here are the examples from that manual - AWS_ACCESS_KEY_ID = '0PN5J17HBGZHT7JJ3X82' - AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o' + AWS_ACCESS_KEY_ID = "0PN5J17HBGZHT7JJ3X82" + AWS_SECRET_ACCESS_KEY = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o" def setUp(self): skip_if_no_boto() @@ -855,7 +879,7 @@ class S3TestCase(unittest.TestCase): httpdownloadhandler=HttpDownloadHandlerMock, ) self.download_request = s3reqh.download_request - self.spider = Spider('foo') + self.spider = Spider("foo") @contextlib.contextmanager def _mocked_date(self, date): @@ -867,7 +891,7 @@ class S3TestCase(unittest.TestCase): # We need to mock botocore.auth.formatdate, because otherwise # botocore overrides Date header with current date and time # and Authorization header is different each time - with mock.patch('botocore.auth.formatdate') as mock_formatdate: + with mock.patch("botocore.auth.formatdate") as mock_formatdate: mock_formatdate.return_value = date yield @@ -887,90 +911,105 @@ class S3TestCase(unittest.TestCase): def test_request_signing1(self): # gets an object from the johnsmith bucket. - date = 'Tue, 27 Mar 2007 19:36:42 +0000' - req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date}) + date = "Tue, 27 Mar 2007 19:36:42 +0000" + req = Request("s3://johnsmith/photos/puppy.jpg", headers={"Date": date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], - b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=') + self.assertEqual( + httpreq.headers["Authorization"], + b"AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=", + ) def test_request_signing2(self): # puts an object into the johnsmith bucket. - date = 'Tue, 27 Mar 2007 21:15:45 +0000' + date = "Tue, 27 Mar 2007 21:15:45 +0000" req = Request( - 's3://johnsmith/photos/puppy.jpg', - method='PUT', + "s3://johnsmith/photos/puppy.jpg", + method="PUT", headers={ - 'Content-Type': 'image/jpeg', - 'Date': date, - 'Content-Length': '94328', + "Content-Type": "image/jpeg", + "Date": date, + "Content-Length": "94328", }, ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], - b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=') + self.assertEqual( + httpreq.headers["Authorization"], + b"AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=", + ) def test_request_signing3(self): # lists the content of the johnsmith bucket. - date = 'Tue, 27 Mar 2007 19:42:41 +0000' + date = "Tue, 27 Mar 2007 19:42:41 +0000" req = Request( - 's3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', - method='GET', headers={ - 'User-Agent': 'Mozilla/5.0', - 'Date': date, - }) - with self._mocked_date(date): - httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], - b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=') - - def test_request_signing4(self): - # fetches the access control policy sub-resource for the 'johnsmith' bucket. - date = 'Tue, 27 Mar 2007 19:44:46 +0000' - req = Request('s3://johnsmith/?acl', method='GET', headers={'Date': date}) - with self._mocked_date(date): - httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], - b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') - - def test_request_signing6(self): - # uploads an object to a CNAME style virtual hosted bucket with metadata. - date = 'Tue, 27 Mar 2007 21:06:08 +0000' - req = Request( - 's3://static.johnsmith.net:8080/db-backup.dat.gz', - method='PUT', headers={ - 'User-Agent': 'curl/7.15.5', - 'Host': 'static.johnsmith.net:8080', - 'Date': date, - 'x-amz-acl': 'public-read', - 'content-type': 'application/x-download', - 'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==', - 'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net', - 'X-Amz-Meta-FileChecksum': '0x02661779', - 'X-Amz-Meta-ChecksumAlgorithm': 'crc32', - 'Content-Disposition': 'attachment; filename=database.dat', - 'Content-Encoding': 'gzip', - 'Content-Length': '5913339', - }) - with self._mocked_date(date): - httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], - b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=') - - def test_request_signing7(self): - # ensure that spaces are quoted properly before signing - date = 'Tue, 27 Mar 2007 19:42:41 +0000' - req = Request( - "s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg", - method='GET', - headers={'Date': date}, + "s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy", + method="GET", + headers={ + "User-Agent": "Mozilla/5.0", + "Date": date, + }, ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) self.assertEqual( - httpreq.headers['Authorization'], - b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=') + httpreq.headers["Authorization"], + b"AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=", + ) + + def test_request_signing4(self): + # fetches the access control policy sub-resource for the 'johnsmith' bucket. + date = "Tue, 27 Mar 2007 19:44:46 +0000" + req = Request("s3://johnsmith/?acl", method="GET", headers={"Date": date}) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) + self.assertEqual( + httpreq.headers["Authorization"], + b"AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=", + ) + + def test_request_signing6(self): + # uploads an object to a CNAME style virtual hosted bucket with metadata. + date = "Tue, 27 Mar 2007 21:06:08 +0000" + req = Request( + "s3://static.johnsmith.net:8080/db-backup.dat.gz", + method="PUT", + headers={ + "User-Agent": "curl/7.15.5", + "Host": "static.johnsmith.net:8080", + "Date": date, + "x-amz-acl": "public-read", + "content-type": "application/x-download", + "Content-MD5": "4gJE4saaMU4BqNR0kLY+lw==", + "X-Amz-Meta-ReviewedBy": "joe@johnsmith.net,jane@johnsmith.net", + "X-Amz-Meta-FileChecksum": "0x02661779", + "X-Amz-Meta-ChecksumAlgorithm": "crc32", + "Content-Disposition": "attachment; filename=database.dat", + "Content-Encoding": "gzip", + "Content-Length": "5913339", + }, + ) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) + self.assertEqual( + httpreq.headers["Authorization"], + b"AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=", + ) + + def test_request_signing7(self): + # ensure that spaces are quoted properly before signing + date = "Tue, 27 Mar 2007 19:42:41 +0000" + req = Request( + "s3://johnsmith/photos/my puppy.jpg?response-content-disposition=my puppy.jpg", + method="GET", + headers={"Date": date}, + ) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) + self.assertEqual( + httpreq.headers["Authorization"], + b"AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=", + ) class BaseFTPTestCase(unittest.TestCase): @@ -979,9 +1018,9 @@ class BaseFTPTestCase(unittest.TestCase): req_meta = {"ftp_user": username, "ftp_password": password} test_files = ( - ('file.txt', b"I have the power!"), - ('file with spaces.txt', b"Moooooooooo power!"), - ('html-file-without-extension', b"\n."), + ("file.txt", b"I have the power!"), + ("file with spaces.txt", b"Moooooooooo power!"), + ("html-file-without-extension", b"\n."), ) def setUp(self): @@ -997,7 +1036,9 @@ class BaseFTPTestCase(unittest.TestCase): (userdir / filename).write_bytes(content) # setup server - realm = FTPRealm(anonymousRoot=str(self.directory), userHome=str(self.directory)) + realm = FTPRealm( + anonymousRoot=str(self.directory), userHome=str(self.directory) + ) p = portal.Portal(realm) users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() users_checker.addUser(self.username, self.password) @@ -1006,7 +1047,9 @@ class BaseFTPTestCase(unittest.TestCase): self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port crawler = get_crawler() - self.download_handler = create_instance(FTPDownloadHandler, crawler.settings, crawler) + self.download_handler = create_instance( + FTPDownloadHandler, crawler.settings, crawler + ) self.addCleanup(self.port.stopListening) def tearDown(self): @@ -1025,34 +1068,37 @@ class BaseFTPTestCase(unittest.TestCase): return deferred def test_ftp_download_success(self): - request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", - meta=self.req_meta) - d = self.download_handler.download_request(request, None) - - def _test(r): - self.assertEqual(r.status, 200) - self.assertEqual(r.body, b'I have the power!') - self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) - self.assertIsNone(r.protocol) - return self._add_test_callbacks(d, _test) - - def test_ftp_download_path_with_spaces(self): request = Request( - url=f"ftp://127.0.0.1:{self.portNum}/file with spaces.txt", - meta=self.req_meta + url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=self.req_meta ) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.status, 200) - self.assertEqual(r.body, b'Moooooooooo power!') - self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']}) + self.assertEqual(r.body, b"I have the power!") + self.assertEqual(r.headers, {b"Local Filename": [b""], b"Size": [b"17"]}) + self.assertIsNone(r.protocol) + + return self._add_test_callbacks(d, _test) + + def test_ftp_download_path_with_spaces(self): + request = Request( + url=f"ftp://127.0.0.1:{self.portNum}/file with spaces.txt", + meta=self.req_meta, + ) + d = self.download_handler.download_request(request, None) + + def _test(r): + self.assertEqual(r.status, 200) + self.assertEqual(r.body, b"Moooooooooo power!") + self.assertEqual(r.headers, {b"Local Filename": [b""], b"Size": [b"18"]}) return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): - request = Request(url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt", - meta=self.req_meta) + request = Request( + url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt", meta=self.req_meta + ) d = self.download_handler.download_request(request, None) def _test(r): @@ -1067,14 +1113,14 @@ class BaseFTPTestCase(unittest.TestCase): os.close(f) meta = {"ftp_local_filename": fname_bytes} meta.update(self.req_meta) - request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", - meta=meta) + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.body, fname_bytes) - self.assertEqual(r.headers, {b'Local Filename': [fname_bytes], - b'Size': [b'17']}) + self.assertEqual( + r.headers, {b"Local Filename": [fname_bytes], b"Size": [b"17"]} + ) self.assertTrue(local_fname.exists()) self.assertEqual(local_fname.read_bytes(), b"I have the power!") local_fname.unlink() @@ -1087,24 +1133,23 @@ class BaseFTPTestCase(unittest.TestCase): os.close(f) meta = {} meta.update(self.req_meta) - request = Request(url=f"ftp://127.0.0.1:{self.portNum}/{filename}", - meta=meta) + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/{filename}", meta=meta) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(type(r), response_class) local_fname.unlink() + return self._add_test_callbacks(d, _test) def test_response_class_from_url(self): - return self._test_response_class('file.txt', TextResponse) + return self._test_response_class("file.txt", TextResponse) def test_response_class_from_body(self): - return self._test_response_class('html-file-without-extension', HtmlResponse) + return self._test_response_class("html-file-without-extension", HtmlResponse) class FTPTestCase(BaseFTPTestCase): - def test_invalid_credentials(self): if self.reactor_pytest == "asyncio" and sys.platform == "win32": raise unittest.SkipTest( @@ -1113,9 +1158,8 @@ class FTPTestCase(BaseFTPTestCase): from twisted.protocols.ftp import ConnectionLost meta = dict(self.req_meta) - meta.update({"ftp_password": 'invalid'}) - request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", - meta=meta) + meta.update({"ftp_password": "invalid"}) + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) d = self.download_handler.download_request(request, None) def _test(r): @@ -1142,15 +1186,15 @@ class AnonymousFTPTestCase(BaseFTPTestCase): # setup server for anonymous access realm = FTPRealm(anonymousRoot=str(self.directory)) p = portal.Portal(realm) - p.registerChecker(checkers.AllowAnonymousAccess(), - credentials.IAnonymous) + p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous) - self.factory = FTPFactory(portal=p, - userAnonymous=self.username) + self.factory = FTPFactory(portal=p, userAnonymous=self.username) self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port crawler = get_crawler() - self.download_handler = create_instance(FTPDownloadHandler, crawler.settings, crawler) + self.download_handler = create_instance( + FTPDownloadHandler, crawler.settings, crawler + ) self.addCleanup(self.port.stopListening) def tearDown(self): @@ -1158,12 +1202,13 @@ class AnonymousFTPTestCase(BaseFTPTestCase): class DataURITestCase(unittest.TestCase): - def setUp(self): crawler = get_crawler() - self.download_handler = create_instance(DataURIDownloadHandler, crawler.settings, crawler) + self.download_handler = create_instance( + DataURIDownloadHandler, crawler.settings, crawler + ) self.download_request = self.download_handler.download_request - self.spider = Spider('foo') + self.spider = Spider("foo") def test_response_attrs(self): uri = "data:,A%20brief%20note" @@ -1177,7 +1222,7 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype_encoding(self): def _test(response): - self.assertEqual(response.text, 'A brief note') + self.assertEqual(response.text, "A brief note") self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "US-ASCII") @@ -1186,7 +1231,7 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype(self): def _test(response): - self.assertEqual(response.text, '\u038e\u03a3\u038e') + self.assertEqual(response.text, "\u038e\u03a3\u038e") self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "iso-8859-7") @@ -1195,8 +1240,8 @@ class DataURITestCase(unittest.TestCase): def test_text_charset(self): def _test(response): - self.assertEqual(response.text, '\u038e\u03a3\u038e') - self.assertEqual(response.body, b'\xbe\xd3\xbe') + self.assertEqual(response.text, "\u038e\u03a3\u038e") + self.assertEqual(response.body, b"\xbe\xd3\xbe") self.assertEqual(response.encoding, "iso-8859-7") request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") @@ -1204,20 +1249,22 @@ class DataURITestCase(unittest.TestCase): def test_mediatype_parameters(self): def _test(response): - self.assertEqual(response.text, '\u038e\u03a3\u038e') + self.assertEqual(response.text, "\u038e\u03a3\u038e") self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "utf-8") - request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;' - 'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22' - ',%CE%8E%CE%A3%CE%8E') + request = Request( + "data:text/plain;foo=%22foo;bar%5C%22%22;" + "charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22" + ",%CE%8E%CE%A3%CE%8E" + ) return self.download_request(request, self.spider).addCallback(_test) def test_base64(self): def _test(response): - self.assertEqual(response.text, 'Hello, world.') + self.assertEqual(response.text, "Hello, world.") - request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') + request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D") return self.download_request(request, self.spider).addCallback(_test) def test_protocol(self): diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index c2fa3ec57..fd765089a 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -15,21 +15,24 @@ from scrapy.utils.misc import create_instance from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory from tests.test_downloader_handlers import ( - Https11TestCase, Https11CustomCiphers, - Http11MockServerTestCase, Http11ProxyTestCase, - UriResource + Https11TestCase, + Https11CustomCiphers, + Http11MockServerTestCase, + Http11ProxyTestCase, + UriResource, ) @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2TestCase(Https11TestCase): - scheme = 'https' + scheme = "https" HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" @classmethod def setUpClass(cls): from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler def test_protocol(self): @@ -41,13 +44,13 @@ class Https2TestCase(Https11TestCase): @defer.inlineCallbacks def test_download_with_maxsize_very_large_file(self): - with mock.patch('scrapy.core.http2.stream.logger') as logger: - request = Request(self.getURL('largechunkedfile')) + with mock.patch("scrapy.core.http2.stream.logger") as logger: + request = Request(self.getURL("largechunkedfile")) def check(logger): logger.error.assert_called_once_with(mock.ANY) - d = self.download_request(request, Spider('foo', download_maxsize=1500)) + d = self.download_request(request, Spider("foo", download_maxsize=1500)) yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) # As the error message is logged in the dataReceived callback, we @@ -55,7 +58,7 @@ class Https2TestCase(Https11TestCase): # after closing the connection. d = defer.Deferred() d.addCallback(check) - reactor.callLater(.1, d.callback, logger) + reactor.callLater(0.1, d.callback, logger) yield d @defer.inlineCallbacks @@ -64,88 +67,86 @@ class Https2TestCase(Https11TestCase): d = self.download_request(request, Spider("foo")) yield self.assertFailure(d, SchemeNotSupported) - def test_download_broken_content_cause_data_loss(self, url='broken'): + def test_download_broken_content_cause_data_loss(self, url="broken"): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_chunked_content_cause_data_loss(self): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - def test_download_broken_content_allow_data_loss(self, url='broken'): + def test_download_broken_content_allow_data_loss(self, url="broken"): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_chunked_content_allow_data_loss(self): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) - def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): + def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_chunked_content_allow_data_loss_via_setting(self): raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) def test_concurrent_requests_same_domain(self): - spider = Spider('foo') + spider = Spider("foo") - request1 = Request(self.getURL('file')) + request1 = Request(self.getURL("file")) d1 = self.download_request(request1, spider) d1.addCallback(lambda r: r.body) d1.addCallback(self.assertEqual, b"0123456789") - request2 = Request(self.getURL('echo'), method='POST') + request2 = Request(self.getURL("echo"), method="POST") d2 = self.download_request(request2, spider) - d2.addCallback(lambda r: r.headers['Content-Length']) + d2.addCallback(lambda r: r.headers["Content-Length"]) d2.addCallback(self.assertEqual, b"79") return defer.DeferredList([d1, d2]) @mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") def test_connect_request(self): - request = Request(self.getURL('file'), method='CONNECT') - d = self.download_request(request, Spider('foo')) + request = Request(self.getURL("file"), method="CONNECT") + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b'') + d.addCallback(self.assertEqual, b"") return d def test_custom_content_length_good(self): - request = Request(self.getURL('contentlength')) + request = Request(self.getURL("contentlength")) custom_content_length = str(len(request.body)) - request.headers['Content-Length'] = custom_content_length - d = self.download_request(request, Spider('foo')) + request.headers["Content-Length"] = custom_content_length + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.text) d.addCallback(self.assertEqual, custom_content_length) return d def test_custom_content_length_bad(self): - request = Request(self.getURL('contentlength')) + request = Request(self.getURL("contentlength")) actual_content_length = str(len(request.body)) bad_content_length = str(len(request.body) + 1) - request.headers['Content-Length'] = bad_content_length + request.headers["Content-Length"] = bad_content_length log = LogCapture() - d = self.download_request(request, Spider('foo')) + d = self.download_request(request, Spider("foo")) d.addCallback(lambda r: r.text) d.addCallback(self.assertEqual, actual_content_length) d.addCallback( lambda _: log.check_present( ( - 'scrapy.core.http2.stream', - 'WARNING', - f'Ignoring bad Content-Length header ' - f'{bad_content_length!r} of request {request}, sending ' - f'{actual_content_length!r} instead', + "scrapy.core.http2.stream", + "WARNING", + f"Ignoring bad Content-Length header " + f"{bad_content_length!r} of request {request}, sending " + f"{actual_content_length!r} instead", ) ) ) - d.addCallback( - lambda _: log.uninstall() - ) + d.addCallback(lambda _: log.uninstall()) return d def test_duplicate_header(self): - request = Request(self.getURL('echo')) - header, value1, value2 = 'Custom-Header', 'foo', 'bar' + request = Request(self.getURL("echo")) + header, value1, value2 = "Custom-Header", "foo", "bar" request.headers.appendlist(header, value1) request.headers.appendlist(header, value2) - d = self.download_request(request, Spider('foo')) - d.addCallback(lambda r: json.loads(r.text)['headers'][header]) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: json.loads(r.text)["headers"][header]) d.addCallback(self.assertEqual, [value1, value2]) return d @@ -163,8 +164,8 @@ class Https2WrongHostnameTestCase(Https2TestCase): # here we test that even if the server certificate is for another domain, # "www.example.com" in this case, # the tests still pass - keyfile = 'keys/example-com.key.pem' - certfile = 'keys/example-com.cert.pem' + keyfile = "keys/example-com.key.pem" + certfile = "keys/example-com.cert.pem" class Https2InvalidDNSId(Https2TestCase): @@ -172,14 +173,14 @@ class Https2InvalidDNSId(Https2TestCase): def setUp(self): super().setUp() - self.host = '127.0.0.1' + self.host = "127.0.0.1" class Https2InvalidDNSPattern(Https2TestCase): """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - keyfile = 'keys/localhost.ip.key' - certfile = 'keys/localhost.ip.crt' + keyfile = "keys/localhost.ip.key" + certfile = "keys/localhost.ip.crt" def setUp(self): try: @@ -195,19 +196,21 @@ class Https2InvalidDNSPattern(Https2TestCase): @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2CustomCiphers(Https11CustomCiphers): - scheme = 'https' + scheme = "https" @classmethod def setUpClass(cls): from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler class Http2MockServerTestCase(Http11MockServerTestCase): """HTTP 2.0 test case with MockServer""" + settings_dict = { - 'DOWNLOAD_HANDLERS': { - 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler' + "DOWNLOAD_HANDLERS": { + "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler" } } @@ -215,28 +218,32 @@ class Http2MockServerTestCase(Http11MockServerTestCase): @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2ProxyTestCase(Http11ProxyTestCase): # only used for HTTPS tests - keyfile = 'keys/localhost.key' - certfile = 'keys/localhost.crt' + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" - scheme = 'https' - host = '127.0.0.1' + scheme = "https" + host = "127.0.0.1" - expected_http_proxy_request_body = b'/' + expected_http_proxy_request_body = b"/" @classmethod def setUpClass(cls): from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler def setUp(self): site = server.Site(UriResource(), timeout=None) self.port = reactor.listenSSL( - 0, site, + 0, + site, ssl_context_factory(self.keyfile, self.certfile), - interface=self.host + interface=self.host, ) self.portno = self.port.getHost().port - self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) + self.download_handler = create_instance( + self.download_handler_cls, None, get_crawler() + ) self.download_request = self.download_handler.download_request def getURL(self, path): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 38be915f2..d8e377519 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -21,14 +21,14 @@ class ManagerTestCase(TestCase): def setUp(self): self.crawler = get_crawler(Spider, self.settings_dict) - self.spider = self.crawler._create_spider('foo') + self.spider = self.crawler._create_spider("foo") self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) # some mw depends on stats collector self.crawler.stats.open_spider(self.spider) return self.mwman.open_spider(self.spider) def tearDown(self): - self.crawler.stats.close_spider(self.spider, '') + self.crawler.stats.close_spider(self.spider, "") return self.mwman.close_spider(self.spider) def _download(self, request, response=None): @@ -57,7 +57,7 @@ class DefaultsTest(ManagerTestCase): """Tests default behavior with default settings""" def test_request_response(self): - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") resp = Response(req.url, status=200) ret = self._download(req, resp) self.assertTrue(isinstance(ret, Response), "Non-response returned") @@ -74,29 +74,41 @@ class DefaultsTest(ManagerTestCase): exceptions.IOError: Not a gzipped file """ - req = Request('http://example.com') - body = b'

You are being redirected

' - resp = Response(req.url, status=302, body=body, headers={ - 'Content-Length': str(len(body)), - 'Content-Type': 'text/html', - 'Content-Encoding': 'gzip', - 'Location': 'http://example.com/login', - }) + req = Request("http://example.com") + body = b"

You are being redirected

" + resp = Response( + req.url, + status=302, + body=body, + headers={ + "Content-Length": str(len(body)), + "Content-Type": "text/html", + "Content-Encoding": "gzip", + "Location": "http://example.com/login", + }, + ) ret = self._download(request=req, response=resp) - self.assertTrue(isinstance(ret, Request), - f"Not redirected: {ret!r}") - self.assertEqual(to_bytes(ret.url), resp.headers['Location'], - "Not redirected to location header") + self.assertTrue(isinstance(ret, Request), f"Not redirected: {ret!r}") + self.assertEqual( + to_bytes(ret.url), + resp.headers["Location"], + "Not redirected to location header", + ) def test_200_and_invalid_gzipped_body_must_fail(self): - req = Request('http://example.com') - body = b'

You are being redirected

' - resp = Response(req.url, status=200, body=body, headers={ - 'Content-Length': str(len(body)), - 'Content-Type': 'text/html', - 'Content-Encoding': 'gzip', - 'Location': 'http://example.com/login', - }) + req = Request("http://example.com") + body = b"

You are being redirected

" + resp = Response( + req.url, + status=200, + body=body, + headers={ + "Content-Length": str(len(body)), + "Content-Type": "text/html", + "Content-Encoding": "gzip", + "Location": "http://example.com/login", + }, + ) self.assertRaises(IOError, self._download, request=req, response=resp) @@ -104,7 +116,7 @@ class ResponseFromProcessRequestTest(ManagerTestCase): """Tests middleware returning a response from process_request.""" def test_download_func_not_called(self): - resp = Response('http://example.com/index.html') + resp = Response("http://example.com/index.html") class ResponseMiddleware: def process_request(self, request, spider): @@ -112,7 +124,7 @@ class ResponseFromProcessRequestTest(ManagerTestCase): self.mwman._add_middleware(ResponseMiddleware()) - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") download_func = mock.MagicMock() dfd = self.mwman.download(download_func, req, self.spider) results = [] @@ -127,7 +139,7 @@ class ProcessRequestInvalidOutput(ManagerTestCase): """Invalid return value for process_request method should raise an exception""" def test_invalid_process_request(self): - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") class InvalidProcessRequestMiddleware: def process_request(self, request, spider): @@ -146,7 +158,7 @@ class ProcessResponseInvalidOutput(ManagerTestCase): """Invalid return value for process_response method should raise an exception""" def test_invalid_process_response(self): - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") class InvalidProcessResponseMiddleware: def process_response(self, request, response, spider): @@ -165,7 +177,7 @@ class ProcessExceptionInvalidOutput(ManagerTestCase): """Invalid return value for process_exception method should raise an exception""" def test_invalid_process_exception(self): - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") class InvalidProcessExceptionMiddleware: def process_request(self, request, spider): @@ -187,7 +199,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase): """Middlewares using Deferreds should work""" def test_deferred(self): - resp = Response('http://example.com/index.html') + resp = Response("http://example.com/index.html") class DeferredMiddleware: def cb(self, result): @@ -200,7 +212,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase): return d self.mwman._add_middleware(DeferredMiddleware()) - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") download_func = mock.MagicMock() dfd = self.mwman.download(download_func, req, self.spider) results = [] @@ -211,12 +223,12 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.assertFalse(download_func.called) -@mark.usefixtures('reactor_pytest') +@mark.usefixtures("reactor_pytest") class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): - resp = Response('http://example.com/index.html') + resp = Response("http://example.com/index.html") class CoroMiddleware: async def process_request(self, request, spider): @@ -224,7 +236,7 @@ class MiddlewareUsingCoro(ManagerTestCase): return resp self.mwman._add_middleware(CoroMiddleware()) - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") download_func = mock.MagicMock() dfd = self.mwman.download(download_func, req, self.spider) results = [] @@ -236,7 +248,7 @@ class MiddlewareUsingCoro(ManagerTestCase): @mark.only_asyncio() def test_asyncdef_asyncio(self): - resp = Response('http://example.com/index.html') + resp = Response("http://example.com/index.html") class CoroMiddleware: async def process_request(self, request, spider): @@ -245,7 +257,7 @@ class MiddlewareUsingCoro(ManagerTestCase): return result self.mwman._add_middleware(CoroMiddleware()) - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") download_func = mock.MagicMock() dfd = self.mwman.download(download_func, req, self.spider) results = [] diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 5a56c9db2..6be107f6f 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -6,13 +6,13 @@ from scrapy.http import Request, HtmlResponse, Response from scrapy.utils.test import get_crawler -__doctests__ = ['scrapy.downloadermiddlewares.ajaxcrawl'] +__doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"] class AjaxCrawlMiddlewareTest(unittest.TestCase): def setUp(self): - crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True}) - self.spider = crawler._create_spider('foo') + crawler = get_crawler(Spider, {"AJAXCRAWL_ENABLED": True}) + self.spider = crawler._create_spider("foo") self.mw = AjaxCrawlMiddleware.from_crawler(crawler) def _ajaxcrawlable_body(self): @@ -24,37 +24,41 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase): return req, resp def test_non_get(self): - req, resp = self._req_resp('http://example.com/', {'method': 'HEAD'}) + req, resp = self._req_resp("http://example.com/", {"method": "HEAD"}) resp2 = self.mw.process_response(req, resp, self.spider) self.assertEqual(resp, resp2) def test_binary_response(self): - req = Request('http://example.com/') - resp = Response('http://example.com/', body=b'foobar\x00\x01\x02', request=req) + req = Request("http://example.com/") + resp = Response("http://example.com/", body=b"foobar\x00\x01\x02", request=req) resp2 = self.mw.process_response(req, resp, self.spider) self.assertIs(resp, resp2) def test_ajaxcrawl(self): req, resp = self._req_resp( - 'http://example.com/', - {'meta': {'foo': 'bar'}}, - {'body': self._ajaxcrawlable_body()} + "http://example.com/", + {"meta": {"foo": "bar"}}, + {"body": self._ajaxcrawlable_body()}, ) req2 = self.mw.process_response(req, resp, self.spider) - self.assertEqual(req2.url, 'http://example.com/?_escaped_fragment_=') - self.assertEqual(req2.meta['foo'], 'bar') + self.assertEqual(req2.url, "http://example.com/?_escaped_fragment_=") + self.assertEqual(req2.meta["foo"], "bar") def test_ajaxcrawl_loop(self): - req, resp = self._req_resp('http://example.com/', {}, {'body': self._ajaxcrawlable_body()}) + req, resp = self._req_resp( + "http://example.com/", {}, {"body": self._ajaxcrawlable_body()} + ) req2 = self.mw.process_response(req, resp, self.spider) resp2 = HtmlResponse(req2.url, body=resp.body, request=req2) resp3 = self.mw.process_response(req2, resp2, self.spider) assert isinstance(resp3, HtmlResponse), (resp3.__class__, resp3) - self.assertEqual(resp3.request.url, 'http://example.com/?_escaped_fragment_=') + self.assertEqual(resp3.request.url, "http://example.com/?_escaped_fragment_=") assert resp3 is resp2 def test_noncrawlable_body(self): - req, resp = self._req_resp('http://example.com/', {}, {'body': b''}) + req, resp = self._req_resp( + "http://example.com/", {}, {"body": b""} + ) resp2 = self.mw.process_response(req, resp, self.spider) self.assertIs(resp, resp2) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index dd835b9c9..812c003da 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -48,24 +48,18 @@ def _cookies_to_set_cookie_list(cookies): return [] if isinstance(cookies, dict): cookies = ({"name": k, "value": v} for k, v in cookies.items()) - return filter( - None, - ( - _cookie_to_set_cookie_value(cookie) - for cookie in cookies - ) - ) + return filter(None, (_cookie_to_set_cookie_value(cookie) for cookie in cookies)) class CookiesMiddlewareTest(TestCase): - def assertCookieValEqual(self, first, second, msg=None): def split_cookies(cookies): return sorted([s.strip() for s in to_bytes(cookies).split(b";")]) + return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg) def setUp(self): - self.spider = Spider('foo') + self.spider = Spider("foo") self.mw = CookiesMiddleware() self.redirect_middleware = RedirectMiddleware(settings=Settings()) @@ -74,346 +68,391 @@ class CookiesMiddlewareTest(TestCase): del self.redirect_middleware def test_basic(self): - req = Request('http://scrapytest.org/') + req = Request("http://scrapytest.org/") assert self.mw.process_request(req, self.spider) is None - assert 'Cookie' not in req.headers + assert "Cookie" not in req.headers - headers = {'Set-Cookie': 'C1=value1; path=/'} - res = Response('http://scrapytest.org/', headers=headers) + headers = {"Set-Cookie": "C1=value1; path=/"} + res = Response("http://scrapytest.org/", headers=headers) assert self.mw.process_response(req, res, self.spider) is res - req2 = Request('http://scrapytest.org/sub1/') + req2 = Request("http://scrapytest.org/sub1/") assert self.mw.process_request(req2, self.spider) is None - self.assertEqual(req2.headers.get('Cookie'), b"C1=value1") + self.assertEqual(req2.headers.get("Cookie"), b"C1=value1") def test_setting_false_cookies_enabled(self): self.assertRaises( NotConfigured, CookiesMiddleware.from_crawler, - get_crawler(settings_dict={'COOKIES_ENABLED': False}) + get_crawler(settings_dict={"COOKIES_ENABLED": False}), ) def test_setting_default_cookies_enabled(self): self.assertIsInstance( - CookiesMiddleware.from_crawler(get_crawler()), - CookiesMiddleware + CookiesMiddleware.from_crawler(get_crawler()), CookiesMiddleware ) def test_setting_true_cookies_enabled(self): self.assertIsInstance( CookiesMiddleware.from_crawler( - get_crawler(settings_dict={'COOKIES_ENABLED': True}) + get_crawler(settings_dict={"COOKIES_ENABLED": True}) ), - CookiesMiddleware + CookiesMiddleware, ) def test_setting_enabled_cookies_debug(self): - crawler = get_crawler(settings_dict={'COOKIES_DEBUG': True}) + crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True}) mw = CookiesMiddleware.from_crawler(crawler) with LogCapture( - 'scrapy.downloadermiddlewares.cookies', + "scrapy.downloadermiddlewares.cookies", propagate=False, level=logging.DEBUG, ) as log: - req = Request('http://scrapytest.org/') - res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) + req = Request("http://scrapytest.org/") + res = Response( + "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} + ) mw.process_response(req, res, crawler.spider) - req2 = Request('http://scrapytest.org/sub1/') + req2 = Request("http://scrapytest.org/sub1/") mw.process_request(req2, crawler.spider) log.check( - ('scrapy.downloadermiddlewares.cookies', - 'DEBUG', - 'Received cookies from: <200 http://scrapytest.org/>\n' - 'Set-Cookie: C1=value1; path=/\n'), - ('scrapy.downloadermiddlewares.cookies', - 'DEBUG', - 'Sending cookies to: \n' - 'Cookie: C1=value1\n'), + ( + "scrapy.downloadermiddlewares.cookies", + "DEBUG", + "Received cookies from: <200 http://scrapytest.org/>\n" + "Set-Cookie: C1=value1; path=/\n", + ), + ( + "scrapy.downloadermiddlewares.cookies", + "DEBUG", + "Sending cookies to: \n" + "Cookie: C1=value1\n", + ), ) def test_setting_disabled_cookies_debug(self): - crawler = get_crawler(settings_dict={'COOKIES_DEBUG': False}) + crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False}) mw = CookiesMiddleware.from_crawler(crawler) with LogCapture( - 'scrapy.downloadermiddlewares.cookies', + "scrapy.downloadermiddlewares.cookies", propagate=False, level=logging.DEBUG, ) as log: - req = Request('http://scrapytest.org/') - res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) + req = Request("http://scrapytest.org/") + res = Response( + "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} + ) mw.process_response(req, res, crawler.spider) - req2 = Request('http://scrapytest.org/sub1/') + req2 = Request("http://scrapytest.org/sub1/") mw.process_request(req2, crawler.spider) log.check() def test_do_not_break_on_non_utf8_header(self): - req = Request('http://scrapytest.org/') + req = Request("http://scrapytest.org/") assert self.mw.process_request(req, self.spider) is None - assert 'Cookie' not in req.headers + assert "Cookie" not in req.headers - headers = {'Set-Cookie': b'C1=in\xa3valid; path=/', 'Other': b'ignore\xa3me'} - res = Response('http://scrapytest.org/', headers=headers) + headers = {"Set-Cookie": b"C1=in\xa3valid; path=/", "Other": b"ignore\xa3me"} + res = Response("http://scrapytest.org/", headers=headers) assert self.mw.process_response(req, res, self.spider) is res - req2 = Request('http://scrapytest.org/sub1/') + req2 = Request("http://scrapytest.org/sub1/") assert self.mw.process_request(req2, self.spider) is None - self.assertIn('Cookie', req2.headers) + self.assertIn("Cookie", req2.headers) def test_dont_merge_cookies(self): # merge some cookies into jar - headers = {'Set-Cookie': 'C1=value1; path=/'} - req = Request('http://scrapytest.org/') - res = Response('http://scrapytest.org/', headers=headers) + headers = {"Set-Cookie": "C1=value1; path=/"} + req = Request("http://scrapytest.org/") + res = Response("http://scrapytest.org/", headers=headers) assert self.mw.process_response(req, res, self.spider) is res # test Cookie header is not seted to request - req = Request('http://scrapytest.org/dontmerge', meta={'dont_merge_cookies': 1}) + req = Request("http://scrapytest.org/dontmerge", meta={"dont_merge_cookies": 1}) assert self.mw.process_request(req, self.spider) is None - assert 'Cookie' not in req.headers + assert "Cookie" not in req.headers # check that returned cookies are not merged back to jar res = Response( - 'http://scrapytest.org/dontmerge', - headers={'Set-Cookie': 'dont=mergeme; path=/'}, + "http://scrapytest.org/dontmerge", + headers={"Set-Cookie": "dont=mergeme; path=/"}, ) assert self.mw.process_response(req, res, self.spider) is res # check that cookies are merged back - req = Request('http://scrapytest.org/mergeme') + req = Request("http://scrapytest.org/mergeme") assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req.headers.get("Cookie"), b"C1=value1") # check that cookies are merged when dont_merge_cookies is passed as 0 - req = Request('http://scrapytest.org/mergeme', meta={'dont_merge_cookies': 0}) + req = Request("http://scrapytest.org/mergeme", meta={"dont_merge_cookies": 0}) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req.headers.get("Cookie"), b"C1=value1") def test_complex_cookies(self): # merge some cookies into jar cookies = [ - {'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'}, - {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'}, - {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'}, - {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}, + { + "name": "C1", + "value": "value1", + "path": "/foo", + "domain": "scrapytest.org", + }, + { + "name": "C2", + "value": "value2", + "path": "/bar", + "domain": "scrapytest.org", + }, + { + "name": "C3", + "value": "value3", + "path": "/foo", + "domain": "scrapytest.org", + }, + {"name": "C4", "value": "value4", "path": "/foo", "domain": "scrapy.org"}, ] - req = Request('http://scrapytest.org/', cookies=cookies) + req = Request("http://scrapytest.org/", cookies=cookies) self.mw.process_request(req, self.spider) # embed C1 and C3 for scrapytest.org/foo - req = Request('http://scrapytest.org/foo') + req = Request("http://scrapytest.org/foo") self.mw.process_request(req, self.spider) - assert req.headers.get('Cookie') in (b'C1=value1; C3=value3', b'C3=value3; C1=value1') + assert req.headers.get("Cookie") in ( + b"C1=value1; C3=value3", + b"C3=value3; C1=value1", + ) # embed C2 for scrapytest.org/bar - req = Request('http://scrapytest.org/bar') + req = Request("http://scrapytest.org/bar") self.mw.process_request(req, self.spider) - self.assertEqual(req.headers.get('Cookie'), b'C2=value2') + self.assertEqual(req.headers.get("Cookie"), b"C2=value2") # embed nothing for scrapytest.org/baz - req = Request('http://scrapytest.org/baz') + req = Request("http://scrapytest.org/baz") self.mw.process_request(req, self.spider) - assert 'Cookie' not in req.headers + assert "Cookie" not in req.headers def test_merge_request_cookies(self): - req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}) + req = Request("http://scrapytest.org/", cookies={"galleta": "salada"}) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get('Cookie'), b'galleta=salada') + self.assertEqual(req.headers.get("Cookie"), b"galleta=salada") - headers = {'Set-Cookie': 'C1=value1; path=/'} - res = Response('http://scrapytest.org/', headers=headers) + headers = {"Set-Cookie": "C1=value1; path=/"} + res = Response("http://scrapytest.org/", headers=headers) assert self.mw.process_response(req, res, self.spider) is res - req2 = Request('http://scrapytest.org/sub1/') + req2 = Request("http://scrapytest.org/sub1/") assert self.mw.process_request(req2, self.spider) is None - self.assertCookieValEqual(req2.headers.get('Cookie'), b"C1=value1; galleta=salada") + self.assertCookieValEqual( + req2.headers.get("Cookie"), b"C1=value1; galleta=salada" + ) def test_cookiejar_key(self): req = Request( - 'http://scrapytest.org/', - cookies={'galleta': 'salada'}, - meta={'cookiejar': "store1"}, + "http://scrapytest.org/", + cookies={"galleta": "salada"}, + meta={"cookiejar": "store1"}, ) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get('Cookie'), b'galleta=salada') + self.assertEqual(req.headers.get("Cookie"), b"galleta=salada") - headers = {'Set-Cookie': 'C1=value1; path=/'} - res = Response('http://scrapytest.org/', headers=headers, request=req) + headers = {"Set-Cookie": "C1=value1; path=/"} + res = Response("http://scrapytest.org/", headers=headers, request=req) assert self.mw.process_response(req, res, self.spider) is res - req2 = Request('http://scrapytest.org/', meta=res.meta) + req2 = Request("http://scrapytest.org/", meta=res.meta) assert self.mw.process_request(req2, self.spider) is None - self.assertCookieValEqual(req2.headers.get('Cookie'), b'C1=value1; galleta=salada') + self.assertCookieValEqual( + req2.headers.get("Cookie"), b"C1=value1; galleta=salada" + ) req3 = Request( - 'http://scrapytest.org/', - cookies={'galleta': 'dulce'}, - meta={'cookiejar': "store2"}, + "http://scrapytest.org/", + cookies={"galleta": "dulce"}, + meta={"cookiejar": "store2"}, ) assert self.mw.process_request(req3, self.spider) is None - self.assertEqual(req3.headers.get('Cookie'), b'galleta=dulce') + self.assertEqual(req3.headers.get("Cookie"), b"galleta=dulce") - headers = {'Set-Cookie': 'C2=value2; path=/'} - res2 = Response('http://scrapytest.org/', headers=headers, request=req3) + headers = {"Set-Cookie": "C2=value2; path=/"} + res2 = Response("http://scrapytest.org/", headers=headers, request=req3) assert self.mw.process_response(req3, res2, self.spider) is res2 - req4 = Request('http://scrapytest.org/', meta=res2.meta) + req4 = Request("http://scrapytest.org/", meta=res2.meta) assert self.mw.process_request(req4, self.spider) is None - self.assertCookieValEqual(req4.headers.get('Cookie'), b'C2=value2; galleta=dulce') + self.assertCookieValEqual( + req4.headers.get("Cookie"), b"C2=value2; galleta=dulce" + ) # cookies from hosts with port - req5_1 = Request('http://scrapytest.org:1104/') + req5_1 = Request("http://scrapytest.org:1104/") assert self.mw.process_request(req5_1, self.spider) is None - headers = {'Set-Cookie': 'C1=value1; path=/'} - res5_1 = Response('http://scrapytest.org:1104/', headers=headers, request=req5_1) + headers = {"Set-Cookie": "C1=value1; path=/"} + res5_1 = Response( + "http://scrapytest.org:1104/", headers=headers, request=req5_1 + ) assert self.mw.process_response(req5_1, res5_1, self.spider) is res5_1 - req5_2 = Request('http://scrapytest.org:1104/some-redirected-path') + req5_2 = Request("http://scrapytest.org:1104/some-redirected-path") assert self.mw.process_request(req5_2, self.spider) is None - self.assertEqual(req5_2.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req5_2.headers.get("Cookie"), b"C1=value1") - req5_3 = Request('http://scrapytest.org/some-redirected-path') + req5_3 = Request("http://scrapytest.org/some-redirected-path") assert self.mw.process_request(req5_3, self.spider) is None - self.assertEqual(req5_3.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req5_3.headers.get("Cookie"), b"C1=value1") # skip cookie retrieval for not http request - req6 = Request('file:///scrapy/sometempfile') + req6 = Request("file:///scrapy/sometempfile") assert self.mw.process_request(req6, self.spider) is None - self.assertEqual(req6.headers.get('Cookie'), None) + self.assertEqual(req6.headers.get("Cookie"), None) def test_local_domain(self): - request = Request("http://example-host/", cookies={'currencyCookie': 'USD'}) + request = Request("http://example-host/", cookies={"currencyCookie": "USD"}) assert self.mw.process_request(request, self.spider) is None - self.assertIn('Cookie', request.headers) - self.assertEqual(b'currencyCookie=USD', request.headers['Cookie']) + self.assertIn("Cookie", request.headers) + self.assertEqual(b"currencyCookie=USD", request.headers["Cookie"]) @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): - DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty') + DEFAULT_REQUEST_HEADERS = dict(Cookie="default=value; asdf=qwerty") mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) # overwrite with values from 'cookies' request argument - req1 = Request('http://example.org', cookies={'default': 'something'}) + req1 = Request("http://example.org", cookies={"default": "something"}) assert mw_default_headers.process_request(req1, self.spider) is None assert self.mw.process_request(req1, self.spider) is None - self.assertCookieValEqual(req1.headers['Cookie'], b'default=something; asdf=qwerty') + self.assertCookieValEqual( + req1.headers["Cookie"], b"default=something; asdf=qwerty" + ) # keep both - req2 = Request('http://example.com', cookies={'a': 'b'}) + req2 = Request("http://example.com", cookies={"a": "b"}) assert mw_default_headers.process_request(req2, self.spider) is None assert self.mw.process_request(req2, self.spider) is None - self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty') + self.assertCookieValEqual( + req2.headers["Cookie"], b"default=value; a=b; asdf=qwerty" + ) @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_header(self): # keep only cookies from 'Cookie' request header - req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}) + req1 = Request("http://scrapytest.org", headers={"Cookie": "a=b; c=d"}) assert self.mw.process_request(req1, self.spider) is None - self.assertCookieValEqual(req1.headers['Cookie'], 'a=b; c=d') + self.assertCookieValEqual(req1.headers["Cookie"], "a=b; c=d") # keep cookies from both 'Cookie' request header and 'cookies' keyword - req2 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}, cookies={'e': 'f'}) + req2 = Request( + "http://scrapytest.org", headers={"Cookie": "a=b; c=d"}, cookies={"e": "f"} + ) assert self.mw.process_request(req2, self.spider) is None - self.assertCookieValEqual(req2.headers['Cookie'], 'a=b; c=d; e=f') + self.assertCookieValEqual(req2.headers["Cookie"], "a=b; c=d; e=f") # overwrite values from 'Cookie' request header with 'cookies' keyword req3 = Request( - 'http://scrapytest.org', - headers={'Cookie': 'a=b; c=d'}, - cookies={'a': 'new', 'e': 'f'}, + "http://scrapytest.org", + headers={"Cookie": "a=b; c=d"}, + cookies={"a": "new", "e": "f"}, ) assert self.mw.process_request(req3, self.spider) is None - self.assertCookieValEqual(req3.headers['Cookie'], 'a=new; c=d; e=f') + self.assertCookieValEqual(req3.headers["Cookie"], "a=new; c=d; e=f") def test_request_cookies_encoding(self): # 1) UTF8-encoded bytes - req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')}) + req1 = Request("http://example.org", cookies={"a": "á".encode("utf8")}) assert self.mw.process_request(req1, self.spider) is None - self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') + self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") # 2) Non UTF8-encoded bytes - req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')}) + req2 = Request("http://example.org", cookies={"a": "á".encode("latin1")}) assert self.mw.process_request(req2, self.spider) is None - self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') + self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1") # 3) String - req3 = Request('http://example.org', cookies={'a': 'á'}) + req3 = Request("http://example.org", cookies={"a": "á"}) assert self.mw.process_request(req3, self.spider) is None - self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') + self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1") @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes - req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')}) + req1 = Request("http://example.org", headers={"Cookie": "a=á".encode("utf8")}) assert self.mw.process_request(req1, self.spider) is None - self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') + self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") # 2) Non UTF8-encoded bytes - req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')}) + req2 = Request("http://example.org", headers={"Cookie": "a=á".encode("latin1")}) assert self.mw.process_request(req2, self.spider) is None - self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') + self.assertCookieValEqual(req2.headers["Cookie"], b"a=\xc3\xa1") # 3) String - req3 = Request('http://example.org', headers={'Cookie': 'a=á'}) + req3 = Request("http://example.org", headers={"Cookie": "a=á"}) assert self.mw.process_request(req3, self.spider) is None - self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') + self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1") def test_invalid_cookies(self): """ Invalid cookies are logged as warnings and discarded """ with LogCapture( - 'scrapy.downloadermiddlewares.cookies', + "scrapy.downloadermiddlewares.cookies", propagate=False, level=logging.INFO, ) as lc: - cookies1 = [{'value': 'bar'}, {'name': 'key', 'value': 'value1'}] - req1 = Request('http://example.org/1', cookies=cookies1) + cookies1 = [{"value": "bar"}, {"name": "key", "value": "value1"}] + req1 = Request("http://example.org/1", cookies=cookies1) assert self.mw.process_request(req1, self.spider) is None - cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}] - req2 = Request('http://example.org/2', cookies=cookies2) + cookies2 = [{"name": "foo"}, {"name": "key", "value": "value2"}] + req2 = Request("http://example.org/2", cookies=cookies2) assert self.mw.process_request(req2, self.spider) is None - cookies3 = [{'name': 'foo', 'value': None}, {'name': 'key', 'value': ''}] - req3 = Request('http://example.org/3', cookies=cookies3) + cookies3 = [{"name": "foo", "value": None}, {"name": "key", "value": ""}] + req3 = Request("http://example.org/3", cookies=cookies3) assert self.mw.process_request(req3, self.spider) is None lc.check( - ("scrapy.downloadermiddlewares.cookies", - "WARNING", - "Invalid cookie found in request :" - " {'value': 'bar'} ('name' is missing)"), - ("scrapy.downloadermiddlewares.cookies", - "WARNING", - "Invalid cookie found in request :" - " {'name': 'foo'} ('value' is missing)"), - ("scrapy.downloadermiddlewares.cookies", - "WARNING", - "Invalid cookie found in request :" - " {'name': 'foo', 'value': None} ('value' is missing)"), + ( + "scrapy.downloadermiddlewares.cookies", + "WARNING", + "Invalid cookie found in request :" + " {'value': 'bar'} ('name' is missing)", + ), + ( + "scrapy.downloadermiddlewares.cookies", + "WARNING", + "Invalid cookie found in request :" + " {'name': 'foo'} ('value' is missing)", + ), + ( + "scrapy.downloadermiddlewares.cookies", + "WARNING", + "Invalid cookie found in request :" + " {'name': 'foo', 'value': None} ('value' is missing)", + ), ) - self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1') - self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2') - self.assertCookieValEqual(req3.headers['Cookie'], 'key=') + self.assertCookieValEqual(req1.headers["Cookie"], "key=value1") + self.assertCookieValEqual(req2.headers["Cookie"], "key=value2") + self.assertCookieValEqual(req3.headers["Cookie"], "key=") def test_primitive_type_cookies(self): # Boolean - req1 = Request('http://example.org', cookies={'a': True}) + req1 = Request("http://example.org", cookies={"a": True}) assert self.mw.process_request(req1, self.spider) is None - self.assertCookieValEqual(req1.headers['Cookie'], b'a=True') + self.assertCookieValEqual(req1.headers["Cookie"], b"a=True") # Float - req2 = Request('http://example.org', cookies={'a': 9.5}) + req2 = Request("http://example.org", cookies={"a": 9.5}) assert self.mw.process_request(req2, self.spider) is None - self.assertCookieValEqual(req2.headers['Cookie'], b'a=9.5') + self.assertCookieValEqual(req2.headers["Cookie"], b"a=9.5") # Integer - req3 = Request('http://example.org', cookies={'a': 10}) + req3 = Request("http://example.org", cookies={"a": 10}) assert self.mw.process_request(req3, self.spider) is None - self.assertCookieValEqual(req3.headers['Cookie'], b'a=10') + self.assertCookieValEqual(req3.headers["Cookie"], b"a=10") # String - req4 = Request('http://example.org', cookies={'a': 'b'}) + req4 = Request("http://example.org", cookies={"a": "b"}) assert self.mw.process_request(req4, self.spider) is None - self.assertCookieValEqual(req4.headers['Cookie'], b'a=b') + self.assertCookieValEqual(req4.headers["Cookie"], b"a=b") def _test_cookie_redirect( self, @@ -423,22 +462,22 @@ class CookiesMiddlewareTest(TestCase): cookies1, cookies2, ): - input_cookies = {'a': 'b'} + input_cookies = {"a": "b"} if not isinstance(source, dict): - source = {'url': source} + source = {"url": source} if not isinstance(target, dict): - target = {'url': target} - target.setdefault('status', 301) + target = {"url": target} + target.setdefault("status", 301) request1 = Request(cookies=input_cookies, **source) self.mw.process_request(request1, self.spider) - cookies = request1.headers.get('Cookie') + cookies = request1.headers.get("Cookie") self.assertEqual(cookies, b"a=b" if cookies1 else None) response = Response( headers={ - 'Location': target['url'], + "Location": target["url"], }, **target, ) @@ -455,37 +494,37 @@ class CookiesMiddlewareTest(TestCase): self.assertIsInstance(request2, Request) self.mw.process_request(request2, self.spider) - cookies = request2.headers.get('Cookie') + cookies = request2.headers.get("Cookie") self.assertEqual(cookies, b"a=b" if cookies2 else None) def test_cookie_redirect_same_domain(self): self._test_cookie_redirect( - 'https://toscrape.com', - 'https://toscrape.com', + "https://toscrape.com", + "https://toscrape.com", cookies1=True, cookies2=True, ) def test_cookie_redirect_same_domain_forcing_get(self): self._test_cookie_redirect( - 'https://toscrape.com', - {'url': 'https://toscrape.com', 'status': 302}, + "https://toscrape.com", + {"url": "https://toscrape.com", "status": 302}, cookies1=True, cookies2=True, ) def test_cookie_redirect_different_domain(self): self._test_cookie_redirect( - 'https://toscrape.com', - 'https://example.com', + "https://toscrape.com", + "https://example.com", cookies1=True, cookies2=False, ) def test_cookie_redirect_different_domain_forcing_get(self): self._test_cookie_redirect( - 'https://toscrape.com', - {'url': 'https://example.com', 'status': 302}, + "https://toscrape.com", + {"url": "https://example.com", "status": 302}, cookies1=True, cookies2=False, ) @@ -514,16 +553,16 @@ class CookiesMiddlewareTest(TestCase): because the middleware empties the header from every request. """ if not isinstance(source, dict): - source = {'url': source} + source = {"url": source} if not isinstance(target, dict): - target = {'url': target} - target.setdefault('status', 301) + target = {"url": target} + target.setdefault("status", 301) - request1 = Request(headers={'Cookie': b'a=b'}, **source) + request1 = Request(headers={"Cookie": b"a=b"}, **source) response = Response( headers={ - 'Location': target['url'], + "Location": target["url"], }, **target, ) @@ -535,34 +574,34 @@ class CookiesMiddlewareTest(TestCase): ) self.assertIsInstance(request2, Request) - cookies = request2.headers.get('Cookie') + cookies = request2.headers.get("Cookie") self.assertEqual(cookies, b"a=b" if cookies2 else None) def test_cookie_header_redirect_same_domain(self): self._test_cookie_header_redirect( - 'https://toscrape.com', - 'https://toscrape.com', + "https://toscrape.com", + "https://toscrape.com", cookies2=True, ) def test_cookie_header_redirect_same_domain_forcing_get(self): self._test_cookie_header_redirect( - 'https://toscrape.com', - {'url': 'https://toscrape.com', 'status': 302}, + "https://toscrape.com", + {"url": "https://toscrape.com", "status": 302}, cookies2=True, ) def test_cookie_header_redirect_different_domain(self): self._test_cookie_header_redirect( - 'https://toscrape.com', - 'https://example.com', + "https://toscrape.com", + "https://example.com", cookies2=False, ) def test_cookie_header_redirect_different_domain_forcing_get(self): self._test_cookie_header_redirect( - 'https://toscrape.com', - {'url': 'https://example.com', 'status': 302}, + "https://toscrape.com", + {"url": "https://example.com", "status": 302}, cookies2=False, ) @@ -577,54 +616,54 @@ class CookiesMiddlewareTest(TestCase): ): input_cookies = [ { - 'name': 'a', - 'value': 'b', - 'domain': domain, + "name": "a", + "value": "b", + "domain": domain, } ] request1 = Request(url1, cookies=input_cookies) self.mw.process_request(request1, self.spider) - cookies = request1.headers.get('Cookie') + cookies = request1.headers.get("Cookie") self.assertEqual(cookies, b"a=b" if cookies1 else None) request2 = Request(url2) self.mw.process_request(request2, self.spider) - cookies = request2.headers.get('Cookie') + cookies = request2.headers.get("Cookie") self.assertEqual(cookies, b"a=b" if cookies2 else None) def test_user_set_cookie_domain_suffix_private(self): self._test_user_set_cookie_domain_followup( - 'https://books.toscrape.com', - 'https://quotes.toscrape.com', - 'toscrape.com', + "https://books.toscrape.com", + "https://quotes.toscrape.com", + "toscrape.com", cookies1=True, cookies2=True, ) def test_user_set_cookie_domain_suffix_public_period(self): self._test_user_set_cookie_domain_followup( - 'https://foo.co.uk', - 'https://bar.co.uk', - 'co.uk', + "https://foo.co.uk", + "https://bar.co.uk", + "co.uk", cookies1=False, cookies2=False, ) def test_user_set_cookie_domain_suffix_public_private(self): self._test_user_set_cookie_domain_followup( - 'https://foo.blogspot.com', - 'https://bar.blogspot.com', - 'blogspot.com', + "https://foo.blogspot.com", + "https://bar.blogspot.com", + "blogspot.com", cookies1=False, cookies2=False, ) def test_user_set_cookie_domain_public_period(self): self._test_user_set_cookie_domain_followup( - 'https://co.uk', - 'https://co.uk', - 'co.uk', + "https://co.uk", + "https://co.uk", + "co.uk", cookies1=True, cookies2=True, ) @@ -642,14 +681,14 @@ class CookiesMiddlewareTest(TestCase): input_cookies = [ { - 'name': 'a', - 'value': 'b', - 'domain': domain, + "name": "a", + "value": "b", + "domain": domain, } ] headers = { - 'Set-Cookie': _cookies_to_set_cookie_list(input_cookies), + "Set-Cookie": _cookies_to_set_cookie_list(input_cookies), } response = Response(url1, status=200, headers=headers) self.assertEqual( @@ -659,37 +698,37 @@ class CookiesMiddlewareTest(TestCase): request2 = Request(url2) self.mw.process_request(request2, self.spider) - actual_cookies = request2.headers.get('Cookie') + actual_cookies = request2.headers.get("Cookie") self.assertEqual(actual_cookies, b"a=b" if cookies else None) def test_server_set_cookie_domain_suffix_private(self): self._test_server_set_cookie_domain_followup( - 'https://books.toscrape.com', - 'https://quotes.toscrape.com', - 'toscrape.com', + "https://books.toscrape.com", + "https://quotes.toscrape.com", + "toscrape.com", cookies=True, ) def test_server_set_cookie_domain_suffix_public_period(self): self._test_server_set_cookie_domain_followup( - 'https://foo.co.uk', - 'https://bar.co.uk', - 'co.uk', + "https://foo.co.uk", + "https://bar.co.uk", + "co.uk", cookies=False, ) def test_server_set_cookie_domain_suffix_public_private(self): self._test_server_set_cookie_domain_followup( - 'https://foo.blogspot.com', - 'https://bar.blogspot.com', - 'blogspot.com', + "https://foo.blogspot.com", + "https://bar.blogspot.com", + "blogspot.com", cookies=False, ) def test_server_set_cookie_domain_public_period(self): self._test_server_set_cookie_domain_followup( - 'https://co.uk', - 'https://co.uk', - 'co.uk', + "https://co.uk", + "https://co.uk", + "co.uk", cookies=True, ) diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index 1c4cae6d1..16ae9ed75 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -7,39 +7,39 @@ from tests import get_testdata def _test_data(formats): - uncompressed_body = get_testdata('compressed', 'feed-sample1.xml') + uncompressed_body = get_testdata("compressed", "feed-sample1.xml") test_responses = {} for format in formats: - body = get_testdata('compressed', 'feed-sample1.' + format) - test_responses[format] = Response('http://foo.com/bar', body=body) + body = get_testdata("compressed", "feed-sample1." + format) + test_responses[format] = Response("http://foo.com/bar", body=body) return uncompressed_body, test_responses class DecompressionMiddlewareTest(TestCase): - test_formats = ['tar', 'xml.bz2', 'xml.gz', 'zip'] + test_formats = ["tar", "xml.bz2", "xml.gz", "zip"] uncompressed_body, test_responses = _test_data(test_formats) def setUp(self): self.mw = DecompressionMiddleware() - self.spider = Spider('foo') + self.spider = Spider("foo") def test_known_compression_formats(self): for fmt in self.test_formats: rsp = self.test_responses[fmt] new = self.mw.process_response(None, rsp, self.spider) - error_msg = f'Failed {fmt}, response type {type(new).__name__}' + error_msg = f"Failed {fmt}, response type {type(new).__name__}" assert isinstance(new, XmlResponse), error_msg assert_samelines(self, new.body, self.uncompressed_body, fmt) def test_plain_response(self): - rsp = Response(url='http://test.com', body=self.uncompressed_body) + rsp = Response(url="http://test.com", body=self.uncompressed_body) new = self.mw.process_response(None, rsp, self.spider) assert new is rsp assert_samelines(self, new.body, rsp.body) def test_empty_response(self): - rsp = Response(url='http://test.com', body=b'') + rsp = Response(url="http://test.com", body=b"") new = self.mw.process_response(None, rsp, self.spider) assert new is rsp assert not rsp.body @@ -49,5 +49,5 @@ class DecompressionMiddlewareTest(TestCase): del self.mw -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 6a31dfcf8..601e85799 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -8,27 +8,26 @@ from scrapy.utils.python import to_bytes class TestDefaultHeadersMiddleware(TestCase): - def get_defaults_spider_mw(self): crawler = get_crawler(Spider) - spider = crawler._create_spider('foo') + spider = crawler._create_spider("foo") defaults = { to_bytes(k): [to_bytes(v)] - for k, v in crawler.settings.get('DEFAULT_REQUEST_HEADERS').items() + for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items() } return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler) def test_process_request(self): defaults, spider, mw = self.get_defaults_spider_mw() - req = Request('http://www.scrapytest.org') + req = Request("http://www.scrapytest.org") mw.process_request(req, spider) self.assertEqual(req.headers, defaults) def test_update_headers(self): defaults, spider, mw = self.get_defaults_spider_mw() - headers = {'Accept-Language': ['es'], 'Test-Header': ['test']} - bytes_headers = {b'Accept-Language': [b'es'], b'Test-Header': [b'test']} - req = Request('http://www.scrapytest.org', headers=headers) + headers = {"Accept-Language": ["es"], "Test-Header": ["test"]} + bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]} + req = Request("http://www.scrapytest.org", headers=headers) self.assertEqual(req.headers, bytes_headers) mw.process_request(req, spider) diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 586bdc0d1..8d2b821b0 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -7,36 +7,35 @@ from scrapy.utils.test import get_crawler class DownloadTimeoutMiddlewareTest(unittest.TestCase): - def get_request_spider_mw(self, settings=None): crawler = get_crawler(Spider, settings) - spider = crawler._create_spider('foo') - request = Request('http://scrapytest.org/') + spider = crawler._create_spider("foo") + request = Request("http://scrapytest.org/") return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler) def test_default_download_timeout(self): req, spider, mw = self.get_request_spider_mw() mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get('download_timeout'), 180) + self.assertEqual(req.meta.get("download_timeout"), 180) def test_string_download_timeout(self): - req, spider, mw = self.get_request_spider_mw({'DOWNLOAD_TIMEOUT': '20.1'}) + req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"}) mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get('download_timeout'), 20.1) + self.assertEqual(req.meta.get("download_timeout"), 20.1) def test_spider_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() spider.download_timeout = 2 mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get('download_timeout'), 2) + self.assertEqual(req.meta.get("download_timeout"), 2) def test_request_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() spider.download_timeout = 2 mw.spider_opened(spider) - req.meta['download_timeout'] = 1 + req.meta["download_timeout"] = 1 assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get('download_timeout'), 1) + self.assertEqual(req.meta.get("download_timeout"), 1) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index b9f3e24a4..1320bded2 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -10,108 +10,106 @@ from scrapy.spiders import Spider class TestSpiderLegacy(Spider): - http_user = 'foo' - http_pass = 'bar' + http_user = "foo" + http_pass = "bar" class TestSpider(Spider): - http_user = 'foo' - http_pass = 'bar' - http_auth_domain = 'example.com' + http_user = "foo" + http_pass = "bar" + http_auth_domain = "example.com" class TestSpiderAny(Spider): - http_user = 'foo' - http_pass = 'bar' + http_user = "foo" + http_pass = "bar" http_auth_domain = None class HttpAuthMiddlewareLegacyTest(unittest.TestCase): - def setUp(self): - self.spider = TestSpiderLegacy('foo') + self.spider = TestSpiderLegacy("foo") def test_auth(self): - with pytest.warns(ScrapyDeprecationWarning, - match="Using HttpAuthMiddleware without http_auth_domain is deprecated"): + with pytest.warns( + ScrapyDeprecationWarning, + match="Using HttpAuthMiddleware without http_auth_domain is deprecated", + ): mw = HttpAuthMiddleware() mw.spider_opened(self.spider) # initial request, sets the domain and sends the header - req = Request('http://example.com/') + req = Request("http://example.com/") assert mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) # subsequent request to the same domain, should send the header - req = Request('http://example.com/') + req = Request("http://example.com/") assert mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) # subsequent request to a different domain, shouldn't send the header - req = Request('http://example-noauth.com/') + req = Request("http://example-noauth.com/") assert mw.process_request(req, self.spider) is None - self.assertNotIn('Authorization', req.headers) + self.assertNotIn("Authorization", req.headers) def test_auth_already_set(self): - with pytest.warns(ScrapyDeprecationWarning, - match="Using HttpAuthMiddleware without http_auth_domain is deprecated"): + with pytest.warns( + ScrapyDeprecationWarning, + match="Using HttpAuthMiddleware without http_auth_domain is deprecated", + ): mw = HttpAuthMiddleware() mw.spider_opened(self.spider) - req = Request('http://example.com/', - headers=dict(Authorization='Digest 123')) + req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) assert mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], b'Digest 123') + self.assertEqual(req.headers["Authorization"], b"Digest 123") class HttpAuthMiddlewareTest(unittest.TestCase): - def setUp(self): self.mw = HttpAuthMiddleware() - self.spider = TestSpider('foo') + self.spider = TestSpider("foo") self.mw.spider_opened(self.spider) def tearDown(self): del self.mw def test_no_auth(self): - req = Request('http://example-noauth.com/') + req = Request("http://example-noauth.com/") assert self.mw.process_request(req, self.spider) is None - self.assertNotIn('Authorization', req.headers) + self.assertNotIn("Authorization", req.headers) def test_auth_domain(self): - req = Request('http://example.com/') + req = Request("http://example.com/") assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) def test_auth_subdomain(self): - req = Request('http://foo.example.com/') + req = Request("http://foo.example.com/") assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) def test_auth_already_set(self): - req = Request('http://example.com/', - headers=dict(Authorization='Digest 123')) + req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], b'Digest 123') + self.assertEqual(req.headers["Authorization"], b"Digest 123") class HttpAuthAnyMiddlewareTest(unittest.TestCase): - def setUp(self): self.mw = HttpAuthMiddleware() - self.spider = TestSpiderAny('foo') + self.spider = TestSpiderAny("foo") self.mw.spider_opened(self.spider) def tearDown(self): del self.mw def test_auth(self): - req = Request('http://example.com/') + req = Request("http://example.com/") assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) def test_auth_already_set(self): - req = Request('http://example.com/', - headers=dict(Authorization='Digest 123')) + req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], b'Digest 123') + self.assertEqual(req.headers["Authorization"], b"Digest 123") diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index b3d8264ba..caa89b6bd 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -15,36 +15,37 @@ from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware class _BaseTest(unittest.TestCase): - storage_class = 'scrapy.extensions.httpcache.DbmCacheStorage' - policy_class = 'scrapy.extensions.httpcache.RFC2616Policy' + storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + policy_class = "scrapy.extensions.httpcache.RFC2616Policy" def setUp(self): self.yesterday = email.utils.formatdate(time.time() - 86400) self.today = email.utils.formatdate() self.tomorrow = email.utils.formatdate(time.time() + 86400) self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider('example.com') + self.spider = self.crawler._create_spider("example.com") self.tmpdir = tempfile.mkdtemp() - self.request = Request('http://www.example.com', - headers={'User-Agent': 'test'}) - self.response = Response('http://www.example.com', - headers={'Content-Type': 'text/html'}, - body=b'test body', - status=202) + self.request = Request("http://www.example.com", headers={"User-Agent": "test"}) + self.response = Response( + "http://www.example.com", + headers={"Content-Type": "text/html"}, + body=b"test body", + status=202, + ) self.crawler.stats.open_spider(self.spider) def tearDown(self): - self.crawler.stats.close_spider(self.spider, '') + self.crawler.stats.close_spider(self.spider, "") shutil.rmtree(self.tmpdir) def _get_settings(self, **new_settings): settings = { - 'HTTPCACHE_ENABLED': True, - 'HTTPCACHE_DIR': self.tmpdir, - 'HTTPCACHE_EXPIRATION_SECS': 1, - 'HTTPCACHE_IGNORE_HTTP_CODES': [], - 'HTTPCACHE_POLICY': self.policy_class, - 'HTTPCACHE_STORAGE': self.storage_class, + "HTTPCACHE_ENABLED": True, + "HTTPCACHE_DIR": self.tmpdir, + "HTTPCACHE_EXPIRATION_SECS": 1, + "HTTPCACHE_IGNORE_HTTP_CODES": [], + "HTTPCACHE_POLICY": self.policy_class, + "HTTPCACHE_STORAGE": self.storage_class, } settings.update(new_settings) return Settings(settings) @@ -82,26 +83,32 @@ class _BaseTest(unittest.TestCase): def assertEqualRequestButWithCacheValidators(self, request1, request2): self.assertEqual(request1.url, request2.url) - assert b'If-None-Match' not in request1.headers - assert b'If-Modified-Since' not in request1.headers - assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since')) + assert b"If-None-Match" not in request1.headers + assert b"If-Modified-Since" not in request1.headers + assert any( + h in request2.headers for h in (b"If-None-Match", b"If-Modified-Since") + ) self.assertEqual(request1.body, request2.body) def test_dont_cache(self): with self._middleware() as mw: - self.request.meta['dont_cache'] = True + self.request.meta["dont_cache"] = True mw.process_response(self.request, self.response, self.spider) - self.assertEqual(mw.storage.retrieve_response(self.spider, self.request), None) + self.assertEqual( + mw.storage.retrieve_response(self.spider, self.request), None + ) with self._middleware() as mw: - self.request.meta['dont_cache'] = False + self.request.meta["dont_cache"] = False mw.process_response(self.request, self.response, self.spider) if mw.policy.should_cache_response(self.response, self.request): - self.assertIsInstance(mw.storage.retrieve_response(self.spider, self.request), self.response.__class__) + self.assertIsInstance( + mw.storage.retrieve_response(self.spider, self.request), + self.response.__class__, + ) class DefaultStorageTest(_BaseTest): - def test_storage(self): with self._storage() as storage: request2 = self.request.copy() @@ -128,8 +135,8 @@ class DefaultStorageTest(_BaseTest): with self._storage() as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response( - 'http://www.example.com', - body=b'\n.', + "http://www.example.com", + body=b"\n.", status=202, ) storage.store_response(self.spider, self.request, response) @@ -140,15 +147,15 @@ class DefaultStorageTest(_BaseTest): class DbmStorageTest(DefaultStorageTest): - storage_class = 'scrapy.extensions.httpcache.DbmCacheStorage' + storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): - dbm_module = 'tests.mocks.dummydbm' + dbm_module = "tests.mocks.dummydbm" def _get_settings(self, **new_settings): - new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module) + new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module) return super()._get_settings(**new_settings) def test_custom_dbm_module_loaded(self): @@ -159,19 +166,18 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): class FilesystemStorageTest(DefaultStorageTest): - storage_class = 'scrapy.extensions.httpcache.FilesystemCacheStorage' + storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" class FilesystemStorageGzipTest(FilesystemStorageTest): - def _get_settings(self, **new_settings): - new_settings.setdefault('HTTPCACHE_GZIP', True) + new_settings.setdefault("HTTPCACHE_GZIP", True) return super()._get_settings(**new_settings) class DummyPolicyTest(_BaseTest): - policy_class = 'scrapy.extensions.httpcache.DummyPolicy' + policy_class = "scrapy.extensions.httpcache.DummyPolicy" def test_middleware(self): with self._middleware() as mw: @@ -180,31 +186,33 @@ class DummyPolicyTest(_BaseTest): response = mw.process_request(self.request, self.spider) assert isinstance(response, HtmlResponse) self.assertEqualResponse(self.response, response) - assert 'cached' in response.flags + assert "cached" in response.flags def test_different_request_response_urls(self): with self._middleware() as mw: - req = Request('http://host.com/path') - res = Response('http://host2.net/test.html') + req = Request("http://host.com/path") + res = Response("http://host2.net/test.html") assert mw.process_request(req, self.spider) is None mw.process_response(req, res, self.spider) cached = mw.process_request(req, self.spider) assert isinstance(cached, Response) self.assertEqualResponse(res, cached) - assert 'cached' in cached.flags + assert "cached" in cached.flags def test_middleware_ignore_missing(self): with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: - self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider) + self.assertRaises( + IgnoreRequest, mw.process_request, self.request, self.spider + ) mw.process_response(self.request, self.response, self.spider) response = mw.process_request(self.request, self.spider) assert isinstance(response, HtmlResponse) self.assertEqualResponse(self.response, response) - assert 'cached' in response.flags + assert "cached" in response.flags def test_middleware_ignore_schemes(self): # http responses are cached by default - req, res = Request('http://test.com/'), Response('http://test.com/') + req, res = Request("http://test.com/"), Response("http://test.com/") with self._middleware() as mw: assert mw.process_request(req, self.spider) is None mw.process_response(req, res, self.spider) @@ -212,10 +220,10 @@ class DummyPolicyTest(_BaseTest): cached = mw.process_request(req, self.spider) assert isinstance(cached, Response), type(cached) self.assertEqualResponse(res, cached) - assert 'cached' in cached.flags + assert "cached" in cached.flags # file response is not cached by default - req, res = Request('file:///tmp/t.txt'), Response('file:///tmp/t.txt') + req, res = Request("file:///tmp/t.txt"), Response("file:///tmp/t.txt") with self._middleware() as mw: assert mw.process_request(req, self.spider) is None mw.process_response(req, res, self.spider) @@ -224,7 +232,7 @@ class DummyPolicyTest(_BaseTest): assert mw.process_request(req, self.spider) is None # s3 scheme response is cached by default - req, res = Request('s3://bucket/key'), Response('http://bucket/key') + req, res = Request("s3://bucket/key"), Response("http://bucket/key") with self._middleware() as mw: assert mw.process_request(req, self.spider) is None mw.process_response(req, res, self.spider) @@ -232,11 +240,11 @@ class DummyPolicyTest(_BaseTest): cached = mw.process_request(req, self.spider) assert isinstance(cached, Response), type(cached) self.assertEqualResponse(res, cached) - assert 'cached' in cached.flags + assert "cached" in cached.flags # ignore s3 scheme - req, res = Request('s3://bucket/key2'), Response('http://bucket/key2') - with self._middleware(HTTPCACHE_IGNORE_SCHEMES=['s3']) as mw: + req, res = Request("s3://bucket/key2"), Response("http://bucket/key2") + with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw: assert mw.process_request(req, self.spider) is None mw.process_response(req, res, self.spider) @@ -258,12 +266,12 @@ class DummyPolicyTest(_BaseTest): response = mw.process_request(self.request, self.spider) assert isinstance(response, HtmlResponse) self.assertEqualResponse(self.response, response) - assert 'cached' in response.flags + assert "cached" in response.flags class RFC2616PolicyTest(DefaultStorageTest): - policy_class = 'scrapy.extensions.httpcache.RFC2616Policy' + policy_class = "scrapy.extensions.httpcache.RFC2616Policy" def _process_requestresponse(self, mw, request, response): result = None @@ -276,17 +284,18 @@ class RFC2616PolicyTest(DefaultStorageTest): assert isinstance(result, Response) return result except Exception: - print('Request', request) - print('Response', response) - print('Result', result) + print("Request", request) + print("Response", response) + print("Result", result) raise def test_request_cacheability(self): - res0 = Response(self.request.url, status=200, - headers={'Expires': self.tomorrow}) - req0 = Request('http://example.com') - req1 = req0.replace(headers={'Cache-Control': 'no-store'}) - req2 = req0.replace(headers={'Cache-Control': 'no-cache'}) + res0 = Response( + self.request.url, status=200, headers={"Expires": self.tomorrow} + ) + req0 = Request("http://example.com") + req1 = req0.replace(headers={"Cache-Control": "no-store"}) + req2 = req0.replace(headers={"Cache-Control": "no-cache"}) with self._middleware() as mw: # response for a request with no-store must not be cached res1 = self._process_requestresponse(mw, req1, res0) @@ -294,227 +303,276 @@ class RFC2616PolicyTest(DefaultStorageTest): assert mw.storage.retrieve_response(self.spider, req1) is None # Re-do request without no-store and expect it to be cached res2 = self._process_requestresponse(mw, req0, res0) - assert 'cached' not in res2.flags + assert "cached" not in res2.flags res3 = mw.process_request(req0, self.spider) - assert 'cached' in res3.flags + assert "cached" in res3.flags self.assertEqualResponse(res2, res3) # request with no-cache directive must not return cached response # but it allows new response to be stored - res0b = res0.replace(body=b'foo') + res0b = res0.replace(body=b"foo") res4 = self._process_requestresponse(mw, req2, res0b) self.assertEqualResponse(res4, res0b) - assert 'cached' not in res4.flags + assert "cached" not in res4.flags res5 = self._process_requestresponse(mw, req0, None) self.assertEqualResponse(res5, res0b) - assert 'cached' in res5.flags + assert "cached" in res5.flags def test_response_cacheability(self): responses = [ # 304 is not cacheable no matter what servers sends (False, 304, {}), - (False, 304, {'Last-Modified': self.yesterday}), - (False, 304, {'Expires': self.tomorrow}), - (False, 304, {'Etag': 'bar'}), - (False, 304, {'Cache-Control': 'max-age=3600'}), + (False, 304, {"Last-Modified": self.yesterday}), + (False, 304, {"Expires": self.tomorrow}), + (False, 304, {"Etag": "bar"}), + (False, 304, {"Cache-Control": "max-age=3600"}), # Always obey no-store cache control - (False, 200, {'Cache-Control': 'no-store'}), - (False, 200, {'Cache-Control': 'no-store, max-age=300'}), # invalid - (False, 200, {'Cache-Control': 'no-store', 'Expires': self.tomorrow}), # invalid + (False, 200, {"Cache-Control": "no-store"}), + (False, 200, {"Cache-Control": "no-store, max-age=300"}), # invalid + ( + False, + 200, + {"Cache-Control": "no-store", "Expires": self.tomorrow}, + ), # invalid # Ignore responses missing expiration and/or validation headers (False, 200, {}), (False, 302, {}), (False, 307, {}), (False, 404, {}), # Cache responses with expiration and/or validation headers - (True, 200, {'Last-Modified': self.yesterday}), - (True, 203, {'Last-Modified': self.yesterday}), - (True, 300, {'Last-Modified': self.yesterday}), - (True, 301, {'Last-Modified': self.yesterday}), - (True, 308, {'Last-Modified': self.yesterday}), - (True, 401, {'Last-Modified': self.yesterday}), - (True, 404, {'Cache-Control': 'public, max-age=600'}), - (True, 302, {'Expires': self.tomorrow}), - (True, 200, {'Etag': 'foo'}), + (True, 200, {"Last-Modified": self.yesterday}), + (True, 203, {"Last-Modified": self.yesterday}), + (True, 300, {"Last-Modified": self.yesterday}), + (True, 301, {"Last-Modified": self.yesterday}), + (True, 308, {"Last-Modified": self.yesterday}), + (True, 401, {"Last-Modified": self.yesterday}), + (True, 404, {"Cache-Control": "public, max-age=600"}), + (True, 302, {"Expires": self.tomorrow}), + (True, 200, {"Etag": "foo"}), ] with self._middleware() as mw: for idx, (shouldcache, status, headers) in enumerate(responses): - req0 = Request(f'http://example-{idx}.com') + req0 = Request(f"http://example-{idx}.com") res0 = Response(req0.url, status=status, headers=headers) res1 = self._process_requestresponse(mw, req0, res0) res304 = res0.replace(status=304) - res2 = self._process_requestresponse(mw, req0, res304 if shouldcache else res0) + res2 = self._process_requestresponse( + mw, req0, res304 if shouldcache else res0 + ) self.assertEqualResponse(res1, res0) self.assertEqualResponse(res2, res0) resc = mw.storage.retrieve_response(self.spider, req0) if shouldcache: self.assertEqualResponse(resc, res1) - assert 'cached' in res2.flags and res2.status != 304 + assert "cached" in res2.flags and res2.status != 304 else: self.assertFalse(resc) - assert 'cached' not in res2.flags + assert "cached" not in res2.flags # cache unconditionally unless response contains no-store or is a 304 with self._middleware(HTTPCACHE_ALWAYS_STORE=True) as mw: for idx, (_, status, headers) in enumerate(responses): - shouldcache = 'no-store' not in headers.get('Cache-Control', '') and status != 304 - req0 = Request(f'http://example2-{idx}.com') + shouldcache = ( + "no-store" not in headers.get("Cache-Control", "") and status != 304 + ) + req0 = Request(f"http://example2-{idx}.com") res0 = Response(req0.url, status=status, headers=headers) res1 = self._process_requestresponse(mw, req0, res0) res304 = res0.replace(status=304) - res2 = self._process_requestresponse(mw, req0, res304 if shouldcache else res0) + res2 = self._process_requestresponse( + mw, req0, res304 if shouldcache else res0 + ) self.assertEqualResponse(res1, res0) self.assertEqualResponse(res2, res0) resc = mw.storage.retrieve_response(self.spider, req0) if shouldcache: self.assertEqualResponse(resc, res1) - assert 'cached' in res2.flags and res2.status != 304 + assert "cached" in res2.flags and res2.status != 304 else: self.assertFalse(resc) - assert 'cached' not in res2.flags + assert "cached" not in res2.flags def test_cached_and_fresh(self): sampledata = [ - (200, {'Date': self.yesterday, 'Expires': self.tomorrow}), - (200, {'Date': self.yesterday, 'Cache-Control': 'max-age=86405'}), - (200, {'Age': '299', 'Cache-Control': 'max-age=300'}), + (200, {"Date": self.yesterday, "Expires": self.tomorrow}), + (200, {"Date": self.yesterday, "Cache-Control": "max-age=86405"}), + (200, {"Age": "299", "Cache-Control": "max-age=300"}), # Obey max-age if present over any others - (200, {'Date': self.today, - 'Age': '86405', - 'Cache-Control': 'max-age=' + str(86400 * 3), - 'Expires': self.yesterday, - 'Last-Modified': self.yesterday, - }), + ( + 200, + { + "Date": self.today, + "Age": "86405", + "Cache-Control": "max-age=" + str(86400 * 3), + "Expires": self.yesterday, + "Last-Modified": self.yesterday, + }, + ), # obey Expires if max-age is not present - (200, {'Date': self.yesterday, - 'Age': '86400', - 'Cache-Control': 'public', - 'Expires': self.tomorrow, - 'Last-Modified': self.yesterday, - }), + ( + 200, + { + "Date": self.yesterday, + "Age": "86400", + "Cache-Control": "public", + "Expires": self.tomorrow, + "Last-Modified": self.yesterday, + }, + ), # Default missing Date header to right now - (200, {'Expires': self.tomorrow}), + (200, {"Expires": self.tomorrow}), # Firefox - Expires if age is greater than 10% of (Date - Last-Modified) - (200, {'Date': self.today, 'Last-Modified': self.yesterday, 'Age': str(86400 / 10 - 1)}), + ( + 200, + { + "Date": self.today, + "Last-Modified": self.yesterday, + "Age": str(86400 / 10 - 1), + }, + ), # Firefox - Set one year maxage to permanent redirects missing expiration info - (300, {}), (301, {}), (308, {}), + (300, {}), + (301, {}), + (308, {}), ] with self._middleware() as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request(f'http://example-{idx}.com') + req0 = Request(f"http://example-{idx}.com") res0 = Response(req0.url, status=status, headers=headers) # cache fresh response res1 = self._process_requestresponse(mw, req0, res0) self.assertEqualResponse(res1, res0) - assert 'cached' not in res1.flags + assert "cached" not in res1.flags # return fresh cached response without network interaction res2 = self._process_requestresponse(mw, req0, None) self.assertEqualResponse(res1, res2) - assert 'cached' in res2.flags + assert "cached" in res2.flags # validate cached response if request max-age set as 0 - req1 = req0.replace(headers={'Cache-Control': 'max-age=0'}) + req1 = req0.replace(headers={"Cache-Control": "max-age=0"}) res304 = res0.replace(status=304) assert mw.process_request(req1, self.spider) is None res3 = self._process_requestresponse(mw, req1, res304) self.assertEqualResponse(res1, res3) - assert 'cached' in res3.flags + assert "cached" in res3.flags def test_cached_and_stale(self): sampledata = [ - (200, {'Date': self.today, 'Expires': self.yesterday}), - (200, {'Date': self.today, 'Expires': self.yesterday, 'Last-Modified': self.yesterday}), - (200, {'Expires': self.yesterday}), - (200, {'Expires': self.yesterday, 'ETag': 'foo'}), - (200, {'Expires': self.yesterday, 'Last-Modified': self.yesterday}), - (200, {'Expires': self.tomorrow, 'Age': '86405'}), - (200, {'Cache-Control': 'max-age=86400', 'Age': '86405'}), + (200, {"Date": self.today, "Expires": self.yesterday}), + ( + 200, + { + "Date": self.today, + "Expires": self.yesterday, + "Last-Modified": self.yesterday, + }, + ), + (200, {"Expires": self.yesterday}), + (200, {"Expires": self.yesterday, "ETag": "foo"}), + (200, {"Expires": self.yesterday, "Last-Modified": self.yesterday}), + (200, {"Expires": self.tomorrow, "Age": "86405"}), + (200, {"Cache-Control": "max-age=86400", "Age": "86405"}), # no-cache forces expiration, also revalidation if validators exists - (200, {'Cache-Control': 'no-cache'}), - (200, {'Cache-Control': 'no-cache', 'ETag': 'foo'}), - (200, {'Cache-Control': 'no-cache', 'Last-Modified': self.yesterday}), - (200, {'Cache-Control': 'no-cache,must-revalidate', 'Last-Modified': self.yesterday}), - (200, {'Cache-Control': 'must-revalidate', 'Expires': self.yesterday, 'Last-Modified': self.yesterday}), - (200, {'Cache-Control': 'max-age=86400,must-revalidate', 'Age': '86405'}), + (200, {"Cache-Control": "no-cache"}), + (200, {"Cache-Control": "no-cache", "ETag": "foo"}), + (200, {"Cache-Control": "no-cache", "Last-Modified": self.yesterday}), + ( + 200, + { + "Cache-Control": "no-cache,must-revalidate", + "Last-Modified": self.yesterday, + }, + ), + ( + 200, + { + "Cache-Control": "must-revalidate", + "Expires": self.yesterday, + "Last-Modified": self.yesterday, + }, + ), + (200, {"Cache-Control": "max-age=86400,must-revalidate", "Age": "86405"}), ] with self._middleware() as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request(f'http://example-{idx}.com') + req0 = Request(f"http://example-{idx}.com") res0a = Response(req0.url, status=status, headers=headers) # cache expired response res1 = self._process_requestresponse(mw, req0, res0a) self.assertEqualResponse(res1, res0a) - assert 'cached' not in res1.flags + assert "cached" not in res1.flags # Same request but as cached response is stale a new response must # be returned - res0b = res0a.replace(body=b'bar') + res0b = res0a.replace(body=b"bar") res2 = self._process_requestresponse(mw, req0, res0b) self.assertEqualResponse(res2, res0b) - assert 'cached' not in res2.flags - cc = headers.get('Cache-Control', '') + assert "cached" not in res2.flags + cc = headers.get("Cache-Control", "") # Previous response expired too, subsequent request to same # resource must revalidate and succeed on 304 if validators # are present - if 'ETag' in headers or 'Last-Modified' in headers: + if "ETag" in headers or "Last-Modified" in headers: res0c = res0b.replace(status=304) res3 = self._process_requestresponse(mw, req0, res0c) self.assertEqualResponse(res3, res0b) - assert 'cached' in res3.flags + assert "cached" in res3.flags # get cached response on server errors unless must-revalidate # in cached response res0d = res0b.replace(status=500) res4 = self._process_requestresponse(mw, req0, res0d) - if 'must-revalidate' in cc: - assert 'cached' not in res4.flags + if "must-revalidate" in cc: + assert "cached" not in res4.flags self.assertEqualResponse(res4, res0d) else: - assert 'cached' in res4.flags + assert "cached" in res4.flags self.assertEqualResponse(res4, res0b) # Requests with max-stale can fetch expired cached responses # unless cached response has must-revalidate - req1 = req0.replace(headers={'Cache-Control': 'max-stale'}) + req1 = req0.replace(headers={"Cache-Control": "max-stale"}) res5 = self._process_requestresponse(mw, req1, res0b) self.assertEqualResponse(res5, res0b) - if 'no-cache' in cc or 'must-revalidate' in cc: - assert 'cached' not in res5.flags + if "no-cache" in cc or "must-revalidate" in cc: + assert "cached" not in res5.flags else: - assert 'cached' in res5.flags + assert "cached" in res5.flags def test_process_exception(self): with self._middleware() as mw: - res0 = Response(self.request.url, headers={'Expires': self.yesterday}) + res0 = Response(self.request.url, headers={"Expires": self.yesterday}) req0 = Request(self.request.url) self._process_requestresponse(mw, req0, res0) for e in mw.DOWNLOAD_EXCEPTIONS: # Simulate encountering an error on download attempts assert mw.process_request(req0, self.spider) is None - res1 = mw.process_exception(req0, e('foo'), self.spider) + res1 = mw.process_exception(req0, e("foo"), self.spider) # Use cached response as recovery - assert 'cached' in res1.flags + assert "cached" in res1.flags self.assertEqualResponse(res0, res1) # Do not use cached response for unhandled exceptions mw.process_request(req0, self.spider) - assert mw.process_exception(req0, Exception('foo'), self.spider) is None + assert mw.process_exception(req0, Exception("foo"), self.spider) is None def test_ignore_response_cache_controls(self): sampledata = [ - (200, {'Date': self.yesterday, 'Expires': self.tomorrow}), - (200, {'Date': self.yesterday, 'Cache-Control': 'no-store,max-age=86405'}), - (200, {'Age': '299', 'Cache-Control': 'max-age=300,no-cache'}), - (300, {'Cache-Control': 'no-cache'}), - (200, {'Expires': self.tomorrow, 'Cache-Control': 'no-store'}), + (200, {"Date": self.yesterday, "Expires": self.tomorrow}), + (200, {"Date": self.yesterday, "Cache-Control": "no-store,max-age=86405"}), + (200, {"Age": "299", "Cache-Control": "max-age=300,no-cache"}), + (300, {"Cache-Control": "no-cache"}), + (200, {"Expires": self.tomorrow, "Cache-Control": "no-store"}), ] - with self._middleware(HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS=['no-cache', 'no-store']) as mw: + with self._middleware( + HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS=["no-cache", "no-store"] + ) as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request(f'http://example-{idx}.com') + req0 = Request(f"http://example-{idx}.com") res0 = Response(req0.url, status=status, headers=headers) # cache fresh response res1 = self._process_requestresponse(mw, req0, res0) self.assertEqualResponse(res1, res0) - assert 'cached' not in res1.flags + assert "cached" not in res1.flags # return fresh cached response without network interaction res2 = self._process_requestresponse(mw, req0, None) self.assertEqualResponse(res1, res2) - assert 'cached' in res2.flags + assert "cached" in res2.flags -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index b0272143d..efae7c4e0 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -7,35 +7,40 @@ from warnings import catch_warnings from w3lib.encoding import resolve_encoding from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse -from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS +from scrapy.downloadermiddlewares.httpcompression import ( + HttpCompressionMiddleware, + ACCEPTED_ENCODINGS, +) from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir -SAMPLEDIR = Path(tests_datadir, 'compressed') +SAMPLEDIR = Path(tests_datadir, "compressed") FORMAT = { - 'gzip': ('html-gzip.bin', 'gzip'), - 'x-gzip': ('html-gzip.bin', 'gzip'), - 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), - 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), - 'br': ('html-br.bin', 'br'), + "gzip": ("html-gzip.bin", "gzip"), + "x-gzip": ("html-gzip.bin", "gzip"), + "rawdeflate": ("html-rawdeflate.bin", "deflate"), + "zlibdeflate": ("html-zlibdeflate.bin", "deflate"), + "br": ("html-br.bin", "br"), # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin - 'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'), + "zstd-static-content-size": ("html-zstd-static-content-size.bin", "zstd"), # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin - 'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), + "zstd-static-no-content-size": ("html-zstd-static-no-content-size.bin", "zstd"), # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin - 'zstd-streaming-no-content-size': ('html-zstd-streaming-no-content-size.bin', 'zstd'), + "zstd-streaming-no-content-size": ( + "html-zstd-streaming-no-content-size.bin", + "zstd", + ), } class HttpCompressionTest(TestCase): - def setUp(self): self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider('scrapytest.org') + self.spider = self.crawler._create_spider("scrapytest.org") self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) self.crawler.stats.open_spider(self.spider) @@ -48,90 +53,93 @@ class HttpCompressionTest(TestCase): body = (SAMPLEDIR / samplefile).read_bytes() headers = { - 'Server': 'Yaws/1.49 Yet Another Web Server', - 'Date': 'Sun, 08 Mar 2009 00:41:03 GMT', - 'Content-Length': len(body), - 'Content-Type': 'text/html', - 'Content-Encoding': contentencoding, + "Server": "Yaws/1.49 Yet Another Web Server", + "Date": "Sun, 08 Mar 2009 00:41:03 GMT", + "Content-Length": len(body), + "Content-Type": "text/html", + "Content-Encoding": contentencoding, } - response = Response('http://scrapytest.org/', body=body, headers=headers) - response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'}) + response = Response("http://scrapytest.org/", body=body, headers=headers) + response.request = Request( + "http://scrapytest.org", headers={"Accept-Encoding": "gzip, deflate"} + ) return response def assertStatsEqual(self, key, value): self.assertEqual( self.crawler.stats.get_value(key, spider=self.spider), value, - str(self.crawler.stats.get_stats(self.spider)) + str(self.crawler.stats.get_stats(self.spider)), ) def test_setting_false_compression_enabled(self): self.assertRaises( NotConfigured, HttpCompressionMiddleware.from_crawler, - get_crawler(settings_dict={'COMPRESSION_ENABLED': False}) + get_crawler(settings_dict={"COMPRESSION_ENABLED": False}), ) def test_setting_default_compression_enabled(self): self.assertIsInstance( HttpCompressionMiddleware.from_crawler(get_crawler()), - HttpCompressionMiddleware + HttpCompressionMiddleware, ) def test_setting_true_compression_enabled(self): self.assertIsInstance( HttpCompressionMiddleware.from_crawler( - get_crawler(settings_dict={'COMPRESSION_ENABLED': True}) + get_crawler(settings_dict={"COMPRESSION_ENABLED": True}) ), - HttpCompressionMiddleware + HttpCompressionMiddleware, ) def test_process_request(self): - request = Request('http://scrapytest.org') - assert 'Accept-Encoding' not in request.headers + request = Request("http://scrapytest.org") + assert "Accept-Encoding" not in request.headers self.mw.process_request(request, self.spider) - self.assertEqual(request.headers.get('Accept-Encoding'), - b', '.join(ACCEPTED_ENCODINGS)) + self.assertEqual( + request.headers.get("Accept-Encoding"), b", ".join(ACCEPTED_ENCODINGS) + ) def test_process_response_gzip(self): - response = self._getresponse('gzip') + response = self._getresponse("gzip") request = response.request - self.assertEqual(response.headers['Content-Encoding'], b'gzip') + self.assertEqual(response.headers["Content-Encoding"], b"gzip") newresponse = self.mw.process_response(request, response, self.spider) assert newresponse is not response - assert newresponse.body.startswith(b'Some page' - b'') - zf = GzipFile(fileobj=f, mode='wb') + plainbody = ( + b"Some page" + b'' + ) + zf = GzipFile(fileobj=f, mode="wb") zf.write(plainbody) zf.close() - response = Response("http;//www.example.com/", headers=headers, body=f.getvalue()) + response = Response( + "http;//www.example.com/", headers=headers, body=f.getvalue() + ) request = Request("http://www.example.com/") newresponse = self.mw.process_response(request, response, self.spider) assert isinstance(newresponse, HtmlResponse) self.assertEqual(newresponse.body, plainbody) - self.assertEqual(newresponse.encoding, resolve_encoding('gb2312')) - self.assertStatsEqual('httpcompression/response_count', 1) - self.assertStatsEqual('httpcompression/response_bytes', 104) + self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) + self.assertStatsEqual("httpcompression/response_count", 1) + self.assertStatsEqual("httpcompression/response_bytes", 104) def test_process_response_force_recalculate_encoding(self): headers = { - 'Content-Type': 'text/html', - 'Content-Encoding': 'gzip', + "Content-Type": "text/html", + "Content-Encoding": "gzip", } f = BytesIO() - plainbody = (b'Some page' - b'') - zf = GzipFile(fileobj=f, mode='wb') + plainbody = ( + b"Some page" + b'' + ) + zf = GzipFile(fileobj=f, mode="wb") zf.write(plainbody) zf.close() - response = HtmlResponse("http;//www.example.com/page.html", headers=headers, body=f.getvalue()) + response = HtmlResponse( + "http;//www.example.com/page.html", headers=headers, body=f.getvalue() + ) request = Request("http://www.example.com/") newresponse = self.mw.process_response(request, response, self.spider) assert isinstance(newresponse, HtmlResponse) self.assertEqual(newresponse.body, plainbody) - self.assertEqual(newresponse.encoding, resolve_encoding('gb2312')) - self.assertStatsEqual('httpcompression/response_count', 1) - self.assertStatsEqual('httpcompression/response_bytes', 104) + self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) + self.assertStatsEqual("httpcompression/response_count", 1) + self.assertStatsEqual("httpcompression/response_bytes", 104) def test_process_response_no_content_type_header(self): headers = { - 'Content-Encoding': 'identity', + "Content-Encoding": "identity", } - plainbody = (b'Some page' - b'') - respcls = responsetypes.from_args(url="http://www.example.com/index", headers=headers, body=plainbody) - response = respcls("http://www.example.com/index", headers=headers, body=plainbody) + plainbody = ( + b"Some page" + b'' + ) + respcls = responsetypes.from_args( + url="http://www.example.com/index", headers=headers, body=plainbody + ) + response = respcls( + "http://www.example.com/index", headers=headers, body=plainbody + ) request = Request("http://www.example.com/index") newresponse = self.mw.process_response(request, response, self.spider) assert isinstance(newresponse, respcls) self.assertEqual(newresponse.body, plainbody) - self.assertEqual(newresponse.encoding, resolve_encoding('gb2312')) - self.assertStatsEqual('httpcompression/response_count', 1) - self.assertStatsEqual('httpcompression/response_bytes', 104) + self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) + self.assertStatsEqual("httpcompression/response_count", 1) + self.assertStatsEqual("httpcompression/response_bytes", 104) def test_process_response_gzipped_contenttype(self): - response = self._getresponse('gzip') - response.headers['Content-Type'] = 'application/gzip' + response = self._getresponse("gzip") + response.headers["Content-Type"] = "application/gzip" request = response.request newresponse = self.mw.process_response(request, response, self.spider) self.assertIsNot(newresponse, response) - self.assertTrue(newresponse.body.startswith(b'0.8 """ - gz_file = GzipFile(fileobj=f, mode='wb') + gz_file = GzipFile(fileobj=f, mode="wb") gz_file.write(plainbody) gz_file.close() # build a gzipped response body containing this gzipped file r = BytesIO() - gz_resp = GzipFile(fileobj=r, mode='wb') + gz_resp = GzipFile(fileobj=r, mode="wb") gz_resp.write(f.getvalue()) gz_resp.close() - response = Response("http;//www.example.com/", headers=headers, body=r.getvalue()) + response = Response( + "http;//www.example.com/", headers=headers, body=r.getvalue() + ) request = Request("http://www.example.com/") newresponse = self.mw.process_response(request, response, self.spider) self.assertEqual(gunzip(newresponse.body), plainbody) - self.assertStatsEqual('httpcompression/response_count', 1) - self.assertStatsEqual('httpcompression/response_bytes', 230) + self.assertStatsEqual("httpcompression/response_count", 1) + self.assertStatsEqual("httpcompression/response_bytes", 230) def test_process_response_head_request_no_decode_required(self): - response = self._getresponse('gzip') - response.headers['Content-Type'] = 'application/gzip' + response = self._getresponse("gzip") + response.headers["Content-Type"] = "application/gzip" request = response.request - request.method = 'HEAD' + request.method = "HEAD" response = response.replace(body=None) newresponse = self.mw.process_response(request, response, self.spider) self.assertIs(newresponse, response) - self.assertEqual(response.body, b'') - self.assertStatsEqual('httpcompression/response_count', None) - self.assertStatsEqual('httpcompression/response_bytes', None) + self.assertEqual(response.body, b"") + self.assertStatsEqual("httpcompression/response_count", None) + self.assertStatsEqual("httpcompression/response_bytes", None) class HttpCompressionSubclassTest(TestCase): - def test_init_missing_stats(self): class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware): - def __init__(self): super().__init__() @@ -361,7 +383,8 @@ class HttpCompressionSubclassTest(TestCase): with catch_warnings(record=True) as caught_warnings: HttpCompressionMiddlewareSubclass.from_crawler(crawler) messages = tuple( - str(warning.message) for warning in caught_warnings + str(warning.message) + for warning in caught_warnings if warning.category is ScrapyDeprecationWarning ) self.assertEqual( @@ -372,5 +395,5 @@ class HttpCompressionSubclassTest(TestCase): "their '__init__' method to support a 'stats' parameter " "or reimplement the 'from_crawler' method." ), - ) + ), ) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 44434f90e..ca125ba36 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -9,7 +9,7 @@ from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -spider = Spider('foo') +spider = Spider("foo") class TestHttpProxyMiddleware(TestCase): @@ -23,184 +23,197 @@ class TestHttpProxyMiddleware(TestCase): os.environ = self._oldenv def test_not_enabled(self): - crawler = get_crawler(Spider, {'HTTPPROXY_ENABLED': False}) + crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False}) with pytest.raises(NotConfigured): HttpProxyMiddleware.from_crawler(crawler) def test_no_environment_proxies(self): - os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} + os.environ = {"dummy_proxy": "reset_env_and_do_not_raise"} mw = HttpProxyMiddleware() - for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'): + for url in ("http://e.com", "https://e.com", "file:///tmp/a"): req = Request(url) assert mw.process_request(req, spider) is None self.assertEqual(req.url, url) self.assertEqual(req.meta, {}) def test_environment_proxies(self): - os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' - os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080' - os.environ.pop('file_proxy', None) + os.environ["http_proxy"] = http_proxy = "https://proxy.for.http:3128" + os.environ["https_proxy"] = https_proxy = "http://proxy.for.https:8080" + os.environ.pop("file_proxy", None) mw = HttpProxyMiddleware() for url, proxy in [ - ('http://e.com', http_proxy), - ('https://e.com', https_proxy), - ('file://tmp/a', None), + ("http://e.com", http_proxy), + ("https://e.com", https_proxy), + ("file://tmp/a", None), ]: req = Request(url) assert mw.process_request(req, spider) is None self.assertEqual(req.url, url) - self.assertEqual(req.meta.get('proxy'), proxy) + self.assertEqual(req.meta.get("proxy"), proxy) def test_proxy_precedence_meta(self): - os.environ['http_proxy'] = 'https://proxy.com' + os.environ["http_proxy"] = "https://proxy.com" mw = HttpProxyMiddleware() - req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'}) + req = Request("http://scrapytest.org", meta={"proxy": "https://new.proxy:3128"}) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://new.proxy:3128'}) + self.assertEqual(req.meta, {"proxy": "https://new.proxy:3128"}) def test_proxy_auth(self): - os.environ['http_proxy'] = 'https://user:pass@proxy:3128' + os.environ["http_proxy"] = "https://user:pass@proxy:3128" mw = HttpProxyMiddleware() - req = Request('http://scrapytest.org') + req = Request("http://scrapytest.org") assert mw.process_request(req, spider) is None - self.assertEqual(req.meta['proxy'], 'https://proxy:3128') - self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') + self.assertEqual(req.meta["proxy"], "https://proxy:3128") + self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic dXNlcjpwYXNz") # proxy from request.meta - req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'}) + req = Request( + "http://scrapytest.org", + meta={"proxy": "https://username:password@proxy:3128"}, + ) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta['proxy'], 'https://proxy:3128') - self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') + self.assertEqual(req.meta["proxy"], "https://proxy:3128") + self.assertEqual( + req.headers.get("Proxy-Authorization"), b"Basic dXNlcm5hbWU6cGFzc3dvcmQ=" + ) def test_proxy_auth_empty_passwd(self): - os.environ['http_proxy'] = 'https://user:@proxy:3128' + os.environ["http_proxy"] = "https://user:@proxy:3128" mw = HttpProxyMiddleware() - req = Request('http://scrapytest.org') + req = Request("http://scrapytest.org") assert mw.process_request(req, spider) is None - self.assertEqual(req.meta['proxy'], 'https://proxy:3128') - self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') + self.assertEqual(req.meta["proxy"], "https://proxy:3128") + self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic dXNlcjo=") # proxy from request.meta - req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'}) + req = Request( + "http://scrapytest.org", meta={"proxy": "https://username:@proxy:3128"} + ) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta['proxy'], 'https://proxy:3128') - self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') + self.assertEqual(req.meta["proxy"], "https://proxy:3128") + self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic dXNlcm5hbWU6") def test_proxy_auth_encoding(self): # utf-8 encoding - os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128' - mw = HttpProxyMiddleware(auth_encoding='utf-8') - req = Request('http://scrapytest.org') + os.environ["http_proxy"] = "https://m\u00E1n:pass@proxy:3128" + mw = HttpProxyMiddleware(auth_encoding="utf-8") + req = Request("http://scrapytest.org") assert mw.process_request(req, spider) is None - self.assertEqual(req.meta['proxy'], 'https://proxy:3128') - self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') + self.assertEqual(req.meta["proxy"], "https://proxy:3128") + self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic bcOhbjpwYXNz") # proxy from request.meta - req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) + req = Request( + "http://scrapytest.org", meta={"proxy": "https://\u00FCser:pass@proxy:3128"} + ) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta['proxy'], 'https://proxy:3128') - self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') + self.assertEqual(req.meta["proxy"], "https://proxy:3128") + self.assertEqual( + req.headers.get("Proxy-Authorization"), b"Basic w7xzZXI6cGFzcw==" + ) # default latin-1 encoding - mw = HttpProxyMiddleware(auth_encoding='latin-1') - req = Request('http://scrapytest.org') + mw = HttpProxyMiddleware(auth_encoding="latin-1") + req = Request("http://scrapytest.org") assert mw.process_request(req, spider) is None - self.assertEqual(req.meta['proxy'], 'https://proxy:3128') - self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') + self.assertEqual(req.meta["proxy"], "https://proxy:3128") + self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic beFuOnBhc3M=") # proxy from request.meta, latin-1 encoding - req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) + req = Request( + "http://scrapytest.org", meta={"proxy": "https://\u00FCser:pass@proxy:3128"} + ) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta['proxy'], 'https://proxy:3128') - self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') + self.assertEqual(req.meta["proxy"], "https://proxy:3128") + self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic /HNlcjpwYXNz") def test_proxy_already_seted(self): - os.environ['http_proxy'] = 'https://proxy.for.http:3128' + os.environ["http_proxy"] = "https://proxy.for.http:3128" mw = HttpProxyMiddleware() - req = Request('http://noproxy.com', meta={'proxy': None}) + req = Request("http://noproxy.com", meta={"proxy": None}) assert mw.process_request(req, spider) is None - assert 'proxy' in req.meta and req.meta['proxy'] is None + assert "proxy" in req.meta and req.meta["proxy"] is None def test_no_proxy(self): - os.environ['http_proxy'] = 'https://proxy.for.http:3128' + os.environ["http_proxy"] = "https://proxy.for.http:3128" mw = HttpProxyMiddleware() - os.environ['no_proxy'] = '*' - req = Request('http://noproxy.com') + os.environ["no_proxy"] = "*" + req = Request("http://noproxy.com") assert mw.process_request(req, spider) is None - assert 'proxy' not in req.meta + assert "proxy" not in req.meta - os.environ['no_proxy'] = 'other.com' - req = Request('http://noproxy.com') + os.environ["no_proxy"] = "other.com" + req = Request("http://noproxy.com") assert mw.process_request(req, spider) is None - assert 'proxy' in req.meta + assert "proxy" in req.meta - os.environ['no_proxy'] = 'other.com,noproxy.com' - req = Request('http://noproxy.com') + os.environ["no_proxy"] = "other.com,noproxy.com" + req = Request("http://noproxy.com") assert mw.process_request(req, spider) is None - assert 'proxy' not in req.meta + assert "proxy" not in req.meta # proxy from meta['proxy'] takes precedence - os.environ['no_proxy'] = '*' - req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'}) + os.environ["no_proxy"] = "*" + req = Request("http://noproxy.com", meta={"proxy": "http://proxy.com"}) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'http://proxy.com'}) + self.assertEqual(req.meta, {"proxy": "http://proxy.com"}) def test_no_proxy_invalid_values(self): - os.environ['no_proxy'] = '/var/run/docker.sock' + os.environ["no_proxy"] = "/var/run/docker.sock" mw = HttpProxyMiddleware() # '/var/run/docker.sock' may be used by the user for # no_proxy value but is not parseable and should be skipped - assert 'no' not in mw.proxies + assert "no" not in mw.proxies def test_add_proxy_without_credentials(self): middleware = HttpProxyMiddleware() - request = Request('https://example.com') + request = Request("https://example.com") assert middleware.process_request(request, spider) is None - request.meta['proxy'] = 'https://example.com' + request.meta["proxy"] = "https://example.com" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.com') - self.assertNotIn(b'Proxy-Authorization', request.headers) + self.assertEqual(request.meta["proxy"], "https://example.com") + self.assertNotIn(b"Proxy-Authorization", request.headers) def test_add_proxy_with_credentials(self): middleware = HttpProxyMiddleware() - request = Request('https://example.com') + request = Request("https://example.com") assert middleware.process_request(request, spider) is None - request.meta['proxy'] = 'https://user1:password1@example.com' + request.meta["proxy"] = "https://user1:password1@example.com" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual(request.meta["proxy"], "https://example.com") encoded_credentials = middleware._basic_auth_header( - 'user1', - 'password1', + "user1", + "password1", ) self.assertEqual( - request.headers['Proxy-Authorization'], - b'Basic ' + encoded_credentials, + request.headers["Proxy-Authorization"], + b"Basic " + encoded_credentials, ) def test_remove_proxy_without_credentials(self): middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - meta={'proxy': 'https://example.com'}, + "https://example.com", + meta={"proxy": "https://example.com"}, ) assert middleware.process_request(request, spider) is None - request.meta['proxy'] = None + request.meta["proxy"] = None assert middleware.process_request(request, spider) is None - self.assertIsNone(request.meta['proxy']) - self.assertNotIn(b'Proxy-Authorization', request.headers) + self.assertIsNone(request.meta["proxy"]) + self.assertNotIn(b"Proxy-Authorization", request.headers) def test_remove_proxy_with_credentials(self): middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - meta={'proxy': 'https://user1:password1@example.com'}, + "https://example.com", + meta={"proxy": "https://user1:password1@example.com"}, ) assert middleware.process_request(request, spider) is None - request.meta['proxy'] = None + request.meta["proxy"] = None assert middleware.process_request(request, spider) is None - self.assertIsNone(request.meta['proxy']) - self.assertNotIn(b'Proxy-Authorization', request.headers) + self.assertIsNone(request.meta["proxy"]) + self.assertNotIn(b"Proxy-Authorization", request.headers) def test_add_credentials(self): """If the proxy request meta switches to a proxy URL with the same @@ -208,21 +221,21 @@ class TestHttpProxyMiddleware(TestCase): credentials must be used.""" middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - meta={'proxy': 'https://example.com'}, + "https://example.com", + meta={"proxy": "https://example.com"}, ) assert middleware.process_request(request, spider) is None - request.meta['proxy'] = 'https://user1:password1@example.com' + request.meta["proxy"] = "https://user1:password1@example.com" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual(request.meta["proxy"], "https://example.com") encoded_credentials = middleware._basic_auth_header( - 'user1', - 'password1', + "user1", + "password1", ) self.assertEqual( - request.headers['Proxy-Authorization'], - b'Basic ' + encoded_credentials, + request.headers["Proxy-Authorization"], + b"Basic " + encoded_credentials, ) def test_change_credentials(self): @@ -230,20 +243,20 @@ class TestHttpProxyMiddleware(TestCase): credentials, those new credentials must be used.""" middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - meta={'proxy': 'https://user1:password1@example.com'}, + "https://example.com", + meta={"proxy": "https://user1:password1@example.com"}, ) assert middleware.process_request(request, spider) is None - request.meta['proxy'] = 'https://user2:password2@example.com' + request.meta["proxy"] = "https://user2:password2@example.com" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual(request.meta["proxy"], "https://example.com") encoded_credentials = middleware._basic_auth_header( - 'user2', - 'password2', + "user2", + "password2", ) self.assertEqual( - request.headers['Proxy-Authorization'], - b'Basic ' + encoded_credentials, + request.headers["Proxy-Authorization"], + b"Basic " + encoded_credentials, ) def test_remove_credentials(self): @@ -256,93 +269,93 @@ class TestHttpProxyMiddleware(TestCase): """ middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - meta={'proxy': 'https://user1:password1@example.com'}, + "https://example.com", + meta={"proxy": "https://user1:password1@example.com"}, ) assert middleware.process_request(request, spider) is None - request.meta['proxy'] = 'https://example.com' + request.meta["proxy"] = "https://example.com" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual(request.meta["proxy"], "https://example.com") encoded_credentials = middleware._basic_auth_header( - 'user1', - 'password1', + "user1", + "password1", ) self.assertEqual( - request.headers['Proxy-Authorization'], - b'Basic ' + encoded_credentials, + request.headers["Proxy-Authorization"], + b"Basic " + encoded_credentials, ) - request.meta['proxy'] = 'https://example.com' - del request.headers[b'Proxy-Authorization'] + request.meta["proxy"] = "https://example.com" + del request.headers[b"Proxy-Authorization"] assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.com') - self.assertNotIn(b'Proxy-Authorization', request.headers) + self.assertEqual(request.meta["proxy"], "https://example.com") + self.assertNotIn(b"Proxy-Authorization", request.headers) def test_change_proxy_add_credentials(self): middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - meta={'proxy': 'https://example.com'}, + "https://example.com", + meta={"proxy": "https://example.com"}, ) assert middleware.process_request(request, spider) is None - request.meta['proxy'] = 'https://user1:password1@example.org' + request.meta["proxy"] = "https://user1:password1@example.org" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.org') + self.assertEqual(request.meta["proxy"], "https://example.org") encoded_credentials = middleware._basic_auth_header( - 'user1', - 'password1', + "user1", + "password1", ) self.assertEqual( - request.headers['Proxy-Authorization'], - b'Basic ' + encoded_credentials, + request.headers["Proxy-Authorization"], + b"Basic " + encoded_credentials, ) def test_change_proxy_keep_credentials(self): middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - meta={'proxy': 'https://user1:password1@example.com'}, + "https://example.com", + meta={"proxy": "https://user1:password1@example.com"}, ) assert middleware.process_request(request, spider) is None - request.meta['proxy'] = 'https://user1:password1@example.org' + request.meta["proxy"] = "https://user1:password1@example.org" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.org') + self.assertEqual(request.meta["proxy"], "https://example.org") encoded_credentials = middleware._basic_auth_header( - 'user1', - 'password1', + "user1", + "password1", ) self.assertEqual( - request.headers['Proxy-Authorization'], - b'Basic ' + encoded_credentials, + request.headers["Proxy-Authorization"], + b"Basic " + encoded_credentials, ) # Make sure, indirectly, that _auth_proxy is updated. - request.meta['proxy'] = 'https://example.com' + request.meta["proxy"] = "https://example.com" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.com') - self.assertNotIn(b'Proxy-Authorization', request.headers) + self.assertEqual(request.meta["proxy"], "https://example.com") + self.assertNotIn(b"Proxy-Authorization", request.headers) def test_change_proxy_change_credentials(self): middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - meta={'proxy': 'https://user1:password1@example.com'}, + "https://example.com", + meta={"proxy": "https://user1:password1@example.com"}, ) assert middleware.process_request(request, spider) is None - request.meta['proxy'] = 'https://user2:password2@example.org' + request.meta["proxy"] = "https://user2:password2@example.org" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.org') + self.assertEqual(request.meta["proxy"], "https://example.org") encoded_credentials = middleware._basic_auth_header( - 'user2', - 'password2', + "user2", + "password2", ) self.assertEqual( - request.headers['Proxy-Authorization'], - b'Basic ' + encoded_credentials, + request.headers["Proxy-Authorization"], + b"Basic " + encoded_credentials, ) def test_change_proxy_remove_credentials(self): @@ -350,14 +363,14 @@ class TestHttpProxyMiddleware(TestCase): proxy and no credentials, no credentials must be used.""" middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - meta={'proxy': 'https://user1:password1@example.com'}, + "https://example.com", + meta={"proxy": "https://user1:password1@example.com"}, ) assert middleware.process_request(request, spider) is None - request.meta['proxy'] = 'https://example.org' + request.meta["proxy"] = "https://example.org" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta, {'proxy': 'https://example.org'}) - self.assertNotIn(b'Proxy-Authorization', request.headers) + self.assertEqual(request.meta, {"proxy": "https://example.org"}) + self.assertNotIn(b"Proxy-Authorization", request.headers) def test_change_proxy_remove_credentials_preremoved_header(self): """Corner case of proxy switch with credentials removal where the @@ -368,36 +381,36 @@ class TestHttpProxyMiddleware(TestCase): """ middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - meta={'proxy': 'https://user1:password1@example.com'}, + "https://example.com", + meta={"proxy": "https://user1:password1@example.com"}, ) assert middleware.process_request(request, spider) is None - request.meta['proxy'] = 'https://example.org' - del request.headers[b'Proxy-Authorization'] + request.meta["proxy"] = "https://example.org" + del request.headers[b"Proxy-Authorization"] assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta, {'proxy': 'https://example.org'}) - self.assertNotIn(b'Proxy-Authorization', request.headers) + self.assertEqual(request.meta, {"proxy": "https://example.org"}) + self.assertNotIn(b"Proxy-Authorization", request.headers) def test_proxy_authentication_header_undefined_proxy(self): middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - headers={'Proxy-Authorization': 'Basic foo'}, + "https://example.com", + headers={"Proxy-Authorization": "Basic foo"}, ) assert middleware.process_request(request, spider) is None - self.assertNotIn('proxy', request.meta) - self.assertNotIn(b'Proxy-Authorization', request.headers) + self.assertNotIn("proxy", request.meta) + self.assertNotIn(b"Proxy-Authorization", request.headers) def test_proxy_authentication_header_disabled_proxy(self): middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - headers={'Proxy-Authorization': 'Basic foo'}, - meta={'proxy': None}, + "https://example.com", + headers={"Proxy-Authorization": "Basic foo"}, + meta={"proxy": None}, ) assert middleware.process_request(request, spider) is None - self.assertIsNone(request.meta['proxy']) - self.assertNotIn(b'Proxy-Authorization', request.headers) + self.assertIsNone(request.meta["proxy"]) + self.assertNotIn(b"Proxy-Authorization", request.headers) def test_proxy_authentication_header_proxy_without_credentials(self): """As long as the proxy URL in request metadata remains the same, the @@ -405,59 +418,59 @@ class TestHttpProxyMiddleware(TestCase): changed.""" middleware = HttpProxyMiddleware() request = Request( - 'https://example.com', - headers={'Proxy-Authorization': 'Basic foo'}, - meta={'proxy': 'https://example.com'}, + "https://example.com", + headers={"Proxy-Authorization": "Basic foo"}, + meta={"proxy": "https://example.com"}, ) assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.com') - self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo') + self.assertEqual(request.meta["proxy"], "https://example.com") + self.assertEqual(request.headers["Proxy-Authorization"], b"Basic foo") assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.com') - self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo') + self.assertEqual(request.meta["proxy"], "https://example.com") + self.assertEqual(request.headers["Proxy-Authorization"], b"Basic foo") - request.headers['Proxy-Authorization'] = b'Basic bar' + request.headers["Proxy-Authorization"] = b"Basic bar" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.com') - self.assertEqual(request.headers['Proxy-Authorization'], b'Basic bar') + self.assertEqual(request.meta["proxy"], "https://example.com") + self.assertEqual(request.headers["Proxy-Authorization"], b"Basic bar") def test_proxy_authentication_header_proxy_with_same_credentials(self): middleware = HttpProxyMiddleware() encoded_credentials = middleware._basic_auth_header( - 'user1', - 'password1', + "user1", + "password1", ) request = Request( - 'https://example.com', - headers={'Proxy-Authorization': b'Basic ' + encoded_credentials}, - meta={'proxy': 'https://user1:password1@example.com'}, + "https://example.com", + headers={"Proxy-Authorization": b"Basic " + encoded_credentials}, + meta={"proxy": "https://user1:password1@example.com"}, ) assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual(request.meta["proxy"], "https://example.com") self.assertEqual( - request.headers['Proxy-Authorization'], - b'Basic ' + encoded_credentials, + request.headers["Proxy-Authorization"], + b"Basic " + encoded_credentials, ) def test_proxy_authentication_header_proxy_with_different_credentials(self): middleware = HttpProxyMiddleware() encoded_credentials1 = middleware._basic_auth_header( - 'user1', - 'password1', + "user1", + "password1", ) request = Request( - 'https://example.com', - headers={'Proxy-Authorization': b'Basic ' + encoded_credentials1}, - meta={'proxy': 'https://user2:password2@example.com'}, + "https://example.com", + headers={"Proxy-Authorization": b"Basic " + encoded_credentials1}, + meta={"proxy": "https://user2:password2@example.com"}, ) assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual(request.meta["proxy"], "https://example.com") encoded_credentials2 = middleware._basic_auth_header( - 'user2', - 'password2', + "user2", + "password2", ) self.assertEqual( - request.headers['Proxy-Authorization'], - b'Basic ' + encoded_credentials2, + request.headers["Proxy-Authorization"], + b"Basic " + encoded_credentials2, ) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 816ac1440..e2ff9ec2b 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,6 +1,9 @@ import unittest -from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware +from scrapy.downloadermiddlewares.redirect import ( + RedirectMiddleware, + MetaRefreshMiddleware, +) from scrapy.spiders import Spider from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response, HtmlResponse @@ -8,24 +11,25 @@ from scrapy.utils.test import get_crawler class RedirectMiddlewareTest(unittest.TestCase): - def setUp(self): self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider('foo') + self.spider = self.crawler._create_spider("foo") self.mw = RedirectMiddleware.from_crawler(self.crawler) def test_priority_adjust(self): - req = Request('http://a.com') - rsp = Response('http://a.com', headers={'Location': 'http://a.com/redirected'}, status=301) + req = Request("http://a.com") + rsp = Response( + "http://a.com", headers={"Location": "http://a.com/redirected"}, status=301 + ) req2 = self.mw.process_response(req, rsp, self.spider) assert req2.priority > req.priority def test_redirect_3xx_permanent(self): def _test(method, status=301): - url = f'http://www.example.com/{status}' - url2 = 'http://www.example.com/redirected' + url = f"http://www.example.com/{status}" + url2 = "http://www.example.com/redirected" req = Request(url, method=method) - rsp = Response(url, headers={'Location': url2}, status=status) + rsp = Response(url, headers={"Location": url2}, status=status) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) @@ -33,33 +37,33 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.method, method) # response without Location header but with status code is 3XX should be ignored - del rsp.headers['Location'] + del rsp.headers["Location"] assert self.mw.process_response(req, rsp, self.spider) is rsp - _test('GET') - _test('POST') - _test('HEAD') + _test("GET") + _test("POST") + _test("HEAD") - _test('GET', status=307) - _test('POST', status=307) - _test('HEAD', status=307) + _test("GET", status=307) + _test("POST", status=307) + _test("HEAD", status=307) - _test('GET', status=308) - _test('POST', status=308) - _test('HEAD', status=308) + _test("GET", status=308) + _test("POST", status=308) + _test("HEAD", status=308) def test_dont_redirect(self): - url = 'http://www.example.com/301' - url2 = 'http://www.example.com/redirected' - req = Request(url, meta={'dont_redirect': True}) - rsp = Response(url, headers={'Location': url2}, status=301) + url = "http://www.example.com/301" + url2 = "http://www.example.com/redirected" + req = Request(url, meta={"dont_redirect": True}) + rsp = Response(url, headers={"Location": url2}, status=301) r = self.mw.process_response(req, rsp, self.spider) assert isinstance(r, Response) assert r is rsp # Test that it redirects when dont_redirect is False - req = Request(url, meta={'dont_redirect': False}) + req = Request(url, meta={"dont_redirect": False}) rsp = Response(url2, status=200) r = self.mw.process_response(req, rsp, self.spider) @@ -67,247 +71,313 @@ class RedirectMiddlewareTest(unittest.TestCase): assert r is rsp def test_redirect_302(self): - url = 'http://www.example.com/302' - url2 = 'http://www.example.com/redirected2' - req = Request(url, method='POST', body='test', - headers={'Content-Type': 'text/plain', 'Content-length': '4'}) - rsp = Response(url, headers={'Location': url2}, status=302) + url = "http://www.example.com/302" + url2 = "http://www.example.com/redirected2" + req = Request( + url, + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = Response(url, headers={"Location": url2}, status=302) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, 'GET') - assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" - assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" + self.assertEqual(req2.method, "GET") + assert ( + "Content-Type" not in req2.headers + ), "Content-Type header must not be present in redirected request" + assert ( + "Content-Length" not in req2.headers + ), "Content-Length header must not be present in redirected request" assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" # response without Location header but with status code is 3XX should be ignored - del rsp.headers['Location'] + del rsp.headers["Location"] assert self.mw.process_response(req, rsp, self.spider) is rsp def test_redirect_302_head(self): - url = 'http://www.example.com/302' - url2 = 'http://www.example.com/redirected2' - req = Request(url, method='HEAD') - rsp = Response(url, headers={'Location': url2}, status=302) + url = "http://www.example.com/302" + url2 = "http://www.example.com/redirected2" + req = Request(url, method="HEAD") + rsp = Response(url, headers={"Location": url2}, status=302) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, 'HEAD') + self.assertEqual(req2.method, "HEAD") # response without Location header but with status code is 3XX should be ignored - del rsp.headers['Location'] + del rsp.headers["Location"] assert self.mw.process_response(req, rsp, self.spider) is rsp def test_redirect_302_relative(self): - url = 'http://www.example.com/302' - url2 = '///i8n.example2.com/302' - url3 = 'http://i8n.example2.com/302' - req = Request(url, method='HEAD') - rsp = Response(url, headers={'Location': url2}, status=302) + url = "http://www.example.com/302" + url2 = "///i8n.example2.com/302" + url3 = "http://i8n.example2.com/302" + req = Request(url, method="HEAD") + rsp = Response(url, headers={"Location": url2}, status=302) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) self.assertEqual(req2.url, url3) - self.assertEqual(req2.method, 'HEAD') + self.assertEqual(req2.method, "HEAD") # response without Location header but with status code is 3XX should be ignored - del rsp.headers['Location'] + del rsp.headers["Location"] assert self.mw.process_response(req, rsp, self.spider) is rsp def test_max_redirect_times(self): self.mw.max_redirect_times = 1 - req = Request('http://scrapytest.org/302') - rsp = Response('http://scrapytest.org/302', headers={'Location': '/redirected'}, status=302) + req = Request("http://scrapytest.org/302") + rsp = Response( + "http://scrapytest.org/302", headers={"Location": "/redirected"}, status=302 + ) req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - assert 'redirect_times' in req.meta - self.assertEqual(req.meta['redirect_times'], 1) - self.assertRaises(IgnoreRequest, self.mw.process_response, req, rsp, self.spider) + assert "redirect_times" in req.meta + self.assertEqual(req.meta["redirect_times"], 1) + self.assertRaises( + IgnoreRequest, self.mw.process_response, req, rsp, self.spider + ) def test_ttl(self): self.mw.max_redirect_times = 100 - req = Request('http://scrapytest.org/302', meta={'redirect_ttl': 1}) - rsp = Response('http://www.scrapytest.org/302', headers={'Location': '/redirected'}, status=302) + req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) + rsp = Response( + "http://www.scrapytest.org/302", + headers={"Location": "/redirected"}, + status=302, + ) req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - self.assertRaises(IgnoreRequest, self.mw.process_response, req, rsp, self.spider) + self.assertRaises( + IgnoreRequest, self.mw.process_response, req, rsp, self.spider + ) def test_redirect_urls(self): - req1 = Request('http://scrapytest.org/first') - rsp1 = Response('http://scrapytest.org/first', headers={'Location': '/redirected'}, status=302) + req1 = Request("http://scrapytest.org/first") + rsp1 = Response( + "http://scrapytest.org/first", + headers={"Location": "/redirected"}, + status=302, + ) req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = Response('http://scrapytest.org/redirected', headers={'Location': '/redirected2'}, status=302) + rsp2 = Response( + "http://scrapytest.org/redirected", + headers={"Location": "/redirected2"}, + status=302, + ) req3 = self.mw.process_response(req2, rsp2, self.spider) - self.assertEqual(req2.url, 'http://scrapytest.org/redirected') - self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first']) - self.assertEqual(req3.url, 'http://scrapytest.org/redirected2') + self.assertEqual(req2.url, "http://scrapytest.org/redirected") + self.assertEqual(req2.meta["redirect_urls"], ["http://scrapytest.org/first"]) + self.assertEqual(req3.url, "http://scrapytest.org/redirected2") self.assertEqual( - req3.meta['redirect_urls'], - ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'] + req3.meta["redirect_urls"], + ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], ) def test_redirect_reasons(self): - req1 = Request('http://scrapytest.org/first') - rsp1 = Response('http://scrapytest.org/first', headers={'Location': '/redirected1'}, status=301) + req1 = Request("http://scrapytest.org/first") + rsp1 = Response( + "http://scrapytest.org/first", + headers={"Location": "/redirected1"}, + status=301, + ) req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = Response('http://scrapytest.org/redirected1', headers={'Location': '/redirected2'}, status=301) + rsp2 = Response( + "http://scrapytest.org/redirected1", + headers={"Location": "/redirected2"}, + status=301, + ) req3 = self.mw.process_response(req2, rsp2, self.spider) - self.assertEqual(req2.meta['redirect_reasons'], [301]) - self.assertEqual(req3.meta['redirect_reasons'], [301, 301]) + self.assertEqual(req2.meta["redirect_reasons"], [301]) + self.assertEqual(req3.meta["redirect_reasons"], [301, 301]) def test_spider_handling(self): - smartspider = self.crawler._create_spider('smarty') + smartspider = self.crawler._create_spider("smarty") smartspider.handle_httpstatus_list = [404, 301, 302] - url = 'http://www.example.com/301' - url2 = 'http://www.example.com/redirected' + url = "http://www.example.com/301" + url2 = "http://www.example.com/redirected" req = Request(url) - rsp = Response(url, headers={'Location': url2}, status=301) + rsp = Response(url, headers={"Location": url2}, status=301) r = self.mw.process_response(req, rsp, smartspider) self.assertIs(r, rsp) def test_request_meta_handling(self): - url = 'http://www.example.com/301' - url2 = 'http://www.example.com/redirected' + url = "http://www.example.com/301" + url2 = "http://www.example.com/redirected" def _test_passthrough(req): - rsp = Response(url, headers={'Location': url2}, status=301, request=req) + rsp = Response(url, headers={"Location": url2}, status=301, request=req) r = self.mw.process_response(req, rsp, self.spider) self.assertIs(r, rsp) - _test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]})) - _test_passthrough(Request(url, meta={'handle_httpstatus_all': True})) + + _test_passthrough( + Request(url, meta={"handle_httpstatus_list": [404, 301, 302]}) + ) + _test_passthrough(Request(url, meta={"handle_httpstatus_all": True})) def test_latin1_location(self): - req = Request('http://scrapytest.org/first') - latin1_location = '/ação'.encode('latin1') # HTTP historically supports latin1 - resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302) + req = Request("http://scrapytest.org/first") + latin1_location = "/ação".encode("latin1") # HTTP historically supports latin1 + resp = Response( + "http://scrapytest.org/first", + headers={"Location": latin1_location}, + status=302, + ) req_result = self.mw.process_response(req, resp, self.spider) - perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o' + perc_encoded_utf8_url = "http://scrapytest.org/a%E7%E3o" self.assertEqual(perc_encoded_utf8_url, req_result.url) def test_utf8_location(self): - req = Request('http://scrapytest.org/first') - utf8_location = '/ação'.encode('utf-8') # header using UTF-8 encoding - resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302) + req = Request("http://scrapytest.org/first") + utf8_location = "/ação".encode("utf-8") # header using UTF-8 encoding + resp = Response( + "http://scrapytest.org/first", + headers={"Location": utf8_location}, + status=302, + ) req_result = self.mw.process_response(req, resp, self.spider) - perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' + perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o" self.assertEqual(perc_encoded_utf8_url, req_result.url) class MetaRefreshMiddlewareTest(unittest.TestCase): - def setUp(self): crawler = get_crawler(Spider) - self.spider = crawler._create_spider('foo') + self.spider = crawler._create_spider("foo") self.mw = MetaRefreshMiddleware.from_crawler(crawler) - def _body(self, interval=5, url='http://example.org/newpage'): + def _body(self, interval=5, url="http://example.org/newpage"): html = f"""""" - return html.encode('utf-8') + return html.encode("utf-8") def test_priority_adjust(self): - req = Request('http://a.com') + req = Request("http://a.com") rsp = HtmlResponse(req.url, body=self._body()) req2 = self.mw.process_response(req, rsp, self.spider) assert req2.priority > req.priority def test_meta_refresh(self): - req = Request(url='http://example.org') + req = Request(url="http://example.org") rsp = HtmlResponse(req.url, body=self._body()) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, 'http://example.org/newpage') + self.assertEqual(req2.url, "http://example.org/newpage") def test_meta_refresh_with_high_interval(self): # meta-refresh with high intervals don't trigger redirects - req = Request(url='http://example.org') - rsp = HtmlResponse(url='http://example.org', - body=self._body(interval=1000), - encoding='utf-8') + req = Request(url="http://example.org") + rsp = HtmlResponse( + url="http://example.org", body=self._body(interval=1000), encoding="utf-8" + ) rsp2 = self.mw.process_response(req, rsp, self.spider) assert rsp is rsp2 def test_meta_refresh_trough_posted_request(self): - req = Request(url='http://example.org', method='POST', body='test', - headers={'Content-Type': 'text/plain', 'Content-length': '4'}) + req = Request( + url="http://example.org", + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) rsp = HtmlResponse(req.url, body=self._body()) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, 'http://example.org/newpage') - self.assertEqual(req2.method, 'GET') - assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" - assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" + self.assertEqual(req2.url, "http://example.org/newpage") + self.assertEqual(req2.method, "GET") + assert ( + "Content-Type" not in req2.headers + ), "Content-Type header must not be present in redirected request" + assert ( + "Content-Length" not in req2.headers + ), "Content-Length header must not be present in redirected request" assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" def test_max_redirect_times(self): self.mw.max_redirect_times = 1 - req = Request('http://scrapytest.org/max') + req = Request("http://scrapytest.org/max") rsp = HtmlResponse(req.url, body=self._body()) req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - assert 'redirect_times' in req.meta - self.assertEqual(req.meta['redirect_times'], 1) - self.assertRaises(IgnoreRequest, self.mw.process_response, req, rsp, self.spider) + assert "redirect_times" in req.meta + self.assertEqual(req.meta["redirect_times"], 1) + self.assertRaises( + IgnoreRequest, self.mw.process_response, req, rsp, self.spider + ) def test_ttl(self): self.mw.max_redirect_times = 100 - req = Request('http://scrapytest.org/302', meta={'redirect_ttl': 1}) + req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) rsp = HtmlResponse(req.url, body=self._body()) req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - self.assertRaises(IgnoreRequest, self.mw.process_response, req, rsp, self.spider) + self.assertRaises( + IgnoreRequest, self.mw.process_response, req, rsp, self.spider + ) def test_redirect_urls(self): - req1 = Request('http://scrapytest.org/first') - rsp1 = HtmlResponse(req1.url, body=self._body(url='/redirected')) + req1 = Request("http://scrapytest.org/first") + rsp1 = HtmlResponse(req1.url, body=self._body(url="/redirected")) req2 = self.mw.process_response(req1, rsp1, self.spider) assert isinstance(req2, Request), req2 - rsp2 = HtmlResponse(req2.url, body=self._body(url='/redirected2')) + rsp2 = HtmlResponse(req2.url, body=self._body(url="/redirected2")) req3 = self.mw.process_response(req2, rsp2, self.spider) assert isinstance(req3, Request), req3 - self.assertEqual(req2.url, 'http://scrapytest.org/redirected') - self.assertEqual(req2.meta['redirect_urls'], ['http://scrapytest.org/first']) - self.assertEqual(req3.url, 'http://scrapytest.org/redirected2') + self.assertEqual(req2.url, "http://scrapytest.org/redirected") + self.assertEqual(req2.meta["redirect_urls"], ["http://scrapytest.org/first"]) + self.assertEqual(req3.url, "http://scrapytest.org/redirected2") self.assertEqual( - req3.meta['redirect_urls'], - ['http://scrapytest.org/first', 'http://scrapytest.org/redirected'] + req3.meta["redirect_urls"], + ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], ) def test_redirect_reasons(self): - req1 = Request('http://scrapytest.org/first') - rsp1 = HtmlResponse('http://scrapytest.org/first', body=self._body(url='/redirected')) + req1 = Request("http://scrapytest.org/first") + rsp1 = HtmlResponse( + "http://scrapytest.org/first", body=self._body(url="/redirected") + ) req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = HtmlResponse('http://scrapytest.org/redirected', body=self._body(url='/redirected1')) + rsp2 = HtmlResponse( + "http://scrapytest.org/redirected", body=self._body(url="/redirected1") + ) req3 = self.mw.process_response(req2, rsp2, self.spider) - self.assertEqual(req2.meta['redirect_reasons'], ['meta refresh']) - self.assertEqual(req3.meta['redirect_reasons'], ['meta refresh', 'meta refresh']) + self.assertEqual(req2.meta["redirect_reasons"], ["meta refresh"]) + self.assertEqual( + req3.meta["redirect_reasons"], ["meta refresh", "meta refresh"] + ) def test_ignore_tags_default(self): - req = Request(url='http://example.org') - body = ('''''') + req = Request(url="http://example.org") + body = ( + """""" + ) rsp = HtmlResponse(req.url, body=body.encode()) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, 'http://example.org/newpage') + self.assertEqual(req2.url, "http://example.org/newpage") def test_ignore_tags_1_x_list(self): """Test that Scrapy 1.x behavior remains possible""" - settings = {'METAREFRESH_IGNORE_TAGS': ['script', 'noscript']} + settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]} crawler = get_crawler(Spider, settings) mw = MetaRefreshMiddleware.from_crawler(crawler) - req = Request(url='http://example.org') - body = ('''''') + req = Request(url="http://example.org") + body = ( + """""" + ) rsp = HtmlResponse(req.url, body=body.encode()) response = mw.process_response(req, rsp, self.spider) assert isinstance(response, Response) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 915bd3a3e..cadd647ad 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -22,65 +22,68 @@ from scrapy.utils.test import get_crawler class RetryTest(unittest.TestCase): def setUp(self): self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider('foo') + self.spider = self.crawler._create_spider("foo") self.mw = RetryMiddleware.from_crawler(self.crawler) self.mw.max_retry_times = 2 def test_priority_adjust(self): - req = Request('http://www.scrapytest.org/503') - rsp = Response('http://www.scrapytest.org/503', body=b'', status=503) + req = Request("http://www.scrapytest.org/503") + rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) req2 = self.mw.process_response(req, rsp, self.spider) assert req2.priority < req.priority def test_404(self): - req = Request('http://www.scrapytest.org/404') - rsp = Response('http://www.scrapytest.org/404', body=b'', status=404) + req = Request("http://www.scrapytest.org/404") + rsp = Response("http://www.scrapytest.org/404", body=b"", status=404) # dont retry 404s assert self.mw.process_response(req, rsp, self.spider) is rsp def test_dont_retry(self): - req = Request('http://www.scrapytest.org/503', meta={'dont_retry': True}) - rsp = Response('http://www.scrapytest.org/503', body=b'', status=503) + req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) + rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry r = self.mw.process_response(req, rsp, self.spider) assert r is rsp # Test retry when dont_retry set to False - req = Request('http://www.scrapytest.org/503', meta={'dont_retry': False}) - rsp = Response('http://www.scrapytest.org/503') + req = Request("http://www.scrapytest.org/503", meta={"dont_retry": False}) + rsp = Response("http://www.scrapytest.org/503") # first retry r = self.mw.process_response(req, rsp, self.spider) assert r is rsp def test_dont_retry_exc(self): - req = Request('http://www.scrapytest.org/503', meta={'dont_retry': True}) + req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) r = self.mw.process_exception(req, DNSLookupError(), self.spider) assert r is None def test_503(self): - req = Request('http://www.scrapytest.org/503') - rsp = Response('http://www.scrapytest.org/503', body=b'', status=503) + req = Request("http://www.scrapytest.org/503") + rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - self.assertEqual(req.meta['retry_times'], 1) + self.assertEqual(req.meta["retry_times"], 1) # second retry req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - self.assertEqual(req.meta['retry_times'], 2) + self.assertEqual(req.meta["retry_times"], 2) # discard it assert self.mw.process_response(req, rsp, self.spider) is rsp - assert self.crawler.stats.get_value('retry/max_reached') == 1 - assert self.crawler.stats.get_value('retry/reason_count/503 Service Unavailable') == 2 - assert self.crawler.stats.get_value('retry/count') == 2 + assert self.crawler.stats.get_value("retry/max_reached") == 1 + assert ( + self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable") + == 2 + ) + assert self.crawler.stats.get_value("retry/count") == 2 def test_twistederrors(self): exceptions = [ @@ -96,24 +99,27 @@ class RetryTest(unittest.TestCase): ] for exc in exceptions: - req = Request(f'http://www.scrapytest.org/{exc.__name__}') - self._test_retry_exception(req, exc('foo')) + req = Request(f"http://www.scrapytest.org/{exc.__name__}") + self._test_retry_exception(req, exc("foo")) stats = self.crawler.stats - assert stats.get_value('retry/max_reached') == len(exceptions) - assert stats.get_value('retry/count') == len(exceptions) * 2 - assert stats.get_value('retry/reason_count/twisted.internet.defer.TimeoutError') == 2 + assert stats.get_value("retry/max_reached") == len(exceptions) + assert stats.get_value("retry/count") == len(exceptions) * 2 + assert ( + stats.get_value("retry/reason_count/twisted.internet.defer.TimeoutError") + == 2 + ) def _test_retry_exception(self, req, exception): # first retry req = self.mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) - self.assertEqual(req.meta['retry_times'], 1) + self.assertEqual(req.meta["retry_times"], 1) # second retry req = self.mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) - self.assertEqual(req.meta['retry_times'], 2) + self.assertEqual(req.meta["retry_times"], 2) # discard it req = self.mw.process_exception(req, exception, self.spider) @@ -122,22 +128,22 @@ class RetryTest(unittest.TestCase): class MaxRetryTimesTest(unittest.TestCase): - invalid_url = 'http://www.scrapytest.org/invalid_url' + invalid_url = "http://www.scrapytest.org/invalid_url" def get_spider_and_middleware(self, settings=None): crawler = get_crawler(Spider, settings or {}) - spider = crawler._create_spider('foo') + spider = crawler._create_spider("foo") middleware = RetryMiddleware.from_crawler(crawler) return spider, middleware def test_with_settings_zero(self): max_retry_times = 0 - settings = {'RETRY_TIMES': max_retry_times} + settings = {"RETRY_TIMES": max_retry_times} spider, middleware = self.get_spider_and_middleware(settings) req = Request(self.invalid_url) self._test_retry( req, - DNSLookupError('foo'), + DNSLookupError("foo"), max_retry_times, spider=spider, middleware=middleware, @@ -146,11 +152,11 @@ class MaxRetryTimesTest(unittest.TestCase): def test_with_metakey_zero(self): max_retry_times = 0 spider, middleware = self.get_spider_and_middleware() - meta = {'max_retry_times': max_retry_times} + meta = {"max_retry_times": max_retry_times} req = Request(self.invalid_url, meta=meta) self._test_retry( req, - DNSLookupError('foo'), + DNSLookupError("foo"), max_retry_times, spider=spider, middleware=middleware, @@ -158,12 +164,12 @@ class MaxRetryTimesTest(unittest.TestCase): def test_without_metakey(self): max_retry_times = 5 - settings = {'RETRY_TIMES': max_retry_times} + settings = {"RETRY_TIMES": max_retry_times} spider, middleware = self.get_spider_and_middleware(settings) req = Request(self.invalid_url) self._test_retry( req, - DNSLookupError('foo'), + DNSLookupError("foo"), max_retry_times, spider=spider, middleware=middleware, @@ -173,22 +179,22 @@ class MaxRetryTimesTest(unittest.TestCase): meta_max_retry_times = 3 middleware_max_retry_times = 2 - req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) + req1 = Request(self.invalid_url, meta={"max_retry_times": meta_max_retry_times}) req2 = Request(self.invalid_url) - settings = {'RETRY_TIMES': middleware_max_retry_times} + settings = {"RETRY_TIMES": middleware_max_retry_times} spider, middleware = self.get_spider_and_middleware(settings) self._test_retry( req1, - DNSLookupError('foo'), + DNSLookupError("foo"), meta_max_retry_times, spider=spider, middleware=middleware, ) self._test_retry( req2, - DNSLookupError('foo'), + DNSLookupError("foo"), middleware_max_retry_times, spider=spider, middleware=middleware, @@ -198,22 +204,22 @@ class MaxRetryTimesTest(unittest.TestCase): meta_max_retry_times = 4 middleware_max_retry_times = 5 - req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) + req1 = Request(self.invalid_url, meta={"max_retry_times": meta_max_retry_times}) req2 = Request(self.invalid_url) - settings = {'RETRY_TIMES': middleware_max_retry_times} + settings = {"RETRY_TIMES": middleware_max_retry_times} spider, middleware = self.get_spider_and_middleware(settings) self._test_retry( req1, - DNSLookupError('foo'), + DNSLookupError("foo"), meta_max_retry_times, spider=spider, middleware=middleware, ) self._test_retry( req2, - DNSLookupError('foo'), + DNSLookupError("foo"), middleware_max_retry_times, spider=spider, middleware=middleware, @@ -223,13 +229,13 @@ class MaxRetryTimesTest(unittest.TestCase): max_retry_times = 4 spider, middleware = self.get_spider_and_middleware() meta = { - 'max_retry_times': max_retry_times, - 'dont_retry': True, + "max_retry_times": max_retry_times, + "dont_retry": True, } req = Request(self.invalid_url, meta=meta) self._test_retry( req, - DNSLookupError('foo'), + DNSLookupError("foo"), 0, spider=spider, middleware=middleware, @@ -256,13 +262,12 @@ class MaxRetryTimesTest(unittest.TestCase): class GetRetryRequestTest(unittest.TestCase): - def get_spider(self, settings=None): crawler = get_crawler(Spider, settings or {}) - return crawler._create_spider('foo') + return crawler._create_spider("foo") def test_basic_usage(self): - request = Request('https://example.com') + request = Request("https://example.com") spider = self.get_spider() with LogCapture() as log: new_request = get_retry_request( @@ -273,10 +278,10 @@ class GetRetryRequestTest(unittest.TestCase): self.assertNotEqual(new_request, request) self.assertEqual(new_request.dont_filter, True) expected_retry_times = 1 - self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.meta["retry_times"], expected_retry_times) self.assertEqual(new_request.priority, -1) expected_reason = "unspecified" - for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): self.assertEqual(spider.crawler.stats.get_value(stat), 1) log.check_present( ( @@ -288,7 +293,7 @@ class GetRetryRequestTest(unittest.TestCase): ) def test_max_retries_reached(self): - request = Request('https://example.com') + request = Request("https://example.com") spider = self.get_spider() max_retry_times = 0 with LogCapture() as log: @@ -298,10 +303,7 @@ class GetRetryRequestTest(unittest.TestCase): max_retry_times=max_retry_times, ) self.assertEqual(new_request, None) - self.assertEqual( - spider.crawler.stats.get_value('retry/max_reached'), - 1 - ) + self.assertEqual(spider.crawler.stats.get_value("retry/max_reached"), 1) failure_count = max_retry_times + 1 expected_reason = "unspecified" log.check_present( @@ -314,7 +316,7 @@ class GetRetryRequestTest(unittest.TestCase): ) def test_one_retry(self): - request = Request('https://example.com') + request = Request("https://example.com") spider = self.get_spider() with LogCapture() as log: new_request = get_retry_request( @@ -326,10 +328,10 @@ class GetRetryRequestTest(unittest.TestCase): self.assertNotEqual(new_request, request) self.assertEqual(new_request.dont_filter, True) expected_retry_times = 1 - self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.meta["retry_times"], expected_retry_times) self.assertEqual(new_request.priority, -1) expected_reason = "unspecified" - for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): self.assertEqual(spider.crawler.stats.get_value(stat), 1) log.check_present( ( @@ -342,7 +344,7 @@ class GetRetryRequestTest(unittest.TestCase): def test_two_retries(self): spider = self.get_spider() - request = Request('https://example.com') + request = Request("https://example.com") new_request = request max_retry_times = 2 for index in range(max_retry_times): @@ -356,10 +358,10 @@ class GetRetryRequestTest(unittest.TestCase): self.assertNotEqual(new_request, request) self.assertEqual(new_request.dont_filter, True) expected_retry_times = index + 1 - self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.meta["retry_times"], expected_retry_times) self.assertEqual(new_request.priority, -expected_retry_times) expected_reason = "unspecified" - for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): value = spider.crawler.stats.get_value(stat) self.assertEqual(value, expected_retry_times) log.check_present( @@ -378,10 +380,7 @@ class GetRetryRequestTest(unittest.TestCase): max_retry_times=max_retry_times, ) self.assertEqual(new_request, None) - self.assertEqual( - spider.crawler.stats.get_value('retry/max_reached'), - 1 - ) + self.assertEqual(spider.crawler.stats.get_value("retry/max_reached"), 1) failure_count = max_retry_times + 1 expected_reason = "unspecified" log.check_present( @@ -394,14 +393,14 @@ class GetRetryRequestTest(unittest.TestCase): ) def test_no_spider(self): - request = Request('https://example.com') + request = Request("https://example.com") with self.assertRaises(TypeError): get_retry_request(request) # pylint: disable=missing-kwoa def test_max_retry_times_setting(self): max_retry_times = 0 - spider = self.get_spider({'RETRY_TIMES': max_retry_times}) - request = Request('https://example.com') + spider = self.get_spider({"RETRY_TIMES": max_retry_times}) + request = Request("https://example.com") new_request = get_retry_request( request, spider=spider, @@ -410,9 +409,9 @@ class GetRetryRequestTest(unittest.TestCase): def test_max_retry_times_meta(self): max_retry_times = 0 - spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) - meta = {'max_retry_times': max_retry_times} - request = Request('https://example.com', meta=meta) + spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1}) + meta = {"max_retry_times": max_retry_times} + request = Request("https://example.com", meta=meta) new_request = get_retry_request( request, spider=spider, @@ -421,9 +420,9 @@ class GetRetryRequestTest(unittest.TestCase): def test_max_retry_times_argument(self): max_retry_times = 0 - spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) - meta = {'max_retry_times': max_retry_times + 1} - request = Request('https://example.com', meta=meta) + spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1}) + meta = {"max_retry_times": max_retry_times + 1} + request = Request("https://example.com", meta=meta) new_request = get_retry_request( request, spider=spider, @@ -433,8 +432,8 @@ class GetRetryRequestTest(unittest.TestCase): def test_priority_adjust_setting(self): priority_adjust = 1 - spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust}) - request = Request('https://example.com') + spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust}) + request = Request("https://example.com") new_request = get_retry_request( request, spider=spider, @@ -443,8 +442,8 @@ class GetRetryRequestTest(unittest.TestCase): def test_priority_adjust_argument(self): priority_adjust = 1 - spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust + 1}) - request = Request('https://example.com') + spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust + 1}) + request = Request("https://example.com") new_request = get_retry_request( request, spider=spider, @@ -453,9 +452,9 @@ class GetRetryRequestTest(unittest.TestCase): self.assertEqual(new_request.priority, priority_adjust) def test_log_extra_retry_success(self): - request = Request('https://example.com') + request = Request("https://example.com") spider = self.get_spider() - with LogCapture(attributes=('spider',)) as log: + with LogCapture(attributes=("spider",)) as log: get_retry_request( request, spider=spider, @@ -463,9 +462,9 @@ class GetRetryRequestTest(unittest.TestCase): log.check_present(spider) def test_log_extra_retries_exceeded(self): - request = Request('https://example.com') + request = Request("https://example.com") spider = self.get_spider() - with LogCapture(attributes=('spider',)) as log: + with LogCapture(attributes=("spider",)) as log: get_retry_request( request, spider=spider, @@ -474,9 +473,9 @@ class GetRetryRequestTest(unittest.TestCase): log.check_present(spider) def test_reason_string(self): - request = Request('https://example.com') + request = Request("https://example.com") spider = self.get_spider() - expected_reason = 'because' + expected_reason = "because" with LogCapture() as log: get_retry_request( request, @@ -484,7 +483,7 @@ class GetRetryRequestTest(unittest.TestCase): reason=expected_reason, ) expected_retry_times = 1 - for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): self.assertEqual(spider.crawler.stats.get_value(stat), 1) log.check_present( ( @@ -496,10 +495,10 @@ class GetRetryRequestTest(unittest.TestCase): ) def test_reason_builtin_exception(self): - request = Request('https://example.com') + request = Request("https://example.com") spider = self.get_spider() expected_reason = NotImplementedError() - expected_reason_string = 'builtins.NotImplementedError' + expected_reason_string = "builtins.NotImplementedError" with LogCapture() as log: get_retry_request( request, @@ -508,7 +507,7 @@ class GetRetryRequestTest(unittest.TestCase): ) expected_retry_times = 1 stat = spider.crawler.stats.get_value( - f'retry/reason_count/{expected_reason_string}' + f"retry/reason_count/{expected_reason_string}" ) self.assertEqual(stat, 1) log.check_present( @@ -521,10 +520,10 @@ class GetRetryRequestTest(unittest.TestCase): ) def test_reason_builtin_exception_class(self): - request = Request('https://example.com') + request = Request("https://example.com") spider = self.get_spider() expected_reason = NotImplementedError - expected_reason_string = 'builtins.NotImplementedError' + expected_reason_string = "builtins.NotImplementedError" with LogCapture() as log: get_retry_request( request, @@ -533,7 +532,7 @@ class GetRetryRequestTest(unittest.TestCase): ) expected_retry_times = 1 stat = spider.crawler.stats.get_value( - f'retry/reason_count/{expected_reason_string}' + f"retry/reason_count/{expected_reason_string}" ) self.assertEqual(stat, 1) log.check_present( @@ -546,10 +545,10 @@ class GetRetryRequestTest(unittest.TestCase): ) def test_reason_custom_exception(self): - request = Request('https://example.com') + request = Request("https://example.com") spider = self.get_spider() expected_reason = IgnoreRequest() - expected_reason_string = 'scrapy.exceptions.IgnoreRequest' + expected_reason_string = "scrapy.exceptions.IgnoreRequest" with LogCapture() as log: get_retry_request( request, @@ -558,7 +557,7 @@ class GetRetryRequestTest(unittest.TestCase): ) expected_retry_times = 1 stat = spider.crawler.stats.get_value( - f'retry/reason_count/{expected_reason_string}' + f"retry/reason_count/{expected_reason_string}" ) self.assertEqual(stat, 1) log.check_present( @@ -571,10 +570,10 @@ class GetRetryRequestTest(unittest.TestCase): ) def test_reason_custom_exception_class(self): - request = Request('https://example.com') + request = Request("https://example.com") spider = self.get_spider() expected_reason = IgnoreRequest - expected_reason_string = 'scrapy.exceptions.IgnoreRequest' + expected_reason_string = "scrapy.exceptions.IgnoreRequest" with LogCapture() as log: get_retry_request( request, @@ -583,7 +582,7 @@ class GetRetryRequestTest(unittest.TestCase): ) expected_retry_times = 1 stat = spider.crawler.stats.get_value( - f'retry/reason_count/{expected_reason_string}' + f"retry/reason_count/{expected_reason_string}" ) self.assertEqual(stat, 1) log.check_present( @@ -626,7 +625,10 @@ class GetRetryRequestTest(unittest.TestCase): reason=expected_reason, stats_base_key=stats_key, ) - for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"): + for stat in ( + f"{stats_key}/count", + f"{stats_key}/reason_count/{expected_reason}", + ): self.assertEqual(spider.crawler.stats.get_value(stat), 1) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 1460d88eb..ac08c6006 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -4,8 +4,10 @@ from twisted.internet import reactor, error from twisted.internet.defer import Deferred, DeferredList, maybeDeferred from twisted.python import failure from twisted.trial import unittest -from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware, - logger as mw_module_logger) +from scrapy.downloadermiddlewares.robotstxt import ( + RobotsTxtMiddleware, + logger as mw_module_logger, +) from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.settings import Settings @@ -13,7 +15,6 @@ from tests.test_robotstxt_interface import rerp_available, reppy_available class RobotsTxtMiddlewareTest(unittest.TestCase): - def setUp(self): self.crawler = mock.MagicMock() self.crawler.settings = Settings() @@ -24,12 +25,12 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def test_robotstxt_settings(self): self.crawler.settings = Settings() - self.crawler.settings.set('USER_AGENT', 'CustomAgent') + self.crawler.settings.set("USER_AGENT", "CustomAgent") self.assertRaises(NotConfigured, RobotsTxtMiddleware, self.crawler) def _get_successful_crawler(self): crawler = self.crawler - crawler.settings.set('ROBOTSTXT_OBEY', True) + crawler.settings.set("ROBOTSTXT_OBEY", True) ROBOTS = """ User-Agent: * Disallow: /admin/ @@ -39,138 +40,179 @@ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: Disallow: /wiki/Käyttäjä: User-Agent: UnicödeBöt Disallow: /some/randome/page.html -""".encode('utf-8') - response = TextResponse('http://site.local/robots.txt', body=ROBOTS) +""".encode( + "utf-8" + ) + response = TextResponse("http://site.local/robots.txt", body=ROBOTS) def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred + crawler.engine.download.side_effect = return_response return crawler def test_robotstxt(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) - return DeferredList([ - self.assertNotIgnored(Request('http://site.local/allowed'), middleware), - self.assertIgnored(Request('http://site.local/admin/main'), middleware), - self.assertIgnored(Request('http://site.local/static/'), middleware), - self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware), - self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware) - ], fireOnOneErrback=True) + return DeferredList( + [ + self.assertNotIgnored(Request("http://site.local/allowed"), middleware), + self.assertIgnored(Request("http://site.local/admin/main"), middleware), + self.assertIgnored(Request("http://site.local/static/"), middleware), + self.assertIgnored( + Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"), + middleware, + ), + self.assertIgnored( + Request("http://site.local/wiki/Käyttäjä:"), middleware + ), + ], + fireOnOneErrback=True, + ) def test_robotstxt_ready_parser(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) - d = self.assertNotIgnored(Request('http://site.local/allowed'), middleware) - d.addCallback(lambda _: self.assertNotIgnored(Request('http://site.local/allowed'), middleware)) + d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + d.addCallback( + lambda _: self.assertNotIgnored( + Request("http://site.local/allowed"), middleware + ) + ) return d def test_robotstxt_meta(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) - meta = {'dont_obey_robotstxt': True} - return DeferredList([ - self.assertNotIgnored(Request('http://site.local/allowed', meta=meta), middleware), - self.assertNotIgnored(Request('http://site.local/admin/main', meta=meta), middleware), - self.assertNotIgnored(Request('http://site.local/static/', meta=meta), middleware) - ], fireOnOneErrback=True) + meta = {"dont_obey_robotstxt": True} + return DeferredList( + [ + self.assertNotIgnored( + Request("http://site.local/allowed", meta=meta), middleware + ), + self.assertNotIgnored( + Request("http://site.local/admin/main", meta=meta), middleware + ), + self.assertNotIgnored( + Request("http://site.local/static/", meta=meta), middleware + ), + ], + fireOnOneErrback=True, + ) def _get_garbage_crawler(self): crawler = self.crawler - crawler.settings.set('ROBOTSTXT_OBEY', True) - response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2') + crawler.settings.set("ROBOTSTXT_OBEY", True) + response = Response( + "http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2" + ) def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred + crawler.engine.download.side_effect = return_response return crawler def test_robotstxt_garbage(self): # garbage response should be discarded, equal 'allow all' middleware = RobotsTxtMiddleware(self._get_garbage_crawler()) - deferred = DeferredList([ - self.assertNotIgnored(Request('http://site.local'), middleware), - self.assertNotIgnored(Request('http://site.local/allowed'), middleware), - self.assertNotIgnored(Request('http://site.local/admin/main'), middleware), - self.assertNotIgnored(Request('http://site.local/static/'), middleware) - ], fireOnOneErrback=True) + deferred = DeferredList( + [ + self.assertNotIgnored(Request("http://site.local"), middleware), + self.assertNotIgnored(Request("http://site.local/allowed"), middleware), + self.assertNotIgnored( + Request("http://site.local/admin/main"), middleware + ), + self.assertNotIgnored(Request("http://site.local/static/"), middleware), + ], + fireOnOneErrback=True, + ) return deferred def _get_emptybody_crawler(self): crawler = self.crawler - crawler.settings.set('ROBOTSTXT_OBEY', True) - response = Response('http://site.local/robots.txt') + crawler.settings.set("ROBOTSTXT_OBEY", True) + response = Response("http://site.local/robots.txt") def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred + crawler.engine.download.side_effect = return_response return crawler def test_robotstxt_empty_response(self): # empty response should equal 'allow all' middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) - return DeferredList([ - self.assertNotIgnored(Request('http://site.local/allowed'), middleware), - self.assertNotIgnored(Request('http://site.local/admin/main'), middleware), - self.assertNotIgnored(Request('http://site.local/static/'), middleware) - ], fireOnOneErrback=True) + return DeferredList( + [ + self.assertNotIgnored(Request("http://site.local/allowed"), middleware), + self.assertNotIgnored( + Request("http://site.local/admin/main"), middleware + ), + self.assertNotIgnored(Request("http://site.local/static/"), middleware), + ], + fireOnOneErrback=True, + ) def test_robotstxt_error(self): - self.crawler.settings.set('ROBOTSTXT_OBEY', True) - err = error.DNSLookupError('Robotstxt address not found') + self.crawler.settings.set("ROBOTSTXT_OBEY", True) + err = error.DNSLookupError("Robotstxt address not found") def return_failure(request): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(err)) return deferred + self.crawler.engine.download.side_effect = return_failure middleware = RobotsTxtMiddleware(self.crawler) middleware._logerror = mock.MagicMock(side_effect=middleware._logerror) - deferred = middleware.process_request(Request('http://site.local'), None) + deferred = middleware.process_request(Request("http://site.local"), None) deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called)) return deferred def test_robotstxt_immediate_error(self): - self.crawler.settings.set('ROBOTSTXT_OBEY', True) - err = error.DNSLookupError('Robotstxt address not found') + self.crawler.settings.set("ROBOTSTXT_OBEY", True) + err = error.DNSLookupError("Robotstxt address not found") def immediate_failure(request): deferred = Deferred() deferred.errback(failure.Failure(err)) return deferred + self.crawler.engine.download.side_effect = immediate_failure middleware = RobotsTxtMiddleware(self.crawler) - return self.assertNotIgnored(Request('http://site.local'), middleware) + return self.assertNotIgnored(Request("http://site.local"), middleware) def test_ignore_robotstxt_request(self): - self.crawler.settings.set('ROBOTSTXT_OBEY', True) + self.crawler.settings.set("ROBOTSTXT_OBEY", True) def ignore_request(request): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest())) return deferred + self.crawler.engine.download.side_effect = ignore_request middleware = RobotsTxtMiddleware(self.crawler) mw_module_logger.error = mock.MagicMock() - d = self.assertNotIgnored(Request('http://site.local/allowed'), middleware) + d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called)) return d def test_robotstxt_user_agent_setting(self): crawler = self._get_successful_crawler() - crawler.settings.set('ROBOTSTXT_USER_AGENT', 'Examplebot') - crawler.settings.set('USER_AGENT', 'Mozilla/5.0 (X11; Linux x86_64)') + crawler.settings.set("ROBOTSTXT_USER_AGENT", "Examplebot") + crawler.settings.set("USER_AGENT", "Mozilla/5.0 (X11; Linux x86_64)") middleware = RobotsTxtMiddleware(crawler) rp = mock.MagicMock(return_value=True) - middleware.process_request_2(rp, Request('http://site.local/allowed'), None) - rp.allowed.assert_called_once_with('http://site.local/allowed', 'Examplebot') + middleware.process_request_2(rp, Request("http://site.local/allowed"), None) + rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot") def assertNotIgnored(self, request, middleware): spider = None # not actually used @@ -180,8 +222,9 @@ Disallow: /some/randome/page.html def assertIgnored(self, request, middleware): spider = None # not actually used - return self.assertFailure(maybeDeferred(middleware.process_request, request, spider), - IgnoreRequest) + return self.assertFailure( + maybeDeferred(middleware.process_request, request, spider), IgnoreRequest + ) class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): @@ -190,7 +233,9 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): def setUp(self): super().setUp() - self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser') + self.crawler.settings.set( + "ROBOTSTXT_PARSER", "scrapy.robotstxt.RerpRobotParser" + ) class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest): @@ -199,4 +244,6 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest): def setUp(self): super().setUp() - self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser') + self.crawler.settings.set( + "ROBOTSTXT_PARSER", "scrapy.robotstxt.ReppyRobotParser" + ) diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 7d88ba4d2..39dfe9ab5 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -15,59 +15,57 @@ class MyException(Exception): class TestDownloaderStats(TestCase): - def setUp(self): self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider('scrapytest.org') + self.spider = self.crawler._create_spider("scrapytest.org") self.mw = DownloaderStats(self.crawler.stats) self.crawler.stats.open_spider(self.spider) - self.req = Request('http://scrapytest.org') - self.res = Response('scrapytest.org', status=400) + self.req = Request("http://scrapytest.org") + self.res = Response("scrapytest.org", status=400) def assertStatsEqual(self, key, value): self.assertEqual( self.crawler.stats.get_value(key, spider=self.spider), value, - str(self.crawler.stats.get_stats(self.spider)) + str(self.crawler.stats.get_stats(self.spider)), ) def test_process_request(self): self.mw.process_request(self.req, self.spider) - self.assertStatsEqual('downloader/request_count', 1) + self.assertStatsEqual("downloader/request_count", 1) def test_process_response(self): self.mw.process_response(self.req, self.res, self.spider) - self.assertStatsEqual('downloader/response_count', 1) + self.assertStatsEqual("downloader/response_count", 1) def test_response_len(self): - body = (b'', b'not_empty') # empty/notempty body - headers = ({}, {'lang': 'en'}, {'lang': 'en', 'User-Agent': 'scrapy'}) # 0 headers, 1h and 2h + body = (b"", b"not_empty") # empty/notempty body + headers = ( + {}, + {"lang": "en"}, + {"lang": "en", "User-Agent": "scrapy"}, + ) # 0 headers, 1h and 2h test_responses = [ # form test responses with all combinations of body/headers - Response( - url='scrapytest.org', - status=200, - body=r[0], - headers=r[1] - ) + Response(url="scrapytest.org", status=200, body=r[0], headers=r[1]) for r in product(body, headers) ] for test_response in test_responses: - self.crawler.stats.set_value('downloader/response_bytes', 0) + self.crawler.stats.set_value("downloader/response_bytes", 0) self.mw.process_response(self.req, test_response, self.spider) with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) resp_size = len(response_httprepr(test_response)) - self.assertStatsEqual('downloader/response_bytes', resp_size) + self.assertStatsEqual("downloader/response_bytes", resp_size) def test_process_exception(self): self.mw.process_exception(self.req, MyException(), self.spider) - self.assertStatsEqual('downloader/exception_count', 1) + self.assertStatsEqual("downloader/exception_count", 1) self.assertStatsEqual( - 'downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException', - 1 + "downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException", + 1, ) def tearDown(self): - self.crawler.stats.close_spider(self.spider, '') + self.crawler.stats.close_spider(self.spider, "") diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index a286764fd..927f0ae4d 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -7,48 +7,48 @@ from scrapy.utils.test import get_crawler class UserAgentMiddlewareTest(TestCase): - def get_spider_and_mw(self, default_useragent): - crawler = get_crawler(Spider, {'USER_AGENT': default_useragent}) - spider = crawler._create_spider('foo') + crawler = get_crawler(Spider, {"USER_AGENT": default_useragent}) + spider = crawler._create_spider("foo") return spider, UserAgentMiddleware.from_crawler(crawler) def test_default_agent(self): - spider, mw = self.get_spider_and_mw('default_useragent') - req = Request('http://scrapytest.org/') + spider, mw = self.get_spider_and_mw("default_useragent") + req = Request("http://scrapytest.org/") assert mw.process_request(req, spider) is None - self.assertEqual(req.headers['User-Agent'], b'default_useragent') + self.assertEqual(req.headers["User-Agent"], b"default_useragent") def test_remove_agent(self): # settings UESR_AGENT to None should remove the user agent - spider, mw = self.get_spider_and_mw('default_useragent') + spider, mw = self.get_spider_and_mw("default_useragent") spider.user_agent = None mw.spider_opened(spider) - req = Request('http://scrapytest.org/') + req = Request("http://scrapytest.org/") assert mw.process_request(req, spider) is None - assert req.headers.get('User-Agent') is None + assert req.headers.get("User-Agent") is None def test_spider_agent(self): - spider, mw = self.get_spider_and_mw('default_useragent') - spider.user_agent = 'spider_useragent' + spider, mw = self.get_spider_and_mw("default_useragent") + spider.user_agent = "spider_useragent" mw.spider_opened(spider) - req = Request('http://scrapytest.org/') + req = Request("http://scrapytest.org/") assert mw.process_request(req, spider) is None - self.assertEqual(req.headers['User-Agent'], b'spider_useragent') + self.assertEqual(req.headers["User-Agent"], b"spider_useragent") def test_header_agent(self): - spider, mw = self.get_spider_and_mw('default_useragent') - spider.user_agent = 'spider_useragent' + spider, mw = self.get_spider_and_mw("default_useragent") + spider.user_agent = "spider_useragent" mw.spider_opened(spider) - req = Request('http://scrapytest.org/', - headers={'User-Agent': 'header_useragent'}) + req = Request( + "http://scrapytest.org/", headers={"User-Agent": "header_useragent"} + ) assert mw.process_request(req, spider) is None - self.assertEqual(req.headers['User-Agent'], b'header_useragent') + self.assertEqual(req.headers["User-Agent"], b"header_useragent") def test_no_agent(self): spider, mw = self.get_spider_and_mw(None) spider.user_agent = None mw.spider_opened(spider) - req = Request('http://scrapytest.org/') + req = Request("http://scrapytest.org/") assert mw.process_request(req, spider) is None - assert 'User-Agent' not in req.headers + assert "User-Agent" not in req.headers diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index e028dea4d..4019012d1 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -25,59 +25,62 @@ def _get_dupefilter(*, crawler=None, settings=None, open=True): class FromCrawlerRFPDupeFilter(RFPDupeFilter): - @classmethod def from_crawler(cls, crawler): df = super().from_crawler(crawler) - df.method = 'from_crawler' + df.method = "from_crawler" return df class FromSettingsRFPDupeFilter(RFPDupeFilter): - @classmethod def from_settings(cls, settings, *, fingerprinter=None): df = super().from_settings(settings, fingerprinter=fingerprinter) - df.method = 'from_settings' + df.method = "from_settings" return df class DirectDupeFilter: - method = 'n/a' + method = "n/a" class RFPDupeFilterTest(unittest.TestCase): - def test_df_from_crawler_scheduler(self): - settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} + settings = { + "DUPEFILTER_DEBUG": True, + "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) - self.assertEqual(scheduler.df.method, 'from_crawler') + self.assertEqual(scheduler.df.method, "from_crawler") def test_df_from_settings_scheduler(self): - settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} + settings = { + "DUPEFILTER_DEBUG": True, + "DUPEFILTER_CLASS": FromSettingsRFPDupeFilter, + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) - self.assertEqual(scheduler.df.method, 'from_settings') + self.assertEqual(scheduler.df.method, "from_settings") def test_df_direct_scheduler(self): - settings = {'DUPEFILTER_CLASS': DirectDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} + settings = { + "DUPEFILTER_CLASS": DirectDupeFilter, + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) - self.assertEqual(scheduler.df.method, 'n/a') + self.assertEqual(scheduler.df.method, "n/a") def test_filter(self): dupefilter = _get_dupefilter() - r1 = Request('http://scrapytest.org/1') - r2 = Request('http://scrapytest.org/2') - r3 = Request('http://scrapytest.org/2') + r1 = Request("http://scrapytest.org/1") + r2 = Request("http://scrapytest.org/2") + r3 = Request("http://scrapytest.org/2") assert not dupefilter.request_seen(r1) assert dupefilter.request_seen(r1) @@ -85,23 +88,23 @@ class RFPDupeFilterTest(unittest.TestCase): assert not dupefilter.request_seen(r2) assert dupefilter.request_seen(r3) - dupefilter.close('finished') + dupefilter.close("finished") def test_dupefilter_path(self): - r1 = Request('http://scrapytest.org/1') - r2 = Request('http://scrapytest.org/2') + r1 = Request("http://scrapytest.org/1") + r2 = Request("http://scrapytest.org/2") path = tempfile.mkdtemp() try: - df = _get_dupefilter(settings={'JOBDIR': path}, open=False) + df = _get_dupefilter(settings={"JOBDIR": path}, open=False) try: df.open() assert not df.request_seen(r1) assert df.request_seen(r1) finally: - df.close('finished') + df.close("finished") - df2 = _get_dupefilter(settings={'JOBDIR': path}, open=False) + df2 = _get_dupefilter(settings={"JOBDIR": path}, open=False) assert df != df2 try: df2.open() @@ -109,7 +112,7 @@ class RFPDupeFilterTest(unittest.TestCase): assert not df2.request_seen(r2) assert df2.request_seen(r2) finally: - df2.close('finished') + df2.close("finished") finally: shutil.rmtree(path) @@ -119,147 +122,156 @@ class RFPDupeFilterTest(unittest.TestCase): """ dupefilter = _get_dupefilter() - r1 = Request('http://scrapytest.org/index.html') - r2 = Request('http://scrapytest.org/INDEX.html') + r1 = Request("http://scrapytest.org/index.html") + r2 = Request("http://scrapytest.org/INDEX.html") assert not dupefilter.request_seen(r1) assert not dupefilter.request_seen(r2) - dupefilter.close('finished') + dupefilter.close("finished") class RequestFingerprinter: - def fingerprint(self, request): fp = hashlib.sha1() fp.update(to_bytes(request.url.lower())) return fp.digest() - settings = {'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter} + settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter} case_insensitive_dupefilter = _get_dupefilter(settings=settings) assert not case_insensitive_dupefilter.request_seen(r1) assert case_insensitive_dupefilter.request_seen(r2) - case_insensitive_dupefilter.close('finished') + case_insensitive_dupefilter.close("finished") def test_seenreq_newlines(self): - """ Checks against adding duplicate \r to - line endings on Windows platforms. """ + """Checks against adding duplicate \r to + line endings on Windows platforms.""" - r1 = Request('http://scrapytest.org/1') + r1 = Request("http://scrapytest.org/1") path = tempfile.mkdtemp() - crawler = get_crawler(settings_dict={'JOBDIR': path}) + crawler = get_crawler(settings_dict={"JOBDIR": path}) try: scheduler = Scheduler.from_crawler(crawler) df = scheduler.df df.open() df.request_seen(r1) - df.close('finished') + df.close("finished") - with Path(path, 'requests.seen').open('rb') as seen_file: + with Path(path, "requests.seen").open("rb") as seen_file: line = next(seen_file).decode() - assert not line.endswith('\r\r\n') - if sys.platform == 'win32': - assert line.endswith('\r\n') + assert not line.endswith("\r\r\n") + if sys.platform == "win32": + assert line.endswith("\r\n") else: - assert line.endswith('\n') + assert line.endswith("\n") finally: shutil.rmtree(path) def test_log(self): with LogCapture() as log: - settings = {'DUPEFILTER_DEBUG': False, - 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} + settings = { + "DUPEFILTER_DEBUG": False, + "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) - r1 = Request('http://scrapytest.org/index.html') - r2 = Request('http://scrapytest.org/index.html') + r1 = Request("http://scrapytest.org/index.html") + r2 = Request("http://scrapytest.org/index.html") dupefilter.log(r1, spider) dupefilter.log(r2, spider) - assert crawler.stats.get_value('dupefilter/filtered') == 2 + assert crawler.stats.get_value("dupefilter/filtered") == 2 log.check_present( ( - 'scrapy.dupefilters', - 'DEBUG', - 'Filtered duplicate request: - no more' - ' duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)' + "scrapy.dupefilters", + "DEBUG", + "Filtered duplicate request: - no more" + " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)", ) ) - dupefilter.close('finished') + dupefilter.close("finished") def test_log_debug(self): with LogCapture() as log: - settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} + settings = { + "DUPEFILTER_DEBUG": True, + "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) - r1 = Request('http://scrapytest.org/index.html') - r2 = Request('http://scrapytest.org/index.html', - headers={'Referer': 'http://scrapytest.org/INDEX.html'}) + r1 = Request("http://scrapytest.org/index.html") + r2 = Request( + "http://scrapytest.org/index.html", + headers={"Referer": "http://scrapytest.org/INDEX.html"}, + ) dupefilter.log(r1, spider) dupefilter.log(r2, spider) - assert crawler.stats.get_value('dupefilter/filtered') == 2 + assert crawler.stats.get_value("dupefilter/filtered") == 2 log.check_present( ( - 'scrapy.dupefilters', - 'DEBUG', - 'Filtered duplicate request: (referer: None)' + "scrapy.dupefilters", + "DEBUG", + "Filtered duplicate request: (referer: None)", ) ) log.check_present( ( - 'scrapy.dupefilters', - 'DEBUG', - 'Filtered duplicate request: ' - ' (referer: http://scrapytest.org/INDEX.html)' + "scrapy.dupefilters", + "DEBUG", + "Filtered duplicate request: " + " (referer: http://scrapytest.org/INDEX.html)", ) ) - dupefilter.close('finished') + dupefilter.close("finished") def test_log_debug_default_dupefilter(self): with LogCapture() as log: - settings = {'DUPEFILTER_DEBUG': True, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} + settings = { + "DUPEFILTER_DEBUG": True, + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) - r1 = Request('http://scrapytest.org/index.html') - r2 = Request('http://scrapytest.org/index.html', - headers={'Referer': 'http://scrapytest.org/INDEX.html'}) + r1 = Request("http://scrapytest.org/index.html") + r2 = Request( + "http://scrapytest.org/index.html", + headers={"Referer": "http://scrapytest.org/INDEX.html"}, + ) dupefilter.log(r1, spider) dupefilter.log(r2, spider) - assert crawler.stats.get_value('dupefilter/filtered') == 2 + assert crawler.stats.get_value("dupefilter/filtered") == 2 log.check_present( ( - 'scrapy.dupefilters', - 'DEBUG', - 'Filtered duplicate request: (referer: None)' + "scrapy.dupefilters", + "DEBUG", + "Filtered duplicate request: (referer: None)", ) ) log.check_present( ( - 'scrapy.dupefilters', - 'DEBUG', - 'Filtered duplicate request: ' - ' (referer: http://scrapytest.org/INDEX.html)' + "scrapy.dupefilters", + "DEBUG", + "Filtered duplicate request: " + " (referer: http://scrapytest.org/INDEX.html)", ) ) - dupefilter.close('finished') + dupefilter.close("finished") diff --git a/tests/test_engine.py b/tests/test_engine.py index aa3313659..7b861332e 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -81,11 +81,11 @@ class TestSpider(Spider): adapter = ItemAdapter(self.item_cls()) m = self.name_re.search(response.text) if m: - adapter['name'] = m.group(1) - adapter['url'] = response.url + adapter["name"] = m.group(1) + adapter["url"] = response.url m = self.price_re.search(response.text) if m: - adapter['price'] = m.group(1) + adapter["price"] = m.group(1) return adapter.item @@ -136,8 +136,10 @@ def start_test_site(debug=False): port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: - print(f"Test server running at http://localhost:{port.getHost().port}/ " - "- hit Ctrl-C to finish.") + print( + f"Test server running at http://localhost:{port.getHost().port}/ " + "- hit Ctrl-C to finish." + ) return port @@ -169,7 +171,7 @@ class CrawlerRun: ] for name, signal in vars(signals).items(): - if not name.startswith('_'): + if not name.startswith("_"): dispatcher.connect(self.record_signal, signal) self.crawler = get_crawler(self.spider_class) @@ -179,8 +181,12 @@ class CrawlerRun: self.crawler.signals.connect(self.bytes_received, signals.bytes_received) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) - self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader) - self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) + self.crawler.signals.connect( + self.request_reached, signals.request_reached_downloader + ) + self.crawler.signals.connect( + self.response_downloaded, signals.response_downloaded + ) self.crawler.crawl(start_urls=start_urls) self.spider = self.crawler.spider @@ -191,7 +197,7 @@ class CrawlerRun: def stop(self): self.port.stopListening() # FIXME: wait for this Deferred for name, signal in vars(signals).items(): - if not name.startswith('_'): + if not name.startswith("_"): disconnect_all(signal) self.deferred.callback(None) return self.crawler.stop() @@ -230,8 +236,8 @@ class CrawlerRun: def record_signal(self, *args, **kwargs): """Record a signal and its parameters""" signalargs = kwargs.copy() - sig = signalargs.pop('signal') - signalargs.pop('sender', None) + sig = signalargs.pop("signal") + signalargs.pop("sender", None) self.signals_caught[sig] = signalargs @@ -239,7 +245,12 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_crawler(self): - for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + for spider in ( + TestSpider, + DictItemsSpider, + AttrsItemsSpider, + DataClassItemsSpider, + ): run = CrawlerRun(spider) yield run.run() self._assert_visited_urls(run) @@ -266,20 +277,30 @@ class EngineTest(unittest.TestCase): def test_crawler_change_close_reason_on_idle(self): run = CrawlerRun(ChangeCloseReasonSpider) yield run.run() - self.assertEqual({'spider': run.spider, 'reason': 'custom_reason'}, - run.signals_caught[signals.spider_closed]) + self.assertEqual( + {"spider": run.spider, "reason": "custom_reason"}, + run.signals_caught[signals.spider_closed], + ) def _assert_visited_urls(self, run: CrawlerRun): - must_be_visited = ["/", "/redirect", "/redirected", - "/item1.html", "/item2.html", "/item999.html"] + must_be_visited = [ + "/", + "/redirect", + "/redirected", + "/item1.html", + "/item2.html", + "/item999.html", + ] urls_visited = {rp[0].url for rp in run.respplug} urls_expected = {run.geturl(p) for p in must_be_visited} - assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" + assert ( + urls_expected <= urls_visited + ), f"URLs not visited: {list(urls_expected - urls_visited)}" def _assert_scheduled_requests(self, run: CrawlerRun, count=None): self.assertEqual(count, len(run.reqplug)) - paths_expected = ['/item999.html', '/item2.html', '/item1.html'] + paths_expected = ["/item999.html", "/item2.html", "/item1.html"] urls_requested = {rq[0].url for rq in run.reqplug} urls_expected = {run.geturl(p) for p in paths_expected} @@ -287,10 +308,10 @@ class EngineTest(unittest.TestCase): scheduled_requests_count = len(run.reqplug) dropped_requests_count = len(run.reqdropped) responses_count = len(run.respplug) - self.assertEqual(scheduled_requests_count, - dropped_requests_count + responses_count) - self.assertEqual(len(run.reqreached), - responses_count) + self.assertEqual( + scheduled_requests_count, dropped_requests_count + responses_count + ) + self.assertEqual(len(run.reqreached), responses_count) def _assert_dropped_requests(self, run: CrawlerRun): self.assertEqual(len(run.reqdropped), 1) @@ -301,9 +322,9 @@ class EngineTest(unittest.TestCase): self.assertEqual(count, len(run.reqreached)) for response, _ in run.respplug: - if run.getpath(response.url) == '/item999.html': + if run.getpath(response.url) == "/item999.html": self.assertEqual(404, response.status) - if run.getpath(response.url) == '/redirect': + if run.getpath(response.url) == "/redirect": self.assertEqual(302, response.status) def _assert_items_error(self, run: CrawlerRun): @@ -312,25 +333,25 @@ class EngineTest(unittest.TestCase): self.assertEqual(failure.value.__class__, ZeroDivisionError) self.assertEqual(spider, run.spider) - self.assertEqual(item['url'], response.url) - if 'item1.html' in item['url']: - self.assertEqual('Item 1 name', item['name']) - self.assertEqual('100', item['price']) - if 'item2.html' in item['url']: - self.assertEqual('Item 2 name', item['name']) - self.assertEqual('200', item['price']) + self.assertEqual(item["url"], response.url) + if "item1.html" in item["url"]: + self.assertEqual("Item 1 name", item["name"]) + self.assertEqual("100", item["price"]) + if "item2.html" in item["url"]: + self.assertEqual("Item 2 name", item["name"]) + self.assertEqual("200", item["price"]) def _assert_scraped_items(self, run: CrawlerRun): self.assertEqual(2, len(run.itemresp)) for item, response in run.itemresp: item = ItemAdapter(item) - self.assertEqual(item['url'], response.url) - if 'item1.html' in item['url']: - self.assertEqual('Item 1 name', item['name']) - self.assertEqual('100', item['price']) - if 'item2.html' in item['url']: - self.assertEqual('Item 2 name', item['name']) - self.assertEqual('200', item['price']) + self.assertEqual(item["url"], response.url) + if "item1.html" in item["url"]: + self.assertEqual("Item 1 name", item["name"]) + self.assertEqual("100", item["price"]) + if "item2.html" in item["url"]: + self.assertEqual("Item 2 name", item["name"]) + self.assertEqual("200", item["price"]) def _assert_headers_received(self, run: CrawlerRun): for headers in run.headers.values(): @@ -351,17 +372,17 @@ class EngineTest(unittest.TestCase): self.assertEqual(joined_data, get_testdata("test_site", "item2.html")) elif run.getpath(request.url) == "/redirected": self.assertEqual(joined_data, b"Redirected here") - elif run.getpath(request.url) == '/redirect': + elif run.getpath(request.url) == "/redirect": self.assertEqual( joined_data, b"\n\n" b" \n" - b" \n" + b' \n' b" \n" - b" \n" - b" click here\n" + b' \n' + b' click here\n' b" \n" - b"\n" + b"\n", ) elif run.getpath(request.url) == "/tem999.html": self.assertEqual( @@ -372,7 +393,7 @@ class EngineTest(unittest.TestCase): b"

No Such Resource

\n" b"

File not found.

\n" b" \n" - b"\n" + b"\n", ) elif run.getpath(request.url) == "/numbers": # signal was fired multiple times @@ -389,12 +410,16 @@ class EngineTest(unittest.TestCase): assert signals.spider_closed in run.signals_caught assert signals.headers_received in run.signals_caught - self.assertEqual({'spider': run.spider}, - run.signals_caught[signals.spider_opened]) - self.assertEqual({'spider': run.spider}, - run.signals_caught[signals.spider_idle]) - self.assertEqual({'spider': run.spider, 'reason': 'finished'}, - run.signals_caught[signals.spider_closed]) + self.assertEqual( + {"spider": run.spider}, run.signals_caught[signals.spider_opened] + ) + self.assertEqual( + {"spider": run.spider}, run.signals_caught[signals.spider_idle] + ) + self.assertEqual( + {"spider": run.spider, "reason": "finished"}, + run.signals_caught[signals.spider_closed], + ) @defer.inlineCallbacks def test_close_downloader(self): @@ -415,9 +440,11 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_close_spiders_downloader(self): - with pytest.warns(ScrapyDeprecationWarning, - match="ExecutionEngine.open_spiders is deprecated, " - "please use ExecutionEngine.spider instead"): + with pytest.warns( + ScrapyDeprecationWarning, + match="ExecutionEngine.open_spiders is deprecated, " + "please use ExecutionEngine.spider instead", + ): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) self.assertEqual(len(e.open_spiders), 1) @@ -426,9 +453,11 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_close_engine_spiders_downloader(self): - with pytest.warns(ScrapyDeprecationWarning, - match="ExecutionEngine.open_spiders is deprecated, " - "please use ExecutionEngine.spider instead"): + with pytest.warns( + ScrapyDeprecationWarning, + match="ExecutionEngine.open_spiders is deprecated, " + "please use ExecutionEngine.spider instead", + ): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() @@ -439,9 +468,11 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_crawl_deprecated_spider_arg(self): - with pytest.warns(ScrapyDeprecationWarning, - match="Passing a 'spider' argument to " - "ExecutionEngine.crawl is deprecated"): + with pytest.warns( + ScrapyDeprecationWarning, + match="Passing a 'spider' argument to " + "ExecutionEngine.crawl is deprecated", + ): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) spider = TestSpider() yield e.open_spider(spider, []) @@ -451,9 +482,11 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_download_deprecated_spider_arg(self): - with pytest.warns(ScrapyDeprecationWarning, - match="Passing a 'spider' argument to " - "ExecutionEngine.download is deprecated"): + with pytest.warns( + ScrapyDeprecationWarning, + match="Passing a 'spider' argument to " + "ExecutionEngine.download is deprecated", + ): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) spider = TestSpider() yield e.open_spider(spider, []) @@ -463,9 +496,11 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_deprecated_schedule(self): - with pytest.warns(ScrapyDeprecationWarning, - match="ExecutionEngine.schedule is deprecated, please use " - "ExecutionEngine.crawl or ExecutionEngine.download instead"): + with pytest.warns( + ScrapyDeprecationWarning, + match="ExecutionEngine.schedule is deprecated, please use " + "ExecutionEngine.crawl or ExecutionEngine.download instead", + ): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) spider = TestSpider() yield e.open_spider(spider, []) @@ -475,8 +510,9 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_deprecated_has_capacity(self): - with pytest.warns(ScrapyDeprecationWarning, - match="ExecutionEngine.has_capacity is deprecated"): + with pytest.warns( + ScrapyDeprecationWarning, match="ExecutionEngine.has_capacity is deprecated" + ): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) self.assertTrue(e.has_capacity()) spider = TestSpider() @@ -489,14 +525,14 @@ class EngineTest(unittest.TestCase): def test_short_timeout(self): args = ( sys.executable, - '-m', - 'scrapy.cmdline', - 'fetch', - '-s', - 'CLOSESPIDER_TIMEOUT=0.001', - '-s', - 'LOG_LEVEL=DEBUG', - 'http://toscrape.com', + "-m", + "scrapy.cmdline", + "fetch", + "-s", + "CLOSESPIDER_TIMEOUT=0.001", + "-s", + "LOG_LEVEL=DEBUG", + "http://toscrape.com", ) p = subprocess.Popen( args, @@ -506,7 +542,7 @@ class EngineTest(unittest.TestCase): def kill_proc(): p.kill() p.communicate() - assert False, 'Command took too much time to complete' + assert False, "Command took too much time to complete" timer = Timer(15, kill_proc) try: @@ -515,10 +551,10 @@ class EngineTest(unittest.TestCase): finally: timer.cancel() - self.assertNotIn(b'Traceback', stderr) + self.assertNotIn(b"Traceback", stderr) if __name__ == "__main__": - if len(sys.argv) > 1 and sys.argv[1] == 'runserver': + if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) reactor.run() diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 933e4067d..fb8dd4313 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -22,22 +22,39 @@ class BytesReceivedCrawlerRun(CrawlerRun): class BytesReceivedEngineTest(EngineTest): @defer.inlineCallbacks def test_crawler(self): - for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + for spider in ( + TestSpider, + DictItemsSpider, + AttrsItemsSpider, + DataClassItemsSpider, + ): run = BytesReceivedCrawlerRun(spider) with LogCapture() as log: yield run.run() - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler BytesReceivedCrawlerRun.bytes_received")) - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler BytesReceivedCrawlerRun.bytes_received")) - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler BytesReceivedCrawlerRun.bytes_received")) + log.check_present( + ( + "scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received", + ) + ) + log.check_present( + ( + "scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received", + ) + ) + log.check_present( + ( + "scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received", + ) + ) self._assert_visited_urls(run) self._assert_scheduled_requests(run, count=9) self._assert_downloaded_responses(run, count=9) diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 8975d0e3f..93437559d 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -22,22 +22,39 @@ class HeadersReceivedCrawlerRun(CrawlerRun): class HeadersReceivedEngineTest(EngineTest): @defer.inlineCallbacks def test_crawler(self): - for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + for spider in ( + TestSpider, + DictItemsSpider, + AttrsItemsSpider, + DataClassItemsSpider, + ): run = HeadersReceivedCrawlerRun(spider) with LogCapture() as log: yield run.run() - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for from" - " signal handler HeadersReceivedCrawlerRun.headers_received")) - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for from signal" - " handler HeadersReceivedCrawlerRun.headers_received")) - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for from" - " signal handler HeadersReceivedCrawlerRun.headers_received")) + log.check_present( + ( + "scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from" + " signal handler HeadersReceivedCrawlerRun.headers_received", + ) + ) + log.check_present( + ( + "scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from signal" + " handler HeadersReceivedCrawlerRun.headers_received", + ) + ) + log.check_present( + ( + "scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from" + " signal handler HeadersReceivedCrawlerRun.headers_received", + ) + ) self._assert_visited_urls(run) self._assert_downloaded_responses(run, count=6) self._assert_signals_caught(run) @@ -51,4 +68,6 @@ class HeadersReceivedEngineTest(EngineTest): must_be_visited = ["/", "/redirect", "/redirected"] urls_visited = {rp[0].url for rp in run.respplug} urls_expected = {run.geturl(p) for p in must_be_visited} - assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" + assert ( + urls_expected <= urls_visited + ), f"URLs not visited: {list(urls_expected - urls_visited)}" diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 86c5d8b5e..7689045b7 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -16,9 +16,15 @@ from scrapy.item import Item, Field from scrapy.utils.python import to_unicode from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exporters import ( - BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter, - XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, - PythonItemExporter, MarshalItemExporter + BaseItemExporter, + PprintItemExporter, + PickleItemExporter, + CsvItemExporter, + XmlItemExporter, + JsonLinesItemExporter, + JsonItemExporter, + PythonItemExporter, + MarshalItemExporter, ) @@ -54,7 +60,7 @@ class BaseItemExporterTest(unittest.TestCase): custom_field_item_class = CustomFieldItem def setUp(self): - self.i = self.item_class(name='John\xa3', age='22') + self.i = self.item_class(name="John\xa3", age="22") self.output = BytesIO() self.ie = self._get_exporter() @@ -71,10 +77,10 @@ class BaseItemExporterTest(unittest.TestCase): def _get_nonstring_types_item(self): return { - 'boolean': False, - 'number': 22, - 'time': datetime(2015, 1, 1, 1, 1, 1), - 'float': 3.14, + "boolean": False, + "number": 22, + "time": datetime(2015, 1, 1, 1, 1, 1), + "float": 3.14, } def assertItemExportWorks(self, item): @@ -95,35 +101,36 @@ class BaseItemExporterTest(unittest.TestCase): def test_serialize_field(self): a = ItemAdapter(self.i) - res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name']) - self.assertEqual(res, 'John\xa3') + res = self.ie.serialize_field(a.get_field_meta("name"), "name", a["name"]) + self.assertEqual(res, "John\xa3") - res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age']) - self.assertEqual(res, '22') + res = self.ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) + self.assertEqual(res, "22") def test_fields_to_export(self): - ie = self._get_exporter(fields_to_export=['name']) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')]) + ie = self._get_exporter(fields_to_export=["name"]) + self.assertEqual( + list(ie._get_serialized_fields(self.i)), [("name", "John\xa3")] + ) - ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') + ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1") _, name = list(ie._get_serialized_fields(self.i))[0] assert isinstance(name, str) - self.assertEqual(name, 'John\xa3') + self.assertEqual(name, "John\xa3") - ie = self._get_exporter( - fields_to_export={'name': 'å稱'} - ) - self.assertEqual( - list(ie._get_serialized_fields(self.i)), - [('å稱', 'John\xa3')] - ) + ie = self._get_exporter(fields_to_export={"name": "å稱"}) + self.assertEqual(list(ie._get_serialized_fields(self.i)), [("å稱", "John\xa3")]) def test_field_custom_serializer(self): - i = self.custom_field_item_class(name='John\xa3', age='22') + i = self.custom_field_item_class(name="John\xa3", age="22") a = ItemAdapter(i) ie = self._get_exporter() - self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3') - self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24') + self.assertEqual( + ie.serialize_field(a.get_field_meta("name"), "name", a["name"]), "John\xa3" + ) + self.assertEqual( + ie.serialize_field(a.get_field_meta("age"), "age", a["age"]), "24" + ) class BaseItemExporterDataclassTest(BaseItemExporterTest): @@ -137,54 +144,63 @@ class PythonItemExporterTest(BaseItemExporterTest): def test_invalid_option(self): with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"): - PythonItemExporter(invalid_option='something') + PythonItemExporter(invalid_option="something") def test_nested_item(self): - i1 = self.item_class(name='Joseph', age='22') - i2 = dict(name='Maria', age=i1) - i3 = self.item_class(name='Jesus', age=i2) + i1 = self.item_class(name="Joseph", age="22") + i2 = dict(name="Maria", age=i1) + i3 = self.item_class(name="Jesus", age=i2) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual(type(exported), dict) self.assertEqual( exported, - {'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'} + { + "age": {"age": {"age": "22", "name": "Joseph"}, "name": "Maria"}, + "name": "Jesus", + }, ) - self.assertEqual(type(exported['age']), dict) - self.assertEqual(type(exported['age']['age']), dict) + self.assertEqual(type(exported["age"]), dict) + self.assertEqual(type(exported["age"]["age"]), dict) def test_export_list(self): - i1 = self.item_class(name='Joseph', age='22') - i2 = self.item_class(name='Maria', age=[i1]) - i3 = self.item_class(name='Jesus', age=[i2]) + i1 = self.item_class(name="Joseph", age="22") + i2 = self.item_class(name="Maria", age=[i1]) + i3 = self.item_class(name="Jesus", age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( exported, - {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'} + { + "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], + "name": "Jesus", + }, ) - self.assertEqual(type(exported['age'][0]), dict) - self.assertEqual(type(exported['age'][0]['age'][0]), dict) + self.assertEqual(type(exported["age"][0]), dict) + self.assertEqual(type(exported["age"][0]["age"][0]), dict) def test_export_item_dict_list(self): - i1 = self.item_class(name='Joseph', age='22') - i2 = dict(name='Maria', age=[i1]) - i3 = self.item_class(name='Jesus', age=[i2]) + i1 = self.item_class(name="Joseph", age="22") + i2 = dict(name="Maria", age=[i1]) + i3 = self.item_class(name="Jesus", age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( exported, - {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'} + { + "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], + "name": "Jesus", + }, ) - self.assertEqual(type(exported['age'][0]), dict) - self.assertEqual(type(exported['age'][0]['age'][0]), dict) + self.assertEqual(type(exported["age"][0]), dict) + self.assertEqual(type(exported["age"][0]["age"][0]), dict) def test_export_binary(self): with catch_warnings(): - filterwarnings('ignore', category=ScrapyDeprecationWarning) + filterwarnings("ignore", category=ScrapyDeprecationWarning) exporter = PythonItemExporter(binary=True) - value = self.item_class(name='John\xa3', age='22') - expected = {b'name': b'John\xc2\xa3', b'age': b'22'} + value = self.item_class(name="John\xa3", age="22") + expected = {b"name": b"John\xc2\xa3", b"age": b"22"} self.assertEqual(expected, exporter.export_item(value)) def test_nonstring_types_item(self): @@ -200,7 +216,6 @@ class PythonItemExporterDataclassTest(PythonItemExporterTest): class PprintItemExporterTest(BaseItemExporterTest): - def _get_exporter(self, **kwargs): return PprintItemExporter(self.output, **kwargs) @@ -214,7 +229,6 @@ class PprintItemExporterDataclassTest(PprintItemExporterTest): class PickleItemExporterTest(BaseItemExporterTest): - def _get_exporter(self, **kwargs): return PickleItemExporter(self.output, **kwargs) @@ -222,8 +236,8 @@ class PickleItemExporterTest(BaseItemExporterTest): self._assert_expected_item(pickle.loads(self.output.getvalue())) def test_export_multiple_items(self): - i1 = self.item_class(name='hello', age='world') - i2 = self.item_class(name='bye', age='world') + i1 = self.item_class(name="hello", age="world") + i2 = self.item_class(name="bye", age="world") f = BytesIO() ie = PickleItemExporter(f) ie.start_exporting() @@ -250,7 +264,6 @@ class PickleItemExporterDataclassTest(PickleItemExporterTest): class MarshalItemExporterTest(BaseItemExporterTest): - def _get_exporter(self, **kwargs): self.output = tempfile.TemporaryFile() return MarshalItemExporter(self.output, **kwargs) @@ -261,7 +274,7 @@ class MarshalItemExporterTest(BaseItemExporterTest): def test_nonstring_types_item(self): item = self._get_nonstring_types_item() - item.pop('time') # datetime is not marshallable + item.pop("time") # datetime is not marshallable fp = tempfile.TemporaryFile() ie = MarshalItemExporter(fp) ie.start_exporting() @@ -287,11 +300,14 @@ class CsvItemExporterTest(BaseItemExporterTest): sorted(re.split(r"(,|\s+)", line)) for line in to_unicode(csv).splitlines(True) ] + return self.assertEqual(split_csv(first), split_csv(second), msg=msg) def _check_output(self): self.output.seek(0) - self.assertCsvEqual(to_unicode(self.output.read()), 'age,name\r\n22,John\xa3\r\n') + self.assertCsvEqual( + to_unicode(self.output.read()), "age,name\r\n22,John\xa3\r\n" + ) def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() @@ -305,21 +321,21 @@ class CsvItemExporterTest(BaseItemExporterTest): self.assertExportResult( item=self.i, fields_to_export=ItemAdapter(self.i).field_names(), - expected=b'age,name\r\n22,John\xc2\xa3\r\n', + expected=b"age,name\r\n22,John\xc2\xa3\r\n", ) def test_header_export_all_dict(self): self.assertExportResult( item=ItemAdapter(self.i).asdict(), - expected=b'age,name\r\n22,John\xc2\xa3\r\n', + expected=b"age,name\r\n22,John\xc2\xa3\r\n", ) def test_header_export_single_field(self): for item in [self.i, ItemAdapter(self.i).asdict()]: self.assertExportResult( item=item, - fields_to_export=['age'], - expected=b'age\r\n22\r\n', + fields_to_export=["age"], + expected=b"age\r\n22\r\n", ) def test_header_export_two_items(self): @@ -330,15 +346,16 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.export_item(item) ie.export_item(item) ie.finish_exporting() - self.assertCsvEqual(output.getvalue(), - b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') + self.assertCsvEqual( + output.getvalue(), b"age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n" + ) def test_header_no_header_line(self): for item in [self.i, ItemAdapter(self.i).asdict()]: self.assertExportResult( item=item, include_headers_line=False, - expected=b'22,John\xc2\xa3\r\n', + expected=b"22,John\xc2\xa3\r\n", ) def test_join_multivalue(self): @@ -348,14 +365,14 @@ class CsvItemExporterTest(BaseItemExporterTest): for cls in TestItem2, dict: self.assertExportResult( - item=cls(name='John', friends=['Mary', 'Paul']), + item=cls(name="John", friends=["Mary", "Paul"]), include_headers_line=False, expected='"Mary,Paul",John\r\n', ) def test_join_multivalue_not_strings(self): self.assertExportResult( - item=dict(name='John', friends=[4, 8]), + item=dict(name="John", friends=[4, 8]), include_headers_line=False, expected='"[4, 8]",John\r\n', ) @@ -364,24 +381,24 @@ class CsvItemExporterTest(BaseItemExporterTest): self.assertExportResult( item=self._get_nonstring_types_item(), include_headers_line=False, - expected='22,False,3.14,2015-01-01 01:01:01\r\n' + expected="22,False,3.14,2015-01-01 01:01:01\r\n", ) def test_errors_default(self): with self.assertRaises(UnicodeEncodeError): self.assertExportResult( - item=dict(text='W\u0275\u200Brd'), + item=dict(text="W\u0275\u200Brd"), expected=None, - encoding='windows-1251', + encoding="windows-1251", ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item=dict(text='W\u0275\u200Brd'), + item=dict(text="W\u0275\u200Brd"), include_headers_line=False, - expected='Wɵ​rd\r\n', - encoding='windows-1251', - errors='xmlcharrefreplace', + expected="Wɵ​rd\r\n", + encoding="windows-1251", + errors="xmlcharrefreplace", ) @@ -391,7 +408,6 @@ class CsvItemExporterDataclassTest(CsvItemExporterTest): class XmlItemExporterTest(BaseItemExporterTest): - def _get_exporter(self, **kwargs): return XmlItemExporter(self.output, **kwargs) @@ -405,6 +421,7 @@ class XmlItemExporterTest(BaseItemExporterTest): def xmlsplit(xmlcontent): doc = lxml.etree.fromstring(xmlcontent) return xmltuple(doc) + return self.assertEqual(xmlsplit(first), xmlsplit(second), msg) def assertExportResult(self, item, expected_value): @@ -418,13 +435,13 @@ class XmlItemExporterTest(BaseItemExporterTest): def _check_output(self): expected_value = ( b'\n' - b'22John\xc2\xa3' + b"22John\xc2\xa3" ) self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): self.assertExportResult( - self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]), + self.item_class(name=["John\xa3", "Doe"], age=[1, 2, 3]), b"""\n @@ -432,13 +449,13 @@ class XmlItemExporterTest(BaseItemExporterTest): 123 - """ + """, ) def test_nested_item(self): - i1 = dict(name='foo\xa3hoo', age='22') - i2 = dict(name='bar', age=i1) - i3 = self.item_class(name='buz', age=i2) + i1 = dict(name="foo\xa3hoo", age="22") + i2 = dict(name="bar", age=i1) + i3 = self.item_class(name="buz", age=i2) self.assertExportResult( i3, @@ -455,13 +472,13 @@ class XmlItemExporterTest(BaseItemExporterTest): buz - """ + """, ) def test_nested_list_item(self): - i1 = dict(name='foo') - i2 = dict(name='bar', v2={"egg": ["spam"]}) - i3 = self.item_class(name='buz', age=[i1, i2]) + i1 = dict(name="foo") + i2 = dict(name="bar", v2={"egg": ["spam"]}) + i3 = self.item_class(name="buz", age=[i1, i2]) self.assertExportResult( i3, @@ -475,7 +492,7 @@ class XmlItemExporterTest(BaseItemExporterTest): buz - """ + """, ) def test_nonstring_types_item(self): @@ -491,7 +508,7 @@ class XmlItemExporterTest(BaseItemExporterTest): - """ + """, ) @@ -503,7 +520,10 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest): class JsonLinesItemExporterTest(BaseItemExporterTest): - _expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}} + _expected_nested = { + "name": "Jesus", + "age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}}, + } def _get_exporter(self, **kwargs): return JsonLinesItemExporter(self.output, **kwargs) @@ -513,9 +533,9 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.assertEqual(exported, ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = self.item_class(name='Joseph', age='22') - i2 = dict(name='Maria', age=i1) - i3 = self.item_class(name='Jesus', age=i2) + i1 = self.item_class(name="Joseph", age="22") + i2 = dict(name="Maria", age=i1) + i3 = self.item_class(name="Jesus", age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() @@ -534,7 +554,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.export_item(item) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - item['time'] = str(item['time']) + item["time"] = str(item["time"]) self.assertEqual(exported, item) @@ -561,7 +581,9 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()]) + self.assertEqual( + exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()] + ) def test_two_items(self): self.assertTwoItemsExported(self.i) @@ -570,25 +592,28 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = self.item_class(name='Joseph\xa3', age='22') - i2 = self.item_class(name='Maria', age=i1) - i3 = self.item_class(name='Jesus', age=i2) + i1 = self.item_class(name="Joseph\xa3", age="22") + i2 = self.item_class(name="Maria", age=i1) + i3 = self.item_class(name="Jesus", age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} + expected = { + "name": "Jesus", + "age": {"name": "Maria", "age": ItemAdapter(i1).asdict()}, + } self.assertEqual(exported, [expected]) def test_nested_dict_item(self): - i1 = dict(name='Joseph\xa3', age='22') - i2 = self.item_class(name='Maria', age=i1) - i3 = dict(name='Jesus', age=i2) + i1 = dict(name="Joseph\xa3", age="22") + i2 = self.item_class(name="Maria", age=i1) + i3 = dict(name="Jesus", age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}} + expected = {"name": "Jesus", "age": {"name": "Maria", "age": i1}} self.assertEqual(exported, [expected]) def test_nonstring_types_item(self): @@ -597,7 +622,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - item['time'] = str(item['time']) + item["time"] = str(item["time"]) self.assertEqual(exported, [item]) @@ -618,20 +643,24 @@ class CustomExporterItemTest(unittest.TestCase): def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): def serialize_field(self, field, name, value): - if name == 'age': + if name == "age": return str(int(value) + 1) return super().serialize_field(field, name, value) - i = self.item_class(name='John', age='22') + i = self.item_class(name="John", age="22") a = ItemAdapter(i) ie = CustomItemExporter() - self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John') - self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23') + self.assertEqual( + ie.serialize_field(a.get_field_meta("name"), "name", a["name"]), "John" + ) + self.assertEqual( + ie.serialize_field(a.get_field_meta("age"), "age", a["age"]), "23" + ) - i2 = {'name': 'John', 'age': '22'} - self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John') - self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23') + i2 = {"name": "John", "age": "22"} + self.assertEqual(ie.serialize_field({}, "name", i2["name"]), "John") + self.assertEqual(ie.serialize_field({}, "age", i2["age"]), "23") class CustomExporterDataclassTest(CustomExporterItemTest): @@ -639,5 +668,5 @@ class CustomExporterDataclassTest(CustomExporterItemTest): item_class = TestDataClass -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 1e716b94a..e36c45d8e 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -24,7 +24,7 @@ class TelnetExtensionTest(unittest.TestCase): @defer.inlineCallbacks def test_bad_credentials(self): console, portal = self._get_console_and_portal() - creds = credentials.UsernamePassword(b'username', b'password') + creds = credentials.UsernamePassword(b"username", b"password") d = portal.login(creds, None, ITelnetProtocol) yield self.assertFailure(d, ValueError) console.stop_listening() @@ -33,8 +33,7 @@ class TelnetExtensionTest(unittest.TestCase): def test_good_credentials(self): console, portal = self._get_console_and_portal() creds = credentials.UsernamePassword( - console.username.encode('utf8'), - console.password.encode('utf8') + console.username.encode("utf8"), console.password.encode("utf8") ) d = portal.login(creds, None, ITelnetProtocol) yield d @@ -43,11 +42,11 @@ class TelnetExtensionTest(unittest.TestCase): @defer.inlineCallbacks def test_custom_credentials(self): settings = { - 'TELNETCONSOLE_USERNAME': 'user', - 'TELNETCONSOLE_PASSWORD': 'pass', + "TELNETCONSOLE_USERNAME": "user", + "TELNETCONSOLE_PASSWORD": "pass", } console, portal = self._get_console_and_portal(settings=settings) - creds = credentials.UsernamePassword(b'user', b'pass') + creds = credentials.UsernamePassword(b"user", b"pass") d = portal.login(creds, None, ITelnetProtocol) yield d console.stop_listening() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 97c3a74b3..890c88c7e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -57,29 +57,28 @@ from tests.spiders import ItemSpider def path_to_url(path): - return urljoin('file:', pathname2url(str(path))) + return urljoin("file:", pathname2url(str(path))) def printf_escape(string): - return string.replace('%', '%%') + return string.replace("%", "%%") def build_url(path: Union[str, PathLike]) -> str: path_str = str(path) - if path_str[0] != '/': - path_str = '/' + path_str - return urljoin('file:', path_str) + if path_str[0] != "/": + path_str = "/" + path_str + return urljoin("file:", path_str) class FileFeedStorageTest(unittest.TestCase): - def test_store_file_uri(self): path = Path(self.mktemp()).resolve() uri = path_to_file_uri(str(path)) return self._assert_stores(FileFeedStorage(uri), path) def test_store_file_uri_makedirs(self): - path = Path(self.mktemp()).resolve() / 'more' / 'paths' / 'file.txt' + path = Path(self.mktemp()).resolve() / "more" / "paths" / "file.txt" uri = path_to_file_uri(str(path)) return self._assert_stores(FileFeedStorage(uri), path) @@ -112,8 +111,7 @@ class FileFeedStorageTest(unittest.TestCase): def test_overwrite(self): path = self._store({"overwrite": True}) return self._assert_stores( - FileFeedStorage(str(path), feed_options={"overwrite": True}), - path + FileFeedStorage(str(path), feed_options={"overwrite": True}), path ) @defer.inlineCallbacks @@ -130,10 +128,9 @@ class FileFeedStorageTest(unittest.TestCase): class FTPFeedStorageTest(unittest.TestCase): - def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): - name = 'test_spider' + name = "test_spider" crawler = get_crawler(settings_dict=settings) spider = TestSpider.from_crawler(crawler) @@ -162,9 +159,9 @@ class FTPFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_append(self): with MockFTPServer() as ftp_server: - filename = 'file' + filename = "file" url = ftp_server.url(filename) - feed_options = {'overwrite': False} + feed_options = {"overwrite": False} yield self._store(url, b"foo", feed_options=feed_options) yield self._store(url, b"bar", feed_options=feed_options) self._assert_stored(ftp_server.path / filename, b"foobar") @@ -172,7 +169,7 @@ class FTPFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_overwrite(self): with MockFTPServer() as ftp_server: - filename = 'file' + filename = "file" url = ftp_server.url(filename) yield self._store(url, b"foo") yield self._store(url, b"bar") @@ -181,10 +178,10 @@ class FTPFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_append_active_mode(self): with MockFTPServer() as ftp_server: - settings = {'FEED_STORAGE_FTP_ACTIVE': True} - filename = 'file' + settings = {"FEED_STORAGE_FTP_ACTIVE": True} + filename = "file" url = ftp_server.url(filename) - feed_options = {'overwrite': False} + feed_options = {"overwrite": False} yield self._store(url, b"foo", feed_options=feed_options, settings=settings) yield self._store(url, b"bar", feed_options=feed_options, settings=settings) self._assert_stored(ftp_server.path / filename, b"foobar") @@ -192,8 +189,8 @@ class FTPFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_overwrite_active_mode(self): with MockFTPServer() as ftp_server: - settings = {'FEED_STORAGE_FTP_ACTIVE': True} - filename = 'file' + settings = {"FEED_STORAGE_FTP_ACTIVE": True} + filename = "file" url = ftp_server.url(filename) yield self._store(url, b"foo", settings=settings) yield self._store(url, b"bar", settings=settings) @@ -201,16 +198,15 @@ class FTPFeedStorageTest(unittest.TestCase): def test_uri_auth_quote(self): # RFC3986: 3.2.1. User Information - pw_quoted = quote(string.punctuation, safe='') - st = FTPFeedStorage(f'ftp://foo:{pw_quoted}@example.com/some_path', {}) + pw_quoted = quote(string.punctuation, safe="") + st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) self.assertEqual(st.password, string.punctuation) class BlockingFeedStorageTest(unittest.TestCase): - def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): - name = 'test_spider' + name = "test_spider" crawler = get_crawler(settings_dict=settings) spider = TestSpider.from_crawler(crawler) @@ -227,7 +223,7 @@ class BlockingFeedStorageTest(unittest.TestCase): b = BlockingFeedStorage() tests_path = Path(__file__).resolve().parent - spider = self.get_test_spider({'FEED_TEMPDIR': str(tests_path)}) + spider = self.get_test_spider({"FEED_TEMPDIR": str(tests_path)}) tmp = b.open(spider) tmp_path = Path(tmp.name).parent self.assertEqual(tmp_path, tests_path) @@ -236,66 +232,72 @@ class BlockingFeedStorageTest(unittest.TestCase): b = BlockingFeedStorage() tests_path = Path(__file__).resolve().parent - invalid_path = tests_path / 'invalid_path' - spider = self.get_test_spider({'FEED_TEMPDIR': str(invalid_path)}) + invalid_path = tests_path / "invalid_path" + spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) self.assertRaises(OSError, b.open, spider=spider) class S3FeedStorageTest(unittest.TestCase): - def test_parse_credentials(self): skip_if_no_boto() - aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', - 'AWS_SECRET_ACCESS_KEY': 'settings_secret', - 'AWS_SESSION_TOKEN': 'settings_token'} + aws_credentials = { + "AWS_ACCESS_KEY_ID": "settings_key", + "AWS_SECRET_ACCESS_KEY": "settings_secret", + "AWS_SESSION_TOKEN": "settings_token", + } crawler = get_crawler(settings_dict=aws_credentials) # Instantiate with crawler storage = S3FeedStorage.from_crawler( crawler, - 's3://mybucket/export.csv', + "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, 'settings_key') - self.assertEqual(storage.secret_key, 'settings_secret') - self.assertEqual(storage.session_token, 'settings_token') + self.assertEqual(storage.access_key, "settings_key") + self.assertEqual(storage.secret_key, "settings_secret") + self.assertEqual(storage.session_token, "settings_token") # Instantiate directly - storage = S3FeedStorage('s3://mybucket/export.csv', - aws_credentials['AWS_ACCESS_KEY_ID'], - aws_credentials['AWS_SECRET_ACCESS_KEY'], - session_token=aws_credentials['AWS_SESSION_TOKEN']) - self.assertEqual(storage.access_key, 'settings_key') - self.assertEqual(storage.secret_key, 'settings_secret') - self.assertEqual(storage.session_token, 'settings_token') + storage = S3FeedStorage( + "s3://mybucket/export.csv", + aws_credentials["AWS_ACCESS_KEY_ID"], + aws_credentials["AWS_SECRET_ACCESS_KEY"], + session_token=aws_credentials["AWS_SESSION_TOKEN"], + ) + self.assertEqual(storage.access_key, "settings_key") + self.assertEqual(storage.secret_key, "settings_secret") + self.assertEqual(storage.session_token, "settings_token") # URI priority > settings priority - storage = S3FeedStorage('s3://uri_key:uri_secret@mybucket/export.csv', - aws_credentials['AWS_ACCESS_KEY_ID'], - aws_credentials['AWS_SECRET_ACCESS_KEY']) - self.assertEqual(storage.access_key, 'uri_key') - self.assertEqual(storage.secret_key, 'uri_secret') + storage = S3FeedStorage( + "s3://uri_key:uri_secret@mybucket/export.csv", + aws_credentials["AWS_ACCESS_KEY_ID"], + aws_credentials["AWS_SECRET_ACCESS_KEY"], + ) + self.assertEqual(storage.access_key, "uri_key") + self.assertEqual(storage.secret_key, "uri_secret") @defer.inlineCallbacks def test_store(self): skip_if_no_boto() settings = { - 'AWS_ACCESS_KEY_ID': 'access_key', - 'AWS_SECRET_ACCESS_KEY': 'secret_key', + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", } crawler = get_crawler(settings_dict=settings) - bucket = 'mybucket' - key = 'export.csv' - storage = S3FeedStorage.from_crawler(crawler, f's3://{bucket}/{key}') + bucket = "mybucket" + key = "export.csv" + storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}") verifyObject(IFeedStorage, storage) file = mock.MagicMock() from botocore.stub import Stubber + with Stubber(storage.s3_client) as stub: stub.add_response( - 'put_object', + "put_object", expected_params={ - 'Body': file, - 'Bucket': bucket, - 'Key': key, + "Body": file, + "Bucket": bucket, + "Key": key, }, service_response={}, ) @@ -309,172 +311,154 @@ class S3FeedStorageTest(unittest.TestCase): mock.call.seek(0), # The call to read does not happen with Stubber mock.call.close(), - ] + ], ) def test_init_without_acl(self): - storage = S3FeedStorage( - 's3://mybucket/export.csv', - 'access_key', - 'secret_key' - ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') + storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.acl, None) def test_init_with_acl(self): storage = S3FeedStorage( - 's3://mybucket/export.csv', - 'access_key', - 'secret_key', - 'custom-acl' + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') - self.assertEqual(storage.acl, 'custom-acl') + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.acl, "custom-acl") def test_init_with_endpoint_url(self): storage = S3FeedStorage( - 's3://mybucket/export.csv', - 'access_key', - 'secret_key', - endpoint_url='https://example.com' + "s3://mybucket/export.csv", + "access_key", + "secret_key", + endpoint_url="https://example.com", ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') - self.assertEqual(storage.endpoint_url, 'https://example.com') + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.endpoint_url, "https://example.com") def test_from_crawler_without_acl(self): settings = { - 'AWS_ACCESS_KEY_ID': 'access_key', - 'AWS_SECRET_ACCESS_KEY': 'secret_key', + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", } crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler( crawler, - 's3://mybucket/export.csv', + "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.acl, None) def test_without_endpoint_url(self): settings = { - 'AWS_ACCESS_KEY_ID': 'access_key', - 'AWS_SECRET_ACCESS_KEY': 'secret_key', + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", } crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler( crawler, - 's3://mybucket/export.csv', + "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, None) def test_from_crawler_with_acl(self): settings = { - 'AWS_ACCESS_KEY_ID': 'access_key', - 'AWS_SECRET_ACCESS_KEY': 'secret_key', - 'FEED_STORAGE_S3_ACL': 'custom-acl', + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "FEED_STORAGE_S3_ACL": "custom-acl", } crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler( crawler, - 's3://mybucket/export.csv', + "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') - self.assertEqual(storage.acl, 'custom-acl') + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.acl, "custom-acl") def test_from_crawler_with_endpoint_url(self): settings = { - 'AWS_ACCESS_KEY_ID': 'access_key', - 'AWS_SECRET_ACCESS_KEY': 'secret_key', - 'AWS_ENDPOINT_URL': 'https://example.com', + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_ENDPOINT_URL": "https://example.com", } crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - 's3://mybucket/export.csv' - ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') - self.assertEqual(storage.endpoint_url, 'https://example.com') + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.endpoint_url, "https://example.com") @defer.inlineCallbacks def test_store_botocore_without_acl(self): skip_if_no_boto() storage = S3FeedStorage( - 's3://mybucket/export.csv', - 'access_key', - 'secret_key', + "s3://mybucket/export.csv", + "access_key", + "secret_key", ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.acl, None) storage.s3_client = mock.MagicMock() - yield storage.store(BytesIO(b'test file')) - self.assertNotIn('ACL', storage.s3_client.put_object.call_args[1]) + yield storage.store(BytesIO(b"test file")) + self.assertNotIn("ACL", storage.s3_client.put_object.call_args[1]) @defer.inlineCallbacks def test_store_botocore_with_acl(self): skip_if_no_boto() storage = S3FeedStorage( - 's3://mybucket/export.csv', - 'access_key', - 'secret_key', - 'custom-acl' + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') - self.assertEqual(storage.acl, 'custom-acl') + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.acl, "custom-acl") storage.s3_client = mock.MagicMock() - yield storage.store(BytesIO(b'test file')) + yield storage.store(BytesIO(b"test file")) self.assertEqual( - storage.s3_client.put_object.call_args[1].get('ACL'), - 'custom-acl' + storage.s3_client.put_object.call_args[1].get("ACL"), "custom-acl" ) def test_overwrite_default(self): with LogCapture() as log: S3FeedStorage( - 's3://mybucket/export.csv', - 'access_key', - 'secret_key', - 'custom-acl' + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) - self.assertNotIn('S3 does not support appending to files', str(log)) + self.assertNotIn("S3 does not support appending to files", str(log)) def test_overwrite_false(self): with LogCapture() as log: S3FeedStorage( - 's3://mybucket/export.csv', - 'access_key', - 'secret_key', - 'custom-acl', - feed_options={'overwrite': False}, + "s3://mybucket/export.csv", + "access_key", + "secret_key", + "custom-acl", + feed_options={"overwrite": False}, ) - self.assertIn('S3 does not support appending to files', str(log)) + self.assertIn("S3 does not support appending to files", str(log)) class GCSFeedStorageTest(unittest.TestCase): - def test_parse_settings(self): try: from google.cloud.storage import Client # noqa except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead'} + settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"} crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') - assert storage.project_id == '123' - assert storage.acl == 'publicRead' - assert storage.bucket_name == 'mybucket' - assert storage.blob_name == 'export.csv' + storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") + assert storage.project_id == "123" + assert storage.acl == "publicRead" + assert storage.bucket_name == "mybucket" + assert storage.blob_name == "export.csv" def test_parse_empty_acl(self): try: @@ -482,14 +466,14 @@ class GCSFeedStorageTest(unittest.TestCase): except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': ''} + settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') + storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") assert storage.acl is None - settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None} + settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": None} crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') + storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") assert storage.acl is None @defer.inlineCallbacks @@ -499,11 +483,11 @@ class GCSFeedStorageTest(unittest.TestCase): except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - uri = 'gs://mybucket/export.csv' - project_id = 'myproject-123' - acl = 'publicRead' + uri = "gs://mybucket/export.csv" + project_id = "myproject-123" + acl = "publicRead" (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() - with mock.patch('google.cloud.storage.Client') as m: + with mock.patch("google.cloud.storage.Client") as m: m.return_value = client_mock f = mock.Mock() @@ -512,17 +496,16 @@ class GCSFeedStorageTest(unittest.TestCase): f.seek.assert_called_once_with(0) m.assert_called_once_with(project=project_id) - client_mock.get_bucket.assert_called_once_with('mybucket') - bucket_mock.blob.assert_called_once_with('export.csv') + client_mock.get_bucket.assert_called_once_with("mybucket") + bucket_mock.blob.assert_called_once_with("export.csv") blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) class StdoutFeedStorageTest(unittest.TestCase): - @defer.inlineCallbacks def test_store(self): out = BytesIO() - storage = StdoutFeedStorage('stdout:', _stdout=out) + storage = StdoutFeedStorage("stdout:", _stdout=out) file = storage.open(scrapy.Spider("default")) file.write(b"content") yield storage.store(file) @@ -530,13 +513,17 @@ class StdoutFeedStorageTest(unittest.TestCase): def test_overwrite_default(self): with LogCapture() as log: - StdoutFeedStorage('stdout:') - self.assertNotIn('Standard output (stdout) storage does not support overwriting', str(log)) + StdoutFeedStorage("stdout:") + self.assertNotIn( + "Standard output (stdout) storage does not support overwriting", str(log) + ) def test_overwrite_true(self): with LogCapture() as log: - StdoutFeedStorage('stdout:', feed_options={'overwrite': True}) - self.assertIn('Standard output (stdout) storage does not support overwriting', str(log)) + StdoutFeedStorage("stdout:", feed_options={"overwrite": True}) + self.assertIn( + "Standard output (stdout) storage does not support overwriting", str(log) + ) class FromCrawlerMixin: @@ -553,7 +540,6 @@ class FromCrawlerCsvItemExporter(CsvItemExporter, FromCrawlerMixin): class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): - @classmethod def from_crawler(cls, crawler, *args, feed_options=None, **kwargs): cls.init_with_crawler = True @@ -561,7 +547,6 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): class DummyBlockingFeedStorage(BlockingFeedStorage): - def __init__(self, uri, *args, feed_options=None): self.path = Path(file_uri_to_path(uri)) @@ -569,16 +554,15 @@ class DummyBlockingFeedStorage(BlockingFeedStorage): dirname = self.path.parent if dirname and not dirname.exists(): dirname.mkdir(parents=True) - with self.path.open('ab') as output_file: + with self.path.open("ab") as output_file: output_file.write(file.read()) file.close() class FailingBlockingFeedStorage(DummyBlockingFeedStorage): - def _store_in_thread(self, file): - raise OSError('Cannot store') + raise OSError("Cannot store") @implementer(IFeedStorage) @@ -593,10 +577,10 @@ class LogOnStoreFileStorage: self.logger = getLogger() def open(self, spider): - return tempfile.NamedTemporaryFile(prefix='feed-') + return tempfile.NamedTemporaryFile(prefix="feed-") def store(self, file): - self.logger.info('Storage.store is called') + self.logger.info("Storage.store is called") file.close() @@ -612,9 +596,9 @@ class FeedExportTestBase(ABC, unittest.TestCase): foo = scrapy.Field() hello = scrapy.Field() - def _random_temp_filename(self, inter_dir='') -> Path: + def _random_temp_filename(self, inter_dir="") -> Path: chars = [random.choice(ascii_letters + digits) for _ in range(15)] - filename = ''.join(chars) + filename = "".join(chars) return Path(self.temp_dir, inter_dir, filename) def setUp(self): @@ -630,7 +614,7 @@ class FeedExportTestBase(ABC, unittest.TestCase): """ class TestSpider(scrapy.Spider): - name = 'testspider' + name = "testspider" def parse(self, response): for item in items: @@ -646,7 +630,7 @@ class FeedExportTestBase(ABC, unittest.TestCase): """ class TestSpider(scrapy.Spider): - name = 'testspider' + name = "testspider" def parse(self, response): pass @@ -685,10 +669,10 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): - """ Run spider with specified settings; return exported data. """ + """Run spider with specified settings; return exported data.""" - FEEDS = settings.get('FEEDS') or {} - settings['FEEDS'] = { + FEEDS = settings.get("FEEDS") or {} + settings["FEEDS"] = { printf_escape(path_to_url(file_path)): feed_options for file_path, feed_options in FEEDS.items() } @@ -696,7 +680,7 @@ class FeedExportTest(FeedExportTestBase): content = {} try: with MockServer() as s: - spider_cls.start_urls = [s.url('/')] + spider_cls.start_urls = [s.url("/")] crawler = get_crawler(spider_cls, settings) yield crawler.crawl() @@ -704,7 +688,7 @@ class FeedExportTest(FeedExportTestBase): if not Path(file_path).exists(): continue - content[feed_options['format']] = Path(file_path).read_bytes() + content[feed_options["format"]] = Path(file_path).read_bytes() finally: for file_path in FEEDS.keys(): @@ -718,88 +702,102 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} - settings.update({ - 'FEEDS': { - self._random_temp_filename(): {'format': 'csv'}, - }, - }) + settings.update( + { + "FEEDS": { + self._random_temp_filename(): {"format": "csv"}, + }, + } + ) data = yield self.exported_data(items, settings) - reader = csv.DictReader(to_unicode(data['csv']).splitlines()) + reader = csv.DictReader(to_unicode(data["csv"]).splitlines()) self.assertEqual(reader.fieldnames, list(header)) self.assertEqual(rows, list(reader)) @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} - settings.update({ - 'FEEDS': { - self._random_temp_filename(): {'format': 'jl'}, - }, - }) + settings.update( + { + "FEEDS": { + self._random_temp_filename(): {"format": "jl"}, + }, + } + ) data = yield self.exported_data(items, settings) - parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()] + parsed = [json.loads(to_unicode(line)) for line in data["jl"].splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] self.assertEqual(rows, parsed) @defer.inlineCallbacks def assertExportedXml(self, items, rows, settings=None): settings = settings or {} - settings.update({ - 'FEEDS': { - self._random_temp_filename(): {'format': 'xml'}, - }, - }) + settings.update( + { + "FEEDS": { + self._random_temp_filename(): {"format": "xml"}, + }, + } + ) data = yield self.exported_data(items, settings) rows = [{k: v for k, v in row.items() if v} for row in rows] - root = lxml.etree.fromstring(data['xml']) - got_rows = [{e.tag: e.text for e in it} for it in root.findall('item')] + root = lxml.etree.fromstring(data["xml"]) + got_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] self.assertEqual(rows, got_rows) @defer.inlineCallbacks def assertExportedMultiple(self, items, rows, settings=None): settings = settings or {} - settings.update({ - 'FEEDS': { - self._random_temp_filename(): {'format': 'xml'}, - self._random_temp_filename(): {'format': 'json'}, - }, - }) + settings.update( + { + "FEEDS": { + self._random_temp_filename(): {"format": "xml"}, + self._random_temp_filename(): {"format": "json"}, + }, + } + ) data = yield self.exported_data(items, settings) rows = [{k: v for k, v in row.items() if v} for row in rows] # XML - root = lxml.etree.fromstring(data['xml']) - xml_rows = [{e.tag: e.text for e in it} for it in root.findall('item')] + root = lxml.etree.fromstring(data["xml"]) + xml_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] self.assertEqual(rows, xml_rows) # JSON - json_rows = json.loads(to_unicode(data['json'])) + json_rows = json.loads(to_unicode(data["json"])) self.assertEqual(rows, json_rows) @defer.inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} - settings.update({ - 'FEEDS': { - self._random_temp_filename(): {'format': 'pickle'}, - }, - }) + settings.update( + { + "FEEDS": { + self._random_temp_filename(): {"format": "pickle"}, + }, + } + ) data = yield self.exported_data(items, settings) expected = [{k: v for k, v in row.items() if v} for row in rows] import pickle - result = self._load_until_eof(data['pickle'], load_func=pickle.load) + + result = self._load_until_eof(data["pickle"], load_func=pickle.load) self.assertEqual(expected, result) @defer.inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): settings = settings or {} - settings.update({ - 'FEEDS': { - self._random_temp_filename(): {'format': 'marshal'}, - }, - }) + settings.update( + { + "FEEDS": { + self._random_temp_filename(): {"format": "marshal"}, + }, + } + ) data = yield self.exported_data(items, settings) expected = [{k: v for k, v in row.items() if v} for row in rows] import marshal - result = self._load_until_eof(data['marshal'], load_func=marshal.load) + + result = self._load_until_eof(data["marshal"], load_func=marshal.load) self.assertEqual(expected, result) @defer.inlineCallbacks @@ -814,8 +812,12 @@ class FeedExportTest(FeedExportTestBase): crawler = get_crawler(ItemSpider, settings) with MockServer() as mockserver: yield crawler.crawl(mockserver=mockserver) - self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()) - self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1) + self.assertIn( + "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() + ) + self.assertEqual( + crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1 + ) @defer.inlineCallbacks def test_stats_file_failed(self): @@ -832,17 +834,22 @@ class FeedExportTest(FeedExportTestBase): stack.enter_context( mock.patch( "scrapy.extensions.feedexport.FileFeedStorage.store", - side_effect=KeyError("foo")) + side_effect=KeyError("foo"), + ) ) yield crawler.crawl(mockserver=mockserver) - self.assertIn("feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats()) - self.assertEqual(crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1) + self.assertIn( + "feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats() + ) + self.assertEqual( + crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1 + ) @defer.inlineCallbacks def test_stats_multiple_file(self): settings = { - 'AWS_ACCESS_KEY_ID': 'access_key', - 'AWS_SECRET_ACCESS_KEY': 'secret_key', + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", "FEEDS": { printf_escape(path_to_url(str(self._random_temp_filename()))): { "format": "json", @@ -852,101 +859,113 @@ class FeedExportTest(FeedExportTestBase): }, "stdout:": { "format": "xml", - } + }, }, } crawler = get_crawler(ItemSpider, settings) with MockServer() as mockserver, mock.patch.object(S3FeedStorage, "store"): yield crawler.crawl(mockserver=mockserver) - self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()) - self.assertIn("feedexport/success_count/S3FeedStorage", crawler.stats.get_stats()) - self.assertIn("feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats()) - self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1) - self.assertEqual(crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1) - self.assertEqual(crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1) + self.assertIn( + "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() + ) + self.assertIn( + "feedexport/success_count/S3FeedStorage", crawler.stats.get_stats() + ) + self.assertIn( + "feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats() + ) + self.assertEqual( + crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1 + ) + self.assertEqual( + crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1 + ) + self.assertEqual( + crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1 + ) @defer.inlineCallbacks def test_export_items(self): # feed exporters use field names from Item items = [ - self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), - self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), ] rows = [ - {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, - {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} + {"egg": "spam1", "foo": "bar1", "baz": ""}, + {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, ] header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows) @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): - for fmt in ('json', 'jsonlines', 'xml', 'csv'): + for fmt in ("json", "jsonlines", "xml", "csv"): settings = { - 'FEEDS': { - self._random_temp_filename(): {'format': fmt}, + "FEEDS": { + self._random_temp_filename(): {"format": fmt}, }, } data = yield self.exported_no_data(settings) - self.assertEqual(b'', data[fmt]) + self.assertEqual(b"", data[fmt]) @defer.inlineCallbacks def test_export_no_items_store_empty(self): formats = ( - ('json', b'[]'), - ('jsonlines', b''), - ('xml', b'\n'), - ('csv', b''), + ("json", b"[]"), + ("jsonlines", b""), + ("xml", b'\n'), + ("csv", b""), ) for fmt, expctd in formats: settings = { - 'FEEDS': { - self._random_temp_filename(): {'format': fmt}, + "FEEDS": { + self._random_temp_filename(): {"format": fmt}, }, - 'FEED_STORE_EMPTY': True, - 'FEED_EXPORT_INDENT': None, + "FEED_STORE_EMPTY": True, + "FEED_EXPORT_INDENT": None, } data = yield self.exported_no_data(settings) self.assertEqual(expctd, data[fmt]) @defer.inlineCallbacks def test_export_no_items_multiple_feeds(self): - """ Make sure that `storage.store` is called for every feed. """ + """Make sure that `storage.store` is called for every feed.""" settings = { - 'FEEDS': { - self._random_temp_filename(): {'format': 'json'}, - self._random_temp_filename(): {'format': 'xml'}, - self._random_temp_filename(): {'format': 'csv'}, + "FEEDS": { + self._random_temp_filename(): {"format": "json"}, + self._random_temp_filename(): {"format": "xml"}, + self._random_temp_filename(): {"format": "csv"}, }, - 'FEED_STORAGES': {'file': LogOnStoreFileStorage}, - 'FEED_STORE_EMPTY': False + "FEED_STORAGES": {"file": LogOnStoreFileStorage}, + "FEED_STORE_EMPTY": False, } with LogCapture() as log: yield self.exported_no_data(settings) print(log) - self.assertEqual(str(log).count('Storage.store is called'), 3) + self.assertEqual(str(log).count("Storage.store is called"), 3) @defer.inlineCallbacks def test_export_multiple_item_classes(self): items = [ - self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), - self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), - self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}), - {'hello': 'world4', 'egg': 'spam4'}, + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem2({"hello": "world2", "foo": "bar2"}), + self.MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}), + {"hello": "world4", "egg": "spam4"}, ] # by default, Scrapy uses fields of the first Item for CSV and # all fields for JSON Lines header = self.MyItem.fields.keys() rows_csv = [ - {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, - {'egg': '', 'foo': 'bar2', 'baz': ''}, - {'egg': 'spam3', 'foo': 'bar3', 'baz': 'quux3'}, - {'egg': 'spam4', 'foo': '', 'baz': ''}, + {"egg": "spam1", "foo": "bar1", "baz": ""}, + {"egg": "", "foo": "bar2", "baz": ""}, + {"egg": "spam3", "foo": "bar3", "baz": "quux3"}, + {"egg": "spam4", "foo": "", "baz": ""}, ] rows_jl = [dict(row) for row in items] yield self.assertExportedCsv(items, header, rows_csv) @@ -955,106 +974,103 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_items_empty_field_list(self): # FEED_EXPORT_FIELDS==[] means the same as default None - items = [{'foo': 'bar'}] + items = [{"foo": "bar"}] header = ["foo"] - rows = [{'foo': 'bar'}] - settings = {'FEED_EXPORT_FIELDS': []} + rows = [{"foo": "bar"}] + settings = {"FEED_EXPORT_FIELDS": []} yield self.assertExportedCsv(items, header, rows) yield self.assertExportedJsonLines(items, rows, settings) @defer.inlineCallbacks def test_export_items_field_list(self): - items = [{'foo': 'bar'}] + items = [{"foo": "bar"}] header = ["foo", "baz"] - rows = [{'foo': 'bar', 'baz': ''}] - settings = {'FEED_EXPORT_FIELDS': header} + rows = [{"foo": "bar", "baz": ""}] + settings = {"FEED_EXPORT_FIELDS": header} yield self.assertExported(items, header, rows, settings=settings) @defer.inlineCallbacks def test_export_items_comma_separated_field_list(self): - items = [{'foo': 'bar'}] + items = [{"foo": "bar"}] header = ["foo", "baz"] - rows = [{'foo': 'bar', 'baz': ''}] - settings = {'FEED_EXPORT_FIELDS': ",".join(header)} + rows = [{"foo": "bar", "baz": ""}] + settings = {"FEED_EXPORT_FIELDS": ",".join(header)} yield self.assertExported(items, header, rows, settings=settings) @defer.inlineCallbacks def test_export_items_json_field_list(self): - items = [{'foo': 'bar'}] + items = [{"foo": "bar"}] header = ["foo", "baz"] - rows = [{'foo': 'bar', 'baz': ''}] - settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} + rows = [{"foo": "bar", "baz": ""}] + settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} yield self.assertExported(items, header, rows, settings=settings) @defer.inlineCallbacks def test_export_items_field_names(self): - items = [{'foo': 'bar'}] - header = {'foo': 'Foo'} - rows = [{'Foo': 'bar'}] - settings = {'FEED_EXPORT_FIELDS': header} - yield self.assertExported(items, list(header.values()), rows, - settings=settings) + items = [{"foo": "bar"}] + header = {"foo": "Foo"} + rows = [{"Foo": "bar"}] + settings = {"FEED_EXPORT_FIELDS": header} + yield self.assertExported(items, list(header.values()), rows, settings=settings) @defer.inlineCallbacks def test_export_items_dict_field_names(self): - items = [{'foo': 'bar'}] + items = [{"foo": "bar"}] header = { - 'baz': 'Baz', - 'foo': 'Foo', + "baz": "Baz", + "foo": "Foo", } - rows = [{'Baz': '', 'Foo': 'bar'}] - settings = {'FEED_EXPORT_FIELDS': header} - yield self.assertExported(items, ['Baz', 'Foo'], rows, - settings=settings) + rows = [{"Baz": "", "Foo": "bar"}] + settings = {"FEED_EXPORT_FIELDS": header} + yield self.assertExported(items, ["Baz", "Foo"], rows, settings=settings) @defer.inlineCallbacks def test_export_items_json_field_names(self): - items = [{'foo': 'bar'}] - header = {'foo': 'Foo'} - rows = [{'Foo': 'bar'}] - settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} - yield self.assertExported(items, list(header.values()), rows, - settings=settings) + items = [{"foo": "bar"}] + header = {"foo": "Foo"} + rows = [{"Foo": "bar"}] + settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} + yield self.assertExported(items, list(header.values()), rows, settings=settings) @defer.inlineCallbacks def test_export_based_on_item_classes(self): items = [ - self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), - self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), - {'hello': 'world3', 'egg': 'spam3'}, + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem2({"hello": "world2", "foo": "bar2"}), + {"hello": "world3", "egg": "spam3"}, ] formats = { - 'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n', - 'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]', - 'jsonlines': ( + "csv": b"baz,egg,foo\r\n,spam1,bar1\r\n", + "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]', + "jsonlines": ( b'{"foo": "bar1", "egg": "spam1"}\n' b'{"hello": "world2", "foo": "bar2"}\n' ), - 'xml': ( + "xml": ( b'\n\n' - b'bar1spam1\n' - b'world2bar2\nworld3' - b'spam3\n' + b"bar1spam1\n" + b"world2bar2\nworld3" + b"spam3\n" ), } settings = { - 'FEEDS': { + "FEEDS": { self._random_temp_filename(): { - 'format': 'csv', - 'item_classes': [self.MyItem], + "format": "csv", + "item_classes": [self.MyItem], }, self._random_temp_filename(): { - 'format': 'json', - 'item_classes': [self.MyItem2], + "format": "json", + "item_classes": [self.MyItem2], }, self._random_temp_filename(): { - 'format': 'jsonlines', - 'item_classes': [self.MyItem, self.MyItem2], + "format": "jsonlines", + "item_classes": [self.MyItem, self.MyItem2], }, self._random_temp_filename(): { - 'format': 'xml', + "format": "xml", }, }, } @@ -1066,9 +1082,9 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_based_on_custom_filters(self): items = [ - self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), - self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), - {'hello': 'world3', 'egg': 'spam3'}, + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem2({"hello": "world2", "foo": "bar2"}), + {"hello": "world3", "egg": "spam3"}, ] MyItem = self.MyItem @@ -1082,40 +1098,40 @@ class FeedExportTest(FeedExportTestBase): class CustomFilter2(scrapy.extensions.feedexport.ItemFilter): def accepts(self, item): - if 'foo' not in item.fields: + if "foo" not in item.fields: return False return True class CustomFilter3(scrapy.extensions.feedexport.ItemFilter): def accepts(self, item): - if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1": + if isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1": return True return False formats = { - 'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]', - 'xml': ( + "json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]', + "xml": ( b'\n\n' - b'bar1spam1\n' - b'world2bar2\n' + b"bar1spam1\n" + b"world2bar2\n" ), - 'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n', + "jsonlines": b'{"foo": "bar1", "egg": "spam1"}\n', } settings = { - 'FEEDS': { + "FEEDS": { self._random_temp_filename(): { - 'format': 'json', - 'item_filter': CustomFilter1, + "format": "json", + "item_filter": CustomFilter1, }, self._random_temp_filename(): { - 'format': 'xml', - 'item_filter': CustomFilter2, + "format": "xml", + "item_filter": CustomFilter2, }, self._random_temp_filename(): { - 'format': 'jsonlines', - 'item_classes': [self.MyItem, self.MyItem2], - 'item_filter': CustomFilter3, + "format": "jsonlines", + "item_classes": [self.MyItem, self.MyItem2], + "item_filter": CustomFilter3, }, }, } @@ -1129,15 +1145,12 @@ class FeedExportTest(FeedExportTestBase): # When dicts are used, only keys from the first row are used as # a header for CSV, and all fields are used for JSON Lines. items = [ - {'foo': 'bar', 'egg': 'spam'}, - {'foo': 'bar', 'egg': 'spam', 'baz': 'quux'}, - ] - rows_csv = [ - {'egg': 'spam', 'foo': 'bar'}, - {'egg': 'spam', 'foo': 'bar'} + {"foo": "bar", "egg": "spam"}, + {"foo": "bar", "egg": "spam", "baz": "quux"}, ] + rows_csv = [{"egg": "spam", "foo": "bar"}, {"egg": "spam", "foo": "bar"}] rows_jl = items - yield self.assertExportedCsv(items, ['foo', 'egg'], rows_csv) + yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -1147,105 +1160,102 @@ class FeedExportTest(FeedExportTestBase): for item_cls in [self.MyItem, dict]: items = [ - item_cls({'foo': 'bar1', 'egg': 'spam1'}), - item_cls({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + item_cls({"foo": "bar1", "egg": "spam1"}), + item_cls({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), ] # export all columns - settings = {'FEED_EXPORT_FIELDS': 'foo,baz,egg'} + settings = {"FEED_EXPORT_FIELDS": "foo,baz,egg"} rows = [ - {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, - {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} + {"egg": "spam1", "foo": "bar1", "baz": ""}, + {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, ] - yield self.assertExported(items, ['foo', 'baz', 'egg'], rows, - settings=settings) + yield self.assertExported( + items, ["foo", "baz", "egg"], rows, settings=settings + ) # export a subset of columns - settings = {'FEED_EXPORT_FIELDS': 'egg,baz'} - rows = [ - {'egg': 'spam1', 'baz': ''}, - {'egg': 'spam2', 'baz': 'quux2'} - ] - yield self.assertExported(items, ['egg', 'baz'], rows, - settings=settings) + settings = {"FEED_EXPORT_FIELDS": "egg,baz"} + rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}] + yield self.assertExported(items, ["egg", "baz"], rows, settings=settings) @defer.inlineCallbacks def test_export_encoding(self): - items = [dict({'foo': 'Test\xd6'})] + items = [dict({"foo": "Test\xd6"})] formats = { - 'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'), - 'jsonlines': '{"foo": "Test\\u00d6"}\n'.encode('utf-8'), - 'xml': ( + "json": '[{"foo": "Test\\u00d6"}]'.encode("utf-8"), + "jsonlines": '{"foo": "Test\\u00d6"}\n'.encode("utf-8"), + "xml": ( '\n' - 'Test\xd6' - ).encode('utf-8'), - 'csv': 'foo\r\nTest\xd6\r\n'.encode('utf-8'), + "Test\xd6" + ).encode("utf-8"), + "csv": "foo\r\nTest\xd6\r\n".encode("utf-8"), } for fmt, expected in formats.items(): settings = { - 'FEEDS': { - self._random_temp_filename(): {'format': fmt}, + "FEEDS": { + self._random_temp_filename(): {"format": fmt}, }, - 'FEED_EXPORT_INDENT': None, + "FEED_EXPORT_INDENT": None, } data = yield self.exported_data(items, settings) self.assertEqual(expected, data[fmt]) formats = { - 'json': '[{"foo": "Test\xd6"}]'.encode('latin-1'), - 'jsonlines': '{"foo": "Test\xd6"}\n'.encode('latin-1'), - 'xml': ( + "json": '[{"foo": "Test\xd6"}]'.encode("latin-1"), + "jsonlines": '{"foo": "Test\xd6"}\n'.encode("latin-1"), + "xml": ( '\n' - 'Test\xd6' - ).encode('latin-1'), - 'csv': 'foo\r\nTest\xd6\r\n'.encode('latin-1'), + "Test\xd6" + ).encode("latin-1"), + "csv": "foo\r\nTest\xd6\r\n".encode("latin-1"), } for fmt, expected in formats.items(): settings = { - 'FEEDS': { - self._random_temp_filename(): {'format': fmt}, + "FEEDS": { + self._random_temp_filename(): {"format": fmt}, }, - 'FEED_EXPORT_INDENT': None, - 'FEED_EXPORT_ENCODING': 'latin-1', + "FEED_EXPORT_INDENT": None, + "FEED_EXPORT_ENCODING": "latin-1", } data = yield self.exported_data(items, settings) self.assertEqual(expected, data[fmt]) @defer.inlineCallbacks def test_export_multiple_configs(self): - items = [dict({'foo': 'FOO', 'bar': 'BAR'})] + items = [dict({"foo": "FOO", "bar": "BAR"})] formats = { - 'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'), - 'xml': ( + "json": '[\n{"bar": "BAR"}\n]'.encode("utf-8"), + "xml": ( '\n' - '\n \n FOO\n \n' - ).encode('latin-1'), - 'csv': 'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), + "\n \n FOO\n \n" + ).encode("latin-1"), + "csv": "bar,foo\r\nBAR,FOO\r\n".encode("utf-8"), } settings = { - 'FEEDS': { + "FEEDS": { self._random_temp_filename(): { - 'format': 'json', - 'indent': 0, - 'fields': ['bar'], - 'encoding': 'utf-8', + "format": "json", + "indent": 0, + "fields": ["bar"], + "encoding": "utf-8", }, self._random_temp_filename(): { - 'format': 'xml', - 'indent': 2, - 'fields': ['foo'], - 'encoding': 'latin-1', + "format": "xml", + "indent": 2, + "fields": ["foo"], + "encoding": "latin-1", }, self._random_temp_filename(): { - 'format': 'csv', - 'indent': None, - 'fields': ['bar', 'foo'], - 'encoding': 'utf-8', + "format": "csv", + "indent": None, + "fields": ["bar", "foo"], + "encoding": "utf-8", }, }, } @@ -1257,37 +1267,37 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_indentation(self): items = [ - {'foo': ['bar']}, - {'key': 'value'}, + {"foo": ["bar"]}, + {"key": "value"}, ] test_cases = [ # JSON { - 'format': 'json', - 'indent': None, - 'expected': b'[{"foo": ["bar"]},{"key": "value"}]', + "format": "json", + "indent": None, + "expected": b'[{"foo": ["bar"]},{"key": "value"}]', }, { - 'format': 'json', - 'indent': -1, - 'expected': b"""[ + "format": "json", + "indent": -1, + "expected": b"""[ {"foo": ["bar"]}, {"key": "value"} ]""", }, { - 'format': 'json', - 'indent': 0, - 'expected': b"""[ + "format": "json", + "indent": 0, + "expected": b"""[ {"foo": ["bar"]}, {"key": "value"} ]""", }, { - 'format': 'json', - 'indent': 2, - 'expected': b"""[ + "format": "json", + "indent": 2, + "expected": b"""[ { "foo": [ "bar" @@ -1299,9 +1309,9 @@ class FeedExportTest(FeedExportTestBase): ]""", }, { - 'format': 'json', - 'indent': 4, - 'expected': b"""[ + "format": "json", + "indent": 4, + "expected": b"""[ { "foo": [ "bar" @@ -1313,9 +1323,9 @@ class FeedExportTest(FeedExportTestBase): ]""", }, { - 'format': 'json', - 'indent': 5, - 'expected': b"""[ + "format": "json", + "indent": 5, + "expected": b"""[ { "foo": [ "bar" @@ -1326,36 +1336,35 @@ class FeedExportTest(FeedExportTestBase): } ]""", }, - # XML { - 'format': 'xml', - 'indent': None, - 'expected': b""" + "format": "xml", + "indent": None, + "expected": b""" barvalue""", }, { - 'format': 'xml', - 'indent': -1, - 'expected': b""" + "format": "xml", + "indent": -1, + "expected": b""" bar value """, }, { - 'format': 'xml', - 'indent': 0, - 'expected': b""" + "format": "xml", + "indent": 0, + "expected": b""" bar value """, }, { - 'format': 'xml', - 'indent': 2, - 'expected': b""" + "format": "xml", + "indent": 2, + "expected": b""" @@ -1368,9 +1377,9 @@ class FeedExportTest(FeedExportTestBase): """, }, { - 'format': 'xml', - 'indent': 4, - 'expected': b""" + "format": "xml", + "indent": 4, + "expected": b""" @@ -1383,9 +1392,9 @@ class FeedExportTest(FeedExportTestBase): """, }, { - 'format': 'xml', - 'indent': 5, - 'expected': b""" + "format": "xml", + "indent": 5, + "expected": b""" @@ -1401,23 +1410,23 @@ class FeedExportTest(FeedExportTestBase): for row in test_cases: settings = { - 'FEEDS': { + "FEEDS": { self._random_temp_filename(): { - 'format': row['format'], - 'indent': row['indent'], + "format": row["format"], + "indent": row["indent"], }, }, } data = yield self.exported_data(items, settings) - self.assertEqual(row['expected'], data[row['format']]) + self.assertEqual(row["expected"], data[row["format"]]) @defer.inlineCallbacks def test_init_exporters_storages_with_crawler(self): settings = { - 'FEED_EXPORTERS': {'csv': FromCrawlerCsvItemExporter}, - 'FEED_STORAGES': {'file': FromCrawlerFileFeedStorage}, - 'FEEDS': { - self._random_temp_filename(): {'format': 'csv'}, + "FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter}, + "FEED_STORAGES": {"file": FromCrawlerFileFeedStorage}, + "FEEDS": { + self._random_temp_filename(): {"format": "csv"}, }, } yield self.exported_data(items=[], settings=settings) @@ -1427,105 +1436,103 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_str_uri(self): settings = { - 'FEED_STORE_EMPTY': True, - 'FEEDS': { - str(self._random_temp_filename()): {'format': 'csv'} - }, + "FEED_STORE_EMPTY": True, + "FEEDS": {str(self._random_temp_filename()): {"format": "csv"}}, } data = yield self.exported_no_data(settings) - self.assertEqual(data['csv'], b'') + self.assertEqual(data["csv"], b"") @defer.inlineCallbacks def test_multiple_feeds_success_logs_blocking_feed_storage(self): settings = { - 'FEEDS': { - self._random_temp_filename(): {'format': 'json'}, - self._random_temp_filename(): {'format': 'xml'}, - self._random_temp_filename(): {'format': 'csv'}, + "FEEDS": { + self._random_temp_filename(): {"format": "json"}, + self._random_temp_filename(): {"format": "xml"}, + self._random_temp_filename(): {"format": "csv"}, }, - 'FEED_STORAGES': {'file': DummyBlockingFeedStorage}, + "FEED_STORAGES": {"file": DummyBlockingFeedStorage}, } items = [ - {'foo': 'bar1', 'baz': ''}, - {'foo': 'bar2', 'baz': 'quux'}, + {"foo": "bar1", "baz": ""}, + {"foo": "bar2", "baz": "quux"}, ] with LogCapture() as log: yield self.exported_data(items, settings) print(log) - for fmt in ['json', 'xml', 'csv']: - self.assertIn(f'Stored {fmt} feed (2 items)', str(log)) + for fmt in ["json", "xml", "csv"]: + self.assertIn(f"Stored {fmt} feed (2 items)", str(log)) @defer.inlineCallbacks def test_multiple_feeds_failing_logs_blocking_feed_storage(self): settings = { - 'FEEDS': { - self._random_temp_filename(): {'format': 'json'}, - self._random_temp_filename(): {'format': 'xml'}, - self._random_temp_filename(): {'format': 'csv'}, + "FEEDS": { + self._random_temp_filename(): {"format": "json"}, + self._random_temp_filename(): {"format": "xml"}, + self._random_temp_filename(): {"format": "csv"}, }, - 'FEED_STORAGES': {'file': FailingBlockingFeedStorage}, + "FEED_STORAGES": {"file": FailingBlockingFeedStorage}, } items = [ - {'foo': 'bar1', 'baz': ''}, - {'foo': 'bar2', 'baz': 'quux'}, + {"foo": "bar1", "baz": ""}, + {"foo": "bar2", "baz": "quux"}, ] with LogCapture() as log: yield self.exported_data(items, settings) print(log) - for fmt in ['json', 'xml', 'csv']: - self.assertIn(f'Error storing {fmt} feed (2 items)', str(log)) + for fmt in ["json", "xml", "csv"]: + self.assertIn(f"Error storing {fmt} feed (2 items)", str(log)) @defer.inlineCallbacks def test_extend_kwargs(self): - items = [{'foo': 'FOO', 'bar': 'BAR'}] + items = [{"foo": "FOO", "bar": "BAR"}] - expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8') - expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8') + expected_with_title_csv = "foo,bar\r\nFOO,BAR\r\n".encode("utf-8") + expected_without_title_csv = "FOO,BAR\r\n".encode("utf-8") test_cases = [ # with title { - 'options': { - 'format': 'csv', - 'item_export_kwargs': {'include_headers_line': True}, + "options": { + "format": "csv", + "item_export_kwargs": {"include_headers_line": True}, }, - 'expected': expected_with_title_csv, + "expected": expected_with_title_csv, }, # without title { - 'options': { - 'format': 'csv', - 'item_export_kwargs': {'include_headers_line': False}, + "options": { + "format": "csv", + "item_export_kwargs": {"include_headers_line": False}, }, - 'expected': expected_without_title_csv, + "expected": expected_without_title_csv, }, ] for row in test_cases: - feed_options = row['options'] + feed_options = row["options"] settings = { - 'FEEDS': { + "FEEDS": { self._random_temp_filename(): feed_options, }, - 'FEED_EXPORT_INDENT': None, + "FEED_EXPORT_INDENT": None, } data = yield self.exported_data(items, settings) - self.assertEqual(row['expected'], data[feed_options['format']]) + self.assertEqual(row["expected"], data[feed_options["format"]]) class FeedPostProcessedExportsTest(FeedExportTestBase): __test__ = True - items = [{'foo': 'bar'}] - expected = b'foo\r\nbar\r\n' + items = [{"foo": "bar"}] + expected = b"foo\r\nbar\r\n" class MyPlugin1: def __init__(self, file, feed_options): self.file = file self.feed_options = feed_options - self.char = self.feed_options.get('plugin1_char', b'') + self.char = self.feed_options.get("plugin1_char", b"") def write(self, data): written_count = self.file.write(data) @@ -1540,10 +1547,10 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): - """ Run spider with specified settings; return exported data with filename. """ + """Run spider with specified settings; return exported data with filename.""" - FEEDS = settings.get('FEEDS') or {} - settings['FEEDS'] = { + FEEDS = settings.get("FEEDS") or {} + settings["FEEDS"] = { printf_escape(path_to_url(file_path)): feed_options for file_path, feed_options in FEEDS.items() } @@ -1551,7 +1558,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): content = {} try: with MockServer() as s: - spider_cls.start_urls = [s.url('/')] + spider_cls.start_urls = [s.url("/")] crawler = get_crawler(spider_cls, settings) yield crawler.crawl() @@ -1570,10 +1577,15 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): return content - def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=''): + def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=""): data_stream = BytesIO() - gzipf = gzip.GzipFile(fileobj=data_stream, filename=filename, mtime=mtime, - compresslevel=compresslevel, mode="wb") + gzipf = gzip.GzipFile( + fileobj=data_stream, + filename=filename, + mtime=mtime, + compresslevel=compresslevel, + mode="wb", + ) gzipf.write(data) gzipf.close() data_stream.seek(0) @@ -1582,13 +1594,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_gzip_plugin(self): - filename = self._named_tempfile('gzip_file') + filename = self._named_tempfile("gzip_file") settings = { - 'FEEDS': { + "FEEDS": { filename: { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], }, }, } @@ -1603,25 +1615,29 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): def test_gzip_plugin_compresslevel(self): filename_to_compressed = { - self._named_tempfile('compresslevel_0'): self.get_gzip_compressed(self.expected, compresslevel=0), - self._named_tempfile('compresslevel_9'): self.get_gzip_compressed(self.expected, compresslevel=9), + self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( + self.expected, compresslevel=0 + ), + self._named_tempfile("compresslevel_9"): self.get_gzip_compressed( + self.expected, compresslevel=9 + ), } settings = { - 'FEEDS': { - self._named_tempfile('compresslevel_0'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], - 'gzip_compresslevel': 0, - 'gzip_mtime': 0, - 'gzip_filename': "", + "FEEDS": { + self._named_tempfile("compresslevel_0"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_compresslevel": 0, + "gzip_mtime": 0, + "gzip_filename": "", }, - self._named_tempfile('compresslevel_9'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], - 'gzip_compresslevel': 9, - 'gzip_mtime': 0, - 'gzip_filename': "", + self._named_tempfile("compresslevel_9"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_compresslevel": 9, + "gzip_mtime": 0, + "gzip_filename": "", }, }, } @@ -1636,23 +1652,27 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_gzip_plugin_mtime(self): filename_to_compressed = { - self._named_tempfile('mtime_123'): self.get_gzip_compressed(self.expected, mtime=123), - self._named_tempfile('mtime_123456789'): self.get_gzip_compressed(self.expected, mtime=123456789), + self._named_tempfile("mtime_123"): self.get_gzip_compressed( + self.expected, mtime=123 + ), + self._named_tempfile("mtime_123456789"): self.get_gzip_compressed( + self.expected, mtime=123456789 + ), } settings = { - 'FEEDS': { - self._named_tempfile('mtime_123'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], - 'gzip_mtime': 123, - 'gzip_filename': "", + "FEEDS": { + self._named_tempfile("mtime_123"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 123, + "gzip_filename": "", }, - self._named_tempfile('mtime_123456789'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], - 'gzip_mtime': 123456789, - 'gzip_filename': "", + self._named_tempfile("mtime_123456789"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 123456789, + "gzip_filename": "", }, }, } @@ -1667,23 +1687,27 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_gzip_plugin_filename(self): filename_to_compressed = { - self._named_tempfile('filename_FILE1'): self.get_gzip_compressed(self.expected, filename="FILE1"), - self._named_tempfile('filename_FILE2'): self.get_gzip_compressed(self.expected, filename="FILE2"), + self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( + self.expected, filename="FILE1" + ), + self._named_tempfile("filename_FILE2"): self.get_gzip_compressed( + self.expected, filename="FILE2" + ), } settings = { - 'FEEDS': { - self._named_tempfile('filename_FILE1'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], - 'gzip_mtime': 0, - 'gzip_filename': "FILE1", + "FEEDS": { + self._named_tempfile("filename_FILE1"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 0, + "gzip_filename": "FILE1", }, - self._named_tempfile('filename_FILE2'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], - 'gzip_mtime': 0, - 'gzip_filename': "FILE2", + self._named_tempfile("filename_FILE2"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 0, + "gzip_filename": "FILE2", }, }, } @@ -1698,13 +1722,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_lzma_plugin(self): - filename = self._named_tempfile('lzma_file') + filename = self._named_tempfile("lzma_file") settings = { - 'FEEDS': { + "FEEDS": { filename: { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], }, }, } @@ -1719,21 +1743,25 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): def test_lzma_plugin_format(self): filename_to_compressed = { - self._named_tempfile('format_FORMAT_XZ'): lzma.compress(self.expected, format=lzma.FORMAT_XZ), - self._named_tempfile('format_FORMAT_ALONE'): lzma.compress(self.expected, format=lzma.FORMAT_ALONE), + self._named_tempfile("format_FORMAT_XZ"): lzma.compress( + self.expected, format=lzma.FORMAT_XZ + ), + self._named_tempfile("format_FORMAT_ALONE"): lzma.compress( + self.expected, format=lzma.FORMAT_ALONE + ), } settings = { - 'FEEDS': { - self._named_tempfile('format_FORMAT_XZ'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], - 'lzma_format': lzma.FORMAT_XZ, + "FEEDS": { + self._named_tempfile("format_FORMAT_XZ"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_format": lzma.FORMAT_XZ, }, - self._named_tempfile('format_FORMAT_ALONE'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], - 'lzma_format': lzma.FORMAT_ALONE, + self._named_tempfile("format_FORMAT_ALONE"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_format": lzma.FORMAT_ALONE, }, }, } @@ -1749,21 +1777,25 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): def test_lzma_plugin_check(self): filename_to_compressed = { - self._named_tempfile('check_CHECK_NONE'): lzma.compress(self.expected, check=lzma.CHECK_NONE), - self._named_tempfile('check_CHECK_CRC256'): lzma.compress(self.expected, check=lzma.CHECK_SHA256), + self._named_tempfile("check_CHECK_NONE"): lzma.compress( + self.expected, check=lzma.CHECK_NONE + ), + self._named_tempfile("check_CHECK_CRC256"): lzma.compress( + self.expected, check=lzma.CHECK_SHA256 + ), } settings = { - 'FEEDS': { - self._named_tempfile('check_CHECK_NONE'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], - 'lzma_check': lzma.CHECK_NONE, + "FEEDS": { + self._named_tempfile("check_CHECK_NONE"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_check": lzma.CHECK_NONE, }, - self._named_tempfile('check_CHECK_CRC256'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], - 'lzma_check': lzma.CHECK_SHA256, + self._named_tempfile("check_CHECK_CRC256"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_check": lzma.CHECK_SHA256, }, }, } @@ -1779,21 +1811,25 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): def test_lzma_plugin_preset(self): filename_to_compressed = { - self._named_tempfile('preset_PRESET_0'): lzma.compress(self.expected, preset=0), - self._named_tempfile('preset_PRESET_9'): lzma.compress(self.expected, preset=9), + self._named_tempfile("preset_PRESET_0"): lzma.compress( + self.expected, preset=0 + ), + self._named_tempfile("preset_PRESET_9"): lzma.compress( + self.expected, preset=9 + ), } settings = { - 'FEEDS': { - self._named_tempfile('preset_PRESET_0'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], - 'lzma_preset': 0, + "FEEDS": { + self._named_tempfile("preset_PRESET_0"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_preset": 0, }, - self._named_tempfile('preset_PRESET_9'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], - 'lzma_preset': 9, + self._named_tempfile("preset_PRESET_9"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_preset": 9, }, }, } @@ -1811,16 +1847,16 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): # https://foss.heptapod.net/pypy/pypy/-/issues/3527 raise unittest.SkipTest("lzma filters doesn't work in PyPy") - filters = [{'id': lzma.FILTER_LZMA2}] + filters = [{"id": lzma.FILTER_LZMA2}] compressed = lzma.compress(self.expected, filters=filters) - filename = self._named_tempfile('filters') + filename = self._named_tempfile("filters") settings = { - 'FEEDS': { + "FEEDS": { filename: { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], - 'lzma_filters': filters, + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_filters": filters, }, }, } @@ -1833,13 +1869,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_bz2_plugin(self): - filename = self._named_tempfile('bz2_file') + filename = self._named_tempfile("bz2_file") settings = { - 'FEEDS': { + "FEEDS": { filename: { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], }, }, } @@ -1854,21 +1890,25 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): def test_bz2_plugin_compresslevel(self): filename_to_compressed = { - self._named_tempfile('compresslevel_1'): bz2.compress(self.expected, compresslevel=1), - self._named_tempfile('compresslevel_9'): bz2.compress(self.expected, compresslevel=9), + self._named_tempfile("compresslevel_1"): bz2.compress( + self.expected, compresslevel=1 + ), + self._named_tempfile("compresslevel_9"): bz2.compress( + self.expected, compresslevel=9 + ), } settings = { - 'FEEDS': { - self._named_tempfile('compresslevel_1'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], - 'bz2_compresslevel': 1, + "FEEDS": { + self._named_tempfile("compresslevel_1"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], + "bz2_compresslevel": 1, }, - self._named_tempfile('compresslevel_9'): { - 'format': 'csv', - 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], - 'bz2_compresslevel': 9, + self._named_tempfile("compresslevel_9"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], + "bz2_compresslevel": 9, }, }, } @@ -1882,13 +1922,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_custom_plugin(self): - filename = self._named_tempfile('csv_file') + filename = self._named_tempfile("csv_file") settings = { - 'FEEDS': { + "FEEDS": { filename: { - 'format': 'csv', - 'postprocessing': [self.MyPlugin1], + "format": "csv", + "postprocessing": [self.MyPlugin1], }, }, } @@ -1899,15 +1939,15 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_custom_plugin_with_parameter(self): - expected = b'foo\r\n\nbar\r\n\n' - filename = self._named_tempfile('newline') + expected = b"foo\r\n\nbar\r\n\n" + filename = self._named_tempfile("newline") settings = { - 'FEEDS': { + "FEEDS": { filename: { - 'format': 'csv', - 'postprocessing': [self.MyPlugin1], - 'plugin1_char': b'\n' + "format": "csv", + "postprocessing": [self.MyPlugin1], + "plugin1_char": b"\n", }, }, } @@ -1918,30 +1958,39 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_custom_plugin_with_compression(self): - expected = b'foo\r\n\nbar\r\n\n' + expected = b"foo\r\n\nbar\r\n\n" filename_to_decompressor = { - self._named_tempfile('bz2'): bz2.decompress, - self._named_tempfile('lzma'): lzma.decompress, - self._named_tempfile('gzip'): gzip.decompress, + self._named_tempfile("bz2"): bz2.decompress, + self._named_tempfile("lzma"): lzma.decompress, + self._named_tempfile("gzip"): gzip.decompress, } settings = { - 'FEEDS': { - self._named_tempfile('bz2'): { - 'format': 'csv', - 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.Bz2Plugin'], - 'plugin1_char': b'\n', + "FEEDS": { + self._named_tempfile("bz2"): { + "format": "csv", + "postprocessing": [ + self.MyPlugin1, + "scrapy.extensions.postprocessing.Bz2Plugin", + ], + "plugin1_char": b"\n", }, - self._named_tempfile('lzma'): { - 'format': 'csv', - 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.LZMAPlugin'], - 'plugin1_char': b'\n', + self._named_tempfile("lzma"): { + "format": "csv", + "postprocessing": [ + self.MyPlugin1, + "scrapy.extensions.postprocessing.LZMAPlugin", + ], + "plugin1_char": b"\n", }, - self._named_tempfile('gzip'): { - 'format': 'csv', - 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'], - 'plugin1_char': b'\n', + self._named_tempfile("gzip"): { + "format": "csv", + "postprocessing": [ + self.MyPlugin1, + "scrapy.extensions.postprocessing.GzipPlugin", + ], + "plugin1_char": b"\n", }, }, } @@ -1956,40 +2005,41 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): def test_exports_compatibility_with_postproc(self): import marshal import pickle + filename_to_expected = { - self._named_tempfile('csv'): b'foo\r\nbar\r\n', - self._named_tempfile('json'): b'[\n{"foo": "bar"}\n]', - self._named_tempfile('jsonlines'): b'{"foo": "bar"}\n', - self._named_tempfile('xml'): b'\n' - b'\nbar\n', + self._named_tempfile("csv"): b"foo\r\nbar\r\n", + self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]', + self._named_tempfile("jsonlines"): b'{"foo": "bar"}\n', + self._named_tempfile("xml"): b'\n' + b"\nbar\n", } settings = { - 'FEEDS': { - self._named_tempfile('csv'): { - 'format': 'csv', - 'postprocessing': [self.MyPlugin1], + "FEEDS": { + self._named_tempfile("csv"): { + "format": "csv", + "postprocessing": [self.MyPlugin1], # empty plugin to activate postprocessing.PostProcessingManager }, - self._named_tempfile('json'): { - 'format': 'json', - 'postprocessing': [self.MyPlugin1], + self._named_tempfile("json"): { + "format": "json", + "postprocessing": [self.MyPlugin1], }, - self._named_tempfile('jsonlines'): { - 'format': 'jsonlines', - 'postprocessing': [self.MyPlugin1], + self._named_tempfile("jsonlines"): { + "format": "jsonlines", + "postprocessing": [self.MyPlugin1], }, - self._named_tempfile('xml'): { - 'format': 'xml', - 'postprocessing': [self.MyPlugin1], + self._named_tempfile("xml"): { + "format": "xml", + "postprocessing": [self.MyPlugin1], }, - self._named_tempfile('marshal'): { - 'format': 'marshal', - 'postprocessing': [self.MyPlugin1], + self._named_tempfile("marshal"): { + "format": "marshal", + "postprocessing": [self.MyPlugin1], }, - self._named_tempfile('pickle'): { - 'format': 'pickle', - 'postprocessing': [self.MyPlugin1], + self._named_tempfile("pickle"): { + "format": "pickle", + "postprocessing": [self.MyPlugin1], }, }, } @@ -1997,9 +2047,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): data = yield self.exported_data(self.items, settings) for filename, result in data.items(): - if 'pickle' in filename: + if "pickle" in filename: expected, result = self.items[0], pickle.loads(result) - elif 'marshal' in filename: + elif "marshal" in filename: expected, result = self.items[0], marshal.loads(result) else: expected = filename_to_expected[filename] @@ -2008,28 +2058,27 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): class BatchDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(batch_time)s_#%(batch_id)02d_' + _file_mark = "_%(batch_time)s_#%(batch_id)02d_" @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): - """ Run spider with specified settings; return exported data. """ + """Run spider with specified settings; return exported data.""" - FEEDS = settings.get('FEEDS') or {} - settings['FEEDS'] = { - build_url(file_path): feed - for file_path, feed in FEEDS.items() + FEEDS = settings.get("FEEDS") or {} + settings["FEEDS"] = { + build_url(file_path): feed for file_path, feed in FEEDS.items() } content = defaultdict(list) try: with MockServer() as s: - spider_cls.start_urls = [s.url('/')] + spider_cls.start_urls = [s.url("/")] crawler = get_crawler(spider_cls, settings) yield crawler.crawl() for path, feed in FEEDS.items(): dir_name = Path(path).parent for file in sorted(dir_name.iterdir()): - content[feed['format']].append(file.read_bytes()) + content[feed["format"]].append(file.read_bytes()) finally: self.tearDown() defer.returnValue(content) @@ -2037,30 +2086,40 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} - settings.update({ - 'FEEDS': { - self._random_temp_filename() / 'jl' / self._file_mark: {'format': 'jl'}, - }, - }) - batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') + settings.update( + { + "FEEDS": { + self._random_temp_filename() + / "jl" + / self._file_mark: {"format": "jl"}, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) - for batch in data['jl']: - got_batch = [json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()] + for batch in data["jl"]: + got_batch = [ + json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines() + ] expected_batch, rows = rows[:batch_size], rows[batch_size:] self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} - settings.update({ - 'FEEDS': { - self._random_temp_filename() / 'csv' / self._file_mark: {'format': 'csv'}, - }, - }) - batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') + settings.update( + { + "FEEDS": { + self._random_temp_filename() + / "csv" + / self._file_mark: {"format": "csv"}, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") data = yield self.exported_data(items, settings) - for batch in data['csv']: + for batch in data["csv"]: got_batch = csv.DictReader(to_unicode(batch).splitlines()) self.assertEqual(list(header), got_batch.fieldnames) expected_batch, rows = rows[:batch_size], rows[batch_size:] @@ -2069,59 +2128,74 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def assertExportedXml(self, items, rows, settings=None): settings = settings or {} - settings.update({ - 'FEEDS': { - self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'}, - }, - }) - batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') + settings.update( + { + "FEEDS": { + self._random_temp_filename() + / "xml" + / self._file_mark: {"format": "xml"}, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) - for batch in data['xml']: + for batch in data["xml"]: root = lxml.etree.fromstring(batch) - got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')] + got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] expected_batch, rows = rows[:batch_size], rows[batch_size:] self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks def assertExportedMultiple(self, items, rows, settings=None): settings = settings or {} - settings.update({ - 'FEEDS': { - self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'}, - self._random_temp_filename() / 'json' / self._file_mark: {'format': 'json'}, - }, - }) - batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') + settings.update( + { + "FEEDS": { + self._random_temp_filename() + / "xml" + / self._file_mark: {"format": "xml"}, + self._random_temp_filename() + / "json" + / self._file_mark: {"format": "json"}, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML xml_rows = rows.copy() - for batch in data['xml']: + for batch in data["xml"]: root = lxml.etree.fromstring(batch) - got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')] + got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] self.assertEqual(expected_batch, got_batch) # JSON json_rows = rows.copy() - for batch in data['json']: - got_batch = json.loads(batch.decode('utf-8')) + for batch in data["json"]: + got_batch = json.loads(batch.decode("utf-8")) expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} - settings.update({ - 'FEEDS': { - self._random_temp_filename() / 'pickle' / self._file_mark: {'format': 'pickle'}, - }, - }) - batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') + settings.update( + { + "FEEDS": { + self._random_temp_filename() + / "pickle" + / self._file_mark: {"format": "pickle"}, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle - for batch in data['pickle']: + + for batch in data["pickle"]: got_batch = self._load_until_eof(batch, load_func=pickle.load) expected_batch, rows = rows[:batch_size], rows[batch_size:] self.assertEqual(expected_batch, got_batch) @@ -2129,80 +2203,87 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): settings = settings or {} - settings.update({ - 'FEEDS': { - self._random_temp_filename() / 'marshal' / self._file_mark: {'format': 'marshal'}, - }, - }) - batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') + settings.update( + { + "FEEDS": { + self._random_temp_filename() + / "marshal" + / self._file_mark: {"format": "marshal"}, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal - for batch in data['marshal']: + + for batch in data["marshal"]: got_batch = self._load_until_eof(batch, load_func=marshal.load) expected_batch, rows = rows[:batch_size], rows[batch_size:] self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks def test_export_items(self): - """ Test partial deliveries in all supported formats """ + """Test partial deliveries in all supported formats""" items = [ - self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), - self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), - self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + self.MyItem({"foo": "bar3", "baz": "quux3"}), ] rows = [ - {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, - {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}, - {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} + {"egg": "spam1", "foo": "bar1", "baz": ""}, + {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, + {"foo": "bar3", "baz": "quux3", "egg": ""}, ] - settings = { - 'FEED_EXPORT_BATCH_ITEM_COUNT': 2 - } + settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2} header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """ + """If path is without %(batch_time)s and %(batch_id) an exception must be raised""" settings = { - 'FEEDS': { - self._random_temp_filename(): {'format': 'xml'}, + "FEEDS": { + self._random_temp_filename(): {"format": "xml"}, }, - 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } crawler = get_crawler(settings_dict=settings) self.assertRaises(NotConfigured, FeedExporter, crawler) @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): - for fmt in ('json', 'jsonlines', 'xml', 'csv'): + for fmt in ("json", "jsonlines", "xml", "csv"): settings = { - 'FEEDS': { - self._random_temp_filename() / fmt / self._file_mark: {'format': fmt}, + "FEEDS": { + self._random_temp_filename() + / fmt + / self._file_mark: {"format": fmt}, }, - 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } data = yield self.exported_no_data(settings) data = dict(data) - self.assertEqual(b'', data[fmt][0]) + self.assertEqual(b"", data[fmt][0]) @defer.inlineCallbacks def test_export_no_items_store_empty(self): formats = ( - ('json', b'[]'), - ('jsonlines', b''), - ('xml', b'\n'), - ('csv', b''), + ("json", b"[]"), + ("jsonlines", b""), + ("xml", b'\n'), + ("csv", b""), ) for fmt, expctd in formats: settings = { - 'FEEDS': { - self._random_temp_filename() / fmt / self._file_mark: {'format': fmt}, + "FEEDS": { + self._random_temp_filename() + / fmt + / self._file_mark: {"format": fmt}, }, - 'FEED_STORE_EMPTY': True, - 'FEED_EXPORT_INDENT': None, - 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, + "FEED_STORE_EMPTY": True, + "FEED_EXPORT_INDENT": None, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } data = yield self.exported_no_data(settings) data = dict(data) @@ -2210,47 +2291,60 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): - items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})] + items = [ + dict({"foo": "FOO", "bar": "BAR"}), + dict({"foo": "FOO1", "bar": "BAR1"}), + ] formats = { - 'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'), - '[\n{"bar": "BAR1"}\n]'.encode('utf-8')], - 'xml': [ - ( - '\n' - '\n \n FOO\n \n' - ).encode('latin-1'), - ( - '\n' - '\n \n FOO1\n \n' - ).encode('latin-1') + "json": [ + '[\n{"bar": "BAR"}\n]'.encode("utf-8"), + '[\n{"bar": "BAR1"}\n]'.encode("utf-8"), + ], + "xml": [ + ( + '\n' + "\n \n FOO\n \n" + ).encode("latin-1"), + ( + '\n' + "\n \n FOO1\n \n" + ).encode("latin-1"), + ], + "csv": [ + "foo,bar\r\nFOO,BAR\r\n".encode("utf-8"), + "foo,bar\r\nFOO1,BAR1\r\n".encode("utf-8"), ], - 'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'), - 'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')], } settings = { - 'FEEDS': { - self._random_temp_filename() / 'json' / self._file_mark: { - 'format': 'json', - 'indent': 0, - 'fields': ['bar'], - 'encoding': 'utf-8', + "FEEDS": { + self._random_temp_filename() + / "json" + / self._file_mark: { + "format": "json", + "indent": 0, + "fields": ["bar"], + "encoding": "utf-8", }, - self._random_temp_filename() / 'xml' / self._file_mark: { - 'format': 'xml', - 'indent': 2, - 'fields': ['foo'], - 'encoding': 'latin-1', + self._random_temp_filename() + / "xml" + / self._file_mark: { + "format": "xml", + "indent": 2, + "fields": ["foo"], + "encoding": "latin-1", }, - self._random_temp_filename() / 'csv' / self._file_mark: { - 'format': 'csv', - 'indent': None, - 'fields': ['foo', 'bar'], - 'encoding': 'utf-8', + self._random_temp_filename() + / "csv" + / self._file_mark: { + "format": "csv", + "indent": None, + "fields": ["foo", "bar"], + "encoding": "utf-8", }, }, - 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): @@ -2259,18 +2353,22 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): - items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})] + items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})] formats = { - 'json': ['[{"foo": "FOO"}]'.encode('utf-8'), - '[{"foo": "FOO1"}]'.encode('utf-8')], + "json": [ + '[{"foo": "FOO"}]'.encode("utf-8"), + '[{"foo": "FOO1"}]'.encode("utf-8"), + ], } settings = { - 'FEEDS': { - self._random_temp_filename() / 'json' / self._file_mark: { - 'format': 'json', - 'indent': None, - 'encoding': 'utf-8', - 'batch_item_count': 1, + "FEEDS": { + self._random_temp_filename() + / "json" + / self._file_mark: { + "format": "json", + "indent": None, + "encoding": "utf-8", + "batch_item_count": 1, }, }, } @@ -2279,7 +2377,9 @@ class BatchDeliveriesTest(FeedExportTestBase): for expected_batch, got_batch in zip(expected, data[fmt]): self.assertEqual(expected_batch, got_batch) - @pytest.mark.skipif(sys.platform == 'win32', reason='Odd behaviour on file creation/output') + @pytest.mark.skipif( + sys.platform == "win32", reason="Odd behaviour on file creation/output" + ) @defer.inlineCallbacks def test_batch_path_differ(self): """ @@ -2287,26 +2387,29 @@ class BatchDeliveriesTest(FeedExportTestBase): So %(batch_time)s replaced with the current date. """ items = [ - self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), - self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), - self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + self.MyItem({"foo": "bar3", "baz": "quux3"}), ] settings = { - 'FEEDS': { - self._random_temp_filename() / '%(batch_time)s': { - 'format': 'json', + "FEEDS": { + self._random_temp_filename() + / "%(batch_time)s": { + "format": "json", }, }, - 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } data = yield self.exported_data(items, settings) - self.assertEqual(len(items), len([_ for _ in data['json'] if _])) + self.assertEqual(len(items), len([_ for _ in data["json"] if _])) @defer.inlineCallbacks def test_stats_batch_file_success(self): settings = { "FEEDS": { - build_url(str(self._random_temp_filename() / "json" / self._file_mark)): { + build_url( + str(self._random_temp_filename() / "json" / self._file_mark) + ): { "format": "json", } }, @@ -2315,18 +2418,22 @@ class BatchDeliveriesTest(FeedExportTestBase): crawler = get_crawler(ItemSpider, settings) with MockServer() as mockserver: yield crawler.crawl(total=2, mockserver=mockserver) - self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()) - self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12) + self.assertIn( + "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() + ) + self.assertEqual( + crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12 + ) @defer.inlineCallbacks def test_s3_export(self): skip_if_no_boto() - bucket = 'mybucket' + bucket = "mybucket" items = [ - self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), - self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), - self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + self.MyItem({"foo": "bar3", "baz": "quux3"}), ] class CustomS3FeedStorage(S3FeedStorage): @@ -2335,33 +2442,34 @@ class BatchDeliveriesTest(FeedExportTestBase): def open(self, *args, **kwargs): from botocore.stub import ANY, Stubber + stub = Stubber(self.s3_client) stub.activate() CustomS3FeedStorage.stubs.append(stub) stub.add_response( - 'put_object', + "put_object", expected_params={ - 'Body': ANY, - 'Bucket': bucket, - 'Key': ANY, + "Body": ANY, + "Bucket": bucket, + "Key": ANY, }, service_response={}, ) return super().open(*args, **kwargs) - key = 'export.csv' - uri = f's3://{bucket}/{key}/%(batch_time)s.json' + key = "export.csv" + uri = f"s3://{bucket}/{key}/%(batch_time)s.json" batch_item_count = 1 settings = { - 'AWS_ACCESS_KEY_ID': 'access_key', - 'AWS_SECRET_ACCESS_KEY': 'secret_key', - 'FEED_EXPORT_BATCH_ITEM_COUNT': batch_item_count, - 'FEED_STORAGES': { - 's3': CustomS3FeedStorage, + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "FEED_EXPORT_BATCH_ITEM_COUNT": batch_item_count, + "FEED_STORAGES": { + "s3": CustomS3FeedStorage, }, - 'FEEDS': { + "FEEDS": { uri: { - 'format': 'json', + "format": "json", }, }, } @@ -2370,14 +2478,14 @@ class BatchDeliveriesTest(FeedExportTestBase): verifyObject(IFeedStorage, storage) class TestSpider(scrapy.Spider): - name = 'testspider' + name = "testspider" def parse(self, response): for item in items: yield item with MockServer() as server: - TestSpider.start_urls = [server.url('/')] + TestSpider.start_urls = [server.url("/")] crawler = get_crawler(TestSpider, settings) yield crawler.crawl() @@ -2387,11 +2495,10 @@ class BatchDeliveriesTest(FeedExportTestBase): class FeedExportInitTest(unittest.TestCase): - def test_unsupported_storage(self): settings = { - 'FEEDS': { - 'unsupported://uri': {}, + "FEEDS": { + "unsupported://uri": {}, }, } crawler = get_crawler(settings_dict=settings) @@ -2400,9 +2507,9 @@ class FeedExportInitTest(unittest.TestCase): def test_unsupported_format(self): settings = { - 'FEEDS': { - 'file://path': { - 'format': 'unsupported_format', + "FEEDS": { + "file://path": { + "format": "unsupported_format", }, }, } @@ -2412,7 +2519,6 @@ class FeedExportInitTest(unittest.TestCase): class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage): - def __init__(self, uri): super().__init__(uri) @@ -2424,25 +2530,26 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): def test_init(self): settings_dict = { - 'FEED_URI': 'file:///tmp/foobar', - 'FEED_STORAGES': { - 'file': StdoutFeedStorageWithoutFeedOptions - }, + "FEED_URI": "file:///tmp/foobar", + "FEED_STORAGES": {"file": StdoutFeedStorageWithoutFeedOptions}, } - with pytest.warns(ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + with pytest.warns( + ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", + ): crawler = get_crawler(settings_dict=settings_dict) feed_exporter = FeedExporter.from_crawler(crawler) spider = scrapy.Spider("default") - with pytest.warns(ScrapyDeprecationWarning, - match="StdoutFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument."): + with pytest.warns( + ScrapyDeprecationWarning, + match="StdoutFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument.", + ): feed_exporter.open_spider(spider) class FileFeedStorageWithoutFeedOptions(FileFeedStorage): - def __init__(self, uri): super().__init__(uri) @@ -2457,31 +2564,31 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): def test_init(self): with tempfile.NamedTemporaryFile() as temp: settings_dict = { - 'FEED_URI': f'file:///{temp.name}', - 'FEED_STORAGES': { - 'file': FileFeedStorageWithoutFeedOptions - }, + "FEED_URI": f"file:///{temp.name}", + "FEED_STORAGES": {"file": FileFeedStorageWithoutFeedOptions}, } - with pytest.warns(ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + with pytest.warns( + ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", + ): crawler = get_crawler(settings_dict=settings_dict) feed_exporter = FeedExporter.from_crawler(crawler) spider = scrapy.Spider("default") - with pytest.warns(ScrapyDeprecationWarning, - match="FileFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument."): + with pytest.warns( + ScrapyDeprecationWarning, + match="FileFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument.", + ): feed_exporter.open_spider(spider) class S3FeedStorageWithoutFeedOptions(S3FeedStorage): - def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs): super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs) class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): - @classmethod def from_crawler(cls, crawler, uri): return super().from_crawler(crawler, uri) @@ -2496,53 +2603,55 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): def test_init(self): settings_dict = { - 'FEED_URI': 'file:///tmp/foobar', - 'FEED_STORAGES': { - 'file': S3FeedStorageWithoutFeedOptions - }, + "FEED_URI": "file:///tmp/foobar", + "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptions}, } - with pytest.warns(ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + with pytest.warns( + ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", + ): crawler = get_crawler(settings_dict=settings_dict) feed_exporter = FeedExporter.from_crawler(crawler) spider = scrapy.Spider("default") spider.crawler = crawler - with pytest.warns(ScrapyDeprecationWarning, - match="S3FeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument."): + with pytest.warns( + ScrapyDeprecationWarning, + match="S3FeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument.", + ): feed_exporter.open_spider(spider) def test_from_crawler(self): settings_dict = { - 'FEED_URI': 'file:///tmp/foobar', - 'FEED_STORAGES': { - 'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler - }, + "FEED_URI": "file:///tmp/foobar", + "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptionsWithFromCrawler}, } - with pytest.warns(ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + with pytest.warns( + ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", + ): crawler = get_crawler(settings_dict=settings_dict) feed_exporter = FeedExporter.from_crawler(crawler) spider = scrapy.Spider("default") spider.crawler = crawler - with pytest.warns(ScrapyDeprecationWarning, - match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " - "the 'feed_options' keyword argument."): + with pytest.warns( + ScrapyDeprecationWarning, + match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " + "the 'feed_options' keyword argument.", + ): feed_exporter.open_spider(spider) class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage): - def __init__(self, uri, use_active_mode=False): super().__init__(uri) class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage): - @classmethod def from_crawler(cls, crawler, uri): return super().from_crawler(crawler, uri) @@ -2557,42 +2666,46 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): def test_init(self): settings_dict = { - 'FEED_URI': 'file:///tmp/foobar', - 'FEED_STORAGES': { - 'file': FTPFeedStorageWithoutFeedOptions - }, + "FEED_URI": "file:///tmp/foobar", + "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptions}, } - with pytest.warns(ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + with pytest.warns( + ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", + ): crawler = get_crawler(settings_dict=settings_dict) feed_exporter = FeedExporter.from_crawler(crawler) spider = scrapy.Spider("default") spider.crawler = crawler - with pytest.warns(ScrapyDeprecationWarning, - match="FTPFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument."): + with pytest.warns( + ScrapyDeprecationWarning, + match="FTPFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument.", + ): feed_exporter.open_spider(spider) def test_from_crawler(self): settings_dict = { - 'FEED_URI': 'file:///tmp/foobar', - 'FEED_STORAGES': { - 'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler - }, + "FEED_URI": "file:///tmp/foobar", + "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptionsWithFromCrawler}, } - with pytest.warns(ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + with pytest.warns( + ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", + ): crawler = get_crawler(settings_dict=settings_dict) feed_exporter = FeedExporter.from_crawler(crawler) spider = scrapy.Spider("default") spider.crawler = crawler - with pytest.warns(ScrapyDeprecationWarning, - match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " - "the 'feed_options' keyword argument."): + with pytest.warns( + ScrapyDeprecationWarning, + match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " + "the 'feed_options' keyword argument.", + ): feed_exporter.open_spider(spider) @@ -2601,13 +2714,15 @@ class URIParamsTest: spider_name = "uri_params_spider" deprecated_options = False - def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + def build_settings(self, uri="file:///tmp/foobar", uri_params=None): raise NotImplementedError def _crawler_feed_exporter(self, settings): if self.deprecated_options: - with pytest.warns(ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + with pytest.warns( + ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", + ): crawler = get_crawler(settings_dict=settings) feed_exporter = FeedExporter.from_crawler(crawler) else: @@ -2617,7 +2732,7 @@ class URIParamsTest: def test_default(self): settings = self.build_settings( - uri='file:///tmp/%(name)s', + uri="file:///tmp/%(name)s", ) crawler, feed_exporter = self._crawler_feed_exporter(settings) spider = scrapy.Spider(self.spider_name) @@ -2627,38 +2742,33 @@ class URIParamsTest: warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - self.assertEqual( - feed_exporter.slots[0].uri, - f'file:///tmp/{self.spider_name}' - ) + self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") def test_none(self): def uri_params(params, spider): pass settings = self.build_settings( - uri='file:///tmp/%(name)s', + uri="file:///tmp/%(name)s", uri_params=uri_params, ) crawler, feed_exporter = self._crawler_feed_exporter(settings) spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with pytest.warns(ScrapyDeprecationWarning, - match="Modifying the params dictionary in-place"): + with pytest.warns( + ScrapyDeprecationWarning, match="Modifying the params dictionary in-place" + ): feed_exporter.open_spider(spider) - self.assertEqual( - feed_exporter.slots[0].uri, - f'file:///tmp/{self.spider_name}' - ) + self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") def test_empty_dict(self): def uri_params(params, spider): return {} settings = self.build_settings( - uri='file:///tmp/%(name)s', + uri="file:///tmp/%(name)s", uri_params=uri_params, ) crawler, feed_exporter = self._crawler_feed_exporter(settings) @@ -2675,7 +2785,7 @@ class URIParamsTest: return params settings = self.build_settings( - uri='file:///tmp/%(name)s', + uri="file:///tmp/%(name)s", uri_params=uri_params, ) crawler, feed_exporter = self._crawler_feed_exporter(settings) @@ -2685,17 +2795,14 @@ class URIParamsTest: warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - self.assertEqual( - feed_exporter.slots[0].uri, - f'file:///tmp/{self.spider_name}' - ) + self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") def test_custom_param(self): def uri_params(params, spider): - return {**params, 'foo': self.spider_name} + return {**params, "foo": self.spider_name} settings = self.build_settings( - uri='file:///tmp/%(foo)s', + uri="file:///tmp/%(foo)s", uri_params=uri_params, ) crawler, feed_exporter = self._crawler_feed_exporter(settings) @@ -2705,21 +2812,18 @@ class URIParamsTest: warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - self.assertEqual( - feed_exporter.slots[0].uri, - f'file:///tmp/{self.spider_name}' - ) + self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") class URIParamsSettingTest(URIParamsTest, unittest.TestCase): deprecated_options = True - def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + def build_settings(self, uri="file:///tmp/foobar", uri_params=None): extra_settings = {} if uri_params: - extra_settings['FEED_URI_PARAMS'] = uri_params + extra_settings["FEED_URI_PARAMS"] = uri_params return { - 'FEED_URI': uri, + "FEED_URI": uri, **extra_settings, } @@ -2727,14 +2831,14 @@ class URIParamsSettingTest(URIParamsTest, unittest.TestCase): class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase): deprecated_options = False - def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + def build_settings(self, uri="file:///tmp/foobar", uri_params=None): options = { - 'format': 'jl', + "format": "jl", } if uri_params: - options['uri_params'] = uri_params + options["uri_params"] = uri_params return { - 'FEEDS': { + "FEEDS": { uri: options, }, } diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index e461bcf2c..88345d2bc 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -9,7 +9,12 @@ from unittest import mock, skipIf from urllib.parse import urlencode from twisted.internet import reactor -from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks +from twisted.internet.defer import ( + CancelledError, + Deferred, + DeferredList, + inlineCallbacks, +) from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.error import TimeoutError from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate @@ -27,22 +32,19 @@ from tests.mockserver import ssl_context_factory, LeafResource, Status def generate_random_string(size): - return ''.join(random.choices( - string.ascii_uppercase + string.digits, - k=size - )) + return "".join(random.choices(string.ascii_uppercase + string.digits, k=size)) def make_html_body(val): - response = f''' + response = f"""

Hello from HTTP2

{val}

-''' - return bytes(response, 'utf-8') +""" + return bytes(response, "utf-8") class DummySpider(Spider): - name = 'dummy' + name = "dummy" start_urls: list = [] def parse(self, response): @@ -51,33 +53,33 @@ class DummySpider(Spider): class Data: SMALL_SIZE = 1024 # 1 KB - LARGE_SIZE = 1024 ** 2 # 1 MB + LARGE_SIZE = 1024**2 # 1 MB STR_SMALL = generate_random_string(SMALL_SIZE) STR_LARGE = generate_random_string(LARGE_SIZE) EXTRA_SMALL = generate_random_string(1024 * 15) - EXTRA_LARGE = generate_random_string((1024 ** 2) * 15) + EXTRA_LARGE = generate_random_string((1024**2) * 15) HTML_SMALL = make_html_body(STR_SMALL) HTML_LARGE = make_html_body(STR_LARGE) - JSON_SMALL = {'data': STR_SMALL} - JSON_LARGE = {'data': STR_LARGE} + JSON_SMALL = {"data": STR_SMALL} + JSON_LARGE = {"data": STR_LARGE} - DATALOSS = b'Dataloss Content' - NO_CONTENT_LENGTH = b'This response do not have any content-length header' + DATALOSS = b"Dataloss Content" + NO_CONTENT_LENGTH = b"This response do not have any content-length header" class GetDataHtmlSmall(LeafResource): def render_GET(self, request: TxRequest): - request.setHeader('Content-Type', 'text/html; charset=UTF-8') + request.setHeader("Content-Type", "text/html; charset=UTF-8") return Data.HTML_SMALL class GetDataHtmlLarge(LeafResource): def render_GET(self, request: TxRequest): - request.setHeader('Content-Type', 'text/html; charset=UTF-8') + request.setHeader("Content-Type", "text/html; charset=UTF-8") return Data.HTML_LARGE @@ -85,16 +87,16 @@ class PostDataJsonMixin: @staticmethod def make_response(request: TxRequest, extra_data: str): response = { - 'request-headers': {}, - 'request-body': json.loads(request.content.read()), - 'extra-data': extra_data + "request-headers": {}, + "request-body": json.loads(request.content.read()), + "extra-data": extra_data, } for k, v in request.requestHeaders.getAllRawHeaders(): - response['request-headers'][str(k, 'utf-8')] = str(v[0], 'utf-8') + response["request-headers"][str(k, "utf-8")] = str(v[0], "utf-8") - response_bytes = bytes(json.dumps(response), 'utf-8') - request.setHeader('Content-Type', 'application/json; charset=UTF-8') - request.setHeader('Content-Encoding', 'UTF-8') + response_bytes = bytes(json.dumps(response), "utf-8") + request.setHeader("Content-Type", "application/json; charset=UTF-8") + request.setHeader("Content-Encoding", "UTF-8") return response_bytes @@ -109,7 +111,6 @@ class PostDataJsonLarge(LeafResource, PostDataJsonMixin): class Dataloss(LeafResource): - def render_GET(self, request: TxRequest): request.setHeader(b"Content-Length", b"1024") self.deferRequest(request, 0, self._delayed_render, request) @@ -123,7 +124,7 @@ class Dataloss(LeafResource): class NoContentLengthHeader(LeafResource): def render_GET(self, request: TxRequest): - request.requestHeaders.removeHeader('Content-Length') + request.requestHeaders.removeHeader("Content-Length") self.deferRequest(request, 0, self._delayed_render, request) return NOT_DONE_YET @@ -140,57 +141,61 @@ class TimeoutResponse(LeafResource): class QueryParams(LeafResource): def render_GET(self, request: TxRequest): - request.setHeader('Content-Type', 'application/json; charset=UTF-8') - request.setHeader('Content-Encoding', 'UTF-8') + request.setHeader("Content-Type", "application/json; charset=UTF-8") + request.setHeader("Content-Encoding", "UTF-8") query_params = {} for k, v in request.args.items(): - query_params[str(k, 'utf-8')] = str(v[0], 'utf-8') + query_params[str(k, "utf-8")] = str(v[0], "utf-8") - return bytes(json.dumps(query_params), 'utf-8') + return bytes(json.dumps(query_params), "utf-8") class RequestHeaders(LeafResource): """Sends all the headers received as a response""" def render_GET(self, request: TxRequest): - request.setHeader('Content-Type', 'application/json; charset=UTF-8') - request.setHeader('Content-Encoding', 'UTF-8') + request.setHeader("Content-Type", "application/json; charset=UTF-8") + request.setHeader("Content-Encoding", "UTF-8") headers = {} for k, v in request.requestHeaders.getAllRawHeaders(): - headers[str(k, 'utf-8')] = str(v[0], 'utf-8') + headers[str(k, "utf-8")] = str(v[0], "utf-8") - return bytes(json.dumps(headers), 'utf-8') + return bytes(json.dumps(headers), "utf-8") -def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate: - pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text(encoding="utf-8") +def get_client_certificate( + key_file: Path, certificate_file: Path +) -> PrivateCertificate: + pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text( + encoding="utf-8" + ) return PrivateCertificate.loadPEM(pem) @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2ClientProtocolTestCase(TestCase): - scheme = 'https' - key_file = Path(__file__).parent / 'keys' / 'localhost.key' - certificate_file = Path(__file__).parent / 'keys' / 'localhost.crt' + scheme = "https" + key_file = Path(__file__).parent / "keys" / "localhost.key" + certificate_file = Path(__file__).parent / "keys" / "localhost.crt" def _init_resource(self): self.temp_directory = self.mktemp() Path(self.temp_directory).mkdir() r = File(self.temp_directory) - r.putChild(b'get-data-html-small', GetDataHtmlSmall()) - r.putChild(b'get-data-html-large', GetDataHtmlLarge()) + r.putChild(b"get-data-html-small", GetDataHtmlSmall()) + r.putChild(b"get-data-html-large", GetDataHtmlLarge()) - r.putChild(b'post-data-json-small', PostDataJsonSmall()) - r.putChild(b'post-data-json-large', PostDataJsonLarge()) + r.putChild(b"post-data-json-small", PostDataJsonSmall()) + r.putChild(b"post-data-json-large", PostDataJsonLarge()) - r.putChild(b'dataloss', Dataloss()) - r.putChild(b'no-content-length-header', NoContentLengthHeader()) - r.putChild(b'status', Status()) - r.putChild(b'query-params', QueryParams()) - r.putChild(b'timeout', TimeoutResponse()) - r.putChild(b'request-headers', RequestHeaders()) + r.putChild(b"dataloss", Dataloss()) + r.putChild(b"no-content-length-header", NoContentLengthHeader()) + r.putChild(b"status", Status()) + r.putChild(b"query-params", QueryParams()) + r.putChild(b"timeout", TimeoutResponse()) + r.putChild(b"request-headers", RequestHeaders()) return r @inlineCallbacks @@ -200,26 +205,35 @@ class Https2ClientProtocolTestCase(TestCase): self.site = Site(root, timeout=None) # Start server for testing - self.hostname = 'localhost' - context_factory = ssl_context_factory(str(self.key_file), str(self.certificate_file)) + self.hostname = "localhost" + context_factory = ssl_context_factory( + str(self.key_file), str(self.certificate_file) + ) - server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) + server_endpoint = SSL4ServerEndpoint( + reactor, 0, context_factory, interface=self.hostname + ) self.server = yield server_endpoint.listen(self.site) self.port_number = self.server.getHost().port # Connect H2 client with server - self.client_certificate = get_client_certificate(self.key_file, self.certificate_file) + self.client_certificate = get_client_certificate( + self.key_file, self.certificate_file + ) client_options = optionsForClientTLS( hostname=self.hostname, trustRoot=self.client_certificate, - acceptableProtocols=[b'h2'] + acceptableProtocols=[b"h2"], ) - uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) + uri = URI.fromBytes(bytes(self.get_url("/"), "utf-8")) self.conn_closed_deferred = Deferred() from scrapy.core.http2.protocol import H2ClientFactory + h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) - client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) + client_endpoint = SSL4ClientEndpoint( + reactor, self.hostname, self.port_number, client_options + ) self.client = yield client_endpoint.connect(h2_client_factory) @inlineCallbacks @@ -236,8 +250,8 @@ class Https2ClientProtocolTestCase(TestCase): :param path: Should have / at the starting compulsorily if not empty :return: Complete url """ - assert len(path) > 0 and (path[0] == '/' or path[0] == '&') - return f'{self.scheme}://{self.hostname}:{self.port_number}{path}' + assert len(path) > 0 and (path[0] == "/" or path[0] == "&") + return f"{self.scheme}://{self.hostname}:{self.port_number}{path}" def make_request(self, request: Request) -> Deferred: return self.client.request(request, DummySpider()) @@ -251,18 +265,13 @@ class Https2ClientProtocolTestCase(TestCase): return DeferredList(d_list, fireOnOneErrback=True) - def _check_GET( - self, - request: Request, - expected_body, - expected_status - ): + def _check_GET(self, request: Request, expected_body, expected_status): def check_response(response: Response): self.assertEqual(response.status, expected_status) self.assertEqual(response.body, expected_body) self.assertEqual(response.request, request) - content_length = int(response.headers.get('Content-Length')) + content_length = int(response.headers.get("Content-Length")) self.assertEqual(len(response.body), content_length) d = self.make_request(request) @@ -271,11 +280,11 @@ class Https2ClientProtocolTestCase(TestCase): return d def test_GET_small_body(self): - request = Request(self.get_url('/get-data-html-small')) + request = Request(self.get_url("/get-data-html-small")) return self._check_GET(request, Data.HTML_SMALL, 200) def test_GET_large_body(self): - request = Request(self.get_url('/get-data-html-large')) + request = Request(self.get_url("/get-data-html-large")) return self._check_GET(request, Data.HTML_LARGE, 200) def _check_GET_x10(self, *args, **kwargs): @@ -286,16 +295,12 @@ class Https2ClientProtocolTestCase(TestCase): def test_GET_small_body_x10(self): return self._check_GET_x10( - Request(self.get_url('/get-data-html-small')), - Data.HTML_SMALL, - 200 + Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200 ) def test_GET_large_body_x10(self): return self._check_GET_x10( - Request(self.get_url('/get-data-html-large')), - Data.HTML_LARGE, - 200 + Request(self.get_url("/get-data-html-large")), Data.HTML_LARGE, 200 ) def _check_POST_json( @@ -303,7 +308,7 @@ class Https2ClientProtocolTestCase(TestCase): request: Request, expected_request_body, expected_extra_data, - expected_status: int + expected_status: int, ): d = self.make_request(request) @@ -311,50 +316,48 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, expected_status) self.assertEqual(response.request, request) - content_length = int(response.headers.get('Content-Length')) + content_length = int(response.headers.get("Content-Length")) self.assertEqual(len(response.body), content_length) # Parse the body - content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + content_encoding = str(response.headers[b"Content-Encoding"], "utf-8") body = json.loads(str(response.body, content_encoding)) - self.assertIn('request-body', body) - self.assertIn('extra-data', body) - self.assertIn('request-headers', body) + self.assertIn("request-body", body) + self.assertIn("extra-data", body) + self.assertIn("request-headers", body) - request_body = body['request-body'] + request_body = body["request-body"] self.assertEqual(request_body, expected_request_body) - extra_data = body['extra-data'] + extra_data = body["extra-data"] self.assertEqual(extra_data, expected_extra_data) # Check if headers were sent successfully - request_headers = body['request-headers'] + request_headers = body["request-headers"] for k, v in request.headers.items(): - k_str = str(k, 'utf-8') + k_str = str(k, "utf-8") self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], str(v[0], 'utf-8')) + self.assertEqual(request_headers[k_str], str(v[0], "utf-8")) d.addCallback(assert_response) d.addErrback(self.fail) return d def test_POST_small_json(self): - request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) - return self._check_POST_json( - request, - Data.JSON_SMALL, - Data.EXTRA_SMALL, - 200 + request = JsonRequest( + url=self.get_url("/post-data-json-small"), + method="POST", + data=Data.JSON_SMALL, ) + return self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200) def test_POST_large_json(self): - request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) - return self._check_POST_json( - request, - Data.JSON_LARGE, - Data.EXTRA_LARGE, - 200 + request = JsonRequest( + url=self.get_url("/post-data-json-large"), + method="POST", + data=Data.JSON_LARGE, ) + return self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200) def _check_POST_json_x10(self, *args, **kwargs): def get_deferred(): @@ -363,32 +366,36 @@ class Https2ClientProtocolTestCase(TestCase): return self._check_repeat(get_deferred, 10) def test_POST_small_json_x10(self): - request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) + request = JsonRequest( + url=self.get_url("/post-data-json-small"), + method="POST", + data=Data.JSON_SMALL, + ) return self._check_POST_json_x10( - request, - Data.JSON_SMALL, - Data.EXTRA_SMALL, - 200 + request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200 ) def test_POST_large_json_x10(self): - request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) + request = JsonRequest( + url=self.get_url("/post-data-json-large"), + method="POST", + data=Data.JSON_LARGE, + ) return self._check_POST_json_x10( - request, - Data.JSON_LARGE, - Data.EXTRA_LARGE, - 200 + request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200 ) @inlineCallbacks def test_invalid_negotiated_protocol(self): - with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"): - request = Request(url=self.get_url('/status?n=200')) + with mock.patch( + "scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2" + ): + request = Request(url=self.get_url("/status?n=200")) with self.assertRaises(ResponseFailed): yield self.make_request(request) def test_cancel_request(self): - request = Request(url=self.get_url('/get-data-html-large')) + request = Request(url=self.get_url("/get-data-html-large")) def assert_response(response: Response): self.assertEqual(response.status, 499) @@ -402,13 +409,15 @@ class Https2ClientProtocolTestCase(TestCase): return d def test_download_maxsize_exceeded(self): - request = Request(url=self.get_url('/get-data-html-large'), meta={'download_maxsize': 1000}) + request = Request( + url=self.get_url("/get-data-html-large"), meta={"download_maxsize": 1000} + ) def assert_cancelled_error(failure): self.assertIsInstance(failure.value, CancelledError) error_pattern = re.compile( - rf'Cancelling download of {request.url}: received response ' - rf'size \(\d*\) larger than download max size \(1000\)' + rf"Cancelling download of {request.url}: received response " + rf"size \(\d*\) larger than download max size \(1000\)" ) self.assertEqual(len(re.findall(error_pattern, str(failure.value))), 1) @@ -420,15 +429,18 @@ class Https2ClientProtocolTestCase(TestCase): def test_received_dataloss_response(self): """In case when value of Header Content-Length != len(Received Data) ProtocolError is raised""" - request = Request(url=self.get_url('/dataloss')) + request = Request(url=self.get_url("/dataloss")) def assert_failure(failure: Failure): self.assertTrue(len(failure.value.reasons) > 0) from h2.exceptions import InvalidBodyLengthError - self.assertTrue(any( - isinstance(error, InvalidBodyLengthError) - for error in failure.value.reasons - )) + + self.assertTrue( + any( + isinstance(error, InvalidBodyLengthError) + for error in failure.value.reasons + ) + ) d = self.make_request(request) d.addCallback(self.fail) @@ -436,13 +448,13 @@ class Https2ClientProtocolTestCase(TestCase): return d def test_missing_content_length_header(self): - request = Request(url=self.get_url('/no-content-length-header')) + request = Request(url=self.get_url("/no-content-length-header")) def assert_content_length(response: Response): self.assertEqual(response.status, 200) self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) self.assertEqual(response.request, request) - self.assertNotIn('Content-Length', response.headers) + self.assertNotIn("Content-Length", response.headers) d = self.make_request(request) d.addCallback(assert_content_length) @@ -450,40 +462,39 @@ class Https2ClientProtocolTestCase(TestCase): return d @inlineCallbacks - def _check_log_warnsize( - self, - request, - warn_pattern, - expected_body - ): - with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm: + def _check_log_warnsize(self, request, warn_pattern, expected_body): + with self.assertLogs("scrapy.core.http2.stream", level="WARNING") as cm: response = yield self.make_request(request) self.assertEqual(response.status, 200) self.assertEqual(response.request, request) self.assertEqual(response.body, expected_body) # Check the warning is raised only once for this request - self.assertEqual(sum( - len(re.findall(warn_pattern, log)) - for log in cm.output - ), 1) + self.assertEqual( + sum(len(re.findall(warn_pattern, log)) for log in cm.output), 1 + ) @inlineCallbacks def test_log_expected_warnsize(self): - request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000}) + request = Request( + url=self.get_url("/get-data-html-large"), meta={"download_warnsize": 1000} + ) warn_pattern = re.compile( - rf'Expected response size \(\d*\) larger than ' - rf'download warn size \(1000\) in request {request}' + rf"Expected response size \(\d*\) larger than " + rf"download warn size \(1000\) in request {request}" ) yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) @inlineCallbacks def test_log_received_warnsize(self): - request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10}) + request = Request( + url=self.get_url("/no-content-length-header"), + meta={"download_warnsize": 10}, + ) warn_pattern = re.compile( - rf'Received more \(\d*\) bytes than download ' - rf'warn size \(10\) in request {request}' + rf"Received more \(\d*\) bytes than download " + rf"warn size \(10\) in request {request}" ) yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) @@ -495,9 +506,7 @@ class Https2ClientProtocolTestCase(TestCase): def get_deferred(): return self._check_GET( - Request(self.get_url('/get-data-html-small')), - Data.HTML_SMALL, - 200 + Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200 ) return self._check_repeat(get_deferred, 500) @@ -511,20 +520,20 @@ class Https2ClientProtocolTestCase(TestCase): def assert_inactive_stream(failure): self.assertIsNotNone(failure.check(ResponseFailed)) from scrapy.core.http2.stream import InactiveStreamClosed - self.assertTrue(any( - isinstance(e, InactiveStreamClosed) - for e in failure.value.reasons - )) + + self.assertTrue( + any(isinstance(e, InactiveStreamClosed) for e in failure.value.reasons) + ) # Send 100 request (we do not check the result) for _ in range(100): - d = self.make_request(Request(self.get_url('/get-data-html-small'))) + d = self.make_request(Request(self.get_url("/get-data-html-small"))) d.addBoth(lambda _: None) d_list.append(d) # Now send 10 extra request and save the response deferred in a list for _ in range(10): - d = self.make_request(Request(self.get_url('/get-data-html-small'))) + d = self.make_request(Request(self.get_url("/get-data-html-small"))) d.addCallback(self.fail) d.addErrback(assert_inactive_stream) d_list.append(d) @@ -537,19 +546,19 @@ class Https2ClientProtocolTestCase(TestCase): def test_invalid_request_type(self): with self.assertRaises(TypeError): - self.make_request('https://InvalidDataTypePassed.com') + self.make_request("https://InvalidDataTypePassed.com") def test_query_parameters(self): params = { - 'a': generate_random_string(20), - 'b': generate_random_string(20), - 'c': generate_random_string(20), - 'd': generate_random_string(20) + "a": generate_random_string(20), + "b": generate_random_string(20), + "c": generate_random_string(20), + "d": generate_random_string(20), } - request = Request(self.get_url(f'/query-params?{urlencode(params)}')) + request = Request(self.get_url(f"/query-params?{urlencode(params)}")) def assert_query_params(response: Response): - content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + content_encoding = str(response.headers[b"Content-Encoding"], "utf-8") data = json.loads(str(response.body, content_encoding)) self.assertEqual(data, params) @@ -565,7 +574,7 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [] for status in [200, 404]: - request = Request(self.get_url(f'/status?n={status}')) + request = Request(self.get_url(f"/status?n={status}")) d = self.make_request(request) d.addCallback(assert_response_status, status) d.addErrback(self.fail) @@ -574,17 +583,23 @@ class Https2ClientProtocolTestCase(TestCase): return DeferredList(d_list, fireOnOneErrback=True) def test_response_has_correct_certificate_ip_address(self): - request = Request(self.get_url('/status?n=200')) + request = Request(self.get_url("/status?n=200")) def assert_metadata(response: Response): self.assertEqual(response.request, request) self.assertIsInstance(response.certificate, Certificate) self.assertIsNotNone(response.certificate.original) - self.assertEqual(response.certificate.getIssuer(), self.client_certificate.getIssuer()) - self.assertTrue(response.certificate.getPublicKey().matches(self.client_certificate.getPublicKey())) + self.assertEqual( + response.certificate.getIssuer(), self.client_certificate.getIssuer() + ) + self.assertTrue( + response.certificate.getPublicKey().matches( + self.client_certificate.getPublicKey() + ) + ) self.assertIsInstance(response.ip_address, IPv4Address) - self.assertEqual(str(response.ip_address), '127.0.0.1') + self.assertEqual(str(response.ip_address), "127.0.0.1") d = self.make_request(request) d.addCallback(assert_metadata) @@ -597,10 +612,11 @@ class Https2ClientProtocolTestCase(TestCase): def assert_invalid_hostname(failure: Failure): from scrapy.core.http2.stream import InvalidHostname + self.assertIsNotNone(failure.check(InvalidHostname)) error_msg = str(failure.value) - self.assertIn('localhost', error_msg) - self.assertIn('127.0.0.1', error_msg) + self.assertIn("localhost", error_msg) + self.assertIn("127.0.0.1", error_msg) self.assertIn(str(request), error_msg) d = self.make_request(request) @@ -609,24 +625,24 @@ class Https2ClientProtocolTestCase(TestCase): return d def test_invalid_hostname(self): - return self._check_invalid_netloc('https://notlocalhost.notlocalhostdomain') + return self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain") def test_invalid_host_port(self): port = self.port_number + 1 - return self._check_invalid_netloc(f'https://127.0.0.1:{port}') + return self._check_invalid_netloc(f"https://127.0.0.1:{port}") def test_connection_stays_with_invalid_requests(self): d_list = [ self.test_invalid_hostname(), self.test_invalid_host_port(), self.test_GET_small_body(), - self.test_POST_small_json() + self.test_POST_small_json(), ] return DeferredList(d_list, fireOnOneErrback=True) def test_connection_timeout(self): - request = Request(self.get_url('/timeout')) + request = Request(self.get_url("/timeout")) d = self.make_request(request) # Update the timer to 1s to test connection timeout @@ -635,8 +651,12 @@ class Https2ClientProtocolTestCase(TestCase): def assert_timeout_error(failure: Failure): for err in failure.value.reasons: from scrapy.core.http2.protocol import H2ClientProtocol + if isinstance(err, TimeoutError): - self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err)) + self.assertIn( + f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", + str(err), + ) break else: self.fail() @@ -646,20 +666,20 @@ class Https2ClientProtocolTestCase(TestCase): return d def test_request_headers_received(self): - request = Request(self.get_url('/request-headers'), headers={ - 'header-1': 'header value 1', - 'header-2': 'header value 2' - }) + request = Request( + self.get_url("/request-headers"), + headers={"header-1": "header value 1", "header-2": "header value 2"}, + ) d = self.make_request(request) def assert_request_headers(response: Response): self.assertEqual(response.status, 200) self.assertEqual(response.request, request) - response_headers = json.loads(str(response.body, 'utf-8')) + response_headers = json.loads(str(response.body, "utf-8")) self.assertIsInstance(response_headers, dict) for k, v in request.headers.items(): - k, v = str(k, 'utf-8'), str(v[0], 'utf-8') + k, v = str(k, "utf-8"), str(v[0], "utf-8") self.assertIn(k, response_headers) self.assertEqual(v, response_headers[k]) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 08420332c..ea42cadcd 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -6,10 +6,10 @@ from scrapy.http.cookies import WrappedRequest, WrappedResponse class WrappedRequestTest(TestCase): - def setUp(self): - self.request = Request("http://www.example.com/page.html", - headers={"Content-Type": "text/html"}) + self.request = Request( + "http://www.example.com/page.html", headers={"Content-Type": "text/html"} + ) self.wrapped = WrappedRequest(self.request) def test_get_full_url(self): @@ -29,35 +29,34 @@ class WrappedRequestTest(TestCase): self.assertFalse(self.wrapped.unverifiable) def test_is_unverifiable2(self): - self.request.meta['is_unverifiable'] = True + self.request.meta["is_unverifiable"] = True self.assertTrue(self.wrapped.is_unverifiable()) self.assertTrue(self.wrapped.unverifiable) def test_get_origin_req_host(self): - self.assertEqual(self.wrapped.origin_req_host, 'www.example.com') + self.assertEqual(self.wrapped.origin_req_host, "www.example.com") def test_has_header(self): - self.assertTrue(self.wrapped.has_header('content-type')) - self.assertFalse(self.wrapped.has_header('xxxxx')) + self.assertTrue(self.wrapped.has_header("content-type")) + self.assertFalse(self.wrapped.has_header("xxxxx")) def test_get_header(self): - self.assertEqual(self.wrapped.get_header('content-type'), 'text/html') - self.assertEqual(self.wrapped.get_header('xxxxx', 'def'), 'def') + self.assertEqual(self.wrapped.get_header("content-type"), "text/html") + self.assertEqual(self.wrapped.get_header("xxxxx", "def"), "def") def test_header_items(self): - self.assertEqual(self.wrapped.header_items(), - [('Content-Type', ['text/html'])]) + self.assertEqual(self.wrapped.header_items(), [("Content-Type", ["text/html"])]) def test_add_unredirected_header(self): - self.wrapped.add_unredirected_header('hello', 'world') - self.assertEqual(self.request.headers['hello'], b'world') + self.wrapped.add_unredirected_header("hello", "world") + self.assertEqual(self.request.headers["hello"], b"world") class WrappedResponseTest(TestCase): - def setUp(self): - self.response = Response("http://www.example.com/page.html", - headers={"Content-TYpe": "text/html"}) + self.response = Response( + "http://www.example.com/page.html", headers={"Content-TYpe": "text/html"} + ) self.wrapped = WrappedResponse(self.response) def test_info(self): @@ -65,4 +64,4 @@ class WrappedResponseTest(TestCase): def test_get_all(self): # get_all result must be native string - self.assertEqual(self.wrapped.get_all('content-type'), ['text/html']) + self.assertEqual(self.wrapped.get_all("content-type"), ["text/html"]) diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 1ca936247..566bb302d 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -5,38 +5,39 @@ from scrapy.http import Headers class HeadersTest(unittest.TestCase): - def assertSortedEqual(self, first, second, msg=None): return self.assertEqual(sorted(first), sorted(second), msg) def test_basics(self): - h = Headers({'Content-Type': 'text/html', 'Content-Length': 1234}) - assert h['Content-Type'] - assert h['Content-Length'] + h = Headers({"Content-Type": "text/html", "Content-Length": 1234}) + assert h["Content-Type"] + assert h["Content-Length"] - self.assertRaises(KeyError, h.__getitem__, 'Accept') - self.assertEqual(h.get('Accept'), None) - self.assertEqual(h.getlist('Accept'), []) + self.assertRaises(KeyError, h.__getitem__, "Accept") + self.assertEqual(h.get("Accept"), None) + self.assertEqual(h.getlist("Accept"), []) - self.assertEqual(h.get('Accept', '*/*'), b'*/*') - self.assertEqual(h.getlist('Accept', '*/*'), [b'*/*']) - self.assertEqual(h.getlist('Accept', ['text/html', 'images/jpeg']), - [b'text/html', b'images/jpeg']) + self.assertEqual(h.get("Accept", "*/*"), b"*/*") + self.assertEqual(h.getlist("Accept", "*/*"), [b"*/*"]) + self.assertEqual( + h.getlist("Accept", ["text/html", "images/jpeg"]), + [b"text/html", b"images/jpeg"], + ) def test_single_value(self): h = Headers() - h['Content-Type'] = 'text/html' - self.assertEqual(h['Content-Type'], b'text/html') - self.assertEqual(h.get('Content-Type'), b'text/html') - self.assertEqual(h.getlist('Content-Type'), [b'text/html']) + h["Content-Type"] = "text/html" + self.assertEqual(h["Content-Type"], b"text/html") + self.assertEqual(h.get("Content-Type"), b"text/html") + self.assertEqual(h.getlist("Content-Type"), [b"text/html"]) def test_multivalue(self): h = Headers() - h['X-Forwarded-For'] = hlist = ['ip1', 'ip2'] - self.assertEqual(h['X-Forwarded-For'], b'ip2') - self.assertEqual(h.get('X-Forwarded-For'), b'ip2') - self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2']) - assert h.getlist('X-Forwarded-For') is not hlist + h["X-Forwarded-For"] = hlist = ["ip1", "ip2"] + self.assertEqual(h["X-Forwarded-For"], b"ip2") + self.assertEqual(h.get("X-Forwarded-For"), b"ip2") + self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"]) + assert h.getlist("X-Forwarded-For") is not hlist def test_multivalue_for_one_header(self): h = Headers((("a", "b"), ("a", "c"))) @@ -45,117 +46,121 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h.getlist("a"), [b"b", b"c"]) def test_encode_utf8(self): - h = Headers({'key': '\xa3'}, encoding='utf-8') + h = Headers({"key": "\xa3"}, encoding="utf-8") key, val = dict(h).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] - self.assertEqual(val[0], b'\xc2\xa3') + self.assertEqual(val[0], b"\xc2\xa3") def test_encode_latin1(self): - h = Headers({'key': '\xa3'}, encoding='latin1') + h = Headers({"key": "\xa3"}, encoding="latin1") key, val = dict(h).popitem() - self.assertEqual(val[0], b'\xa3') + self.assertEqual(val[0], b"\xa3") def test_encode_multiple(self): - h = Headers({'key': ['\xa3']}, encoding='utf-8') + h = Headers({"key": ["\xa3"]}, encoding="utf-8") key, val = dict(h).popitem() - self.assertEqual(val[0], b'\xc2\xa3') + self.assertEqual(val[0], b"\xc2\xa3") def test_delete_and_contains(self): h = Headers() - h['Content-Type'] = 'text/html' - assert 'Content-Type' in h - del h['Content-Type'] - assert 'Content-Type' not in h + h["Content-Type"] = "text/html" + assert "Content-Type" in h + del h["Content-Type"] + assert "Content-Type" not in h def test_setdefault(self): h = Headers() - hlist = ['ip1', 'ip2'] - olist = h.setdefault('X-Forwarded-For', hlist) - assert h.getlist('X-Forwarded-For') is not hlist - assert h.getlist('X-Forwarded-For') is olist + hlist = ["ip1", "ip2"] + olist = h.setdefault("X-Forwarded-For", hlist) + assert h.getlist("X-Forwarded-For") is not hlist + assert h.getlist("X-Forwarded-For") is olist h = Headers() - olist = h.setdefault('X-Forwarded-For', 'ip1') - self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1']) - assert h.getlist('X-Forwarded-For') is olist + olist = h.setdefault("X-Forwarded-For", "ip1") + self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1"]) + assert h.getlist("X-Forwarded-For") is olist def test_iterables(self): - idict = {'Content-Type': 'text/html', 'X-Forwarded-For': ['ip1', 'ip2']} + idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]} h = Headers(idict) - self.assertDictEqual(dict(h), - {b'Content-Type': [b'text/html'], - b'X-Forwarded-For': [b'ip1', b'ip2']}) - self.assertSortedEqual(h.keys(), - [b'X-Forwarded-For', b'Content-Type']) - self.assertSortedEqual(h.items(), - [(b'X-Forwarded-For', [b'ip1', b'ip2']), - (b'Content-Type', [b'text/html'])]) - self.assertSortedEqual(h.values(), [b'ip2', b'text/html']) + self.assertDictEqual( + dict(h), + {b"Content-Type": [b"text/html"], b"X-Forwarded-For": [b"ip1", b"ip2"]}, + ) + self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"]) + self.assertSortedEqual( + h.items(), + [(b"X-Forwarded-For", [b"ip1", b"ip2"]), (b"Content-Type", [b"text/html"])], + ) + self.assertSortedEqual(h.values(), [b"ip2", b"text/html"]) def test_update(self): h = Headers() - h.update({'Content-Type': 'text/html', - 'X-Forwarded-For': ['ip1', 'ip2']}) - self.assertEqual(h.getlist('Content-Type'), [b'text/html']) - self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2']) + h.update({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}) + self.assertEqual(h.getlist("Content-Type"), [b"text/html"]) + self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"]) def test_copy(self): - h1 = Headers({'header1': ['value1', 'value2']}) + h1 = Headers({"header1": ["value1", "value2"]}) h2 = copy.copy(h1) self.assertEqual(h1, h2) - self.assertEqual(h1.getlist('header1'), h2.getlist('header1')) - assert h1.getlist('header1') is not h2.getlist('header1') + self.assertEqual(h1.getlist("header1"), h2.getlist("header1")) + assert h1.getlist("header1") is not h2.getlist("header1") assert isinstance(h2, Headers) def test_appendlist(self): - h1 = Headers({'header1': 'value1'}) - h1.appendlist('header1', 'value3') - self.assertEqual(h1.getlist('header1'), [b'value1', b'value3']) + h1 = Headers({"header1": "value1"}) + h1.appendlist("header1", "value3") + self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"]) h1 = Headers() - h1.appendlist('header1', 'value1') - h1.appendlist('header1', 'value3') - self.assertEqual(h1.getlist('header1'), [b'value1', b'value3']) + h1.appendlist("header1", "value1") + h1.appendlist("header1", "value3") + self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"]) def test_setlist(self): - h1 = Headers({'header1': 'value1'}) - self.assertEqual(h1.getlist('header1'), [b'value1']) - h1.setlist('header1', [b'value2', b'value3']) - self.assertEqual(h1.getlist('header1'), [b'value2', b'value3']) + h1 = Headers({"header1": "value1"}) + self.assertEqual(h1.getlist("header1"), [b"value1"]) + h1.setlist("header1", [b"value2", b"value3"]) + self.assertEqual(h1.getlist("header1"), [b"value2", b"value3"]) def test_setlistdefault(self): - h1 = Headers({'header1': 'value1'}) - h1.setlistdefault('header1', ['value2', 'value3']) - h1.setlistdefault('header2', ['value2', 'value3']) - self.assertEqual(h1.getlist('header1'), [b'value1']) - self.assertEqual(h1.getlist('header2'), [b'value2', b'value3']) + h1 = Headers({"header1": "value1"}) + h1.setlistdefault("header1", ["value2", "value3"]) + h1.setlistdefault("header2", ["value2", "value3"]) + self.assertEqual(h1.getlist("header1"), [b"value1"]) + self.assertEqual(h1.getlist("header2"), [b"value2", b"value3"]) def test_none_value(self): h1 = Headers() - h1['foo'] = 'bar' - h1['foo'] = None - h1.setdefault('foo', 'bar') - self.assertEqual(h1.get('foo'), None) - self.assertEqual(h1.getlist('foo'), []) + h1["foo"] = "bar" + h1["foo"] = None + h1.setdefault("foo", "bar") + self.assertEqual(h1.get("foo"), None) + self.assertEqual(h1.getlist("foo"), []) def test_int_value(self): - h1 = Headers({'hey': 5}) - h1['foo'] = 1 - h1.setdefault('bar', 2) - h1.setlist('buz', [1, 'dos', 3]) - self.assertEqual(h1.getlist('foo'), [b'1']) - self.assertEqual(h1.getlist('bar'), [b'2']) - self.assertEqual(h1.getlist('buz'), [b'1', b'dos', b'3']) - self.assertEqual(h1.getlist('hey'), [b'5']) + h1 = Headers({"hey": 5}) + h1["foo"] = 1 + h1.setdefault("bar", 2) + h1.setlist("buz", [1, "dos", 3]) + self.assertEqual(h1.getlist("foo"), [b"1"]) + self.assertEqual(h1.getlist("bar"), [b"2"]) + self.assertEqual(h1.getlist("buz"), [b"1", b"dos", b"3"]) + self.assertEqual(h1.getlist("hey"), [b"5"]) def test_invalid_value(self): - self.assertRaisesRegex(TypeError, 'Unsupported value type', - Headers, {'foo': object()}) - self.assertRaisesRegex(TypeError, 'Unsupported value type', - Headers().__setitem__, 'foo', object()) - self.assertRaisesRegex(TypeError, 'Unsupported value type', - Headers().setdefault, 'foo', object()) - self.assertRaisesRegex(TypeError, 'Unsupported value type', - Headers().setlist, 'foo', [object()]) + self.assertRaisesRegex( + TypeError, "Unsupported value type", Headers, {"foo": object()} + ) + self.assertRaisesRegex( + TypeError, "Unsupported value type", Headers().__setitem__, "foo", object() + ) + self.assertRaisesRegex( + TypeError, "Unsupported value type", Headers().setdefault, "foo", object() + ) + self.assertRaisesRegex( + TypeError, "Unsupported value type", Headers().setlist, "foo", [object()] + ) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9f7f1854f..b6f999b29 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -6,14 +6,21 @@ import warnings from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes, urlparse -from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse +from scrapy.http import ( + Request, + FormRequest, + XmlRpcRequest, + JsonRequest, + Headers, + HtmlResponse, +) from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): request_class = Request - default_method = 'GET' + default_method = "GET" default_headers = {} default_meta = {} @@ -23,7 +30,7 @@ class RequestTest(unittest.TestCase): # url argument must be basestring self.assertRaises(TypeError, self.request_class, 123) - r = self.request_class('http://www.example.com') + r = self.request_class("http://www.example.com") r = self.request_class("http://www.example.com") assert isinstance(r.url, str) @@ -36,7 +43,9 @@ class RequestTest(unittest.TestCase): meta = {"lala": "lolo"} headers = {b"caca": b"coco"} - r = self.request_class("http://www.example.com", meta=meta, headers=headers, body="a body") + r = self.request_class( + "http://www.example.com", meta=meta, headers=headers, body="a body" + ) assert r.meta is not meta self.assertEqual(r.meta, meta) @@ -45,22 +54,22 @@ class RequestTest(unittest.TestCase): def test_url_scheme(self): # This test passes by not raising any (ValueError) exception - self.request_class('http://example.org') - self.request_class('https://example.org') - self.request_class('s3://example.org') - self.request_class('ftp://example.org') - self.request_class('about:config') - self.request_class('data:,Hello%2C%20World!') + self.request_class("http://example.org") + self.request_class("https://example.org") + self.request_class("s3://example.org") + self.request_class("ftp://example.org") + self.request_class("about:config") + self.request_class("data:,Hello%2C%20World!") def test_url_no_scheme(self): - self.assertRaises(ValueError, self.request_class, 'foo') - self.assertRaises(ValueError, self.request_class, '/foo/') - self.assertRaises(ValueError, self.request_class, '/foo:bar') + self.assertRaises(ValueError, self.request_class, "foo") + self.assertRaises(ValueError, self.request_class, "/foo/") + self.assertRaises(ValueError, self.request_class, "/foo:bar") def test_headers(self): # Different ways of setting headers attribute - url = 'http://www.scrapy.org' - headers = {b'Accept': 'gzip', b'Custom-Header': 'nothing to tell you'} + url = "http://www.scrapy.org" + headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"} r = self.request_class(url=url, headers=headers) p = self.request_class(url=url, headers=r.headers) @@ -69,15 +78,15 @@ class RequestTest(unittest.TestCase): self.assertFalse(p.headers is r.headers) # headers must not be unicode - h = Headers({'key1': 'val1', 'key2': 'val2'}) - h['newkey'] = 'newval' + h = Headers({"key1": "val1", "key2": "val2"}) + h["newkey"] = "newval" for k, v in h.items(): self.assertIsInstance(k, bytes) for s in v: self.assertIsInstance(s, bytes) def test_eq(self): - url = 'http://www.scrapy.org' + url = "http://www.scrapy.org" r1 = self.request_class(url=url) r2 = self.request_class(url=url) self.assertNotEqual(r1, r2) @@ -115,13 +124,17 @@ class RequestTest(unittest.TestCase): self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") # should be same as above - r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8") + r2 = self.request_class( + url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8" + ) self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") def test_url_encoding_query_latin1(self): # encoding is used for encoding query-string before percent-escaping; # path is still UTF-8 encoded before percent-escaping - r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1") + r3 = self.request_class( + url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1" + ) self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3") def test_url_encoding_nonutf8_untouched(self): @@ -154,27 +167,35 @@ class RequestTest(unittest.TestCase): def test_body(self): r1 = self.request_class(url="http://www.example.com/") - assert r1.body == b'' + assert r1.body == b"" r2 = self.request_class(url="http://www.example.com/", body=b"") assert isinstance(r2.body, bytes) - self.assertEqual(r2.encoding, 'utf-8') # default encoding + self.assertEqual(r2.encoding, "utf-8") # default encoding - r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8') + r3 = self.request_class( + url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8" + ) assert isinstance(r3.body, bytes) self.assertEqual(r3.body, b"Price: \xc2\xa3100") - r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1') + r4 = self.request_class( + url="http://www.example.com/", body="Price: \xa3100", encoding="latin1" + ) assert isinstance(r4.body, bytes) self.assertEqual(r4.body, b"Price: \xa3100") def test_ajax_url(self): # ascii url r = self.request_class(url="http://www.example.com/ajax.html#!key=value") - self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue") + self.assertEqual( + r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue" + ) # unicode url r = self.request_class(url="http://www.example.com/ajax.html#!key=value") - self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue") + self.assertEqual( + r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue" + ) def test_copy(self): """Test Request copy""" @@ -182,10 +203,14 @@ class RequestTest(unittest.TestCase): def somecallback(): pass - r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'], - callback=somecallback, errback=somecallback) - r1.meta['foo'] = 'bar' - r1.cb_kwargs['key'] = 'value' + r1 = self.request_class( + "http://www.example.com", + flags=["f1", "f2"], + callback=somecallback, + errback=somecallback, + ) + r1.meta["foo"] = "bar" + r1.cb_kwargs["key"] = "value" r2 = r1.copy() # make sure copy does not propagate callbacks @@ -199,7 +224,9 @@ class RequestTest(unittest.TestCase): self.assertEqual(r1.flags, r2.flags) # make sure cb_kwargs dict is shallow copied - assert r1.cb_kwargs is not r2.cb_kwargs, "cb_kwargs must be a shallow copy, not identical" + assert ( + r1.cb_kwargs is not r2.cb_kwargs + ), "cb_kwargs must be a shallow copy, not identical" self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) # make sure meta dict is shallow copied @@ -207,7 +234,9 @@ class RequestTest(unittest.TestCase): self.assertEqual(r1.meta, r2.meta) # make sure headers attribute is shallow copied - assert r1.headers is not r2.headers, "headers must be a shallow copy, not identical" + assert ( + r1.headers is not r2.headers + ), "headers must be a shallow copy, not identical" self.assertEqual(r1.headers, r2.headers) self.assertEqual(r1.encoding, r2.encoding) self.assertEqual(r1.dont_filter, r2.dont_filter) @@ -220,27 +249,31 @@ class RequestTest(unittest.TestCase): class CustomRequest(self.request_class): pass - r1 = CustomRequest('http://www.example.com') + r1 = CustomRequest("http://www.example.com") r2 = r1.copy() assert isinstance(r2, CustomRequest) def test_replace(self): """Test Request.replace() method""" - r1 = self.request_class("http://www.example.com", method='GET') + r1 = self.request_class("http://www.example.com", method="GET") hdrs = Headers(r1.headers) - hdrs[b'key'] = b'value' + hdrs[b"key"] = b"value" r2 = r1.replace(method="POST", body="New body", headers=hdrs) self.assertEqual(r1.url, r2.url) self.assertEqual((r1.method, r2.method), ("GET", "POST")) - self.assertEqual((r1.body, r2.body), (b'', b"New body")) + self.assertEqual((r1.body, r2.body), (b"", b"New body")) self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs)) # Empty attributes (which may fail if not compared properly) - r3 = self.request_class("http://www.example.com", meta={'a': 1}, dont_filter=True) - r4 = r3.replace(url="http://www.example.com/2", body=b'', meta={}, dont_filter=False) + r3 = self.request_class( + "http://www.example.com", meta={"a": 1}, dont_filter=True + ) + r4 = r3.replace( + url="http://www.example.com/2", body=b"", meta={}, dont_filter=False + ) self.assertEqual(r4.url, "http://www.example.com/2") - self.assertEqual(r4.body, b'') + self.assertEqual(r4.body, b"") self.assertEqual(r4.meta, {}) assert r4.dont_filter is False @@ -250,27 +283,27 @@ class RequestTest(unittest.TestCase): def test_immutable_attributes(self): r = self.request_class("http://example.com") - self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com') - self.assertRaises(AttributeError, setattr, r, 'body', 'xxx') + self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com") + self.assertRaises(AttributeError, setattr, r, "body", "xxx") def test_callback_and_errback(self): def a_function(): pass - r1 = self.request_class('http://example.com') + r1 = self.request_class("http://example.com") self.assertIsNone(r1.callback) self.assertIsNone(r1.errback) - r2 = self.request_class('http://example.com', callback=a_function) + r2 = self.request_class("http://example.com", callback=a_function) self.assertIs(r2.callback, a_function) self.assertIsNone(r2.errback) - r3 = self.request_class('http://example.com', errback=a_function) + r3 = self.request_class("http://example.com", errback=a_function) self.assertIsNone(r3.callback) self.assertIs(r3.errback, a_function) r4 = self.request_class( - url='http://example.com', + url="http://example.com", callback=a_function, errback=a_function, ) @@ -279,14 +312,14 @@ class RequestTest(unittest.TestCase): def test_callback_and_errback_type(self): with self.assertRaises(TypeError): - self.request_class('http://example.com', callback='a_function') + self.request_class("http://example.com", callback="a_function") with self.assertRaises(TypeError): - self.request_class('http://example.com', errback='a_function') + self.request_class("http://example.com", errback="a_function") with self.assertRaises(TypeError): self.request_class( - url='http://example.com', - callback='a_function', - errback='a_function', + url="http://example.com", + callback="a_function", + errback="a_function", ) def test_from_curl(self): @@ -311,37 +344,48 @@ class RequestTest(unittest.TestCase): r = self.request_class.from_curl(curl_command) self.assertEqual(r.method, "POST") self.assertEqual(r.url, "http://httpbin.org/post") - self.assertEqual(r.body, - b"custname=John+Smith&custtel=500&custemail=jsmith%40" - b"example.org&size=small&topping=cheese&topping=onion" - b"&delivery=12%3A15&comments=") - self.assertEqual(r.cookies, { - '_gauges_unique_year': '1', - '_gauges_unique': '1', - '_gauges_unique_month': '1', - '_gauges_unique_hour': '1', - '_gauges_unique_day': '1' - }) - self.assertEqual(r.headers, { - b'Origin': [b'http://httpbin.org'], - b'Accept-Encoding': [b'gzip, deflate'], - b'Accept-Language': [b'en-US,en;q=0.9,ru;q=0.8,es;q=0.7'], - b'Upgrade-Insecure-Requests': [b'1'], - b'User-Agent': [b'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.' - b'36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202' - b'.75 Chrome/62.0.3202.75 Safari/537.36'], - b'Content-Type': [b'application /x-www-form-urlencoded'], - b'Accept': [b'text/html,application/xhtml+xml,application/xml;q=0.' - b'9,image/webp,image/apng,*/*;q=0.8'], - b'Cache-Control': [b'max-age=0'], - b'Referer': [b'http://httpbin.org/forms/post'], - b'Connection': [b'keep-alive']}) + self.assertEqual( + r.body, + b"custname=John+Smith&custtel=500&custemail=jsmith%40" + b"example.org&size=small&topping=cheese&topping=onion" + b"&delivery=12%3A15&comments=", + ) + self.assertEqual( + r.cookies, + { + "_gauges_unique_year": "1", + "_gauges_unique": "1", + "_gauges_unique_month": "1", + "_gauges_unique_hour": "1", + "_gauges_unique_day": "1", + }, + ) + self.assertEqual( + r.headers, + { + b"Origin": [b"http://httpbin.org"], + b"Accept-Encoding": [b"gzip, deflate"], + b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], + b"Upgrade-Insecure-Requests": [b"1"], + b"User-Agent": [ + b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." + b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" + b".75 Chrome/62.0.3202.75 Safari/537.36" + ], + b"Content-Type": [b"application /x-www-form-urlencoded"], + b"Accept": [ + b"text/html,application/xhtml+xml,application/xml;q=0." + b"9,image/webp,image/apng,*/*;q=0.8" + ], + b"Cache-Control": [b"max-age=0"], + b"Referer": [b"http://httpbin.org/forms/post"], + b"Connection": [b"keep-alive"], + }, + ) def test_from_curl_with_kwargs(self): r = self.request_class.from_curl( - 'curl -X PATCH "http://example.org"', - method="POST", - meta={'key': 'value'} + 'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"} ) self.assertEqual(r.method, "POST") self.assertEqual(r.meta, {"key": "value"}) @@ -349,7 +393,7 @@ class RequestTest(unittest.TestCase): def test_from_curl_ignore_unknown_options(self): # By default: it works and ignores the unknown options: --foo and -z with warnings.catch_warnings(): # avoid warning when executing tests - warnings.simplefilter('ignore') + warnings.simplefilter("ignore") r = self.request_class.from_curl( 'curl -X DELETE "http://example.org" --foo -z', ) @@ -377,68 +421,85 @@ class FormRequestTest(RequestTest): def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) - self.assertEqual(r1.body, b'') + self.assertEqual(r1.body, b"") def test_formdata_overrides_querystring(self): - data = (('a', 'one'), ('a', 'two'), ('b', '2')) - url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', - method='GET', formdata=data).url.split('#')[0] - fs = _qs(self.request_class(url, method='GET', formdata=data)) - self.assertEqual(set(fs[b'a']), {b'one', b'two'}) - self.assertEqual(fs[b'b'], [b'2']) - self.assertIsNone(fs.get(b'c')) + data = (("a", "one"), ("a", "two"), ("b", "2")) + url = self.request_class( + "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data + ).url.split("#")[0] + fs = _qs(self.request_class(url, method="GET", formdata=data)) + self.assertEqual(set(fs[b"a"]), {b"one", b"two"}) + self.assertEqual(fs[b"b"], [b"2"]) + self.assertIsNone(fs.get(b"c")) - data = {'a': '1', 'b': '2'} - fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) - self.assertEqual(fs[b'b'], [b'2']) + data = {"a": "1", "b": "2"} + fs = _qs( + self.request_class("http://www.example.com/", method="GET", formdata=data) + ) + self.assertEqual(fs[b"a"], [b"1"]) + self.assertEqual(fs[b"b"], [b"2"]) def test_default_encoding_bytes(self): # using default encoding (utf-8) - data = {b'one': b'two', b'price': b'\xc2\xa3 100'} + data = {b"one": b"two", b"price": b"\xc2\xa3 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, 'POST') - self.assertEqual(r2.encoding, 'utf-8') - self.assertQueryEqual(r2.body, b'price=%C2%A3+100&one=two') - self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') + self.assertEqual(r2.method, "POST") + self.assertEqual(r2.encoding, "utf-8") + self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") + self.assertEqual( + r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" + ) def test_default_encoding_textual_data(self): # using default encoding (utf-8) - data = {'µ one': 'two', 'price': '£ 100'} + data = {"µ one": "two", "price": "£ 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, 'POST') - self.assertEqual(r2.encoding, 'utf-8') - self.assertQueryEqual(r2.body, b'price=%C2%A3+100&%C2%B5+one=two') - self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') + self.assertEqual(r2.method, "POST") + self.assertEqual(r2.encoding, "utf-8") + self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") + self.assertEqual( + r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" + ) def test_default_encoding_mixed_data(self): # using default encoding (utf-8) - data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'} + data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, 'POST') - self.assertEqual(r2.encoding, 'utf-8') - self.assertQueryEqual(r2.body, b'%C2%B5one=two&price%C2%A3=%C2%A3+100') - self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') + self.assertEqual(r2.method, "POST") + self.assertEqual(r2.encoding, "utf-8") + self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") + self.assertEqual( + r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" + ) def test_custom_encoding_bytes(self): - data = {b'\xb5 one': b'two', b'price': b'\xa3 100'} - r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') - self.assertEqual(r2.method, 'POST') - self.assertEqual(r2.encoding, 'latin1') - self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two') - self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') + data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} + r2 = self.request_class( + "http://www.example.com", formdata=data, encoding="latin1" + ) + self.assertEqual(r2.method, "POST") + self.assertEqual(r2.encoding, "latin1") + self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") + self.assertEqual( + r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" + ) def test_custom_encoding_textual_data(self): - data = {'price': '£ 100'} - r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') - self.assertEqual(r3.encoding, 'latin1') - self.assertEqual(r3.body, b'price=%A3+100') + data = {"price": "£ 100"} + r3 = self.request_class( + "http://www.example.com", formdata=data, encoding="latin1" + ) + self.assertEqual(r3.encoding, "latin1") + self.assertEqual(r3.body, b"price=%A3+100") def test_multi_key_values(self): # using multiples values for a single key - data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']} + data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]} r3 = self.request_class("http://www.example.com", formdata=data) - self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100') + self.assertQueryEqual( + r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100" + ) def test_from_response_post(self): response = _buildresponse( @@ -447,17 +508,22 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers[b"Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req) - self.assertEqual(set(fs[b'test']), {b'val1', b'val2'}) - self.assertEqual(set(fs[b'one']), {b'two', b'three'}) - self.assertEqual(fs[b'test2'], [b'xxx']) - self.assertEqual(fs[b'six'], [b'seven']) + self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) + self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) + self.assertEqual(fs[b"test2"], [b"xxx"]) + self.assertEqual(fs[b"six"], [b"seven"]) def test_from_response_post_nonascii_bytes_utf8(self): response = _buildresponse( @@ -466,17 +532,22 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers[b"Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs['test £']), {'val1', 'val2'}) - self.assertEqual(set(fs['one']), {'two', 'three'}) - self.assertEqual(fs['test2'], ['xxx µ']) - self.assertEqual(fs['six'], ['seven']) + self.assertEqual(set(fs["test £"]), {"val1", "val2"}) + self.assertEqual(set(fs["one"]), {"two", "three"}) + self.assertEqual(fs["test2"], ["xxx µ"]) + self.assertEqual(fs["six"], ["seven"]) def test_from_response_post_nonascii_bytes_latin1(self): response = _buildresponse( @@ -486,18 +557,22 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html", - encoding='latin1', + encoding="latin1", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} ) - req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers[b"Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") - fs = _qs(req, to_unicode=True, encoding='latin1') - self.assertEqual(set(fs['test £']), {'val1', 'val2'}) - self.assertEqual(set(fs['one']), {'two', 'three'}) - self.assertEqual(fs['test2'], ['xxx µ']) - self.assertEqual(fs['six'], ['seven']) + fs = _qs(req, to_unicode=True, encoding="latin1") + self.assertEqual(set(fs["test £"]), {"val1", "val2"}) + self.assertEqual(set(fs["one"]), {"two", "three"}) + self.assertEqual(fs["test2"], ["xxx µ"]) + self.assertEqual(fs["six"], ["seven"]) def test_from_response_post_nonascii_unicode(self): response = _buildresponse( @@ -506,42 +581,46 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers[b"Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs['test £']), {'val1', 'val2'}) - self.assertEqual(set(fs['one']), {'two', 'three'}) - self.assertEqual(fs['test2'], ['xxx µ']) - self.assertEqual(fs['six'], ['seven']) + self.assertEqual(set(fs["test £"]), {"val1", "val2"}) + self.assertEqual(set(fs["one"]), {"two", "three"}) + self.assertEqual(fs["test2"], ["xxx µ"]) + self.assertEqual(fs["six"], ["seven"]) def test_from_response_duplicate_form_key(self): - response = _buildresponse( - '
', - url='http://www.example.com') + response = _buildresponse("
", url="http://www.example.com") req = self.request_class.from_response( response=response, - method='GET', - formdata=(('foo', 'bar'), ('foo', 'baz')), + method="GET", + formdata=(("foo", "bar"), ("foo", "baz")), ) - self.assertEqual(urlparse(req.url).hostname, 'www.example.com') - self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz') + self.assertEqual(urlparse(req.url).hostname, "www.example.com") + self.assertEqual(urlparse(req.url).query, "foo=bar&foo=baz") def test_from_response_override_duplicate_form_key(self): response = _buildresponse( """
-
""") + """ + ) req = self.request_class.from_response( - response, - formdata=(('two', '2'), ('two', '4'))) + response, formdata=(("two", "2"), ("two", "4")) + ) fs = _qs(req) - self.assertEqual(fs[b'one'], [b'1']) - self.assertEqual(fs[b'two'], [b'2', b'4']) + self.assertEqual(fs[b"one"], [b"1"]) + self.assertEqual(fs[b"two"], [b"2", b"4"]) def test_from_response_extra_headers(self): response = _buildresponse( @@ -549,15 +628,18 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response( response=response, - formdata={'one': ['two', 'three'], 'six': 'seven'}, + formdata={"one": ["two", "three"], "six": "seven"}, headers={"Accept-Encoding": "gzip,deflate"}, ) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') - self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers["Content-type"], b"application/x-www-form-urlencoded" + ) + self.assertEqual(req.headers["Accept-Encoding"], b"gzip,deflate") def test_from_response_get(self): response = _buildresponse( @@ -566,60 +648,67 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") - r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) - self.assertEqual(r1.method, 'GET') + url="http://www.example.com/this/list.html", + ) + r1 = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + self.assertEqual(r1.method, "GET") self.assertEqual(urlparse(r1.url).hostname, "www.example.com") self.assertEqual(urlparse(r1.url).path, "/this/get.php") fs = _qs(r1) - self.assertEqual(set(fs[b'test']), {b'val1', b'val2'}) - self.assertEqual(set(fs[b'one']), {b'two', b'three'}) - self.assertEqual(fs[b'test2'], [b'xxx']) - self.assertEqual(fs[b'six'], [b'seven']) + self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) + self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) + self.assertEqual(fs[b"test2"], [b"xxx"]) + self.assertEqual(fs[b"six"], [b"seven"]) def test_from_response_override_params(self): response = _buildresponse( """
-
""") - req = self.request_class.from_response(response, formdata={'two': '2'}) + """ + ) + req = self.request_class.from_response(response, formdata={"two": "2"}) fs = _qs(req) - self.assertEqual(fs[b'one'], [b'1']) - self.assertEqual(fs[b'two'], [b'2']) + self.assertEqual(fs[b"one"], [b"1"]) + self.assertEqual(fs[b"two"], [b"2"]) def test_from_response_drop_params(self): response = _buildresponse( """
-
""") - req = self.request_class.from_response(response, formdata={'two': None}) + """ + ) + req = self.request_class.from_response(response, formdata={"two": None}) fs = _qs(req) - self.assertEqual(fs[b'one'], [b'1']) - self.assertNotIn(b'two', fs) + self.assertEqual(fs[b"one"], [b"1"]) + self.assertNotIn(b"two", fs) def test_from_response_override_method(self): response = _buildresponse( - ''' + """
- ''') + """ + ) request = FormRequest.from_response(response) - self.assertEqual(request.method, 'GET') - request = FormRequest.from_response(response, method='POST') - self.assertEqual(request.method, 'POST') + self.assertEqual(request.method, "GET") + request = FormRequest.from_response(response, method="POST") + self.assertEqual(request.method, "POST") def test_from_response_override_url(self): response = _buildresponse( - ''' + """
- ''') + """ + ) request = FormRequest.from_response(response) - self.assertEqual(request.url, 'http://example.com/app') - request = FormRequest.from_response(response, url='http://foo.bar/absolute') - self.assertEqual(request.url, 'http://foo.bar/absolute') - request = FormRequest.from_response(response, url='/relative') - self.assertEqual(request.url, 'http://example.com/relative') + self.assertEqual(request.url, "http://example.com/app") + request = FormRequest.from_response(response, url="http://foo.bar/absolute") + self.assertEqual(request.url, "http://foo.bar/absolute") + request = FormRequest.from_response(response, url="/relative") + self.assertEqual(request.url, "http://example.com/relative") def test_from_response_case_insensitive(self): response = _buildresponse( @@ -627,12 +716,13 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response(response) fs = _qs(req) - self.assertEqual(fs[b'clickable1'], [b'clicked1']) - self.assertFalse(b'i1' in fs, fs) # xpath in _get_inputs() - self.assertFalse(b'clickable2' in fs, fs) # xpath in _get_clickable() + self.assertEqual(fs[b"clickable1"], [b"clicked1"]) + self.assertFalse(b"i1" in fs, fs) # xpath in _get_inputs() + self.assertFalse(b"clickable2" in fs, fs) # xpath in _get_clickable() def test_from_response_submit_first_clickable(self): response = _buildresponse( @@ -641,13 +731,14 @@ class FormRequestTest(RequestTest): - """) - req = self.request_class.from_response(response, formdata={'two': '2'}) + """ + ) + req = self.request_class.from_response(response, formdata={"two": "2"}) fs = _qs(req) - self.assertEqual(fs[b'clickable1'], [b'clicked1']) - self.assertFalse(b'clickable2' in fs, fs) - self.assertEqual(fs[b'one'], [b'1']) - self.assertEqual(fs[b'two'], [b'2']) + self.assertEqual(fs[b"clickable1"], [b"clicked1"]) + self.assertFalse(b"clickable2" in fs, fs) + self.assertEqual(fs[b"one"], [b"1"]) + self.assertEqual(fs[b"two"], [b"2"]) def test_from_response_submit_not_first_clickable(self): response = _buildresponse( @@ -656,15 +747,16 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response( - response, formdata={'two': '2'}, clickdata={'name': 'clickable2'} + response, formdata={"two": "2"}, clickdata={"name": "clickable2"} ) fs = _qs(req) - self.assertEqual(fs[b'clickable2'], [b'clicked2']) - self.assertFalse(b'clickable1' in fs, fs) - self.assertEqual(fs[b'one'], [b'1']) - self.assertEqual(fs[b'two'], [b'2']) + self.assertEqual(fs[b"clickable2"], [b"clicked2"]) + self.assertFalse(b"clickable1" in fs, fs) + self.assertEqual(fs[b"one"], [b"1"]) + self.assertEqual(fs[b"two"], [b"2"]) def test_from_response_dont_submit_image_as_input(self): response = _buildresponse( @@ -672,10 +764,11 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v']}) + self.assertEqual(fs, {b"i1": [b"i1v"]}) def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( @@ -684,20 +777,22 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) + self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]}) def test_from_response_clickdata_does_not_ignore_image(self): response = _buildresponse( """
-
""") + """ + ) req = self.request_class.from_response(response) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) + self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]}) def test_from_response_multiple_clickdata(self): response = _buildresponse( @@ -706,14 +801,15 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response( - response, clickdata={'name': 'clickable', 'value': 'clicked2'} + response, clickdata={"name": "clickable", "value": "clicked2"} ) fs = _qs(req) - self.assertEqual(fs[b'clickable'], [b'clicked2']) - self.assertEqual(fs[b'one'], [b'clicked1']) - self.assertEqual(fs[b'two'], [b'clicked2']) + self.assertEqual(fs[b"clickable"], [b"clicked2"]) + self.assertEqual(fs[b"one"], [b"clicked1"]) + self.assertEqual(fs[b"two"], [b"clicked2"]) def test_from_response_unicode_clickdata(self): response = _buildresponse( @@ -722,12 +818,13 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response( - response, clickdata={'name': 'price in \u00a3'} + response, clickdata={"name": "price in \u00a3"} ) fs = _qs(req, to_unicode=True) - self.assertTrue(fs['price in \u00a3']) + self.assertTrue(fs["price in \u00a3"]) def test_from_response_unicode_clickdata_latin1(self): response = _buildresponse( @@ -737,12 +834,13 @@ class FormRequestTest(RequestTest): """, - encoding='latin1') - req = self.request_class.from_response( - response, clickdata={'name': 'price in \u00a5'} + encoding="latin1", ) - fs = _qs(req, to_unicode=True, encoding='latin1') - self.assertTrue(fs['price in \u00a5']) + req = self.request_class.from_response( + response, clickdata={"name": "price in \u00a5"} + ) + fs = _qs(req, to_unicode=True, encoding="latin1") + self.assertTrue(fs["price in \u00a5"]) def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( @@ -754,22 +852,25 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response( - response, formname='form2', clickdata={'name': 'clickable'} + response, formname="form2", clickdata={"name": "clickable"} ) fs = _qs(req) - self.assertEqual(fs[b'clickable'], [b'clicked2']) - self.assertEqual(fs[b'field2'], [b'value2']) - self.assertFalse(b'field1' in fs, fs) + self.assertEqual(fs[b"clickable"], [b"clicked2"]) + self.assertEqual(fs[b"field2"], [b"value2"]) + self.assertFalse(b"field1" in fs, fs) def test_from_response_override_clickable(self): - response = _buildresponse('''
''') + response = _buildresponse( + """
""" + ) req = self.request_class.from_response( - response, formdata={'clickme': 'two'}, clickdata={'name': 'clickme'} + response, formdata={"clickme": "two"}, clickdata={"name": "clickme"} ) fs = _qs(req) - self.assertEqual(fs[b'clickme'], [b'two']) + self.assertEqual(fs[b"clickme"], [b"two"]) def test_from_response_dont_click(self): response = _buildresponse( @@ -778,11 +879,12 @@ class FormRequestTest(RequestTest): - """) + """ + ) r1 = self.request_class.from_response(response, dont_click=True) fs = _qs(r1) - self.assertFalse(b'clickable1' in fs, fs) - self.assertFalse(b'clickable2' in fs, fs) + self.assertFalse(b"clickable1" in fs, fs) + self.assertFalse(b"clickable2" in fs, fs) def test_from_response_ambiguous_clickdata(self): response = _buildresponse( @@ -792,17 +894,27 @@ class FormRequestTest(RequestTest): - """) - self.assertRaises(ValueError, self.request_class.from_response, - response, clickdata={'type': 'submit'}) + """ + ) + self.assertRaises( + ValueError, + self.request_class.from_response, + response, + clickdata={"type": "submit"}, + ) def test_from_response_non_matching_clickdata(self): response = _buildresponse( """
-
""") - self.assertRaises(ValueError, self.request_class.from_response, - response, clickdata={'nonexistent': 'notme'}) + """ + ) + self.assertRaises( + ValueError, + self.request_class.from_response, + response, + clickdata={"nonexistent": "notme"}, + ) def test_from_response_nr_index_clickdata(self): response = _buildresponse( @@ -810,32 +922,37 @@ class FormRequestTest(RequestTest): - """) - req = self.request_class.from_response(response, clickdata={'nr': 1}) + """ + ) + req = self.request_class.from_response(response, clickdata={"nr": 1}) fs = _qs(req) - self.assertIn(b'clickable2', fs) - self.assertNotIn(b'clickable1', fs) + self.assertIn(b"clickable2", fs) + self.assertNotIn(b"clickable1", fs) def test_from_response_invalid_nr_index_clickdata(self): response = _buildresponse( """
- """) - self.assertRaises(ValueError, self.request_class.from_response, - response, clickdata={'nr': 1}) + """ + ) + self.assertRaises( + ValueError, self.request_class.from_response, response, clickdata={"nr": 1} + ) def test_from_response_errors_noform(self): response = _buildresponse("""""") self.assertRaises(ValueError, self.request_class.from_response, response) def test_from_response_invalid_html5(self): - response = _buildresponse("""
""" - """""" - """
""") - req = self.request_class.from_response(response, formdata={'bar': 'buz'}) + response = _buildresponse( + """
""" + """""" + """
""" + ) + req = self.request_class.from_response(response, formdata={"bar": "buz"}) fs = _qs(req) - self.assertEqual(fs, {b'foo': [b'xxx'], b'bar': [b'buz']}) + self.assertEqual(fs, {b"foo": [b"xxx"], b"bar": [b"buz"]}) def test_from_response_errors_formnumber(self): response = _buildresponse( @@ -843,20 +960,26 @@ class FormRequestTest(RequestTest): - """) - self.assertRaises(IndexError, self.request_class.from_response, response, formnumber=1) + """ + ) + self.assertRaises( + IndexError, self.request_class.from_response, response, formnumber=1 + ) def test_from_response_noformname(self): response = _buildresponse( """
-
""") - r1 = self.request_class.from_response(response, formdata={'two': '3'}) - self.assertEqual(r1.method, 'POST') - self.assertEqual(r1.headers['Content-type'], b'application/x-www-form-urlencoded') + """ + ) + r1 = self.request_class.from_response(response, formdata={"two": "3"}) + self.assertEqual(r1.method, "POST") + self.assertEqual( + r1.headers["Content-type"], b"application/x-www-form-urlencoded" + ) fs = _qs(r1) - self.assertEqual(fs, {b'one': [b'1'], b'two': [b'3']}) + self.assertEqual(fs, {b"one": [b"1"], b"two": [b"3"]}) def test_from_response_formname_exists(self): response = _buildresponse( @@ -867,11 +990,12 @@ class FormRequestTest(RequestTest):
-
""") + """ + ) r1 = self.request_class.from_response(response, formname="form2") - self.assertEqual(r1.method, 'POST') + self.assertEqual(r1.method, "POST") fs = _qs(r1) - self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']}) + self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) def test_from_response_formname_notexist(self): response = _buildresponse( @@ -880,11 +1004,12 @@ class FormRequestTest(RequestTest):
-
""") + """ + ) r1 = self.request_class.from_response(response, formname="form3") - self.assertEqual(r1.method, 'POST') + self.assertEqual(r1.method, "POST") fs = _qs(r1) - self.assertEqual(fs, {b'one': [b'1']}) + self.assertEqual(fs, {b"one": [b"1"]}) def test_from_response_formname_errors_formnumber(self): response = _buildresponse( @@ -893,9 +1018,15 @@ class FormRequestTest(RequestTest):
-
""") - self.assertRaises(IndexError, self.request_class.from_response, - response, formname="form3", formnumber=2) + """ + ) + self.assertRaises( + IndexError, + self.request_class.from_response, + response, + formname="form3", + formnumber=2, + ) def test_from_response_formid_exists(self): response = _buildresponse( @@ -906,11 +1037,12 @@ class FormRequestTest(RequestTest):
-
""") + """ + ) r1 = self.request_class.from_response(response, formid="form2") - self.assertEqual(r1.method, 'POST') + self.assertEqual(r1.method, "POST") fs = _qs(r1) - self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']}) + self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) def test_from_response_formname_notexists_fallback_formid(self): response = _buildresponse( @@ -921,11 +1053,14 @@ class FormRequestTest(RequestTest):
-
""") - r1 = self.request_class.from_response(response, formname="form3", formid="form2") - self.assertEqual(r1.method, 'POST') + """ + ) + r1 = self.request_class.from_response( + response, formname="form3", formid="form2" + ) + self.assertEqual(r1.method, "POST") fs = _qs(r1) - self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']}) + self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) def test_from_response_formid_notexist(self): response = _buildresponse( @@ -934,11 +1069,12 @@ class FormRequestTest(RequestTest):
-
""") + """ + ) r1 = self.request_class.from_response(response, formid="form3") - self.assertEqual(r1.method, 'POST') + self.assertEqual(r1.method, "POST") fs = _qs(r1) - self.assertEqual(fs, {b'one': [b'1']}) + self.assertEqual(fs, {b"one": [b"1"]}) def test_from_response_formid_errors_formnumber(self): response = _buildresponse( @@ -947,13 +1083,19 @@ class FormRequestTest(RequestTest):
-
""") - self.assertRaises(IndexError, self.request_class.from_response, - response, formid="form3", formnumber=2) + """ + ) + self.assertRaises( + IndexError, + self.request_class.from_response, + response, + formid="form3", + formnumber=2, + ) def test_from_response_select(self): res = _buildresponse( - '''
+ """ @@ -993,14 +1136,15 @@ class FormRequestTest(RequestTest): -
''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'iv2'], b'i2': [b'on']}) + self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]}) def test_from_response_checkbox(self): res = _buildresponse( - '''
+ """ @@ -1009,51 +1153,55 @@ class FormRequestTest(RequestTest): -
''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'iv2'], b'i2': [b'on']}) + self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]}) def test_from_response_input_text(self): res = _buildresponse( - '''
+ """ -
''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v1'], b'i2': [b''], b'i4': [b'i4v1']}) + self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]}) def test_from_response_input_hidden(self): res = _buildresponse( - '''
+ """ -
''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v1'], b'i2': [b'']}) + self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""]}) def test_from_response_input_textarea(self): res = _buildresponse( - '''
+ """ -
''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b''], b'i3': [b'']}) + self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]}) def test_from_response_descendants(self): res = _buildresponse( - '''
+ """
@@ -1067,10 +1215,11 @@ class FormRequestTest(RequestTest):
-
''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(set(fs), {b'h2', b'i2', b'i1', b'i3', b'h1', b'i5', b'i4'}) + self.assertEqual(set(fs), {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"}) def test_from_response_xpath(self): response = _buildresponse( @@ -1081,28 +1230,43 @@ class FormRequestTest(RequestTest):
-
""") - r1 = self.request_class.from_response(response, formxpath="//form[@action='post.php']") + """ + ) + r1 = self.request_class.from_response( + response, formxpath="//form[@action='post.php']" + ) fs = _qs(r1) - self.assertEqual(fs[b'one'], [b'1']) + self.assertEqual(fs[b"one"], [b"1"]) - r1 = self.request_class.from_response(response, formxpath="//form/input[@name='four']") + r1 = self.request_class.from_response( + response, formxpath="//form/input[@name='four']" + ) fs = _qs(r1) - self.assertEqual(fs[b'three'], [b'3']) + self.assertEqual(fs[b"three"], [b"3"]) - self.assertRaises(ValueError, self.request_class.from_response, - response, formxpath="//form/input[@name='abc']") + self.assertRaises( + ValueError, + self.request_class.from_response, + response, + formxpath="//form/input[@name='abc']", + ) def test_from_response_unicode_xpath(self): response = _buildresponse(b'
') - r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']") + r = self.request_class.from_response( + response, formxpath="//form[@name='\u044a']" + ) fs = _qs(r) self.assertEqual(fs, {}) xpath = "//form[@name='\u03b1']" - self.assertRaisesRegex(ValueError, re.escape(xpath), - self.request_class.from_response, - response, formxpath=xpath) + self.assertRaisesRegex( + ValueError, + re.escape(xpath), + self.request_class.from_response, + response, + formxpath=xpath, + ) def test_from_response_button_submit(self): response = _buildresponse( @@ -1111,15 +1275,18 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") + url="http://www.example.com/this/list.html", + ) req = self.request_class.from_response(response) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers["Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req) - self.assertEqual(fs[b'test1'], [b'val1']) - self.assertEqual(fs[b'test2'], [b'val2']) - self.assertEqual(fs[b'button1'], [b'submit1']) + self.assertEqual(fs[b"test1"], [b"val1"]) + self.assertEqual(fs[b"test2"], [b"val2"]) + self.assertEqual(fs[b"button1"], [b"submit1"]) def test_from_response_button_notype(self): response = _buildresponse( @@ -1128,15 +1295,18 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") + url="http://www.example.com/this/list.html", + ) req = self.request_class.from_response(response) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers["Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req) - self.assertEqual(fs[b'test1'], [b'val1']) - self.assertEqual(fs[b'test2'], [b'val2']) - self.assertEqual(fs[b'button1'], [b'submit1']) + self.assertEqual(fs[b"test1"], [b"val1"]) + self.assertEqual(fs[b"test2"], [b"val2"]) + self.assertEqual(fs[b"button1"], [b"submit1"]) def test_from_response_submit_novalue(self): response = _buildresponse( @@ -1145,15 +1315,18 @@ class FormRequestTest(RequestTest): Submit """, - url="http://www.example.com/this/list.html") + url="http://www.example.com/this/list.html", + ) req = self.request_class.from_response(response) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers["Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req) - self.assertEqual(fs[b'test1'], [b'val1']) - self.assertEqual(fs[b'test2'], [b'val2']) - self.assertEqual(fs[b'button1'], [b'']) + self.assertEqual(fs[b"test1"], [b"val1"]) + self.assertEqual(fs[b"test2"], [b"val2"]) + self.assertEqual(fs[b"button1"], [b""]) def test_from_response_button_novalue(self): response = _buildresponse( @@ -1162,15 +1335,18 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") + url="http://www.example.com/this/list.html", + ) req = self.request_class.from_response(response) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers["Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req) - self.assertEqual(fs[b'test1'], [b'val1']) - self.assertEqual(fs[b'test2'], [b'val2']) - self.assertEqual(fs[b'button1'], [b'']) + self.assertEqual(fs[b"test1"], [b"val1"]) + self.assertEqual(fs[b"test2"], [b"val2"]) + self.assertEqual(fs[b"button1"], [b""]) def test_html_base_form_action(self): response = _buildresponse( @@ -1185,15 +1361,15 @@ class FormRequestTest(RequestTest): """, - url='http://a.com/' + url="http://a.com/", ) req = self.request_class.from_response(response) - self.assertEqual(req.url, 'http://b.com/test_form') + self.assertEqual(req.url, "http://b.com/test_form") def test_spaces_in_action(self): resp = _buildresponse('
') req = self.request_class.from_response(resp) - self.assertEqual(req.url, 'http://example.com/path') + self.assertEqual(req.url, "http://example.com/path") def test_from_response_css(self): response = _buildresponse( @@ -1204,44 +1380,53 @@ class FormRequestTest(RequestTest):
-
""") - r1 = self.request_class.from_response(response, formcss="form[action='post.php']") + """ + ) + r1 = self.request_class.from_response( + response, formcss="form[action='post.php']" + ) fs = _qs(r1) - self.assertEqual(fs[b'one'], [b'1']) + self.assertEqual(fs[b"one"], [b"1"]) r1 = self.request_class.from_response(response, formcss="input[name='four']") fs = _qs(r1) - self.assertEqual(fs[b'three'], [b'3']) + self.assertEqual(fs[b"three"], [b"3"]) - self.assertRaises(ValueError, self.request_class.from_response, - response, formcss="input[name='abc']") + self.assertRaises( + ValueError, + self.request_class.from_response, + response, + formcss="input[name='abc']", + ) def test_from_response_valid_form_methods(self): - form_methods = [[method, method] for method in self.request_class.valid_form_methods] - form_methods.append(['UNKNOWN', 'GET']) + form_methods = [ + [method, method] for method in self.request_class.valid_form_methods + ] + form_methods.append(["UNKNOWN", "GET"]) for method, expected in form_methods: response = _buildresponse( f'
' '' - '
' + "" ) r = self.request_class.from_response(response) self.assertEqual(r.method, expected) def _buildresponse(body, **kwargs): - kwargs.setdefault('body', body) - kwargs.setdefault('url', 'http://example.com') - kwargs.setdefault('encoding', 'utf-8') + kwargs.setdefault("body", body) + kwargs.setdefault("url", "http://example.com") + kwargs.setdefault("encoding", "utf-8") return HtmlResponse(**kwargs) -def _qs(req, encoding='utf-8', to_unicode=False): - if req.method == 'POST': +def _qs(req, encoding="utf-8", to_unicode=False): + if req.method == "POST": qs = req.body else: - qs = req.url.partition('?')[2] + qs = req.url.partition("?")[2] uqs = unquote_to_bytes(qs) if to_unicode: uqs = uqs.decode(encoding) @@ -1251,38 +1436,41 @@ def _qs(req, encoding='utf-8', to_unicode=False): class XmlRpcRequestTest(RequestTest): request_class = XmlRpcRequest - default_method = 'POST' - default_headers = {b'Content-Type': [b'text/xml']} + default_method = "POST" + default_headers = {b"Content-Type": [b"text/xml"]} def _test_request(self, **kwargs): - r = self.request_class('http://scrapytest.org/rpc2', **kwargs) - self.assertEqual(r.headers[b'Content-Type'], b'text/xml') - self.assertEqual(r.body, - to_bytes(xmlrpc.client.dumps(**kwargs), - encoding=kwargs.get('encoding', 'utf-8'))) - self.assertEqual(r.method, 'POST') - self.assertEqual(r.encoding, kwargs.get('encoding', 'utf-8')) + r = self.request_class("http://scrapytest.org/rpc2", **kwargs) + self.assertEqual(r.headers[b"Content-Type"], b"text/xml") + self.assertEqual( + r.body, + to_bytes( + xmlrpc.client.dumps(**kwargs), encoding=kwargs.get("encoding", "utf-8") + ), + ) + self.assertEqual(r.method, "POST") + self.assertEqual(r.encoding, kwargs.get("encoding", "utf-8")) self.assertTrue(r.dont_filter, True) def test_xmlrpc_dumps(self): - self._test_request(params=('value',)) - self._test_request(params=('username', 'password'), methodname='login') - self._test_request(params=('response', ), methodresponse='login') - self._test_request(params=('pas£',), encoding='utf-8') + self._test_request(params=("value",)) + self._test_request(params=("username", "password"), methodname="login") + self._test_request(params=("response",), methodresponse="login") + self._test_request(params=("pas£",), encoding="utf-8") self._test_request(params=(None,), allow_none=1) self.assertRaises(TypeError, self._test_request) self.assertRaises(TypeError, self._test_request, params=(None,)) def test_latin1(self): - self._test_request(params=('pas£',), encoding='latin1') + self._test_request(params=("pas£",), encoding="latin1") class JsonRequestTest(RequestTest): request_class = JsonRequest - default_method = 'GET' + default_method = "GET" default_headers = { - b'Content-Type': [b'application/json'], - b'Accept': [b'application/json, text/javascript, */*; q=0.01'], + b"Content-Type": [b"application/json"], + b"Accept": [b"application/json, text/javascript, */*; q=0.01"], } def setUp(self): @@ -1291,14 +1479,14 @@ class JsonRequestTest(RequestTest): def test_data(self): r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.body, b'') + self.assertEqual(r1.body, b"") - body = b'body' + body = b"body" r2 = self.request_class(url="http://www.example.com/", body=body) self.assertEqual(r2.body, body) data = { - 'name': 'value', + "name": "value", } r3 = self.request_class(url="http://www.example.com/", data=data) self.assertEqual(r3.body, to_bytes(json.dumps(data))) @@ -1310,134 +1498,138 @@ class JsonRequestTest(RequestTest): def test_data_method(self): # data is not passed r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.method, 'GET') + self.assertEqual(r1.method, "GET") - body = b'body' + body = b"body" r2 = self.request_class(url="http://www.example.com/", body=body) - self.assertEqual(r2.method, 'GET') + self.assertEqual(r2.method, "GET") data = { - 'name': 'value', + "name": "value", } r3 = self.request_class(url="http://www.example.com/", data=data) - self.assertEqual(r3.method, 'POST') + self.assertEqual(r3.method, "POST") # method passed explicitly - r4 = self.request_class(url="http://www.example.com/", data=data, method='GET') - self.assertEqual(r4.method, 'GET') + r4 = self.request_class(url="http://www.example.com/", data=data, method="GET") + self.assertEqual(r4.method, "GET") r5 = self.request_class(url="http://www.example.com/", data=[]) - self.assertEqual(r5.method, 'POST') + self.assertEqual(r5.method, "POST") def test_body_data(self): - """ passing both body and data should result a warning """ - body = b'body' + """passing both body and data should result a warning""" + body = b"body" data = { - 'name': 'value', + "name": "value", } with warnings.catch_warnings(record=True) as _warnings: r5 = self.request_class(url="http://www.example.com/", body=body, data=data) self.assertEqual(r5.body, body) - self.assertEqual(r5.method, 'GET') + self.assertEqual(r5.method, "GET") self.assertEqual(len(_warnings), 1) - self.assertIn('data will be ignored', str(_warnings[0].message)) + self.assertIn("data will be ignored", str(_warnings[0].message)) def test_empty_body_data(self): - """ passing any body value and data should result a warning """ + """passing any body value and data should result a warning""" data = { - 'name': 'value', + "name": "value", } with warnings.catch_warnings(record=True) as _warnings: - r6 = self.request_class(url="http://www.example.com/", body=b'', data=data) - self.assertEqual(r6.body, b'') - self.assertEqual(r6.method, 'GET') + r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) + self.assertEqual(r6.body, b"") + self.assertEqual(r6.method, "GET") self.assertEqual(len(_warnings), 1) - self.assertIn('data will be ignored', str(_warnings[0].message)) + self.assertIn("data will be ignored", str(_warnings[0].message)) def test_body_none_data(self): data = { - 'name': 'value', + "name": "value", } with warnings.catch_warnings(record=True) as _warnings: r7 = self.request_class(url="http://www.example.com/", body=None, data=data) self.assertEqual(r7.body, to_bytes(json.dumps(data))) - self.assertEqual(r7.method, 'POST') + self.assertEqual(r7.method, "POST") self.assertEqual(len(_warnings), 0) def test_body_data_none(self): with warnings.catch_warnings(record=True) as _warnings: r8 = self.request_class(url="http://www.example.com/", body=None, data=None) - self.assertEqual(r8.method, 'GET') + self.assertEqual(r8.method, "GET") self.assertEqual(len(_warnings), 0) def test_dumps_sort_keys(self): - """ Test that sort_keys=True is passed to json.dumps by default """ + """Test that sort_keys=True is passed to json.dumps by default""" data = { - 'name': 'value', + "name": "value", } - with mock.patch('json.dumps', return_value=b'') as mock_dumps: + with mock.patch("json.dumps", return_value=b"") as mock_dumps: self.request_class(url="http://www.example.com/", data=data) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs['sort_keys'], True) + self.assertEqual(kwargs["sort_keys"], True) def test_dumps_kwargs(self): - """ Test that dumps_kwargs are passed to json.dumps """ + """Test that dumps_kwargs are passed to json.dumps""" data = { - 'name': 'value', + "name": "value", } dumps_kwargs = { - 'ensure_ascii': True, - 'allow_nan': True, + "ensure_ascii": True, + "allow_nan": True, } - with mock.patch('json.dumps', return_value=b'') as mock_dumps: - self.request_class(url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs) + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + self.request_class( + url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs + ) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs['ensure_ascii'], True) - self.assertEqual(kwargs['allow_nan'], True) + self.assertEqual(kwargs["ensure_ascii"], True) + self.assertEqual(kwargs["allow_nan"], True) def test_replace_data(self): data1 = { - 'name1': 'value1', + "name1": "value1", } data2 = { - 'name2': 'value2', + "name2": "value2", } r1 = self.request_class(url="http://www.example.com/", data=data1) r2 = r1.replace(data=data2) self.assertEqual(r2.body, to_bytes(json.dumps(data2))) def test_replace_sort_keys(self): - """ Test that replace provides sort_keys=True to json.dumps """ + """Test that replace provides sort_keys=True to json.dumps""" data1 = { - 'name1': 'value1', + "name1": "value1", } data2 = { - 'name2': 'value2', + "name2": "value2", } r1 = self.request_class(url="http://www.example.com/", data=data1) - with mock.patch('json.dumps', return_value=b'') as mock_dumps: + with mock.patch("json.dumps", return_value=b"") as mock_dumps: r1.replace(data=data2) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs['sort_keys'], True) + self.assertEqual(kwargs["sort_keys"], True) def test_replace_dumps_kwargs(self): - """ Test that dumps_kwargs are provided to json.dumps when replace is called """ + """Test that dumps_kwargs are provided to json.dumps when replace is called""" data1 = { - 'name1': 'value1', + "name1": "value1", } data2 = { - 'name2': 'value2', + "name2": "value2", } dumps_kwargs = { - 'ensure_ascii': True, - 'allow_nan': True, + "ensure_ascii": True, + "allow_nan": True, } - r1 = self.request_class(url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs) - with mock.patch('json.dumps', return_value=b'') as mock_dumps: + r1 = self.request_class( + url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs + ) + with mock.patch("json.dumps", return_value=b"") as mock_dumps: r1.replace(data=data2) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs['ensure_ascii'], True) - self.assertEqual(kwargs['allow_nan'], True) + self.assertEqual(kwargs["ensure_ascii"], True) + self.assertEqual(kwargs["allow_nan"], True) def tearDown(self): warnings.resetwarnings() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index f51f3d988..c52fcce09 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -4,8 +4,14 @@ from unittest import mock from w3lib.encoding import resolve_encoding -from scrapy.http import (Request, Response, TextResponse, HtmlResponse, - XmlResponse, Headers) +from scrapy.http import ( + Request, + Response, + TextResponse, + HtmlResponse, + XmlResponse, + Headers, +) from scrapy.selector import Selector from scrapy.utils.python import to_unicode from scrapy.exceptions import NotSupported @@ -20,15 +26,30 @@ class BaseResponseTest(unittest.TestCase): def test_init(self): # Response requires url in the constructor self.assertRaises(Exception, self.response_class) - self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class)) + self.assertTrue( + isinstance(self.response_class("http://example.com/"), self.response_class) + ) self.assertRaises(TypeError, self.response_class, b"http://example.com") # body can be str or None - self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class)) - self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class)) + self.assertTrue( + isinstance( + self.response_class("http://example.com/", body=b""), + self.response_class, + ) + ) + self.assertTrue( + isinstance( + self.response_class("http://example.com/", body=b"body"), + self.response_class, + ) + ) # test presence of all optional parameters self.assertTrue( isinstance( - self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class + self.response_class( + "http://example.com/", body=b"", headers={}, status=200 + ), + self.response_class, ) ) @@ -49,15 +70,17 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://www.example.com", status=301) self.assertEqual(r.status, 301) - r = self.response_class("http://www.example.com", status='301') + r = self.response_class("http://www.example.com", status="301") self.assertEqual(r.status, 301) - self.assertRaises(ValueError, self.response_class, "http://example.com", status='lala200') + self.assertRaises( + ValueError, self.response_class, "http://example.com", status="lala200" + ) def test_copy(self): """Test Response copy""" r1 = self.response_class("http://www.example.com", body=b"Some body") - r1.flags.append('cached') + r1.flags.append("cached") r2 = r1.copy() self.assertEqual(r1.status, r2.status) @@ -68,29 +91,37 @@ class BaseResponseTest(unittest.TestCase): self.assertEqual(r1.flags, r2.flags) # make sure headers attribute is shallow copied - assert r1.headers is not r2.headers, "headers must be a shallow copy, not identical" + assert ( + r1.headers is not r2.headers + ), "headers must be a shallow copy, not identical" self.assertEqual(r1.headers, r2.headers) def test_copy_meta(self): req = Request("http://www.example.com") - req.meta['foo'] = 'bar' - r1 = self.response_class("http://www.example.com", body=b"Some body", request=req) + req.meta["foo"] = "bar" + r1 = self.response_class( + "http://www.example.com", body=b"Some body", request=req + ) assert r1.meta is req.meta def test_copy_cb_kwargs(self): req = Request("http://www.example.com") - req.cb_kwargs['foo'] = 'bar' - r1 = self.response_class("http://www.example.com", body=b"Some body", request=req) + req.cb_kwargs["foo"] = "bar" + r1 = self.response_class( + "http://www.example.com", body=b"Some body", request=req + ) assert r1.cb_kwargs is req.cb_kwargs def test_unavailable_meta(self): r1 = self.response_class("http://www.example.com", body=b"Some body") - with self.assertRaisesRegex(AttributeError, r'Response\.meta not available'): + with self.assertRaisesRegex(AttributeError, r"Response\.meta not available"): r1.meta def test_unavailable_cb_kwargs(self): r1 = self.response_class("http://www.example.com", body=b"Some body") - with self.assertRaisesRegex(AttributeError, r'Response\.cb_kwargs not available'): + with self.assertRaisesRegex( + AttributeError, r"Response\.cb_kwargs not available" + ): r1.cb_kwargs def test_copy_inherited_classes(self): @@ -99,7 +130,7 @@ class BaseResponseTest(unittest.TestCase): class CustomResponse(self.response_class): pass - r1 = CustomResponse('http://www.example.com') + r1 = CustomResponse("http://www.example.com") r2 = r1.copy() assert isinstance(r2, CustomResponse) @@ -109,16 +140,16 @@ class BaseResponseTest(unittest.TestCase): hdrs = Headers({"key": "value"}) r1 = self.response_class("http://www.example.com") r2 = r1.replace(status=301, body=b"New body", headers=hdrs) - assert r1.body == b'' + assert r1.body == b"" self.assertEqual(r1.url, r2.url) self.assertEqual((r1.status, r2.status), (200, 301)) - self.assertEqual((r1.body, r2.body), (b'', b"New body")) + self.assertEqual((r1.body, r2.body), (b"", b"New body")) self.assertEqual((r1.headers, r2.headers), ({}, hdrs)) # Empty attributes (which may fail if not compared properly) - r3 = self.response_class("http://www.example.com", flags=['cached']) - r4 = r3.replace(body=b'', flags=[]) - self.assertEqual(r4.body, b'') + r3 = self.response_class("http://www.example.com", flags=["cached"]) + r4 = r3.replace(body=b"", flags=[]) + self.assertEqual(r4.body, b"") self.assertEqual(r4.flags, []) def _assert_response_values(self, response, encoding, body): @@ -140,81 +171,84 @@ class BaseResponseTest(unittest.TestCase): def test_immutable_attributes(self): r = self.response_class("http://example.com") - self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com') - self.assertRaises(AttributeError, setattr, r, 'body', 'xxx') + self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com") + self.assertRaises(AttributeError, setattr, r, "body", "xxx") def test_urljoin(self): """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" - joined = self.response_class('http://www.example.com').urljoin('/test') - absolute = 'http://www.example.com/test' + joined = self.response_class("http://www.example.com").urljoin("/test") + absolute = "http://www.example.com/test" self.assertEqual(joined, absolute) def test_shortcut_attributes(self): - r = self.response_class("http://example.com", body=b'hello') + r = self.response_class("http://example.com", body=b"hello") if self.response_class == Response: msg = "Response content isn't text" - self.assertRaisesRegex(AttributeError, msg, getattr, r, 'text') - self.assertRaisesRegex(NotSupported, msg, r.css, 'body') - self.assertRaisesRegex(NotSupported, msg, r.xpath, '//body') + self.assertRaisesRegex(AttributeError, msg, getattr, r, "text") + self.assertRaisesRegex(NotSupported, msg, r.css, "body") + self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body") else: r.text - r.css('body') - r.xpath('//body') + r.css("body") + r.xpath("//body") # Response.follow def test_follow_url_absolute(self): - self._assert_followed_url('http://foo.example.com', - 'http://foo.example.com') + self._assert_followed_url("http://foo.example.com", "http://foo.example.com") def test_follow_url_relative(self): - self._assert_followed_url('foo', - 'http://example.com/foo') + self._assert_followed_url("foo", "http://example.com/foo") def test_follow_link(self): - self._assert_followed_url(Link('http://example.com/foo'), - 'http://example.com/foo') + self._assert_followed_url( + Link("http://example.com/foo"), "http://example.com/foo" + ) def test_follow_None_url(self): r = self.response_class("http://example.com") self.assertRaises(ValueError, r.follow, None) def test_follow_whitespace_url(self): - self._assert_followed_url('foo ', - 'http://example.com/foo%20') + self._assert_followed_url("foo ", "http://example.com/foo%20") def test_follow_whitespace_link(self): - self._assert_followed_url(Link('http://example.com/foo '), - 'http://example.com/foo%20') + self._assert_followed_url( + Link("http://example.com/foo "), "http://example.com/foo%20" + ) def test_follow_flags(self): - res = self.response_class('http://example.com/') - fol = res.follow('http://example.com/', flags=['cached', 'allowed']) - self.assertEqual(fol.flags, ['cached', 'allowed']) + res = self.response_class("http://example.com/") + fol = res.follow("http://example.com/", flags=["cached", "allowed"]) + self.assertEqual(fol.flags, ["cached", "allowed"]) # Response.follow_all def test_follow_all_absolute(self): - url_list = ['http://example.org', 'http://www.example.org', - 'http://example.com', 'http://www.example.com'] + url_list = [ + "http://example.org", + "http://www.example.org", + "http://example.com", + "http://www.example.com", + ] self._assert_followed_all_urls(url_list, url_list) def test_follow_all_relative(self): - relative = ['foo', 'bar', 'foo/bar', 'bar/foo'] + relative = ["foo", "bar", "foo/bar", "bar/foo"] absolute = [ - 'http://example.com/foo', - 'http://example.com/bar', - 'http://example.com/foo/bar', - 'http://example.com/bar/foo', + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", ] self._assert_followed_all_urls(relative, absolute) def test_follow_all_links(self): absolute = [ - 'http://example.com/foo', - 'http://example.com/bar', - 'http://example.com/foo/bar', - 'http://example.com/bar/foo', + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", ] links = map(Link, absolute) self._assert_followed_all_urls(links, absolute) @@ -241,36 +275,36 @@ class BaseResponseTest(unittest.TestCase): list(r.follow_all(urls=[None])) def test_follow_all_whitespace(self): - relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo '] + relative = ["foo ", "bar ", "foo/bar ", "bar/foo "] absolute = [ - 'http://example.com/foo%20', - 'http://example.com/bar%20', - 'http://example.com/foo/bar%20', - 'http://example.com/bar/foo%20', + "http://example.com/foo%20", + "http://example.com/bar%20", + "http://example.com/foo/bar%20", + "http://example.com/bar/foo%20", ] self._assert_followed_all_urls(relative, absolute) def test_follow_all_whitespace_links(self): absolute = [ - 'http://example.com/foo ', - 'http://example.com/bar ', - 'http://example.com/foo/bar ', - 'http://example.com/bar/foo ', + "http://example.com/foo ", + "http://example.com/bar ", + "http://example.com/foo/bar ", + "http://example.com/bar/foo ", ] links = map(Link, absolute) - expected = [u.replace(' ', '%20') for u in absolute] + expected = [u.replace(" ", "%20") for u in absolute] self._assert_followed_all_urls(links, expected) def test_follow_all_flags(self): - re = self.response_class('http://www.example.com/') + re = self.response_class("http://www.example.com/") urls = [ - 'http://www.example.com/', - 'http://www.example.com/2', - 'http://www.example.com/foo', + "http://www.example.com/", + "http://www.example.com/2", + "http://www.example.com/foo", ] - fol = re.follow_all(urls, flags=['cached', 'allowed']) + fol = re.follow_all(urls, flags=["cached", "allowed"]) for req in fol: - self.assertEqual(req.flags, ['cached', 'allowed']) + self.assertEqual(req.flags, ["cached", "allowed"]) def _assert_followed_url(self, follow_obj, target_url, response=None): if response is None: @@ -288,13 +322,13 @@ class BaseResponseTest(unittest.TestCase): yield req def _links_response(self): - body = get_testdata('link_extractor', 'linkextractor.html') - resp = self.response_class('http://example.com/index', body=body) + body = get_testdata("link_extractor", "linkextractor.html") + resp = self.response_class("http://example.com/index", body=body) return resp def _links_response_no_href(self): - body = get_testdata('link_extractor', 'linkextractor_no_href.html') - resp = self.response_class('http://example.com/index', body=body) + body = get_testdata("link_extractor", "linkextractor_no_href.html") + resp = self.response_class("http://example.com/index", body=body) return resp @@ -304,7 +338,9 @@ class TextResponseTest(BaseResponseTest): def test_replace(self): super().test_replace() - r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852") + r1 = self.response_class( + "http://www.example.com", body="hello", encoding="cp852" + ) r2 = r1.replace(url="http://www.example.com/other") r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") @@ -320,52 +356,89 @@ class TextResponseTest(BaseResponseTest): self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) # make sure urls are converted to str - resp = self.response_class(url="http://www.example.com/", encoding='utf-8') + resp = self.response_class(url="http://www.example.com/", encoding="utf-8") assert isinstance(resp.url, str) - resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8') - self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1') - self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') - resp = self.response_class("http://www.example.com/price/\xa3", - headers={"Content-type": ["text/html; charset=utf-8"]}) - self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class("http://www.example.com/price/\xa3", - headers={"Content-type": ["text/html; charset=iso-8859-1"]}) - self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') + resp = self.response_class( + url="http://www.example.com/price/\xa3", encoding="utf-8" + ) + self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")) + resp = self.response_class( + url="http://www.example.com/price/\xa3", encoding="latin-1" + ) + self.assertEqual(resp.url, "http://www.example.com/price/\xa3") + resp = self.response_class( + "http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}, + ) + self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")) + resp = self.response_class( + "http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) + self.assertEqual(resp.url, "http://www.example.com/price/\xa3") def test_unicode_body(self): - unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 ' - '\u0442\u0435\u043a\u0441\u0442') - self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body') + unicode_string = ( + "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 " + "\u0442\u0435\u043a\u0441\u0442" + ) + self.assertRaises( + TypeError, + self.response_class, + "http://www.example.com", + body="unicode body", + ) - original_string = unicode_string.encode('cp1251') - r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') + original_string = unicode_string.encode("cp1251") + r1 = self.response_class( + "http://www.example.com", body=original_string, encoding="cp1251" + ) # check response.text self.assertTrue(isinstance(r1.text, str)) self.assertEqual(r1.text, unicode_string) def test_encoding(self): - r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=utf-8"]}) - r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3") - r3 = self.response_class("http://www.example.com", body=b"\xa3", - headers={"Content-type": ["text/html; charset=iso-8859-1"]}) + r1 = self.response_class( + "http://www.example.com", + body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}, + ) + r2 = self.response_class( + "http://www.example.com", encoding="utf-8", body="\xa3" + ) + r3 = self.response_class( + "http://www.example.com", + body=b"\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") - r5 = self.response_class("http://www.example.com", body=b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=None"]}) - r6 = self.response_class("http://www.example.com", body=b"\xa8D", - headers={"Content-type": ["text/html; charset=gb2312"]}) - r7 = self.response_class("http://www.example.com", body=b"\xa8D", - headers={"Content-type": ["text/html; charset=gbk"]}) - r8 = self.response_class("http://www.example.com", body=codecs.BOM_UTF8 + b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=cp1251"]}) + r5 = self.response_class( + "http://www.example.com", + body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=None"]}, + ) + r6 = self.response_class( + "http://www.example.com", + body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gb2312"]}, + ) + r7 = self.response_class( + "http://www.example.com", + body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gbk"]}, + ) + r8 = self.response_class( + "http://www.example.com", + body=codecs.BOM_UTF8 + b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=cp1251"]}, + ) self.assertEqual(r1._headers_encoding(), "utf-8") self.assertEqual(r2._headers_encoding(), None) - self.assertEqual(r2._declared_encoding(), 'utf-8') - self._assert_response_encoding(r2, 'utf-8') + self.assertEqual(r2._declared_encoding(), "utf-8") + self._assert_response_encoding(r2, "utf-8") self.assertEqual(r3._headers_encoding(), "cp1252") self.assertEqual(r3._declared_encoding(), "cp1252") self.assertEqual(r4._headers_encoding(), None) @@ -374,81 +447,96 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r8._declared_encoding(), "utf-8") self._assert_response_encoding(r5, "utf-8") self._assert_response_encoding(r8, "utf-8") - assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' - self._assert_response_values(r1, 'utf-8', "\xa3") - self._assert_response_values(r2, 'utf-8', "\xa3") - self._assert_response_values(r3, 'iso-8859-1', "\xa3") - self._assert_response_values(r6, 'gb18030', "\u2015") - self._assert_response_values(r7, 'gb18030', "\u2015") + assert ( + r4._body_inferred_encoding() is not None + and r4._body_inferred_encoding() != "ascii" + ) + self._assert_response_values(r1, "utf-8", "\xa3") + self._assert_response_values(r2, "utf-8", "\xa3") + self._assert_response_values(r3, "iso-8859-1", "\xa3") + self._assert_response_values(r6, "gb18030", "\u2015") + self._assert_response_values(r7, "gb18030", "\u2015") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies - self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3") + self.assertRaises( + TypeError, self.response_class, "http://www.example.com", body="\xa3" + ) def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" - r = self.response_class("http://www.example.com", - headers={"Content-type": ["text/html; charset=UNKNOWN"]}, - body=b"\xc2\xa3") + r = self.response_class( + "http://www.example.com", + headers={"Content-type": ["text/html; charset=UNKNOWN"]}, + body=b"\xc2\xa3", + ) self.assertEqual(r._declared_encoding(), None) - self._assert_response_values(r, 'utf-8', "\xa3") + self._assert_response_values(r, "utf-8", "\xa3") def test_utf16(self): """Test utf-16 because UnicodeDammit is known to have problems with""" - r = self.response_class("http://www.example.com", - body=b'\xff\xfeh\x00i\x00', - encoding='utf-16') - self._assert_response_values(r, 'utf-16', "hi") + r = self.response_class( + "http://www.example.com", body=b"\xff\xfeh\x00i\x00", encoding="utf-16" + ) + self._assert_response_values(r, "utf-16", "hi") def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): - r6 = self.response_class("http://www.example.com", - headers={"Content-type": ["text/html; charset=utf-8"]}, - body=b"\xef\xbb\xbfWORD\xe3\xab") - self.assertEqual(r6.encoding, 'utf-8') - self.assertIn(r6.text, { - 'WORD\ufffd\ufffd', # w3lib < 1.19.0 - 'WORD\ufffd', # w3lib >= 1.19.0 - }) + r6 = self.response_class( + "http://www.example.com", + headers={"Content-type": ["text/html; charset=utf-8"]}, + body=b"\xef\xbb\xbfWORD\xe3\xab", + ) + self.assertEqual(r6.encoding, "utf-8") + self.assertIn( + r6.text, + { + "WORD\ufffd\ufffd", # w3lib < 1.19.0 + "WORD\ufffd", # w3lib >= 1.19.0 + }, + ) def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, # this test tries to ensure that calling response.encoding and # response.text in indistint order doesn't affect final # values for encoding and decoded body. - url = 'http://example.com' + url = "http://example.com" body = b"\xef\xbb\xbfWORD" headers = {"Content-type": ["text/html; charset=utf-8"]} # Test response without content-type and BOM encoding response = self.response_class(url, body=body) - self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.text, 'WORD') + self.assertEqual(response.encoding, "utf-8") + self.assertEqual(response.text, "WORD") response = self.response_class(url, body=body) - self.assertEqual(response.text, 'WORD') - self.assertEqual(response.encoding, 'utf-8') + self.assertEqual(response.text, "WORD") + self.assertEqual(response.encoding, "utf-8") # Body caching sideeffect isn't triggered when encoding is declared in # content-type header but BOM still need to be removed from decoded # body response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.text, 'WORD') + self.assertEqual(response.encoding, "utf-8") + self.assertEqual(response.text, "WORD") response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.text, 'WORD') - self.assertEqual(response.encoding, 'utf-8') + self.assertEqual(response.text, "WORD") + self.assertEqual(response.encoding, "utf-8") def test_replace_wrong_encoding(self): """Test invalid chars are replaced properly""" - r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX') + r = self.response_class( + "http://www.example.com", encoding="utf-8", body=b"PREFIX\xe3\xabSUFFIX" + ) # XXX: Policy for replacing invalid chars may suffer minor variations # but it should always contain the unicode replacement char ('\ufffd') - assert '\ufffd' in r.text, repr(r.text) - assert 'PREFIX' in r.text, repr(r.text) - assert 'SUFFIX' in r.text, repr(r.text) + assert "\ufffd" in r.text, repr(r.text) + assert "PREFIX" in r.text, repr(r.text) + assert "SUFFIX" in r.text, repr(r.text) # Do not destroy html tags due to encoding bugs - r = self.response_class("http://example.com", encoding='utf-8', - body=b'\xf0value') - assert 'value' in r.text, repr(r.text) + r = self.response_class( + "http://example.com", encoding="utf-8", body=b"\xf0value" + ) + assert "value" in r.text, repr(r.text) # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') @@ -459,22 +547,15 @@ class TextResponseTest(BaseResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertIsInstance(response.selector, Selector) - self.assertEqual(response.selector.type, 'html') + self.assertEqual(response.selector.type, "html") self.assertIs(response.selector, response.selector) # property is cached self.assertIs(response.selector.response, response) self.assertEqual( - response.selector.xpath("//title/text()").getall(), - ['Some page'] - ) - self.assertEqual( - response.selector.css("title::text").getall(), - ['Some page'] - ) - self.assertEqual( - response.selector.re("Some (.*)"), - ['page'] + response.selector.xpath("//title/text()").getall(), ["Some page"] ) + self.assertEqual(response.selector.css("title::text").getall(), ["Some page"]) + self.assertEqual(response.selector.re("Some (.*)"), ["page"]) def test_selector_shortcuts(self): body = b"Some page" @@ -490,151 +571,163 @@ class TextResponseTest(BaseResponseTest): ) def test_selector_shortcuts_kwargs(self): - body = b"Some page

A nice paragraph.

" + body = b'Some page

A nice paragraph.

' response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(), - response.xpath("normalize-space(//p[@class=\"content\"])").getall(), + response.xpath( + "normalize-space(//p[@class=$pclass])", pclass="content" + ).getall(), + response.xpath('normalize-space(//p[@class="content"])').getall(), ) self.assertEqual( response.xpath( "//title[count(following::p[@class=$pclass])=$pcount]/text()", - pclass="content", pcount=1, + pclass="content", + pcount=1, + ).getall(), + response.xpath( + '//title[count(following::p[@class="content"])=1]/text()' ).getall(), - response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(), ) def test_urljoin_with_base_url(self): """Test urljoin shortcut which also evaluates base-url through get_base_url().""" body = b'' - joined = self.response_class('http://www.example.com', body=body).urljoin('/test') - absolute = 'https://example.net/test' + joined = self.response_class("http://www.example.com", body=body).urljoin( + "/test" + ) + absolute = "https://example.net/test" self.assertEqual(joined, absolute) body = b'' - joined = self.response_class('http://www.example.com', body=body).urljoin('test') - absolute = 'http://www.example.com/test' + joined = self.response_class("http://www.example.com", body=body).urljoin( + "test" + ) + absolute = "http://www.example.com/test" self.assertEqual(joined, absolute) body = b'' - joined = self.response_class('http://www.example.com', body=body).urljoin('test') - absolute = 'http://www.example.com/elsewhere/test' + joined = self.response_class("http://www.example.com", body=body).urljoin( + "test" + ) + absolute = "http://www.example.com/elsewhere/test" self.assertEqual(joined, absolute) def test_follow_selector(self): resp = self._links_response() urls = [ - 'http://example.com/sample2.html', - 'http://example.com/sample3.html', - 'http://example.com/sample3.html', - 'http://example.com/sample3.html#foo', - 'http://www.google.com/something', - 'http://example.com/innertag.html' + "http://example.com/sample2.html", + "http://example.com/sample3.html", + "http://example.com/sample3.html", + "http://example.com/sample3.html#foo", + "http://www.google.com/something", + "http://example.com/innertag.html", ] # select elements - for sellist in [resp.css('a'), resp.xpath('//a')]: + for sellist in [resp.css("a"), resp.xpath("//a")]: for sel, url in zip(sellist, urls): self._assert_followed_url(sel, url, response=resp) # select elements self._assert_followed_url( - Selector(text='').css('link')[0], - 'http://example.com/foo', - response=resp + Selector(text='').css("link")[0], + "http://example.com/foo", + response=resp, ) # href attributes should work - for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]: + for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]: for sel, url in zip(sellist, urls): self._assert_followed_url(sel, url, response=resp) # non-a elements are not supported - self.assertRaises(ValueError, resp.follow, resp.css('div')[0]) + self.assertRaises(ValueError, resp.follow, resp.css("div")[0]) def test_follow_selector_list(self): resp = self._links_response() - self.assertRaisesRegex(ValueError, 'SelectorList', - resp.follow, resp.css('a')) + self.assertRaisesRegex(ValueError, "SelectorList", resp.follow, resp.css("a")) def test_follow_selector_invalid(self): resp = self._links_response() - self.assertRaisesRegex(ValueError, 'Unsupported', - resp.follow, resp.xpath('count(//div)')[0]) + self.assertRaisesRegex( + ValueError, "Unsupported", resp.follow, resp.xpath("count(//div)")[0] + ) def test_follow_selector_attribute(self): resp = self._links_response() - for src in resp.css('img::attr(src)'): - self._assert_followed_url(src, 'http://example.com/sample2.jpg') + for src in resp.css("img::attr(src)"): + self._assert_followed_url(src, "http://example.com/sample2.jpg") def test_follow_selector_no_href(self): resp = self.response_class( - url='http://example.com', - body=b'click me', + url="http://example.com", + body=b"click me", ) - self.assertRaisesRegex(ValueError, 'no href', - resp.follow, resp.css('a')[0]) + self.assertRaisesRegex(ValueError, "no href", resp.follow, resp.css("a")[0]) def test_follow_whitespace_selector(self): resp = self.response_class( - 'http://example.com', - body=b'''click me''' + "http://example.com", + body=b"""click me""", ) self._assert_followed_url( - resp.css('a')[0], - 'http://example.com/foo', - response=resp) + resp.css("a")[0], "http://example.com/foo", response=resp + ) self._assert_followed_url( - resp.css('a::attr(href)')[0], - 'http://example.com/foo', - response=resp) + resp.css("a::attr(href)")[0], "http://example.com/foo", response=resp + ) def test_follow_encoding(self): resp1 = self.response_class( - 'http://example.com', - encoding='utf8', - body='click me'.encode('utf8') + "http://example.com", + encoding="utf8", + body='click me'.encode( + "utf8" + ), ) req = self._assert_followed_url( - resp1.css('a')[0], - 'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82', + resp1.css("a")[0], + "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82", response=resp1, ) - self.assertEqual(req.encoding, 'utf8') + self.assertEqual(req.encoding, "utf8") resp2 = self.response_class( - 'http://example.com', - encoding='cp1251', - body='click me'.encode('cp1251') + "http://example.com", + encoding="cp1251", + body='click me'.encode( + "cp1251" + ), ) req = self._assert_followed_url( - resp2.css('a')[0], - 'http://example.com/foo?%EF%F0%E8%E2%E5%F2', + resp2.css("a")[0], + "http://example.com/foo?%EF%F0%E8%E2%E5%F2", response=resp2, ) - self.assertEqual(req.encoding, 'cp1251') + self.assertEqual(req.encoding, "cp1251") def test_follow_flags(self): - res = self.response_class('http://example.com/') - fol = res.follow('http://example.com/', flags=['cached', 'allowed']) - self.assertEqual(fol.flags, ['cached', 'allowed']) + res = self.response_class("http://example.com/") + fol = res.follow("http://example.com/", flags=["cached", "allowed"]) + self.assertEqual(fol.flags, ["cached", "allowed"]) def test_follow_all_flags(self): - re = self.response_class('http://www.example.com/') + re = self.response_class("http://www.example.com/") urls = [ - 'http://www.example.com/', - 'http://www.example.com/2', - 'http://www.example.com/foo', + "http://www.example.com/", + "http://www.example.com/2", + "http://www.example.com/foo", ] - fol = re.follow_all(urls, flags=['cached', 'allowed']) + fol = re.follow_all(urls, flags=["cached", "allowed"]) for req in fol: - self.assertEqual(req.flags, ['cached', 'allowed']) + self.assertEqual(req.flags, ["cached", "allowed"]) def test_follow_all_css(self): expected = [ - 'http://example.com/sample3.html', - 'http://example.com/innertag.html', + "http://example.com/sample3.html", + "http://example.com/innertag.html", ] response = self._links_response() extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')] @@ -642,20 +735,20 @@ class TextResponseTest(BaseResponseTest): def test_follow_all_css_skip_invalid(self): expected = [ - 'http://example.com/page/1/', - 'http://example.com/page/3/', - 'http://example.com/page/4/', + "http://example.com/page/1/", + "http://example.com/page/3/", + "http://example.com/page/4/", ] response = self._links_response_no_href() - extracted1 = [r.url for r in response.follow_all(css='.pagination a')] + extracted1 = [r.url for r in response.follow_all(css=".pagination a")] self.assertEqual(expected, extracted1) - extracted2 = [r.url for r in response.follow_all(response.css('.pagination a'))] + extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))] self.assertEqual(expected, extracted2) def test_follow_all_xpath(self): expected = [ - 'http://example.com/sample3.html', - 'http://example.com/innertag.html', + "http://example.com/sample3.html", + "http://example.com/innertag.html", ] response = self._links_response() extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]') @@ -663,25 +756,33 @@ class TextResponseTest(BaseResponseTest): def test_follow_all_xpath_skip_invalid(self): expected = [ - 'http://example.com/page/1/', - 'http://example.com/page/3/', - 'http://example.com/page/4/', + "http://example.com/page/1/", + "http://example.com/page/3/", + "http://example.com/page/4/", ] response = self._links_response_no_href() - extracted1 = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')] + extracted1 = [ + r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a') + ] self.assertEqual(expected, extracted1) - extracted2 = [r.url for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))] + extracted2 = [ + r.url + for r in response.follow_all(response.xpath('//div[@id="pagination"]/a')) + ] self.assertEqual(expected, extracted2) def test_follow_all_too_many_arguments(self): response = self._links_response() with self.assertRaises(ValueError): - response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]') + response.follow_all( + css='a[href*="example.com"]', + xpath='//a[contains(@href, "example.com")]', + ) def test_json_response(self): json_body = b"""{"ip": "109.187.217.200"}""" json_response = self.response_class("http://www.example.com", body=json_body) - self.assertEqual(json_response.json(), {'ip': '109.187.217.200'}) + self.assertEqual(json_response.json(), {"ip": "109.187.217.200"}) text_body = b"""text""" text_response = self.response_class("http://www.example.com", body=text_body) @@ -691,9 +792,11 @@ class TextResponseTest(BaseResponseTest): def test_cache_json_response(self): json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""] for json_body in json_valid_bodies: - json_response = self.response_class("http://www.example.com", body=json_body) + json_response = self.response_class( + "http://www.example.com", body=json_body + ) - with mock.patch('json.loads') as mock_json: + with mock.patch("json.loads") as mock_json: for _ in range(2): json_response.json() mock_json.assert_called_once_with(json_body.decode()) @@ -710,33 +813,36 @@ class HtmlResponseTest(TextResponseTest): Price: \xa3100' """ r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, 'iso-8859-1', body) + self._assert_response_values(r1, "iso-8859-1", body) body = b""" Price: \xa3100 """ r2 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r2, 'iso-8859-1', body) + self._assert_response_values(r2, "iso-8859-1", body) # for conflicting declarations headers must take precedence body = b"""Some page Price: \xa3100' """ - r3 = self.response_class("http://www.example.com", body=body, - headers={"Content-type": ["text/html; charset=iso-8859-1"]}) - self._assert_response_values(r3, 'iso-8859-1', body) + r3 = self.response_class( + "http://www.example.com", + body=body, + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) + self._assert_response_values(r3, "iso-8859-1", body) # make sure replace() preserves the encoding of the original response body = b"New body \xa3" r4 = r3.replace(body=body) - self._assert_response_values(r4, 'iso-8859-1', body) + self._assert_response_values(r4, "iso-8859-1", body) def test_html5_meta_charset(self): body = b"""Some pagebla bla""" r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, 'gb2312', body) + self._assert_response_values(r1, "gb2312", body) class XmlResponseTest(TextResponseTest): @@ -750,14 +856,14 @@ class XmlResponseTest(TextResponseTest): body = b"""""" r2 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r2, 'iso-8859-1', body) + self._assert_response_values(r2, "iso-8859-1", body) # make sure replace() preserves the explicit encoding passed in the __init__ method body = b"""""" - r3 = self.response_class("http://www.example.com", body=body, encoding='utf-8') + r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8") body2 = b"New body" r4 = r3.replace(body=body2) - self._assert_response_values(r4, 'utf-8', body2) + self._assert_response_values(r4, "utf-8", body2) def test_replace_encoding(self): # make sure replace() keeps the previous encoding unless overridden explicitly @@ -765,24 +871,21 @@ class XmlResponseTest(TextResponseTest): body2 = b"""""" r5 = self.response_class("http://www.example.com", body=body) r6 = r5.replace(body=body2) - r7 = r5.replace(body=body2, encoding='utf-8') - self._assert_response_values(r5, 'iso-8859-1', body) - self._assert_response_values(r6, 'iso-8859-1', body2) - self._assert_response_values(r7, 'utf-8', body2) + r7 = r5.replace(body=body2, encoding="utf-8") + self._assert_response_values(r5, "iso-8859-1", body) + self._assert_response_values(r6, "iso-8859-1", body2) + self._assert_response_values(r7, "utf-8", body2) def test_selector(self): body = b'value' response = self.response_class("http://www.example.com", body=body) self.assertIsInstance(response.selector, Selector) - self.assertEqual(response.selector.type, 'xml') + self.assertEqual(response.selector.type, "xml") self.assertIs(response.selector, response.selector) # property is cached self.assertIs(response.selector.response, response) - self.assertEqual( - response.selector.xpath("//elem/text()").getall(), - ['value'] - ) + self.assertEqual(response.selector.xpath("//elem/text()").getall(), ["value"]) def test_selector_shortcuts(self): body = b'value' @@ -794,20 +897,26 @@ class XmlResponseTest(TextResponseTest): ) def test_selector_shortcuts_kwargs(self): - body = b''' + body = b""" value - ''' + """ response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(), - response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(), + response.xpath( + "//s:elem/text()", namespaces={"s": "http://scrapy.org"} + ).getall(), + response.selector.xpath( + "//s:elem/text()", namespaces={"s": "http://scrapy.org"} + ).getall(), ) - response.selector.register_namespace('s2', 'http://scrapy.org') + response.selector.register_namespace("s2", "http://scrapy.org") self.assertEqual( - response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(), + response.xpath( + "//s1:elem/text()", namespaces={"s1": "http://scrapy.org"} + ).getall(), response.selector.xpath("//s2:elem/text()").getall(), ) @@ -827,7 +936,9 @@ class CustomResponseTest(TextResponseTest): def test_copy(self): super().test_copy() - r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost") + r1 = self.response_class( + url="https://example.org", status=200, foo="foo", bar="bar", lost="lost" + ) r2 = r1.copy() self.assertIsInstance(r2, self.response_class) self.assertEqual(r1.foo, r2.foo) @@ -837,7 +948,9 @@ class CustomResponseTest(TextResponseTest): def test_replace(self): super().test_replace() - r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost") + r1 = self.response_class( + url="https://example.org", status=200, foo="foo", bar="bar", lost="lost" + ) r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") self.assertIsInstance(r2, self.response_class) @@ -868,4 +981,8 @@ class CustomResponseTest(TextResponseTest): with self.assertRaises(TypeError) as ctx: r1.replace(unknown="unknown") - self.assertTrue(str(ctx.exception).endswith("__init__() got an unexpected keyword argument 'unknown'")) + self.assertTrue( + str(ctx.exception).endswith( + "__init__() got an unexpected keyword argument 'unknown'" + ) + ) diff --git a/tests/test_item.py b/tests/test_item.py index 25f2aea0a..ce2b4fd15 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -5,7 +5,6 @@ from scrapy.item import ABCMeta, Field, Item, ItemMeta class ItemTest(unittest.TestCase): - def assertSortedEqual(self, first, second, msg=None): return self.assertEqual(sorted(first), sorted(second), msg) @@ -14,35 +13,34 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - i['name'] = 'name' - self.assertEqual(i['name'], 'name') + i["name"] = "name" + self.assertEqual(i["name"], "name") def test_init(self): class TestItem(Item): name = Field() i = TestItem() - self.assertRaises(KeyError, i.__getitem__, 'name') + self.assertRaises(KeyError, i.__getitem__, "name") - i2 = TestItem(name='john doe') - self.assertEqual(i2['name'], 'john doe') + i2 = TestItem(name="john doe") + self.assertEqual(i2["name"], "john doe") - i3 = TestItem({'name': 'john doe'}) - self.assertEqual(i3['name'], 'john doe') + i3 = TestItem({"name": "john doe"}) + self.assertEqual(i3["name"], "john doe") i4 = TestItem(i3) - self.assertEqual(i4['name'], 'john doe') + self.assertEqual(i4["name"], "john doe") - self.assertRaises(KeyError, TestItem, {'name': 'john doe', - 'other': 'foo'}) + self.assertRaises(KeyError, TestItem, {"name": "john doe", "other": "foo"}) def test_invalid_field(self): class TestItem(Item): pass i = TestItem() - self.assertRaises(KeyError, i.__setitem__, 'field', 'text') - self.assertRaises(KeyError, i.__getitem__, 'field') + self.assertRaises(KeyError, i.__setitem__, "field", "text") + self.assertRaises(KeyError, i.__getitem__, "field") def test_repr(self): class TestItem(Item): @@ -50,55 +48,54 @@ class ItemTest(unittest.TestCase): number = Field() i = TestItem() - i['name'] = 'John Doe' - i['number'] = 123 + i["name"] = "John Doe" + i["number"] = 123 itemrepr = repr(i) - self.assertEqual(itemrepr, - "{'name': 'John Doe', 'number': 123}") + self.assertEqual(itemrepr, "{'name': 'John Doe', 'number': 123}") i2 = eval(itemrepr) - self.assertEqual(i2['name'], 'John Doe') - self.assertEqual(i2['number'], 123) + self.assertEqual(i2["name"], "John Doe") + self.assertEqual(i2["number"], 123) def test_private_attr(self): class TestItem(Item): name = Field() i = TestItem() - i._private = 'test' - self.assertEqual(i._private, 'test') + i._private = "test" + self.assertEqual(i._private, "test") def test_raise_getattr(self): class TestItem(Item): name = Field() i = TestItem() - self.assertRaises(AttributeError, getattr, i, 'name') + self.assertRaises(AttributeError, getattr, i, "name") def test_raise_setattr(self): class TestItem(Item): name = Field() i = TestItem() - self.assertRaises(AttributeError, setattr, i, 'name', 'john') + self.assertRaises(AttributeError, setattr, i, "name", "john") def test_custom_methods(self): class TestItem(Item): name = Field() def get_name(self): - return self['name'] + return self["name"] def change_name(self, name): - self['name'] = name + self["name"] = name i = TestItem() self.assertRaises(KeyError, i.get_name) - i['name'] = 'lala' - self.assertEqual(i.get_name(), 'lala') - i.change_name('other') - self.assertEqual(i.get_name(), 'other') + i["name"] = "lala" + self.assertEqual(i.get_name(), "lala") + i.change_name("other") + self.assertEqual(i.get_name(), "other") def test_metaclass(self): class TestItem(Item): @@ -107,22 +104,22 @@ class ItemTest(unittest.TestCase): values = Field() i = TestItem() - i['name'] = 'John' - self.assertEqual(list(i.keys()), ['name']) - self.assertEqual(list(i.values()), ['John']) + i["name"] = "John" + self.assertEqual(list(i.keys()), ["name"]) + self.assertEqual(list(i.values()), ["John"]) - i['keys'] = 'Keys' - i['values'] = 'Values' - self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name']) - self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John']) + i["keys"] = "Keys" + i["values"] = "Values" + self.assertSortedEqual(list(i.keys()), ["keys", "values", "name"]) + self.assertSortedEqual(list(i.values()), ["Keys", "Values", "John"]) def test_metaclass_with_fields_attribute(self): class TestItem(Item): - fields = {'new': Field(default='X')} + fields = {"new": Field(default="X")} - item = TestItem(new='New') - self.assertSortedEqual(list(item.keys()), ['new']) - self.assertSortedEqual(list(item.values()), ['New']) + item = TestItem(new="New") + self.assertSortedEqual(list(item.keys()), ["new"]) + self.assertSortedEqual(list(item.values()), ["New"]) def test_metaclass_inheritance(self): class ParentItem(Item): @@ -134,151 +131,163 @@ class ItemTest(unittest.TestCase): keys = Field() i = TestItem() - i['keys'] = 3 - self.assertEqual(list(i.keys()), ['keys']) + i["keys"] = 3 + self.assertEqual(list(i.keys()), ["keys"]) self.assertEqual(list(i.values()), [3]) def test_metaclass_multiple_inheritance_simple(self): class A(Item): - fields = {'load': Field(default='A')} - save = Field(default='A') + fields = {"load": Field(default="A")} + save = Field(default="A") class B(A): pass class C(Item): - fields = {'load': Field(default='C')} - save = Field(default='C') + fields = {"load": Field(default="C")} + save = Field(default="C") class D(B, C): pass - item = D(save='X', load='Y') - self.assertEqual(item['save'], 'X') - self.assertEqual(item['load'], 'Y') - self.assertEqual(D.fields, {'load': {'default': 'A'}, 'save': {'default': 'A'}}) + item = D(save="X", load="Y") + self.assertEqual(item["save"], "X") + self.assertEqual(item["load"], "Y") + self.assertEqual(D.fields, {"load": {"default": "A"}, "save": {"default": "A"}}) # D class inverted class E(C, B): pass - self.assertEqual(E(save='X')['save'], 'X') - self.assertEqual(E(load='X')['load'], 'X') - self.assertEqual(E.fields, {'load': {'default': 'C'}, 'save': {'default': 'C'}}) + self.assertEqual(E(save="X")["save"], "X") + self.assertEqual(E(load="X")["load"], "X") + self.assertEqual(E.fields, {"load": {"default": "C"}, "save": {"default": "C"}}) def test_metaclass_multiple_inheritance_diamond(self): class A(Item): - fields = {'update': Field(default='A')} - save = Field(default='A') - load = Field(default='A') + fields = {"update": Field(default="A")} + save = Field(default="A") + load = Field(default="A") class B(A): pass class C(A): - fields = {'update': Field(default='C')} - save = Field(default='C') + fields = {"update": Field(default="C")} + save = Field(default="C") class D(B, C): - fields = {'update': Field(default='D')} - load = Field(default='D') + fields = {"update": Field(default="D")} + load = Field(default="D") - self.assertEqual(D(save='X')['save'], 'X') - self.assertEqual(D(load='X')['load'], 'X') + self.assertEqual(D(save="X")["save"], "X") + self.assertEqual(D(load="X")["load"], "X") self.assertEqual( D.fields, - {'save': {'default': 'C'}, 'load': {'default': 'D'}, 'update': {'default': 'D'}}) + { + "save": {"default": "C"}, + "load": {"default": "D"}, + "update": {"default": "D"}, + }, + ) # D class inverted class E(C, B): - load = Field(default='E') + load = Field(default="E") - self.assertEqual(E(save='X')['save'], 'X') - self.assertEqual(E(load='X')['load'], 'X') + self.assertEqual(E(save="X")["save"], "X") + self.assertEqual(E(load="X")["load"], "X") self.assertEqual( E.fields, - {'save': {'default': 'C'}, 'load': {'default': 'E'}, 'update': {'default': 'C'}}) + { + "save": {"default": "C"}, + "load": {"default": "E"}, + "update": {"default": "C"}, + }, + ) def test_metaclass_multiple_inheritance_without_metaclass(self): class A(Item): - fields = {'load': Field(default='A')} - save = Field(default='A') + fields = {"load": Field(default="A")} + save = Field(default="A") class B(A): pass class C: - fields = {'load': Field(default='C')} - not_allowed = Field(default='not_allowed') - save = Field(default='C') + fields = {"load": Field(default="C")} + not_allowed = Field(default="not_allowed") + save = Field(default="C") class D(B, C): pass - self.assertRaises(KeyError, D, not_allowed='value') - self.assertEqual(D(save='X')['save'], 'X') - self.assertEqual(D.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}}) + self.assertRaises(KeyError, D, not_allowed="value") + self.assertEqual(D(save="X")["save"], "X") + self.assertEqual(D.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) # D class inverted class E(C, B): pass - self.assertRaises(KeyError, E, not_allowed='value') - self.assertEqual(E(save='X')['save'], 'X') - self.assertEqual(E.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}}) + self.assertRaises(KeyError, E, not_allowed="value") + self.assertEqual(E(save="X")["save"], "X") + self.assertEqual(E.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) def test_to_dict(self): class TestItem(Item): name = Field() i = TestItem() - i['name'] = 'John' - self.assertEqual(dict(i), {'name': 'John'}) + i["name"] = "John" + self.assertEqual(dict(i), {"name": "John"}) def test_copy(self): class TestItem(Item): name = Field() - item = TestItem({'name': 'lower'}) + + item = TestItem({"name": "lower"}) copied_item = item.copy() self.assertNotEqual(id(item), id(copied_item)) - copied_item['name'] = copied_item['name'].upper() - self.assertNotEqual(item['name'], copied_item['name']) + copied_item["name"] = copied_item["name"].upper() + self.assertNotEqual(item["name"], copied_item["name"]) def test_deepcopy(self): class TestItem(Item): tags = Field() - item = TestItem({'tags': ['tag1']}) + + item = TestItem({"tags": ["tag1"]}) copied_item = item.deepcopy() - item['tags'].append('tag2') - assert item['tags'] != copied_item['tags'] + item["tags"].append("tag2") + assert item["tags"] != copied_item["tags"] class ItemMetaTest(unittest.TestCase): - def test_new_method_propagates_classcell(self): new_mock = mock.Mock(side_effect=ABCMeta.__new__) base = ItemMeta.__bases__[0] - with mock.patch.object(base, '__new__', new_mock): + with mock.patch.object(base, "__new__", new_mock): class MyItem(Item): def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 - return __class__ # noqa https://github.com/scrapy/scrapy/issues/2836 + return ( + __class__ # noqa https://github.com/scrapy/scrapy/issues/2836 + ) MyItem() (first_call, second_call) = new_mock.call_args_list[-2:] mcs, class_name, bases, attrs = first_call[0] - assert '__classcell__' not in attrs + assert "__classcell__" not in attrs mcs, class_name, bases, attrs = second_call[0] - assert '__classcell__' in attrs + assert "__classcell__" in attrs class ItemMetaClassCellRegression(unittest.TestCase): - def test_item_meta_classcell_regression(self): class MyItem(Item, metaclass=ItemMeta): def __init__(self, *args, **kwargs): diff --git a/tests/test_link.py b/tests/test_link.py index e0f1efffa..7ba0851ae 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -4,7 +4,6 @@ from scrapy.link import Link class LinkTest(unittest.TestCase): - def _assert_same_links(self, link1, link2): self.assertEqual(link1, link2) self.assertEqual(hash(link1), hash(link2)) @@ -30,16 +29,26 @@ class LinkTest(unittest.TestCase): self._assert_different_links(l4, l5) self._assert_same_links(l4, l6) - l7 = Link("http://www.example.com", text="test", fragment='something', nofollow=False) - l8 = Link("http://www.example.com", text="test", fragment='something', nofollow=False) - l9 = Link("http://www.example.com", text="test", fragment='something', nofollow=True) - l10 = Link("http://www.example.com", text="test", fragment='other', nofollow=False) + l7 = Link( + "http://www.example.com", text="test", fragment="something", nofollow=False + ) + l8 = Link( + "http://www.example.com", text="test", fragment="something", nofollow=False + ) + l9 = Link( + "http://www.example.com", text="test", fragment="something", nofollow=True + ) + l10 = Link( + "http://www.example.com", text="test", fragment="other", nofollow=False + ) self._assert_same_links(l7, l8) self._assert_different_links(l7, l9) self._assert_different_links(l7, l10) def test_repr(self): - l1 = Link("http://www.example.com", text="test", fragment='something', nofollow=True) + l1 = Link( + "http://www.example.com", text="test", fragment="something", nofollow=True + ) l2 = eval(repr(l1)) self._assert_same_links(l1, l2) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index e28dc9bdb..520effedb 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -14,108 +14,163 @@ class Base: extractor_cls = None def setUp(self): - body = get_testdata('link_extractor', 'linkextractor.html') - self.response = HtmlResponse(url='http://example.com/index', body=body) + body = get_testdata("link_extractor", "linkextractor.html") + self.response = HtmlResponse(url="http://example.com/index", body=body) def test_urls_type(self): - ''' Test that the resulting urls are str objects ''' + """Test that the resulting urls are str objects""" lx = self.extractor_cls() - self.assertTrue(all(isinstance(link.url, str) - for link in lx.extract_links(self.response))) + self.assertTrue( + all( + isinstance(link.url, str) + for link in lx.extract_links(self.response) + ) + ) def test_extract_all_links(self): lx = self.extractor_cls() - page4_url = 'http://example.com/page%204.html' + page4_url = "http://example.com/page%204.html" - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - Link(url='http://example.com/sample3.html', text='sample 3 text'), - Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=''), - Link(url='http://example.com/innertag.html', text='inner tag'), - Link(url=page4_url, text='href with whitespaces'), - ]) + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + Link(url="http://www.google.com/something", text=""), + Link(url="http://example.com/innertag.html", text="inner tag"), + Link(url=page4_url, text="href with whitespaces"), + ], + ) def test_extract_filter_allow(self): - lx = self.extractor_cls(allow=('sample', )) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - Link(url='http://example.com/sample3.html', text='sample 3 text'), - Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') - ]) + lx = self.extractor_cls(allow=("sample",)) + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + ], + ) def test_extract_filter_allow_with_duplicates(self): - lx = self.extractor_cls(allow=('sample', ), unique=False) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - Link(url='http://example.com/sample3.html', text='sample 3 text'), - Link(url='http://example.com/sample3.html', text='sample 3 repetition'), - Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') - ]) + lx = self.extractor_cls(allow=("sample",), unique=False) + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + ], + ) def test_extract_filter_allow_with_duplicates_canonicalize(self): - lx = self.extractor_cls(allow=('sample', ), unique=False, - canonicalize=True) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - Link(url='http://example.com/sample3.html', text='sample 3 text'), - Link(url='http://example.com/sample3.html', text='sample 3 repetition'), - Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment') - ]) + lx = self.extractor_cls(allow=("sample",), unique=False, canonicalize=True) + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition with fragment", + ), + ], + ) def test_extract_filter_allow_no_duplicates_canonicalize(self): - lx = self.extractor_cls(allow=('sample',), unique=True, - canonicalize=True) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - Link(url='http://example.com/sample3.html', text='sample 3 text'), - ]) + lx = self.extractor_cls(allow=("sample",), unique=True, canonicalize=True) + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + ], + ) def test_extract_filter_allow_and_deny(self): - lx = self.extractor_cls(allow=('sample', ), deny=('3', )) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - ]) + lx = self.extractor_cls(allow=("sample",), deny=("3",)) + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ], + ) def test_extract_filter_allowed_domains(self): - lx = self.extractor_cls(allow_domains=('google.com', )) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=''), - ]) + lx = self.extractor_cls(allow_domains=("google.com",)) + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://www.google.com/something", text=""), + ], + ) def test_extraction_using_single_values(self): - '''Test the extractor's behaviour among different situations''' + """Test the extractor's behaviour among different situations""" - lx = self.extractor_cls(allow='sample') - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - Link(url='http://example.com/sample3.html', text='sample 3 text'), - Link(url='http://example.com/sample3.html#foo', - text='sample 3 repetition with fragment') - ]) + lx = self.extractor_cls(allow="sample") + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + ], + ) - lx = self.extractor_cls(allow='sample', deny='3') - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - ]) + lx = self.extractor_cls(allow="sample", deny="3") + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ], + ) - lx = self.extractor_cls(allow_domains='google.com') - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=''), - ]) + lx = self.extractor_cls(allow_domains="google.com") + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://www.google.com/something", text=""), + ], + ) - lx = self.extractor_cls(deny_domains='example.com') - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=''), - ]) + lx = self.extractor_cls(deny_domains="example.com") + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://www.google.com/something", text=""), + ], + ) def test_nofollow(self): '''Test the extractor's behaviour for links with rel="nofollow"''' @@ -141,48 +196,68 @@ class Base: response = HtmlResponse("http://example.org/somepage/index.html", body=html) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/about.html', text='About us'), - Link(url='http://example.org/follow.html', text='Follow this link'), - Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True), - Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'), - Link(url='http://google.com/something', text='External link not to follow', nofollow=True), - ]) + self.assertEqual( + lx.extract_links(response), + [ + Link(url="http://example.org/about.html", text="About us"), + Link(url="http://example.org/follow.html", text="Follow this link"), + Link( + url="http://example.org/nofollow.html", + text="Dont follow this one", + nofollow=True, + ), + Link( + url="http://example.org/nofollow2.html", + text="Choose to follow or not", + ), + Link( + url="http://google.com/something", + text="External link not to follow", + nofollow=True, + ), + ], + ) def test_matches(self): - url1 = 'http://lotsofstuff.com/stuff1/index' - url2 = 'http://evenmorestuff.com/uglystuff/index' + url1 = "http://lotsofstuff.com/stuff1/index" + url2 = "http://evenmorestuff.com/uglystuff/index" - lx = self.extractor_cls(allow=(r'stuff1', )) + lx = self.extractor_cls(allow=(r"stuff1",)) self.assertEqual(lx.matches(url1), True) self.assertEqual(lx.matches(url2), False) - lx = self.extractor_cls(deny=(r'uglystuff', )) + lx = self.extractor_cls(deny=(r"uglystuff",)) self.assertEqual(lx.matches(url1), True) self.assertEqual(lx.matches(url2), False) - lx = self.extractor_cls(allow_domains=('evenmorestuff.com', )) + lx = self.extractor_cls(allow_domains=("evenmorestuff.com",)) self.assertEqual(lx.matches(url1), False) self.assertEqual(lx.matches(url2), True) - lx = self.extractor_cls(deny_domains=('lotsofstuff.com', )) + lx = self.extractor_cls(deny_domains=("lotsofstuff.com",)) self.assertEqual(lx.matches(url1), False) self.assertEqual(lx.matches(url2), True) - lx = self.extractor_cls(allow=['blah1'], deny=['blah2'], - allow_domains=['blah1.com'], - deny_domains=['blah2.com']) - self.assertEqual(lx.matches('http://blah1.com/blah1'), True) - self.assertEqual(lx.matches('http://blah1.com/blah2'), False) - self.assertEqual(lx.matches('http://blah2.com/blah1'), False) - self.assertEqual(lx.matches('http://blah2.com/blah2'), False) + lx = self.extractor_cls( + allow=["blah1"], + deny=["blah2"], + allow_domains=["blah1.com"], + deny_domains=["blah2.com"], + ) + self.assertEqual(lx.matches("http://blah1.com/blah1"), True) + self.assertEqual(lx.matches("http://blah1.com/blah2"), False) + self.assertEqual(lx.matches("http://blah2.com/blah1"), False) + self.assertEqual(lx.matches("http://blah2.com/blah2"), False) def test_restrict_xpaths(self): - lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', )) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - ]) + lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',)) + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ], + ) def test_restrict_xpaths_encoding(self): """Test restrict_xpaths with encodings""" @@ -195,84 +270,143 @@ class Base:

This shouldn't be followed

""" - response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') + response = HtmlResponse( + "http://example.org/somepage/index.html", + body=html, + encoding="windows-1252", + ) lx = self.extractor_cls(restrict_xpaths="//div[@class='links']") - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/about.html', text='About us\xa3')]) + self.assertEqual( + lx.extract_links(response), + [Link(url="http://example.org/about.html", text="About us\xa3")], + ) def test_restrict_xpaths_with_html_entities(self): html = b'

text

' - response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15') - links = self.extractor_cls(restrict_xpaths='//p').extract_links(response) - self.assertEqual(links, - [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')]) + response = HtmlResponse( + "http://example.org/somepage/index.html", + body=html, + encoding="iso8859-15", + ) + links = self.extractor_cls(restrict_xpaths="//p").extract_links(response) + self.assertEqual( + links, [Link(url="http://example.org/%E2%99%A5/you?c=%A4", text="text")] + ) def test_restrict_xpaths_concat_in_handle_data(self): """html entities cause SGMLParser to call handle_data hook twice""" body = b"""
>\xbe\xa9<\xb6\xab""" - response = HtmlResponse("http://example.org", body=body, encoding='gb18030') + response = HtmlResponse("http://example.org", body=body, encoding="gb18030") lx = self.extractor_cls(restrict_xpaths="//div") - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/foo', text='>\u4eac<\u4e1c', - fragment='', nofollow=False)]) + self.assertEqual( + lx.extract_links(response), + [ + Link( + url="http://example.org/foo", + text=">\u4eac<\u4e1c", + fragment="", + nofollow=False, + ) + ], + ) def test_restrict_css(self): - lx = self.extractor_cls(restrict_css=('#subwrapper a',)) - self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample2.html', text='sample 2') - ]) + lx = self.extractor_cls(restrict_css=("#subwrapper a",)) + self.assertEqual( + lx.extract_links(self.response), + [Link(url="http://example.com/sample2.html", text="sample 2")], + ) def test_restrict_css_and_restrict_xpaths_together(self): - lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ), - restrict_css=('#subwrapper + a', )) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - Link(url='http://example.com/sample3.html', text='sample 3 text'), - ]) + lx = self.extractor_cls( + restrict_xpaths=('//div[@id="subwrapper"]',), + restrict_css=("#subwrapper + a",), + ) + self.assertEqual( + [link for link in lx.extract_links(self.response)], + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + ], + ) def test_area_tag_with_unicode_present(self): body = b"""\xbe\xa9""" - response = HtmlResponse("http://example.org", body=body, encoding='utf-8') + response = HtmlResponse("http://example.org", body=body, encoding="utf-8") lx = self.extractor_cls() lx.extract_links(response) lx.extract_links(response) lx.extract_links(response) - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/foo', text='', - fragment='', nofollow=False)]) + self.assertEqual( + lx.extract_links(response), + [ + Link( + url="http://example.org/foo", + text="", + fragment="", + nofollow=False, + ) + ], + ) def test_encoded_url(self): body = b"""
BinB""" - response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') + response = HtmlResponse( + "http://known.fm/AC%2FDC/", body=body, encoding="utf8" + ) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False), - ]) + self.assertEqual( + lx.extract_links(response), + [ + Link( + url="http://known.fm/AC%2FDC/?page=2", + text="BinB", + fragment="", + nofollow=False, + ), + ], + ) def test_encoded_url_in_restricted_xpath(self): body = b"""
BinB""" - response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') + response = HtmlResponse( + "http://known.fm/AC%2FDC/", body=body, encoding="utf8" + ) lx = self.extractor_cls(restrict_xpaths="//div") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False), - ]) + self.assertEqual( + lx.extract_links(response), + [ + Link( + url="http://known.fm/AC%2FDC/?page=2", + text="BinB", + fragment="", + nofollow=False, + ), + ], + ) def test_ignored_extensions(self): # jpg is ignored by default html = b"""asd and """ response = HtmlResponse("http://example.org/", body=html) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/page.html', text='asd'), - ]) + self.assertEqual( + lx.extract_links(response), + [ + Link(url="http://example.org/page.html", text="asd"), + ], + ) # override denied extensions - lx = self.extractor_cls(deny_extensions=['html']) - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/photo.jpg'), - ]) + lx = self.extractor_cls(deny_extensions=["html"]) + self.assertEqual( + lx.extract_links(response), + [ + Link(url="http://example.org/photo.jpg"), + ], + ) def test_process_value(self): """Test restrict_xpaths with encodings""" @@ -280,7 +414,11 @@ class Base: Text About us """ - response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') + response = HtmlResponse( + "http://example.org/somepage/index.html", + body=html, + encoding="windows-1252", + ) def process_value(value): m = re.search(r"javascript:goToPage\('(.*?)'", value) @@ -288,8 +426,10 @@ class Base: return m.group(1) lx = self.extractor_cls(process_value=process_value) - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/other/page.html', text='Text')]) + self.assertEqual( + lx.extract_links(response), + [Link(url="http://example.org/other/page.html", text="Text")], + ) def test_base_url_with_restrict_xpaths(self): html = b"""Page title<title><base href="http://otherdomain.com/base/" /> @@ -297,34 +437,50 @@ class Base: </body></html>""" response = HtmlResponse("http://example.org/somepage/index.html", body=html) lx = self.extractor_cls(restrict_xpaths="//p") - self.assertEqual(lx.extract_links(response), - [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) + self.assertEqual( + lx.extract_links(response), + [Link(url="http://otherdomain.com/base/item/12.html", text="Item 12")], + ) def test_attrs(self): lx = self.extractor_cls(attrs="href") - page4_url = 'http://example.com/page%204.html' + page4_url = "http://example.com/page%204.html" - self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - Link(url='http://example.com/sample3.html', text='sample 3 text'), - Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=''), - Link(url='http://example.com/innertag.html', text='inner tag'), - Link(url=page4_url, text='href with whitespaces'), - ]) + self.assertEqual( + lx.extract_links(self.response), + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + Link(url="http://www.google.com/something", text=""), + Link(url="http://example.com/innertag.html", text="inner tag"), + Link(url=page4_url, text="href with whitespaces"), + ], + ) - lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=()) - self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - Link(url='http://example.com/sample2.jpg', text=''), - Link(url='http://example.com/sample3.html', text='sample 3 text'), - Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=''), - Link(url='http://example.com/innertag.html', text='inner tag'), - Link(url=page4_url, text='href with whitespaces'), - ]) + lx = self.extractor_cls( + attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=() + ) + self.assertEqual( + lx.extract_links(self.response), + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample2.jpg", text=""), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + Link(url="http://www.google.com/something", text=""), + Link(url="http://example.com/innertag.html", text="inner tag"), + Link(url=page4_url, text="href with whitespaces"), + ], + ) lx = self.extractor_cls(attrs=None) self.assertEqual(lx.extract_links(self.response), []) @@ -340,26 +496,40 @@ class Base: self.assertEqual(lx.extract_links(response), []) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=''), - Link(url='http://example.com/sample2.html', text='sample 2'), - ]) + self.assertEqual( + lx.extract_links(response), + [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ], + ) lx = self.extractor_cls(tags="area") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=''), - ]) + self.assertEqual( + lx.extract_links(response), + [ + Link(url="http://example.com/sample1.html", text=""), + ], + ) lx = self.extractor_cls(tags="a") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample2.html', text='sample 2'), - ]) + self.assertEqual( + lx.extract_links(response), + [ + Link(url="http://example.com/sample2.html", text="sample 2"), + ], + ) - lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=()) - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample2.html', text='sample 2'), - Link(url='http://example.com/sample2.jpg', text=''), - ]) + lx = self.extractor_cls( + tags=("a", "img"), attrs=("href", "src"), deny_extensions=() + ) + self.assertEqual( + lx.extract_links(response), + [ + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample2.jpg", text=""), + ], + ) def test_tags_attrs(self): html = b""" @@ -370,17 +540,43 @@ class Base: """ response = HtmlResponse("http://example.com/index.html", body=html) - lx = self.extractor_cls(tags='div', attrs='data-url') - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False), - Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False) - ]) + lx = self.extractor_cls(tags="div", attrs="data-url") + self.assertEqual( + lx.extract_links(response), + [ + Link( + url="http://example.com/get?id=1", + text="Item 1", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/get?id=2", + text="Item 2", + fragment="", + nofollow=False, + ), + ], + ) - lx = self.extractor_cls(tags=('div',), attrs=('data-url',)) - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False), - Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False) - ]) + lx = self.extractor_cls(tags=("div",), attrs=("data-url",)) + self.assertEqual( + lx.extract_links(response), + [ + Link( + url="http://example.com/get?id=1", + text="Item 1", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/get?id=2", + text="Item 2", + fragment="", + nofollow=False, + ), + ], + ) def test_xhtml(self): xhtml = b""" @@ -417,14 +613,36 @@ class Base: self.assertEqual( lx.extract_links(response), [ - Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text='Dont follow this one', - fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text='Choose to follow or not', - fragment='', nofollow=False), - Link(url='http://google.com/something', text='External link not to follow', nofollow=True), - ] + Link( + url="http://example.com/about.html", + text="About us", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/follow.html", + text="Follow this link", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/nofollow.html", + text="Dont follow this one", + fragment="", + nofollow=True, + ), + Link( + url="http://example.com/nofollow2.html", + text="Choose to follow or not", + fragment="", + nofollow=False, + ), + Link( + url="http://google.com/something", + text="External link not to follow", + nofollow=True, + ), + ], ) response = XmlResponse("http://example.com/index.xhtml", body=xhtml) @@ -433,14 +651,36 @@ class Base: self.assertEqual( lx.extract_links(response), [ - Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text='Dont follow this one', - fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text='Choose to follow or not', - fragment='', nofollow=False), - Link(url='http://google.com/something', text='External link not to follow', nofollow=True), - ] + Link( + url="http://example.com/about.html", + text="About us", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/follow.html", + text="Follow this link", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/nofollow.html", + text="Dont follow this one", + fragment="", + nofollow=True, + ), + Link( + url="http://example.com/nofollow2.html", + text="Choose to follow or not", + fragment="", + nofollow=False, + ), + Link( + url="http://google.com/something", + text="External link not to follow", + nofollow=True, + ), + ], ) def test_link_wrong_href(self): @@ -451,21 +691,42 @@ class Base: """ response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text='Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text='Item 3', nofollow=False), - ]) + self.assertEqual( + [link for link in lx.extract_links(response)], + [ + Link( + url="http://example.org/item1.html", + text="Item 1", + nofollow=False, + ), + Link( + url="http://example.org/item3.html", + text="Item 3", + nofollow=False, + ), + ], + ) def test_ftp_links(self): body = b""" <html><body> <div><a href="ftp://www.external.com/">An Item</a></div> </body></html>""" - response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8') + response = HtmlResponse( + "http://www.example.com/index.html", body=body, encoding="utf8" + ) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), [ - Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False), - ]) + self.assertEqual( + lx.extract_links(response), + [ + Link( + url="ftp://www.external.com/", + text="An Item", + fragment="", + nofollow=False, + ), + ], + ) def test_pickle_extractor(self): lx = self.extractor_cls() @@ -483,10 +744,17 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): """ response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text='Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text='Item 3', nofollow=False), - ]) + self.assertEqual( + [link for link in lx.extract_links(response)], + [ + Link( + url="http://example.org/item1.html", text="Item 1", nofollow=False + ), + Link( + url="http://example.org/item3.html", text="Item 3", nofollow=False + ), + ], + ) def test_link_restrict_text(self): html = b""" @@ -496,21 +764,46 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): """ response = HtmlResponse("http://example.org/index.html", body=html) # Simple text inclusion test - lx = self.extractor_cls(restrict_text='dog') - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), - ]) + lx = self.extractor_cls(restrict_text="dog") + self.assertEqual( + [link for link in lx.extract_links(response)], + [ + Link( + url="http://example.org/item2.html", + text="Pic of a dog", + nofollow=False, + ), + ], + ) # Unique regex test - lx = self.extractor_cls(restrict_text=r'of.*dog') - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), - ]) + lx = self.extractor_cls(restrict_text=r"of.*dog") + self.assertEqual( + [link for link in lx.extract_links(response)], + [ + Link( + url="http://example.org/item2.html", + text="Pic of a dog", + nofollow=False, + ), + ], + ) # Multiple regex test - lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat']) - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False), - Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), - ]) + lx = self.extractor_cls(restrict_text=[r"of.*dog", r"of.*cat"]) + self.assertEqual( + [link for link in lx.extract_links(response)], + [ + Link( + url="http://example.org/item1.html", + text="Pic of a cat", + nofollow=False, + ), + Link( + url="http://example.org/item2.html", + text="Pic of a dog", + nofollow=False, + ), + ], + ) def test_restrict_xpaths_with_html_entities(self): super().test_restrict_xpaths_with_html_entities() diff --git a/tests/test_loader.py b/tests/test_loader.py index b3e44d36b..9dd298864 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -59,32 +59,31 @@ class DefaultedItemLoader(NameItemLoader): # test processors def processor_with_args(value, other=None, loader_context=None): - if 'key' in loader_context: - return loader_context['key'] + if "key" in loader_context: + return loader_context["key"] return value class BasicItemLoaderTest(unittest.TestCase): - def test_add_value_on_unknown_field(self): il = TestItemLoader() - self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo']) + self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"]) def test_load_item_using_default_loader(self): i = TestItem() - i['summary'] = 'lala' + i["summary"] = "lala" il = ItemLoader(item=i) - il.add_value('name', 'marta') + il.add_value("name", "marta") item = il.load_item() assert item is i - self.assertEqual(item['summary'], ['lala']) - self.assertEqual(item['name'], ['marta']) + self.assertEqual(item["summary"], ["lala"]) + self.assertEqual(item["name"], ["marta"]) def test_load_item_using_custom_loader(self): il = TestItemLoader() - il.add_value('name', 'marta') + il.add_value("name", "marta") item = il.load_item() - self.assertEqual(item['name'], ['Marta']) + self.assertEqual(item["name"], ["Marta"]) class InitializationTestMixin: @@ -93,85 +92,94 @@ class InitializationTestMixin: def test_keep_single_value(self): """Loaded item should contain values from the initial item""" - input_item = self.item_class(name='foo') + input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo']}) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]}) def test_keep_list(self): """Loaded item should contain values from the initial item""" - input_item = self.item_class(name=['foo', 'bar']) + input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']}) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) def test_add_value_singlevalue_singlevalue(self): """Values added after initialization should be appended""" - input_item = self.item_class(name='foo') + input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - il.add_value('name', 'bar') + il.add_value("name", "bar") loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']}) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) def test_add_value_singlevalue_list(self): """Values added after initialization should be appended""" - input_item = self.item_class(name='foo') + input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - il.add_value('name', ['item', 'loader']) + il.add_value("name", ["item", "loader"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'item', 'loader']}) + self.assertEqual( + ItemAdapter(loaded_item).asdict(), {"name": ["foo", "item", "loader"]} + ) def test_add_value_list_singlevalue(self): """Values added after initialization should be appended""" - input_item = self.item_class(name=['foo', 'bar']) + input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - il.add_value('name', 'qwerty') + il.add_value("name", "qwerty") loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'qwerty']}) + self.assertEqual( + ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar", "qwerty"]} + ) def test_add_value_list_list(self): """Values added after initialization should be appended""" - input_item = self.item_class(name=['foo', 'bar']) + input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - il.add_value('name', ['item', 'loader']) + il.add_value("name", ["item", "loader"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'item', 'loader']}) + self.assertEqual( + ItemAdapter(loaded_item).asdict(), + {"name": ["foo", "bar", "item", "loader"]}, + ) def test_get_output_value_singlevalue(self): """Getting output value must not remove value from item""" - input_item = self.item_class(name='foo') + input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value('name'), ['foo']) + self.assertEqual(il.get_output_value("name"), ["foo"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo']})) + self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({"name": ["foo"]})) def test_get_output_value_list(self): """Getting output value must not remove value from item""" - input_item = self.item_class(name=['foo', 'bar']) + input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value('name'), ['foo', 'bar']) + self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo', 'bar']})) + self.assertEqual( + ItemAdapter(loaded_item).asdict(), dict({"name": ["foo", "bar"]}) + ) def test_values_single(self): """Values from initial item must be added to loader._values""" - input_item = self.item_class(name='foo') + input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - self.assertEqual(il._values.get('name'), ['foo']) + self.assertEqual(il._values.get("name"), ["foo"]) def test_values_list(self): """Values from initial item must be added to loader._values""" - input_item = self.item_class(name=['foo', 'bar']) + input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - self.assertEqual(il._values.get('name'), ['foo', 'bar']) + self.assertEqual(il._values.get("name"), ["foo", "bar"]) class InitializationFromDictTest(InitializationTestMixin, unittest.TestCase): @@ -207,42 +215,52 @@ class NoInputReprocessingFromItemTest(unittest.TestCase): """ Loaders initialized from loaded items must not reprocess fields (Item instances) """ + def test_avoid_reprocessing_with_initial_values_single(self): - il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title='foo')) + il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title="foo")) il_loaded = il.load_item() - self.assertEqual(il_loaded, {'title': 'foo'}) - self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'foo'}) + self.assertEqual(il_loaded, {"title": "foo"}) + self.assertEqual( + NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"} + ) def test_avoid_reprocessing_with_initial_values_list(self): - il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title=['foo', 'bar'])) + il = NoInputReprocessingItemLoader( + item=NoInputReprocessingItem(title=["foo", "bar"]) + ) il_loaded = il.load_item() - self.assertEqual(il_loaded, {'title': 'foo'}) - self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'foo'}) + self.assertEqual(il_loaded, {"title": "foo"}) + self.assertEqual( + NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"} + ) def test_avoid_reprocessing_without_initial_values_single(self): il = NoInputReprocessingItemLoader() - il.add_value('title', 'FOO') + il.add_value("title", "FOO") il_loaded = il.load_item() - self.assertEqual(il_loaded, {'title': 'FOO'}) - self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'}) + self.assertEqual(il_loaded, {"title": "FOO"}) + self.assertEqual( + NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"} + ) def test_avoid_reprocessing_without_initial_values_list(self): il = NoInputReprocessingItemLoader() - il.add_value('title', ['foo', 'bar']) + il.add_value("title", ["foo", "bar"]) il_loaded = il.load_item() - self.assertEqual(il_loaded, {'title': 'FOO'}) - self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'}) + self.assertEqual(il_loaded, {"title": "FOO"}) + self.assertEqual( + NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"} + ) class TestOutputProcessorItem(unittest.TestCase): def test_output_processor(self): - class TempItem(Item): temp = Field() def __init__(self, *args, **kwargs): super().__init__(self, *args, **kwargs) - self.setdefault('temp', 0.3) + self.setdefault("temp", 0.3) class TempLoader(ItemLoader): default_item_class = TempItem @@ -252,11 +270,14 @@ class TestOutputProcessorItem(unittest.TestCase): loader = TempLoader() item = loader.load_item() self.assertIsInstance(item, TempItem) - self.assertEqual(dict(item), {'temp': 0.3}) + self.assertEqual(dict(item), {"temp": 0.3}) class SelectortemLoaderTest(unittest.TestCase): - response = HtmlResponse(url="", encoding='utf-8', body=b""" + response = HtmlResponse( + url="", + encoding="utf-8", + body=b""" <html> <body> <div id="id">marta</div> @@ -265,7 +286,8 @@ class SelectortemLoaderTest(unittest.TestCase): <img src="/images/logo.png" width="244" height="65" alt="Scrapy"> </body> </html> - """) + """, + ) def test_init_method(self): l = TestItemLoader() @@ -273,28 +295,28 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_errors(self): l = TestItemLoader() - self.assertRaises(RuntimeError, l.add_xpath, 'url', '//a/@href') - self.assertRaises(RuntimeError, l.replace_xpath, 'url', '//a/@href') - self.assertRaises(RuntimeError, l.get_xpath, '//a/@href') - self.assertRaises(RuntimeError, l.add_css, 'name', '#name::text') - self.assertRaises(RuntimeError, l.replace_css, 'name', '#name::text') - self.assertRaises(RuntimeError, l.get_css, '#name::text') + self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href") + self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href") + self.assertRaises(RuntimeError, l.get_xpath, "//a/@href") + self.assertRaises(RuntimeError, l.add_css, "name", "#name::text") + self.assertRaises(RuntimeError, l.replace_css, "name", "#name::text") + self.assertRaises(RuntimeError, l.get_css, "#name::text") def test_init_method_with_selector(self): sel = Selector(text="<html><body><div>marta</div></body></html>") l = TestItemLoader(selector=sel) self.assertIs(l.selector, sel) - l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), ['Marta']) + l.add_xpath("name", "//div/text()") + self.assertEqual(l.get_output_value("name"), ["Marta"]) def test_init_method_with_selector_css(self): sel = Selector(text="<html><body><div>marta</div></body></html>") l = TestItemLoader(selector=sel) self.assertIs(l.selector, sel) - l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), ['Marta']) + l.add_css("name", "div::text") + self.assertEqual(l.get_output_value("name"), ["Marta"]) def test_init_method_with_base_response(self): """Selector should be None after initialization""" @@ -306,122 +328,131 @@ class SelectortemLoaderTest(unittest.TestCase): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) - l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), ['Marta']) + l.add_xpath("name", "//div/text()") + self.assertEqual(l.get_output_value("name"), ["Marta"]) def test_init_method_with_response_css(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) - l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), ['Marta']) + l.add_css("name", "div::text") + self.assertEqual(l.get_output_value("name"), ["Marta"]) - l.add_css('url', 'a::attr(href)') - self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) + l.add_css("url", "a::attr(href)") + self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) # combining/accumulating CSS selectors and XPath expressions - l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta']) + l.add_xpath("name", "//div/text()") + self.assertEqual(l.get_output_value("name"), ["Marta", "Marta"]) - l.add_xpath('url', '//img/@src') - self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png']) + l.add_xpath("url", "//img/@src") + self.assertEqual( + l.get_output_value("url"), ["http://www.scrapy.org", "/images/logo.png"] + ) def test_add_xpath_re(self): l = TestItemLoader(response=self.response) - l.add_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_output_value('name'), ['Ma']) + l.add_xpath("name", "//div/text()", re="ma") + self.assertEqual(l.get_output_value("name"), ["Ma"]) def test_replace_xpath(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) - l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), ['Marta']) - l.replace_xpath('name', '//p/text()') - self.assertEqual(l.get_output_value('name'), ['Paragraph']) + l.add_xpath("name", "//div/text()") + self.assertEqual(l.get_output_value("name"), ["Marta"]) + l.replace_xpath("name", "//p/text()") + self.assertEqual(l.get_output_value("name"), ["Paragraph"]) - l.replace_xpath('name', ['//p/text()', '//div/text()']) - self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta']) + l.replace_xpath("name", ["//p/text()", "//div/text()"]) + self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) def test_get_xpath(self): l = TestItemLoader(response=self.response) - self.assertEqual(l.get_xpath('//p/text()'), ['paragraph']) - self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph') - self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa') + self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"]) + self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph") + self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa") - self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta']) + self.assertEqual( + l.get_xpath(["//p/text()", "//div/text()"]), ["paragraph", "marta"] + ) def test_replace_xpath_multi_fields(self): l = TestItemLoader(response=self.response) - l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), ['Marta']) - l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), ['Paragraph']) + l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x}) + self.assertEqual(l.get_output_value("name"), ["Marta"]) + l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x}) + self.assertEqual(l.get_output_value("name"), ["Paragraph"]) def test_replace_xpath_re(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) - l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), ['Marta']) - l.replace_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_output_value('name'), ['Ma']) + l.add_xpath("name", "//div/text()") + self.assertEqual(l.get_output_value("name"), ["Marta"]) + l.replace_xpath("name", "//div/text()", re="ma") + self.assertEqual(l.get_output_value("name"), ["Ma"]) def test_add_css_re(self): l = TestItemLoader(response=self.response) - l.add_css('name', 'div::text', re='ma') - self.assertEqual(l.get_output_value('name'), ['Ma']) + l.add_css("name", "div::text", re="ma") + self.assertEqual(l.get_output_value("name"), ["Ma"]) - l.add_css('url', 'a::attr(href)', re='http://(.+)') - self.assertEqual(l.get_output_value('url'), ['www.scrapy.org']) + l.add_css("url", "a::attr(href)", re="http://(.+)") + self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) def test_replace_css(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) - l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), ['Marta']) - l.replace_css('name', 'p::text') - self.assertEqual(l.get_output_value('name'), ['Paragraph']) + l.add_css("name", "div::text") + self.assertEqual(l.get_output_value("name"), ["Marta"]) + l.replace_css("name", "p::text") + self.assertEqual(l.get_output_value("name"), ["Paragraph"]) - l.replace_css('name', ['p::text', 'div::text']) - self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta']) + l.replace_css("name", ["p::text", "div::text"]) + self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) - l.add_css('url', 'a::attr(href)', re='http://(.+)') - self.assertEqual(l.get_output_value('url'), ['www.scrapy.org']) - l.replace_css('url', 'img::attr(src)') - self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) + l.add_css("url", "a::attr(href)", re="http://(.+)") + self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) + l.replace_css("url", "img::attr(src)") + self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) def test_get_css(self): l = TestItemLoader(response=self.response) - self.assertEqual(l.get_css('p::text'), ['paragraph']) - self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph') - self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa') + self.assertEqual(l.get_css("p::text"), ["paragraph"]) + self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph") + self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa") - self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta']) - self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']), - ['http://www.scrapy.org', '/images/logo.png']) + self.assertEqual(l.get_css(["p::text", "div::text"]), ["paragraph", "marta"]) + self.assertEqual( + l.get_css(["a::attr(href)", "img::attr(src)"]), + ["http://www.scrapy.org", "/images/logo.png"], + ) def test_replace_css_multi_fields(self): l = TestItemLoader(response=self.response) - l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), ['Marta']) - l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), ['Paragraph']) + l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x}) + self.assertEqual(l.get_output_value("name"), ["Marta"]) + l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x}) + self.assertEqual(l.get_output_value("name"), ["Paragraph"]) - l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x}) - self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) - l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x}) - self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) + l.add_css(None, "a::attr(href)", TakeFirst(), lambda x: {"url": x}) + self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + l.replace_css(None, "img::attr(src)", TakeFirst(), lambda x: {"url": x}) + self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) def test_replace_css_re(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) - l.add_css('url', 'a::attr(href)') - self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) - l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)') - self.assertEqual(l.get_output_value('url'), ['scrapy.org']) + l.add_css("url", "a::attr(href)") + self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + l.replace_css("url", "a::attr(href)", re=r"http://www\.(.+)") + self.assertEqual(l.get_output_value("url"), ["scrapy.org"]) class SubselectorLoaderTest(unittest.TestCase): - response = HtmlResponse(url="", encoding='utf-8', body=b""" + response = HtmlResponse( + url="", + encoding="utf-8", + body=b""" <html> <body> <header> @@ -434,76 +465,88 @@ class SubselectorLoaderTest(unittest.TestCase): </footer> </body> </html> - """) + """, + ) def test_nested_xpath(self): l = NestedItemLoader(response=self.response) nl = l.nested_xpath("//header") - nl.add_xpath('name', 'div/text()') - nl.add_css('name_div', '#id') - nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) + nl.add_xpath("name", "div/text()") + nl.add_css("name_div", "#id") + nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value('name'), ['marta']) - self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>']) - self.assertEqual(l.get_output_value('name_value'), ['marta']) + self.assertEqual(l.get_output_value("name"), ["marta"]) + self.assertEqual(l.get_output_value("name_div"), ['<div id="id">marta</div>']) + self.assertEqual(l.get_output_value("name_value"), ["marta"]) - self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) - self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) - self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value')) + self.assertEqual(l.get_output_value("name"), nl.get_output_value("name")) + self.assertEqual( + l.get_output_value("name_div"), nl.get_output_value("name_div") + ) + self.assertEqual( + l.get_output_value("name_value"), nl.get_output_value("name_value") + ) def test_nested_css(self): l = NestedItemLoader(response=self.response) nl = l.nested_css("header") - nl.add_xpath('name', 'div/text()') - nl.add_css('name_div', '#id') - nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) + nl.add_xpath("name", "div/text()") + nl.add_css("name_div", "#id") + nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value('name'), ['marta']) - self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>']) - self.assertEqual(l.get_output_value('name_value'), ['marta']) + self.assertEqual(l.get_output_value("name"), ["marta"]) + self.assertEqual(l.get_output_value("name_div"), ['<div id="id">marta</div>']) + self.assertEqual(l.get_output_value("name_value"), ["marta"]) - self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) - self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) - self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value')) + self.assertEqual(l.get_output_value("name"), nl.get_output_value("name")) + self.assertEqual( + l.get_output_value("name_div"), nl.get_output_value("name_div") + ) + self.assertEqual( + l.get_output_value("name_value"), nl.get_output_value("name_value") + ) def test_nested_replace(self): l = NestedItemLoader(response=self.response) - nl1 = l.nested_xpath('//footer') - nl2 = nl1.nested_xpath('a') + nl1 = l.nested_xpath("//footer") + nl2 = nl1.nested_xpath("a") - l.add_xpath('url', '//footer/a/@href') - self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) - nl1.replace_xpath('url', 'img/@src') - self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) - nl2.replace_xpath('url', '@href') - self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) + l.add_xpath("url", "//footer/a/@href") + self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + nl1.replace_xpath("url", "img/@src") + self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + nl2.replace_xpath("url", "@href") + self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) def test_nested_ordering(self): l = NestedItemLoader(response=self.response) - nl1 = l.nested_xpath('//footer') - nl2 = nl1.nested_xpath('a') + nl1 = l.nested_xpath("//footer") + nl2 = nl1.nested_xpath("a") - nl1.add_xpath('url', 'img/@src') - l.add_xpath('url', '//footer/a/@href') - nl2.add_xpath('url', 'text()') - l.add_xpath('url', '//footer/a/@href') + nl1.add_xpath("url", "img/@src") + l.add_xpath("url", "//footer/a/@href") + nl2.add_xpath("url", "text()") + l.add_xpath("url", "//footer/a/@href") - self.assertEqual(l.get_output_value('url'), [ - '/images/logo.png', - 'http://www.scrapy.org', - 'homepage', - 'http://www.scrapy.org', - ]) + self.assertEqual( + l.get_output_value("url"), + [ + "/images/logo.png", + "http://www.scrapy.org", + "homepage", + "http://www.scrapy.org", + ], + ) def test_nested_load_item(self): l = NestedItemLoader(response=self.response) - nl1 = l.nested_xpath('//footer') - nl2 = nl1.nested_xpath('img') + nl1 = l.nested_xpath("//footer") + nl2 = nl1.nested_xpath("img") - l.add_xpath('name', '//header/div/text()') - nl1.add_xpath('url', 'a/@href') - nl2.add_xpath('image', '@src') + l.add_xpath("name", "//header/div/text()") + nl1.add_xpath("url", "a/@href") + nl2.add_xpath("image", "@src") item = l.load_item() @@ -511,13 +554,14 @@ class SubselectorLoaderTest(unittest.TestCase): assert item is nl1.item assert item is nl2.item - self.assertEqual(item['name'], ['marta']) - self.assertEqual(item['url'], ['http://www.scrapy.org']) - self.assertEqual(item['image'], ['/images/logo.png']) + self.assertEqual(item["name"], ["marta"]) + self.assertEqual(item["url"], ["http://www.scrapy.org"]) + self.assertEqual(item["image"], ["/images/logo.png"]) # Functions as processors + def function_processor_strip(iterable): return [x.strip() for x in iterable] @@ -538,15 +582,11 @@ class FunctionProcessorItemLoader(ItemLoader): class FunctionProcessorTestCase(unittest.TestCase): - def test_processor_defined_in_item(self): lo = FunctionProcessorItemLoader() - lo.add_value('foo', ' bar ') - lo.add_value('foo', [' asdf ', ' qwerty ']) - self.assertEqual( - dict(lo.load_item()), - {'foo': ['BAR', 'ASDF', 'QWERTY']} - ) + lo.add_value("foo", " bar ") + lo.add_value("foo", [" asdf ", " qwerty "]) + self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) if __name__ == "__main__": diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 14b3b5568..8757db0ce 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -7,8 +7,14 @@ import unittest import warnings from functools import partial -from itemloaders.processors import (Compose, Identity, Join, - MapCompose, SelectJmes, TakeFirst) +from itemloaders.processors import ( + Compose, + Identity, + Join, + MapCompose, + SelectJmes, + TakeFirst, +) from scrapy.item import Item, Field from scrapy.loader import ItemLoader @@ -42,28 +48,27 @@ class DefaultedItemLoader(NameItemLoader): # test processors def processor_with_args(value, other=None, loader_context=None): - if 'key' in loader_context: - return loader_context['key'] + if "key" in loader_context: + return loader_context["key"] return value class BasicItemLoaderTest(unittest.TestCase): - def test_load_item_using_default_loader(self): i = TestItem() - i['summary'] = 'lala' + i["summary"] = "lala" il = ItemLoader(item=i) - il.add_value('name', 'marta') + il.add_value("name", "marta") item = il.load_item() assert item is i - self.assertEqual(item['summary'], ['lala']) - self.assertEqual(item['name'], ['marta']) + self.assertEqual(item["summary"], ["lala"]) + self.assertEqual(item["name"], ["marta"]) def test_load_item_using_custom_loader(self): il = TestItemLoader() - il.add_value('name', 'marta') + il.add_value("name", "marta") item = il.load_item() - self.assertEqual(item['name'], ['Marta']) + self.assertEqual(item["name"], ["Marta"]) def test_load_item_ignore_none_field_values(self): def validate_sku(value): @@ -76,206 +81,223 @@ class BasicItemLoaderTest(unittest.TestCase): price_out = Compose(TakeFirst(), float) sku_out = Compose(TakeFirst(), validate_sku) - valid_fragment = 'SKU: 1234' - invalid_fragment = 'SKU: not available' - sku_re = 'SKU: (.+)' + valid_fragment = "SKU: 1234" + invalid_fragment = "SKU: not available" + sku_re = "SKU: (.+)" il = MyLoader(item={}) # Should not return "sku: None". - il.add_value('sku', [invalid_fragment], re=sku_re) + il.add_value("sku", [invalid_fragment], re=sku_re) # Should not ignore empty values. - il.add_value('name', '') - il.add_value('price', ['0']) - self.assertEqual(il.load_item(), { - 'name': '', - 'price': 0.0, - }) + il.add_value("name", "") + il.add_value("price", ["0"]) + self.assertEqual( + il.load_item(), + { + "name": "", + "price": 0.0, + }, + ) - il.replace_value('sku', [valid_fragment], re=sku_re) - self.assertEqual(il.load_item()['sku'], '1234') + il.replace_value("sku", [valid_fragment], re=sku_re) + self.assertEqual(il.load_item()["sku"], "1234") def test_self_referencing_loader(self): class MyLoader(ItemLoader): url_out = TakeFirst() def img_url_out(self, values): - return (self.get_output_value('url') or '') + values[0] + return (self.get_output_value("url") or "") + values[0] il = MyLoader(item={}) - il.add_value('url', 'http://example.com/') - il.add_value('img_url', '1234.png') - self.assertEqual(il.load_item(), { - 'url': 'http://example.com/', - 'img_url': 'http://example.com/1234.png', - }) + il.add_value("url", "http://example.com/") + il.add_value("img_url", "1234.png") + self.assertEqual( + il.load_item(), + { + "url": "http://example.com/", + "img_url": "http://example.com/1234.png", + }, + ) il = MyLoader(item={}) - il.add_value('img_url', '1234.png') - self.assertEqual(il.load_item(), { - 'img_url': '1234.png', - }) + il.add_value("img_url", "1234.png") + self.assertEqual( + il.load_item(), + { + "img_url": "1234.png", + }, + ) def test_add_value(self): il = TestItemLoader() - il.add_value('name', 'marta') - self.assertEqual(il.get_collected_values('name'), ['Marta']) - self.assertEqual(il.get_output_value('name'), ['Marta']) - il.add_value('name', 'pepe') - self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe']) - self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe']) + il.add_value("name", "marta") + self.assertEqual(il.get_collected_values("name"), ["Marta"]) + self.assertEqual(il.get_output_value("name"), ["Marta"]) + il.add_value("name", "pepe") + self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe"]) + self.assertEqual(il.get_output_value("name"), ["Marta", "Pepe"]) # test add object value - il.add_value('summary', {'key': 1}) - self.assertEqual(il.get_collected_values('summary'), [{'key': 1}]) + il.add_value("summary", {"key": 1}) + self.assertEqual(il.get_collected_values("summary"), [{"key": 1}]) - il.add_value(None, 'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim']) + il.add_value(None, "Jim", lambda x: {"name": x}) + self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe", "Jim"]) def test_add_zero(self): il = NameItemLoader() - il.add_value('name', 0) - self.assertEqual(il.get_collected_values('name'), [0]) + il.add_value("name", 0) + self.assertEqual(il.get_collected_values("name"), [0]) def test_replace_value(self): il = TestItemLoader() - il.replace_value('name', 'marta') - self.assertEqual(il.get_collected_values('name'), ['Marta']) - self.assertEqual(il.get_output_value('name'), ['Marta']) - il.replace_value('name', 'pepe') - self.assertEqual(il.get_collected_values('name'), ['Pepe']) - self.assertEqual(il.get_output_value('name'), ['Pepe']) + il.replace_value("name", "marta") + self.assertEqual(il.get_collected_values("name"), ["Marta"]) + self.assertEqual(il.get_output_value("name"), ["Marta"]) + il.replace_value("name", "pepe") + self.assertEqual(il.get_collected_values("name"), ["Pepe"]) + self.assertEqual(il.get_output_value("name"), ["Pepe"]) - il.replace_value(None, 'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), ['Jim']) + il.replace_value(None, "Jim", lambda x: {"name": x}) + self.assertEqual(il.get_collected_values("name"), ["Jim"]) def test_get_value(self): il = NameItemLoader() - self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper)) - self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$')) - self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')) + self.assertEqual("FOO", il.get_value(["foo", "bar"], TakeFirst(), str.upper)) + self.assertEqual( + ["foo", "bar"], il.get_value(["name:foo", "name:bar"], re="name:(.*)$") + ) + self.assertEqual( + "foo", il.get_value(["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$") + ) - il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$') - self.assertEqual(['foo'], il.get_collected_values('name')) - il.replace_value('name', 'name:bar', re='name:(.*)$') - self.assertEqual(['bar'], il.get_collected_values('name')) + il.add_value("name", ["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$") + self.assertEqual(["foo"], il.get_collected_values("name")) + il.replace_value("name", "name:bar", re="name:(.*)$") + self.assertEqual(["bar"], il.get_collected_values("name")) def test_iter_on_input_processor_input(self): class NameFirstItemLoader(NameItemLoader): name_in = TakeFirst() il = NameFirstItemLoader() - il.add_value('name', 'marta') - self.assertEqual(il.get_collected_values('name'), ['marta']) + il.add_value("name", "marta") + self.assertEqual(il.get_collected_values("name"), ["marta"]) il = NameFirstItemLoader() - il.add_value('name', ['marta', 'jose']) - self.assertEqual(il.get_collected_values('name'), ['marta']) + il.add_value("name", ["marta", "jose"]) + self.assertEqual(il.get_collected_values("name"), ["marta"]) il = NameFirstItemLoader() - il.replace_value('name', 'marta') - self.assertEqual(il.get_collected_values('name'), ['marta']) + il.replace_value("name", "marta") + self.assertEqual(il.get_collected_values("name"), ["marta"]) il = NameFirstItemLoader() - il.replace_value('name', ['marta', 'jose']) - self.assertEqual(il.get_collected_values('name'), ['marta']) + il.replace_value("name", ["marta", "jose"]) + self.assertEqual(il.get_collected_values("name"), ["marta"]) il = NameFirstItemLoader() - il.add_value('name', 'marta') - il.add_value('name', ['jose', 'pedro']) - self.assertEqual(il.get_collected_values('name'), ['marta', 'jose']) + il.add_value("name", "marta") + il.add_value("name", ["jose", "pedro"]) + self.assertEqual(il.get_collected_values("name"), ["marta", "jose"]) def test_map_compose_filter(self): def filter_world(x): - return None if x == 'world' else x + return None if x == "world" else x proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), - ['HELLO', 'THIS', 'IS', 'SCRAPY']) + self.assertEqual( + proc(["hello", "world", "this", "is", "scrapy"]), + ["HELLO", "THIS", "IS", "SCRAPY"], + ) def test_map_compose_filter_multil(self): class TestItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) il = TestItemLoader() - il.add_value('name', 'marta') - self.assertEqual(il.get_output_value('name'), ['Mart']) + il.add_value("name", "marta") + self.assertEqual(il.get_output_value("name"), ["Mart"]) item = il.load_item() - self.assertEqual(item['name'], ['Mart']) + self.assertEqual(item["name"], ["Mart"]) def test_default_input_processor(self): il = DefaultedItemLoader() - il.add_value('name', 'marta') - self.assertEqual(il.get_output_value('name'), ['mart']) + il.add_value("name", "marta") + self.assertEqual(il.get_output_value("name"), ["mart"]) def test_inherited_default_input_processor(self): class InheritDefaultedItemLoader(DefaultedItemLoader): pass il = InheritDefaultedItemLoader() - il.add_value('name', 'marta') - self.assertEqual(il.get_output_value('name'), ['mart']) + il.add_value("name", "marta") + self.assertEqual(il.get_output_value("name"), ["mart"]) def test_input_processor_inheritance(self): class ChildItemLoader(TestItemLoader): url_in = MapCompose(lambda v: v.lower()) il = ChildItemLoader() - il.add_value('url', 'HTTP://scrapy.ORG') - self.assertEqual(il.get_output_value('url'), ['http://scrapy.org']) - il.add_value('name', 'marta') - self.assertEqual(il.get_output_value('name'), ['Marta']) + il.add_value("url", "HTTP://scrapy.ORG") + self.assertEqual(il.get_output_value("url"), ["http://scrapy.org"]) + il.add_value("name", "marta") + self.assertEqual(il.get_output_value("name"), ["Marta"]) class ChildChildItemLoader(ChildItemLoader): url_in = MapCompose(lambda v: v.upper()) summary_in = MapCompose(lambda v: v) il = ChildChildItemLoader() - il.add_value('url', 'http://scrapy.org') - self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG']) - il.add_value('name', 'marta') - self.assertEqual(il.get_output_value('name'), ['Marta']) + il.add_value("url", "http://scrapy.org") + self.assertEqual(il.get_output_value("url"), ["HTTP://SCRAPY.ORG"]) + il.add_value("name", "marta") + self.assertEqual(il.get_output_value("name"), ["Marta"]) def test_empty_map_compose(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): name_in = MapCompose() il = IdentityDefaultedItemLoader() - il.add_value('name', 'marta') - self.assertEqual(il.get_output_value('name'), ['marta']) + il.add_value("name", "marta") + self.assertEqual(il.get_output_value("name"), ["marta"]) def test_identity_input_processor(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): name_in = Identity() il = IdentityDefaultedItemLoader() - il.add_value('name', 'marta') - self.assertEqual(il.get_output_value('name'), ['marta']) + il.add_value("name", "marta") + self.assertEqual(il.get_output_value("name"), ["marta"]) def test_extend_custom_input_processors(self): class ChildItemLoader(TestItemLoader): name_in = MapCompose(TestItemLoader.name_in, str.swapcase) il = ChildItemLoader() - il.add_value('name', 'marta') - self.assertEqual(il.get_output_value('name'), ['mARTA']) + il.add_value("name", "marta") + self.assertEqual(il.get_output_value("name"), ["mARTA"]) def test_extend_default_input_processors(self): class ChildDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase) + name_in = MapCompose( + DefaultedItemLoader.default_input_processor, str.swapcase + ) il = ChildDefaultedItemLoader() - il.add_value('name', 'marta') - self.assertEqual(il.get_output_value('name'), ['MART']) + il.add_value("name", "marta") + self.assertEqual(il.get_output_value("name"), ["MART"]) def test_output_processor_using_function(self): il = TestItemLoader() - il.add_value('name', ['mar', 'ta']) - self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) + il.add_value("name", ["mar", "ta"]) + self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) class TakeFirstItemLoader(TestItemLoader): name_out = " ".join il = TakeFirstItemLoader() - il.add_value('name', ['mar', 'ta']) - self.assertEqual(il.get_output_value('name'), 'Mar Ta') + il.add_value("name", ["mar", "ta"]) + self.assertEqual(il.get_output_value("name"), "Mar Ta") def test_output_processor_error(self): class TestItemLoader(ItemLoader): @@ -283,9 +305,9 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = MapCompose(float) il = TestItemLoader() - il.add_value('name', ['$10']) + il.add_value("name", ["$10"]) try: - float('$10') + float("$10") except Exception as e: expected_exc_str = str(e) @@ -296,118 +318,118 @@ class BasicItemLoaderTest(unittest.TestCase): exc = e assert isinstance(exc, ValueError) s = str(exc) - assert 'name' in s, s - assert '$10' in s, s - assert 'ValueError' in s, s + assert "name" in s, s + assert "$10" in s, s + assert "ValueError" in s, s assert expected_exc_str in s, s def test_output_processor_using_classes(self): il = TestItemLoader() - il.add_value('name', ['mar', 'ta']) - self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) + il.add_value("name", ["mar", "ta"]) + self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) class TakeFirstItemLoader(TestItemLoader): name_out = Join() il = TakeFirstItemLoader() - il.add_value('name', ['mar', 'ta']) - self.assertEqual(il.get_output_value('name'), 'Mar Ta') + il.add_value("name", ["mar", "ta"]) + self.assertEqual(il.get_output_value("name"), "Mar Ta") class TakeFirstItemLoader(TestItemLoader): name_out = Join("<br>") il = TakeFirstItemLoader() - il.add_value('name', ['mar', 'ta']) - self.assertEqual(il.get_output_value('name'), 'Mar<br>Ta') + il.add_value("name", ["mar", "ta"]) + self.assertEqual(il.get_output_value("name"), "Mar<br>Ta") def test_default_output_processor(self): il = TestItemLoader() - il.add_value('name', ['mar', 'ta']) - self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) + il.add_value("name", ["mar", "ta"]) + self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) class LalaItemLoader(TestItemLoader): default_output_processor = Identity() il = LalaItemLoader() - il.add_value('name', ['mar', 'ta']) - self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) + il.add_value("name", ["mar", "ta"]) + self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) def test_loader_context_on_declaration(self): class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args, key='val') + url_in = MapCompose(processor_with_args, key="val") il = ChildItemLoader() - il.add_value('url', 'text') - self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', 'text2') - self.assertEqual(il.get_output_value('url'), ['val']) + il.add_value("url", "text") + self.assertEqual(il.get_output_value("url"), ["val"]) + il.replace_value("url", "text2") + self.assertEqual(il.get_output_value("url"), ["val"]) def test_loader_context_on_instantiation(self): class ChildItemLoader(TestItemLoader): url_in = MapCompose(processor_with_args) - il = ChildItemLoader(key='val') - il.add_value('url', 'text') - self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', 'text2') - self.assertEqual(il.get_output_value('url'), ['val']) + il = ChildItemLoader(key="val") + il.add_value("url", "text") + self.assertEqual(il.get_output_value("url"), ["val"]) + il.replace_value("url", "text2") + self.assertEqual(il.get_output_value("url"), ["val"]) def test_loader_context_on_assign(self): class ChildItemLoader(TestItemLoader): url_in = MapCompose(processor_with_args) il = ChildItemLoader() - il.context['key'] = 'val' - il.add_value('url', 'text') - self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', 'text2') - self.assertEqual(il.get_output_value('url'), ['val']) + il.context["key"] = "val" + il.add_value("url", "text") + self.assertEqual(il.get_output_value("url"), ["val"]) + il.replace_value("url", "text2") + self.assertEqual(il.get_output_value("url"), ["val"]) def test_item_passed_to_input_processor_functions(self): def processor(value, loader_context): - return loader_context['item']['name'] + return loader_context["item"]["name"] class ChildItemLoader(TestItemLoader): url_in = MapCompose(processor) - it = TestItem(name='marta') + it = TestItem(name="marta") il = ChildItemLoader(item=it) - il.add_value('url', 'text') - self.assertEqual(il.get_output_value('url'), ['marta']) - il.replace_value('url', 'text2') - self.assertEqual(il.get_output_value('url'), ['marta']) + il.add_value("url", "text") + self.assertEqual(il.get_output_value("url"), ["marta"]) + il.replace_value("url", "text2") + self.assertEqual(il.get_output_value("url"), ["marta"]) def test_compose_processor(self): class TestItemLoader(NameItemLoader): name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) il = TestItemLoader() - il.add_value('name', ['marta', 'other']) - self.assertEqual(il.get_output_value('name'), 'Mart') + il.add_value("name", ["marta", "other"]) + self.assertEqual(il.get_output_value("name"), "Mart") item = il.load_item() - self.assertEqual(item['name'], 'Mart') + self.assertEqual(item["name"], "Mart") def test_partial_processor(self): def join(values, sep=None, loader_context=None, ignored=None): if sep is not None: return sep.join(values) - if loader_context and 'sep' in loader_context: - return loader_context['sep'].join(values) - return ''.join(values) + if loader_context and "sep" in loader_context: + return loader_context["sep"].join(values) + return "".join(values) class TestItemLoader(NameItemLoader): - name_out = Compose(partial(join, sep='+')) - url_out = Compose(partial(join, loader_context={'sep': '.'})) - summary_out = Compose(partial(join, ignored='foo')) + name_out = Compose(partial(join, sep="+")) + url_out = Compose(partial(join, loader_context={"sep": "."})) + summary_out = Compose(partial(join, ignored="foo")) il = TestItemLoader() - il.add_value('name', ['rabbit', 'hole']) - il.add_value('url', ['rabbit', 'hole']) - il.add_value('summary', ['rabbit', 'hole']) + il.add_value("name", ["rabbit", "hole"]) + il.add_value("url", ["rabbit", "hole"]) + il.add_value("summary", ["rabbit", "hole"]) item = il.load_item() - self.assertEqual(item['name'], 'rabbit+hole') - self.assertEqual(item['url'], 'rabbit.hole') - self.assertEqual(item['summary'], 'rabbithole') + self.assertEqual(item["name"], "rabbit+hole") + self.assertEqual(item["url"], "rabbit.hole") + self.assertEqual(item["summary"], "rabbithole") def test_error_input_processor(self): class TestItem(Item): @@ -418,8 +440,7 @@ class BasicItemLoaderTest(unittest.TestCase): name_in = MapCompose(float) il = TestItemLoader() - self.assertRaises(ValueError, il.add_value, 'name', - ['marta', 'other']) + self.assertRaises(ValueError, il.add_value, "name", ["marta", "other"]) def test_error_output_processor(self): class TestItem(Item): @@ -430,7 +451,7 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = Compose(Join(), float) il = TestItemLoader() - il.add_value('name', 'marta') + il.add_value("name", "marta") with self.assertRaises(ValueError): il.load_item() @@ -442,8 +463,9 @@ class BasicItemLoaderTest(unittest.TestCase): default_item_class = TestItem il = TestItemLoader() - self.assertRaises(ValueError, il.add_value, 'name', - ['marta', 'other'], Compose(float)) + self.assertRaises( + ValueError, il.add_value, "name", ["marta", "other"], Compose(float) + ) class InitializationFromDictTest(unittest.TestCase): @@ -452,85 +474,85 @@ class InitializationFromDictTest(unittest.TestCase): def test_keep_single_value(self): """Loaded item should contain values from the initial item""" - input_item = self.item_class(name='foo') + input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {'name': ['foo']}) + self.assertEqual(dict(loaded_item), {"name": ["foo"]}) def test_keep_list(self): """Loaded item should contain values from the initial item""" - input_item = self.item_class(name=['foo', 'bar']) + input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']}) + self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]}) def test_add_value_singlevalue_singlevalue(self): """Values added after initialization should be appended""" - input_item = self.item_class(name='foo') + input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - il.add_value('name', 'bar') + il.add_value("name", "bar") loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']}) + self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]}) def test_add_value_singlevalue_list(self): """Values added after initialization should be appended""" - input_item = self.item_class(name='foo') + input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - il.add_value('name', ['item', 'loader']) + il.add_value("name", ["item", "loader"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']}) + self.assertEqual(dict(loaded_item), {"name": ["foo", "item", "loader"]}) def test_add_value_list_singlevalue(self): """Values added after initialization should be appended""" - input_item = self.item_class(name=['foo', 'bar']) + input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - il.add_value('name', 'qwerty') + il.add_value("name", "qwerty") loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']}) + self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "qwerty"]}) def test_add_value_list_list(self): """Values added after initialization should be appended""" - input_item = self.item_class(name=['foo', 'bar']) + input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - il.add_value('name', ['item', 'loader']) + il.add_value("name", ["item", "loader"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']}) + self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "item", "loader"]}) def test_get_output_value_singlevalue(self): """Getting output value must not remove value from item""" - input_item = self.item_class(name='foo') + input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value('name'), ['foo']) + self.assertEqual(il.get_output_value("name"), ["foo"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, dict({'name': ['foo']})) + self.assertEqual(loaded_item, dict({"name": ["foo"]})) def test_get_output_value_list(self): """Getting output value must not remove value from item""" - input_item = self.item_class(name=['foo', 'bar']) + input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value('name'), ['foo', 'bar']) + self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']})) + self.assertEqual(loaded_item, dict({"name": ["foo", "bar"]})) def test_values_single(self): """Values from initial item must be added to loader._values""" - input_item = self.item_class(name='foo') + input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - self.assertEqual(il._values.get('name'), ['foo']) + self.assertEqual(il._values.get("name"), ["foo"]) def test_values_list(self): """Values from initial item must be added to loader._values""" - input_item = self.item_class(name=['foo', 'bar']) + input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - self.assertEqual(il._values.get('name'), ['foo', 'bar']) + self.assertEqual(il._values.get("name"), ["foo", "bar"]) class BaseNoInputReprocessingLoader(ItemLoader): @@ -546,40 +568,48 @@ class NoInputReprocessingFromDictTest(unittest.TestCase): """ Loaders initialized from loaded items must not reprocess fields (dict instances) """ + def test_avoid_reprocessing_with_initial_values_single(self): - il = NoInputReprocessingDictLoader(item=dict(title='foo')) + il = NoInputReprocessingDictLoader(item=dict(title="foo")) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title='foo')) - self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo')) + self.assertEqual(il_loaded, dict(title="foo")) + self.assertEqual( + NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo") + ) def test_avoid_reprocessing_with_initial_values_list(self): - il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar'])) + il = NoInputReprocessingDictLoader(item=dict(title=["foo", "bar"])) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title='foo')) - self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo')) + self.assertEqual(il_loaded, dict(title="foo")) + self.assertEqual( + NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo") + ) def test_avoid_reprocessing_without_initial_values_single(self): il = NoInputReprocessingDictLoader() - il.add_value('title', 'foo') + il.add_value("title", "foo") il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title='FOO')) - self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO')) + self.assertEqual(il_loaded, dict(title="FOO")) + self.assertEqual( + NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO") + ) def test_avoid_reprocessing_without_initial_values_list(self): il = NoInputReprocessingDictLoader() - il.add_value('title', ['foo', 'bar']) + il.add_value("title", ["foo", "bar"]) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title='FOO')) - self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO')) + self.assertEqual(il_loaded, dict(title="FOO")) + self.assertEqual( + NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO") + ) class TestOutputProcessorDict(unittest.TestCase): def test_output_processor(self): - class TempDict(dict): def __init__(self, *args, **kwargs): super().__init__(self, *args, **kwargs) - self.setdefault('temp', 0.3) + self.setdefault("temp", 0.3) class TempLoader(ItemLoader): default_item_class = TempDict @@ -589,64 +619,67 @@ class TestOutputProcessorDict(unittest.TestCase): loader = TempLoader() item = loader.load_item() self.assertIsInstance(item, TempDict) - self.assertEqual(dict(item), {'temp': 0.3}) + self.assertEqual(dict(item), {"temp": 0.3}) class ProcessorsTest(unittest.TestCase): - def test_take_first(self): proc = TakeFirst() - self.assertEqual(proc([None, '', 'hello', 'world']), 'hello') - self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0) + self.assertEqual(proc([None, "", "hello", "world"]), "hello") + self.assertEqual(proc([None, "", 0, "hello", "world"]), 0) def test_identity(self): proc = Identity() - self.assertEqual(proc([None, '', 'hello', 'world']), - [None, '', 'hello', 'world']) + self.assertEqual( + proc([None, "", "hello", "world"]), [None, "", "hello", "world"] + ) def test_join(self): proc = Join() - self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) - self.assertEqual(proc(['', 'hello', 'world']), ' hello world') - self.assertEqual(proc(['hello', 'world']), 'hello world') - self.assertIsInstance(proc(['hello', 'world']), str) + self.assertRaises(TypeError, proc, [None, "", "hello", "world"]) + self.assertEqual(proc(["", "hello", "world"]), " hello world") + self.assertEqual(proc(["hello", "world"]), "hello world") + self.assertIsInstance(proc(["hello", "world"]), str) def test_compose(self): proc = Compose(lambda v: v[0], str.upper) - self.assertEqual(proc(['hello', 'world']), 'HELLO') + self.assertEqual(proc(["hello", "world"]), "HELLO") proc = Compose(str.upper) self.assertEqual(proc(None), None) proc = Compose(str.upper, stop_on_none=False) self.assertRaises(ValueError, proc, None) proc = Compose(str.upper, lambda x: x + 1) - self.assertRaises(ValueError, proc, 'hello') + self.assertRaises(ValueError, proc, "hello") def test_mapcompose(self): def filter_world(x): - return None if x == 'world' else x + return None if x == "world" else x + proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), - ['HELLO', 'THIS', 'IS', 'SCRAPY']) + self.assertEqual( + proc(["hello", "world", "this", "is", "scrapy"]), + ["HELLO", "THIS", "IS", "SCRAPY"], + ) proc = MapCompose(filter_world, str.upper) self.assertEqual(proc(None), []) proc = MapCompose(filter_world, str.upper) self.assertRaises(ValueError, proc, [1]) proc = MapCompose(filter_world, lambda x: x + 1) - self.assertRaises(ValueError, proc, 'hello') + self.assertRaises(ValueError, proc, "hello") class SelectJmesTestCase(unittest.TestCase): test_list_equals = { - 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), - 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), - 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'dict': ( - 'foo.bar[*].name', + "simple": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"), + "invalid": ("foo.bar.baz", {"foo": {"bar": "baz"}}, None), + "top_level": ("foo", {"foo": {"bar": "baz"}}, {"bar": "baz"}), + "double_vs_single_quote_string": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"), + "dict": ( + "foo.bar[*].name", {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ['one', 'two'] + ["one", "two"], ), - 'list': ('[1]', [1, 2], 2) + "list": ("[1]", [1, 2], 2), } def test_output(self): @@ -654,14 +687,13 @@ class SelectJmesTestCase(unittest.TestCase): expr, test_list, expected = self.test_list_equals[tl] test = SelectJmes(expr)(test_list) self.assertEqual( - test, - expected, - msg=f'test "{tl}" got {test} expected {expected}' + test, expected, msg=f'test "{tl}" got {test} expected {expected}' ) # Functions as processors + def function_processor_strip(iterable): return [x.strip() for x in iterable] @@ -684,19 +716,14 @@ class FunctionProcessorDictLoader(ItemLoader): class FunctionProcessorTestCase(unittest.TestCase): - def test_processor_defined_in_item_loader(self): lo = FunctionProcessorDictLoader() - lo.add_value('foo', ' bar ') - lo.add_value('foo', [' asdf ', ' qwerty ']) - self.assertEqual( - dict(lo.load_item()), - {'foo': ['BAR', 'ASDF', 'QWERTY']} - ) + lo.add_value("foo", " bar ") + lo.add_value("foo", [" asdf ", " qwerty "]) + self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) class DeprecatedUtilityFunctionsTestCase(unittest.TestCase): - def test_deprecated_wrap_loader_context(self): def function(*args): return None @@ -709,7 +736,7 @@ class DeprecatedUtilityFunctionsTestCase(unittest.TestCase): def test_deprecated_extract_regex(self): with warnings.catch_warnings(record=True) as w: - extract_regex(r'\w+', 'this is a test') + extract_regex(r"\w+", "this is a test") assert len(w) == 1 assert issubclass(w[0].category, ScrapyDeprecationWarning) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index f3bb23bda..11cf6d81a 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -24,67 +24,74 @@ class CustomItem(Item): class LogFormatterTestCase(unittest.TestCase): - def setUp(self): self.formatter = LogFormatter() - self.spider = Spider('default') + self.spider = Spider("default") def test_crawled_with_referer(self): req = Request("http://www.example.com") res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) - logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None)") + logline = logkws["msg"] % logkws["args"] + self.assertEqual( + logline, "Crawled (200) <GET http://www.example.com> (referer: None)" + ) def test_crawled_without_referer(self): - req = Request("http://www.example.com", headers={'referer': 'http://example.com'}) - res = Response("http://www.example.com", flags=['cached']) + req = Request( + "http://www.example.com", headers={"referer": "http://example.com"} + ) + res = Response("http://www.example.com", flags=["cached"]) logkws = self.formatter.crawled(req, res, self.spider) - logline = logkws['msg'] % logkws['args'] + logline = logkws["msg"] % logkws["args"] self.assertEqual( logline, - "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']") + "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']", + ) def test_flags_in_request(self): - req = Request("http://www.example.com", flags=['test', 'flag']) + req = Request("http://www.example.com", flags=["test", "flag"]) res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) - logline = logkws['msg'] % logkws['args'] + logline = logkws["msg"] % logkws["args"] self.assertEqual( logline, - "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)") + "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)", + ) def test_dropped(self): item = {} exception = Exception("\u2018") response = Response("http://www.example.com") logkws = self.formatter.dropped(item, exception, response, self.spider) - logline = logkws['msg'] % logkws['args'] + logline = logkws["msg"] % logkws["args"] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, ["Dropped: \u2018", '{}']) + self.assertEqual(lines, ["Dropped: \u2018", "{}"]) def test_item_error(self): # In practice, the complete traceback is shown by passing the # 'exc_info' argument to the logging function - item = {'key': 'value'} + item = {"key": "value"} exception = Exception() response = Response("http://www.example.com") logkws = self.formatter.item_error(item, exception, response, self.spider) - logline = logkws['msg'] % logkws['args'] + logline = logkws["msg"] % logkws["args"] self.assertEqual(logline, "Error processing {'key': 'value'}") def test_spider_error(self): # In practice, the complete traceback is shown by passing the # 'exc_info' argument to the logging function failure = Failure(Exception()) - request = Request("http://www.example.com", headers={'Referer': 'http://example.org'}) + request = Request( + "http://www.example.com", headers={"Referer": "http://example.org"} + ) response = Response("http://www.example.com", request=request) logkws = self.formatter.spider_error(failure, request, response, self.spider) - logline = logkws['msg'] % logkws['args'] + logline = logkws["msg"] % logkws["args"] self.assertEqual( logline, - "Spider error processing <GET http://www.example.com> (referer: http://example.org)" + "Spider error processing <GET http://www.example.com> (referer: http://example.org)", ) def test_download_error_short(self): @@ -93,7 +100,7 @@ class LogFormatterTestCase(unittest.TestCase): failure = Failure(Exception()) request = Request("http://www.example.com") logkws = self.formatter.download_error(failure, request, self.spider) - logline = logkws['msg'] % logkws['args'] + logline = logkws["msg"] % logkws["args"] self.assertEqual(logline, "Error downloading <GET http://www.example.com>") def test_download_error_long(self): @@ -101,67 +108,77 @@ class LogFormatterTestCase(unittest.TestCase): # 'exc_info' argument to the logging function failure = Failure(Exception()) request = Request("http://www.example.com") - logkws = self.formatter.download_error(failure, request, self.spider, "Some message") - logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, "Error downloading <GET http://www.example.com>: Some message") + logkws = self.formatter.download_error( + failure, request, self.spider, "Some message" + ) + logline = logkws["msg"] % logkws["args"] + self.assertEqual( + logline, "Error downloading <GET http://www.example.com>: Some message" + ) def test_scraped(self): item = CustomItem() - item['name'] = '\xa3' + item["name"] = "\xa3" response = Response("http://www.example.com") logkws = self.formatter.scraped(item, response, self.spider) - logline = logkws['msg'] % logkws['args'] + logline = logkws["msg"] % logkws["args"] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3']) + self.assertEqual( + lines, ["Scraped from <200 http://www.example.com>", "name: \xa3"] + ) class LogFormatterSubclass(LogFormatter): def crawled(self, request, response, spider): kwargs = super().crawled(request, response, spider) - CRAWLEDMSG = ( - "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" - ) - log_args = kwargs['args'] - log_args['flags'] = str(request.flags) + CRAWLEDMSG = "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" + log_args = kwargs["args"] + log_args["flags"] = str(request.flags) return { - 'level': kwargs['level'], - 'msg': CRAWLEDMSG, - 'args': log_args, + "level": kwargs["level"], + "msg": CRAWLEDMSG, + "args": log_args, } class LogformatterSubclassTest(LogFormatterTestCase): def setUp(self): self.formatter = LogFormatterSubclass() - self.spider = Spider('default') + self.spider = Spider("default") def test_crawled_with_referer(self): req = Request("http://www.example.com") res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) - logline = logkws['msg'] % logkws['args'] + logline = logkws["msg"] % logkws["args"] self.assertEqual( - logline, - "Crawled (200) <GET http://www.example.com> (referer: None) []") + logline, "Crawled (200) <GET http://www.example.com> (referer: None) []" + ) def test_crawled_without_referer(self): - req = Request("http://www.example.com", headers={'referer': 'http://example.com'}, flags=['cached']) + req = Request( + "http://www.example.com", + headers={"referer": "http://example.com"}, + flags=["cached"], + ) res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) - logline = logkws['msg'] % logkws['args'] + logline = logkws["msg"] % logkws["args"] self.assertEqual( logline, - "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']") + "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']", + ) def test_flags_in_request(self): - req = Request("http://www.example.com", flags=['test', 'flag']) + req = Request("http://www.example.com", flags=["test", "flag"]) res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) - logline = logkws['msg'] % logkws['args'] + logline = logkws["msg"] % logkws["args"] self.assertEqual( logline, - "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']") + "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']", + ) class SkipMessagesLogFormatter(LogFormatter): @@ -191,8 +208,8 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): self.mockserver = MockServer() self.mockserver.__enter__() self.base_settings = { - 'LOG_LEVEL': 'DEBUG', - 'ITEM_PIPELINES': { + "LOG_LEVEL": "DEBUG", + "ITEM_PIPELINES": { DropSomeItemsPipeline: 300, }, } @@ -212,7 +229,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): @defer.inlineCallbacks def test_skip_messages(self): settings = self.base_settings.copy() - settings['LOG_FORMATTER'] = SkipMessagesLogFormatter + settings["LOG_FORMATTER"] = SkipMessagesLogFormatter crawler = get_crawler(ItemSpider, settings) with LogCapture() as lc: yield crawler.crawl(mockserver=self.mockserver) diff --git a/tests/test_mail.py b/tests/test_mail.py index 9b248fbfa..c78980d57 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -8,118 +8,144 @@ from scrapy.mail import MailSender class MailSenderTest(unittest.TestCase): - def test_send(self): mailsender = MailSender(debug=True) - mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', - _callback=self._catch_mail_sent) + mailsender.send( + to=["test@scrapy.org"], + subject="subject", + body="body", + _callback=self._catch_mail_sent, + ) assert self.catched_msg - self.assertEqual(self.catched_msg['to'], ['test@scrapy.org']) - self.assertEqual(self.catched_msg['subject'], 'subject') - self.assertEqual(self.catched_msg['body'], 'body') + self.assertEqual(self.catched_msg["to"], ["test@scrapy.org"]) + self.assertEqual(self.catched_msg["subject"], "subject") + self.assertEqual(self.catched_msg["body"], "body") - msg = self.catched_msg['msg'] - self.assertEqual(msg['to'], 'test@scrapy.org') - self.assertEqual(msg['subject'], 'subject') - self.assertEqual(msg.get_payload(), 'body') - self.assertEqual(msg.get('Content-Type'), 'text/plain') + msg = self.catched_msg["msg"] + self.assertEqual(msg["to"], "test@scrapy.org") + self.assertEqual(msg["subject"], "subject") + self.assertEqual(msg.get_payload(), "body") + self.assertEqual(msg.get("Content-Type"), "text/plain") def test_send_single_values_to_and_cc(self): mailsender = MailSender(debug=True) - mailsender.send(to='test@scrapy.org', subject='subject', body='body', - cc='test@scrapy.org', _callback=self._catch_mail_sent) + mailsender.send( + to="test@scrapy.org", + subject="subject", + body="body", + cc="test@scrapy.org", + _callback=self._catch_mail_sent, + ) def test_send_html(self): mailsender = MailSender(debug=True) - mailsender.send(to=['test@scrapy.org'], subject='subject', - body='<p>body</p>', mimetype='text/html', - _callback=self._catch_mail_sent) + mailsender.send( + to=["test@scrapy.org"], + subject="subject", + body="<p>body</p>", + mimetype="text/html", + _callback=self._catch_mail_sent, + ) - msg = self.catched_msg['msg'] - self.assertEqual(msg.get_payload(), '<p>body</p>') - self.assertEqual(msg.get('Content-Type'), 'text/html') + msg = self.catched_msg["msg"] + self.assertEqual(msg.get_payload(), "<p>body</p>") + self.assertEqual(msg.get("Content-Type"), "text/html") def test_send_attach(self): attach = BytesIO() - attach.write(b'content') + attach.write(b"content") attach.seek(0) - attachs = [('attachment', 'text/plain', attach)] + attachs = [("attachment", "text/plain", attach)] mailsender = MailSender(debug=True) - mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', - attachs=attachs, _callback=self._catch_mail_sent) + mailsender.send( + to=["test@scrapy.org"], + subject="subject", + body="body", + attachs=attachs, + _callback=self._catch_mail_sent, + ) assert self.catched_msg - self.assertEqual(self.catched_msg['to'], ['test@scrapy.org']) - self.assertEqual(self.catched_msg['subject'], 'subject') - self.assertEqual(self.catched_msg['body'], 'body') + self.assertEqual(self.catched_msg["to"], ["test@scrapy.org"]) + self.assertEqual(self.catched_msg["subject"], "subject") + self.assertEqual(self.catched_msg["body"], "body") - msg = self.catched_msg['msg'] - self.assertEqual(msg['to'], 'test@scrapy.org') - self.assertEqual(msg['subject'], 'subject') + msg = self.catched_msg["msg"] + self.assertEqual(msg["to"], "test@scrapy.org") + self.assertEqual(msg["subject"], "subject") payload = msg.get_payload() assert isinstance(payload, list) self.assertEqual(len(payload), 2) text, attach = payload - self.assertEqual(text.get_payload(decode=True), b'body') - self.assertEqual(text.get_charset(), Charset('us-ascii')) - self.assertEqual(attach.get_payload(decode=True), b'content') + self.assertEqual(text.get_payload(decode=True), b"body") + self.assertEqual(text.get_charset(), Charset("us-ascii")) + self.assertEqual(attach.get_payload(decode=True), b"content") def _catch_mail_sent(self, **kwargs): self.catched_msg = dict(**kwargs) def test_send_utf8(self): - subject = 'sübjèçt' - body = 'bödÿ-àéïöñß' + subject = "sübjèçt" + body = "bödÿ-àéïöñß" mailsender = MailSender(debug=True) - mailsender.send(to=['test@scrapy.org'], subject=subject, body=body, - charset='utf-8', _callback=self._catch_mail_sent) + mailsender.send( + to=["test@scrapy.org"], + subject=subject, + body=body, + charset="utf-8", + _callback=self._catch_mail_sent, + ) assert self.catched_msg - self.assertEqual(self.catched_msg['subject'], subject) - self.assertEqual(self.catched_msg['body'], body) + self.assertEqual(self.catched_msg["subject"], subject) + self.assertEqual(self.catched_msg["body"], body) - msg = self.catched_msg['msg'] - self.assertEqual(msg['subject'], subject) + msg = self.catched_msg["msg"] + self.assertEqual(msg["subject"], subject) self.assertEqual(msg.get_payload(), body) - self.assertEqual(msg.get_charset(), Charset('utf-8')) - self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"') + self.assertEqual(msg.get_charset(), Charset("utf-8")) + self.assertEqual(msg.get("Content-Type"), 'text/plain; charset="utf-8"') def test_send_attach_utf8(self): - subject = 'sübjèçt' - body = 'bödÿ-àéïöñß' + subject = "sübjèçt" + body = "bödÿ-àéïöñß" attach = BytesIO() - attach.write(body.encode('utf-8')) + attach.write(body.encode("utf-8")) attach.seek(0) - attachs = [('attachment', 'text/plain', attach)] + attachs = [("attachment", "text/plain", attach)] mailsender = MailSender(debug=True) - mailsender.send(to=['test@scrapy.org'], subject=subject, body=body, - attachs=attachs, charset='utf-8', - _callback=self._catch_mail_sent) + mailsender.send( + to=["test@scrapy.org"], + subject=subject, + body=body, + attachs=attachs, + charset="utf-8", + _callback=self._catch_mail_sent, + ) assert self.catched_msg - self.assertEqual(self.catched_msg['subject'], subject) - self.assertEqual(self.catched_msg['body'], body) + self.assertEqual(self.catched_msg["subject"], subject) + self.assertEqual(self.catched_msg["body"], body) - msg = self.catched_msg['msg'] - self.assertEqual(msg['subject'], subject) - self.assertEqual(msg.get_charset(), Charset('utf-8')) - self.assertEqual(msg.get('Content-Type'), - 'multipart/mixed; charset="utf-8"') + msg = self.catched_msg["msg"] + self.assertEqual(msg["subject"], subject) + self.assertEqual(msg.get_charset(), Charset("utf-8")) + self.assertEqual(msg.get("Content-Type"), 'multipart/mixed; charset="utf-8"') payload = msg.get_payload() assert isinstance(payload, list) self.assertEqual(len(payload), 2) text, attach = payload - self.assertEqual(text.get_payload(decode=True).decode('utf-8'), body) - self.assertEqual(text.get_charset(), Charset('utf-8')) - self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body) + self.assertEqual(text.get_payload(decode=True).decode("utf-8"), body) + self.assertEqual(text.get_charset(), Charset("utf-8")) + self.assertEqual(attach.get_payload(decode=True).decode("utf-8"), body) if __name__ == "__main__": diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 8651431b5..a84cf4c28 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -6,7 +6,6 @@ from scrapy.middleware import MiddlewareManager class M1: - def open_spider(self, spider): pass @@ -18,7 +17,6 @@ class M1: class M2: - def open_spider(self, spider): pass @@ -29,13 +27,11 @@ class M2: class M3: - def process(self, response, request, spider): pass class MOff: - def open_spider(self, spider): pass @@ -47,37 +43,39 @@ class MOff: class TestMiddlewareManager(MiddlewareManager): - @classmethod def _get_mwlist_from_settings(cls, settings): return [M1, MOff, M3] def _add_middleware(self, mw): super()._add_middleware(mw) - if hasattr(mw, 'process'): - self.methods['process'].append(mw.process) + if hasattr(mw, "process"): + self.methods["process"].append(mw.process) class MiddlewareManagerTest(unittest.TestCase): - def test_init(self): m1, m2, m3 = M1(), M2(), M3() mwman = TestMiddlewareManager(m1, m2, m3) - self.assertEqual(list(mwman.methods['open_spider']), [m1.open_spider, m2.open_spider]) - self.assertEqual(list(mwman.methods['close_spider']), [m2.close_spider, m1.close_spider]) - self.assertEqual(list(mwman.methods['process']), [m1.process, m3.process]) + self.assertEqual( + list(mwman.methods["open_spider"]), [m1.open_spider, m2.open_spider] + ) + self.assertEqual( + list(mwman.methods["close_spider"]), [m2.close_spider, m1.close_spider] + ) + self.assertEqual(list(mwman.methods["process"]), [m1.process, m3.process]) def test_methods(self): mwman = TestMiddlewareManager(M1(), M2(), M3()) self.assertEqual( - [x.__self__.__class__ for x in mwman.methods['open_spider']], - [M1, M2]) + [x.__self__.__class__ for x in mwman.methods["open_spider"]], [M1, M2] + ) self.assertEqual( - [x.__self__.__class__ for x in mwman.methods['close_spider']], - [M2, M1]) + [x.__self__.__class__ for x in mwman.methods["close_spider"]], [M2, M1] + ) self.assertEqual( - [x.__self__.__class__ for x in mwman.methods['process']], - [M1, M3]) + [x.__self__.__class__ for x in mwman.methods["process"]], [M1, M3] + ) def test_enabled(self): m1, m2, m3 = M1(), M2(), M3() diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 39bca7bf2..b04da22be 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -13,7 +13,7 @@ from tests.spiders import SimpleSpider class MediaDownloadSpider(SimpleSpider): - name = 'mediadownload' + name = "mediadownload" def _process_url(self, url): return url @@ -27,34 +27,38 @@ class MediaDownloadSpider(SimpleSpider): self._process_url(response.urljoin(href)) for href in response.xpath( '//table[thead/tr/th="Filename"]/tbody//a/@href' - ).getall()], + ).getall() + ], } yield item class BrokenLinksMediaDownloadSpider(MediaDownloadSpider): - name = 'brokenmedia' + name = "brokenmedia" def _process_url(self, url): - return url + '.foo' + return url + ".foo" class RedirectedMediaDownloadSpider(MediaDownloadSpider): - name = 'redirectedmedia' + name = "redirectedmedia" def _process_url(self, url): - return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url) + return add_or_replace_parameter( + self.mockserver.url("/redirect-to"), "goto", url + ) class FileDownloadCrawlTestCase(TestCase): - pipeline_class = 'scrapy.pipelines.files.FilesPipeline' - store_setting_key = 'FILES_STORE' - media_key = 'files' - media_urls_key = 'file_urls' + pipeline_class = "scrapy.pipelines.files.FilesPipeline" + store_setting_key = "FILES_STORE" + media_key = "files" + media_urls_key = "file_urls" expected_checksums = { - '5547178b89448faf0015a13f904c936e', - 'c2281c83670e31d8aaab7cb642b824db', - 'ed3f6538dc15d4d9179dae57319edc5f'} + "5547178b89448faf0015a13f904c936e", + "c2281c83670e31d8aaab7cb642b824db", + "ed3f6538dc15d4d9179dae57319edc5f", + } def setUp(self): self.mockserver = MockServer() @@ -64,8 +68,8 @@ class FileDownloadCrawlTestCase(TestCase): self.tmpmediastore = Path(self.mktemp()) self.tmpmediastore.mkdir() self.settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', - 'ITEM_PIPELINES': {self.pipeline_class: 1}, + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + "ITEM_PIPELINES": {self.pipeline_class: 1}, self.store_setting_key: str(self.tmpmediastore), } self.runner = CrawlerRunner(self.settings) @@ -91,27 +95,25 @@ class FileDownloadCrawlTestCase(TestCase): self.assertIn(self.media_key, items[0]) # check that logs show the expected number of successful file downloads - file_dl_success = 'File (downloaded): Downloaded file from' + file_dl_success = "File (downloaded): Downloaded file from" self.assertEqual(logs.count(file_dl_success), 3) # check that the images/files status is `downloaded` for item in items: for i in item[self.media_key]: - self.assertEqual(i['status'], 'downloaded') + self.assertEqual(i["status"], "downloaded") # check that the images/files checksums are what we know they should be if self.expected_checksums is not None: checksums = set( - i['checksum'] - for item in items - for i in item[self.media_key] + i["checksum"] for item in items for i in item[self.media_key] ) self.assertEqual(checksums, self.expected_checksums) # check that the image files where actually written to the media store for item in items: for i in item[self.media_key]: - self.assertTrue((self.tmpmediastore / i['path']).exists()) + self.assertTrue((self.tmpmediastore / i["path"]).exists()) def _assert_files_download_failure(self, crawler, items, code, logs): @@ -121,13 +123,19 @@ class FileDownloadCrawlTestCase(TestCase): self.assertFalse(items[0][self.media_key]) # check that there was 1 successful fetch and 3 other responses with non-200 code - self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4) - self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4) - self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1) - self.assertEqual(crawler.stats.get_value(f'downloader/response_status_count/{code}'), 3) + self.assertEqual( + crawler.stats.get_value("downloader/request_method_count/GET"), 4 + ) + self.assertEqual(crawler.stats.get_value("downloader/response_count"), 4) + self.assertEqual( + crawler.stats.get_value("downloader/response_status_count/200"), 1 + ) + self.assertEqual( + crawler.stats.get_value(f"downloader/response_status_count/{code}"), 3 + ) # check that logs do show the failure on the file downloads - file_dl_failure = f'File (code: {code}): Error downloading file from' + file_dl_failure = f"File (code: {code}): Error downloading file from" self.assertEqual(logs.count(file_dl_failure), 3) # check that no files were written to the media store @@ -140,7 +148,8 @@ class FileDownloadCrawlTestCase(TestCase): yield crawler.crawl( self.mockserver.url("/files/images/"), media_key=self.media_key, - media_urls_key=self.media_urls_key) + media_urls_key=self.media_urls_key, + ) self._assert_files_downloaded(self.items, str(log)) @defer.inlineCallbacks @@ -150,7 +159,8 @@ class FileDownloadCrawlTestCase(TestCase): yield crawler.crawl( self.mockserver.url("/files/images/"), media_key=self.media_key, - media_urls_key=self.media_urls_key) + media_urls_key=self.media_urls_key, + ) self._assert_files_download_failure(crawler, self.items, 404, str(log)) @defer.inlineCallbacks @@ -161,13 +171,14 @@ class FileDownloadCrawlTestCase(TestCase): self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, - mockserver=self.mockserver) + mockserver=self.mockserver, + ) self._assert_files_download_failure(crawler, self.items, 302, str(log)) @defer.inlineCallbacks def test_download_media_redirected_allowed(self): settings = dict(self.settings) - settings.update({'MEDIA_ALLOW_REDIRECTS': True}) + settings.update({"MEDIA_ALLOW_REDIRECTS": True}) runner = CrawlerRunner(settings) crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner) with LogCapture() as log: @@ -175,15 +186,20 @@ class FileDownloadCrawlTestCase(TestCase): self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, - mockserver=self.mockserver) + mockserver=self.mockserver, + ) self._assert_files_downloaded(self.items, str(log)) - self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) + self.assertEqual( + crawler.stats.get_value("downloader/response_status_count/302"), 3 + ) try: from PIL import Image # noqa: imported just to check for the import error except ImportError: - skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' + skip_pillow = ( + "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + ) else: skip_pillow = None @@ -192,10 +208,10 @@ class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): skip = skip_pillow - pipeline_class = 'scrapy.pipelines.images.ImagesPipeline' - store_setting_key = 'IMAGES_STORE' - media_key = 'images' - media_urls_key = 'image_urls' + pipeline_class = "scrapy.pipelines.images.ImagesPipeline" + store_setting_key = "IMAGES_STORE" + media_key = "images" + media_urls_key = "image_urls" # somehow checksums for images are different for Python 3.3 expected_checksums = None diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4acd29bf7..43942e53e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -35,15 +35,14 @@ from scrapy.utils.test import ( def _mocked_download_func(request, info): - response = request.meta.get('response') + response = request.meta.get("response") return response() if callable(response) else response class FilesPipelineTestCase(unittest.TestCase): - def setUp(self): self.tempdir = mkdtemp() - settings_dict = {'FILES_STORE': self.tempdir} + settings_dict = {"FILES_STORE": self.tempdir} crawler = get_crawler(spidercls=None, settings_dict=settings_dict) self.pipeline = FilesPipeline.from_crawler(crawler) self.pipeline.download_func = _mocked_download_func @@ -56,41 +55,70 @@ class FilesPipelineTestCase(unittest.TestCase): file_path = self.pipeline.file_path self.assertEqual( file_path(Request("https://dev.mydeco.com/mydeco.pdf")), - 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') + "full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf", + ) self.assertEqual( - file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")), - 'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt') + file_path( + Request( + "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt" + ) + ), + "full/4ce274dd83db0368bafd7e406f382ae088e39219.txt", + ) self.assertEqual( - file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")), - 'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc') + file_path( + Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc") + ), + "full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc", + ) self.assertEqual( - file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), - 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') + file_path( + Request( + "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg" + ) + ), + "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg", + ) self.assertEqual( file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2') + "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2", + ) self.assertEqual( file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1", + ) self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), - response=Response("http://www.dorma.co.uk/images/product_details/2532"), - info=object()), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + file_path( + Request("http://www.dorma.co.uk/images/product_details/2532"), + response=Response("http://www.dorma.co.uk/images/product_details/2532"), + info=object(), + ), + "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1", + ) self.assertEqual( - file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")), - 'full/76c00cef2ef669ae65052661f68d451162829507') + file_path( + Request( + "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha" + ) + ), + "full/76c00cef2ef669ae65052661f68d451162829507", + ) self.assertEqual( - file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ - //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")), - 'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png') + file_path( + Request( + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ + //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y" + ) + ), + "full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png", + ) def test_fs_store(self): assert isinstance(self.pipeline.store, FSFilesStore) self.assertEqual(self.pipeline.store.basedir, self.tempdir) - path = 'some/image/key.jpg' - fullpath = Path(self.tempdir, 'some', 'image', 'key.jpg') + path = "some/image/key.jpg" + fullpath = Path(self.tempdir, "some", "image", "key.jpg") self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath) @defer.inlineCallbacks @@ -98,18 +126,24 @@ class FilesPipelineTestCase(unittest.TestCase): item_url = "http://example.com/file.pdf" item = _create_item_with_files(item_url) patchers = [ - mock.patch.object(FilesPipeline, 'inc_stats', return_value=True), - mock.patch.object(FSFilesStore, 'stat_file', return_value={ - 'checksum': 'abc', 'last_modified': time.time()}), - mock.patch.object(FilesPipeline, 'get_media_requests', - return_value=[_prepare_request_object(item_url)]) + mock.patch.object(FilesPipeline, "inc_stats", return_value=True), + mock.patch.object( + FSFilesStore, + "stat_file", + return_value={"checksum": "abc", "last_modified": time.time()}, + ), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), ] for p in patchers: p.start() result = yield self.pipeline.process_item(item, None) - self.assertEqual(result['files'][0]['checksum'], 'abc') - self.assertEqual(result['files'][0]['status'], 'uptodate') + self.assertEqual(result["files"][0]["checksum"], "abc") + self.assertEqual(result["files"][0]["status"], "uptodate") for p in patchers: p.stop() @@ -119,19 +153,28 @@ class FilesPipelineTestCase(unittest.TestCase): item_url = "http://example.com/file2.pdf" item = _create_item_with_files(item_url) patchers = [ - mock.patch.object(FSFilesStore, 'stat_file', return_value={ - 'checksum': 'abc', - 'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}), - mock.patch.object(FilesPipeline, 'get_media_requests', - return_value=[_prepare_request_object(item_url)]), - mock.patch.object(FilesPipeline, 'inc_stats', return_value=True) + mock.patch.object( + FSFilesStore, + "stat_file", + return_value={ + "checksum": "abc", + "last_modified": time.time() + - (self.pipeline.expires * 60 * 60 * 24 * 2), + }, + ), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + mock.patch.object(FilesPipeline, "inc_stats", return_value=True), ] for p in patchers: p.start() result = yield self.pipeline.process_item(item, None) - self.assertNotEqual(result['files'][0]['checksum'], 'abc') - self.assertEqual(result['files'][0]['status'], 'downloaded') + self.assertNotEqual(result["files"][0]["checksum"], "abc") + self.assertEqual(result["files"][0]["status"], "downloaded") for p in patchers: p.stop() @@ -141,19 +184,28 @@ class FilesPipelineTestCase(unittest.TestCase): item_url = "http://example.com/file3.pdf" item = _create_item_with_files(item_url) patchers = [ - mock.patch.object(FilesPipeline, 'inc_stats', return_value=True), - mock.patch.object(FSFilesStore, 'stat_file', return_value={ - 'checksum': 'abc', - 'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}), - mock.patch.object(FilesPipeline, 'get_media_requests', - return_value=[_prepare_request_object(item_url, flags=['cached'])]) + mock.patch.object(FilesPipeline, "inc_stats", return_value=True), + mock.patch.object( + FSFilesStore, + "stat_file", + return_value={ + "checksum": "abc", + "last_modified": time.time() + - (self.pipeline.expires * 60 * 60 * 24 * 2), + }, + ), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url, flags=["cached"])], + ), ] for p in patchers: p.start() result = yield self.pipeline.process_item(item, None) - self.assertNotEqual(result['files'][0]['checksum'], 'abc') - self.assertEqual(result['files'][0]['status'], 'cached') + self.assertNotEqual(result["files"][0]["checksum"], "abc") + self.assertEqual(result["files"][0]["status"], "cached") for p in patchers: p.stop() @@ -162,48 +214,58 @@ class FilesPipelineTestCase(unittest.TestCase): """ Custom file path based on item data, overriding default implementation """ + class CustomFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, item=None): return f'full/{item.get("path")}' - file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path - item = dict(path='path-to-store-file') + file_path = CustomFilesPipeline.from_settings( + Settings({"FILES_STORE": self.tempdir}) + ).file_path + item = dict(path="path-to-store-file") request = Request("http://example.com") - self.assertEqual(file_path(request, item=item), 'full/path-to-store-file') + self.assertEqual(file_path(request, item=item), "full/path-to-store-file") class FilesPipelineTestCaseFieldsMixin: - def test_item_fields_default(self): - url = 'http://www.example.com/files/1.txt' - item = self.item_class(name='item1', file_urls=[url]) - pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'})) + url = "http://www.example.com/files/1.txt" + item = self.item_class(name="item1", file_urls=[url]) + pipeline = FilesPipeline.from_settings( + Settings({"FILES_STORE": "s3://example/files/"}) + ) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) - results = [(True, {'url': url})] + results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) files = ItemAdapter(item).get("files") self.assertEqual(files, [results[0][1]]) self.assertIsInstance(item, self.item_class) def test_item_fields_override_settings(self): - url = 'http://www.example.com/files/1.txt' - item = self.item_class(name='item1', custom_file_urls=[url]) - pipeline = FilesPipeline.from_settings(Settings({ - 'FILES_STORE': 's3://example/files/', - 'FILES_URLS_FIELD': 'custom_file_urls', - 'FILES_RESULT_FIELD': 'custom_files' - })) + url = "http://www.example.com/files/1.txt" + item = self.item_class(name="item1", custom_file_urls=[url]) + pipeline = FilesPipeline.from_settings( + Settings( + { + "FILES_STORE": "s3://example/files/", + "FILES_URLS_FIELD": "custom_file_urls", + "FILES_RESULT_FIELD": "custom_files", + } + ) + ) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) - results = [(True, {'url': url})] + results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) custom_files = ItemAdapter(item).get("custom_files") self.assertEqual(custom_files, [results[0][1]]) self.assertIsInstance(item, self.item_class) -class FilesPipelineTestCaseFieldsDict(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): +class FilesPipelineTestCaseFieldsDict( + FilesPipelineTestCaseFieldsMixin, unittest.TestCase +): item_class = dict @@ -217,7 +279,9 @@ class FilesPipelineTestItem(Item): custom_files = Field() -class FilesPipelineTestCaseFieldsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): +class FilesPipelineTestCaseFieldsItem( + FilesPipelineTestCaseFieldsMixin, unittest.TestCase +): item_class = FilesPipelineTestItem @@ -232,7 +296,9 @@ class FilesPipelineTestDataClass: custom_files: list = dataclasses.field(default_factory=list) -class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): +class FilesPipelineTestCaseFieldsDataClass( + FilesPipelineTestCaseFieldsMixin, unittest.TestCase +): item_class = FilesPipelineTestDataClass @@ -247,7 +313,9 @@ class FilesPipelineTestAttrsItem: custom_files = attr.ib(default=lambda: []) -class FilesPipelineTestCaseFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): +class FilesPipelineTestCaseFieldsAttrsItem( + FilesPipelineTestCaseFieldsMixin, unittest.TestCase +): item_class = FilesPipelineTestAttrsItem @@ -255,12 +323,12 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): default_cls_settings = { "EXPIRES": 90, "FILES_URLS_FIELD": "file_urls", - "FILES_RESULT_FIELD": "files" + "FILES_RESULT_FIELD": "files", } file_cls_attr_settings_map = { ("EXPIRES", "FILES_EXPIRES", "expires"), ("FILES_URLS_FIELD", "FILES_URLS_FIELD", "files_urls_field"), - ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field") + ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"), } def setUp(self): @@ -270,7 +338,6 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): rmtree(self.tempdir) def _generate_fake_settings(self, prefix=None): - def random_string(): return "".join([chr(random.randint(97, 123)) for _ in range(10)]) @@ -278,15 +345,17 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): "FILES_EXPIRES": random.randint(100, 1000), "FILES_URLS_FIELD": random_string(), "FILES_RESULT_FIELD": random_string(), - "FILES_STORE": self.tempdir + "FILES_STORE": self.tempdir, } if not prefix: return settings - return {prefix.upper() + "_" + k if k != "FILES_STORE" else k: v for k, v in settings.items()} + return { + prefix.upper() + "_" + k if k != "FILES_STORE" else k: v + for k, v in settings.items() + } def _generate_fake_pipeline(self): - class UserDefinedFilePipeline(FilesPipeline): EXPIRES = 1001 FILES_URLS_FIELD = "alfa" @@ -339,10 +408,13 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): If there are no settings for subclass and no subclass attributes, pipeline should use attributes of base class. """ + class UserDefinedFilesPipeline(FilesPipeline): pass - user_pipeline = UserDefinedFilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir})) + user_pipeline = UserDefinedFilesPipeline.from_settings( + Settings({"FILES_STORE": self.tempdir}) + ) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_cls_settings.get(pipe_attr.upper()) @@ -353,6 +425,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): If there are custom settings for subclass and NO class attributes, pipeline should use custom settings. """ + class UserDefinedFilesPipeline(FilesPipeline): pass @@ -374,7 +447,11 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): prefix = pipeline_cls.__name__.upper() settings = self._generate_fake_settings(prefix=prefix) user_pipeline = pipeline_cls.from_settings(Settings(settings)) - for pipe_cls_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: + for ( + pipe_cls_attr, + settings_attr, + pipe_inst_attr, + ) in self.file_cls_attr_settings_map: custom_value = settings.get(prefix + "_" + settings_attr) self.assertNotEqual(custom_value, self.default_cls_settings[pipe_cls_attr]) self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value) @@ -384,7 +461,9 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): DEFAULT_FILES_RESULT_FIELD = "this" DEFAULT_FILES_URLS_FIELD = "that" - pipeline = UserDefinedFilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir})) + pipeline = UserDefinedFilesPipeline.from_settings( + Settings({"FILES_STORE": self.tempdir}) + ) self.assertEqual(pipeline.files_result_field, "this") self.assertEqual(pipeline.files_urls_field, "that") @@ -402,37 +481,36 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: expected_value = settings.get(settings_attr) - self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), - expected_value) + self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value) class TestS3FilesStore(unittest.TestCase): - @defer.inlineCallbacks def test_persist(self): skip_if_no_boto() - bucket = 'mybucket' - key = 'export.csv' - uri = f's3://{bucket}/{key}' + bucket = "mybucket" + key = "export.csv" + uri = f"s3://{bucket}/{key}" buffer = mock.MagicMock() - meta = {'foo': 'bar'} - path = '' - content_type = 'image/png' + meta = {"foo": "bar"} + path = "" + content_type = "image/png" store = S3FilesStore(uri) from botocore.stub import Stubber + with Stubber(store.s3_client) as stub: stub.add_response( - 'put_object', + "put_object", expected_params={ - 'ACL': S3FilesStore.POLICY, - 'Body': buffer, - 'Bucket': bucket, - 'CacheControl': S3FilesStore.HEADERS['Cache-Control'], - 'ContentType': content_type, - 'Key': key, - 'Metadata': meta, + "ACL": S3FilesStore.POLICY, + "Body": buffer, + "Bucket": bucket, + "CacheControl": S3FilesStore.HEADERS["Cache-Control"], + "ContentType": content_type, + "Key": key, + "Metadata": meta, }, service_response={}, ) @@ -442,7 +520,7 @@ class TestS3FilesStore(unittest.TestCase): buffer, info=None, meta=meta, - headers={'Content-Type': content_type}, + headers={"Content-Type": content_type}, ) stub.assert_no_pending_responses() @@ -451,40 +529,41 @@ class TestS3FilesStore(unittest.TestCase): [ mock.call.seek(0), # The call to read does not happen with Stubber - ] + ], ) @defer.inlineCallbacks def test_stat(self): skip_if_no_boto() - bucket = 'mybucket' - key = 'export.csv' - uri = f's3://{bucket}/{key}' - checksum = '3187896a9657a28163abb31667df64c8' + bucket = "mybucket" + key = "export.csv" + uri = f"s3://{bucket}/{key}" + checksum = "3187896a9657a28163abb31667df64c8" last_modified = datetime(2019, 12, 1) store = S3FilesStore(uri) from botocore.stub import Stubber + with Stubber(store.s3_client) as stub: stub.add_response( - 'head_object', + "head_object", expected_params={ - 'Bucket': bucket, - 'Key': key, + "Bucket": bucket, + "Key": key, }, service_response={ - 'ETag': f'"{checksum}"', - 'LastModified': last_modified, + "ETag": f'"{checksum}"', + "LastModified": last_modified, }, ) - file_stats = yield store.stat_file('', info=None) + file_stats = yield store.stat_file("", info=None) self.assertEqual( file_stats, { - 'checksum': checksum, - 'last_modified': last_modified.timestamp(), + "checksum": checksum, + "last_modified": last_modified.timestamp(), }, ) @@ -495,27 +574,27 @@ class TestGCSFilesStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): assert_gcs_environ() - uri = os.environ.get('GCS_TEST_FILE_URI') + uri = os.environ.get("GCS_TEST_FILE_URI") if not uri: raise unittest.SkipTest("No GCS URI available for testing") data = b"TestGCSFilesStore: \xe2\x98\x83" buf = BytesIO(data) - meta = {'foo': 'bar'} - path = 'full/filename' + meta = {"foo": "bar"} + path = "full/filename" store = GCSFilesStore(uri) - store.POLICY = 'authenticatedRead' - expected_policy = {'role': 'READER', 'entity': 'allAuthenticatedUsers'} + store.POLICY = "authenticatedRead" + expected_policy = {"role": "READER", "entity": "allAuthenticatedUsers"} yield store.persist_file(path, buf, info=None, meta=meta, headers=None) s = yield store.stat_file(path, info=None) - self.assertIn('last_modified', s) - self.assertIn('checksum', s) - self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==') + self.assertIn("last_modified", s) + self.assertIn("checksum", s) + self.assertEqual(s["checksum"], "zc2oVgXkbQr2EQdSdw3OPA==") u = urlparse(uri) content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path) self.assertEqual(content, data) - self.assertEqual(blob.metadata, {'foo': 'bar'}) + self.assertEqual(blob.metadata, {"foo": "bar"}) self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) - self.assertEqual(blob.content_type, 'application/octet-stream') + self.assertEqual(blob.content_type, "application/octet-stream") self.assertIn(expected_policy, acl) @defer.inlineCallbacks @@ -525,17 +604,19 @@ class TestGCSFilesStore(unittest.TestCase): """ assert_gcs_environ() try: - import google.cloud.storage # noqa + import google.cloud.storage # noqa except ModuleNotFoundError: raise unittest.SkipTest("google-cloud-storage is not installed") else: - with mock.patch('google.cloud.storage') as _: - with mock.patch('scrapy.pipelines.files.time') as _: - uri = 'gs://my_bucket/my_prefix/' + with mock.patch("google.cloud.storage") as _: + with mock.patch("scrapy.pipelines.files.time") as _: + uri = "gs://my_bucket/my_prefix/" store = GCSFilesStore(uri) store.bucket = mock.Mock() - path = 'full/my_data.txt' - yield store.persist_file(path, mock.Mock(), info=None, meta=None, headers=None) + path = "full/my_data.txt" + yield store.persist_file( + path, mock.Mock(), info=None, meta=None, headers=None + ) yield store.stat_file(path, info=None) expected_blob_path = store.prefix + path store.bucket.blob.assert_called_with(expected_blob_path) @@ -545,25 +626,30 @@ class TestGCSFilesStore(unittest.TestCase): class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): - uri = os.environ.get('FTP_TEST_FILE_URI') + uri = os.environ.get("FTP_TEST_FILE_URI") if not uri: raise unittest.SkipTest("No FTP URI available for testing") data = b"TestFTPFilesStore: \xe2\x98\x83" buf = BytesIO(data) - meta = {'foo': 'bar'} - path = 'full/filename' + meta = {"foo": "bar"} + path = "full/filename" store = FTPFilesStore(uri) empty_dict = yield store.stat_file(path, info=None) self.assertEqual(empty_dict, {}) yield store.persist_file(path, buf, info=None, meta=meta, headers=None) stat = yield store.stat_file(path, info=None) - self.assertIn('last_modified', stat) - self.assertIn('checksum', stat) - self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6') - path = f'{store.basedir}/{path}' + self.assertIn("last_modified", stat) + self.assertIn("checksum", stat) + self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6") + path = f"{store.basedir}/{path}" content = get_ftp_content_and_delete( - path, store.host, store.port, - store.username, store.password, store.USE_ACTIVE_MODE) + path, + store.host, + store.port, + store.username, + store.password, + store.USE_ACTIVE_MODE, + ) self.assertEqual(data.decode(), content) @@ -574,14 +660,15 @@ class ItemWithFiles(Item): def _create_item_with_files(*files): item = ItemWithFiles() - item['file_urls'] = files + item["file_urls"] = files return item def _prepare_request_object(item_url, flags=None): return Request( item_url, - meta={'response': Response(item_url, status=200, body=b'data', flags=flags)}) + meta={"response": Response(item_url, status=200, body=b"data", flags=flags)}, + ) if __name__ == "__main__": diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index f98d40fda..3cd3e5966 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -22,17 +22,19 @@ from scrapy.utils.python import to_bytes try: from PIL import Image except ImportError: - skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' + skip_pillow = ( + "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + ) else: - encoders = {'jpeg_encoder', 'jpeg_decoder'} + encoders = {"jpeg_encoder", "jpeg_decoder"} if not encoders.issubset(set(Image.core.__dict__)): - skip_pillow = 'Missing JPEG encoders' + skip_pillow = "Missing JPEG encoders" else: skip_pillow = None def _mocked_download_func(request, info): - response = request.meta.get('response') + response = request.meta.get("response") return response() if callable(response) else response @@ -42,7 +44,9 @@ class ImagesPipelineTestCase(unittest.TestCase): def setUp(self): self.tempdir = mkdtemp() - self.pipeline = ImagesPipeline(self.tempdir, download_func=_mocked_download_func) + self.pipeline = ImagesPipeline( + self.tempdir, download_func=_mocked_download_func + ) def tearDown(self): rmtree(self.tempdir) @@ -51,43 +55,75 @@ class ImagesPipelineTestCase(unittest.TestCase): file_path = self.pipeline.file_path self.assertEqual( file_path(Request("https://dev.mydeco.com/mydeco.gif")), - 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg", + ) self.assertEqual( - file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")), - 'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg') + file_path( + Request( + "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg" + ) + ), + "full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg", + ) self.assertEqual( - file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")), - 'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg') + file_path( + Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif") + ), + "full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg", + ) self.assertEqual( - file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")), - 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') + file_path( + Request( + "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg" + ) + ), + "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg", + ) self.assertEqual( file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg') + "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg", + ) self.assertEqual( file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') + "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg", + ) self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532"), - response=Response("http://www.dorma.co.uk/images/product_details/2532"), - info=object()), - 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg') + file_path( + Request("http://www.dorma.co.uk/images/product_details/2532"), + response=Response("http://www.dorma.co.uk/images/product_details/2532"), + info=object(), + ), + "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg", + ) def test_thumbnail_name(self): thumb_path = self.pipeline.thumb_path - name = '50' - self.assertEqual(thumb_path(Request("file:///tmp/foo.jpg"), name), - 'thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg') - self.assertEqual(thumb_path(Request("file://foo.png"), name), - 'thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg') - self.assertEqual(thumb_path(Request("file:///tmp/foo"), name), - 'thumbs/50/0329ad83ebb8e93ea7c7906d46e9ed55f7349a50.jpg') - self.assertEqual(thumb_path(Request("file:///tmp/some.name/foo"), name), - 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') - self.assertEqual(thumb_path(Request("file:///tmp/some.name/foo"), name, - response=Response("file:///tmp/some.name/foo"), - info=object()), - 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') + name = "50" + self.assertEqual( + thumb_path(Request("file:///tmp/foo.jpg"), name), + "thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg", + ) + self.assertEqual( + thumb_path(Request("file://foo.png"), name), + "thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg", + ) + self.assertEqual( + thumb_path(Request("file:///tmp/foo"), name), + "thumbs/50/0329ad83ebb8e93ea7c7906d46e9ed55f7349a50.jpg", + ) + self.assertEqual( + thumb_path(Request("file:///tmp/some.name/foo"), name), + "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg", + ) + self.assertEqual( + thumb_path( + Request("file:///tmp/some.name/foo"), + name, + response=Response("file:///tmp/some.name/foo"), + info=object(), + ), + "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg", + ) def test_thumbnail_name_from_item(self): """ @@ -95,23 +131,27 @@ class ImagesPipelineTestCase(unittest.TestCase): """ class CustomImagesPipeline(ImagesPipeline): - def thumb_path(self, request, thumb_id, response=None, info=None, item=None): + def thumb_path( + self, request, thumb_id, response=None, info=None, item=None + ): return f"thumb/{thumb_id}/{item.get('path')}" - thumb_path = CustomImagesPipeline.from_settings(Settings( - {'IMAGES_STORE': self.tempdir} - )).thumb_path - item = dict(path='path-to-store-file') + thumb_path = CustomImagesPipeline.from_settings( + Settings({"IMAGES_STORE": self.tempdir}) + ).thumb_path + item = dict(path="path-to-store-file") request = Request("http://example.com") - self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file') + self.assertEqual( + thumb_path(request, "small", item=item), "thumb/small/path-to-store-file" + ) def test_get_images_exception(self): self.pipeline.min_width = 100 self.pipeline.min_height = 100 - _, buf1 = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) - _, buf2 = _create_image('JPEG', 'RGB', (150, 50), (0, 0, 0)) - _, buf3 = _create_image('JPEG', 'RGB', (50, 150), (0, 0, 0)) + _, buf1 = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) + _, buf2 = _create_image("JPEG", "RGB", (150, 50), (0, 0, 0)) + _, buf3 = _create_image("JPEG", "RGB", (50, 150), (0, 0, 0)) resp1 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf1.getvalue()) resp2 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf2.getvalue()) @@ -128,121 +168,154 @@ class ImagesPipelineTestCase(unittest.TestCase): def test_get_images_new(self): self.pipeline.min_width = 0 self.pipeline.min_height = 0 - self.pipeline.thumbs = {'small': (20, 20)} + self.pipeline.thumbs = {"small": (20, 20)} - orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) - orig_thumb, orig_thumb_buf = _create_image('JPEG', 'RGB', (20, 20), (0, 0, 0)) + orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) + orig_thumb, orig_thumb_buf = _create_image("JPEG", "RGB", (20, 20), (0, 0, 0)) resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) req = Request(url="https://dev.mydeco.com/mydeco.gif") - get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object()) + get_images_gen = self.pipeline.get_images( + response=resp, request=req, info=object() + ) path, new_im, new_buf = next(get_images_gen) - self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg") self.assertEqual(orig_im, new_im) self.assertEqual(buf.getvalue(), new_buf.getvalue()) thumb_path, thumb_img, thumb_buf = next(get_images_gen) - self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual( + thumb_path, "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + ) self.assertEqual(thumb_img, thumb_img) self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue()) def test_get_images_old(self): - self.pipeline.thumbs = {'small': (20, 20)} - orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + self.pipeline.thumbs = {"small": (20, 20)} + orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) req = Request(url="https://dev.mydeco.com/mydeco.gif") def overridden_convert_image(image, size=None): - im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) return im, buf - with patch.object(self.pipeline, 'convert_image', overridden_convert_image): + with patch.object(self.pipeline, "convert_image", overridden_convert_image): with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object()) + warnings.simplefilter("always") + get_images_gen = self.pipeline.get_images( + response=resp, request=req, info=object() + ) path, new_im, new_buf = next(get_images_gen) - self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual( + path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + ) self.assertEqual(orig_im.mode, new_im.mode) self.assertEqual(orig_im.getcolors(), new_im.getcolors()) self.assertEqual(buf.getvalue(), new_buf.getvalue()) thumb_path, thumb_img, thumb_buf = next(get_images_gen) - self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual( + thumb_path, + "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg", + ) self.assertEqual(orig_im.mode, thumb_img.mode) self.assertEqual(orig_im.getcolors(), thumb_img.getcolors()) self.assertEqual(buf.getvalue(), thumb_buf.getvalue()) - expected_warning_msg = ('.convert_image() method overriden in a deprecated way, ' - 'overriden method does not accept response_body argument.') - self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1) + expected_warning_msg = ( + ".convert_image() method overriden in a deprecated way, " + "overriden method does not accept response_body argument." + ) + self.assertEqual( + len( + [ + warning + for warning in w + if expected_warning_msg in str(warning.message) + ] + ), + 1, + ) def test_convert_image_old(self): # tests for old API with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') + warnings.simplefilter("always") SIZE = (100, 100) # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) - im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR) + im, _ = _create_image("JPEG", "RGB", SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.mode, "RGB") self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) # check that thumbnail keep image ratio thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) - self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.mode, "RGB") self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) - im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.mode, "RGB") self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) - im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) - im = im.convert('P') + im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR) + im = im.convert("P") converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.mode, "RGB") self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # ensure that we recieved deprecation warnings - expected_warning_msg = '.convert_image() method called in a deprecated way' - self.assertTrue(len([warning for warning in w if expected_warning_msg in str(warning.message)]) == 4) + expected_warning_msg = ".convert_image() method called in a deprecated way" + self.assertTrue( + len( + [ + warning + for warning in w + if expected_warning_msg in str(warning.message) + ] + ) + == 4 + ) def test_convert_image_new(self): # tests for new API SIZE = (100, 100) # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) - im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) + im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.mode, "RGB") self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) # check that we don't convert JPEGs again self.assertEqual(converted_buf, buf) # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf) - self.assertEqual(thumbnail.mode, 'RGB') + thumbnail, _ = self.pipeline.convert_image( + converted, size=(10, 25), response_body=converted_buf + ) + self.assertEqual(thumbnail.mode, "RGB") self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.mode, "RGB") self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - im = im.convert('P') + im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR) + im = im.convert("P") converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.mode, "RGB") self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) @@ -252,11 +325,11 @@ class DeprecatedImagesPipeline(ImagesPipeline): def image_key(self, url): image_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return f'empty/{image_guid}.jpg' + return f"empty/{image_guid}.jpg" def thumb_key(self, url, thumb_id): thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return f'thumbsup/{thumb_id}/{thumb_guid}.jpg' + return f"thumbsup/{thumb_id}/{thumb_guid}.jpg" class ImagesPipelineTestCaseFieldsMixin: @@ -264,35 +337,43 @@ class ImagesPipelineTestCaseFieldsMixin: skip = skip_pillow def test_item_fields_default(self): - url = 'http://www.example.com/images/1.jpg' - item = self.item_class(name='item1', image_urls=[url]) - pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'})) + url = "http://www.example.com/images/1.jpg" + item = self.item_class(name="item1", image_urls=[url]) + pipeline = ImagesPipeline.from_settings( + Settings({"IMAGES_STORE": "s3://example/images/"}) + ) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) - results = [(True, {'url': url})] + results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) images = ItemAdapter(item).get("images") self.assertEqual(images, [results[0][1]]) self.assertIsInstance(item, self.item_class) def test_item_fields_override_settings(self): - url = 'http://www.example.com/images/1.jpg' - item = self.item_class(name='item1', custom_image_urls=[url]) - pipeline = ImagesPipeline.from_settings(Settings({ - 'IMAGES_STORE': 's3://example/images/', - 'IMAGES_URLS_FIELD': 'custom_image_urls', - 'IMAGES_RESULT_FIELD': 'custom_images' - })) + url = "http://www.example.com/images/1.jpg" + item = self.item_class(name="item1", custom_image_urls=[url]) + pipeline = ImagesPipeline.from_settings( + Settings( + { + "IMAGES_STORE": "s3://example/images/", + "IMAGES_URLS_FIELD": "custom_image_urls", + "IMAGES_RESULT_FIELD": "custom_images", + } + ) + ) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) - results = [(True, {'url': url})] + results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) custom_images = ItemAdapter(item).get("custom_images") self.assertEqual(custom_images, [results[0][1]]) self.assertIsInstance(item, self.item_class) -class ImagesPipelineTestCaseFieldsDict(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): +class ImagesPipelineTestCaseFieldsDict( + ImagesPipelineTestCaseFieldsMixin, unittest.TestCase +): item_class = dict @@ -306,7 +387,9 @@ class ImagesPipelineTestItem(Item): custom_images = Field() -class ImagesPipelineTestCaseFieldsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): +class ImagesPipelineTestCaseFieldsItem( + ImagesPipelineTestCaseFieldsMixin, unittest.TestCase +): item_class = ImagesPipelineTestItem @@ -321,7 +404,9 @@ class ImagesPipelineTestDataClass: custom_images: list = dataclasses.field(default_factory=list) -class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): +class ImagesPipelineTestCaseFieldsDataClass( + ImagesPipelineTestCaseFieldsMixin, unittest.TestCase +): item_class = ImagesPipelineTestDataClass @@ -336,7 +421,9 @@ class ImagesPipelineTestAttrsItem: custom_images = attr.ib(default=lambda: []) -class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): +class ImagesPipelineTestCaseFieldsAttrsItem( + ImagesPipelineTestCaseFieldsMixin, unittest.TestCase +): item_class = ImagesPipelineTestAttrsItem @@ -351,7 +438,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): ("MIN_HEIGHT", "IMAGES_MIN_HEIGHT"), ("IMAGES_URLS_FIELD", "IMAGES_URLS_FIELD"), ("IMAGES_RESULT_FIELD", "IMAGES_RESULT_FIELD"), - ("THUMBS", "IMAGES_THUMBS") + ("THUMBS", "IMAGES_THUMBS"), ] # This should match what is defined in ImagesPipeline. @@ -360,8 +447,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): MIN_HEIGHT=0, EXPIRES=90, THUMBS={}, - IMAGES_URLS_FIELD='image_urls', - IMAGES_RESULT_FIELD='images' + IMAGES_URLS_FIELD="image_urls", + IMAGES_RESULT_FIELD="images", ) def setUp(self): @@ -387,26 +474,30 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): "IMAGES_MIN_WIDTH": random.randint(1, 1000), "IMAGES_MIN_HEIGHT": random.randint(1, 1000), "IMAGES_THUMBS": { - 'small': (random.randint(1, 1000), random.randint(1, 1000)), - 'big': (random.randint(1, 1000), random.randint(1, 1000)) - } + "small": (random.randint(1, 1000), random.randint(1, 1000)), + "big": (random.randint(1, 1000), random.randint(1, 1000)), + }, } if not prefix: return settings - return {prefix.upper() + "_" + k if k != "IMAGES_STORE" else k: v for k, v in settings.items()} + return { + prefix.upper() + "_" + k if k != "IMAGES_STORE" else k: v + for k, v in settings.items() + } def _generate_fake_pipeline_subclass(self): """ :return: ImagePipeline class will all uppercase attributes set. """ + class UserDefinedImagePipeline(ImagesPipeline): # Values should be in different range than fake_settings. MIN_WIDTH = random.randint(1000, 2000) MIN_HEIGHT = random.randint(1000, 2000) THUMBS = { - 'small': (random.randint(1000, 2000), random.randint(1000, 2000)), - 'big': (random.randint(1000, 2000), random.randint(1000, 2000)) + "small": (random.randint(1000, 2000), random.randint(1000, 2000)), + "big": (random.randint(1000, 2000), random.randint(1000, 2000)), } EXPIRES = random.randint(1000, 2000) IMAGES_URLS_FIELD = "field_one" @@ -427,7 +518,9 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): expected_default_value = self.default_pipeline_settings.get(pipe_attr) custom_value = custom_settings.get(settings_attr) self.assertNotEqual(expected_default_value, custom_value) - self.assertEqual(getattr(default_sts_pipe, pipe_attr.lower()), expected_default_value) + self.assertEqual( + getattr(default_sts_pipe, pipe_attr.lower()), expected_default_value + ) self.assertEqual(getattr(user_sts_pipe, pipe_attr.lower()), custom_value) def test_subclass_attrs_preserved_default_settings(self): @@ -464,10 +557,13 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): If there are no settings for subclass and no subclass attributes, pipeline should use attributes of base class. """ + class UserDefinedImagePipeline(ImagesPipeline): pass - user_pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir})) + user_pipeline = UserDefinedImagePipeline.from_settings( + Settings({"IMAGES_STORE": self.tempdir}) + ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_pipeline_settings.get(pipe_attr.upper()) @@ -478,6 +574,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): If there are custom settings for subclass and NO class attributes, pipeline should use custom settings. """ + class UserDefinedImagePipeline(ImagesPipeline): pass @@ -508,7 +605,10 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): class UserDefinedImagePipeline(ImagesPipeline): DEFAULT_IMAGES_URLS_FIELD = "something" DEFAULT_IMAGES_RESULT_FIELD = "something_else" - pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir})) + + pipeline = UserDefinedImagePipeline.from_settings( + Settings({"IMAGES_STORE": self.tempdir}) + ) self.assertEqual(pipeline.images_result_field, "something_else") self.assertEqual(pipeline.images_urls_field, "something") @@ -526,12 +626,10 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_value = settings.get(settings_attr) - self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), - expected_value) + self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value) class NoimagesDropTestCase(unittest.TestCase): - def test_deprecation_warning(self): arg = str() with warnings.catch_warnings(record=True) as w: @@ -539,8 +637,10 @@ class NoimagesDropTestCase(unittest.TestCase): self.assertEqual(len(w), 1) self.assertEqual(w[0].category, ScrapyDeprecationWarning) with warnings.catch_warnings(record=True) as w: + class SubclassedNoimagesDrop(NoimagesDrop): pass + SubclassedNoimagesDrop(arg) self.assertEqual(len(w), 1) self.assertEqual(w[0].category, ScrapyDeprecationWarning) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 0a94ae699..87ab03395 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -23,13 +23,15 @@ from scrapy.utils.test import get_crawler try: from PIL import Image # noqa: imported just to check for the import error except ImportError: - skip_pillow: Optional[str] = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' + skip_pillow: Optional[ + str + ] = "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" else: skip_pillow = None def _mocked_download_func(request, info): - response = request.meta.get('response') + response = request.meta.get("response") return response() if callable(response) else response @@ -40,7 +42,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): def setUp(self): spider_cls = Spider - self.spider = spider_cls('media.com') + self.spider = spider_cls("media.com") crawler = get_crawler(spider_cls, self.settings) self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe.download_func = _mocked_download_func @@ -50,29 +52,29 @@ class BaseMediaPipelineTestCase(unittest.TestCase): def tearDown(self): for name, signal in vars(signals).items(): - if not name.startswith('_'): + if not name.startswith("_"): disconnect_all(signal) def test_default_media_to_download(self): - request = Request('http://url') + request = Request("http://url") assert self.pipe.media_to_download(request, self.info) is None def test_default_get_media_requests(self): - item = dict(name='name') + item = dict(name="name") assert self.pipe.get_media_requests(item, self.info) is None def test_default_media_downloaded(self): - request = Request('http://url') - response = Response('http://url', body=b'') + request = Request("http://url") + response = Response("http://url", body=b"") assert self.pipe.media_downloaded(response, request, self.info) is response def test_default_media_failed(self): - request = Request('http://url') + request = Request("http://url") fail = Failure(Exception()) assert self.pipe.media_failed(fail, request, self.info) is fail def test_default_item_completed(self): - item = dict(name='name') + item = dict(name="name") assert self.pipe.item_completed([], item, self.info) is item # Check that failures are logged by default @@ -85,7 +87,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert new_item is item assert len(log.records) == 1 record = log.records[0] - assert record.levelname == 'ERROR' + assert record.levelname == "ERROR" self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) # disable failure logging and check again @@ -97,14 +99,14 @@ class BaseMediaPipelineTestCase(unittest.TestCase): @inlineCallbacks def test_default_process_item(self): - item = dict(name='name') + item = dict(name="name") new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item def test_modify_media_request(self): - request = Request('http://url') + request = Request("http://url") self.pipe._modify_media_request(request) - assert request.meta == {'handle_httpstatus_all': True} + assert request.meta == {"handle_httpstatus_all": True} def test_should_remove_req_res_references_before_caching_the_results(self): """Regression test case to prevent a memory leak in the Media Pipeline. @@ -134,8 +136,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase): Chaining (https://www.python.org/dev/peps/pep-3134/). """ # Create sample pair of Request and Response objects - request = Request('http://url') - response = Response('http://url', body=b'', request=request) + request = Request("http://url") + response = Response("http://url", body=b"", request=request) # Simulate the Media Pipeline behavior to produce a Twisted Failure try: @@ -146,7 +148,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): try: # Simulate the media_downloaded callback raising a FileException # This usually happens when the status code is not 200 OK - raise FileException('download-error') + raise FileException("download-error") except Exception as exc: file_exc = exc # Simulate Twisted capturing the FileException @@ -171,42 +173,41 @@ class BaseMediaPipelineTestCase(unittest.TestCase): # ... encapsulating the original FileException ... self.assertEqual(info.downloaded[fp].value, file_exc) # ... but it should not store the StopIteration exception on its context - context = getattr(info.downloaded[fp].value, '__context__', None) + context = getattr(info.downloaded[fp].value, "__context__", None) self.assertIsNone(context) class MockedMediaPipeline(MediaPipeline): - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._mockcalled = [] def download(self, request, info): - self._mockcalled.append('download') + self._mockcalled.append("download") return super().download(request, info) def media_to_download(self, request, info, *, item=None): - self._mockcalled.append('media_to_download') - if 'result' in request.meta: - return request.meta.get('result') + self._mockcalled.append("media_to_download") + if "result" in request.meta: + return request.meta.get("result") return super().media_to_download(request, info) def get_media_requests(self, item, info): - self._mockcalled.append('get_media_requests') - return item.get('requests') + self._mockcalled.append("get_media_requests") + return item.get("requests") def media_downloaded(self, response, request, info, *, item=None): - self._mockcalled.append('media_downloaded') + self._mockcalled.append("media_downloaded") return super().media_downloaded(response, request, info) def media_failed(self, failure, request, info): - self._mockcalled.append('media_failed') + self._mockcalled.append("media_failed") return super().media_failed(failure, request, info) def item_completed(self, results, item, info): - self._mockcalled.append('item_completed') + self._mockcalled.append("item_completed") item = super().item_completed(results, item, info) - item['results'] = results + item["results"] = results return item @@ -215,72 +216,94 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): pipeline_class = MockedMediaPipeline def _callback(self, result): - self.pipe._mockcalled.append('request_callback') + self.pipe._mockcalled.append("request_callback") return result def _errback(self, result): - self.pipe._mockcalled.append('request_errback') + self.pipe._mockcalled.append("request_errback") return result @inlineCallbacks def test_result_succeed(self): - rsp = Response('http://url1') - req = Request('http://url1', meta=dict(response=rsp), - callback=self._callback, errback=self._errback) + rsp = Response("http://url1") + req = Request( + "http://url1", + meta=dict(response=rsp), + callback=self._callback, + errback=self._errback, + ) item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item['results'], [(True, rsp)]) + self.assertEqual(new_item["results"], [(True, rsp)]) self.assertEqual( self.pipe._mockcalled, - ['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed']) + [ + "get_media_requests", + "media_to_download", + "media_downloaded", + "request_callback", + "item_completed", + ], + ) @inlineCallbacks def test_result_failure(self): self.pipe.LOG_FAILED_RESULTS = False fail = Failure(Exception()) - req = Request('http://url1', meta=dict(response=fail), - callback=self._callback, errback=self._errback) + req = Request( + "http://url1", + meta=dict(response=fail), + callback=self._callback, + errback=self._errback, + ) item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item['results'], [(False, fail)]) + self.assertEqual(new_item["results"], [(False, fail)]) self.assertEqual( self.pipe._mockcalled, - ['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed']) + [ + "get_media_requests", + "media_to_download", + "media_failed", + "request_errback", + "item_completed", + ], + ) @inlineCallbacks def test_mix_of_success_and_failure(self): self.pipe.LOG_FAILED_RESULTS = False - rsp1 = Response('http://url1') - req1 = Request('http://url1', meta=dict(response=rsp1)) + rsp1 = Response("http://url1") + req1 = Request("http://url1", meta=dict(response=rsp1)) fail = Failure(Exception()) - req2 = Request('http://url2', meta=dict(response=fail)) + req2 = Request("http://url2", meta=dict(response=fail)) item = dict(requests=[req1, req2]) new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item['results'], [(True, rsp1), (False, fail)]) + self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)]) m = self.pipe._mockcalled # only once - self.assertEqual(m[0], 'get_media_requests') # first hook called - self.assertEqual(m.count('get_media_requests'), 1) - self.assertEqual(m.count('item_completed'), 1) - self.assertEqual(m[-1], 'item_completed') # last hook called + self.assertEqual(m[0], "get_media_requests") # first hook called + self.assertEqual(m.count("get_media_requests"), 1) + self.assertEqual(m.count("item_completed"), 1) + self.assertEqual(m[-1], "item_completed") # last hook called # twice, one per request - self.assertEqual(m.count('media_to_download'), 2) + self.assertEqual(m.count("media_to_download"), 2) # one to handle success and other for failure - self.assertEqual(m.count('media_downloaded'), 1) - self.assertEqual(m.count('media_failed'), 1) + self.assertEqual(m.count("media_downloaded"), 1) + self.assertEqual(m.count("media_failed"), 1) @inlineCallbacks def test_get_media_requests(self): # returns single Request (without callback) - req = Request('http://url') + req = Request("http://url") item = dict(requests=req) # pass a single item new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item self.assertIn(self.fingerprint(req), self.info.downloaded) # returns iterable of Requests - req1 = Request('http://url1') - req2 = Request('http://url2') + req1 = Request("http://url1") + req2 = Request("http://url2") item = dict(requests=iter([req1, req2])) new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item @@ -289,30 +312,34 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): @inlineCallbacks def test_results_are_cached_across_multiple_items(self): - rsp1 = Response('http://url1') - req1 = Request('http://url1', meta=dict(response=rsp1)) + rsp1 = Response("http://url1") + req1 = Request("http://url1", meta=dict(response=rsp1)) item = dict(requests=req1) new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) - self.assertEqual(new_item['results'], [(True, rsp1)]) + self.assertEqual(new_item["results"], [(True, rsp1)]) # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same - req2 = Request(req1.url, meta=dict(response=Response('http://donot.download.me'))) + req2 = Request( + req1.url, meta=dict(response=Response("http://donot.download.me")) + ) item = dict(requests=req2) new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) - self.assertEqual(new_item['results'], [(True, rsp1)]) + self.assertEqual(new_item["results"], [(True, rsp1)]) @inlineCallbacks def test_results_are_cached_for_requests_of_single_item(self): - rsp1 = Response('http://url1') - req1 = Request('http://url1', meta=dict(response=rsp1)) - req2 = Request(req1.url, meta=dict(response=Response('http://donot.download.me'))) + rsp1 = Response("http://url1") + req1 = Request("http://url1", meta=dict(response=rsp1)) + req2 = Request( + req1.url, meta=dict(response=Response("http://donot.download.me")) + ) item = dict(requests=[req1, req2]) new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) - self.assertEqual(new_item['results'], [(True, rsp1), (True, rsp1)]) + self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) @inlineCallbacks def test_wait_if_request_is_downloading(self): @@ -324,78 +351,80 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): self.assertEqual(len(self.info.waiting[fp]), 2) return response - rsp1 = Response('http://url') + rsp1 = Response("http://url") def rsp1_func(): dfd = Deferred().addCallback(_check_downloading) - reactor.callLater(.1, dfd.callback, rsp1) + reactor.callLater(0.1, dfd.callback, rsp1) return dfd def rsp2_func(): - self.fail('it must cache rsp1 result and must not try to redownload') + self.fail("it must cache rsp1 result and must not try to redownload") - req1 = Request('http://url', meta=dict(response=rsp1_func)) + req1 = Request("http://url", meta=dict(response=rsp1_func)) req2 = Request(req1.url, meta=dict(response=rsp2_func)) item = dict(requests=[req1, req2]) new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item['results'], [(True, rsp1), (True, rsp1)]) + self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) @inlineCallbacks def test_use_media_to_download_result(self): - req = Request('http://url', meta=dict(result='ITSME', response=self.fail)) + req = Request("http://url", meta=dict(result="ITSME", response=self.fail)) item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item['results'], [(True, 'ITSME')]) + self.assertEqual(new_item["results"], [(True, "ITSME")]) self.assertEqual( self.pipe._mockcalled, - ['get_media_requests', 'media_to_download', 'item_completed']) + ["get_media_requests", "media_to_download", "item_completed"], + ) class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._mockcalled = [] def get_media_requests(self, item, info): - item_url = item['image_urls'][0] + item_url = item["image_urls"][0] output_img = io.BytesIO() - img = Image.new('RGB', (60, 30), color='red') - img.save(output_img, format='JPEG') + img = Image.new("RGB", (60, 30), color="red") + img.save(output_img, format="JPEG") return Request( item_url, - meta={'response': Response(item_url, status=200, body=output_img.getvalue())} + meta={ + "response": Response(item_url, status=200, body=output_img.getvalue()) + }, ) def inc_stats(self, *args, **kwargs): return True def media_to_download(self, request, info): - self._mockcalled.append('media_to_download') + self._mockcalled.append("media_to_download") return super().media_to_download(request, info) def media_downloaded(self, response, request, info): - self._mockcalled.append('media_downloaded') + self._mockcalled.append("media_downloaded") return super().media_downloaded(response, request, info) def file_downloaded(self, response, request, info): - self._mockcalled.append('file_downloaded') + self._mockcalled.append("file_downloaded") return super().file_downloaded(response, request, info) def file_path(self, request, response=None, info=None): - self._mockcalled.append('file_path') + self._mockcalled.append("file_path") return super().file_path(request, response, info) def thumb_path(self, request, thumb_id, response=None, info=None): - self._mockcalled.append('thumb_path') + self._mockcalled.append("thumb_path") return super().thumb_path(request, thumb_id, response, info) def get_images(self, response, request, info): - self._mockcalled.append('get_images') + self._mockcalled.append("get_images") return super().get_images(response, request, info) def image_downloaded(self, response, request, info): - self._mockcalled.append('image_downloaded') + self._mockcalled.append("image_downloaded") return super().image_downloaded(response, request, info) @@ -404,20 +433,23 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): def setUp(self): settings_dict = { - 'IMAGES_STORE': 'store-uri', - 'IMAGES_THUMBS': {'small': (50, 50)}, + "IMAGES_STORE": "store-uri", + "IMAGES_THUMBS": {"small": (50, 50)}, } crawler = get_crawler(spidercls=None, settings_dict=settings_dict) self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler) self.pipe.download_func = _mocked_download_func self.pipe.open_spider(None) - self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[]) + self.item = dict(image_urls=["http://picsum.photos/id/1014/200/300"], images=[]) def _assert_method_called_with_warnings(self, method, message, warnings): self.assertIn(method, self.pipe._mockcalled) warningShown = False for warning in warnings: - if warning['message'] == message and warning['category'] == ScrapyDeprecationWarning: + if ( + warning["message"] == message + and warning["category"] == ScrapyDeprecationWarning + ): warningShown = True self.assertTrue(warningShown) @@ -426,125 +458,111 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): yield self.pipe.process_item(self.item, None) warnings = self.flushWarnings([MediaPipeline._compatible]) message = ( - 'media_to_download(self, request, info) is deprecated, ' - 'please use media_to_download(self, request, info, *, item=None)' + "media_to_download(self, request, info) is deprecated, " + "please use media_to_download(self, request, info, *, item=None)" ) - self._assert_method_called_with_warnings('media_to_download', message, warnings) + self._assert_method_called_with_warnings("media_to_download", message, warnings) @inlineCallbacks def test_media_downloaded_called(self): yield self.pipe.process_item(self.item, None) warnings = self.flushWarnings([MediaPipeline._compatible]) message = ( - 'media_downloaded(self, response, request, info) is deprecated, ' - 'please use media_downloaded(self, response, request, info, *, item=None)' + "media_downloaded(self, response, request, info) is deprecated, " + "please use media_downloaded(self, response, request, info, *, item=None)" ) - self._assert_method_called_with_warnings('media_downloaded', message, warnings) + self._assert_method_called_with_warnings("media_downloaded", message, warnings) @inlineCallbacks def test_file_downloaded_called(self): yield self.pipe.process_item(self.item, None) warnings = self.flushWarnings([MediaPipeline._compatible]) message = ( - 'file_downloaded(self, response, request, info) is deprecated, ' - 'please use file_downloaded(self, response, request, info, *, item=None)' + "file_downloaded(self, response, request, info) is deprecated, " + "please use file_downloaded(self, response, request, info, *, item=None)" ) - self._assert_method_called_with_warnings('file_downloaded', message, warnings) + self._assert_method_called_with_warnings("file_downloaded", message, warnings) @inlineCallbacks def test_file_path_called(self): yield self.pipe.process_item(self.item, None) warnings = self.flushWarnings([MediaPipeline._compatible]) message = ( - 'file_path(self, request, response=None, info=None) is deprecated, ' - 'please use file_path(self, request, response=None, info=None, *, item=None)' + "file_path(self, request, response=None, info=None) is deprecated, " + "please use file_path(self, request, response=None, info=None, *, item=None)" ) - self._assert_method_called_with_warnings('file_path', message, warnings) + self._assert_method_called_with_warnings("file_path", message, warnings) @inlineCallbacks def test_thumb_path_called(self): yield self.pipe.process_item(self.item, None) warnings = self.flushWarnings([MediaPipeline._compatible]) message = ( - 'thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, ' - 'please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)' + "thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, " + "please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)" ) - self._assert_method_called_with_warnings('thumb_path', message, warnings) + self._assert_method_called_with_warnings("thumb_path", message, warnings) @inlineCallbacks def test_get_images_called(self): yield self.pipe.process_item(self.item, None) warnings = self.flushWarnings([MediaPipeline._compatible]) message = ( - 'get_images(self, response, request, info) is deprecated, ' - 'please use get_images(self, response, request, info, *, item=None)' + "get_images(self, response, request, info) is deprecated, " + "please use get_images(self, response, request, info, *, item=None)" ) - self._assert_method_called_with_warnings('get_images', message, warnings) + self._assert_method_called_with_warnings("get_images", message, warnings) @inlineCallbacks def test_image_downloaded_called(self): yield self.pipe.process_item(self.item, None) warnings = self.flushWarnings([MediaPipeline._compatible]) message = ( - 'image_downloaded(self, response, request, info) is deprecated, ' - 'please use image_downloaded(self, response, request, info, *, item=None)' + "image_downloaded(self, response, request, info) is deprecated, " + "please use image_downloaded(self, response, request, info, *, item=None)" ) - self._assert_method_called_with_warnings('image_downloaded', message, warnings) + self._assert_method_called_with_warnings("image_downloaded", message, warnings) class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): - def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(settings=Settings(settings)) - request = Request('http://url') + request = Request("http://url") pipe._modify_media_request(request) - self.assertIn('handle_httpstatus_list', request.meta) + self.assertIn("handle_httpstatus_list", request.meta) for status, check in [ - (200, True), - - # These are the status codes we want - # the downloader to handle itself - (301, False), - (302, False), - (302, False), - (307, False), - (308, False), - - # we still want to get 4xx and 5xx - (400, True), - (404, True), - (500, True)]: + (200, True), + # These are the status codes we want + # the downloader to handle itself + (301, False), + (302, False), + (302, False), + (307, False), + (308, False), + # we still want to get 4xx and 5xx + (400, True), + (404, True), + (500, True), + ]: if check: - self.assertIn(status, request.meta['handle_httpstatus_list']) + self.assertIn(status, request.meta["handle_httpstatus_list"]) else: - self.assertNotIn(status, request.meta['handle_httpstatus_list']) + self.assertNotIn(status, request.meta["handle_httpstatus_list"]) def test_standard_setting(self): - self._assert_request_no3xx( - MediaPipeline, - { - 'MEDIA_ALLOW_REDIRECTS': True - }) + self._assert_request_no3xx(MediaPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) def test_subclass_standard_setting(self): - class UserDefinedPipeline(MediaPipeline): pass - self._assert_request_no3xx( - UserDefinedPipeline, - { - 'MEDIA_ALLOW_REDIRECTS': True - }) + self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) def test_subclass_specific_setting(self): - class UserDefinedPipeline(MediaPipeline): pass self._assert_request_no3xx( - UserDefinedPipeline, - { - 'USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS': True - }) + UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True} + ) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 8e432b913..7b905d321 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -14,13 +14,13 @@ from tests.mockserver import MockServer class SimplePipeline: def process_item(self, item, spider): - item['pipeline_passed'] = True + item["pipeline_passed"] = True return item class DeferredPipeline: def cb(self, item): - item['pipeline_passed'] = True + item["pipeline_passed"] = True return item def process_item(self, item, spider): @@ -34,9 +34,10 @@ class AsyncDefPipeline: async def process_item(self, item, spider): d = Deferred() from twisted.internet import reactor + reactor.callLater(0, d.callback, None) await maybe_deferred_to_future(d) - item['pipeline_passed'] = True + item["pipeline_passed"] = True return item @@ -44,10 +45,11 @@ class AsyncDefAsyncioPipeline: async def process_item(self, item, spider): d = Deferred() from twisted.internet import reactor + reactor.callLater(0, d.callback, None) await deferred_to_future(d) await asyncio.sleep(0.2) - item['pipeline_passed'] = await get_from_asyncio_queue(True) + item["pipeline_passed"] = await get_from_asyncio_queue(True) return item @@ -55,23 +57,24 @@ class AsyncDefNotAsyncioPipeline: async def process_item(self, item, spider): d1 = Deferred() from twisted.internet import reactor + reactor.callLater(0, d1.callback, None) await d1 d2 = Deferred() reactor.callLater(0, d2.callback, None) await maybe_deferred_to_future(d2) - item['pipeline_passed'] = True + item["pipeline_passed"] = True return item class ItemSpider(Spider): - name = 'itemspider' + name = "itemspider" def start_requests(self): - yield Request(self.mockserver.url('/status?n=200')) + yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): - return {'field': 42} + return {"field": 42} class PipelineTestCase(unittest.TestCase): @@ -84,12 +87,12 @@ class PipelineTestCase(unittest.TestCase): def _on_item_scraped(self, item): self.assertIsInstance(item, dict) - self.assertTrue(item.get('pipeline_passed')) + self.assertTrue(item.get("pipeline_passed")) self.items.append(item) def _create_crawler(self, pipeline_class): settings = { - 'ITEM_PIPELINES': {pipeline_class: 1}, + "ITEM_PIPELINES": {pipeline_class: 1}, } crawler = get_crawler(ItemSpider, settings) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index ec55033d1..96a64c19d 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -19,7 +19,9 @@ class PriorityQueueTest(unittest.TestCase): def test_queue_push_pop_one(self): temp_dir = tempfile.mkdtemp() - queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + queue = ScrapyPriorityQueue.from_crawler( + self.crawler, FifoMemoryQueue, temp_dir + ) self.assertIsNone(queue.pop()) self.assertEqual(len(queue), 0) req1 = Request("https://example.org/1", priority=1) @@ -35,9 +37,14 @@ class PriorityQueueTest(unittest.TestCase): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") temp_dir = tempfile.mkdtemp() - queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + queue = ScrapyPriorityQueue.from_crawler( + self.crawler, FifoMemoryQueue, temp_dir + ) queue.push(Request("https://example.org")) - with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + with self.assertRaises( + NotImplementedError, + msg="The underlying queue class does not implement 'peek'", + ): queue.peek() queue.close() @@ -45,7 +52,9 @@ class PriorityQueueTest(unittest.TestCase): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") temp_dir = tempfile.mkdtemp() - queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + queue = ScrapyPriorityQueue.from_crawler( + self.crawler, FifoMemoryQueue, temp_dir + ) self.assertEqual(len(queue), 0) self.assertIsNone(queue.peek()) req1 = Request("https://example.org/1") @@ -67,7 +76,9 @@ class PriorityQueueTest(unittest.TestCase): def test_queue_push_pop_priorities(self): temp_dir = tempfile.mkdtemp() - queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]) + queue = ScrapyPriorityQueue.from_crawler( + self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3] + ) self.assertIsNone(queue.pop()) self.assertEqual(len(queue), 0) req1 = Request("https://example.org/1", priority=1) @@ -119,7 +130,10 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") self.queue.push(Request("https://example.org")) - with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + with self.assertRaises( + NotImplementedError, + msg="The underlying queue class does not implement 'peek'", + ): self.queue.peek() def test_peek(self): diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index ea7701b5d..5aeae7546 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -17,30 +17,40 @@ from tests.spiders import SimpleSpider, SingleRequestSpider class MitmProxy: - auth_user = 'scrapy' - auth_pass = 'scrapy' + auth_user = "scrapy" + auth_pass = "scrapy" def start(self): from scrapy.utils.test import get_testenv + script = """ import sys from mitmproxy.tools.main import mitmdump sys.argv[0] = "mitmdump" sys.exit(mitmdump()) """ - cert_path = Path(__file__).parent.resolve() / 'keys' / 'mitmproxy-ca.pem' - self.proc = Popen([sys.executable, - '-c', script, - '--listen-host', '127.0.0.1', - '--listen-port', '0', - '--proxyauth', f'{self.auth_user}:{self.auth_pass}', - '--certs', str(cert_path), - '--ssl-insecure', - ], - stdout=PIPE, env=get_testenv()) - line = self.proc.stdout.readline().decode('utf-8') - host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1) - address = f'http://{self.auth_user}:{self.auth_pass}@{host_port}' + cert_path = Path(__file__).parent.resolve() / "keys" / "mitmproxy-ca.pem" + self.proc = Popen( + [ + sys.executable, + "-c", + script, + "--listen-host", + "127.0.0.1", + "--listen-port", + "0", + "--proxyauth", + f"{self.auth_user}:{self.auth_pass}", + "--certs", + str(cert_path), + "--ssl-insecure", + ], + stdout=PIPE, + env=get_testenv(), + ) + line = self.proc.stdout.readline().decode("utf-8") + host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1) + address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}" return address def stop(self): @@ -50,17 +60,16 @@ sys.exit(mitmdump()) def _wrong_credentials(proxy_url): bad_auth_proxy = list(urlsplit(proxy_url)) - bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@') + bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@") return urlunsplit(bad_auth_proxy) class ProxyConnectTestCase(TestCase): - def setUp(self): try: import mitmproxy # noqa: F401 except ImportError: - self.skipTest('mitmproxy is not installed') + self.skipTest("mitmproxy is not installed") self.mockserver = MockServer() self.mockserver.__enter__() @@ -68,8 +77,8 @@ class ProxyConnectTestCase(TestCase): self._proxy = MitmProxy() proxy_url = self._proxy.start() - os.environ['https_proxy'] = proxy_url - os.environ['http_proxy'] = proxy_url + os.environ["https_proxy"] = proxy_url + os.environ["http_proxy"] = proxy_url def tearDown(self): self.mockserver.__exit__(None, None, None) @@ -85,7 +94,7 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_tunnel_auth_error(self): - os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"]) crawler = get_crawler(SimpleSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) @@ -100,13 +109,13 @@ class ProxyConnectTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(seed=request) self._assert_got_response_code(200, log) - echo = json.loads(crawler.spider.meta['responses'][0].text) - self.assertTrue('Proxy-Authorization' not in echo['headers']) + echo = json.loads(crawler.spider.meta["responses"][0].text) + self.assertTrue("Proxy-Authorization" not in echo["headers"]) def _assert_got_response_code(self, code, log): print(log) - self.assertEqual(str(log).count(f'Crawled ({code})'), 1) + self.assertEqual(str(log).count(f"Crawled ({code})"), 1) def _assert_got_tunnel_error(self, log): print(log) - self.assertIn('TunnelError', str(log)) + self.assertIn("TunnelError", str(log)) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 15e400327..f7540743d 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -60,7 +60,6 @@ class AlternativeCallbacksMiddleware: class CrawlTestCase(TestCase): - def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() @@ -90,11 +89,14 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_downloader_middleware_raise_exception(self): url = self.mockserver.url("/status?n=200") - crawler = get_crawler(SingleRequestSpider, { - "DOWNLOADER_MIDDLEWARES": { - RaiseExceptionRequestMiddleware: 590, + crawler = get_crawler( + SingleRequestSpider, + { + "DOWNLOADER_MIDDLEWARES": { + RaiseExceptionRequestMiddleware: 590, + }, }, - }) + ) yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] self.assertEqual(failure.request.url, url) @@ -116,11 +118,14 @@ class CrawlTestCase(TestCase): signal_params["request"] = request url = self.mockserver.url("/status?n=200") - crawler = get_crawler(SingleRequestSpider, { - "DOWNLOADER_MIDDLEWARES": { - ProcessResponseMiddleware: 595, - } - }) + crawler = get_crawler( + SingleRequestSpider, + { + "DOWNLOADER_MIDDLEWARES": { + ProcessResponseMiddleware: 595, + } + }, + ) crawler.signals.connect(signal_handler, signal=signals.response_received) with LogCapture() as log: @@ -133,7 +138,11 @@ class CrawlTestCase(TestCase): self.assertEqual(signal_params["request"].url, OVERRIDEN_URL) log.check_present( - ("scrapy.core.engine", "DEBUG", f"Crawled (200) <GET {OVERRIDEN_URL}> (referer: None)"), + ( + "scrapy.core.engine", + "DEBUG", + f"Crawled (200) <GET {OVERRIDEN_URL}> (referer: None)", + ), ) @defer.inlineCallbacks @@ -145,12 +154,15 @@ class CrawlTestCase(TestCase): The spider callback should receive the overridden response.request """ url = self.mockserver.url("/status?n=200") - crawler = get_crawler(SingleRequestSpider, { - "DOWNLOADER_MIDDLEWARES": { - RaiseExceptionRequestMiddleware: 590, - CatchExceptionOverrideRequestMiddleware: 595, + crawler = get_crawler( + SingleRequestSpider, + { + "DOWNLOADER_MIDDLEWARES": { + RaiseExceptionRequestMiddleware: 590, + CatchExceptionOverrideRequestMiddleware: 595, + }, }, - }) + ) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] self.assertEqual(response.body, b"Caught ZeroDivisionError") @@ -165,12 +177,15 @@ class CrawlTestCase(TestCase): The spider callback should receive the original response.request """ url = self.mockserver.url("/status?n=200") - crawler = get_crawler(SingleRequestSpider, { - "DOWNLOADER_MIDDLEWARES": { - RaiseExceptionRequestMiddleware: 590, - CatchExceptionDoNotOverrideRequestMiddleware: 595, + crawler = get_crawler( + SingleRequestSpider, + { + "DOWNLOADER_MIDDLEWARES": { + RaiseExceptionRequestMiddleware: 590, + CatchExceptionDoNotOverrideRequestMiddleware: 595, + }, }, - }) + ) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] self.assertEqual(response.body, b"Caught ZeroDivisionError") @@ -182,16 +197,23 @@ class CrawlTestCase(TestCase): Downloader middleware which returns a response with a specific 'request' attribute, with an alternative callback """ - crawler = get_crawler(AlternativeCallbacksSpider, { - "DOWNLOADER_MIDDLEWARES": { - AlternativeCallbacksMiddleware: 595, - } - }) + crawler = get_crawler( + AlternativeCallbacksSpider, + { + "DOWNLOADER_MIDDLEWARES": { + AlternativeCallbacksMiddleware: 595, + } + }, + ) with LogCapture() as log: url = self.mockserver.url("/status?n=200") yield crawler.crawl(seed=url, mockserver=self.mockserver) log.check_present( - ("alternative_callbacks_spider", "INFO", "alt_callback was invoked with foo=bar"), + ( + "alternative_callbacks_spider", + "INFO", + "alt_callback was invoked with foo=bar", + ), ) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 63e3aac00..5a3dae103 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -12,14 +12,15 @@ class InjectArgumentsDownloaderMiddleware: """ Make sure downloader middlewares are able to update the keyword arguments """ + def process_request(self, request, spider): - if request.callback.__name__ == 'parse_downloader_mw': - request.cb_kwargs['from_process_request'] = True + if request.callback.__name__ == "parse_downloader_mw": + request.cb_kwargs["from_process_request"] = True return None def process_response(self, request, response, spider): - if request.callback.__name__ == 'parse_downloader_mw': - request.cb_kwargs['from_process_response'] = True + if request.callback.__name__ == "parse_downloader_mw": + request.cb_kwargs["from_process_response"] = True return response @@ -27,32 +28,36 @@ class InjectArgumentsSpiderMiddleware: """ Make sure spider middlewares are able to update the keyword arguments """ + def process_start_requests(self, start_requests, spider): for request in start_requests: - if request.callback.__name__ == 'parse_spider_mw': - request.cb_kwargs['from_process_start_requests'] = True + if request.callback.__name__ == "parse_spider_mw": + request.cb_kwargs["from_process_start_requests"] = True yield request def process_spider_input(self, response, spider): request = response.request - if request.callback.__name__ == 'parse_spider_mw': - request.cb_kwargs['from_process_spider_input'] = True + if request.callback.__name__ == "parse_spider_mw": + request.cb_kwargs["from_process_spider_input"] = True return None def process_spider_output(self, response, result, spider): for element in result: - if isinstance(element, Request) and element.callback.__name__ == 'parse_spider_mw_2': - element.cb_kwargs['from_process_spider_output'] = True + if ( + isinstance(element, Request) + and element.callback.__name__ == "parse_spider_mw_2" + ): + element.cb_kwargs["from_process_spider_output"] = True yield element class KeywordArgumentsSpider(MockServerSpider): - name = 'kwargs' + name = "kwargs" custom_settings = { - 'DOWNLOADER_MIDDLEWARES': { + "DOWNLOADER_MIDDLEWARES": { InjectArgumentsDownloaderMiddleware: 750, }, - 'SPIDER_MIDDLEWARES': { + "SPIDER_MIDDLEWARES": { InjectArgumentsSpiderMiddleware: 750, }, } @@ -60,50 +65,61 @@ class KeywordArgumentsSpider(MockServerSpider): checks = [] def start_requests(self): - data = {'key': 'value', 'number': 123, 'callback': 'some_callback'} - yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data) - yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data) - yield Request(self.mockserver.url('/general_without'), self.parse_general) - yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs) - yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data) - yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data) - yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data) - yield Request(self.mockserver.url('/downloader_mw'), self.parse_downloader_mw) - yield Request(self.mockserver.url('/spider_mw'), self.parse_spider_mw) + data = {"key": "value", "number": 123, "callback": "some_callback"} + yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data) + yield Request( + self.mockserver.url("/general_with"), self.parse_general, cb_kwargs=data + ) + yield Request(self.mockserver.url("/general_without"), self.parse_general) + yield Request(self.mockserver.url("/no_kwargs"), self.parse_no_kwargs) + yield Request( + self.mockserver.url("/default"), self.parse_default, cb_kwargs=data + ) + yield Request( + self.mockserver.url("/takes_less"), self.parse_takes_less, cb_kwargs=data + ) + yield Request( + self.mockserver.url("/takes_more"), self.parse_takes_more, cb_kwargs=data + ) + yield Request(self.mockserver.url("/downloader_mw"), self.parse_downloader_mw) + yield Request(self.mockserver.url("/spider_mw"), self.parse_spider_mw) def parse_first(self, response, key, number): - self.checks.append(key == 'value') + self.checks.append(key == "value") self.checks.append(number == 123) - self.crawler.stats.inc_value('boolean_checks', 2) + self.crawler.stats.inc_value("boolean_checks", 2) yield response.follow( - self.mockserver.url('/two'), + self.mockserver.url("/two"), self.parse_second, - cb_kwargs={'new_key': 'new_value'}) + cb_kwargs={"new_key": "new_value"}, + ) def parse_second(self, response, new_key): - self.checks.append(new_key == 'new_value') - self.crawler.stats.inc_value('boolean_checks') + self.checks.append(new_key == "new_value") + self.crawler.stats.inc_value("boolean_checks") def parse_general(self, response, **kwargs): - if response.url.endswith('/general_with'): - self.checks.append(kwargs['key'] == 'value') - self.checks.append(kwargs['number'] == 123) - self.checks.append(kwargs['callback'] == 'some_callback') - self.crawler.stats.inc_value('boolean_checks', 3) - elif response.url.endswith('/general_without'): - self.checks.append(kwargs == {}) # pylint: disable=use-implicit-booleaness-not-comparison - self.crawler.stats.inc_value('boolean_checks') + if response.url.endswith("/general_with"): + self.checks.append(kwargs["key"] == "value") + self.checks.append(kwargs["number"] == 123) + self.checks.append(kwargs["callback"] == "some_callback") + self.crawler.stats.inc_value("boolean_checks", 3) + elif response.url.endswith("/general_without"): + self.checks.append( + kwargs == {} + ) # pylint: disable=use-implicit-booleaness-not-comparison + self.crawler.stats.inc_value("boolean_checks") def parse_no_kwargs(self, response): - self.checks.append(response.url.endswith('/no_kwargs')) - self.crawler.stats.inc_value('boolean_checks') + self.checks.append(response.url.endswith("/no_kwargs")) + self.crawler.stats.inc_value("boolean_checks") def parse_default(self, response, key, number=None, default=99): - self.checks.append(response.url.endswith('/default')) - self.checks.append(key == 'value') + self.checks.append(response.url.endswith("/default")) + self.checks.append(key == "value") self.checks.append(number == 123) self.checks.append(default == 99) - self.crawler.stats.inc_value('boolean_checks', 4) + self.crawler.stats.inc_value("boolean_checks", 4) def parse_takes_less(self, response, key, callback): """ @@ -117,20 +133,24 @@ class KeywordArgumentsSpider(MockServerSpider): TypeError: parse_takes_more() missing 1 required positional argument: 'other' """ - def parse_downloader_mw(self, response, from_process_request, from_process_response): + def parse_downloader_mw( + self, response, from_process_request, from_process_response + ): self.checks.append(bool(from_process_request)) self.checks.append(bool(from_process_response)) - self.crawler.stats.inc_value('boolean_checks', 2) + self.crawler.stats.inc_value("boolean_checks", 2) - def parse_spider_mw(self, response, from_process_spider_input, from_process_start_requests): + def parse_spider_mw( + self, response, from_process_spider_input, from_process_start_requests + ): self.checks.append(bool(from_process_spider_input)) self.checks.append(bool(from_process_start_requests)) - self.crawler.stats.inc_value('boolean_checks', 2) - return Request(self.mockserver.url('/spider_mw_2'), self.parse_spider_mw_2) + self.crawler.stats.inc_value("boolean_checks", 2) + return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) def parse_spider_mw_2(self, response, from_process_spider_output): self.checks.append(bool(from_process_spider_output)) - self.crawler.stats.inc_value('boolean_checks', 1) + self.crawler.stats.inc_value("boolean_checks", 1) class CallbackKeywordArgumentsTestCase(TestCase): @@ -150,24 +170,26 @@ class CallbackKeywordArgumentsTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) self.assertTrue(all(crawler.spider.checks)) - self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks')) + self.assertEqual( + len(crawler.spider.checks), crawler.stats.get_value("boolean_checks") + ) # check exceptions for argument mismatch exceptions = {} for line in log.records: - for key in ('takes_less', 'takes_more'): + for key in ("takes_less", "takes_more"): if key in line.getMessage(): exceptions[key] = line - self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError) + self.assertEqual(exceptions["takes_less"].exc_info[0], TypeError) self.assertTrue( - str(exceptions['takes_less'].exc_info[1]).endswith( + str(exceptions["takes_less"].exc_info[1]).endswith( "parse_takes_less() got an unexpected keyword argument 'number'" ), - msg="Exception message: " + str(exceptions['takes_less'].exc_info[1]), + msg="Exception message: " + str(exceptions["takes_less"].exc_info[1]), ) - self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) + self.assertEqual(exceptions["takes_more"].exc_info[0], TypeError) self.assertTrue( - str(exceptions['takes_more'].exc_info[1]).endswith( + str(exceptions["takes_more"].exc_info[1]).endswith( "parse_takes_more() missing 1 required positional argument: 'other'" ), - msg="Exception message: " + str(exceptions['takes_more'].exc_info[1]), + msg="Exception message: " + str(exceptions["takes_more"].exc_info[1]), ) diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 5bdcb975b..d9067610e 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -14,7 +14,6 @@ class CustomRequest(Request): class RequestSerializationTest(unittest.TestCase): - def setUp(self): self.spider = TestSpider() @@ -29,13 +28,13 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error, method="POST", body=b"some body", - headers={'content-encoding': 'text/html; charset=latin-1'}, - cookies={'currency': 'руб'}, - encoding='latin-1', + headers={"content-encoding": "text/html; charset=latin-1"}, + cookies={"currency": "руб"}, + encoding="latin-1", priority=20, - meta={'a': 'b'}, - cb_kwargs={'k': 'v'}, - flags=['testFlag'], + meta={"a": "b"}, + cb_kwargs={"k": "v"}, + flags=["testFlag"], ) self._assert_serializes_ok(r, spider=self.spider) @@ -80,46 +79,57 @@ class RequestSerializationTest(unittest.TestCase): self._assert_serializes_ok(r3, spider=self.spider) def test_callback_serialization(self): - r = Request("http://www.example.com", callback=self.spider.parse_item, - errback=self.spider.handle_error) + r = Request( + "http://www.example.com", + callback=self.spider.parse_item, + errback=self.spider.handle_error, + ) self._assert_serializes_ok(r, spider=self.spider) def test_reference_callback_serialization(self): - r = Request("http://www.example.com", - callback=self.spider.parse_item_reference, - errback=self.spider.handle_error_reference) + r = Request( + "http://www.example.com", + callback=self.spider.parse_item_reference, + errback=self.spider.handle_error_reference, + ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) - self.assertEqual(request_dict['callback'], 'parse_item_reference') - self.assertEqual(request_dict['errback'], 'handle_error_reference') + self.assertEqual(request_dict["callback"], "parse_item_reference") + self.assertEqual(request_dict["errback"], "handle_error_reference") def test_private_reference_callback_serialization(self): - r = Request("http://www.example.com", - callback=self.spider._TestSpider__parse_item_reference, - errback=self.spider._TestSpider__handle_error_reference) + r = Request( + "http://www.example.com", + callback=self.spider._TestSpider__parse_item_reference, + errback=self.spider._TestSpider__handle_error_reference, + ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) - self.assertEqual(request_dict['callback'], - '_TestSpider__parse_item_reference') - self.assertEqual(request_dict['errback'], - '_TestSpider__handle_error_reference') + self.assertEqual(request_dict["callback"], "_TestSpider__parse_item_reference") + self.assertEqual(request_dict["errback"], "_TestSpider__handle_error_reference") def test_private_callback_serialization(self): - r = Request("http://www.example.com", - callback=self.spider._TestSpider__parse_item_private, - errback=self.spider.handle_error) + r = Request( + "http://www.example.com", + callback=self.spider._TestSpider__parse_item_private, + errback=self.spider.handle_error, + ) self._assert_serializes_ok(r, spider=self.spider) def test_mixin_private_callback_serialization(self): - r = Request("http://www.example.com", - callback=self.spider._TestSpiderMixin__mixin_callback, - errback=self.spider.handle_error) + r = Request( + "http://www.example.com", + callback=self.spider._TestSpiderMixin__mixin_callback, + errback=self.spider.handle_error, + ) self._assert_serializes_ok(r, spider=self.spider) def test_delegated_callback_serialization(self): - r = Request("http://www.example.com", - callback=self.spider.delegated_callback, - errback=self.spider.handle_error) + r = Request( + "http://www.example.com", + callback=self.spider.delegated_callback, + errback=self.spider.handle_error, + ) self._assert_serializes_ok(r, spider=self.spider) def test_unserializable_callback1(self): @@ -134,14 +144,14 @@ class RequestSerializationTest(unittest.TestCase): """Parser method is removed or replaced dynamically.""" class MySpider(Spider): - name = 'my_spider' + name = "my_spider" def parse(self, response): pass spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) - setattr(spider, 'parse', None) + setattr(spider, "parse", None) self.assertRaises(ValueError, r.to_dict, spider=spider) def test_callback_not_available(self): @@ -157,9 +167,14 @@ class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest): with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") with suppress(KeyError): - del sys.modules["scrapy.utils.reqser"] # delete module to reset the deprecation warning + del sys.modules[ + "scrapy.utils.reqser" + ] # delete module to reset the deprecation warning - from scrapy.utils.reqser import request_from_dict as _from_dict, request_to_dict as _to_dict + from scrapy.utils.reqser import ( + request_from_dict as _from_dict, + request_to_dict as _to_dict, + ) request_copy = _from_dict(_to_dict(request, spider), spider) self._assert_same_request(request, request_copy) @@ -200,7 +215,7 @@ def private_handle_error(failure): class TestSpider(Spider, TestSpiderMixin): - name = 'test' + name = "test" parse_item_reference = parse_item handle_error_reference = handle_error __parse_item_reference = private_parse_item diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 4d4483881..d08ed0f68 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -7,12 +7,11 @@ from tests.mockserver import MockServer class SignalCatcherSpider(Spider): - name = 'signal_catcher' + name = "signal_catcher" def __init__(self, crawler, url, *args, **kwargs): super().__init__(*args, **kwargs) - crawler.signals.connect(self.on_request_left, - signal=request_left_downloader) + crawler.signals.connect(self.on_request_left, signal=request_left_downloader) self.caught_times = 0 self.start_urls = [url] @@ -26,7 +25,6 @@ class SignalCatcherSpider(Spider): class TestCatching(TestCase): - def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() @@ -42,8 +40,7 @@ class TestCatching(TestCase): @defer.inlineCallbacks def test_timeout(self): - crawler = get_crawler(SignalCatcherSpider, - {'DOWNLOAD_TIMEOUT': 0.1}) + crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1}) yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) self.assertEqual(crawler.spider.caught_times, 1) @@ -56,5 +53,5 @@ class TestCatching(TestCase): @defer.inlineCallbacks def test_noconnect(self): crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl('http://thereisdefinetelynosuchdomain.com') + yield crawler.crawl("http://thereisdefinetelynosuchdomain.com") self.assertEqual(crawler.spider.caught_times, 1) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 4b4095fb0..57484a2a1 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -5,15 +5,14 @@ from scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse, Heade class ResponseTypesTest(unittest.TestCase): - def test_from_filename(self): mappings = [ - ('data.bin', Response), - ('file.txt', TextResponse), - ('file.xml.gz', Response), - ('file.xml', XmlResponse), - ('file.html', HtmlResponse), - ('file.unknownext', Response), + ("data.bin", Response), + ("file.txt", TextResponse), + ("file.xml.gz", Response), + ("file.xml", XmlResponse), + ("file.html", HtmlResponse), + ("file.unknownext", Response), ] for source, cls in mappings: retcls = responsetypes.from_filename(source) @@ -22,13 +21,12 @@ class ResponseTypesTest(unittest.TestCase): def test_from_content_disposition(self): mappings = [ (b'attachment; filename="data.xml"', XmlResponse), - (b'attachment; filename=data.xml', XmlResponse), - ('attachment;filename=data£.tar.gz'.encode('utf-8'), Response), - ('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response), - ('attachment;filename=data高.doc'.encode('gbk'), Response), - ('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse), - ('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse), - + (b"attachment; filename=data.xml", XmlResponse), + ("attachment;filename=data£.tar.gz".encode("utf-8"), Response), + ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response), + ("attachment;filename=data高.doc".encode("gbk"), Response), + ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse), + ("attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), XmlResponse), ] for source, cls in mappings: retcls = responsetypes.from_content_disposition(source) @@ -36,14 +34,14 @@ class ResponseTypesTest(unittest.TestCase): def test_from_content_type(self): mappings = [ - ('text/html; charset=UTF-8', HtmlResponse), - ('text/xml; charset=UTF-8', XmlResponse), - ('application/xhtml+xml; charset=UTF-8', HtmlResponse), - ('application/vnd.wap.xhtml+xml; charset=utf-8', HtmlResponse), - ('application/xml; charset=UTF-8', XmlResponse), - ('application/octet-stream', Response), - ('application/x-json; encoding=UTF8;charset=UTF-8', TextResponse), - ('application/json-amazonui-streaming;charset=UTF-8', TextResponse), + ("text/html; charset=UTF-8", HtmlResponse), + ("text/xml; charset=UTF-8", XmlResponse), + ("application/xhtml+xml; charset=UTF-8", HtmlResponse), + ("application/vnd.wap.xhtml+xml; charset=utf-8", HtmlResponse), + ("application/xml; charset=UTF-8", XmlResponse), + ("application/octet-stream", Response), + ("application/x-json; encoding=UTF8;charset=UTF-8", TextResponse), + ("application/json-amazonui-streaming;charset=UTF-8", TextResponse), ] for source, cls in mappings: retcls = responsetypes.from_content_type(source) @@ -51,11 +49,11 @@ class ResponseTypesTest(unittest.TestCase): def test_from_body(self): mappings = [ - (b'\x03\x02\xdf\xdd\x23', Response), - (b'Some plain text\ndata with tabs\t and null bytes\0', TextResponse), - (b'<html><head><title>Hello', HtmlResponse), + (b"\x03\x02\xdf\xdd\x23", Response), + (b"Some plain text\ndata with tabs\t and null bytes\0", TextResponse), + (b"Hello", HtmlResponse), # https://codersblock.com/blog/the-smallest-valid-html5-page/ - (b'\n.', HtmlResponse), + (b"\n.", HtmlResponse), (b'

" - response = TextResponse(url="http://example.com", body=body, encoding='utf-8') + response = TextResponse(url="http://example.com", body=body, encoding="utf-8") sel = Selector(response) - xl = sel.xpath('//input') + xl = sel.xpath("//input") self.assertEqual(2, len(xl)) for x in xl: assert isinstance(x, Selector) self.assertEqual( - sel.xpath('//input').getall(), - [x.get() for x in sel.xpath('//input')] + sel.xpath("//input").getall(), [x.get() for x in sel.xpath("//input")] ) self.assertEqual( - [x.get() for x in sel.xpath("//input[@name='a']/@name")], - ['a'] + [x.get() for x in sel.xpath("//input[@name='a']/@name")], ["a"] ) self.assertEqual( - [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], - ['12.0'] + [ + x.get() + for x in sel.xpath( + "number(concat(//input[@name='a']/@value, //input[@name='b']/@value))" + ) + ], + ["12.0"], ) + self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(), ["xpathrules"]) self.assertEqual( - sel.xpath("concat('xpath', 'rules')").getall(), - ['xpathrules'] - ) - self.assertEqual( - [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], - ['12'] + [ + x.get() + for x in sel.xpath( + "concat(//input[@name='a']/@value, //input[@name='b']/@value)" + ) + ], + ["12"], ) def test_root_base_url(self): body = b'
' url = "http://example.com" - response = TextResponse(url=url, body=body, encoding='utf-8') + response = TextResponse(url=url, body=body, encoding="utf-8") sel = Selector(response) self.assertEqual(url, sel.root.base) def test_flavor_detection(self): text = b'

Hello

' - sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8')) - self.assertEqual(sel.type, 'xml') - self.assertEqual(sel.xpath("//div").getall(), - ['

Hello

']) + sel = Selector(XmlResponse("http://example.com", body=text, encoding="utf-8")) + self.assertEqual(sel.type, "xml") + self.assertEqual( + sel.xpath("//div").getall(), + ['

Hello

'], + ) - sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8')) - self.assertEqual(sel.type, 'html') - self.assertEqual(sel.xpath("//div").getall(), - ['

Hello

']) + sel = Selector(HtmlResponse("http://example.com", body=text, encoding="utf-8")) + self.assertEqual(sel.type, "html") + self.assertEqual( + sel.xpath("//div").getall(), ['

Hello

'] + ) def test_http_header_encoding_precedence(self): # '\xa3' = pound symbol in unicode # '\xc2\xa3' = pound symbol in utf-8 # '\xa3' = pound symbol in latin-1 (iso-8859-1) - meta = '' - head = '' + meta + '' + meta = ( + '' + ) + head = "" + meta + "" body_content = '\xa3' - body = '' + body_content + '' - html = '' + head + body + '' - encoding = 'utf-8' + body = "" + body_content + "" + html = "" + head + body + "" + encoding = "utf-8" html_utf8 = html.encode(encoding) - headers = {'Content-Type': ['text/html; charset=utf-8']} - response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) + headers = {"Content-Type": ["text/html; charset=utf-8"]} + response = HtmlResponse( + url="http://example.com", headers=headers, body=html_utf8 + ) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3']) + self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ["\xa3"]) def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence - r1 = TextResponse('http://www.example.com', - body=b'

an Jos\xe9 de

', - encoding='utf-8') - Selector(r1).xpath('//text()').getall() + r1 = TextResponse( + "http://www.example.com", + body=b"

an Jos\xe9 de

", + encoding="utf-8", + ) + Selector(r1).xpath("//text()").getall() def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" - x = Selector(text='') + x = Selector(text="") weakref.ref(x) - assert not hasattr(x, '__dict__'), f"{x.__class__.__name__} does not use __slots__" + assert not hasattr( + x, "__dict__" + ), f"{x.__class__.__name__} does not use __slots__" def test_selector_bad_args(self): - with self.assertRaisesRegex(ValueError, 'received both response and text'): - Selector(TextResponse(url='http://example.com', body=b''), text='') + with self.assertRaisesRegex(ValueError, "received both response and text"): + Selector(TextResponse(url="http://example.com", body=b""), text="") diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 916fe012a..9a01fd433 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,13 +1,17 @@ import unittest from unittest import mock -from scrapy.settings import (BaseSettings, Settings, SettingsAttribute, - SETTINGS_PRIORITIES, get_settings_priority) +from scrapy.settings import ( + BaseSettings, + Settings, + SettingsAttribute, + SETTINGS_PRIORITIES, + get_settings_priority, +) from . import default_settings class SettingsGlobalFuncsTest(unittest.TestCase): - def test_get_settings_priority(self): for prio_str, prio_num in SETTINGS_PRIORITIES.items(): self.assertEqual(get_settings_priority(prio_str), prio_num) @@ -15,127 +19,130 @@ class SettingsGlobalFuncsTest(unittest.TestCase): class SettingsAttributeTest(unittest.TestCase): - def setUp(self): - self.attribute = SettingsAttribute('value', 10) + self.attribute = SettingsAttribute("value", 10) def test_set_greater_priority(self): - self.attribute.set('value2', 20) - self.assertEqual(self.attribute.value, 'value2') + self.attribute.set("value2", 20) + self.assertEqual(self.attribute.value, "value2") self.assertEqual(self.attribute.priority, 20) def test_set_equal_priority(self): - self.attribute.set('value2', 10) - self.assertEqual(self.attribute.value, 'value2') + self.attribute.set("value2", 10) + self.assertEqual(self.attribute.value, "value2") self.assertEqual(self.attribute.priority, 10) def test_set_less_priority(self): - self.attribute.set('value2', 0) - self.assertEqual(self.attribute.value, 'value') + self.attribute.set("value2", 0) + self.assertEqual(self.attribute.value, "value") self.assertEqual(self.attribute.priority, 10) def test_overwrite_basesettings(self): - original_dict = {'one': 10, 'two': 20} + original_dict = {"one": 10, "two": 20} original_settings = BaseSettings(original_dict, 0) attribute = SettingsAttribute(original_settings, 0) - new_dict = {'three': 11, 'four': 21} + new_dict = {"three": 11, "four": 21} attribute.set(new_dict, 10) self.assertIsInstance(attribute.value, BaseSettings) self.assertCountEqual(attribute.value, new_dict) self.assertCountEqual(original_settings, original_dict) - new_settings = BaseSettings({'five': 12}, 0) + new_settings = BaseSettings({"five": 12}, 0) attribute.set(new_settings, 0) # Insufficient priority self.assertCountEqual(attribute.value, new_dict) attribute.set(new_settings, 10) self.assertCountEqual(attribute.value, new_settings) def test_repr(self): - self.assertEqual(repr(self.attribute), - "") + self.assertEqual( + repr(self.attribute), "" + ) class BaseSettingsTest(unittest.TestCase): - def setUp(self): self.settings = BaseSettings() def test_set_new_attribute(self): - self.settings.set('TEST_OPTION', 'value', 0) - self.assertIn('TEST_OPTION', self.settings.attributes) + self.settings.set("TEST_OPTION", "value", 0) + self.assertIn("TEST_OPTION", self.settings.attributes) - attr = self.settings.attributes['TEST_OPTION'] + attr = self.settings.attributes["TEST_OPTION"] self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, 'value') + self.assertEqual(attr.value, "value") self.assertEqual(attr.priority, 0) def test_set_settingsattribute(self): myattr = SettingsAttribute(0, 30) # Note priority 30 - self.settings.set('TEST_ATTR', myattr, 10) - self.assertEqual(self.settings.get('TEST_ATTR'), 0) - self.assertEqual(self.settings.getpriority('TEST_ATTR'), 30) + self.settings.set("TEST_ATTR", myattr, 10) + self.assertEqual(self.settings.get("TEST_ATTR"), 0) + self.assertEqual(self.settings.getpriority("TEST_ATTR"), 30) def test_set_instance_identity_on_update(self): - attr = SettingsAttribute('value', 0) - self.settings.attributes = {'TEST_OPTION': attr} - self.settings.set('TEST_OPTION', 'othervalue', 10) + attr = SettingsAttribute("value", 0) + self.settings.attributes = {"TEST_OPTION": attr} + self.settings.set("TEST_OPTION", "othervalue", 10) - self.assertIn('TEST_OPTION', self.settings.attributes) - self.assertIs(attr, self.settings.attributes['TEST_OPTION']) + self.assertIn("TEST_OPTION", self.settings.attributes) + self.assertIs(attr, self.settings.attributes["TEST_OPTION"]) def test_set_calls_settings_attributes_methods_on_update(self): - attr = SettingsAttribute('value', 10) - with mock.patch.object(attr, '__setattr__') as mock_setattr, mock.patch.object(attr, 'set') as mock_set: + attr = SettingsAttribute("value", 10) + with mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object( + attr, "set" + ) as mock_set: - self.settings.attributes = {'TEST_OPTION': attr} + self.settings.attributes = {"TEST_OPTION": attr} for priority in (0, 10, 20): - self.settings.set('TEST_OPTION', 'othervalue', priority) - mock_set.assert_called_once_with('othervalue', priority) + self.settings.set("TEST_OPTION", "othervalue", priority) + mock_set.assert_called_once_with("othervalue", priority) self.assertFalse(mock_setattr.called) mock_set.reset_mock() mock_setattr.reset_mock() def test_setitem(self): settings = BaseSettings() - settings.set('key', 'a', 'default') - settings['key'] = 'b' - self.assertEqual(settings['key'], 'b') - self.assertEqual(settings.getpriority('key'), 20) - settings['key'] = 'c' - self.assertEqual(settings['key'], 'c') - settings['key2'] = 'x' - self.assertIn('key2', settings) - self.assertEqual(settings['key2'], 'x') - self.assertEqual(settings.getpriority('key2'), 20) + settings.set("key", "a", "default") + settings["key"] = "b" + self.assertEqual(settings["key"], "b") + self.assertEqual(settings.getpriority("key"), 20) + settings["key"] = "c" + self.assertEqual(settings["key"], "c") + settings["key2"] = "x" + self.assertIn("key2", settings) + self.assertEqual(settings["key2"], "x") + self.assertEqual(settings.getpriority("key2"), 20) def test_setdict_alias(self): - with mock.patch.object(self.settings, 'set') as mock_set: - self.settings.setdict({'TEST_1': 'value1', 'TEST_2': 'value2'}, 10) + with mock.patch.object(self.settings, "set") as mock_set: + self.settings.setdict({"TEST_1": "value1", "TEST_2": "value2"}, 10) self.assertEqual(mock_set.call_count, 2) - calls = [mock.call('TEST_1', 'value1', 10), - mock.call('TEST_2', 'value2', 10)] + calls = [ + mock.call("TEST_1", "value1", 10), + mock.call("TEST_2", "value2", 10), + ] mock_set.assert_has_calls(calls, any_order=True) def test_setmodule_only_load_uppercase_vars(self): - class ModuleMock(): - UPPERCASE_VAR = 'value' - MIXEDcase_VAR = 'othervalue' - lowercase_var = 'anothervalue' + class ModuleMock: + UPPERCASE_VAR = "value" + MIXEDcase_VAR = "othervalue" + lowercase_var = "anothervalue" self.settings.attributes = {} self.settings.setmodule(ModuleMock(), 10) - self.assertIn('UPPERCASE_VAR', self.settings.attributes) - self.assertNotIn('MIXEDcase_VAR', self.settings.attributes) - self.assertNotIn('lowercase_var', self.settings.attributes) + self.assertIn("UPPERCASE_VAR", self.settings.attributes) + self.assertNotIn("MIXEDcase_VAR", self.settings.attributes) + self.assertNotIn("lowercase_var", self.settings.attributes) self.assertEqual(len(self.settings.attributes), 1) def test_setmodule_alias(self): - with mock.patch.object(self.settings, 'set') as mock_set: + with mock.patch.object(self.settings, "set") as mock_set: self.settings.setmodule(default_settings, 10) - mock_set.assert_any_call('TEST_DEFAULT', 'defvalue', 10) - mock_set.assert_any_call('TEST_DICT', {'key': 'val'}, 10) + mock_set.assert_any_call("TEST_DEFAULT", "defvalue", 10) + mock_set.assert_any_call("TEST_DICT", {"key": "val"}, 10) def test_setmodule_by_path(self): self.settings.attributes = {} @@ -143,11 +150,9 @@ class BaseSettingsTest(unittest.TestCase): ctrl_attributes = self.settings.attributes.copy() self.settings.attributes = {} - self.settings.setmodule( - 'tests.test_settings.default_settings', 10) + self.settings.setmodule("tests.test_settings.default_settings", 10) - self.assertCountEqual(self.settings.attributes.keys(), - ctrl_attributes.keys()) + self.assertCountEqual(self.settings.attributes.keys(), ctrl_attributes.keys()) for key in ctrl_attributes.keys(): attr = self.settings.attributes[key] @@ -156,181 +161,196 @@ class BaseSettingsTest(unittest.TestCase): self.assertEqual(attr.priority, ctrl_attr.priority) def test_update(self): - settings = BaseSettings({'key_lowprio': 0}, priority=0) - settings.set('key_highprio', 10, priority=50) - custom_settings = BaseSettings({'key_lowprio': 1, 'key_highprio': 11}, - priority=30) - custom_settings.set('newkey_one', None, priority=50) - custom_dict = {'key_lowprio': 2, 'key_highprio': 12, 'newkey_two': None} + settings = BaseSettings({"key_lowprio": 0}, priority=0) + settings.set("key_highprio", 10, priority=50) + custom_settings = BaseSettings( + {"key_lowprio": 1, "key_highprio": 11}, priority=30 + ) + custom_settings.set("newkey_one", None, priority=50) + custom_dict = {"key_lowprio": 2, "key_highprio": 12, "newkey_two": None} settings.update(custom_dict, priority=20) - self.assertEqual(settings['key_lowprio'], 2) - self.assertEqual(settings.getpriority('key_lowprio'), 20) - self.assertEqual(settings['key_highprio'], 10) - self.assertIn('newkey_two', settings) - self.assertEqual(settings.getpriority('newkey_two'), 20) + self.assertEqual(settings["key_lowprio"], 2) + self.assertEqual(settings.getpriority("key_lowprio"), 20) + self.assertEqual(settings["key_highprio"], 10) + self.assertIn("newkey_two", settings) + self.assertEqual(settings.getpriority("newkey_two"), 20) settings.update(custom_settings) - self.assertEqual(settings['key_lowprio'], 1) - self.assertEqual(settings.getpriority('key_lowprio'), 30) - self.assertEqual(settings['key_highprio'], 10) - self.assertIn('newkey_one', settings) - self.assertEqual(settings.getpriority('newkey_one'), 50) + self.assertEqual(settings["key_lowprio"], 1) + self.assertEqual(settings.getpriority("key_lowprio"), 30) + self.assertEqual(settings["key_highprio"], 10) + self.assertIn("newkey_one", settings) + self.assertEqual(settings.getpriority("newkey_one"), 50) - settings.update({'key_lowprio': 3}, priority=20) - self.assertEqual(settings['key_lowprio'], 1) + settings.update({"key_lowprio": 3}, priority=20) + self.assertEqual(settings["key_lowprio"], 1) def test_update_jsonstring(self): - settings = BaseSettings({'number': 0, 'dict': BaseSettings({'key': 'val'})}) + settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})}) settings.update('{"number": 1, "newnumber": 2}') - self.assertEqual(settings['number'], 1) - self.assertEqual(settings['newnumber'], 2) + self.assertEqual(settings["number"], 1) + self.assertEqual(settings["newnumber"], 2) settings.set("dict", '{"key": "newval", "newkey": "newval2"}') - self.assertEqual(settings['dict']['key'], "newval") - self.assertEqual(settings['dict']['newkey'], "newval2") + self.assertEqual(settings["dict"]["key"], "newval") + self.assertEqual(settings["dict"]["newkey"], "newval2") def test_delete(self): - settings = BaseSettings({'key': None}) - settings.set('key_highprio', None, priority=50) - settings.delete('key') - settings.delete('key_highprio') - self.assertNotIn('key', settings) - self.assertIn('key_highprio', settings) - del settings['key_highprio'] - self.assertNotIn('key_highprio', settings) + settings = BaseSettings({"key": None}) + settings.set("key_highprio", None, priority=50) + settings.delete("key") + settings.delete("key_highprio") + self.assertNotIn("key", settings) + self.assertIn("key_highprio", settings) + del settings["key_highprio"] + self.assertNotIn("key_highprio", settings) def test_get(self): test_configuration = { - 'TEST_ENABLED1': '1', - 'TEST_ENABLED2': True, - 'TEST_ENABLED3': 1, - 'TEST_ENABLED4': 'True', - 'TEST_ENABLED5': 'true', - 'TEST_ENABLED_WRONG': 'on', - 'TEST_DISABLED1': '0', - 'TEST_DISABLED2': False, - 'TEST_DISABLED3': 0, - 'TEST_DISABLED4': 'False', - 'TEST_DISABLED5': 'false', - 'TEST_DISABLED_WRONG': 'off', - 'TEST_INT1': 123, - 'TEST_INT2': '123', - 'TEST_FLOAT1': 123.45, - 'TEST_FLOAT2': '123.45', - 'TEST_LIST1': ['one', 'two'], - 'TEST_LIST2': 'one,two', - 'TEST_STR': 'value', - 'TEST_DICT1': {'key1': 'val1', 'ke2': 3}, - 'TEST_DICT2': '{"key1": "val1", "ke2": 3}', + "TEST_ENABLED1": "1", + "TEST_ENABLED2": True, + "TEST_ENABLED3": 1, + "TEST_ENABLED4": "True", + "TEST_ENABLED5": "true", + "TEST_ENABLED_WRONG": "on", + "TEST_DISABLED1": "0", + "TEST_DISABLED2": False, + "TEST_DISABLED3": 0, + "TEST_DISABLED4": "False", + "TEST_DISABLED5": "false", + "TEST_DISABLED_WRONG": "off", + "TEST_INT1": 123, + "TEST_INT2": "123", + "TEST_FLOAT1": 123.45, + "TEST_FLOAT2": "123.45", + "TEST_LIST1": ["one", "two"], + "TEST_LIST2": "one,two", + "TEST_STR": "value", + "TEST_DICT1": {"key1": "val1", "ke2": 3}, + "TEST_DICT2": '{"key1": "val1", "ke2": 3}', } settings = self.settings - settings.attributes = {key: SettingsAttribute(value, 0) for key, value - in test_configuration.items()} + settings.attributes = { + key: SettingsAttribute(value, 0) + for key, value in test_configuration.items() + } - self.assertTrue(settings.getbool('TEST_ENABLED1')) - self.assertTrue(settings.getbool('TEST_ENABLED2')) - self.assertTrue(settings.getbool('TEST_ENABLED3')) - self.assertTrue(settings.getbool('TEST_ENABLED4')) - self.assertTrue(settings.getbool('TEST_ENABLED5')) - self.assertFalse(settings.getbool('TEST_ENABLEDx')) - self.assertTrue(settings.getbool('TEST_ENABLEDx', True)) - self.assertFalse(settings.getbool('TEST_DISABLED1')) - self.assertFalse(settings.getbool('TEST_DISABLED2')) - self.assertFalse(settings.getbool('TEST_DISABLED3')) - self.assertFalse(settings.getbool('TEST_DISABLED4')) - self.assertFalse(settings.getbool('TEST_DISABLED5')) - self.assertEqual(settings.getint('TEST_INT1'), 123) - self.assertEqual(settings.getint('TEST_INT2'), 123) - self.assertEqual(settings.getint('TEST_INTx'), 0) - self.assertEqual(settings.getint('TEST_INTx', 45), 45) - self.assertEqual(settings.getfloat('TEST_FLOAT1'), 123.45) - self.assertEqual(settings.getfloat('TEST_FLOAT2'), 123.45) - self.assertEqual(settings.getfloat('TEST_FLOATx'), 0.0) - self.assertEqual(settings.getfloat('TEST_FLOATx', 55.0), 55.0) - self.assertEqual(settings.getlist('TEST_LIST1'), ['one', 'two']) - self.assertEqual(settings.getlist('TEST_LIST2'), ['one', 'two']) - self.assertEqual(settings.getlist('TEST_LISTx'), []) - self.assertEqual(settings.getlist('TEST_LISTx', ['default']), ['default']) - self.assertEqual(settings['TEST_STR'], 'value') - self.assertEqual(settings.get('TEST_STR'), 'value') - self.assertEqual(settings['TEST_STRx'], None) - self.assertEqual(settings.get('TEST_STRx'), None) - self.assertEqual(settings.get('TEST_STRx', 'default'), 'default') - self.assertEqual(settings.getdict('TEST_DICT1'), {'key1': 'val1', 'ke2': 3}) - self.assertEqual(settings.getdict('TEST_DICT2'), {'key1': 'val1', 'ke2': 3}) - self.assertEqual(settings.getdict('TEST_DICT3'), {}) - self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5}) - self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1') - self.assertRaises(ValueError, settings.getbool, 'TEST_ENABLED_WRONG') - self.assertRaises(ValueError, settings.getbool, 'TEST_DISABLED_WRONG') + self.assertTrue(settings.getbool("TEST_ENABLED1")) + self.assertTrue(settings.getbool("TEST_ENABLED2")) + self.assertTrue(settings.getbool("TEST_ENABLED3")) + self.assertTrue(settings.getbool("TEST_ENABLED4")) + self.assertTrue(settings.getbool("TEST_ENABLED5")) + self.assertFalse(settings.getbool("TEST_ENABLEDx")) + self.assertTrue(settings.getbool("TEST_ENABLEDx", True)) + self.assertFalse(settings.getbool("TEST_DISABLED1")) + self.assertFalse(settings.getbool("TEST_DISABLED2")) + self.assertFalse(settings.getbool("TEST_DISABLED3")) + self.assertFalse(settings.getbool("TEST_DISABLED4")) + self.assertFalse(settings.getbool("TEST_DISABLED5")) + self.assertEqual(settings.getint("TEST_INT1"), 123) + self.assertEqual(settings.getint("TEST_INT2"), 123) + self.assertEqual(settings.getint("TEST_INTx"), 0) + self.assertEqual(settings.getint("TEST_INTx", 45), 45) + self.assertEqual(settings.getfloat("TEST_FLOAT1"), 123.45) + self.assertEqual(settings.getfloat("TEST_FLOAT2"), 123.45) + self.assertEqual(settings.getfloat("TEST_FLOATx"), 0.0) + self.assertEqual(settings.getfloat("TEST_FLOATx", 55.0), 55.0) + self.assertEqual(settings.getlist("TEST_LIST1"), ["one", "two"]) + self.assertEqual(settings.getlist("TEST_LIST2"), ["one", "two"]) + self.assertEqual(settings.getlist("TEST_LISTx"), []) + self.assertEqual(settings.getlist("TEST_LISTx", ["default"]), ["default"]) + self.assertEqual(settings["TEST_STR"], "value") + self.assertEqual(settings.get("TEST_STR"), "value") + self.assertEqual(settings["TEST_STRx"], None) + self.assertEqual(settings.get("TEST_STRx"), None) + self.assertEqual(settings.get("TEST_STRx", "default"), "default") + self.assertEqual(settings.getdict("TEST_DICT1"), {"key1": "val1", "ke2": 3}) + self.assertEqual(settings.getdict("TEST_DICT2"), {"key1": "val1", "ke2": 3}) + self.assertEqual(settings.getdict("TEST_DICT3"), {}) + self.assertEqual(settings.getdict("TEST_DICT3", {"key1": 5}), {"key1": 5}) + self.assertRaises(ValueError, settings.getdict, "TEST_LIST1") + self.assertRaises(ValueError, settings.getbool, "TEST_ENABLED_WRONG") + self.assertRaises(ValueError, settings.getbool, "TEST_DISABLED_WRONG") def test_getpriority(self): - settings = BaseSettings({'key': 'value'}, priority=99) - self.assertEqual(settings.getpriority('key'), 99) - self.assertEqual(settings.getpriority('nonexistentkey'), None) + settings = BaseSettings({"key": "value"}, priority=99) + self.assertEqual(settings.getpriority("key"), 99) + self.assertEqual(settings.getpriority("nonexistentkey"), None) def test_getwithbase(self): - s = BaseSettings({'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'), - 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), - 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) - s['TEST'].set(2, 200, 'cmdline') - self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30}) - self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE']) - self.assertEqual(s.getwithbase('NONEXISTENT'), {}) + s = BaseSettings( + { + "TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"), + "TEST": BaseSettings({1: 10, 3: 30}, "default"), + "HASNOBASE": BaseSettings({3: 3000}, "default"), + } + ) + s["TEST"].set(2, 200, "cmdline") + self.assertCountEqual(s.getwithbase("TEST"), {1: 1, 2: 200, 3: 30}) + self.assertCountEqual(s.getwithbase("HASNOBASE"), s["HASNOBASE"]) + self.assertEqual(s.getwithbase("NONEXISTENT"), {}) def test_maxpriority(self): # Empty settings should return 'default' self.assertEqual(self.settings.maxpriority(), 0) - self.settings.set('A', 0, 10) - self.settings.set('B', 0, 30) + self.settings.set("A", 0, 10) + self.settings.set("B", 0, 30) self.assertEqual(self.settings.maxpriority(), 30) def test_copy(self): values = { - 'TEST_BOOL': True, - 'TEST_LIST': ['one', 'two'], - 'TEST_LIST_OF_LISTS': [['first_one', 'first_two'], - ['second_one', 'second_two']] + "TEST_BOOL": True, + "TEST_LIST": ["one", "two"], + "TEST_LIST_OF_LISTS": [ + ["first_one", "first_two"], + ["second_one", "second_two"], + ], } self.settings.setdict(values) copy = self.settings.copy() - self.settings.set('TEST_BOOL', False) - self.assertTrue(copy.get('TEST_BOOL')) + self.settings.set("TEST_BOOL", False) + self.assertTrue(copy.get("TEST_BOOL")) - test_list = self.settings.get('TEST_LIST') - test_list.append('three') - self.assertListEqual(copy.get('TEST_LIST'), ['one', 'two']) + test_list = self.settings.get("TEST_LIST") + test_list.append("three") + self.assertListEqual(copy.get("TEST_LIST"), ["one", "two"]) - test_list_of_lists = self.settings.get('TEST_LIST_OF_LISTS') - test_list_of_lists[0].append('first_three') - self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0], - ['first_one', 'first_two']) + test_list_of_lists = self.settings.get("TEST_LIST_OF_LISTS") + test_list_of_lists[0].append("first_three") + self.assertListEqual( + copy.get("TEST_LIST_OF_LISTS")[0], ["first_one", "first_two"] + ) def test_copy_to_dict(self): - s = BaseSettings({'TEST_STRING': 'a string', - 'TEST_LIST': [1, 2], - 'TEST_BOOLEAN': False, - 'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'), - 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), - 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) + s = BaseSettings( + { + "TEST_STRING": "a string", + "TEST_LIST": [1, 2], + "TEST_BOOLEAN": False, + "TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"), + "TEST": BaseSettings({1: 10, 3: 30}, "default"), + "HASNOBASE": BaseSettings({3: 3000}, "default"), + } + ) self.assertDictEqual( s.copy_to_dict(), { - 'HASNOBASE': {3: 3000}, - 'TEST': {1: 10, 3: 30}, - 'TEST_BASE': {1: 1, 2: 2}, - 'TEST_LIST': [1, 2], - 'TEST_BOOLEAN': False, - 'TEST_STRING': 'a string', - } + "HASNOBASE": {3: 3000}, + "TEST": {1: 10, 3: 30}, + "TEST_BASE": {1: 1, 2: 2}, + "TEST_LIST": [1, 2], + "TEST_BOOLEAN": False, + "TEST_STRING": "a string", + }, ) def test_freeze(self): self.settings.freeze() with self.assertRaises(TypeError) as cm: - self.settings.set('TEST_BOOL', False) - self.assertEqual(str(cm.exception), - "Trying to modify an immutable Settings object") + self.settings.set("TEST_BOOL", False) + self.assertEqual( + str(cm.exception), "Trying to modify an immutable Settings object" + ) def test_frozencopy(self): frozencopy = self.settings.frozencopy() @@ -339,83 +359,84 @@ class BaseSettingsTest(unittest.TestCase): class SettingsTest(unittest.TestCase): - def setUp(self): self.settings = Settings() - @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {'default': 10}) - @mock.patch('scrapy.settings.default_settings', default_settings) + @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {"default": 10}) + @mock.patch("scrapy.settings.default_settings", default_settings) def test_initial_defaults(self): settings = Settings() self.assertEqual(len(settings.attributes), 2) - self.assertIn('TEST_DEFAULT', settings.attributes) + self.assertIn("TEST_DEFAULT", settings.attributes) - attr = settings.attributes['TEST_DEFAULT'] + attr = settings.attributes["TEST_DEFAULT"] self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, 'defvalue') + self.assertEqual(attr.value, "defvalue") self.assertEqual(attr.priority, 10) - @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {}) - @mock.patch('scrapy.settings.default_settings', {}) + @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {}) + @mock.patch("scrapy.settings.default_settings", {}) def test_initial_values(self): - settings = Settings({'TEST_OPTION': 'value'}, 10) + settings = Settings({"TEST_OPTION": "value"}, 10) self.assertEqual(len(settings.attributes), 1) - self.assertIn('TEST_OPTION', settings.attributes) + self.assertIn("TEST_OPTION", settings.attributes) - attr = settings.attributes['TEST_OPTION'] + attr = settings.attributes["TEST_OPTION"] self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, 'value') + self.assertEqual(attr.value, "value") self.assertEqual(attr.priority, 10) - @mock.patch('scrapy.settings.default_settings', default_settings) + @mock.patch("scrapy.settings.default_settings", default_settings) def test_autopromote_dicts(self): settings = Settings() - mydict = settings.get('TEST_DICT') + mydict = settings.get("TEST_DICT") self.assertIsInstance(mydict, BaseSettings) - self.assertIn('key', mydict) - self.assertEqual(mydict['key'], 'val') - self.assertEqual(mydict.getpriority('key'), 0) + self.assertIn("key", mydict) + self.assertEqual(mydict["key"], "val") + self.assertEqual(mydict.getpriority("key"), 0) - @mock.patch('scrapy.settings.default_settings', default_settings) + @mock.patch("scrapy.settings.default_settings", default_settings) def test_getdict_autodegrade_basesettings(self): settings = Settings() - mydict = settings.getdict('TEST_DICT') + mydict = settings.getdict("TEST_DICT") self.assertIsInstance(mydict, dict) self.assertEqual(len(mydict), 1) - self.assertIn('key', mydict) - self.assertEqual(mydict['key'], 'val') + self.assertIn("key", mydict) + self.assertEqual(mydict["key"], "val") def test_passing_objects_as_values(self): from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.utils.misc import create_instance from scrapy.utils.test import get_crawler - class TestPipeline(): + class TestPipeline: def process_item(self, i, s): return i - settings = Settings({ - 'ITEM_PIPELINES': { - TestPipeline: 800, - }, - 'DOWNLOAD_HANDLERS': { - 'ftp': FileDownloadHandler, - }, - }) + settings = Settings( + { + "ITEM_PIPELINES": { + TestPipeline: 800, + }, + "DOWNLOAD_HANDLERS": { + "ftp": FileDownloadHandler, + }, + } + ) - self.assertIn('ITEM_PIPELINES', settings.attributes) + self.assertIn("ITEM_PIPELINES", settings.attributes) - mypipeline, priority = settings.getdict('ITEM_PIPELINES').popitem() + mypipeline, priority = settings.getdict("ITEM_PIPELINES").popitem() self.assertEqual(priority, 800) self.assertEqual(mypipeline, TestPipeline) self.assertIsInstance(mypipeline(), TestPipeline) - self.assertEqual(mypipeline().process_item('item', None), 'item') + self.assertEqual(mypipeline().process_item("item", None), "item") - myhandler = settings.getdict('DOWNLOAD_HANDLERS').pop('ftp') + myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp") self.assertEqual(myhandler, FileDownloadHandler) myhandler_instance = create_instance(myhandler, None, get_crawler()) self.assertIsInstance(myhandler_instance, FileDownloadHandler) - self.assertTrue(hasattr(myhandler_instance, 'download_request')) + self.assertTrue(hasattr(myhandler_instance, "download_request")) if __name__ == "__main__": diff --git a/tests/test_settings/default_settings.py b/tests/test_settings/default_settings.py index 26a555275..2a2142b32 100644 --- a/tests/test_settings/default_settings.py +++ b/tests/test_settings/default_settings.py @@ -1,4 +1,3 @@ +TEST_DEFAULT = "defvalue" -TEST_DEFAULT = 'defvalue' - -TEST_DICT = {'key': 'val'} +TEST_DICT = {"key": "val"} diff --git a/tests/test_signals.py b/tests/test_signals.py index a43f00b27..4c6ffabdc 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -9,15 +9,16 @@ from tests.mockserver import MockServer class ItemSpider(Spider): - name = 'itemspider' + name = "itemspider" def start_requests(self): for index in range(10): - yield Request(self.mockserver.url(f'/status?n=200&id={index}'), - meta={'index': index}) + yield Request( + self.mockserver.url(f"/status?n=200&id={index}"), meta={"index": index} + ) def parse(self, response): - return {'index': response.meta['index']} + return {"index": response.meta["index"]} class AsyncSignalTestCase(unittest.TestCase): @@ -41,4 +42,4 @@ class AsyncSignalTestCase(unittest.TestCase): yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(self.items), 10) for index in range(10): - self.assertIn({'index': index}, self.items) + self.assertIn({"index": index}, self.items) diff --git a/tests/test_spider.py b/tests/test_spider.py index cb66066b0..540091516 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -37,39 +37,39 @@ class SpiderTest(unittest.TestCase): def test_base_spider(self): spider = self.spider_class("example.com") - self.assertEqual(spider.name, 'example.com') + self.assertEqual(spider.name, "example.com") self.assertEqual(spider.start_urls, []) def test_start_requests(self): - spider = self.spider_class('example.com') + spider = self.spider_class("example.com") start_requests = spider.start_requests() self.assertTrue(inspect.isgenerator(start_requests)) self.assertEqual(list(start_requests), []) def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" - spider = self.spider_class('example.com', foo='bar') - self.assertEqual(spider.foo, 'bar') + spider = self.spider_class("example.com", foo="bar") + self.assertEqual(spider.foo, "bar") def test_spider_without_name(self): """``__init__`` method arguments are assigned to spider attributes""" self.assertRaises(ValueError, self.spider_class) - self.assertRaises(ValueError, self.spider_class, somearg='foo') + self.assertRaises(ValueError, self.spider_class, somearg="foo") def test_from_crawler_crawler_and_settings_population(self): crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, 'example.com') - self.assertTrue(hasattr(spider, 'crawler')) + spider = self.spider_class.from_crawler(crawler, "example.com") + self.assertTrue(hasattr(spider, "crawler")) self.assertIs(spider.crawler, crawler) - self.assertTrue(hasattr(spider, 'settings')) + self.assertTrue(hasattr(spider, "settings")) self.assertIs(spider.settings, crawler.settings) def test_from_crawler_init_call(self): - with mock.patch.object(self.spider_class, '__init__', - return_value=None) as mock_init: - self.spider_class.from_crawler(get_crawler(), 'example.com', - foo='bar') - mock_init.assert_called_once_with('example.com', foo='bar') + with mock.patch.object( + self.spider_class, "__init__", return_value=None + ) as mock_init: + self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar") + mock_init.assert_called_once_with("example.com", foo="bar") def test_closed_signal_call(self): class TestSpider(self.spider_class): @@ -79,39 +79,39 @@ class SpiderTest(unittest.TestCase): self.closed_called = True crawler = get_crawler() - spider = TestSpider.from_crawler(crawler, 'example.com') - crawler.signals.send_catch_log(signal=signals.spider_opened, - spider=spider) - crawler.signals.send_catch_log(signal=signals.spider_closed, - spider=spider, reason=None) + spider = TestSpider.from_crawler(crawler, "example.com") + crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider) + crawler.signals.send_catch_log( + signal=signals.spider_closed, spider=spider, reason=None + ) self.assertTrue(spider.closed_called) def test_update_settings(self): - spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'} - project_settings = {'TEST1': 'project', 'TEST3': 'project'} + spider_settings = {"TEST1": "spider", "TEST2": "spider"} + project_settings = {"TEST1": "project", "TEST3": "project"} self.spider_class.custom_settings = spider_settings - settings = Settings(project_settings, priority='project') + settings = Settings(project_settings, priority="project") self.spider_class.update_settings(settings) - self.assertEqual(settings.get('TEST1'), 'spider') - self.assertEqual(settings.get('TEST2'), 'spider') - self.assertEqual(settings.get('TEST3'), 'project') + self.assertEqual(settings.get("TEST1"), "spider") + self.assertEqual(settings.get("TEST2"), "spider") + self.assertEqual(settings.get("TEST3"), "project") def test_logger(self): - spider = self.spider_class('example.com') + spider = self.spider_class("example.com") with LogCapture() as lc: - spider.logger.info('test log msg') - lc.check(('example.com', 'INFO', 'test log msg')) + spider.logger.info("test log msg") + lc.check(("example.com", "INFO", "test log msg")) record = lc.records[0] - self.assertIn('spider', record.__dict__) + self.assertIn("spider", record.__dict__) self.assertIs(record.spider, spider) def test_log(self): - spider = self.spider_class('example.com') - with mock.patch('scrapy.spiders.Spider.logger') as mock_logger: - spider.log('test log msg', 'INFO') - mock_logger.log.assert_called_once_with('INFO', 'test log msg') + spider = self.spider_class("example.com") + with mock.patch("scrapy.spiders.Spider.logger") as mock_logger: + spider.log("test log msg", "INFO") + mock_logger.log.assert_called_once_with("INFO", "test log msg") class InitSpiderTest(SpiderTest): @@ -132,37 +132,45 @@ class XMLFeedSpiderTest(SpiderTest): http://www.example.com/2009-08-16 """ - response = XmlResponse(url='http://example.com/sitemap.xml', body=body) + response = XmlResponse(url="http://example.com/sitemap.xml", body=body) class _XMLSpider(self.spider_class): - itertag = 'url' + itertag = "url" namespaces = ( - ('a', 'http://www.google.com/schemas/sitemap/0.84'), - ('b', 'http://www.example.com/schemas/extras/1.0'), + ("a", "http://www.google.com/schemas/sitemap/0.84"), + ("b", "http://www.example.com/schemas/extras/1.0"), ) def parse_node(self, response, selector): yield { - 'loc': selector.xpath('a:loc/text()').getall(), - 'updated': selector.xpath('b:updated/text()').getall(), - 'other': selector.xpath('other/@value').getall(), - 'custom': selector.xpath('other/@b:custom').getall(), + "loc": selector.xpath("a:loc/text()").getall(), + "updated": selector.xpath("b:updated/text()").getall(), + "other": selector.xpath("other/@value").getall(), + "custom": selector.xpath("other/@b:custom").getall(), } - for iterator in ('iternodes', 'xml'): - spider = _XMLSpider('example', iterator=iterator) + for iterator in ("iternodes", "xml"): + spider = _XMLSpider("example", iterator=iterator) output = list(spider._parse(response)) self.assertEqual(len(output), 2, iterator) - self.assertEqual(output, [ - {'loc': ['http://www.example.com/Special-Offers.html'], - 'updated': ['2009-08-16'], - 'custom': ['fuu'], - 'other': ['bar']}, - {'loc': [], - 'updated': ['2009-08-16'], - 'other': ['foo'], - 'custom': []}, - ], iterator) + self.assertEqual( + output, + [ + { + "loc": ["http://www.example.com/Special-Offers.html"], + "updated": ["2009-08-16"], + "custom": ["fuu"], + "other": ["bar"], + }, + { + "loc": [], + "updated": ["2009-08-16"], + "other": ["foo"], + "custom": [], + }, + ], + iterator, + ) class CSVFeedSpiderTest(SpiderTest): @@ -170,7 +178,7 @@ class CSVFeedSpiderTest(SpiderTest): spider_class = CSVFeedSpider def test_parse_rows(self): - body = get_testdata('feeds', 'feed-sample6.csv') + body = get_testdata("feeds", "feed-sample6.csv") response = Response("http://example.org/dummy.csv", body=body) class _CrawlSpider(self.spider_class): @@ -183,7 +191,7 @@ class CSVFeedSpiderTest(SpiderTest): spider = _CrawlSpider() rows = list(spider.parse_rows(response)) - assert rows[0] == {'id': '1', 'name': 'alpha', 'value': 'foobar'} + assert rows[0] == {"id": "1", "name": "alpha", "value": "foobar"} assert len(rows) == 4 @@ -203,34 +211,38 @@ class CrawlSpiderTest(SpiderTest): def test_rule_without_link_extractor(self): - response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) class _CrawlSpider(self.spider_class): name = "test" - allowed_domains = ['example.org'] - rules = ( - Rule(), - ) + allowed_domains = ["example.org"] + rules = (Rule(),) spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html', - 'http://example.org/nofollow.html']) + self.assertEqual( + [r.url for r in output], + [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ], + ) def test_process_links(self): - response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) class _CrawlSpider(self.spider_class): name = "test" - allowed_domains = ['example.org'] - rules = ( - Rule(LinkExtractor(), process_links="dummy_process_links"), - ) + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) def dummy_process_links(self, links): return links @@ -239,47 +251,54 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html', - 'http://example.org/nofollow.html']) + self.assertEqual( + [r.url for r in output], + [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ], + ) def test_process_links_filter(self): - response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) class _CrawlSpider(self.spider_class): import re name = "test" - allowed_domains = ['example.org'] - rules = ( - Rule(LinkExtractor(), process_links="filter_process_links"), - ) - _test_regex = re.compile('nofollow') + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_links="filter_process_links"),) + _test_regex = re.compile("nofollow") def filter_process_links(self, links): - return [link for link in links - if not self._test_regex.search(link.url)] + return [link for link in links if not self._test_regex.search(link.url)] spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 2) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html']) + self.assertEqual( + [r.url for r in output], + [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + ], + ) def test_process_links_generator(self): - response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) class _CrawlSpider(self.spider_class): name = "test" - allowed_domains = ['example.org'] - rules = ( - Rule(LinkExtractor(), process_links="dummy_process_links"), - ) + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) def dummy_process_links(self, links): for link in links: @@ -289,21 +308,27 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html', - 'http://example.org/nofollow.html']) + self.assertEqual( + [r.url for r in output], + [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ], + ) def test_process_request(self): - response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) def process_request_change_domain(request, response): - return request.replace(url=request.url.replace('.org', '.com')) + return request.replace(url=request.url.replace(".org", ".com")) class _CrawlSpider(self.spider_class): name = "test" - allowed_domains = ['example.org'] + allowed_domains = ["example.org"] rules = ( Rule(LinkExtractor(), process_request=process_request_change_domain), ) @@ -312,47 +337,61 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://example.com/somepage/item/12.html', - 'http://example.com/about.html', - 'http://example.com/nofollow.html']) + self.assertEqual( + [r.url for r in output], + [ + "http://example.com/somepage/item/12.html", + "http://example.com/about.html", + "http://example.com/nofollow.html", + ], + ) def test_process_request_with_response(self): - response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) def process_request_meta_response_class(request, response): - request.meta['response_class'] = response.__class__.__name__ + request.meta["response_class"] = response.__class__.__name__ return request class _CrawlSpider(self.spider_class): name = "test" - allowed_domains = ['example.org'] + allowed_domains = ["example.org"] rules = ( - Rule(LinkExtractor(), process_request=process_request_meta_response_class), + Rule( + LinkExtractor(), process_request=process_request_meta_response_class + ), ) spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html', - 'http://example.org/nofollow.html']) - self.assertEqual([r.meta['response_class'] for r in output], - ['HtmlResponse', 'HtmlResponse', 'HtmlResponse']) + self.assertEqual( + [r.url for r in output], + [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ], + ) + self.assertEqual( + [r.meta["response_class"] for r in output], + ["HtmlResponse", "HtmlResponse", "HtmlResponse"], + ) def test_process_request_instance_method(self): - response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) class _CrawlSpider(self.spider_class): name = "test" - allowed_domains = ['example.org'] - rules = ( - Rule(LinkExtractor(), process_request='process_request_upper'), - ) + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_request="process_request_upper"),) def process_request_upper(self, request, response): return request.replace(url=request.url.upper()) @@ -361,55 +400,69 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - [safe_url_string('http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML'), - safe_url_string('http://EXAMPLE.ORG/ABOUT.HTML'), - safe_url_string('http://EXAMPLE.ORG/NOFOLLOW.HTML')]) + self.assertEqual( + [r.url for r in output], + [ + safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), + safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), + safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), + ], + ) def test_process_request_instance_method_with_response(self): - response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) class _CrawlSpider(self.spider_class): name = "test" - allowed_domains = ['example.org'] + allowed_domains = ["example.org"] rules = ( - Rule(LinkExtractor(), process_request='process_request_meta_response_class'), + Rule( + LinkExtractor(), + process_request="process_request_meta_response_class", + ), ) def process_request_meta_response_class(self, request, response): - request.meta['response_class'] = response.__class__.__name__ + request.meta["response_class"] = response.__class__.__name__ return request spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html', - 'http://example.org/nofollow.html']) - self.assertEqual([r.meta['response_class'] for r in output], - ['HtmlResponse', 'HtmlResponse', 'HtmlResponse']) + self.assertEqual( + [r.url for r in output], + [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ], + ) + self.assertEqual( + [r.meta["response_class"] for r in output], + ["HtmlResponse", "HtmlResponse", "HtmlResponse"], + ) def test_follow_links_attribute_population(self): crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, 'example.com') - self.assertTrue(hasattr(spider, '_follow_links')) + spider = self.spider_class.from_crawler(crawler, "example.com") + self.assertTrue(hasattr(spider, "_follow_links")) self.assertTrue(spider._follow_links) - settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False} + settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False} crawler = get_crawler(settings_dict=settings_dict) - spider = self.spider_class.from_crawler(crawler, 'example.com') - self.assertTrue(hasattr(spider, '_follow_links')) + spider = self.spider_class.from_crawler(crawler, "example.com") + self.assertTrue(hasattr(spider, "_follow_links")) self.assertFalse(spider._follow_links) def test_start_url(self): spider = self.spider_class("example.com") - spider.start_url = 'https://www.example.com' + spider.start_url = "https://www.example.com" - with self.assertRaisesRegex(AttributeError, - r'^Crawling could not start.*$'): + with self.assertRaisesRegex(AttributeError, r"^Crawling could not start.*$"): list(spider.start_requests()) @@ -419,7 +472,7 @@ class SitemapSpiderTest(SpiderTest): BODY = b"SITEMAP" f = BytesIO() - g = gzip.GzipFile(fileobj=f, mode='w+b') + g = gzip.GzipFile(fileobj=f, mode="w+b") g.write(BODY) g.close() GZBODY = f.getvalue() @@ -439,8 +492,11 @@ class SitemapSpiderTest(SpiderTest): self.assertSitemapBody(r, None) def test_get_sitemap_body_gzip_headers(self): - r = Response(url="http://www.example.com/sitemap", body=self.GZBODY, - headers={"content-type": "application/gzip"}) + r = Response( + url="http://www.example.com/sitemap", + body=self.GZBODY, + headers={"content-type": "application/gzip"}, + ) self.assertSitemapBody(r, self.BODY) def test_get_sitemap_body_xml_url(self): @@ -465,11 +521,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/robots.txt", body=robots) spider = self.spider_class("example.com") - self.assertEqual([req.url for req in spider._parse_sitemap(r)], - ['http://example.com/sitemap.xml', - 'http://example.com/sitemap-product-index.xml', - 'http://example.com/sitemap-uppercase.xml', - 'http://www.example.com/sitemap-relative-url.xml']) + self.assertEqual( + [req.url for req in spider._parse_sitemap(r)], + [ + "http://example.com/sitemap.xml", + "http://example.com/sitemap-product-index.xml", + "http://example.com/sitemap-uppercase.xml", + "http://www.example.com/sitemap-relative-url.xml", + ], + ) def test_alternate_url_locs(self): sitemap = b""" @@ -488,15 +548,21 @@ Sitemap: /sitemap-relative-url.xml """ r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual([req.url for req in spider._parse_sitemap(r)], - ['http://www.example.com/english/']) + self.assertEqual( + [req.url for req in spider._parse_sitemap(r)], + ["http://www.example.com/english/"], + ) spider.sitemap_alternate_links = True - self.assertEqual([req.url for req in spider._parse_sitemap(r)], - ['http://www.example.com/english/', - 'http://www.example.com/deutsch/', - 'http://www.example.com/schweiz-deutsch/', - 'http://www.example.com/italiano/']) + self.assertEqual( + [req.url for req in spider._parse_sitemap(r)], + [ + "http://www.example.com/english/", + "http://www.example.com/deutsch/", + "http://www.example.com/schweiz-deutsch/", + "http://www.example.com/italiano/", + ], + ) def test_sitemap_filter(self): sitemap = b""" @@ -515,20 +581,24 @@ Sitemap: /sitemap-relative-url.xml class FilteredSitemapSpider(self.spider_class): def sitemap_filter(self, entries): from datetime import datetime + for entry in entries: - date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') + date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d") if date_time.year > 2008: yield entry r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual([req.url for req in spider._parse_sitemap(r)], - ['http://www.example.com/english/', - 'http://www.example.com/portuguese/']) + self.assertEqual( + [req.url for req in spider._parse_sitemap(r)], + ["http://www.example.com/english/", "http://www.example.com/portuguese/"], + ) spider = FilteredSitemapSpider("example.com") - self.assertEqual([req.url for req in spider._parse_sitemap(r)], - ['http://www.example.com/english/']) + self.assertEqual( + [req.url for req in spider._parse_sitemap(r)], + ["http://www.example.com/english/"], + ) def test_sitemap_filter_with_alternate_links(self): sitemap = b""" @@ -549,21 +619,27 @@ Sitemap: /sitemap-relative-url.xml class FilteredSitemapSpider(self.spider_class): def sitemap_filter(self, entries): for entry in entries: - alternate_links = entry.get('alternate', tuple()) + alternate_links = entry.get("alternate", tuple()) for link in alternate_links: - if '/deutsch/' in link: - entry['loc'] = link + if "/deutsch/" in link: + entry["loc"] = link yield entry r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual([req.url for req in spider._parse_sitemap(r)], - ['http://www.example.com/english/article_1/', - 'http://www.example.com/english/article_2/']) + self.assertEqual( + [req.url for req in spider._parse_sitemap(r)], + [ + "http://www.example.com/english/article_1/", + "http://www.example.com/english/article_2/", + ], + ) spider = FilteredSitemapSpider("example.com") - self.assertEqual([req.url for req in spider._parse_sitemap(r)], - ['http://www.example.com/deutsch/article_1/']) + self.assertEqual( + [req.url for req in spider._parse_sitemap(r)], + ["http://www.example.com/deutsch/article_1/"], + ) def test_sitemapindex_filter(self): sitemap = b""" @@ -581,27 +657,35 @@ Sitemap: /sitemap-relative-url.xml class FilteredSitemapSpider(self.spider_class): def sitemap_filter(self, entries): from datetime import datetime + for entry in entries: - date_time = datetime.strptime(entry['lastmod'].split('T')[0], '%Y-%m-%d') + date_time = datetime.strptime( + entry["lastmod"].split("T")[0], "%Y-%m-%d" + ) if date_time.year > 2004: yield entry r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual([req.url for req in spider._parse_sitemap(r)], - ['http://www.example.com/sitemap1.xml', - 'http://www.example.com/sitemap2.xml']) + self.assertEqual( + [req.url for req in spider._parse_sitemap(r)], + [ + "http://www.example.com/sitemap1.xml", + "http://www.example.com/sitemap2.xml", + ], + ) spider = FilteredSitemapSpider("example.com") - self.assertEqual([req.url for req in spider._parse_sitemap(r)], - ['http://www.example.com/sitemap2.xml']) + self.assertEqual( + [req.url for req in spider._parse_sitemap(r)], + ["http://www.example.com/sitemap2.xml"], + ) class DeprecationTest(unittest.TestCase): - def test_crawl_spider(self): assert issubclass(CrawlSpider, Spider) - assert isinstance(CrawlSpider(name='foo'), Spider) + assert isinstance(CrawlSpider(name="foo"), Spider) class NoParseMethodSpiderTest(unittest.TestCase): @@ -609,10 +693,10 @@ class NoParseMethodSpiderTest(unittest.TestCase): spider_class = Spider def test_undefined_parse_method(self): - spider = self.spider_class('example.com') - text = b'Random text' + spider = self.spider_class("example.com") + text = b"Random text" resp = TextResponse(url="http://www.example.com/random_url", body=text) - exc_msg = 'Spider.parse callback is not defined' + exc_msg = "Spider.parse callback is not defined" with self.assertRaisesRegex(NotImplementedError, exc_msg): spider.parse(resp) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 0b6b51a5b..3745355a0 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -28,19 +28,18 @@ def _copytree(source: Path, target: Path): class SpiderLoaderTest(unittest.TestCase): - def setUp(self): - orig_spiders_dir = module_dir / 'test_spiders' + orig_spiders_dir = module_dir / "test_spiders" self.tmpdir = Path(tempfile.mkdtemp()) - self.spiders_dir = self.tmpdir / 'test_spiders_xxx' + self.spiders_dir = self.tmpdir / "test_spiders_xxx" _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(str(self.tmpdir)) - settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) + settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) self.spider_loader = SpiderLoader.from_settings(settings) def tearDown(self): del self.spider_loader - del sys.modules['test_spiders_xxx'] + del sys.modules["test_spiders_xxx"] sys.path.remove(str(self.tmpdir)) def test_interface(self): @@ -48,78 +47,86 @@ class SpiderLoaderTest(unittest.TestCase): def test_list(self): self.assertEqual( - set(self.spider_loader.list()), - {'spider1', 'spider2', 'spider3', 'spider4'}) + set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"} + ) def test_load(self): spider1 = self.spider_loader.load("spider1") - self.assertEqual(spider1.__name__, 'Spider1') + self.assertEqual(spider1.__name__, "Spider1") def test_find_by_request(self): self.assertEqual( - self.spider_loader.find_by_request(Request('http://scrapy1.org/test')), - ['spider1']) + self.spider_loader.find_by_request(Request("http://scrapy1.org/test")), + ["spider1"], + ) self.assertEqual( - self.spider_loader.find_by_request(Request('http://scrapy2.org/test')), - ['spider2']) + self.spider_loader.find_by_request(Request("http://scrapy2.org/test")), + ["spider2"], + ) self.assertEqual( - set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))), - {'spider1', 'spider2'}) + set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))), + {"spider1", "spider2"}, + ) self.assertEqual( - self.spider_loader.find_by_request(Request('http://scrapy999.org/test')), - []) + self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), [] + ) self.assertEqual( - self.spider_loader.find_by_request(Request('http://spider3.com')), - []) + self.spider_loader.find_by_request(Request("http://spider3.com")), [] + ) self.assertEqual( - self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')), - ['spider3']) + self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")), + ["spider3"], + ) def test_load_spider_module(self): - module = 'tests.test_spiderloader.test_spiders.spider1' - settings = Settings({'SPIDER_MODULES': [module]}) + module = "tests.test_spiderloader.test_spiders.spider1" + settings = Settings({"SPIDER_MODULES": [module]}) self.spider_loader = SpiderLoader.from_settings(settings) assert len(self.spider_loader._spiders) == 1 def test_load_spider_module_multiple(self): - prefix = 'tests.test_spiderloader.test_spiders.' - module = ','.join(prefix + s for s in ('spider1', 'spider2')) - settings = Settings({'SPIDER_MODULES': module}) + prefix = "tests.test_spiderloader.test_spiders." + module = ",".join(prefix + s for s in ("spider1", "spider2")) + settings = Settings({"SPIDER_MODULES": module}) self.spider_loader = SpiderLoader.from_settings(settings) assert len(self.spider_loader._spiders) == 2 def test_load_base_spider(self): - module = 'tests.test_spiderloader.test_spiders.spider0' - settings = Settings({'SPIDER_MODULES': [module]}) + module = "tests.test_spiderloader.test_spiders.spider0" + settings = Settings({"SPIDER_MODULES": [module]}) self.spider_loader = SpiderLoader.from_settings(settings) assert len(self.spider_loader._spiders) == 0 def test_crawler_runner_loading(self): - module = 'tests.test_spiderloader.test_spiders.spider1' - runner = CrawlerRunner({ - 'SPIDER_MODULES': [module], - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', - }) + module = "tests.test_spiderloader.test_spiders.spider1" + runner = CrawlerRunner( + { + "SPIDER_MODULES": [module], + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } + ) - self.assertRaisesRegex(KeyError, 'Spider not found', - runner.create_crawler, 'spider2') + self.assertRaisesRegex( + KeyError, "Spider not found", runner.create_crawler, "spider2" + ) - crawler = runner.create_crawler('spider1') + crawler = runner.create_crawler("spider1") self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) - self.assertEqual(crawler.spidercls.name, 'spider1') + self.assertEqual(crawler.spidercls.name, "spider1") def test_bad_spider_modules_exception(self): - module = 'tests.test_spiderloader.test_spiders.doesnotexist' - settings = Settings({'SPIDER_MODULES': [module]}) + module = "tests.test_spiderloader.test_spiders.doesnotexist" + settings = Settings({"SPIDER_MODULES": [module]}) self.assertRaises(ImportError, SpiderLoader.from_settings, settings) def test_bad_spider_modules_warning(self): with warnings.catch_warnings(record=True) as w: - module = 'tests.test_spiderloader.test_spiders.doesnotexist' - settings = Settings({'SPIDER_MODULES': [module], - 'SPIDER_LOADER_WARN_ONLY': True}) + module = "tests.test_spiderloader.test_spiders.doesnotexist" + settings = Settings( + {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} + ) spider_loader = SpiderLoader.from_settings(settings) if str(w[0].message).startswith("_SixMetaPathImporter"): # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, @@ -133,24 +140,25 @@ class SpiderLoaderTest(unittest.TestCase): class DuplicateSpiderNameLoaderTest(unittest.TestCase): - def setUp(self): - orig_spiders_dir = module_dir / 'test_spiders' + orig_spiders_dir = module_dir / "test_spiders" self.tmpdir = Path(self.mktemp()) self.tmpdir.mkdir() - self.spiders_dir = self.tmpdir / 'test_spiders_xxx' + self.spiders_dir = self.tmpdir / "test_spiders_xxx" _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(str(self.tmpdir)) - self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) + self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) def tearDown(self): - del sys.modules['test_spiders_xxx'] + del sys.modules["test_spiders_xxx"] sys.path.remove(str(self.tmpdir)) def test_dupename_warning(self): # copy 1 spider module so as to have duplicate spider name - shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider3.py', - self.tmpdir / 'test_spiders_xxx' / 'spider3dupe.py') + shutil.copyfile( + self.tmpdir / "test_spiders_xxx" / "spider3.py", + self.tmpdir / "test_spiders_xxx" / "spider3dupe.py", + ) with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) @@ -166,15 +174,19 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.assertNotIn("'spider4'", msg) spiders = set(spider_loader.list()) - self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'}) + self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"}) def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name # This should issue 2 warning, 1 for each duplicate spider name - shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider1.py', - self.tmpdir / 'test_spiders_xxx' / 'spider1dupe.py') - shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider2.py', - self.tmpdir / 'test_spiders_xxx' / 'spider2dupe.py') + shutil.copyfile( + self.tmpdir / "test_spiders_xxx" / "spider1.py", + self.tmpdir / "test_spiders_xxx" / "spider1dupe.py", + ) + shutil.copyfile( + self.tmpdir / "test_spiders_xxx" / "spider2.py", + self.tmpdir / "test_spiders_xxx" / "spider2dupe.py", + ) with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) @@ -192,4 +204,4 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.assertNotIn("'spider4'", msg) spiders = set(spider_loader.list()) - self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'}) + self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"}) diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.py index dbd1fb123..6c8ea4fe4 100644 --- a/tests/test_spiderloader/test_spiders/nested/spider4.py +++ b/tests/test_spiderloader/test_spiders/nested/spider4.py @@ -3,8 +3,8 @@ from scrapy.spiders import Spider class Spider4(Spider): name = "spider4" - allowed_domains = ['spider4.com'] + allowed_domains = ["spider4.com"] @classmethod def handles_request(cls, request): - return request.url == 'http://spider4.com/onlythis' + return request.url == "http://spider4.com/onlythis" diff --git a/tests/test_spiderloader/test_spiders/spider3.py b/tests/test_spiderloader/test_spiders/spider3.py index 84998ba35..d99878276 100644 --- a/tests/test_spiderloader/test_spiders/spider3.py +++ b/tests/test_spiderloader/test_spiders/spider3.py @@ -3,8 +3,8 @@ from scrapy.spiders import Spider class Spider3(Spider): name = "spider3" - allowed_domains = ['spider3.com'] + allowed_domains = ["spider3.com"] @classmethod def handles_request(cls, request): - return request.url == 'http://spider3.com/onlythis' + return request.url == "http://spider3.com/onlythis" diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index edde6f682..760ee43df 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -17,12 +17,11 @@ from scrapy.core.spidermw import SpiderMiddlewareManager class SpiderMiddlewareTestCase(TestCase): - def setUp(self): - self.request = Request('http://example.com/index.html') + self.request = Request("http://example.com/index.html") self.response = Response(self.request.url, request=self.request) - self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}}) - self.spider = self.crawler._create_spider('foo') + self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES_BASE": {}}) + self.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) def _scrape_response(self): @@ -30,7 +29,9 @@ class SpiderMiddlewareTestCase(TestCase): Raise exception in case of failure. """ scrape_func = mock.MagicMock() - dfd = self.mwman.scrape_response(scrape_func, self.response, self.request, self.spider) + dfd = self.mwman.scrape_response( + scrape_func, self.response, self.request, self.spider + ) # catch deferred result and return the value results = [] dfd.addBoth(results.append) @@ -43,7 +44,6 @@ class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): """Invalid return value for process_spider_input method""" def test_invalid_process_spider_input(self): - class InvalidProcessSpiderInputMiddleware: def process_spider_input(self, response, spider): return 1 @@ -58,7 +58,6 @@ class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase): """Invalid return value for process_spider_output method""" def test_invalid_process_spider_output(self): - class InvalidProcessSpiderOutputMiddleware: def process_spider_output(self, response, result, spider): return 1 @@ -73,7 +72,6 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): """Invalid return value for process_spider_exception method""" def test_invalid_process_spider_exception(self): - class InvalidProcessSpiderOutputExceptionMiddleware: def process_spider_exception(self, response, exception, spider): return 1 @@ -93,7 +91,6 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): """Re raise the exception by returning None""" def test_process_spider_exception_return_none(self): - class ProcessSpiderExceptionReturnNoneMiddleware: def process_spider_exception(self, response, exception, spider): return None @@ -110,7 +107,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): - """ Helpers for testing sync, async and mixed middlewares. + """Helpers for testing sync, async and mixed middlewares. Should work for process_spider_output and, when it's supported, process_start_requests. """ @@ -124,23 +121,31 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): return {i: c for c, i in enumerate(mw_classes, start=start_index)} def _scrape_func(self, *args, **kwargs): - yield {'foo': 1} - yield {'foo': 2} - yield {'foo': 3} + yield {"foo": 1} + yield {"foo": 2} + yield {"foo": 3} @defer.inlineCallbacks def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) - self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting}) - self.spider = self.crawler._create_spider('foo') + self.crawler = get_crawler( + Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} + ) + self.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider) + result = yield self.mwman.scrape_response( + self._scrape_func, self.response, self.request, self.spider + ) return result @defer.inlineCallbacks - def _test_simple_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None): + def _test_simple_base( + self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None + ): with LogCapture() as log: - result = yield self._get_middleware_result(*mw_classes, start_index=start_index) + result = yield self._get_middleware_result( + *mw_classes, start_index=start_index + ) self.assertIsInstance(result, collections.abc.Iterable) result_list = list(result) self.assertEqual(len(result_list), self.RESULT_COUNT) @@ -148,9 +153,13 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): self.assertEqual("downgraded to a non-async" in str(log), downgrade) @defer.inlineCallbacks - def _test_asyncgen_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None): + def _test_asyncgen_base( + self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None + ): with LogCapture() as log: - result = yield self._get_middleware_result(*mw_classes, start_index=start_index) + result = yield self._get_middleware_result( + *mw_classes, start_index=start_index + ) self.assertIsInstance(result, collections.abc.AsyncIterator) result_list = yield deferred_from_coro(collect_asyncgen(result)) self.assertEqual(len(result_list), self.RESULT_COUNT) @@ -182,24 +191,25 @@ class ProcessSpiderOutputUniversalMiddleware: class ProcessSpiderExceptionSimpleIterableMiddleware: def process_spider_exception(self, response, exception, spider): - yield {'foo': 1} - yield {'foo': 2} - yield {'foo': 3} + yield {"foo": 1} + yield {"foo": 2} + yield {"foo": 3} class ProcessSpiderExceptionAsyncIterableMiddleware: async def process_spider_exception(self, response, exception, spider): - yield {'foo': 1} + yield {"foo": 1} d = defer.Deferred() from twisted.internet import reactor + reactor.callLater(0, d.callback, None) await maybe_deferred_to_future(d) - yield {'foo': 2} - yield {'foo': 3} + yield {"foo": 2} + yield {"foo": 3} class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): - """ process_spider_output tests for simple callbacks""" + """process_spider_output tests for simple callbacks""" ITEM_TYPE = dict MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware @@ -207,82 +217,70 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware def test_simple(self): - """ Simple mw """ + """Simple mw""" return self._test_simple_base(self.MW_SIMPLE) def test_asyncgen(self): - """ Asyncgen mw; upgrade """ + """Asyncgen mw; upgrade""" return self._test_asyncgen_base(self.MW_ASYNCGEN) def test_simple_asyncgen(self): - """ Simple mw -> asyncgen mw; upgrade """ - return self._test_asyncgen_base(self.MW_ASYNCGEN, - self.MW_SIMPLE) + """Simple mw -> asyncgen mw; upgrade""" + return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE) def test_asyncgen_simple(self): - """ Asyncgen mw -> simple mw; upgrade then downgrade """ - return self._test_simple_base(self.MW_SIMPLE, - self.MW_ASYNCGEN, - downgrade=True) + """Asyncgen mw -> simple mw; upgrade then downgrade""" + return self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True) def test_universal(self): - """ Universal mw """ + """Universal mw""" return self._test_simple_base(self.MW_UNIVERSAL) def test_universal_simple(self): - """ Universal mw -> simple mw """ - return self._test_simple_base(self.MW_SIMPLE, - self.MW_UNIVERSAL) + """Universal mw -> simple mw""" + return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL) def test_simple_universal(self): - """ Simple mw -> universal mw """ - return self._test_simple_base(self.MW_UNIVERSAL, - self.MW_SIMPLE) + """Simple mw -> universal mw""" + return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE) def test_universal_asyncgen(self): - """ Universal mw -> asyncgen mw; upgrade """ - return self._test_asyncgen_base(self.MW_ASYNCGEN, - self.MW_UNIVERSAL) + """Universal mw -> asyncgen mw; upgrade""" + return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) def test_asyncgen_universal(self): - """ Asyncgen mw -> universal mw; upgrade """ - return self._test_asyncgen_base(self.MW_UNIVERSAL, - self.MW_ASYNCGEN) + """Asyncgen mw -> universal mw; upgrade""" + return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): - """ process_spider_output tests for async generator callbacks """ + """process_spider_output tests for async generator callbacks""" async def _scrape_func(self, *args, **kwargs): for item in super()._scrape_func(): yield item def test_simple(self): - """ Simple mw; downgrade """ - return self._test_simple_base(self.MW_SIMPLE, - downgrade=True) + """Simple mw; downgrade""" + return self._test_simple_base(self.MW_SIMPLE, downgrade=True) def test_simple_asyncgen(self): - """ Simple mw -> asyncgen mw; downgrade then upgrade """ - return self._test_asyncgen_base(self.MW_ASYNCGEN, - self.MW_SIMPLE, - downgrade=True) + """Simple mw -> asyncgen mw; downgrade then upgrade""" + return self._test_asyncgen_base( + self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True + ) def test_universal(self): - """ Universal mw """ + """Universal mw""" return self._test_asyncgen_base(self.MW_UNIVERSAL) def test_universal_simple(self): - """ Universal mw -> simple mw; downgrade """ - return self._test_simple_base(self.MW_SIMPLE, - self.MW_UNIVERSAL, - downgrade=True) + """Universal mw -> simple mw; downgrade""" + return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True) def test_simple_universal(self): - """ Simple mw -> universal mw; downgrade """ - return self._test_simple_base(self.MW_UNIVERSAL, - self.MW_SIMPLE, - downgrade=True) + """Simple mw -> universal mw; downgrade""" + return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True) class ProcessSpiderOutputNonIterableMiddleware: @@ -299,7 +297,6 @@ class ProcessSpiderOutputCoroutineMiddleware: class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): - @defer.inlineCallbacks def test_non_iterable(self): with self.assertRaisesRegex( @@ -331,27 +328,29 @@ class ProcessStartRequestsSimpleMiddleware: class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): - """ process_start_requests tests for simple start_requests""" + """process_start_requests tests for simple start_requests""" ITEM_TYPE = Request MW_SIMPLE = ProcessStartRequestsSimpleMiddleware def _start_requests(self): for i in range(3): - yield Request(f'https://example.com/{i}', dont_filter=True) + yield Request(f"https://example.com/{i}", dont_filter=True) @defer.inlineCallbacks def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) - self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting}) - self.spider = self.crawler._create_spider('foo') + self.crawler = get_crawler( + Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} + ) + self.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) start_requests = iter(self._start_requests()) results = yield self.mwman.process_start_requests(start_requests, self.spider) return results def test_simple(self): - """ Simple mw """ + """Simple mw""" return self._test_simple_base(self.MW_SIMPLE) @@ -383,41 +382,57 @@ class UniversalMiddlewareManagerTest(TestCase): def test_simple_mw(self): mw = ProcessSpiderOutputSimpleMiddleware self.mwman._add_middleware(mw) - self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output) + self.assertEqual( + self.mwman.methods["process_spider_output"][0], mw.process_spider_output + ) def test_async_mw(self): mw = ProcessSpiderOutputAsyncGenMiddleware self.mwman._add_middleware(mw) - self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output) + self.assertEqual( + self.mwman.methods["process_spider_output"][0], mw.process_spider_output + ) def test_universal_mw(self): mw = ProcessSpiderOutputUniversalMiddleware self.mwman._add_middleware(mw) - self.assertEqual(self.mwman.methods['process_spider_output'][0], - (mw.process_spider_output, mw.process_spider_output_async)) + self.assertEqual( + self.mwman.methods["process_spider_output"][0], + (mw.process_spider_output, mw.process_spider_output_async), + ) def test_universal_mw_no_sync(self): with LogCapture() as log: self.mwman._add_middleware(UniversalMiddlewareNoSync) - self.assertIn("UniversalMiddlewareNoSync has process_spider_output_async" - " without process_spider_output", str(log)) - self.assertEqual(self.mwman.methods['process_spider_output'][0], None) + self.assertIn( + "UniversalMiddlewareNoSync has process_spider_output_async" + " without process_spider_output", + str(log), + ) + self.assertEqual(self.mwman.methods["process_spider_output"][0], None) def test_universal_mw_both_sync(self): mw = UniversalMiddlewareBothSync with LogCapture() as log: self.mwman._add_middleware(mw) - self.assertIn("UniversalMiddlewareBothSync.process_spider_output_async " - "is not an async generator function", str(log)) - self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output) + self.assertIn( + "UniversalMiddlewareBothSync.process_spider_output_async " + "is not an async generator function", + str(log), + ) + self.assertEqual( + self.mwman.methods["process_spider_output"][0], mw.process_spider_output + ) def test_universal_mw_both_async(self): with LogCapture() as log: self.mwman._add_middleware(UniversalMiddlewareBothAsync) - self.assertIn("UniversalMiddlewareBothAsync.process_spider_output " - "is an async generator function while process_spider_output_async exists", - str(log)) - self.assertEqual(self.mwman.methods['process_spider_output'][0], None) + self.assertIn( + "UniversalMiddlewareBothAsync.process_spider_output " + "is an async generator function while process_spider_output_async exists", + str(log), + ) + self.assertEqual(self.mwman.methods["process_spider_output"][0], None) class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): @@ -429,10 +444,12 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): @defer.inlineCallbacks def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) - self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES': setting}) - self.spider = self.crawler._create_spider('foo') + self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting}) + self.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider) + result = yield self.mwman.scrape_response( + self._scrape_func, self.response, self.request, self.spider + ) return result def test_just_builtin(self): @@ -442,7 +459,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): return self._test_simple_base(self.MW_SIMPLE, start_index=1000) def test_builtin_async(self): - """ Upgrade """ + """Upgrade""" return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) def test_builtin_universal(self): @@ -452,7 +469,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): return self._test_simple_base(self.MW_SIMPLE) def test_async_builtin(self): - """ Upgrade """ + """Upgrade""" return self._test_asyncgen_base(self.MW_ASYNCGEN) def test_universal_builtin(self): @@ -468,7 +485,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): return self._test_asyncgen_base() def test_builtin_simple(self): - """ Downgrade """ + """Downgrade""" return self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000) def test_builtin_async(self): @@ -478,7 +495,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): return self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) def test_simple_builtin(self): - """ Downgrade """ + """Downgrade""" return self._test_simple_base(self.MW_SIMPLE, downgrade=True) def test_async_builtin(self): @@ -501,33 +518,31 @@ class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase): @defer.inlineCallbacks def _test_asyncgen_nodowngrade(self, *mw_classes): - with self.assertRaisesRegex(_InvalidOutput, "Async iterable returned from .+ cannot be downgraded"): + with self.assertRaisesRegex( + _InvalidOutput, "Async iterable returned from .+ cannot be downgraded" + ): yield self._get_middleware_result(*mw_classes) def test_exc_simple(self): - """ Simple exc mw """ + """Simple exc mw""" return self._test_simple_base(self.MW_EXC_SIMPLE) def test_exc_async(self): - """ Async exc mw """ + """Async exc mw""" return self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) def test_exc_simple_simple(self): - """ Simple exc mw -> simple output mw """ - return self._test_simple_base(self.MW_SIMPLE, - self.MW_EXC_SIMPLE) + """Simple exc mw -> simple output mw""" + return self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE) def test_exc_async_async(self): - """ Async exc mw -> async output mw """ - return self._test_asyncgen_base(self.MW_ASYNCGEN, - self.MW_EXC_ASYNCGEN) + """Async exc mw -> async output mw""" + return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN) def test_exc_simple_async(self): - """ Simple exc mw -> async output mw; upgrade """ - return self._test_asyncgen_base(self.MW_ASYNCGEN, - self.MW_EXC_SIMPLE) + """Simple exc mw -> async output mw; upgrade""" + return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE) def test_exc_async_simple(self): - """ Async exc mw -> simple output mw; cannot work as downgrading is not supported """ - return self._test_asyncgen_nodowngrade(self.MW_SIMPLE, - self.MW_EXC_ASYNCGEN) + """Async exc mw -> simple output mw; cannot work as downgrading is not supported""" + return self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN) diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 71cca2472..af17c13a0 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -8,10 +8,9 @@ from scrapy.utils.test import get_crawler class TestDepthMiddleware(TestCase): - def setUp(self): crawler = get_crawler(Spider) - self.spider = crawler._create_spider('scrapytest.org') + self.spider = crawler._create_spider("scrapytest.org") self.stats = StatsCollector(crawler) self.stats.open_spider(self.spider) @@ -19,24 +18,24 @@ class TestDepthMiddleware(TestCase): self.mw = DepthMiddleware(1, self.stats, True) def test_process_spider_output(self): - req = Request('http://scrapytest.org') - resp = Response('http://scrapytest.org') + req = Request("http://scrapytest.org") + resp = Response("http://scrapytest.org") resp.request = req - result = [Request('http://scrapytest.org')] + result = [Request("http://scrapytest.org")] out = list(self.mw.process_spider_output(resp, result, self.spider)) self.assertEqual(out, result) - rdc = self.stats.get_value('request_depth_count/1', spider=self.spider) + rdc = self.stats.get_value("request_depth_count/1", spider=self.spider) self.assertEqual(rdc, 1) - req.meta['depth'] = 1 + req.meta["depth"] = 1 out2 = list(self.mw.process_spider_output(resp, result, self.spider)) self.assertEqual(out2, []) - rdm = self.stats.get_value('request_depth_max', spider=self.spider) + rdm = self.stats.get_value("request_depth_max", spider=self.spider) self.assertEqual(rdm, 1) def tearDown(self): - self.stats.close_spider(self.spider, '') + self.stats.close_spider(self.spider, "") diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index ee11ee492..faa8e9091 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -15,7 +15,7 @@ from tests.spiders import MockServerSpider class _HttpErrorSpider(MockServerSpider): - name = 'httperror' + name = "httperror" bypass_status_codes = set() def __init__(self, *args, **kwargs): @@ -59,28 +59,35 @@ def _responses(request, status_codes): class TestHttpErrorMiddleware(TestCase): - def setUp(self): crawler = get_crawler(Spider) - self.spider = Spider.from_crawler(crawler, name='foo') + self.spider = Spider.from_crawler(crawler, name="foo") self.mw = HttpErrorMiddleware(Settings({})) - self.req = Request('http://scrapytest.org') + self.req = Request("http://scrapytest.org") self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) + self.assertRaises( + HttpError, self.mw.process_spider_input, self.res404, self.spider + ) def test_process_spider_exception(self): self.assertEqual( [], - self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider)) - self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider)) + self.mw.process_spider_exception( + self.res404, HttpError(self.res404), self.spider + ), + ) + self.assertIsNone( + self.mw.process_spider_exception(self.res404, Exception(), self.spider) + ) def test_handle_httpstatus_list(self): res = self.res404.copy() - res.request = Request('http://scrapytest.org', - meta={'handle_httpstatus_list': [404]}) + res.request = Request( + "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} + ) self.assertIsNone(self.mw.process_spider_input(res, self.spider)) self.spider.handle_httpstatus_list = [404] @@ -91,18 +98,22 @@ class TestHttpErrorMiddlewareSettings(TestCase): """Similar test, but with settings""" def setUp(self): - self.spider = Spider('foo') - self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOWED_CODES': (402,)})) - self.req = Request('http://scrapytest.org') + self.spider = Spider("foo") + self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)})) + self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) + self.assertRaises( + HttpError, self.mw.process_spider_input, self.res404, self.spider + ) self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): - request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) + request = Request( + "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} + ) res404 = self.res404.copy() res404.request = request res402 = self.res402.copy() @@ -114,15 +125,16 @@ class TestHttpErrorMiddlewareSettings(TestCase): def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) - self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider) + self.assertRaises( + HttpError, self.mw.process_spider_input, self.res402, self.spider + ) class TestHttpErrorMiddlewareHandleAll(TestCase): - def setUp(self): - self.spider = Spider('foo') - self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True})) - self.req = Request('http://scrapytest.org') + self.spider = Spider("foo") + self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True})) + self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): @@ -130,7 +142,9 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): - request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) + request = Request( + "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} + ) res404 = self.res404.copy() res404.request = request res402 = self.res402.copy() @@ -142,8 +156,12 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): def test_httperror_allow_all_false(self): crawler = get_crawler(_HttpErrorSpider) mw = HttpErrorMiddleware.from_crawler(crawler) - request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False}) - request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True}) + request_httpstatus_false = Request( + "http://scrapytest.org", meta={"handle_httpstatus_all": False} + ) + request_httpstatus_true = Request( + "http://scrapytest.org", meta={"handle_httpstatus_all": True} + ) res404 = self.res404.copy() res404.request = request_httpstatus_false res402 = self.res402.copy() @@ -166,28 +184,28 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): crawler = get_crawler(_HttpErrorSpider) yield crawler.crawl(mockserver=self.mockserver) assert not crawler.spider.skipped, crawler.spider.skipped - self.assertEqual(crawler.spider.parsed, {'200'}) - self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) + self.assertEqual(crawler.spider.parsed, {"200"}) + self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) get_value = crawler.stats.get_value - self.assertEqual(get_value('httperror/response_ignored_count'), 3) - self.assertEqual(get_value('httperror/response_ignored_status_count/404'), 1) - self.assertEqual(get_value('httperror/response_ignored_status_count/402'), 1) - self.assertEqual(get_value('httperror/response_ignored_status_count/500'), 1) + self.assertEqual(get_value("httperror/response_ignored_count"), 3) + self.assertEqual(get_value("httperror/response_ignored_status_count/404"), 1) + self.assertEqual(get_value("httperror/response_ignored_status_count/402"), 1) + self.assertEqual(get_value("httperror/response_ignored_status_count/500"), 1) @defer.inlineCallbacks def test_logging(self): crawler = get_crawler(_HttpErrorSpider) with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) - self.assertEqual(crawler.spider.parsed, {'200', '402'}) - self.assertEqual(crawler.spider.skipped, {'402'}) - self.assertEqual(crawler.spider.failed, {'404', '500'}) + self.assertEqual(crawler.spider.parsed, {"200", "402"}) + self.assertEqual(crawler.spider.skipped, {"402"}) + self.assertEqual(crawler.spider.failed, {"404", "500"}) - self.assertIn('Ignoring response <404', str(log)) - self.assertIn('Ignoring response <500', str(log)) - self.assertNotIn('Ignoring response <200', str(log)) - self.assertNotIn('Ignoring response <402', str(log)) + self.assertIn("Ignoring response <404", str(log)) + self.assertIn("Ignoring response <500", str(log)) + self.assertNotIn("Ignoring response <200", str(log)) + self.assertNotIn("Ignoring response <402", str(log)) @defer.inlineCallbacks def test_logging_level(self): @@ -195,22 +213,22 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.INFO) as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.parsed, {'200'}) - self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) + self.assertEqual(crawler.spider.parsed, {"200"}) + self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) - self.assertIn('Ignoring response <402', str(log)) - self.assertIn('Ignoring response <404', str(log)) - self.assertIn('Ignoring response <500', str(log)) - self.assertNotIn('Ignoring response <200', str(log)) + self.assertIn("Ignoring response <402", str(log)) + self.assertIn("Ignoring response <404", str(log)) + self.assertIn("Ignoring response <500", str(log)) + self.assertNotIn("Ignoring response <200", str(log)) # with level WARNING, we shouldn't capture anything from HttpError crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.WARNING) as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.parsed, {'200'}) - self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) + self.assertEqual(crawler.spider.parsed, {"200"}) + self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) - self.assertNotIn('Ignoring response <402', str(log)) - self.assertNotIn('Ignoring response <404', str(log)) - self.assertNotIn('Ignoring response <500', str(log)) - self.assertNotIn('Ignoring response <200', str(log)) + self.assertNotIn("Ignoring response <402", str(log)) + self.assertNotIn("Ignoring response <404", str(log)) + self.assertNotIn("Ignoring response <500", str(log)) + self.assertNotIn("Ignoring response <200", str(log)) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 0f4b98a07..380bafe04 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -9,7 +9,6 @@ from scrapy.utils.test import get_crawler class TestOffsiteMiddleware(TestCase): - def setUp(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider(**self._get_spiderargs()) @@ -17,28 +16,31 @@ class TestOffsiteMiddleware(TestCase): self.mw.spider_opened(self.spider) def _get_spiderargs(self): - return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org', 'scrapy.test.org']) + return dict( + name="foo", + allowed_domains=["scrapytest.org", "scrapy.org", "scrapy.test.org"], + ) def test_process_spider_output(self): - res = Response('http://scrapytest.org') + res = Response("http://scrapytest.org") onsite_reqs = [ - Request('http://scrapytest.org/1'), - Request('http://scrapy.org/1'), - Request('http://sub.scrapy.org/1'), - Request('http://offsite.tld/letmepass', dont_filter=True), - Request('http://scrapy.test.org/'), - Request('http://scrapy.test.org:8000/'), + Request("http://scrapytest.org/1"), + Request("http://scrapy.org/1"), + Request("http://sub.scrapy.org/1"), + Request("http://offsite.tld/letmepass", dont_filter=True), + Request("http://scrapy.test.org/"), + Request("http://scrapy.test.org:8000/"), ] offsite_reqs = [ - Request('http://scrapy2.org'), - Request('http://offsite.tld/'), - Request('http://offsite.tld/scrapytest.org'), - Request('http://offsite.tld/rogue.scrapytest.org'), - Request('http://rogue.scrapytest.org.haha.com'), - Request('http://roguescrapytest.org'), - Request('http://test.org/'), - Request('http://notscrapy.test.org/'), + Request("http://scrapy2.org"), + Request("http://offsite.tld/"), + Request("http://offsite.tld/scrapytest.org"), + Request("http://offsite.tld/rogue.scrapytest.org"), + Request("http://rogue.scrapytest.org.haha.com"), + Request("http://roguescrapytest.org"), + Request("http://test.org/"), + Request("http://notscrapy.test.org/"), ] reqs = onsite_reqs + offsite_reqs @@ -47,40 +49,40 @@ class TestOffsiteMiddleware(TestCase): class TestOffsiteMiddleware2(TestOffsiteMiddleware): - def _get_spiderargs(self): - return dict(name='foo', allowed_domains=None) + return dict(name="foo", allowed_domains=None) def test_process_spider_output(self): - res = Response('http://scrapytest.org') - reqs = [Request('http://a.com/b.html'), Request('http://b.com/1')] + res = Response("http://scrapytest.org") + reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")] out = list(self.mw.process_spider_output(res, reqs, self.spider)) self.assertEqual(out, reqs) class TestOffsiteMiddleware3(TestOffsiteMiddleware2): - def _get_spiderargs(self): - return dict(name='foo') + return dict(name="foo") class TestOffsiteMiddleware4(TestOffsiteMiddleware3): - def _get_spiderargs(self): - bad_hostname = urlparse('http:////scrapytest.org').hostname - return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname]) + bad_hostname = urlparse("http:////scrapytest.org").hostname + return dict(name="foo", allowed_domains=["scrapytest.org", None, bad_hostname]) def test_process_spider_output(self): - res = Response('http://scrapytest.org') - reqs = [Request('http://scrapytest.org/1')] + res = Response("http://scrapytest.org") + reqs = [Request("http://scrapytest.org/1")] out = list(self.mw.process_spider_output(res, reqs, self.spider)) self.assertEqual(out, reqs) class TestOffsiteMiddleware5(TestOffsiteMiddleware4): - def test_get_host_regex(self): - self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org'] + self.spider.allowed_domains = [ + "http://scrapytest.org", + "scrapy.org", + "scrapy.test.org", + ] with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") self.mw.get_host_regex(self.spider) @@ -88,9 +90,12 @@ class TestOffsiteMiddleware5(TestOffsiteMiddleware4): class TestOffsiteMiddleware6(TestOffsiteMiddleware4): - def test_get_host_regex(self): - self.spider.allowed_domains = ['scrapytest.org:8000', 'scrapy.org', 'scrapy.test.org'] + self.spider.allowed_domains = [ + "scrapytest.org:8000", + "scrapy.org", + "scrapy.test.org", + ] with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") self.mw.get_host_regex(self.spider) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 8dd1def17..fad5dcaac 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -9,7 +9,9 @@ from tests.mockserver import MockServer class LogExceptionMiddleware: def process_spider_exception(self, response, exception, spider): - spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) + spider.logger.info( + "Middleware: %s exception caught", exception.__class__.__name__ + ) return None @@ -17,34 +19,36 @@ class LogExceptionMiddleware: # (0) recover from an exception on a spider callback class RecoveryMiddleware: def process_spider_exception(self, response, exception, spider): - spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) + spider.logger.info( + "Middleware: %s exception caught", exception.__class__.__name__ + ) return [ - {'from': 'process_spider_exception'}, - Request(response.url, meta={'dont_fail': True}, dont_filter=True), + {"from": "process_spider_exception"}, + Request(response.url, meta={"dont_fail": True}, dont_filter=True), ] class RecoverySpider(Spider): - name = 'RecoverySpider' + name = "RecoverySpider" custom_settings = { - 'SPIDER_MIDDLEWARES_BASE': {}, - 'SPIDER_MIDDLEWARES': { + "SPIDER_MIDDLEWARES_BASE": {}, + "SPIDER_MIDDLEWARES": { RecoveryMiddleware: 10, }, } def start_requests(self): - yield Request(self.mockserver.url('/status?n=200')) + yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): - yield {'test': 1} - self.logger.info('DONT_FAIL: %s', response.meta.get('dont_fail')) - if not response.meta.get('dont_fail'): + yield {"test": 1} + self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail")) + if not response.meta.get("dont_fail"): raise TabError() class RecoveryAsyncGenSpider(RecoverySpider): - name = 'RecoveryAsyncGenSpider' + name = "RecoveryAsyncGenSpider" async def parse(self, response): for r in super().parse(response): @@ -55,14 +59,14 @@ class RecoveryAsyncGenSpider(RecoverySpider): # (1) exceptions from a spider middleware's process_spider_input method class FailProcessSpiderInputMiddleware: def process_spider_input(self, response, spider): - spider.logger.info('Middleware: will raise IndexError') + spider.logger.info("Middleware: will raise IndexError") raise IndexError() class ProcessSpiderInputSpiderWithoutErrback(Spider): - name = 'ProcessSpiderInputSpiderWithoutErrback' + name = "ProcessSpiderInputSpiderWithoutErrback" custom_settings = { - 'SPIDER_MIDDLEWARES': { + "SPIDER_MIDDLEWARES": { # spider FailProcessSpiderInputMiddleware: 8, LogExceptionMiddleware: 6, @@ -71,55 +75,57 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): } def start_requests(self): - yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse) + yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse) def parse(self, response): - return {'from': 'callback'} + return {"from": "callback"} class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback): - name = 'ProcessSpiderInputSpiderWithErrback' + name = "ProcessSpiderInputSpiderWithErrback" def start_requests(self): - yield Request(self.mockserver.url('/status?n=200'), self.parse, errback=self.errback) + yield Request( + self.mockserver.url("/status?n=200"), self.parse, errback=self.errback + ) def errback(self, failure): - self.logger.info('Got a Failure on the Request errback') - return {'from': 'errback'} + self.logger.info("Got a Failure on the Request errback") + return {"from": "errback"} # ================================================================================ # (2) exceptions from a spider callback (generator) class GeneratorCallbackSpider(Spider): - name = 'GeneratorCallbackSpider' + name = "GeneratorCallbackSpider" custom_settings = { - 'SPIDER_MIDDLEWARES': { + "SPIDER_MIDDLEWARES": { LogExceptionMiddleware: 10, }, } def start_requests(self): - yield Request(self.mockserver.url('/status?n=200')) + yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): - yield {'test': 1} - yield {'test': 2} + yield {"test": 1} + yield {"test": 2} raise ImportError() class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider): async def parse(self, response): - yield {'test': 1} - yield {'test': 2} + yield {"test": 1} + yield {"test": 2} raise ImportError() # ================================================================================ # (2.1) exceptions from a spider callback (generator, middleware right after callback) class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider): - name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider' + name = "GeneratorCallbackSpiderMiddlewareRightAfterSpider" custom_settings = { - 'SPIDER_MIDDLEWARES': { + "SPIDER_MIDDLEWARES": { LogExceptionMiddleware: 100000, }, } @@ -128,26 +134,26 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider) # ================================================================================ # (3) exceptions from a spider callback (not a generator) class NotGeneratorCallbackSpider(Spider): - name = 'NotGeneratorCallbackSpider' + name = "NotGeneratorCallbackSpider" custom_settings = { - 'SPIDER_MIDDLEWARES': { + "SPIDER_MIDDLEWARES": { LogExceptionMiddleware: 10, }, } def start_requests(self): - yield Request(self.mockserver.url('/status?n=200')) + yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): - return [{'test': 1}, {'test': 1 / 0}] + return [{"test": 1}, {"test": 1 / 0}] # ================================================================================ # (3.1) exceptions from a spider callback (not a generator, middleware right after callback) class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackSpider): - name = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider' + name = "NotGeneratorCallbackSpiderMiddlewareRightAfterSpider" custom_settings = { - 'SPIDER_MIDDLEWARES': { + "SPIDER_MIDDLEWARES": { LogExceptionMiddleware: 100000, }, } @@ -158,26 +164,26 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS class _GeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append(f'{self.__class__.__name__}.process_spider_output') + r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r def process_spider_exception(self, response, exception, spider): - method = f'{self.__class__.__name__}.process_spider_exception' - spider.logger.info('%s: %s caught', method, exception.__class__.__name__) + method = f"{self.__class__.__name__}.process_spider_exception" + spider.logger.info("%s: %s caught", method, exception.__class__.__name__) return None class GeneratorFailMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append(f'{self.__class__.__name__}.process_spider_output') + r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r raise LookupError() def process_spider_exception(self, response, exception, spider): - method = f'{self.__class__.__name__}.process_spider_exception' - spider.logger.info('%s: %s caught', method, exception.__class__.__name__) - yield {'processed': [method]} + method = f"{self.__class__.__name__}.process_spider_exception" + spider.logger.info("%s: %s caught", method, exception.__class__.__name__) + yield {"processed": [method]} class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware): @@ -187,13 +193,13 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware): class GeneratorRecoverMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append(f'{self.__class__.__name__}.process_spider_output') + r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r def process_spider_exception(self, response, exception, spider): - method = f'{self.__class__.__name__}.process_spider_exception' - spider.logger.info('%s: %s caught', method, exception.__class__.__name__) - yield {'processed': [method]} + method = f"{self.__class__.__name__}.process_spider_exception" + spider.logger.info("%s: %s caught", method, exception.__class__.__name__) + yield {"processed": [method]} class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): @@ -201,9 +207,9 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): class GeneratorOutputChainSpider(Spider): - name = 'GeneratorOutputChainSpider' + name = "GeneratorOutputChainSpider" custom_settings = { - 'SPIDER_MIDDLEWARES': { + "SPIDER_MIDDLEWARES": { GeneratorFailMiddleware: 10, GeneratorDoNothingAfterFailureMiddleware: 8, GeneratorRecoverMiddleware: 5, @@ -212,27 +218,28 @@ class GeneratorOutputChainSpider(Spider): } def start_requests(self): - yield Request(self.mockserver.url('/status?n=200')) + yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): - yield {'processed': ['parse-first-item']} - yield {'processed': ['parse-second-item']} + yield {"processed": ["parse-first-item"]} + yield {"processed": ["parse-second-item"]} # ================================================================================ # (5) exceptions from a middleware process_spider_output method (not generator) + class _NotGeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append(f'{self.__class__.__name__}.process_spider_output') + r["processed"].append(f"{self.__class__.__name__}.process_spider_output") out.append(r) return out def process_spider_exception(self, response, exception, spider): - method = f'{self.__class__.__name__}.process_spider_exception' - spider.logger.info('%s: %s caught', method, exception.__class__.__name__) + method = f"{self.__class__.__name__}.process_spider_exception" + spider.logger.info("%s: %s caught", method, exception.__class__.__name__) return None @@ -240,15 +247,15 @@ class NotGeneratorFailMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append(f'{self.__class__.__name__}.process_spider_output') + r["processed"].append(f"{self.__class__.__name__}.process_spider_output") out.append(r) raise ReferenceError() return out def process_spider_exception(self, response, exception, spider): - method = f'{self.__class__.__name__}.process_spider_exception' - spider.logger.info('%s: %s caught', method, exception.__class__.__name__) - return [{'processed': [method]}] + method = f"{self.__class__.__name__}.process_spider_exception" + spider.logger.info("%s: %s caught", method, exception.__class__.__name__) + return [{"processed": [method]}] class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddleware): @@ -259,14 +266,14 @@ class NotGeneratorRecoverMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append(f'{self.__class__.__name__}.process_spider_output') + r["processed"].append(f"{self.__class__.__name__}.process_spider_output") out.append(r) return out def process_spider_exception(self, response, exception, spider): - method = f'{self.__class__.__name__}.process_spider_exception' - spider.logger.info('%s: %s caught', method, exception.__class__.__name__) - return [{'processed': [method]}] + method = f"{self.__class__.__name__}.process_spider_exception" + spider.logger.info("%s: %s caught", method, exception.__class__.__name__) + return [{"processed": [method]}] class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware): @@ -274,9 +281,9 @@ class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddlew class NotGeneratorOutputChainSpider(Spider): - name = 'NotGeneratorOutputChainSpider' + name = "NotGeneratorOutputChainSpider" custom_settings = { - 'SPIDER_MIDDLEWARES': { + "SPIDER_MIDDLEWARES": { NotGeneratorFailMiddleware: 10, NotGeneratorDoNothingAfterFailureMiddleware: 8, NotGeneratorRecoverMiddleware: 5, @@ -285,10 +292,13 @@ class NotGeneratorOutputChainSpider(Spider): } def start_requests(self): - return [Request(self.mockserver.url('/status?n=200'))] + return [Request(self.mockserver.url("/status?n=200"))] def parse(self, response): - return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}] + return [ + {"processed": ["parse-first-item"]}, + {"processed": ["parse-second-item"]}, + ] # ================================================================================ @@ -402,7 +412,9 @@ class TestSpiderMiddleware(TestCase): (3.1) Special case of (3): Exceptions should be caught even if the middleware is placed right after the spider """ - log31 = yield self.crawl_log(NotGeneratorCallbackSpiderMiddlewareRightAfterSpider) + log31 = yield self.crawl_log( + NotGeneratorCallbackSpiderMiddlewareRightAfterSpider + ) self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31)) self.assertNotIn("item_scraped_count", str(log31)) @@ -418,28 +430,40 @@ class TestSpiderMiddleware(TestCase): """ log4 = yield self.crawl_log(GeneratorOutputChainSpider) self.assertIn("'item_scraped_count': 2", str(log4)) - self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertIn( + "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", + str(log4), + ) self.assertIn( "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", - str(log4)) + str(log4), + ) self.assertNotIn( "GeneratorFailMiddleware.process_spider_exception: LookupError caught", - str(log4)) + str(log4), + ) self.assertNotIn( "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", - str(log4)) - item_from_callback = {'processed': [ - 'parse-first-item', - 'GeneratorFailMiddleware.process_spider_output', - 'GeneratorDoNothingAfterFailureMiddleware.process_spider_output', - 'GeneratorRecoverMiddleware.process_spider_output', - 'GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']} - item_recovered = {'processed': [ - 'GeneratorRecoverMiddleware.process_spider_exception', - 'GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']} + str(log4), + ) + item_from_callback = { + "processed": [ + "parse-first-item", + "GeneratorFailMiddleware.process_spider_output", + "GeneratorDoNothingAfterFailureMiddleware.process_spider_output", + "GeneratorRecoverMiddleware.process_spider_output", + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", + ] + } + item_recovered = { + "processed": [ + "GeneratorRecoverMiddleware.process_spider_exception", + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", + ] + } self.assertIn(str(item_from_callback), str(log4)) self.assertIn(str(item_recovered), str(log4)) - self.assertNotIn('parse-second-item', str(log4)) + self.assertNotIn("parse-second-item", str(log4)) @defer.inlineCallbacks def test_not_a_generator_output_chain(self): @@ -453,17 +477,28 @@ class TestSpiderMiddleware(TestCase): """ log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) self.assertIn("'item_scraped_count': 1", str(log5)) - self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertIn( + "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", + str(log5), + ) self.assertIn( "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", - str(log5)) - self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + str(log5), + ) + self.assertNotIn( + "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", + str(log5), + ) self.assertNotIn( "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", - str(log5)) - item_recovered = {'processed': [ - 'NotGeneratorRecoverMiddleware.process_spider_exception', - 'NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']} + str(log5), + ) + item_recovered = { + "processed": [ + "NotGeneratorRecoverMiddleware.process_spider_exception", + "NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", + ] + } self.assertIn(str(item_recovered), str(log5)) - self.assertNotIn('parse-first-item', str(log5)) - self.assertNotIn('parse-second-item', str(log5)) + self.assertNotIn("parse-first-item", str(log5)) + self.assertNotIn("parse-second-item", str(log5)) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 63daf0b8a..db5a22adb 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -36,11 +36,11 @@ class TestRefererMiddleware(TestCase): resp_headers = {} settings = {} scenarii = [ - ('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'), + ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] def setUp(self): - self.spider = Spider('foo') + self.spider = Spider("foo") settings = Settings(self.settings) self.mw = RefererMiddleware(settings) @@ -56,7 +56,7 @@ class TestRefererMiddleware(TestCase): response = self.get_response(origin) request = self.get_request(target) out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEqual(out[0].headers.get('Referer'), referrer) + self.assertEqual(out[0].headers.get("Referer"), referrer) class MixinDefault: @@ -65,101 +65,184 @@ class MixinDefault: with some additional filtering of s3:// """ + scenarii = [ - ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), - ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), - ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), - ('https://example.com/', 'http://scrapy.org/', None), - + ("https://example.com/", "https://scrapy.org/", b"https://example.com/"), + ("http://example.com/", "http://scrapy.org/", b"http://example.com/"), + ("http://example.com/", "https://scrapy.org/", b"http://example.com/"), + ("https://example.com/", "http://scrapy.org/", None), # no credentials leak - ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), - + ( + "http://user:password@example.com/", + "https://scrapy.org/", + b"http://example.com/", + ), # no referrer leak for local schemes - ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), - ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), - + ("file:///home/path/to/somefile.html", "https://scrapy.org/", None), + ("file:///home/path/to/somefile.html", "http://scrapy.org/", None), # no referrer leak for s3 origins - ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), - ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), + ("s3://mybucket/path/to/data.csv", "https://scrapy.org/", None), + ("s3://mybucket/path/to/data.csv", "http://scrapy.org/", None), ] class MixinNoReferrer: scenarii = [ - ('https://example.com/page.html', 'https://example.com/', None), - ('http://www.example.com/', 'https://scrapy.org/', None), - ('http://www.example.com/', 'http://scrapy.org/', None), - ('https://www.example.com/', 'http://scrapy.org/', None), - ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + ("https://example.com/page.html", "https://example.com/", None), + ("http://www.example.com/", "https://scrapy.org/", None), + ("http://www.example.com/", "http://scrapy.org/", None), + ("https://www.example.com/", "http://scrapy.org/", None), + ("file:///home/path/to/somefile.html", "http://scrapy.org/", None), ] class MixinNoReferrerWhenDowngrade: scenarii = [ # TLS to TLS: send non-empty referrer - ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), - ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), - ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), - ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), - ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), - + ( + "https://example.com/page.html", + "https://not.example.com/", + b"https://example.com/page.html", + ), + ( + "https://example.com/page.html", + "https://scrapy.org/", + b"https://example.com/page.html", + ), + ( + "https://example.com:443/page.html", + "https://scrapy.org/", + b"https://example.com/page.html", + ), + ( + "https://example.com:444/page.html", + "https://scrapy.org/", + b"https://example.com:444/page.html", + ), + ( + "ftps://example.com/urls.zip", + "https://scrapy.org/", + b"ftps://example.com/urls.zip", + ), # TLS to non-TLS: do not send referrer - ('https://example.com/page.html', 'http://not.example.com/', None), - ('https://example.com/page.html', 'http://scrapy.org/', None), - ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), - + ("https://example.com/page.html", "http://not.example.com/", None), + ("https://example.com/page.html", "http://scrapy.org/", None), + ("ftps://example.com/urls.zip", "http://scrapy.org/", None), # non-TLS to TLS or non-TLS: send referrer - ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), - ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), - ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), - ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), - ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), - ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), - ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), - + ( + "http://example.com/page.html", + "https://not.example.com/", + b"http://example.com/page.html", + ), + ( + "http://example.com/page.html", + "https://scrapy.org/", + b"http://example.com/page.html", + ), + ( + "http://example.com:8080/page.html", + "https://scrapy.org/", + b"http://example.com:8080/page.html", + ), + ( + "http://example.com:80/page.html", + "http://not.example.com/", + b"http://example.com/page.html", + ), + ( + "http://example.com/page.html", + "http://scrapy.org/", + b"http://example.com/page.html", + ), + ( + "http://example.com:443/page.html", + "http://scrapy.org/", + b"http://example.com:443/page.html", + ), + ( + "ftp://example.com/urls.zip", + "http://scrapy.org/", + b"ftp://example.com/urls.zip", + ), + ( + "ftp://example.com/urls.zip", + "https://scrapy.org/", + b"ftp://example.com/urls.zip", + ), # test for user/password stripping - ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ( + "http://user:password@example.com/page.html", + "https://not.example.com/", + b"http://example.com/page.html", + ), ] class MixinSameOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), ( - 'http://example.com:8888/page.html', - 'http://example.com:8888/not-page.html', - b'http://example.com:8888/page.html', + "https://example.com/page.html", + "https://example.com/not-page.html", + b"https://example.com/page.html", ), - - # Different host: do NOT send referrer - ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), - ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), - ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), - - # Different port: do NOT send referrer - ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), - ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), - ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), - - # Different protocols: do NOT send refferer - ('https://example.com/page.html', 'http://example.com/not-page.html', None), - ('https://example.com/page.html', 'http://not.example.com/', None), - ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), - ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), - ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), - - # test for user/password stripping - ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), ( - 'https://user:password@example.com/page.html', - 'https://example.com/not-page.html', - b'https://example.com/page.html', + "http://example.com/page.html", + "http://example.com/not-page.html", + b"http://example.com/page.html", + ), + ( + "https://example.com:443/page.html", + "https://example.com/not-page.html", + b"https://example.com/page.html", + ), + ( + "http://example.com:80/page.html", + "http://example.com/not-page.html", + b"http://example.com/page.html", + ), + ( + "http://example.com/page.html", + "http://example.com:80/not-page.html", + b"http://example.com/page.html", + ), + ( + "http://example.com:8888/page.html", + "http://example.com:8888/not-page.html", + b"http://example.com:8888/page.html", + ), + # Different host: do NOT send referrer + ( + "https://example.com/page.html", + "https://not.example.com/otherpage.html", + None, + ), + ("http://example.com/page.html", "http://not.example.com/otherpage.html", None), + ("http://example.com/page.html", "http://www.example.com/otherpage.html", None), + # Different port: do NOT send referrer + ( + "https://example.com:444/page.html", + "https://example.com/not-page.html", + None, + ), + ("http://example.com:81/page.html", "http://example.com/not-page.html", None), + ("http://example.com/page.html", "http://example.com:81/not-page.html", None), + # Different protocols: do NOT send refferer + ("https://example.com/page.html", "http://example.com/not-page.html", None), + ("https://example.com/page.html", "http://not.example.com/", None), + ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None), + ("ftp://example.com/urls.zip", "http://example.com/not-page.html", None), + ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None), + # test for user/password stripping + ( + "https://user:password@example.com/page.html", + "http://example.com/not-page.html", + None, + ), + ( + "https://user:password@example.com/page.html", + "https://example.com/not-page.html", + b"https://example.com/page.html", ), ] @@ -167,78 +250,158 @@ class MixinSameOrigin: class MixinOrigin: scenarii = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), - ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), - ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), - ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), - + ( + "https://example.com/page.html", + "https://example.com/not-page.html", + b"https://example.com/", + ), + ( + "https://example.com/page.html", + "https://scrapy.org", + b"https://example.com/", + ), + ("https://example.com/page.html", "http://scrapy.org", b"https://example.com/"), + ("http://example.com/page.html", "http://scrapy.org", b"http://example.com/"), # test for user/password stripping - ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ( + "https://user:password@example.com/page.html", + "http://scrapy.org", + b"https://example.com/", + ), ] class MixinStrictOrigin: scenarii = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), - ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), - ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), - + ( + "https://example.com/page.html", + "https://example.com/not-page.html", + b"https://example.com/", + ), + ( + "https://example.com/page.html", + "https://scrapy.org", + b"https://example.com/", + ), + ("http://example.com/page.html", "http://scrapy.org", b"http://example.com/"), # downgrade: send nothing - ('https://example.com/page.html', 'http://scrapy.org', None), - + ("https://example.com/page.html", "http://scrapy.org", None), # upgrade: send origin - ('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'), - + ("http://example.com/page.html", "https://scrapy.org", b"http://example.com/"), # test for user/password stripping - ('https://user:password@example.com/page.html', 'https://scrapy.org', b'https://example.com/'), - ('https://user:password@example.com/page.html', 'http://scrapy.org', None), + ( + "https://user:password@example.com/page.html", + "https://scrapy.org", + b"https://example.com/", + ), + ("https://user:password@example.com/page.html", "http://scrapy.org", None), ] class MixinOriginWhenCrossOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), ( - 'http://example.com:8888/page.html', - 'http://example.com:8888/not-page.html', - b'http://example.com:8888/page.html', + "https://example.com/page.html", + "https://example.com/not-page.html", + b"https://example.com/page.html", + ), + ( + "http://example.com/page.html", + "http://example.com/not-page.html", + b"http://example.com/page.html", + ), + ( + "https://example.com:443/page.html", + "https://example.com/not-page.html", + b"https://example.com/page.html", + ), + ( + "http://example.com:80/page.html", + "http://example.com/not-page.html", + b"http://example.com/page.html", + ), + ( + "http://example.com/page.html", + "http://example.com:80/not-page.html", + b"http://example.com/page.html", + ), + ( + "http://example.com:8888/page.html", + "http://example.com:8888/not-page.html", + b"http://example.com:8888/page.html", ), - # Different host: send origin as referrer - ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), - ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), - ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + ( + "https://example2.com/page.html", + "https://scrapy.org/otherpage.html", + b"https://example2.com/", + ), + ( + "https://example2.com/page.html", + "https://not.example2.com/otherpage.html", + b"https://example2.com/", + ), + ( + "http://example2.com/page.html", + "http://not.example2.com/otherpage.html", + b"http://example2.com/", + ), # exact match required - ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), - + ( + "http://example2.com/page.html", + "http://www.example2.com/otherpage.html", + b"http://example2.com/", + ), # Different port: send origin as referrer - ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), - ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), - + ( + "https://example3.com:444/page.html", + "https://example3.com/not-page.html", + b"https://example3.com:444/", + ), + ( + "http://example3.com:81/page.html", + "http://example3.com/not-page.html", + b"http://example3.com:81/", + ), # Different protocols: send origin as referrer - ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), - ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), - ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), - ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), - ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), - + ( + "https://example4.com/page.html", + "http://example4.com/not-page.html", + b"https://example4.com/", + ), + ( + "https://example4.com/page.html", + "http://not.example4.com/", + b"https://example4.com/", + ), + ( + "ftps://example4.com/urls.zip", + "https://example4.com/not-page.html", + b"ftps://example4.com/", + ), + ( + "ftp://example4.com/urls.zip", + "http://example4.com/not-page.html", + b"ftp://example4.com/", + ), + ( + "ftps://example4.com/urls.zip", + "https://example4.com/not-page.html", + b"ftps://example4.com/", + ), # test for user/password stripping ( - 'https://user:password@example5.com/page.html', - 'https://example5.com/not-page.html', - b'https://example5.com/page.html', + "https://user:password@example5.com/page.html", + "https://example5.com/not-page.html", + b"https://example5.com/page.html", ), # TLS to non-TLS downgrade: send origin ( - 'https://user:password@example5.com/page.html', - 'http://example5.com/not-page.html', - b'https://example5.com/', + "https://user:password@example5.com/page.html", + "http://example5.com/not-page.html", + b"https://example5.com/", ), ] @@ -246,90 +409,215 @@ class MixinOriginWhenCrossOrigin: class MixinStrictOriginWhenCrossOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), ( - 'http://example.com:8888/page.html', - 'http://example.com:8888/not-page.html', - b'http://example.com:8888/page.html', + "https://example.com/page.html", + "https://example.com/not-page.html", + b"https://example.com/page.html", + ), + ( + "http://example.com/page.html", + "http://example.com/not-page.html", + b"http://example.com/page.html", + ), + ( + "https://example.com:443/page.html", + "https://example.com/not-page.html", + b"https://example.com/page.html", + ), + ( + "http://example.com:80/page.html", + "http://example.com/not-page.html", + b"http://example.com/page.html", + ), + ( + "http://example.com/page.html", + "http://example.com:80/not-page.html", + b"http://example.com/page.html", + ), + ( + "http://example.com:8888/page.html", + "http://example.com:8888/not-page.html", + b"http://example.com:8888/page.html", ), - # Different host: send origin as referrer - ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), - ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), - ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + ( + "https://example2.com/page.html", + "https://scrapy.org/otherpage.html", + b"https://example2.com/", + ), + ( + "https://example2.com/page.html", + "https://not.example2.com/otherpage.html", + b"https://example2.com/", + ), + ( + "http://example2.com/page.html", + "http://not.example2.com/otherpage.html", + b"http://example2.com/", + ), # exact match required - ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), - + ( + "http://example2.com/page.html", + "http://www.example2.com/otherpage.html", + b"http://example2.com/", + ), # Different port: send origin as referrer - ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), - ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), - + ( + "https://example3.com:444/page.html", + "https://example3.com/not-page.html", + b"https://example3.com:444/", + ), + ( + "http://example3.com:81/page.html", + "http://example3.com/not-page.html", + b"http://example3.com:81/", + ), # downgrade - ('https://example4.com/page.html', 'http://example4.com/not-page.html', None), - ('https://example4.com/page.html', 'http://not.example4.com/', None), - + ("https://example4.com/page.html", "http://example4.com/not-page.html", None), + ("https://example4.com/page.html", "http://not.example4.com/", None), # non-TLS to non-TLS - ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), - + ( + "ftp://example4.com/urls.zip", + "http://example4.com/not-page.html", + b"ftp://example4.com/", + ), # upgrade - ('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'), - ('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'), - + ( + "http://example4.com/page.html", + "https://example4.com/not-page.html", + b"http://example4.com/", + ), + ( + "http://example4.com/page.html", + "https://not.example4.com/", + b"http://example4.com/", + ), # Different protocols: send origin as referrer - ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), - ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), - + ( + "ftps://example4.com/urls.zip", + "https://example4.com/not-page.html", + b"ftps://example4.com/", + ), + ( + "ftps://example4.com/urls.zip", + "https://example4.com/not-page.html", + b"ftps://example4.com/", + ), # test for user/password stripping ( - 'https://user:password@example5.com/page.html', - 'https://example5.com/not-page.html', - b'https://example5.com/page.html', + "https://user:password@example5.com/page.html", + "https://example5.com/not-page.html", + b"https://example5.com/page.html", ), - # TLS to non-TLS downgrade: send nothing - ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None), + ( + "https://user:password@example5.com/page.html", + "http://example5.com/not-page.html", + None, + ), ] class MixinUnsafeUrl: scenarii = [ # TLS to TLS: send referrer - ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), - ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), - ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), - ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), - ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), - ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), - - # TLS to non-TLS: send referrer (yes, it's unsafe) - ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), - ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), - ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), - - # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) - ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), - ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), - ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), - ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), - ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), - ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), - ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), - ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), - - # test for user/password stripping ( - 'http://user:password@example4.com/page.html', - 'https://not.example4.com/', - b'http://example4.com/page.html', + "https://example.com/sekrit.html", + "http://not.example.com/", + b"https://example.com/sekrit.html", ), ( - 'https://user:password@example4.com/page.html', - 'http://scrapy.org/', - b'https://example4.com/page.html', + "https://example1.com/page.html", + "https://not.example1.com/", + b"https://example1.com/page.html", + ), + ( + "https://example1.com/page.html", + "https://scrapy.org/", + b"https://example1.com/page.html", + ), + ( + "https://example1.com:443/page.html", + "https://scrapy.org/", + b"https://example1.com/page.html", + ), + ( + "https://example1.com:444/page.html", + "https://scrapy.org/", + b"https://example1.com:444/page.html", + ), + ( + "ftps://example1.com/urls.zip", + "https://scrapy.org/", + b"ftps://example1.com/urls.zip", + ), + # TLS to non-TLS: send referrer (yes, it's unsafe) + ( + "https://example2.com/page.html", + "http://not.example2.com/", + b"https://example2.com/page.html", + ), + ( + "https://example2.com/page.html", + "http://scrapy.org/", + b"https://example2.com/page.html", + ), + ( + "ftps://example2.com/urls.zip", + "http://scrapy.org/", + b"ftps://example2.com/urls.zip", + ), + # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) + ( + "http://example3.com/page.html", + "https://not.example3.com/", + b"http://example3.com/page.html", + ), + ( + "http://example3.com/page.html", + "https://scrapy.org/", + b"http://example3.com/page.html", + ), + ( + "http://example3.com:8080/page.html", + "https://scrapy.org/", + b"http://example3.com:8080/page.html", + ), + ( + "http://example3.com:80/page.html", + "http://not.example3.com/", + b"http://example3.com/page.html", + ), + ( + "http://example3.com/page.html", + "http://scrapy.org/", + b"http://example3.com/page.html", + ), + ( + "http://example3.com:443/page.html", + "http://scrapy.org/", + b"http://example3.com:443/page.html", + ), + ( + "ftp://example3.com/urls.zip", + "http://scrapy.org/", + b"ftp://example3.com/urls.zip", + ), + ( + "ftp://example3.com/urls.zip", + "https://scrapy.org/", + b"ftp://example3.com/urls.zip", + ), + # test for user/password stripping + ( + "http://user:password@example4.com/page.html", + "https://not.example4.com/", + b"http://example4.com/page.html", + ), + ( + "https://user:password@example4.com/page.html", + "http://scrapy.org/", + b"https://example4.com/page.html", ), ] @@ -340,35 +628,49 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): # --- Tests using settings to set policy using class path class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} + settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerPolicy"} -class TestSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} +class TestSettingsNoReferrerWhenDowngrade( + MixinNoReferrerWhenDowngrade, TestRefererMiddleware +): + settings = { + "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy" + } class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"} class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} + settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginPolicy"} class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} + settings = { + "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.StrictOriginPolicy" + } -class TestSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} +class TestSettingsOriginWhenCrossOrigin( + MixinOriginWhenCrossOrigin, TestRefererMiddleware +): + settings = { + "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy" + } -class TestSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} +class TestSettingsStrictOriginWhenCrossOrigin( + MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware +): + settings = { + "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy" + } class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} class CustomPythonOrgPolicy(ReferrerPolicy): @@ -376,77 +678,95 @@ class CustomPythonOrgPolicy(ReferrerPolicy): A dummy policy that returns referrer as http(s)://python.org depending on the scheme of the target URL. """ + def referrer(self, response, request): scheme = urlparse(request).scheme - if scheme == 'https': - return b'https://python.org/' - if scheme == 'http': - return b'http://python.org/' + if scheme == "https": + return b"https://python.org/" + if scheme == "http": + return b"http://python.org/" class TestSettingsCustomPolicy(TestRefererMiddleware): - settings = {'REFERRER_POLICY': CustomPythonOrgPolicy} + settings = {"REFERRER_POLICY": CustomPythonOrgPolicy} scenarii = [ - ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), - ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), - ('http://example.com/', 'https://scrapy.org/', b'https://python.org/'), - ('https://example.com/', 'http://scrapy.org/', b'http://python.org/'), - ('file:///home/path/to/somefile.html', 'https://scrapy.org/', b'https://python.org/'), - ('file:///home/path/to/somefile.html', 'http://scrapy.org/', b'http://python.org/'), - + ("https://example.com/", "https://scrapy.org/", b"https://python.org/"), + ("http://example.com/", "http://scrapy.org/", b"http://python.org/"), + ("http://example.com/", "https://scrapy.org/", b"https://python.org/"), + ("https://example.com/", "http://scrapy.org/", b"http://python.org/"), + ( + "file:///home/path/to/somefile.html", + "https://scrapy.org/", + b"https://python.org/", + ), + ( + "file:///home/path/to/somefile.html", + "http://scrapy.org/", + b"http://python.org/", + ), ] # --- Tests using Request meta dict to set policy class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware): - req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} + req_meta = {"referrer_policy": POLICY_SCRAPY_DEFAULT} class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware): - req_meta = {'referrer_policy': POLICY_NO_REFERRER} + req_meta = {"referrer_policy": POLICY_NO_REFERRER} -class TestRequestMetaNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): - req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE} +class TestRequestMetaNoReferrerWhenDowngrade( + MixinNoReferrerWhenDowngrade, TestRefererMiddleware +): + req_meta = {"referrer_policy": POLICY_NO_REFERRER_WHEN_DOWNGRADE} class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware): - req_meta = {'referrer_policy': POLICY_SAME_ORIGIN} + req_meta = {"referrer_policy": POLICY_SAME_ORIGIN} class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware): - req_meta = {'referrer_policy': POLICY_ORIGIN} + req_meta = {"referrer_policy": POLICY_ORIGIN} class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware): - req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN} + req_meta = {"referrer_policy": POLICY_STRICT_ORIGIN} -class TestRequestMetaOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): - req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} +class TestRequestMetaOriginWhenCrossOrigin( + MixinOriginWhenCrossOrigin, TestRefererMiddleware +): + req_meta = {"referrer_policy": POLICY_ORIGIN_WHEN_CROSS_ORIGIN} -class TestRequestMetaStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): - req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} +class TestRequestMetaStrictOriginWhenCrossOrigin( + MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware +): + req_meta = {"referrer_policy": POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): - req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + req_meta = {"referrer_policy": POLICY_UNSAFE_URL} class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} - req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"} + req_meta = {"referrer_policy": POLICY_UNSAFE_URL} class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} - req_meta = {'referrer_policy': POLICY_NO_REFERRER} + settings = { + "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy" + } + req_meta = {"referrer_policy": POLICY_NO_REFERRER} class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} - req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + settings = { + "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy" + } + req_meta = {"referrer_policy": POLICY_UNSAFE_URL} class TestRequestMetaSettingFallback(TestCase): @@ -456,57 +776,65 @@ class TestRequestMetaSettingFallback(TestCase): # When an unknown policy is referenced in Request.meta # (here, a typo error), # the policy defined in settings takes precedence - {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}, + { + "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy" + }, {}, - {'referrer_policy': 'ssscrapy-default'}, + {"referrer_policy": "ssscrapy-default"}, OriginWhenCrossOriginPolicy, - True + True, ), ( # same as above but with string value for settings policy - {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {"REFERRER_POLICY": "origin-when-cross-origin"}, {}, - {'referrer_policy': 'ssscrapy-default'}, + {"referrer_policy": "ssscrapy-default"}, OriginWhenCrossOriginPolicy, - True + True, ), ( # request meta references a wrong policy but it is set, # so the Referrer-Policy header in response is not used, # and the settings' policy is applied - {'REFERRER_POLICY': 'origin-when-cross-origin'}, - {'Referrer-Policy': 'unsafe-url'}, - {'referrer_policy': 'ssscrapy-default'}, + {"REFERRER_POLICY": "origin-when-cross-origin"}, + {"Referrer-Policy": "unsafe-url"}, + {"referrer_policy": "ssscrapy-default"}, OriginWhenCrossOriginPolicy, - True + True, ), ( # here, request meta does not set the policy # so response headers take precedence - {'REFERRER_POLICY': 'origin-when-cross-origin'}, - {'Referrer-Policy': 'unsafe-url'}, + {"REFERRER_POLICY": "origin-when-cross-origin"}, + {"Referrer-Policy": "unsafe-url"}, {}, UnsafeUrlPolicy, - False + False, ), ( # here, request meta does not set the policy, # but response headers also use an unknown policy, # so the settings' policy is used - {'REFERRER_POLICY': 'origin-when-cross-origin'}, - {'Referrer-Policy': 'unknown'}, + {"REFERRER_POLICY": "origin-when-cross-origin"}, + {"Referrer-Policy": "unknown"}, {}, OriginWhenCrossOriginPolicy, - True - ) + True, + ), ] def test(self): - origin = 'http://www.scrapy.org' - target = 'http://www.example.com' + origin = "http://www.scrapy.org" + target = "http://www.example.com" - for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]: + for ( + settings, + response_headers, + request_meta, + policy_class, + check_warning, + ) in self.params[3:]: mw = RefererMiddleware(Settings(settings)) response = Response(origin, headers=response_headers) @@ -522,7 +850,6 @@ class TestRequestMetaSettingFallback(TestCase): class TestSettingsPolicyByName(TestCase): - def test_valid_name(self): for s, p in [ (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), @@ -535,7 +862,7 @@ class TestSettingsPolicyByName(TestCase): (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - settings = Settings({'REFERRER_POLICY': s}) + settings = Settings({"REFERRER_POLICY": s}) mw = RefererMiddleware(settings) self.assertEqual(mw.default_policy, p) @@ -551,98 +878,121 @@ class TestSettingsPolicyByName(TestCase): (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - settings = Settings({'REFERRER_POLICY': s.upper()}) + settings = Settings({"REFERRER_POLICY": s.upper()}) mw = RefererMiddleware(settings) self.assertEqual(mw.default_policy, p) def test_invalid_name(self): - settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) + settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"}) with self.assertRaises(RuntimeError): RefererMiddleware(settings) class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} - resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} + settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"} + resp_headers = {"Referrer-Policy": POLICY_UNSAFE_URL.upper()} class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} - resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} + settings = { + "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy" + } + resp_headers = {"Referrer-Policy": POLICY_NO_REFERRER.swapcase()} -class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} - resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} +class TestPolicyHeaderPredecence003( + MixinNoReferrerWhenDowngrade, TestRefererMiddleware +): + settings = { + "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy" + } + resp_headers = {"Referrer-Policy": POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} -class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): +class TestPolicyHeaderPredecence004( + MixinNoReferrerWhenDowngrade, TestRefererMiddleware +): """ The empty string means "no-referrer-when-downgrade" """ - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} - resp_headers = {'Referrer-Policy': ''} + + settings = { + "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy" + } + resp_headers = {"Referrer-Policy": ""} class TestReferrerOnRedirect(TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} scenarii = [ ( - 'http://scrapytest.org/1', # parent - 'http://scrapytest.org/2', # target + "http://scrapytest.org/1", # parent + "http://scrapytest.org/2", # target ( # redirections: code, URL - (301, 'http://scrapytest.org/3'), - (301, 'http://scrapytest.org/4'), + (301, "http://scrapytest.org/3"), + (301, "http://scrapytest.org/4"), ), - b'http://scrapytest.org/1', # expected initial referer - b'http://scrapytest.org/1', # expected referer for the redirection request + b"http://scrapytest.org/1", # expected initial referer + b"http://scrapytest.org/1", # expected referer for the redirection request ), ( - 'https://scrapytest.org/1', - 'https://scrapytest.org/2', + "https://scrapytest.org/1", + "https://scrapytest.org/2", ( # redirecting to non-secure URL - (301, 'http://scrapytest.org/3'), + (301, "http://scrapytest.org/3"), ), - b'https://scrapytest.org/1', - b'https://scrapytest.org/1', + b"https://scrapytest.org/1", + b"https://scrapytest.org/1", ), ( - 'https://scrapytest.org/1', - 'https://scrapytest.com/2', + "https://scrapytest.org/1", + "https://scrapytest.com/2", ( # redirecting to non-secure URL: different origin - (301, 'http://scrapytest.com/3'), + (301, "http://scrapytest.com/3"), ), - b'https://scrapytest.org/1', - b'https://scrapytest.org/1', + b"https://scrapytest.org/1", + b"https://scrapytest.org/1", ), ] def setUp(self): - self.spider = Spider('foo') + self.spider = Spider("foo") settings = Settings(self.settings) self.referrermw = RefererMiddleware(settings) self.redirectmw = RedirectMiddleware(settings) def test(self): - for parent, target, redirections, init_referrer, final_referrer in self.scenarii: + for ( + parent, + target, + redirections, + init_referrer, + final_referrer, + ) in self.scenarii: response = self.get_response(parent) request = self.get_request(target) - out = list(self.referrermw.process_spider_output(response, [request], self.spider)) - self.assertEqual(out[0].headers.get('Referer'), init_referrer) + out = list( + self.referrermw.process_spider_output(response, [request], self.spider) + ) + self.assertEqual(out[0].headers.get("Referer"), init_referrer) for status, url in redirections: - response = Response(request.url, headers={'Location': url}, status=status) - request = self.redirectmw.process_response(request, response, self.spider) + response = Response( + request.url, headers={"Location": url}, status=status + ) + request = self.redirectmw.process_response( + request, response, self.spider + ) self.referrermw.request_scheduled(request, self.spider) assert isinstance(request, Request) - self.assertEqual(request.headers.get('Referer'), final_referrer) + self.assertEqual(request.headers.get("Referer"), final_referrer) class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): @@ -650,34 +1000,31 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): No Referrer policy never sets the "Referer" header. HTTP redirections should not change that. """ - settings = {'REFERRER_POLICY': 'no-referrer'} + + settings = {"REFERRER_POLICY": "no-referrer"} scenarii = [ ( - 'http://scrapytest.org/1', # parent - 'http://scrapytest.org/2', # target + "http://scrapytest.org/1", # parent + "http://scrapytest.org/2", # target ( # redirections: code, URL - (301, 'http://scrapytest.org/3'), - (301, 'http://scrapytest.org/4'), + (301, "http://scrapytest.org/3"), + (301, "http://scrapytest.org/4"), ), None, # expected initial "Referer" None, # expected "Referer" for the redirection request ), ( - 'https://scrapytest.org/1', - 'https://scrapytest.org/2', - ( - (301, 'http://scrapytest.org/3'), - ), + "https://scrapytest.org/1", + "https://scrapytest.org/2", + ((301, "http://scrapytest.org/3"),), None, None, ), ( - 'https://scrapytest.org/1', - 'https://example.com/2', # different origin - ( - (301, 'http://scrapytest.com/3'), - ), + "https://scrapytest.org/1", + "https://example.com/2", # different origin + ((301, "http://scrapytest.com/3"),), None, None, ), @@ -692,37 +1039,38 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect): HTTP redirections to a different domain or a lower secure level should have the "Referer" removed. """ - settings = {'REFERRER_POLICY': 'same-origin'} + + settings = {"REFERRER_POLICY": "same-origin"} scenarii = [ ( - 'http://scrapytest.org/101', # origin - 'http://scrapytest.org/102', # target + "http://scrapytest.org/101", # origin + "http://scrapytest.org/102", # target ( # redirections: code, URL - (301, 'http://scrapytest.org/103'), - (301, 'http://scrapytest.org/104'), + (301, "http://scrapytest.org/103"), + (301, "http://scrapytest.org/104"), ), - b'http://scrapytest.org/101', # expected initial "Referer" - b'http://scrapytest.org/101', # expected referer for the redirection request + b"http://scrapytest.org/101", # expected initial "Referer" + b"http://scrapytest.org/101", # expected referer for the redirection request ), ( - 'https://scrapytest.org/201', - 'https://scrapytest.org/202', + "https://scrapytest.org/201", + "https://scrapytest.org/202", ( # redirecting from secure to non-secure URL == different origin - (301, 'http://scrapytest.org/203'), + (301, "http://scrapytest.org/203"), ), - b'https://scrapytest.org/201', + b"https://scrapytest.org/201", None, ), ( - 'https://scrapytest.org/301', - 'https://scrapytest.org/302', + "https://scrapytest.org/301", + "https://scrapytest.org/302", ( # different domain == different origin - (301, 'http://example.com/303'), + (301, "http://example.com/303"), ), - b'https://scrapytest.org/301', + b"https://scrapytest.org/301", None, ), ] @@ -737,63 +1085,65 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect): Redirections from secure to non-secure URLs should have the "Referrer" header removed if necessary. """ - settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN} + + settings = {"REFERRER_POLICY": POLICY_STRICT_ORIGIN} scenarii = [ ( - 'http://scrapytest.org/101', - 'http://scrapytest.org/102', + "http://scrapytest.org/101", + "http://scrapytest.org/102", ( - (301, 'http://scrapytest.org/103'), - (301, 'http://scrapytest.org/104'), + (301, "http://scrapytest.org/103"), + (301, "http://scrapytest.org/104"), ), - b'http://scrapytest.org/', # send origin - b'http://scrapytest.org/', # redirects to same origin: send origin + b"http://scrapytest.org/", # send origin + b"http://scrapytest.org/", # redirects to same origin: send origin ), ( - 'https://scrapytest.org/201', - 'https://scrapytest.org/202', + "https://scrapytest.org/201", + "https://scrapytest.org/202", ( # redirecting to non-secure URL: no referrer - (301, 'http://scrapytest.org/203'), + (301, "http://scrapytest.org/203"), ), - b'https://scrapytest.org/', + b"https://scrapytest.org/", None, ), ( - 'https://scrapytest.org/301', - 'https://scrapytest.org/302', + "https://scrapytest.org/301", + "https://scrapytest.org/302", ( # redirecting to non-secure URL (different domain): no referrer - (301, 'http://example.com/303'), + (301, "http://example.com/303"), ), - b'https://scrapytest.org/', + b"https://scrapytest.org/", None, ), ( - 'http://scrapy.org/401', - 'http://example.com/402', - ( - (301, 'http://scrapytest.org/403'), - ), - b'http://scrapy.org/', - b'http://scrapy.org/', + "http://scrapy.org/401", + "http://example.com/402", + ((301, "http://scrapytest.org/403"),), + b"http://scrapy.org/", + b"http://scrapy.org/", ), ( - 'https://scrapy.org/501', - 'https://example.com/502', + "https://scrapy.org/501", + "https://example.com/502", ( # HTTPS all along, so origin referrer is kept as-is - (301, 'https://google.com/503'), - (301, 'https://facebook.com/504'), + (301, "https://google.com/503"), + (301, "https://facebook.com/504"), ), - b'https://scrapy.org/', - b'https://scrapy.org/', + b"https://scrapy.org/", + b"https://scrapy.org/", ), ( - 'https://scrapytest.org/601', - 'http://scrapytest.org/602', # TLS to non-TLS: no referrer + "https://scrapytest.org/601", + "http://scrapytest.org/602", # TLS to non-TLS: no referrer ( - (301, 'https://scrapytest.org/603'), # TLS URL again: (still) no referrer + ( + 301, + "https://scrapytest.org/603", + ), # TLS URL again: (still) no referrer ), None, None, @@ -810,67 +1160,64 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect): Redirections to a different origin should strip the "Referer" to the parent origin. """ - settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} + + settings = {"REFERRER_POLICY": POLICY_ORIGIN_WHEN_CROSS_ORIGIN} scenarii = [ ( - 'http://scrapytest.org/101', # origin - 'http://scrapytest.org/102', # target + redirection + "http://scrapytest.org/101", # origin + "http://scrapytest.org/102", # target + redirection ( # redirections: code, URL - (301, 'http://scrapytest.org/103'), - (301, 'http://scrapytest.org/104'), + (301, "http://scrapytest.org/103"), + (301, "http://scrapytest.org/104"), ), - b'http://scrapytest.org/101', # expected initial referer - b'http://scrapytest.org/101', # expected referer for the redirection request + b"http://scrapytest.org/101", # expected initial referer + b"http://scrapytest.org/101", # expected referer for the redirection request ), ( - 'https://scrapytest.org/201', - 'https://scrapytest.org/202', + "https://scrapytest.org/201", + "https://scrapytest.org/202", ( # redirecting to non-secure URL: send origin - (301, 'http://scrapytest.org/203'), + (301, "http://scrapytest.org/203"), ), - b'https://scrapytest.org/201', - b'https://scrapytest.org/', + b"https://scrapytest.org/201", + b"https://scrapytest.org/", ), ( - 'https://scrapytest.org/301', - 'https://scrapytest.org/302', + "https://scrapytest.org/301", + "https://scrapytest.org/302", ( # redirecting to non-secure URL (different domain): send origin - (301, 'http://example.com/303'), + (301, "http://example.com/303"), ), - b'https://scrapytest.org/301', - b'https://scrapytest.org/', + b"https://scrapytest.org/301", + b"https://scrapytest.org/", ), ( - 'http://scrapy.org/401', - 'http://example.com/402', - ( - (301, 'http://scrapytest.org/403'), - ), - b'http://scrapy.org/', - b'http://scrapy.org/', + "http://scrapy.org/401", + "http://example.com/402", + ((301, "http://scrapytest.org/403"),), + b"http://scrapy.org/", + b"http://scrapy.org/", ), ( - 'https://scrapy.org/501', - 'https://example.com/502', + "https://scrapy.org/501", + "https://example.com/502", ( # all different domains: send origin - (301, 'https://google.com/503'), - (301, 'https://facebook.com/504'), + (301, "https://google.com/503"), + (301, "https://facebook.com/504"), ), - b'https://scrapy.org/', - b'https://scrapy.org/', + b"https://scrapy.org/", + b"https://scrapy.org/", ), ( - 'https://scrapytest.org/301', - 'http://scrapytest.org/302', # TLS to non-TLS: send origin - ( - (301, 'https://scrapytest.org/303'), # TLS URL again: send origin (also) - ), - b'https://scrapytest.org/', - b'https://scrapytest.org/', + "https://scrapytest.org/301", + "http://scrapytest.org/302", # TLS to non-TLS: send origin + ((301, "https://scrapytest.org/303"),), # TLS URL again: send origin (also) + b"https://scrapytest.org/", + b"https://scrapytest.org/", ), ] @@ -886,64 +1233,66 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect): Redirections to a different origin should strip the "Referer" to the parent origin, and from https:// to http:// will remove the "Referer" header. """ - settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} + + settings = {"REFERRER_POLICY": POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} scenarii = [ ( - 'http://scrapytest.org/101', # origin - 'http://scrapytest.org/102', # target + redirection + "http://scrapytest.org/101", # origin + "http://scrapytest.org/102", # target + redirection ( # redirections: code, URL - (301, 'http://scrapytest.org/103'), - (301, 'http://scrapytest.org/104'), + (301, "http://scrapytest.org/103"), + (301, "http://scrapytest.org/104"), ), - b'http://scrapytest.org/101', # expected initial referer - b'http://scrapytest.org/101', # expected referer for the redirection request + b"http://scrapytest.org/101", # expected initial referer + b"http://scrapytest.org/101", # expected referer for the redirection request ), ( - 'https://scrapytest.org/201', - 'https://scrapytest.org/202', + "https://scrapytest.org/201", + "https://scrapytest.org/202", ( # redirecting to non-secure URL: do not send the "Referer" header - (301, 'http://scrapytest.org/203'), + (301, "http://scrapytest.org/203"), ), - b'https://scrapytest.org/201', + b"https://scrapytest.org/201", None, ), ( - 'https://scrapytest.org/301', - 'https://scrapytest.org/302', + "https://scrapytest.org/301", + "https://scrapytest.org/302", ( # redirecting to non-secure URL (different domain): send origin - (301, 'http://example.com/303'), + (301, "http://example.com/303"), ), - b'https://scrapytest.org/301', + b"https://scrapytest.org/301", None, ), ( - 'http://scrapy.org/401', - 'http://example.com/402', - ( - (301, 'http://scrapytest.org/403'), - ), - b'http://scrapy.org/', - b'http://scrapy.org/', + "http://scrapy.org/401", + "http://example.com/402", + ((301, "http://scrapytest.org/403"),), + b"http://scrapy.org/", + b"http://scrapy.org/", ), ( - 'https://scrapy.org/501', - 'https://example.com/502', + "https://scrapy.org/501", + "https://example.com/502", ( # all different domains: send origin - (301, 'https://google.com/503'), - (301, 'https://facebook.com/504'), + (301, "https://google.com/503"), + (301, "https://facebook.com/504"), ), - b'https://scrapy.org/', - b'https://scrapy.org/', + b"https://scrapy.org/", + b"https://scrapy.org/", ), ( - 'https://scrapytest.org/601', - 'http://scrapytest.org/602', # TLS to non-TLS: do not send "Referer" + "https://scrapytest.org/601", + "http://scrapytest.org/602", # TLS to non-TLS: do not send "Referer" ( - (301, 'https://scrapytest.org/603'), # TLS URL again: (still) send nothing + ( + 301, + "https://scrapytest.org/603", + ), # TLS URL again: (still) send nothing ), None, None, diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 171f4ddfd..22716bdda 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -10,23 +10,24 @@ from scrapy.settings import Settings class TestUrlLengthMiddleware(TestCase): - def setUp(self): self.maxlength = 25 - settings = Settings({'URLLENGTH_LIMIT': self.maxlength}) + settings = Settings({"URLLENGTH_LIMIT": self.maxlength}) crawler = get_crawler(Spider) - self.spider = crawler._create_spider('foo') + self.spider = crawler._create_spider("foo") self.stats = crawler.stats self.mw = UrlLengthMiddleware.from_settings(settings) - self.response = Response('http://scrapytest.org') - self.short_url_req = Request('http://scrapytest.org/') - self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url') + self.response = Response("http://scrapytest.org") + self.short_url_req = Request("http://scrapytest.org/") + self.long_url_req = Request("http://scrapytest.org/this_is_a_long_url") self.reqs = [self.short_url_req, self.long_url_req] def process_spider_output(self): - return list(self.mw.process_spider_output(self.response, self.reqs, self.spider)) + return list( + self.mw.process_spider_output(self.response, self.reqs, self.spider) + ) def test_middleware_works(self): self.assertEqual(self.process_spider_output(), [self.short_url_req]) @@ -35,7 +36,9 @@ class TestUrlLengthMiddleware(TestCase): with LogCapture() as log: self.process_spider_output() - ric = self.stats.get_value('urllength/request_ignored_count', spider=self.spider) + ric = self.stats.get_value( + "urllength/request_ignored_count", spider=self.spider + ) self.assertEqual(ric, 1) - self.assertIn(f'Ignoring link (url length > {self.maxlength})', str(log)) + self.assertIn(f"Ignoring link (url length > {self.maxlength})", str(log)) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index ab215576e..5c6dccf11 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -10,24 +10,23 @@ from scrapy.utils.test import get_crawler class SpiderStateTest(unittest.TestCase): - def test_store_load(self): jobdir = self.mktemp() Path(jobdir).mkdir() try: - spider = Spider(name='default') + spider = Spider(name="default") dt = datetime.now() ss = SpiderState(jobdir) ss.spider_opened(spider) - spider.state['one'] = 1 - spider.state['dt'] = dt + spider.state["one"] = 1 + spider.state["dt"] = dt ss.spider_closed(spider) - spider2 = Spider(name='default') + spider2 = Spider(name="default") ss2 = SpiderState(jobdir) ss2.spider_opened(spider2) - self.assertEqual(spider.state, {'one': 1, 'dt': dt}) + self.assertEqual(spider.state, {"one": 1, "dt": dt}) ss2.spider_closed(spider2) finally: shutil.rmtree(jobdir) @@ -35,7 +34,7 @@ class SpiderStateTest(unittest.TestCase): def test_state_attribute(self): # state attribute must be present if jobdir is not set, to provide a # consistent interface - spider = Spider(name='default') + spider = Spider(name="default") ss = SpiderState() ss.spider_opened(spider) self.assertEqual(spider.state, {}) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index acc821b83..0e2441f90 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -31,20 +31,19 @@ def nonserializable_object_test(self): q = self.queue() self.assertRaises(ValueError, q.push, lambda x: x) # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) - sel = Selector(text='

some text

') + sel = Selector(text="

some text

") self.assertRaises(ValueError, q.push, sel) class FifoDiskQueueTestMixin: - def test_serialize(self): q = self.queue() - q.push('a') + q.push("a") q.push(123) - q.push({'a': 'dict'}) - self.assertEqual(q.pop(), 'a') + q.push({"a": "dict"}) + self.assertEqual(q.pop(), "a") self.assertEqual(q.pop(), 123) - self.assertEqual(q.pop(), {'a': 'dict'}) + self.assertEqual(q.pop(), {"a": "dict"}) test_nonserializable_object = nonserializable_object_test @@ -81,7 +80,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): def test_serialize_item(self): q = self.queue() - i = TestItem(name='foo') + i = TestItem(name="foo") q.push(i) i2 = q.pop() assert isinstance(i2, TestItem) @@ -94,17 +93,17 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): loader2 = q.pop() assert isinstance(loader2, TestLoader) assert loader2.default_item_class is TestItem - self.assertEqual(loader2.name_out('x'), 'xx') + self.assertEqual(loader2.name_out("x"), "xx") def test_serialize_request_recursive(self): q = self.queue() - r = Request('http://www.example.com') - r.meta['request'] = r + r = Request("http://www.example.com") + r.meta["request"] = r q.push(r) r2 = q.pop() assert isinstance(r2, Request) self.assertEqual(r.url, r2.url) - assert r2.meta['request'] is r2 + assert r2.meta["request"] is r2 def test_non_pickable_object(self): q = self.queue() @@ -115,7 +114,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): self.assertIsInstance(exc.__context__, pickle.PicklingError) else: self.assertIsInstance(exc.__context__, AttributeError) - sel = Selector(text='

some text

') + sel = Selector(text="

some text

") try: q.push(sel) except ValueError as exc: @@ -139,33 +138,30 @@ class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): class LifoDiskQueueTestMixin: - def test_serialize(self): q = self.queue() - q.push('a') + q.push("a") q.push(123) - q.push({'a': 'dict'}) - self.assertEqual(q.pop(), {'a': 'dict'}) + q.push({"a": "dict"}) + self.assertEqual(q.pop(), {"a": "dict"}) self.assertEqual(q.pop(), 123) - self.assertEqual(q.pop(), 'a') + self.assertEqual(q.pop(), "a") test_nonserializable_object = nonserializable_object_test class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): - def queue(self): return _MarshalLifoSerializationDiskQueue(self.qpath) class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): - def queue(self): return _PickleLifoSerializationDiskQueue(self.qpath) def test_serialize_item(self): q = self.queue() - i = TestItem(name='foo') + i = TestItem(name="foo") q.push(i) i2 = q.pop() assert isinstance(i2, TestItem) @@ -178,14 +174,14 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): loader2 = q.pop() assert isinstance(loader2, TestLoader) assert loader2.default_item_class is TestItem - self.assertEqual(loader2.name_out('x'), 'xx') + self.assertEqual(loader2.name_out("x"), "xx") def test_serialize_request_recursive(self): q = self.queue() - r = Request('http://www.example.com') - r.meta['request'] = r + r = Request("http://www.example.com") + r.meta["request"] = r q.push(r) r2 = q.pop() assert isinstance(r2, Request) self.assertEqual(r.url, r2.url) - assert r2.meta['request'] is r2 + assert r2.meta["request"] is r2 diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 22dafb2d2..5d9001bb0 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -69,7 +69,10 @@ class RequestQueueTestMixin: req = Request("http://www.example.com") q.push(req) self.assertEqual(len(q), 1) - with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + with self.assertRaises( + NotImplementedError, + msg="The underlying queue class does not implement 'peek'", + ): q.peek() self.assertEqual(q.pop().url, req.url) self.assertEqual(len(q), 0) @@ -117,7 +120,10 @@ class FifoQueueMixin(RequestQueueTestMixin): q.push(req1) q.push(req2) q.push(req3) - with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + with self.assertRaises( + NotImplementedError, + msg="The underlying queue class does not implement 'peek'", + ): q.peek() self.assertEqual(len(q), 3) self.assertEqual(q.pop().url, req1.url) @@ -170,7 +176,10 @@ class LifoQueueMixin(RequestQueueTestMixin): q.push(req1) q.push(req2) q.push(req3) - with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + with self.assertRaises( + NotImplementedError, + msg="The underlying queue class does not implement 'peek'", + ): q.peek() self.assertEqual(len(q), 3) self.assertEqual(q.pop().url, req3.url) @@ -195,12 +204,16 @@ class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): def queue(self): - return MarshalFifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/fifo") + return MarshalFifoDiskQueue.from_crawler( + crawler=self.crawler, key="marshal/fifo" + ) class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): def queue(self): - return MarshalLifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/lifo") + return MarshalLifoDiskQueue.from_crawler( + crawler=self.crawler, key="marshal/lifo" + ) class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): diff --git a/tests/test_stats.py b/tests/test_stats.py index 2bbbb9e2c..2ee04429a 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -9,12 +9,11 @@ from scrapy.utils.test import get_crawler class CoreStatsExtensionTest(unittest.TestCase): - def setUp(self): self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider('foo') + self.spider = self.crawler._create_spider("foo") - @mock.patch('scrapy.extensions.corestats.datetime') + @mock.patch("scrapy.extensions.corestats.datetime") def test_core_stats_default_stats_collector(self, mock_datetime): fixed_datetime = datetime(2019, 12, 1, 11, 38) mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime) @@ -24,19 +23,19 @@ class CoreStatsExtensionTest(unittest.TestCase): ext.item_scraped({}, self.spider) ext.response_received(self.spider) ext.item_dropped({}, self.spider, ZeroDivisionError()) - ext.spider_closed(self.spider, 'finished') + ext.spider_closed(self.spider, "finished") self.assertEqual( ext.stats._stats, { - 'start_time': fixed_datetime, - 'finish_time': fixed_datetime, - 'item_scraped_count': 1, - 'response_received_count': 1, - 'item_dropped_count': 1, - 'item_dropped_reasons_count/ZeroDivisionError': 1, - 'finish_reason': 'finished', - 'elapsed_time_seconds': 0.0, - } + "start_time": fixed_datetime, + "finish_time": fixed_datetime, + "item_scraped_count": 1, + "response_received_count": 1, + "item_dropped_count": 1, + "item_dropped_reasons_count/ZeroDivisionError": 1, + "finish_reason": "finished", + "elapsed_time_seconds": 0.0, + }, ) def test_core_stats_dummy_stats_collector(self): @@ -46,53 +45,52 @@ class CoreStatsExtensionTest(unittest.TestCase): ext.item_scraped({}, self.spider) ext.response_received(self.spider) ext.item_dropped({}, self.spider, ZeroDivisionError()) - ext.spider_closed(self.spider, 'finished') + ext.spider_closed(self.spider, "finished") self.assertEqual(ext.stats._stats, {}) class StatsCollectorTest(unittest.TestCase): - def setUp(self): self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider('foo') + self.spider = self.crawler._create_spider("foo") def test_collector(self): stats = StatsCollector(self.crawler) self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_value('anything'), None) - self.assertEqual(stats.get_value('anything', 'default'), 'default') - stats.set_value('test', 'value') - self.assertEqual(stats.get_stats(), {'test': 'value'}) - stats.set_value('test2', 23) - self.assertEqual(stats.get_stats(), {'test': 'value', 'test2': 23}) - self.assertEqual(stats.get_value('test2'), 23) - stats.inc_value('test2') - self.assertEqual(stats.get_value('test2'), 24) - stats.inc_value('test2', 6) - self.assertEqual(stats.get_value('test2'), 30) - stats.max_value('test2', 6) - self.assertEqual(stats.get_value('test2'), 30) - stats.max_value('test2', 40) - self.assertEqual(stats.get_value('test2'), 40) - stats.max_value('test3', 1) - self.assertEqual(stats.get_value('test3'), 1) - stats.min_value('test2', 60) - self.assertEqual(stats.get_value('test2'), 40) - stats.min_value('test2', 35) - self.assertEqual(stats.get_value('test2'), 35) - stats.min_value('test4', 7) - self.assertEqual(stats.get_value('test4'), 7) + self.assertEqual(stats.get_value("anything"), None) + self.assertEqual(stats.get_value("anything", "default"), "default") + stats.set_value("test", "value") + self.assertEqual(stats.get_stats(), {"test": "value"}) + stats.set_value("test2", 23) + self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23}) + self.assertEqual(stats.get_value("test2"), 23) + stats.inc_value("test2") + self.assertEqual(stats.get_value("test2"), 24) + stats.inc_value("test2", 6) + self.assertEqual(stats.get_value("test2"), 30) + stats.max_value("test2", 6) + self.assertEqual(stats.get_value("test2"), 30) + stats.max_value("test2", 40) + self.assertEqual(stats.get_value("test2"), 40) + stats.max_value("test3", 1) + self.assertEqual(stats.get_value("test3"), 1) + stats.min_value("test2", 60) + self.assertEqual(stats.get_value("test2"), 40) + stats.min_value("test2", 35) + self.assertEqual(stats.get_value("test2"), 35) + stats.min_value("test4", 7) + self.assertEqual(stats.get_value("test4"), 7) def test_dummy_collector(self): stats = DummyStatsCollector(self.crawler) self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_value('anything'), None) - self.assertEqual(stats.get_value('anything', 'default'), 'default') - stats.set_value('test', 'value') - stats.inc_value('v1') - stats.max_value('v2', 100) - stats.min_value('v3', 100) - stats.open_spider('a') - stats.set_value('test', 'value', spider=self.spider) + self.assertEqual(stats.get_value("anything"), None) + self.assertEqual(stats.get_value("anything", "default"), "default") + stats.set_value("test", "value") + stats.inc_value("v1") + stats.max_value("v2", 100) + stats.min_value("v3", 100) + stats.open_spider("a") + stats.set_value("test", "value", spider=self.spider) self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_stats('a'), {}) + self.assertEqual(stats.get_stats("a"), {}) diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index fdc5df166..9a4eeb04e 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -4,7 +4,6 @@ import scrapy class ToplevelTestCase(TestCase): - def test_version(self): self.assertIs(type(scrapy.__version__), str) @@ -13,18 +12,22 @@ class ToplevelTestCase(TestCase): def test_request_shortcut(self): from scrapy.http import Request, FormRequest + self.assertIs(scrapy.Request, Request) self.assertIs(scrapy.FormRequest, FormRequest) def test_spider_shortcut(self): from scrapy.spiders import Spider + self.assertIs(scrapy.Spider, Spider) def test_selector_shortcut(self): from scrapy.selector import Selector + self.assertIs(scrapy.Selector, Selector) def test_item_shortcut(self): from scrapy.item import Item, Field + self.assertIs(scrapy.Item, Item) self.assertIs(scrapy.Field, Field) diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py index bea0cf3e5..3b6428686 100644 --- a/tests/test_urlparse_monkeypatches.py +++ b/tests/test_urlparse_monkeypatches.py @@ -3,10 +3,9 @@ import unittest class UrlparseTestCase(unittest.TestCase): - def test_s3_url(self): - p = urlparse('s3://bucket/key/name?param=value') - self.assertEqual(p.scheme, 's3') - self.assertEqual(p.hostname, 'bucket') - self.assertEqual(p.path, '/key/name') - self.assertEqual(p.query, 'param=value') + p = urlparse("s3://bucket/key/name?param=value") + self.assertEqual(p.scheme, "s3") + self.assertEqual(p.hostname, "bucket") + self.assertEqual(p.path, "/key/name") + self.assertEqual(p.query, "param=value") diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 741c6a505..25cef2b26 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -6,12 +6,13 @@ from pytest import mark from scrapy.utils.reactor import is_asyncio_reactor_installed, install_reactor -@mark.usefixtures('reactor_pytest') +@mark.usefixtures("reactor_pytest") class AsyncioTest(TestCase): - def test_is_asyncio_reactor_installed(self): # the result should depend only on the pytest --reactor argument - self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') + self.assertEqual( + is_asyncio_reactor_installed(), self.reactor_pytest == "asyncio" + ) def test_install_asyncio_reactor(self): with warnings.catch_warnings(record=True) as w: diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index a92880626..61a683318 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -7,200 +7,242 @@ from scrapy.utils.conf import ( arglist_to_dict, build_component_list, feed_complete_default_values_from_settings, - feed_process_params_from_cli + feed_process_params_from_cli, ) class BuildComponentListTest(unittest.TestCase): - def test_build_dict(self): - d = {'one': 1, 'two': None, 'three': 8, 'four': 4} - self.assertEqual(build_component_list(d, convert=lambda x: x), - ['one', 'four', 'three']) + d = {"one": 1, "two": None, "three": 8, "four": 4} + self.assertEqual( + build_component_list(d, convert=lambda x: x), ["one", "four", "three"] + ) def test_backward_compatible_build_dict(self): - base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None} - custom = {'two': None, 'three': 8, 'four': 4} - self.assertEqual(build_component_list(base, custom, - convert=lambda x: x), - ['one', 'four', 'five', 'three']) + base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None} + custom = {"two": None, "three": 8, "four": 4} + self.assertEqual( + build_component_list(base, custom, convert=lambda x: x), + ["one", "four", "five", "three"], + ) def test_return_list(self): - custom = ['a', 'b', 'c'] - self.assertEqual(build_component_list(None, custom, - convert=lambda x: x), - custom) + custom = ["a", "b", "c"] + self.assertEqual( + build_component_list(None, custom, convert=lambda x: x), custom + ) def test_map_dict(self): - custom = {'one': 1, 'two': 2, 'three': 3} - self.assertEqual(build_component_list({}, custom, - convert=lambda x: x.upper()), - ['ONE', 'TWO', 'THREE']) + custom = {"one": 1, "two": 2, "three": 3} + self.assertEqual( + build_component_list({}, custom, convert=lambda x: x.upper()), + ["ONE", "TWO", "THREE"], + ) def test_map_list(self): - custom = ['a', 'b', 'c'] - self.assertEqual(build_component_list(None, custom, - lambda x: x.upper()), - ['A', 'B', 'C']) + custom = ["a", "b", "c"] + self.assertEqual( + build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"] + ) def test_duplicate_components_in_dict(self): - duplicate_dict = {'one': 1, 'two': 2, 'ONE': 4} - self.assertRaises(ValueError, build_component_list, {}, duplicate_dict, - convert=lambda x: x.lower()) + duplicate_dict = {"one": 1, "two": 2, "ONE": 4} + self.assertRaises( + ValueError, + build_component_list, + {}, + duplicate_dict, + convert=lambda x: x.lower(), + ) def test_duplicate_components_in_list(self): - duplicate_list = ['a', 'b', 'a'] + duplicate_list = ["a", "b", "a"] with self.assertRaises(ValueError) as cm: build_component_list(None, duplicate_list, convert=lambda x: x) self.assertIn(str(duplicate_list), str(cm.exception)) def test_duplicate_components_in_basesettings(self): # Higher priority takes precedence - duplicate_bs = BaseSettings({'one': 1, 'two': 2}, priority=0) - duplicate_bs.set('ONE', 4, priority=10) - self.assertEqual(build_component_list(duplicate_bs, - convert=lambda x: x.lower()), - ['two', 'one']) - duplicate_bs.set('one', duplicate_bs['one'], priority=20) - self.assertEqual(build_component_list(duplicate_bs, - convert=lambda x: x.lower()), - ['one', 'two']) + duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0) + duplicate_bs.set("ONE", 4, priority=10) + self.assertEqual( + build_component_list(duplicate_bs, convert=lambda x: x.lower()), + ["two", "one"], + ) + duplicate_bs.set("one", duplicate_bs["one"], priority=20) + self.assertEqual( + build_component_list(duplicate_bs, convert=lambda x: x.lower()), + ["one", "two"], + ) # Same priority raises ValueError - duplicate_bs.set('ONE', duplicate_bs['ONE'], priority=20) - self.assertRaises(ValueError, build_component_list, duplicate_bs, - convert=lambda x: x.lower()) + duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20) + self.assertRaises( + ValueError, build_component_list, duplicate_bs, convert=lambda x: x.lower() + ) def test_valid_numbers(self): # work well with None and numeric values - d = {'a': 10, 'b': None, 'c': 15, 'd': 5.0} - self.assertEqual(build_component_list(d, convert=lambda x: x), - ['d', 'a', 'c']) - d = {'a': 33333333333333333333, 'b': 11111111111111111111, 'c': 22222222222222222222} - self.assertEqual(build_component_list(d, convert=lambda x: x), - ['b', 'c', 'a']) + d = {"a": 10, "b": None, "c": 15, "d": 5.0} + self.assertEqual(build_component_list(d, convert=lambda x: x), ["d", "a", "c"]) + d = { + "a": 33333333333333333333, + "b": 11111111111111111111, + "c": 22222222222222222222, + } + self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"]) # raise exception for invalid values - d = {'one': '5'} + d = {"one": "5"} self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {'one': '1.0'} + d = {"one": "1.0"} self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {'one': [1, 2, 3]} + d = {"one": [1, 2, 3]} self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {'one': {'a': 'a', 'b': 2}} + d = {"one": {"a": "a", "b": 2}} self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {'one': 'lorem ipsum'} + d = {"one": "lorem ipsum"} self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) class UtilsConfTestCase(unittest.TestCase): - def test_arglist_to_dict(self): self.assertEqual( - arglist_to_dict(['arg1=val1', 'arg2=val2']), - {'arg1': 'val1', 'arg2': 'val2'}) + arglist_to_dict(["arg1=val1", "arg2=val2"]), + {"arg1": "val1", "arg2": "val2"}, + ) class FeedExportConfigTestCase(unittest.TestCase): - def test_feed_export_config_invalid_format(self): settings = Settings() - self.assertRaises(UsageError, feed_process_params_from_cli, settings, ['items.dat'], 'noformat') + self.assertRaises( + UsageError, + feed_process_params_from_cli, + settings, + ["items.dat"], + "noformat", + ) def test_feed_export_config_mismatch(self): settings = Settings() self.assertRaises( UsageError, - feed_process_params_from_cli, settings, ['items1.dat', 'items2.dat'], 'noformat' + feed_process_params_from_cli, + settings, + ["items1.dat", "items2.dat"], + "noformat", ) def test_feed_export_config_backward_compatible(self): with warnings.catch_warnings(record=True) as cw: settings = Settings() self.assertEqual( - {'items.dat': {'format': 'csv'}}, - feed_process_params_from_cli(settings, ['items.dat'], 'csv') + {"items.dat": {"format": "csv"}}, + feed_process_params_from_cli(settings, ["items.dat"], "csv"), ) self.assertEqual(cw[0].category, ScrapyDeprecationWarning) def test_feed_export_config_explicit_formats(self): settings = Settings() self.assertEqual( - {'items_1.dat': {'format': 'json'}, 'items_2.dat': {'format': 'xml'}, 'items_3.dat': {'format': 'csv'}}, - feed_process_params_from_cli(settings, ['items_1.dat:json', 'items_2.dat:xml', 'items_3.dat:csv']) + { + "items_1.dat": {"format": "json"}, + "items_2.dat": {"format": "xml"}, + "items_3.dat": {"format": "csv"}, + }, + feed_process_params_from_cli( + settings, ["items_1.dat:json", "items_2.dat:xml", "items_3.dat:csv"] + ), ) def test_feed_export_config_implicit_formats(self): settings = Settings() self.assertEqual( - {'items_1.json': {'format': 'json'}, 'items_2.xml': {'format': 'xml'}, 'items_3.csv': {'format': 'csv'}}, - feed_process_params_from_cli(settings, ['items_1.json', 'items_2.xml', 'items_3.csv']) + { + "items_1.json": {"format": "json"}, + "items_2.xml": {"format": "xml"}, + "items_3.csv": {"format": "csv"}, + }, + feed_process_params_from_cli( + settings, ["items_1.json", "items_2.xml", "items_3.csv"] + ), ) def test_feed_export_config_stdout(self): settings = Settings() self.assertEqual( - {'stdout:': {'format': 'pickle'}}, - feed_process_params_from_cli(settings, ['-:pickle']) + {"stdout:": {"format": "pickle"}}, + feed_process_params_from_cli(settings, ["-:pickle"]), ) def test_feed_export_config_overwrite(self): settings = Settings() self.assertEqual( - {'output.json': {'format': 'json', 'overwrite': True}}, - feed_process_params_from_cli(settings, [], None, ['output.json']) + {"output.json": {"format": "json", "overwrite": True}}, + feed_process_params_from_cli(settings, [], None, ["output.json"]), ) def test_output_and_overwrite_output(self): with self.assertRaises(UsageError): feed_process_params_from_cli( Settings(), - ['output1.json'], + ["output1.json"], None, - ['output2.json'], + ["output2.json"], ) def test_feed_complete_default_values_from_settings_empty(self): feed = {} - settings = Settings({ - "FEED_EXPORT_ENCODING": "custom encoding", - "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], - "FEED_EXPORT_INDENT": 42, - "FEED_STORE_EMPTY": True, - "FEED_URI_PARAMS": (1, 2, 3, 4), - "FEED_EXPORT_BATCH_ITEM_COUNT": 2, - }) + settings = Settings( + { + "FEED_EXPORT_ENCODING": "custom encoding", + "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], + "FEED_EXPORT_INDENT": 42, + "FEED_STORE_EMPTY": True, + "FEED_URI_PARAMS": (1, 2, 3, 4), + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, + } + ) new_feed = feed_complete_default_values_from_settings(feed, settings) - self.assertEqual(new_feed, { - "encoding": "custom encoding", - "fields": ["f1", "f2", "f3"], - "indent": 42, - "store_empty": True, - "uri_params": (1, 2, 3, 4), - "batch_item_count": 2, - "item_export_kwargs": {}, - }) + self.assertEqual( + new_feed, + { + "encoding": "custom encoding", + "fields": ["f1", "f2", "f3"], + "indent": 42, + "store_empty": True, + "uri_params": (1, 2, 3, 4), + "batch_item_count": 2, + "item_export_kwargs": {}, + }, + ) def test_feed_complete_default_values_from_settings_non_empty(self): feed = { "encoding": "other encoding", "fields": None, } - settings = Settings({ - "FEED_EXPORT_ENCODING": "custom encoding", - "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], - "FEED_EXPORT_INDENT": 42, - "FEED_STORE_EMPTY": True, - "FEED_EXPORT_BATCH_ITEM_COUNT": 2, - }) + settings = Settings( + { + "FEED_EXPORT_ENCODING": "custom encoding", + "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], + "FEED_EXPORT_INDENT": 42, + "FEED_STORE_EMPTY": True, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, + } + ) new_feed = feed_complete_default_values_from_settings(feed, settings) - self.assertEqual(new_feed, { - "encoding": "other encoding", - "fields": None, - "indent": 42, - "store_empty": True, - "uri_params": None, - "batch_item_count": 2, - "item_export_kwargs": {}, - }) + self.assertEqual( + new_feed, + { + "encoding": "other encoding", + "fields": None, + "indent": 42, + "store_empty": True, + "uri_params": None, + "batch_item_count": 2, + "item_export_kwargs": {}, + }, + ) if __name__ == "__main__": diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index 380c41367..18f63b540 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -1,14 +1,17 @@ import unittest from scrapy.utils.console import get_shell_embed_func + try: import bpython + bpy = True del bpython except ImportError: bpy = False try: import IPython + ipy = True del IPython except ImportError: @@ -16,29 +19,28 @@ except ImportError: class UtilsConsoleTestCase(unittest.TestCase): - def test_get_shell_embed_func(self): - shell = get_shell_embed_func(['invalid']) + shell = get_shell_embed_func(["invalid"]) self.assertEqual(shell, None) - shell = get_shell_embed_func(['invalid', 'python']) + shell = get_shell_embed_func(["invalid", "python"]) self.assertTrue(callable(shell)) - self.assertEqual(shell.__name__, '_embed_standard_shell') + self.assertEqual(shell.__name__, "_embed_standard_shell") - @unittest.skipIf(not bpy, 'bpython not available in testenv') + @unittest.skipIf(not bpy, "bpython not available in testenv") def test_get_shell_embed_func2(self): - shell = get_shell_embed_func(['bpython']) + shell = get_shell_embed_func(["bpython"]) self.assertTrue(callable(shell)) - self.assertEqual(shell.__name__, '_embed_bpython_shell') + self.assertEqual(shell.__name__, "_embed_bpython_shell") - @unittest.skipIf(not ipy, 'IPython not available in testenv') + @unittest.skipIf(not ipy, "IPython not available in testenv") def test_get_shell_embed_func3(self): # default shell should be 'ipython' shell = get_shell_embed_func() - self.assertEqual(shell.__name__, '_embed_ipython_shell') + self.assertEqual(shell.__name__, "_embed_ipython_shell") if __name__ == "__main__": diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index f5d684d3f..fd4612eba 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -34,10 +34,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): "method": "GET", "url": "https://api.test.com/", "headers": [ - ( - "Authorization", - basic_auth_header("some_username", "some_password") - ) + ("Authorization", basic_auth_header("some_username", "some_password")) ], } self._test_command(curl_command, expected_result) @@ -77,11 +74,11 @@ class CurlToRequestKwargsTest(unittest.TestCase): ("Connection", "keep-alive"), ], "cookies": { - '_gauges_unique_year': '1', - '_gauges_unique_hour': '1', - '_gauges_unique_day': '1', - '_gauges_unique': '1', - '_gauges_unique_month': '1' + "_gauges_unique_year": "1", + "_gauges_unique_hour": "1", + "_gauges_unique_day": "1", + "_gauges_unique": "1", + "_gauges_unique_month": "1", }, } self._test_command(curl_command, expected_result) @@ -107,14 +104,14 @@ class CurlToRequestKwargsTest(unittest.TestCase): "method": "POST", "url": "http://httpbin.org/post", "body": "custname=John+Smith&custtel=500&custemail=jsmith%40exampl" - "e.org&size=small&topping=cheese&topping=onion&delivery=12" - "%3A15&comments=", + "e.org&size=small&topping=cheese&topping=onion&delivery=12" + "%3A15&comments=", "cookies": { - '_gauges_unique_year': '1', - '_gauges_unique_hour': '1', - '_gauges_unique_day': '1', - '_gauges_unique': '1', - '_gauges_unique_month': '1' + "_gauges_unique_year": "1", + "_gauges_unique_hour": "1", + "_gauges_unique_day": "1", + "_gauges_unique": "1", + "_gauges_unique_month": "1", }, "headers": [ ("Origin", "http://httpbin.org"), @@ -152,16 +149,17 @@ class CurlToRequestKwargsTest(unittest.TestCase): "body": ( "excerptLength=200&enableDidYouMean=true&sortCriteria=ffirstz3" "2xnamez32x201740686%20ascending&queryFunctions=%5B%5D&ranking" - "Functions=%5B%5D") + "Functions=%5B%5D" + ), } self._test_command(curl_command, expected_result) def test_explicit_get_with_data(self): - curl_command = 'curl httpbin.org/anything -X GET --data asdf' + curl_command = "curl httpbin.org/anything -X GET --data asdf" expected_result = { "method": "GET", "url": "http://httpbin.org/anything", - "body": "asdf" + "body": "asdf", } self._test_command(curl_command, expected_result) @@ -182,16 +180,14 @@ class CurlToRequestKwargsTest(unittest.TestCase): ("Authorization", basic_auth_header("username", "password")), ], "body": '{"hostname": "agent02.example.com", "agent_config_state"' - ': "Enabled", "resources": ["Java","Linux"], "environments' - '": ["Dev"]}', + ': "Enabled", "resources": ["Java","Linux"], "environments' + '": ["Dev"]}', } self._test_command(curl_command, expected_result) def test_delete(self): curl_command = 'curl -X "DELETE" https://www.url.com/page' - expected_result = { - "method": "DELETE", "url": "https://www.url.com/page" - } + expected_result = {"method": "DELETE", "url": "https://www.url.com/page"} self._test_command(curl_command, expected_result) def test_get_silent(self): @@ -209,8 +205,8 @@ class CurlToRequestKwargsTest(unittest.TestCase): def test_ignore_unknown_options(self): # case 1: ignore_unknown_options=True: with warnings.catch_warnings(): # avoid warning when executing tests - warnings.simplefilter('ignore') - curl_command = 'curl --bar --baz http://www.example.com' + warnings.simplefilter("ignore") + curl_command = "curl --bar --baz http://www.example.com" expected_result = {"method": "GET", "url": "http://www.example.com"} self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) @@ -219,13 +215,12 @@ class CurlToRequestKwargsTest(unittest.TestCase): ValueError, "Unrecognized options:.*--bar.*--baz", lambda: curl_to_request_kwargs( - "curl --bar --baz http://www.example.com", - ignore_unknown_options=False + "curl --bar --baz http://www.example.com", ignore_unknown_options=False ), ) def test_must_start_with_curl_error(self): self.assertRaises( ValueError, - lambda: curl_to_request_kwargs("carl -X POST http://example.org") + lambda: curl_to_request_kwargs("carl -X POST http://example.org"), ) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index e4bccf30e..0c86c7e7a 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -3,26 +3,30 @@ import unittest from collections.abc import Mapping, MutableMapping from scrapy.http import Request -from scrapy.utils.datatypes import CaselessDict, LocalCache, LocalWeakReferencedCache, SequenceExclude +from scrapy.utils.datatypes import ( + CaselessDict, + LocalCache, + LocalWeakReferencedCache, + SequenceExclude, +) from scrapy.utils.python import garbage_collect -__doctests__ = ['scrapy.utils.datatypes'] +__doctests__ = ["scrapy.utils.datatypes"] class CaselessDictTest(unittest.TestCase): - def test_init_dict(self): - seq = {'red': 1, 'black': 3} + seq = {"red": 1, "black": 3} d = CaselessDict(seq) - self.assertEqual(d['red'], 1) - self.assertEqual(d['black'], 3) + self.assertEqual(d["red"], 1) + self.assertEqual(d["black"], 3) def test_init_pair_sequence(self): - seq = (('red', 1), ('black', 3)) + seq = (("red", 1), ("black", 3)) d = CaselessDict(seq) - self.assertEqual(d['red'], 1) - self.assertEqual(d['black'], 3) + self.assertEqual(d["red"], 1) + self.assertEqual(d["black"], 3) def test_init_mapping(self): class MyMapping(Mapping): @@ -40,8 +44,8 @@ class CaselessDictTest(unittest.TestCase): seq = MyMapping(red=1, black=3) d = CaselessDict(seq) - self.assertEqual(d['red'], 1) - self.assertEqual(d['black'], 3) + self.assertEqual(d["red"], 1) + self.assertEqual(d["black"], 3) def test_init_mutable_mapping(self): class MyMutableMapping(MutableMapping): @@ -65,72 +69,72 @@ class CaselessDictTest(unittest.TestCase): seq = MyMutableMapping(red=1, black=3) d = CaselessDict(seq) - self.assertEqual(d['red'], 1) - self.assertEqual(d['black'], 3) + self.assertEqual(d["red"], 1) + self.assertEqual(d["black"], 3) def test_caseless(self): d = CaselessDict() - d['key_Lower'] = 1 - self.assertEqual(d['KEy_loWer'], 1) - self.assertEqual(d.get('KEy_loWer'), 1) + d["key_Lower"] = 1 + self.assertEqual(d["KEy_loWer"], 1) + self.assertEqual(d.get("KEy_loWer"), 1) - d['KEY_LOWER'] = 3 - self.assertEqual(d['key_Lower'], 3) - self.assertEqual(d.get('key_Lower'), 3) + d["KEY_LOWER"] = 3 + self.assertEqual(d["key_Lower"], 3) + self.assertEqual(d.get("key_Lower"), 3) def test_delete(self): - d = CaselessDict({'key_lower': 1}) - del d['key_LOWER'] - self.assertRaises(KeyError, d.__getitem__, 'key_LOWER') - self.assertRaises(KeyError, d.__getitem__, 'key_lower') + d = CaselessDict({"key_lower": 1}) + del d["key_LOWER"] + self.assertRaises(KeyError, d.__getitem__, "key_LOWER") + self.assertRaises(KeyError, d.__getitem__, "key_lower") def test_getdefault(self): d = CaselessDict() - self.assertEqual(d.get('c', 5), 5) - d['c'] = 10 - self.assertEqual(d.get('c', 5), 10) + self.assertEqual(d.get("c", 5), 5) + d["c"] = 10 + self.assertEqual(d.get("c", 5), 10) def test_setdefault(self): - d = CaselessDict({'a': 1, 'b': 2}) + d = CaselessDict({"a": 1, "b": 2}) - r = d.setdefault('A', 5) + r = d.setdefault("A", 5) self.assertEqual(r, 1) - self.assertEqual(d['A'], 1) + self.assertEqual(d["A"], 1) - r = d.setdefault('c', 5) + r = d.setdefault("c", 5) self.assertEqual(r, 5) - self.assertEqual(d['C'], 5) + self.assertEqual(d["C"], 5) def test_fromkeys(self): - keys = ('a', 'b') + keys = ("a", "b") d = CaselessDict.fromkeys(keys) - self.assertEqual(d['A'], None) - self.assertEqual(d['B'], None) + self.assertEqual(d["A"], None) + self.assertEqual(d["B"], None) d = CaselessDict.fromkeys(keys, 1) - self.assertEqual(d['A'], 1) - self.assertEqual(d['B'], 1) + self.assertEqual(d["A"], 1) + self.assertEqual(d["B"], 1) instance = CaselessDict() d = instance.fromkeys(keys) - self.assertEqual(d['A'], None) - self.assertEqual(d['B'], None) + self.assertEqual(d["A"], None) + self.assertEqual(d["B"], None) d = instance.fromkeys(keys, 1) - self.assertEqual(d['A'], 1) - self.assertEqual(d['B'], 1) + self.assertEqual(d["A"], 1) + self.assertEqual(d["B"], 1) def test_contains(self): d = CaselessDict() - d['a'] = 1 - assert 'a' in d + d["a"] = 1 + assert "a" in d def test_pop(self): d = CaselessDict() - d['a'] = 1 - self.assertEqual(d.pop('A'), 1) - self.assertRaises(KeyError, d.pop, 'A') + d["a"] = 1 + self.assertEqual(d.pop("A"), 1) + self.assertRaises(KeyError, d.pop, "A") def test_normkey(self): class MyDict(CaselessDict): @@ -138,8 +142,8 @@ class CaselessDictTest(unittest.TestCase): return key.title() d = MyDict() - d['key-one'] = 2 - self.assertEqual(list(d.keys()), ['Key-One']) + d["key-one"] = 2 + self.assertEqual(list(d.keys()), ["Key-One"]) def test_normvalue(self): class MyDict(CaselessDict): @@ -147,39 +151,38 @@ class CaselessDictTest(unittest.TestCase): if value is not None: return value + 1 - d = MyDict({'key': 1}) - self.assertEqual(d['key'], 2) - self.assertEqual(d.get('key'), 2) + d = MyDict({"key": 1}) + self.assertEqual(d["key"], 2) + self.assertEqual(d.get("key"), 2) d = MyDict() - d['key'] = 1 - self.assertEqual(d['key'], 2) - self.assertEqual(d.get('key'), 2) + d["key"] = 1 + self.assertEqual(d["key"], 2) + self.assertEqual(d.get("key"), 2) d = MyDict() - d.setdefault('key', 1) - self.assertEqual(d['key'], 2) - self.assertEqual(d.get('key'), 2) + d.setdefault("key", 1) + self.assertEqual(d["key"], 2) + self.assertEqual(d.get("key"), 2) d = MyDict() - d.update({'key': 1}) - self.assertEqual(d['key'], 2) - self.assertEqual(d.get('key'), 2) + d.update({"key": 1}) + self.assertEqual(d["key"], 2) + self.assertEqual(d.get("key"), 2) - d = MyDict.fromkeys(('key',), 1) - self.assertEqual(d['key'], 2) - self.assertEqual(d.get('key'), 2) + d = MyDict.fromkeys(("key",), 1) + self.assertEqual(d["key"], 2) + self.assertEqual(d.get("key"), 2) def test_copy(self): - h1 = CaselessDict({'header1': 'value'}) + h1 = CaselessDict({"header1": "value"}) h2 = copy.copy(h1) self.assertEqual(h1, h2) - self.assertEqual(h1.get('header1'), h2.get('header1')) + self.assertEqual(h1.get("header1"), h2.get("header1")) assert isinstance(h2, CaselessDict) class SequenceExcludeTest(unittest.TestCase): - def test_list(self): seq = [1, 2, 3] d = SequenceExclude(seq) @@ -226,25 +229,24 @@ class SequenceExcludeTest(unittest.TestCase): # supplied sequence is a set, so checking for list (non)inclusion fails self.assertRaises(TypeError, (0, 1, 2) in d) - self.assertRaises(TypeError, d.__contains__, ['a', 'b', 'c']) + self.assertRaises(TypeError, d.__contains__, ["a", "b", "c"]) for v in [-3, "test", 1.1]: self.assertNotIn(v, d) class LocalCacheTest(unittest.TestCase): - def test_cache_with_limit(self): cache = LocalCache(limit=2) - cache['a'] = 1 - cache['b'] = 2 - cache['c'] = 3 + cache["a"] = 1 + cache["b"] = 2 + cache["c"] = 3 self.assertEqual(len(cache), 2) - self.assertNotIn('a', cache) - self.assertIn('b', cache) - self.assertIn('c', cache) - self.assertEqual(cache['b'], 2) - self.assertEqual(cache['c'], 3) + self.assertNotIn("a", cache) + self.assertIn("b", cache) + self.assertIn("c", cache) + self.assertEqual(cache["b"], 2) + self.assertEqual(cache["c"], 3) def test_cache_without_limit(self): maximum = 10**4 @@ -258,12 +260,11 @@ class LocalCacheTest(unittest.TestCase): class LocalWeakReferencedCacheTest(unittest.TestCase): - def test_cache_with_limit(self): cache = LocalWeakReferencedCache(limit=2) - r1 = Request('https://example.org') - r2 = Request('https://example.com') - r3 = Request('https://example.net') + r1 = Request("https://example.org") + r2 = Request("https://example.com") + r3 = Request("https://example.net") cache[r1] = 1 cache[r2] = 2 cache[r3] = 3 @@ -299,7 +300,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): cache = LocalWeakReferencedCache() refs = [] for x in range(max): - refs.append(Request(f'https://example.org/{x}')) + refs.append(Request(f"https://example.org/{x}")) cache[refs[-1]] = x self.assertEqual(len(cache), max) for i, r in enumerate(refs): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 97c1c60d1..8d7f33c9a 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -68,41 +68,43 @@ def eb1(failure, arg1, arg2): class DeferUtilsTest(unittest.TestCase): - @defer.inlineCallbacks def test_process_chain(self): - x = yield process_chain([cb1, cb2, cb3], 'res', 'v1', 'v2') + x = yield process_chain([cb1, cb2, cb3], "res", "v1", "v2") self.assertEqual(x, "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)") gotexc = False try: - yield process_chain([cb1, cb_fail, cb3], 'res', 'v1', 'v2') + yield process_chain([cb1, cb_fail, cb3], "res", "v1", "v2") except TypeError: gotexc = True self.assertTrue(gotexc) @defer.inlineCallbacks def test_process_chain_both(self): - x = yield process_chain_both([cb_fail, cb2, cb3], [None, eb1, None], 'res', 'v1', 'v2') + x = yield process_chain_both( + [cb_fail, cb2, cb3], [None, eb1, None], "res", "v1", "v2" + ) self.assertEqual(x, "(cb3 (eb1 TypeError v1 v2) v1 v2)") fail = Failure(ZeroDivisionError()) - x = yield process_chain_both([eb1, cb2, cb3], [eb1, None, None], fail, 'v1', 'v2') + x = yield process_chain_both( + [eb1, cb2, cb3], [eb1, None, None], fail, "v1", "v2" + ) self.assertEqual(x, "(cb3 (cb2 (eb1 ZeroDivisionError v1 v2) v1 v2) v1 v2)") @defer.inlineCallbacks def test_process_parallel(self): - x = yield process_parallel([cb1, cb2, cb3], 'res', 'v1', 'v2') - self.assertEqual(x, ['(cb1 res v1 v2)', '(cb2 res v1 v2)', '(cb3 res v1 v2)']) + x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") + self.assertEqual(x, ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"]) def test_process_parallel_failure(self): - d = process_parallel([cb1, cb_fail, cb3], 'res', 'v1', 'v2') + d = process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2") self.failUnlessFailure(d, TypeError) return d class IterErrbackTest(unittest.TestCase): - def test_iter_errback_good(self): def itergood(): for x in range(10): @@ -128,7 +130,6 @@ class IterErrbackTest(unittest.TestCase): class AiterErrbackTest(unittest.TestCase): - @deferred_f_from_coro_f async def test_aiter_errback_good(self): async def itergood(): @@ -171,7 +172,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase): class AsyncCooperatorTest(unittest.TestCase): - """ This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage. + """This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage. parallel_async is called with the results of a callback (so an iterable of items, requests and None, with arbitrary delays between values), and it uses Scraper._process_spidermw_output as the callable @@ -182,6 +183,7 @@ class AsyncCooperatorTest(unittest.TestCase): We also want to simulate the real usage, with arbitrary delays between getting the values from the iterable. We also want to simulate sync and async results from the callable. """ + CONCURRENT_ITEMS = 50 @staticmethod diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 50c63dfab..1a8c96e7a 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -20,22 +20,23 @@ class NewName(SomeBaseClass): class WarnWhenSubclassedTest(unittest.TestCase): - def _mywarnings(self, w, category=MyWarning): return [x for x in w if x.category is MyWarning] def test_no_warning_on_definition(self): with warnings.catch_warnings(record=True) as w: - create_deprecated_class('Deprecated', NewName) + create_deprecated_class("Deprecated", NewName) w = self._mywarnings(w) self.assertEqual(w, []) def test_subclassing_warning_message(self): - Deprecated = create_deprecated_class('Deprecated', NewName, - warn_category=MyWarning) + Deprecated = create_deprecated_class( + "Deprecated", NewName, warn_category=MyWarning + ) with warnings.catch_warnings(record=True) as w: + class UserClass(Deprecated): pass @@ -46,17 +47,21 @@ class WarnWhenSubclassedTest(unittest.TestCase): "tests.test_utils_deprecate.UserClass inherits from " "deprecated class tests.test_utils_deprecate.Deprecated, " "please inherit from tests.test_utils_deprecate.NewName." - " (warning only on first subclass, there may be others)" + " (warning only on first subclass, there may be others)", ) self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1]) def test_custom_class_paths(self): - Deprecated = create_deprecated_class('Deprecated', NewName, - new_class_path='foo.NewClass', - old_class_path='bar.OldClass', - warn_category=MyWarning) + Deprecated = create_deprecated_class( + "Deprecated", + NewName, + new_class_path="foo.NewClass", + old_class_path="bar.OldClass", + warn_category=MyWarning, + ) with warnings.catch_warnings(record=True) as w: + class UserClass(Deprecated): pass @@ -64,17 +69,18 @@ class WarnWhenSubclassedTest(unittest.TestCase): w = self._mywarnings(w) self.assertEqual(len(w), 2) - self.assertIn('foo.NewClass', str(w[0].message)) - self.assertIn('bar.OldClass', str(w[0].message)) - self.assertIn('foo.NewClass', str(w[1].message)) - self.assertIn('bar.OldClass', str(w[1].message)) + self.assertIn("foo.NewClass", str(w[0].message)) + self.assertIn("bar.OldClass", str(w[0].message)) + self.assertIn("foo.NewClass", str(w[1].message)) + self.assertIn("bar.OldClass", str(w[1].message)) def test_subclassing_warns_only_on_direct_childs(self): - Deprecated = create_deprecated_class('Deprecated', NewName, - warn_once=False, - warn_category=MyWarning) + Deprecated = create_deprecated_class( + "Deprecated", NewName, warn_once=False, warn_category=MyWarning + ) with warnings.catch_warnings(record=True) as w: + class UserClass(Deprecated): pass @@ -83,13 +89,15 @@ class WarnWhenSubclassedTest(unittest.TestCase): w = self._mywarnings(w) self.assertEqual(len(w), 1) - self.assertIn('UserClass', str(w[0].message)) + self.assertIn("UserClass", str(w[0].message)) def test_subclassing_warns_once_by_default(self): - Deprecated = create_deprecated_class('Deprecated', NewName, - warn_category=MyWarning) + Deprecated = create_deprecated_class( + "Deprecated", NewName, warn_category=MyWarning + ) with warnings.catch_warnings(record=True) as w: + class UserClass(Deprecated): pass @@ -101,15 +109,16 @@ class WarnWhenSubclassedTest(unittest.TestCase): w = self._mywarnings(w) self.assertEqual(len(w), 1) - self.assertIn('UserClass', str(w[0].message)) + self.assertIn("UserClass", str(w[0].message)) def test_warning_on_instance(self): - Deprecated = create_deprecated_class('Deprecated', NewName, - warn_category=MyWarning) + Deprecated = create_deprecated_class( + "Deprecated", NewName, warn_category=MyWarning + ) # ignore subclassing warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', MyWarning) + warnings.simplefilter("ignore", MyWarning) class UserClass(Deprecated): pass @@ -123,13 +132,13 @@ class WarnWhenSubclassedTest(unittest.TestCase): self.assertEqual( str(w[0].message), "tests.test_utils_deprecate.Deprecated is deprecated, " - "instantiate tests.test_utils_deprecate.NewName instead." + "instantiate tests.test_utils_deprecate.NewName instead.", ) self.assertEqual(w[0].lineno, lineno) def test_warning_auto_message(self): with warnings.catch_warnings(record=True) as w: - Deprecated = create_deprecated_class('Deprecated', NewName) + Deprecated = create_deprecated_class("Deprecated", NewName) class UserClass2(Deprecated): pass @@ -140,8 +149,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): def test_issubclass(self): with warnings.catch_warnings(): - warnings.simplefilter('ignore', ScrapyDeprecationWarning) - DeprecatedName = create_deprecated_class('DeprecatedName', NewName) + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + DeprecatedName = create_deprecated_class("DeprecatedName", NewName) class UpdatedUserClass1(NewName): pass @@ -176,8 +185,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): def test_isinstance(self): with warnings.catch_warnings(): - warnings.simplefilter('ignore', ScrapyDeprecationWarning) - DeprecatedName = create_deprecated_class('DeprecatedName', NewName) + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + DeprecatedName = create_deprecated_class("DeprecatedName", NewName) class UpdatedUserClass2(NewName): pass @@ -210,24 +219,28 @@ class WarnWhenSubclassedTest(unittest.TestCase): def test_clsdict(self): with warnings.catch_warnings(): - warnings.simplefilter('ignore', ScrapyDeprecationWarning) - Deprecated = create_deprecated_class('Deprecated', NewName, {'foo': 'bar'}) + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"}) - self.assertEqual(Deprecated.foo, 'bar') + self.assertEqual(Deprecated.foo, "bar") def test_deprecate_a_class_with_custom_metaclass(self): - Meta1 = type('Meta1', (type,), {}) - New = Meta1('New', (), {}) - create_deprecated_class('Deprecated', New) + Meta1 = type("Meta1", (type,), {}) + New = Meta1("New", (), {}) + create_deprecated_class("Deprecated", New) def test_deprecate_subclass_of_deprecated_class(self): with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - Deprecated = create_deprecated_class('Deprecated', NewName, - warn_category=MyWarning) - AlsoDeprecated = create_deprecated_class('AlsoDeprecated', Deprecated, - new_class_path='foo.Bar', - warn_category=MyWarning) + warnings.simplefilter("always") + Deprecated = create_deprecated_class( + "Deprecated", NewName, warn_category=MyWarning + ) + AlsoDeprecated = create_deprecated_class( + "AlsoDeprecated", + Deprecated, + new_class_path="foo.Bar", + warn_category=MyWarning, + ) w = self._mywarnings(w) self.assertEqual(len(w), 0, str(map(str, w))) @@ -240,15 +253,15 @@ class WarnWhenSubclassedTest(unittest.TestCase): w = self._mywarnings(w) self.assertEqual(len(w), 2) - self.assertIn('AlsoDeprecated', str(w[0].message)) - self.assertIn('foo.Bar', str(w[0].message)) - self.assertIn('AlsoDeprecated', str(w[1].message)) - self.assertIn('foo.Bar', str(w[1].message)) + self.assertIn("AlsoDeprecated", str(w[0].message)) + self.assertIn("foo.Bar", str(w[0].message)) + self.assertIn("AlsoDeprecated", str(w[1].message)) + self.assertIn("foo.Bar", str(w[1].message)) def test_inspect_stack(self): - with mock.patch('inspect.stack', side_effect=IndexError): + with mock.patch("inspect.stack", side_effect=IndexError): with warnings.catch_warnings(record=True) as w: - DeprecatedName = create_deprecated_class('DeprecatedName', NewName) + DeprecatedName = create_deprecated_class("DeprecatedName", NewName) class SubClass(DeprecatedName): pass @@ -256,27 +269,30 @@ class WarnWhenSubclassedTest(unittest.TestCase): self.assertIn("Error detecting parent module", str(w[0].message)) -@mock.patch('scrapy.utils.deprecate.DEPRECATION_RULES', - [('scrapy.contrib.pipeline.', 'scrapy.pipelines.'), - ('scrapy.contrib.', 'scrapy.extensions.')]) +@mock.patch( + "scrapy.utils.deprecate.DEPRECATION_RULES", + [ + ("scrapy.contrib.pipeline.", "scrapy.pipelines."), + ("scrapy.contrib.", "scrapy.extensions."), + ], +) class UpdateClassPathTest(unittest.TestCase): - def test_old_path_gets_fixed(self): with warnings.catch_warnings(record=True) as w: - output = update_classpath('scrapy.contrib.debug.Debug') - self.assertEqual(output, 'scrapy.extensions.debug.Debug') + output = update_classpath("scrapy.contrib.debug.Debug") + self.assertEqual(output, "scrapy.extensions.debug.Debug") self.assertEqual(len(w), 1) self.assertIn("scrapy.contrib.debug.Debug", str(w[0].message)) self.assertIn("scrapy.extensions.debug.Debug", str(w[0].message)) def test_sorted_replacement(self): with warnings.catch_warnings(): - warnings.simplefilter('ignore', ScrapyDeprecationWarning) - output = update_classpath('scrapy.contrib.pipeline.Pipeline') - self.assertEqual(output, 'scrapy.pipelines.Pipeline') + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + output = update_classpath("scrapy.contrib.pipeline.Pipeline") + self.assertEqual(output, "scrapy.pipelines.Pipeline") def test_unmatched_path_stays_the_same(self): with warnings.catch_warnings(record=True) as w: - output = update_classpath('scrapy.unmatched.Path') - self.assertEqual(output, 'scrapy.unmatched.Path') + output = update_classpath("scrapy.unmatched.Path") + self.assertEqual(output, "scrapy.unmatched.Path") self.assertEqual(len(w), 0) diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index 9ec8311d9..43236acec 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -6,14 +6,14 @@ from scrapy.utils.display import pformat, pprint class TestDisplay(TestCase): - object = {'a': 1} + object = {"a": 1} colorized_string = ( "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n" ) plain_string = "{'a': 1}" - @mock.patch('sys.platform', 'linux') + @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") def test_pformat(self, isatty): isatty.return_value = True @@ -27,44 +27,47 @@ class TestDisplay(TestCase): def test_pformat_not_tty(self): self.assertEqual(pformat(self.object), self.plain_string) - @mock.patch('sys.platform', 'win32') - @mock.patch('platform.version') + @mock.patch("sys.platform", "win32") + @mock.patch("platform.version") @mock.patch("sys.stdout.isatty") def test_pformat_old_windows(self, isatty, version): isatty.return_value = True - version.return_value = '10.0.14392' + version.return_value = "10.0.14392" self.assertEqual(pformat(self.object), self.colorized_string) - @mock.patch('sys.platform', 'win32') - @mock.patch('scrapy.utils.display._enable_windows_terminal_processing') - @mock.patch('platform.version') + @mock.patch("sys.platform", "win32") + @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") + @mock.patch("platform.version") @mock.patch("sys.stdout.isatty") - def test_pformat_windows_no_terminal_processing(self, isatty, version, terminal_processing): + def test_pformat_windows_no_terminal_processing( + self, isatty, version, terminal_processing + ): isatty.return_value = True - version.return_value = '10.0.14393' + version.return_value = "10.0.14393" terminal_processing.return_value = False self.assertEqual(pformat(self.object), self.plain_string) - @mock.patch('sys.platform', 'win32') - @mock.patch('scrapy.utils.display._enable_windows_terminal_processing') - @mock.patch('platform.version') + @mock.patch("sys.platform", "win32") + @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") + @mock.patch("platform.version") @mock.patch("sys.stdout.isatty") def test_pformat_windows(self, isatty, version, terminal_processing): isatty.return_value = True - version.return_value = '10.0.14393' + version.return_value = "10.0.14393" terminal_processing.return_value = True self.assertEqual(pformat(self.object), self.colorized_string) - @mock.patch('sys.platform', 'linux') + @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") def test_pformat_no_pygments(self, isatty): isatty.return_value = True import builtins + real_import = builtins.__import__ def mock_import(name, globals, locals, fromlist, level): - if 'pygments' in name: + if "pygments" in name: raise ImportError return real_import(name, globals, locals, fromlist, level) @@ -73,6 +76,6 @@ class TestDisplay(TestCase): builtins.__import__ = real_import def test_pprint(self): - with mock.patch('sys.stdout', new=StringIO()) as mock_out: + with mock.patch("sys.stdout", new=StringIO()) as mock_out: pprint(self.object) self.assertEqual(mock_out.getvalue(), "{'a': 1}\n") diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index ca98bff21..a34664956 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -8,13 +8,15 @@ from scrapy.http import Response from tests import tests_datadir -SAMPLEDIR = Path(tests_datadir, 'compressed') +SAMPLEDIR = Path(tests_datadir, "compressed") class GunzipTest(unittest.TestCase): - def test_gunzip_basic(self): - r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'feed-sample1.xml.gz').read_bytes()) + r1 = Response( + "http://www.example.com", + body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(), + ) self.assertTrue(gzip_magic_number(r1)) r2 = Response("http://www.example.com", body=gunzip(r1.body)) @@ -22,18 +24,23 @@ class GunzipTest(unittest.TestCase): self.assertEqual(len(r2.body), 9950) def test_gunzip_truncated(self): - text = gunzip((SAMPLEDIR / 'truncated-crc-error.gz').read_bytes()) - assert text.endswith(b'') + assert r2.body.endswith(b"") self.assertFalse(gzip_magic_number(r2)) def test_is_gzipped_empty(self): @@ -41,7 +48,11 @@ class GunzipTest(unittest.TestCase): self.assertFalse(gzip_magic_number(r1)) def test_gunzip_illegal_eof(self): - text = html_to_unicode('charset=cp1252', gunzip((SAMPLEDIR / 'unexpected-eof.gz').read_bytes()))[1] - expected_text = (SAMPLEDIR / 'unexpected-eof-output.txt').read_text(encoding="utf-8") + text = html_to_unicode( + "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes()) + )[1] + expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text( + encoding="utf-8" + ) self.assertEqual(len(text), len(expected_text)) self.assertEqual(text, expected_text) diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index cf8ad1f23..b824972d5 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -6,7 +6,6 @@ from scrapy.utils.httpobj import urlparse_cached class HttpobjUtilsTest(unittest.TestCase): - def test_urlparse_cached(self): url = "http://www.example.com/index.html" request1 = Request(url) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ba3136b96..893582a32 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -28,14 +28,18 @@ class XmliterTestCase(unittest.TestCase): response = XmlResponse(url="http://example.com", body=body) attrs = [] - for x in self.xmliter(response, 'product'): - attrs.append(( - x.attrib['id'], - x.xpath("name/text()").getall(), - x.xpath("./type/text()").getall())) + for x in self.xmliter(response, "product"): + attrs.append( + ( + x.attrib["id"], + x.xpath("name/text()").getall(), + x.xpath("./type/text()").getall(), + ) + ) - self.assertEqual(attrs, - [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])]) + self.assertEqual( + attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])] + ) def test_xmliter_unusual_node(self): body = b""" @@ -45,8 +49,10 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url="http://example.com", body=body) - nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')] - self.assertEqual(nodenames, [['matchme...']]) + nodenames = [ + e.xpath("name()").getall() for e in self.xmliter(response, "matchme...") + ] + self.assertEqual(nodenames, [["matchme..."]]) def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 @@ -88,29 +94,35 @@ class XmliterTestCase(unittest.TestCase): for r in ( # with bytes - XmlResponse(url="http://example.com", body=body.encode('utf-8')), + XmlResponse(url="http://example.com", body=body.encode("utf-8")), # Unicode body needs encoding information - XmlResponse(url="http://example.com", body=body, encoding='utf-8'), + XmlResponse(url="http://example.com", body=body, encoding="utf-8"), ): attrs = [] - for x in self.xmliter(r, 'þingflokkur'): - attrs.append((x.attrib['id'], - x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(), - x.xpath('./tímabil/fyrstaþing/text()').getall())) + for x in self.xmliter(r, "þingflokkur"): + attrs.append( + ( + x.attrib["id"], + x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(), + x.xpath("./tímabil/fyrstaþing/text()").getall(), + ) + ) - self.assertEqual(attrs, - [('26', ['-'], ['80']), - ('21', ['Ab'], ['76']), - ('27', ['A'], ['27'])]) + self.assertEqual( + attrs, + [("26", ["-"], ["80"]), ("21", ["Ab"], ["76"]), ("27", ["A"], ["27"])], + ) def test_xmliter_text(self): body = ( '' - 'onetwo' + "onetwo" ) - self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')], - [['one'], ['two']]) + self.assertEqual( + [x.xpath("text()").getall() for x in self.xmliter(body, "product")], + [["one"], ["two"]], + ) def test_xmliter_namespaces(self): body = b""" @@ -131,22 +143,25 @@ class XmliterTestCase(unittest.TestCase): """ - response = XmlResponse(url='http://mydummycompany.com', body=body) - my_iter = self.xmliter(response, 'item') + response = XmlResponse(url="http://mydummycompany.com", body=body) + my_iter = self.xmliter(response, "item") node = next(my_iter) - node.register_namespace('g', 'http://base.google.com/ns/1.0') - self.assertEqual(node.xpath('title/text()').getall(), ['Item 1']) - self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1']) - self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1']) + node.register_namespace("g", "http://base.google.com/ns/1.0") + self.assertEqual(node.xpath("title/text()").getall(), ["Item 1"]) + self.assertEqual(node.xpath("description/text()").getall(), ["This is item 1"]) self.assertEqual( - node.xpath('g:image_link/text()').getall(), - ['http://www.mydummycompany.com/images/item1.jpg'] + node.xpath("link/text()").getall(), + ["http://www.mydummycompany.com/items/1"], ) - self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1']) - self.assertEqual(node.xpath('g:price/text()').getall(), ['400']) - self.assertEqual(node.xpath('image_link/text()').getall(), []) - self.assertEqual(node.xpath('id/text()').getall(), []) - self.assertEqual(node.xpath('price/text()').getall(), []) + self.assertEqual( + node.xpath("g:image_link/text()").getall(), + ["http://www.mydummycompany.com/images/item1.jpg"], + ) + self.assertEqual(node.xpath("g:id/text()").getall(), ["ITEM_1"]) + self.assertEqual(node.xpath("g:price/text()").getall(), ["400"]) + self.assertEqual(node.xpath("image_link/text()").getall(), []) + self.assertEqual(node.xpath("id/text()").getall(), []) + self.assertEqual(node.xpath("price/text()").getall(), []) def test_xmliter_namespaced_nodename(self): body = b""" @@ -167,11 +182,14 @@ class XmliterTestCase(unittest.TestCase): """ - response = XmlResponse(url='http://mydummycompany.com', body=body) - my_iter = self.xmliter(response, 'g:image_link') + response = XmlResponse(url="http://mydummycompany.com", body=body) + my_iter = self.xmliter(response, "g:image_link") node = next(my_iter) - node.register_namespace('g', 'http://base.google.com/ns/1.0') - self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + node.register_namespace("g", "http://base.google.com/ns/1.0") + self.assertEqual( + node.xpath("text()").extract(), + ["http://www.mydummycompany.com/images/item1.jpg"], + ) def test_xmliter_namespaced_nodename_missing(self): body = b""" @@ -192,45 +210,45 @@ class XmliterTestCase(unittest.TestCase): """ - response = XmlResponse(url='http://mydummycompany.com', body=body) - my_iter = self.xmliter(response, 'g:link_image') + response = XmlResponse(url="http://mydummycompany.com", body=body) + my_iter = self.xmliter(response, "g:link_image") with self.assertRaises(StopIteration): next(my_iter) def test_xmliter_exception(self): body = ( '' - 'onetwo' + "onetwo" ) - iter = self.xmliter(body, 'product') + iter = self.xmliter(body, "product") next(iter) next(iter) self.assertRaises(StopIteration, next, iter) def test_xmliter_objtype_exception(self): - i = self.xmliter(42, 'product') + i = self.xmliter(42, "product") self.assertRaises(TypeError, next, i) def test_xmliter_encoding(self): body = ( b'\n' - b'\n' - b' Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n' - b'\n\n' + b"\n" + b" Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n" + b"\n\n" ) - response = XmlResponse('http://www.example.com', body=body) + response = XmlResponse("http://www.example.com", body=body) self.assertEqual( - next(self.xmliter(response, 'item')).get(), - 'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' + next(self.xmliter(response, "item")).get(), + "Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6", ) class LxmlXmliterTestCase(XmliterTestCase): xmliter = staticmethod(xmliter_lxml) - @mark.xfail(reason='known bug of the current implementation') + @mark.xfail(reason="known bug of the current implementation") def test_xmliter_namespaced_nodename(self): super().test_xmliter_namespaced_nodename() @@ -252,16 +270,24 @@ class LxmlXmliterTestCase(XmliterTestCase): """ - response = XmlResponse(url='http://mydummycompany.com', body=body) + response = XmlResponse(url="http://mydummycompany.com", body=body) - no_namespace_iter = self.xmliter(response, 'image_link') + no_namespace_iter = self.xmliter(response, "image_link") self.assertEqual(len(list(no_namespace_iter)), 0) - namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0') + namespace_iter = self.xmliter( + response, "image_link", "http://base.google.com/ns/1.0" + ) node = next(namespace_iter) - self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual( + node.xpath("text()").getall(), + ["http://www.mydummycompany.com/images/item1.jpg"], + ) node = next(namespace_iter) - self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg']) + self.assertEqual( + node.xpath("text()").getall(), + ["http://www.mydummycompany.com/images/item2.jpg"], + ) def test_xmliter_namespaces_prefix(self): body = b""" @@ -282,36 +308,42 @@ class LxmlXmliterTestCase(XmliterTestCase): """ - response = XmlResponse(url='http://mydummycompany.com', body=body) - my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h') + response = XmlResponse(url="http://mydummycompany.com", body=body) + my_iter = self.xmliter(response, "table", "http://www.w3.org/TR/html4/", "h") node = next(my_iter) - self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2) - self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples']) - self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas']) + self.assertEqual(len(node.xpath("h:tr/h:td").getall()), 2) + self.assertEqual(node.xpath("h:tr/h:td[1]/text()").getall(), ["Apples"]) + self.assertEqual(node.xpath("h:tr/h:td[2]/text()").getall(), ["Bananas"]) - my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f') + my_iter = self.xmliter( + response, "table", "http://www.w3schools.com/furniture", "f" + ) node = next(my_iter) - self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table']) + self.assertEqual(node.xpath("f:name/text()").getall(), ["African Coffee Table"]) def test_xmliter_objtype_exception(self): - i = self.xmliter(42, 'product') + i = self.xmliter(42, "product") self.assertRaises(TypeError, next, i) class UtilsCsvTestCase(unittest.TestCase): def test_csviter_defaults(self): - body = get_testdata('feeds', 'feed-sample3.csv') + body = get_testdata("feeds", "feed-sample3.csv") response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) result = [row for row in csv] - self.assertEqual(result, - [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, - {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, - {'id': '4', 'name': 'empty', 'value': ''}]) + self.assertEqual( + result, + [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ], + ) # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: @@ -319,88 +351,120 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertTrue(all((isinstance(v, str) for v in result_row.values()))) def test_csviter_delimiter(self): - body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') + body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t") response = TextResponse(url="http://example.com/", body=body) - csv = csviter(response, delimiter='\t') + csv = csviter(response, delimiter="\t") - self.assertEqual([row for row in csv], - [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, - {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, - {'id': '4', 'name': 'empty', 'value': ''}]) + self.assertEqual( + [row for row in csv], + [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ], + ) def test_csviter_quotechar(self): - body1 = get_testdata('feeds', 'feed-sample6.csv') - body2 = get_testdata('feeds', 'feed-sample6.csv').replace(b',', b'|') + body1 = get_testdata("feeds", "feed-sample6.csv") + body2 = get_testdata("feeds", "feed-sample6.csv").replace(b",", b"|") response1 = TextResponse(url="http://example.com/", body=body1) csv1 = csviter(response1, quotechar="'") - self.assertEqual([row for row in csv1], - [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, - {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, - {'id': '4', 'name': 'empty', 'value': ''}]) + self.assertEqual( + [row for row in csv1], + [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ], + ) response2 = TextResponse(url="http://example.com/", body=body2) csv2 = csviter(response2, delimiter="|", quotechar="'") - self.assertEqual([row for row in csv2], - [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, - {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, - {'id': '4', 'name': 'empty', 'value': ''}]) + self.assertEqual( + [row for row in csv2], + [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ], + ) def test_csviter_wrong_quotechar(self): - body = get_testdata('feeds', 'feed-sample6.csv') + body = get_testdata("feeds", "feed-sample6.csv") response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - self.assertEqual([row for row in csv], - [{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, - {"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"}, - {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"}, - {"'id'": "4", "'name'": "'empty'", "'value'": ""}]) + self.assertEqual( + [row for row in csv], + [ + {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, + { + "'id'": "2", + "'name'": "'unicode'", + "'value'": "'\xfan\xedc\xf3d\xe9\u203d'", + }, + {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"}, + {"'id'": "4", "'name'": "'empty'", "'value'": ""}, + ], + ) def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): - body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') + body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t") response = Response(url="http://example.com/", body=body) - csv = csviter(response, delimiter='\t') + csv = csviter(response, delimiter="\t") - self.assertEqual([row for row in csv], - [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, - {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, - {'id': '4', 'name': 'empty', 'value': ''}]) + self.assertEqual( + [row for row in csv], + [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ], + ) def test_csviter_headers(self): - sample = get_testdata('feeds', 'feed-sample3.csv').splitlines() - headers, body = sample[0].split(b','), b'\n'.join(sample[1:]) + sample = get_testdata("feeds", "feed-sample3.csv").splitlines() + headers, body = sample[0].split(b","), b"\n".join(sample[1:]) response = TextResponse(url="http://example.com/", body=body) - csv = csviter(response, headers=[h.decode('utf-8') for h in headers]) + csv = csviter(response, headers=[h.decode("utf-8") for h in headers]) - self.assertEqual([row for row in csv], - [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, - {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': 'foo\nbar'}, - {'id': '4', 'name': 'empty', 'value': ''}]) + self.assertEqual( + [row for row in csv], + [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ], + ) def test_csviter_falserow(self): - body = get_testdata('feeds', 'feed-sample3.csv') - body = b'\n'.join((body, b'a,b', b'a,b,c,d')) + body = get_testdata("feeds", "feed-sample3.csv") + body = b"\n".join((body, b"a,b", b"a,b,c,d")) response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - self.assertEqual([row for row in csv], - [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, - {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, - {'id': '4', 'name': 'empty', 'value': ''}]) + self.assertEqual( + [row for row in csv], + [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ], + ) def test_csviter_exception(self): - body = get_testdata('feeds', 'feed-sample3.csv') + body = get_testdata("feeds", "feed-sample3.csv") response = TextResponse(url="http://example.com/", body=body) iter = csviter(response) @@ -412,35 +476,41 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertRaises(StopIteration, next, iter) def test_csviter_encoding(self): - body1 = get_testdata('feeds', 'feed-sample4.csv') - body2 = get_testdata('feeds', 'feed-sample5.csv') + body1 = get_testdata("feeds", "feed-sample4.csv") + body2 = get_testdata("feeds", "feed-sample5.csv") - response = TextResponse(url="http://example.com/", body=body1, encoding='latin1') + response = TextResponse( + url="http://example.com/", body=body1, encoding="latin1" + ) csv = csviter(response) self.assertEqual( list(csv), [ - {'id': '1', 'name': 'latin1', 'value': 'test'}, - {'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'}, - ] + {"id": "1", "name": "latin1", "value": "test"}, + {"id": "2", "name": "something", "value": "\xf1\xe1\xe9\xf3"}, + ], ) - response = TextResponse(url="http://example.com/", body=body2, encoding='cp852') + response = TextResponse(url="http://example.com/", body=body2, encoding="cp852") csv = csviter(response) self.assertEqual( list(csv), [ - {'id': '1', 'name': 'cp852', 'value': 'test'}, - {'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}, - ] + {"id": "1", "name": "cp852", "value": "test"}, + { + "id": "2", + "name": "something", + "value": "\u255a\u2569\u2569\u2569\u2550\u2550\u2557", + }, + ], ) class TestHelper(unittest.TestCase): - bbody = b'utf8-body' - ubody = bbody.decode('utf8') - txtresponse = TextResponse(url='http://example.org/', body=bbody, encoding='utf-8') - response = Response(url='http://example.org/', body=bbody) + bbody = b"utf8-body" + ubody = bbody.decode("utf8") + txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8") + response = Response(url="http://example.org/", body=bbody) def test_body_or_str(self): for obj in (self.bbody, self.ubody, self.txtresponse, self.response): @@ -454,8 +524,9 @@ class TestHelper(unittest.TestCase): self.assertTrue(type(r1) is not type(r3)) def _assert_type_and_value(self, a, b, obj): - self.assertTrue(type(a) is type(b), - f'Got {type(a)}, expected {type(b)} for { obj!r}') + self.assertTrue( + type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}" + ) self.assertEqual(a, b) diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 535f56691..438dd0cdc 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -5,14 +5,17 @@ import unittest from testfixtures import LogCapture from twisted.python.failure import Failure -from scrapy.utils.log import (failure_to_exc_info, TopLevelFormatter, - LogCounterHandler, StreamLogger) +from scrapy.utils.log import ( + failure_to_exc_info, + TopLevelFormatter, + LogCounterHandler, + StreamLogger, +) from scrapy.utils.test import get_crawler from scrapy.extensions import telnet class FailureToExcInfoTest(unittest.TestCase): - def test_failure(self): try: 0 / 0 @@ -23,48 +26,46 @@ class FailureToExcInfoTest(unittest.TestCase): self.assertTupleEqual(exc_info, failure_to_exc_info(failure)) def test_non_failure(self): - self.assertIsNone(failure_to_exc_info('test')) + self.assertIsNone(failure_to_exc_info("test")) class TopLevelFormatterTest(unittest.TestCase): - def setUp(self): self.handler = LogCapture() - self.handler.addFilter(TopLevelFormatter(['test'])) + self.handler.addFilter(TopLevelFormatter(["test"])) def test_top_level_logger(self): - logger = logging.getLogger('test') + logger = logging.getLogger("test") with self.handler as log: - logger.warning('test log msg') - log.check(('test', 'WARNING', 'test log msg')) + logger.warning("test log msg") + log.check(("test", "WARNING", "test log msg")) def test_children_logger(self): - logger = logging.getLogger('test.test1') + logger = logging.getLogger("test.test1") with self.handler as log: - logger.warning('test log msg') - log.check(('test', 'WARNING', 'test log msg')) + logger.warning("test log msg") + log.check(("test", "WARNING", "test log msg")) def test_overlapping_name_logger(self): - logger = logging.getLogger('test2') + logger = logging.getLogger("test2") with self.handler as log: - logger.warning('test log msg') - log.check(('test2', 'WARNING', 'test log msg')) + logger.warning("test log msg") + log.check(("test2", "WARNING", "test log msg")) def test_different_name_logger(self): - logger = logging.getLogger('different') + logger = logging.getLogger("different") with self.handler as log: - logger.warning('test log msg') - log.check(('different', 'WARNING', 'test log msg')) + logger.warning("test log msg") + log.check(("different", "WARNING", "test log msg")) class LogCounterHandlerTest(unittest.TestCase): - def setUp(self): - settings = {'LOG_LEVEL': 'WARNING'} + settings = {"LOG_LEVEL": "WARNING"} if not telnet.TWISTED_CONCH_AVAILABLE: # disable it to avoid the extra warning - settings['TELNETCONSOLE_ENABLED'] = False - self.logger = logging.getLogger('test') + settings["TELNETCONSOLE_ENABLED"] = False + self.logger = logging.getLogger("test") self.logger.setLevel(logging.NOTSET) self.logger.propagate = False self.crawler = get_crawler(settings_dict=settings) @@ -76,26 +77,25 @@ class LogCounterHandlerTest(unittest.TestCase): self.logger.removeHandler(self.handler) def test_init(self): - self.assertIsNone(self.crawler.stats.get_value('log_count/DEBUG')) - self.assertIsNone(self.crawler.stats.get_value('log_count/INFO')) - self.assertIsNone(self.crawler.stats.get_value('log_count/WARNING')) - self.assertIsNone(self.crawler.stats.get_value('log_count/ERROR')) - self.assertIsNone(self.crawler.stats.get_value('log_count/CRITICAL')) + self.assertIsNone(self.crawler.stats.get_value("log_count/DEBUG")) + self.assertIsNone(self.crawler.stats.get_value("log_count/INFO")) + self.assertIsNone(self.crawler.stats.get_value("log_count/WARNING")) + self.assertIsNone(self.crawler.stats.get_value("log_count/ERROR")) + self.assertIsNone(self.crawler.stats.get_value("log_count/CRITICAL")) def test_accepted_level(self): - self.logger.error('test log msg') - self.assertEqual(self.crawler.stats.get_value('log_count/ERROR'), 1) + self.logger.error("test log msg") + self.assertEqual(self.crawler.stats.get_value("log_count/ERROR"), 1) def test_filtered_out_level(self): - self.logger.debug('test log msg') - self.assertIsNone(self.crawler.stats.get_value('log_count/INFO')) + self.logger.debug("test log msg") + self.assertIsNone(self.crawler.stats.get_value("log_count/INFO")) class StreamLoggerTest(unittest.TestCase): - def setUp(self): self.stdout = sys.stdout - logger = logging.getLogger('test') + logger = logging.getLogger("test") logger.setLevel(logging.WARNING) sys.stdout = StreamLogger(logger, logging.ERROR) @@ -104,5 +104,5 @@ class StreamLoggerTest(unittest.TestCase): def test_redirect(self): with LogCapture() as log: - print('test log msg') - log.check(('test', 'ERROR', 'test log msg')) + print("test log msg") + log.check(("test", "ERROR", "test log msg")) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index dc5b9e123..38a61036c 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -5,118 +5,122 @@ from pathlib import Path from unittest import mock from scrapy.item import Item, Field -from scrapy.utils.misc import arg_to_iter, create_instance, load_object, rel_has_nofollow, set_environ, walk_modules +from scrapy.utils.misc import ( + arg_to_iter, + create_instance, + load_object, + rel_has_nofollow, + set_environ, + walk_modules, +) -__doctests__ = ['scrapy.utils.misc'] +__doctests__ = ["scrapy.utils.misc"] class UtilsMiscTestCase(unittest.TestCase): - def test_load_object_class(self): obj = load_object(Field) self.assertIs(obj, Field) - obj = load_object('scrapy.item.Field') + obj = load_object("scrapy.item.Field") self.assertIs(obj, Field) def test_load_object_function(self): obj = load_object(load_object) self.assertIs(obj, load_object) - obj = load_object('scrapy.utils.misc.load_object') + obj = load_object("scrapy.utils.misc.load_object") self.assertIs(obj, load_object) def test_load_object_exceptions(self): - self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') - self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') + self.assertRaises(ImportError, load_object, "nomodule999.mod.function") + self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999") self.assertRaises(TypeError, load_object, {}) def test_walk_modules(self): - mods = walk_modules('tests.test_utils_misc.test_walk_modules') + mods = walk_modules("tests.test_utils_misc.test_walk_modules") expected = [ - 'tests.test_utils_misc.test_walk_modules', - 'tests.test_utils_misc.test_walk_modules.mod', - 'tests.test_utils_misc.test_walk_modules.mod.mod0', - 'tests.test_utils_misc.test_walk_modules.mod1', + "tests.test_utils_misc.test_walk_modules", + "tests.test_utils_misc.test_walk_modules.mod", + "tests.test_utils_misc.test_walk_modules.mod.mod0", + "tests.test_utils_misc.test_walk_modules.mod1", ] self.assertEqual({m.__name__ for m in mods}, set(expected)) - mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod') + mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod") expected = [ - 'tests.test_utils_misc.test_walk_modules.mod', - 'tests.test_utils_misc.test_walk_modules.mod.mod0', + "tests.test_utils_misc.test_walk_modules.mod", + "tests.test_utils_misc.test_walk_modules.mod.mod0", ] self.assertEqual({m.__name__ for m in mods}, set(expected)) - mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod1') + mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod1") expected = [ - 'tests.test_utils_misc.test_walk_modules.mod1', + "tests.test_utils_misc.test_walk_modules.mod1", ] self.assertEqual({m.__name__ for m in mods}, set(expected)) - self.assertRaises(ImportError, walk_modules, 'nomodule999') + self.assertRaises(ImportError, walk_modules, "nomodule999") def test_walk_modules_egg(self): - egg = str(Path(__file__).parent / 'test.egg') + egg = str(Path(__file__).parent / "test.egg") sys.path.append(egg) try: - mods = walk_modules('testegg') + mods = walk_modules("testegg") expected = [ - 'testegg.spiders', - 'testegg.spiders.a', - 'testegg.spiders.b', - 'testegg' + "testegg.spiders", + "testegg.spiders.a", + "testegg.spiders.b", + "testegg", ] self.assertEqual({m.__name__ for m in mods}, set(expected)) finally: sys.path.remove(egg) def test_arg_to_iter(self): - class TestItem(Item): name = Field() - assert hasattr(arg_to_iter(None), '__iter__') - assert hasattr(arg_to_iter(100), '__iter__') - assert hasattr(arg_to_iter('lala'), '__iter__') - assert hasattr(arg_to_iter([1, 2, 3]), '__iter__') - assert hasattr(arg_to_iter(c for c in 'abcd'), '__iter__') + assert hasattr(arg_to_iter(None), "__iter__") + assert hasattr(arg_to_iter(100), "__iter__") + assert hasattr(arg_to_iter("lala"), "__iter__") + assert hasattr(arg_to_iter([1, 2, 3]), "__iter__") + assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__") self.assertEqual(list(arg_to_iter(None)), []) - self.assertEqual(list(arg_to_iter('lala')), ['lala']) + self.assertEqual(list(arg_to_iter("lala")), ["lala"]) self.assertEqual(list(arg_to_iter(100)), [100]) - self.assertEqual(list(arg_to_iter(c for c in 'abc')), ['a', 'b', 'c']) + self.assertEqual(list(arg_to_iter(c for c in "abc")), ["a", "b", "c"]) self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3]) - self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}]) - self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")]) + self.assertEqual(list(arg_to_iter({"a": 1})), [{"a": 1}]) + self.assertEqual( + list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")] + ) def test_create_instance(self): settings = mock.MagicMock() - crawler = mock.MagicMock(spec_set=['settings']) - args = (True, 100.) - kwargs = {'key': 'val'} + crawler = mock.MagicMock(spec_set=["settings"]) + args = (True, 100.0) + kwargs = {"key": "val"} def _test_with_settings(mock, settings): create_instance(mock, settings, None, *args, **kwargs) - if hasattr(mock, 'from_crawler'): + if hasattr(mock, "from_crawler"): self.assertEqual(mock.from_crawler.call_count, 0) - if hasattr(mock, 'from_settings'): - mock.from_settings.assert_called_once_with(settings, *args, - **kwargs) + if hasattr(mock, "from_settings"): + mock.from_settings.assert_called_once_with(settings, *args, **kwargs) self.assertEqual(mock.call_count, 0) else: mock.assert_called_once_with(*args, **kwargs) def _test_with_crawler(mock, settings, crawler): create_instance(mock, settings, crawler, *args, **kwargs) - if hasattr(mock, 'from_crawler'): - mock.from_crawler.assert_called_once_with(crawler, *args, - **kwargs) - if hasattr(mock, 'from_settings'): + if hasattr(mock, "from_crawler"): + mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) + if hasattr(mock, "from_settings"): self.assertEqual(mock.from_settings.call_count, 0) self.assertEqual(mock.call_count, 0) - elif hasattr(mock, 'from_settings'): - mock.from_settings.assert_called_once_with(settings, *args, - **kwargs) + elif hasattr(mock, "from_settings"): + mock.from_settings.assert_called_once_with(settings, *args, **kwargs) self.assertEqual(mock.call_count, 0) else: mock.assert_called_once_with(*args, **kwargs) @@ -127,10 +131,10 @@ class UtilsMiscTestCase(unittest.TestCase): # 3. with from_crawler() constructor # 4. with from_settings() and from_crawler() constructor spec_sets = ( - ['__qualname__'], - ['__qualname__', 'from_settings'], - ['__qualname__', 'from_crawler'], - ['__qualname__', 'from_settings', 'from_crawler'], + ["__qualname__"], + ["__qualname__", "from_settings"], + ["__qualname__", "from_crawler"], + ["__qualname__", "from_settings", "from_crawler"], ) for specs in spec_sets: m = mock.MagicMock(spec_set=specs) @@ -139,10 +143,9 @@ class UtilsMiscTestCase(unittest.TestCase): _test_with_crawler(m, settings, crawler) # Check adoption of crawler settings - m = mock.MagicMock(spec_set=['__qualname__', 'from_settings']) + m = mock.MagicMock(spec_set=["__qualname__", "from_settings"]) create_instance(m, None, crawler, *args, **kwargs) - m.from_settings.assert_called_once_with(crawler.settings, *args, - **kwargs) + m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs) with self.assertRaises(ValueError): create_instance(m, None, None) @@ -152,25 +155,25 @@ class UtilsMiscTestCase(unittest.TestCase): create_instance(m, settings, None) def test_set_environ(self): - assert os.environ.get('some_test_environ') is None - with set_environ(some_test_environ='test_value'): - assert os.environ.get('some_test_environ') == 'test_value' - assert os.environ.get('some_test_environ') is None + assert os.environ.get("some_test_environ") is None + with set_environ(some_test_environ="test_value"): + assert os.environ.get("some_test_environ") == "test_value" + assert os.environ.get("some_test_environ") is None - os.environ['some_test_environ'] = 'test' - assert os.environ.get('some_test_environ') == 'test' - with set_environ(some_test_environ='test_value'): - assert os.environ.get('some_test_environ') == 'test_value' - assert os.environ.get('some_test_environ') == 'test' + os.environ["some_test_environ"] = "test" + assert os.environ.get("some_test_environ") == "test" + with set_environ(some_test_environ="test_value"): + assert os.environ.get("some_test_environ") == "test_value" + assert os.environ.get("some_test_environ") == "test" def test_rel_has_nofollow(self): - assert rel_has_nofollow('ugc nofollow') is True - assert rel_has_nofollow('ugc,nofollow') is True - assert rel_has_nofollow('ugc') is False - assert rel_has_nofollow('nofollow') is True - assert rel_has_nofollow('nofollowfoo') is False - assert rel_has_nofollow('foonofollow') is False - assert rel_has_nofollow('ugc, , nofollow') is True + assert rel_has_nofollow("ugc nofollow") is True + assert rel_has_nofollow("ugc,nofollow") is True + assert rel_has_nofollow("ugc") is False + assert rel_has_nofollow("nofollow") is True + assert rel_has_nofollow("nofollowfoo") is False + assert rel_has_nofollow("foonofollow") is False + assert rel_has_nofollow("ugc, , nofollow") is True if __name__ == "__main__": diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 562f72fee..484757035 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -3,7 +3,10 @@ import warnings from functools import partial from unittest import mock -from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value +from scrapy.utils.misc import ( + is_generator_with_return_value, + warn_on_generator_with_return_value, +) def _indentation_error(*args, **kwargs): @@ -12,7 +15,7 @@ def _indentation_error(*args, **kwargs): def top_level_return_something(): """ -docstring + docstring """ url = """ https://example.org @@ -23,7 +26,7 @@ https://example.org def top_level_return_none(): """ -docstring + docstring """ url = """ https://example.org @@ -39,7 +42,6 @@ def generator_that_returns_stuff(): class UtilsMiscPy3TestCase(unittest.TestCase): - def test_generators_return_something(self): def f1(): yield 1 @@ -60,7 +62,7 @@ class UtilsMiscPy3TestCase(unittest.TestCase): def i1(): """ -docstring + docstring """ url = """ https://example.org @@ -77,7 +79,10 @@ https://example.org with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_something) self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.top_level_return_something" method is a generator', str(w[0].message)) + self.assertIn( + 'The "NoneType.top_level_return_something" method is a generator', + str(w[0].message), + ) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, f1) self.assertEqual(len(w), 1) @@ -121,7 +126,7 @@ https://example.org def k2(): """ -docstring + docstring """ url = """ https://example.org @@ -170,6 +175,7 @@ https://example.org def decorator(func): def inner_func(): func() + return inner_func @decorator @@ -203,7 +209,7 @@ https://example.org @decorator def k3(): """ -docstring + docstring """ url = """ https://example.org @@ -249,12 +255,14 @@ https://example.org warn_on_generator_with_return_value(None, l3) self.assertEqual(len(w), 0) - @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error) + @mock.patch( + "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error + ) def test_indentation_error(self): with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_none) self.assertEqual(len(w), 1) - self.assertIn('Unable to determine', str(w[0].message)) + self.assertIn("Unable to determine", str(w[0].message)) def test_partial(self): def cb(arg1, arg2): diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 867e12103..b08e5f475 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -16,7 +16,7 @@ def inside_a_project(): try: os.chdir(project_dir) - Path('scrapy.cfg').touch() + Path("scrapy.cfg").touch() yield project_dir finally: @@ -26,21 +26,15 @@ def inside_a_project(): class ProjectUtilsTest(unittest.TestCase): def test_data_path_outside_project(self): - self.assertEqual( - str(Path('.scrapy', 'somepath')), - data_path('somepath') - ) - abspath = str(Path(os.path.sep, 'absolute', 'path')) + self.assertEqual(str(Path(".scrapy", "somepath")), data_path("somepath")) + abspath = str(Path(os.path.sep, "absolute", "path")) self.assertEqual(abspath, data_path(abspath)) def test_data_path_inside_project(self): with inside_a_project() as proj_path: - expected = Path(proj_path, '.scrapy', 'somepath') - self.assertEqual( - expected.resolve(), - Path(data_path('somepath')).resolve() - ) - abspath = str(Path(os.path.sep, 'absolute', 'path').resolve()) + expected = Path(proj_path, ".scrapy", "somepath") + self.assertEqual(expected.resolve(), Path(data_path("somepath")).resolve()) + abspath = str(Path(os.path.sep, "absolute", "path").resolve()) self.assertEqual(abspath, data_path(abspath)) @@ -59,22 +53,21 @@ def set_env(**update): class GetProjectSettingsTestCase(unittest.TestCase): - def test_valid_envvar(self): - value = 'tests.test_cmdline.settings' + value = "tests.test_cmdline.settings" envvars = { - 'SCRAPY_SETTINGS_MODULE': value, + "SCRAPY_SETTINGS_MODULE": value, } with warnings.catch_warnings(): warnings.simplefilter("error") with set_env(**envvars): settings = get_project_settings() - assert settings.get('SETTINGS_MODULE') == value + assert settings.get("SETTINGS_MODULE") == value def test_invalid_envvar(self): envvars = { - 'SCRAPY_FOO': 'bar', + "SCRAPY_FOO": "bar", } with set_env(**envvars): settings = get_project_settings() @@ -82,12 +75,12 @@ class GetProjectSettingsTestCase(unittest.TestCase): assert settings.get("SCRAPY_FOO") is None def test_valid_and_invalid_envvars(self): - value = 'tests.test_cmdline.settings' + value = "tests.test_cmdline.settings" envvars = { - 'SCRAPY_FOO': 'bar', - 'SCRAPY_SETTINGS_MODULE': value, + "SCRAPY_FOO": "bar", + "SCRAPY_SETTINGS_MODULE": value, } with set_env(**envvars): settings = get_project_settings() - assert settings.get('SETTINGS_MODULE') == value - assert settings.get('SCRAPY_FOO') is None + assert settings.get("SETTINGS_MODULE") == value + assert settings.get("SCRAPY_FOO") is None diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 403e4f8fe..3c6270864 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -7,12 +7,19 @@ from twisted.trial import unittest from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback from scrapy.utils.python import ( - memoizemethod_noargs, binary_is_text, equal_attributes, - get_func_args, to_bytes, to_unicode, - without_none_values, MutableChain, MutableAsyncChain) + memoizemethod_noargs, + binary_is_text, + equal_attributes, + get_func_args, + to_bytes, + to_unicode, + without_none_values, + MutableChain, + MutableAsyncChain, +) -__doctests__ = ['scrapy.utils.python'] +__doctests__ = ["scrapy.utils.python"] class MutableChainTest(unittest.TestCase): @@ -79,48 +86,41 @@ class MutableAsyncChainTest(unittest.TestCase): class ToUnicodeTest(unittest.TestCase): def test_converting_an_utf8_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e') + self.assertEqual(to_unicode(b"lel\xc3\xb1e"), "lel\xf1e") def test_converting_a_latin_1_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e') + self.assertEqual(to_unicode(b"lel\xf1e", "latin-1"), "lel\xf1e") def test_converting_a_unicode_to_unicode_should_return_the_same_object(self): - self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e') + self.assertEqual(to_unicode("\xf1e\xf1e\xf1e"), "\xf1e\xf1e\xf1e") def test_converting_a_strange_object_should_raise_TypeError(self): self.assertRaises(TypeError, to_unicode, 423) def test_errors_argument(self): - self.assertEqual( - to_unicode(b'a\xedb', 'utf-8', errors='replace'), - 'a\ufffdb' - ) + self.assertEqual(to_unicode(b"a\xedb", "utf-8", errors="replace"), "a\ufffdb") class ToBytesTest(unittest.TestCase): def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self): - self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49') + self.assertEqual(to_bytes("\xa3 49"), b"\xc2\xa3 49") def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self): - self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49') + self.assertEqual(to_bytes("\xa3 49", "latin-1"), b"\xa3 49") def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self): - self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e') + self.assertEqual(to_bytes(b"lel\xf1e"), b"lel\xf1e") def test_converting_a_strange_object_should_raise_TypeError(self): self.assertRaises(TypeError, to_bytes, unittest) def test_errors_argument(self): - self.assertEqual( - to_bytes('a\ufffdb', 'latin-1', errors='replace'), - b'a?b' - ) + self.assertEqual(to_bytes("a\ufffdb", "latin-1", errors="replace"), b"a?b") class MemoizedMethodTest(unittest.TestCase): def test_memoizemethod_noargs(self): class A: - @memoizemethod_noargs def cached(self): return object() @@ -141,7 +141,7 @@ class BinaryIsTextTest(unittest.TestCase): assert binary_is_text(b"hello") def test_utf_16_strings_contain_null_bytes(self): - assert binary_is_text("hello".encode('utf-16')) + assert binary_is_text("hello".encode("utf-16")) def test_one_with_encoding(self): assert binary_is_text(b"
Price \xa3
") @@ -151,7 +151,6 @@ class BinaryIsTextTest(unittest.TestCase): class UtilsPythonTestCase(unittest.TestCase): - def test_equal_attributes(self): class Obj: pass @@ -161,44 +160,44 @@ class UtilsPythonTestCase(unittest.TestCase): # no attributes given return False self.assertFalse(equal_attributes(a, b, [])) # not existent attributes - self.assertFalse(equal_attributes(a, b, ['x', 'y'])) + self.assertFalse(equal_attributes(a, b, ["x", "y"])) a.x = 1 b.x = 1 # equal attribute - self.assertTrue(equal_attributes(a, b, ['x'])) + self.assertTrue(equal_attributes(a, b, ["x"])) b.y = 2 # obj1 has no attribute y - self.assertFalse(equal_attributes(a, b, ['x', 'y'])) + self.assertFalse(equal_attributes(a, b, ["x", "y"])) a.y = 2 # equal attributes - self.assertTrue(equal_attributes(a, b, ['x', 'y'])) + self.assertTrue(equal_attributes(a, b, ["x", "y"])) a.y = 1 # differente attributes - self.assertFalse(equal_attributes(a, b, ['x', 'y'])) + self.assertFalse(equal_attributes(a, b, ["x", "y"])) # test callable a.meta = {} b.meta = {} - self.assertTrue(equal_attributes(a, b, ['meta'])) + self.assertTrue(equal_attributes(a, b, ["meta"])) # compare ['meta']['a'] - a.meta['z'] = 1 - b.meta['z'] = 1 + a.meta["z"] = 1 + b.meta["z"] = 1 - get_z = operator.itemgetter('z') - get_meta = operator.attrgetter('meta') + get_z = operator.itemgetter("z") + get_meta = operator.attrgetter("meta") def compare_z(obj): return get_z(get_meta(obj)) - self.assertTrue(equal_attributes(a, b, [compare_z, 'x'])) + self.assertTrue(equal_attributes(a, b, [compare_z, "x"])) # fail z equality - a.meta['z'] = 2 - self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) + a.meta["z"] = 2 + self.assertFalse(equal_attributes(a, b, [compare_z, "x"])) def test_get_func_args(self): def f1(a, b, c): @@ -218,7 +217,6 @@ class UtilsPythonTestCase(unittest.TestCase): pass class Callable: - def __call__(self, a, b, c): pass @@ -228,33 +226,38 @@ class UtilsPythonTestCase(unittest.TestCase): partial_f2 = functools.partial(f1, b=None) partial_f3 = functools.partial(partial_f2, None) - self.assertEqual(get_func_args(f1), ['a', 'b', 'c']) - self.assertEqual(get_func_args(f2), ['a', 'b', 'c']) - self.assertEqual(get_func_args(f3), ['a', 'b', 'c']) - self.assertEqual(get_func_args(A), ['a', 'b', 'c']) - self.assertEqual(get_func_args(a.method), ['a', 'b', 'c']) - self.assertEqual(get_func_args(partial_f1), ['b', 'c']) - self.assertEqual(get_func_args(partial_f2), ['a', 'c']) - self.assertEqual(get_func_args(partial_f3), ['c']) - self.assertEqual(get_func_args(cal), ['a', 'b', 'c']) + self.assertEqual(get_func_args(f1), ["a", "b", "c"]) + self.assertEqual(get_func_args(f2), ["a", "b", "c"]) + self.assertEqual(get_func_args(f3), ["a", "b", "c"]) + self.assertEqual(get_func_args(A), ["a", "b", "c"]) + self.assertEqual(get_func_args(a.method), ["a", "b", "c"]) + self.assertEqual(get_func_args(partial_f1), ["b", "c"]) + self.assertEqual(get_func_args(partial_f2), ["a", "c"]) + self.assertEqual(get_func_args(partial_f3), ["c"]) + self.assertEqual(get_func_args(cal), ["a", "b", "c"]) self.assertEqual(get_func_args(object), []) - if platform.python_implementation() == 'CPython': + if platform.python_implementation() == "CPython": # TODO: how do we fix this to return the actual argument names? self.assertEqual(get_func_args(str.split), []) self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) - elif platform.python_implementation() == 'PyPy': - self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) - self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj']) - self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable']) + elif platform.python_implementation() == "PyPy": + self.assertEqual( + get_func_args(str.split, stripself=True), ["sep", "maxsplit"] + ) + self.assertEqual( + get_func_args(operator.itemgetter(2), stripself=True), ["obj"] + ) + self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"]) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4)) self.assertEqual( - without_none_values({'one': 1, 'none': None, 'three': 3, 'four': 4}), - {'one': 1, 'three': 3, 'four': 4}) + without_none_values({"one": 1, "none": None, "three": 3, "four": 4}), + {"one": 1, "three": 3, "four": 4}, + ) if __name__ == "__main__": diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index a92d9a0ac..4760bf92b 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -23,24 +23,32 @@ from scrapy.utils.test import get_crawler class UtilsRequestTest(unittest.TestCase): - def test_request_authenticate(self): r = Request("http://www.example.com") - request_authenticate(r, 'someuser', 'somepass') - self.assertEqual(r.headers['Authorization'], b'Basic c29tZXVzZXI6c29tZXBhc3M=') + request_authenticate(r, "someuser", "somepass") + self.assertEqual(r.headers["Authorization"], b"Basic c29tZXVzZXI6c29tZXBhc3M=") def test_request_httprepr(self): r1 = Request("http://www.example.com") - self.assertEqual(request_httprepr(r1), b'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n') + self.assertEqual( + request_httprepr(r1), b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n" + ) r1 = Request("http://www.example.com/some/page.html?arg=1") - self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n') - - r1 = Request("http://www.example.com", method='POST', - headers={"Content-type": b"text/html"}, body=b"Some body") self.assertEqual( request_httprepr(r1), - b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body' + b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + ) + + r1 = Request( + "http://www.example.com", + method="POST", + headers={"Content-type": b"text/html"}, + body=b"Some body", + ) + self.assertEqual( + request_httprepr(r1), + b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body", ) def test_request_httprepr_for_non_http_request(self): @@ -61,77 +69,77 @@ class FingerprintTest(unittest.TestCase): known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( ( Request("http://example.org"), - b'xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD', + b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD", {}, ), ( Request("https://example.org"), - b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l', + b"\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l", {}, ), ( Request("https://example.org?a"), - b'G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c', + b"G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c", {}, ), ( Request("https://example.org?a=b"), - b'\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17', + b"\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17", {}, ), ( Request("https://example.org?a=b&a"), - b't+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96', + b"t+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96", {}, ), ( Request("https://example.org?a=b&a=c"), - b'\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef', + b"\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef", {}, ), ( - Request("https://example.org", method='POST'), - b'\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d', + Request("https://example.org", method="POST"), + b"\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d", {}, ), ( - Request("https://example.org", body=b'a'), - b'\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_', + Request("https://example.org", body=b"a"), + b"\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_", {}, ), ( - Request("https://example.org", method='POST', body=b'a'), - b'5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7', + Request("https://example.org", method="POST", body=b"a"), + b"5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7", {}, ), ( - Request("https://example.org#a", headers={'A': b'B'}), - b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l', + Request("https://example.org#a", headers={"A": b"B"}), + b"\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l", {}, ), ( - Request("https://example.org#a", headers={'A': b'B'}), - b']\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.', - {'include_headers': ['A']}, + Request("https://example.org#a", headers={"A": b"B"}), + b"]\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.", + {"include_headers": ["A"]}, ), ( - Request("https://example.org#a", headers={'A': b'B'}), - b'<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f', - {'keep_fragments': True}, + Request("https://example.org#a", headers={"A": b"B"}), + b"<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f", + {"keep_fragments": True}, ), ( - Request("https://example.org#a", headers={'A': b'B'}), - b'\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11', - {'include_headers': ['A'], 'keep_fragments': True}, + Request("https://example.org#a", headers={"A": b"B"}), + b"\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11", + {"include_headers": ["A"], "keep_fragments": True}, ), ( Request("https://example.org/ab"), - b'N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6', + b"N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6", {}, ), ( - Request("https://example.org/a", body=b'b'), - b'_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5', + Request("https://example.org/a", body=b"b"), + b"_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5", {}, ), ) @@ -143,42 +151,42 @@ class FingerprintTest(unittest.TestCase): self.assertEqual(self.function(r1), self.function(r2)) def test_query_string_key_without_value(self): - r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199') - r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199') + r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78132,199") + r2 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") self.assertNotEqual(self.function(r1), self.function(r2)) def test_caching(self): - r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199') - self.assertEqual( - self.function(r1), - self.cache[r1][self.default_cache_key] - ) + r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") + self.assertEqual(self.function(r1), self.cache[r1][self.default_cache_key]) def test_header(self): r1 = Request("http://www.example.com/members/offers.html") r2 = Request("http://www.example.com/members/offers.html") - r2.headers['SESSIONID'] = b"somehash" + r2.headers["SESSIONID"] = b"somehash" self.assertEqual(self.function(r1), self.function(r2)) def test_headers(self): r1 = Request("http://www.example.com/") r2 = Request("http://www.example.com/") - r2.headers['Accept-Language'] = b'en' + r2.headers["Accept-Language"] = b"en" r3 = Request("http://www.example.com/") - r3.headers['Accept-Language'] = b'en' - r3.headers['SESSIONID'] = b"somehash" + r3.headers["Accept-Language"] = b"en" + r3.headers["SESSIONID"] = b"somehash" self.assertEqual(self.function(r1), self.function(r2), self.function(r3)) - self.assertEqual(self.function(r1), - self.function(r1, include_headers=['Accept-Language'])) + self.assertEqual( + self.function(r1), self.function(r1, include_headers=["Accept-Language"]) + ) self.assertNotEqual( - self.function(r1), - self.function(r2, include_headers=['Accept-Language'])) + self.function(r1), self.function(r2, include_headers=["Accept-Language"]) + ) - self.assertEqual(self.function(r3, include_headers=['accept-language', 'sessionid']), - self.function(r3, include_headers=['SESSIONID', 'Accept-Language'])) + self.assertEqual( + self.function(r3, include_headers=["accept-language", "sessionid"]), + self.function(r3, include_headers=["SESSIONID", "Accept-Language"]), + ) def test_fragment(self): r1 = Request("http://www.example.com/test.html") @@ -190,8 +198,8 @@ class FingerprintTest(unittest.TestCase): def test_method_and_body(self): r1 = Request("http://www.example.com") - r2 = Request("http://www.example.com", method='POST') - r3 = Request("http://www.example.com", method='POST', body=b'request body') + r2 = Request("http://www.example.com", method="POST") + r3 = Request("http://www.example.com", method="POST", body=b"request body") self.assertNotEqual(self.function(r1), self.function(r2)) self.assertNotEqual(self.function(r2), self.function(r3)) @@ -209,7 +217,7 @@ class FingerprintTest(unittest.TestCase): # would put the body right after the URL. r1 = Request("http://www.example.com/foo") fp1 = self.function(r1) - r2 = Request("http://www.example.com/f", body=b'oo') + r2 = Request("http://www.example.com/f", body=b"oo") fp2 = self.function(r2) self.assertNotEqual(fp1, fp2) @@ -217,13 +225,9 @@ class FingerprintTest(unittest.TestCase): """Test hardcoded hashes, to make sure future changes to not introduce backward incompatibilities.""" actual = [ - self.function(request, **kwargs) - for request, _, kwargs in self.known_hashes - ] - expected = [ - _fingerprint - for _, _fingerprint, _ in self.known_hashes + self.function(request, **kwargs) for request, _, kwargs in self.known_hashes ] + expected = [_fingerprint for _, _fingerprint, _ in self.known_hashes] self.assertEqual(actual, expected) @@ -233,77 +237,77 @@ class RequestFingerprintTest(FingerprintTest): known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( ( Request("http://example.org"), - 'b2e5245ef826fd9576c93bd6e392fce3133fab62', + "b2e5245ef826fd9576c93bd6e392fce3133fab62", {}, ), ( Request("https://example.org"), - 'bd10a0a89ea32cdee77917320f1309b0da87e892', + "bd10a0a89ea32cdee77917320f1309b0da87e892", {}, ), ( Request("https://example.org?a"), - '2fb7d48ae02f04b749f40caa969c0bc3c43204ce', + "2fb7d48ae02f04b749f40caa969c0bc3c43204ce", {}, ), ( Request("https://example.org?a=b"), - '42e5fe149b147476e3f67ad0670c57b4cc57856a', + "42e5fe149b147476e3f67ad0670c57b4cc57856a", {}, ), ( Request("https://example.org?a=b&a"), - 'd23a9787cb56c6375c2cae4453c5a8c634526942', + "d23a9787cb56c6375c2cae4453c5a8c634526942", {}, ), ( Request("https://example.org?a=b&a=c"), - '9a18a7a8552a9182b7f1e05d33876409e421e5c5', + "9a18a7a8552a9182b7f1e05d33876409e421e5c5", {}, ), ( - Request("https://example.org", method='POST'), - 'ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6', + Request("https://example.org", method="POST"), + "ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6", {}, ), ( - Request("https://example.org", body=b'a'), - '4bb136e54e715a4ea7a9dd1101831765d33f2d60', + Request("https://example.org", body=b"a"), + "4bb136e54e715a4ea7a9dd1101831765d33f2d60", {}, ), ( - Request("https://example.org", method='POST', body=b'a'), - '6c6595374a304b293be762f7b7be3f54e9947c65', + Request("https://example.org", method="POST", body=b"a"), + "6c6595374a304b293be762f7b7be3f54e9947c65", {}, ), ( - Request("https://example.org#a", headers={'A': b'B'}), - 'bd10a0a89ea32cdee77917320f1309b0da87e892', + Request("https://example.org#a", headers={"A": b"B"}), + "bd10a0a89ea32cdee77917320f1309b0da87e892", {}, ), ( - Request("https://example.org#a", headers={'A': b'B'}), - '515b633cb3ca502a33a9d8c890e889ec1e425e65', - {'include_headers': ['A']}, + Request("https://example.org#a", headers={"A": b"B"}), + "515b633cb3ca502a33a9d8c890e889ec1e425e65", + {"include_headers": ["A"]}, ), ( - Request("https://example.org#a", headers={'A': b'B'}), - '505c96e7da675920dfef58725e8c957dfdb38f47', - {'keep_fragments': True}, + Request("https://example.org#a", headers={"A": b"B"}), + "505c96e7da675920dfef58725e8c957dfdb38f47", + {"keep_fragments": True}, ), ( - Request("https://example.org#a", headers={'A': b'B'}), - 'd6f673cdcb661b7970c2b9a00ee63e87d1e2e5da', - {'include_headers': ['A'], 'keep_fragments': True}, + Request("https://example.org#a", headers={"A": b"B"}), + "d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da", + {"include_headers": ["A"], "keep_fragments": True}, ), ( Request("https://example.org/ab"), - '4e2870fee58582d6f81755e9b8fdefe3cba0c951', + "4e2870fee58582d6f81755e9b8fdefe3cba0c951", {}, ), ( - Request("https://example.org/a", body=b'b'), - '4e2870fee58582d6f81755e9b8fdefe3cba0c951', + Request("https://example.org/a", body=b"b"), + "4e2870fee58582d6f81755e9b8fdefe3cba0c951", {}, ), ) @@ -314,36 +318,29 @@ class RequestFingerprintTest(FingerprintTest): def tearDown(self) -> None: warnings.simplefilter("default", ScrapyDeprecationWarning) - @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True) + @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True) def test_part_separation(self): super().test_part_separation() class RequestFingerprintDeprecationTest(unittest.TestCase): - def test_deprecation_default_parameters(self): with pytest.warns(ScrapyDeprecationWarning) as warnings: request_fingerprint(Request("http://www.example.com")) messages = [str(warning.message) for warning in warnings] self.assertTrue( - any( - 'Call to deprecated function' in message - for message in messages - ) + any("Call to deprecated function" in message for message in messages) ) - self.assertFalse(any('non-default' in message for message in messages)) + self.assertFalse(any("non-default" in message for message in messages)) def test_deprecation_non_default_parameters(self): with pytest.warns(ScrapyDeprecationWarning) as warnings: request_fingerprint(Request("http://www.example.com"), keep_fragments=True) messages = [str(warning.message) for warning in warnings] self.assertTrue( - any( - 'Call to deprecated function' in message - for message in messages - ) + any("Call to deprecated function" in message for message in messages) ) - self.assertTrue(any('non-default' in message for message in messages)) + self.assertTrue(any("non-default" in message for message in messages)) class RequestFingerprintAsBytesTest(FingerprintTest): @@ -352,24 +349,21 @@ class RequestFingerprintAsBytesTest(FingerprintTest): known_hashes = RequestFingerprintTest.known_hashes def test_caching(self): - r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199') + r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") self.assertEqual( - self.function(r1), - bytes.fromhex(self.cache[r1][self.default_cache_key]) + self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key]) ) - @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True) + @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True) def test_part_separation(self): super().test_part_separation() def test_hashes(self): actual = [ - self.function(request, **kwargs) - for request, _, kwargs in self.known_hashes + self.function(request, **kwargs) for request, _, kwargs in self.known_hashes ] expected = [ - bytes.fromhex(_fingerprint) - for _, _fingerprint, _ in self.known_hashes + bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes ] self.assertEqual(actual, expected) @@ -385,8 +379,10 @@ def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False) if cache_key not in cache: fp = sha1() fp.update(to_bytes(request.method)) - fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) - fp.update(request.body or b'') + fp.update( + to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)) + ) + fp.update(request.body or b"") if include_headers: for hdr in include_headers: if hdr in request.headers: @@ -401,39 +397,38 @@ REQUEST_OBJECTS_TO_TEST = ( Request("http://www.example.com/"), Request("http://www.example.com/query?id=111&cat=222"), Request("http://www.example.com/query?cat=222&id=111"), - Request('http://www.example.com/hnnoticiaj1.aspx?78132,199'), - Request('http://www.example.com/hnnoticiaj1.aspx?78160,199'), + Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"), + Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"), Request("http://www.example.com/members/offers.html"), Request( "http://www.example.com/members/offers.html", - headers={'SESSIONID': b"somehash"}, + headers={"SESSIONID": b"somehash"}, ), Request( "http://www.example.com/", - headers={'Accept-Language': b"en"}, + headers={"Accept-Language": b"en"}, ), Request( "http://www.example.com/", headers={ - 'Accept-Language': b"en", - 'SESSIONID': b"somehash", + "Accept-Language": b"en", + "SESSIONID": b"somehash", }, ), Request("http://www.example.com/test.html"), Request("http://www.example.com/test.html#fragment"), - Request("http://www.example.com", method='POST'), - Request("http://www.example.com", method='POST', body=b'request body'), + Request("http://www.example.com", method="POST"), + Request("http://www.example.com", method="POST", body=b"request body"), ) class BackwardCompatibilityTestCase(unittest.TestCase): - def test_function_backward_compatibility(self): include_headers_to_test = ( None, - ['Accept-Language'], - ['accept-language', 'sessionid'], - ['SESSIONID', 'Accept-Language'], + ["Accept-Language"], + ["accept-language", "sessionid"], + ["SESSIONID", "Accept-Language"], ) for request_object in REQUEST_OBJECTS_TO_TEST: for include_headers in include_headers_to_test: @@ -475,14 +470,14 @@ class BackwardCompatibilityTestCase(unittest.TestCase): fp = sha1() fp.update(to_bytes(request.method)) fp.update(to_bytes(canonicalize_url(request.url))) - fp.update(request.body or b'') + fp.update(request.body or b"") self.cache[request] = fp.digest() return self.cache[request] for request_object in REQUEST_OBJECTS_TO_TEST: with warnings.catch_warnings() as logged_warnings: settings = { - 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } crawler = get_crawler(settings_dict=settings) fp = crawler.request_fingerprinter.fingerprint(request_object) @@ -492,11 +487,10 @@ class BackwardCompatibilityTestCase(unittest.TestCase): class RequestFingerprinterTestCase(unittest.TestCase): - def test_default_implementation(self): with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(prevent_warnings=False) - request = Request('https://example.com') + request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), _request_fingerprint_as_bytes(request), @@ -505,11 +499,11 @@ class RequestFingerprinterTestCase(unittest.TestCase): def test_deprecated_implementation(self): settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.6', + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6", } with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) - request = Request('https://example.com') + request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), _request_fingerprint_as_bytes(request), @@ -518,11 +512,11 @@ class RequestFingerprinterTestCase(unittest.TestCase): def test_recommended_implementation(self): settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) - request = Request('https://example.com') + request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), fingerprint(request), @@ -531,34 +525,30 @@ class RequestFingerprinterTestCase(unittest.TestCase): def test_unknown_implementation(self): settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.5', + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5", } with self.assertRaises(ValueError): get_crawler(settings_dict=settings) class CustomRequestFingerprinterTestCase(unittest.TestCase): - def test_include_headers(self): - class RequestFingerprinter: - def fingerprint(self, request): - return fingerprint(request, include_headers=['X-ID']) + return fingerprint(request, include_headers=["X-ID"]) settings = { - 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } crawler = get_crawler(settings_dict=settings) - r1 = Request("http://www.example.com", headers={'X-ID': '1'}) + r1 = Request("http://www.example.com", headers={"X-ID": "1"}) fp1 = crawler.request_fingerprinter.fingerprint(r1) - r2 = Request("http://www.example.com", headers={'X-ID': '2'}) + r2 = Request("http://www.example.com", headers={"X-ID": "2"}) fp2 = crawler.request_fingerprinter.fingerprint(r2) self.assertNotEqual(fp1, fp2) def test_dont_canonicalize(self): - class RequestFingerprinter: cache = WeakKeyDictionary() @@ -570,7 +560,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): return self.cache[request] settings = { - 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } crawler = get_crawler(settings_dict=settings) @@ -581,26 +571,24 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): self.assertNotEqual(fp1, fp2) def test_meta(self): - class RequestFingerprinter: - def fingerprint(self, request): - if 'fingerprint' in request.meta: - return request.meta['fingerprint'] + if "fingerprint" in request.meta: + return request.meta["fingerprint"] return fingerprint(request) settings = { - 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } crawler = get_crawler(settings_dict=settings) r1 = Request("http://www.example.com") fp1 = crawler.request_fingerprinter.fingerprint(r1) - r2 = Request("http://www.example.com", meta={'fingerprint': 'a'}) + r2 = Request("http://www.example.com", meta={"fingerprint": "a"}) fp2 = crawler.request_fingerprinter.fingerprint(r2) - r3 = Request("http://www.example.com", meta={'fingerprint': 'a'}) + r3 = Request("http://www.example.com", meta={"fingerprint": "a"}) fp3 = crawler.request_fingerprinter.fingerprint(r3) - r4 = Request("http://www.example.com", meta={'fingerprint': 'b'}) + r4 = Request("http://www.example.com", meta={"fingerprint": "b"}) fp4 = crawler.request_fingerprinter.fingerprint(r4) self.assertNotEqual(fp1, fp2) self.assertNotEqual(fp1, fp4) @@ -608,55 +596,50 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): self.assertEqual(fp2, fp3) def test_from_crawler(self): - class RequestFingerprinter: - @classmethod def from_crawler(cls, crawler): return cls(crawler) def __init__(self, crawler): - self._fingerprint = crawler.settings['FINGERPRINT'] + self._fingerprint = crawler.settings["FINGERPRINT"] def fingerprint(self, request): return self._fingerprint settings = { - 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, - 'FINGERPRINT': b'fingerprint', + "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, + "FINGERPRINT": b"fingerprint", } crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) - self.assertEqual(fingerprint, settings['FINGERPRINT']) + self.assertEqual(fingerprint, settings["FINGERPRINT"]) def test_from_settings(self): - class RequestFingerprinter: - @classmethod def from_settings(cls, settings): return cls(settings) def __init__(self, settings): - self._fingerprint = settings['FINGERPRINT'] + self._fingerprint = settings["FINGERPRINT"] def fingerprint(self, request): return self._fingerprint settings = { - 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, - 'FINGERPRINT': b'fingerprint', + "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, + "FINGERPRINT": b"fingerprint", } crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) - self.assertEqual(fingerprint, settings['FINGERPRINT']) + self.assertEqual(fingerprint, settings["FINGERPRINT"]) def test_from_crawler_and_settings(self): - class RequestFingerprinter: # This method is ignored due to the presence of from_crawler @@ -669,20 +652,20 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): return cls(crawler) def __init__(self, crawler): - self._fingerprint = crawler.settings['FINGERPRINT'] + self._fingerprint = crawler.settings["FINGERPRINT"] def fingerprint(self, request): return self._fingerprint settings = { - 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, - 'FINGERPRINT': b'fingerprint', + "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, + "FINGERPRINT": b"fingerprint", } crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) - self.assertEqual(fingerprint, settings['FINGERPRINT']) + self.assertEqual(fingerprint, settings["FINGERPRINT"]) if __name__ == "__main__": diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index cdf972933..d82aa19c6 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -6,32 +6,49 @@ from urllib.parse import urlparse from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse, HtmlResponse from scrapy.utils.python import to_bytes -from scrapy.utils.response import (response_httprepr, open_in_browser, - get_meta_refresh, get_base_url, response_status_message) +from scrapy.utils.response import ( + response_httprepr, + open_in_browser, + get_meta_refresh, + get_base_url, + response_status_message, +) -__doctests__ = ['scrapy.utils.response'] +__doctests__ = ["scrapy.utils.response"] class ResponseUtilsTest(unittest.TestCase): - dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response') + dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") def test_response_httprepr(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) r1 = Response("http://www.example.com") - self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n') + self.assertEqual(response_httprepr(r1), b"HTTP/1.1 200 OK\r\n\r\n") - r1 = Response("http://www.example.com", status=404, - headers={"Content-type": "text/html"}, body=b"Some body") - self.assertEqual(response_httprepr(r1), - b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body') + r1 = Response( + "http://www.example.com", + status=404, + headers={"Content-type": "text/html"}, + body=b"Some body", + ) + self.assertEqual( + response_httprepr(r1), + b"HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body", + ) - r1 = Response("http://www.example.com", status=6666, - headers={"Content-type": "text/html"}, body=b"Some body") - self.assertEqual(response_httprepr(r1), - b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body') + r1 = Response( + "http://www.example.com", + status=6666, + headers={"Content-type": "text/html"}, + body=b"Some body", + ) + self.assertEqual( + response_httprepr(r1), + b"HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body", + ) def test_open_in_browser(self): url = "http:///www.example.com/some/page.html" @@ -40,10 +57,11 @@ class ResponseUtilsTest(unittest.TestCase): def browser_open(burl): path = urlparse(burl).path if not path or not Path(path).exists(): - path = burl.replace('file://', '') + path = burl.replace("file://", "") bbody = Path(path).read_bytes() self.assertIn(b'', bbody) return True + response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=browser_open), "Browser not called" @@ -51,45 +69,60 @@ class ResponseUtilsTest(unittest.TestCase): self.assertRaises(TypeError, open_in_browser, resp, debug=True) def test_get_meta_refresh(self): - r1 = HtmlResponse("http://www.example.com", body=b""" + r1 = HtmlResponse( + "http://www.example.com", + body=b""" Dummy blahablsdfsal& - """) - r2 = HtmlResponse("http://www.example.com", body=b""" + """, + ) + r2 = HtmlResponse( + "http://www.example.com", + body=b""" Dummy blahablsdfsal& - """) - r3 = HtmlResponse("http://www.example.com", body=b""" + """, + ) + r3 = HtmlResponse( + "http://www.example.com", + body=b"""
sample 3 text sample 3 repetition + sample 3 repetition sample 3 repetition with fragment inner tag diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 74e170ec0..07aef2ee1 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -541,6 +541,7 @@ class TextResponseTest(BaseResponseTest): 'http://example.com/sample2.html', 'http://example.com/sample3.html', 'http://example.com/sample3.html', + 'http://example.com/sample3.html', 'http://example.com/sample3.html#foo', 'http://www.google.com/something', 'http://example.com/innertag.html' diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index e28dc9bdb..6c34a96a0 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -53,6 +53,7 @@ class Base: Link(url='http://example.com/sample2.html', text='sample 2'), Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html', text='sample 3 repetition'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) @@ -64,6 +65,7 @@ class Base: Link(url='http://example.com/sample2.html', text='sample 2'), Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html', text='sample 3 repetition'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment') ]) From faa5bd0f6b688eaef18a6245f2d703e8fc8ff684 Mon Sep 17 00:00:00 2001 From: silviopavanetto Date: Wed, 11 Jan 2023 20:30:57 +0100 Subject: [PATCH 0770/2083] Update scrapy/linkextractors/lxmlhtml.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/linkextractors/lxmlhtml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 8a4175d49..f772df987 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -195,7 +195,7 @@ class LxmlLinkExtractor: Only links that match the settings passed to the ``__init__`` method of the link extractor are returned. - Duplicate links are omitted if the ``unique`` parameter is set to ``True``, + Duplicate links are omitted if the ``unique`` attribute is set to ``True``, otherwise they are returned. """ base_url = get_base_url(response) From 36b89a4b20f12a84123930664eab888dfdf3dd8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 12 Jan 2023 08:36:31 +0100 Subject: [PATCH 0771/2083] Remove trailing whitespace --- scrapy/linkextractors/lxmlhtml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index f772df987..3f90ed84a 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -195,7 +195,7 @@ class LxmlLinkExtractor: Only links that match the settings passed to the ``__init__`` method of the link extractor are returned. - Duplicate links are omitted if the ``unique`` attribute is set to ``True``, + Duplicate links are omitted if the ``unique`` attribute is set to ``True``, otherwise they are returned. """ base_url = get_base_url(response) From 93ad6a4bc2fd2d453a37961b817a8a2a85f589c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 13 Jan 2023 20:39:44 +0400 Subject: [PATCH 0772/2083] Simplify code for modern pyOpenSSL. --- scrapy/core/downloader/tls.py | 8 ++++---- scrapy/utils/ssl.py | 14 ++------------ tests/mockserver.py | 3 +-- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 7d67a426f..65028d21f 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -17,10 +17,10 @@ METHOD_TLSv12 = 'TLSv1.2' openssl_methods = { - METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only - METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only - METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only + METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) + METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only + METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only + METHOD_TLSv12: SSL.TLSv1_2_METHOD, # TLS 1.2 only } diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index ea4dde882..98efd91c7 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,14 +1,9 @@ -import OpenSSL +import OpenSSL.SSL import OpenSSL._util as pyOpenSSLutil from scrapy.utils.python import to_unicode -# The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL. -# Using the binding directly, as this code does, requires cryptography 2.4. -SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0) - - def ffi_buf_to_string(buf): return to_unicode(pyOpenSSLutil.ffi.string(buf)) @@ -22,9 +17,6 @@ def x509name_to_string(x509name): def get_temp_key_info(ssl_object): - if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'): # requires OpenSSL 1.0.2 - return None - # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **") if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p): @@ -55,7 +47,5 @@ def get_temp_key_info(ssl_object): def get_openssl_version(): - system_openssl = OpenSSL.SSL.SSLeay_version( - OpenSSL.SSL.SSLEAY_VERSION - ).decode('ascii', errors='replace') + system_openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) return f'{OpenSSL.version.__version__} ({system_openssl})' diff --git a/tests/mockserver.py b/tests/mockserver.py index 6d2d95692..4fd3adce7 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -19,7 +19,6 @@ from twisted.web.static import File from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3 from scrapy.utils.test import get_testenv @@ -350,7 +349,7 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c if cipher_string: ctx = factory.getContext() # disabling TLS1.3 because it unconditionally enables some strong ciphers - ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3) + ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_3) ctx.set_cipher_list(to_bytes(cipher_string)) return factory From fb52918d23b37cd2581aa39ebbd422b017ec8051 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 13 Jan 2023 20:46:55 +0400 Subject: [PATCH 0773/2083] Set OP_LEGACY_SERVER_CONNECT to support some old servers when using OpenSSL 3. --- scrapy/core/downloader/contextfactory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 4abde2238..bc6ad34d8 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -61,7 +61,9 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() def getContext(self, hostname=None, port=None): - return self.getCertificateOptions().getContext() + ctx = self.getCertificateOptions().getContext() + ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT + return ctx def creatorForNetloc(self, hostname, port): return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(), From 43ab8bd16acdabc8be64b421b030fb6e1751e47b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 13 Jan 2023 20:51:07 +0400 Subject: [PATCH 0774/2083] Roll back the get_openssl_version() type change. --- scrapy/utils/ssl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 98efd91c7..ce211bf9b 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -47,5 +47,7 @@ def get_temp_key_info(ssl_object): def get_openssl_version(): - system_openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) + system_openssl = OpenSSL.SSL.SSLeay_version( + OpenSSL.SSL.SSLEAY_VERSION + ).decode('ascii', errors='replace') return f'{OpenSSL.version.__version__} ({system_openssl})' From caaeb235a08a250a9438c97259f2a74ae2ca8bbd Mon Sep 17 00:00:00 2001 From: Serhii A Date: Tue, 17 Jan 2023 13:52:41 +0200 Subject: [PATCH 0775/2083] =?UTF-8?q?scrapy.utils.console.DEFAULT=5FPYTHON?= =?UTF-8?q?=5FSHELLS:=20OrderedDict=20=E2=86=92=20dict=20(#5795)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/utils/console.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 1bc0bd45f..4828c7767 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,5 +1,4 @@ from functools import wraps -from collections import OrderedDict def _embed_ipython_shell(namespace={}, banner=''): @@ -63,12 +62,12 @@ def _embed_standard_shell(namespace={}, banner=''): return wrapper -DEFAULT_PYTHON_SHELLS = OrderedDict([ - ('ptpython', _embed_ptpython_shell), - ('ipython', _embed_ipython_shell), - ('bpython', _embed_bpython_shell), - ('python', _embed_standard_shell), -]) +DEFAULT_PYTHON_SHELLS = { + 'ptpython': _embed_ptpython_shell, + 'ipython': _embed_ipython_shell, + 'bpython': _embed_bpython_shell, + 'python': _embed_standard_shell, +} def get_shell_embed_func(shells=None, known_shells=None): From 482a0b79e3e5ca274cf30584eb8a29c1b79791a8 Mon Sep 17 00:00:00 2001 From: Emmanuel Rondan Date: Tue, 17 Jan 2023 14:26:12 -0300 Subject: [PATCH 0776/2083] explaining pre-commit on documentation --- docs/contributing.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/contributing.rst b/docs/contributing.rst index 9cfe10012..dca635f15 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -179,6 +179,25 @@ Scrapy: See https://help.github.com/en/github/using-git/setting-your-username-in-git for setup instructions. +.. _pre-commit: + +Pre-commit +========== + +Pre-commit is a tool that allows developers to specify a set of checks to be run +automatically every time they make a commit. This can include code style checks, +linting, and automated tests. +The checks are defined in a configuration file called .pre-commit-config.yaml. + +By using pre-commit, developers can ensure that their code adheres to a consistent +style and passes certain tests before they make a commit. +This can help catch errors early in the development process and prevent them +from being pushed to the main branch. + +To use pre-commit, developers first need to install it on their local machine. +Once it is installed, they can specify the checks they want to run in the .pre-commit-config.yaml file. +After that, pre-commit will run automatically every time they make a commit. + .. _documentation-policies: Documentation policies From 23e8b553b4db0c33645f371fd161470545733320 Mon Sep 17 00:00:00 2001 From: Emmanuel Rondan Date: Wed, 18 Jan 2023 11:42:30 -0300 Subject: [PATCH 0777/2083] fixing docs with suggestions --- docs/contributing.rst | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index dca635f15..f7f1218c8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -169,10 +169,10 @@ Coding style Please follow these coding conventions when writing code for inclusion in Scrapy: -* Unless otherwise specified, follow :pep:`8`. - -* It's OK to use lines longer than 79 chars if it improves the code - readability. +* We use `black `_ for code formatting. + There is a hook in the pre-commit config + that will automatically format your code before every commit. You can also + run black manually with ``tox -e black``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. @@ -184,19 +184,21 @@ Scrapy: Pre-commit ========== -Pre-commit is a tool that allows developers to specify a set of checks to be run -automatically every time they make a commit. This can include code style checks, -linting, and automated tests. -The checks are defined in a configuration file called .pre-commit-config.yaml. +We use `pre-commit`_ to automatically address simple code issues before every +commit. -By using pre-commit, developers can ensure that their code adheres to a consistent -style and passes certain tests before they make a commit. -This can help catch errors early in the development process and prevent them -from being pushed to the main branch. +Before you start writing a patch: +#. `Install pre-commit `_. +#. On the root of your local clone of the Scrapy repository, run the following command: + .. code-block:: bash + + pre-commit install -To use pre-commit, developers first need to install it on their local machine. -Once it is installed, they can specify the checks they want to run in the .pre-commit-config.yaml file. -After that, pre-commit will run automatically every time they make a commit. +Now our pre-commit hooks will run every time you create a Git commit. Upon +finding issues, pre-commit hooks aborts your commit, and they either fix +the corresponding issues automatically or only report them to you. If they fix +the issues automatically, creating your commit again should succeed. Otherwise, +you may need to address the corresponding issues manually first. .. _documentation-policies: From 50500a6b2897418405b459b6a2a80a7a5b0a6f29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 19 Jan 2023 17:14:18 +0100 Subject: [PATCH 0778/2083] Implement a NO_CALLBACK value for Request.callback --- docs/topics/request-response.rst | 43 ++++++++++++++------ scrapy/downloadermiddlewares/robotstxt.py | 4 +- scrapy/http/request/__init__.py | 28 ++++++++++--- scrapy/pipelines/media.py | 3 +- tests/test_downloadermiddleware_robotstxt.py | 8 ++++ tests/test_http_request.py | 9 ++++ tests/test_pipeline_files.py | 5 +-- tests/test_pipeline_images.py | 7 +--- tests/test_pipeline_media.py | 2 + 9 files changed, 79 insertions(+), 30 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a0d9fc03e..766710d66 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -32,11 +32,22 @@ Request objects :type url: str :param callback: the function that will be called with the response of this - request (once it's downloaded) as its first parameter. For more information - see :ref:`topics-request-response-ref-request-callback-arguments` below. - If a Request doesn't specify a callback, the spider's - :meth:`~scrapy.Spider.parse` method will be used. - Note that if exceptions are raised during processing, errback is called instead. + request (once it's downloaded) as its first parameter. + + In addition to a function, the following values are supported: + + - ``None`` (default), which indicates that the spider's + :meth:`~scrapy.Spider.parse` method must be used. + + - :py:data:`scrapy.http.request.NO_CALLBACK` + + .. autodata:: scrapy.http.request.NO_CALLBACK + + For more information, see + :ref:`topics-request-response-ref-request-callback-arguments`. + + .. note:: If exceptions are raised during processing, ``errback`` is + called instead. :type callback: collections.abc.Callable @@ -69,16 +80,24 @@ Request objects 1. Using a dict:: - request_with_cookies = Request(url="http://www.example.com", - cookies={'currency': 'USD', 'country': 'UY'}) + request_with_cookies = Request( + url="http://www.example.com", + cookies={'currency': 'USD', 'country': 'UY'}, + ) 2. Using a list of dicts:: - request_with_cookies = Request(url="http://www.example.com", - cookies=[{'name': 'currency', - 'value': 'USD', - 'domain': 'example.com', - 'path': '/currency'}]) + request_with_cookies = Request( + url="http://www.example.com", + cookies=[ + { + 'name': 'currency', + 'value': 'USD', + 'domain': 'example.com', + 'path': '/currency', + }, + ], + ) The latter form allows for customizing the ``domain`` and ``path`` attributes of the cookie. This is only useful if the cookies are saved diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7bd39aa43..67e14b7b5 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -9,6 +9,7 @@ import logging from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request +from scrapy.http.request import NO_CALLBACK from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object @@ -65,7 +66,8 @@ class RobotsTxtMiddleware: robotsreq = Request( robotsurl, priority=self.DOWNLOAD_PRIORITY, - meta={'dont_obey_robotstxt': True} + meta={'dont_obey_robotstxt': True}, + callback=NO_CALLBACK, ) dfd = self.crawler.engine.download(robotsreq) dfd.addCallback(self._parse_robots, netloc, spider) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a1001fc4a..b57faf121 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -20,6 +20,17 @@ from scrapy.utils.url import escape_ajax RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") +#: When assigned to the ``callback`` parameter of +#: :class:`~scrapy.http.Request`, it indicates that the request it not meant to +#: have a spider callback at all. +#: +#: This value should be used by :ref:`components ` +#: that create and handle their own requests, e.g. through +#: :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download +#: middlewares handling such requests can treat them differently from requests +#: intended for the :meth:`~scrapy.Spider.parse` callback. +NO_CALLBACK = object() + class Request(object_ref): """Represents an HTTP request, which is usually generated in a Spider and @@ -63,12 +74,8 @@ class Request(object_ref): raise TypeError(f"Request priority not an integer: {priority!r}") self.priority = priority - if callback is not None and not callable(callback): - raise TypeError(f'callback must be a callable, got {type(callback).__name__}') - if errback is not None and not callable(errback): - raise TypeError(f'errback must be a callable, got {type(errback).__name__}') - self.callback = callback - self.errback = errback + self._set_xback("callback", callback) + self._set_xback("errback", errback) self.cookies = cookies or {} self.headers = Headers(headers or {}, encoding=encoding) @@ -78,6 +85,15 @@ class Request(object_ref): self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None self.flags = [] if flags is None else list(flags) + def _set_xback(self, name: str, value: Optional[Callable]) -> None: + if ( + value is not None + and (name != "callback" or value is not NO_CALLBACK) + and not callable(value) + ): + raise TypeError(f'{name} must be a callable, got {type(value).__name__}') + setattr(self, name, value) + @property def cb_kwargs(self) -> dict: if self._cb_kwargs is None: diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 5308a9793..fc5db58e8 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -7,6 +7,7 @@ from warnings import warn from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure +from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result @@ -93,7 +94,7 @@ class MediaPipeline: fp = self._fingerprinter.fingerprint(request) cb = request.callback or (lambda _: _) eb = request.errback - request.callback = None + request.callback = NO_CALLBACK request.errback = None # Return cached result if request was already seen diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 1460d88eb..71d53ff1a 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -8,6 +8,7 @@ from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware, logger as mw_module_logger) from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse +from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings from tests.test_robotstxt_interface import rerp_available, reppy_available @@ -53,6 +54,7 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self._get_successful_crawler()) return DeferredList([ self.assertNotIgnored(Request('http://site.local/allowed'), middleware), + maybeDeferred(self.assertRobotsTxtRequested, "http://site.local"), self.assertIgnored(Request('http://site.local/admin/main'), middleware), self.assertIgnored(Request('http://site.local/static/'), middleware), self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware), @@ -183,6 +185,12 @@ Disallow: /some/randome/page.html return self.assertFailure(maybeDeferred(middleware.process_request, request, spider), IgnoreRequest) + def assertRobotsTxtRequested(self, base_url): + calls = self.crawler.engine.download.call_args_list + request = calls[0][0][0] + self.assertEqual(request.url, f"{base_url}/robots.txt") + self.assertEqual(request.callback, NO_CALLBACK) + class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): if not rerp_available(): diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 81cebdc7b..e14f8c8f4 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -7,6 +7,7 @@ from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes, urlparse from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse +from scrapy.http.request import NO_CALLBACK from scrapy.utils.python import to_bytes, to_unicode @@ -277,6 +278,12 @@ class RequestTest(unittest.TestCase): self.assertIs(r4.callback, a_function) self.assertIs(r4.errback, a_function) + r5 = self.request_class( + url='http://example.com', + callback=NO_CALLBACK, + ) + self.assertIs(r5.callback, NO_CALLBACK) + def test_callback_and_errback_type(self): with self.assertRaises(TypeError): self.request_class('http://example.com', callback='a_function') @@ -288,6 +295,8 @@ class RequestTest(unittest.TestCase): callback='a_function', errback='a_function', ) + with self.assertRaises(TypeError): + self.request_class('http://example.com', errback=NO_CALLBACK) def test_from_curl(self): # Note: more curated tests regarding curl conversion are in diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4acd29bf7..83572e74f 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -33,10 +33,7 @@ from scrapy.utils.test import ( skip_if_no_boto, ) - -def _mocked_download_func(request, info): - response = request.meta.get('response') - return response() if callable(response) else response +from .test_pipeline_media import _mocked_download_func class FilesPipelineTestCase(unittest.TestCase): diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 6f5466191..3b39212bd 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -31,18 +31,13 @@ else: skip_pillow = None -def _mocked_download_func(request, info): - response = request.meta.get('response') - return response() if callable(response) else response - - class ImagesPipelineTestCase(unittest.TestCase): skip = skip_pillow def setUp(self): self.tempdir = mkdtemp() - self.pipeline = ImagesPipeline(self.tempdir, download_func=_mocked_download_func) + self.pipeline = ImagesPipeline(self.tempdir) def tearDown(self): rmtree(self.tempdir) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 0a94ae699..99fb424f4 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -9,6 +9,7 @@ from twisted.internet.defer import Deferred, inlineCallbacks from scrapy import signals from scrapy.http import Request, Response +from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.pipelines.files import FileException @@ -29,6 +30,7 @@ else: def _mocked_download_func(request, info): + assert request.callback is NO_CALLBACK response = request.meta.get('response') return response() if callable(response) else response From f449ee53778b038f9ecb1feccb2fa9baa40e1f56 Mon Sep 17 00:00:00 2001 From: Tobias Mayr Date: Thu, 19 Jan 2023 18:44:55 +0000 Subject: [PATCH 0779/2083] Fix SMTP STARTTLS for Twisted >= 21.2.0 (#5406) --- scrapy/mail.py | 28 +++++++++++++++++++++------- tests/test_mail.py | 16 ++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 2a25ccd44..b8cc28335 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -12,7 +12,9 @@ from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO +from twisted.python.versions import Version from twisted.internet import defer, ssl +from twisted import version as twisted_version from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes @@ -126,16 +128,11 @@ class MailSender: 'mailattachs': nattachs, 'mailerr': errstr}) def _sendmail(self, to_addrs, msg): - # Import twisted.mail here because it is not available in python3 from twisted.internet import reactor - from twisted.mail.smtp import ESMTPSenderFactory msg = BytesIO(msg) d = defer.Deferred() - factory = ESMTPSenderFactory( - self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, - heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls, - ) - factory.noisy = False + + factory = self._create_sender_factory(to_addrs, msg, d) if self.smtpssl: reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory()) @@ -143,3 +140,20 @@ class MailSender: reactor.connectTCP(self.smtphost, self.smtpport, factory) return d + + def _create_sender_factory(self, to_addrs, msg, d): + from twisted.mail.smtp import ESMTPSenderFactory + + factory_keywords = { + 'heloFallback': True, + 'requireAuthentication': False, + 'requireTransportSecurity': self.smtptls + } + + # Newer versions of twisted require the hostname to use STARTTLS + if twisted_version >= Version('twisted', 21, 2, 0): + factory_keywords['hostname'] = self.smtphost + + factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, **factory_keywords) + factory.noisy = False + return factory diff --git a/tests/test_mail.py b/tests/test_mail.py index 9b248fbfa..fd02020ee 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -4,6 +4,11 @@ import unittest from io import BytesIO from email.charset import Charset +from twisted.internet._sslverify import ClientTLSOptions +from twisted.internet.ssl import ClientContextFactory +from twisted.python.versions import Version +from twisted.internet import defer +from twisted import version as twisted_version from scrapy.mail import MailSender @@ -121,6 +126,17 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(text.get_charset(), Charset('utf-8')) self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body) + def test_create_sender_factory_with_host(self): + mailsender = MailSender(debug=False, smtphost='smtp.testhost.com') + + factory = mailsender._create_sender_factory(to_addrs=['test@scrapy.org'], msg='test', d=defer.Deferred()) + + context = factory.buildProtocol('test@scrapy.org').context + if twisted_version >= Version('twisted', 21, 2, 0): + self.assertIsInstance(context, ClientTLSOptions) + else: + self.assertIsInstance(context, ClientContextFactory) + if __name__ == "__main__": unittest.main() From a49346494201d3d7a9d017f16fd64fa2d9042b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 19 Jan 2023 19:53:53 +0100 Subject: [PATCH 0780/2083] Update the screenshot pipeline code example --- docs/topics/item-pipeline.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 1672ccbcc..fa19d2f4c 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -191,6 +191,7 @@ item. import scrapy from itemadapter import ItemAdapter + from scrapy.http.request import NO_CALLBACK from scrapy.utils.defer import maybe_deferred_to_future @@ -204,8 +205,10 @@ item. adapter = ItemAdapter(item) encoded_item_url = quote(adapter["url"]) screenshot_url = self.SPLASH_URL.format(encoded_item_url) - request = scrapy.Request(screenshot_url) - response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider)) + request = scrapy.Request(screenshot_url, callback=NO_CALLBACK) + response = await maybe_deferred_to_future( + spider.crawler.engine.download(request, spider) + ) if response.status != 200: # Error happened, return item. From 5c1559f60e459a9678eecfe24c4fea06e272dbab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 19 Jan 2023 20:30:22 +0100 Subject: [PATCH 0781/2083] Address typing issues --- scrapy/http/request/__init__.py | 12 ++++++++++-- tox.ini | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index b57faf121..ea73781c8 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -5,7 +5,8 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ import inspect -from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union +from enum import Enum +from typing import Any, Callable, Final, List, Optional, Tuple, Type, TypeVar, Union from w3lib.url import safe_url_string @@ -20,6 +21,11 @@ from scrapy.utils.url import escape_ajax RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") + +# https://github.com/python/typing/issues/689#issuecomment-561425237 +class _NoCallback(Enum): + NO_CALLBACK = 0 + #: When assigned to the ``callback`` parameter of #: :class:`~scrapy.http.Request`, it indicates that the request it not meant to #: have a spider callback at all. @@ -29,7 +35,7 @@ RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") #: :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download #: middlewares handling such requests can treat them differently from requests #: intended for the :meth:`~scrapy.Spider.parse` callback. -NO_CALLBACK = object() +NO_CALLBACK: Final = _NoCallback.NO_CALLBACK class Request(object_ref): @@ -49,6 +55,8 @@ class Request(object_ref): Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and :func:`~scrapy.utils.request.request_from_dict`. """ + callback: Union[None, _NoCallback, Callable] + errback: Optional[Callable] def __init__( self, diff --git a/tox.ini b/tox.ini index 520d90303..076178d8e 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = security,flake8,py +envlist = security,flake8,typing,py minversion = 1.7.0 [testenv] From 4242ae405d9775098607a67612b5640a4daaf840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 19 Jan 2023 20:37:24 +0100 Subject: [PATCH 0782/2083] Restore Python 3.7 support --- scrapy/http/request/__init__.py | 4 +++- setup.py | 1 + tox.ini | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index ea73781c8..936afb007 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -6,8 +6,9 @@ See documentation in docs/topics/request-response.rst """ import inspect from enum import Enum -from typing import Any, Callable, Final, List, Optional, Tuple, Type, TypeVar, Union +from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union +from typing_extensions import Final from w3lib.url import safe_url_string import scrapy @@ -26,6 +27,7 @@ RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") class _NoCallback(Enum): NO_CALLBACK = 0 + #: When assigned to the ``callback`` parameter of #: :class:`~scrapy.http.Request`, it indicates that the request it not meant to #: have a spider callback at all. diff --git a/setup.py b/setup.py index bdae28047..049c4f9a6 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ install_requires = [ 'packaging', 'tldextract', 'lxml>=4.3.0', + 'typing-extensions>=3.10.0.0', ] extras_require = {} cpython_dependencies = [ diff --git a/tox.ini b/tox.ini index 076178d8e..1ff5f7a4e 100644 --- a/tox.ini +++ b/tox.ini @@ -94,6 +94,7 @@ deps = w3lib==1.17.0 zope.interface==5.1.0 lxml==4.3.0 + typing-extensions==3.10.0.0 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies From b1dd893fbb4d2efed3342e6c98a6bbf4b393d48e Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 20 Jan 2023 02:17:02 -0800 Subject: [PATCH 0783/2083] Support Path Objects Issue #5739 --- scrapy/pipelines/files.py | 9 +++++---- tests/test_pipeline_files.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 51aedafe8..4b0c96b25 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -14,7 +14,7 @@ from contextlib import suppress from ftplib import FTP from io import BytesIO from pathlib import Path -from typing import DefaultDict, Optional, Set +from typing import DefaultDict, Optional, Set, Union from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -41,7 +41,8 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: str): + def __init__(self, basedir: Union[str, os.PathLike]): + basedir = str(basedir) # support Path object if '://' in basedir: basedir = basedir.split('://', 1)[1] self.basedir = basedir @@ -65,8 +66,8 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: str) -> Path: - path_comps = path.split('/') + def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: + path_comps = str(path).split('/') return Path(self.basedir, *path_comps) def _mkdir(self, dirname: Path, domain: Optional[str] = None): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4acd29bf7..5280ab0d2 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -405,6 +405,23 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value) + def test_file_pipeline_using_pathlike_objects(self): + + class CustomFilesPipelineWithPathLikeDir(FilesPipeline): + def file_path(self, request, response=None, info=None, *, item=None): + return Path('subdir') / Path(request.url).name + + pipeline = CustomFilesPipelineWithPathLikeDir.from_settings( + Settings({'FILES_STORE': Path('./Temp')}) + ) + request = Request("http://example.com/image01.jpg") + self.assertEqual(pipeline.file_path(request), Path('subdir/image01.jpg')) + + def test_files_store_constructor_with_pathlike_object(self): + path = Path('./FileDir') + fs_store = FSFilesStore(path) + self.assertEqual(fs_store.basedir, str(path)) + class TestS3FilesStore(unittest.TestCase): From 8270df754d5caa7e8115432923197f09b4ebc78f Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Fri, 20 Jan 2023 07:55:16 -0300 Subject: [PATCH 0784/2083] Set `FEED_EXPORT_ENCODING='utf-8'` in the default template --- docs/topics/feed-exports.rst | 5 +++++ scrapy/templates/project/module/settings.py.tmpl | 1 + 2 files changed, 6 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index a620e2c04..7b662f34d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -515,6 +515,11 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons. Use ``utf-8`` if you want UTF-8 for JSON too. +.. versionchanged:: 2.8 + The :command:`startproject` command now sets this setting to + ``utf-8`` in the generated + ``settings.py`` file. + .. setting:: FEED_EXPORT_FIELDS FEED_EXPORT_FIELDS diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index bbf60982c..2f6df5abc 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -90,3 +90,4 @@ ROBOTSTXT_OBEY = True # Set settings whose default value is deprecated to a future-proof value REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7' TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' +FEED_EXPORT_ENCODING = 'utf-8' From 973f0cf5678adcec36aaf5d0ceb860198f34de4a Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Fri, 20 Jan 2023 08:23:05 -0300 Subject: [PATCH 0785/2083] fix: line break --- docs/topics/feed-exports.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7b662f34d..8775a99d0 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -517,8 +517,7 @@ Use ``utf-8`` if you want UTF-8 for JSON too. .. versionchanged:: 2.8 The :command:`startproject` command now sets this setting to - ``utf-8`` in the generated - ``settings.py`` file. + ``utf-8`` in the generated ``settings.py`` file. .. setting:: FEED_EXPORT_FIELDS From b6118480299384c88d194bf09ad60b67a36395ed Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Fri, 20 Jan 2023 08:33:35 -0300 Subject: [PATCH 0786/2083] fix(docs): Change `versionchanged` value --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 8775a99d0..8f96b1154 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -515,7 +515,7 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons. Use ``utf-8`` if you want UTF-8 for JSON too. -.. versionchanged:: 2.8 +.. versionchanged:: VERSION The :command:`startproject` command now sets this setting to ``utf-8`` in the generated ``settings.py`` file. From 818d69fa003a14a2f2058e76258e6269d227efb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 20 Jan 2023 12:38:07 +0100 Subject: [PATCH 0787/2083] =?UTF-8?q?Fix=20typo:=20it=20=E2=86=92=20is?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrey Rakhmatullin --- scrapy/http/request/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 936afb007..302895781 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -29,7 +29,7 @@ class _NoCallback(Enum): #: When assigned to the ``callback`` parameter of -#: :class:`~scrapy.http.Request`, it indicates that the request it not meant to +#: :class:`~scrapy.http.Request`, it indicates that the request is not meant to #: have a spider callback at all. #: #: This value should be used by :ref:`components ` From 8ee48174717591bbf74497fded060d7ae83d67ba Mon Sep 17 00:00:00 2001 From: Emmanuel Rondan Date: Fri, 20 Jan 2023 10:54:46 -0300 Subject: [PATCH 0788/2083] reaplying black, fixing conflicts and ignoring bandit checks on test directory --- .bandit.yml | 1 + .github/workflows/checks.yml | 4 +- .github/workflows/publish.yml | 4 +- .github/workflows/tests-ubuntu.yml | 5 +- docs/news.rst | 18 +-- docs/topics/settings.rst | 32 +++-- docs/topics/spiders.rst | 2 +- extras/qpsclient.py | 5 +- scrapy/cmdline.py | 2 +- scrapy/core/downloader/contextfactory.py | 4 +- scrapy/core/downloader/tls.py | 4 +- scrapy/core/downloader/webclient.py | 2 +- scrapy/core/http2/stream.py | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/extensions/httpcache.py | 2 +- scrapy/http/request/form.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 7 +- scrapy/pipelines/images.py | 4 +- scrapy/shell.py | 4 + scrapy/utils/console.py | 15 +-- scrapy/utils/defer.py | 10 +- scrapy/utils/reactor.py | 28 +++- scrapy/utils/response.py | 2 +- scrapy/utils/ssl.py | 12 +- scrapy/utils/url.py | 2 +- sep/sep-016.rst | 2 +- sep/sep-018.rst | 12 +- sep/sep-021.rst | 2 +- setup.py | 2 +- tests/mockserver.py | 3 +- tests/pipelines.py | 2 +- .../link_extractor/linkextractor.html | 1 + tests/test_commands.py | 2 +- tests/test_crawl.py | 4 +- tests/test_downloader_handlers.py | 21 ++- tests/test_downloadermiddleware_useragent.py | 2 +- tests/test_engine.py | 2 +- tests/test_feedexport.py | 125 +++++++++++++++++- tests/test_http_request.py | 8 +- tests/test_http_response.py | 3 +- tests/test_linkextractors.py | 8 ++ tests/test_pipeline_images.py | 10 +- tests/test_request_attribute_binding.py | 16 +-- tests/test_spidermiddleware_referer.py | 16 +-- tests/test_utils_asyncio.py | 5 + tests/test_utils_deprecate.py | 2 +- tests/test_utils_display.py | 24 +++- tests/test_utils_python.py | 2 +- tox.ini | 16 +-- 49 files changed, 326 insertions(+), 139 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 41f1bb597..c8e84cc2e 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -17,3 +17,4 @@ skips: - B503 - B603 - B605 +exclude_dirs: ['tests'] diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 90b6ade4a..740092dab 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -14,9 +14,7 @@ jobs: - python-version: "3.11" env: TOXENV: flake8 - # Pylint requires installing reppy, which does not support Python 3.9 - # https://github.com/seomoz/reppy/issues/122 - - python-version: 3.8 + - python-version: "3.11" env: TOXENV: pylint - python-version: 3.7 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 991b0b6e8..eee9a4f02 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,8 +24,8 @@ jobs: - name: Publish to PyPI if: steps.check-release-tag.outputs.release_tag == 'true' run: | - pip install --upgrade setuptools wheel twine - python setup.py sdist bdist_wheel + pip install --upgrade build twine + python -m build export TWINE_USERNAME=__token__ export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} twine upload dist/* diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 9c3ce8115..8fcf90a18 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -38,10 +38,7 @@ jobs: env: TOXENV: pypy3-pinned - # extras - # extra-deps includes reppy, which does not support Python 3.9 - # https://github.com/seomoz/reppy/issues/122 - - python-version: 3.8 + - python-version: "3.11" env: TOXENV: extra-deps diff --git a/docs/news.rst b/docs/news.rst index c97de0ed8..07264827b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -4700,7 +4700,7 @@ Scrapy 0.22.1 (released 2014-02-08) - BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`) - BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`) - Fix tests for Travis-CI build (:commit:`76c7e20`) -- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`) +- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`) - RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`) - Updated the tutorial crawl output with latest output. (:commit:`8da65de`) - Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`) @@ -4725,7 +4725,7 @@ Enhancements - [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage`` - Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`) -- Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`) +- Add a middleware to crawl ajax crawlable pages as defined by google (:issue:`343`) - Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`) - Selectors register EXSLT namespaces by default (:issue:`472`) - Unify item loaders similar to selectors renaming (:issue:`461`) @@ -4905,7 +4905,7 @@ Scrapy 0.18.0 (released 2013-08-09) ----------------------------------- - Lot of improvements to testsuite run using Tox, including a way to test on pypi -- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`) +- Handle GET parameters for AJAX crawlable urls (:commit:`3fe2a32`) - Use lxml recover option to parse sitemaps (:issue:`347`) - Bugfix cookie merging by hostname and not by netloc (:issue:`352`) - Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`) @@ -4939,8 +4939,8 @@ Scrapy 0.18.0 (released 2013-08-09) - Added ``--pdb`` option to ``scrapy`` command line tool - Added :meth:`XPathSelector.remove_namespaces ` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. - Several improvements to spider contracts -- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections, -- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62 +- New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections, +- MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62 - added from_crawler method to spiders - added system tests with mock server - more improvements to macOS compatibility (thanks Alex Cepoi) @@ -5082,7 +5082,7 @@ Scrapy changes: - promoted :ref:`topics-djangoitem` to main contrib - LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`) - downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method -- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module +- replaced memory usage accounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module - removed signal: ``scrapy.mail.mail_sent`` - removed ``TRACK_REFS`` setting, now :ref:`trackrefs ` is always enabled - DBM is now the default storage backend for HTTP cache middleware @@ -5148,7 +5148,7 @@ Scrapy 0.14 New features and settings ~~~~~~~~~~~~~~~~~~~~~~~~~ -- Support for `AJAX crawleable urls`_ +- Support for `AJAX crawlable urls`_ - New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`) - added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``) - Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`) @@ -5408,7 +5408,7 @@ Backward-incompatible changes - Renamed setting: ``REQUESTS_PER_DOMAIN`` to ``CONCURRENT_REQUESTS_PER_SPIDER`` (:rev:`1830`, :rev:`1844`) - Renamed setting: ``CONCURRENT_DOMAINS`` to ``CONCURRENT_SPIDERS`` (:rev:`1830`) - Refactored HTTP Cache middleware -- HTTP Cache middleware has been heavilty refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` ) +- HTTP Cache middleware has been heavily refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` ) - Renamed exception: ``DontCloseDomain`` to ``DontCloseSpider`` (:rev:`1859` | #120) - Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121) - Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`) @@ -5419,7 +5419,7 @@ Scrapy 0.7 First release of Scrapy. -.. _AJAX crawleable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 +.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 .. _botocore: https://github.com/boto/botocore .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 022265992..e147a943b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -636,19 +636,30 @@ DOWNLOAD_DELAY Default: ``0`` -The amount of time (in secs) that the downloader should wait before downloading -consecutive pages from the same website. This can be used to throttle the -crawling speed to avoid hitting servers too hard. Decimal numbers are -supported. Example:: +Minimum seconds to wait between 2 consecutive requests to the same domain. - DOWNLOAD_DELAY = 0.25 # 250 ms of delay +Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting +servers too hard. + +Decimal numbers are supported. For example, to send a maximum of 4 requests +every 10 seconds:: + + DOWNLOAD_DELAY = 2.5 This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY` -setting (which is enabled by default). By default, Scrapy doesn't wait a fixed -amount of time between requests, but uses a random interval between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`. +setting, which is enabled by default. When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced -per ip address instead of per domain. +per IP address instead of per domain. + +Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain +concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response +time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective +concurrency for that domain is 1. When testing throttling configurations, it +usually makes sense to lower :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` first, +and only increase :setting:`DOWNLOAD_DELAY` once +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is 1 but a higher throttling is +desired. .. _spider-download_delay-attribute: @@ -656,6 +667,11 @@ per ip address instead of per domain. This delay can be set per spider using :attr:`download_delay` spider attribute. +It is also possible to change this setting per domain, although it requires +non-trivial code. See the implementation of the :ref:`AutoThrottle +` extension for an example. + + .. setting:: DOWNLOAD_HANDLERS DOWNLOAD_HANDLERS diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index ffe41cf3e..7aa8555d5 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -99,7 +99,7 @@ scrapy.Spider .. attribute:: crawler This attribute is set by the :meth:`from_crawler` class method after - initializating the class, and links to the + initializing the class, and links to the :class:`~scrapy.crawler.Crawler` object to which this spider instance is bound. diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 83bb08561..bb8527af2 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -1,9 +1,10 @@ """ -A spider that generate light requests to meassure QPS throughput +A spider that generate light requests to measure QPS throughput usage: - scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0 --set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3 + scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0 + --set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3 """ diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 1d74923a9..ffb40e1c5 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -24,7 +24,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser): def _iter_command_classes(module_name): - # TODO: add `name` attribute to commands and and merge this function with + # TODO: add `name` attribute to commands and merge this function with # scrapy.utils.spider.iter_spider_classes for module in walk_modules(module_name): for obj in vars(module).values(): diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 9dd2df638..1513638df 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -83,7 +83,9 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() def getContext(self, hostname=None, port=None): - return self.getCertificateOptions().getContext() + ctx = self.getCertificateOptions().getContext() + ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT + return ctx def creatorForNetloc(self, hostname, port): return ScrapyClientTLSOptions( diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index db2639a75..d1c511db0 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -23,8 +23,8 @@ METHOD_TLSv12 = "TLSv1.2" openssl_methods = { METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only - METHOD_TLSv11: getattr(SSL, "TLSv1_1_METHOD", 5), # TLS 1.1 only - METHOD_TLSv12: getattr(SSL, "TLSv1_2_METHOD", 6), # TLS 1.2 only + METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only + METHOD_TLSv12: SSL.TLSv1_2_METHOD, # TLS 1.2 only } diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index a00afb7af..6421391d0 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -101,7 +101,7 @@ class ScrapyHTTPPageGetter(HTTPClient): # This class used to inherit from Twisted’s # twisted.web.client.HTTPClientFactory. When that class was deprecated in # Twisted (https://github.com/twisted/twisted/pull/643), we merged its -# non-overriden code into this class. +# non-overridden code into this class. class ScrapyHTTPClientFactory(ClientFactory): protocol = ScrapyHTTPPageGetter diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 1b185cd8c..2b5c98e5f 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -348,7 +348,7 @@ class Stream: def receive_headers(self, headers: List[HeaderTuple]) -> None: for name, value in headers: - self._response["headers"][name] = value + self._response["headers"].appendlist(name, value) # Check if we exceed the allowed max data size which can be received expected_size = int(self._response["headers"].get(b"Content-Length", -1)) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index ca4d77a35..823955aa3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -384,11 +384,11 @@ class FeedExporter: return defer.DeferredList(deferred_list) if deferred_list else None def _close_slot(self, slot, spider): + slot.finish_exporting() if not slot.itemcount and not slot.store_empty: # We need to call slot.storage.store nonetheless to get the file # properly closed. return defer.maybeDeferred(slot.storage.store, slot.file) - slot.finish_exporting() logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" d = defer.maybeDeferred(slot.storage.store, slot.file) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 3bfbddc54..bbddaac40 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -196,7 +196,7 @@ class RFC2616Policy: if response.status in (300, 301, 308): return self.MAXAGE - # Insufficient information to compute fresshness lifetime + # Insufficient information to compute freshness lifetime return 0 def _compute_current_age(self, response, request, now): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ea519fb19..993219745 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -200,7 +200,7 @@ def _select_value(ele: SelectElement, n: str, v: str): o = ele.value_options return (n, o[0]) if o else (None, None) if v is not None and multiple: - # This is a workround to bug in lxml fixed 2.3.1 + # This is a workaround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath(".//option[@selected]") values = [(o.get("value") or o.text or "").strip() for o in selected_options] diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 0bd28a882..66cf58d89 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -226,7 +226,8 @@ class LxmlLinkExtractor: Only links that match the settings passed to the ``__init__`` method of the link extractor are returned. - Duplicate links are omitted. + Duplicate links are omitted if the ``unique`` attribute is set to ``True``, + otherwise they are returned. """ base_url = get_base_url(response) if self.restrict_xpaths: @@ -239,4 +240,6 @@ class LxmlLinkExtractor: for doc in docs: links = self._extract_links(doc, response.url, response.encoding, base_url) all_links.extend(self._process_links(links)) - return unique_list(all_links) + if self.link_extractor.unique: + return unique_list(all_links) + return all_links diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index fcecb5fbb..0cfa5665a 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -151,8 +151,8 @@ class ImagesPipeline(FilesPipeline): ) if self._deprecated_convert_image: warnings.warn( - f"{self.__class__.__name__}.convert_image() method overriden in a deprecated way, " - "overriden method does not accept response_body argument.", + f"{self.__class__.__name__}.convert_image() method overridden in a deprecated way, " + "overridden method does not accept response_body argument.", category=ScrapyDeprecationWarning, ) diff --git a/scrapy/shell.py b/scrapy/shell.py index a42dedc93..084a27141 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -177,7 +177,11 @@ class Shell: def inspect_response(response, spider): """Open a shell to inspect the given response""" + # Shell.start removes the SIGINT handler, so save it and re-add it after + # the shell has closed + sigint_handler = signal.getsignal(signal.SIGINT) Shell(spider.crawler).start(response=response, spider=spider) + signal.signal(signal.SIGINT, sigint_handler) def _request_deferred(request): diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index b64098d7b..100f040bb 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,5 +1,4 @@ from functools import wraps -from collections import OrderedDict def _embed_ipython_shell(namespace={}, banner=""): @@ -70,14 +69,12 @@ def _embed_standard_shell(namespace={}, banner=""): return wrapper -DEFAULT_PYTHON_SHELLS = OrderedDict( - [ - ("ptpython", _embed_ptpython_shell), - ("ipython", _embed_ipython_shell), - ("bpython", _embed_bpython_shell), - ("python", _embed_standard_shell), - ] -) +DEFAULT_PYTHON_SHELLS = { + "ptpython": _embed_ptpython_shell, + "ipython": _embed_ipython_shell, + "bpython": _embed_bpython_shell, + "python": _embed_standard_shell, +} def get_shell_embed_func(shells=None, known_shells=None): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 4af2a647a..21cd5e78f 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -26,10 +26,7 @@ from twisted.python import failure from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest -from scrapy.utils.reactor import ( - is_asyncio_reactor_installed, - get_asyncio_event_loop_policy, -) +from scrapy.utils.reactor import is_asyncio_reactor_installed, _get_asyncio_event_loop def defer_fail(_failure: Failure) -> Deferred: @@ -290,7 +287,7 @@ def deferred_from_coro(o) -> Any: # that use asyncio, e.g. "await asyncio.sleep(1)" return ensureDeferred(o) # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - event_loop = get_asyncio_event_loop_policy().get_event_loop() + event_loop = _get_asyncio_event_loop() return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) return o @@ -343,8 +340,7 @@ def deferred_to_future(d: Deferred) -> Future: d = treq.get('https://example.com/additional') additional_response = await deferred_to_future(d) """ - policy = get_asyncio_event_loop_policy() - return d.asFuture(policy.get_event_loop()) + return d.asFuture(_get_asyncio_event_loop()) def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index e40016031..c20948fd3 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,6 +1,7 @@ import asyncio import sys from contextlib import suppress +from warnings import catch_warnings, filterwarnings from twisted.internet import asyncioreactor, error @@ -83,6 +84,10 @@ def install_reactor(reactor_path, event_loop_path=None): installer() +def _get_asyncio_event_loop(): + return set_asyncio_event_loop(None) + + def set_asyncio_event_loop(event_loop_path): """Sets and returns the event loop with specified import path.""" policy = get_asyncio_event_loop_policy() @@ -92,11 +97,26 @@ def set_asyncio_event_loop(event_loop_path): asyncio.set_event_loop(event_loop) else: try: - event_loop = policy.get_event_loop() + with catch_warnings(): + # In Python 3.10.9, 3.11.1, 3.12 and 3.13, a DeprecationWarning + # is emitted about the lack of a current event loop, because in + # Python 3.14 and later `get_event_loop` will raise a + # RuntimeError in that event. Because our code is already + # prepared for that future behavior, we ignore the deprecation + # warning. + filterwarnings( + "ignore", + message="There is no current event loop", + category=DeprecationWarning, + ) + event_loop = policy.get_event_loop() except RuntimeError: - # `get_event_loop` is expected to fail when called from a new thread - # with no asyncio event loop yet installed. Such is the case when - # called from `scrapy shell` + # `get_event_loop` raises RuntimeError when called with no asyncio + # event loop yet installed in the following scenarios: + # - From a thread other than the main thread. For example, when + # using ``scrapy shell``. + # - Previsibly on Python 3.14 and later. + # https://github.com/python/cpython/issues/100160#issuecomment-1345581902 event_loop = policy.new_event_loop() asyncio.set_event_loop(event_loop) return event_loop diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 3693177e4..a91a49170 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -40,7 +40,7 @@ def get_meta_refresh( response: "scrapy.http.response.text.TextResponse", ignore_tags: Optional[Iterable[str]] = ("script", "noscript"), ) -> Union[Tuple[None, None], Tuple[float, str]]: - """Parse the http-equiv refrsh parameter from the given response""" + """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] _metaref_cache[response] = html.get_meta_refresh( diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index b73261723..9f03621c1 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,14 +1,9 @@ -import OpenSSL +import OpenSSL.SSL import OpenSSL._util as pyOpenSSLutil from scrapy.utils.python import to_unicode -# The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL. -# Using the binding directly, as this code does, requires cryptography 2.4. -SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, "SSL_OP_NO_TLSv1_3", 0) - - def ffi_buf_to_string(buf): return to_unicode(pyOpenSSLutil.ffi.string(buf)) @@ -24,11 +19,6 @@ def x509name_to_string(x509name): def get_temp_key_info(ssl_object): - if not hasattr( - pyOpenSSLutil.lib, "SSL_get_server_tmp_key" - ): # requires OpenSSL 1.0.2 - return None - # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **") if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p): diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index ad3bff4f0..cd8a6a05a 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -48,7 +48,7 @@ def parse_url(url, encoding=None): def escape_ajax(url): """ - Return the crawleable url according to: + Return the crawlable url according to: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started >>> escape_ajax("www.example.com/ajax.html#!key=value") diff --git a/sep/sep-016.rst b/sep/sep-016.rst index 335f09f45..a60ab30dd 100644 --- a/sep/sep-016.rst +++ b/sep/sep-016.rst @@ -148,7 +148,7 @@ Another example could be for building URL canonicalizers: :: #!python - class CanonializeUrl(LegSpider): + class CanonicalizeUrl(LegSpider): def process_request(self, request): curl = canonicalize_url(request.url, rules=self.spider.canonicalization_rules) diff --git a/sep/sep-018.rst b/sep/sep-018.rst index fe707923a..96df82f6b 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -321,7 +321,7 @@ Another example could be for building URL canonicalizers: :: #!python - class CanonializeUrl(object): + class CanonicalizeUrl(object): def process_request(self, request, response, spider): curl = canonicalize_url(request.url, @@ -594,18 +594,18 @@ A middleware to Scrape data using Parsley as described in UsingParsley class ParsleyExtractor(object): - def __init__(self, parslet_json_code): - parslet = json.loads(parselet_json_code) + def __init__(self, parsley_json_code): + parsley = json.loads(parselet_json_code) class ParsleyItem(Item): def __init__(self, *a, **kw): - for name in parslet.keys(): + for name in parsley.keys(): self.fields[name] = Field() super(ParsleyItem, self).__init__(*a, **kw) self.item_class = ParsleyItem - self.parsley = PyParsley(parslet, output='python') + self.parsley = PyParsley(parsley, output='python') def process_response(self, response, request, spider): - return self.item_class(self.parsly.parse(string=response.body)) + return self.item_class(self.parsley.parse(string=response.body)) diff --git a/sep/sep-021.rst b/sep/sep-021.rst index c1ec16f7f..d56bc26af 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -79,7 +79,7 @@ If it raises an exception, Scrapy will print it and exit. Examples:: def addon_configure(settings): - settings.overrides['DOWNLADER_MIDDLEWARES'].update({ + settings.overrides['DOWNLOADER_MIDDLEWARES'].update({ 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, }) diff --git a/setup.py b/setup.py index 5d1245f37..f53334d4e 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def has_environment_marker_platform_impl_support(): install_requires = [ "Twisted>=18.9.0", - "cryptography>=3.3", + "cryptography>=3.4.6", "cssselect>=0.9.1", "itemloaders>=1.0.1", "parsel>=1.5.0", diff --git a/tests/mockserver.py b/tests/mockserver.py index 9f1a707c9..e07ae8797 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -19,7 +19,6 @@ from twisted.web.static import File from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3 from scrapy.utils.test import get_testenv @@ -358,7 +357,7 @@ def ssl_context_factory( if cipher_string: ctx = factory.getContext() # disabling TLS1.3 because it unconditionally enables some strong ciphers - ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3) + ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_3) ctx.set_cipher_list(to_bytes(cipher_string)) return factory diff --git a/tests/pipelines.py b/tests/pipelines.py index 4e73178d5..d9fc12676 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -11,6 +11,6 @@ class ZeroDivisionErrorPipeline: return item -class ProcessWithZeroDivisionErrorPipiline: +class ProcessWithZeroDivisionErrorPipeline: def process_item(self, item, spider): 1 / 0 diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html index e3a2a4145..29075602d 100644 --- a/tests/sample_data/link_extractor/linkextractor.html +++ b/tests/sample_data/link_extractor/linkextractor.html @@ -13,6 +13,7 @@
sample 3 text sample 3 repetition + sample 3 repetition sample 3 repetition with fragment inner tag diff --git a/tests/test_commands.py b/tests/test_commands.py index ed0eb85f5..363e87aa7 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -336,7 +336,7 @@ class StartprojectTemplatesTest(ProjectTest): self.assertEqual(actual_permissions, expected_permissions) def test_startproject_permissions_unchanged_in_destination(self): - """Check that pre-existing folders and files in the destination folder + """Check that preexisting folders and files in the destination folder do not see their permissions modified.""" scrapy_path = scrapy.__path__[0] project_template = Path(scrapy_path, "templates", "project") diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3f19bef22..4139f1b11 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -154,7 +154,7 @@ class CrawlTestCase(TestCase): raise unittest.SkipTest("Non-existing hosts are resolvable") crawler = get_crawler(SimpleSpider) with LogCapture() as log: - # try to fetch the homepage of a non-existent domain + # try to fetch the homepage of a nonexistent domain yield crawler.crawl( "http://dns.resolution.invalid./", mockserver=self.mockserver ) @@ -183,7 +183,7 @@ class CrawlTestCase(TestCase): self.assertIs(record.exc_info[0], ZeroDivisionError) @defer.inlineCallbacks - def test_start_requests_lazyness(self): + def test_start_requests_laziness(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(BrokenStartRequestsSpider, settings) yield crawler.crawl(mockserver=self.mockserver) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 536d09f18..4f953439d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -209,6 +209,12 @@ class LargeChunkedFileResource(resource.Resource): return server.NOT_DONE_YET +class DuplicateHeaderResource(resource.Resource): + def render(self, request): + request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) + return b"" + + class HttpTestCase(unittest.TestCase): scheme = "http" download_handler_cls: Type = HTTPDownloadHandler @@ -234,6 +240,7 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"contentlength", ContentLengthHeaderResource()) r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) r.putChild(b"largechunkedfile", LargeChunkedFileResource()) + r.putChild(b"duplicate-header", DuplicateHeaderResource()) r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) @@ -407,6 +414,16 @@ class HttpTestCase(unittest.TestCase): HtmlResponse, ) + def test_get_duplicate_header(self): + def _test(response): + self.assertEqual( + response.headers.getlist(b"Set-Cookie"), + [b"a=b", b"c=d"], + ) + + request = Request(self.getURL("duplicate-header")) + return self.download_request(request, Spider("foo")).addCallback(_test) + class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" @@ -1095,9 +1112,9 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) - def test_ftp_download_notexist(self): + def test_ftp_download_nonexistent(self): request = Request( - url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt", meta=self.req_meta + url=f"ftp://127.0.0.1:{self.portNum}/nonexistent.txt", meta=self.req_meta ) d = self.download_handler.download_request(request, None) diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index 927f0ae4d..0702dd042 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -19,7 +19,7 @@ class UserAgentMiddlewareTest(TestCase): self.assertEqual(req.headers["User-Agent"], b"default_useragent") def test_remove_agent(self): - # settings UESR_AGENT to None should remove the user agent + # settings USER_AGENT to None should remove the user agent spider, mw = self.get_spider_and_mw("default_useragent") spider.user_agent = None mw.spider_opened(spider) diff --git a/tests/test_engine.py b/tests/test_engine.py index 7b861332e..7ddb420ba 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -109,7 +109,7 @@ class DataClassItemsSpider(TestSpider): class ItemZeroDivisionErrorSpider(TestSpider): custom_settings = { "ITEM_PIPELINES": { - "tests.pipelines.ProcessWithZeroDivisionErrorPipiline": 300, + "tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300, } } diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 890c88c7e..09a4aa823 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -33,8 +33,9 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.exporters import CsvItemExporter +from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( + _FeedSlot, BlockingFeedStorage, FeedExporter, FileFeedStorage, @@ -664,6 +665,50 @@ class FeedExportTestBase(ABC, unittest.TestCase): return result +class InstrumentedFeedSlot(_FeedSlot): + """Instrumented _FeedSlot subclass for keeping track of calls to + start_exporting and finish_exporting.""" + + def start_exporting(self): + self.update_listener("start") + super().start_exporting() + + def finish_exporting(self): + self.update_listener("finish") + super().finish_exporting() + + @classmethod + def subscribe__listener(cls, listener): + cls.update_listener = listener.update + + +class IsExportingListener: + """When subscribed to InstrumentedFeedSlot, keeps track of when + a call to start_exporting has been made without a closing call to + finish_exporting and when a call to finish_exporting has been made + before a call to start_exporting.""" + + def __init__(self): + self.start_without_finish = False + self.finish_without_start = False + + def update(self, method): + if method == "start": + self.start_without_finish = True + elif method == "finish": + if self.start_without_finish: + self.start_without_finish = False + else: + self.finish_before_start = True + + +class ExceptionJsonItemExporter(JsonItemExporter): + """JsonItemExporter that throws an exception every time export_item is called.""" + + def export_item(self, _): + raise Exception("foo") + + class FeedExportTest(FeedExportTestBase): __test__ = True @@ -909,6 +954,84 @@ class FeedExportTest(FeedExportTestBase): data = yield self.exported_no_data(settings) self.assertEqual(b"", data[fmt]) + @defer.inlineCallbacks + def test_start_finish_exporting_items(self): + items = [ + self.MyItem({"foo": "bar1", "egg": "spam1"}), + ] + settings = { + "FEEDS": { + self._random_temp_filename(): {"format": "json"}, + }, + "FEED_EXPORT_INDENT": None, + } + + listener = IsExportingListener() + InstrumentedFeedSlot.subscribe__listener(listener) + + with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot): + _ = yield self.exported_data(items, settings) + self.assertFalse(listener.start_without_finish) + self.assertFalse(listener.finish_without_start) + + @defer.inlineCallbacks + def test_start_finish_exporting_no_items(self): + items = [] + settings = { + "FEEDS": { + self._random_temp_filename(): {"format": "json"}, + }, + "FEED_EXPORT_INDENT": None, + } + + listener = IsExportingListener() + InstrumentedFeedSlot.subscribe__listener(listener) + + with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot): + _ = yield self.exported_data(items, settings) + self.assertFalse(listener.start_without_finish) + self.assertFalse(listener.finish_without_start) + + @defer.inlineCallbacks + def test_start_finish_exporting_items_exception(self): + items = [ + self.MyItem({"foo": "bar1", "egg": "spam1"}), + ] + settings = { + "FEEDS": { + self._random_temp_filename(): {"format": "json"}, + }, + "FEED_EXPORTERS": {"json": ExceptionJsonItemExporter}, + "FEED_EXPORT_INDENT": None, + } + + listener = IsExportingListener() + InstrumentedFeedSlot.subscribe__listener(listener) + + with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot): + _ = yield self.exported_data(items, settings) + self.assertFalse(listener.start_without_finish) + self.assertFalse(listener.finish_without_start) + + @defer.inlineCallbacks + def test_start_finish_exporting_no_items_exception(self): + items = [] + settings = { + "FEEDS": { + self._random_temp_filename(): {"format": "json"}, + }, + "FEED_EXPORTERS": {"json": ExceptionJsonItemExporter}, + "FEED_EXPORT_INDENT": None, + } + + listener = IsExportingListener() + InstrumentedFeedSlot.subscribe__listener(listener) + + with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot): + _ = yield self.exported_data(items, settings) + self.assertFalse(listener.start_without_finish) + self.assertFalse(listener.finish_without_start) + @defer.inlineCallbacks def test_export_no_items_store_empty(self): formats = ( diff --git a/tests/test_http_request.py b/tests/test_http_request.py index b6f999b29..0c10b27a0 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -399,7 +399,7 @@ class RequestTest(unittest.TestCase): ) self.assertEqual(r.method, "DELETE") - # If `ignore_unknon_options` is set to `False` it raises an error with + # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z self.assertRaises( ValueError, @@ -997,7 +997,7 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) - def test_from_response_formname_notexist(self): + def test_from_response_formname_nonexistent(self): response = _buildresponse( """ @@ -1044,7 +1044,7 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) - def test_from_response_formname_notexists_fallback_formid(self): + def test_from_response_formname_nonexistent_fallback_formid(self): response = _buildresponse( """ @@ -1062,7 +1062,7 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) - def test_from_response_formid_notexist(self): + def test_from_response_formid_nonexistent(self): response = _buildresponse( """ diff --git a/tests/test_http_response.py b/tests/test_http_response.py index ce32092c1..8c422bb4f 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -518,7 +518,7 @@ class TextResponseTest(BaseResponseTest): def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, # this test tries to ensure that calling response.encoding and - # response.text in indistint order doesn't affect final + # response.text in indistinct order doesn't affect final # values for encoding and decoded body. url = "http://example.com" body = b"\xef\xbb\xbfWORD" @@ -645,6 +645,7 @@ class TextResponseTest(BaseResponseTest): "http://example.com/sample2.html", "http://example.com/sample3.html", "http://example.com/sample3.html", + "http://example.com/sample3.html", "http://example.com/sample3.html#foo", "http://www.google.com/something", "http://example.com/innertag.html", diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 520effedb..f663013ba 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -74,6 +74,10 @@ class Base: url="http://example.com/sample3.html", text="sample 3 repetition", ), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), Link( url="http://example.com/sample3.html#foo", text="sample 3 repetition with fragment", @@ -93,6 +97,10 @@ class Base: url="http://example.com/sample3.html", text="sample 3 repetition", ), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), Link( url="http://example.com/sample3.html", text="sample 3 repetition with fragment", diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 3cd3e5966..38a2d6c41 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -225,8 +225,8 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(buf.getvalue(), thumb_buf.getvalue()) expected_warning_msg = ( - ".convert_image() method overriden in a deprecated way, " - "overriden method does not accept response_body argument." + ".convert_image() method overridden in a deprecated way, " + "overridden method does not accept response_body argument." ) self.assertEqual( len( @@ -244,7 +244,7 @@ class ImagesPipelineTestCase(unittest.TestCase): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") SIZE = (100, 100) - # straigh forward case: RGB and JPEG + # straight forward case: RGB and JPEG COLOUR = (0, 127, 255) im, _ = _create_image("JPEG", "RGB", SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) @@ -271,7 +271,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.mode, "RGB") self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - # ensure that we recieved deprecation warnings + # ensure that we received deprecation warnings expected_warning_msg = ".convert_image() method called in a deprecated way" self.assertTrue( len( @@ -287,7 +287,7 @@ class ImagesPipelineTestCase(unittest.TestCase): def test_convert_image_new(self): # tests for new API SIZE = (100, 100) - # straigh forward case: RGB and JPEG + # straight forward case: RGB and JPEG COLOUR = (0, 127, 255) im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index f7540743d..17c0309d1 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -11,12 +11,12 @@ from tests.mockserver import MockServer from tests.spiders import SingleRequestSpider -OVERRIDEN_URL = "https://example.org" +OVERRIDDEN_URL = "https://example.org" class ProcessResponseMiddleware: def process_response(self, request, response, spider): - return response.replace(request=Request(OVERRIDEN_URL)) + return response.replace(request=Request(OVERRIDDEN_URL)) class RaiseExceptionRequestMiddleware: @@ -30,7 +30,7 @@ class CatchExceptionOverrideRequestMiddleware: return Response( url="http://localhost/", body=b"Caught " + exception.__class__.__name__.encode("utf-8"), - request=Request(OVERRIDEN_URL), + request=Request(OVERRIDDEN_URL), ) @@ -52,7 +52,7 @@ class AlternativeCallbacksSpider(SingleRequestSpider): class AlternativeCallbacksMiddleware: def process_response(self, request, response, spider): new_request = request.replace( - url=OVERRIDEN_URL, + url=OVERRIDDEN_URL, callback=spider.alt_callback, cb_kwargs={"foo": "bar"}, ) @@ -132,16 +132,16 @@ class CrawlTestCase(TestCase): yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.request.url, OVERRIDEN_URL) + self.assertEqual(response.request.url, OVERRIDDEN_URL) self.assertEqual(signal_params["response"].url, url) - self.assertEqual(signal_params["request"].url, OVERRIDEN_URL) + self.assertEqual(signal_params["request"].url, OVERRIDDEN_URL) log.check_present( ( "scrapy.core.engine", "DEBUG", - f"Crawled (200) (referer: None)", + f"Crawled (200) (referer: None)", ), ) @@ -166,7 +166,7 @@ class CrawlTestCase(TestCase): yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] self.assertEqual(response.body, b"Caught ZeroDivisionError") - self.assertEqual(response.request.url, OVERRIDEN_URL) + self.assertEqual(response.request.url, OVERRIDDEN_URL) @defer.inlineCallbacks def test_downloader_middleware_do_not_override_in_process_exception(self): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index db5a22adb..dad39b6ee 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -227,7 +227,7 @@ class MixinSameOrigin: ), ("http://example.com:81/page.html", "http://example.com/not-page.html", None), ("http://example.com/page.html", "http://example.com:81/not-page.html", None), - # Different protocols: do NOT send refferer + # Different protocols: do NOT send referrer ("https://example.com/page.html", "http://example.com/not-page.html", None), ("https://example.com/page.html", "http://not.example.com/", None), ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None), @@ -750,19 +750,19 @@ class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {"referrer_policy": POLICY_UNSAFE_URL} -class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaPrecedence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"} req_meta = {"referrer_policy": POLICY_UNSAFE_URL} -class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): +class TestRequestMetaPrecedence002(MixinNoReferrer, TestRefererMiddleware): settings = { "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy" } req_meta = {"referrer_policy": POLICY_NO_REFERRER} -class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware): settings = { "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy" } @@ -888,19 +888,19 @@ class TestSettingsPolicyByName(TestCase): RefererMiddleware(settings) -class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): +class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"} resp_headers = {"Referrer-Policy": POLICY_UNSAFE_URL.upper()} -class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): +class TestPolicyHeaderPrecedence002(MixinNoReferrer, TestRefererMiddleware): settings = { "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy" } resp_headers = {"Referrer-Policy": POLICY_NO_REFERRER.swapcase()} -class TestPolicyHeaderPredecence003( +class TestPolicyHeaderPrecedence003( MixinNoReferrerWhenDowngrade, TestRefererMiddleware ): settings = { @@ -909,7 +909,7 @@ class TestPolicyHeaderPredecence003( resp_headers = {"Referrer-Policy": POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} -class TestPolicyHeaderPredecence004( +class TestPolicyHeaderPrecedence004( MixinNoReferrerWhenDowngrade, TestRefererMiddleware ): """ diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 25cef2b26..d09335651 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -15,6 +15,11 @@ class AsyncioTest(TestCase): ) def test_install_asyncio_reactor(self): + from twisted.internet import reactor as original_reactor + with warnings.catch_warnings(record=True) as w: install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") self.assertEqual(len(w), 0) + from twisted.internet import reactor + + assert original_reactor == reactor diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 1a8c96e7a..214deceb2 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -74,7 +74,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): self.assertIn("foo.NewClass", str(w[1].message)) self.assertIn("bar.OldClass", str(w[1].message)) - def test_subclassing_warns_only_on_direct_childs(self): + def test_subclassing_warns_only_on_direct_children(self): Deprecated = create_deprecated_class( "Deprecated", NewName, warn_once=False, warn_category=MyWarning ) diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index 43236acec..da61f4b0b 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -7,17 +7,27 @@ from scrapy.utils.display import pformat, pprint class TestDisplay(TestCase): object = {"a": 1} - colorized_string = ( - "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" - "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n" - ) + colorized_strings = { + ( + ( + "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" + "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}" + ) + + suffix + ) + for suffix in ( + # https://github.com/pygments/pygments/issues/2313 + "\n", # pygments ≤ 2.13 + "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14 + ) + } plain_string = "{'a': 1}" @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") def test_pformat(self, isatty): isatty.return_value = True - self.assertEqual(pformat(self.object), self.colorized_string) + self.assertIn(pformat(self.object), self.colorized_strings) @mock.patch("sys.stdout.isatty") def test_pformat_dont_colorize(self, isatty): @@ -33,7 +43,7 @@ class TestDisplay(TestCase): def test_pformat_old_windows(self, isatty, version): isatty.return_value = True version.return_value = "10.0.14392" - self.assertEqual(pformat(self.object), self.colorized_string) + self.assertIn(pformat(self.object), self.colorized_strings) @mock.patch("sys.platform", "win32") @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") @@ -55,7 +65,7 @@ class TestDisplay(TestCase): isatty.return_value = True version.return_value = "10.0.14393" terminal_processing.return_value = True - self.assertEqual(pformat(self.object), self.colorized_string) + self.assertIn(pformat(self.object), self.colorized_strings) @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3c6270864..5caa5b8f2 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -159,7 +159,7 @@ class UtilsPythonTestCase(unittest.TestCase): b = Obj() # no attributes given return False self.assertFalse(equal_attributes(a, b, [])) - # not existent attributes + # nonexistent attributes self.assertFalse(equal_attributes(a, b, ["x", "y"])) a.x = 1 diff --git a/tox.ini b/tox.ini index e37ae4579..eaf29173a 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,7 @@ download = true commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = - pip install -U -ctests/upper-constraints.txt {opts} {packages} + python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} [testenv:typing] basepython = python3 @@ -63,8 +63,7 @@ commands = flake8 {posargs:docs scrapy tests} [testenv:pylint] -# reppy does not support Python 3.9+ -basepython = python3.8 +basepython = python3 deps = {[testenv:extra-deps]deps} pylint==2.15.6 @@ -75,13 +74,14 @@ commands = basepython = python3 deps = twine==4.0.1 + build==0.9.0 commands = - python setup.py sdist + python -m build --sdist twine check dist/* [pinned] deps = - cryptography==3.3 + cryptography==3.4.6 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 @@ -106,7 +106,7 @@ deps = setenv = _SCRAPY_PINNED=true install_command = - pip install -U {opts} {packages} + python -I -m pip install {opts} {packages} [testenv:pinned] deps = @@ -126,8 +126,7 @@ setenv = {[pinned]setenv} [testenv:extra-deps] -# reppy does not support Python 3.9+ -basepython = python3.8 +basepython = python3 deps = {[testenv]deps} boto @@ -135,7 +134,6 @@ deps = # Twisted[http2] currently forces old mitmproxy because of h2 version # restrictions in their deps, so we need to pin old markupsafe here too. markupsafe < 2.1.0 - reppy robotexclusionrulesparser Pillow>=4.0.0 Twisted[http2]>=17.9.0 From 0cfe81d1d3978169f6c8ccc90e49315dcd67aca7 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Sat, 21 Jan 2023 16:57:31 -0300 Subject: [PATCH 0789/2083] `set-output` command is deprecated --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index eee9a4f02..02cf0c9fa 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,7 +18,7 @@ jobs: id: check-release-tag run: | if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then - echo ::set-output name=release_tag::true + echo "release_tag=true" >> $GITHUB_OUTPUT fi - name: Publish to PyPI From 72a853c751044172d90704bce1adb6780e50934e Mon Sep 17 00:00:00 2001 From: Emmanuel Rondan Date: Mon, 23 Jan 2023 14:56:48 -0300 Subject: [PATCH 0790/2083] fixing contributing page with suggestions --- docs/contributing.rst | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 12f0e8d21..edc6c2179 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -179,27 +179,30 @@ Scrapy: See https://help.github.com/en/github/using-git/setting-your-username-in-git for setup instructions. -.. _pre-commit: - Pre-commit ========== We use `pre-commit`_ to automatically address simple code issues before every commit. +.. _pre-commit: https://pre-commit.com/ + Before you start writing a patch: - #. `Install pre-commit `_. - #. On the root of your local clone of the Scrapy repository, run the following command: + +#. `Install pre-commit `_. + +#. On the root of your local clone of the Scrapy repository, run the following + command: .. code-block:: bash - - pre-commit install -Now our pre-commit hooks will run every time you create a Git commit. Upon -finding issues, pre-commit hooks aborts your commit, and they either fix -the corresponding issues automatically or only report them to you. If they fix -the issues automatically, creating your commit again should succeed. Otherwise, -you may need to address the corresponding issues manually first. + pre-commit install + +Now pre-commit will check your changes every time you create a Git commit. Upon +finding issues, pre-commit aborts your commit, and either fixes those issues +automatically, or only reports them to you. If it fixes those issues +automatically, creating your commit again should succeed. Otherwise, you may +need to address the corresponding issues manually first. .. _documentation-policies: From 84fb234cae1bffa944d9f5870c2fc87c75d35261 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Jan 2023 22:47:38 -0800 Subject: [PATCH 0791/2083] fixed additional trigger in FilesPipeline --- scrapy/pipelines/files.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4b0c96b25..a5948525a 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,13 +36,17 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) +def _to_string(path: Union[str, os.PathLike]): + return str(path) # convert a Path object to string + + class FileException(Exception): """General media error exception""" class FSFilesStore: def __init__(self, basedir: Union[str, os.PathLike]): - basedir = str(basedir) # support Path object + basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] self.basedir = basedir @@ -67,7 +71,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: - path_comps = str(path).split('/') + path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) def _mkdir(self, dirname: Path, domain: Optional[str] = None): @@ -323,12 +327,12 @@ class FilesPipeline(MediaPipeline): DEFAULT_FILES_RESULT_FIELD = 'files' def __init__(self, store_uri, download_func=None, settings=None): + store_uri = _to_string(store_uri) if not store_uri: raise NotConfigured if isinstance(settings, dict) or settings is None: settings = Settings(settings) - cls_name = "FilesPipeline" self.store = self._get_store(store_uri) resolve = functools.partial(self._key_for_pipe, From 44512bebc8bc74164c73be55a8fd5debafb48f10 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Jan 2023 22:57:25 -0800 Subject: [PATCH 0792/2083] fixed mypy warnings with type declaration for os.PathLike --- scrapy/pipelines/files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index a5948525a..6a32c8b47 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,7 +36,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, os.PathLike]): +def _to_string(path: Union[str, os.PathLike[str]]): return str(path) # convert a Path object to string @@ -45,7 +45,7 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: Union[str, os.PathLike]): + def __init__(self, basedir: Union[str, os.PathLike[str]]): basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -70,7 +70,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: + def _get_filesystem_path(self, path: Union[str, os.PathLike[str]]) -> Path: path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) From 7c753adbe532f1a027d1d285a7c1e4b59fd9e5a8 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 Jan 2023 23:05:17 -0800 Subject: [PATCH 0793/2083] revert mypy fix didn't work --- scrapy/pipelines/files.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 6a32c8b47..a5948525a 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,7 +36,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, os.PathLike[str]]): +def _to_string(path: Union[str, os.PathLike]): return str(path) # convert a Path object to string @@ -45,7 +45,7 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: Union[str, os.PathLike[str]]): + def __init__(self, basedir: Union[str, os.PathLike]): basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -70,7 +70,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: Union[str, os.PathLike[str]]) -> Path: + def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) From 42e8d5a6157a9d2495bd7a2934efe776eee2a7d6 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 25 Jan 2023 01:03:27 -0800 Subject: [PATCH 0794/2083] fixing type declerations --- scrapy/pipelines/files.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index a5948525a..f9dfa53e3 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -9,6 +9,7 @@ import logging import mimetypes import os import time +from os import PathLike from collections import defaultdict from contextlib import suppress from ftplib import FTP @@ -36,7 +37,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, os.PathLike]): +def _to_string(path: Union[str, PathLike]): return str(path) # convert a Path object to string @@ -45,7 +46,7 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: Union[str, os.PathLike]): + def __init__(self, basedir: Union[str, PathLike]): basedir = _to_string(basedir) if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -70,7 +71,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: + def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path: path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) From c883a13006ad98f9f529fe130f7d717ad91f7afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 25 Jan 2023 17:43:10 +0100 Subject: [PATCH 0795/2083] Make the _set_xback condition more readable --- scrapy/http/request/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a78ba5115..065f2daef 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -106,10 +106,10 @@ class Request(object_ref): self.flags = [] if flags is None else list(flags) def _set_xback(self, name: str, value: Optional[Callable]) -> None: - if ( - value is not None - and (name != "callback" or value is not NO_CALLBACK) - and not callable(value) + if not ( + callable(value) + or value is None + or (name == "callback" and value is NO_CALLBACK) ): raise TypeError(f"{name} must be a callable, got {type(value).__name__}") setattr(self, name, value) From 1f3e42897a6697f799d04a4a4320a7c16cbcd939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 25 Jan 2023 18:30:29 +0100 Subject: [PATCH 0796/2083] =?UTF-8?q?=5FNoCallback=20=E2=86=92=20NoCallbac?= =?UTF-8?q?kType?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/http/request/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 924bd1d3e..068a4baa3 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -24,7 +24,7 @@ RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") # https://github.com/python/typing/issues/689#issuecomment-561425237 -class _NoCallback(Enum): +class NoCallbackType(Enum): NO_CALLBACK = 0 @@ -37,7 +37,7 @@ class _NoCallback(Enum): #: :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download #: middlewares handling such requests can treat them differently from requests #: intended for the :meth:`~scrapy.Spider.parse` callback. -NO_CALLBACK: Final = _NoCallback.NO_CALLBACK +NO_CALLBACK: Final = NoCallbackType.NO_CALLBACK class Request(object_ref): @@ -67,7 +67,7 @@ class Request(object_ref): Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and :func:`~scrapy.utils.request.request_from_dict`. """ - callback: Union[None, _NoCallback, Callable] + callback: Union[None, NoCallbackType, Callable] errback: Optional[Callable] def __init__( From 73f697f1db835be09bf5b717c25f3565d0c054b1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jan 2023 01:10:06 +0400 Subject: [PATCH 0797/2083] Re-enable follow_imports. --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1fab6fe22..c711e6e72 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,6 @@ universal=1 [mypy] ignore_missing_imports = true -follow_imports = skip # FIXME: remove the following sections once the issues are solved From c0ea7fd4fd2726494adf6a84a9fd6b6d7fcf116f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 22:14:24 +0400 Subject: [PATCH 0798/2083] Remove obsolete top-level run code in tests. --- tests/test_dependencies.py | 4 ---- tests/test_utils_iterators.py | 4 ---- tests/test_utils_python.py | 4 ---- 3 files changed, 12 deletions(-) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index f9b2d853c..a39ed0694 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -37,7 +37,3 @@ class ScrapyUtilsTest(unittest.TestCase): pinned_twisted_version_string = match[1] self.assertEqual(twisted_version.short(), pinned_twisted_version_string) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 893582a32..778f7162d 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -528,7 +528,3 @@ class TestHelper(unittest.TestCase): type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}" ) self.assertEqual(a, b) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 5caa5b8f2..8d2695b0d 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -258,7 +258,3 @@ class UtilsPythonTestCase(unittest.TestCase): without_none_values({"one": 1, "none": None, "three": 3, "four": 4}), {"one": 1, "three": 3, "four": 4}, ) - - -if __name__ == "__main__": - unittest.main() From 2e33fb812b227b8f5ded8be9913467c31929a87b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 22:15:07 +0400 Subject: [PATCH 0799/2083] Silence improper typing of twisted.internet.reactor in tests. --- scrapy/utils/benchserver.py | 6 +++--- tests/CrawlerRunner/ip_address.py | 6 +++--- tests/mockserver.py | 13 ++++++++----- tests/test_engine.py | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 32bc2e38c..750d3c093 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -37,11 +37,11 @@ if __name__ == "__main__": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(8998, Site(root)) + httpPort = reactor.listenTCP(8998, Site(root)) # type: ignore[attr-defined] def _print_listening(): httpHost = httpPort.getHost() print(f"Bench server at http://{httpHost.host}:{httpHost.port}") - reactor.callWhenRunning(_print_listening) - reactor.run() + reactor.callWhenRunning(_print_listening) # type: ignore[attr-defined] + reactor.run() # type: ignore[attr-defined] diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 26db16dd6..b9a4485a9 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -41,10 +41,10 @@ if __name__ == "__main__": url = f"http://not.a.real.domain:{port}/echo" servers = [(mock_dns_server.host, mock_dns_server.port)] - reactor.installResolver(createResolver(servers=servers)) + reactor.installResolver(createResolver(servers=servers)) # type: ignore[attr-defined] configure_logging() runner = CrawlerRunner() d = runner.crawl(LocalhostSpider, url=url) - d.addBoth(lambda _: reactor.stop()) - reactor.run() + d.addBoth(lambda _: reactor.stop()) # type: ignore[attr-defined] + reactor.run() # type: ignore[attr-defined] diff --git a/tests/mockserver.py b/tests/mockserver.py index e07ae8797..185897373 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -10,6 +10,7 @@ from urllib.parse import urlencode from OpenSSL import SSL from twisted.internet import defer, reactor, ssl +from twisted.internet.protocol import ServerFactory from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory @@ -369,12 +370,14 @@ if __name__ == "__main__": ) args = parser.parse_args() + factory: ServerFactory + if args.type == "http": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(0, factory) + httpPort = reactor.listenTCP(0, factory) # type: ignore[attr-defined] contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(0, factory, contextFactory) + httpsPort = reactor.listenSSL(0, factory, contextFactory) # type: ignore[attr-defined] def print_listening(): httpHost = httpPort.getHost() @@ -388,11 +391,11 @@ if __name__ == "__main__": clients = [MockDNSResolver()] factory = DNSServerFactory(clients=clients) protocol = dns.DNSDatagramProtocol(controller=factory) - listener = reactor.listenUDP(0, protocol) + listener = reactor.listenUDP(0, protocol) # type: ignore[attr-defined] def print_listening(): host = listener.getHost() print(f"{host.host}:{host.port}") - reactor.callWhenRunning(print_listening) - reactor.run() + reactor.callWhenRunning(print_listening) # type: ignore[attr-defined] + reactor.run() # type: ignore[attr-defined] diff --git a/tests/test_engine.py b/tests/test_engine.py index 7ddb420ba..5fb87424b 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -557,4 +557,4 @@ class EngineTest(unittest.TestCase): if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) - reactor.run() + reactor.run() # type: ignore[attr-defined] From afd5d8532093c25a6ed4453e3a10b275335c94bd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jan 2023 01:12:59 +0400 Subject: [PATCH 0800/2083] Fix remaining issues in tests. --- tests/test_downloadermiddleware_httpproxy.py | 2 +- tests/test_http2_client_protocol.py | 5 ++++- tests/test_spidermiddleware.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index ca125ba36..0040f8d23 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -14,7 +14,7 @@ spider = Spider("foo") class TestHttpProxyMiddleware(TestCase): - failureException = AssertionError + failureException = AssertionError # type: ignore[assignment] def setUp(self): self._oldenv = os.environ.copy() diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 88345d2bc..2a928eea0 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,6 +5,7 @@ import shutil import string from ipaddress import IPv4Address from pathlib import Path +from typing import Dict from unittest import mock, skipIf from urllib.parse import urlencode @@ -86,6 +87,7 @@ class GetDataHtmlLarge(LeafResource): class PostDataJsonMixin: @staticmethod def make_response(request: TxRequest, extra_data: str): + assert request.content is not None response = { "request-headers": {}, "request-body": json.loads(request.content.read()), @@ -144,7 +146,8 @@ class QueryParams(LeafResource): request.setHeader("Content-Type", "application/json; charset=UTF-8") request.setHeader("Content-Encoding", "UTF-8") - query_params = {} + query_params: Dict[str, str] = {} + assert request.args is not None for k, v in request.args.items(): query_params[str(k, "utf-8")] = str(v[0], "utf-8") diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 760ee43df..ca6348913 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -112,6 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): Should work for process_spider_output and, when it's supported, process_start_requests. """ + ITEM_TYPE: type RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod From 232aab53b38553262d60259461adf3fa8054e445 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 22:28:15 +0400 Subject: [PATCH 0801/2083] Fix most of the new typing issues in utils. --- scrapy/utils/defer.py | 19 +++++++++++-------- scrapy/utils/response.py | 7 ++++--- scrapy/utils/testsite.py | 4 ++-- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 21cd5e78f..8fee5a7ed 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -10,6 +10,7 @@ from typing import ( AsyncGenerator, AsyncIterable, Callable, + cast, Coroutine, Generator, Iterable, @@ -38,8 +39,8 @@ def defer_fail(_failure: Failure) -> Deferred: """ from twisted.internet import reactor - d = Deferred() - reactor.callLater(0.1, d.errback, _failure) + d: Deferred = Deferred() + reactor.callLater(0.1, d.errback, _failure) # type: ignore[attr-defined] return d @@ -52,8 +53,8 @@ def defer_succeed(result) -> Deferred: """ from twisted.internet import reactor - d = Deferred() - reactor.callLater(0.1, d.callback, result) + d: Deferred = Deferred() + reactor.callLater(0.1, d.callback, result) # type: ignore[attr-defined] return d @@ -182,7 +183,9 @@ class _AsyncCooperatorAdapter(Iterator): def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. - self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) + self.anext_deferred = cast( + Deferred, deferred_from_coro(self.aiterator.__anext__()) + ) self.anext_deferred.addCallbacks(self._callback, self._errback) def __next__(self) -> Deferred: @@ -190,7 +193,7 @@ class _AsyncCooperatorAdapter(Iterator): # It also calls __anext__() if needed. if self.finished: raise StopIteration - d = Deferred() + d: Deferred = Deferred() self.waiting_deferreds.append(d) if not self.anext_deferred: self._call_anext() @@ -209,7 +212,7 @@ def parallel_async( def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks""" - d = Deferred() + d: Deferred = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) @@ -220,7 +223,7 @@ def process_chain_both( callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" - d = Deferred() + d: Deferred = Deferred() for cb, eb in zip(callbacks, errbacks): d.addCallbacks( callback=cb, diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index a91a49170..40a32309a 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -6,7 +6,7 @@ import os import re import tempfile import webbrowser -from typing import Any, Callable, Iterable, Optional, Tuple, Union +from typing import Any, Callable, Iterable, Tuple, Union from weakref import WeakKeyDictionary from twisted.web import http @@ -38,12 +38,13 @@ _metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[floa def get_meta_refresh( response: "scrapy.http.response.text.TextResponse", - ignore_tags: Optional[Iterable[str]] = ("script", "noscript"), + ignore_tags: Iterable[str] = ("script", "noscript"), ) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] - _metaref_cache[response] = html.get_meta_refresh( + # a w3lib typing bug here + _metaref_cache[response] = html.get_meta_refresh( # type: ignore[assignment] text, response.url, response.encoding, ignore_tags=ignore_tags ) return _metaref_cache[response] diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index a47756c4b..119be1dfb 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -50,6 +50,6 @@ def test_site(): if __name__ == "__main__": from twisted.internet import reactor - port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") + port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") # type: ignore[attr-defined] print(f"http://localhost:{port.getHost().port}/") - reactor.run() + reactor.run() # type: ignore[attr-defined] From 764a9d47bb95e87519a4fc2fe20ddf0c97b11739 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 22:31:12 +0400 Subject: [PATCH 0802/2083] Fix typing of inlineCallbacks-decorated functions. --- scrapy/core/engine.py | 6 +++--- scrapy/core/spidermw.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 19696415b..cafa7f6da 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -7,7 +7,7 @@ For more information see docs/topics/architecture.rst import logging import warnings from time import time -from typing import Callable, Iterable, Iterator, Optional, Set, Union +from typing import Any, Callable, Generator, Iterable, Iterator, Optional, Set, Union from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall @@ -96,7 +96,7 @@ class ExecutionEngine: return scheduler_cls @inlineCallbacks - def start(self) -> Deferred: + def start(self) -> Generator[Deferred, Any, None]: if self.running: raise RuntimeError("Engine already running") self.start_time = time() @@ -109,7 +109,7 @@ class ExecutionEngine: """Gracefully stop the execution engine""" @inlineCallbacks - def _finish_stopping_engine(_) -> Deferred: + def _finish_stopping_engine(_) -> Generator[Deferred, Any, None]: yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) self._closewait.callback(None) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1aaed5865..0bc8d54d2 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -182,7 +182,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, result: Union[Iterable, AsyncIterable], start_index: int = 0, - ) -> Deferred: + ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered: Union[MutableChain, MutableAsyncChain] From 5fde6d533903441fe3d986030319de3415a9b956 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 22:54:46 +0400 Subject: [PATCH 0803/2083] Don't type-check Twisted interfaces. --- setup.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.cfg b/setup.cfg index c711e6e72..af9c87945 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,6 +7,10 @@ universal=1 [mypy] ignore_missing_imports = true +# Interface classes are hard to support +[mypy-twisted.internet.interfaces] +follow_imports = skip + # FIXME: remove the following sections once the issues are solved [mypy-scrapy.downloadermiddlewares.httpproxy] From e9094d1f38f2ed2904a6955004a8069054c8e35b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 23:22:18 +0400 Subject: [PATCH 0804/2083] Address remaining typing issues in scrapy.core. --- scrapy/core/downloader/__init__.py | 2 +- scrapy/core/downloader/handlers/http2.py | 8 +++---- scrapy/core/engine.py | 27 ++++++++++++++++++------ scrapy/core/http2/agent.py | 10 +++++---- scrapy/core/http2/protocol.py | 7 ++++++ scrapy/core/http2/stream.py | 2 +- scrapy/core/scraper.py | 8 ++++--- scrapy/core/spidermw.py | 2 +- 8 files changed, 45 insertions(+), 21 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 3a7de8072..e66156ba1 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -95,7 +95,7 @@ class Downloader: dfd = self.middleware.download(self._enqueue_request, request, spider) return dfd.addBoth(_deactivate) - def needs_backout(self): + def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency def _get_slot(self, request, spider): diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 25ac0307b..20cd50c5a 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -28,7 +28,7 @@ class H2DownloadHandler: from twisted.internet import reactor - self._pool = H2ConnectionPool(reactor, settings) + self._pool = H2ConnectionPool(reactor, settings) # type: ignore[arg-type] self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod @@ -82,7 +82,7 @@ class ScrapyH2Agent: "Tunneling via CONNECT method using HTTP/2.0 is not yet supported" ) return self._ProxyAgent( - reactor=reactor, + reactor=reactor, # type: ignore[arg-type] context_factory=self._context_factory, proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")), connect_timeout=timeout, @@ -91,7 +91,7 @@ class ScrapyH2Agent: ) return self._Agent( - reactor=reactor, + reactor=reactor, # type: ignore[arg-type] context_factory=self._context_factory, connect_timeout=timeout, bind_address=bind_address, @@ -108,7 +108,7 @@ class ScrapyH2Agent: d = agent.request(request, spider) d.addCallback(self._cb_latency, request, start_time) - timeout_cl = reactor.callLater(timeout, d.cancel) + timeout_cl = reactor.callLater(timeout, d.cancel) # type: ignore[attr-defined] d.addBoth(self._cb_timeout, request, timeout, timeout_cl) return d diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index cafa7f6da..e44675ef0 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -7,13 +7,24 @@ For more information see docs/topics/architecture.rst import logging import warnings from time import time -from typing import Any, Callable, Generator, Iterable, Iterator, Optional, Set, Union +from typing import ( + Any, + Callable, + cast, + Generator, + Iterable, + Iterator, + Optional, + Set, + Union, +) from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals +from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper from scrapy.exceptions import ( CloseSpider, @@ -80,7 +91,7 @@ class ExecutionEngine: self.paused = False self.scheduler_cls = self._get_scheduler_class(crawler.settings) downloader_cls = load_object(self.settings["DOWNLOADER"]) - self.downloader = downloader_cls(crawler) + self.downloader: Downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback = spider_closed_callback @@ -102,7 +113,7 @@ class ExecutionEngine: self.start_time = time() yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True - self._closewait = Deferred() + self._closewait: Deferred = Deferred() yield self._closewait def stop(self) -> Deferred: @@ -177,11 +188,13 @@ class ExecutionEngine: self._spider_idle() def _needs_backout(self) -> bool: + assert self.slot is not None # typing + assert self.scraper.slot is not None # typing return ( not self.running - or self.slot.closing # type: ignore[union-attr] + or bool(self.slot.closing) or self.downloader.needs_backout() - or self.scraper.slot.needs_backout() # type: ignore[union-attr] + or self.scraper.slot.needs_backout() ) def _next_request_from_scheduler(self) -> Optional[Deferred]: @@ -201,7 +214,7 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - d.addBoth(lambda _: self.slot.remove_request(request)) + d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) d.addErrback( lambda f: logger.info( "Error while removing request from slot", @@ -429,7 +442,7 @@ class ExecutionEngine: dfd.addErrback(log_failure("Scraper close failure")) if hasattr(self.slot.scheduler, "close"): - dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) + dfd.addBoth(lambda _: cast(Slot, self.slot).scheduler.close(reason)) dfd.addErrback(log_failure("Scheduler close failure")) dfd.addBoth( diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 119443c80..3ceda9b61 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -35,7 +35,7 @@ class H2ConnectionPool: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol # instance - d = Deferred() + d: Deferred = Deferred() self._pending_requests[key].append(d) return d @@ -53,14 +53,14 @@ class H2ConnectionPool: ) -> Deferred: self._pending_requests[key] = deque() - conn_lost_deferred = Deferred() + conn_lost_deferred: Deferred = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) conn_d = endpoint.connect(factory) conn_d.addCallback(self.put_connection, key) - d = Deferred() + d: Deferred = Deferred() self._pending_requests[key].append(d) return d @@ -83,7 +83,8 @@ class H2ConnectionPool: pending_requests = self._pending_requests.pop(key, None) while pending_requests: d = pending_requests.popleft() - d.errback(errors) + # TODO: this is incorrect, errback takes a single exception + d.errback(errors) # type: ignore[arg-type] def close_connections(self) -> None: """Close all the HTTP/2 connections and remove them from pool @@ -92,6 +93,7 @@ class H2ConnectionPool: Deferred that fires when all connections have been closed """ for conn in self._connections.values(): + assert conn.transport is not None # typing conn.transport.abortConnection() diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 214deeed0..e44aed43c 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -141,6 +141,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): This is used while initiating pending streams to make sure that we initiate stream only during active HTTP/2 Connection """ + assert self.transport is not None # typing return bool(self.transport.connected) and self.metadata["settings_acknowledged"] @property @@ -197,6 +198,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """Write data to the underlying transport connection from the HTTP2 connection instance if any """ + assert self.transport is not None # typing # Reset the idle timeout as connection is still actively sending data self.resetTimeout() @@ -227,6 +229,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # Initialize the timeout self.setTimeout(self.IDLE_TIMEOUT) + assert self.transport is not None # typing destination = self.transport.getPeer() self.metadata["ip_address"] = ipaddress.ip_address(destination.host) @@ -238,12 +241,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """Helper function to lose the connection with the error sent as a reason""" self._conn_lost_errors += errors + assert self.transport is not None # typing self.transport.loseConnection() def handshakeCompleted(self) -> None: """ Close the connection if it's not made via the expected protocol """ + assert self.transport is not None # typing if ( self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME @@ -276,6 +281,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # hyper-h2 does not drop the connection in this scenario, we # need to abort the connection manually. self._conn_lost_errors += [e] + assert self.transport is not None # typing self.transport.abortConnection() return @@ -389,6 +395,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._send_pending_requests() # Update certificate when our HTTP/2 connection is established + assert self.transport is not None # typing self.metadata["certificate"] = Certificate(self.transport.getPeerCertificate()) def stream_ended(self, event: StreamEnded) -> None: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 2b5c98e5f..5324a5ca1 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -151,7 +151,7 @@ class Stream: else: self.close(StreamCloseReason.CANCELLED) - self._deferred_response = Deferred(_cancel) + self._deferred_response: Deferred = Deferred(_cancel) def __repr__(self) -> str: return f"Stream(id={self.stream_id!r})" diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7c2eefbe6..c1488a46c 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -66,7 +66,7 @@ class Slot: def add_response_request( self, result: Union[Response, Failure], request: Request ) -> Deferred: - deferred = Deferred() + deferred: Deferred = Deferred() self.queue.append((result, request, deferred)) if isinstance(result, Response): self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -205,10 +205,12 @@ class Scraper: callback=callback, callbackKeywords=result.request.cb_kwargs ) else: # result is a Failure - result.request = request + # TODO: properly type adding this attribute to a Failure + result.request = request # type: ignore[attr-defined] warn_on_generator_with_return_value(spider, request.errback) dfd = defer_fail(result) - dfd.addErrback(request.errback) + if request.errback: + dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) def handle_spider_error( diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 0bc8d54d2..31e815b19 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -157,7 +157,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # simplified when downgrading is removed. if dfd.called: # the result is available immediately if _process_spider_output didn't do downgrading - return dfd.result + return cast(MutableChain, dfd.result) # we forbid waiting here because otherwise we would need to return a deferred from # _process_spider_exception too, which complicates the architecture msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" From a4edff31b91c4647a02bb392020110856b669fd9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 23:30:21 +0400 Subject: [PATCH 0805/2083] Replace lxml-stubs with types-lxml. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index eaf29173a..f05ebc45a 100644 --- a/tox.ini +++ b/tox.ini @@ -37,9 +37,9 @@ install_command = [testenv:typing] basepython = python3 deps = - lxml-stubs==0.2.0 mypy==0.991 types-attrs==19.1.0 + types-lxml==2022.11.8 types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 commands = @@ -201,4 +201,4 @@ deps = black==22.12.0 commands = black {posargs:--check .} - \ No newline at end of file + From 69bb9a78594ed864291d45d36108cc0ad960f048 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Jan 2023 23:52:46 +0400 Subject: [PATCH 0806/2083] Install types-Pillow and types-Pygments. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index f05ebc45a..7048391ac 100644 --- a/tox.ini +++ b/tox.ini @@ -40,6 +40,8 @@ deps = mypy==0.991 types-attrs==19.1.0 types-lxml==2022.11.8 + types-Pillow==9.4.0.5 + types-Pygments==2.14.0.1 types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 commands = From 6d0f9df8c1ad5ad4c4bab4be32cdbe8aed325cd4 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 25 Jan 2023 14:22:42 -0600 Subject: [PATCH 0807/2083] added isort.cfg --- .isort.cfg | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 000000000..a29184f0a --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,3 @@ +[settings] +profile = black +multi_line_output = 3 From a5c1ef82762c6c0910abea00c0a6249c40005e44 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 25 Jan 2023 14:25:15 -0600 Subject: [PATCH 0808/2083] sort imports with isort --- conftest.py | 1 - docs/_ext/scrapydocs.py | 3 ++- extras/qps-bench-server.py | 7 ++++--- extras/qpsclient.py | 2 +- scrapy/__init__.py | 8 ++++---- scrapy/cmdline.py | 9 +++++---- scrapy/commands/__init__.py | 6 +++--- scrapy/commands/bench.py | 2 +- scrapy/commands/check.py | 5 +++-- scrapy/commands/edit.py | 2 +- scrapy/commands/fetch.py | 5 +++-- scrapy/commands/genspider.py | 5 ++--- scrapy/commands/parse.py | 6 +++--- scrapy/commands/runspider.py | 6 +++--- scrapy/commands/shell.py | 2 +- scrapy/commands/startproject.py | 7 +++---- scrapy/commands/view.py | 1 + scrapy/contracts/default.py | 2 +- scrapy/core/downloader/__init__.py | 12 +++++------ scrapy/core/downloader/contextfactory.py | 8 ++++---- scrapy/core/downloader/handlers/__init__.py | 1 - scrapy/core/downloader/handlers/http11.py | 6 +++--- scrapy/core/downloader/handlers/http2.py | 1 - scrapy/core/downloader/middleware.py | 2 +- scrapy/core/downloader/tls.py | 4 ++-- scrapy/core/downloader/webclient.py | 6 +++--- scrapy/core/engine.py | 11 +++------- scrapy/core/http2/agent.py | 2 +- scrapy/core/http2/protocol.py | 5 ++--- scrapy/core/http2/stream.py | 4 ++-- scrapy/core/scheduler.py | 1 - scrapy/core/scraper.py | 6 ++---- scrapy/core/spidermw.py | 5 ++--- scrapy/crawler.py | 6 +++--- scrapy/downloadermiddlewares/ajaxcrawl.py | 3 +-- scrapy/downloadermiddlewares/decompression.py | 1 - scrapy/downloadermiddlewares/httpcache.py | 1 - scrapy/downloadermiddlewares/httpproxy.py | 2 +- scrapy/downloadermiddlewares/redirect.py | 2 +- scrapy/downloadermiddlewares/retry.py | 3 +-- scrapy/downloadermiddlewares/robotstxt.py | 3 ++- scrapy/dupefilters.py | 3 +-- scrapy/exporters.py | 3 +-- scrapy/extensions/debug.py | 6 +++--- scrapy/extensions/feedexport.py | 5 ++--- scrapy/extensions/httpcache.py | 2 +- scrapy/extensions/logstats.py | 2 +- scrapy/extensions/memusage.py | 6 +++--- scrapy/extensions/statsmailer.py | 2 +- scrapy/extensions/telnet.py | 12 +++++------ scrapy/extensions/throttle.py | 2 +- scrapy/http/__init__.py | 6 ++---- scrapy/http/cookies.py | 4 ++-- scrapy/http/headers.py | 1 + scrapy/http/request/__init__.py | 1 - scrapy/http/request/form.py | 5 ++--- scrapy/http/request/rpc.py | 1 - scrapy/http/response/text.py | 2 +- scrapy/loader/processors.py | 1 - scrapy/logformatter.py | 2 +- scrapy/mail.py | 5 ++--- scrapy/middleware.py | 2 +- scrapy/pipelines/files.py | 1 - scrapy/pipelines/media.py | 4 ++-- scrapy/pqueues.py | 1 - scrapy/resolver.py | 3 +-- scrapy/responsetypes.py | 2 +- scrapy/robotstxt.py | 2 +- scrapy/selector/unified.py | 6 +++--- scrapy/settings/__init__.py | 3 +-- scrapy/shell.py | 7 ++----- scrapy/signalmanager.py | 1 + scrapy/spidermiddlewares/offsite.py | 2 +- scrapy/spidermiddlewares/referer.py | 1 - scrapy/spidermiddlewares/urllength.py | 2 +- scrapy/spiders/__init__.py | 2 +- scrapy/spiders/crawl.py | 2 +- scrapy/spiders/feed.py | 8 ++++---- scrapy/spiders/sitemap.py | 7 +++---- scrapy/statscollectors.py | 2 +- scrapy/utils/benchserver.py | 2 +- scrapy/utils/conf.py | 1 - scrapy/utils/curl.py | 2 +- scrapy/utils/defer.py | 2 +- scrapy/utils/deprecate.py | 3 ++- scrapy/utils/display.py | 1 + scrapy/utils/ftp.py | 2 +- scrapy/utils/httpobj.py | 3 +-- scrapy/utils/iterators.py | 3 +-- scrapy/utils/log.py | 1 - scrapy/utils/misc.py | 9 ++++----- scrapy/utils/ossignal.py | 1 - scrapy/utils/project.py | 6 ++---- scrapy/utils/reqser.py | 1 - scrapy/utils/request.py | 1 - scrapy/utils/response.py | 5 ++--- scrapy/utils/serialize.py | 4 ++-- scrapy/utils/signal.py | 6 ++---- scrapy/utils/spider.py | 1 - scrapy/utils/ssl.py | 2 +- scrapy/utils/template.py | 2 +- scrapy/utils/test.py | 4 ++-- scrapy/utils/testproc.py | 2 +- scrapy/utils/testsite.py | 2 +- scrapy/utils/trackref.py | 1 - scrapy/utils/url.py | 1 + setup.py | 5 +++-- .../asyncio_enabled_reactor_same_loop.py | 3 +-- ..._select_subclass_twisted_reactor_select.py | 1 + tests/CrawlerRunner/ip_address.py | 8 +++++--- tests/keys/__init__.py | 2 +- tests/mockserver.py | 4 ++-- tests/test_closespider.py | 3 ++- tests/test_cmdline/__init__.py | 2 +- .../__init__.py | 2 +- tests/test_command_fetch.py | 4 ++-- tests/test_command_parse.py | 7 ++++--- tests/test_command_shell.py | 7 +++---- tests/test_command_version.py | 3 ++- tests/test_commands.py | 7 +++---- tests/test_contracts.py | 14 ++++++------- tests/test_crawl.py | 2 +- tests/test_crawler.py | 17 +++++++--------- tests/test_downloader_handlers.py | 8 +++++--- tests/test_downloader_handlers_http2.py | 4 ++-- tests/test_downloadermiddleware.py | 8 ++++---- ...test_downloadermiddleware_ajaxcrawlable.py | 3 +-- tests/test_downloadermiddleware_cookies.py | 4 ++-- ...test_downloadermiddleware_decompression.py | 3 ++- ...est_downloadermiddleware_defaultheaders.py | 2 +- ...st_downloadermiddleware_downloadtimeout.py | 2 +- tests/test_downloadermiddleware_httpauth.py | 2 +- tests/test_downloadermiddleware_httpcache.py | 18 ++++++++--------- ...st_downloadermiddleware_httpcompression.py | 9 +++++---- tests/test_downloadermiddleware_redirect.py | 6 +++--- tests/test_downloadermiddleware_retry.py | 2 +- tests/test_downloadermiddleware_robotstxt.py | 11 +++++----- tests/test_downloadermiddleware_useragent.py | 4 ++-- tests/test_dupefilters.py | 7 ++++--- tests/test_engine.py | 7 +++---- tests/test_engine_stop_download_bytes.py | 5 ++--- tests/test_engine_stop_download_headers.py | 5 ++--- tests/test_exporters.py | 20 +++++++++---------- tests/test_extension_telnet.py | 2 +- tests/test_feedexport.py | 11 +++------- tests/test_http2_client_protocol.py | 13 ++++++------ tests/test_http_cookies.py | 2 +- tests/test_http_headers.py | 2 +- tests/test_http_request.py | 12 +++++------ tests/test_loader.py | 4 ++-- tests/test_loader_deprecated.py | 2 +- tests/test_logformatter.py | 4 ++-- tests/test_mail.py | 7 ++++--- tests/test_middleware.py | 2 +- tests/test_pipeline_crawl.py | 2 +- tests/test_pipeline_files.py | 2 +- tests/test_pipeline_images.py | 1 - tests/test_pipeline_media.py | 11 +++++----- tests/test_pipelines.py | 5 ++--- tests/test_pqueues.py | 3 +-- tests/test_proxy_connect.py | 4 ++-- tests/test_request_attribute_binding.py | 5 +---- tests/test_request_cb_kwargs.py | 2 +- tests/test_request_dict.py | 8 +++----- tests/test_request_left.py | 1 + tests/test_responsetypes.py | 4 ++-- tests/test_scheduler.py | 5 ++--- tests/test_selector.py | 2 +- tests/test_settings/__init__.py | 3 ++- tests/test_signals.py | 3 +-- tests/test_spider.py | 10 +++++----- tests/test_spiderloader/__init__.py | 15 +++++++------- tests/test_spidermiddleware.py | 8 ++++---- tests/test_spidermiddleware_depth.py | 2 +- tests/test_spidermiddleware_httperror.py | 10 +++++----- tests/test_spidermiddleware_offsite.py | 6 +++--- tests/test_spidermiddleware_referer.py | 20 +++++++++---------- tests/test_spidermiddleware_urllength.py | 4 ++-- tests/test_spiderstate.py | 5 +++-- tests/test_squeues.py | 9 +++++---- tests/test_squeues_request.py | 16 +++++++-------- tests/test_stats.py | 4 ++-- tests/test_toplevel.py | 4 ++-- tests/test_urlparse_monkeypatches.py | 2 +- tests/test_utils_asyncio.py | 2 +- tests/test_utils_conf.py | 2 +- tests/test_utils_datatypes.py | 1 - tests/test_utils_defer.py | 6 +++--- tests/test_utils_deprecate.py | 2 +- tests/test_utils_display.py | 3 +-- tests/test_utils_gz.py | 3 +-- tests/test_utils_iterators.py | 4 ++-- tests/test_utils_log.py | 8 ++++---- tests/test_utils_misc/__init__.py | 5 ++--- tests/test_utils_project.py | 8 ++++---- tests/test_utils_python.py | 9 ++++----- tests/test_utils_response.py | 9 ++++----- tests/test_utils_serialize.py | 2 +- tests/test_utils_spider.py | 2 +- tests/test_utils_template.py | 4 ++-- tests/test_utils_url.py | 4 ++-- tests/test_webclient.py | 9 +++++---- 202 files changed, 428 insertions(+), 484 deletions(-) diff --git a/conftest.py b/conftest.py index 585356a3e..e1d4b1213 100644 --- a/conftest.py +++ b/conftest.py @@ -4,7 +4,6 @@ import pytest from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor - from tests.keys import generate_keys diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 1419792fc..c23a89089 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,7 +1,8 @@ from operator import itemgetter -from docutils.parsers.rst.roles import set_classes + from docutils import nodes from docutils.parsers.rst import Directive +from docutils.parsers.rst.roles import set_classes from sphinx.util.nodes import make_refnode diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index 622164c75..70c9003e5 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -1,9 +1,10 @@ #!/usr/bin/env python -from time import time from collections import deque -from twisted.web.server import Site, NOT_DONE_YET -from twisted.web.resource import Resource +from time import time + from twisted.internet import reactor +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET, Site class Root(Resource): diff --git a/extras/qpsclient.py b/extras/qpsclient.py index bb8527af2..acad71e07 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -8,8 +8,8 @@ usage: """ -from scrapy.spiders import Spider from scrapy.http import Request +from scrapy.spiders import Spider class QPSSpider(Spider): diff --git a/scrapy/__init__.py b/scrapy/__init__.py index f0d85198d..44df3d54b 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -8,12 +8,12 @@ import warnings from twisted import version as _txv +from scrapy.http import FormRequest, Request +from scrapy.item import Field, Item +from scrapy.selector import Selector + # Declare top-level shortcuts from scrapy.spiders import Spider -from scrapy.http import Request, FormRequest -from scrapy.selector import Selector -from scrapy.item import Item, Field - __all__ = [ "__version__", diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index ffb40e1c5..730e55350 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,16 +1,17 @@ -import sys -import os import argparse import cProfile import inspect +import os +import sys + import pkg_resources import scrapy +from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess -from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules -from scrapy.utils.project import inside_project, get_project_settings +from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index f37d61321..de68c43a5 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -1,16 +1,16 @@ """ Base class for Scrapy commands """ -import os import argparse +import os from pathlib import Path from typing import Any, Dict, Optional from twisted.python import failure -from scrapy.crawler import CrawlerProcess -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError +from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli class ScrapyCommand: diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 2e2a21f00..911e5afe6 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,6 +1,6 @@ +import subprocess import sys import time -import subprocess from urllib.parse import urlencode import scrapy diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index efc7a46ed..de54ca4d3 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,11 +1,12 @@ import time from collections import defaultdict -from unittest import TextTestRunner, TextTestResult as _TextTestResult +from unittest import TextTestResult as _TextTestResult +from unittest import TextTestRunner from scrapy.commands import ScrapyCommand from scrapy.contracts import ContractsManager -from scrapy.utils.misc import load_object, set_environ from scrapy.utils.conf import build_component_list +from scrapy.utils.misc import load_object, set_environ class TextTestResult(_TextTestResult): diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 537b2013c..ca591011c 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,5 +1,5 @@ -import sys import os +import sys from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 918db55c6..a9076c5b1 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,11 +1,12 @@ import sys + from w3lib.url import is_url from scrapy.commands import ScrapyCommand -from scrapy.http import Request from scrapy.exceptions import UsageError +from scrapy.http import Request from scrapy.utils.datatypes import SequenceExclude -from scrapy.utils.spider import spidercls_for_request, DefaultSpider +from scrapy.utils.spider import DefaultSpider, spidercls_for_request class Command(ScrapyCommand): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index e880e44a9..90dd0874e 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,16 +1,15 @@ import os import shutil import string - -from pathlib import Path from importlib import import_module +from pathlib import Path from typing import Optional, cast from urllib.parse import urlparse import scrapy from scrapy.commands import ScrapyCommand -from scrapy.utils.template import render_templatefile, string_camelcase from scrapy.exceptions import UsageError +from scrapy.utils.template import render_templatefile, string_camelcase def sanitize_module_name(module_name): diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ac97b6193..9c3fc86d4 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -2,15 +2,15 @@ import json import logging from typing import Dict -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter, is_item +from twisted.internet.defer import maybeDeferred from w3lib.url import is_url -from twisted.internet.defer import maybeDeferred from scrapy.commands import BaseRunSpiderCommand +from scrapy.exceptions import UsageError from scrapy.http import Request from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output, spidercls_for_request -from scrapy.exceptions import UsageError logger = logging.getLogger(__name__) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 9751c6c30..8a75f9270 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,13 +1,13 @@ import sys +from importlib import import_module from os import PathLike from pathlib import Path -from importlib import import_module from types import ModuleType from typing import Union -from scrapy.utils.spider import iter_spider_classes -from scrapy.exceptions import UsageError from scrapy.commands import BaseRunSpiderCommand +from scrapy.exceptions import UsageError +from scrapy.utils.spider import iter_spider_classes def _import_file(filepath: Union[str, PathLike]) -> ModuleType: diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 1fad8f328..05c76d1eb 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -8,7 +8,7 @@ from threading import Thread from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.shell import Shell -from scrapy.utils.spider import spidercls_for_request, DefaultSpider +from scrapy.utils.spider import DefaultSpider, spidercls_for_request from scrapy.utils.url import guess_scheme diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 3ed1f5dbc..88bd5bb33 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,16 +1,15 @@ -import re import os +import re import string from importlib.util import find_spec from pathlib import Path -from shutil import ignore_patterns, move, copy2, copystat +from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION import scrapy from scrapy.commands import ScrapyCommand -from scrapy.utils.template import render_templatefile, string_camelcase from scrapy.exceptions import UsageError - +from scrapy.utils.template import render_templatefile, string_camelcase TEMPLATES_TO_RENDER = ( ("scrapy.cfg",), diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index a81af7565..ebdfa10a8 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,4 +1,5 @@ import argparse + from scrapy.commands import fetch from scrapy.utils.response import open_in_browser diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index e41d83960..eac702cef 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,6 +1,6 @@ import json -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter, is_item from scrapy.contracts import Contract from scrapy.exceptions import ContractFail diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 3a7de8072..7e0b62bb0 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,16 +1,16 @@ import random -from time import time -from datetime import datetime from collections import deque +from datetime import datetime +from time import time from twisted.internet import defer, task +from scrapy import signals +from scrapy.core.downloader.handlers import DownloadHandlers +from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.resolver import dnscache from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached -from scrapy.resolver import dnscache -from scrapy import signals -from scrapy.core.downloader.middleware import DownloaderMiddlewareManager -from scrapy.core.downloader.handlers import DownloadHandlers class Slot: diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 1513638df..53ae78918 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -3,10 +3,10 @@ import warnings from OpenSSL import SSL from twisted.internet._sslverify import _setAcceptableProtocols from twisted.internet.ssl import ( - optionsForClientTLS, - CertificateOptions, - platformTrust, AcceptableCiphers, + CertificateOptions, + optionsForClientTLS, + platformTrust, ) from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS @@ -15,8 +15,8 @@ from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import ( DEFAULT_CIPHERS, - openssl_methods, ScrapyClientTLSOptions, + openssl_methods, ) from scrapy.utils.misc import create_instance, load_object diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index bb2141d28..39155efe9 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -10,7 +10,6 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values - logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 201c84ff8..8de5459e9 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -13,15 +13,15 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError from twisted.python.failure import Failure from twisted.web.client import ( + URI, Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, - URI, ) -from twisted.web.http import _DataLoss, PotentialDataLoss +from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer from zope.interface import implementer from scrapy import signals diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 25ac0307b..b2579362c 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -16,7 +16,6 @@ from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes - H2DownloadHandlerOrSubclass = TypeVar( "H2DownloadHandlerOrSubclass", bound="H2DownloadHandler" ) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 3410b4255..5a94e66a6 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -12,8 +12,8 @@ from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager -from scrapy.utils.defer import mustbe_deferred, deferred_from_coro from scrapy.utils.conf import build_component_list +from scrapy.utils.defer import deferred_from_coro, mustbe_deferred class DownloaderMiddlewareManager(MiddlewareManager): diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index d1c511db0..025575fe1 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -4,12 +4,12 @@ from OpenSSL import SSL from service_identity.exceptions import CertificateError from twisted.internet._sslverify import ( ClientTLSOptions, - verifyHostname, VerificationError, + verifyHostname, ) from twisted.internet.ssl import AcceptableCiphers -from scrapy.utils.ssl import x509name_to_string, get_temp_key_info +from scrapy.utils.ssl import get_temp_key_info, x509name_to_string logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 6421391d0..4558402b2 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,15 +1,15 @@ import re from time import time -from urllib.parse import urlparse, urlunparse, urldefrag -from twisted.web.http import HTTPClient +from urllib.parse import urldefrag, urlparse, urlunparse from twisted.internet import defer from twisted.internet.protocol import ClientFactory +from twisted.web.http import HTTPClient from scrapy.http import Headers +from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode -from scrapy.responsetypes import responsetypes def _parsed_url_args(parsed): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 19696415b..1efbdb271 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -15,19 +15,14 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper -from scrapy.exceptions import ( - CloseSpider, - DontCloseSpider, - ScrapyDeprecationWarning, -) -from scrapy.http import Response, Request +from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning +from scrapy.http import Request, Response from scrapy.settings import BaseSettings from scrapy.spiders import Spider -from scrapy.utils.log import logformatter_adapter, failure_to_exc_info +from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import create_instance, load_object from scrapy.utils.reactor import CallLaterOnce - logger = logging.getLogger(__name__) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 119443c80..1c43d241c 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -10,7 +10,7 @@ from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpoint from twisted.web.error import SchemeNotSupported from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory -from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory +from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol from scrapy.http.request import Request from scrapy.settings import Settings from scrapy.spiders import Spider diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 214deeed0..0bf69e513 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -9,9 +9,9 @@ from h2.config import H2Configuration from h2.connection import H2Connection from h2.errors import ErrorCodes from h2.events import ( - Event, ConnectionTerminated, DataReceived, + Event, ResponseReceived, SettingsAcknowledged, StreamEnded, @@ -23,7 +23,7 @@ from h2.exceptions import FrameTooLargeError, H2Error from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory -from twisted.internet.protocol import connectionDone, Factory, Protocol +from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin from twisted.python.failure import Failure @@ -35,7 +35,6 @@ from scrapy.http import Request from scrapy.settings import Settings from scrapy.spiders import Spider - logger = logging.getLogger(__name__) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 2b5c98e5f..87beb41e5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,13 +1,13 @@ import logging from enum import Enum from io import BytesIO +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple from urllib.parse import urlparse -from typing import Dict, List, Optional, Tuple, TYPE_CHECKING from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError from hpack import HeaderTuple -from twisted.internet.defer import Deferred, CancelledError +from twisted.internet.defer import CancelledError, Deferred from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 1e6fc69e1..3c46e3a5f 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -12,7 +12,6 @@ from scrapy.spiders import Spider from scrapy.utils.job import job_dir from scrapy.utils.misc import create_instance, load_object - logger = logging.getLogger(__name__) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7c2eefbe6..1a09f22f7 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -5,6 +5,7 @@ from __future__ import annotations import logging from collections import deque from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, AsyncIterable, @@ -13,7 +14,6 @@ from typing import ( Iterable, Optional, Set, - TYPE_CHECKING, Tuple, Union, ) @@ -22,7 +22,7 @@ from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure -from scrapy import signals, Spider +from scrapy import Spider, signals from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response @@ -34,12 +34,10 @@ from scrapy.utils.defer import ( parallel, parallel_async, ) - from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output - if TYPE_CHECKING: from scrapy.crawler import Crawler diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1aaed5865..ba9c37e38 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -28,14 +28,13 @@ from scrapy.middleware import MiddlewareManager from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( - mustbe_deferred, - deferred_from_coro, deferred_f_from_coro_f, + deferred_from_coro, maybe_deferred_to_future, + mustbe_deferred, ) from scrapy.utils.python import MutableAsyncChain, MutableChain - logger = logging.getLogger(__name__) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index f58cd73d3..397817d6f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -17,20 +17,20 @@ except ImportError: from zope.interface.verify import verifyClass -from scrapy import signals, Spider +from scrapy import Spider, signals from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader -from scrapy.settings import overridden_settings, Settings +from scrapy.settings import Settings, overridden_settings from scrapy.signalmanager import SignalManager from scrapy.utils.log import ( + LogCounterHandler, configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, log_reactor_info, log_scrapy_info, - LogCounterHandler, ) from scrapy.utils.misc import create_instance, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 86ff7b9fe..137ed5b18 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -1,12 +1,11 @@ -import re import logging +import re from w3lib import html from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse - logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 410015281..368ca60f7 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -14,7 +14,6 @@ from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes - warn( "scrapy.downloadermiddlewares.decompression is deprecated", ScrapyDeprecationWarning, diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index eb2754f1d..74c55f6e2 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -23,7 +23,6 @@ from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object - HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware") diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 489867918..f74d84b69 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,6 +1,6 @@ import base64 from urllib.parse import unquote, urlunparse -from urllib.request import getproxies, proxy_bypass, _parse_proxy +from urllib.request import _parse_proxy, getproxies, proxy_bypass from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 003c59fc4..f442a3012 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -3,10 +3,10 @@ from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh -from scrapy.exceptions import IgnoreRequest, NotConfigured logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 8a8f15f9a..11a30911c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,7 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ -from logging import getLogger, Logger +from logging import Logger, getLogger from typing import Optional, Union from twisted.internet import defer @@ -31,7 +31,6 @@ from scrapy.spiders import Spider from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message - retry_logger = getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 326c35290..89f8f7428 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,8 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. import logging from twisted.internet.defer import Deferred, maybeDeferred -from scrapy.exceptions import NotConfigured, IgnoreRequest + +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index fa0f8f846..d796e5cbb 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -10,8 +10,7 @@ from scrapy.settings import BaseSettings from scrapy.spiders import Spider from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.job import job_dir -from scrapy.utils.request import referer_str, RequestFingerprinter - +from scrapy.utils.request import RequestFingerprinter, referer_str BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter") diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 7d9a9b6ff..bb3e3c662 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -11,14 +11,13 @@ import warnings from collections.abc import Mapping from xml.sax.saxutils import XMLGenerator -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter, is_item from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder - __all__ = [ "BaseItemExporter", "PprintItemExporter", diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 8628b4a1e..1b6c7777f 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -4,11 +4,11 @@ Extensions for debugging Scrapy See documentation in docs/topics/extensions.rst """ -import sys -import signal import logging -import traceback +import signal +import sys import threading +import traceback from pdb import Pdb from scrapy.utils.engine import format_engine_status diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 823955aa3..cd26b5778 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -16,9 +16,9 @@ from urllib.parse import unquote, urlparse from twisted.internet import defer, threads from w3lib.url import file_uri_to_path -from zope.interface import implementer, Interface +from zope.interface import Interface, implementer -from scrapy import signals, Spider +from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available @@ -28,7 +28,6 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import get_func_args, without_none_values - logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index bbddaac40..2d120a6ed 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -7,7 +7,7 @@ from pathlib import Path from time import time from weakref import WeakKeyDictionary -from w3lib.http import headers_raw_to_dict, headers_dict_to_raw +from w3lib.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.http import Headers, Response from scrapy.http.request import Request diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 6295dcdb7..78874a6db 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -2,8 +2,8 @@ import logging from twisted.internet import task -from scrapy.exceptions import NotConfigured from scrapy import signals +from scrapy.exceptions import NotConfigured logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 2bba71972..221967bda 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -3,11 +3,11 @@ MemoryUsage extension See documentation in docs/topics/extensions.rst """ -import sys -import socket import logging -from pprint import pformat +import socket +import sys from importlib import import_module +from pprint import pformat from twisted.internet import task diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 8733ad22b..58610c25e 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -5,8 +5,8 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address """ from scrapy import signals -from scrapy.mail import MailSender from scrapy.exceptions import NotConfigured +from scrapy.mail import MailSender class StatsMailer: diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 271f22428..c92b7f5fe 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -4,11 +4,11 @@ Scrapy Telnet Console extension See documentation in docs/topics/telnetconsole.rst """ -import pprint -import logging -import traceback import binascii +import logging import os +import pprint +import traceback from twisted.internet import protocol @@ -21,12 +21,12 @@ except (ImportError, SyntaxError): _TWISTED_CONCH_TRACEBACK = traceback.format_exc() TWISTED_CONCH_AVAILABLE = False -from scrapy.exceptions import NotConfigured from scrapy import signals -from scrapy.utils.trackref import print_live_refs +from scrapy.exceptions import NotConfigured +from scrapy.utils.decorators import defers from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp -from scrapy.utils.decorators import defers +from scrapy.utils.trackref import print_live_refs logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 79e20de2a..396800775 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -1,7 +1,7 @@ import logging -from scrapy.exceptions import NotConfigured from scrapy import signals +from scrapy.exceptions import NotConfigured logger = logging.getLogger(__name__) diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index e6c58e1f1..ac3946302 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -6,13 +6,11 @@ Request and Response outside this module. """ from scrapy.http.headers import Headers - from scrapy.http.request import Request from scrapy.http.request.form import FormRequest -from scrapy.http.request.rpc import XmlRpcRequest from scrapy.http.request.json_request import JsonRequest - +from scrapy.http.request.rpc import XmlRpcRequest from scrapy.http.response import Response from scrapy.http.response.html import HtmlResponse -from scrapy.http.response.xml import XmlResponse from scrapy.http.response.text import TextResponse +from scrapy.http.response.xml import XmlResponse diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 94afedb08..a5329ad51 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,11 +1,11 @@ import re import time -from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy +from http.cookiejar import CookieJar as _CookieJar +from http.cookiejar import DefaultCookiePolicy from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode - # Defined in the http.cookiejar module, but undocumented: # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 IPV4_RE = re.compile(r"\.\d+$", re.ASCII) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index a5db30d6f..2540be01a 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,6 +1,7 @@ from collections.abc import Mapping from w3lib.http import headers_dict_to_raw + from scrapy.utils.datatypes import CaselessDict from scrapy.utils.python import to_unicode diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index e290f2143..0e925301e 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -17,7 +17,6 @@ from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax - RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 993219745..bdc6a3e39 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -6,7 +6,7 @@ See documentation in docs/topics/request-response.rst """ from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union -from urllib.parse import urljoin, urlencode, urlsplit, urlunsplit +from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement from parsel.selector import create_root_node @@ -14,10 +14,9 @@ from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request from scrapy.http.response.text import TextResponse -from scrapy.utils.python import to_bytes, is_listlike +from scrapy.utils.python import is_listlike, to_bytes from scrapy.utils.response import get_base_url - FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") FormdataType = Optional[Union[dict, List[Tuple[str, str]]]] diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index c0a6e86c1..43692923b 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -10,7 +10,6 @@ from typing import Optional from scrapy.http.request import Request from scrapy.utils.python import get_func_args - DUMPS_ARGS = get_func_args(xmlrpclib.dumps) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index f9df4e1b0..e45d95602 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -15,8 +15,8 @@ from w3lib.encoding import ( html_body_declared_encoding, html_to_unicode, http_content_type_encoding, - resolve_encoding, read_bom, + resolve_encoding, ) from w3lib.html import strip_html5_whitespace diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py index f27a669d6..b82c6d5c7 100644 --- a/scrapy/loader/processors.py +++ b/scrapy/loader/processors.py @@ -7,7 +7,6 @@ from itemloaders import processors from scrapy.utils.deprecate import create_deprecated_class - MapCompose = create_deprecated_class("MapCompose", processors.MapCompose) Compose = create_deprecated_class("Compose", processors.Compose) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index e0b93d812..560006c95 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -1,5 +1,5 @@ -import os import logging +import os from twisted.python.failure import Failure diff --git a/scrapy/mail.py b/scrapy/mail.py index fa1e55f1f..43115c53e 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -12,14 +12,13 @@ from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from twisted.python.versions import Version -from twisted.internet import defer, ssl from twisted import version as twisted_version +from twisted.internet import defer, ssl +from twisted.python.versions import Version from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes - logger = logging.getLogger(__name__) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 15f5b23e0..f82d722fa 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -8,8 +8,8 @@ from twisted.internet.defer import Deferred from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.settings import Settings +from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.defer import process_parallel, process_chain logger = logging.getLogger(__name__) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 01a9c41fe..d925fc984 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -32,7 +32,6 @@ from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str - logger = logging.getLogger(__name__) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index af23b4cc8..f6eb5b139 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -9,10 +9,10 @@ from twisted.python.failure import Failure from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude -from scrapy.utils.defer import mustbe_deferred, defer_result +from scrapy.utils.defer import defer_result, mustbe_deferred from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.misc import arg_to_iter from scrapy.utils.log import failure_to_exc_info +from scrapy.utils.misc import arg_to_iter logger = logging.getLogger(__name__) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 6f65184e5..62a9af477 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -3,7 +3,6 @@ import logging from scrapy.utils.misc import create_instance - logger = logging.getLogger(__name__) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f5d2b8e05..6cbe01cbf 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,8 +1,8 @@ from twisted.internet import defer from twisted.internet.base import ThreadedResolver from twisted.internet.interfaces import ( - IHostResolution, IHostnameResolver, + IHostResolution, IResolutionReceiver, IResolverSimple, ) @@ -10,7 +10,6 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache - # TODO: cache misses dnscache = LocalCache(10000) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 6b489bd8b..6af8915c2 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -2,9 +2,9 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ +from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data -from io import StringIO from scrapy.http import Response from scrapy.utils.misc import load_object diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0dadeef92..604b5e314 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -1,5 +1,5 @@ -import sys import logging +import sys from abc import ABCMeta, abstractmethod from scrapy.utils.python import to_unicode diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 6ba87428e..cff97104a 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -3,10 +3,10 @@ XPath selectors based on lxml """ from parsel import Selector as _ParselSelector -from scrapy.utils.trackref import object_ref -from scrapy.utils.python import to_bytes -from scrapy.http import HtmlResponse, XmlResponse +from scrapy.http import HtmlResponse, XmlResponse +from scrapy.utils.python import to_bytes +from scrapy.utils.trackref import object_ref __all__ = ["Selector", "SelectorList"] diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index c0d0741c5..fde8fdde4 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,12 +1,11 @@ -import json import copy +import json from collections.abc import MutableMapping from importlib import import_module from pprint import pformat from scrapy.settings import default_settings - SETTINGS_PRIORITIES = { "default": 0, "command": 10, diff --git a/scrapy/shell.py b/scrapy/shell.py index 084a27141..ae6e641fd 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -7,7 +7,7 @@ import os import signal from itemadapter import is_item -from twisted.internet import threads, defer +from twisted.internet import defer, threads from twisted.python import threadable from w3lib.url import any_to_uri @@ -20,11 +20,8 @@ from scrapy.utils.conf import get_config from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object +from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser -from scrapy.utils.reactor import ( - is_asyncio_reactor_installed, - set_asyncio_event_loop, -) class Shell: diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index f00447a55..d7e3bce91 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,4 +1,5 @@ from pydispatch import dispatcher + from scrapy.utils import signal as _signal diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index c57ec8d48..1a48926b3 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -3,8 +3,8 @@ Offsite Spider Middleware See documentation in docs/topics/spider-middleware.rst """ -import re import logging +import re import warnings from scrapy import signals diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a99b6315b..d86f55a40 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -15,7 +15,6 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url - LOCAL_SCHEMES = ( "about", "blob", diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 9a21379f9..f6d92e53a 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -6,8 +6,8 @@ See documentation in docs/topics/spider-middleware.rst import logging -from scrapy.http import Request from scrapy.exceptions import NotConfigured +from scrapy.http import Request logger = logging.getLogger(__name__) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index f8cac5458..3502f8b27 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -97,5 +97,5 @@ class Spider(object_ref): # Top-level imports from scrapy.spiders.crawl import CrawlSpider, Rule -from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider +from scrapy.spiders.feed import CSVFeedSpider, XMLFeedSpider from scrapy.spiders.sitemap import SitemapSpider diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d75b455ae..05c425948 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,7 +8,7 @@ See documentation in docs/topics/spiders.rst import copy from typing import AsyncIterable, Awaitable, Sequence -from scrapy.http import Request, Response, HtmlResponse +from scrapy.http import HtmlResponse, Request, Response from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index b3c5ff01e..5ec0504a8 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -4,11 +4,11 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ -from scrapy.spiders import Spider -from scrapy.utils.iterators import xmliter, csviter -from scrapy.utils.spider import iterate_spider_output -from scrapy.selector import Selector from scrapy.exceptions import NotConfigured, NotSupported +from scrapy.selector import Selector +from scrapy.spiders import Spider +from scrapy.utils.iterators import csviter, xmliter +from scrapy.utils.spider import iterate_spider_output class XMLFeedSpider(Spider): diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index a1734a3b1..c3cca9699 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,11 +1,10 @@ -import re import logging +import re -from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse -from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots +from scrapy.spiders import Spider from scrapy.utils.gz import gunzip, gzip_magic_number - +from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots logger = logging.getLogger(__name__) diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 4181c7a2f..dd3c32737 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -1,8 +1,8 @@ """ Scrapy extension for collecting scraping stats """ -import pprint import logging +import pprint logger = logging.getLogger(__name__) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 32bc2e38c..1089ba7b8 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -1,8 +1,8 @@ import random from urllib.parse import urlencode -from twisted.web.server import Site from twisted.web.resource import Resource +from twisted.web.server import Site class Root(Resource): diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 2f1569ab6..3ade1d105 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Union from scrapy.exceptions import ScrapyDeprecationWarning, UsageError - from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 3175e5fdc..a2243ae2e 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,7 +1,7 @@ import argparse import warnings -from shlex import split from http.cookies import SimpleCookie +from shlex import split from urllib.parse import urlparse from w3lib.http import basic_auth_header diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 21cd5e78f..ec130d685 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -26,7 +26,7 @@ from twisted.python import failure from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest -from scrapy.utils.reactor import is_asyncio_reactor_installed, _get_asyncio_event_loop +from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed def defer_fail(_failure: Failure) -> Deferred: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 4757fef0a..61a4347ea 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -1,8 +1,9 @@ """Some helpers for deprecation messages""" -import warnings import inspect +import warnings from typing import List, Tuple + from scrapy.exceptions import ScrapyDeprecationWarning diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index f6dceb87f..77c32b002 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -6,6 +6,7 @@ import ctypes import platform import sys from pprint import pformat as pformat_ + from packaging.version import Version as parse_version diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 9dbb4180f..6bf6e9195 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,5 +1,5 @@ import posixpath -from ftplib import error_perm, FTP +from ftplib import FTP, error_perm from posixpath import dirname diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index 540035ca9..d502e8910 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,12 +1,11 @@ """Helper functions for scrapy.http objects (Request, Response)""" from typing import Union -from urllib.parse import urlparse, ParseResult +from urllib.parse import ParseResult, urlparse from weakref import WeakKeyDictionary from scrapy.http import Request, Response - _urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = ( WeakKeyDictionary() ) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 7d52d35c9..170055d5e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -3,11 +3,10 @@ import logging import re from io import StringIO -from scrapy.http import TextResponse, Response +from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode - logger = logging.getLogger(__name__) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2560a421f..6ae27dc29 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -11,7 +11,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.versions import scrapy_components_versions - logger = logging.getLogger(__name__) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index dfd2f767c..f9f9c0d5b 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,23 +1,22 @@ """Helper functions which don't fit anywhere else""" import ast +import hashlib import inspect import os import re -import hashlib import warnings from collections import deque from contextlib import contextmanager +from functools import partial from importlib import import_module from pkgutil import iter_modules -from functools import partial from w3lib.html import replace_entities -from scrapy.utils.datatypes import LocalWeakReferencedCache -from scrapy.utils.python import flatten, to_unicode from scrapy.item import Item +from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.deprecate import ScrapyDeprecationWarning - +from scrapy.utils.python import flatten, to_unicode _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 18d856927..7646264a8 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,6 +1,5 @@ import signal - signal_names = {} for signame in dir(signal): if signame.startswith("SIG") and not signame.startswith("SIG_"): diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 4fbb6bcaf..ab1b8e3ee 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,13 +1,11 @@ import os import warnings - from importlib import import_module from pathlib import Path -from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env -from scrapy.settings import Settings from scrapy.exceptions import NotConfigured - +from scrapy.settings import Settings +from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env ENVVAR = "SCRAPY_SETTINGS_MODULE" DATADIR_CFG_SECTION = "datadir" diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index c818c8700..15705db83 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -5,7 +5,6 @@ import scrapy from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.request import request_from_dict as _from_dict - warnings.warn( ( "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 3e29a9c57..409ca2e52 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -19,7 +19,6 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode - _deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" _deprecated_fingerprint_cache = WeakKeyDictionary() diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index a91a49170..730d005e8 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -11,12 +11,11 @@ from weakref import WeakKeyDictionary from twisted.web import http from w3lib import html + import scrapy from scrapy.http.response import Response - - -from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.decorators import deprecated +from scrapy.utils.python import to_bytes, to_unicode _baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 3602043f3..358f41679 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -1,8 +1,8 @@ -import json import datetime import decimal +import json -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter, is_item from twisted.internet import defer from scrapy.http import Request, Response diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index b7c284174..b95786d35 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -2,9 +2,6 @@ import collections.abc import logging -from twisted.internet.defer import DeferredList, Deferred -from twisted.python.failure import Failure - from pydispatch.dispatcher import ( Anonymous, Any, @@ -13,12 +10,13 @@ from pydispatch.dispatcher import ( liveReceivers, ) from pydispatch.robustapply import robustApply +from twisted.internet.defer import Deferred, DeferredList +from twisted.python.failure import Failure from scrapy.exceptions import StopDownload from scrapy.utils.defer import maybeDeferred_coro from scrapy.utils.log import failure_to_exc_info - logger = logging.getLogger(__name__) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index f829bceb8..86449eeb2 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -5,7 +5,6 @@ from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter - logger = logging.getLogger(__name__) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 9f03621c1..f4b598ac7 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,5 +1,5 @@ -import OpenSSL.SSL import OpenSSL._util as pyOpenSSLutil +import OpenSSL.SSL from scrapy.utils.python import to_unicode diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 89bedfc69..1499aeb3d 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,8 +1,8 @@ """Helper functions for working with templates""" -from os import PathLike import re import string +from os import PathLike from pathlib import Path from typing import Union diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index d21065706..58576903a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -4,11 +4,11 @@ This module contains some assorted functions used in tests import asyncio import os +from importlib import import_module from pathlib import Path from posixpath import split from unittest import mock -from importlib import import_module from twisted.trial.unittest import SkipTest from scrapy.utils.boto import is_botocore_available @@ -109,7 +109,7 @@ def mock_google_cloud_storage(): """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. """ - from google.cloud.storage import Client, Bucket, Blob + from google.cloud.storage import Blob, Bucket, Client client_mock = mock.create_autospec(Client) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index fe5c8d74c..ecb2e31bf 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -1,5 +1,5 @@ -import sys import os +import sys from twisted.internet import defer, protocol diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index a47756c4b..de9ce992a 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -1,6 +1,6 @@ from urllib.parse import urljoin -from twisted.web import server, resource, static, util +from twisted.web import resource, server, static, util class SiteTest: diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 9aa775a1b..01b980c93 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -15,7 +15,6 @@ from time import time from typing import DefaultDict from weakref import WeakKeyDictionary - NoneType = type(None) live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index cd8a6a05a..0a27ccd6d 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -12,6 +12,7 @@ from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # move doesn't break old code from w3lib.url import * from w3lib.url import _safe_chars, _unquotepath # noqa: F401 + from scrapy.utils.python import to_unicode diff --git a/setup.py b/setup.py index f53334d4e..c6bcf2439 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ from pathlib import Path -from pkg_resources import parse_version -from setuptools import setup, find_packages, __version__ as setuptools_version +from pkg_resources import parse_version +from setuptools import __version__ as setuptools_version +from setuptools import find_packages, setup version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index 79dd77bb2..be9c83b95 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -1,9 +1,8 @@ import asyncio import sys -from uvloop import Loop - from twisted.internet import asyncioreactor +from uvloop import Loop if sys.version_info >= (3, 8) and sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 37626c081..a8f707841 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -1,5 +1,6 @@ from twisted.internet.main import installReactor from twisted.internet.selectreactor import SelectReactor + import scrapy from scrapy.crawler import CrawlerProcess diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 26db16dd6..23260ab0d 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,14 +1,16 @@ from urllib.parse import urlparse from twisted.internet import reactor -from twisted.names import cache, hosts as hostsModule, resolve +from twisted.names import cache +from twisted.names import hosts as hostsModule +from twisted.names import resolve from twisted.names.client import Resolver from twisted.python.runtime import platform -from scrapy import Spider, Request +from scrapy import Request, Spider from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging -from tests.mockserver import MockServer, MockDNSServer +from tests.mockserver import MockDNSServer, MockServer # https://stackoverflow.com/a/32784190 diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index b306437db..5cc65a903 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -14,8 +14,8 @@ from cryptography.x509 import ( DNSName, Name, NameAttribute, - random_serial_number, SubjectAlternativeName, + random_serial_number, ) from cryptography.x509.oid import NameOID diff --git a/tests/mockserver.py b/tests/mockserver.py index e07ae8797..7991da9dc 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -4,7 +4,7 @@ import random import sys from pathlib import Path from shutil import rmtree -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen from tempfile import mkdtemp from urllib.parse import urlencode @@ -14,7 +14,7 @@ from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory from twisted.web import resource, server -from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site +from twisted.web.server import NOT_DONE_YET, GzipEncoderFactory, Site from twisted.web.static import File from twisted.web.util import redirectTo diff --git a/tests/test_closespider.py b/tests/test_closespider.py index c497450f7..9b39187d5 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,8 +1,9 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase + from scrapy.utils.test import get_crawler -from tests.spiders import FollowAllSpider, ItemSpider, ErrorSpider from tests.mockserver import MockServer +from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider class TestCloseSpider(TestCase): diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 5aa35a6d9..15833cd19 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -6,7 +6,7 @@ import tempfile import unittest from io import StringIO from pathlib import Path -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen from scrapy.utils.test import get_testenv diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index d5088e817..5cb09b5c0 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -1,7 +1,7 @@ import sys import unittest from pathlib import Path -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen class CmdlineCrawlPipelineTest(unittest.TestCase): diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index bd44fa76e..124c968c2 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -1,8 +1,8 @@ -from twisted.trial import unittest from twisted.internet import defer +from twisted.trial import unittest -from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest +from scrapy.utils.testsite import SiteTest class FetchTest(ProcessTest, SiteTest, unittest.TestCase): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 1ee1bf5a7..b0fb978e9 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,13 +1,14 @@ -import os import argparse +import os from pathlib import Path from twisted.internet import defer + from scrapy.commands import parse from scrapy.settings import Settings -from scrapy.utils.testsite import SiteTest -from scrapy.utils.testproc import ProcessTest from scrapy.utils.python import to_unicode +from scrapy.utils.testproc import ProcessTest +from scrapy.utils.testsite import SiteTest from tests.test_commands import CommandTest diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7e99a7296..8ce82db86 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,12 +1,11 @@ from pathlib import Path -from twisted.trial import unittest from twisted.internet import defer +from twisted.trial import unittest -from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest - -from tests import tests_datadir, NON_EXISTING_RESOLVABLE +from scrapy.utils.testsite import SiteTest +from tests import NON_EXISTING_RESOLVABLE, tests_datadir class ShellTest(ProcessTest, SiteTest, unittest.TestCase): diff --git a/tests/test_command_version.py b/tests/test_command_version.py index f97a088a8..3bf6019b5 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -1,6 +1,7 @@ import sys -from twisted.trial import unittest + from twisted.internet import defer +from twisted.trial import unittest import scrapy from scrapy.utils.testproc import ProcessTest diff --git a/tests/test_commands.py b/tests/test_commands.py index 363e87aa7..5ff2dd482 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,6 +1,6 @@ +import argparse import inspect import json -import argparse import os import platform import re @@ -10,7 +10,7 @@ import tempfile from contextlib import contextmanager from itertools import chain from pathlib import Path -from shutil import rmtree, copytree +from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import mkdtemp from threading import Timer @@ -23,12 +23,11 @@ from twisted.python.versions import Version from twisted.trial import unittest import scrapy -from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv - from tests.test_crawler import ExceptionSpider, NoRequestsSpider diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 7b104f618..813927fc5 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -5,18 +5,18 @@ from twisted.python import failure from twisted.trial import unittest from scrapy import FormRequest -from scrapy.spidermiddlewares.httperror import HttpError -from scrapy.spiders import Spider -from scrapy.http import Request -from scrapy.item import Item, Field -from scrapy.utils.test import get_crawler -from scrapy.contracts import ContractsManager, Contract +from scrapy.contracts import Contract, ContractsManager from scrapy.contracts.default import ( - UrlContract, CallbackKeywordArgumentsContract, ReturnsContract, ScrapesContract, + UrlContract, ) +from scrapy.http import Request +from scrapy.item import Field, Item +from scrapy.spidermiddlewares.httperror import HttpError +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 4139f1b11..ca9084294 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,9 +1,9 @@ import json import logging +import unittest from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse -import unittest from pytest import mark from testfixtures import LogCapture diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c6b93599e..706bfbaa9 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -5,28 +5,25 @@ import sys import warnings from pathlib import Path -from pytest import raises, mark +from pkg_resources import parse_version +from pytest import mark, raises from twisted import version as twisted_version from twisted.internet import defer from twisted.python.versions import Version from twisted.trial import unittest - -from pkg_resources import parse_version from w3lib import __version__ as w3lib_version import scrapy -from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess +from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.extensions import telnet +from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler -from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object -from scrapy.utils.test import get_crawler -from scrapy.extensions.throttle import AutoThrottle -from scrapy.extensions import telnet -from scrapy.utils.test import get_testenv - +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler, get_testenv from tests.mockserver import MockServer diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 4f953439d..fd4176e2f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -5,7 +5,7 @@ import sys import tempfile from pathlib import Path from typing import Optional, Type -from unittest import mock, SkipTest +from unittest import SkipTest, mock from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal @@ -1041,7 +1041,8 @@ class BaseFTPTestCase(unittest.TestCase): ) def setUp(self): - from twisted.protocols.ftp import FTPRealm, FTPFactory + from twisted.protocols.ftp import FTPFactory, FTPRealm + from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dirs and test file @@ -1190,7 +1191,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase): req_meta = {} def setUp(self): - from twisted.protocols.ftp import FTPRealm, FTPFactory + from twisted.protocols.ftp import FTPFactory, FTPRealm + from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dir and test file diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index fd765089a..8090d50b9 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -15,10 +15,10 @@ from scrapy.utils.misc import create_instance from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory from tests.test_downloader_handlers import ( - Https11TestCase, - Https11CustomCiphers, Http11MockServerTestCase, Http11ProxyTestCase, + Https11CustomCiphers, + Https11TestCase, UriResource, ) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index d8e377519..2be32e37b 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -4,15 +4,15 @@ from unittest import mock from pytest import mark from twisted.internet import defer from twisted.internet.defer import Deferred -from twisted.trial.unittest import TestCase from twisted.python.failure import Failure +from twisted.trial.unittest import TestCase +from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider -from scrapy.exceptions import _InvalidOutput -from scrapy.core.downloader.middleware import DownloaderMiddlewareManager -from scrapy.utils.test import get_crawler, get_from_asyncio_queue from scrapy.utils.python import to_bytes +from scrapy.utils.test import get_crawler, get_from_asyncio_queue class ManagerTestCase(TestCase): diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 6be107f6f..043dc0a12 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -1,11 +1,10 @@ import unittest from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware +from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider -from scrapy.http import Request, HtmlResponse, Response from scrapy.utils.test import get_crawler - __doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"] diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 812c003da..4a81a638e 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,14 +1,14 @@ import logging from unittest import TestCase -from testfixtures import LogCapture import pytest +from testfixtures import LogCapture from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.exceptions import NotConfigured -from scrapy.http import Response, Request +from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index 16ae9ed75..412c20a78 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -1,6 +1,7 @@ from unittest import TestCase, main -from scrapy.http import Response, XmlResponse + from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware +from scrapy.http import Response, XmlResponse from scrapy.spiders import Spider from scrapy.utils.test import assert_samelines from tests import get_testdata diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 601e85799..27d6224b4 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -3,8 +3,8 @@ from unittest import TestCase from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.utils.test import get_crawler from scrapy.utils.python import to_bytes +from scrapy.utils.test import get_crawler class TestDefaultHeadersMiddleware(TestCase): diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 8d2b821b0..44458ade8 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -1,8 +1,8 @@ import unittest from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware -from scrapy.spiders import Spider from scrapy.http import Request +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 1320bded2..6b79234d0 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -3,9 +3,9 @@ import unittest import pytest from w3lib.http import basic_auth_header +from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request -from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.spiders import Spider diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index caa89b6bd..a355a9b5b 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -1,16 +1,16 @@ -import time -import tempfile -import shutil -import unittest import email.utils +import shutil +import tempfile +import time +import unittest from contextlib import contextmanager -from scrapy.http import Response, HtmlResponse, Request -from scrapy.spiders import Spider -from scrapy.settings import Settings -from scrapy.exceptions import IgnoreRequest -from scrapy.utils.test import get_crawler from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.http import HtmlResponse, Request, Response +from scrapy.settings import Settings +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler class _BaseTest(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index efae7c4e0..fac5588ff 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,18 +1,19 @@ from gzip import GzipFile from io import BytesIO from pathlib import Path -from unittest import TestCase, SkipTest +from unittest import SkipTest, TestCase from warnings import catch_warnings from w3lib.encoding import resolve_encoding -from scrapy.spiders import Spider -from scrapy.http import Response, Request, HtmlResponse + from scrapy.downloadermiddlewares.httpcompression import ( - HttpCompressionMiddleware, ACCEPTED_ENCODINGS, + HttpCompressionMiddleware, ) from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes +from scrapy.spiders import Spider from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index e2ff9ec2b..dc15b672c 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,12 +1,12 @@ import unittest from scrapy.downloadermiddlewares.redirect import ( - RedirectMiddleware, MetaRefreshMiddleware, + RedirectMiddleware, ) -from scrapy.spiders import Spider from scrapy.exceptions import IgnoreRequest -from scrapy.http import Request, Response, HtmlResponse +from scrapy.http import HtmlResponse, Request, Response +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index cadd647ad..02854c2a7 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -12,7 +12,7 @@ from twisted.internet.error import ( ) from twisted.web.client import ResponseFailed -from scrapy.downloadermiddlewares.retry import get_retry_request, RetryMiddleware +from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response from scrapy.spiders import Spider diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index ac08c6006..f98e0b12e 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,17 +1,16 @@ from unittest import mock -from twisted.internet import reactor, error +from twisted.internet import error, reactor from twisted.internet.defer import Deferred, DeferredList, maybeDeferred from twisted.python import failure from twisted.trial import unittest -from scrapy.downloadermiddlewares.robotstxt import ( - RobotsTxtMiddleware, - logger as mw_module_logger, -) + +from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware +from scrapy.downloadermiddlewares.robotstxt import logger as mw_module_logger from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.settings import Settings -from tests.test_robotstxt_interface import rerp_available, reppy_available +from tests.test_robotstxt_interface import reppy_available, rerp_available class RobotsTxtMiddlewareTest(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index 0702dd042..cad3dea5c 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -1,8 +1,8 @@ from unittest import TestCase -from scrapy.spiders import Spider -from scrapy.http import Request from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware +from scrapy.http import Request +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 4019012d1..aa0975555 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -1,14 +1,15 @@ import hashlib -import tempfile -import unittest import shutil import sys +import tempfile +import unittest from pathlib import Path + from testfixtures import LogCapture +from scrapy.core.scheduler import Scheduler from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request -from scrapy.core.scheduler import Scheduler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider diff --git a/tests/test_engine.py b/tests/test_engine.py index 7ddb420ba..02b59f448 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -14,13 +14,13 @@ import re import subprocess import sys from collections import defaultdict +from dataclasses import dataclass from pathlib import Path from threading import Timer from urllib.parse import urlparse -from dataclasses import dataclass -import pytest import attr +import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer, reactor @@ -31,12 +31,11 @@ from scrapy import signals from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning from scrapy.http import Request -from scrapy.item import Item, Field +from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler - from tests import get_testdata, tests_datadir diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index fb8dd4313..8dbb5b7ea 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -2,14 +2,13 @@ from testfixtures import LogCapture from twisted.internet import defer from scrapy.exceptions import StopDownload - from tests.test_engine import ( AttrsItemsSpider, + CrawlerRun, DataClassItemsSpider, DictItemsSpider, - TestSpider, - CrawlerRun, EngineTest, + TestSpider, ) diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 93437559d..0bad5ba55 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -2,14 +2,13 @@ from testfixtures import LogCapture from twisted.internet import defer from scrapy.exceptions import StopDownload - from tests.test_engine import ( AttrsItemsSpider, + CrawlerRun, DataClassItemsSpider, DictItemsSpider, - TestSpider, - CrawlerRun, EngineTest, + TestSpider, ) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 7689045b7..8e0999348 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1,31 +1,31 @@ -import re +import dataclasses import json import marshal import pickle +import re import tempfile import unittest -import dataclasses -from io import BytesIO from datetime import datetime +from io import BytesIO from warnings import catch_warnings, filterwarnings import lxml.etree from itemadapter import ItemAdapter -from scrapy.item import Item, Field -from scrapy.utils.python import to_unicode from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exporters import ( BaseItemExporter, - PprintItemExporter, - PickleItemExporter, CsvItemExporter, - XmlItemExporter, - JsonLinesItemExporter, JsonItemExporter, - PythonItemExporter, + JsonLinesItemExporter, MarshalItemExporter, + PickleItemExporter, + PprintItemExporter, + PythonItemExporter, + XmlItemExporter, ) +from scrapy.item import Field, Item +from scrapy.utils.python import to_unicode def custom_serializer(value): diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index e36c45d8e..9fd680e9f 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -1,7 +1,7 @@ -from twisted.trial import unittest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials from twisted.internet import defer +from twisted.trial import unittest from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 09a4aa823..96f97ca99 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -19,7 +19,7 @@ from pathlib import Path from string import ascii_letters, digits from typing import Union from unittest import mock -from urllib.parse import urljoin, quote +from urllib.parse import quote, urljoin from urllib.request import pathname2url import lxml.etree @@ -35,7 +35,6 @@ import scrapy from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( - _FeedSlot, BlockingFeedStorage, FeedExporter, FileFeedStorage, @@ -44,15 +43,11 @@ from scrapy.extensions.feedexport import ( IFeedStorage, S3FeedStorage, StdoutFeedStorage, + _FeedSlot, ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.test import ( - get_crawler, - mock_google_cloud_storage, - skip_if_no_boto, -) - +from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 88345d2bc..17a94f036 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -17,18 +17,19 @@ from twisted.internet.defer import ( ) from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.error import TimeoutError -from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate +from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS from twisted.python.failure import Failure from twisted.trial.unittest import TestCase -from twisted.web.client import ResponseFailed, URI -from twisted.web.http import H2_ENABLED, Request as TxRequest -from twisted.web.server import Site, NOT_DONE_YET +from twisted.web.client import URI, ResponseFailed +from twisted.web.http import H2_ENABLED +from twisted.web.http import Request as TxRequest +from twisted.web.server import NOT_DONE_YET, Site from twisted.web.static import File -from scrapy.http import Request, Response, JsonRequest +from scrapy.http import JsonRequest, Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider -from tests.mockserver import ssl_context_factory, LeafResource, Status +from tests.mockserver import LeafResource, Status, ssl_context_factory def generate_random_string(size): diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index ea42cadcd..9e43b72b0 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,5 +1,5 @@ -from urllib.parse import urlparse from unittest import TestCase +from urllib.parse import urlparse from scrapy.http import Request, Response from scrapy.http.cookies import WrappedRequest, WrappedResponse diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 566bb302d..7db1eb8c5 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -1,5 +1,5 @@ -import unittest import copy +import unittest from scrapy.http import Headers diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 0c10b27a0..d02f11f0e 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,18 +1,18 @@ -import unittest -import re import json -import xmlrpc.client +import re +import unittest import warnings +import xmlrpc.client from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes, urlparse from scrapy.http import ( - Request, FormRequest, - XmlRpcRequest, - JsonRequest, Headers, HtmlResponse, + JsonRequest, + Request, + XmlRpcRequest, ) from scrapy.utils.python import to_bytes, to_unicode diff --git a/tests/test_loader.py b/tests/test_loader.py index 9dd298864..5f4750ff3 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,12 +1,12 @@ -import unittest import dataclasses +import unittest import attr from itemadapter import ItemAdapter from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst from scrapy.http import HtmlResponse, Response -from scrapy.item import Item, Field +from scrapy.item import Field, Item from scrapy.loader import ItemLoader from scrapy.selector import Selector diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 8757db0ce..638af825b 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -16,7 +16,7 @@ from itemloaders.processors import ( TakeFirst, ) -from scrapy.item import Item, Field +from scrapy.item import Field, Item from scrapy.loader import ItemLoader from scrapy.loader.common import wrap_loader_context from scrapy.utils.deprecate import ScrapyDeprecationWarning diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 11cf6d81a..0971a5a38 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -6,11 +6,11 @@ from twisted.python.failure import Failure from twisted.trial.unittest import TestCase as TwistedTestCase from scrapy.exceptions import DropItem -from scrapy.utils.test import get_crawler from scrapy.http import Request, Response -from scrapy.item import Item, Field +from scrapy.item import Field, Item from scrapy.logformatter import LogFormatter from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import ItemSpider diff --git a/tests/test_mail.py b/tests/test_mail.py index 0ee0400cd..bc7298e9d 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -1,14 +1,15 @@ # coding=utf-8 import unittest -from io import BytesIO from email.charset import Charset +from io import BytesIO +from twisted import version as twisted_version +from twisted.internet import defer from twisted.internet._sslverify import ClientTLSOptions from twisted.internet.ssl import ClientContextFactory from twisted.python.versions import Version -from twisted.internet import defer -from twisted import version as twisted_version + from scrapy.mail import MailSender diff --git a/tests/test_middleware.py b/tests/test_middleware.py index a84cf4c28..00ff746ee 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -1,8 +1,8 @@ from twisted.trial import unittest -from scrapy.settings import Settings from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager +from scrapy.settings import Settings class M1: diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index b04da22be..8f5d87ebf 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -6,8 +6,8 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase from w3lib.url import add_or_replace_parameter -from scrapy.crawler import CrawlerRunner from scrapy import signals +from scrapy.crawler import CrawlerRunner from tests.mockserver import MockServer from tests.spiders import SimpleSpider diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 43942e53e..13de042a4 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,3 +1,4 @@ +import dataclasses import os import random import time @@ -8,7 +9,6 @@ from shutil import rmtree from tempfile import mkdtemp from unittest import mock from urllib.parse import urlparse -import dataclasses import attr from itemadapter import ItemAdapter diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 38a2d6c41..a5a495393 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -18,7 +18,6 @@ from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop from scrapy.settings import Settings from scrapy.utils.python import to_bytes - try: from PIL import Image except ImportError: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 87ab03395..e6d8ed2a2 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,25 +1,24 @@ -from typing import Optional import io +from typing import Optional from testfixtures import LogCapture -from twisted.trial import unittest -from twisted.python.failure import Failure from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.python.failure import Failure +from twisted.trial import unittest from scrapy import signals from scrapy.http import Request, Response -from scrapy.settings import Settings -from scrapy.spiders import Spider from scrapy.pipelines.files import FileException from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline +from scrapy.settings import Settings +from scrapy.spiders import Spider from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler - try: from PIL import Image # noqa: imported just to check for the import error except ImportError: diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 7b905d321..5ab288c1a 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -5,10 +5,9 @@ from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial import unittest -from scrapy import Spider, signals, Request -from scrapy.utils.defer import maybe_deferred_to_future, deferred_to_future +from scrapy import Request, Spider, signals +from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future from scrapy.utils.test import get_crawler, get_from_asyncio_queue - from tests.mockserver import MockServer diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 96a64c19d..1584014b8 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -4,11 +4,10 @@ import unittest import queuelib from scrapy.http.request import Request -from scrapy.pqueues import ScrapyPriorityQueue, DownloaderAwarePriorityQueue +from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue from scrapy.utils.test import get_crawler - from tests.test_scheduler import MockDownloader, MockEngine diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 5aeae7546..c05f4da91 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -3,15 +3,15 @@ import os import re import sys from pathlib import Path -from subprocess import Popen, PIPE +from subprocess import PIPE, Popen from urllib.parse import urlsplit, urlunsplit + from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.http import Request from scrapy.utils.test import get_crawler - from tests.mockserver import MockServer from tests.spiders import SimpleSpider, SingleRequestSpider diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 17c0309d1..d65d74206 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -1,16 +1,13 @@ +from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase -from testfixtures import LogCapture - from scrapy import Request, signals from scrapy.http.response import Response from scrapy.utils.test import get_crawler - from tests.mockserver import MockServer from tests.spiders import SingleRequestSpider - OVERRIDDEN_URL = "https://example.org" diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 454b68942..577522c6c 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -4,8 +4,8 @@ from twisted.trial.unittest import TestCase from scrapy.http import Request from scrapy.utils.test import get_crawler -from tests.spiders import MockServerSpider from tests.mockserver import MockServer +from tests.spiders import MockServerSpider class InjectArgumentsDownloaderMiddleware: diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index d9067610e..8665a9205 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -3,7 +3,7 @@ import unittest import warnings from contextlib import suppress -from scrapy import Spider, Request +from scrapy import Request, Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import FormRequest, JsonRequest from scrapy.utils.request import request_from_dict @@ -171,10 +171,8 @@ class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest): "scrapy.utils.reqser" ] # delete module to reset the deprecation warning - from scrapy.utils.reqser import ( - request_from_dict as _from_dict, - request_to_dict as _to_dict, - ) + from scrapy.utils.reqser import request_from_dict as _from_dict + from scrapy.utils.reqser import request_to_dict as _to_dict request_copy = _from_dict(_to_dict(request, spider), spider) self._assert_same_request(request, request_copy) diff --git a/tests/test_request_left.py b/tests/test_request_left.py index d08ed0f68..54155f7ef 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -1,5 +1,6 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase + from scrapy.signals import request_left_downloader from scrapy.spiders import Spider from scrapy.utils.test import get_crawler diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 57484a2a1..859960518 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -1,7 +1,7 @@ import unittest -from scrapy.responsetypes import responsetypes -from scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse, Headers +from scrapy.http import Headers, HtmlResponse, Response, TextResponse, XmlResponse +from scrapy.responsetypes import responsetypes class ResponseTypesTest(unittest.TestCase): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 67728321d..5acc412e5 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,21 +1,20 @@ +import collections import shutil import tempfile import unittest -import collections from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.crawler import Crawler from scrapy.core.downloader import Downloader from scrapy.core.scheduler import Scheduler +from scrapy.crawler import Crawler from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.test import get_crawler from tests.mockserver import MockServer - MockEngine = collections.namedtuple("MockEngine", ["downloader"]) MockSlot = collections.namedtuple("MockSlot", ["active"]) diff --git a/tests/test_selector.py b/tests/test_selector.py index ad72e068d..febae46ac 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -2,7 +2,7 @@ import weakref from twisted.trial import unittest -from scrapy.http import TextResponse, HtmlResponse, XmlResponse +from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.selector import Selector diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 9a01fd433..2a3b2d529 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -2,12 +2,13 @@ import unittest from unittest import mock from scrapy.settings import ( + SETTINGS_PRIORITIES, BaseSettings, Settings, SettingsAttribute, - SETTINGS_PRIORITIES, get_settings_priority, ) + from . import default_settings diff --git a/tests/test_signals.py b/tests/test_signals.py index 4c6ffabdc..0df104600 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -2,9 +2,8 @@ from pytest import mark from twisted.internet import defer from twisted.trial import unittest -from scrapy import signals, Request, Spider +from scrapy import Request, Spider, signals from scrapy.utils.test import get_crawler, get_from_asyncio_queue - from tests.mockserver import MockServer diff --git a/tests/test_spider.py b/tests/test_spider.py index 540091516..eb8a1f9f0 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -6,21 +6,21 @@ from unittest import mock from testfixtures import LogCapture from twisted.trial import unittest - from w3lib.url import safe_url_string + from scrapy import signals +from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse +from scrapy.linkextractors import LinkExtractor from scrapy.settings import Settings -from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse -from scrapy.spiders.init import InitSpider from scrapy.spiders import ( - CSVFeedSpider, CrawlSpider, + CSVFeedSpider, Rule, SitemapSpider, Spider, XMLFeedSpider, ) -from scrapy.linkextractors import LinkExtractor +from scrapy.spiders.init import InitSpider from scrapy.utils.test import get_crawler from tests import get_testdata diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 3745355a0..da656303d 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,21 +1,20 @@ -import sys import shutil +import sys +import tempfile import warnings from pathlib import Path -import tempfile -from zope.interface.verify import verifyObject from twisted.trial import unittest - +from zope.interface.verify import verifyObject # ugly hack to avoid cyclic imports of scrapy.spiders when running this test # alone import scrapy -from scrapy.interfaces import ISpiderLoader -from scrapy.spiderloader import SpiderLoader -from scrapy.settings import Settings -from scrapy.http import Request from scrapy.crawler import CrawlerRunner +from scrapy.http import Request +from scrapy.interfaces import ISpiderLoader +from scrapy.settings import Settings +from scrapy.spiderloader import SpiderLoader module_dir = Path(__file__).resolve().parent diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 760ee43df..974a0023d 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -4,16 +4,16 @@ from unittest import mock from testfixtures import LogCapture from twisted.internet import defer -from twisted.trial.unittest import TestCase from twisted.python.failure import Failure +from twisted.trial.unittest import TestCase -from scrapy.spiders import Spider -from scrapy.http import Request, Response +from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import _InvalidOutput +from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.test import get_crawler -from scrapy.core.spidermw import SpiderMiddlewareManager class SpiderMiddlewareTestCase(TestCase): diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index af17c13a0..e359d9cfc 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -1,7 +1,7 @@ from unittest import TestCase +from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware -from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index faa8e9091..1d5a887cc 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -2,14 +2,14 @@ import logging from unittest import TestCase from testfixtures import LogCapture -from twisted.trial.unittest import TestCase as TrialTestCase from twisted.internet import defer +from twisted.trial.unittest import TestCase as TrialTestCase -from scrapy.utils.test import get_crawler -from scrapy.http import Response, Request -from scrapy.spiders import Spider -from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError +from scrapy.http import Request, Response from scrapy.settings import Settings +from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import MockServerSpider diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 380bafe04..ea45b7698 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -1,10 +1,10 @@ +import warnings from unittest import TestCase from urllib.parse import urlparse -import warnings -from scrapy.http import Response, Request +from scrapy.http import Request, Response +from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, PortWarning, URLWarning from scrapy.spiders import Spider -from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning, PortWarning from scrapy.utils.test import get_crawler diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index dad39b6ee..1bc5ccb9a 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,17 +1,11 @@ -from urllib.parse import urlparse -from unittest import TestCase import warnings -from scrapy.http import Response, Request +from unittest import TestCase +from urllib.parse import urlparse -from scrapy.settings import Settings -from scrapy.spiders import Spider from scrapy.downloadermiddlewares.redirect import RedirectMiddleware +from scrapy.http import Request, Response +from scrapy.settings import Settings from scrapy.spidermiddlewares.referer import ( - DefaultReferrerPolicy, - NoReferrerPolicy, - NoReferrerWhenDowngradePolicy, - OriginPolicy, - OriginWhenCrossOriginPolicy, POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, POLICY_ORIGIN, @@ -21,6 +15,11 @@ from scrapy.spidermiddlewares.referer import ( POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, POLICY_UNSAFE_URL, + DefaultReferrerPolicy, + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + OriginPolicy, + OriginWhenCrossOriginPolicy, RefererMiddleware, ReferrerPolicy, SameOriginPolicy, @@ -28,6 +27,7 @@ from scrapy.spidermiddlewares.referer import ( StrictOriginWhenCrossOriginPolicy, UnsafeUrlPolicy, ) +from scrapy.spiders import Spider class TestRefererMiddleware(TestCase): diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 22716bdda..9111e4c82 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -2,11 +2,11 @@ from unittest import TestCase from testfixtures import LogCapture +from scrapy.http import Request, Response +from scrapy.settings import Settings from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware -from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from scrapy.settings import Settings class TestUrlLengthMiddleware(TestCase): diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 5c6dccf11..f645f4cce 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,11 +1,12 @@ +import shutil from datetime import datetime from pathlib import Path -import shutil + from twisted.trial import unittest +from scrapy.exceptions import NotConfigured from scrapy.extensions.spiderstate import SpiderState from scrapy.spiders import Spider -from scrapy.exceptions import NotConfigured from scrapy.utils.test import get_crawler diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 0e2441f90..1586f90c5 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -2,16 +2,17 @@ import pickle import sys from queuelib.tests import test_queue as t + +from scrapy.http import Request +from scrapy.item import Field, Item +from scrapy.loader import ItemLoader +from scrapy.selector import Selector from scrapy.squeues import ( _MarshalFifoSerializationDiskQueue, _MarshalLifoSerializationDiskQueue, _PickleFifoSerializationDiskQueue, _PickleLifoSerializationDiskQueue, ) -from scrapy.item import Item, Field -from scrapy.http import Request -from scrapy.loader import ItemLoader -from scrapy.selector import Selector class TestItem(Item): diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 5d9001bb0..b444c32b7 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -4,16 +4,16 @@ import unittest import queuelib -from scrapy.squeues import ( - PickleFifoDiskQueue, - PickleLifoDiskQueue, - MarshalFifoDiskQueue, - MarshalLifoDiskQueue, - FifoMemoryQueue, - LifoMemoryQueue, -) from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.squeues import ( + FifoMemoryQueue, + LifoMemoryQueue, + MarshalFifoDiskQueue, + MarshalLifoDiskQueue, + PickleFifoDiskQueue, + PickleLifoDiskQueue, +) from scrapy.utils.test import get_crawler """ diff --git a/tests/test_stats.py b/tests/test_stats.py index 2ee04429a..7a8adf638 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,10 +1,10 @@ -from datetime import datetime import unittest +from datetime import datetime from unittest import mock from scrapy.extensions.corestats import CoreStats from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector, DummyStatsCollector +from scrapy.statscollectors import DummyStatsCollector, StatsCollector from scrapy.utils.test import get_crawler diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index 9a4eeb04e..d272101b8 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -11,7 +11,7 @@ class ToplevelTestCase(TestCase): self.assertIs(type(scrapy.version_info), tuple) def test_request_shortcut(self): - from scrapy.http import Request, FormRequest + from scrapy.http import FormRequest, Request self.assertIs(scrapy.Request, Request) self.assertIs(scrapy.FormRequest, FormRequest) @@ -27,7 +27,7 @@ class ToplevelTestCase(TestCase): self.assertIs(scrapy.Selector, Selector) def test_item_shortcut(self): - from scrapy.item import Item, Field + from scrapy.item import Field, Item self.assertIs(scrapy.Item, Item) self.assertIs(scrapy.Field, Field) diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py index 3b6428686..c695968d7 100644 --- a/tests/test_urlparse_monkeypatches.py +++ b/tests/test_urlparse_monkeypatches.py @@ -1,5 +1,5 @@ -from urllib.parse import urlparse import unittest +from urllib.parse import urlparse class UrlparseTestCase(unittest.TestCase): diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index d09335651..746731a2e 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -3,7 +3,7 @@ from unittest import TestCase from pytest import mark -from scrapy.utils.reactor import is_asyncio_reactor_installed, install_reactor +from scrapy.utils.reactor import install_reactor, is_asyncio_reactor_installed @mark.usefixtures("reactor_pytest") diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 61a683318..78ed9a7c9 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,7 +1,7 @@ import unittest import warnings -from scrapy.exceptions import UsageError, ScrapyDeprecationWarning +from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( arglist_to_dict, diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 0c86c7e7a..b6a84ee91 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -11,7 +11,6 @@ from scrapy.utils.datatypes import ( ) from scrapy.utils.python import garbage_collect - __doctests__ = ["scrapy.utils.datatypes"] diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 8d7f33c9a..bb0ebc2a4 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,11 +1,11 @@ import random from pytest import mark -from twisted.trial import unittest -from twisted.internet import reactor, defer +from twisted.internet import defer, reactor from twisted.python.failure import Failure +from twisted.trial import unittest -from scrapy.utils.asyncgen import collect_asyncgen, as_async_generator +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import ( aiter_errback, deferred_f_from_coro_f, diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 214deceb2..2d9210410 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,7 +1,7 @@ import inspect import unittest -from unittest import mock import warnings +from unittest import mock from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import create_deprecated_class, update_classpath diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index da61f4b0b..d1bf64828 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -1,6 +1,5 @@ from io import StringIO - -from unittest import mock, TestCase +from unittest import TestCase, mock from scrapy.utils.display import pformat, pprint diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index a34664956..6b2a458bc 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -3,11 +3,10 @@ from pathlib import Path from w3lib.encoding import html_to_unicode -from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.http import Response +from scrapy.utils.gz import gunzip, gzip_magic_number from tests import tests_datadir - SAMPLEDIR = Path(tests_datadir, "compressed") diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 893582a32..ed077440c 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,8 +1,8 @@ from pytest import mark from twisted.trial import unittest -from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml -from scrapy.http import XmlResponse, TextResponse, Response +from scrapy.http import Response, TextResponse, XmlResponse +from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 438dd0cdc..eae744df5 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,18 +1,18 @@ -import sys import logging +import sys import unittest from testfixtures import LogCapture from twisted.python.failure import Failure +from scrapy.extensions import telnet from scrapy.utils.log import ( - failure_to_exc_info, - TopLevelFormatter, LogCounterHandler, StreamLogger, + TopLevelFormatter, + failure_to_exc_info, ) from scrapy.utils.test import get_crawler -from scrapy.extensions import telnet class FailureToExcInfoTest(unittest.TestCase): diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 38a61036c..69793ee75 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -1,10 +1,10 @@ -import sys import os +import sys import unittest from pathlib import Path from unittest import mock -from scrapy.item import Item, Field +from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, create_instance, @@ -14,7 +14,6 @@ from scrapy.utils.misc import ( walk_modules, ) - __doctests__ = ["scrapy.utils.misc"] diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index b08e5f475..90bd350a5 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -1,8 +1,8 @@ -import unittest -import os -import tempfile -import shutil import contextlib +import os +import shutil +import tempfile +import unittest import warnings from pathlib import Path diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 5caa5b8f2..fbf60ca71 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -5,20 +5,19 @@ import platform from twisted.trial import unittest from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen -from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback +from scrapy.utils.defer import aiter_errback, deferred_f_from_coro_f from scrapy.utils.python import ( - memoizemethod_noargs, + MutableAsyncChain, + MutableChain, binary_is_text, equal_attributes, get_func_args, + memoizemethod_noargs, to_bytes, to_unicode, without_none_values, - MutableChain, - MutableAsyncChain, ) - __doctests__ = ["scrapy.utils.python"] diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index d82aa19c6..80e15a60f 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -4,17 +4,16 @@ from pathlib import Path from urllib.parse import urlparse from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.http import Response, TextResponse, HtmlResponse +from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import ( - response_httprepr, - open_in_browser, - get_meta_refresh, get_base_url, + get_meta_refresh, + open_in_browser, + response_httprepr, response_status_message, ) - __doctests__ = ["scrapy.utils.response"] diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index 20aebc2d7..5cdcc7f7c 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -1,7 +1,7 @@ +import dataclasses import datetime import json import unittest -import dataclasses from decimal import Decimal import attr diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 6fb7b8b82..460ae40c3 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -3,7 +3,7 @@ import unittest from scrapy import Spider from scrapy.http import Request from scrapy.item import Item -from scrapy.utils.spider import iterate_spider_output, iter_spider_classes +from scrapy.utils.spider import iter_spider_classes, iterate_spider_output class MySpider1(Spider): diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 45e23f793..c79a1fdce 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,9 +1,9 @@ +import unittest from pathlib import Path from shutil import rmtree from tempfile import mkdtemp -import unittest -from scrapy.utils.template import render_templatefile +from scrapy.utils.template import render_templatefile __doctests__ = ["scrapy.utils.template"] diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 9133663d9..65522f0fd 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -4,13 +4,13 @@ from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.spiders import Spider from scrapy.utils.misc import arg_to_iter from scrapy.utils.url import ( + _is_filesystem_path, add_http_if_no_scheme, guess_scheme, - _is_filesystem_path, strip_url, + url_has_any_extension, url_is_from_any_domain, url_is_from_spider, - url_has_any_extension, ) __doctests__ = ["scrapy.utils.url"] diff --git a/tests/test_webclient.py b/tests/test_webclient.py index aadfe0f40..0042fe8f0 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -6,9 +6,9 @@ import shutil from pathlib import Path import OpenSSL.SSL +from twisted.internet import defer, reactor from twisted.trial import unittest -from twisted.web import server, static, util, resource -from twisted.internet import reactor, defer +from twisted.web import resource, server, static, util try: from twisted.internet.testing import StringTransport @@ -16,12 +16,13 @@ except ImportError: # deprecated in Twisted 19.7.0 # (remove once we bump our requirement past that version) from twisted.test.proto_helpers import StringTransport -from twisted.protocols.policies import WrappingFactory + from twisted.internet.defer import inlineCallbacks +from twisted.protocols.policies import WrappingFactory from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory -from scrapy.http import Request, Headers +from scrapy.http import Headers, Request from scrapy.settings import Settings from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes, to_unicode From 7f01e1f0ce106bf810501d53217730b754d35edf Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 25 Jan 2023 14:43:25 -0600 Subject: [PATCH 0809/2083] added isort to pre-commit-config --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b93a73453..d67249371 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,3 +17,7 @@ repos: rev: 22.12.0 hooks: - id: black +- repo: https://github.com/pycqa/isort + rev: 5.11.3 + hooks: + - id: isort From 0a21a9457b7aeafef3b9ee1c0206546d6c8fb294 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 26 Jan 2023 00:50:29 -0800 Subject: [PATCH 0810/2083] fixed mypy typing error --- scrapy/pipelines/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index f9dfa53e3..94b0b1b70 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -37,7 +37,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, PathLike]): +def _to_string(path: Union[str, PathLike]) -> str: return str(path) # convert a Path object to string From 3054235dc09b1667c5c897f976eafa18845283e1 Mon Sep 17 00:00:00 2001 From: Cj Malone Date: Thu, 26 Jan 2023 16:10:57 +0000 Subject: [PATCH 0811/2083] Don't check robotstxt for local files --- scrapy/downloadermiddlewares/robotstxt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 326c35290..8e9beeeef 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -38,6 +38,8 @@ class RobotsTxtMiddleware: def process_request(self, request, spider): if request.meta.get("dont_obey_robotstxt"): return + if request.url.startswith("data:") or request.url.startswith("file:"): + return d = maybeDeferred(self.robot_parser, request, spider) d.addCallback(self.process_request_2, request, spider) return d From 33b85a9e2a379b355398e2daf416130bb840167d Mon Sep 17 00:00:00 2001 From: Cj Malone Date: Thu, 26 Jan 2023 19:51:20 +0000 Subject: [PATCH 0812/2083] Test local files aren't processed --- tests/test_downloadermiddleware_robotstxt.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index ac08c6006..fd27e637d 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -214,6 +214,19 @@ Disallow: /some/randome/page.html middleware.process_request_2(rp, Request("http://site.local/allowed"), None) rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot") + def test_robotstxt_local_file(self): + middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) + assert not middleware.process_request( + Request("data:text/plain,Hello World data"), None + ) + assert not middleware.process_request( + Request("file:///tests/sample_data/test_site/nothinghere.html"), None + ) + assert isinstance( + middleware.process_request(Request("http://site.local/allowed"), None), + Deferred, + ) + def assertNotIgnored(self, request, middleware): spider = None # not actually used dfd = maybeDeferred(middleware.process_request, request, spider) From 94161d101cd47d2dac4be7e8325a24f0ddea86cf Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 26 Jan 2023 16:41:06 -0800 Subject: [PATCH 0813/2083] update --- scrapy/pipelines/files.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 6a32c8b47..afa237b3d 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -36,7 +36,7 @@ from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) -def _to_string(path: Union[str, os.PathLike[str]]): +def _to_string(path: Union[str, os.PathLike]) -> str: return str(path) # convert a Path object to string @@ -45,10 +45,10 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir: Union[str, os.PathLike[str]]): + def __init__(self, basedir: Union[str, os.PathLike]): basedir = _to_string(basedir) - if '://' in basedir: - basedir = basedir.split('://', 1)[1] + if "://" in basedir: + basedir = basedir.split("://", 1)[1] self.basedir = basedir self._mkdir(Path(self.basedir)) self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set) @@ -70,7 +70,7 @@ class FSFilesStore: return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path: Union[str, os.PathLike[str]]) -> Path: + def _get_filesystem_path(self, path: Union[str, os.PathLike]) -> Path: path_comps = _to_string(path).split('/') return Path(self.basedir, *path_comps) From a1e2fbafdcaae9b70cf7c4216c5cb80d1df5268e Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 26 Jan 2023 16:46:08 -0800 Subject: [PATCH 0814/2083] applied black to tests --- tests/test_pipeline_files.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 76c16e57f..1e00e6d1d 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -484,19 +484,18 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value) def test_file_pipeline_using_pathlike_objects(self): - class CustomFilesPipelineWithPathLikeDir(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return Path('subdir') / Path(request.url).name + return Path("subdir") / Path(request.url).name pipeline = CustomFilesPipelineWithPathLikeDir.from_settings( - Settings({'FILES_STORE': Path('./Temp')}) + Settings({"FILES_STORE": Path("./Temp")}) ) request = Request("http://example.com/image01.jpg") - self.assertEqual(pipeline.file_path(request), Path('subdir/image01.jpg')) + self.assertEqual(pipeline.file_path(request), Path("subdir/image01.jpg")) def test_files_store_constructor_with_pathlike_object(self): - path = Path('./FileDir') + path = Path("./FileDir") fs_store = FSFilesStore(path) self.assertEqual(fs_store.basedir, str(path)) From f03b47db05e623189cf7719e647d18e8457494f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 27 Jan 2023 17:35:32 +0100 Subject: [PATCH 0815/2083] Make NO_CALLBACK a callable --- scrapy/http/request/__init__.py | 52 +++++++++++++++------------------ setup.py | 1 - tests/test_http_request.py | 8 +++-- 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 068a4baa3..de13cf264 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -5,10 +5,8 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ import inspect -from enum import Enum from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union -from typing_extensions import Final from w3lib.url import safe_url_string import scrapy @@ -23,21 +21,22 @@ from scrapy.utils.url import escape_ajax RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") -# https://github.com/python/typing/issues/689#issuecomment-561425237 -class NoCallbackType(Enum): - NO_CALLBACK = 0 +def NO_CALLBACK(*args, **kwargs): + """When assigned to the ``callback`` parameter of + :class:`~scrapy.http.Request`, it indicates that the request is not meant + to have a spider callback at all. - -#: When assigned to the ``callback`` parameter of -#: :class:`~scrapy.http.Request`, it indicates that the request is not meant to -#: have a spider callback at all. -#: -#: This value should be used by :ref:`components ` -#: that create and handle their own requests, e.g. through -#: :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download -#: middlewares handling such requests can treat them differently from requests -#: intended for the :meth:`~scrapy.Spider.parse` callback. -NO_CALLBACK: Final = NoCallbackType.NO_CALLBACK + This value should be used by :ref:`components ` that + create and handle their own requests, e.g. through + :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download + middlewares handling such requests can treat them differently from requests + intended for the :meth:`~scrapy.Spider.parse` callback. + """ + raise RuntimeError( + "The NO_CALLBACK callback has been called. This is a special callback " + "value intended for requests whose callback is never meant to be " + "called." + ) class Request(object_ref): @@ -67,8 +66,6 @@ class Request(object_ref): Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and :func:`~scrapy.utils.request.request_from_dict`. """ - callback: Union[None, NoCallbackType, Callable] - errback: Optional[Callable] def __init__( self, @@ -94,8 +91,14 @@ class Request(object_ref): raise TypeError(f"Request priority not an integer: {priority!r}") self.priority = priority - self._set_xback("callback", callback) - self._set_xback("errback", errback) + if not (callable(callback) or callback is None): + raise TypeError( + f"callback must be a callable, got {type(callback).__name__}" + ) + if not (callable(errback) or errback is None): + raise TypeError(f"errback must be a callable, got {type(errback).__name__}") + self.callback = callback + self.errback = errback self.cookies = cookies or {} self.headers = Headers(headers or {}, encoding=encoding) @@ -105,15 +108,6 @@ class Request(object_ref): self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None self.flags = [] if flags is None else list(flags) - def _set_xback(self, name: str, value: Optional[Callable]) -> None: - if not ( - callable(value) - or value is None - or (name == "callback" and value is NO_CALLBACK) - ): - raise TypeError(f"{name} must be a callable, got {type(value).__name__}") - setattr(self, name, value) - @property def cb_kwargs(self) -> dict: if self._cb_kwargs is None: diff --git a/setup.py b/setup.py index 9150dac0b..f53334d4e 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ install_requires = [ "packaging", "tldextract", "lxml>=4.3.0", - "typing-extensions>=3.10.0.0", ] extras_require = {} cpython_dependencies = [ diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 233a5f0b2..e800f427f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -314,8 +314,10 @@ class RequestTest(unittest.TestCase): r5 = self.request_class( url="http://example.com", callback=NO_CALLBACK, + errback=NO_CALLBACK, ) self.assertIs(r5.callback, NO_CALLBACK) + self.assertIs(r5.errback, NO_CALLBACK) def test_callback_and_errback_type(self): with self.assertRaises(TypeError): @@ -328,8 +330,10 @@ class RequestTest(unittest.TestCase): callback="a_function", errback="a_function", ) - with self.assertRaises(TypeError): - self.request_class("http://example.com", errback=NO_CALLBACK) + + def test_no_callback(self): + with self.assertRaises(RuntimeError): + NO_CALLBACK() def test_from_curl(self): # Note: more curated tests regarding curl conversion are in From c0efb271a23218656b5f629fde6d23c115a0c3de Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 27 Jan 2023 23:19:44 +0400 Subject: [PATCH 0816/2083] Ignore typing for twisted.internet.reactor globally. --- scrapy/core/downloader/handlers/http2.py | 8 ++++---- scrapy/utils/benchserver.py | 6 +++--- scrapy/utils/defer.py | 4 ++-- scrapy/utils/testsite.py | 4 ++-- setup.cfg | 3 +++ tests/CrawlerRunner/ip_address.py | 6 +++--- tests/mockserver.py | 10 +++++----- tests/test_engine.py | 2 +- 8 files changed, 23 insertions(+), 20 deletions(-) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 20cd50c5a..25ac0307b 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -28,7 +28,7 @@ class H2DownloadHandler: from twisted.internet import reactor - self._pool = H2ConnectionPool(reactor, settings) # type: ignore[arg-type] + self._pool = H2ConnectionPool(reactor, settings) self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod @@ -82,7 +82,7 @@ class ScrapyH2Agent: "Tunneling via CONNECT method using HTTP/2.0 is not yet supported" ) return self._ProxyAgent( - reactor=reactor, # type: ignore[arg-type] + reactor=reactor, context_factory=self._context_factory, proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")), connect_timeout=timeout, @@ -91,7 +91,7 @@ class ScrapyH2Agent: ) return self._Agent( - reactor=reactor, # type: ignore[arg-type] + reactor=reactor, context_factory=self._context_factory, connect_timeout=timeout, bind_address=bind_address, @@ -108,7 +108,7 @@ class ScrapyH2Agent: d = agent.request(request, spider) d.addCallback(self._cb_latency, request, start_time) - timeout_cl = reactor.callLater(timeout, d.cancel) # type: ignore[attr-defined] + timeout_cl = reactor.callLater(timeout, d.cancel) d.addBoth(self._cb_timeout, request, timeout, timeout_cl) return d diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 750d3c093..32bc2e38c 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -37,11 +37,11 @@ if __name__ == "__main__": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(8998, Site(root)) # type: ignore[attr-defined] + httpPort = reactor.listenTCP(8998, Site(root)) def _print_listening(): httpHost = httpPort.getHost() print(f"Bench server at http://{httpHost.host}:{httpHost.port}") - reactor.callWhenRunning(_print_listening) # type: ignore[attr-defined] - reactor.run() # type: ignore[attr-defined] + reactor.callWhenRunning(_print_listening) + reactor.run() diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 8fee5a7ed..7f2211877 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -40,7 +40,7 @@ def defer_fail(_failure: Failure) -> Deferred: from twisted.internet import reactor d: Deferred = Deferred() - reactor.callLater(0.1, d.errback, _failure) # type: ignore[attr-defined] + reactor.callLater(0.1, d.errback, _failure) return d @@ -54,7 +54,7 @@ def defer_succeed(result) -> Deferred: from twisted.internet import reactor d: Deferred = Deferred() - reactor.callLater(0.1, d.callback, result) # type: ignore[attr-defined] + reactor.callLater(0.1, d.callback, result) return d diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 119be1dfb..a47756c4b 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -50,6 +50,6 @@ def test_site(): if __name__ == "__main__": from twisted.internet import reactor - port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") # type: ignore[attr-defined] + port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") print(f"http://localhost:{port.getHost().port}/") - reactor.run() # type: ignore[attr-defined] + reactor.run() diff --git a/setup.cfg b/setup.cfg index af9c87945..db79c5821 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,6 +11,9 @@ ignore_missing_imports = true [mypy-twisted.internet.interfaces] follow_imports = skip +[mypy-twisted.internet.reactor] +follow_imports = skip + # FIXME: remove the following sections once the issues are solved [mypy-scrapy.downloadermiddlewares.httpproxy] diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index b9a4485a9..26db16dd6 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -41,10 +41,10 @@ if __name__ == "__main__": url = f"http://not.a.real.domain:{port}/echo" servers = [(mock_dns_server.host, mock_dns_server.port)] - reactor.installResolver(createResolver(servers=servers)) # type: ignore[attr-defined] + reactor.installResolver(createResolver(servers=servers)) configure_logging() runner = CrawlerRunner() d = runner.crawl(LocalhostSpider, url=url) - d.addBoth(lambda _: reactor.stop()) # type: ignore[attr-defined] - reactor.run() # type: ignore[attr-defined] + d.addBoth(lambda _: reactor.stop()) + reactor.run() diff --git a/tests/mockserver.py b/tests/mockserver.py index 185897373..eb1c03db7 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -375,9 +375,9 @@ if __name__ == "__main__": if args.type == "http": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(0, factory) # type: ignore[attr-defined] + httpPort = reactor.listenTCP(0, factory) contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(0, factory, contextFactory) # type: ignore[attr-defined] + httpsPort = reactor.listenSSL(0, factory, contextFactory) def print_listening(): httpHost = httpPort.getHost() @@ -391,11 +391,11 @@ if __name__ == "__main__": clients = [MockDNSResolver()] factory = DNSServerFactory(clients=clients) protocol = dns.DNSDatagramProtocol(controller=factory) - listener = reactor.listenUDP(0, protocol) # type: ignore[attr-defined] + listener = reactor.listenUDP(0, protocol) def print_listening(): host = listener.getHost() print(f"{host.host}:{host.port}") - reactor.callWhenRunning(print_listening) # type: ignore[attr-defined] - reactor.run() # type: ignore[attr-defined] + reactor.callWhenRunning(print_listening) + reactor.run() diff --git a/tests/test_engine.py b/tests/test_engine.py index 5fb87424b..7ddb420ba 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -557,4 +557,4 @@ class EngineTest(unittest.TestCase): if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) - reactor.run() # type: ignore[attr-defined] + reactor.run() From 5fa0f64db5d4b7bf3f3a7c83578ce15ec534d3d3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 27 Jan 2023 23:29:23 +0400 Subject: [PATCH 0817/2083] Ru typing on 3.8 as types-lxml isn't available for 3.7. --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 740092dab..bd26e8bb2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -17,7 +17,7 @@ jobs: - python-version: "3.11" env: TOXENV: pylint - - python-version: 3.7 + - python-version: 3.8 env: TOXENV: typing - python-version: "3.11" # Keep in sync with .readthedocs.yml From e9ee9454f960d7e14b5cca4527bf2b185235bd89 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 27 Jan 2023 14:59:08 -0600 Subject: [PATCH 0818/2083] fix .isort.cfg --- .isort.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/.isort.cfg b/.isort.cfg index a29184f0a..f238bf7ea 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,2 @@ [settings] profile = black -multi_line_output = 3 From ef794251f6bd238986c4e90763521a0b3ac02dc6 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 27 Jan 2023 15:00:19 -0600 Subject: [PATCH 0819/2083] fix scrapy/__init__.py --- scrapy/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 44df3d54b..a757a9290 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -8,11 +8,10 @@ import warnings from twisted import version as _txv +# Declare top-level shortcuts from scrapy.http import FormRequest, Request from scrapy.item import Field, Item from scrapy.selector import Selector - -# Declare top-level shortcuts from scrapy.spiders import Spider __all__ = [ From 4bd48d26138176c83086180172e1cff245d49648 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 27 Jan 2023 15:06:54 -0600 Subject: [PATCH 0820/2083] added pre-commit action --- .github/workflows/checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 740092dab..6b2f4ef10 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -32,6 +32,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: pre-commit/action@v3.0.0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 From 80453d53b19bbe288dc1e5b721f2f29acb89706a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 28 Jan 2023 01:29:09 +0400 Subject: [PATCH 0821/2083] Type DeferredLists as Deferreds. --- scrapy/utils/defer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 7f2211877..79b96856c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -85,7 +85,7 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: def parallel( iterable: Iterable, count: int, callable: Callable, *args, **named -) -> DeferredList: +) -> Deferred: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -202,11 +202,11 @@ class _AsyncCooperatorAdapter(Iterator): def parallel_async( async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named -) -> DeferredList: +) -> Deferred: """Like parallel but for async iterators""" coop = Cooperator() work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) - dl = DeferredList([coop.coiterate(work) for _ in range(count)]) + dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)]) return dl @@ -245,7 +245,7 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) + d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d From 5dcf8b9015d412919aca99cad0371298f9591b94 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Sun, 29 Jan 2023 00:22:56 -0600 Subject: [PATCH 0822/2083] fix isort version --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d67249371..0534bb142 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,6 @@ repos: hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.11.3 + rev: 5.12.0 hooks: - id: isort From 17354a61b11eb792adbe77fcdfa6b95a5993cc30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 30 Jan 2023 10:04:27 +0100 Subject: [PATCH 0823/2083] Avoid duplicities in CI; remove pylint from pre-commit --- .github/workflows/checks.yml | 16 ++++++---------- .pre-commit-config.yaml | 7 +------ 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6b2f4ef10..aa79cbc0d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -8,12 +8,6 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.11" - env: - TOXENV: security - - python-version: "3.11" - env: - TOXENV: flake8 - python-version: "3.11" env: TOXENV: pylint @@ -26,13 +20,9 @@ jobs: - python-version: "3.11" env: TOXENV: twinecheck - - python-version: "3.11" - env: - TOXENV: black steps: - uses: actions/checkout@v3 - - uses: pre-commit/action@v3.0.0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -44,3 +34,9 @@ jobs: run: | pip install -U tox tox + + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: pre-commit/action@v3.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0534bb142..f5fc1285f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,6 @@ repos: rev: 6.0.0 hooks: - id: flake8 -- repo: https://github.com/PyCQA/pylint - rev: v2.15.6 - hooks: - - id: pylint - args: [conftest.py, docs, extras, scrapy, setup.py, tests] - repo: https://github.com/psf/black.git rev: 22.12.0 hooks: @@ -20,4 +15,4 @@ repos: - repo: https://github.com/pycqa/isort rev: 5.12.0 hooks: - - id: isort + - id: isort From e1699479f6e48ce87dea1e6ed5661fea9ca7b1aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 30 Jan 2023 11:54:31 +0100 Subject: [PATCH 0824/2083] =?UTF-8?q?Fix=20typo:=20download=20middleware?= =?UTF-8?q?=20=E2=86=92=20downloader=20middleware?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrey Rakhmatullin --- scrapy/http/request/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 75dd2a74f..7afb28db5 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -27,7 +27,7 @@ def NO_CALLBACK(*args, **kwargs): This value should be used by :ref:`components ` that create and handle their own requests, e.g. through - :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download + :meth:`scrapy.core.engine.ExecutionEngine.download`, so that downloader middlewares handling such requests can treat them differently from requests intended for the :meth:`~scrapy.Spider.parse` callback. """ From 389fd99e79374bad73faf98424c97ac804eb1a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 30 Jan 2023 12:37:34 +0100 Subject: [PATCH 0825/2083] get_media_requests: support and encourage callback=NO_CALLBACK --- scrapy/pipelines/files.py | 3 ++- scrapy/pipelines/images.py | 3 ++- scrapy/pipelines/media.py | 9 ++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 01a9c41fe..91fc172b2 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -22,6 +22,7 @@ from twisted.internet import defer, threads from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request +from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available @@ -517,7 +518,7 @@ class FilesPipeline(MediaPipeline): # Overridable Interface def get_media_requests(self, item, info): urls = ItemAdapter(item).get(self.files_urls_field, []) - return [Request(u) for u in urls] + return [Request(u, callback=NO_CALLBACK) for u in urls] def file_downloaded(self, response, request, info, *, item=None): path = self.file_path(request, response=response, info=info, item=item) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 0cfa5665a..9d18144ee 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -13,6 +13,7 @@ from itemadapter import ItemAdapter from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request +from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline @@ -214,7 +215,7 @@ class ImagesPipeline(FilesPipeline): def get_media_requests(self, item, info): urls = ItemAdapter(item).get(self.images_urls_field, []) - return [Request(u) for u in urls] + return [Request(u, callback=NO_CALLBACK) for u in urls] def item_completed(self, results, item, info): with suppress(KeyError): diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 1e921f0b5..679035c5d 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -18,6 +18,10 @@ from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) +def _DUMMY_CALLBACK(response): + return response + + class MediaPipeline: LOG_FAILED_RESULTS = True @@ -91,7 +95,10 @@ class MediaPipeline: def _process_request(self, request, info, item): fp = self._fingerprinter.fingerprint(request) - cb = request.callback or (lambda _: _) + if not request.callback or request.callback is NO_CALLBACK: + cb = _DUMMY_CALLBACK + else: + cb = request.callback eb = request.errback request.callback = NO_CALLBACK request.errback = None From 78eaf0671bd50642f68d5b07bec3175298120a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 30 Jan 2023 14:33:11 +0100 Subject: [PATCH 0826/2083] Remove typing-extensions from tox.ini --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index f2268b0f6..453c28c4c 100644 --- a/tox.ini +++ b/tox.ini @@ -94,7 +94,6 @@ deps = w3lib==1.17.0 zope.interface==5.1.0 lxml==4.3.0 - typing-extensions==3.10.0.0 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies From 349fc33cc70aee38d30f7715811829b70ef77492 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 31 Jan 2023 14:28:08 -0800 Subject: [PATCH 0827/2083] added disable_warnings instruction to .coveragerc --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 02acbff8e..ad0ee0f6c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,3 +3,4 @@ branch = true include = scrapy/* omit = tests/* +disable_warnings = include-ignored From c1bbb299d7dc30d03c33cc3eda776ae30ba77d0d Mon Sep 17 00:00:00 2001 From: pankaj1707k <76695979+pankaj1707k@users.noreply.github.com> Date: Wed, 1 Feb 2023 16:30:57 +0530 Subject: [PATCH 0828/2083] Add and run pre-commit hook 'blacken-docs' Change python code snippets to begin with '.. code-block:: python' to be recognized by the hook for formatting. All snippets under '::' (rst literal blocks) are ignored. --- .pre-commit-config.yaml | 6 + docs/faq.rst | 44 ++- docs/intro/overview.rst | 14 +- docs/intro/tutorial.rst | 120 +++--- docs/topics/asyncio.rst | 6 +- docs/topics/broad-crawls.rst | 40 +- docs/topics/commands.rst | 17 +- docs/topics/components.rst | 8 +- docs/topics/contracts.rst | 38 +- docs/topics/coroutines.rst | 29 +- docs/topics/debug.rst | 35 +- docs/topics/developer-tools.rst | 17 +- docs/topics/downloader-middleware.rst | 50 ++- docs/topics/dynamic-content.rst | 15 +- docs/topics/email.rst | 20 +- docs/topics/exceptions.rst | 8 +- docs/topics/exporters.rst | 35 +- docs/topics/extensions.rst | 24 +- docs/topics/feed-exports.rst | 67 ++-- docs/topics/item-pipeline.rst | 50 ++- docs/topics/items.rst | 30 +- docs/topics/jobs.rst | 6 +- docs/topics/link-extractors.rst | 8 +- docs/topics/loaders.rst | 100 +++-- docs/topics/logging.rst | 109 +++-- docs/topics/media-pipeline.rst | 177 ++++++--- docs/topics/practices.rst | 55 ++- docs/topics/request-response.rst | 176 ++++++--- docs/topics/selectors.rst | 33 +- docs/topics/settings.rst | 202 ++++++---- docs/topics/shell.rst | 5 +- docs/topics/signals.rst | 31 +- docs/topics/spider-middleware.rst | 18 +- docs/topics/spiders.rst | 246 +++++++----- docs/topics/stats.rst | 29 +- sep/sep-001.rst | 549 +++++++++++++------------- sep/sep-002.rst | 44 ++- sep/sep-003.rst | 90 +++-- sep/sep-004.rst | 9 +- sep/sep-005.rst | 33 +- sep/sep-008.rst | 5 +- sep/sep-009.rst | 18 +- sep/sep-012.rst | 6 +- sep/sep-014.rst | 170 ++++---- sep/sep-016.rst | 49 ++- sep/sep-017.rst | 10 +- sep/sep-018.rst | 150 +++---- sep/sep-019.rst | 10 +- sep/sep-020.rst | 50 +-- sep/sep-021.rst | 20 +- 50 files changed, 1821 insertions(+), 1260 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5fc1285f..729682392 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,3 +16,9 @@ repos: rev: 5.12.0 hooks: - id: isort +- repo: https://github.com/adamchainz/blacken-docs + rev: 1.13.0 + hooks: + - id: blacken-docs + additional_dependencies: + - black==22.12.0 diff --git a/docs/faq.rst b/docs/faq.rst index 8a9ba809b..836420567 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -35,8 +35,9 @@ for parsing HTML responses in Scrapy callbacks. You just have to feed the response's body into a ``BeautifulSoup`` object and extract whatever data you need from it. -Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser:: +Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser: +.. code-block:: python from bs4 import BeautifulSoup import scrapy @@ -45,17 +46,12 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars class ExampleSpider(scrapy.Spider): name = "example" allowed_domains = ["example.com"] - start_urls = ( - 'http://www.example.com/', - ) + start_urls = ("http://www.example.com/",) def parse(self, response): # use lxml to get decent HTML parsing speed - soup = BeautifulSoup(response.text, 'lxml') - yield { - "url": response.url, - "title": soup.h1.string - } + soup = BeautifulSoup(response.text, "lxml") + yield {"url": response.url, "title": soup.h1.string} .. note:: @@ -109,11 +105,13 @@ basically means that it crawls in `DFO order`_. This order is more convenient in most cases. If you do want to crawl in true `BFO order`_, you can do it by -setting the following settings:: +setting the following settings: + +.. code-block:: python DEPTH_PRIORITY = 1 - SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue' - SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue' + SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" + SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" While pending requests are below the configured values of :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or @@ -159,11 +157,13 @@ See also other suggestions at `StackOverflow`_. .. note:: Remember to disable :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable - your custom implementation:: + your custom implementation: + + .. code-block:: python SPIDER_MIDDLEWARES = { - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, - 'myproject.middlewares.CustomOffsiteMiddleware': 500, + "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, + "myproject.middlewares.CustomOffsiteMiddleware": 500, } .. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation @@ -235,11 +235,13 @@ What does the response status code 999 means? 999 is a custom response status code used by Yahoo sites to throttle requests. Try slowing down the crawling speed by using a download delay of ``2`` (or -higher) in your spider:: +higher) in your spider: + +.. code-block:: python class MySpider(CrawlSpider): - name = 'myspider' + name = "myspider" download_delay = 2 @@ -351,19 +353,21 @@ How to split an item into multiple items in an item pipeline? input item. :ref:`Create a spider middleware ` instead, and use its :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` -method for this purpose. For example:: +method for this purpose. For example: + +.. code-block:: python from copy import deepcopy from itemadapter import is_item, ItemAdapter - class MultiplyItemsMiddleware: + class MultiplyItemsMiddleware: def process_spider_output(self, response, result, spider): for item in result: if is_item(item): adapter = ItemAdapter(item) - for _ in range(adapter['multiply_by']): + for _ in range(adapter["multiply_by"]): yield deepcopy(item) Does Scrapy support IPv6 addresses? diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index cfa6bfa83..495aad091 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -20,22 +20,24 @@ In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. Here's the code for a spider that scrapes famous quotes from website -https://quotes.toscrape.com, following the pagination:: +https://quotes.toscrape.com, following the pagination + +.. code-block:: python import scrapy class QuotesSpider(scrapy.Spider): - name = 'quotes' + name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/tag/humor/', + "https://quotes.toscrape.com/tag/humor/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'author': quote.xpath('span/small/text()').get(), - 'text': quote.css('span.text::text').get(), + "author": quote.xpath("span/small/text()").get(), + "text": quote.css("span.text::text").get(), } next_page = response.css('li.next a::attr("href")').get() diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 901a170b4..f5e9b372e 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -177,7 +177,9 @@ that generates :class:`scrapy.Request ` objects from URLs, you can just define a :attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This list will then be used by the default implementation of :meth:`~scrapy.Spider.start_requests` to create the initial requests -for your spider:: +for your spider. + +.. code-block:: python from pathlib import Path @@ -187,13 +189,13 @@ for your spider:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', - 'https://quotes.toscrape.com/page/2/', + "https://quotes.toscrape.com/page/1/", + "https://quotes.toscrape.com/page/2/", ] def parse(self, response): page = response.url.split("/")[-2] - filename = f'quotes-{page}.html' + filename = f"quotes-{page}.html" Path(filename).write_bytes(response.body) The :meth:`~scrapy.Spider.parse` method will be called to handle each @@ -438,7 +440,9 @@ extraction logic above into our spider. A Scrapy spider typically generates many dictionaries containing the data extracted from the page. To do that, we use the ``yield`` Python keyword -in the callback, as you can see below:: +in the callback, as you can see below: + +.. code-block:: python import scrapy @@ -446,16 +450,16 @@ in the callback, as you can see below:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', - 'https://quotes.toscrape.com/page/2/', + "https://quotes.toscrape.com/page/1/", + "https://quotes.toscrape.com/page/2/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), + "text": quote.css("span.text::text").get(), + "author": quote.css("small.author::text").get(), + "tags": quote.css("div.tags a.tag::text").getall(), } If you run this spider, it will output the extracted data with the log:: @@ -543,7 +547,9 @@ There is also an ``attrib`` property available '/page/2/' Let's see now our spider modified to recursively follow the link to the next -page, extracting data from it:: +page, extracting data from it: + +.. code-block:: python import scrapy @@ -551,18 +557,18 @@ page, extracting data from it:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', + "https://quotes.toscrape.com/page/1/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), + "text": quote.css("span.text::text").get(), + "author": quote.css("small.author::text").get(), + "tags": quote.css("div.tags a.tag::text").getall(), } - next_page = response.css('li.next a::attr(href)').get() + next_page = response.css("li.next a::attr(href)").get() if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse) @@ -594,7 +600,9 @@ A shortcut for creating Requests -------------------------------- As a shortcut for creating Request objects you can use -:meth:`response.follow `:: +:meth:`response.follow ` + +.. code-block:: python import scrapy @@ -602,18 +610,18 @@ As a shortcut for creating Request objects you can use class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', + "https://quotes.toscrape.com/page/1/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('span small::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), + "text": quote.css("span.text::text").get(), + "author": quote.css("span small::text").get(), + "tags": quote.css("div.tags a.tag::text").getall(), } - next_page = response.css('li.next a::attr(href)').get() + next_page = response.css("li.next a::attr(href)").get() if next_page is not None: yield response.follow(next_page, callback=self.parse) @@ -622,57 +630,67 @@ need to call urljoin. Note that ``response.follow`` just returns a Request instance; you still have to yield this Request. You can also pass a selector to ``response.follow`` instead of a string; -this selector should extract necessary attributes:: +this selector should extract necessary attributes: - for href in response.css('ul.pager a::attr(href)'): +.. code-block:: python + + for href in response.css("ul.pager a::attr(href)"): yield response.follow(href, callback=self.parse) For ```` elements there is a shortcut: ``response.follow`` uses their href -attribute automatically. So the code can be shortened further:: +attribute automatically. So the code can be shortened further: - for a in response.css('ul.pager a'): +.. code-block:: python + + for a in response.css("ul.pager a"): yield response.follow(a, callback=self.parse) To create multiple requests from an iterable, you can use -:meth:`response.follow_all ` instead:: +:meth:`response.follow_all ` instead: - anchors = response.css('ul.pager a') +.. code-block:: python + + anchors = response.css("ul.pager a") yield from response.follow_all(anchors, callback=self.parse) -or, shortening it further:: +or, shortening it further: - yield from response.follow_all(css='ul.pager a', callback=self.parse) +.. code-block:: python + + yield from response.follow_all(css="ul.pager a", callback=self.parse) More examples and patterns -------------------------- Here is another spider that illustrates callbacks and following links, -this time for scraping author information:: +this time for scraping author information: + +.. code-block:: python import scrapy class AuthorSpider(scrapy.Spider): - name = 'author' + name = "author" - start_urls = ['https://quotes.toscrape.com/'] + start_urls = ["https://quotes.toscrape.com/"] def parse(self, response): - author_page_links = response.css('.author + a') + author_page_links = response.css(".author + a") yield from response.follow_all(author_page_links, self.parse_author) - pagination_links = response.css('li.next a') + pagination_links = response.css("li.next a") yield from response.follow_all(pagination_links, self.parse) def parse_author(self, response): def extract_with_css(query): - return response.css(query).get(default='').strip() + return response.css(query).get(default="").strip() yield { - 'name': extract_with_css('h3.author-title::text'), - 'birthdate': extract_with_css('.author-born-date::text'), - 'bio': extract_with_css('.author-description::text'), + "name": extract_with_css("h3.author-title::text"), + "birthdate": extract_with_css(".author-born-date::text"), + "bio": extract_with_css(".author-description::text"), } This spider will start from the main page, it will follow all the links to the @@ -720,7 +738,9 @@ spider attributes by default. In this example, the value provided for the ``tag`` argument will be available via ``self.tag``. You can use this to make your spider fetch only quotes -with a specific tag, building the URL based on the argument:: +with a specific tag, building the URL based on the argument: + +.. code-block:: python import scrapy @@ -729,20 +749,20 @@ with a specific tag, building the URL based on the argument:: name = "quotes" def start_requests(self): - url = 'https://quotes.toscrape.com/' - tag = getattr(self, 'tag', None) + url = "https://quotes.toscrape.com/" + tag = getattr(self, "tag", None) if tag is not None: - url = url + 'tag/' + tag + url = url + "tag/" + tag yield scrapy.Request(url, self.parse) def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), + "text": quote.css("span.text::text").get(), + "author": quote.css("small.author::text").get(), } - next_page = response.css('li.next a::attr(href)').get() + next_page = response.css("li.next a::attr(href)").get() if next_page is not None: yield response.follow(next_page, self.parse) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index dbee7146d..7713b1af1 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -106,12 +106,14 @@ Enforcing asyncio as a requirement If you are writing a :ref:`component ` that requires asyncio to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to :ref:`enforce it as a requirement `. For -example:: +example: + +.. code-block:: python from scrapy.utils.reactor import is_asyncio_reactor_installed - class MyComponent: + class MyComponent: def __init__(self): if not is_asyncio_reactor_installed(): raise ValueError( diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 0927ac2d2..8be89feb2 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -48,9 +48,11 @@ Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQ It works best during single-domain crawl. It does not work well with crawling many different domains in parallel -To apply the recommended priority queue use:: +To apply the recommended priority queue use: - SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue' +.. code-block:: python + + SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue" .. _broad-crawls-concurrency: @@ -71,7 +73,9 @@ many different domains in parallel, so you will want to increase it. How much to increase it will depend on how much CPU and memory your crawler will have available. -A good starting point is ``100``:: +A good starting point is ``100``: + +.. code-block:: python CONCURRENT_REQUESTS = 100 @@ -92,7 +96,9 @@ hitting DNS resolver timeouts. Possible solution to increase the number of threads handling DNS queries. The DNS queue will be processed faster speeding up establishing of connection and crawling overall. -To increase maximum thread pool size use:: +To increase maximum thread pool size use: + +.. code-block:: python REACTOR_THREADPOOL_MAXSIZE = 20 @@ -114,9 +120,11 @@ should not use ``DEBUG`` log level when preforming large broad crawls in production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. -To set the log level use:: +To set the log level use: - LOG_LEVEL = 'INFO' +.. code-block:: python + + LOG_LEVEL = "INFO" Disable cookies =============== @@ -126,7 +134,9 @@ doing broad crawls (search engine crawlers ignore them), and they improve performance by saving some CPU cycles and reducing the memory footprint of your Scrapy crawler. -To disable cookies use:: +To disable cookies use: + +.. code-block:: python COOKIES_ENABLED = False @@ -138,7 +148,9 @@ when sites causes are very slow (or fail) to respond, thus causing a timeout error which gets retried many times, unnecessarily, preventing crawler capacity to be reused for other domains. -To disable retries use:: +To disable retries use: + +.. code-block:: python RETRY_ENABLED = False @@ -149,7 +161,9 @@ Unless you are crawling from a very slow connection (which shouldn't be the case for broad crawls) reduce the download timeout so that stuck requests are discarded quickly and free up capacity to process the next ones. -To reduce the download timeout use:: +To reduce the download timeout use: + +.. code-block:: python DOWNLOAD_TIMEOUT = 15 @@ -162,7 +176,9 @@ revisiting the site at a later crawl. This also help to keep the number of request constant per crawl batch, otherwise redirect loops may cause the crawler to dedicate too many resources on any specific domain. -To disable redirects use:: +To disable redirects use: + +.. code-block:: python REDIRECT_ENABLED = False @@ -179,7 +195,9 @@ Pages can indicate it in two ways: "main", "index" website pages. Scrapy handles (1) automatically; to handle (2) enable -:ref:`AjaxCrawlMiddleware `:: +:ref:`AjaxCrawlMiddleware `: + +.. code-block:: python AJAXCRAWL_ENABLED = True diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 362190116..54fd5d663 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -617,7 +617,7 @@ Example: .. code-block:: python - COMMANDS_MODULE = 'mybot.commands' + COMMANDS_MODULE = "mybot.commands" .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html @@ -636,10 +636,11 @@ The following example adds ``my_command`` command: from setuptools import setup, find_packages - setup(name='scrapy-mymodule', - entry_points={ - 'scrapy.commands': [ - 'my_command=my_scrapy_module.commands:MyCommand', - ], - }, - ) + setup( + name="scrapy-mymodule", + entry_points={ + "scrapy.commands": [ + "my_command=my_scrapy_module.commands:MyCommand", + ], + }, + ) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index ca301b827..1ed55f000 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -66,16 +66,18 @@ version mismatch, while :exc:`ValueError` may be better if the issue is the value of a setting. If your requirement is a minimum Scrapy version, you may use -:attr:`scrapy.__version__` to enforce your requirement. For example:: +:attr:`scrapy.__version__` to enforce your requirement. For example: + +.. code-block:: python from pkg_resources import parse_version import scrapy - class MyComponent: + class MyComponent: def __init__(self): - if parse_version(scrapy.__version__) < parse_version('2.7'): + if parse_version(scrapy.__version__) < parse_version("2.7"): raise RuntimeError( f"{MyComponent.__qualname__} requires Scrapy 2.7 or " f"later, which allow defining the process_spider_output " diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index c29a3a410..211a0f5f2 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -11,10 +11,13 @@ integrated way of testing your spiders by the means of contracts. This allows you to test each callback of your spider by hardcoding a sample url and check various constraints for how the callback processes the response. Each contract is prefixed with an ``@`` and included in the docstring. See the -following example:: +following example: + +.. code-block:: python def parse(self, response): - """ This function parses a sample response. Some contracts are mingled + """ + This function parses a sample response. Some contracts are mingled with this docstring. @url http://www.amazon.com/s?field-keywords=selfish+gene @@ -64,11 +67,13 @@ Custom Contracts If you find you need more power than the built-in Scrapy contracts you can create and load your own contracts in the project by using the -:setting:`SPIDER_CONTRACTS` setting:: +:setting:`SPIDER_CONTRACTS` setting: + +.. code-block:: python SPIDER_CONTRACTS = { - 'myproject.contracts.ResponseCheck': 10, - 'myproject.contracts.ItemValidate': 10, + "myproject.contracts.ResponseCheck": 10, + "myproject.contracts.ItemValidate": 10, } Each contract must inherit from :class:`~scrapy.contracts.Contract` and can @@ -111,22 +116,26 @@ Raise :class:`~scrapy.exceptions.ContractFail` from .. autoclass:: scrapy.exceptions.ContractFail Here is a demo contract which checks the presence of a custom header in the -response received:: +response received: + +.. code-block:: python from scrapy.contracts import Contract from scrapy.exceptions import ContractFail + class HasHeaderContract(Contract): - """ Demo contract which checks the presence of a custom header - @has_header X-CustomHeader + """ + Demo contract which checks the presence of a custom header + @has_header X-CustomHeader """ - name = 'has_header' + name = "has_header" def pre_process(self, response): for header in self.args: if header not in response.headers: - raise ContractFail('X-CustomHeader not present') + raise ContractFail("X-CustomHeader not present") .. _detecting-contract-check-runs: @@ -135,14 +144,17 @@ Detecting check runs When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is set to the ``true`` string. You can use :data:`os.environ` to perform any change to -your spiders or your settings when ``scrapy check`` is used:: +your spiders or your settings when ``scrapy check`` is used: + +.. code-block:: python import os import scrapy + class ExampleSpider(scrapy.Spider): - name = 'example' + name = "example" def __init__(self): - if os.environ.get('SCRAPY_CHECK'): + if os.environ.get("SCRAPY_CHECK"): pass # Do some scraper adjustments when a check is running diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a1ba4ba5c..a0c005204 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -58,49 +58,58 @@ There are several use cases for coroutines in Scrapy. Code that would return Deferreds when written for previous Scrapy versions, such as downloader middlewares and signal handlers, can be rewritten to be -shorter and cleaner:: +shorter and cleaner: + +.. code-block:: python from itemadapter import ItemAdapter + class DbPipeline: def _update_item(self, data, item): adapter = ItemAdapter(item) - adapter['field'] = data + adapter["field"] = data return item def process_item(self, item, spider): adapter = ItemAdapter(item) - dfd = db.get_some_data(adapter['id']) + dfd = db.get_some_data(adapter["id"]) dfd.addCallback(self._update_item, item) return dfd -becomes:: +becomes: + +.. code-block:: python from itemadapter import ItemAdapter + class DbPipeline: async def process_item(self, item, spider): adapter = ItemAdapter(item) - adapter['field'] = await db.get_some_data(adapter['id']) + adapter["field"] = await db.get_some_data(adapter["id"]) return item Coroutines may be used to call asynchronous code. This includes other coroutines, functions that return Deferreds and functions that return :term:`awaitable objects ` such as :class:`~asyncio.Future`. -This means you can use many useful Python libraries providing such code:: +This means you can use many useful Python libraries providing such code: + +.. code-block:: python class MySpiderDeferred(Spider): # ... async def parse(self, response): - additional_response = await treq.get('https://additional.url') + additional_response = await treq.get("https://additional.url") additional_data = await treq.content(additional_response) # ... use response and additional_data to yield items and requests + class MySpiderAsyncio(Spider): # ... async def parse(self, response): async with aiohttp.ClientSession() as session: - async with session.get('https://additional.url') as additional_response: + async with session.get("https://additional.url") as additional_response: additional_data = await additional_response.text() # ... use response and additional_data to yield items and requests @@ -192,7 +201,9 @@ while maintaining support for older Scrapy versions, you may define :term:`asynchronous generator` version of that method with an alternative name: ``process_spider_output_async``. -For example:: +For example: + +.. code-block:: python class UniversalSpiderMiddleware: def process_spider_output(self, response, result, spider): diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index edbcaf432..b133fcc1e 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -5,21 +5,24 @@ Debugging Spiders ================= This document explains the most common techniques for debugging spiders. -Consider the following Scrapy spider below:: +Consider the following Scrapy spider below: + +.. code-block:: python import scrapy from myproject.items import MyItem + class MySpider(scrapy.Spider): - name = 'myspider' + name = "myspider" start_urls = ( - 'http://example.com/page1', - 'http://example.com/page2', - ) + "http://example.com/page1", + "http://example.com/page2", + ) def parse(self, response): # - # collect `item_urls` + # collect `item_urls` for item_url in item_urls: yield scrapy.Request(item_url, self.parse_item) @@ -28,7 +31,9 @@ Consider the following Scrapy spider below:: item = MyItem() # populate `item` fields # and extract item_details_url - yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item}) + yield scrapy.Request( + item_details_url, self.parse_details, cb_kwargs={"item": item} + ) def parse_details(self, response, item): # populate more `item` fields @@ -103,10 +108,13 @@ showing the response received and the output. How to debug the situation when .. highlight:: python Fortunately, the :command:`shell` is your bread and butter in this case (see -:ref:`topics-shell-inspect-response`):: +:ref:`topics-shell-inspect-response`): + +.. code-block:: python from scrapy.shell import inspect_response + def parse_details(self, response, item=None): if item: # populate more `item` fields @@ -121,10 +129,13 @@ Open in browser Sometimes you just want to see how a certain response looks in a browser, you can use the ``open_in_browser`` function for that. Here is an example of how -you would use it:: +you would use it: + +.. code-block:: python from scrapy.utils.response import open_in_browser + def parse_details(self, response): if "item name" not in response.body: open_in_browser(response) @@ -138,14 +149,16 @@ Logging Logging is another useful option for getting information about your spider run. Although not as convenient, it comes with the advantage that the logs will be -available in all future runs should they be necessary again:: +available in all future runs should they be necessary again: + +.. code-block:: python def parse_details(self, response, item=None): if item: # populate more `item` fields return item else: - self.logger.warning('No item received for %s', response.url) + self.logger.warning("No item received for %s", response.url) For more information, check the :ref:`topics-logging` section. diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 9bf97c628..39e7b7d3c 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -237,17 +237,19 @@ on the request and open ``Open in new tab`` to get a better overview. :alt: JSON-object returned from the quotes.toscrape API With this response we can now easily parse the JSON-object and -also request each page to get every quote on the site:: +also request each page to get every quote on the site: + +.. code-block:: python import scrapy import json class QuoteSpider(scrapy.Spider): - name = 'quote' - allowed_domains = ['quotes.toscrape.com'] + name = "quote" + allowed_domains = ["quotes.toscrape.com"] page = 1 - start_urls = ['https://quotes.toscrape.com/api/quotes?page=1'] + start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"] def parse(self, response): data = json.loads(response.text) @@ -275,7 +277,9 @@ requests, as we could need to add ``headers`` or ``cookies`` to make it work. In those cases you can export the requests in `cURL `_ format, by right-clicking on each of them in the network tool and using the :meth:`~scrapy.Request.from_curl()` method to generate an equivalent -request:: +request: + +.. code-block:: python from scrapy import Request @@ -286,7 +290,8 @@ request:: "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM" "zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW" "I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http" - "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'") + "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'" + ) Alternatively, if you want to know the arguments needed to recreate that request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs` diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 986da0476..e1c481c37 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -17,10 +17,12 @@ To activate a downloader middleware component, add it to the :setting:`DOWNLOADER_MIDDLEWARES` setting, which is a dict whose keys are the middleware class paths and their values are the middleware orders. -Here's an example:: +Here's an example: + +.. code-block:: python DOWNLOADER_MIDDLEWARES = { - 'myproject.middlewares.CustomDownloaderMiddleware': 543, + "myproject.middlewares.CustomDownloaderMiddleware": 543, } The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the @@ -42,11 +44,13 @@ previous (or subsequent) middleware being applied. If you want to disable a built-in middleware (the ones defined in :setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` -as its value. For example, if you want to disable the user-agent middleware:: +as its value. For example, if you want to disable the user-agent middleware: + +.. code-block:: python DOWNLOADER_MIDDLEWARES = { - 'myproject.middlewares.CustomDownloaderMiddleware': 543, - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None, + "myproject.middlewares.CustomDownloaderMiddleware": 543, + "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None, } Finally, keep in mind that some middlewares may need to be enabled through a @@ -226,20 +230,25 @@ There is support for keeping multiple cookie sessions per spider by using the :reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar (session), but you can pass an identifier to use different ones. -For example:: +For example: + +.. code-block:: python for i, url in enumerate(urls): - yield scrapy.Request(url, meta={'cookiejar': i}, - callback=self.parse_page) + yield scrapy.Request(url, meta={"cookiejar": i}, callback=self.parse_page) Keep in mind that the :reqmeta:`cookiejar` meta key is not "sticky". You need to keep -passing it along on subsequent requests. For example:: +passing it along on subsequent requests. For example: + +.. code-block:: python def parse_page(self, response): # do some processing - return scrapy.Request("http://www.example.com/otherpage", - meta={'cookiejar': response.meta['cookiejar']}, - callback=self.parse_other_page) + return scrapy.Request( + "http://www.example.com/otherpage", + meta={"cookiejar": response.meta["cookiejar"]}, + callback=self.parse_other_page, + ) .. setting:: COOKIES_ENABLED @@ -339,16 +348,19 @@ HttpAuthMiddleware domain of the first request, which will work for some spiders but not for others. In the future the middleware will produce an error instead. - Example:: + Example: + + .. code-block:: python from scrapy.spiders import CrawlSpider + class SomeIntranetSiteSpider(CrawlSpider): - http_user = 'someuser' - http_pass = 'somepass' - http_auth_domain = 'intranet.example.com' - name = 'intranet.example.com' + http_user = "someuser" + http_pass = "somepass" + http_auth_domain = "intranet.example.com" + name = "intranet.example.com" # .. rest of the spider code omitted ... @@ -792,7 +804,9 @@ If you want to handle some redirect status codes in your spider, you can specify these in the ``handle_httpstatus_list`` spider attribute. For example, if you want the redirect middleware to ignore 301 and 302 -responses (and pass them through to your spider) you can do this:: +responses (and pass them through to your spider) you can do this: + +.. code-block:: python class MySpider(CrawlSpider): handle_httpstatus_list = [301, 302] diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index ea5d06210..9be0ed058 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -119,16 +119,20 @@ data from it depends on the type of response: ` as usual. - If the response is JSON, use :func:`json.loads` to load the desired data from - :attr:`response.text `:: + :attr:`response.text `: + + .. code-block:: python data = json.loads(response.text) If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a :class:`~scrapy.Selector` and then - :ref:`use it ` as usual:: + :ref:`use it ` as usual: - selector = Selector(data['html']) + .. code-block:: python + + selector = Selector(data["html"]) - If the response is JavaScript, or HTML with a `` -{% endblock %} From 46bb7b31d1d4ea351615ae890286e40b6d0a82c4 Mon Sep 17 00:00:00 2001 From: karza_abhishek Date: Tue, 28 Mar 2023 23:23:32 +0530 Subject: [PATCH 0903/2083] Fixed Docs Makefile to open build/html/index.html in browser(#5878) --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index 596cb6cef..48401bac8 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -87,7 +87,7 @@ coverage: build htmlview: html $(PYTHON) -c "import webbrowser; from pathlib import Path; \ - webbrowser.open('file://' + Path('build/html/index.html').resolve())" + webbrowser.open(Path('build/html/index.html').resolve().as_uri())" clean: -rm -rf build/* From b83fa60a0a76364a64d4036e826a713a9a47f96d Mon Sep 17 00:00:00 2001 From: Prathm-s Date: Thu, 30 Mar 2023 16:57:56 +0530 Subject: [PATCH 0904/2083] return added to _sent_failed --- scrapy/mail.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/mail.py b/scrapy/mail.py index 43115c53e..c11f3898d 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -164,6 +164,7 @@ class MailSender: "mailerr": errstr, }, ) + return failure def _sendmail(self, to_addrs, msg): from twisted.internet import reactor From 90ce6589eee58e8aca9c368a71907b30250df68d Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Thu, 30 Mar 2023 13:07:51 +0000 Subject: [PATCH 0905/2083] Add try/except to safe_url_string() Added a try catch condition to the safe_url_string() processing in the LxmlParserLinkExtractor class to avoid scrapers crashing unneccessarily --- scrapy/linkextractors/lxmlhtml.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index dd8dcdf7c..1ee81427c 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -88,7 +88,11 @@ class LxmlParserLinkExtractor: url = self.process_attr(attr_val) if url is None: continue - url = safe_url_string(url, encoding=response_encoding) + try: + url = safe_url_string(url, encoding=response_encoding) + except ValueError: + continue # Disregard badly formatted urls + # to fix relative links after process_value url = urljoin(response_url, url) link = Link( From 9ef00c5c0bc16c9b44c878952a2b54310dc25638 Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 08:01:54 +0000 Subject: [PATCH 0906/2083] Add logging Lines Adds an error loggign line to the LinkExtractor to detail encountered bad links --- scrapy/linkextractors/lxmlhtml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 1ee81427c..f7c6937b0 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -2,6 +2,7 @@ Link extractor based on lxml.html """ import operator +import logging from functools import partial from urllib.parse import urljoin, urlparse @@ -23,6 +24,8 @@ from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain +logger = logging.getLogger(__name__) + # from lxml/src/lxml/html/__init__.py XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" @@ -91,6 +94,7 @@ class LxmlParserLinkExtractor: try: url = safe_url_string(url, encoding=response_encoding) except ValueError: + logger.error(f"Skipping extraction of bad link {url}") continue # Disregard badly formatted urls # to fix relative links after process_value From 9cbcf7724df7de9a449659fbf68bc5d532c33499 Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 08:07:43 +0000 Subject: [PATCH 0907/2083] Add test to make sure spider doesn't crash on bad --- tests/test_linkextractors.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index f663013ba..d992a5eae 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -815,3 +815,26 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): def test_restrict_xpaths_with_html_entities(self): super().test_restrict_xpaths_with_html_entities() + + def test_skip_bad_links(self): + html = b""" + Why would you do this? + Good Link + Good Link 2 + """ + response = HtmlResponse("http://example.org/index.html", body=html) + self.assertEqual( + [link for link in lx.extract_links(response)], + [ + Link( + url="http://example.org/item2.html", + text="Good Link", + nofollow=False, + ), + Link( + url="http://example.org/item3.html", + text="Good Link 2", + nofollow=False, + ), + ], + ) From 7cb7cf1ad1aa3d75b494a5b069e5b76b60328daa Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 08:09:02 +0000 Subject: [PATCH 0908/2083] Add link extractor back to test --- tests/test_linkextractors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index d992a5eae..3ad1abea5 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -823,6 +823,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): Good Link 2 """ response = HtmlResponse("http://example.org/index.html", body=html) + lx = self.extractor_cls() self.assertEqual( [link for link in lx.extract_links(response)], [ From 00d93026c8b078d75e4fe43c344f6524f9b45f28 Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 08:30:19 +0000 Subject: [PATCH 0909/2083] Fix bad test case --- tests/test_linkextractors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 3ad1abea5..1ea364d80 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -818,11 +818,11 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): def test_skip_bad_links(self): html = b""" - Why would you do this? + Why would you do this? Good Link Good Link 2 """ - response = HtmlResponse("http://example.org/index.html", body=html) + response = HtmlResponse("http://example.org/index.html", body=html, encoding='utf-8') lx = self.extractor_cls() self.assertEqual( [link for link in lx.extract_links(response)], From 6d94aa061ca96275ab8b83840ac21f0e72ad1583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20DIDIER?= <73602526+sdidier-dev@users.noreply.github.com> Date: Fri, 31 Mar 2023 11:05:20 +0200 Subject: [PATCH 0910/2083] Add missing 'crawl' command in crawl examples --- docs/topics/commands.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 362190116..c2f2da4e2 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -288,13 +288,13 @@ Usage examples:: $ scrapy crawl myspider [ ... myspider starts crawling ... ] - $ scrapy -o myfile:csv myspider + $ scrapy crawl -o myfile:csv myspider [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] - $ scrapy -O myfile:json myspider + $ scrapy crawl -O myfile:json myspider [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ] - $ scrapy -o myfile -t csv myspider + $ scrapy crawl -o myfile -t csv myspider [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] .. command:: check From 4043560547faac0ee4cfadba4d0f02b4be1f72de Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 12:29:22 +0000 Subject: [PATCH 0911/2083] remove utf-8 encoding flag from test --- tests/test_linkextractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 1ea364d80..3673e82cd 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -822,7 +822,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): Good Link Good Link 2 """ - response = HtmlResponse("http://example.org/index.html", body=html, encoding='utf-8') + response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( [link for link in lx.extract_links(response)], From c9a5934494cbb3fe0aa572b536ee222a3e212487 Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 12:29:49 +0000 Subject: [PATCH 0912/2083] Reduce logging level of bad URL --- scrapy/linkextractors/lxmlhtml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index f7c6937b0..0d1b76531 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -94,7 +94,7 @@ class LxmlParserLinkExtractor: try: url = safe_url_string(url, encoding=response_encoding) except ValueError: - logger.error(f"Skipping extraction of bad link {url}") + logger.debug(f"Skipping extraction of bad link {url}") continue # Disregard badly formatted urls # to fix relative links after process_value From 608b7de582af891a37a3fab60423c847af648db8 Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 14:38:06 +0000 Subject: [PATCH 0913/2083] Skip new test if python version less than 3.8 --- tests/test_linkextractors.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 3673e82cd..78219f642 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,6 +1,7 @@ import pickle import re import unittest +import sys from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link @@ -816,6 +817,10 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): def test_restrict_xpaths_with_html_entities(self): super().test_restrict_xpaths_with_html_entities() + @unittest.skipIf( + sys.version_info < (3, 8), + reason="Urllib3 is less strict in versions for python 3.7 so does not cause spider to crash", + ) def test_skip_bad_links(self): html = b""" Why would you do this? From 618e82dbe104c4b97cc4f7b37bf9130a76093734 Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 15:12:47 +0000 Subject: [PATCH 0914/2083] Exclude test for python versionbs less than 3.8 --- tests/test_linkextractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 78219f642..784fdb658 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -819,7 +819,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): @unittest.skipIf( sys.version_info < (3, 8), - reason="Urllib3 is less strict in versions for python 3.7 so does not cause spider to crash", + reason="some library for python 3.7 so is less strict so bad links like htis don't crash scrapy", ) def test_skip_bad_links(self): html = b""" From a0e2e36b52743c9bf1cc03408bf4eb66e68a1edc Mon Sep 17 00:00:00 2001 From: Felipe Boff Nunes Date: Thu, 6 Apr 2023 14:23:19 -0300 Subject: [PATCH 0915/2083] adjustments --- scrapy/extensions/feedexport.py | 2 -- tests/test_feedexport.py | 22 +++++++++------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 67ff26764..0df32083f 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -115,7 +115,6 @@ class BlockingFeedStorage: @implementer(IFeedStorage) class StdoutFeedStorage: - def __init__(self, uri, _stdout=None, *, feed_options=None): if not _stdout: _stdout = sys.stdout.buffer @@ -384,7 +383,6 @@ class FeedExporter: return defer.DeferredList(deferred_list) if deferred_list else None def _close_slot(self, slot, spider): - def get_file(slot_): if isinstance(slot_.file, PostProcessingManager): return slot_.file.file diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 45027171f..3124d9d67 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1640,15 +1640,13 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_storage_file_no_postprocessing(self): - @implementer(IFeedStorage) class Storage: - def __init__(self, uri, *, feed_options=None): pass def open(self, spider): - Storage.open_file = tempfile.NamedTemporaryFile(prefix='feed-') + Storage.open_file = tempfile.NamedTemporaryFile(prefix="feed-") return Storage.open_file def store(self, file): @@ -1656,23 +1654,21 @@ class FeedExportTest(FeedExportTestBase): file.close() settings = { - 'FEEDS': {self._random_temp_filename(): {'format': 'jsonlines'}}, - 'FEED_STORAGES': {'file': Storage}, + "FEEDS": {self._random_temp_filename(): {"format": "jsonlines"}}, + "FEED_STORAGES": {"file": Storage}, } yield self.exported_no_data(settings) self.assertIs(Storage.open_file, Storage.store_file) @defer.inlineCallbacks def test_storage_file_postprocessing(self): - @implementer(IFeedStorage) class Storage: - def __init__(self, uri, *, feed_options=None): pass def open(self, spider): - Storage.open_file = tempfile.NamedTemporaryFile(prefix='feed-') + Storage.open_file = tempfile.NamedTemporaryFile(prefix="feed-") return Storage.open_file def store(self, file): @@ -1680,15 +1676,15 @@ class FeedExportTest(FeedExportTestBase): file.close() settings = { - 'FEEDS': { + "FEEDS": { self._random_temp_filename(): { - 'format': 'jsonlines', - 'postprocessing': [ - 'scrapy.extensions.postprocessing.GzipPlugin', + "format": "jsonlines", + "postprocessing": [ + "scrapy.extensions.postprocessing.GzipPlugin", ], }, }, - 'FEED_STORAGES': {'file': Storage}, + "FEED_STORAGES": {"file": Storage}, } yield self.exported_no_data(settings) self.assertIs(Storage.open_file, Storage.store_file) From 98571eb946e24edfe5b520c0478e72b695d09a9d Mon Sep 17 00:00:00 2001 From: Mojtaba Dashtinejad Date: Mon, 10 Apr 2023 14:14:49 +0200 Subject: [PATCH 0916/2083] Add missing slash in selecting dynamically-loaded content documentation (#5890) --- docs/topics/dynamic-content.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index d01e0a8d4..a0f4b4411 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -276,7 +276,7 @@ The following is a simple snippet to illustrate its usage within a Scrapy spider async with async_playwright() as pw: browser = await pw.chromium.launch() page = await browser.new_page() - await page.goto("https:/example.org") + await page.goto("https://example.org") title = await page.title() return {"title": title} From 23017e6e926116162cc8fbdaf38e02670ad5d5cb Mon Sep 17 00:00:00 2001 From: Felipe Boff Nunes Date: Mon, 10 Apr 2023 12:30:00 -0300 Subject: [PATCH 0917/2083] adjust --- scrapy/extensions/feedexport.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0df32083f..da1a88299 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -385,6 +385,7 @@ class FeedExporter: def _close_slot(self, slot, spider): def get_file(slot_): if isinstance(slot_.file, PostProcessingManager): + slot_.file.close() return slot_.file.file return slot_.file From c7730627a0f99afccca11437b0775757c50ca9e8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Apr 2023 15:47:41 +0400 Subject: [PATCH 0918/2083] Typing improvements for some core components (#5889) --- scrapy/core/downloader/__init__.py | 9 ++-- scrapy/core/engine.py | 68 ++++++++++++++++++------------ scrapy/core/scheduler.py | 31 ++++++++------ scrapy/core/scraper.py | 31 +++++++++----- scrapy/core/spidermw.py | 25 ++++++----- scrapy/crawler.py | 33 +++++++++------ scrapy/exceptions.py | 2 +- scrapy/logformatter.py | 32 +++++++++++--- scrapy/middleware.py | 10 ++--- scrapy/pipelines/__init__.py | 10 +++-- scrapy/signalmanager.py | 23 +++++----- scrapy/statscollectors.py | 65 +++++++++++++++++----------- scrapy/utils/log.py | 5 ++- scrapy/utils/misc.py | 9 +++- scrapy/utils/reactor.py | 18 ++++---- scrapy/utils/signal.py | 9 +++- tox.ini | 12 +++--- 17 files changed, 245 insertions(+), 147 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 1b83c3a8a..dde76a547 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -4,8 +4,9 @@ from datetime import datetime from time import time from twisted.internet import defer, task +from twisted.internet.defer import Deferred -from scrapy import signals +from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.resolver import dnscache @@ -86,7 +87,7 @@ class Downloader: self._slot_gc_loop.start(60) self.per_slot_settings = self.settings.getdict("DOWNLOAD_SLOTS", {}) - def fetch(self, request, spider): + def fetch(self, request: Request, spider: Spider) -> Deferred: def _deactivate(response): self.active.remove(request) return response @@ -206,12 +207,12 @@ class Downloader: return dfd.addBoth(finish_transferring) - def close(self): + def close(self) -> None: self._slot_gc_loop.stop() for slot in self.slots.values(): slot.close() - def _slot_gc(self, age=60): + def _slot_gc(self, age: float = 60) -> None: mintime = time() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index c6738b531..3e5a281b2 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -8,13 +8,16 @@ import logging import warnings from time import time from typing import ( + TYPE_CHECKING, Any, Callable, Generator, Iterable, Iterator, + List, Optional, Set, + Type, Union, cast, ) @@ -28,30 +31,36 @@ from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning from scrapy.http import Request, Response -from scrapy.settings import BaseSettings +from scrapy.logformatter import LogFormatter +from scrapy.settings import BaseSettings, Settings +from scrapy.signalmanager import SignalManager from scrapy.spiders import Spider from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import create_instance, load_object from scrapy.utils.reactor import CallLaterOnce +if TYPE_CHECKING: + from scrapy.core.scheduler import BaseScheduler + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) class Slot: def __init__( self, - start_requests: Iterable, + start_requests: Iterable[Request], close_if_idle: bool, nextcall: CallLaterOnce, - scheduler, + scheduler: "BaseScheduler", ) -> None: self.closing: Optional[Deferred] = None self.inprogress: Set[Request] = set() - self.start_requests: Optional[Iterator] = iter(start_requests) - self.close_if_idle = close_if_idle - self.nextcall = nextcall - self.scheduler = scheduler - self.heartbeat = LoopingCall(nextcall.schedule) + self.start_requests: Optional[Iterator[Request]] = iter(start_requests) + self.close_if_idle: bool = close_if_idle + self.nextcall: CallLaterOnce = nextcall + self.scheduler: "BaseScheduler" = scheduler + self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule) def add_request(self, request: Request) -> None: self.inprogress.add(request) @@ -75,25 +84,28 @@ class Slot: class ExecutionEngine: - def __init__(self, crawler, spider_closed_callback: Callable) -> None: - self.crawler = crawler - self.settings = crawler.settings - self.signals = crawler.signals - self.logformatter = crawler.logformatter + def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None: + self.crawler: "Crawler" = crawler + self.settings: Settings = crawler.settings + self.signals: SignalManager = crawler.signals + self.logformatter: LogFormatter = crawler.logformatter self.slot: Optional[Slot] = None self.spider: Optional[Spider] = None - self.running = False - self.paused = False - self.scheduler_cls = self._get_scheduler_class(crawler.settings) - downloader_cls = load_object(self.settings["DOWNLOADER"]) + self.running: bool = False + self.paused: bool = False + self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class( + crawler.settings + ) + downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) - self._spider_closed_callback = spider_closed_callback + self._spider_closed_callback: Callable = spider_closed_callback + self.start_time: Optional[float] = None - def _get_scheduler_class(self, settings: BaseSettings) -> type: + def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]: from scrapy.core.scheduler import BaseScheduler - scheduler_cls = load_object(settings["SCHEDULER"]) + scheduler_cls: Type = load_object(settings["SCHEDULER"]) if not issubclass(scheduler_cls, BaseScheduler): raise TypeError( f"The provided scheduler class ({settings['SCHEDULER']})" @@ -115,7 +127,7 @@ class ExecutionEngine: """Gracefully stop the execution engine""" @inlineCallbacks - def _finish_stopping_engine(_) -> Generator[Deferred, Any, None]: + def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]: yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) self._closewait.callback(None) @@ -141,7 +153,8 @@ class ExecutionEngine: return self.close_spider( self.spider, reason="shutdown" ) # will also close downloader - return succeed(self.downloader.close()) + self.downloader.close() + return succeed(None) def pause(self) -> None: self.paused = True @@ -209,7 +222,7 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) + d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) # type: ignore[arg-type] d.addErrback( lambda f: logger.info( "Error while removing request from slot", @@ -339,6 +352,7 @@ class ExecutionEngine: if isinstance(result, Response): if result.request is None: result.request = request + assert spider is not None logkws = self.logformatter.crawled(result.request, result, spider) if logkws is not None: logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) @@ -350,10 +364,12 @@ class ExecutionEngine: ) return result - def _on_complete(_): + def _on_complete(_: Any) -> Any: + assert self.slot is not None self.slot.nextcall.schedule() return _ + assert spider is not None dwld = self.downloader.fetch(request, spider) dwld.addCallbacks(_on_success) dwld.addBoth(_on_complete) @@ -362,7 +378,7 @@ class ExecutionEngine: @inlineCallbacks def open_spider( self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True - ): + ) -> Generator[Deferred, Any, None]: if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) @@ -471,7 +487,7 @@ class ExecutionEngine: return dfd @property - def open_spiders(self) -> list: + def open_spiders(self) -> List[Spider]: warnings.warn( "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", category=ScrapyDeprecationWarning, diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 3c46e3a5f..3fb0bbaff 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -2,13 +2,15 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import Optional, Type, TypeVar +from typing import Any, Optional, Type, TypeVar, cast from twisted.internet.defer import Deferred from scrapy.crawler import Crawler +from scrapy.dupefilters import BaseDupeFilter from scrapy.http.request import Request from scrapy.spiders import Spider +from scrapy.statscollectors import StatsCollector from scrapy.utils.job import job_dir from scrapy.utils.misc import create_instance, load_object @@ -20,10 +22,10 @@ class BaseSchedulerMeta(type): Metaclass to check scheduler classes against the necessary interface """ - def __instancecheck__(cls, instance): + def __instancecheck__(cls, instance: Any) -> bool: return cls.__subclasscheck__(type(instance)) - def __subclasscheck__(cls, subclass): + def __subclasscheck__(cls, subclass: type) -> bool: return ( hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests) @@ -168,26 +170,26 @@ class Scheduler(BaseScheduler): def __init__( self, - dupefilter, + dupefilter: BaseDupeFilter, jobdir: Optional[str] = None, dqclass=None, mqclass=None, logunser: bool = False, - stats=None, + stats: Optional[StatsCollector] = None, pqclass=None, crawler: Optional[Crawler] = None, ): - self.df = dupefilter - self.dqdir = self._dqdir(jobdir) + self.df: BaseDupeFilter = dupefilter + self.dqdir: Optional[str] = self._dqdir(jobdir) self.pqclass = pqclass self.dqclass = dqclass self.mqclass = mqclass - self.logunser = logunser - self.stats = stats - self.crawler = crawler + self.logunser: bool = logunser + self.stats: Optional[StatsCollector] = stats + self.crawler: Optional[Crawler] = crawler @classmethod - def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV: + def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV: """ Factory method, initializes the scheduler with arguments taken from the crawl settings """ @@ -242,6 +244,7 @@ class Scheduler(BaseScheduler): self.df.log(request, self.spider) return False dqok = self._dqpush(request) + assert self.stats is not None if dqok: self.stats.inc_value("scheduler/enqueued/disk", spider=self.spider) else: @@ -259,7 +262,8 @@ class Scheduler(BaseScheduler): Increment the appropriate stats, such as: ``scheduler/dequeued``, ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. """ - request = self.mqs.pop() + request: Optional[Request] = self.mqs.pop() + assert self.stats is not None if request is not None: self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider) else: @@ -295,6 +299,7 @@ class Scheduler(BaseScheduler): extra={"spider": self.spider}, ) self.logunser = False + assert self.stats is not None self.stats.inc_value("scheduler/unserializable", spider=self.spider) return False else: @@ -351,7 +356,7 @@ class Scheduler(BaseScheduler): if not path.exists(): return [] with path.open(encoding="utf-8") as f: - return json.load(f) + return cast(list, json.load(f)) def _write_dqs_state(self, dqdir: str, state: list) -> None: with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8468b8419..a85f6a661 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -15,6 +15,7 @@ from typing import ( Optional, Set, Tuple, + Type, Union, ) @@ -26,6 +27,9 @@ from scrapy import Spider, signals from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response +from scrapy.logformatter import LogFormatter +from scrapy.pipelines import ItemPipelineManager +from scrapy.signalmanager import SignalManager from scrapy.utils.defer import ( aiter_errback, defer_fail, @@ -96,16 +100,20 @@ class Slot: class Scraper: def __init__(self, crawler: Crawler) -> None: self.slot: Optional[Slot] = None - self.spidermw = SpiderMiddlewareManager.from_crawler(crawler) - itemproc_cls = load_object(crawler.settings["ITEM_PROCESSOR"]) - self.itemproc = itemproc_cls.from_crawler(crawler) - self.concurrent_items = crawler.settings.getint("CONCURRENT_ITEMS") - self.crawler = crawler - self.signals = crawler.signals - self.logformatter = crawler.logformatter + self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler( + crawler + ) + itemproc_cls: Type[ItemPipelineManager] = load_object( + crawler.settings["ITEM_PROCESSOR"] + ) + self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler) + self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS") + self.crawler: Crawler = crawler + self.signals: SignalManager = crawler.signals + self.logformatter: LogFormatter = crawler.logformatter @inlineCallbacks - def open_spider(self, spider: Spider): + def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]: """Open the given spider for scraping and allocate resources for it""" self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) yield self.itemproc.open_spider(spider) @@ -135,7 +143,8 @@ class Scraper: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) - def finish_scraping(_): + def finish_scraping(_: Any) -> Any: + assert self.slot is not None self.slot.finish_response(result, request) self._check_if_closing(spider) self._scrape_next(spider) @@ -205,9 +214,9 @@ class Scraper: else: # result is a Failure # TODO: properly type adding this attribute to a Failure result.request = request # type: ignore[attr-defined] - warn_on_generator_with_return_value(spider, request.errback) dfd = defer_fail(result) if request.errback: + warn_on_generator_with_return_value(spider, request.errback) dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) @@ -338,7 +347,7 @@ class Scraper: def _itemproc_finished( self, output: Any, item: Any, response: Response, spider: Spider - ) -> None: + ) -> Deferred: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing self.slot.itemproc_size -= 1 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c21985b18..dcf1a6dbc 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -13,6 +13,8 @@ from typing import ( Callable, Generator, Iterable, + List, + Optional, Tuple, Union, cast, @@ -25,6 +27,7 @@ from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager +from scrapy.settings import BaseSettings from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( @@ -41,22 +44,22 @@ logger = logging.getLogger(__name__) ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] -def _isiterable(o) -> bool: +def _isiterable(o: Any) -> bool: return isinstance(o, (Iterable, AsyncIterable)) class SpiderMiddlewareManager(MiddlewareManager): component_name = "spider middleware" - def __init__(self, *middlewares): + def __init__(self, *middlewares: Any): super().__init__(*middlewares) self.downgrade_warning_done = False @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]: return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) - def _add_middleware(self, mw): + def _add_middleware(self, mw: Any) -> None: super()._add_middleware(mw) if hasattr(mw, "process_spider_input"): self.methods["process_spider_input"].append(mw.process_spider_input) @@ -98,7 +101,7 @@ class SpiderMiddlewareManager(MiddlewareManager): exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain], ) -> Union[Generator, AsyncGenerator]: - def process_sync(iterable: Iterable): + def process_sync(iterable: Iterable) -> Generator: try: for r in iterable: yield r @@ -110,7 +113,7 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recover_to.extend(exception_result) - async def process_async(iterable: AsyncIterable): + async def process_async(iterable: AsyncIterable) -> AsyncGenerator: try: async for r in iterable: yield r @@ -280,7 +283,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if isinstance(recovered, AsyncIterable): recovered_collected = await collect_asyncgen(recovered) recovered = MutableChain(recovered_collected) - return MutableChain(result, recovered) # type: ignore[arg-type] + return MutableChain(result, recovered) def scrape_response( self, @@ -306,7 +309,9 @@ class SpiderMiddlewareManager(MiddlewareManager): ) return dfd - def process_start_requests(self, start_requests, spider: Spider) -> Deferred: + def process_start_requests( + self, start_requests: Iterable[Request], spider: Spider + ) -> Deferred: return self._process_chain("process_start_requests", start_requests, spider) # This method is only needed until _async compatibility methods are removed. @@ -314,9 +319,9 @@ class SpiderMiddlewareManager(MiddlewareManager): def _get_async_method_pair( mw: Any, methodname: str ) -> Union[None, Callable, Tuple[Callable, Callable]]: - normal_method = getattr(mw, methodname, None) + normal_method: Optional[Callable] = getattr(mw, methodname, None) methodname_async = methodname + "_async" - async_method = getattr(mw, methodname_async, None) + async_method: Optional[Callable] = getattr(mw, methodname_async, None) if not async_method: return normal_method if not normal_method: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 397817d6f..256f6e2c5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,7 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Optional, Type, Union from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement @@ -22,8 +22,10 @@ from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader +from scrapy.logformatter import LogFormatter from scrapy.settings import Settings, overridden_settings from scrapy.signalmanager import SignalManager +from scrapy.statscollectors import StatsCollector from scrapy.utils.log import ( LogCounterHandler, configure_logging, @@ -49,20 +51,25 @@ logger = logging.getLogger(__name__) class Crawler: - def __init__(self, spidercls, settings=None, init_reactor: bool = False): + def __init__( + self, + spidercls: Type[Spider], + settings: Union[None, dict, Settings] = None, + init_reactor: bool = False, + ): if isinstance(spidercls, Spider): raise ValueError("The spidercls argument must be a class, not an object") if isinstance(settings, dict) or settings is None: settings = Settings(settings) - self.spidercls = spidercls - self.settings = settings.copy() + self.spidercls: Type[Spider] = spidercls + self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self.signals = SignalManager(self) + self.signals: SignalManager = SignalManager(self) - self.stats = load_object(self.settings["STATS_CLASS"])(self) + self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self) handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) logging.root.addHandler(handler) @@ -80,8 +87,8 @@ class Crawler: self.__remove_handler = lambda: logging.root.removeHandler(handler) self.signals.connect(self.__remove_handler, signals.engine_stopped) - lf_cls = load_object(self.settings["LOG_FORMATTER"]) - self.logformatter = lf_cls.from_crawler(self) + lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) + self.logformatter: LogFormatter = lf_cls.from_crawler(self) self.request_fingerprinter: RequestFingerprinter = create_instance( load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), @@ -89,8 +96,8 @@ class Crawler: crawler=self, ) - reactor_class = self.settings["TWISTED_REACTOR"] - event_loop = self.settings["ASYNCIO_EVENT_LOOP"] + reactor_class: str = self.settings["TWISTED_REACTOR"] + event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] if init_reactor: # this needs to be done after the spider settings are merged, # but before something imports twisted.internet.reactor @@ -104,11 +111,11 @@ class Crawler: if is_asyncio_reactor_installed() and event_loop: verify_installed_asyncio_event_loop(event_loop) - self.extensions = ExtensionManager.from_crawler(self) + self.extensions: ExtensionManager = ExtensionManager.from_crawler(self) self.settings.freeze() - self.crawling = False - self.spider = None + self.crawling: bool = False + self.spider: Optional[Spider] = None self.engine: Optional[ExecutionEngine] = None @defer.inlineCallbacks diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 7f202b8b8..fedd02805 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -39,7 +39,7 @@ class DontCloseSpider(Exception): class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" - def __init__(self, reason="cancelled"): + def __init__(self, reason: str = "cancelled"): super().__init__() self.reason = reason diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 560006c95..7cb379b46 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -1,8 +1,11 @@ import logging import os +from typing import Any, Dict, Optional, Union from twisted.python.failure import Failure +from scrapy import Request, Spider +from scrapy.http import Response from scrapy.utils.request import referer_str SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s" @@ -52,7 +55,7 @@ class LogFormatter: } """ - def crawled(self, request, response, spider): + def crawled(self, request: Request, response: Response, spider: Spider) -> dict: """Logs a message when the crawler finds a webpage.""" request_flags = f" {str(request.flags)}" if request.flags else "" response_flags = f" {str(response.flags)}" if response.flags else "" @@ -70,8 +73,11 @@ class LogFormatter: }, } - def scraped(self, item, response, spider): + def scraped( + self, item: Any, response: Union[Response, Failure], spider: Spider + ) -> dict: """Logs a message when an item is scraped by a spider.""" + src: Any if isinstance(response, Failure): src = response.getErrorMessage() else: @@ -85,7 +91,9 @@ class LogFormatter: }, } - def dropped(self, item, exception, response, spider): + def dropped( + self, item: Any, exception: BaseException, response: Response, spider: Spider + ) -> dict: """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { "level": logging.WARNING, @@ -96,7 +104,9 @@ class LogFormatter: }, } - def item_error(self, item, exception, response, spider): + def item_error( + self, item: Any, exception, response: Response, spider: Spider + ) -> dict: """Logs a message when an item causes an error while it is passing through the item pipeline. @@ -110,7 +120,9 @@ class LogFormatter: }, } - def spider_error(self, failure, request, response, spider): + def spider_error( + self, failure: Failure, request: Request, response: Response, spider: Spider + ) -> dict: """Logs an error message from a spider. .. versionadded:: 2.0 @@ -124,13 +136,19 @@ class LogFormatter: }, } - def download_error(self, failure, request, spider, errmsg=None): + def download_error( + self, + failure: Failure, + request: Request, + spider: Spider, + errmsg: Optional[str] = None, + ) -> dict: """Logs a download error message from a spider (typically coming from the engine). .. versionadded:: 2.0 """ - args = {"request": request} + args: Dict[str, Any] = {"request": request} if errmsg: msg = DOWNLOADERRORMSG_LONG args["errmsg"] = errmsg diff --git a/scrapy/middleware.py b/scrapy/middleware.py index f82d722fa..03e92b565 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, cast +from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, cast from twisted.internet.defer import Deferred @@ -30,7 +30,7 @@ class MiddlewareManager: self._add_middleware(mw) @classmethod - def _get_mwlist_from_settings(cls, settings: Settings) -> list: + def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: raise NotImplementedError @classmethod @@ -67,17 +67,17 @@ class MiddlewareManager: def from_crawler(cls, crawler): return cls.from_settings(crawler.settings, crawler) - def _add_middleware(self, mw) -> None: + def _add_middleware(self, mw: Any) -> None: if hasattr(mw, "open_spider"): self.methods["open_spider"].append(mw.open_spider) if hasattr(mw, "close_spider"): self.methods["close_spider"].appendleft(mw.close_spider) - def _process_parallel(self, methodname: str, obj, *args) -> Deferred: + def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred: methods = cast(Iterable[Callable], self.methods[methodname]) return process_parallel(methods, obj, *args) - def _process_chain(self, methodname: str, obj, *args) -> Deferred: + def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred: methods = cast(Iterable[Callable], self.methods[methodname]) return process_chain(methods, obj, *args) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index df574a0a1..c97d71fb6 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -3,7 +3,11 @@ Item pipeline See documentation in docs/item-pipeline.rst """ +from typing import Any, List +from twisted.internet.defer import Deferred + +from scrapy import Spider from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_f_from_coro_f @@ -13,15 +17,15 @@ class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings) -> List[Any]: return build_component_list(settings.getwithbase("ITEM_PIPELINES")) - def _add_middleware(self, pipe): + def _add_middleware(self, pipe: Any) -> None: super()._add_middleware(pipe) if hasattr(pipe, "process_item"): self.methods["process_item"].append( deferred_f_from_coro_f(pipe.process_item) ) - def process_item(self, item, spider): + def process_item(self, item: Any, spider: Spider) -> Deferred: return self._process_chain("process_item", item, spider) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index d7e3bce91..f6df191d8 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,13 +1,16 @@ +from typing import Any, List, Tuple + from pydispatch import dispatcher +from twisted.internet.defer import Deferred from scrapy.utils import signal as _signal class SignalManager: - def __init__(self, sender=dispatcher.Anonymous): - self.sender = sender + def __init__(self, sender: Any = dispatcher.Anonymous): + self.sender: Any = sender - def connect(self, receiver, signal, **kwargs): + def connect(self, receiver: Any, signal: Any, **kwargs: Any) -> None: """ Connect a receiver function to a signal. @@ -22,18 +25,18 @@ class SignalManager: :type signal: object """ kwargs.setdefault("sender", self.sender) - return dispatcher.connect(receiver, signal, **kwargs) + dispatcher.connect(receiver, signal, **kwargs) - def disconnect(self, receiver, signal, **kwargs): + def disconnect(self, receiver: Any, signal: Any, **kwargs: Any) -> None: """ Disconnect a receiver function from a signal. This has the opposite effect of the :meth:`connect` method, and the arguments are the same. """ kwargs.setdefault("sender", self.sender) - return dispatcher.disconnect(receiver, signal, **kwargs) + dispatcher.disconnect(receiver, signal, **kwargs) - def send_catch_log(self, signal, **kwargs): + def send_catch_log(self, signal: Any, **kwargs: Any) -> List[Tuple[Any, Any]]: """ Send a signal, catch exceptions and log them. @@ -43,7 +46,7 @@ class SignalManager: kwargs.setdefault("sender", self.sender) return _signal.send_catch_log(signal, **kwargs) - def send_catch_log_deferred(self, signal, **kwargs): + def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred: """ Like :meth:`send_catch_log` but supports returning :class:`~twisted.internet.defer.Deferred` objects from signal handlers. @@ -57,7 +60,7 @@ class SignalManager: kwargs.setdefault("sender", self.sender) return _signal.send_catch_log_deferred(signal, **kwargs) - def disconnect_all(self, signal, **kwargs): + def disconnect_all(self, signal: Any, **kwargs: Any) -> None: """ Disconnect all receivers from the given signal. @@ -65,4 +68,4 @@ class SignalManager: :type signal: object """ kwargs.setdefault("sender", self.sender) - return _signal.disconnect_all(signal, **kwargs) + _signal.disconnect_all(signal, **kwargs) diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index dd3c32737..15193aac5 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -3,44 +3,57 @@ Scrapy extension for collecting scraping stats """ import logging import pprint +from typing import TYPE_CHECKING, Any, Dict, Optional + +from scrapy import Spider + +if TYPE_CHECKING: + from scrapy.crawler import Crawler logger = logging.getLogger(__name__) -class StatsCollector: - def __init__(self, crawler): - self._dump = crawler.settings.getbool("STATS_DUMP") - self._stats = {} +StatsT = Dict[str, Any] - def get_value(self, key, default=None, spider=None): + +class StatsCollector: + def __init__(self, crawler: "Crawler"): + self._dump: bool = crawler.settings.getbool("STATS_DUMP") + self._stats: StatsT = {} + + def get_value( + self, key: str, default: Any = None, spider: Optional[Spider] = None + ) -> Any: return self._stats.get(key, default) - def get_stats(self, spider=None): + def get_stats(self, spider: Optional[Spider] = None) -> StatsT: return self._stats - def set_value(self, key, value, spider=None): + def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: self._stats[key] = value - def set_stats(self, stats, spider=None): + def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None: self._stats = stats - def inc_value(self, key, count=1, start=0, spider=None): + def inc_value( + self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None + ) -> None: d = self._stats d[key] = d.setdefault(key, start) + count - def max_value(self, key, value, spider=None): + def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: self._stats[key] = max(self._stats.setdefault(key, value), value) - def min_value(self, key, value, spider=None): + def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: self._stats[key] = min(self._stats.setdefault(key, value), value) - def clear_stats(self, spider=None): + def clear_stats(self, spider: Optional[Spider] = None) -> None: self._stats.clear() - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: pass - def close_spider(self, spider, reason): + def close_spider(self, spider: Spider, reason: str) -> None: if self._dump: logger.info( "Dumping Scrapy stats:\n" + pprint.pformat(self._stats), @@ -48,34 +61,38 @@ class StatsCollector: ) self._persist_stats(self._stats, spider) - def _persist_stats(self, stats, spider): + def _persist_stats(self, stats: StatsT, spider: Spider) -> None: pass class MemoryStatsCollector(StatsCollector): - def __init__(self, crawler): + def __init__(self, crawler: "Crawler"): super().__init__(crawler) - self.spider_stats = {} + self.spider_stats: Dict[str, StatsT] = {} - def _persist_stats(self, stats, spider): + def _persist_stats(self, stats: StatsT, spider: Spider) -> None: self.spider_stats[spider.name] = stats class DummyStatsCollector(StatsCollector): - def get_value(self, key, default=None, spider=None): + def get_value( + self, key: str, default: Any = None, spider: Optional[Spider] = None + ) -> Any: return default - def set_value(self, key, value, spider=None): + def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: pass - def set_stats(self, stats, spider=None): + def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None: pass - def inc_value(self, key, count=1, start=0, spider=None): + def inc_value( + self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None + ) -> None: pass - def max_value(self, key, value, spider=None): + def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: pass - def min_value(self, key, value, spider=None): + def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: pass diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 6ae27dc29..2ce4725f4 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -2,6 +2,7 @@ import logging import sys import warnings from logging.config import dictConfig +from typing import Tuple from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -14,7 +15,7 @@ from scrapy.utils.versions import scrapy_components_versions logger = logging.getLogger(__name__) -def failure_to_exc_info(failure): +def failure_to_exc_info(failure: Failure): """Extract exc_info from Failure instances""" if isinstance(failure, Failure): return (failure.type, failure.value, failure.getTracebackObject()) @@ -206,7 +207,7 @@ class LogCounterHandler(logging.Handler): self.crawler.stats.inc_value(sname) -def logformatter_adapter(logkws): +def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: """ Helper that takes the dictionary output from the methods in LogFormatter and adapts it into a tuple of positional arguments for logger.log calls, diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index f9f9c0d5b..d861c9ab6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -10,6 +10,7 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules +from typing import TYPE_CHECKING, Any, Callable, Union from w3lib.html import replace_entities @@ -18,6 +19,10 @@ from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.python import flatten, to_unicode +if TYPE_CHECKING: + from scrapy import Spider + + _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes @@ -34,7 +39,7 @@ def arg_to_iter(arg): return [arg] -def load_object(path): +def load_object(path: Union[str, Callable]) -> Any: """Load an object given its absolute object path, and return it. The object can be the import path of a class, function, variable or an @@ -249,7 +254,7 @@ def is_generator_with_return_value(callable): return _generator_callbacks_cache[callable] -def warn_on_generator_with_return_value(spider, callable): +def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None: """ Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 7f67d036a..f1b9239e6 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,9 +1,11 @@ import asyncio import sys from contextlib import suppress +from typing import Any, Callable, Dict, Optional, Sequence from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error +from twisted.internet.base import DelayedCall from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object @@ -34,23 +36,23 @@ class CallLaterOnce: it hasn't been already scheduled since the last time it ran. """ - def __init__(self, func, *a, **kw): - self._func = func - self._a = a - self._kw = kw - self._call = None + def __init__(self, func: Callable, *a: Any, **kw: Any): + self._func: Callable = func + self._a: Sequence[Any] = a + self._kw: Dict[str, Any] = kw + self._call: Optional[DelayedCall] = None - def schedule(self, delay=0): + def schedule(self, delay: float = 0) -> None: from twisted.internet import reactor if self._call is None: self._call = reactor.callLater(delay, self) - def cancel(self): + def cancel(self) -> None: if self._call: self._call.cancel() - def __call__(self): + def __call__(self) -> Any: self._call = None return self._func(*self._a, **self._kw) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index b95786d35..9e7ddd827 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,6 +1,8 @@ """Helper functions for working with signals""" import collections.abc import logging +from typing import Any as TypingAny +from typing import List, Tuple from pydispatch.dispatcher import ( Anonymous, @@ -20,7 +22,9 @@ from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) -def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): +def send_catch_log( + signal=Any, sender=Anonymous, *arguments, **named +) -> List[Tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. """ @@ -32,8 +36,9 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): ) dont_log += (StopDownload,) spider = named.get("spider", None) - responses = [] + responses: List[Tuple[TypingAny, TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): + result: TypingAny try: response = robustApply( receiver, signal=signal, sender=sender, *arguments, **named diff --git a/tox.ini b/tox.ini index 5a9d9cf29..d96a278ea 100644 --- a/tox.ini +++ b/tox.ini @@ -37,13 +37,13 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.0.1 + mypy==1.2.0 types-attrs==19.1.0 - types-lxml==2023.2.11 - types-Pillow==9.4.0.16 - types-Pygments==2.14.0.5 - types-pyOpenSSL==23.0.0.4 - types-setuptools==67.4.0.1 + types-lxml==2023.3.28 + types-Pillow==9.4.0.19 + types-Pygments==2.14.0.7 + types-pyOpenSSL==23.1.0.1 + types-setuptools==67.6.0.7 commands = mypy --show-error-codes {posargs: scrapy tests} From d47c732ae9ebda84c689563048919923ddb17383 Mon Sep 17 00:00:00 2001 From: Kartik Kumar <130273246+heppymxm@users.noreply.github.com> Date: Tue, 11 Apr 2023 21:55:42 +0530 Subject: [PATCH 0919/2083] base64-decode GCS checksums (#5891) --- scrapy/pipelines/files.py | 3 ++- tests/test_pipeline_files.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 6e9f661e5..4b594ccb7 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -3,6 +3,7 @@ Files Pipeline See documentation in topics/media-pipeline.rst """ +import base64 import functools import hashlib import logging @@ -228,7 +229,7 @@ class GCSFilesStore: def stat_file(self, path, info): def _onsuccess(blob): if blob: - checksum = blob.md5_hash + checksum = base64.b64decode(blob.md5_hash).hex() last_modified = time.mktime(blob.updated.timetuple()) return {"checksum": checksum, "last_modified": last_modified} return {} diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 9701e5d4e..c80666586 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -601,7 +601,7 @@ class TestGCSFilesStore(unittest.TestCase): s = yield store.stat_file(path, info=None) self.assertIn("last_modified", s) self.assertIn("checksum", s) - self.assertEqual(s["checksum"], "zc2oVgXkbQr2EQdSdw3OPA==") + self.assertEqual(s["checksum"], "cdcda85605e46d0af6110752770dce3c") u = urlparse(uri) content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path) self.assertEqual(content, data) From 3f0c2fae5e18c448bd1791920500c976d44fc321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Apr 2023 09:28:28 +0200 Subject: [PATCH 0920/2083] Skip test_skip_bad_links based on the w3lib version --- scrapy/linkextractors/lxmlhtml.py | 6 +++--- tests/test_linkextractors.py | 16 +++++++++++----- tox.ini | 1 + 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 0d1b76531..23cbd0116 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,8 +1,8 @@ """ Link extractor based on lxml.html """ -import operator import logging +import operator from functools import partial from urllib.parse import urljoin, urlparse @@ -94,8 +94,8 @@ class LxmlParserLinkExtractor: try: url = safe_url_string(url, encoding=response_encoding) except ValueError: - logger.debug(f"Skipping extraction of bad link {url}") - continue # Disregard badly formatted urls + logger.debug(f"Skipping extraction of link with bad URL {url!r}") + continue # to fix relative links after process_value url = urljoin(response_url, url) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 784fdb658..e1ec19601 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,7 +1,10 @@ import pickle import re import unittest -import sys + +from packaging.version import Version +from pytest import mark +from w3lib import __version__ as w3lib_version from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link @@ -817,13 +820,16 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): def test_restrict_xpaths_with_html_entities(self): super().test_restrict_xpaths_with_html_entities() - @unittest.skipIf( - sys.version_info < (3, 8), - reason="some library for python 3.7 so is less strict so bad links like htis don't crash scrapy", + @mark.skipif( + Version(w3lib_version) < Version("2.0.0"), + reason=( + "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain " + "about an invalid port value." + ), ) def test_skip_bad_links(self): html = b""" - Why would you do this? + Why would you do this? Good Link Good Link 2 """ diff --git a/tox.ini b/tox.ini index 5a9d9cf29..873e7662b 100644 --- a/tox.ini +++ b/tox.ini @@ -101,6 +101,7 @@ install_command = python -I -m pip install {opts} {packages} [testenv:pinned] +basepython = python3.7 deps = {[pinned]deps} PyDispatcher==2.0.5 From c2a31974ffc06412a5ab8d87fe070c98cd9c856b Mon Sep 17 00:00:00 2001 From: Serhii A Date: Thu, 13 Apr 2023 12:44:20 +0300 Subject: [PATCH 0921/2083] Improve get_func_args (#5885) --- scrapy/utils/python.py | 52 +++++++++++++++++++------------------- tests/test_utils_python.py | 10 +++----- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index fc50e0f12..818fa5d6b 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -174,33 +174,33 @@ def binary_is_text(data): def get_func_args(func, stripself=False): - """Return the argument name list of a callable""" - if inspect.isfunction(func): - spec = inspect.getfullargspec(func) - func_args = spec.args + spec.kwonlyargs - elif inspect.isclass(func): - return get_func_args(func.__init__, True) - elif inspect.ismethod(func): - return get_func_args(func.__func__, True) - elif inspect.ismethoddescriptor(func): - return [] - elif isinstance(func, partial): - return [ - x - for x in get_func_args(func.func)[len(func.args) :] - if not (func.keywords and x in func.keywords) - ] - elif hasattr(func, "__call__"): - if inspect.isroutine(func): - return [] - if getattr(func, "__name__", None) == "__call__": - return [] - return get_func_args(func.__call__, True) + """Return the argument name list of a callable object""" + if not callable(func): + raise TypeError(f"func must be callable, got '{type(func).__name__}'") + + args = [] + try: + sig = inspect.signature(func) + except ValueError: + return args + + if isinstance(func, partial): + partial_args = func.args + partial_kw = func.keywords + + for name, param in sig.parameters.items(): + if param.name in partial_args: + continue + if partial_kw and param.name in partial_kw: + continue + args.append(name) else: - raise TypeError(f"{type(func)} is not callable") - if stripself: - func_args.pop(0) - return func_args + for name in sig.parameters.keys(): + args.append(name) + + if stripself and args and args[0] == "self": + args = args[1:] + return args def get_spec(func): diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 57f40c2e5..80d2e8da1 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -235,20 +235,16 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(partial_f3), ["c"]) self.assertEqual(get_func_args(cal), ["a", "b", "c"]) self.assertEqual(get_func_args(object), []) + self.assertEqual(get_func_args(str.split, stripself=True), ["sep", "maxsplit"]) + self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"]) if platform.python_implementation() == "CPython": - # TODO: how do we fix this to return the actual argument names? - self.assertEqual(get_func_args(str.split), []) - self.assertEqual(get_func_args(" ".join), []) + # doesn't work on CPython: https://bugs.python.org/issue42785 self.assertEqual(get_func_args(operator.itemgetter(2)), []) elif platform.python_implementation() == "PyPy": - self.assertEqual( - get_func_args(str.split, stripself=True), ["sep", "maxsplit"] - ) self.assertEqual( get_func_args(operator.itemgetter(2), stripself=True), ["obj"] ) - self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"]) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) From 441ac196e4151765fa424af59f3938e72b8434c1 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <95530227+guillermo-bondonno@users.noreply.github.com> Date: Thu, 13 Apr 2023 12:46:59 -0300 Subject: [PATCH 0922/2083] Implement a request_to_curl function (#5892) --- scrapy/utils/request.py | 31 ++++++++++++++++++ tests/test_utils_request.py | 64 +++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 409ca2e52..6d8be991d 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -327,3 +327,34 @@ def _get_method(obj, name): return getattr(obj, name) except AttributeError: raise ValueError(f"Method {name!r} not found in: {obj}") + + +def request_to_curl(request: Request) -> str: + """ + Converts a :class:`~scrapy.Request` object to a curl command. + + :param :class:`~scrapy.Request`: Request object to be converted + :return: string containing the curl command + """ + method = request.method + + data = f"--data-raw '{request.body.decode('utf-8')}'" if request.body else "" + + headers = " ".join( + f"-H '{k.decode()}: {v[0].decode()}'" for k, v in request.headers.items() + ) + + url = request.url + cookies = "" + if request.cookies: + if isinstance(request.cookies, dict): + cookie = "; ".join(f"{k}={v}" for k, v in request.cookies.items()) + cookies = f"--cookie '{cookie}'" + elif isinstance(request.cookies, list): + cookie = "; ".join( + f"{list(c.keys())[0]}={list(c.values())[0]}" for c in request.cookies + ) + cookies = f"--cookie '{cookie}'" + + curl_cmd = f"curl -X {method} {url} {data} {headers} {cookies}".strip() + return " ".join(curl_cmd.split()) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 6ca272de1..e6d1abe3f 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,3 +1,4 @@ +import json import unittest import warnings from hashlib import sha1 @@ -18,6 +19,7 @@ from scrapy.utils.request import ( request_authenticate, request_fingerprint, request_httprepr, + request_to_curl, ) from scrapy.utils.test import get_crawler @@ -666,5 +668,67 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): self.assertEqual(fingerprint, settings["FINGERPRINT"]) +class RequestToCurlTest(unittest.TestCase): + def _test_request(self, request_object, expected_curl_command): + curl_command = request_to_curl(request_object) + self.assertEqual(curl_command, expected_curl_command) + + def test_get(self): + request_object = Request("https://www.example.com") + expected_curl_command = "curl -X GET https://www.example.com" + self._test_request(request_object, expected_curl_command) + + def test_post(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + 'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\'' + ) + self._test_request(request_object, expected_curl_command) + + def test_headers(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + headers={"Content-Type": "application/json", "Accept": "application/json"}, + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + ' --data-raw \'{"foo": "bar"}\'' + " -H 'Content-Type: application/json' -H 'Accept: application/json'" + ) + self._test_request(request_object, expected_curl_command) + + def test_cookies_dict(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + cookies={"foo": "bar"}, + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" + ) + self._test_request(request_object, expected_curl_command) + + def test_cookies_list(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + cookies=[{"foo": "bar"}], + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" + ) + self._test_request(request_object, expected_curl_command) + + if __name__ == "__main__": unittest.main() From e1f66620ec7341c55f3eb7f44088224b5f68c1ad Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Apr 2023 18:13:21 +0400 Subject: [PATCH 0923/2083] Fix typo on tutorial.rst (#5893) (#5895) Co-authored-by: Seth Herr --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 064ce05f8..04d73d95a 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -329,7 +329,7 @@ the :meth:`~scrapy.selector.SelectorList.re` method to extract using >>> response.css("title::text").re(r"(\w+) to (\w+)") ['Quotes', 'Scrape'] -In order to find the proper CSS selectors to use, you might find useful opening +In order to find the proper CSS selectors to use, you might find it useful to open the response page from the shell in your web browser using ``view(response)``. You can use your browser's developer tools to inspect the HTML and come up with a selector (see :ref:`topics-developer-tools`). From 02f3e8d413ccdd6a3f0b5828a9cd94e1fb3662b1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Apr 2023 10:37:52 +0400 Subject: [PATCH 0924/2083] Typing for scrapy/core/downloader (#5896) --- scrapy/core/downloader/__init__.py | 95 ++++++++++++--------- scrapy/core/downloader/contextfactory.py | 56 +++++++----- scrapy/core/downloader/handlers/__init__.py | 31 ++++--- scrapy/core/downloader/middleware.py | 29 ++++--- scrapy/core/downloader/tls.py | 26 +++--- scrapy/core/downloader/webclient.py | 41 +++++---- scrapy/utils/python.py | 36 ++++++-- scrapy/utils/ssl.py | 22 +++-- 8 files changed, 211 insertions(+), 125 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index dde76a547..426056dc8 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -2,45 +2,52 @@ import random from collections import deque from datetime import datetime from time import time +from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast -from twisted.internet import defer, task +from twisted.internet import task from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.http import Response from scrapy.resolver import dnscache +from scrapy.settings import BaseSettings +from scrapy.signalmanager import SignalManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached +if TYPE_CHECKING: + from scrapy.crawler import Crawler + class Slot: """Downloader slot""" - def __init__(self, concurrency, delay, randomize_delay): - self.concurrency = concurrency - self.delay = delay - self.randomize_delay = randomize_delay + def __init__(self, concurrency: int, delay: float, randomize_delay: bool): + self.concurrency: int = concurrency + self.delay: float = delay + self.randomize_delay: bool = randomize_delay - self.active = set() - self.queue = deque() - self.transferring = set() - self.lastseen = 0 + self.active: Set[Request] = set() + self.queue: Deque[Tuple[Request, Deferred]] = deque() + self.transferring: Set[Request] = set() + self.lastseen: float = 0 self.latercall = None - def free_transfer_slots(self): + def free_transfer_slots(self) -> int: return self.concurrency - len(self.transferring) - def download_delay(self): + def download_delay(self) -> float: if self.randomize_delay: return random.uniform(0.5 * self.delay, 1.5 * self.delay) return self.delay - def close(self): + def close(self) -> None: if self.latercall and self.latercall.active(): self.latercall.cancel() - def __repr__(self): + def __repr__(self) -> str: cls_name = self.__class__.__name__ return ( f"{cls_name}(concurrency={self.concurrency!r}, " @@ -48,7 +55,7 @@ class Slot: f"randomize_delay={self.randomize_delay!r})" ) - def __str__(self): + def __str__(self) -> str: return ( f" Tuple[int, float]: + delay: float = settings.getfloat("DOWNLOAD_DELAY") if hasattr(spider, "download_delay"): delay = spider.download_delay @@ -72,23 +81,29 @@ def _get_concurrency_delay(concurrency, spider, settings): class Downloader: DOWNLOAD_SLOT = "download_slot" - def __init__(self, crawler): - self.settings = crawler.settings - self.signals = crawler.signals - self.slots = {} - self.active = set() - self.handlers = DownloadHandlers(crawler) - self.total_concurrency = self.settings.getint("CONCURRENT_REQUESTS") - self.domain_concurrency = self.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") - self.ip_concurrency = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") - self.randomize_delay = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") - self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) - self._slot_gc_loop = task.LoopingCall(self._slot_gc) + def __init__(self, crawler: "Crawler"): + self.settings: BaseSettings = crawler.settings + self.signals: SignalManager = crawler.signals + self.slots: Dict[str, Slot] = {} + self.active: Set[Request] = set() + self.handlers: DownloadHandlers = DownloadHandlers(crawler) + self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS") + self.domain_concurrency: int = self.settings.getint( + "CONCURRENT_REQUESTS_PER_DOMAIN" + ) + self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") + self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") + self.middleware: DownloaderMiddlewareManager = ( + DownloaderMiddlewareManager.from_crawler(crawler) + ) + self._slot_gc_loop: task.LoopingCall = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) - self.per_slot_settings = self.settings.getdict("DOWNLOAD_SLOTS", {}) + self.per_slot_settings: Dict[str, Dict[str, Any]] = self.settings.getdict( + "DOWNLOAD_SLOTS", {} + ) def fetch(self, request: Request, spider: Spider) -> Deferred: - def _deactivate(response): + def _deactivate(response: Response) -> Response: self.active.remove(request) return response @@ -99,7 +114,7 @@ class Downloader: def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency - def _get_slot(self, request, spider): + def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]: key = self._get_slot_key(request, spider) if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) @@ -117,9 +132,9 @@ class Downloader: return key, self.slots[key] - def _get_slot_key(self, request, spider): + def _get_slot_key(self, request: Request, spider: Spider) -> str: if self.DOWNLOAD_SLOT in request.meta: - return request.meta[self.DOWNLOAD_SLOT] + return cast(str, request.meta[self.DOWNLOAD_SLOT]) key = urlparse_cached(request).hostname or "" if self.ip_concurrency: @@ -127,11 +142,11 @@ class Downloader: return key - def _enqueue_request(self, request, spider): + def _enqueue_request(self, request: Request, spider: Spider) -> Deferred: key, slot = self._get_slot(request, spider) request.meta[self.DOWNLOAD_SLOT] = key - def _deactivate(response): + def _deactivate(response: Response) -> Response: slot.active.remove(request) return response @@ -139,12 +154,12 @@ class Downloader: self.signals.send_catch_log( signal=signals.request_reached_downloader, request=request, spider=spider ) - deferred = defer.Deferred().addBoth(_deactivate) + deferred = Deferred().addBoth(_deactivate) slot.queue.append((request, deferred)) self._process_queue(spider, slot) return deferred - def _process_queue(self, spider, slot): + def _process_queue(self, spider: Spider, slot: Slot) -> None: from twisted.internet import reactor if slot.latercall and slot.latercall.active(): @@ -172,7 +187,7 @@ class Downloader: self._process_queue(spider, slot) break - def _download(self, slot, request, spider): + def _download(self, slot: Slot, request: Request, spider: Spider) -> Deferred: # The order is very important for the following deferreds. Do not change! # 1. Create the download deferred @@ -180,7 +195,7 @@ class Downloader: # 2. Notify response_downloaded listeners about the recent download # before querying queue for next request - def _downloaded(response): + def _downloaded(response: Response) -> Response: self.signals.send_catch_log( signal=signals.response_downloaded, response=response, @@ -197,7 +212,7 @@ class Downloader: # middleware itself) slot.transferring.add(request) - def finish_transferring(_): + def finish_transferring(_: Any) -> Any: slot.transferring.remove(request) self._process_queue(spider, slot) self.signals.send_catch_log( diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 53ae78918..909cc273f 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,4 +1,5 @@ import warnings +from typing import TYPE_CHECKING, Any, List, Optional from OpenSSL import SSL from twisted.internet._sslverify import _setAcceptableProtocols @@ -18,8 +19,12 @@ from scrapy.core.downloader.tls import ( ScrapyClientTLSOptions, openssl_methods, ) +from scrapy.settings import BaseSettings from scrapy.utils.misc import create_instance, load_object +if TYPE_CHECKING: + from twisted.internet._sslverify import ClientTLSOptions + @implementer(IPolicyForHTTPS) class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): @@ -35,25 +40,34 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): def __init__( self, - method=SSL.SSLv23_METHOD, - tls_verbose_logging=False, - tls_ciphers=None, - *args, - **kwargs, + method: int = SSL.SSLv23_METHOD, + tls_verbose_logging: bool = False, + tls_ciphers: Optional[str] = None, + *args: Any, + **kwargs: Any, ): super().__init__(*args, **kwargs) - self._ssl_method = method - self.tls_verbose_logging = tls_verbose_logging + self._ssl_method: int = method + self.tls_verbose_logging: bool = tls_verbose_logging + self.tls_ciphers: AcceptableCiphers if tls_ciphers: self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers) else: self.tls_ciphers = DEFAULT_CIPHERS @classmethod - def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs): - tls_verbose_logging = settings.getbool("DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING") - tls_ciphers = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] - return cls( + def from_settings( + cls, + settings: BaseSettings, + method: int = SSL.SSLv23_METHOD, + *args: Any, + **kwargs: Any, + ): + tls_verbose_logging: bool = settings.getbool( + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" + ) + tls_ciphers: Optional[str] = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + return cls( # type: ignore[misc] method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, @@ -61,7 +75,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): **kwargs, ) - def getCertificateOptions(self): + def getCertificateOptions(self) -> CertificateOptions: # setting verify=True will require you to provide CAs # to verify against; in other words: it's not that simple @@ -82,12 +96,12 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() - def getContext(self, hostname=None, port=None): + def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: ctx = self.getCertificateOptions().getContext() ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx - def creatorForNetloc(self, hostname, port): + def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": return ScrapyClientTLSOptions( hostname.decode("ascii"), self.getContext(), @@ -114,7 +128,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ - def creatorForNetloc(self, hostname, port): + def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": # trustRoot set to platformTrust() will use the platform's root CAs. # # This means that a website like https://www.cacert.org will be rejected @@ -133,13 +147,15 @@ class AcceptableProtocolsContextFactory: negotiation. """ - def __init__(self, context_factory, acceptable_protocols): + def __init__(self, context_factory: Any, acceptable_protocols: List[bytes]): verifyObject(IPolicyForHTTPS, context_factory) - self._wrapped_context_factory = context_factory - self._acceptable_protocols = acceptable_protocols + self._wrapped_context_factory: Any = context_factory + self._acceptable_protocols: List[bytes] = acceptable_protocols - def creatorForNetloc(self, hostname, port): - options = self._wrapped_context_factory.creatorForNetloc(hostname, port) + def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": + options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc( + hostname, port + ) _setAcceptableProtocols(options._ctx, self._acceptable_protocols) return options diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 39155efe9..6a211aafa 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,25 +1,32 @@ """Download handlers for different schemes""" import logging +from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast from twisted.internet import defer +from twisted.internet.defer import Deferred -from scrapy import signals +from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, NotSupported from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) class DownloadHandlers: - def __init__(self, crawler): - self._crawler = crawler - self._schemes = {} # stores acceptable schemes on instancing - self._handlers = {} # stores instanced handlers for schemes - self._notconfigured = {} # remembers failed handlers - handlers = without_none_values( + def __init__(self, crawler: "Crawler"): + self._crawler: "Crawler" = crawler + self._schemes: Dict[ + str, Union[str, Callable] + ] = {} # stores acceptable schemes on instancing + self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes + self._notconfigured: Dict[str, str] = {} # remembers failed handlers + handlers: Dict[str, Union[str, Callable]] = without_none_values( crawler.settings.getwithbase("DOWNLOAD_HANDLERS") ) for scheme, clspath in handlers.items(): @@ -28,7 +35,7 @@ class DownloadHandlers: crawler.signals.connect(self._close, signals.engine_stopped) - def _get_handler(self, scheme): + def _get_handler(self, scheme: str) -> Any: """Lazy-load the downloadhandler for a scheme only on the first request for that scheme. """ @@ -42,7 +49,7 @@ class DownloadHandlers: return self._load_handler(scheme) - def _load_handler(self, scheme, skip_lazy=False): + def _load_handler(self, scheme: str, skip_lazy: bool = False) -> Any: path = self._schemes[scheme] try: dhcls = load_object(path) @@ -69,17 +76,17 @@ class DownloadHandlers: self._handlers[scheme] = dh return dh - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: scheme = urlparse_cached(request).scheme handler = self._get_handler(scheme) if not handler: raise NotSupported( f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}" ) - return handler.download_request(request, spider) + return cast(Deferred, handler.download_request(request, spider)) @defer.inlineCallbacks - def _close(self, *_a, **_kw): + def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred, Any, None]: for dh in self._handlers.values(): if hasattr(dh, "close"): yield dh.close() diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 56df48b2e..dca13c01e 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,15 +3,16 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from typing import Callable, Union, cast +from typing import Any, Callable, Generator, List, Union, cast -from twisted.internet import defer +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager +from scrapy.settings import BaseSettings from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_from_coro, mustbe_deferred @@ -20,10 +21,10 @@ class DownloaderMiddlewareManager(MiddlewareManager): component_name = "downloader middleware" @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]: return build_component_list(settings.getwithbase("DOWNLOADER_MIDDLEWARES")) - def _add_middleware(self, mw): + def _add_middleware(self, mw: Any) -> None: if hasattr(mw, "process_request"): self.methods["process_request"].append(mw.process_request) if hasattr(mw, "process_response"): @@ -31,9 +32,11 @@ class DownloaderMiddlewareManager(MiddlewareManager): if hasattr(mw, "process_exception"): self.methods["process_exception"].appendleft(mw.process_exception) - def download(self, download_func: Callable, request: Request, spider: Spider): - @defer.inlineCallbacks - def process_request(request: Request): + def download( + self, download_func: Callable, request: Request, spider: Spider + ) -> Deferred: + @inlineCallbacks + def process_request(request: Request) -> Generator[Deferred, Any, Any]: for method in self.methods["process_request"]: method = cast(Callable, method) response = yield deferred_from_coro( @@ -50,8 +53,10 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response return (yield download_func(request=request, spider=spider)) - @defer.inlineCallbacks - def process_response(response: Union[Response, Request]): + @inlineCallbacks + def process_response( + response: Union[Response, Request] + ) -> Generator[Deferred, Any, Union[Response, Request]]: if response is None: raise TypeError("Received None in process_response") elif isinstance(response, Request): @@ -71,8 +76,10 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response return response - @defer.inlineCallbacks - def process_exception(failure: Failure): + @inlineCallbacks + def process_exception( + failure: Failure, + ) -> Generator[Deferred, Any, Union[Failure, Response, Request]]: exception = failure.value for method in self.methods["process_exception"]: method = cast(Callable, method) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 025575fe1..33cea7263 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -1,4 +1,5 @@ import logging +from typing import Any, Dict from OpenSSL import SSL from service_identity.exceptions import CertificateError @@ -20,7 +21,7 @@ METHOD_TLSv11 = "TLSv1.1" METHOD_TLSv12 = "TLSv1.2" -openssl_methods = { +openssl_methods: Dict[str, int] = { METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only @@ -39,11 +40,13 @@ class ScrapyClientTLSOptions(ClientTLSOptions): logging warnings. Also, HTTPS connection parameters logging is added. """ - def __init__(self, hostname, ctx, verbose_logging=False): + def __init__(self, hostname: str, ctx: SSL.Context, verbose_logging: bool = False): super().__init__(hostname, ctx) - self.verbose_logging = verbose_logging + self.verbose_logging: bool = verbose_logging - def _identityVerifyingInfoCallback(self, connection, where, ret): + def _identityVerifyingInfoCallback( + self, connection: SSL.Connection, where: int, ret: Any + ) -> None: if where & SSL.SSL_CB_HANDSHAKE_START: connection.set_tlsext_host_name(self._hostnameBytes) elif where & SSL.SSL_CB_HANDSHAKE_DONE: @@ -55,11 +58,12 @@ class ScrapyClientTLSOptions(ClientTLSOptions): connection.get_cipher_name(), ) server_cert = connection.get_peer_certificate() - logger.debug( - 'SSL connection certificate: issuer "%s", subject "%s"', - x509name_to_string(server_cert.get_issuer()), - x509name_to_string(server_cert.get_subject()), - ) + if server_cert: + logger.debug( + 'SSL connection certificate: issuer "%s", subject "%s"', + x509name_to_string(server_cert.get_issuer()), + x509name_to_string(server_cert.get_subject()), + ) key_info = get_temp_key_info(connection._ssl) if key_info: logger.debug("SSL temp key: %s", key_info) @@ -82,4 +86,6 @@ class ScrapyClientTLSOptions(ClientTLSOptions): ) -DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString("DEFAULT") +DEFAULT_CIPHERS: AcceptableCiphers = AcceptableCiphers.fromOpenSSLCipherString( + "DEFAULT" +) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 3d103652b..bb1f73805 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,22 +1,25 @@ import re from time import time -from urllib.parse import urldefrag, urlparse, urlunparse +from typing import Optional, Tuple +from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from twisted.internet import defer from twisted.internet.protocol import ClientFactory from twisted.web.http import HTTPClient +from scrapy import Request from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode -def _parsed_url_args(parsed): +def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]: # Assume parsed is urlparse-d from Request.url, # which was passed via safe_url_string and is ascii-only. - path = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, "")) - path = to_bytes(path, encoding="ascii") + path_str = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, "")) + path = to_bytes(path_str, encoding="ascii") + assert parsed.hostname is not None host = to_bytes(parsed.hostname, encoding="ascii") port = parsed.port scheme = to_bytes(parsed.scheme, encoding="ascii") @@ -26,7 +29,7 @@ def _parsed_url_args(parsed): return scheme, netloc, host, port, path -def _parse(url): +def _parse(url: str) -> Tuple[bytes, bytes, bytes, int, bytes]: """Return tuple of (scheme, netloc, host, port, path), all in bytes except for port which is int. Assume url is from Request.url, which was passed via safe_url_string @@ -132,17 +135,19 @@ class ScrapyHTTPClientFactory(ClientFactory): self.scheme, _, self.host, self.port, _ = _parse(proxy) self.path = self.url - def __init__(self, request, timeout=180): - self._url = urldefrag(request.url)[0] + def __init__(self, request: Request, timeout: float = 180): + self._url: str = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface - self.url = to_bytes(self._url, encoding="ascii") - self.method = to_bytes(request.method, encoding="ascii") - self.body = request.body or None - self.headers = Headers(request.headers) - self.response_headers = None - self.timeout = request.meta.get("download_timeout") or timeout - self.start_time = time() - self.deferred = defer.Deferred().addCallback(self._build_response, request) + self.url: bytes = to_bytes(self._url, encoding="ascii") + self.method: bytes = to_bytes(request.method, encoding="ascii") + self.body: Optional[bytes] = request.body or None + self.headers: Headers = Headers(request.headers) + self.response_headers: Optional[Headers] = None + self.timeout: float = request.meta.get("download_timeout") or timeout + self.start_time: float = time() + self.deferred: defer.Deferred = defer.Deferred().addCallback( + self._build_response, request + ) # Fixes Twisted 11.1.0+ support as HTTPClientFactory is expected # to have _disconnectedDeferred. See Twisted r32329. @@ -150,7 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory): # needed to add the callback _waitForDisconnect. # Specifically this avoids the AttributeError exception when # clientConnectionFailed method is called. - self._disconnectedDeferred = defer.Deferred() + self._disconnectedDeferred: defer.Deferred = defer.Deferred() self._set_connection_attributes(request) @@ -166,8 +171,8 @@ class ScrapyHTTPClientFactory(ClientFactory): elif self.method == b"POST": self.headers["Content-Length"] = 0 - def __repr__(self): - return f"<{self.__class__.__name__}: {self.url}>" + def __repr__(self) -> str: + return f"<{self.__class__.__name__}: {self._url}>" def _cancelTimeout(self, result, timeoutCall): if timeoutCall.active(): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 818fa5d6b..27816c0df 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,7 +8,16 @@ import sys import weakref from functools import partial, wraps from itertools import chain -from typing import Any, AsyncGenerator, AsyncIterable, Iterable, Union +from typing import ( + Any, + AsyncGenerator, + AsyncIterable, + Iterable, + Mapping, + Optional, + Union, + overload, +) from scrapy.utils.asyncgen import as_async_generator @@ -82,7 +91,9 @@ def unique(list_, key=lambda x: x): return result -def to_unicode(text, encoding=None, errors="strict"): +def to_unicode( + text: Union[str, bytes], encoding: Optional[str] = None, errors: str = "strict" +) -> str: """Return the unicode representation of a bytes object ``text``. If ``text`` is already an unicode object, return it as-is.""" if isinstance(text, str): @@ -97,7 +108,9 @@ def to_unicode(text, encoding=None, errors="strict"): return text.decode(encoding, errors) -def to_bytes(text, encoding=None, errors="strict"): +def to_bytes( + text: Union[str, bytes], encoding: Optional[str] = None, errors: str = "strict" +) -> bytes: """Return the binary representation of ``text``. If ``text`` is already a bytes object, return it as-is.""" if isinstance(text, bytes): @@ -160,11 +173,12 @@ def memoizemethod_noargs(method): return new_method -_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} -_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} +_BINARYCHARS = { + i for i in range(32) if to_bytes(chr(i)) not in {b"\0", b"\t", b"\n", b"\r"} +} -def binary_is_text(data): +def binary_is_text(data: bytes) -> bool: """Returns ``True`` if the given ``data`` argument (a ``bytes`` object) does not contain unprintable control characters. """ @@ -258,6 +272,16 @@ def equal_attributes(obj1, obj2, attributes): return True +@overload +def without_none_values(iterable: Mapping) -> dict: + ... + + +@overload +def without_none_values(iterable: Iterable) -> Iterable: + ... + + def without_none_values(iterable): """Return a copy of ``iterable`` with all ``None`` entries removed. diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 3ddceea35..03ae4ba9e 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,24 +1,28 @@ +from typing import Any, Optional, cast + import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL +import OpenSSL.version +from OpenSSL.crypto import X509Name from scrapy.utils.python import to_unicode -def ffi_buf_to_string(buf): +def ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) -def x509name_to_string(x509name): +def x509name_to_string(x509name: X509Name) -> str: # from OpenSSL.crypto.X509Name.__repr__ - result_buffer = pyOpenSSLutil.ffi.new("char[]", 512) + result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512) pyOpenSSLutil.lib.X509_NAME_oneline( - x509name._name, result_buffer, len(result_buffer) + x509name._name, result_buffer, len(result_buffer) # type: ignore[attr-defined] ) return ffi_buf_to_string(result_buffer) -def get_temp_key_info(ssl_object): +def get_temp_key_info(ssl_object: Any) -> Optional[str]: # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"): # removed in cryptography 40.0.0 @@ -53,8 +57,10 @@ def get_temp_key_info(ssl_object): return ", ".join(key_info) -def get_openssl_version(): - system_openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION).decode( - "ascii", errors="replace" +def get_openssl_version() -> str: + # https://github.com/python/typeshed/issues/10024 + system_openssl_bytes = cast( + bytes, OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) ) + system_openssl = system_openssl_bytes.decode("ascii", errors="replace") return f"{OpenSSL.version.__version__} ({system_openssl})" From f5447f3b4ca2c91a07bfdd5acad9661061b8bbd7 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 18 Apr 2023 21:31:51 -0500 Subject: [PATCH 0925/2083] fix: Request.from_curl() with prefixed string literals --- scrapy/utils/curl.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index a2243ae2e..5e095f933 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,4 +1,5 @@ import argparse +import re import warnings from http.cookies import SimpleCookie from shlex import split @@ -7,6 +8,15 @@ from urllib.parse import urlparse from w3lib.http import basic_auth_header +class DataAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + value = str(values).encode("utf-8").decode("utf-8") + if items := re.findall(r"{.+}", value): + value = items[0] + + setattr(namespace, self.dest, value) + + class CurlParser(argparse.ArgumentParser): def error(self, message): error_msg = f"There was an error parsing the curl command: {message}" @@ -17,7 +27,7 @@ curl_parser = CurlParser() curl_parser.add_argument("url") curl_parser.add_argument("-H", "--header", dest="headers", action="append") curl_parser.add_argument("-X", "--request", dest="method") -curl_parser.add_argument("-d", "--data", "--data-raw", dest="data") +curl_parser.add_argument("-d", "--data", "--data-raw", dest="data", action=DataAction) curl_parser.add_argument("-u", "--user", dest="auth") From 88c58a8c9ca6f8149779bfc92d00499a613e5fb3 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 18 Apr 2023 21:49:05 -0500 Subject: [PATCH 0926/2083] feat: added test_post_data_raw_with_string_prefix --- tests/test_utils_curl.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index fd4612eba..1816db29b 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -154,6 +154,15 @@ class CurlToRequestKwargsTest(unittest.TestCase): } self._test_command(curl_command, expected_result) + def test_post_data_raw_with_string_prefix(self): + curl_command = "curl 'https://www.example.org/' --data-raw $'{\"$filters\":\"Filter\u0021\"}'" + expected_result = { + "method": "POST", + "url": "https://www.example.org/", + "body": '{"$filters":"Filter!"}', + } + self._test_command(curl_command, expected_result) + def test_explicit_get_with_data(self): curl_command = "curl httpbin.org/anything -X GET --data asdf" expected_result = { From 69f96b9e96b1399f68cc61db0fe2d2b6cff5484d Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 18 Apr 2023 22:04:34 -0500 Subject: [PATCH 0927/2083] fix: regex --- scrapy/utils/curl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 5e095f933..b873d2bdf 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -11,7 +11,7 @@ from w3lib.http import basic_auth_header class DataAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): value = str(values).encode("utf-8").decode("utf-8") - if items := re.findall(r"{.+}", value): + if items := re.findall(r"\$(.+)", value): value = items[0] setattr(namespace, self.dest, value) From 3209eac14f430f9cba522c12a615111bcabaecd5 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 18 Apr 2023 22:35:15 -0500 Subject: [PATCH 0928/2083] fix: checks --- scrapy/utils/curl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index b873d2bdf..ecfa292ea 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -11,9 +11,8 @@ from w3lib.http import basic_auth_header class DataAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): value = str(values).encode("utf-8").decode("utf-8") - if items := re.findall(r"\$(.+)", value): - value = items[0] - + items = re.findall(r"\$(.+)", value) + value = items[0] if items else value setattr(namespace, self.dest, value) From 7e1814faf836757933afec4c7c394f43f34c3567 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 18 Apr 2023 23:36:51 -0500 Subject: [PATCH 0929/2083] fix: regex --- scrapy/utils/curl.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index ecfa292ea..9c98e4cb8 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -11,8 +11,7 @@ from w3lib.http import basic_auth_header class DataAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): value = str(values).encode("utf-8").decode("utf-8") - items = re.findall(r"\$(.+)", value) - value = items[0] if items else value + value = value[1::] if re.match(r"^\$(.+)", value) else value setattr(namespace, self.dest, value) From ef61fb5698c93178a57a9d4d67760def9a3039d9 Mon Sep 17 00:00:00 2001 From: tstauder <55719290+tstauder@users.noreply.github.com> Date: Wed, 19 Apr 2023 02:33:32 -0400 Subject: [PATCH 0930/2083] Fix flaky tests involving feed export batches (#5898) Co-authored-by: jmannoop --- tests/test_feedexport.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 3124d9d67..83de0e77e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2542,7 +2542,7 @@ class BatchDeliveriesTest(FeedExportTestBase): def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. - So %(batch_time)s replaced with the current date. + So %(batch_id)d replaced with the current id. """ items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -2552,7 +2552,7 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = { "FEEDS": { self._random_temp_filename() - / "%(batch_time)s": { + / "%(batch_id)d": { "format": "json", }, }, @@ -2615,7 +2615,7 @@ class BatchDeliveriesTest(FeedExportTestBase): return super().open(*args, **kwargs) key = "export.csv" - uri = f"s3://{bucket}/{key}/%(batch_time)s.json" + uri = f"s3://{bucket}/{key}/%(batch_id)d.json" batch_item_count = 1 settings = { "AWS_ACCESS_KEY_ID": "access_key", From b7ecec18099ace6ba77161302cca85c3e58a2ae7 Mon Sep 17 00:00:00 2001 From: Jalil SA Date: Wed, 19 Apr 2023 01:04:03 -0600 Subject: [PATCH 0931/2083] Update scrapy/utils/curl.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/curl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 9c98e4cb8..790c26b1a 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -10,8 +10,9 @@ from w3lib.http import basic_auth_header class DataAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): - value = str(values).encode("utf-8").decode("utf-8") - value = value[1::] if re.match(r"^\$(.+)", value) else value + value = str(values) + if value.startswith("$"): + value = value[1:] setattr(namespace, self.dest, value) From f69ba43f8e5b52af51119c682509167f5cf5517f Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 19 Apr 2023 02:06:00 -0500 Subject: [PATCH 0932/2083] fix: import re --- scrapy/utils/curl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 790c26b1a..f5dbbd64e 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,5 +1,4 @@ import argparse -import re import warnings from http.cookies import SimpleCookie from shlex import split From 87c8c5199902b448e90708e779470494f365c647 Mon Sep 17 00:00:00 2001 From: DeflateAwning <11021263+DeflateAwning@users.noreply.github.com> Date: Thu, 20 Apr 2023 00:23:02 -0600 Subject: [PATCH 0933/2083] Fix a typo in the FAQ (#5904) --- docs/faq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 031f4b942..20dd814df 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -231,8 +231,8 @@ Can I return (Twisted) deferreds from signal handlers? Some signals support returning deferreds from their handlers, others don't. See the :ref:`topics-signals-ref` to know which ones. -What does the response status code 999 means? ---------------------------------------------- +What does the response status code 999 mean? +-------------------------------------------- 999 is a custom response status code used by Yahoo sites to throttle requests. Try slowing down the crawling speed by using a download delay of ``2`` (or From 5a37af146f4f036168ac562918fca43adb4ac65f Mon Sep 17 00:00:00 2001 From: Jalil SA Date: Fri, 21 Apr 2023 01:29:57 -0600 Subject: [PATCH 0934/2083] Update documentation expectations for Parsel 1.8.0 (#5902) --- docs/intro/tutorial.rst | 8 ++++---- pytest.ini | 1 - tox.ini | 11 ++++++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 04d73d95a..19a76fc16 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -252,7 +252,7 @@ object: .. code-block:: pycon >>> response.css("title") - [] + [] The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of @@ -348,7 +348,7 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions: .. code-block:: pycon >>> response.xpath("//title") - [] + [] >>> response.xpath("//title/text()").get() 'Quotes to Scrape' @@ -410,8 +410,8 @@ We get a list of selectors for the quote HTML elements with: .. code-block:: pycon >>> response.css("div.quote") - [, - , + [, + , ...] Each of the selectors returned by the query above allows us to run further diff --git a/pytest.ini b/pytest.ini index f5fbf2529..866f0c950 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,7 +5,6 @@ python_files=test_*.py __init__.py python_classes= addopts = --assert=plain - --doctest-modules --ignore=docs/_ext --ignore=docs/conf.py --ignore=docs/news.rst diff --git a/tox.ini b/tox.ini index 06b52f3dc..9d81ec3e7 100644 --- a/tox.ini +++ b/tox.ini @@ -30,7 +30,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} @@ -99,6 +99,8 @@ setenv = _SCRAPY_PINNED=true install_command = python -I -m pip install {opts} {packages} +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] basepython = python3.7 @@ -108,6 +110,7 @@ deps = install_command = {[pinned]install_command} setenv = {[pinned]setenv} +commands = {[pinned]commands} [testenv:windows-pinned] basepython = python3 @@ -117,6 +120,7 @@ deps = install_command = {[pinned]install_command} setenv = {[pinned]setenv} +commands = {[pinned]commands} [testenv:extra-deps] basepython = python3 @@ -137,7 +141,7 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} -commands = {[testenv:asyncio]commands} +commands = {[pinned]commands} --reactor=asyncio install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -152,7 +156,8 @@ basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} PyPyDispatcher==2.1.0 -commands = {[testenv:pypy3]commands} +commands = + pytest --durations=10 scrapy tests install_command = {[pinned]install_command} setenv = {[pinned]setenv} From 67bfb304cdeb19a0b72b0a09542582f718cf07d6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 21 Apr 2023 18:52:58 +0400 Subject: [PATCH 0935/2083] Release notes for the current master. --- docs/news.rst | 105 ++++++++++++++++++++++++++++++++++++++- docs/topics/settings.rst | 2 +- 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 9b9eeac71..6cf366449 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,108 @@ Release notes ============= +.. _release-2.9.0: + +Scrapy 2.9.0 (YYYY-MM-DD) +------------------------- + +Highlights: + +- Per-domain request settings. +- Compatibility with new cryptography_ and new parsel_. +- TBD + +New features +~~~~~~~~~~~~ + +- Settings correponding to :setting:`DOWNLOAD_DELAY`, + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per domain basis + via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) + +- Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a + curl command from a :class:`~scrapy.Request` object. (:issue:`5892`) + +- Values of :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can now be + :class:`pathlib.Path` instances. (:issue:`5801`) + +- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed + string values for the curl ``--data-raw`` argument, which are produced by + browsers for data that includes certain symbols. (:issue:`5899`, + :issue:`5901`) + +- The ``scrapy parse`` command now also works with async generator callbacks. + (:issue:`5819`, :issue:`5824`) + +- The ``scrapy genspider`` command now properly works with HTTPS URLs. + (:issue:`3553`, :issue:`5808`) + +- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`) + +- :class:`LinkExtractor ` + now skips certain malformed URLs instead of raising an exception. + (:issue:`5881`) + +- :func:`scrapy.utils.python.get_func_args` now supports more types of + callables. (:issue:`5872`, :issue:`5885`) + +Bug fixes +~~~~~~~~~ + +- Fixed an error when using feed postprocessing with S3 storage. + (:issue:`5500`, :issue:`5581`) + +- Added the missing :meth:`scrapy.settings.BaseSettings.setdefault` method. + (:issue:`5811`, :issue:`5821`) + +- Fixed an error when using cryptography_ 40.0.0+ and + :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` is enabled. + (:issue:`5857`, :issue:`5858`) + +- The checksums returned by :class:`~scrapy.pipelines.files.FilesPipeline` + for files on Google Cloud Storage are no longer Base64-encoded. + (:issue:`5874`, :issue:`5891`) + +- Fixed an error breaking user handling of send failures in + :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) + +Documentation +~~~~~~~~~~~~~ + +- Expanded contributing docs. (:issue:`5109`, :issue:`5851`) + +- Added blacken-docs_ to pre-commit and reformatted the docs with it. + (:issue:`5813`, :issue:`5816`) + +- Fixed a JS issue. (:issue:`5875`, :issue:`5877`) + +- Fixed ``make htmlview``. (:issue:`5878`, :issue:`5879`) + +- Fixed typos and other small errors. (:issue:`5827`, :issue:`5839`, + :issue:`5883`, :issue:`5890`, :issue:`5895`, :issue:`5904`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Extended typing hints. (:issue:`5805`, :issue:`5889`, :issue:`5896`) + +- Tests for most of the examples in the docs are now run as a part of CI, + found problems were fixed. (:issue:`5816`, :issue:`5826`) + +- Removed usage of deprecated Python classes. (:issue:`5849`) + +- Silenced ``include-ignored`` warnings from coverage. (:issue:`5820`) + +- Fixed a random failure of the ``test_feedexport.test_batch_path_differ`` + test. (:issue:`5855`, :issue:`5898`) + +- Updated docstrings to match output produced by parsel_ 1.8.1 so that they + don't cause test failures. (:issue:`5902`) + +- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`) + +.. _blacken-docs: https://github.com/adamchainz/blacken-docs + .. _release-2.8.0: Scrapy 2.8.0 (2023-02-02) @@ -4207,8 +4309,6 @@ Relocations + Note: telnet is not enabled on Python 3 (https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595) -.. _parsel: https://github.com/scrapy/parsel - Bugfixes ~~~~~~~~ @@ -5638,6 +5738,7 @@ First release of Scrapy. .. _LevelDB: https://github.com/google/leveldb .. _lxml: https://lxml.de/ .. _marshal: https://docs.python.org/2/library/marshal.html +.. _parsel: https://github.com/scrapy/parsel .. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4412b5c1c..3e06d84f9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -783,7 +783,7 @@ DOWNLOAD_SLOTS Default: ``{}`` -Allows to define concurrency/delay parameters on per slot(domain) basis: +Allows to define concurrency/delay parameters on per slot (domain) basis: .. code-block:: python From 8c8fb67057609d79a31e76587fc66b162d746906 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 24 Apr 2023 11:34:34 +0400 Subject: [PATCH 0936/2083] Update tool versions (#5908) --- .bandit.yml | 1 + .pre-commit-config.yaml | 6 +++--- tox.ini | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index c8e84cc2e..2aae8a0aa 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,5 +1,6 @@ skips: - B101 +- B113 # https://github.com/PyCQA/bandit/issues/1010 - B105 - B301 - B303 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b90233e5..faf8808f2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.7.4 + rev: 1.7.5 hooks: - id: bandit args: [-r, -c, .bandit.yml] @@ -9,7 +9,7 @@ repos: hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 23.1.0 + rev: 23.3.0 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -21,4 +21,4 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==23.1.0 + - black==23.3.0 diff --git a/tox.ini b/tox.ini index 9d81ec3e7..af8f1f57a 100644 --- a/tox.ini +++ b/tox.ini @@ -58,7 +58,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.16.0 + pylint==2.17.2 commands = pylint conftest.py docs extras scrapy setup.py tests From 9af596a6b806a0cd0ba7f0d3bcff9ea6e3a19519 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 25 Apr 2023 10:24:14 -0500 Subject: [PATCH 0937/2083] feat: Add support for the Parsel JMESPath --- scrapy/http/response/__init__.py | 6 ++++++ scrapy/http/response/text.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 4213d491d..a82ed834a 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -142,6 +142,12 @@ class Response(object_ref): """ raise NotSupported("Response content isn't text") + def jmespath(self, *a, **kw): + """Shortcut method implemented only by responses whose content + is text (subclasses of TextResponse). + """ + raise NotSupported("Response content isn't text") + def xpath(self, *a, **kw): """Shortcut method implemented only by responses whose content is text (subclasses of TextResponse). diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 73bb811de..360d6334e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -139,6 +139,9 @@ class TextResponse(Response): self._cached_selector = Selector(self) return self._cached_selector + def jmespath(self, query, **kwargs): + return self.selector.jmespath(query, **kwargs) + def xpath(self, query, **kwargs): return self.selector.xpath(query, **kwargs) From b50c032ee9a75d1c9b42f1126637fdc655b141a8 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <95530227+guillermo-bondonno@users.noreply.github.com> Date: Wed, 26 Apr 2023 03:20:37 -0300 Subject: [PATCH 0938/2083] Add feed_slot_closed and feed_exporter_closed signals (#5876) --- docs/topics/signals.rst | 27 ++++++++++ scrapy/extensions/feedexport.py | 43 +++++++++++---- scrapy/signals.py | 2 + tests/test_feedexport.py | 92 ++++++++++++++++++++++++++++++--- 4 files changed, 148 insertions(+), 16 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3400a205a..9bfd1761c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -307,6 +307,33 @@ spider_error :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object +feed_slot_closed +~~~~~~~~~~~~~~~~ + +.. signal:: feed_slot_closed +.. function:: feed_slot_closed(slot) + + Sent when a :ref:`feed exports ` slot is closed. + + This signal supports returning deferreds from its handlers. + + :param slot: the slot closed + :type slot: scrapy.extensions.feedexport.FeedSlot + + +feed_exporter_closed +~~~~~~~~~~~~~~~~~~~~ + +.. signal:: feed_exporter_closed +.. function:: feed_exporter_closed() + + Sent when the :ref:`feed exports ` extension is closed, + during the handling of the :signal:`spider_closed` signal by the extension, + after all feed exporting has been handled. + + This signal supports returning deferreds from its handlers. + + Request signals --------------- diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index da1a88299..bcf0b779a 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,10 +11,11 @@ import warnings from datetime import datetime from pathlib import Path from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, Optional, Tuple, Union +from typing import IO, Any, Callable, List, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads +from twisted.internet.defer import DeferredList from w3lib.url import file_uri_to_path from zope.interface import Interface, implementer @@ -23,6 +24,8 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object @@ -271,7 +274,7 @@ class FTPFeedStorage(BlockingFeedStorage): ) -class _FeedSlot: +class FeedSlot: def __init__( self, file, @@ -309,7 +312,15 @@ class _FeedSlot: self._exporting = False +_FeedSlot = create_deprecated_class( + name="_FeedSlot", + new_class=FeedSlot, +) + + class FeedExporter: + _pending_deferreds: List[defer.Deferred] = [] + @classmethod def from_crawler(cls, crawler): exporter = cls(crawler) @@ -375,12 +386,18 @@ class FeedExporter: ) ) - def close_spider(self, spider): - deferred_list = [] + async def close_spider(self, spider): for slot in self.slots: - d = self._close_slot(slot, spider) - deferred_list.append(d) - return defer.DeferredList(deferred_list) if deferred_list else None + self._close_slot(slot, spider) + + # Await all deferreds + if self._pending_deferreds: + await maybe_deferred_to_future(DeferredList(self._pending_deferreds)) + + # Send FEED_EXPORTER_CLOSED signal + await maybe_deferred_to_future( + self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) + ) def _close_slot(self, slot, spider): def get_file(slot_): @@ -404,6 +421,14 @@ class FeedExporter: d.addErrback( self._handle_store_error, logmsg, spider, type(slot.storage).__name__ ) + self._pending_deferreds.append(d) + d.addCallback( + lambda _: self.crawler.signals.send_catch_log_deferred( + signals.feed_slot_closed, slot=slot + ) + ) + d.addBoth(lambda _: self._pending_deferreds.remove(d)) + return d def _handle_store_error(self, f, logmsg, spider, slot_type): @@ -444,7 +469,7 @@ class FeedExporter: indent=feed_options["indent"], **feed_options["item_export_kwargs"], ) - slot = _FeedSlot( + slot = FeedSlot( file=file, exporter=exporter, storage=storage, @@ -579,7 +604,7 @@ class FeedExporter: self, spider: Spider, uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]], - slot: Optional[_FeedSlot] = None, + slot: Optional[FeedSlot] = None, ) -> dict: params = {} for k in dir(spider): diff --git a/scrapy/signals.py b/scrapy/signals.py index 8cf2a4d93..0090f1c8b 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -22,6 +22,8 @@ bytes_received = object() item_scraped = object() item_dropped = object() item_error = object() +feed_slot_closed = object() +feed_exporter_closed = object() # for backward compatibility stats_spider_opened = spider_opened diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 83de0e77e..b1059099a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -32,18 +32,19 @@ from zope.interface import implementer from zope.interface.verify import verifyObject import scrapy +from scrapy import signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, FeedExporter, + FeedSlot, FileFeedStorage, FTPFeedStorage, GCSFeedStorage, IFeedStorage, S3FeedStorage, StdoutFeedStorage, - _FeedSlot, ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -660,8 +661,8 @@ class FeedExportTestBase(ABC, unittest.TestCase): return result -class InstrumentedFeedSlot(_FeedSlot): - """Instrumented _FeedSlot subclass for keeping track of calls to +class InstrumentedFeedSlot(FeedSlot): + """Instrumented FeedSlot subclass for keeping track of calls to start_exporting and finish_exporting.""" def start_exporting(self): @@ -964,7 +965,7 @@ class FeedExportTest(FeedExportTestBase): listener = IsExportingListener() InstrumentedFeedSlot.subscribe__listener(listener) - with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot): + with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) self.assertFalse(listener.start_without_finish) self.assertFalse(listener.finish_without_start) @@ -982,7 +983,7 @@ class FeedExportTest(FeedExportTestBase): listener = IsExportingListener() InstrumentedFeedSlot.subscribe__listener(listener) - with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot): + with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) self.assertFalse(listener.start_without_finish) self.assertFalse(listener.finish_without_start) @@ -1003,7 +1004,7 @@ class FeedExportTest(FeedExportTestBase): listener = IsExportingListener() InstrumentedFeedSlot.subscribe__listener(listener) - with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot): + with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) self.assertFalse(listener.start_without_finish) self.assertFalse(listener.finish_without_start) @@ -1022,7 +1023,7 @@ class FeedExportTest(FeedExportTestBase): listener = IsExportingListener() InstrumentedFeedSlot.subscribe__listener(listener) - with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot): + with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) self.assertFalse(listener.start_without_finish) self.assertFalse(listener.finish_without_start) @@ -2651,6 +2652,83 @@ class BatchDeliveriesTest(FeedExportTestBase): stub.assert_no_pending_responses() +# Test that the FeedExporer sends the feed_exporter_closed and feed_slot_closed signals +class FeedExporterSignalsTest(unittest.TestCase): + items = [ + {"foo": "bar1", "egg": "spam1"}, + {"foo": "bar2", "egg": "spam2", "baz": "quux2"}, + {"foo": "bar3", "baz": "quux3"}, + ] + + with tempfile.NamedTemporaryFile(suffix="json") as tmp: + settings = { + "FEEDS": { + f"file:///{tmp.name}": { + "format": "json", + }, + }, + } + + def feed_exporter_closed_signal_handler(self): + self.feed_exporter_closed_received = True + + def feed_slot_closed_signal_handler(self, slot): + self.feed_slot_closed_received = True + + def feed_exporter_closed_signal_handler_deferred(self): + d = defer.Deferred() + d.addCallback(lambda _: setattr(self, "feed_exporter_closed_received", True)) + d.callback(None) + return d + + def feed_slot_closed_signal_handler_deferred(self, slot): + d = defer.Deferred() + d.addCallback(lambda _: setattr(self, "feed_slot_closed_received", True)) + d.callback(None) + return d + + def run_signaled_feed_exporter( + self, feed_exporter_signal_handler, feed_slot_signal_handler + ): + crawler = get_crawler(settings_dict=self.settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + crawler.signals.connect( + feed_exporter_signal_handler, + signal=signals.feed_exporter_closed, + ) + crawler.signals.connect( + feed_slot_signal_handler, signal=signals.feed_slot_closed + ) + feed_exporter.open_spider(spider) + for item in self.items: + feed_exporter.item_scraped(item, spider) + defer.ensureDeferred(feed_exporter.close_spider(spider)) + + def test_feed_exporter_signals_sent(self): + self.feed_exporter_closed_received = False + self.feed_slot_closed_received = False + + self.run_signaled_feed_exporter( + self.feed_exporter_closed_signal_handler, + self.feed_slot_closed_signal_handler, + ) + self.assertTrue(self.feed_slot_closed_received) + self.assertTrue(self.feed_exporter_closed_received) + + def test_feed_exporter_signals_sent_deferred(self): + self.feed_exporter_closed_received = False + self.feed_slot_closed_received = False + + self.run_signaled_feed_exporter( + self.feed_exporter_closed_signal_handler_deferred, + self.feed_slot_closed_signal_handler_deferred, + ) + self.assertTrue(self.feed_slot_closed_received) + self.assertTrue(self.feed_exporter_closed_received) + + class FeedExportInitTest(unittest.TestCase): def test_unsupported_storage(self): settings = { From 865c36bdbbd7af0e5dd9c5c333d2285e88dfbcfd Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 28 Apr 2023 08:56:11 -0600 Subject: [PATCH 0939/2083] update docs --- docs/topics/request-response.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 99c7915df..407df32d2 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1281,6 +1281,12 @@ TextResponse objects :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: + .. method:: TextResponse.jmespath(query) + + A shortcut to ``TextResponse.selector.jmespath(query)``:: + + response.jmespath('object.[*]') + .. method:: TextResponse.xpath(query) A shortcut to ``TextResponse.selector.xpath(query)``:: From 3d29f20fc2021efb80d96389dc02c5a2cac9ef62 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 28 Apr 2023 23:54:09 -0600 Subject: [PATCH 0940/2083] added tests for jmespath --- tests/test_selector.py | 138 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/tests/test_selector.py b/tests/test_selector.py index febae46ac..b0deb5c99 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -108,3 +108,141 @@ class SelectorTestCase(unittest.TestCase): def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, "received both response and text"): Selector(TextResponse(url="http://example.com", body=b""), text="") + + +class JMESPathTestCase(unittest.TestCase): + def test_json_has_html(self) -> None: + """Sometimes the information is returned in a json wrapper""" + body = """ + { + "content": [ + { + "name": "A", + "value": "a" + }, + { + "name": { + "age": 18 + }, + "value": "b" + }, + { + "name": "C", + "value": "c" + }, + { + "name": "D", + "value": "
d
" + } + ], + "html": "
def
" + } + """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + self.assertEqual( + resp.jmespath("html").get(), + "
def
", + ) + self.assertEqual( + resp.jmespath("html").xpath("//div/a/text()").getall(), + ["a", "b", "d"], + ) + self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["f"]) + self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18") + + def test_html_has_json(self) -> None: + body = """ +
+

Information

+ + { + "user": [ + { + "name": "A", + "age": 18 + }, + { + "name": "B", + "age": 32 + }, + { + "name": "C", + "age": 22 + }, + { + "name": "D", + "age": 25 + } + ], + "total": 4, + "status": "ok" + } + +
+ """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + self.assertEqual( + resp.xpath("//div/content/text()").jmespath("user[*].name").getall(), + ["A", "B", "C", "D"], + ) + self.assertEqual( + resp.xpath("//div/content").jmespath("user[*].name").getall(), + ["A", "B", "C", "D"], + ) + self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4") + + def test_jmestpath_with_re(self) -> None: + body = """ +
+

Information

+ + { + "user": [ + { + "name": "A", + "age": 18 + }, + { + "name": "B", + "age": 32 + }, + { + "name": "C", + "age": 22 + }, + { + "name": "D", + "age": 25 + } + ], + "total": 4, + "status": "ok" + } + +
+ """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + self.assertEqual( + resp.xpath("//div/content/text()").jmespath("user[*].name").re(r"(\w+)"), + ["A", "B", "C", "D"], + ) + self.assertEqual( + resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"), + ["A", "B", "C", "D"], + ) + + self.assertEqual( + resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), [] + ) + + self.assertEqual( + resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)"), + None, + ) + + self.assertEqual( + resp.xpath("//div/content") + .jmespath("user[*].age.to_string(@)") + .re(r"(\d+)"), + ["18", "32", "22", "25"], + ) From 578606779d0f127a759f8b1623c1e4be341a17db Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Sat, 29 Apr 2023 00:52:39 -0600 Subject: [PATCH 0941/2083] update tests --- tests/test_http_response.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index dbc9f1fef..cefdb1709 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -32,6 +32,9 @@ class BaseResponseTest(unittest.TestCase): isinstance(self.response_class("http://example.com/"), self.response_class) ) self.assertRaises(TypeError, self.response_class, b"http://example.com") + self.assertRaises( + TypeError, self.response_class, url="http://example.com", body={} + ) # body can be str or None self.assertTrue( isinstance( @@ -192,6 +195,7 @@ class BaseResponseTest(unittest.TestCase): self.assertRaisesRegex(AttributeError, msg, getattr, r, "text") self.assertRaisesRegex(NotSupported, msg, r.css, "body") self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body") + self.assertRaisesRegex(NotSupported, msg, r.jmespath, "body") else: r.text r.css("body") From 8acde511a902515c56f7452f950221657953b92e Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 2 May 2023 12:11:23 -0300 Subject: [PATCH 0942/2083] fix: non-UTF-8 content-type headers --- scrapy/http/response/text.py | 6 ++++-- scrapy/responsetypes.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 73bb811de..d580a7876 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -100,11 +100,13 @@ class TextResponse(Response): @memoizemethod_noargs def _headers_encoding(self): content_type = self.headers.get(b"Content-Type", b"") - return http_content_type_encoding(to_unicode(content_type)) + return http_content_type_encoding(to_unicode(content_type, encoding="latin-1")) def _body_inferred_encoding(self): if self._cached_benc is None: - content_type = to_unicode(self.headers.get(b"Content-Type", b"")) + content_type = to_unicode( + self.headers.get(b"Content-Type", b""), encoding="latin-1" + ) benc, ubody = html_to_unicode( content_type, self.body, diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index f01e9096c..58884f21a 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -51,7 +51,9 @@ class ResponseTypes: header""" if content_encoding: return Response - mimetype = to_unicode(content_type).split(";")[0].strip().lower() + mimetype = ( + to_unicode(content_type, encoding="latin-1").split(";")[0].strip().lower() + ) return self.from_mimetype(mimetype) def from_content_disposition(self, content_disposition): From 7b49aa1b019672d4f2ab7a8d75465381cb5705a4 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 2 May 2023 12:53:04 -0300 Subject: [PATCH 0943/2083] chore: add tests --- tests/test_http_response.py | 11 +++++++++++ tests/test_responsetypes.py | 1 + 2 files changed, 12 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index dbc9f1fef..a05b702aa 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -448,6 +448,13 @@ class TextResponseTest(BaseResponseTest): body=codecs.BOM_UTF8 + b"\xc2\xa3", headers={"Content-type": ["text/html; charset=cp1251"]}, ) + r9 = self.response_class( + "http://www.example.com", + body=b"\x80", + headers={ + "Content-type": [b"application/x-download; filename=\x80dummy.txt"] + }, + ) self.assertEqual(r1._headers_encoding(), "utf-8") self.assertEqual(r2._headers_encoding(), None) @@ -458,9 +465,12 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r4._headers_encoding(), None) self.assertEqual(r5._headers_encoding(), None) self.assertEqual(r8._headers_encoding(), "cp1251") + self.assertEqual(r9._headers_encoding(), None) self.assertEqual(r8._declared_encoding(), "utf-8") + self.assertEqual(r9._declared_encoding(), None) self._assert_response_encoding(r5, "utf-8") self._assert_response_encoding(r8, "utf-8") + self._assert_response_encoding(r9, "cp1252") assert ( r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != "ascii" @@ -470,6 +480,7 @@ class TextResponseTest(BaseResponseTest): self._assert_response_values(r3, "iso-8859-1", "\xa3") self._assert_response_values(r6, "gb18030", "\u2015") self._assert_response_values(r7, "gb18030", "\u2015") + self._assert_response_values(r9, "cp1252", "€") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies self.assertRaises( diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 859960518..6e1ed82f0 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -42,6 +42,7 @@ class ResponseTypesTest(unittest.TestCase): ("application/octet-stream", Response), ("application/x-json; encoding=UTF8;charset=UTF-8", TextResponse), ("application/json-amazonui-streaming;charset=UTF-8", TextResponse), + (b"application/x-download; filename=\x80dummy.txt", Response), ] for source, cls in mappings: retcls = responsetypes.from_content_type(source) From 1eb44604853e24f7b526853d5e95414949fd88c6 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 2 May 2023 18:49:05 -0600 Subject: [PATCH 0944/2083] fix: jmespath --- scrapy/http/response/text.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 360d6334e..dd042a2bd 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -140,7 +140,12 @@ class TextResponse(Response): return self._cached_selector def jmespath(self, query, **kwargs): - return self.selector.jmespath(query, **kwargs) + if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined] + raise AttributeError( + "Please install parsel >= 1.8.1 to get jmespath support" + ) + + return self.selector.jmespath(query, **kwargs) # type: ignore[attr-defined] def xpath(self, query, **kwargs): return self.selector.xpath(query, **kwargs) From a604dfae5c12a55760d0b44d7da06b022cce3615 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 2 May 2023 19:19:00 -0600 Subject: [PATCH 0945/2083] update tests --- tests/test_selector.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_selector.py b/tests/test_selector.py index b0deb5c99..274d63d8d 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -1,10 +1,16 @@ import weakref +import packaging.version as version +import parsel +import pytest from twisted.trial import unittest from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.selector import Selector +PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) +PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0") + class SelectorTestCase(unittest.TestCase): def test_simple_selection(self): @@ -111,8 +117,12 @@ class SelectorTestCase(unittest.TestCase): class JMESPathTestCase(unittest.TestCase): + @pytest.mark.skipif( + not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" + ) def test_json_has_html(self) -> None: """Sometimes the information is returned in a json wrapper""" + body = """ { "content": [ @@ -150,6 +160,9 @@ class JMESPathTestCase(unittest.TestCase): self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["f"]) self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18") + @pytest.mark.skipif( + not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" + ) def test_html_has_json(self) -> None: body = """
@@ -191,6 +204,9 @@ class JMESPathTestCase(unittest.TestCase): ) self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4") + @pytest.mark.skipif( + not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" + ) def test_jmestpath_with_re(self) -> None: body = """
@@ -246,3 +262,14 @@ class JMESPathTestCase(unittest.TestCase): .re(r"(\d+)"), ["18", "32", "22", "25"], ) + + @pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath") + def test_jmespath_not_available(my_json_page) -> None: + body = """ + { + "website": {"name": "Example"} + } + """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + with pytest.raises(AttributeError): + resp.jmespath("website.name").get() From 4bb99fd2f3a957958ed9ea8fa418b6127a6bebee Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 2 May 2023 19:26:20 -0600 Subject: [PATCH 0946/2083] fix: pylint --- tests/test_selector.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_selector.py b/tests/test_selector.py index 274d63d8d..311c09aba 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -1,8 +1,8 @@ import weakref -import packaging.version as version import parsel import pytest +from packaging import version from twisted.trial import unittest from scrapy.http import HtmlResponse, TextResponse, XmlResponse @@ -11,6 +11,9 @@ from scrapy.selector import Selector PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0") +print(PARSEL_VERSION) +print(PARSEL_18_PLUS) + class SelectorTestCase(unittest.TestCase): def test_simple_selection(self): From a038faf11c2bc47921f57954a2405279d427f890 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 2 May 2023 19:40:04 -0600 Subject: [PATCH 0947/2083] fix: tests/tes_selector.py --- tests/test_selector.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_selector.py b/tests/test_selector.py index 311c09aba..85527bba9 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -11,9 +11,6 @@ from scrapy.selector import Selector PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0") -print(PARSEL_VERSION) -print(PARSEL_18_PLUS) - class SelectorTestCase(unittest.TestCase): def test_simple_selection(self): From d907f9e09284367d555c89d2bea862a310f60a19 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 3 May 2023 22:12:21 -0300 Subject: [PATCH 0948/2083] fix: Handle Parsel > 1.7.0 warning --- scrapy/selector/unified.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index cff97104a..208dd807f 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -4,6 +4,11 @@ XPath selectors based on lxml from parsel import Selector as _ParselSelector +try: + from parsel.selector import _NOT_SET +except ImportError: + _NOT_SET = None + from scrapy.http import HtmlResponse, XmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref @@ -63,7 +68,7 @@ class Selector(_ParselSelector, object_ref): __slots__ = ["response"] selectorlist_cls = SelectorList - def __init__(self, response=None, text=None, type=None, root=None, **kwargs): + def __init__(self, response=None, text=None, type=None, root=_NOT_SET, **kwargs): if response is not None and text is not None: raise ValueError( f"{self.__class__.__name__}.__init__() received " From 7317ff11014c4bf20d4e35193cc96a7151d1d5b0 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 4 May 2023 05:55:25 -0300 Subject: [PATCH 0949/2083] refactor: use kwargs strategy --- scrapy/selector/unified.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 208dd807f..caff79e9c 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -4,17 +4,14 @@ XPath selectors based on lxml from parsel import Selector as _ParselSelector -try: - from parsel.selector import _NOT_SET -except ImportError: - _NOT_SET = None - from scrapy.http import HtmlResponse, XmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref __all__ = ["Selector", "SelectorList"] +_NOT_SET = object() + def _st(response, st): if st is None: @@ -85,4 +82,8 @@ class Selector(_ParselSelector, object_ref): kwargs.setdefault("base_url", response.url) self.response = response - super().__init__(text=text, type=st, root=root, **kwargs) + + if root is not _NOT_SET: + kwargs["root"] = root + + super().__init__(text=text, type=st, **kwargs) From cba891a66c2223745d7b8484550ed54ca1e155ec Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 4 May 2023 15:04:33 +0400 Subject: [PATCH 0950/2083] Enable doc tests for selectors.rst, fix issues. --- docs/topics/selectors.rst | 81 ++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 26 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index c25c75d17..4a64d530b 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -48,6 +48,8 @@ Constructing selectors .. highlight:: python +.. skip: start + Response objects expose a :class:`~scrapy.Selector` instance on ``.selector`` attribute: @@ -66,6 +68,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``: >>> response.css("span::text").get() 'good' +.. skip: end + Scrapy selectors are instances of :class:`~scrapy.Selector` class constructed by passing either :class:`~scrapy.http.TextResponse` object or markup as a string (in ``text`` argument). @@ -93,7 +97,7 @@ Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of >>> from scrapy.selector import Selector >>> from scrapy.http import HtmlResponse - >>> response = HtmlResponse(url="http://example.com", body=body) + >>> response = HtmlResponse(url="http://example.com", body=body, encoding="utf-8") >>> Selector(response=response).xpath("//span/text()").get() 'good' @@ -103,6 +107,13 @@ Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of Using selectors --------------- +.. invisible-code-block: python + + html_response = response = load_response( + "https://docs.scrapy.org/en/latest/_static/selectors-sample1.html", + "../_static/selectors-sample1.html", + ) + To explain how to use the selectors we'll use the ``Scrapy shell`` (which provides interactive testing) and an example page located in the Scrapy documentation server: @@ -135,7 +146,7 @@ page, let's construct an XPath for selecting the text inside the title tag: .. code-block:: pycon >>> response.xpath("//title/text()") - [] + [] To actually extract the textual data, you must call the selector ``.get()`` or ``.getall()`` methods, as follows: @@ -363,11 +374,11 @@ too. Here's an example: >>> links = response.xpath('//a[contains(@href, "image")]') >>> links.getall() - ['Name: My image 1
', - 'Name: My image 2
', - 'Name: My image 3
', - 'Name: My image 4
', - 'Name: My image 5
'] + ['Name: My image 1
image1', + 'Name: My image 2
image2', + 'Name: My image 3
image3', + 'Name: My image 4
image4', + 'Name: My image 5
image5'] >>> for index, link in enumerate(links): ... href_xpath = link.xpath("@href").get() @@ -447,11 +458,11 @@ Here's an example used to extract image names from the :ref:`HTML code .. code-block:: pycon >>> response.xpath('//a[contains(@href, "image")]/text()').re(r"Name:\s*(.*)") - ['My image 1', - 'My image 2', - 'My image 3', - 'My image 4', - 'My image 5'] + ['My image 1 ', + 'My image 2 ', + 'My image 3 ', + 'My image 4 ', + 'My image 5 '] There's an additional helper reciprocating ``.get()`` (and its alias ``.extract_first()``) for ``.re()``, named ``.re_first()``. @@ -460,7 +471,7 @@ Use it to extract just the first matching string: .. code-block:: pycon >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r"Name:\s*(.*)") - 'My image 1' + 'My image 1 ' .. _old-extraction-api: @@ -761,6 +772,8 @@ on `XPath variables`_. Removing namespaces ------------------- +.. skip: start + When dealing with scraping projects, it is often quite convenient to get rid of namespaces altogether and just work with element names, to write more simple/convenient XPaths. You can use the @@ -808,8 +821,8 @@ nodes can be accessed directly by their names: >>> response.selector.remove_namespaces() >>> response.xpath("//link") - [, - , + [, + , ... If you wonder why the namespace removal procedure isn't always called by default @@ -824,6 +837,7 @@ of relevance, are: case some element names clash between namespaces. These cases are very rare though. +.. skip: end Using EXSLT extensions ---------------------- @@ -881,6 +895,8 @@ extracting text elements for example. Example extracting microdata (sample content taken from https://schema.org/Product) with groups of itemscopes and corresponding itemprops: +.. skip: next + .. code-block:: pycon >>> doc = """ @@ -977,26 +993,35 @@ Scrapy selectors also provide a sorely missed XPath extension function ``has-class`` that returns ``True`` for nodes that have all of the specified HTML classes. -.. highlight:: html +For the following HTML: -For the following HTML:: +.. code-block:: pycon -

First

-

Second

-

Third

-

Fourth

- -.. highlight:: python + >>> from scrapy.http import HtmlResponse + >>> response = HtmlResponse( + ... url="http://example.com", + ... body=""" + ... + ... + ...

First

+ ...

Second

+ ...

Third

+ ...

Fourth

+ ... + ... + ... """, + ... encoding="utf-8", + ... ) You can use it like this: .. code-block:: pycon >>> response.xpath('//p[has-class("foo")]') - [, - ] + [, + ] >>> response.xpath('//p[has-class("foo", "bar-baz")]') - [] + [] >>> response.xpath('//p[has-class("foo", "bar")]') [] @@ -1132,6 +1157,8 @@ a :class:`~scrapy.http.HtmlResponse` object like this: Selector examples on XML response --------------------------------- +.. skip: start + Here are some examples to illustrate concepts for :class:`Selector` objects instantiated with an :class:`~scrapy.http.XmlResponse` object: @@ -1154,4 +1181,6 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object: sel.register_namespace("g", "http://base.google.com/ns/1.0") sel.xpath("//g:price").getall() +.. skip: end + .. _Google Base XML feed: https://support.google.com/merchants/answer/160589?hl=en&ref_topic=2473799 From d1d6465ef4ea8987efb08e2f9abfd65b36719e04 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 4 May 2023 17:19:01 +0400 Subject: [PATCH 0951/2083] Address feedback. --- docs/news.rst | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 6cf366449..5f189760d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,7 +10,7 @@ Scrapy 2.9.0 (YYYY-MM-DD) Highlights: -- Per-domain request settings. +- Per-domain download settings. - Compatibility with new cryptography_ and new parsel_. - TBD @@ -19,7 +19,7 @@ New features - Settings correponding to :setting:`DOWNLOAD_DELAY`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per domain basis + :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) - Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a @@ -28,26 +28,6 @@ New features - Values of :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can now be :class:`pathlib.Path` instances. (:issue:`5801`) -- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed - string values for the curl ``--data-raw`` argument, which are produced by - browsers for data that includes certain symbols. (:issue:`5899`, - :issue:`5901`) - -- The ``scrapy parse`` command now also works with async generator callbacks. - (:issue:`5819`, :issue:`5824`) - -- The ``scrapy genspider`` command now properly works with HTTPS URLs. - (:issue:`3553`, :issue:`5808`) - -- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`) - -- :class:`LinkExtractor ` - now skips certain malformed URLs instead of raising an exception. - (:issue:`5881`) - -- :func:`scrapy.utils.python.get_func_args` now supports more types of - callables. (:issue:`5872`, :issue:`5885`) - Bug fixes ~~~~~~~~~ @@ -65,6 +45,26 @@ Bug fixes for files on Google Cloud Storage are no longer Base64-encoded. (:issue:`5874`, :issue:`5891`) +- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed + string values for the curl ``--data-raw`` argument, which are produced by + browsers for data that includes certain symbols. (:issue:`5899`, + :issue:`5901`) + +- The :command:`parse` command now also works with async generator callbacks. + (:issue:`5819`, :issue:`5824`) + +- The :command:`genspider` command now properly works with HTTPS URLs. + (:issue:`3553`, :issue:`5808`) + +- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`) + +- :class:`LinkExtractor ` + now skips certain malformed URLs instead of raising an exception. + (:issue:`5881`) + +- :func:`scrapy.utils.python.get_func_args` now supports more types of + callables. (:issue:`5872`, :issue:`5885`) + - Fixed an error breaking user handling of send failures in :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) From 636559f1cc652e839ef42522e7168e3ff9d77921 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 4 May 2023 17:55:07 +0400 Subject: [PATCH 0952/2083] Add newer changes. --- docs/news.rst | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 5f189760d..cbbb376e5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,7 +12,8 @@ Highlights: - Per-domain download settings. - Compatibility with new cryptography_ and new parsel_. -- TBD +- JMESPath selectors from the new parsel_. +- Bug fixes. New features ~~~~~~~~~~~~ @@ -22,6 +23,12 @@ New features :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) +- Added :meth:`TextResponse.jmespath`, a shortcut for JMESPath selectors + available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`) + +- Added :signal:`feed_slot_closed` and :signal:`feed_exporter_closed` + signals. (:issue:`5876`) + - Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a curl command from a :class:`~scrapy.Request` object. (:issue:`5892`) @@ -31,6 +38,8 @@ New features Bug fixes ~~~~~~~~~ +- Fixed a warning with Parsel 1.8.1+. (:issue:`5903`, :issue:`5918`) + - Fixed an error when using feed postprocessing with S3 storage. (:issue:`5500`, :issue:`5581`) @@ -65,6 +74,9 @@ Bug fixes - :func:`scrapy.utils.python.get_func_args` now supports more types of callables. (:issue:`5872`, :issue:`5885`) +- Fixed an error when processing non-UTF8 values of ``Content-Type`` headers. + (:issue:`5914`, :issue:`5917`) + - Fixed an error breaking user handling of send failures in :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) @@ -89,7 +101,7 @@ Quality assurance - Extended typing hints. (:issue:`5805`, :issue:`5889`, :issue:`5896`) - Tests for most of the examples in the docs are now run as a part of CI, - found problems were fixed. (:issue:`5816`, :issue:`5826`) + found problems were fixed. (:issue:`5816`, :issue:`5826`, :issue:`5919`) - Removed usage of deprecated Python classes. (:issue:`5849`) @@ -99,9 +111,10 @@ Quality assurance test. (:issue:`5855`, :issue:`5898`) - Updated docstrings to match output produced by parsel_ 1.8.1 so that they - don't cause test failures. (:issue:`5902`) + don't cause test failures. (:issue:`5902`, :issue:`5919`) -- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`) +- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`, + :issue:`5908`) .. _blacken-docs: https://github.com/adamchainz/blacken-docs From 4596a58a1333b8174b09336ab74c32c2e14c40f4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 16 Apr 2023 01:12:21 +0400 Subject: [PATCH 0953/2083] Typing for smaller scrapy/utils/ modules. --- scrapy/utils/boto.py | 2 +- scrapy/utils/decorators.py | 16 +++++++++------- scrapy/utils/serialize.py | 3 ++- scrapy/utils/versions.py | 3 ++- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 085ee7d25..53cfeddd0 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,7 +1,7 @@ """Boto/botocore helpers""" -def is_botocore_available(): +def is_botocore_available() -> bool: try: import botocore # noqa: F401 diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 4e684645b..04186559f 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -1,19 +1,21 @@ import warnings from functools import wraps +from typing import Any, Callable from twisted.internet import defer, threads +from twisted.internet.defer import Deferred from scrapy.exceptions import ScrapyDeprecationWarning -def deprecated(use_instead=None): +def deprecated(use_instead: Any = None) -> Callable: """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" - def deco(func): + def deco(func: Callable) -> Callable: @wraps(func) - def wrapped(*args, **kwargs): + def wrapped(*args: Any, **kwargs: Any) -> Any: message = f"Call to deprecated function {func.__name__}." if use_instead: message += f" Use {use_instead} instead." @@ -28,23 +30,23 @@ def deprecated(use_instead=None): return deco -def defers(func): +def defers(func: Callable) -> Callable[..., Deferred]: """Decorator to make sure a function always returns a deferred""" @wraps(func) - def wrapped(*a, **kw): + def wrapped(*a: Any, **kw: Any) -> Deferred: return defer.maybeDeferred(func, *a, **kw) return wrapped -def inthread(func): +def inthread(func: Callable) -> Callable[..., Deferred]: """Decorator to call a function in a thread and return a deferred with the result """ @wraps(func) - def wrapped(*a, **kw): + def wrapped(*a: Any, **kw: Any) -> Deferred: return threads.deferToThread(func, *a, **kw) return wrapped diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 414658944..3b4f67f00 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -1,6 +1,7 @@ import datetime import decimal import json +from typing import Any from itemadapter import ItemAdapter, is_item from twisted.internet import defer @@ -12,7 +13,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): DATE_FORMAT = "%Y-%m-%d" TIME_FORMAT = "%H:%M:%S" - def default(self, o): + def default(self, o: Any) -> Any: if isinstance(o, set): return list(o) if isinstance(o, datetime.datetime): diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index b0737d3d5..9b637bdb0 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -1,5 +1,6 @@ import platform import sys +from typing import List, Tuple import cryptography import cssselect @@ -12,7 +13,7 @@ import scrapy from scrapy.utils.ssl import get_openssl_version -def scrapy_components_versions(): +def scrapy_components_versions() -> List[Tuple[str, str]]: lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) From e0dbc83bd269e256e8247525e4d5a1d07658b635 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 16 Apr 2023 01:30:04 +0400 Subject: [PATCH 0954/2083] More typing for scrapy/utils/request.py. --- scrapy/utils/request.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 6d8be991d..6c7f3b345 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -6,7 +6,18 @@ scrapy.http.Request objects import hashlib import json import warnings -from typing import Dict, Iterable, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generator, + Iterable, + List, + Optional, + Tuple, + Type, + Union, +) from urllib.parse import urlunparse from weakref import WeakKeyDictionary @@ -19,11 +30,16 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from scrapy.crawler import Crawler + _deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" _deprecated_fingerprint_cache = WeakKeyDictionary() -def _serialize_headers(headers, request): +def _serialize_headers( + headers: Iterable[bytes], request: Request +) -> Generator[bytes, Any, None]: for header in headers: if header in request.headers: yield header @@ -139,7 +155,7 @@ def request_fingerprint( return cache[cache_key] -def _request_fingerprint_as_bytes(*args, **kwargs): +def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes: with warnings.catch_warnings(): warnings.simplefilter("ignore") return bytes.fromhex(request_fingerprint(*args, **kwargs)) @@ -231,7 +247,7 @@ class RequestFingerprinter: def from_crawler(cls, crawler): return cls(crawler) - def __init__(self, crawler=None): + def __init__(self, crawler: Optional["Crawler"] = None): if crawler: implementation = crawler.settings.get( "REQUEST_FINGERPRINTER_IMPLEMENTATION" @@ -265,7 +281,7 @@ class RequestFingerprinter: f"and '2.7'." ) - def fingerprint(self, request: Request): + def fingerprint(self, request: Request) -> bytes: return self._fingerprint(request) @@ -311,7 +327,7 @@ def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: If a spider is given, it will try to resolve the callbacks looking at the spider for methods with the same name. """ - request_cls = load_object(d["_class"]) if "_class" in d else Request + request_cls: Type[Request] = load_object(d["_class"]) if "_class" in d else Request kwargs = {key: value for key, value in d.items() if key in request_cls.attributes} if d.get("callback") and spider: kwargs["callback"] = _get_method(spider, d["callback"]) @@ -320,7 +336,7 @@ def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: return request_cls(**kwargs) -def _get_method(obj, name): +def _get_method(obj: Any, name: Any) -> Any: """Helper function for request_from_dict""" name = str(name) try: From f64a7dedca6d4bf33f86c633a0381a8017eb79f7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 23 Apr 2023 22:56:27 +0400 Subject: [PATCH 0955/2083] Add typing to scrapy/utils/url.py. --- scrapy/utils/url.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 833aa3e20..22b4197f9 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,6 +6,7 @@ Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ import re +from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this @@ -15,8 +16,14 @@ from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + from scrapy import Spider -def url_is_from_any_domain(url, domains): + +UrlT = Union[str, bytes, ParseResult] + + +def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: """Return True if the url belongs to any of the given domains""" host = parse_url(url).netloc.lower() if not host: @@ -25,29 +32,29 @@ def url_is_from_any_domain(url, domains): return any((host == d) or (host.endswith(f".{d}")) for d in domains) -def url_is_from_spider(url, spider): +def url_is_from_spider(url: UrlT, spider: Type["Spider"]) -> bool: """Return True if the url belongs to the given spider""" return url_is_from_any_domain( url, [spider.name] + list(getattr(spider, "allowed_domains", [])) ) -def url_has_any_extension(url, extensions): +def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: """Return True if the url ends with one of the extensions provided""" lowercase_path = parse_url(url).path.lower() return any(lowercase_path.endswith(ext) for ext in extensions) -def parse_url(url, encoding=None): +def parse_url(url: UrlT, encoding: Optional[str] = None) -> ParseResult: """Return urlparsed url from the given argument (which could be an already parsed url) """ if isinstance(url, ParseResult): return url - return urlparse(to_unicode(url, encoding)) + return cast(ParseResult, urlparse(to_unicode(url, encoding))) -def escape_ajax(url): +def escape_ajax(url: str) -> str: """ Return the crawlable url according to: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started @@ -76,7 +83,7 @@ def escape_ajax(url): return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:]) -def add_http_if_no_scheme(url): +def add_http_if_no_scheme(url: str) -> str: """Add http as the default scheme if it is missing from the url.""" match = re.match(r"^\w+://", url, flags=re.I) if not match: @@ -87,7 +94,7 @@ def add_http_if_no_scheme(url): return url -def _is_posix_path(string): +def _is_posix_path(string: str) -> bool: return bool( re.match( r""" @@ -109,7 +116,7 @@ def _is_posix_path(string): ) -def _is_windows_path(string): +def _is_windows_path(string: str) -> bool: return bool( re.match( r""" @@ -125,11 +132,11 @@ def _is_windows_path(string): ) -def _is_filesystem_path(string): +def _is_filesystem_path(string: str) -> bool: return _is_posix_path(string) or _is_windows_path(string) -def guess_scheme(url): +def guess_scheme(url: str) -> str: """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" if _is_filesystem_path(url): @@ -138,12 +145,12 @@ def guess_scheme(url): def strip_url( - url, - strip_credentials=True, - strip_default_port=True, - origin_only=False, - strip_fragment=True, -): + url: str, + strip_credentials: bool = True, + strip_default_port: bool = True, + origin_only: bool = False, + strip_fragment: bool = True, +) -> str: """Strip URL string from some of its components: - ``strip_credentials`` removes "user:password@" From 7347d021457866ade62a6ed8a0839766e91032e4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 01:12:52 +0400 Subject: [PATCH 0956/2083] Add typing to scrapy/utils/datatypes.py. --- scrapy/resolver.py | 8 +++++--- scrapy/utils/datatypes.py | 28 ++++++++++++++++------------ scrapy/utils/misc.py | 12 +++++++----- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 6cbe01cbf..e2e8beff4 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,3 +1,5 @@ +from typing import Any + from twisted.internet import defer from twisted.internet.base import ThreadedResolver from twisted.internet.interfaces import ( @@ -11,7 +13,7 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache # TODO: cache misses -dnscache = LocalCache(10000) +dnscache: LocalCache[str, Any] = LocalCache(10000) @implementer(IResolverSimple) @@ -36,7 +38,7 @@ class CachingThreadedResolver(ThreadedResolver): def install_on_reactor(self): self.reactor.installResolver(self) - def getHostByName(self, name, timeout=None): + def getHostByName(self, name: str, timeout=None): if name in dnscache: return defer.succeed(dnscache[name]) # in Twisted<=16.6, getHostByName() is always called with @@ -110,7 +112,7 @@ class CachingHostnameResolver: def resolveHostName( self, resolutionReceiver, - hostName, + hostName: str, portNumber=0, addressTypes=None, transportSemantics="TCP", diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index fa57a4f26..599b201ea 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -8,6 +8,10 @@ This module must not depend on any module outside the Standard Library. import collections import weakref from collections.abc import Mapping +from typing import Any, Optional, Sequence, TypeVar + +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") class CaselessDict(dict): @@ -64,24 +68,24 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class LocalCache(collections.OrderedDict): +class LocalCache(collections.OrderedDict[_KT, _VT]): """Dictionary with a finite number of keys. Older items expires first. """ - def __init__(self, limit=None): + def __init__(self, limit: Optional[int] = None): super().__init__() - self.limit = limit + self.limit: Optional[int] = limit - def __setitem__(self, key, value): + def __setitem__(self, key: _KT, value: _VT) -> None: if self.limit: while len(self) >= self.limit: self.popitem(last=False) super().__setitem__(key, value) -class LocalWeakReferencedCache(weakref.WeakKeyDictionary): +class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT]): """ A weakref.WeakKeyDictionary implementation that uses LocalCache as its underlying data structure, making it ordered and capable of being size-limited. @@ -93,17 +97,17 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): it cannot be instantiated with an initial dictionary. """ - def __init__(self, limit=None): + def __init__(self, limit: Optional[int] = None): super().__init__() - self.data = LocalCache(limit=limit) + self.data: LocalCache = LocalCache(limit=limit) - def __setitem__(self, key, value): + def __setitem__(self, key: _KT, value: _VT) -> None: try: super().__setitem__(key, value) except TypeError: pass # key is not weak-referenceable, skip caching - def __getitem__(self, key): + def __getitem__(self, key: _KT) -> Optional[_VT]: # type: ignore[override] try: return super().__getitem__(key) except (TypeError, KeyError): @@ -113,8 +117,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): class SequenceExclude: """Object to test if an item is NOT within some sequence.""" - def __init__(self, seq): - self.seq = seq + def __init__(self, seq: Sequence): + self.seq: Sequence = seq - def __contains__(self, item): + def __contains__(self, item: Any) -> bool: return item not in self.seq diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d861c9ab6..ea3f934c7 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -214,16 +214,18 @@ def walk_callable(node): yield node -_generator_callbacks_cache = LocalWeakReferencedCache(limit=128) +_generator_callbacks_cache: LocalWeakReferencedCache[ + Callable, bool +] = LocalWeakReferencedCache(limit=128) -def is_generator_with_return_value(callable): +def is_generator_with_return_value(callable: Callable) -> bool: """ Returns True if a callable is a generator function which includes a 'return' statement with a value different than None, False otherwise """ if callable in _generator_callbacks_cache: - return _generator_callbacks_cache[callable] + return bool(_generator_callbacks_cache[callable]) def returns_none(return_node): value = return_node.value @@ -248,10 +250,10 @@ def is_generator_with_return_value(callable): for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): _generator_callbacks_cache[callable] = True - return _generator_callbacks_cache[callable] + return bool(_generator_callbacks_cache[callable]) _generator_callbacks_cache[callable] = False - return _generator_callbacks_cache[callable] + return bool(_generator_callbacks_cache[callable]) def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None: From ea299dfd7ce8766c2c9430fe8b297fddad45adee Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 01:21:00 +0400 Subject: [PATCH 0957/2083] Add typing to scrapy/utils/misc.py. --- scrapy/utils/misc.py | 46 +++++++++++++++++++++++++++++------------- scrapy/utils/python.py | 4 ++-- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index ea3f934c7..defc8663d 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -10,7 +10,21 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from typing import TYPE_CHECKING, Any, Callable, Union +from types import ModuleType +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Deque, + Generator, + Iterable, + List, + Optional, + Pattern, + Union, + cast, +) from w3lib.html import replace_entities @@ -26,7 +40,7 @@ if TYPE_CHECKING: _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes -def arg_to_iter(arg): +def arg_to_iter(arg: Any) -> Iterable[Any]: """Convert an argument to an iterable. The argument can be a None, single value, or an iterable. @@ -35,7 +49,7 @@ def arg_to_iter(arg): if arg is None: return [] if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"): - return arg + return cast(Iterable[Any], arg) return [arg] @@ -72,7 +86,7 @@ def load_object(path: Union[str, Callable]) -> Any: return obj -def walk_modules(path): +def walk_modules(path: str) -> List[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that exception is thrown back. @@ -80,7 +94,7 @@ def walk_modules(path): For example: walk_modules('scrapy.utils') """ - mods = [] + mods: List[ModuleType] = [] mod = import_module(path) mods.append(mod) if hasattr(mod, "__path__"): @@ -94,7 +108,9 @@ def walk_modules(path): return mods -def extract_regex(regex, text, encoding="utf-8"): +def extract_regex( + regex: Union[str, Pattern], text: str, encoding: str = "utf-8" +) -> List[str]: """Extract a list of unicode strings from the given text/encoding using the following policies: * if the regex contains a named group called "extract" that will be returned @@ -111,9 +127,11 @@ def extract_regex(regex, text, encoding="utf-8"): regex = re.compile(regex, re.UNICODE) try: - strings = [regex.search(text).group("extract")] # named group + # named group + strings = [regex.search(text).group("extract")] # type: ignore[union-attr] except Exception: - strings = regex.findall(text) # full regex or numbered groups + # full regex or numbered groups + strings = regex.findall(text) strings = flatten(strings) if isinstance(text, str): @@ -123,7 +141,7 @@ def extract_regex(regex, text, encoding="utf-8"): ] -def md5sum(file): +def md5sum(file: IO) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. @@ -140,7 +158,7 @@ def md5sum(file): return m.hexdigest() -def rel_has_nofollow(rel): +def rel_has_nofollow(rel: Optional[str]) -> bool: """Return True if link rel attribute has nofollow type""" return rel is not None and "nofollow" in rel.replace(",", " ").split() @@ -181,7 +199,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): @contextmanager -def set_environ(**kwargs): +def set_environ(**kwargs: str) -> Generator[None, Any, None]: """Temporarily set environment variables inside the context manager and fully restore previous environment afterwards """ @@ -198,11 +216,11 @@ def set_environ(**kwargs): os.environ[k] = v -def walk_callable(node): +def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: """Similar to ``ast.walk``, but walks only function body and skips nested functions defined within the node. """ - todo = deque([node]) + todo: Deque[ast.AST] = deque([node]) walked_func_def = False while todo: node = todo.popleft() @@ -227,7 +245,7 @@ def is_generator_with_return_value(callable: Callable) -> bool: if callable in _generator_callbacks_cache: return bool(_generator_callbacks_cache[callable]) - def returns_none(return_node): + def returns_none(return_node: ast.Return) -> bool: value = return_node.value return ( value is None or isinstance(value, ast.NameConstant) and value.value is None diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 27816c0df..ae8feaf7d 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -22,7 +22,7 @@ from typing import ( from scrapy.utils.asyncgen import as_async_generator -def flatten(x): +def flatten(x: Iterable) -> list: """flatten(sequence) -> list Returns a single, flat list which contains all elements retrieved @@ -42,7 +42,7 @@ def flatten(x): return list(iflatten(x)) -def iflatten(x): +def iflatten(x: Iterable) -> Iterable: """iflatten(sequence) -> iterator Similar to ``.flatten()``, but returns iterator instead""" From 43ee483a0dc6c7883ba9f219ac8b4fd95647b83d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 01:34:54 +0400 Subject: [PATCH 0958/2083] Add typing to scrapy/utils/reactor.py. --- scrapy/utils/reactor.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index f1b9239e6..ad3d1d8bc 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,7 +1,8 @@ import asyncio import sys +from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress -from typing import Any, Callable, Dict, Optional, Sequence +from typing import Any, Callable, Dict, Optional, Sequence, Type from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error @@ -57,7 +58,7 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) -def set_asyncio_event_loop_policy(): +def set_asyncio_event_loop_policy() -> None: """The policy functions from asyncio often behave unexpectedly, so we restrict their use to the absolutely essential case. This should only be used to install the reactor. @@ -65,7 +66,7 @@ def set_asyncio_event_loop_policy(): _get_asyncio_event_loop_policy() -def get_asyncio_event_loop_policy(): +def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: warn( "Call to deprecated function " "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n" @@ -81,7 +82,7 @@ def get_asyncio_event_loop_policy(): return _get_asyncio_event_loop_policy() -def _get_asyncio_event_loop_policy(): +def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: policy = asyncio.get_event_loop_policy() if ( sys.version_info >= (3, 8) @@ -93,7 +94,7 @@ def _get_asyncio_event_loop_policy(): return policy -def install_reactor(reactor_path, event_loop_path=None): +def install_reactor(reactor_path: str, event_loop_path: Optional[str] = None) -> None: """Installs the :mod:`~twisted.internet.reactor` with the specified import path. Also installs the asyncio event loop with the specified import path if the asyncio reactor is enabled""" @@ -111,14 +112,14 @@ def install_reactor(reactor_path, event_loop_path=None): installer() -def _get_asyncio_event_loop(): +def _get_asyncio_event_loop() -> AbstractEventLoop: return set_asyncio_event_loop(None) -def set_asyncio_event_loop(event_loop_path): +def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: - event_loop_class = load_object(event_loop_path) + event_loop_class: Type[AbstractEventLoop] = load_object(event_loop_path) event_loop = event_loop_class() asyncio.set_event_loop(event_loop) else: @@ -146,7 +147,7 @@ def set_asyncio_event_loop(event_loop_path): return event_loop -def verify_installed_reactor(reactor_path): +def verify_installed_reactor(reactor_path: str) -> None: """Raises :exc:`Exception` if the installed :mod:`~twisted.internet.reactor` does not match the specified import path.""" @@ -162,7 +163,7 @@ def verify_installed_reactor(reactor_path): raise Exception(msg) -def verify_installed_asyncio_event_loop(loop_path): +def verify_installed_asyncio_event_loop(loop_path: str) -> None: from twisted.internet import reactor loop_class = load_object(loop_path) @@ -181,7 +182,7 @@ def verify_installed_asyncio_event_loop(loop_path): ) -def is_asyncio_reactor_installed(): +def is_asyncio_reactor_installed() -> bool: from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) From 4da86915109f77066c306130ecd122a17430191d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 02:02:33 +0400 Subject: [PATCH 0959/2083] Add typing to scrapy/utils/gz.py and remove dead code. --- scrapy/utils/gz.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index e5df34d2e..98ca510ed 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,15 +1,18 @@ import struct from gzip import GzipFile from io import BytesIO +from typing import List + +from scrapy.http import Response -def gunzip(data): +def gunzip(data: bytes) -> bytes: """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output_list = [] + output_list: List[bytes] = [] chunk = b"." while chunk: try: @@ -18,17 +21,13 @@ def gunzip(data): except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output_list is empty and f.extrabuf - # contains the whole page content - if output_list or getattr(f, "extrabuf", None): - try: - output_list.append(f.extrabuf[-f.extrasize :]) - finally: - break + # some pages are quite small so output_list is empty + if output_list: + break else: raise return b"".join(output_list) -def gzip_magic_number(response): +def gzip_magic_number(response: Response) -> bool: return response.body[:3] == b"\x1f\x8b\x08" From d400f1ac0664768d04985f0a00bc6d47a54957fe Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 18:35:24 +0400 Subject: [PATCH 0960/2083] Add more typing to scrapy/utils/python.py. --- scrapy/utils/python.py | 58 ++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index ae8feaf7d..0b5dc324f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,6 +1,7 @@ """ This module contains essential stuff that should've come with Python itself ;) """ +import collections.abc import gc import inspect import re @@ -12,9 +13,17 @@ from typing import ( Any, AsyncGenerator, AsyncIterable, + AsyncIterator, + Callable, + Dict, + Generator, Iterable, + Iterator, + List, Mapping, Optional, + Pattern, + Tuple, Union, overload, ) @@ -78,7 +87,7 @@ def is_listlike(x: Any) -> bool: return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) -def unique(list_, key=lambda x: x): +def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list: """efficient function to uniquify a list preserving item order""" seen = set() result = [] @@ -124,7 +133,9 @@ def to_bytes( return text.encode(encoding, errors) -def re_rsearch(pattern, text, chunk_size=1024): +def re_rsearch( + pattern: Union[str, Pattern], text: str, chunk_size: int = 1024 +) -> Optional[Tuple[int, int]]: """ This function does a reverse search in a text using a regular expression given in the attribute 'pattern'. @@ -138,7 +149,7 @@ def re_rsearch(pattern, text, chunk_size=1024): the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter(): + def _chunk_iter() -> Generator[Tuple[str, int], Any, None]: offset = len(text) while True: offset -= chunk_size * 1024 @@ -158,14 +169,14 @@ def re_rsearch(pattern, text, chunk_size=1024): return None -def memoizemethod_noargs(method): +def memoizemethod_noargs(method: Callable) -> Callable: """Decorator to cache the result of a method (without arguments) using a weak reference to its object """ - cache = weakref.WeakKeyDictionary() + cache: weakref.WeakKeyDictionary[Any, Any] = weakref.WeakKeyDictionary() @wraps(method) - def new_method(self, *args, **kwargs): + def new_method(self: Any, *args: Any, **kwargs: Any) -> Any: if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] @@ -187,12 +198,12 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) -def get_func_args(func, stripself=False): +def get_func_args(func: Callable, stripself: bool = False) -> List[str]: """Return the argument name list of a callable object""" if not callable(func): raise TypeError(f"func must be callable, got '{type(func).__name__}'") - args = [] + args: List[str] = [] try: sig = inspect.signature(func) except ValueError: @@ -217,7 +228,7 @@ def get_func_args(func, stripself=False): return args -def get_spec(func): +def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: """Returns (args, kwargs) tuple for a function >>> import re >>> get_spec(re.match) @@ -246,7 +257,7 @@ def get_spec(func): else: raise TypeError(f"{type(func)} is not callable") - defaults = spec.defaults or [] + defaults: Tuple[Any, ...] = spec.defaults or () firstdefault = len(spec.args) - len(defaults) args = spec.args[:firstdefault] @@ -254,7 +265,9 @@ def get_spec(func): return args, kwargs -def equal_attributes(obj1, obj2, attributes): +def equal_attributes( + obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]] +) -> bool: """Compare two objects attributes""" # not attributes given return False by default if not attributes: @@ -282,19 +295,20 @@ def without_none_values(iterable: Iterable) -> Iterable: ... -def without_none_values(iterable): +def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have value ``None`` have been removed. """ - try: + if isinstance(iterable, collections.abc.Mapping): return {k: v for k, v in iterable.items() if v is not None} - except AttributeError: - return type(iterable)((v for v in iterable if v is not None)) + else: + # the iterable __init__ must take another iterable + return type(iterable)(v for v in iterable if v is not None) # type: ignore[call-arg] -def global_object_name(obj): +def global_object_name(obj: Any) -> str: """ Return full name of a global object. @@ -307,14 +321,14 @@ def global_object_name(obj): if hasattr(sys, "pypy_version_info"): - def garbage_collect(): + def garbage_collect() -> None: # Collecting weakreferences can take two collections on PyPy. gc.collect() gc.collect() else: - def garbage_collect(): + def garbage_collect() -> None: gc.collect() @@ -329,10 +343,10 @@ class MutableChain(Iterable): def extend(self, *iterables: Iterable) -> None: self.data = chain(self.data, chain.from_iterable(iterables)) - def __iter__(self): + def __iter__(self) -> Iterator: return self - def __next__(self): + def __next__(self) -> Any: return next(self.data) @@ -353,8 +367,8 @@ class MutableAsyncChain(AsyncIterable): def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) - def __aiter__(self): + def __aiter__(self) -> AsyncIterator: return self - async def __anext__(self): + async def __anext__(self) -> Any: return await self.data.__anext__() From 9661a5c4913e4be314572e12a49c215e7631db88 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 18:50:23 +0400 Subject: [PATCH 0961/2083] Add typing to scrapy/utils/deprecate.py. --- scrapy/utils/deprecate.py | 59 ++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index f4d6e0451..ab2719bb3 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -2,12 +2,12 @@ import inspect import warnings -from typing import List, Tuple +from typing import Any, List, Optional, Tuple, Type, overload from scrapy.exceptions import ScrapyDeprecationWarning -def attribute(obj, oldattr, newattr, version="0.12"): +def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> None: cname = obj.__class__.__name__ warnings.warn( f"{cname}.{oldattr} attribute is deprecated and will be no longer supported " @@ -18,16 +18,16 @@ def attribute(obj, oldattr, newattr, version="0.12"): def create_deprecated_class( - name, - new_class, - clsdict=None, - warn_category=ScrapyDeprecationWarning, - warn_once=True, - old_class_path=None, - new_class_path=None, - subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.", - instance_warn_message="{cls} is deprecated, instantiate {new} instead.", -): + name: str, + new_class: type, + clsdict: Optional[dict[str, Any]] = None, + warn_category: Type[Warning] = ScrapyDeprecationWarning, + warn_once: bool = True, + old_class_path: Optional[str] = None, + new_class_path: Optional[str] = None, + subclass_warn_message: str = "{cls} inherits from deprecated class {old}, please inherit from {new}.", + instance_warn_message: str = "{cls} is deprecated, instantiate {new} instead.", +) -> type: """ Return a "deprecated" class that causes its subclasses to issue a warning. Subclasses of ``new_class`` are considered subclasses of this class. @@ -53,17 +53,20 @@ def create_deprecated_class( OldName. """ - class DeprecatedClass(new_class.__class__): - deprecated_class = None - warned_on_subclass = False + # https://github.com/python/mypy/issues/4177 + class DeprecatedClass(new_class.__class__): # type: ignore[misc, name-defined] + deprecated_class: Optional[type] = None + warned_on_subclass: bool = False - def __new__(metacls, name, bases, clsdict_): + def __new__( + metacls, name: str, bases: Tuple[type, ...], clsdict_: dict[str, Any] + ) -> type: cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls - def __init__(cls, name, bases, clsdict_): + def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: dict[str, Any]): meta = cls.__class__ old = meta.deprecated_class if old in bases and not (warn_once and meta.warned_on_subclass): @@ -81,10 +84,10 @@ def create_deprecated_class( # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks # for implementation details - def __instancecheck__(cls, inst): + def __instancecheck__(cls, inst: Any) -> bool: return any(cls.__subclasscheck__(c) for c in (type(inst), inst.__class__)) - def __subclasscheck__(cls, sub): + def __subclasscheck__(cls, sub: type) -> bool: if cls is not DeprecatedClass.deprecated_class: # we should do the magic only if second `issubclass` argument # is the deprecated class itself - subclasses of the @@ -98,7 +101,7 @@ def create_deprecated_class( mro = getattr(sub, "__mro__", ()) return any(c in {cls, new_class} for c in mro) - def __call__(cls, *args, **kwargs): + def __call__(cls, *args: Any, **kwargs: Any) -> Any: old = DeprecatedClass.deprecated_class if cls is old: msg = instance_warn_message.format( @@ -125,7 +128,7 @@ def create_deprecated_class( return deprecated_cls -def _clspath(cls, forced=None): +def _clspath(cls: type, forced: Optional[str] = None) -> str: if forced is not None: return forced return f"{cls.__module__}.{cls.__name__}" @@ -134,7 +137,17 @@ def _clspath(cls, forced=None): DEPRECATION_RULES: List[Tuple[str, str]] = [] -def update_classpath(path): +@overload +def update_classpath(path: str) -> str: + ... + + +@overload +def update_classpath(path: Any) -> Any: + ... + + +def update_classpath(path: Any) -> Any: """Update a deprecated path from an object with its new location""" for prefix, replacement in DEPRECATION_RULES: if isinstance(path, str) and path.startswith(prefix): @@ -147,7 +160,7 @@ def update_classpath(path): return path -def method_is_overridden(subclass, base_class, method_name): +def method_is_overridden(subclass: type, base_class: type, method_name: str) -> bool: """ Return True if a method named ``method_name`` of a ``base_class`` is overridden in a ``subclass``. From b8277f4cab7941989402a8339634a91dcd3500c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 19:08:46 +0400 Subject: [PATCH 0962/2083] Add more typing to scrapy/utils/defer.py. --- scrapy/utils/defer.py | 57 ++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d25ebbdf4..307707bf5 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -9,13 +9,16 @@ from typing import ( Any, AsyncGenerator, AsyncIterable, + AsyncIterator, Callable, Coroutine, + Dict, Generator, Iterable, Iterator, List, Optional, + Tuple, Union, cast, ) @@ -44,7 +47,7 @@ def defer_fail(_failure: Failure) -> Deferred: return d -def defer_succeed(result) -> Deferred: +def defer_succeed(result: Any) -> Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -58,7 +61,7 @@ def defer_succeed(result) -> Deferred: return d -def defer_result(result) -> Deferred: +def defer_result(result: Any) -> Deferred: if isinstance(result, Deferred): return result if isinstance(result, failure.Failure): @@ -66,7 +69,7 @@ def defer_result(result) -> Deferred: return defer_succeed(result) -def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: +def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -84,7 +87,7 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: def parallel( - iterable: Iterable, count: int, callable: Callable, *args, **named + iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any ) -> Deferred: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -146,14 +149,14 @@ class _AsyncCooperatorAdapter(Iterator): self, aiterable: AsyncIterable, callable: Callable, - *callable_args, - **callable_kwargs + *callable_args: Any, + **callable_kwargs: Any, ): - self.aiterator = aiterable.__aiter__() - self.callable = callable - self.callable_args = callable_args - self.callable_kwargs = callable_kwargs - self.finished = False + self.aiterator: AsyncIterator = aiterable.__aiter__() + self.callable: Callable = callable + self.callable_args: Tuple[Any, ...] = callable_args + self.callable_kwargs: Dict[str, Any] = callable_kwargs + self.finished: bool = False self.waiting_deferreds: List[Deferred] = [] self.anext_deferred: Optional[Deferred] = None @@ -201,7 +204,11 @@ class _AsyncCooperatorAdapter(Iterator): def parallel_async( - async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named + async_iterable: AsyncIterable, + count: int, + callable: Callable, + *args: Any, + **named: Any, ) -> Deferred: """Like parallel but for async iterators""" coop = Cooperator() @@ -210,7 +217,9 @@ def parallel_async( return dl -def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: +def process_chain( + callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any +) -> Deferred: """Return a Deferred built by chaining the given callbacks""" d: Deferred = Deferred() for x in callbacks: @@ -220,7 +229,11 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: def process_chain_both( - callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw + callbacks: Iterable[Callable], + errbacks: Iterable[Callable], + input: Any, + *a: Any, + **kw: Any, ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" d: Deferred = Deferred() @@ -240,7 +253,9 @@ def process_chain_both( return d -def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: +def process_parallel( + callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any +) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ @@ -250,7 +265,9 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred return d -def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: +def iter_errback( + iterable: Iterable, errback: Callable, *a: Any, **kw: Any +) -> Generator: """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -265,7 +282,7 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: async def aiter_errback( - aiterable: AsyncIterable, errback: Callable, *a, **kw + aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any ) -> AsyncGenerator: """Wraps an async iterable calling an errback if an error is caught while iterating it. Similar to scrapy.utils.defer.iter_errback() @@ -280,7 +297,7 @@ async def aiter_errback( errback(failure.Failure(), *a, **kw) -def deferred_from_coro(o) -> Any: +def deferred_from_coro(o: Any) -> Any: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, Deferred): return o @@ -303,13 +320,13 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: """ @wraps(coro_f) - def f(*coro_args, **coro_kwargs): + def f(*coro_args: Any, **coro_kwargs: Any) -> Any: return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) return f -def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: +def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) From c04b9ba19de85154c7bfec536f584f31456e44b3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 19:10:42 +0400 Subject: [PATCH 0963/2083] Add typing to scrapy/utils/template.py. --- scrapy/utils/template.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 1499aeb3d..6b22f3bfa 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -4,10 +4,10 @@ import re import string from os import PathLike from pathlib import Path -from typing import Union +from typing import Any, Union -def render_templatefile(path: Union[str, PathLike], **kwargs): +def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None: path_obj = Path(path) raw = path_obj.read_text("utf8") @@ -24,7 +24,7 @@ def render_templatefile(path: Union[str, PathLike], **kwargs): CAMELCASE_INVALID_CHARS = re.compile(r"[^a-zA-Z\d]") -def string_camelcase(string): +def string_camelcase(string: str) -> str: """Convert a word to its CamelCase version and remove invalid chars >>> string_camelcase('lost-pound') From 36507ddb7b55d6fc4bfdd19286d82057ef3fb5cf Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 19:16:57 +0400 Subject: [PATCH 0964/2083] Add typing to scrapy/utils/engine.py. --- scrapy/utils/engine.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 8e3ec2c37..a5f2a8c6e 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -2,9 +2,13 @@ # used in global tests code from time import time # noqa: F401 +from typing import TYPE_CHECKING, Any, List, Tuple + +if TYPE_CHECKING: + from scrapy.core.engine import ExecutionEngine -def get_engine_status(engine): +def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: """Return a report of the current engine status""" tests = [ "time()-engine.start_time", @@ -23,7 +27,7 @@ def get_engine_status(engine): "engine.scraper.slot.needs_backout()", ] - checks = [] + checks: List[Tuple[str, Any]] = [] for test in tests: try: checks += [(test, eval(test))] @@ -33,7 +37,7 @@ def get_engine_status(engine): return checks -def format_engine_status(engine=None): +def format_engine_status(engine: "ExecutionEngine") -> str: checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: @@ -43,5 +47,5 @@ def format_engine_status(engine=None): return s -def print_engine_status(engine): +def print_engine_status(engine: "ExecutionEngine") -> None: print(format_engine_status(engine)) From f38cea9c8c5410e0553c666f090fb150a68ae591 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 19:22:57 +0400 Subject: [PATCH 0965/2083] Add typing to scrapy/utils/display.py. --- scrapy/utils/display.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 77c32b002..596cf89e4 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -6,17 +6,18 @@ import ctypes import platform import sys from pprint import pformat as pformat_ +from typing import Any from packaging.version import Version as parse_version -def _enable_windows_terminal_processing(): +def _enable_windows_terminal_processing() -> bool: # https://stackoverflow.com/a/36760881 - kernel32 = ctypes.windll.kernel32 + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7)) -def _tty_supports_color(): +def _tty_supports_color() -> bool: if sys.platform != "win32": return True @@ -28,7 +29,7 @@ def _tty_supports_color(): return _enable_windows_terminal_processing() -def _colorize(text, colorize=True): +def _colorize(text: str, colorize: bool = True) -> str: if not colorize or not sys.stdout.isatty() or not _tty_supports_color(): return text try: @@ -42,9 +43,9 @@ def _colorize(text, colorize=True): return highlight(text, PythonLexer(), TerminalFormatter()) -def pformat(obj, *args, **kwargs): +def pformat(obj: Any, *args: Any, **kwargs: Any) -> str: return _colorize(pformat_(obj), kwargs.pop("colorize", True)) -def pprint(obj, *args, **kwargs): +def pprint(obj: Any, *args: Any, **kwargs: Any) -> None: print(pformat(obj, *args, **kwargs)) From 54fa04aa0a47314f121ff5a7627e7e47d4d2c013 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 20:32:34 +0400 Subject: [PATCH 0966/2083] Add typing to scrapy/utils/test.py, fix a FTP test. --- scrapy/crawler.py | 18 +++++++------ scrapy/spiderloader.py | 11 +++++--- scrapy/utils/test.py | 49 ++++++++++++++++++++++++------------ tests/test_pipeline_files.py | 41 +++++++++++++++--------------- 4 files changed, 71 insertions(+), 48 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 256f6e2c5..9631c73d6 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,11 +4,13 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Type, Union from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement +from scrapy.spiderloader import SpiderLoader + try: # zope >= 5.0 only supports MultipleInvalid from zope.interface.exceptions import MultipleInvalid @@ -171,7 +173,7 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings): + def _get_spider_loader(settings) -> SpiderLoader: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) @@ -190,13 +192,13 @@ class CrawlerRunner: ) return loader_cls.from_settings(settings.frozencopy()) - def __init__(self, settings=None): + def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings self.spider_loader = self._get_spider_loader(settings) - self._crawlers = set() - self._active = set() + self._crawlers: Set[Crawler] = set() + self._active: Set[defer.Deferred] = set() self.bootstrap_failed = False @property @@ -252,7 +254,9 @@ class CrawlerRunner: return d.addBoth(_done) - def create_crawler(self, crawler_or_spidercls): + def create_crawler( + self, crawler_or_spidercls: Union[Type[Spider], str, Crawler] + ) -> Crawler: """ Return a :class:`~scrapy.crawler.Crawler` object. @@ -272,7 +276,7 @@ class CrawlerRunner: return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) - def _create_crawler(self, spidercls): + def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 02a451a2b..ea5a26e77 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,10 +1,13 @@ import traceback import warnings from collections import defaultdict +from typing import DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer +from scrapy import Spider from scrapy.interfaces import ISpiderLoader +from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes @@ -16,11 +19,11 @@ class SpiderLoader: in a Scrapy project. """ - def __init__(self, settings): + def __init__(self, settings: BaseSettings): self.spider_modules = settings.getlist("SPIDER_MODULES") self.warn_only = settings.getbool("SPIDER_LOADER_WARN_ONLY") - self._spiders = {} - self._found = defaultdict(list) + self._spiders: Dict[str, Type[Spider]] = {} + self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list) self._load_all_spiders() def _check_name_duplicates(self): @@ -68,7 +71,7 @@ class SpiderLoader: def from_settings(cls, settings): return cls(settings) - def load(self, spider_name): + def load(self, spider_name: str) -> Type[Spider]: """ Return the Spider class for the given spider name. If the spider name is not found, raise a KeyError. diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 58576903a..97de8d25a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -7,24 +7,30 @@ import os from importlib import import_module from pathlib import Path from posixpath import split -from unittest import mock +from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type +from unittest import TestCase, mock +from twisted.internet.defer import Deferred from twisted.trial.unittest import SkipTest +from scrapy import Spider +from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available -def assert_gcs_environ(): +def assert_gcs_environ() -> None: if "GCS_PROJECT_ID" not in os.environ: raise SkipTest("GCS_PROJECT_ID not found") -def skip_if_no_boto(): +def skip_if_no_boto() -> None: if not is_botocore_available(): raise SkipTest("missing botocore library") -def get_gcs_content_and_delete(bucket, path): +def get_gcs_content_and_delete( + bucket: Any, path: str +) -> Tuple[bytes, List[Dict[str, str]], Any]: from google.cloud import storage client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) @@ -37,8 +43,13 @@ def get_gcs_content_and_delete(bucket, path): def get_ftp_content_and_delete( - path, host, port, username, password, use_active_mode=False -): + path: str, + host: str, + port: int, + username: str, + password: str, + use_active_mode: bool = False, +) -> bytes: from ftplib import FTP ftp = FTP() @@ -46,19 +57,23 @@ def get_ftp_content_and_delete( ftp.login(username, password) if use_active_mode: ftp.set_pasv(False) - ftp_data = [] + ftp_data: List[bytes] = [] - def buffer_data(data): + def buffer_data(data: bytes) -> None: ftp_data.append(data) ftp.retrbinary(f"RETR {path}", buffer_data) dirname, filename = split(path) ftp.cwd(dirname) ftp.delete(filename) - return "".join(ftp_data) + return b"".join(ftp_data) -def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): +def get_crawler( + spidercls: Optional[Type[Spider]] = None, + settings_dict: Optional[Dict[str, Any]] = None, + prevent_warnings: bool = True, +) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. @@ -82,7 +97,7 @@ def get_pythonpath() -> str: return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get("PYTHONPATH", "") -def get_testenv(): +def get_testenv() -> Dict[str, str]: """Return a OS environment dict suitable to fork processes that need to import this installation of Scrapy, instead of a system installed one. """ @@ -91,21 +106,23 @@ def get_testenv(): return env -def assert_samelines(testcase, text1, text2, msg=None): +def assert_samelines( + testcase: TestCase, text1: str, text2: str, msg: Optional[str] = None +) -> None: """Asserts text1 and text2 have the same lines, ignoring differences in line endings between platforms """ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) -def get_from_asyncio_queue(value): - q = asyncio.Queue() +def get_from_asyncio_queue(value: Any) -> Coroutine: + q: asyncio.Queue = asyncio.Queue() getter = q.get() q.put_nowait(value) return getter -def mock_google_cloud_storage(): +def mock_google_cloud_storage() -> Tuple[Any, Any, Any]: """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. """ @@ -122,7 +139,7 @@ def mock_google_cloud_storage(): return (client_mock, bucket_mock, blob_mock) -def get_web_client_agent_req(url): +def get_web_client_agent_req(url: str) -> Deferred: from twisted.internet import reactor from twisted.web.client import Agent # imports twisted.internet.reactor diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c80666586..859ad6f9c 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -32,6 +32,7 @@ from scrapy.utils.test import ( get_gcs_content_and_delete, skip_if_no_boto, ) +from tests.mockserver import MockFTPServer from .test_pipeline_media import _mocked_download_func @@ -639,31 +640,29 @@ class TestGCSFilesStore(unittest.TestCase): class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): - uri = os.environ.get("FTP_TEST_FILE_URI") - if not uri: - raise unittest.SkipTest("No FTP URI available for testing") data = b"TestFTPFilesStore: \xe2\x98\x83" buf = BytesIO(data) meta = {"foo": "bar"} path = "full/filename" - store = FTPFilesStore(uri) - empty_dict = yield store.stat_file(path, info=None) - self.assertEqual(empty_dict, {}) - yield store.persist_file(path, buf, info=None, meta=meta, headers=None) - stat = yield store.stat_file(path, info=None) - self.assertIn("last_modified", stat) - self.assertIn("checksum", stat) - self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6") - path = f"{store.basedir}/{path}" - content = get_ftp_content_and_delete( - path, - store.host, - store.port, - store.username, - store.password, - store.USE_ACTIVE_MODE, - ) - self.assertEqual(data.decode(), content) + with MockFTPServer() as ftp_server: + store = FTPFilesStore(ftp_server.url("/")) + empty_dict = yield store.stat_file(path, info=None) + self.assertEqual(empty_dict, {}) + yield store.persist_file(path, buf, info=None, meta=meta, headers=None) + stat = yield store.stat_file(path, info=None) + self.assertIn("last_modified", stat) + self.assertIn("checksum", stat) + self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6") + path = f"{store.basedir}/{path}" + content = get_ftp_content_and_delete( + path, + store.host, + store.port, + store.username, + store.password, + store.USE_ACTIVE_MODE, + ) + self.assertEqual(data, content) class ItemWithFiles(Item): From 048812ba350d9f7fe6831d4c6f0d60d222b7f131 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 22:40:29 +0400 Subject: [PATCH 0967/2083] Bump types-* versions. --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index af8f1f57a..a1e956bf4 100644 --- a/tox.ini +++ b/tox.ini @@ -40,10 +40,10 @@ deps = mypy==1.2.0 types-attrs==19.1.0 types-lxml==2023.3.28 - types-Pillow==9.4.0.19 - types-Pygments==2.14.0.7 - types-pyOpenSSL==23.1.0.1 - types-setuptools==67.6.0.7 + types-Pillow==9.5.0.2 + types-Pygments==2.15.0.0 + types-pyOpenSSL==23.1.0.2 + types-setuptools==67.7.0.1 commands = mypy --show-error-codes {posargs: scrapy tests} From 0ec79e316619c1c98b0a1dd4fb0edea6e6de803d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 23:01:27 +0400 Subject: [PATCH 0968/2083] Fix compatibility with Python 3.8. --- scrapy/utils/deprecate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index ab2719bb3..ea577c44a 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -2,7 +2,7 @@ import inspect import warnings -from typing import Any, List, Optional, Tuple, Type, overload +from typing import Any, Dict, List, Optional, Tuple, Type, overload from scrapy.exceptions import ScrapyDeprecationWarning @@ -20,7 +20,7 @@ def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> No def create_deprecated_class( name: str, new_class: type, - clsdict: Optional[dict[str, Any]] = None, + clsdict: Optional[Dict[str, Any]] = None, warn_category: Type[Warning] = ScrapyDeprecationWarning, warn_once: bool = True, old_class_path: Optional[str] = None, @@ -59,14 +59,14 @@ def create_deprecated_class( warned_on_subclass: bool = False def __new__( - metacls, name: str, bases: Tuple[type, ...], clsdict_: dict[str, Any] + metacls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any] ) -> type: cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls - def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: dict[str, Any]): + def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any]): meta = cls.__class__ old = meta.deprecated_class if old in bases and not (warn_once and meta.warned_on_subclass): From e03c6bb70a8915f997771472ef05efc0c3ad6bd6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 23:03:35 +0400 Subject: [PATCH 0969/2083] Fix pylint issues. --- scrapy/crawler.py | 3 +-- scrapy/utils/gz.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 9631c73d6..69ff07bb7 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -9,8 +9,6 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Type, Union from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement -from scrapy.spiderloader import SpiderLoader - try: # zope >= 5.0 only supports MultipleInvalid from zope.interface.exceptions import MultipleInvalid @@ -27,6 +25,7 @@ from scrapy.interfaces import ISpiderLoader from scrapy.logformatter import LogFormatter from scrapy.settings import Settings, overridden_settings from scrapy.signalmanager import SignalManager +from scrapy.spiderloader import SpiderLoader from scrapy.statscollectors import StatsCollector from scrapy.utils.log import ( LogCounterHandler, diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 98ca510ed..c0eb77e07 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -24,8 +24,7 @@ def gunzip(data: bytes) -> bytes: # some pages are quite small so output_list is empty if output_list: break - else: - raise + raise return b"".join(output_list) From 33153855ea04d95545970970c3be08226b679f10 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Sun, 7 May 2023 22:29:23 +0300 Subject: [PATCH 0970/2083] periodic_log: implemented as separate extension --- scrapy/extensions/logstats_extended.py | 112 +++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 scrapy/extensions/logstats_extended.py diff --git a/scrapy/extensions/logstats_extended.py b/scrapy/extensions/logstats_extended.py new file mode 100644 index 000000000..7cba0dc2b --- /dev/null +++ b/scrapy/extensions/logstats_extended.py @@ -0,0 +1,112 @@ +import logging +from datetime import datetime + +from twisted.internet import task + +from scrapy import signals +from scrapy.exceptions import NotConfigured +from scrapy.utils.serialize import ScrapyJSONEncoder + +logger = logging.getLogger(__name__) + + +class LogStatsExtended: + """Log basic scraping stats periodically""" + + def __init__( + self, stats, interval=60.0, extended=False, ext_include=None, ext_exclude=None + ): + self.stats = stats + self.interval = interval + self.multiplier = 60.0 / self.interval + self.task = None + self.extended = extended + self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4) + self.ext_include = ext_include + self.ext_exclude = ext_exclude + + @classmethod + def from_crawler(cls, crawler): + interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + extended = crawler.settings.getbool("LOGSTATS_EXTENDED_ENABLED") + ext_include = crawler.settings.getlist("LOGSTATS_EXTENDED_INCLUDE", []) + ext_exclude = crawler.settings.getlist("LOGSTATS_EXTENDED_EXCLUDE", []) + if not interval: + raise NotConfigured + o = cls(crawler.stats, interval, extended, ext_include, ext_exclude) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) + return o + + def spider_opened(self, spider): + self.time_prev = datetime.utcnow() + self.delta_prev = {} + self.stats_prev = {} + + self.task = task.LoopingCall(self.log, spider) + self.task.start(self.interval) + + def log(self, spider): + data = {} + data.update(self.log_timing()) + data.update(self.log_delta()) + data.update(self.log_crawler_stats()) + logger.info(self.encoder.encode(data)) + + + def log_delta(self): + num_stats = { + k: v + for k, v in self.stats._stats.items() + if isinstance(v, (int, float)) and self.delta_param_allowed(k) + } + delta = {k: v - self.delta_prev.get(k, 0) for k, v in num_stats.items()} + self.delta_prev = num_stats + return {"delta": delta} + + def log_timing(self): + now = datetime.utcnow() + time = { + "log_interval": self.interval, + "start_time": self.stats._stats["start_time"], + "utcnow": now, + "log_interval_real": (now - self.time_prev).total_seconds(), + "elapsed": (now - self.stats._stats["start_time"]).total_seconds(), + } + self.time_prev = now + return {"time": time} + + def log_time(self): + num_stats = { + k: v + for k, v in self.stats._stats.items() + if isinstance(v, (int, float)) and self.delta_param_allowed(k) + } + delta = {k: v - self.stats_prev.get(k, 0) for k, v in num_stats.items()} + self.stats_prev = num_stats + return {"delta": delta} + + def log_crawler_stats(self): + return {"stats": self.stats.get_stats()} + + def delta_param_allowed(self, stat_name): + for p in self.ext_exclude: + if p in stat_name: + return False + for p in self.ext_include: + if p in stat_name: + return True + if self.ext_include: + return False + else: + return True + + def spider_closed(self, spider, reason): + data = {} + data.update(self.log_timing()) + data.update(self.log_delta()) + data.update(self.log_crawler_stats()) + logger.info(self.encoder.encode(data)) + + if self.task and self.task.running: + self.task.stop() \ No newline at end of file From caa66fa15ae5d353434e5bb1d0c1cc2bdf857b6c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 8 May 2023 13:27:01 +0400 Subject: [PATCH 0971/2083] Mention deprecating _FeedSlot. --- docs/news.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index cbbb376e5..4e198c893 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -15,6 +15,13 @@ Highlights: - JMESPath selectors from the new parsel_. - Bug fixes. +Deprecations +~~~~~~~~~~~~ + +- :class:`scrapy.extensions.feedexport._FeedSlot` is renamed to + :class:`scrapy.extensions.feedexport.FeedSlot` and the old name is + deprecated. (:issue:`5876`) + New features ~~~~~~~~~~~~ From 52c072640aa61884de05214cb1bdda07c2a87bef Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 8 May 2023 14:30:06 +0400 Subject: [PATCH 0972/2083] =?UTF-8?q?Bump=20version:=202.8.0=20=E2=86=92?= =?UTF-8?q?=202.9.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 4cfba674d..a00b7cfb3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.8.0 +current_version = 2.9.0 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index 4e198c893..c7ad11862 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.9.0: -Scrapy 2.9.0 (YYYY-MM-DD) +Scrapy 2.9.0 (2023-05-08) ------------------------- Highlights: diff --git a/scrapy/VERSION b/scrapy/VERSION index 834f26295..c8e38b614 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.8.0 +2.9.0 From c327a92e971411e50e49dded772a73b293f9f0a9 Mon Sep 17 00:00:00 2001 From: bulat Date: Tue, 9 May 2023 18:04:18 +0500 Subject: [PATCH 0973/2083] add additional requests examples. --- docs/topics/asyncio.rst | 47 +++++++++++++++++++++++++++++++++++++++++ scrapy/utils/defer.py | 10 +++++---- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7713b1af1..7aa83f505 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -98,6 +98,53 @@ Futures. Scrapy provides two helpers for this: into your own code. +Async additional requests +===================== + +The spider below shows a single use-case of scraping page and gathering price from a separate url:: + + + class SingleRequestSpider(scrapy.Spider): + name = "single" + start_urls = ["https://example.org/product"] + + async def parse(self, response, **kwargs): + additional_request = scrapy.Request('https://example.org/price') + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) + yield { + 'h1': response.css('h1').get(), + 'price': additional_response.css('#price').get(), + } + + +Spider with gathering batch requests:: + + class BatchRequestsSpider(scrapy.Spider): + name = "batch" + start_urls = ["https://example.com/product"] + + async def parse(self, response, **kwargs): + additional_requests = [ + scrapy.Request("https://example.com/price1"), + scrapy.Request("https://example.com/price2"), + ] + coroutines = [] + for r in additional_requests: + deffered = self.crawler.engine.download(r) + coroutines.append(maybe_deferred_to_future(deffered)) + + responses = await asyncio.gather( + *coroutines, return_exceptions=True + ) + yield { + 'h1': response.css('h1::text').get(), + 'price': responses[0].css('.price_color::text').get(), + 'price2': responses[1].css('.price_color::text').get(), + } + + + .. _enforce-asyncio-requirement: Enforcing asyncio as a requirement diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d25ebbdf4..a46274fef 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -340,8 +340,9 @@ def deferred_to_future(d: Deferred) -> Future: class MySpider(Spider): ... async def parse(self, response): - d = treq.get('https://example.com/additional') - additional_response = await deferred_to_future(d) + additional_request = scrapy.Request('https://example.org/price') + deferred = self.crawler.engine.download(additional_request) + additional_response = await deferred_to_future(deferred) """ return d.asFuture(_get_asyncio_event_loop()) @@ -368,8 +369,9 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: class MySpider(Spider): ... async def parse(self, response): - d = treq.get('https://example.com/additional') - extra_response = await maybe_deferred_to_future(d) + additional_request = scrapy.Request('https://example.org/price') + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) """ if not is_asyncio_reactor_installed(): return d From a75231a1ecd9a08b31ea3c1d5b59e457ac85ccf2 Mon Sep 17 00:00:00 2001 From: bulat Date: Tue, 9 May 2023 18:57:43 +0500 Subject: [PATCH 0974/2083] fix underline. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7aa83f505..d46527cfc 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -99,7 +99,7 @@ Futures. Scrapy provides two helpers for this: Async additional requests -===================== +========================= The spider below shows a single use-case of scraping page and gathering price from a separate url:: From d32c6782347c97086e804da0da01f45622743198 Mon Sep 17 00:00:00 2001 From: bulat Date: Tue, 9 May 2023 19:02:34 +0500 Subject: [PATCH 0975/2083] Update description. --- docs/topics/asyncio.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index d46527cfc..d439e0ab8 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -101,7 +101,7 @@ Futures. Scrapy provides two helpers for this: Async additional requests ========================= -The spider below shows a single use-case of scraping page and gathering price from a separate url:: +The spider below shows a single use-case of scraping a page and gathering a price from a separate URL:: class SingleRequestSpider(scrapy.Spider): @@ -118,7 +118,7 @@ The spider below shows a single use-case of scraping page and gathering price fr } -Spider with gathering batch requests:: +The spider gathering batch requests:: class BatchRequestsSpider(scrapy.Spider): name = "batch" From 99b0ece165ff27b70a6d7375a86bcc67da111df7 Mon Sep 17 00:00:00 2001 From: bulat Date: Tue, 9 May 2023 20:27:46 +0500 Subject: [PATCH 0976/2083] remove extra line. --- docs/topics/asyncio.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index d439e0ab8..0dab0ac5e 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -144,7 +144,6 @@ The spider gathering batch requests:: } - .. _enforce-asyncio-requirement: Enforcing asyncio as a requirement From 6998e1c905ef6e5fa737b32ac0b6e7f1b7701c14 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 10 May 2023 14:21:18 +0400 Subject: [PATCH 0977/2083] Fix typing-related issued on Python < 3.9. --- scrapy/utils/datatypes.py | 7 +++---- scrapy/utils/misc.py | 4 +--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 599b201ea..0f6bdc5ab 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -5,10 +5,9 @@ Python Standard Library. This module must not depend on any module outside the Standard Library. """ -import collections import weakref from collections.abc import Mapping -from typing import Any, Optional, Sequence, TypeVar +from typing import Any, Optional, OrderedDict, Sequence, TypeVar _KT = TypeVar("_KT") _VT = TypeVar("_VT") @@ -68,7 +67,7 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class LocalCache(collections.OrderedDict[_KT, _VT]): +class LocalCache(OrderedDict[_KT, _VT]): """Dictionary with a finite number of keys. Older items expires first. @@ -85,7 +84,7 @@ class LocalCache(collections.OrderedDict[_KT, _VT]): super().__setitem__(key, value) -class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT]): +class LocalWeakReferencedCache(weakref.WeakKeyDictionary): """ A weakref.WeakKeyDictionary implementation that uses LocalCache as its underlying data structure, making it ordered and capable of being size-limited. diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index defc8663d..70187ba74 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -232,9 +232,7 @@ def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: yield node -_generator_callbacks_cache: LocalWeakReferencedCache[ - Callable, bool -] = LocalWeakReferencedCache(limit=128) +_generator_callbacks_cache = LocalWeakReferencedCache(limit=128) def is_generator_with_return_value(callable: Callable) -> bool: From b1f4017788877ba2139f8621ecb5e821c62c111d Mon Sep 17 00:00:00 2001 From: bulat Date: Wed, 10 May 2023 15:34:58 +0500 Subject: [PATCH 0978/2083] Refactor batch sample. --- docs/topics/asyncio.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 0dab0ac5e..dc83148f5 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -126,8 +126,8 @@ The spider gathering batch requests:: async def parse(self, response, **kwargs): additional_requests = [ - scrapy.Request("https://example.com/price1"), - scrapy.Request("https://example.com/price2"), + scrapy.Request("https://example.com/price"), + scrapy.Request("https://example.com/color"), ] coroutines = [] for r in additional_requests: @@ -139,8 +139,8 @@ The spider gathering batch requests:: ) yield { 'h1': response.css('h1::text').get(), - 'price': responses[0].css('.price_color::text').get(), - 'price2': responses[1].css('.price_color::text').get(), + 'price': responses[0].css('.price::text').get(), + 'color': responses[1].css('color::text').get(), } From 87d10161cd413353eba2abf8ebdc2a8656927c43 Mon Sep 17 00:00:00 2001 From: bulat Date: Wed, 10 May 2023 15:35:48 +0500 Subject: [PATCH 0979/2083] Add selector as class. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index dc83148f5..f00ba0ff8 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -140,7 +140,7 @@ The spider gathering batch requests:: yield { 'h1': response.css('h1::text').get(), 'price': responses[0].css('.price::text').get(), - 'color': responses[1].css('color::text').get(), + 'color': responses[1].css('.color::text').get(), } From 57f3140daaa0166f924fcca42d3f3d3ef178bf92 Mon Sep 17 00:00:00 2001 From: Bulat Khabibullin Date: Wed, 10 May 2023 18:31:54 +0500 Subject: [PATCH 0980/2083] Update docs/topics/asyncio.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/asyncio.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index f00ba0ff8..f9efef108 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -101,8 +101,10 @@ Futures. Scrapy provides two helpers for this: Async additional requests ========================= -The spider below shows a single use-case of scraping a page and gathering a price from a separate URL:: +The spider below shows how to send a request and await its response all from +within a spider callback: +.. code-block:: python class SingleRequestSpider(scrapy.Spider): name = "single" From 26374e21f81eb53f5a21e1cc68a65e54fbb62cb6 Mon Sep 17 00:00:00 2001 From: Bulat Khabibullin Date: Wed, 10 May 2023 18:32:36 +0500 Subject: [PATCH 0981/2083] Update docs/topics/asyncio.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/asyncio.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index f9efef108..7fe78585a 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -119,8 +119,9 @@ within a spider callback: 'price': additional_response.css('#price').get(), } +You can also send multiple requests in parallel: -The spider gathering batch requests:: +.. code-block:: python class BatchRequestsSpider(scrapy.Spider): name = "batch" From 85103b493289011161731e4fafec4320cd85e0af Mon Sep 17 00:00:00 2001 From: Bulat Khabibullin Date: Thu, 11 May 2023 12:53:43 +0500 Subject: [PATCH 0982/2083] add proper example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/asyncio.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7fe78585a..eeec76157 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -123,6 +123,8 @@ You can also send multiple requests in parallel: .. code-block:: python + from twisted.internet.defer import DeferredList + class BatchRequestsSpider(scrapy.Spider): name = "batch" start_urls = ["https://example.com/product"] @@ -132,14 +134,11 @@ You can also send multiple requests in parallel: scrapy.Request("https://example.com/price"), scrapy.Request("https://example.com/color"), ] - coroutines = [] + deferreds = [] for r in additional_requests: - deffered = self.crawler.engine.download(r) - coroutines.append(maybe_deferred_to_future(deffered)) - - responses = await asyncio.gather( - *coroutines, return_exceptions=True - ) + deferred = self.crawler.engine.download(r) + deferreds.append(deferred) + responses = await maybe_deferred_to_future(DeferredList(deferreds)) yield { 'h1': response.css('h1::text').get(), 'price': responses[0].css('.price::text').get(), From 6194db133518b07b92bfdae35332c69e95f5c415 Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 12:54:01 +0500 Subject: [PATCH 0983/2083] Update title. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7fe78585a..1b670aaab 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -98,7 +98,7 @@ Futures. Scrapy provides two helpers for this: into your own code. -Async additional requests +Inline requests ========================= The spider below shows how to send a request and await its response all from From fc2d1b217130ebf605636049ea773196a50303a0 Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 12:56:29 +0500 Subject: [PATCH 0984/2083] make example reachable. --- docs/topics/asyncio.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 12bf548df..fcf44c0cb 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -125,8 +125,8 @@ You can also send multiple requests in parallel: from twisted.internet.defer import DeferredList - class BatchRequestsSpider(scrapy.Spider): - name = "batch" + class MultipleRequestsSpider(scrapy.Spider): + name = "multiple" start_urls = ["https://example.com/product"] async def parse(self, response, **kwargs): @@ -141,8 +141,8 @@ You can also send multiple requests in parallel: responses = await maybe_deferred_to_future(DeferredList(deferreds)) yield { 'h1': response.css('h1::text').get(), - 'price': responses[0].css('.price::text').get(), - 'color': responses[1].css('.color::text').get(), + 'price': responses[0][1].css('.price::text').get(), + 'price2': responses[1][1].css('.color::text').get(), } From b62c1263de4b026e8529416d5dede1795d47f7ad Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 13:15:30 +0500 Subject: [PATCH 0985/2083] add import to the example. --- docs/topics/asyncio.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index fcf44c0cb..2ad784335 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -106,6 +106,8 @@ within a spider callback: .. code-block:: python + from scrapy.utils.defer import maybe_deferred_to_future + class SingleRequestSpider(scrapy.Spider): name = "single" start_urls = ["https://example.org/product"] From 4878cc7ef04ae40279d80fec5d5234d17569ce11 Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 13:19:40 +0500 Subject: [PATCH 0986/2083] Add proper imports. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 2ad784335..a3f45a84b 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -125,7 +125,7 @@ You can also send multiple requests in parallel: .. code-block:: python - from twisted.internet.defer import DeferredList + from scrapy.utils.defer import DeferredList class MultipleRequestsSpider(scrapy.Spider): name = "multiple" From 8de2064ba33d6e0b8e0a22a6b5f6928a35eb44b7 Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 13:22:33 +0500 Subject: [PATCH 0987/2083] add import. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index a3f45a84b..5e0063be0 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -125,7 +125,7 @@ You can also send multiple requests in parallel: .. code-block:: python - from scrapy.utils.defer import DeferredList + from scrapy.utils.defer import DeferredList, maybe_deferred_to_future class MultipleRequestsSpider(scrapy.Spider): name = "multiple" From 5adada5d19e9e275462330b070b746305115112e Mon Sep 17 00:00:00 2001 From: isabela_catanante Date: Fri, 12 May 2023 12:55:24 +0200 Subject: [PATCH 0988/2083] Improve the overwrite feed option documentation --- docs/topics/feed-exports.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index eef0bb5ca..b4ac93b1d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -175,6 +175,12 @@ FTP supports two different connection modes: `active or passive mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +storage backend is: ``True``. + +.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the + previous version of your data. + This storage backend uses :ref:`delayed file delivery `. @@ -209,6 +215,12 @@ You can also define a custom ACL and custom endpoint for exported feeds using th - :setting:`FEED_STORAGE_S3_ACL` - :setting:`AWS_ENDPOINT_URL` +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +storage backend is: ``True``. + +.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the + previous version of your data. + This storage backend uses :ref:`delayed file delivery `. @@ -236,6 +248,12 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following - :setting:`FEED_STORAGE_GCS_ACL` - :setting:`GCS_PROJECT_ID` +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +storage backend is: ``True``. + +.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the + previous version of your data. + This storage backend uses :ref:`delayed file delivery `. .. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python @@ -488,6 +506,8 @@ as a fallback value if that key is not provided for a specific feed definition: - :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported `_) + - :ref:`topics-feed-storage-gcs`: ``True`` (appending is not supported) + - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) .. versionadded:: 2.4.0 From e4cf8fc121fc89d70949a9159bfe67cbd0429e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 15 May 2023 18:51:58 +0200 Subject: [PATCH 0989/2083] Update asyncio.rst --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 5e0063be0..efb93c844 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -99,7 +99,7 @@ Futures. Scrapy provides two helpers for this: Inline requests -========================= +=============== The spider below shows how to send a request and await its response all from within a spider callback: From d362699fa3855c7fd6e11204ccd8668128e38a5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 16 May 2023 13:39:02 +0200 Subject: [PATCH 0990/2083] Move inline request examples to the coroutines documentation --- docs/topics/asyncio.rst | 50 --------------------------------- docs/topics/coroutines.rst | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 50 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index efb93c844..7713b1af1 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -98,56 +98,6 @@ Futures. Scrapy provides two helpers for this: into your own code. -Inline requests -=============== - -The spider below shows how to send a request and await its response all from -within a spider callback: - -.. code-block:: python - - from scrapy.utils.defer import maybe_deferred_to_future - - class SingleRequestSpider(scrapy.Spider): - name = "single" - start_urls = ["https://example.org/product"] - - async def parse(self, response, **kwargs): - additional_request = scrapy.Request('https://example.org/price') - deferred = self.crawler.engine.download(additional_request) - additional_response = await maybe_deferred_to_future(deferred) - yield { - 'h1': response.css('h1').get(), - 'price': additional_response.css('#price').get(), - } - -You can also send multiple requests in parallel: - -.. code-block:: python - - from scrapy.utils.defer import DeferredList, maybe_deferred_to_future - - class MultipleRequestsSpider(scrapy.Spider): - name = "multiple" - start_urls = ["https://example.com/product"] - - async def parse(self, response, **kwargs): - additional_requests = [ - scrapy.Request("https://example.com/price"), - scrapy.Request("https://example.com/color"), - ] - deferreds = [] - for r in additional_requests: - deferred = self.crawler.engine.download(r) - deferreds.append(deferred) - responses = await maybe_deferred_to_future(DeferredList(deferreds)) - yield { - 'h1': response.css('h1::text').get(), - 'price': responses[0][1].css('.price::text').get(), - 'price2': responses[1][1].css('.color::text').get(), - } - - .. _enforce-asyncio-requirement: Enforcing asyncio as a requirement diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 3916bd295..a65bab3ca 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -134,6 +134,63 @@ Common use cases for asynchronous code include: .. _aio-libs: https://github.com/aio-libs +.. _inline-requests: + +Inline requests +=============== + +The spider below shows how to send a request and await its response all from +within a spider callback: + +.. code-block:: python + + from scrapy import Spider, Request + from scrapy.utils.defer import maybe_deferred_to_future + + + class SingleRequestSpider(Spider): + name = "single" + start_urls = ["https://example.org/product"] + + async def parse(self, response, **kwargs): + additional_request = Request("https://example.org/price") + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) + yield { + "h1": response.css("h1").get(), + "price": additional_response.css("#price").get(), + } + +You can also send multiple requests in parallel: + +.. code-block:: python + + from scrapy import Spider, Request + from scrapy.utils.defer import maybe_deferred_to_future + from twisted.internet.defer import DeferredList + + + class MultipleRequestsSpider(Spider): + name = "multiple" + start_urls = ["https://example.com/product"] + + async def parse(self, response, **kwargs): + additional_requests = [ + Request("https://example.com/price"), + Request("https://example.com/color"), + ] + deferreds = [] + for r in additional_requests: + deferred = self.crawler.engine.download(r) + deferreds.append(deferred) + responses = await maybe_deferred_to_future(DeferredList(deferreds)) + yield { + "h1": response.css("h1::text").get(), + "price": responses[0][1].css(".price::text").get(), + "price2": responses[1][1].css(".color::text").get(), + } + + .. _sync-async-spider-middleware: Mixing synchronous and asynchronous spider middlewares From 84fb0edd5fc9c994931dc8931bf3d8a44a63aa57 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Sun, 21 May 2023 20:22:35 +0300 Subject: [PATCH 0991/2083] periodic_log: extension updated --- scrapy/extensions/logstats_extended.py | 94 ++++++++++++++++---------- 1 file changed, 57 insertions(+), 37 deletions(-) diff --git a/scrapy/extensions/logstats_extended.py b/scrapy/extensions/logstats_extended.py index 7cba0dc2b..63703dace 100644 --- a/scrapy/extensions/logstats_extended.py +++ b/scrapy/extensions/logstats_extended.py @@ -14,26 +14,57 @@ class LogStatsExtended: """Log basic scraping stats periodically""" def __init__( - self, stats, interval=60.0, extended=False, ext_include=None, ext_exclude=None + self, stats, interval=60.0, + ext_stats_enabled=False, + ext_stats_include=None, + ext_stats_exclude=None, + ext_delta_enabled=False, + ext_delta_include=None, + ext_delta_exclude=None, + ext_timing_enabled=False, + ext_timing_include=None, + ext_timing_exclude=None, ): self.stats = stats self.interval = interval self.multiplier = 60.0 / self.interval self.task = None - self.extended = extended self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4) - self.ext_include = ext_include - self.ext_exclude = ext_exclude + self.ext_stats_enabled = ext_stats_enabled + self.ext_stats_include = ext_stats_include + self.ext_stats_exclude = ext_stats_exclude + self.ext_delta_enabled = ext_delta_enabled + self.ext_delta_include = ext_delta_include + self.ext_delta_exclude = ext_delta_exclude + self.ext_timing_enabled = ext_timing_enabled + self.ext_timing_include = ext_timing_include + self.ext_timing_exclude = ext_timing_exclude + @classmethod def from_crawler(cls, crawler): interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") - extended = crawler.settings.getbool("LOGSTATS_EXTENDED_ENABLED") - ext_include = crawler.settings.getlist("LOGSTATS_EXTENDED_INCLUDE", []) - ext_exclude = crawler.settings.getlist("LOGSTATS_EXTENDED_EXCLUDE", []) + ext_stats_enabled = crawler.settings.getbool("LOGSTATS_EXT_STATS_ENABLED") + ext_stats_include = crawler.settings.getlist("LOGSTATS_EXT_STATS_INCLUDE", []) + ext_stats_exclude = crawler.settings.getlist("LOGSTATS_EXT_STATS_EXCLUDE", []) + ext_delta_enabled = crawler.settings.getbool("LOGSTATS_EXT_DELTA_ENABLED") + ext_delta_include = crawler.settings.getlist("LOGSTATS_EXT_DELTA_INCLUDE", []) + ext_delta_exclude = crawler.settings.getlist("LOGSTATS_EXT_DELTA_EXCLUDE", []) + ext_timing_enabled = crawler.settings.getbool("LOGSTATS_EXT_TIMING_ENABLED") if not interval: raise NotConfigured - o = cls(crawler.stats, interval, extended, ext_include, ext_exclude) + if not (ext_stats_enabled or ext_delta_enabled or ext_timing_enabled): + raise NotConfigured + o = cls(crawler.stats, + interval, + ext_stats_enabled, + ext_stats_include, + ext_stats_exclude, + ext_delta_enabled, + ext_delta_include, + ext_delta_exclude, + ext_timing_enabled, + ) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o @@ -48,17 +79,19 @@ class LogStatsExtended: def log(self, spider): data = {} - data.update(self.log_timing()) - data.update(self.log_delta()) - data.update(self.log_crawler_stats()) + if self.ext_timing_enabled: + data.update(self.log_timing()) + if self.ext_delta_enabled: + data.update(self.log_delta()) + if self.ext_stats_enabled: + data.update(self.log_crawler_stats()) logger.info(self.encoder.encode(data)) - def log_delta(self): num_stats = { k: v for k, v in self.stats._stats.items() - if isinstance(v, (int, float)) and self.delta_param_allowed(k) + if isinstance(v, (int, float)) and self.param_allowed(k,self.ext_delta_include,self.ext_delta_exclude) } delta = {k: v - self.delta_prev.get(k, 0) for k, v in num_stats.items()} self.delta_prev = num_stats @@ -76,37 +109,24 @@ class LogStatsExtended: self.time_prev = now return {"time": time} - def log_time(self): - num_stats = { + def log_crawler_stats(self): + stats = { k: v for k, v in self.stats._stats.items() - if isinstance(v, (int, float)) and self.delta_param_allowed(k) - } - delta = {k: v - self.stats_prev.get(k, 0) for k, v in num_stats.items()} - self.stats_prev = num_stats - return {"delta": delta} + if self.param_allowed(k,self.ext_stats_include, self.ext_stats_exclude)} + return {"stats": stats} - def log_crawler_stats(self): - return {"stats": self.stats.get_stats()} - - def delta_param_allowed(self, stat_name): - for p in self.ext_exclude: + def param_allowed(self, stat_name, include, exclude): + for p in exclude: if p in stat_name: return False - for p in self.ext_include: + for p in include: if p in stat_name: return True - if self.ext_include: - return False - else: - return True + return False + def spider_closed(self, spider, reason): - data = {} - data.update(self.log_timing()) - data.update(self.log_delta()) - data.update(self.log_crawler_stats()) - logger.info(self.encoder.encode(data)) - + self.log(spider) if self.task and self.task.running: - self.task.stop() \ No newline at end of file + self.task.stop() From a2f238d927329ac308edcf09fde4e85dd9f61e44 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Mon, 22 May 2023 23:24:53 +0300 Subject: [PATCH 0992/2083] periodic_log: settings input in extension updated --- scrapy/extensions/logstats_extended.py | 48 +++++++++++++------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/scrapy/extensions/logstats_extended.py b/scrapy/extensions/logstats_extended.py index 63703dace..4f970d3d7 100644 --- a/scrapy/extensions/logstats_extended.py +++ b/scrapy/extensions/logstats_extended.py @@ -15,54 +15,52 @@ class LogStatsExtended: def __init__( self, stats, interval=60.0, - ext_stats_enabled=False, - ext_stats_include=None, - ext_stats_exclude=None, - ext_delta_enabled=False, - ext_delta_include=None, - ext_delta_exclude=None, + ext_stats={}, + ext_delta={}, ext_timing_enabled=False, - ext_timing_include=None, - ext_timing_exclude=None, ): self.stats = stats self.interval = interval self.multiplier = 60.0 / self.interval self.task = None self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4) - self.ext_stats_enabled = ext_stats_enabled - self.ext_stats_include = ext_stats_include - self.ext_stats_exclude = ext_stats_exclude - self.ext_delta_enabled = ext_delta_enabled - self.ext_delta_include = ext_delta_include - self.ext_delta_exclude = ext_delta_exclude + self.ext_stats_enabled = bool(ext_stats) + self.ext_stats_include = ext_stats.get("include", []) + self.ext_stats_exclude = ext_stats.get("exclude", []) + self.ext_delta_enabled = bool(ext_delta) + self.ext_delta_include = ext_delta.get("include", []) + self.ext_delta_exclude = ext_delta.get("exclude", []) self.ext_timing_enabled = ext_timing_enabled - self.ext_timing_include = ext_timing_include - self.ext_timing_exclude = ext_timing_exclude @classmethod def from_crawler(cls, crawler): interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + try: + ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS") + except: + ext_stats = {"enabled": True} if crawler.settings.getbool("PERIODIC_LOG_STATS") else None + + try: + ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA") + except: + ext_delta = {"enabled": True} if crawler.settings.getdict("PERIODIC_LOG_DELTA") else None + ext_timing_enabled = crawler.settings.getbool("PERIODIC_LOG_TIMING_ENABLED", False) ext_stats_enabled = crawler.settings.getbool("LOGSTATS_EXT_STATS_ENABLED") ext_stats_include = crawler.settings.getlist("LOGSTATS_EXT_STATS_INCLUDE", []) ext_stats_exclude = crawler.settings.getlist("LOGSTATS_EXT_STATS_EXCLUDE", []) ext_delta_enabled = crawler.settings.getbool("LOGSTATS_EXT_DELTA_ENABLED") ext_delta_include = crawler.settings.getlist("LOGSTATS_EXT_DELTA_INCLUDE", []) ext_delta_exclude = crawler.settings.getlist("LOGSTATS_EXT_DELTA_EXCLUDE", []) - ext_timing_enabled = crawler.settings.getbool("LOGSTATS_EXT_TIMING_ENABLED") + #ext_timing_enabled = crawler.settings.getbool("LOGSTATS_EXT_TIMING_ENABLED") if not interval: raise NotConfigured - if not (ext_stats_enabled or ext_delta_enabled or ext_timing_enabled): + if not (ext_stats or ext_delta or ext_timing_enabled): raise NotConfigured o = cls(crawler.stats, interval, - ext_stats_enabled, - ext_stats_include, - ext_stats_exclude, - ext_delta_enabled, - ext_delta_include, - ext_delta_exclude, + ext_stats, + ext_delta, ext_timing_enabled, ) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) @@ -117,6 +115,8 @@ class LogStatsExtended: return {"stats": stats} def param_allowed(self, stat_name, include, exclude): + if not include and not exclude: + return True for p in exclude: if p in stat_name: return False From db794d351c6218c8b849682bb1aa0048788e7f1d Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Mon, 22 May 2023 23:30:00 +0300 Subject: [PATCH 0993/2083] periodic_log: not used code deleted --- scrapy/extensions/logstats_extended.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/scrapy/extensions/logstats_extended.py b/scrapy/extensions/logstats_extended.py index 4f970d3d7..9c9a4fe4a 100644 --- a/scrapy/extensions/logstats_extended.py +++ b/scrapy/extensions/logstats_extended.py @@ -46,13 +46,6 @@ class LogStatsExtended: except: ext_delta = {"enabled": True} if crawler.settings.getdict("PERIODIC_LOG_DELTA") else None ext_timing_enabled = crawler.settings.getbool("PERIODIC_LOG_TIMING_ENABLED", False) - ext_stats_enabled = crawler.settings.getbool("LOGSTATS_EXT_STATS_ENABLED") - ext_stats_include = crawler.settings.getlist("LOGSTATS_EXT_STATS_INCLUDE", []) - ext_stats_exclude = crawler.settings.getlist("LOGSTATS_EXT_STATS_EXCLUDE", []) - ext_delta_enabled = crawler.settings.getbool("LOGSTATS_EXT_DELTA_ENABLED") - ext_delta_include = crawler.settings.getlist("LOGSTATS_EXT_DELTA_INCLUDE", []) - ext_delta_exclude = crawler.settings.getlist("LOGSTATS_EXT_DELTA_EXCLUDE", []) - #ext_timing_enabled = crawler.settings.getbool("LOGSTATS_EXT_TIMING_ENABLED") if not interval: raise NotConfigured if not (ext_stats or ext_delta or ext_timing_enabled): @@ -72,10 +65,10 @@ class LogStatsExtended: self.delta_prev = {} self.stats_prev = {} - self.task = task.LoopingCall(self.log, spider) + self.task = task.LoopingCall(self.log) self.task.start(self.interval) - def log(self, spider): + def log(self): data = {} if self.ext_timing_enabled: data.update(self.log_timing()) @@ -127,6 +120,6 @@ class LogStatsExtended: def spider_closed(self, spider, reason): - self.log(spider) + self.log() if self.task and self.task.running: self.task.stop() From 5c91f1bb43c00f9d64d16ecc1a5a976e14895ea5 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Tue, 23 May 2023 23:00:26 +0300 Subject: [PATCH 0994/2083] periodic_log: added settings to default settings --- scrapy/settings/default_settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 260ec1701..9660e0bcd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -240,6 +240,10 @@ METAREFRESH_MAXDELAY = 100 NEWSPIDER_MODULE = "" +PERIODIC_LOG_DELTA = None +PERIODIC_LOG_STATS = None +PERIODIC_LOG_TIMING_ENABLED = None + RANDOMIZE_DOWNLOAD_DELAY = True REACTOR_THREADPOOL_MAXSIZE = 10 From a0c84903b703a9405f2807873837ebf274873a60 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Tue, 23 May 2023 23:07:12 +0300 Subject: [PATCH 0995/2083] periodic_log: codestyle fix (from pre-commit) --- scrapy/extensions/logstats_extended.py | 51 ++++++++++++++++---------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/scrapy/extensions/logstats_extended.py b/scrapy/extensions/logstats_extended.py index 9c9a4fe4a..158c82868 100644 --- a/scrapy/extensions/logstats_extended.py +++ b/scrapy/extensions/logstats_extended.py @@ -14,10 +14,12 @@ class LogStatsExtended: """Log basic scraping stats periodically""" def __init__( - self, stats, interval=60.0, - ext_stats={}, - ext_delta={}, - ext_timing_enabled=False, + self, + stats, + interval=60.0, + ext_stats={}, + ext_delta={}, + ext_timing_enabled=False, ): self.stats = stats self.interval = interval @@ -32,30 +34,40 @@ class LogStatsExtended: self.ext_delta_exclude = ext_delta.get("exclude", []) self.ext_timing_enabled = ext_timing_enabled - @classmethod def from_crawler(cls, crawler): interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") try: ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS") - except: - ext_stats = {"enabled": True} if crawler.settings.getbool("PERIODIC_LOG_STATS") else None + except ValueError: + ext_stats = ( + {"enabled": True} + if crawler.settings.getbool("PERIODIC_LOG_STATS") + else None + ) try: ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA") - except: - ext_delta = {"enabled": True} if crawler.settings.getdict("PERIODIC_LOG_DELTA") else None - ext_timing_enabled = crawler.settings.getbool("PERIODIC_LOG_TIMING_ENABLED", False) + except ValueError: + ext_delta = ( + {"enabled": True} + if crawler.settings.getdict("PERIODIC_LOG_DELTA") + else None + ) + ext_timing_enabled = crawler.settings.getbool( + "PERIODIC_LOG_TIMING_ENABLED", False + ) if not interval: raise NotConfigured if not (ext_stats or ext_delta or ext_timing_enabled): raise NotConfigured - o = cls(crawler.stats, - interval, - ext_stats, - ext_delta, - ext_timing_enabled, - ) + o = cls( + crawler.stats, + interval, + ext_stats, + ext_delta, + ext_timing_enabled, + ) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o @@ -82,7 +94,8 @@ class LogStatsExtended: num_stats = { k: v for k, v in self.stats._stats.items() - if isinstance(v, (int, float)) and self.param_allowed(k,self.ext_delta_include,self.ext_delta_exclude) + if isinstance(v, (int, float)) + and self.param_allowed(k, self.ext_delta_include, self.ext_delta_exclude) } delta = {k: v - self.delta_prev.get(k, 0) for k, v in num_stats.items()} self.delta_prev = num_stats @@ -104,7 +117,8 @@ class LogStatsExtended: stats = { k: v for k, v in self.stats._stats.items() - if self.param_allowed(k,self.ext_stats_include, self.ext_stats_exclude)} + if self.param_allowed(k, self.ext_stats_include, self.ext_stats_exclude) + } return {"stats": stats} def param_allowed(self, stat_name, include, exclude): @@ -118,7 +132,6 @@ class LogStatsExtended: return True return False - def spider_closed(self, spider, reason): self.log() if self.task and self.task.running: From b60e0faf22eaee9cb26357c4c6637fba7e20f125 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Wed, 24 May 2023 23:10:30 +0300 Subject: [PATCH 0996/2083] periodic_log: fixed naming --- scrapy/extensions/{logstats_extended.py => periodic_log.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename scrapy/extensions/{logstats_extended.py => periodic_log.py} (99%) diff --git a/scrapy/extensions/logstats_extended.py b/scrapy/extensions/periodic_log.py similarity index 99% rename from scrapy/extensions/logstats_extended.py rename to scrapy/extensions/periodic_log.py index 158c82868..3e496096c 100644 --- a/scrapy/extensions/logstats_extended.py +++ b/scrapy/extensions/periodic_log.py @@ -10,7 +10,7 @@ from scrapy.utils.serialize import ScrapyJSONEncoder logger = logging.getLogger(__name__) -class LogStatsExtended: +class PeriodicLog: """Log basic scraping stats periodically""" def __init__( From 49839d6071832aab23093c34fa6ceb961fcdf9d0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 4 Jun 2023 19:59:58 +0400 Subject: [PATCH 0997/2083] Don't rely on get_testenv() for running mockserver. --- tests/mockserver.py | 19 +++++++++++++++---- tests/test_crawler.py | 11 ++++++++--- tests/test_proxy_connect.py | 3 --- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index eb4f8334e..647b0682e 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,11 +1,13 @@ import argparse import json +import os import random import sys from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp +from typing import Dict from urllib.parse import urlencode from OpenSSL import SSL @@ -20,7 +22,6 @@ from twisted.web.static import File from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.test import get_testenv def getarg(request, name, default=None, type=None): @@ -32,6 +33,16 @@ def getarg(request, name, default=None, type=None): return default +def get_mockserver_env() -> Dict[str, str]: + """Return a OS environment dict suitable to run mockserver processes.""" + + tests_path = Path(__file__).parent.parent + pythonpath = str(tests_path) + os.pathsep + os.environ.get("PYTHONPATH", "") + env = os.environ.copy() + env["PYTHONPATH"] = pythonpath + return env + + # most of the following resources are copied from twisted.web.test.test_webclient class ForeverTakingResource(resource.Resource): """ @@ -264,7 +275,7 @@ class MockServer: self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver", "-t", "http"], stdout=PIPE, - env=get_testenv(), + env=get_mockserver_env(), ) http_address = self.proc.stdout.readline().strip().decode("ascii") https_address = self.proc.stdout.readline().strip().decode("ascii") @@ -308,7 +319,7 @@ class MockDNSServer: self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver", "-t", "dns"], stdout=PIPE, - env=get_testenv(), + env=get_mockserver_env(), ) self.host = "127.0.0.1" self.port = int( @@ -331,7 +342,7 @@ class MockFTPServer: self.proc = Popen( [sys.executable, "-u", "-m", "tests.ftpserver", "-d", str(self.path)], stderr=PIPE, - env=get_testenv(), + env=get_mockserver_env(), ) for line in self.proc.stderr: if b"starting FTP server" in line: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 706bfbaa9..ecb9c9b62 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,4 +1,5 @@ import logging +import os import platform import subprocess import sys @@ -23,8 +24,8 @@ from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.misc import load_object from scrapy.utils.spider import DefaultSpider -from scrapy.utils.test import get_crawler, get_testenv -from tests.mockserver import MockServer +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer, get_mockserver_env class BaseCrawlerTest(unittest.TestCase): @@ -289,12 +290,16 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: script_dir: Path + cwd = os.getcwd() def run_script(self, script_name: str, *script_args): script_path = self.script_dir / script_name args = [sys.executable, str(script_path)] + list(script_args) p = subprocess.Popen( - args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE + args, + env=get_mockserver_env(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) stdout, stderr = p.communicate() return stderr.decode("utf-8") diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index c05f4da91..dc0a82086 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -21,8 +21,6 @@ class MitmProxy: auth_pass = "scrapy" def start(self): - from scrapy.utils.test import get_testenv - script = """ import sys from mitmproxy.tools.main import mitmdump @@ -46,7 +44,6 @@ sys.exit(mitmdump()) "--ssl-insecure", ], stdout=PIPE, - env=get_testenv(), ) line = self.proc.stdout.readline().decode("utf-8") host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1) From 493ea435384d4b5891129cbcf61d07b09a00ce7f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 12 Jun 2023 21:50:29 +0400 Subject: [PATCH 0998/2083] Improve finding tests.test_cmdline.settings. --- tests/test_cmdline/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 15833cd19..25ded143c 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,4 +1,5 @@ import json +import os import pstats import shutil import sys @@ -14,6 +15,8 @@ from scrapy.utils.test import get_testenv class CmdlineTest(unittest.TestCase): def setUp(self): self.env = get_testenv() + tests_path = Path(__file__).parent.parent + self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent) self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings" def _execute(self, *new_args, **kwargs): From 3f92882be4b35cf476f0c7c284e4fe1ba498e873 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Jun 2023 19:13:58 +0400 Subject: [PATCH 0999/2083] Fix a wrong merge. --- scrapy/extensions/feedexport.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index de3ed093c..7e93bc366 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -492,20 +492,6 @@ class FeedExporter: :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ storage = self._get_storage(uri, feed_options) - file = storage.open(spider) - if "postprocessing" in feed_options: - file = PostProcessingManager( - feed_options["postprocessing"], file, feed_options - ) - - exporter = self._get_exporter( - file=file, - format=feed_options["format"], - fields_to_export=feed_options["fields"], - encoding=feed_options["encoding"], - indent=feed_options["indent"], - **feed_options["item_export_kwargs"], - ) slot = FeedSlot( storage=storage, uri=uri, From 0adbd210acc9d1af75e42df2b17ee28e3fecdfc8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Jun 2023 19:34:26 +0400 Subject: [PATCH 1000/2083] Fix extra-deps-pinned tests. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 8a6693a49..5f8bf85f2 100644 --- a/tox.ini +++ b/tox.ini @@ -139,6 +139,7 @@ deps = install_command = {[pinned]install_command} setenv = {[pinned]setenv} +commands = {[pinned]commands} [testenv:asyncio] commands = From 075ad6f196e7936edfd75b57d10b7313da7fe4f4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 16:34:18 +0400 Subject: [PATCH 1001/2083] Test, linter etc. fixes. --- docs/topics/addons.rst | 24 ++- scrapy/addons/__init__.py | 106 +++++----- scrapy/addons/builtins.py | 135 +++++++----- scrapy/utils/misc.py | 2 +- tests/test_addons/__init__.py | 329 ++++++++++++++--------------- tests/test_addons/addonmod.py | 2 + tests/test_addons/addons.py | 11 +- tests/test_addons/test_builtins.py | 18 +- tests/test_crawl.py | 2 +- tests/test_middleware.py | 4 +- 10 files changed, 329 insertions(+), 304 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 4dab15a2a..ba6e839a5 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -350,29 +350,31 @@ Check configuration of fully initialized crawler (see Provide add-on interface through a module: -.. No idea why just using '::' doesn't work for this one .. code-block:: python - name = 'AddonModule' - version = '1.0' + name = "AddonModule" + version = "1.0" + class MyPipeline(object): - # ... + ... + class MyDownloaderMiddleware(object): - # ... + ... + def update_settings(config, settings): settings.set( - 'ITEM_PIPELINES', + "ITEM_PIPELINES", {MyPipeline(): 200}, - priority='addon', - } + priority="addon", + ) settings.set( - 'DOWNLOADER_MIDDLEWARES', + "DOWNLOADER_MIDDLEWARES", {MyDownloaderMiddleware(): 800}, - priority='addon', - } + priority="addon", + ) Forward to other add-ons depending on Python version:: diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 169c79eac..aad9ef789 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -1,10 +1,10 @@ -from collections import defaultdict, Mapping, OrderedDict -from inspect import isclass -import six import warnings +from collections import OrderedDict, defaultdict +from collections.abc import Mapping +from inspect import isclass -from pkg_resources import WorkingSet, Distribution, Requirement import zope.interface +from pkg_resources import Distribution, Requirement, WorkingSet from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon @@ -14,7 +14,6 @@ from scrapy.utils.misc import load_module_or_object @zope.interface.implementer(IAddon) class Addon(object): - basic_settings = None """``dict`` of settings that will be exported via :meth:`export_basics`.""" @@ -83,8 +82,8 @@ class Addon(object): else: # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} k = comp - v = config.get('order', self.component_order) - settings[self.component_type].update({k: v}, 'addon') + v = config.get("order", self.component_order) + settings[self.component_type].update({k: v}, "addon") def export_basics(self, settings): """Export the :attr:`basic_settings` attribute into the settings object. @@ -95,8 +94,8 @@ class Addon(object): :param settings: Settings object into which to expose the basic settings :type settings: :class:`~scrapy.settings.Settings` """ - for setting, value in six.iteritems(self.basic_settings or {}): - settings.set(setting, value, 'addon') + for setting, value in (self.basic_settings or {}).items(): + settings.set(setting, value, "addon") def export_config(self, config, settings): """Export the add-on configuration, all keys in caps and with @@ -121,14 +120,13 @@ class Addon(object): prefix = self.settings_prefix or self.name # Since default exported config is case-insensitive (everything will be # uppercased), make mapped config case-insensitive as well - conf_mapping = {k.lower(): v - for k, v in six.iteritems(self.config_mapping or {})} - for key, val in six.iteritems(conf): + conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()} + for key, val in conf.items(): if key.lower() in conf_mapping: key = conf_mapping[key.lower()] else: - key = (prefix + '_' + key).upper() - settings.set(key, val, 'addon') + key = (prefix + "_" + key).upper() + settings.set(key, val, "addon") def update_settings(self, config, settings): """Export both the basic settings and the add-on configuration. I.e., @@ -210,11 +208,11 @@ class AddonManager(Mapping): verifyObject(IAddon, addon) name = addon.name if name in self: - raise ValueError("Addon '{}' already loaded".format(name)) + raise ValueError(f"Addon '{name}' already loaded") self._addons[name] = addon self.configs[name] = config or {} if name in self._disable_on_add: - self.configs[name]['_enabled'] = False + self.configs[name]["_enabled"] = False self._disable_on_add.remove(name) def remove(self, addon): @@ -229,7 +227,7 @@ class AddonManager(Mapping): """ if addon in self: del self[addon] - elif hasattr(addon, 'name') and addon.name in self: + elif hasattr(addon, "name") and addon.name in self: del self[addon.name] else: try: @@ -250,14 +248,14 @@ class AddonManager(Mapping): :param path: Python or file path to an add-on :type path: ``str`` """ - if isinstance(path, six.string_types): + if isinstance(path, str): try: obj = load_module_or_object(path) except NameError: - raise NameError("Could not find add-on '%s'" % path) + raise NameError(f"Could not find add-on '{path}'") else: obj = path - if hasattr(obj, '_addon'): + if hasattr(obj, "_addon"): obj = AddonManager.get_addon(obj._addon) return obj @@ -284,7 +282,7 @@ class AddonManager(Mapping): and values correspond to their configuration :type addonsdict: ``dict`` """ - for addonpath, addoncfg in six.iteritems(addonsdict): + for addonpath, addoncfg in addonsdict.items(): self.add(addonpath, addoncfg) def load_settings(self, settings): @@ -299,7 +297,7 @@ class AddonManager(Mapping): which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - paths = build_component_list(settings['ADDONS']) + paths = build_component_list(settings["ADDONS"]) addons = [self.get_addon(path) for path in paths] configs = [settings.getdict(addon.name.upper()) for addon in addons] for a, c in zip(addons, configs): @@ -322,20 +320,23 @@ class AddonManager(Mapping): add-on. """ # Collect all active add-ons and the components they provide - ws = WorkingSet('') + ws = WorkingSet("") def add_dist(project_name, version, **kwargs): - if project_name in ws.entry_keys.get('scrapy', []): - raise ImportError("Component {} provided by multiple add-ons" - "".format(project_name)) + if project_name in ws.entry_keys.get("scrapy", []): + raise ImportError( + f"Component {project_name} provided by multiple add-ons" + ) else: - dist = Distribution(project_name=project_name, version=version, - **kwargs) - ws.add(dist, entry='scrapy') + dist = Distribution( + project_name=project_name, version=version, **kwargs + ) + ws.add(dist, entry="scrapy") + for name in self: ver = self[name].version add_dist(name, ver) - for provides_name in getattr(self[name], 'provides', []): + for provides_name in getattr(self[name], "provides", []): add_dist(provides_name, ver) # Collect all required and modified components @@ -345,8 +346,9 @@ class AddonManager(Mapping): for entry in getattr(self[name], attribute_name, []): attrs[entry].append(name) return attrs - modified = compile_attribute_dict('modifies') - required = compile_attribute_dict('requires') + + modified = compile_attribute_dict("modifies") + required = compile_attribute_dict("requires") req_or_mod = set(required.keys()).union(modified.keys()) for reqstr in req_or_mod: @@ -355,15 +357,16 @@ class AddonManager(Mapping): # our own exception or is it helpful enough? if ws.find(req) is None: raise ImportError( - "Add-ons {} require or modify missing component {}" - "".format(required[reqstr]+modified[reqstr], reqstr) + f"Add-ons {required[reqstr] + modified[reqstr]} require" + f" or modify missing component {reqstr}" ) mod_and_req = set(required.keys()).intersection(modified.keys()) for conflict in mod_and_req: - warnings.warn("Component '{}', required by add-ons {}, is modified " - "by add-ons {}".format(conflict, required[conflict], - modified[conflict])) + warnings.warn( + f"Component '{conflict}', required by add-ons {required[conflict]}," + f" is modified by add-ons {modified[conflict]}" + ) def disable(self, addon): """Disable an add-on, i.e. prevent its callbacks from being called. @@ -375,7 +378,7 @@ class AddonManager(Mapping): :type addon: ``str`` """ if addon in self: - self.configs[addon]['_enabled'] = False + self.configs[addon]["_enabled"] = False else: self._disable_on_add.append(addon) @@ -389,23 +392,23 @@ class AddonManager(Mapping): :type addon: ``str`` """ if addon in self: - self.configs[addon]['_enabled'] = True + self.configs[addon]["_enabled"] = True elif addon in self._disable_on_add: self._disable_on_add.remove(addon) else: - raise ValueError("Add-ons need to be added before they can be " - "enabled") + raise ValueError("Add-ons need to be added before they can be " "enabled") @property def disabled(self): """Names of disabled add-ons""" - return ([a for a in self if not self.configs[a].get('_enabled', True)] + - self._disable_on_add) + return [ + a for a in self if not self.configs[a].get("_enabled", True) + ] + self._disable_on_add @property def enabled(self): """Names of enabled add-ons""" - return [a for a in self if self.configs[a].get('_enabled', True)] + return [a for a in self if self.configs[a].get("_enabled", True)] def _call_if_exists(self, obj, cbname, *args, **kwargs): if obj is None: @@ -418,9 +421,10 @@ class AddonManager(Mapping): cb(*args, **kwargs) def _call_addon(self, addonname, cbname, *args, **kwargs): - if self.configs[addonname].get('_enabled', True): - self._call_if_exists(self[addonname], cbname, - self.configs[addonname], *args, **kwargs) + if self.configs[addonname].get("_enabled", True): + self._call_if_exists( + self[addonname], cbname, self.configs[addonname], *args, **kwargs + ) def update_addons(self): """Call ``update_addons()`` of all held add-ons. @@ -432,7 +436,7 @@ class AddonManager(Mapping): while called_addons != set(self): for name in set(self).difference(called_addons): called_addons.add(name) - self._call_addon(name, 'update_addons', self) + self._call_addon(name, "update_addons", self) def update_settings(self, settings): """Call ``update_settings()`` of all held add-ons. @@ -442,7 +446,7 @@ class AddonManager(Mapping): :type settings: :class:`~scrapy.settings.Settings` """ for name in self: - self._call_addon(name, 'update_settings', settings) + self._call_addon(name, "update_settings", settings) def check_configuration(self, crawler): """Call ``check_configuration()`` of all held add-ons. @@ -451,7 +455,7 @@ class AddonManager(Mapping): :type crawler: :class:`~scrapy.crawler.Crawler` """ for name in self: - self._call_addon(name, 'check_configuration', crawler) + self._call_addon(name, "check_configuration", crawler) -from scrapy.addons.builtins import * +from scrapy.addons.builtins import * # noqa diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py index 9babdeb6f..ea3afbf99 100644 --- a/scrapy/addons/builtins.py +++ b/scrapy/addons/builtins.py @@ -1,23 +1,44 @@ import scrapy from scrapy.addons import Addon -__all__ = ['make_builtin_addon', - - 'depth', 'httperror', 'offsite', 'referer', 'urllength', - - 'ajaxcrawl', 'chunked', 'cookies', 'defaultheaders', - 'downloadtimeout', 'httpauth', 'httpcache', 'httpcompression', - 'httpproxy', 'metarefresh', 'redirect', 'retry', 'robotstxt', - 'stats', 'useragent', - - 'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport', - 'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump', - 'statsmailer', 'telnetconsole', - ] +__all__ = [ + "make_builtin_addon", + "depth", + "httperror", + "offsite", + "referer", + "urllength", + "ajaxcrawl", + "chunked", + "cookies", + "defaultheaders", + "downloadtimeout", + "httpauth", + "httpcache", + "httpcompression", + "httpproxy", + "metarefresh", + "redirect", + "retry", + "robotstxt", + "stats", + "useragent", + "autothrottle", + "corestats", + "closespider", + "debugger", + "feedexport", + "logstats", + "memdebug", + "memusage", + "spiderstate", + "stacktracedump", + "statsmailer", + "telnetconsole", +] -def make_builtin_addon(addon_name, addon_default_config=None, - addon_version=None): +def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None): class ThisAddon(Addon): name = addon_name version = addon_version or scrapy.__version__ @@ -33,59 +54,65 @@ def make_builtin_addon(addon_name, addon_default_config=None, # SPIDER MIDDLEWARES -depth = make_builtin_addon('depth') +depth = make_builtin_addon("depth") -httperror = make_builtin_addon('httperror') +httperror = make_builtin_addon("httperror") -offsite = make_builtin_addon('offsite') +offsite = make_builtin_addon("offsite") -referer = make_builtin_addon('referer') +referer = make_builtin_addon("referer") -urllength = make_builtin_addon('urllength') +urllength = make_builtin_addon("urllength") # DOWNLOADER MIDDLEWARES -ajaxcrawl = make_builtin_addon('ajaxcrawl', {'enabled': True}) +ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True}) -chunked = make_builtin_addon('chunked') +chunked = make_builtin_addon("chunked") + +cookies = make_builtin_addon("cookies") + +defaultheaders = make_builtin_addon("defaultheaders") -cookies = make_builtin_addon('cookies') -defaultheaders = make_builtin_addon('defaultheaders') # Assume every config entry is a header def defaultheaders_export_config(self, config, settings): conf = self.default_config or {} conf.update(config) - settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon') + settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon") + + defaultheaders.export_config = defaultheaders_export_config -downloadtimeout = make_builtin_addon('downloadtimeout') -downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT', - 'download_timeout': 'DOWNLOAD_TIMEOUT'} +downloadtimeout = make_builtin_addon("downloadtimeout") +downloadtimeout.config_mapping = { + "timeout": "DOWNLOAD_TIMEOUT", + "download_timeout": "DOWNLOAD_TIMEOUT", +} -httpauth = make_builtin_addon('httpauth') +httpauth = make_builtin_addon("httpauth") -httpcache = make_builtin_addon('httpcache', {'enabled': True}) +httpcache = make_builtin_addon("httpcache", {"enabled": True}) -httpcompression = make_builtin_addon('httpcompression') -httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'} +httpcompression = make_builtin_addon("httpcompression") +httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"} -httpproxy = make_builtin_addon('httpproxy') +httpproxy = make_builtin_addon("httpproxy") -metarefresh = make_builtin_addon('metarefresh') -metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'} +metarefresh = make_builtin_addon("metarefresh") +metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"} -redirect = make_builtin_addon('redirect') +redirect = make_builtin_addon("redirect") -retry = make_builtin_addon('retry') +retry = make_builtin_addon("retry") -robotstxt = make_builtin_addon('robotstxt', {'obey': True}) +robotstxt = make_builtin_addon("robotstxt", {"obey": True}) -stats = make_builtin_addon('stats') +stats = make_builtin_addon("stats") -useragent = make_builtin_addon('useragent') -useragent.config_mapping = {'user_agent': 'USER_AGENT'} +useragent = make_builtin_addon("useragent") +useragent.config_mapping = {"user_agent": "USER_AGENT"} # ITEM PIPELINES @@ -93,27 +120,27 @@ useragent.config_mapping = {'user_agent': 'USER_AGENT'} # EXTENSIONS -autothrottle = make_builtin_addon('autothrottle', {'enabled': True}) +autothrottle = make_builtin_addon("autothrottle", {"enabled": True}) -corestats = make_builtin_addon('corestats') +corestats = make_builtin_addon("corestats") -closespider = make_builtin_addon('closespider') +closespider = make_builtin_addon("closespider") -debugger = make_builtin_addon('debugger') +debugger = make_builtin_addon("debugger") -feedexport = make_builtin_addon('feedexport') -feedexport.settings_prefix = 'FEED' +feedexport = make_builtin_addon("feedexport") +feedexport.settings_prefix = "FEED" -logstats = make_builtin_addon('logstats') +logstats = make_builtin_addon("logstats") -memdebug = make_builtin_addon('memdebug', {'enabled': True}) +memdebug = make_builtin_addon("memdebug", {"enabled": True}) -memusage = make_builtin_addon('memusage', {'enabled': True}) +memusage = make_builtin_addon("memusage", {"enabled": True}) -spiderstate = make_builtin_addon('spiderstate') +spiderstate = make_builtin_addon("spiderstate") -stacktracedump = make_builtin_addon('stacktracedump') +stacktracedump = make_builtin_addon("stacktracedump") -statsmailer = make_builtin_addon('statsmailer') +statsmailer = make_builtin_addon("statsmailer") -telnetconsole = make_builtin_addon('telnetconsole') +telnetconsole = make_builtin_addon("telnetconsole") diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 4bf7e7e66..8577cce02 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -85,7 +85,7 @@ def load_module_or_object(path): return load_object(path) except (ValueError, NameError, ImportError): pass - raise NameError("Could not load '%s'" % path) + raise NameError(f"Could not load '{path}'") def walk_modules(path): diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index a4e278fa5..741dd81cf 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,145 +1,143 @@ -from collections import OrderedDict import itertools -import os.path -import six -from tests import mock import unittest import warnings +from collections import OrderedDict +from unittest import mock from pkg_resources import VersionConflict -import zope.interface +from zope.interface import directlyProvides +from zope.interface.exceptions import BrokenImplementation, MultipleInvalid from zope.interface.verify import verifyObject -from zope.interface.exceptions import BrokenImplementation from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon from scrapy.settings import BaseSettings -from . import addons -from . import addonmod +from . import addonmod, addons class AddonTest(unittest.TestCase): - def setUp(self): self.rawaddon = Addon() class AddonWithAttributes(Addon): - name = 'Test' - version = '1.0' + name = "Test" + version = "1.0" + self.testaddon = AddonWithAttributes() def test_interface(self): # Raw Addon should fail exactly b/c name and version are not given - self.assertFalse(hasattr(self.rawaddon, 'name')) - self.assertFalse(hasattr(self.rawaddon, 'version')) - self.assertRaises(BrokenImplementation, verifyObject, IAddon, - self.rawaddon) + self.assertFalse(hasattr(self.rawaddon, "name")) + self.assertFalse(hasattr(self.rawaddon, "version")) + self.assertRaises(MultipleInvalid, verifyObject, IAddon, self.rawaddon) verifyObject(IAddon, self.testaddon) def test_export_component(self): - settings = BaseSettings({'ITEM_PIPELINES': BaseSettings(), - 'DOWNLOAD_HANDLERS': BaseSettings()}, - 'default') + settings = BaseSettings( + {"ITEM_PIPELINES": BaseSettings(), "DOWNLOAD_HANDLERS": BaseSettings()}, + "default", + ) self.testaddon.component_type = None self.testaddon.export_component({}, settings) - self.assertEqual(len(settings['ITEM_PIPELINES']), 0) - self.testaddon.component_type = 'ITEM_PIPELINES' - self.testaddon.component = 'test.component' + self.assertEqual(len(settings["ITEM_PIPELINES"]), 0) + self.testaddon.component_type = "ITEM_PIPELINES" + self.testaddon.component = "test.component" self.testaddon.export_component({}, settings) - six.assertCountEqual(self, settings['ITEM_PIPELINES'], - ['test.component']) - self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 0) + self.assertCountEqual(settings["ITEM_PIPELINES"], ["test.component"]) + self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 0) self.testaddon.component_order = 313 self.testaddon.export_component({}, settings) - self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 313) - self.testaddon.component_type = 'DOWNLOAD_HANDLERS' - self.testaddon.component_key = 'http' + self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 313) + self.testaddon.component_type = "DOWNLOAD_HANDLERS" + self.testaddon.component_key = "http" self.testaddon.export_component({}, settings) - self.assertEqual(settings['DOWNLOAD_HANDLERS']['http'], - 'test.component') + self.assertEqual(settings["DOWNLOAD_HANDLERS"]["http"], "test.component") def test_export_basics(self): settings = BaseSettings() - self.testaddon.basic_settings = {'TESTKEY': 313, 'OTHERKEY': True} + self.testaddon.basic_settings = {"TESTKEY": 313, "OTHERKEY": True} self.testaddon.export_basics(settings) - self.assertEqual(settings['TESTKEY'], 313) - self.assertEqual(settings['OTHERKEY'], True) - self.assertEqual(settings.getpriority('TESTKEY'), 15) + self.assertEqual(settings["TESTKEY"], 313) + self.assertEqual(settings["OTHERKEY"], True) + self.assertEqual(settings.getpriority("TESTKEY"), 15) def test_export_config(self): settings = BaseSettings() self.testaddon.settings_prefix = None - self.testaddon.config_mapping = {'MAPPED_key': 'MAPPING_WORKED'} - self.testaddon.default_config = {'key': 55, 'defaultkey': 100} - self.testaddon.export_config({'key': 313, 'OTHERKEY': True, - 'mapped_KEY': 99}, settings) - self.assertEqual(settings['TEST_KEY'], 313) - self.assertEqual(settings['TEST_DEFAULTKEY'], 100) - self.assertEqual(settings['TEST_OTHERKEY'], True) - self.assertNotIn('MAPPED_key', settings) - self.assertNotIn('MAPPED_KEY', settings) - self.assertEqual(settings['MAPPING_WORKED'], 99) - self.assertEqual(settings.getpriority('TEST_KEY'), 15) + self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} + self.testaddon.default_config = {"key": 55, "defaultkey": 100} + self.testaddon.export_config( + {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings + ) + self.assertEqual(settings["TEST_KEY"], 313) + self.assertEqual(settings["TEST_DEFAULTKEY"], 100) + self.assertEqual(settings["TEST_OTHERKEY"], True) + self.assertNotIn("MAPPED_key", settings) + self.assertNotIn("MAPPED_KEY", settings) + self.assertEqual(settings["MAPPING_WORKED"], 99) + self.assertEqual(settings.getpriority("TEST_KEY"), 15) - self.testaddon.settings_prefix = 'PREF' - self.testaddon.export_config({'newkey': 99}, settings) - self.assertEqual(settings['PREF_NEWKEY'], 99) + self.testaddon.settings_prefix = "PREF" + self.testaddon.export_config({"newkey": 99}, settings) + self.assertEqual(settings["PREF_NEWKEY"], 99) - with mock.patch.object(settings, 'set') as mock_set: + with mock.patch.object(settings, "set") as mock_set: self.testaddon.settings_prefix = False - self.testaddon.export_config({'thirdnewkey': 99}, settings) + self.testaddon.export_config({"thirdnewkey": 99}, settings) self.assertEqual(mock_set.call_count, 0) def test_update_settings(self): settings = BaseSettings() - settings.set('TEST_KEY1', 'default', priority='default') - settings.set('TEST_KEY2', 'project', priority='project') + settings.set("TEST_KEY1", "default", priority="default") + settings.set("TEST_KEY2", "project", priority="project") self.testaddon.settings_prefix = None - self.testaddon.basic_settings = {'OTHERTEST_KEY': 'addon'} - addon_config = {'key1': 'addon', 'key2': 'addon', 'key3': 'addon'} + self.testaddon.basic_settings = {"OTHERTEST_KEY": "addon"} + addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} self.testaddon.update_settings(addon_config, settings) - self.assertEqual(settings['OTHERTEST_KEY'], 'addon') - self.assertEqual(settings['TEST_KEY1'], 'addon') - self.assertEqual(settings['TEST_KEY2'], 'project') - self.assertEqual(settings['TEST_KEY3'], 'addon') + self.assertEqual(settings["OTHERTEST_KEY"], "addon") + self.assertEqual(settings["TEST_KEY1"], "addon") + self.assertEqual(settings["TEST_KEY2"], "project") + self.assertEqual(settings["TEST_KEY3"], "addon") class AddonManagerTest(unittest.TestCase): - def setUp(self): self.manager = AddonManager() def test_add(self): manager = AddonManager() - manager.add(addonmod, {'key': 'val1'}) - manager.add('tests.test_addons.addons.GoodAddon') - six.assertCountEqual(self, manager, ['AddonModule', 'GoodAddon']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + manager.add(addonmod, {"key": "val1"}) + manager.add("tests.test_addons.addons.GoodAddon") + self.assertCountEqual(manager, ["AddonModule", "GoodAddon"]) + self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) + self.assertCountEqual(manager.configs["AddonModule"], ["key"]) + self.assertEqual(manager.configs["AddonModule"]["key"], "val1") self.assertRaises(ValueError, manager.add, addonmod) def test_add_dont_instantiate_providing_classes(self): class ProviderGoodAddon(addons.GoodAddon): pass - zope.interface.directlyProvides(ProviderGoodAddon, IAddon) + + directlyProvides(ProviderGoodAddon, IAddon) manager = AddonManager() manager.add(ProviderGoodAddon) - self.assertIs(manager['GoodAddon'], ProviderGoodAddon) + self.assertIs(manager["GoodAddon"], ProviderGoodAddon) def test_add_verifies(self): - brokenaddon = self.manager.get_addon( - 'tests.test_addons.addons.BrokenAddon') - self.assertRaises(zope.interface.exceptions.BrokenImplementation, - self.manager.add, - brokenaddon) + brokenaddon = self.manager.get_addon("tests.test_addons.addons.BrokenAddon") + self.assertRaises( + BrokenImplementation, + self.manager.add, + brokenaddon, + ) def test_add_adds_missing_interface_declaration(self): class GoodAddonWithoutDeclaration(object): - name = 'GoodAddonWithoutDeclaration' - version = '1.0' + name = "GoodAddonWithoutDeclaration" + version = "1.0" + self.manager.add(GoodAddonWithoutDeclaration) def test_remove(self): @@ -147,64 +145,66 @@ class AddonManagerTest(unittest.TestCase): def test_gets_removed(removearg): manager.add(addonmod) - self.assertIn('AddonModule', manager) + self.assertIn("AddonModule", manager) manager.remove(removearg) - self.assertNotIn('AddonModule', manager) + self.assertNotIn("AddonModule", manager) - test_gets_removed('AddonModule') + test_gets_removed("AddonModule") test_gets_removed(addonmod) - test_gets_removed('tests.test_addons.addonmod') - self.assertRaises(KeyError, manager.remove, 'nonexistent') + test_gets_removed("tests.test_addons.addonmod") + self.assertRaises(KeyError, manager.remove, "nonexistent") self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) def test_get_addon(self): - goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon') + goodaddon = self.manager.get_addon("tests.test_addons.addons.GoodAddon") self.assertIs(goodaddon, addons.GoodAddon) loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod") self.assertIs(loaded_addonmod, addonmod) - addonspath = os.path.join(os.path.dirname(__file__), 'addons.py') goodaddon = self.manager.get_addon("tests.test_addons.addons") self.assertIsInstance(goodaddon, addons.GoodAddon) - self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent') + self.assertRaises(NameError, self.manager.get_addon, "xy.n_onexistent") def test_get_addon_forward(self): class SomeCls(object): - _addon = 'tests.test_addons.addons.GoodAddon' + _addon = "tests.test_addons.addons.GoodAddon" + self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon) def test_get_addon_nested(self): - x = addons.GoodAddon('outer') - x._addon = addons.GoodAddon('middle') - x._addon._addon = addons.GoodAddon('inner') + x = addons.GoodAddon("outer") + x._addon = addons.GoodAddon("middle") + x._addon._addon = addons.GoodAddon("inner") self.assertIs(self.manager.get_addon(x), x._addon._addon) def test_load_dict_load_settings(self): def _test_load_method(func, *args, **kwargs): manager = AddonManager() getattr(manager, func)(*args, **kwargs) - six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) - self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') - self.assertEqual(manager['AddonModule'], addonmod) - self.assertIn('key', manager.configs['AddonModule']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) + self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) + self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) + self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") + self.assertEqual(manager["AddonModule"], addonmod) + self.assertIn("key", manager.configs["AddonModule"]) + self.assertEqual(manager.configs["AddonModule"]["key"], "val1") addonsdict = { - "tests.test_addons.addonmod": {'key': 'val1'}, - 'tests.test_addons.addons.GoodAddon': {'key': 'val2'}, - } - _test_load_method('load_dict', addonsdict) + "tests.test_addons.addonmod": {"key": "val1"}, + "tests.test_addons.addons.GoodAddon": {"key": "val2"}, + } + _test_load_method("load_dict", addonsdict) settings = BaseSettings() - settings.set('ADDONS', {"tests.test_addons.addonmod": 0, - 'tests.test_addons.addons.GoodAddon': 0}) - settings.set('ADDONMODULE', {'key': 'val1'}) - settings.set('GOODADDON', {'key': 'val2'}) - _test_load_method('load_settings', settings) + settings.set( + "ADDONS", + {"tests.test_addons.addonmod": 0, "tests.test_addons.addons.GoodAddon": 0}, + ) + settings.set("ADDONMODULE", {"key": "val1"}) + settings.set("GOODADDON", {"key": "val2"}) + _test_load_method("load_settings", settings) def test_load_dict_load_settings_order(self): def _test_load_method(expected_order, func, *args, **kwargs): @@ -218,72 +218,71 @@ class AddonManagerTest(unittest.TestCase): for ordered_addons in itertools.permutations(addonlist): expected_order = [a.name for a in ordered_addons] addonsdict = OrderedDict((a, {}) for a in ordered_addons) - _test_load_method(expected_order, 'load_dict', addonsdict) - settings = BaseSettings({ - 'ADDONS': {a: i for i, a in enumerate(ordered_addons)} - }) - _test_load_method(expected_order, 'load_settings', settings) + _test_load_method(expected_order, "load_dict", addonsdict) + settings = BaseSettings( + {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} + ) + _test_load_method(expected_order, "load_settings", settings) def test_enabled_disabled(self): manager = AddonManager() - manager.add(addons.GoodAddon('FirstAddon')) - manager.add(addons.GoodAddon('SecondAddon')) - self.assertEqual(set(manager.enabled), - set(('FirstAddon', 'SecondAddon'))) + manager.add(addons.GoodAddon("FirstAddon")) + manager.add(addons.GoodAddon("SecondAddon")) + self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) self.assertEqual(manager.disabled, []) - manager.disable('FirstAddon') - self.assertEqual(manager.enabled, ['SecondAddon']) - self.assertEqual(manager.disabled, ['FirstAddon']) - manager.enable('FirstAddon') - self.assertEqual(set(manager.enabled), - set(('FirstAddon', 'SecondAddon'))) + manager.disable("FirstAddon") + self.assertEqual(manager.enabled, ["SecondAddon"]) + self.assertEqual(manager.disabled, ["FirstAddon"]) + manager.enable("FirstAddon") + self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) self.assertEqual(manager.disabled, []) def test_enable_before_add(self): manager = AddonManager() - self.assertRaises(ValueError, manager.enable, 'FirstAddon') - manager.disable('FirstAddon') - manager.enable('FirstAddon') - manager.add(addons.GoodAddon('FirstAddon')) - self.assertIn('FirstAddon', manager.enabled) + self.assertRaises(ValueError, manager.enable, "FirstAddon") + manager.disable("FirstAddon") + manager.enable("FirstAddon") + manager.add(addons.GoodAddon("FirstAddon")) + self.assertIn("FirstAddon", manager.enabled) def test_disable_before_add(self): manager = AddonManager() - manager.disable('FirstAddon') - manager.add(addons.GoodAddon('FirstAddon')) - self.assertEqual(manager.disabled, ['FirstAddon']) + manager.disable("FirstAddon") + manager.add(addons.GoodAddon("FirstAddon")) + self.assertEqual(manager.disabled, ["FirstAddon"]) def test_callbacks(self): - first_addon = addons.GoodAddon('FirstAddon') - second_addon = addons.GoodAddon('SecondAddon') + first_addon = addons.GoodAddon("FirstAddon") + second_addon = addons.GoodAddon("SecondAddon") manager = AddonManager() - manager.add(first_addon, {'test': 'first'}) - manager.add(second_addon, {'test': 'second'}) + manager.add(first_addon, {"test": "first"}) + manager.add(second_addon, {"test": "second"}) crawler = mock.create_autospec(Crawler) settings = BaseSettings() - with mock.patch.object(first_addon, 'update_addons') as ua_first, \ - mock.patch.object(second_addon, 'update_addons') as ua_second, \ - mock.patch.object(first_addon, 'update_settings') as us_first, \ - mock.patch.object(second_addon, 'update_settings') as us_second, \ - mock.patch.object(first_addon, 'check_configuration') as cc_first, \ - mock.patch.object(second_addon, 'check_configuration') as cc_second: + with mock.patch.object( + first_addon, "update_addons" + ) as ua_first, mock.patch.object( + second_addon, "update_addons" + ) as ua_second, mock.patch.object( + first_addon, "update_settings" + ) as us_first, mock.patch.object( + second_addon, "update_settings" + ) as us_second, mock.patch.object( + first_addon, "check_configuration" + ) as cc_first, mock.patch.object( + second_addon, "check_configuration" + ) as cc_second: manager.update_addons() - ua_first.assert_called_once_with(manager.configs['FirstAddon'], - manager) - ua_second.assert_called_once_with(manager.configs['SecondAddon'], - manager) + ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) + ua_second.assert_called_once_with(manager.configs["SecondAddon"], manager) manager.update_settings(settings) - us_first.assert_called_once_with(manager.configs['FirstAddon'], - settings) - us_second.assert_called_once_with(manager.configs['SecondAddon'], - settings) + us_first.assert_called_once_with(manager.configs["FirstAddon"], settings) + us_second.assert_called_once_with(manager.configs["SecondAddon"], settings) manager.check_configuration(crawler) - cc_first.assert_called_once_with(manager.configs['FirstAddon'], - crawler) - cc_second.assert_called_once_with(manager.configs['SecondAddon'], - crawler) + cc_first.assert_called_once_with(manager.configs["FirstAddon"], crawler) + cc_second.assert_called_once_with(manager.configs["SecondAddon"], crawler) self.assertEqual(ua_first.call_count, 1) self.assertEqual(ua_second.call_count, 1) self.assertEqual(us_first.call_count, 1) @@ -291,10 +290,10 @@ class AddonManagerTest(unittest.TestCase): us_first.reset_mock() us_second.reset_mock() - manager.disable('FirstAddon') + manager.disable("FirstAddon") manager.update_settings(settings) self.assertEqual(us_first.call_count, 0) - manager.enable('FirstAddon') + manager.enable("FirstAddon") manager.update_settings(settings) self.assertEqual(us_first.call_count, 1) self.assertEqual(us_second.call_count, 2) @@ -302,44 +301,42 @@ class AddonManagerTest(unittest.TestCase): # This will become relevant when we let spiders implement the add-on # interface and should be replaced with a test where # AddonManager.spidercls = None then. - manager._call_if_exists(None, 'irrelevant') + manager._call_if_exists(None, "irrelevant") def test_update_addons_last_minute_add(self): class AddedAddon(addons.GoodAddon): - name = 'AddedAddon' + name = "AddedAddon" class FirstAddon(addons.GoodAddon): - name = 'FirstAddon' + name = "FirstAddon" def update_addons(self, config, addons): addons.add(AddedAddon()) manager = AddonManager() first_addon = FirstAddon() - with mock.patch.object(first_addon, 'update_addons', - wraps=first_addon.update_addons) as ua_first, \ - mock.patch.object(AddedAddon, 'update_addons') as ua_added: - manager.add(first_addon, {'non-empty': 'dict'}) + with mock.patch.object( + first_addon, "update_addons", wraps=first_addon.update_addons + ) as ua_first, mock.patch.object(AddedAddon, "update_addons") as ua_added: + manager.add(first_addon, {"non-empty": "dict"}) manager.update_addons() - six.assertCountEqual(self, manager, ['FirstAddon', 'AddedAddon']) - ua_first.assert_called_once_with(manager.configs['FirstAddon'], - manager) - ua_added.assert_called_once_with(manager.configs['AddedAddon'], - manager) + self.assertCountEqual(manager, ["FirstAddon", "AddedAddon"]) + ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) + ua_added.assert_called_once_with(manager.configs["AddedAddon"], manager) def test_check_dependency_clashes_attributes(self): provides = addons.GoodAddon("ProvidesAddon") - provides.provides = ('test', ) + provides.provides = ("test",) provides2 = addons.GoodAddon("ProvidesAddon2") - provides2.provides = ('test', ) + provides2.provides = ("test",) requires = addons.GoodAddon("RequiresAddon") - requires.requires = ('test', ) + requires.requires = ("test",) requires_name = addons.GoodAddon("RequiresNameAddon") - requires_name.requires = ('ProvidesAddon', ) + requires_name.requires = ("ProvidesAddon",) requires_newer = addons.GoodAddon("RequiresNewerAddon") - requires_newer.requires = ('test>=2.0', ) + requires_newer.requires = ("test>=2.0",) modifies = addons.GoodAddon("ModifiesAddon") - modifies.modifies = ('test', ) + modifies.modifies = ("test",) def check_with(*addons): manager = AddonManager() diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py index 8ecf4b81d..c59f6737f 100644 --- a/tests/test_addons/addonmod.py +++ b/tests/test_addons/addonmod.py @@ -9,8 +9,10 @@ FROM = "test_addons.addonmod" name = "AddonModule" version = "1.0" + def update_settings(config, settings): pass + def check_configuration(config, crawler): pass diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py index f3442b192..4adb9fe8f 100644 --- a/tests/test_addons/addons.py +++ b/tests/test_addons/addons.py @@ -1,18 +1,16 @@ import zope.interface -from scrapy.addons import Addon from scrapy.interfaces import IAddon class Addon(object): - FROM = 'test_addons.addons' + FROM = "test_addons.addons" @zope.interface.declarations.implementer(IAddon) class GoodAddon(object): - - name = 'GoodAddon' - version = '1.0' + name = "GoodAddon" + version = "1.0" def __init__(self, name=None, version=None): if name is not None: @@ -32,8 +30,7 @@ class GoodAddon(object): @zope.interface.declarations.implementer(IAddon) class BrokenAddon(object): - - name = 'BrokenAddon' + name = "BrokenAddon" # No version diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py index c89876950..1050cbbed 100644 --- a/tests/test_addons/test_builtins.py +++ b/tests/test_addons/test_builtins.py @@ -7,19 +7,17 @@ from scrapy.settings import Settings class BuiltinAddonsTest(unittest.TestCase): - def test_make_builtin_addon(self): - httpcache = make_builtin_addon('httpcache', {'enabled': True}) - self.assertEqual(httpcache.name, 'httpcache') - self.assertEqual(httpcache.default_config, {'enabled': True}) + httpcache = make_builtin_addon("httpcache", {"enabled": True}) + self.assertEqual(httpcache.name, "httpcache") + self.assertEqual(httpcache.default_config, {"enabled": True}) self.assertEqual(httpcache.version, scrapy.__version__) - httpcache = make_builtin_addon('httpcache', {'enabled': True}, '99.9') - self.assertEqual(httpcache.version, '99.9') + httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9") + self.assertEqual(httpcache.version, "99.9") def test_defaultheaders_export_config(self): settings = Settings() dh = scrapy.addons.defaultheaders() - dh.export_config({'X-Test-Header': 'val'}, settings) - self.assertIn('X-Test-Header', settings['DEFAULT_REQUEST_HEADERS']) - self.assertEqual(settings['DEFAULT_REQUEST_HEADERS']['X-Test-Header'], - 'val') + dh.export_config({"X-Test-Header": "val"}, settings) + self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"]) + self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val") diff --git a/tests/test_crawl.py b/tests/test_crawl.py index e353f80cb..1e97863b0 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -419,7 +419,7 @@ with multiples lines addonmgr = AddonManager() addonmgr.add(FailedCheckAddon()) - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) crawler.addons = addonmgr # Doesn't work in 'precise' test environment: # with self.assertRaises(ValueError): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 7ddf952f7..fd57d846e 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -90,14 +90,12 @@ class MiddlewareManagerTest(unittest.TestCase): def test_instances_from_settings(self): settings = Settings() - myM3 = M3() class InstanceTestMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return ["tests.test_middleware.M1", M2, myM3] + return ["tests.test_middleware.M1", M2] mwman = InstanceTestMiddlewareManager.from_settings(settings) self.assertIsInstance(mwman.middlewares[0], M1) self.assertIsInstance(mwman.middlewares[1], M2) - self.assertIs(mwman.middlewares[2], myM3) From 55ac26228b853aec6d543bfaf3ab85ebbf5ab002 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 16:50:43 +0400 Subject: [PATCH 1002/2083] Remove builtin addons. --- scrapy/{addons/__init__.py => addons.py} | 3 - scrapy/addons/builtins.py | 146 ----------------------- sep/sep-021.rst | 25 ---- tests/test_addons/test_builtins.py | 23 ---- 4 files changed, 197 deletions(-) rename scrapy/{addons/__init__.py => addons.py} (99%) delete mode 100644 scrapy/addons/builtins.py delete mode 100644 tests/test_addons/test_builtins.py diff --git a/scrapy/addons/__init__.py b/scrapy/addons.py similarity index 99% rename from scrapy/addons/__init__.py rename to scrapy/addons.py index aad9ef789..c9b873039 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons.py @@ -456,6 +456,3 @@ class AddonManager(Mapping): """ for name in self: self._call_addon(name, "check_configuration", crawler) - - -from scrapy.addons.builtins import * # noqa diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py deleted file mode 100644 index ea3afbf99..000000000 --- a/scrapy/addons/builtins.py +++ /dev/null @@ -1,146 +0,0 @@ -import scrapy -from scrapy.addons import Addon - -__all__ = [ - "make_builtin_addon", - "depth", - "httperror", - "offsite", - "referer", - "urllength", - "ajaxcrawl", - "chunked", - "cookies", - "defaultheaders", - "downloadtimeout", - "httpauth", - "httpcache", - "httpcompression", - "httpproxy", - "metarefresh", - "redirect", - "retry", - "robotstxt", - "stats", - "useragent", - "autothrottle", - "corestats", - "closespider", - "debugger", - "feedexport", - "logstats", - "memdebug", - "memusage", - "spiderstate", - "stacktracedump", - "statsmailer", - "telnetconsole", -] - - -def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None): - class ThisAddon(Addon): - name = addon_name - version = addon_version or scrapy.__version__ - default_config = addon_default_config or {} - - return ThisAddon - - -# XXX: Below are CLASSES that have lowercase names. This is in line with the -# original SEP-021 but violates PEP8. -# We might consider prepending all built-in addon names with scrapy_ or similar -# to reduce the chance of name clashes. - -# SPIDER MIDDLEWARES - -depth = make_builtin_addon("depth") - -httperror = make_builtin_addon("httperror") - -offsite = make_builtin_addon("offsite") - -referer = make_builtin_addon("referer") - -urllength = make_builtin_addon("urllength") - - -# DOWNLOADER MIDDLEWARES - -ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True}) - -chunked = make_builtin_addon("chunked") - -cookies = make_builtin_addon("cookies") - -defaultheaders = make_builtin_addon("defaultheaders") - - -# Assume every config entry is a header -def defaultheaders_export_config(self, config, settings): - conf = self.default_config or {} - conf.update(config) - settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon") - - -defaultheaders.export_config = defaultheaders_export_config - -downloadtimeout = make_builtin_addon("downloadtimeout") -downloadtimeout.config_mapping = { - "timeout": "DOWNLOAD_TIMEOUT", - "download_timeout": "DOWNLOAD_TIMEOUT", -} - -httpauth = make_builtin_addon("httpauth") - -httpcache = make_builtin_addon("httpcache", {"enabled": True}) - -httpcompression = make_builtin_addon("httpcompression") -httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"} - -httpproxy = make_builtin_addon("httpproxy") - -metarefresh = make_builtin_addon("metarefresh") -metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"} - -redirect = make_builtin_addon("redirect") - -retry = make_builtin_addon("retry") - -robotstxt = make_builtin_addon("robotstxt", {"obey": True}) - -stats = make_builtin_addon("stats") - -useragent = make_builtin_addon("useragent") -useragent.config_mapping = {"user_agent": "USER_AGENT"} - - -# ITEM PIPELINES - - -# EXTENSIONS - -autothrottle = make_builtin_addon("autothrottle", {"enabled": True}) - -corestats = make_builtin_addon("corestats") - -closespider = make_builtin_addon("closespider") - -debugger = make_builtin_addon("debugger") - -feedexport = make_builtin_addon("feedexport") -feedexport.settings_prefix = "FEED" - -logstats = make_builtin_addon("logstats") - -memdebug = make_builtin_addon("memdebug", {"enabled": True}) - -memusage = make_builtin_addon("memusage", {"enabled": True}) - -spiderstate = make_builtin_addon("spiderstate") - -stacktracedump = make_builtin_addon("stacktracedump") - -statsmailer = make_builtin_addon("statsmailer") - -telnetconsole = make_builtin_addon("telnetconsole") diff --git a/sep/sep-021.rst b/sep/sep-021.rst index cb1701014..47cba004c 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -322,28 +322,3 @@ with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons`` submodule). If the object found has an ``_addon`` attribute, that attribute will be treated as the found add-on. This allows, for example, to change the add-on based on the Python version. - -Updating existing extensions ----------------------------- - -An ``Addon`` class is introduced that add-on developers may or may not subclass -depending on how much of the 'default functionality' they want. Naturally, it -does not provide ``NAME`` and ``VERSION``. Its default ``update_settings()`` -exposes the add-on configuration into the global settings namespace with an -appropriate name, e.g. this section from ``scrapy.cfg``:: - - [httpcache] - dir = /some/dir - -would expose ``HTTPCACHE_DIR``. - -Add-on modules will be written for all built-in extensions and placed in -``scrapy.addons``. For many default Scrapy components, it will be sufficient to -create a subclass of ``Addon`` with minor or no method modifications. The -component code remains where it is (i.e. in ``scrapy.pipelines``, etc.). - -Later, the global settings namespace could be cleaned up in a backwards --incompatible fashion by deprecating support for the global setting names, e.g. -``HTTPCACHE_DIR``, and instead instantiate the components with the add-on -configuration in ``update_settings()``. - diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py deleted file mode 100644 index 1050cbbed..000000000 --- a/tests/test_addons/test_builtins.py +++ /dev/null @@ -1,23 +0,0 @@ -import unittest - -import scrapy -import scrapy.addons -from scrapy.addons.builtins import make_builtin_addon -from scrapy.settings import Settings - - -class BuiltinAddonsTest(unittest.TestCase): - def test_make_builtin_addon(self): - httpcache = make_builtin_addon("httpcache", {"enabled": True}) - self.assertEqual(httpcache.name, "httpcache") - self.assertEqual(httpcache.default_config, {"enabled": True}) - self.assertEqual(httpcache.version, scrapy.__version__) - httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9") - self.assertEqual(httpcache.version, "99.9") - - def test_defaultheaders_export_config(self): - settings = Settings() - dh = scrapy.addons.defaultheaders() - dh.export_config({"X-Test-Header": "val"}, settings) - self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"]) - self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val") From fdbc141b23b95f8f744712f86c28a4d4d4d8c52f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 17:22:51 +0400 Subject: [PATCH 1003/2083] Replace pkg_resources with packaging. --- scrapy/addons.py | 28 ++++++++++++---------------- tests/test_addons/__init__.py | 3 +-- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index c9b873039..8f0f39889 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -2,9 +2,11 @@ import warnings from collections import OrderedDict, defaultdict from collections.abc import Mapping from inspect import isclass +from typing import Dict import zope.interface -from pkg_resources import Distribution, Requirement, WorkingSet +from packaging.requirements import Requirement +from packaging.version import Version from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon @@ -303,7 +305,7 @@ class AddonManager(Mapping): for a, c in zip(addons, configs): self.add(a, c) - def check_dependency_clashes(self): + def check_dependency_clashes(self) -> None: """Check for incompatibilities in add-on dependencies. Add-ons can provide information about their dependencies in their @@ -320,24 +322,20 @@ class AddonManager(Mapping): add-on. """ # Collect all active add-ons and the components they provide - ws = WorkingSet("") + versions: Dict[str, Version] = {} - def add_dist(project_name, version, **kwargs): - if project_name in ws.entry_keys.get("scrapy", []): + def add_version(project_name, version): + if project_name in versions: raise ImportError( f"Component {project_name} provided by multiple add-ons" ) - else: - dist = Distribution( - project_name=project_name, version=version, **kwargs - ) - ws.add(dist, entry="scrapy") + versions[project_name] = Version(version) for name in self: ver = self[name].version - add_dist(name, ver) + add_version(name, ver) for provides_name in getattr(self[name], "provides", []): - add_dist(provides_name, ver) + add_version(provides_name, ver) # Collect all required and modified components def compile_attribute_dict(attribute_name): @@ -352,10 +350,8 @@ class AddonManager(Mapping): req_or_mod = set(required.keys()).union(modified.keys()) for reqstr in req_or_mod: - req = Requirement.parse(reqstr) - # May raise VersionConflict. Do we want to catch it and raise - # our own exception or is it helpful enough? - if ws.find(req) is None: + req = Requirement(reqstr) + if req.name not in versions or versions[req.name] not in req.specifier: raise ImportError( f"Add-ons {required[reqstr] + modified[reqstr]} require" f" or modify missing component {reqstr}" diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 741dd81cf..451ffed0f 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -4,7 +4,6 @@ import warnings from collections import OrderedDict from unittest import mock -from pkg_resources import VersionConflict from zope.interface import directlyProvides from zope.interface.exceptions import BrokenImplementation, MultipleInvalid from zope.interface.verify import verifyObject @@ -347,7 +346,7 @@ class AddonManagerTest(unittest.TestCase): self.assertRaises(ImportError, check_with, requires) self.assertRaises(ImportError, check_with, modifies) self.assertRaises(ImportError, check_with, provides, provides2) - self.assertRaises(VersionConflict, check_with, provides, requires_newer) + self.assertRaises(ImportError, check_with, provides, requires_newer) with warnings.catch_warnings(record=True) as w: check_with(provides, modifies) check_with(provides) From 7ebb8256f029b2027a3603685569378ccce73070 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 17:52:11 +0400 Subject: [PATCH 1004/2083] Some cleanup. --- docs/topics/addons.rst | 3 --- scrapy/addons.py | 6 +++--- tests/test_addons/addonmod.py | 6 ++---- tests/test_addons/addons.py | 10 +++------- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index ba6e839a5..b86058dc4 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -43,7 +43,6 @@ case with one requiring no configuration) are enabled/configured in a project's INSTALLED_ADDONS = ( 'httpcache', 'path.to.some.addon', - 'path/to/other/addon.py', ) HTTPCACHE = { @@ -73,8 +72,6 @@ dictionary keys. I.e., the configuration from above would look like this: [addon:path.to.some.addon] some_config = true - [addon:path/to/other/addon.py] - Enabling and configuring add-ons within Python code --------------------------------------------------- diff --git a/scrapy/addons.py b/scrapy/addons.py index 8f0f39889..43a36e5e5 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -156,10 +156,10 @@ class AddonManager(Mapping): addons = AddonManager() # ... load some add-ons here - print addons.enabled # prints names of all enabled add-ons - print addons['TestAddon'].version # prints version of add-on with name + print(addons.enabled) # prints names of all enabled add-ons + print(addons['TestAddon'].version) # prints version of add-on with name # 'TestAddon' - print addons.configs['TestAddon'] # prints configuration of 'TestAddon' + print(addons.configs['TestAddon']) # prints configuration of 'TestAddon' """ diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py index c59f6737f..092c3c0eb 100644 --- a/tests/test_addons/addonmod.py +++ b/tests/test_addons/addonmod.py @@ -1,10 +1,8 @@ -import zope.interface +from zope.interface import moduleProvides from scrapy.interfaces import IAddon -zope.interface.moduleProvides(IAddon) - -FROM = "test_addons.addonmod" +moduleProvides(IAddon) name = "AddonModule" version = "1.0" diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py index 4adb9fe8f..d878f37ea 100644 --- a/tests/test_addons/addons.py +++ b/tests/test_addons/addons.py @@ -1,13 +1,9 @@ -import zope.interface +from zope.interface import implementer from scrapy.interfaces import IAddon -class Addon(object): - FROM = "test_addons.addons" - - -@zope.interface.declarations.implementer(IAddon) +@implementer(IAddon) class GoodAddon(object): name = "GoodAddon" version = "1.0" @@ -28,7 +24,7 @@ class GoodAddon(object): pass -@zope.interface.declarations.implementer(IAddon) +@implementer(IAddon) class BrokenAddon(object): name = "BrokenAddon" # No version From 282fe3dd4fd4a87b27f8e9793760f877778d2914 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 17:52:25 +0400 Subject: [PATCH 1005/2083] Drop the SEP as we decided we don't want to update it. --- sep/sep-021.rst | 324 ------------------------------------------------ 1 file changed, 324 deletions(-) delete mode 100644 sep/sep-021.rst diff --git a/sep/sep-021.rst b/sep/sep-021.rst deleted file mode 100644 index 47cba004c..000000000 --- a/sep/sep-021.rst +++ /dev/null @@ -1,324 +0,0 @@ -======= =================== -SEP 21 -Title Add-ons -Author Pablo Hoffman -Created 2014-02-14 -Status Draft -======= =================== - -================ -SEP-021: Add-ons -================ - -This proposal introduces add-ons, a unified way to manage Scrapy extensions, -middlewares and pipelines. - -Scrapy currently supports many hooks and mechanisms for extending its -functionality, but no single entry point for enabling and configuring them. -Instead, the hooks are spread over: - -* Spider middlewares (``SPIDER_MIDDLEWARES``) -* Downloader middlewares (``DOWNLOADER_MIDDLEWARES``) -* Downloader handlers (``DOWNLOADER_HANDLERS``) -* Item pipelines (``ITEM_PIPELINES``) -* Feed exporters and storages (``FEED_EXPORTERS``, ``FEED_STORAGES``) -* Overridable components (``DUPEFILTER_CLASS``, ``STATS_CLASS``, - ``SCHEDULER``, ``SPIDER_MANAGER_CLASS``, ``ITEM_PROCESSOR``, etc.) -* Generic extensions (``EXTENSIONS``) -* CLI commands (``COMMANDS_MODULE``) - -This approach has several shortfalls: - -* Enabling an extension often requires modifying many settings, often in a - coordinated way, which is complex and error prone. -* Extension developers have little control over ensuring their library - dependencies and configuration requirements are met, especially since most - extensions never 'see' a fully-configured crawler before it starts running. -* The user is burdened with supervising potential interplay of extensions, - especially non-included ones, ranging from setting name clashes to mutually - excluding dependencies/configuration requirements. - -*Add-ons* search to remedy these shortcomings by enhancing Scrapy's extension -management, making it easy-to-use and transparent for users while giving more -configuration control to developers. - - -Design goals and non-goals -========================== - -Goals: - -* simple to manage: adding or removing extensions should be just a matter of - adding or removing lines in a configuration file -* backward compatibility with enabling extension the "old way" (i.e. modifying - settings directly) - -Non-goals: - -* a way to publish, distribute or discover extensions (use pypi for that) - - -User experience: managing add-ons -================================= - -Add-ons are enabled and configured either via Scrapy's settings, or (for add-ons -not bound to any project) in ``scrapy.cfg``. - -In the settings, add-ons can be enabled by adding either their name (for -built-in add-ons), their Python path, or their file path, to a -``INSTALLED_ADDONS`` setting. If necessary, each add-on can be configured by -providing a dictionary-valued setting with the uppercase add-on name. For -example, to enable and configure the built-in ``httpcache`` add-on and enable -(without configuring) two custom add-ons, one via Python path and one via file -path, add these entries to your settings module:: - - INSTALLED_ADDONS = ( - 'httpcache', - 'mymodule.filters.myfilter', - 'mymodule/filters/otherfilter.py', - ) - - HTTPCACHE = { - 'ignore_http_codes': [404, 503], - } - -In ``scrapy.cfg``, add-ons are enabled and configured with one section per -add-on. The section names correspond to the entries of ``INSTALLED_ADDONS``. -The configuration from above could look like this:: - - [addon:httpcache] - ignore_http_codes = 404,503 - - [addon:mymodule.filters.myfilter] - - [addon:mymodule/filters/otherfilter.py] - - -Developer experience: writing add-ons -===================================== - -Add-ons are (any) Python *objects* that implement Scrapy's *add-on interface*. -The interface is enforced through ``zope.interface``. This leaves the choice of -Python object up the developer. Examples: - -* for a small pipeline, the add-on interface could be implemented in the same - class that also implements the ``open/close_spider`` and ``process_item`` - callbacks -* for larger add-ons, or for clearer structure, the interface could be provided - by a stand-alone module - -The absolute minimum interface consists of just two attributes: - -* ``NAME``: string with add-on name -* ``VERSION``: PEP-440 style version string - -To be any useful, an add-on should implement at least one of the following -callback methods: - -* ``update_addons()``: adds and configures other add-ons -* ``update_settings()``: sets configuration (such as default values for this - add-on and required settings for other extensions) and enables needed - components. -* ``check_configuration()``: receives the fully-initialized ``Crawler`` - instance before it starts running, performs additional dependency and - configuration requirement checks - -Additionally, an add-on may (and should, where appropriate) provide one or more -variables that can be used for automated detection of possible dependency -clashes: - -* ``REQUIRES``: list of built-in or custom components required by this add-on, - as PEP-440 strings -* ``MODIFIES``: list of components whose functionality is affected or replaced - by this add-on (a custom HTTP cache should list ``httpcache`` here) -* ``PROVIDES``: list of components provided by this add-on (e.g. ``mongodb`` - for an extension that provides generic read/write access to a MongoDB - database, releasing other components from having to provide their own - database access methods) - -update_addons() ------------------ - -Called: -~~~~~~~ - -Shortly after initialisation of the ``Crawler`` object. - -Arguments: -~~~~~~~~~~ - -* ``config``: configuration of this add-on -* ``addons``: the add-on manager, providing methods to add and configure add-ons - -Purpose: -~~~~~~~~ - -* Configure and enable related add-ons, useful for 'umbrella add-ons' which - chain-load other add-ons based on the configuration - -Examples: -~~~~~~~~~ - -.. code-block:: python - - def update_addons(config, addons): - if "httpcache" not in addons.enabled: - addons.add("httpcache", {"expiration_secs": 60}) - -or: - -.. code-block:: python - - def update_addons(config, addons): - if "otheraddon" in addons.enabled: - addons.configs["otheraddon"]["some_config_name"] = True - -update_settings() ------------------ - -Called: -~~~~~~~ - -Directly after the ``update_addons()`` callback of all add-ons has been called. - -Arguments: -~~~~~~~~~~ - -* ``config``: configuration of this add-on -* ``settings``: the crawler's ``Settings`` instance containing all project - settings - -Purpose: -~~~~~~~~ - -* Modify ``settings`` to enable required components -* Expose some add-on specific configuration (``config``) into the global - settings namespace (``settings``) if necessary -* Raise exception if components can not be properly configured (e.g. on missing - dependencies); Scrapy will print this exception *and exit* (making users - explicitly acknowledge that the add-on does not work by forcing them to - disable it). - -Side note: -~~~~~~~~~~ - -The ``MiddlewareManager.from_settings()`` method will receive a slight -modification to allow directly placing Python objects instead of class paths -in the middleware dict settings. This way, add-ons can place already -instantiated components into the settings. This allows keeping configuration -as local to components as possible and avoids cluttering up the global -settings namespace. Furthermore, it allows reusing components (e.g. using -two instances of the same mongodb pipeline to write to different locations). - -Examples: -~~~~~~~~~ - -:: - - def update_settings(config, settings): - # Don't care where this module is located - settings.set['DOWNLADER_MIDDLEWARES']({ - __name__ + '.downloadermw.coolmw': 900, - }) - - # Instantiate components to not expose settings into - # the global namespace - from .pipelines import MySQLPipeline - mysqlpl = MySQLPipeline(password = config['password']) - settings.set['ITEM_PIPELINES']({ - mysqlpl: 200, - }) - -or:: - - def update_settings(config, settings): - # Assuming this class also has a process_item() method - settings.set['ITEM_PIPELINES']({ - self: 200, - }) - -or:: - - def update_settings(config, settings): - try: - import boto - except ImportError: - raise RuntimeError("boto library is required") - -check_configuration() ---------------------- - -Called: -~~~~~~~ - -Shortly before the crawler starts crawling. - -Arguments: -~~~~~~~~~~ - -* ``config``: configuration of this add-on -* ``crawler``: fully-initialized ``Crawler`` object, ready to start crawling - -Purpose: -~~~~~~~~ - -* Perform post-initialization checks like making sure the extension and its - dependencies were configured properly. -* Raise exception if a critical check failed; Scrapy will print this exception - *and exit* (see ``update_settings()`` purpose for rationale on this). - -Examples: -~~~~~~~~~ - -:: - - def check_configuration(config, crawler): - if 'some.other.addon' not in crawler.addons.enabled: - raise RuntimeError("Some other add-on required to use this add-on") - - -Implementation -============== - -A new core component, the *add-on manager*, is introduced to Scrapy. It -facilitates loading add-ons, gathering and providing information on them, -calling their callbacks at appropriate times, and performing basic checks for -dependency and configuration clashes. - -Layout ------- - -A new ``AddonManager`` class is introduced, providing methods to - -* add and remove add-ons, -* search for add-ons by name -* read enabled add-ons and their configurations from the settings module and - from ``settings.py``, -* enable and disable add-ons -* check for possible dependency incompatibilites by inspecting the collected - ``REQUIRES``, ``MODIFIES`` and ``PROVIDES`` add-on variables -* call the add-on callbacks - -Integration into start-up process ---------------------------------- - -The settings used to crawl are not complete until the spider-specific settings -have been loaded in ``Crawler.__init__()``. Add-on management follows this -approach and only starts loading add-ons when the crawler is initialised. - -Instantiation and the calls ``update_addons()`` and ``update_settings()`` happen -in ``Crawler.__init__()``. The final checks (i.e. the callback to -``check_configuration()``) is coded into the ``Crawler.crawl()`` method after -creating the engine. - -Finding add-ons ---------------- - -Add-on localisation is governed by the add-on paths given in -``INSTALLED_ADDONS`` (or by the section names if using ``scrapy.cfg``). If -nothing is found at the given path, it is tried again with ``addons.`` -prepended (i.e. pointing to the project's ``addons`` folder or module), then -with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons`` -submodule). If the object found has an ``_addon`` attribute, that attribute -will be treated as the found add-on. This allows, for example, to change the -add-on based on the Python version. From 22bd0d9a796cca980f3d2c2d951d0f3960ab8a18 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 17:58:11 +0400 Subject: [PATCH 1006/2083] Fix docs build. --- docs/topics/addons.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index b86058dc4..198c48bcc 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -236,6 +236,7 @@ convenience functions by overwriting :meth:`~scrapy.addons.Addon.update_settings`. .. module:: scrapy.addons + :noindex: .. autoclass:: Addon :members: From fdfab17438647b10a7cb74b808ddf4acfd89c6ab Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:00:18 +0400 Subject: [PATCH 1007/2083] Fix docs for the renamed ADDONS setting. --- docs/topics/addons.rst | 18 +++++++++--------- docs/topics/settings.rst | 20 ++++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 198c48bcc..25d7da50b 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -20,15 +20,15 @@ only then, the add-on manager will read a list of enabled add-ons and their configurations from your settings. There are two places where you can provide the paths to add-ons you want to enable: -* the ``INSTALLED_ADDONS`` setting, and +* the ``ADDONS`` setting, and * the ``scrapy.cfg`` file. As Scrapy settings can be modified from many places, e.g. in a project's ``settings.py``, in a Spider's ``custom_settings`` attribute, or from the -command line, using the ``INSTALLED_ADDONS`` setting is the preferred way to +command line, using the ``ADDONS`` setting is the preferred way to manage add-ons. -The ``INSTALLED_ADDONS`` setting a tuple in which every item is a path to an +The ``ADDONS`` setting a tuple in which every item is a path to an add-on. The path can be both a Python or a file path. While more precise, it is not necessary to specify the full add-on Python path if it is either built into Scrapy or lives in your project's ``addons`` submodule. @@ -40,7 +40,7 @@ This is an example where an internal add-on and two third-party add-ons (in this case with one requiring no configuration) are enabled/configured in a project's ``settings.py``:: - INSTALLED_ADDONS = ( + ADDONS = ( 'httpcache', 'path.to.some.addon', ) @@ -79,7 +79,7 @@ Enabling and configuring add-ons within Python code The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings and from ``scrapy.cfg`` *at the beginning* of Scrapy's start-up process. Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is -populated, changing the ``INSTALLED_ADDONS`` setting or any of the add-on +populated, changing the ``ADDONS`` setting or any of the add-on configuration dictionary settings will have no effect. If you want to enable, disable, or configure add-ons in Python code, for example @@ -202,7 +202,7 @@ specify :pep:`440`-style information about required versions. Examples:: requires = ['otheraddon >= 2.0', 'yetanotheraddon'] The Python object or module that is pointed to by an add-on path (e.g. given in -the ``INSTALLED_ADDONS`` setting, or given to +the ``ADDONS`` setting, or given to :meth:`~scrapy.addons.AddonManager.add`) does not necessarily have to be an add-on. Instead, it can provide an ``_addon`` attribute. This attribute can be either an add-on or another add-on path. @@ -401,7 +401,7 @@ of placing this in your ``settings.py``:: you can also use the add-on framework:: - INSTALLED_ADDONS = ( + ADDONS = ( # ..., 'httpcache', ) @@ -412,9 +412,9 @@ you can also use the add-on framework:: } Note that you *must* enable built-in addons by placing them in your -``INSTALLED_ADDONS`` setting before you can use them for configuring built-in +``ADDONS`` setting before you can use them for configuring built-in components. I.e., configuring the ``HTTPCACHE`` setting will have no effect -when ``httpcache`` is not listed in ``INSTALLED_ADDONS``. +when ``httpcache`` is not listed in ``ADDONS``. In general, the add-on names match the lowercase name of the component, with its type suffix removed (i.e. the add-on configuring the diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 9e8c30efe..aa09abbd5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -201,6 +201,16 @@ to any particular component. In that case the module of that component will be shown, typically an extension, middleware or pipeline. It also means that the component must be enabled in order for the setting to have any effect. +.. setting:: ADDONS + +ADDONS +------ + +Default: ``()`` + +A tuple containing paths to the add-ons enabled in your project. For more +information, see :ref:`topics-addons`. + .. setting:: AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_ID @@ -964,16 +974,6 @@ some of them need to be enabled through a setting. For more information See the :ref:`extensions user guide ` and the :ref:`list of available extensions `. -.. setting:: INSTALLED_ADDONS - -INSTALLED_ADDONS ----------------- - -Default: ``()`` - -A tuple containing paths to the add-ons enabled in your project. For more -information, see :ref:`topics-addons`. - .. setting:: FEED_TEMPDIR FEED_TEMPDIR From 5e76464fbf1338bb791b4ae2ad97dd33f72dd0a1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:01:41 +0400 Subject: [PATCH 1008/2083] Fix a merge error. --- scrapy/interfaces.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index d8e5f9866..b8aa77ced 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -17,11 +17,6 @@ class ISpiderLoader(Interface): """Return the list of spiders names that can handle the given request""" -# ISpiderManager is deprecated, don't use it! -# An alias is kept for backwards compatibility. -ISpiderManager = ISpiderLoader - - class IAddon(Interface): """Scrapy add-on""" From 815af431209686018b1bc2cbb80fd946cfe93614 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:11:14 +0400 Subject: [PATCH 1009/2083] Remove load_module_or_object. --- scrapy/addons.py | 6 +++--- scrapy/utils/misc.py | 16 ---------------- tests/test_utils_misc/__init__.py | 13 ++++++------- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 43a36e5e5..153de66d2 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -11,7 +11,7 @@ from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_module_or_object +from scrapy.utils.misc import load_object @zope.interface.implementer(IAddon) @@ -252,8 +252,8 @@ class AddonManager(Mapping): """ if isinstance(path, str): try: - obj = load_module_or_object(path) - except NameError: + obj = load_object(path) + except (ValueError, NameError, ImportError): raise NameError(f"Could not find add-on '{path}'") else: obj = path diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 8577cce02..d861c9ab6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -72,22 +72,6 @@ def load_object(path: Union[str, Callable]) -> Any: return obj -def load_module_or_object(path): - """Load python module or (non-module) object from given path. - - Path can be both a Python or a file path. - """ - try: - return import_module(path) - except ImportError: - pass - try: - return load_object(path) - except (ValueError, NameError, ImportError): - pass - raise NameError(f"Could not load '{path}'") - - def walk_modules(path): """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 7932ca04c..4f6e0d02c 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -8,7 +8,6 @@ from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, create_instance, - load_module_or_object, load_object, rel_has_nofollow, set_environ, @@ -31,17 +30,17 @@ class UtilsMiscTestCase(unittest.TestCase): obj = load_object("scrapy.utils.misc.load_object") self.assertIs(obj, load_object) + def test_load_object_module(self): + testmod = load_object(__name__ + ".testmod") + self.assertTrue(hasattr(testmod, "TESTVAR")) + obj = load_object("scrapy.utils.misc.load_object") + self.assertIs(obj, load_object) + def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, "nomodule999.mod.function") self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999") self.assertRaises(TypeError, load_object, {}) - def test_load_module_or_object(self): - testmod = load_module_or_object(__name__ + ".testmod") - self.assertTrue(hasattr(testmod, "TESTVAR")) - obj = load_object("scrapy.utils.misc.load_object") - self.assertIs(obj, load_object) - def test_walk_modules(self): mods = walk_modules("tests.test_utils_misc.test_walk_modules") expected = [ From 7cfdca8f9b1b1f16aacae958f7a5c8824df056e7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:23:29 +0400 Subject: [PATCH 1010/2083] Actually run test_set_asyncio_event_loop(). --- tests/test_utils_asyncio.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 01d0ee043..65e352053 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -4,6 +4,7 @@ from unittest import TestCase from pytest import mark +from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( install_reactor, is_asyncio_reactor_installed, @@ -29,6 +30,8 @@ class AsyncioTest(TestCase): assert original_reactor == reactor + @mark.only_asyncio() + @deferred_f_from_coro_f async def test_set_asyncio_event_loop(self): install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - assert set_asyncio_event_loop() is asyncio.get_running_loop() + assert set_asyncio_event_loop(None) is asyncio.get_running_loop() From 27f5f3513437466d4e67df7dc3e0e57f959cc03b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:30:33 +0400 Subject: [PATCH 1011/2083] More quick doc fixes. --- docs/topics/addons.rst | 131 ++------------------------------------- docs/topics/settings.rst | 6 +- 2 files changed, 7 insertions(+), 130 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 25d7da50b..523f6e86e 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -40,10 +40,10 @@ This is an example where an internal add-on and two third-party add-ons (in this case with one requiring no configuration) are enabled/configured in a project's ``settings.py``:: - ADDONS = ( - 'httpcache', - 'path.to.some.addon', - ) + ADDONS = { + 'httpcache': 0, + 'path.to.some.addon': 0, + } HTTPCACHE = { 'expiration_secs': 60, @@ -385,126 +385,3 @@ Forward to other add-ons depending on Python version:: _addon = 'path.to.addon' else: _addon = 'path.to.other.addon' - - -Built-in add-on reference -========================= - -Scrapy comes with gateway add-ons that you can use to configure the built-in -middlewares and extensions. For example, to activate and configure the -:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`, instead -of placing this in your ``settings.py``:: - - HTTPCACHE_ENABLED = True - HTTPCACHE_EXPIRATION_SECS = 60 - HTTPCACHE_IGNORE_HTTP_CODES = [404] - -you can also use the add-on framework:: - - ADDONS = ( - # ..., - 'httpcache', - ) - - HTTPCACHE = { - 'expiration_secs': 60, - 'ignore_http_codes': [404], - } - -Note that you *must* enable built-in addons by placing them in your -``ADDONS`` setting before you can use them for configuring built-in -components. I.e., configuring the ``HTTPCACHE`` setting will have no effect -when ``httpcache`` is not listed in ``ADDONS``. - -In general, the add-on names match the lowercase name of the component, with its -type suffix removed (i.e. the add-on configuring the -:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` is called -``httpcache``), and the configuration option names match the names of the -settings they map to, with the component prefix removed (i.e. -``expiration_secs`` maps to :setting:`HTTPCACHE_EXPIRATION_SECS`, as above). -The available add-ons are: - - -+--------------------------------------+--------------------------------------+ -| Add-on | Notes | -+======================================+======================================+ -| **Spider middlewares** | -+--------------------------------------+--------------------------------------+ -| depth (:class:`~scrapy.spidermi\ | | -| ddlewares.depth.DepthMiddleware`) | | -+--------------------------------------+--------------------------------------+ -| httperror (:class:`~scrapy.spid\ | | -| ermiddlewares.httperror.HttpErrorMi\ | | -| ddleware`) | | -+--------------------------------------+--------------------------------------+ -| offsite (:class:`~scrapy.spid\ | | -| ermiddlewares.offsite.OffsiteMiddle\ | | -| ware`) | | -+--------------------------------------+--------------------------------------+ -| referer (:class:`~scrapy.spid\ | | -| ermiddlewares.referer.RefererMiddle\ | | -| ware`) | | -+--------------------------------------+--------------------------------------+ -| urllength (:class:`~scrapy.spid\ | | -| ermiddlewares.urllength.UrlLengthMi\ | | -| ddleware`) | | -+--------------------------------------+--------------------------------------+ -| **Downloader middlewares** | -+--------------------------------------+--------------------------------------+ -| ajaxcrawl (:class:`~scrapy.download\ | | -| ermiddlewares.ajaxcrawl.AjaxCrawlMi\ | | -| ddleware`) | | -+--------------------------------------+--------------------------------------+ -| chunked (:class:`~scrapy.download\ | | -| ermiddlewares.chunked.ChunkedTrans\ | | -| ferMiddleware`) | | -+--------------------------------------+--------------------------------------+ -| cookies (:class:`~scrapy.download\ | | -| ermiddlewares.cookies.CookiesMiddle\ | | -| ware`) | | -+--------------------------------------+--------------------------------------+ -| defaultheaders (:class:`~scrapy.down\| Every configuration entry is treated | -| loadermiddlewares.defaultheaders.Def\| as a default header. | -| aultHeadersMiddleware`) | | -+--------------------------------------+--------------------------------------+ -| **Extensions** | -+--------------------------------------+--------------------------------------+ -| autothrottle | Installing sets | -| (:ref:`topics-autothrottle`) | :setting:`AUTOTHROTTLE_ENABLED` to | -| | ``True``. | -+--------------------------------------+--------------------------------------+ -| corestats (:class:`~scrapy.exten\ | | -| sions.corestats.CoreStats`) | | -+--------------------------------------+--------------------------------------+ -| closespider (:class:`~scrapy.exten\ | | -| sions.closespider.CloseSpider`) | | -+--------------------------------------+--------------------------------------+ -| debugger (:class:`~scrapy.exten\ | | -| sions.debug.Debugger`) | | -+--------------------------------------+--------------------------------------+ -| feedexport (:ref:`topics-feed-expor\ | | -| ts`) | | -+--------------------------------------+--------------------------------------+ -| logstats (:class:`~scrapy.exten\ | | -| sions.logstats.LogStats`) | | -+--------------------------------------+--------------------------------------+ -| memdebug (:class:`~scrapy.exten\ | Installing sets | -| sions.memdebug.MemoryDebugger`) | :setting:`MEMDEBUG_ENABLED` to | -| | ``True``. | -+--------------------------------------+--------------------------------------+ -| memusage (:class:`~scrapy.exten\ | Installing sets | -| sions.memusage.MemoryUsage`) | :setting:`MEMUSAGE_ENABLED` to | -| | ``True``. | -+--------------------------------------+--------------------------------------+ -| spiderstate (:class:`~scrapy.exten\ | | -| sions.spiderstate.SpiderState`) | | -+--------------------------------------+--------------------------------------+ -| stacktracedump (:class:`~scrapy.ext\ | | -| ensions.debug.StackTraceDump`) | | -+--------------------------------------+--------------------------------------+ -| statsmailer (:class:`~scrapy.exten\ | | -| sions.statsmailer.StatsMailer`) | | -+--------------------------------------+--------------------------------------+ -| telnetconsole (:ref:`topics-telnet\ | | -| console`) | | -+--------------------------------------+--------------------------------------+ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index aa09abbd5..143002360 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -206,10 +206,10 @@ component must be enabled in order for the setting to have any effect. ADDONS ------ -Default: ``()`` +Default: ``{}`` -A tuple containing paths to the add-ons enabled in your project. For more -information, see :ref:`topics-addons`. +A dict containing paths to the add-ons enabled in your project and their +priorities. For more information, see :ref:`topics-addons`. .. setting:: AWS_ACCESS_KEY_ID From c7f78a8305fd7c08705cc0125e0f6e99dbaa0d10 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 20:43:18 +0400 Subject: [PATCH 1012/2083] Revert "Remove load_module_or_object." This reverts commit 815af431209686018b1bc2cbb80fd946cfe93614. --- scrapy/addons.py | 6 +++--- scrapy/utils/misc.py | 16 ++++++++++++++++ tests/test_utils_misc/__init__.py | 13 +++++++------ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 153de66d2..43a36e5e5 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -11,7 +11,7 @@ from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_module_or_object @zope.interface.implementer(IAddon) @@ -252,8 +252,8 @@ class AddonManager(Mapping): """ if isinstance(path, str): try: - obj = load_object(path) - except (ValueError, NameError, ImportError): + obj = load_module_or_object(path) + except NameError: raise NameError(f"Could not find add-on '{path}'") else: obj = path diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d861c9ab6..8577cce02 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -72,6 +72,22 @@ def load_object(path: Union[str, Callable]) -> Any: return obj +def load_module_or_object(path): + """Load python module or (non-module) object from given path. + + Path can be both a Python or a file path. + """ + try: + return import_module(path) + except ImportError: + pass + try: + return load_object(path) + except (ValueError, NameError, ImportError): + pass + raise NameError(f"Could not load '{path}'") + + def walk_modules(path): """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 4f6e0d02c..7932ca04c 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -8,6 +8,7 @@ from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, create_instance, + load_module_or_object, load_object, rel_has_nofollow, set_environ, @@ -30,17 +31,17 @@ class UtilsMiscTestCase(unittest.TestCase): obj = load_object("scrapy.utils.misc.load_object") self.assertIs(obj, load_object) - def test_load_object_module(self): - testmod = load_object(__name__ + ".testmod") - self.assertTrue(hasattr(testmod, "TESTVAR")) - obj = load_object("scrapy.utils.misc.load_object") - self.assertIs(obj, load_object) - def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, "nomodule999.mod.function") self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999") self.assertRaises(TypeError, load_object, {}) + def test_load_module_or_object(self): + testmod = load_module_or_object(__name__ + ".testmod") + self.assertTrue(hasattr(testmod, "TESTVAR")) + obj = load_object("scrapy.utils.misc.load_object") + self.assertIs(obj, load_object) + def test_walk_modules(self): mods = walk_modules("tests.test_utils_misc.test_walk_modules") expected = [ From 2f9ebb66c32fc11eee86261d55e232b7e12f6a09 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 17:00:38 +0400 Subject: [PATCH 1013/2083] Remove some dead code/docs. --- docs/topics/addons.rst | 32 ++------------------------ scrapy/addons.py | 26 --------------------- tests/test_addons/__init__.py | 43 ++++++++++++----------------------- 3 files changed, 16 insertions(+), 85 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 523f6e86e..35e40ca15 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -17,16 +17,7 @@ Activating and configuring add-ons Add-ons and their configuration live in Scrapy's :class:`~scrapy.addons.AddonManager`. During Scrapy's start-up process, and only then, the add-on manager will read a list of enabled add-ons and their -configurations from your settings. There are two places where you can provide -the paths to add-ons you want to enable: - -* the ``ADDONS`` setting, and -* the ``scrapy.cfg`` file. - -As Scrapy settings can be modified from many places, e.g. in a project's -``settings.py``, in a Spider's ``custom_settings`` attribute, or from the -command line, using the ``ADDONS`` setting is the preferred way to -manage add-ons. +configurations from your ``ADDONS`` setting. The ``ADDONS`` setting a tuple in which every item is a path to an add-on. The path can be both a Python or a file path. While more precise, it is @@ -54,30 +45,11 @@ case with one requiring no configuration) are enabled/configured in a project's 'some_config': True, } -It is also possible to manage add-ons from ``scrapy.cfg``. While the syntax is -a little friendlier, be aware that this file, and therefore the configuration in -it, is not bound to a particular Scrapy project. While this should not pose a -problem when you use the project on your development machine only, a common -stumbling block is that ``scrapy.cfg`` is not deployed via ``scrapyd-deploy``. - -In ``scrapy.cfg``, section names, prepended with ``addon:``, replace the -dictionary keys. I.e., the configuration from above would look like this: - -.. code-block:: cfg - - [addon:httpcache] - expiration_secs = 60 - ignore_http_codes = 404,405 - - [addon:path.to.some.addon] - some_config = true - - Enabling and configuring add-ons within Python code --------------------------------------------------- The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings -and from ``scrapy.cfg`` *at the beginning* of Scrapy's start-up process. +*at the beginning* of Scrapy's start-up process. Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is populated, changing the ``ADDONS`` setting or any of the add-on configuration dictionary settings will have no effect. diff --git a/scrapy/addons.py b/scrapy/addons.py index 43a36e5e5..66463afdb 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -261,32 +261,6 @@ class AddonManager(Mapping): obj = AddonManager.get_addon(obj._addon) return obj - def load_dict(self, addonsdict): - """Load add-ons and configurations from given dictionary. - - Each add-on should be an entry in the dictionary, where the key - corresponds to the add-on path. The value should be a dictionary - representing the add-on configuration. - - Example add-on dictionary:: - - addonsdict = { - 'path.to.addon1': { - 'setting1': 'value', - 'setting2': 42, - }, - 'path/to/addon2.py': { - 'addon2setting': True, - }, - } - - :param addonsdict: dictionary where keys correspond to add-on paths \ - and values correspond to their configuration - :type addonsdict: ``dict`` - """ - for addonpath, addoncfg in addonsdict.items(): - self.add(addonpath, addoncfg) - def load_settings(self, settings): """Load add-ons and configurations from settings object. diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 451ffed0f..d1c17a044 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,7 +1,6 @@ import itertools import unittest import warnings -from collections import OrderedDict from unittest import mock from zope.interface import directlyProvides @@ -178,24 +177,7 @@ class AddonManagerTest(unittest.TestCase): x._addon._addon = addons.GoodAddon("inner") self.assertIs(self.manager.get_addon(x), x._addon._addon) - def test_load_dict_load_settings(self): - def _test_load_method(func, *args, **kwargs): - manager = AddonManager() - getattr(manager, func)(*args, **kwargs) - self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) - self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) - self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) - self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") - self.assertEqual(manager["AddonModule"], addonmod) - self.assertIn("key", manager.configs["AddonModule"]) - self.assertEqual(manager.configs["AddonModule"]["key"], "val1") - - addonsdict = { - "tests.test_addons.addonmod": {"key": "val1"}, - "tests.test_addons.addons.GoodAddon": {"key": "val2"}, - } - _test_load_method("load_dict", addonsdict) - + def test_load_settings(self): settings = BaseSettings() settings.set( "ADDONS", @@ -203,25 +185,28 @@ class AddonManagerTest(unittest.TestCase): ) settings.set("ADDONMODULE", {"key": "val1"}) settings.set("GOODADDON", {"key": "val2"}) - _test_load_method("load_settings", settings) - - def test_load_dict_load_settings_order(self): - def _test_load_method(expected_order, func, *args, **kwargs): - manager = AddonManager() - getattr(manager, func)(*args, **kwargs) - self.assertEqual(list(manager.keys()), expected_order) + manager = AddonManager() + manager.load_settings(settings) + self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) + self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) + self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) + self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") + self.assertEqual(manager["AddonModule"], addonmod) + self.assertIn("key", manager.configs["AddonModule"]) + self.assertEqual(manager.configs["AddonModule"]["key"], "val1") + def test_load_settings_order(self): # Get three addons named 0, 1, 2 addonlist = [addons.GoodAddon(str(x)) for x in range(3)] # Test both methods for every possible mutation for ordered_addons in itertools.permutations(addonlist): expected_order = [a.name for a in ordered_addons] - addonsdict = OrderedDict((a, {}) for a in ordered_addons) - _test_load_method(expected_order, "load_dict", addonsdict) settings = BaseSettings( {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} ) - _test_load_method(expected_order, "load_settings", settings) + manager = AddonManager() + manager.load_settings(settings) + self.assertEqual(list(manager.keys()), expected_order) def test_enabled_disabled(self): manager = AddonManager() From 0258c87dab6d7336263ff53c7747f6400d8cd527 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 18:40:59 +0400 Subject: [PATCH 1014/2083] Add typing for Crawler.addons. --- scrapy/crawler.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 5e7499f9c..fdcc6354d 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,7 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Optional, Type, Union +from typing import TYPE_CHECKING, Optional, Set, Type, Union from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement @@ -57,7 +57,7 @@ class Crawler: spidercls: Type[Spider], settings: Union[None, dict, Settings] = None, init_reactor: bool = False, - addons=None, + addons: Optional[AddonManager] = None, ): if isinstance(spidercls, Spider): raise ValueError("The spidercls argument must be a class, not an object") @@ -69,7 +69,7 @@ class Crawler: self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self.addons = addons if addons is not None else AddonManager() + self.addons: AddonManager = addons if addons is not None else AddonManager() self.addons.load_settings(self.settings) self.addons.update_addons() self.addons.check_dependency_clashes() @@ -199,14 +199,14 @@ class CrawlerRunner: ) return loader_cls.from_settings(settings.frozencopy()) - def __init__(self, settings=None, addons=None): + def __init__(self, settings=None, addons: Optional[AddonManager] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings - self.addons = addons + self.addons: Optional[AddonManager] = addons self.spider_loader = self._get_spider_loader(settings) - self._crawlers = set() - self._active = set() + self._crawlers: Set[Crawler] = set() + self._active: Set[defer.Deferred] = set() self.bootstrap_failed = False @property @@ -285,7 +285,7 @@ class CrawlerRunner: def _create_crawler(self, spidercls): if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings, self.addons) + return Crawler(spidercls, self.settings, addons=self.addons) def stop(self): """ @@ -331,7 +331,12 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None, install_root_handler=True, addons=None): + def __init__( + self, + settings=None, + install_root_handler: bool = True, + addons: Optional[AddonManager] = None, + ): super().__init__(settings, addons) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) From f582246d7b0a70c9ace5642002c30040f7182d7c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 18:49:08 +0400 Subject: [PATCH 1015/2083] More doc fixes. --- docs/index.rst | 2 +- docs/topics/addons.rst | 33 +++++++++++---------------------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index ace5a2eb7..8798aebd1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -237,7 +237,7 @@ Extending Scrapy Understand the Scrapy architecture. :doc:`topics/addons` - Enable and configure built-in and third-party extensions. + Enable and configure third-party extensions. :doc:`topics/downloader-middleware` Customize how pages get requested and downloaded. diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 35e40ca15..5d1a4f753 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -19,26 +19,18 @@ Add-ons and their configuration live in Scrapy's only then, the add-on manager will read a list of enabled add-ons and their configurations from your ``ADDONS`` setting. -The ``ADDONS`` setting a tuple in which every item is a path to an -add-on. The path can be both a Python or a file path. While more precise, it is -not necessary to specify the full add-on Python path if it is either built into -Scrapy or lives in your project's ``addons`` submodule. +The ``ADDONS`` setting is a dict in which every key is an addon class or its +import path and the vaoue is its priority. The configuration of an add-on, if necessary at all, is stored as a dictionary setting whose name is the uppercase add-on name. -This is an example where an internal add-on and two third-party add-ons (in this -case with one requiring no configuration) are enabled/configured in a project's -``settings.py``:: +This is an example where two add-ons (in this case with one requiring no +configuration) are enabled/configured in a project's ``settings.py``:: ADDONS = { - 'httpcache': 0, - 'path.to.some.addon': 0, - } - - HTTPCACHE = { - 'expiration_secs': 60, - 'ignore_http_codes': [404, 405], + 'path.to.someaddon': 0, + path.to.someaddon2: 1, } SOMEADDON = { @@ -67,7 +59,7 @@ add-ons framework, e.g.: * :meth:`~scrapy.addons.AddonManager.enable` and :meth:`~scrapy.addons.AddonManager.disable` methods, * the :attr:`~scrapy.addons.AddonManager.configs` dictionary which holds the - configuration of all add-ons + configuration of all add-ons. In this example, we ensure that the ``httpcache`` add-on is loaded, and that its ``expiration_secs`` configuration is set to ``60``:: @@ -88,9 +80,9 @@ Python object up the developer. Examples: * for a small pipeline, the add-on interface could be implemented in the same class that also implements the ``open/close_spider`` and ``process_item`` - callbacks + callbacks, * for larger add-ons, or for clearer structure, the interface could be provided - by a stand-alone module + by a stand-alone module. The absolute minimum interface consists of two attributes: @@ -137,9 +129,7 @@ crawling process: This method is called immediately before :meth:`update_settings`, and should be used to enable and configure other *add-ons* only. - When using this callback, be aware that there is no guarantee in which order - the :meth:`update_addons` callbacks of enabled add-ons will be called. - Add-ons that are added to the :class:`~scrapy.addons.AddonManager` during + Add-ons that are added to the :class:`~scrapy.addons.AddonManager` during this callback will also have their :meth:`update_addons` method called. :param config: Configuration of this add-on @@ -244,8 +234,7 @@ Check dependencies:: import boto except ImportError: raise RuntimeError("myaddon requires the boto library") - else: - self.export_config(config, settings) + self.export_config(config, settings) Enable a component that lives relative to the add-on (see :ref:`topics-api-settings`):: From 79bf8b1f2e8eae5dfb02443b4be5d474502793ca Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 19:23:28 +0400 Subject: [PATCH 1016/2083] Test cleanup. --- tests/test_addons/__init__.py | 2 +- tests/test_crawl.py | 6 ++---- tests/test_middleware.py | 12 ------------ 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index d1c17a044..fa3f706e5 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -198,7 +198,7 @@ class AddonManagerTest(unittest.TestCase): def test_load_settings_order(self): # Get three addons named 0, 1, 2 addonlist = [addons.GoodAddon(str(x)) for x in range(3)] - # Test both methods for every possible mutation + # Test for every possible ordering for ordered_addons in itertools.permutations(addonlist): expected_order = [a.name for a in ordered_addons] settings = BaseSettings( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1e97863b0..920e5f4ae 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -421,10 +421,8 @@ with multiples lines addonmgr.add(FailedCheckAddon()) crawler = get_crawler(SimpleSpider) crawler.addons = addonmgr - # Doesn't work in 'precise' test environment: - # with self.assertRaises(ValueError): - # yield crawler.crawl() - yield self.assertFailure(crawler.crawl(), ValueError) + with self.assertRaises(ValueError): + yield crawler.crawl() class CrawlSpiderTestCase(TestCase): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index fd57d846e..00ff746ee 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -87,15 +87,3 @@ class MiddlewareManagerTest(unittest.TestCase): mwman = TestMiddlewareManager.from_settings(settings) classes = [x.__class__ for x in mwman.middlewares] self.assertEqual(classes, [M1, M3]) - - def test_instances_from_settings(self): - settings = Settings() - - class InstanceTestMiddlewareManager(MiddlewareManager): - @classmethod - def _get_mwlist_from_settings(cls, settings): - return ["tests.test_middleware.M1", M2] - - mwman = InstanceTestMiddlewareManager.from_settings(settings) - self.assertIsInstance(mwman.middlewares[0], M1) - self.assertIsInstance(mwman.middlewares[1], M2) From 54287f733972dcaf13706664ba89ab1c52e3a290 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 19:53:34 +0400 Subject: [PATCH 1017/2083] Docs cleanup. --- scrapy/addons.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 66463afdb..aced6092a 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -30,9 +30,7 @@ class Addon(object): component_type = None """Component setting into which to export via :meth:`export_component`. Can - be any of the dictionary-like component setting names (e.g. - ``DOWNLOADER_MIDDLEWARES``) or any of their abbreviations in - :attr:`~scrapy.addons.COMPONENT_TYPE_ABBR`. If ``None``, + be any of the dictionary-like component setting names. If ``None``, :meth:`export_component` will do nothing. """ @@ -50,9 +48,8 @@ class Addon(object): component = None """Component to be inserted via :meth:`export_component`. This can be anything that can be used in the dictionary-like component settings, i.e. - a class path, a class, or an instance. If ``None``, it is assumed that the - add-on itself is also provides the component interface, and ``self`` will be - used. + a class path or a class. If ``None``, it is assumed that the add-on itself + also provides the component interface, and ``self`` will be used. """ settings_prefix = None @@ -241,13 +238,13 @@ class AddonManager(Mapping): def get_addon(path): """Get an add-on object by its Python or file path. - ``path`` is assumed to be either a Python or a file path of a Scrapy - add-on. If the object or module pointed to by ``path`` has an attribute - named ``_addon`` that attribute will be assumed to be the add-on. - :meth:`get_addon` will keep following ``_addon`` attributes until it - finds an object that does not have an attribute named ``_addon``. + ``path`` is assumed to be an import path of an add-on. If the object or + module pointed to by ``path`` has an attribute named ``_addon`` that + attribute will be assumed to be the add-on. :meth:`get_addon` will keep + following ``_addon`` attributes until it finds an object that does not + have an attribute named ``_addon``. - :param path: Python or file path to an add-on + :param path: Import path of an add-on :type path: ``str`` """ if isinstance(path, str): @@ -366,7 +363,7 @@ class AddonManager(Mapping): elif addon in self._disable_on_add: self._disable_on_add.remove(addon) else: - raise ValueError("Add-ons need to be added before they can be " "enabled") + raise ValueError("Add-ons need to be added before they can be enabled") @property def disabled(self): From 639c2bcc47ecd700917aa2a48b9c63f41327f977 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 16 Jun 2023 13:37:26 +0300 Subject: [PATCH 1018/2083] periodic_log: TypeError except added --- scrapy/extensions/periodic_log.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 3e496096c..0f01c441c 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -45,6 +45,12 @@ class PeriodicLog: if crawler.settings.getbool("PERIODIC_LOG_STATS") else None ) + except TypeError: + ext_stats = ( + {"enabled": True} + if crawler.settings.getbool("PERIODIC_LOG_STATS") + else None + ) try: ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA") @@ -54,6 +60,13 @@ class PeriodicLog: if crawler.settings.getdict("PERIODIC_LOG_DELTA") else None ) + except TypeError: + ext_stats = ( + {"enabled": True} + if crawler.settings.getbool("PERIODIC_LOG_DELTA") + else None + ) + ext_timing_enabled = crawler.settings.getbool( "PERIODIC_LOG_TIMING_ENABLED", False ) From 6e65eeb07beb6805d7f6d0a139c98f58c41a7bc3 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 16 Jun 2023 14:30:50 +0300 Subject: [PATCH 1019/2083] periodic_log: tests [wip] added --- tests/test_extension_periodic_log.py | 79 ++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 tests/test_extension_periodic_log.py diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py new file mode 100644 index 000000000..9f7ec7b23 --- /dev/null +++ b/tests/test_extension_periodic_log.py @@ -0,0 +1,79 @@ +import datetime +import unittest + +from scrapy.crawler import Crawler +from scrapy.exceptions import NotConfigured +from scrapy.extensions.periodic_log import PeriodicLog + +from .spiders import MetaSpider + +stats_dump_1 = { + "log_count/INFO": 10, + "log_count/WARNING": 1, + "start_time": datetime.datetime(2023, 6, 16, 8, 59, 18, 993170), + "scheduler/enqueued/memory": 190, + "scheduler/enqueued": 190, + "scheduler/dequeued/memory": 166, + "scheduler/dequeued": 166, + "downloader/request_count": 166, + "downloader/request_method_count/GET": 166, + "downloader/request_bytes": 56803, + "downloader/response_count": 150, + "downloader/response_status_count/200": 150, + "downloader/response_bytes": 595698, + "httpcompression/response_bytes": 3186068, + "httpcompression/response_count": 150, + "response_received_count": 150, + "request_depth_max": 9, + "dupefilter/filtered": 180, + "item_scraped_count": 140, +} +stats_dump_2 = { + "log_count/INFO": 12, + "log_count/WARNING": 1, + "start_time": datetime.datetime(2023, 6, 16, 8, 59, 18, 993170), + "scheduler/enqueued/memory": 337, + "scheduler/enqueued": 337, + "scheduler/dequeued/memory": 280, + "scheduler/dequeued": 280, + "downloader/request_count": 280, + "downloader/request_method_count/GET": 280, + "downloader/request_bytes": 95754, + "downloader/response_count": 264, + "downloader/response_status_count/200": 264, + "downloader/response_bytes": 1046274, + "httpcompression/response_bytes": 5614484, + "httpcompression/response_count": 264, + "response_received_count": 264, + "request_depth_max": 16, + "dupefilter/filtered": 320, + "item_scraped_count": 248, +} + + +class TestPeriodicLog(unittest.TestCase): + def test_extension_enabled(self): + extension = PeriodicLog.from_crawler( + Crawler( + MetaSpider, + settings={"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60}, + ) + ) + # Test enabled + assert extension + + # Raise not configured if not set by settings + with self.assertRaises(NotConfigured): + PeriodicLog.from_crawler(Crawler(MetaSpider)) + + def test_periodic_log_stats(self): + pass + + def test_log_delta(self): + pass + + def test_settings_include(self): + pass + + def test_settings_exclude(self): + pass From 777a6ea4128cf00d53fa71dc48385bf036cb64fc Mon Sep 17 00:00:00 2001 From: Serhii A Date: Fri, 16 Jun 2023 16:46:06 +0300 Subject: [PATCH 1020/2083] Make the retry middleware exception list configurable (#5929) --- docs/topics/downloader-middleware.rst | 32 ++++++++++++ scrapy/downloadermiddlewares/retry.py | 63 ++++++++++++------------ scrapy/settings/default_settings.py | 15 ++++++ tests/test_downloadermiddleware_retry.py | 39 +++++++++++++-- 4 files changed, 113 insertions(+), 36 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 7665a901a..a8e5b23bf 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -915,6 +915,7 @@ settings (see the settings documentation for more info): * :setting:`RETRY_ENABLED` * :setting:`RETRY_TIMES` * :setting:`RETRY_HTTP_CODES` +* :setting:`RETRY_EXCEPTIONS` .. reqmeta:: dont_retry @@ -966,6 +967,37 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because it is a common code used to indicate server overload. It is not included by default because HTTP specs say so. +.. setting:: RETRY_EXCEPTIONS + +RETRY_EXCEPTIONS +^^^^^^^^^^^^^^^^ + +Default:: + + [ + 'twisted.internet.defer.TimeoutError', + 'twisted.internet.error.TimeoutError', + 'twisted.internet.error.DNSLookupError', + 'twisted.internet.error.ConnectionRefusedError', + 'twisted.internet.error.ConnectionDone', + 'twisted.internet.error.ConnectError', + 'twisted.internet.error.ConnectionLost', + 'twisted.internet.error.TCPTimedOutError', + 'twisted.web.client.ResponseFailed', + IOError, + 'scrapy.core.downloader.handlers.http11.TunnelError', + ] + +List of exceptions to retry. + +Each list entry may be an exception type or its import path as a string. + +An exception will not be caught when the exception type is not in +:setting:`RETRY_EXCEPTIONS` or when the maximum number of retries for a request +has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught +exception propagation, see +:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`. + .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 081642a4b..50cbc3111 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,31 +9,36 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ +import warnings from logging import Logger, getLogger from typing import Optional, Union -from twisted.internet import defer -from twisted.internet.error import ( - ConnectError, - ConnectionDone, - ConnectionLost, - ConnectionRefusedError, - DNSLookupError, - TCPTimedOutError, - TimeoutError, -) -from twisted.web.client import ResponseFailed - -from scrapy.core.downloader.handlers.http11 import TunnelError -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http.request import Request +from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message retry_logger = getLogger(__name__) +class BackwardsCompatibilityMetaclass(type): + @property + def EXCEPTIONS_TO_RETRY(cls): + warnings.warn( + "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " + "Use the RETRY_EXCEPTIONS setting instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return tuple( + load_object(x) if isinstance(x, str) else x + for x in Settings().getlist("RETRY_EXCEPTIONS") + ) + + def get_retry_request( request: Request, *, @@ -121,23 +126,7 @@ def get_retry_request( return None -class RetryMiddleware: - # IOError is raised by the HttpCompression middleware when trying to - # decompress an empty response - EXCEPTIONS_TO_RETRY = ( - defer.TimeoutError, - TimeoutError, - DNSLookupError, - ConnectionRefusedError, - ConnectionDone, - ConnectError, - ConnectionLost, - TCPTimedOutError, - ResponseFailed, - IOError, - TunnelError, - ) - +class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): def __init__(self, settings): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured @@ -147,6 +136,16 @@ class RetryMiddleware: ) self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") + if not hasattr( + self, "EXCEPTIONS_TO_RETRY" + ): # If EXCEPTIONS_TO_RETRY is not "overriden" + self.exceptions_to_retry = tuple( + load_object(x) if isinstance(x, str) else x + for x in settings.getlist("RETRY_EXCEPTIONS") + ) + else: + self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY + @classmethod def from_crawler(cls, crawler): return cls(crawler.settings) @@ -160,7 +159,7 @@ class RetryMiddleware: return response def process_exception(self, request, exception, spider): - if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get( + if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( "dont_retry", False ): return self._retry(request, exception, spider) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 260ec1701..89837b4ab 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -258,6 +258,21 @@ RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] RETRY_PRIORITY_ADJUST = -1 +RETRY_EXCEPTIONS = [ + "twisted.internet.defer.TimeoutError", + "twisted.internet.error.TimeoutError", + "twisted.internet.error.DNSLookupError", + "twisted.internet.error.ConnectionRefusedError", + "twisted.internet.error.ConnectionDone", + "twisted.internet.error.ConnectError", + "twisted.internet.error.ConnectionLost", + "twisted.internet.error.TCPTimedOutError", + "twisted.web.client.ResponseFailed", + # IOError is raised by the HttpCompression middleware when trying to + # decompress an empty response + IOError, + "scrapy.core.downloader.handlers.http11.TunnelError", +] ROBOTSTXT_OBEY = False ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser" diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 63bd61848..97ae1e29a 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,5 +1,6 @@ import logging import unittest +import warnings from testfixtures import LogCapture from twisted.internet import defer @@ -15,6 +16,7 @@ from twisted.web.client import ResponseFailed from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response +from scrapy.settings.default_settings import RETRY_EXCEPTIONS from scrapy.spiders import Spider from scrapy.utils.test import get_crawler @@ -110,19 +112,48 @@ class RetryTest(unittest.TestCase): == 2 ) - def _test_retry_exception(self, req, exception): + def test_exception_to_retry_added(self): + exc = ValueError + settings_dict = { + "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc], + } + crawler = get_crawler(Spider, settings_dict=settings_dict) + mw = RetryMiddleware.from_crawler(crawler) + req = Request(f"http://www.scrapytest.org/{exc.__name__}") + self._test_retry_exception(req, exc("foo"), mw) + + def test_exception_to_retry_customMiddleware(self): + exc = ValueError + + with warnings.catch_warnings(record=True) as warns: + + class MyRetryMiddleware(RetryMiddleware): + EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,) + + self.assertEqual(len(warns), 1) + + mw2 = MyRetryMiddleware.from_crawler(self.crawler) + req = Request(f"http://www.scrapytest.org/{exc.__name__}") + req = mw2.process_exception(req, exc("foo"), self.spider) + assert isinstance(req, Request) + self.assertEqual(req.meta["retry_times"], 1) + + def _test_retry_exception(self, req, exception, mw=None): + if mw is None: + mw = self.mw + # first retry - req = self.mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) self.assertEqual(req.meta["retry_times"], 1) # second retry - req = self.mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) self.assertEqual(req.meta["retry_times"], 2) # discard it - req = self.mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception, self.spider) self.assertEqual(req, None) From 2122278d4bb7177a550bc0b04dd96d1fc1fad1a0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 18 Jun 2023 18:37:50 +0400 Subject: [PATCH 1021/2083] Drop Python 3.7 support. --- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 13 +++++-------- .github/workflows/tests-windows.yml | 5 +---- README.rst | 2 +- docs/contributing.rst | 12 ++++++------ docs/intro/install.rst | 2 +- scrapy/__init__.py | 4 ++-- setup.py | 3 +-- tox.ini | 8 +++----- 9 files changed, 21 insertions(+), 30 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 174d245ca..3044a1af3 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 96b26a1f8..39e3b0af7 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -8,9 +8,6 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.8 - env: - TOXENV: py - python-version: 3.9 env: TOXENV: py @@ -28,19 +25,19 @@ jobs: TOXENV: pypy3 # pinned deps - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: pinned - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: asyncio-pinned - - python-version: pypy3.7 + - python-version: pypy3.8 env: TOXENV: pypy3-pinned - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: extra-deps-pinned - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: botocore-pinned diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index f60c48841..5bcf74d5e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -8,12 +8,9 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.7 - env: - TOXENV: windows-pinned - python-version: 3.8 env: - TOXENV: py + TOXENV: windows-pinned - python-version: 3.9 env: TOXENV: py diff --git a/README.rst b/README.rst index 970bf2c35..1918850d6 100644 --- a/README.rst +++ b/README.rst @@ -58,7 +58,7 @@ including a list of features. Requirements ============ -* Python 3.7+ +* Python 3.8+ * Works on Linux, Windows, macOS, BSD Install diff --git a/docs/contributing.rst b/docs/contributing.rst index eef92e148..2b3249601 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -265,15 +265,15 @@ To run a specific test (say ``tests/test_loader.py``) use: To run the tests on a specific :doc:`tox ` environment, use ``-e `` with an environment name from ``tox.ini``. For example, to run -the tests with Python 3.7 use:: +the tests with Python 3.10 use:: - tox -e py37 + tox -e py310 You can also specify a comma-separated list of environments, and use :ref:`tox’s parallel mode ` to run the tests on multiple environments in parallel:: - tox -e py37,py38 -p auto + tox -e py39,py310 -p auto To pass command-line options to :doc:`pytest `, add them after ``--`` in your call to :doc:`tox `. Using ``--`` overrides the @@ -283,9 +283,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well:: tox -- scrapy tests -x # stop after first failure You can also use the `pytest-xdist`_ plugin. For example, to run all tests on -the Python 3.7 :doc:`tox ` environment using all your CPU cores:: +the Python 3.10 :doc:`tox ` environment using all your CPU cores:: - tox -e py37 -- scrapy tests -n auto + tox -e py310 -- scrapy tests -n auto To see coverage report install :doc:`coverage ` (``pip install coverage``) and run: @@ -322,4 +322,4 @@ And their unit-tests are in:: .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist .. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 -.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy \ No newline at end of file +.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 2c2079f68..c90c1d2bf 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,7 +9,7 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.7+, either the CPython implementation (default) or +Scrapy requires Python 3.8+, either the CPython implementation (default) or the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: diff --git a/scrapy/__init__.py b/scrapy/__init__.py index a757a9290..cc0e539c4 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -34,8 +34,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version -if sys.version_info < (3, 7): - print(f"Scrapy {__version__} requires Python 3.7+") +if sys.version_info < (3, 8): + print(f"Scrapy {__version__} requires Python 3.8+") sys.exit(1) diff --git a/setup.py b/setup.py index c6bcf2439..f1cd4c5e2 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,6 @@ setup( "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -91,7 +90,7 @@ setup( "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ], - python_requires=">=3.7", + python_requires=">=3.8", install_requires=install_requires, extras_require=extras_require, ) diff --git a/tox.ini b/tox.ini index 5f8bf85f2..d5b6118f5 100644 --- a/tox.ini +++ b/tox.ini @@ -16,8 +16,6 @@ deps = #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' - # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) - markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy' passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -96,7 +94,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] -basepython = python3.7 +basepython = python3.8 deps = {[pinned]deps} PyDispatcher==2.0.5 @@ -129,7 +127,7 @@ deps = Twisted[http2] [testenv:extra-deps-pinned] -basepython = python3.7 +basepython = python3.8 deps = {[pinned]deps} boto3==1.20.0 @@ -211,7 +209,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} [testenv:botocore-pinned] -basepython = python3.7 +basepython = python3.8 deps = {[pinned]deps} botocore==1.4.87 From 1b2c9a3e0ae9766623a06ce7e739a3dfda399159 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 18 Jun 2023 18:38:56 +0400 Subject: [PATCH 1022/2083] Bump isort and flake8 versions. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index faf8808f2..31e9ed1ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 # 6.0.0 drops Python 3.7 support + rev: 6.0.0 hooks: - id: flake8 - repo: https://github.com/psf/black.git @@ -13,7 +13,7 @@ repos: hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.11.5 # 5.12 drops Python 3.7 support + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs From 075b89eab5e201246a5bddc4a6ce9c7921de082b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 18 Jun 2023 19:08:41 +0400 Subject: [PATCH 1023/2083] Bump lxml and cryptography to versions with 3.8 wheels available. --- setup.py | 4 ++-- tox.ini | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index f1cd4c5e2..ccfe20ae5 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ def has_environment_marker_platform_impl_support(): install_requires = [ "Twisted>=18.9.0", - "cryptography>=3.4.6", + "cryptography>=36.0.0", "cssselect>=0.9.1", "itemloaders>=1.0.1", "parsel>=1.5.0", @@ -34,7 +34,7 @@ install_requires = [ "setuptools", "packaging", "tldextract", - "lxml>=4.3.0", + "lxml>=4.4.1", ] extras_require = {} cpython_dependencies = [ diff --git a/tox.ini b/tox.ini index d5b6118f5..ec3a59366 100644 --- a/tox.ini +++ b/tox.ini @@ -69,7 +69,7 @@ commands = [pinned] deps = - cryptography==3.4.6 + cryptography==36.0.0 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 @@ -81,7 +81,7 @@ deps = Twisted[http2]==18.9.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.3.0 + lxml==4.4.1 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies From 0097b4c0bb4de6e651e8b9d064aae140e11698d5 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 16:40:38 -0300 Subject: [PATCH 1024/2083] cleanup: Remove `pkg_resources` usage --- docs/topics/components.rst | 2 +- scrapy/cmdline.py | 5 ++--- setup.py | 2 +- tests/test_crawler.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 1ed55f000..478dd9647 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -70,7 +70,7 @@ If your requirement is a minimum Scrapy version, you may use .. code-block:: python - from pkg_resources import parse_version + from packaging.version import parse as parse_version import scrapy diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 730e55350..cfa771104 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -3,8 +3,7 @@ import cProfile import inspect import os import sys - -import pkg_resources +from importlib.metadata import entry_points import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -49,7 +48,7 @@ def _get_commands_from_module(module, inproject): def _get_commands_from_entry_points(inproject, group="scrapy.commands"): cmds = {} - for entry_point in pkg_resources.iter_entry_points(group): + for entry_point in entry_points(group): obj = entry_point.load() if inspect.isclass(obj): cmds[entry_point.name] = obj() diff --git a/setup.py b/setup.py index ccfe20ae5..f918db09e 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from pathlib import Path -from pkg_resources import parse_version +from packaging.version import parse as parse_version from setuptools import __version__ as setuptools_version from setuptools import find_packages, setup diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ecb9c9b62..d54a2cb7e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -6,7 +6,7 @@ import sys import warnings from pathlib import Path -from pkg_resources import parse_version +from packaging.version import parse as parse_version from pytest import mark, raises from twisted import version as twisted_version from twisted.internet import defer From 6afb31b82b5a0a5d2f37962c250fbf34c21d8580 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 16:48:07 -0300 Subject: [PATCH 1025/2083] chore: Add `packaging` to tests deps --- tests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements.txt b/tests/requirements.txt index 618949795..72350b216 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -14,3 +14,4 @@ brotli # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" +packaging \ No newline at end of file From 6e1af20ac4dd537a4643df5c022f948cf07d05ec Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 17:00:01 -0300 Subject: [PATCH 1026/2083] fix: add `build-system` --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index ec3a59366..79d692599 100644 --- a/tox.ini +++ b/tox.ini @@ -218,3 +218,6 @@ setenv = {[pinned]setenv} commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + +[build-system] +build-backend = 'setuptools.build_meta' \ No newline at end of file From a93a63c208af1d13d5ea84623d160337c7fec6c5 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 17:05:49 -0300 Subject: [PATCH 1027/2083] fix: move import to inside function --- setup.py | 3 ++- tests/requirements.txt | 1 - tox.ini | 3 --- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index f918db09e..dfe5b80ec 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ from pathlib import Path -from packaging.version import parse as parse_version from setuptools import __version__ as setuptools_version from setuptools import find_packages, setup @@ -15,6 +14,8 @@ def has_environment_marker_platform_impl_support(): it is 18.5, see: https://setuptools.readthedocs.io/en/latest/history.html#id235 """ + from packaging.version import parse as parse_version + return parse_version(setuptools_version) >= parse_version("18.5") diff --git a/tests/requirements.txt b/tests/requirements.txt index 72350b216..618949795 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -14,4 +14,3 @@ brotli # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" -packaging \ No newline at end of file diff --git a/tox.ini b/tox.ini index 79d692599..ec3a59366 100644 --- a/tox.ini +++ b/tox.ini @@ -218,6 +218,3 @@ setenv = {[pinned]setenv} commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} - -[build-system] -build-backend = 'setuptools.build_meta' \ No newline at end of file From 0b1da44a05cc64970aa11ccc4d7a4a3bec143443 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 17:14:21 -0300 Subject: [PATCH 1028/2083] chore: Remove deprecated code --- setup.py | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/setup.py b/setup.py index dfe5b80ec..1f214571b 100644 --- a/setup.py +++ b/setup.py @@ -1,24 +1,10 @@ from pathlib import Path -from setuptools import __version__ as setuptools_version from setuptools import find_packages, setup version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() -def has_environment_marker_platform_impl_support(): - """Code extracted from 'pytest/setup.py' - https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31 - - The first known release to support environment marker with range operators - it is 18.5, see: - https://setuptools.readthedocs.io/en/latest/history.html#id235 - """ - from packaging.version import parse as parse_version - - return parse_version(setuptools_version) >= parse_version("18.5") - - install_requires = [ "Twisted>=18.9.0", "cryptography>=36.0.0", @@ -37,19 +23,10 @@ install_requires = [ "tldextract", "lxml>=4.4.1", ] -extras_require = {} -cpython_dependencies = [ - "PyDispatcher>=2.0.5", -] -if has_environment_marker_platform_impl_support(): - extras_require[ - ':platform_python_implementation == "CPython"' - ] = cpython_dependencies - extras_require[':platform_python_implementation == "PyPy"'] = [ - "PyPyDispatcher>=2.1.0", - ] -else: - install_requires.extend(cpython_dependencies) +extras_require = { + ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], + ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"], +} setup( From 82cf00bbc931723320c9aca3cf0305372027d0a0 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 18:27:03 -0300 Subject: [PATCH 1029/2083] fix: default value --- scrapy/cmdline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index cfa771104..efc9b36ea 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -48,7 +48,7 @@ def _get_commands_from_module(module, inproject): def _get_commands_from_entry_points(inproject, group="scrapy.commands"): cmds = {} - for entry_point in entry_points(group): + for entry_point in entry_points().get(group, {}): obj = entry_point.load() if inspect.isclass(obj): cmds[entry_point.name] = obj() From ee215a29704adbc61a40baeca1d27f6179a1735e Mon Sep 17 00:00:00 2001 From: Aaron Smith <60046611+medic-code@users.noreply.github.com> Date: Wed, 21 Jun 2023 19:05:39 +0100 Subject: [PATCH 1030/2083] Change redirect text from Response.request docs (#5937) --- docs/topics/request-response.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 407df32d2..41df51589 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1103,9 +1103,10 @@ Response objects through all :ref:`Downloader Middlewares `. In particular, this means that: - - HTTP redirections will cause the original request (to the URL before - redirection) to be assigned to the redirected response (with the final - URL after redirection). + - HTTP redirections will create a new request from the request before + redirection. It has the majority of the same metadata and original + request attributes and gets assigned to the redirected response + instead of the propagation of the original request. - Response.request.url doesn't always equal Response.url From 5360ba34bc345667f77a4d4256f15fd648e42e18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= Date: Wed, 21 Jun 2023 11:08:53 -0700 Subject: [PATCH 1031/2083] IOError and other cleanup (#4716) --- docs/utils/linkfix.py | 2 +- scrapy/downloadermiddlewares/decompression.py | 4 ++-- scrapy/downloadermiddlewares/httpcache.py | 2 +- scrapy/settings/default_settings.py | 4 ++-- scrapy/utils/gz.py | 2 +- scrapy/utils/python.py | 2 +- tests/test_downloader_handlers.py | 2 +- tests/test_downloadermiddleware.py | 4 ++-- tests/test_mail.py | 2 -- tests/test_robotstxt_interface.py | 1 - tests/test_utils_gz.py | 2 +- tests/test_utils_iterators.py | 4 ++-- 12 files changed, 14 insertions(+), 17 deletions(-) diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 1f270837c..c17b9d511 100644 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -30,7 +30,7 @@ def main(): try: with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out: output_lines = out.readlines() - except IOError: + except OSError: print("linkcheck output not found; please run linkcheck first.") sys.exit(1) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 5839dc243..3b8702419 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -63,7 +63,7 @@ class DecompressionMiddleware: archive = BytesIO(response.body) try: body = gzip.GzipFile(fileobj=archive).read() - except IOError: + except OSError: return respcls = responsetypes.from_args(body=body) @@ -72,7 +72,7 @@ class DecompressionMiddleware: def _is_bzip2(self, response): try: body = bz2.decompress(response.body) - except IOError: + except OSError: return respcls = responsetypes.from_args(body=body) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index b9316c43a..ac87d4a4e 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -37,7 +37,7 @@ class HttpCacheMiddleware: ConnectionLost, TCPTimedOutError, ResponseFailed, - IOError, + OSError, ) def __init__(self, settings: Settings, stats: StatsCollector) -> None: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 89837b4ab..a4cb555bd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -268,9 +268,9 @@ RETRY_EXCEPTIONS = [ "twisted.internet.error.ConnectionLost", "twisted.internet.error.TCPTimedOutError", "twisted.web.client.ResponseFailed", - # IOError is raised by the HttpCompression middleware when trying to + # OSError is raised by the HttpCompression middleware when trying to # decompress an empty response - IOError, + OSError, "scrapy.core.downloader.handlers.http11.TunnelError", ] diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index e5df34d2e..77e0197d8 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -15,7 +15,7 @@ def gunzip(data): try: chunk = f.read1(8196) output_list.append(chunk) - except (IOError, EOFError, struct.error): + except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output_list is empty and f.extrabuf diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 27816c0df..bb5dbebbc 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -291,7 +291,7 @@ def without_none_values(iterable): try: return {k: v for k, v in iterable.items() if v is not None} except AttributeError: - return type(iterable)((v for v in iterable if v is not None)) + return type(iterable)(v for v in iterable if v is not None) def global_object_name(obj): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index fd4176e2f..9731b62c4 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -129,7 +129,7 @@ class FileTestCase(unittest.TestCase): def test_non_existent(self): request = Request(f"file://{self.mktemp()}") d = self.download_request(request, Spider("foo")) - return self.assertFailure(d, IOError) + return self.assertFailure(d, OSError) class ContentLengthHeaderResource(resource.Resource): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 060cfe08b..062e8a8b4 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -70,7 +70,7 @@ class DefaultsTest(ManagerTestCase): In particular when some website returns a 30x response with header 'Content-Encoding: gzip' giving as result the error below: - exceptions.IOError: Not a gzipped file + BadGzipFile: Not a gzipped file (...) """ req = Request("http://example.com") @@ -108,7 +108,7 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - self.assertRaises(IOError, self._download, request=req, response=resp) + self.assertRaises(OSError, self._download, request=req, response=resp) class ResponseFromProcessRequestTest(ManagerTestCase): diff --git a/tests/test_mail.py b/tests/test_mail.py index bc7298e9d..504c78486 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -1,5 +1,3 @@ -# coding=utf-8 - import unittest from email.charset import Charset from io import BytesIO diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 8d87a322a..d7a923085 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,4 +1,3 @@ -# coding=utf-8 from twisted.trial import unittest diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 6b2a458bc..7b7a25db8 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -28,7 +28,7 @@ class GunzipTest(unittest.TestCase): def test_gunzip_no_gzip_file_raises(self): self.assertRaises( - IOError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes() + OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes() ) def test_gunzip_truncated_short(self): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index faf7d2709..3598fa0bb 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -346,8 +346,8 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all((isinstance(k, str) for k in result_row.keys()))) - self.assertTrue(all((isinstance(v, str) for v in result_row.values()))) + self.assertTrue(all(isinstance(k, str) for k in result_row.keys())) + self.assertTrue(all(isinstance(v, str) for v in result_row.values())) def test_csviter_delimiter(self): body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t") From 04ee3303e4487270a433f5c3a087bda9a87d7008 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 21 Jun 2023 22:04:06 -0700 Subject: [PATCH 1032/2083] Adding support for Windows of absolute pathlib.Path objects in FeedExporter (#5939) --- docs/topics/feed-exports.rst | 4 ++-- scrapy/extensions/feedexport.py | 7 +++++-- tests/test_feedexport.py | 25 +++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index b31dc069e..aba47d998 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -156,8 +156,8 @@ The feeds are stored in the local filesystem. - Required external libraries: none Note that for the local filesystem storage (only) you can omit the scheme if -you specify an absolute path like ``/tmp/export.csv``. This only works on Unix -systems though. +you specify an absolute path like ``/tmp/export.csv`` (Unix systems only). +Alternatively you can also use a :class:`pathlib.Path` object. .. _topics-feed-storage-ftp: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 39934cbf3..1cdc78f59 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -382,7 +382,9 @@ class FeedExporter: category=ScrapyDeprecationWarning, stacklevel=2, ) - uri = str(self.settings["FEED_URI"]) # handle pathlib.Path objects + uri = self.settings["FEED_URI"] + # handle pathlib.Path objects + uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")} self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings @@ -392,7 +394,8 @@ class FeedExporter: # 'FEEDS' setting takes precedence over 'FEED_URI' for uri, feed_options in self.settings.getdict("FEEDS").items(): - uri = str(uri) # handle pathlib.Path objects + # handle pathlib.Path objects + uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 62a5697cd..8df86dbd7 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2758,6 +2758,31 @@ class FeedExportInitTest(unittest.TestCase): with self.assertRaises(NotConfigured): FeedExporter.from_crawler(crawler) + def test_absolute_pathlib_as_uri(self): + with tempfile.NamedTemporaryFile(suffix="json") as tmp: + settings = { + "FEEDS": { + Path(tmp.name).resolve(): { + "format": "json", + }, + }, + } + crawler = get_crawler(settings_dict=settings) + exporter = FeedExporter.from_crawler(crawler) + self.assertIsInstance(exporter, FeedExporter) + + def test_relative_pathlib_as_uri(self): + settings = { + "FEEDS": { + Path("./items.json"): { + "format": "json", + }, + }, + } + crawler = get_crawler(settings_dict=settings) + exporter = FeedExporter.from_crawler(crawler) + self.assertIsInstance(exporter, FeedExporter) + class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage): def __init__(self, uri): From e71d6d67e56e35642fddc226e34e4d523041ad17 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 22 Jun 2023 21:10:50 +0400 Subject: [PATCH 1033/2083] Apply suggestions from code review --- scrapy/extensions/feedexport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 7e93bc366..d088450a7 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -439,10 +439,10 @@ class FeedExporter: return slot_.file if slot.itemcount: - # Nomal case + # Normal case slot.finish_exporting() elif slot.store_empty and slot.batch_id == 1: - # Need Store Empty + # Need to store the empty file slot.start_exporting() slot.finish_exporting() else: From 080b9bd0b8f65dcf09ea8ad94505fec6c77f820c Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 22 Jun 2023 23:58:03 -0300 Subject: [PATCH 1034/2083] chore: Implement `pop` method on `BaseSettings` class --- scrapy/settings/__init__.py | 14 ++++++++++++++ tests/test_settings/__init__.py | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index a3b849f7b..57fe1d17a 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -75,6 +75,8 @@ class BaseSettings(MutableMapping): highest priority will be retrieved. """ + __default = object() + def __init__(self, values=None, priority="project"): self.frozen = False self.attributes = {} @@ -445,6 +447,18 @@ class BaseSettings(MutableMapping): else: p.text(pformat(self.copy_to_dict())) + def pop(self, name, default=__default): + try: + value = self.attributes[name] + except KeyError: + if default is self.__default: + raise + + return SettingsAttribute(default, get_settings_priority("project")) + else: + del self.attributes[name] + return value + class Settings(BaseSettings): """ diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 4a577cd8c..0e2f4aa98 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -451,6 +451,19 @@ class SettingsTest(unittest.TestCase): self.assertIsInstance(myhandler_instance, FileDownloadHandler) self.assertTrue(hasattr(myhandler_instance, "download_request")) + def test_pop_item_with_default_value(self): + settings = Settings() + + with self.assertRaises(KeyError): + settings.pop("DUMMY_CONFIG") + + dummy_config = settings.pop("DUMMY_CONFIG", "dummy_value") + + self.assertEqual( + repr(dummy_config), "" + ) + self.assertEqual(dummy_config.value, "dummy_value") + if __name__ == "__main__": unittest.main() From 876feaf339e181c9c5a6b9a5f8ffedc03a9ed3d2 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Fri, 23 Jun 2023 00:14:31 -0300 Subject: [PATCH 1035/2083] chore: Use dunder to delete item instead of del keyword to handle immutable settings --- scrapy/settings/__init__.py | 2 +- tests/test_settings/__init__.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 57fe1d17a..8b3bdbabe 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -456,7 +456,7 @@ class BaseSettings(MutableMapping): return SettingsAttribute(default, get_settings_priority("project")) else: - del self.attributes[name] + self.__delitem__(name) return value diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 0e2f4aa98..125b1d96f 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -464,6 +464,22 @@ class SettingsTest(unittest.TestCase): ) self.assertEqual(dummy_config.value, "dummy_value") + def test_pop_item_with_frozen_settings(self): + settings = Settings( + {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} + ) + + self.assertEqual(settings.pop("DUMMY_CONFIG").value, "dummy_value") + + settings.freeze() + + with self.assertRaises(TypeError) as error: + settings.pop("OTHER_DUMMY_CONFIG") + + self.assertEqual( + str(error.exception), "Trying to modify an immutable Settings object" + ) + if __name__ == "__main__": unittest.main() From a3f8912d69eacdd2208617e6afb418e4e1847e36 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Fri, 23 Jun 2023 00:15:32 -0300 Subject: [PATCH 1036/2083] chore: Rename test --- tests/test_settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 125b1d96f..bb6dc67fa 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -464,7 +464,7 @@ class SettingsTest(unittest.TestCase): ) self.assertEqual(dummy_config.value, "dummy_value") - def test_pop_item_with_frozen_settings(self): + def test_pop_item_with_immutable_settings(self): settings = Settings( {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} ) From 2ce4856508f7092368349c0ebf296dd1391a229a Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Sun, 25 Jun 2023 12:16:59 +0300 Subject: [PATCH 1037/2083] periodic_log: Exception handling on init updated --- scrapy/extensions/periodic_log.py | 17 +------ tests/test_extension_periodic_log.py | 76 ++++++++++++++++++++++++---- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 0f01c441c..b1f5b8894 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -39,29 +39,16 @@ class PeriodicLog: interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") try: ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS") - except ValueError: + except (TypeError, ValueError): ext_stats = ( {"enabled": True} if crawler.settings.getbool("PERIODIC_LOG_STATS") else None ) - except TypeError: - ext_stats = ( - {"enabled": True} - if crawler.settings.getbool("PERIODIC_LOG_STATS") - else None - ) - try: ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA") - except ValueError: + except (TypeError, ValueError): ext_delta = ( - {"enabled": True} - if crawler.settings.getdict("PERIODIC_LOG_DELTA") - else None - ) - except TypeError: - ext_stats = ( {"enabled": True} if crawler.settings.getbool("PERIODIC_LOG_DELTA") else None diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 9f7ec7b23..4dccc687b 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -51,20 +51,78 @@ stats_dump_2 = { } +def extension(settings=None): + return PeriodicLog.from_crawler( + Crawler( + MetaSpider, + settings=settings, + ) + ) + + class TestPeriodicLog(unittest.TestCase): def test_extension_enabled(self): - extension = PeriodicLog.from_crawler( - Crawler( - MetaSpider, - settings={"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60}, - ) - ) - # Test enabled - assert extension + # Expected that settings for this extension loaded succesfully + # And on certain conditions - extension raising NotConfigured + + # "PERIODIC_LOG_STATS": True -> set to {"enabled": True} + # due to TypeError exception from settings.getdict + assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60}) + + # "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True} + # due to JSONDecodeError(ValueError) exception from settings.getdict + assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60}) + + # The ame for PERIODIC_LOG_DELTA: + assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) + assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) # Raise not configured if not set by settings with self.assertRaises(NotConfigured): - PeriodicLog.from_crawler(Crawler(MetaSpider)) + extension() + + # Regular use cases: + assert extension( + { + "PERIODIC_LOG_STATS": { + "include": [ + "downloader/", + "scheduler/", + "log_count/", + "item_scraped_count/", + ], + "exclude": ["scheduler/"], + } + } + ) + + assert extension( + { + "PERIODIC_LOG_DELTA": {"include": ["downloader/"]}, + "PERIODIC_LOG_TIMING_ENABLED": True, + } + ) + + assert extension( + { + "PERIODIC_LOG_TIMING_ENABLED": True, + } + ) + + assert extension( + { + "PERIODIC_LOG_STATS": { + "include": [ + "downloader/", + "scheduler/", + "log_count/", + "item_scraped_count/", + ], + "exclude": ["scheduler/"], + }, + "PERIODIC_LOG_DELTA": {"include": ["downloader/"]}, + } + ) def test_periodic_log_stats(self): pass From e7124447f7e86ee93ad78ffd0ebfa6acbebc73ce Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 26 Jun 2023 16:57:46 +0400 Subject: [PATCH 1038/2083] Remove unneeded code. --- docs/topics/addons.rst | 241 ++------------------- scrapy/addons.py | 347 ++++-------------------------- scrapy/cmdline.py | 5 +- scrapy/crawler.py | 23 +- scrapy/interfaces.py | 21 +- scrapy/utils/conf.py | 12 +- scrapy/utils/misc.py | 18 +- scrapy/utils/test.py | 3 +- tests/test_addons.py | 72 +++++++ tests/test_addons/__init__.py | 342 ----------------------------- tests/test_addons/addonmod.py | 16 -- tests/test_addons/addons.py | 33 --- tests/test_crawl.py | 1 - tests/test_crawler.py | 27 --- tests/test_utils_misc/__init__.py | 7 - tests/test_utils_misc/testmod.py | 1 - 16 files changed, 140 insertions(+), 1029 deletions(-) create mode 100644 tests/test_addons.py delete mode 100644 tests/test_addons/__init__.py delete mode 100644 tests/test_addons/addonmod.py delete mode 100644 tests/test_addons/addons.py delete mode 100644 tests/test_utils_misc/testmod.py diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 5d1a4f753..6a9590f33 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -15,12 +15,13 @@ Activating and configuring add-ons ================================== Add-ons and their configuration live in Scrapy's -:class:`~scrapy.addons.AddonManager`. During Scrapy's start-up process, and -only then, the add-on manager will read a list of enabled add-ons and their -configurations from your ``ADDONS`` setting. +:class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler` +initialization the add-on manager will read a list of enabled add-ons from your +``ADDONS`` setting and their optional configuration from the respective +settings. The ``ADDONS`` setting is a dict in which every key is an addon class or its -import path and the vaoue is its priority. +import path and the value is its priority. The configuration of an add-on, if necessary at all, is stored as a dictionary setting whose name is the uppercase add-on name. @@ -37,66 +38,17 @@ configuration) are enabled/configured in a project's ``settings.py``:: 'some_config': True, } -Enabling and configuring add-ons within Python code ---------------------------------------------------- - -The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings -*at the beginning* of Scrapy's start-up process. -Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is -populated, changing the ``ADDONS`` setting or any of the add-on -configuration dictionary settings will have no effect. - -If you want to enable, disable, or configure add-ons in Python code, for example -when writing your own add-on, you will have to use the -:class:`~scrapy.addons.AddonManager`. You can access the add-on manager through -either ``crawler.addons`` or, if you are writing an add-on, through the -``addons`` argument of the :meth:`update_addons` callback. The add-on manager -provides many useful methods and attributes to facilitate interacting with the -add-ons framework, e.g.: - -* an :meth:`~scrapy.addons.AddonManager.add` method to load add-ons, -* the :attr:`~scrapy.addons.AddonManager.enabled` list of enabled add-ons, -* :meth:`~scrapy.addons.AddonManager.enable` and - :meth:`~scrapy.addons.AddonManager.disable` methods, -* the :attr:`~scrapy.addons.AddonManager.configs` dictionary which holds the - configuration of all add-ons. - -In this example, we ensure that the ``httpcache`` add-on is loaded, and that -its ``expiration_secs`` configuration is set to ``60``:: - - # addons is an instance of AddonManager - if 'httpcache' not in addons: - addons.add('httpcache', {'expiration_secs': 60}) - else: - addons.configs['httpcache']['expiration_secs'] = 60 - Writing your own add-ons ======================== -Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*. -The interface is enforced through ``zope.interface``. This leaves the choice of -Python object up the developer. Examples: - -* for a small pipeline, the add-on interface could be implemented in the same - class that also implements the ``open/close_spider`` and ``process_item`` - callbacks, -* for larger add-ons, or for clearer structure, the interface could be provided - by a stand-alone module. - -The absolute minimum interface consists of two attributes: +Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*: .. attribute:: name string with add-on name -.. attribute:: version - - version string (PEP-404, e.g. ``'1.0.1'``) - -Of course, stating just these two attributes will not get you very far. Add-ons -can provide three callback methods that are called at various stages before the -crawling process: + :type: ``str`` .. method:: update_settings(config, settings) @@ -124,75 +76,18 @@ crawling process: :param crawler: Fully initialized Scrapy crawler :type crawler: :class:`~scrapy.crawler.Crawler` -.. method:: update_addons(config, addons) - - This method is called immediately before :meth:`update_settings`, and should - be used to enable and configure other *add-ons* only. - - Add-ons that are added to the :class:`~scrapy.addons.AddonManager` during - this callback will also have their :meth:`update_addons` method called. - - :param config: Configuration of this add-on - :type config: ``dict`` - - :param addons: Add-on manager holding all loaded add-ons - :type addons: :class:`~scrapy.addons.AddonManager` - -Additionally, add-ons may (and should, where appropriate) provide one or more -attributes that can be used for limited automated detection of possible -dependency clashes: - -.. attribute:: requires - - list of built-in or custom components needed by this add-on, as strings. - -.. attribute:: modifies - - list of built-in or custom components whose functionality is affected or - replaced by this add-on (a custom HTTP cache should list ``httpcache`` here) - -.. attribute:: provides - - list of components provided by this add-on (e.g. ``mongodb`` for an - extension that provides generic read/write access to a MongoDB database) - -The entries in the :attr:`requires` and :attr:`modifies` attributes can be add-on -names or components from other add-ons' :attr:`provides` attribute. You can -specify :pep:`440`-style information about required versions. Examples:: - - requires = ['httpcache'] - requires = ['otheraddon >= 2.0', 'yetanotheraddon'] - -The Python object or module that is pointed to by an add-on path (e.g. given in -the ``ADDONS`` setting, or given to -:meth:`~scrapy.addons.AddonManager.add`) does not necessarily have to be an -add-on. Instead, it can provide an ``_addon`` attribute. This attribute can be -either an add-on or another add-on path. - Add-on base class ================= Scrapy comes with a built-in base class for add-ons which provides some -convenience functionality: - -* basic settings can be exported via :meth:`~scrapy.addons.Addon.export_basics`, - configurable via :attr:`~scrapy.addons.Addon.basic_settings`. -* a single component (e.g. an item pipeline or a downloader middleware) can be - inserted into Scrapy's settings via - :meth:`~scrapy.addons.Addon.export_component`, configurable via - :attr:`~scrapy.addons.Addon.component_type`, - :attr:`~scrapy.addons.Addon.component_key`, - :attr:`~scrapy.addons.Addon.component`, and the ``order`` key in - :attr:`~scrapy.addons.Addon.default_config`. -* the add-on configuration can be exposed into Scrapy's settings via - :meth:`~scrapy.addons.Addon.export_config`, configurable via - :attr:`~scrapy.addons.Addon.default_config`, - :attr:`~scrapy.addons.Addon.config_mapping`, and - :attr:`~scrapy.addons.Addon.settings_prefix`. +convenience functionality: the add-on configuration can be exposed into +Scrapy's settings via :meth:`~scrapy.addons.Addon.export_config`, configurable +via :attr:`~scrapy.addons.Addon.default_config` and +:attr:`~scrapy.addons.Addon.config_mapping`. By default, the base add-on class will expose the add-on configuration into -Scrapy's settings namespace, in caps and with the add-on name prepended. It is +Scrapy's settings namespace, in upper case. It is easy to write your own functionality while still being able to use the convenience functions by overwriting :meth:`~scrapy.addons.Addon.update_settings`. @@ -213,13 +108,11 @@ Set some basic configuration using the :class:`Addon` base class:: class MyAddon(Addon): name = 'myaddon' - version = '1.0' - component = 'path.to.mypipeline' - component_type = 'ITEM_PIPELINES' - component_order = 200 - basic_settings = { - 'DNSCACHE_ENABLED': False, - } + + def update_settings(self, config, settings): + super().update_settings(settings) + settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 + settings["DNSCACHE_ENABLED"] = True Check dependencies:: @@ -227,78 +120,22 @@ Check dependencies:: class MyAddon(Addon): name = 'myaddon' - version = '1.0' def update_settings(self, config, settings): try: import boto except ImportError: raise RuntimeError("myaddon requires the boto library") - self.export_config(config, settings) - -Enable a component that lives relative to the add-on (see -:ref:`topics-api-settings`):: - - from scrapy.addons import Addon - - class MyAddon(Addon): - name = 'myaddon' - version = '1.0' - component = __name__ + '.downloadermw.coolmw' - component_type = 'DOWNLOADER_MIDDLEWARES' - component_order = 900 - -Instantiate components ad hoc:: - - from path.to.my.pipelines import MySQLPipeline - - class MyAddon(object): - name = 'myaddon' - version = '1.0' - - def update_settings(self, config, settings): - mysqlpl = MySQLPipeline(password=config['password']) - settings.set( - 'ITEM_PIPELINES', - {mysqlpl: 200}, - priority='addon', - ) - -Provide add-on interface along component interface:: - - class MyPipeline(object): - name = 'mypipeline' - version = '1.0' - - def process_item(self, item, spider): - # Do some processing here - return item - - def update_settings(self, config, settings): - settings.set( - 'ITEM_PIPELINES', - {self: 200}, - priority='addon', - ) - -Enable another addon (see :ref:`topics-api-addonmanager`):: - - class MyAddon(object): - name = 'myaddon' - version = '1.0' - - def update_addons(self, config, addons): - if 'httpcache' not in addons.enabled: - addons.add('httpcache', {'expiration_secs': 60}) + super().update_settings(settings) Check configuration of fully initialized crawler (see :ref:`topics-api-crawler`):: class MyAddon(object): name = 'myaddon' - version = '1.0' def update_settings(self, config, settings): + super().update_settings(settings) settings.set('DNSCACHE_ENABLED', False, priority='addon') def check_configuration(self, config, crawler): @@ -306,43 +143,3 @@ Check configuration of fully initialized crawler (see # The spider, some other add-on, or the user messed with the # DNS cache setting raise ValueError("myaddon is incompatible with DNS cache") - -Provide add-on interface through a module: - -.. code-block:: python - - name = "AddonModule" - version = "1.0" - - - class MyPipeline(object): - ... - - - class MyDownloaderMiddleware(object): - ... - - - def update_settings(config, settings): - settings.set( - "ITEM_PIPELINES", - {MyPipeline(): 200}, - priority="addon", - ) - settings.set( - "DOWNLOADER_MIDDLEWARES", - {MyDownloaderMiddleware(): 800}, - priority="addon", - ) - -Forward to other add-ons depending on Python version:: - - # This could be a Python module, say project/pipelines/mypipeline.py, but - # could also be done inside a class, etc. - import six - - if six.PY3: - # We're running Python 3 - _addon = 'path.to.addon' - else: - _addon = 'path.to.other.addon' diff --git a/scrapy/addons.py b/scrapy/addons.py index aced6092a..a54086fda 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,108 +1,27 @@ -import warnings -from collections import OrderedDict, defaultdict from collections.abc import Mapping -from inspect import isclass -from typing import Dict +from typing import Any, Dict, Iterator, Optional, OrderedDict -import zope.interface -from packaging.requirements import Requirement -from packaging.version import Version -from zope.interface.verify import verifyObject - -from scrapy.interfaces import IAddon from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_module_or_object +from scrapy.utils.misc import load_object -@zope.interface.implementer(IAddon) class Addon(object): - basic_settings = None - """``dict`` of settings that will be exported via :meth:`export_basics`.""" + name: str default_config = None """``dict`` with default configuration.""" config_mapping = None """``dict`` with mappings from config names to setting names. The given - setting names will be taken as given, i.e. they will be neither prefixed - nor uppercased. + setting names will be taken as given, not uppercased. """ - component_type = None - """Component setting into which to export via :meth:`export_component`. Can - be any of the dictionary-like component setting names. If ``None``, - :meth:`export_component` will do nothing. - """ - - component_key = None - """Key to be used in the component dictionary setting when exporting via - :meth:`export_component`. This is only useful for the settings that have - no order, e.g. ``DOWNLOAD_HANDLERS`` or ``FEED_EXPORTERS``. - """ - - component_order = 0 - """Component order to use when not given in the add-on configuration. Has - no effect for component types that use :attr:`component_key`. - """ - - component = None - """Component to be inserted via :meth:`export_component`. This can be - anything that can be used in the dictionary-like component settings, i.e. - a class path or a class. If ``None``, it is assumed that the add-on itself - also provides the component interface, and ``self`` will be used. - """ - - settings_prefix = None - """Prefix with which the add-on configuration will be exported into the - global settings namespace via :meth:`export_config`. If ``None``, - :attr:`name` will be used. If ``False``, no configuration will be exported. - """ - - def export_component(self, config, settings): - """Export the component in :attr:`component` into the dictionary-like - component setting derived from :attr:`component_type`. - - Where applicable, the order parameter of the component (i.e. the - dictionary value) will be retrieved from the ``order`` add-on - configuration value. - - :param config: Add-on configuration from which to read component order - :type config: ``dict`` - - :param settings: Settings object into which to export component - :type settings: :class:`~scrapy.settings.Settings` - """ - if self.component_type: - comp = self.component or self - if self.component_key: - # e.g. for DOWNLOAD_HANDLERS: {'http': 'myclass'} - k = self.component_key - v = comp - else: - # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} - k = comp - v = config.get("order", self.component_order) - settings[self.component_type].update({k: v}, "addon") - - def export_basics(self, settings): - """Export the :attr:`basic_settings` attribute into the settings object. - - All settings will be exported with ``addon`` priority (see - :ref:`topics-api-settings`). - - :param settings: Settings object into which to expose the basic settings - :type settings: :class:`~scrapy.settings.Settings` - """ - for setting, value in (self.basic_settings or {}).items(): - settings.set(setting, value, "addon") - def export_config(self, config, settings): - """Export the add-on configuration, all keys in caps and with - :attr:`settings_prefix` or :attr:`name` prepended, into the settings + """Export the add-on configuration, all keys in caps, into the settings object. For example, the add-on configuration ``{'key': 'value'}`` will export - the setting ``ADDONNAME_KEY`` with a value of ``value``. All settings + the setting ``KEY`` with a value of ``value``. All settings will be exported with ``addon`` priority (see :ref:`topics-api-settings`). @@ -112,11 +31,8 @@ class Addon(object): :param settings: Settings object into which to export the configuration :type settings: :class:`~scrapy.settings.Settings` """ - if self.settings_prefix is False: - return conf = self.default_config or {} conf.update(config) - prefix = self.settings_prefix or self.name # Since default exported config is case-insensitive (everything will be # uppercased), make mapped config case-insensitive as well conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()} @@ -124,14 +40,11 @@ class Addon(object): if key.lower() in conf_mapping: key = conf_mapping[key.lower()] else: - key = (prefix + "_" + key).upper() + key = key.upper() settings.set(key, val, "addon") def update_settings(self, config, settings): - """Export both the basic settings and the add-on configuration. I.e., - call :meth:`export_basics` and :meth:`export_config`. - - For more advanced add-ons, you may want to override this callback. + """Modifiy `settings` to enable and configure required components. :param config: Add-on configuration :type config: ``dict`` @@ -139,12 +52,21 @@ class Addon(object): :param settings: Crawler settings object :type settings: :class:`~scrapy.settings.Settings` """ - self.export_component(config, settings) - self.export_basics(settings) self.export_config(config, settings) + def check_configuration(self, config, crawler): + """Perform post-initialization checks on fully configured `crawler`. -class AddonManager(Mapping): + :param config: Add-on configuration + :type config: ``dict`` + + :param crawler: the fully-initialized crawler + :type crawler: :class:`~scrapy.crawler.Crawler` + """ + pass + + +class AddonManager(Mapping[str, Addon]): """This class facilitates loading and storing :ref:`topics-addons`. You can treat it like a read-only dictionary in which keys correspond to @@ -160,108 +82,42 @@ class AddonManager(Mapping): """ - def __init__(self): - self._addons = OrderedDict() - self.configs = {} - self._disable_on_add = [] + def __init__(self) -> None: + self._addons: OrderedDict[str, Addon] = OrderedDict[str, Addon]() + self.configs: Dict[str, Dict[str, Any]] = {} - def __getitem__(self, name): + def __getitem__(self, name: str) -> Addon: return self._addons[name] - def __delitem__(self, name): - del self._addons[name] - del self.configs[name] - - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._addons) - def __len__(self): + def __len__(self) -> int: return len(self._addons) - def add(self, addon, config=None): + def add(self, addon: Any, config: Optional[Dict[str, Any]] = None): """Store an add-on. - If ``addon`` is a string, it will be treated as add-on path and passed - to :meth:`get_addon`. Otherwise, ``addon`` must be a Python object - implementing or providing Scrapy's add-on interface. The interface - will be enforced through ``zope.interface``'s ``verifyObject()``. - - If ``addon`` is a class, it will be instantiated. You can avoid this - (for example if you have implemented the add-on callbacks as class - methods) by declaring -- via ``zope.interface`` -- that your class - directly *provides* ``scrapy.interfaces.IAddon``. - :param addon: The add-on object (or path) to be stored - :type addon: Any Python object providing the add-on interface or ``str`` + :type addon: Python object, class or ``str`` :param config: The add-on configuration dictionary :type config: ``dict`` """ - addon = self.get_addon(addon) - if isclass(addon) and not IAddon.providedBy(addon): + if isinstance(addon, (type, str)): + addon = load_object(addon) + if isinstance(addon, type): addon = addon() - if not IAddon.providedBy(addon): - zope.interface.alsoProvides(addon, IAddon) - # zope.interface's exceptions are already quite helpful. Still, should - # we catch them and log an error message? - verifyObject(IAddon, addon) name = addon.name if name in self: raise ValueError(f"Addon '{name}' already loaded") self._addons[name] = addon self.configs[name] = config or {} - if name in self._disable_on_add: - self.configs[name]["_enabled"] = False - self._disable_on_add.remove(name) - - def remove(self, addon): - """Remove an add-on. - - If ``addon`` is the name of a stored add-on, that add-on will be - removed. Otherwise, you can use the argument in the same fashion as - in :meth:`add`. - - :param addon: The add-on name, object, or path to be removed - :type addon: Any Python object providing the add-on interface or ``str`` - """ - if addon in self: - del self[addon] - elif hasattr(addon, "name") and addon.name in self: - del self[addon.name] - else: - try: - del self[self.get_addon(addon).name] - except NameError: - raise KeyError - - @staticmethod - def get_addon(path): - """Get an add-on object by its Python or file path. - - ``path`` is assumed to be an import path of an add-on. If the object or - module pointed to by ``path`` has an attribute named ``_addon`` that - attribute will be assumed to be the add-on. :meth:`get_addon` will keep - following ``_addon`` attributes until it finds an object that does not - have an attribute named ``_addon``. - - :param path: Import path of an add-on - :type path: ``str`` - """ - if isinstance(path, str): - try: - obj = load_module_or_object(path) - except NameError: - raise NameError(f"Could not find add-on '{path}'") - else: - obj = path - if hasattr(obj, "_addon"): - obj = AddonManager.get_addon(obj._addon) - return obj def load_settings(self, settings): """Load add-ons and configurations from settings object. - This will invoke :meth:`get_addon` for every add-on path in the + This will load the addon for every add-on path in the ``ADDONS`` setting. For each of these add-ons, the configuration will be read from the dictionary setting whose name matches the uppercase add-on name. @@ -271,141 +127,12 @@ class AddonManager(Mapping): :type settings: :class:`~scrapy.settings.Settings` """ paths = build_component_list(settings["ADDONS"]) - addons = [self.get_addon(path) for path in paths] + addons = [load_object(path) for path in paths] configs = [settings.getdict(addon.name.upper()) for addon in addons] for a, c in zip(addons, configs): self.add(a, c) - def check_dependency_clashes(self) -> None: - """Check for incompatibilities in add-on dependencies. - - Add-ons can provide information about their dependencies in their - ``provides``, ``modifies`` and ``requires`` attributes. This method will - raise an ``ImportError`` if - - * a component required by an add-on is not provided by any other add-on, - or - * a component modified by an add-on is not provided by any other add-on, - or - * the same component is provided by more than one add-on, - - and warn when a component required by an add-on is modified by any other - add-on. - """ - # Collect all active add-ons and the components they provide - versions: Dict[str, Version] = {} - - def add_version(project_name, version): - if project_name in versions: - raise ImportError( - f"Component {project_name} provided by multiple add-ons" - ) - versions[project_name] = Version(version) - - for name in self: - ver = self[name].version - add_version(name, ver) - for provides_name in getattr(self[name], "provides", []): - add_version(provides_name, ver) - - # Collect all required and modified components - def compile_attribute_dict(attribute_name): - attrs = defaultdict(list) - for name in self: - for entry in getattr(self[name], attribute_name, []): - attrs[entry].append(name) - return attrs - - modified = compile_attribute_dict("modifies") - required = compile_attribute_dict("requires") - - req_or_mod = set(required.keys()).union(modified.keys()) - for reqstr in req_or_mod: - req = Requirement(reqstr) - if req.name not in versions or versions[req.name] not in req.specifier: - raise ImportError( - f"Add-ons {required[reqstr] + modified[reqstr]} require" - f" or modify missing component {reqstr}" - ) - - mod_and_req = set(required.keys()).intersection(modified.keys()) - for conflict in mod_and_req: - warnings.warn( - f"Component '{conflict}', required by add-ons {required[conflict]}," - f" is modified by add-ons {modified[conflict]}" - ) - - def disable(self, addon): - """Disable an add-on, i.e. prevent its callbacks from being called. - - If you disable an add-on before it is loaded, it will be disabled as - soon as it is added to the :class:`AddonManager`. - - :param addon: Name of the add-on to be disabled - :type addon: ``str`` - """ - if addon in self: - self.configs[addon]["_enabled"] = False - else: - self._disable_on_add.append(addon) - - def enable(self, addon): - """Re-enable a disabled add-on. - - Will raise ``ValueError`` if the add-on is neither already loaded nor - marked for being disabled on adding. - - :param addon: Name of the add-on to be enabled - :type addon: ``str`` - """ - if addon in self: - self.configs[addon]["_enabled"] = True - elif addon in self._disable_on_add: - self._disable_on_add.remove(addon) - else: - raise ValueError("Add-ons need to be added before they can be enabled") - - @property - def disabled(self): - """Names of disabled add-ons""" - return [ - a for a in self if not self.configs[a].get("_enabled", True) - ] + self._disable_on_add - - @property - def enabled(self): - """Names of enabled add-ons""" - return [a for a in self if self.configs[a].get("_enabled", True)] - - def _call_if_exists(self, obj, cbname, *args, **kwargs): - if obj is None: - return - try: - cb = getattr(obj, cbname) - except AttributeError: - return - else: - cb(*args, **kwargs) - - def _call_addon(self, addonname, cbname, *args, **kwargs): - if self.configs[addonname].get("_enabled", True): - self._call_if_exists( - self[addonname], cbname, self.configs[addonname], *args, **kwargs - ) - - def update_addons(self): - """Call ``update_addons()`` of all held add-ons. - - This will also call ``update_addons()`` of all add-ons that are added - last minute during the ``update_addons()`` routine of other add-ons. - """ - called_addons = set() - while called_addons != set(self): - for name in set(self).difference(called_addons): - called_addons.add(name) - self._call_addon(name, "update_addons", self) - - def update_settings(self, settings): + def update_settings(self, settings) -> None: """Call ``update_settings()`` of all held add-ons. :param settings: The :class:`~scrapy.settings.Settings` object to be \ @@ -413,13 +140,13 @@ class AddonManager(Mapping): :type settings: :class:`~scrapy.settings.Settings` """ for name in self: - self._call_addon(name, "update_settings", settings) + self[name].update_settings(self.configs[name], settings) - def check_configuration(self, crawler): + def check_configuration(self, crawler) -> None: """Call ``check_configuration()`` of all held add-ons. :param crawler: the fully-initialized crawler :type crawler: :class:`~scrapy.crawler.Crawler` """ for name in self: - self._call_addon(name, "check_configuration", crawler) + self[name].check_configuration(self.configs[name], crawler) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 1b579f10e..efc9b36ea 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,7 +6,6 @@ import sys from importlib.metadata import entry_points import scrapy -from scrapy.addons import AddonManager from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError @@ -131,8 +130,6 @@ def execute(argv=None, settings=None): else: settings["EDITOR"] = editor - addons = AddonManager() - inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) @@ -156,7 +153,7 @@ def execute(argv=None, settings=None): opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) - cmd.crawler_process = CrawlerProcess(settings, addons=addons) + cmd.crawler_process = CrawlerProcess(settings) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 0c9861dbd..a0cb368ed 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -58,7 +58,6 @@ class Crawler: spidercls: Type[Spider], settings: Union[None, dict, Settings] = None, init_reactor: bool = False, - addons: Optional[AddonManager] = None, ): if isinstance(spidercls, Spider): raise ValueError("The spidercls argument must be a class, not an object") @@ -70,10 +69,8 @@ class Crawler: self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self.addons: AddonManager = addons if addons is not None else AddonManager() + self.addons: AddonManager = AddonManager() self.addons.load_settings(self.settings) - self.addons.update_addons() - self.addons.check_dependency_clashes() self.addons.update_settings(self.settings) self.signals: SignalManager = SignalManager(self) @@ -200,15 +197,10 @@ class CrawlerRunner: ) return loader_cls.from_settings(settings.frozencopy()) - def __init__( - self, - settings: Union[Dict[str, Any], Settings, None] = None, - addons: Optional[AddonManager] = None, - ): + def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings - self.addons: Optional[AddonManager] = addons self.spider_loader = self._get_spider_loader(settings) self._crawlers: Set[Crawler] = set() self._active: Set[defer.Deferred] = set() @@ -292,7 +284,7 @@ class CrawlerRunner: def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings, addons=self.addons) + return Crawler(spidercls, self.settings) def stop(self): """ @@ -338,13 +330,8 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__( - self, - settings=None, - install_root_handler: bool = True, - addons: Optional[AddonManager] = None, - ): - super().__init__(settings, addons) + def __init__(self, settings=None, install_root_handler=True): + super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) self._initialized_reactor = False diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index b8aa77ced..9a2c5f170 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,4 +1,4 @@ -from zope.interface import Attribute, Interface +from zope.interface import Interface class ISpiderLoader(Interface): @@ -15,22 +15,3 @@ class ISpiderLoader(Interface): def find_by_request(request): """Return the list of spiders names that can handle the given request""" - - -class IAddon(Interface): - """Scrapy add-on""" - - name = Attribute("""Add-on name""") - version = Attribute("""Add-on version string (PEP440)""") - - # XXX: Can methods be declared optional? I.e., can I enforce the signature - # but not the existence of a method? - - # def update_addons(config, addons): - # """Enables and configures other add-ons""" - - # def update_settings(config, settings): - # """Modifies `settings` to enable and configure required components""" - - # def check_configuration(config, crawler): - # """Performs post-initialization checks on fully configured `crawler`""" diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 8d1544c68..3ade1d105 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -99,18 +99,12 @@ def init_env(project="default", set_syspath=True): sys.path.append(projdir) -def config_from_filepath(sources): - """Create a ConfigParser and read in the given `sources`, which can be - either a filename or a list of filenames.""" - cfg = ConfigParser() - cfg.read(sources) - return cfg - - def get_config(use_closest=True): """Get Scrapy config file as a ConfigParser""" sources = get_sources(use_closest) - return config_from_filepath(sources) + cfg = ConfigParser() + cfg.read(sources) + return cfg def get_sources(use_closest=True) -> List[str]: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 4e23b01c3..b3c28da92 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -67,7 +67,7 @@ def load_object(path: Union[str, Callable]) -> Any: if callable(path): return path raise TypeError( - "Unexpected argument type, expected string " f"or object, got: {type(path)}" + f"Unexpected argument type, expected string or object, got: {type(path)}" ) try: @@ -86,22 +86,6 @@ def load_object(path: Union[str, Callable]) -> Any: return obj -def load_module_or_object(path): - """Load python module or (non-module) object from given path. - - Path can be both a Python or a file path. - """ - try: - return import_module(path) - except ImportError: - pass - try: - return load_object(path) - except (ValueError, NameError, ImportError): - pass - raise NameError(f"Could not load '{path}'") - - def walk_modules(path: str) -> List[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index fe26e1708..97de8d25a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -73,7 +73,6 @@ def get_crawler( spidercls: Optional[Type[Spider]] = None, settings_dict: Optional[Dict[str, Any]] = None, prevent_warnings: bool = True, - addons=None, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level @@ -87,7 +86,7 @@ def get_crawler( if prevent_warnings: settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7" settings.update(settings_dict or {}) - runner = CrawlerRunner(settings, addons) + runner = CrawlerRunner(settings) return runner.create_crawler(spidercls or Spider) diff --git a/tests/test_addons.py b/tests/test_addons.py new file mode 100644 index 000000000..ecdc0426c --- /dev/null +++ b/tests/test_addons.py @@ -0,0 +1,72 @@ +import unittest + +from scrapy.addons import Addon, AddonManager +from scrapy.settings import BaseSettings + + +class GoodAddon(object): + name = "GoodAddon" + + def update_settings(self, config, settings): + pass + + def check_configuration(self, config, crawler): + pass + + +class AddonTest(unittest.TestCase): + def setUp(self): + class AddonWithAttributes(Addon): + name = "Test" + + self.testaddon = AddonWithAttributes() + + def test_export_config(self): + settings = BaseSettings() + self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} + self.testaddon.default_config = {"key": 55, "defaultkey": 100} + self.testaddon.export_config( + {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings + ) + self.assertEqual(settings["KEY"], 313) + self.assertEqual(settings["DEFAULTKEY"], 100) + self.assertEqual(settings["OTHERKEY"], True) + self.assertNotIn("MAPPED_key", settings) + self.assertNotIn("MAPPED_KEY", settings) + self.assertEqual(settings["MAPPING_WORKED"], 99) + self.assertEqual(settings.getpriority("KEY"), 15) + + def test_update_settings(self): + settings = BaseSettings() + settings.set("KEY1", "default", priority="default") + settings.set("KEY2", "project", priority="project") + addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} + self.testaddon.update_settings(addon_config, settings) + self.assertEqual(settings["KEY1"], "addon") + self.assertEqual(settings["KEY2"], "project") + self.assertEqual(settings["KEY3"], "addon") + + +class AddonManagerTest(unittest.TestCase): + def setUp(self): + self.manager = AddonManager() + + def test_add(self): + manager = AddonManager() + manager.add("tests.test_addons.GoodAddon") + self.assertCountEqual(manager, ["GoodAddon"]) + self.assertIsInstance(manager["GoodAddon"], GoodAddon) + + def test_load_settings(self): + settings = BaseSettings() + settings.set( + "ADDONS", + {"tests.test_addons.GoodAddon": 0}, + ) + settings.set("GOODADDON", {"key": "val2"}) + manager = AddonManager() + manager.load_settings(settings) + self.assertCountEqual(manager, ["GoodAddon"]) + self.assertIsInstance(manager["GoodAddon"], GoodAddon) + self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) + self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py deleted file mode 100644 index fa3f706e5..000000000 --- a/tests/test_addons/__init__.py +++ /dev/null @@ -1,342 +0,0 @@ -import itertools -import unittest -import warnings -from unittest import mock - -from zope.interface import directlyProvides -from zope.interface.exceptions import BrokenImplementation, MultipleInvalid -from zope.interface.verify import verifyObject - -from scrapy.addons import Addon, AddonManager -from scrapy.crawler import Crawler -from scrapy.interfaces import IAddon -from scrapy.settings import BaseSettings - -from . import addonmod, addons - - -class AddonTest(unittest.TestCase): - def setUp(self): - self.rawaddon = Addon() - - class AddonWithAttributes(Addon): - name = "Test" - version = "1.0" - - self.testaddon = AddonWithAttributes() - - def test_interface(self): - # Raw Addon should fail exactly b/c name and version are not given - self.assertFalse(hasattr(self.rawaddon, "name")) - self.assertFalse(hasattr(self.rawaddon, "version")) - self.assertRaises(MultipleInvalid, verifyObject, IAddon, self.rawaddon) - verifyObject(IAddon, self.testaddon) - - def test_export_component(self): - settings = BaseSettings( - {"ITEM_PIPELINES": BaseSettings(), "DOWNLOAD_HANDLERS": BaseSettings()}, - "default", - ) - self.testaddon.component_type = None - self.testaddon.export_component({}, settings) - self.assertEqual(len(settings["ITEM_PIPELINES"]), 0) - self.testaddon.component_type = "ITEM_PIPELINES" - self.testaddon.component = "test.component" - self.testaddon.export_component({}, settings) - self.assertCountEqual(settings["ITEM_PIPELINES"], ["test.component"]) - self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 0) - self.testaddon.component_order = 313 - self.testaddon.export_component({}, settings) - self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 313) - self.testaddon.component_type = "DOWNLOAD_HANDLERS" - self.testaddon.component_key = "http" - self.testaddon.export_component({}, settings) - self.assertEqual(settings["DOWNLOAD_HANDLERS"]["http"], "test.component") - - def test_export_basics(self): - settings = BaseSettings() - self.testaddon.basic_settings = {"TESTKEY": 313, "OTHERKEY": True} - self.testaddon.export_basics(settings) - self.assertEqual(settings["TESTKEY"], 313) - self.assertEqual(settings["OTHERKEY"], True) - self.assertEqual(settings.getpriority("TESTKEY"), 15) - - def test_export_config(self): - settings = BaseSettings() - self.testaddon.settings_prefix = None - self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} - self.testaddon.default_config = {"key": 55, "defaultkey": 100} - self.testaddon.export_config( - {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings - ) - self.assertEqual(settings["TEST_KEY"], 313) - self.assertEqual(settings["TEST_DEFAULTKEY"], 100) - self.assertEqual(settings["TEST_OTHERKEY"], True) - self.assertNotIn("MAPPED_key", settings) - self.assertNotIn("MAPPED_KEY", settings) - self.assertEqual(settings["MAPPING_WORKED"], 99) - self.assertEqual(settings.getpriority("TEST_KEY"), 15) - - self.testaddon.settings_prefix = "PREF" - self.testaddon.export_config({"newkey": 99}, settings) - self.assertEqual(settings["PREF_NEWKEY"], 99) - - with mock.patch.object(settings, "set") as mock_set: - self.testaddon.settings_prefix = False - self.testaddon.export_config({"thirdnewkey": 99}, settings) - self.assertEqual(mock_set.call_count, 0) - - def test_update_settings(self): - settings = BaseSettings() - settings.set("TEST_KEY1", "default", priority="default") - settings.set("TEST_KEY2", "project", priority="project") - self.testaddon.settings_prefix = None - self.testaddon.basic_settings = {"OTHERTEST_KEY": "addon"} - addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} - self.testaddon.update_settings(addon_config, settings) - self.assertEqual(settings["OTHERTEST_KEY"], "addon") - self.assertEqual(settings["TEST_KEY1"], "addon") - self.assertEqual(settings["TEST_KEY2"], "project") - self.assertEqual(settings["TEST_KEY3"], "addon") - - -class AddonManagerTest(unittest.TestCase): - def setUp(self): - self.manager = AddonManager() - - def test_add(self): - manager = AddonManager() - manager.add(addonmod, {"key": "val1"}) - manager.add("tests.test_addons.addons.GoodAddon") - self.assertCountEqual(manager, ["AddonModule", "GoodAddon"]) - self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) - self.assertCountEqual(manager.configs["AddonModule"], ["key"]) - self.assertEqual(manager.configs["AddonModule"]["key"], "val1") - self.assertRaises(ValueError, manager.add, addonmod) - - def test_add_dont_instantiate_providing_classes(self): - class ProviderGoodAddon(addons.GoodAddon): - pass - - directlyProvides(ProviderGoodAddon, IAddon) - manager = AddonManager() - manager.add(ProviderGoodAddon) - self.assertIs(manager["GoodAddon"], ProviderGoodAddon) - - def test_add_verifies(self): - brokenaddon = self.manager.get_addon("tests.test_addons.addons.BrokenAddon") - self.assertRaises( - BrokenImplementation, - self.manager.add, - brokenaddon, - ) - - def test_add_adds_missing_interface_declaration(self): - class GoodAddonWithoutDeclaration(object): - name = "GoodAddonWithoutDeclaration" - version = "1.0" - - self.manager.add(GoodAddonWithoutDeclaration) - - def test_remove(self): - manager = AddonManager() - - def test_gets_removed(removearg): - manager.add(addonmod) - self.assertIn("AddonModule", manager) - manager.remove(removearg) - self.assertNotIn("AddonModule", manager) - - test_gets_removed("AddonModule") - test_gets_removed(addonmod) - test_gets_removed("tests.test_addons.addonmod") - self.assertRaises(KeyError, manager.remove, "nonexistent") - self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) - - def test_get_addon(self): - goodaddon = self.manager.get_addon("tests.test_addons.addons.GoodAddon") - self.assertIs(goodaddon, addons.GoodAddon) - - loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod") - self.assertIs(loaded_addonmod, addonmod) - - goodaddon = self.manager.get_addon("tests.test_addons.addons") - self.assertIsInstance(goodaddon, addons.GoodAddon) - - self.assertRaises(NameError, self.manager.get_addon, "xy.n_onexistent") - - def test_get_addon_forward(self): - class SomeCls(object): - _addon = "tests.test_addons.addons.GoodAddon" - - self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon) - - def test_get_addon_nested(self): - x = addons.GoodAddon("outer") - x._addon = addons.GoodAddon("middle") - x._addon._addon = addons.GoodAddon("inner") - self.assertIs(self.manager.get_addon(x), x._addon._addon) - - def test_load_settings(self): - settings = BaseSettings() - settings.set( - "ADDONS", - {"tests.test_addons.addonmod": 0, "tests.test_addons.addons.GoodAddon": 0}, - ) - settings.set("ADDONMODULE", {"key": "val1"}) - settings.set("GOODADDON", {"key": "val2"}) - manager = AddonManager() - manager.load_settings(settings) - self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) - self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) - self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) - self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") - self.assertEqual(manager["AddonModule"], addonmod) - self.assertIn("key", manager.configs["AddonModule"]) - self.assertEqual(manager.configs["AddonModule"]["key"], "val1") - - def test_load_settings_order(self): - # Get three addons named 0, 1, 2 - addonlist = [addons.GoodAddon(str(x)) for x in range(3)] - # Test for every possible ordering - for ordered_addons in itertools.permutations(addonlist): - expected_order = [a.name for a in ordered_addons] - settings = BaseSettings( - {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} - ) - manager = AddonManager() - manager.load_settings(settings) - self.assertEqual(list(manager.keys()), expected_order) - - def test_enabled_disabled(self): - manager = AddonManager() - manager.add(addons.GoodAddon("FirstAddon")) - manager.add(addons.GoodAddon("SecondAddon")) - self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) - self.assertEqual(manager.disabled, []) - manager.disable("FirstAddon") - self.assertEqual(manager.enabled, ["SecondAddon"]) - self.assertEqual(manager.disabled, ["FirstAddon"]) - manager.enable("FirstAddon") - self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) - self.assertEqual(manager.disabled, []) - - def test_enable_before_add(self): - manager = AddonManager() - self.assertRaises(ValueError, manager.enable, "FirstAddon") - manager.disable("FirstAddon") - manager.enable("FirstAddon") - manager.add(addons.GoodAddon("FirstAddon")) - self.assertIn("FirstAddon", manager.enabled) - - def test_disable_before_add(self): - manager = AddonManager() - manager.disable("FirstAddon") - manager.add(addons.GoodAddon("FirstAddon")) - self.assertEqual(manager.disabled, ["FirstAddon"]) - - def test_callbacks(self): - first_addon = addons.GoodAddon("FirstAddon") - second_addon = addons.GoodAddon("SecondAddon") - - manager = AddonManager() - manager.add(first_addon, {"test": "first"}) - manager.add(second_addon, {"test": "second"}) - crawler = mock.create_autospec(Crawler) - settings = BaseSettings() - - with mock.patch.object( - first_addon, "update_addons" - ) as ua_first, mock.patch.object( - second_addon, "update_addons" - ) as ua_second, mock.patch.object( - first_addon, "update_settings" - ) as us_first, mock.patch.object( - second_addon, "update_settings" - ) as us_second, mock.patch.object( - first_addon, "check_configuration" - ) as cc_first, mock.patch.object( - second_addon, "check_configuration" - ) as cc_second: - manager.update_addons() - ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) - ua_second.assert_called_once_with(manager.configs["SecondAddon"], manager) - manager.update_settings(settings) - us_first.assert_called_once_with(manager.configs["FirstAddon"], settings) - us_second.assert_called_once_with(manager.configs["SecondAddon"], settings) - manager.check_configuration(crawler) - cc_first.assert_called_once_with(manager.configs["FirstAddon"], crawler) - cc_second.assert_called_once_with(manager.configs["SecondAddon"], crawler) - self.assertEqual(ua_first.call_count, 1) - self.assertEqual(ua_second.call_count, 1) - self.assertEqual(us_first.call_count, 1) - self.assertEqual(us_second.call_count, 1) - - us_first.reset_mock() - us_second.reset_mock() - manager.disable("FirstAddon") - manager.update_settings(settings) - self.assertEqual(us_first.call_count, 0) - manager.enable("FirstAddon") - manager.update_settings(settings) - self.assertEqual(us_first.call_count, 1) - self.assertEqual(us_second.call_count, 2) - - # This will become relevant when we let spiders implement the add-on - # interface and should be replaced with a test where - # AddonManager.spidercls = None then. - manager._call_if_exists(None, "irrelevant") - - def test_update_addons_last_minute_add(self): - class AddedAddon(addons.GoodAddon): - name = "AddedAddon" - - class FirstAddon(addons.GoodAddon): - name = "FirstAddon" - - def update_addons(self, config, addons): - addons.add(AddedAddon()) - - manager = AddonManager() - first_addon = FirstAddon() - with mock.patch.object( - first_addon, "update_addons", wraps=first_addon.update_addons - ) as ua_first, mock.patch.object(AddedAddon, "update_addons") as ua_added: - manager.add(first_addon, {"non-empty": "dict"}) - manager.update_addons() - self.assertCountEqual(manager, ["FirstAddon", "AddedAddon"]) - ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) - ua_added.assert_called_once_with(manager.configs["AddedAddon"], manager) - - def test_check_dependency_clashes_attributes(self): - provides = addons.GoodAddon("ProvidesAddon") - provides.provides = ("test",) - provides2 = addons.GoodAddon("ProvidesAddon2") - provides2.provides = ("test",) - requires = addons.GoodAddon("RequiresAddon") - requires.requires = ("test",) - requires_name = addons.GoodAddon("RequiresNameAddon") - requires_name.requires = ("ProvidesAddon",) - requires_newer = addons.GoodAddon("RequiresNewerAddon") - requires_newer.requires = ("test>=2.0",) - modifies = addons.GoodAddon("ModifiesAddon") - modifies.modifies = ("test",) - - def check_with(*addons): - manager = AddonManager() - for a in addons: - manager.add(a) - return manager.check_dependency_clashes() - - self.assertRaises(ImportError, check_with, requires) - self.assertRaises(ImportError, check_with, modifies) - self.assertRaises(ImportError, check_with, provides, provides2) - self.assertRaises(ImportError, check_with, provides, requires_newer) - with warnings.catch_warnings(record=True) as w: - check_with(provides, modifies) - check_with(provides) - check_with(provides, requires) - check_with(provides, requires_name) - self.assertEqual(len(w), 0) - check_with(requires, provides, modifies) - self.assertEqual(len(w), 1) diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py deleted file mode 100644 index 092c3c0eb..000000000 --- a/tests/test_addons/addonmod.py +++ /dev/null @@ -1,16 +0,0 @@ -from zope.interface import moduleProvides - -from scrapy.interfaces import IAddon - -moduleProvides(IAddon) - -name = "AddonModule" -version = "1.0" - - -def update_settings(config, settings): - pass - - -def check_configuration(config, crawler): - pass diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py deleted file mode 100644 index d878f37ea..000000000 --- a/tests/test_addons/addons.py +++ /dev/null @@ -1,33 +0,0 @@ -from zope.interface import implementer - -from scrapy.interfaces import IAddon - - -@implementer(IAddon) -class GoodAddon(object): - name = "GoodAddon" - version = "1.0" - - def __init__(self, name=None, version=None): - if name is not None: - self.name = name - if version is not None: - self.version = version - - def update_addons(self, config, addons): - pass - - def update_settings(self, config, settings): - pass - - def check_configuration(self, config, crawler): - pass - - -@implementer(IAddon) -class BrokenAddon(object): - name = "BrokenAddon" - # No version - - -_addon = GoodAddon() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 920e5f4ae..d844a645f 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -412,7 +412,6 @@ with multiples lines def test_abort_on_addon_failed_check(self): class FailedCheckAddon(Addon): name = "FailedCheckAddon" - version = "1.0" def check_configuration(self, config, crawler): raise ValueError diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ed34d9e58..d54a2cb7e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -15,7 +15,6 @@ from twisted.trial import unittest from w3lib import __version__ as w3lib_version import scrapy -from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions import telnet @@ -56,32 +55,6 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertFalse(settings.frozen) self.assertTrue(crawler.settings.frozen) - def test_populate_addons_settings(self): - class TestAddon(Addon): - name = "TestAddon" - version = "1.0" - - addonconfig = {"TEST1": "addon", "TEST2": "addon", "TEST3": "addon"} - - class TestAddon2(Addon): - name = "testAddon2" - version = "1.0" - - addonconfig2 = {"TEST": "addon2"} - - settings = Settings() - settings.set("TESTADDON_TEST1", "project", priority="project") - settings.set("TESTADDON_TEST2", "default", priority="default") - addonmgr = AddonManager() - addonmgr.add(TestAddon(), addonconfig) - addonmgr.add(TestAddon2(), addonconfig2) - crawler = Crawler(DefaultSpider, settings, addons=addonmgr) - - self.assertEqual(crawler.settings["TESTADDON_TEST1"], "project") - self.assertEqual(crawler.settings["TESTADDON_TEST2"], "addon") - self.assertEqual(crawler.settings["TESTADDON_TEST3"], "addon") - self.assertEqual(crawler.settings["TESTADDON2_TEST"], "addon2") - def test_crawler_accepts_dict(self): crawler = get_crawler(DefaultSpider, {"foo": "bar"}) self.assertEqual(crawler.settings["foo"], "bar") diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 7932ca04c..69793ee75 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -8,7 +8,6 @@ from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, create_instance, - load_module_or_object, load_object, rel_has_nofollow, set_environ, @@ -36,12 +35,6 @@ class UtilsMiscTestCase(unittest.TestCase): self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999") self.assertRaises(TypeError, load_object, {}) - def test_load_module_or_object(self): - testmod = load_module_or_object(__name__ + ".testmod") - self.assertTrue(hasattr(testmod, "TESTVAR")) - obj = load_object("scrapy.utils.misc.load_object") - self.assertIs(obj, load_object) - def test_walk_modules(self): mods = walk_modules("tests.test_utils_misc.test_walk_modules") expected = [ diff --git a/tests/test_utils_misc/testmod.py b/tests/test_utils_misc/testmod.py deleted file mode 100644 index eb540335f..000000000 --- a/tests/test_utils_misc/testmod.py +++ /dev/null @@ -1 +0,0 @@ -TESTVAR = True From 760c0db094b3147236294f656f4168c16502de97 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 26 Jun 2023 17:15:13 +0400 Subject: [PATCH 1039/2083] Fix typing on 3.8. --- scrapy/addons.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index a54086fda..523c36e9c 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,5 +1,4 @@ -from collections.abc import Mapping -from typing import Any, Dict, Iterator, Optional, OrderedDict +from typing import Any, Dict, Iterator, Mapping, Optional, OrderedDict from scrapy.utils.conf import build_component_list from scrapy.utils.misc import load_object From ebce5b4bcb8b0af172dce063d9a220359202cf36 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Tue, 27 Jun 2023 09:22:50 +0300 Subject: [PATCH 1040/2083] periodic_log: tests for logging deltas added --- tests/test_extension_periodic_log.py | 69 ++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 4dccc687b..1df030548 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -51,8 +51,16 @@ stats_dump_2 = { } +class TestExtPeriodicLog(PeriodicLog): + def set_a(self): + self.stats._stats = stats_dump_1 + + def set_b(self): + self.stats._stats = stats_dump_2 + + def extension(settings=None): - return PeriodicLog.from_crawler( + return TestExtPeriodicLog.from_crawler( Crawler( MetaSpider, settings=settings, @@ -124,14 +132,57 @@ class TestPeriodicLog(unittest.TestCase): } ) - def test_periodic_log_stats(self): - pass - def test_log_delta(self): - pass + def emulate(settings=None): + ext = extension(settings) + ext.spider_opened(MetaSpider) + ext.set_a() + a = ext.log_delta() + ext.set_a() + b = ext.log_delta() + return ext, a, b - def test_settings_include(self): - pass + def check(settings: dict, condition: callable): + ext, a, b = emulate(settings) + assert list(a["delta"].keys()) == [ + k for k, v in ext.stats._stats.items() if condition(k, v) + ] + assert list(b["delta"].keys()) == [ + k for k, v in ext.stats._stats.items() if condition(k, v) + ] - def test_settings_exclude(self): - pass + # Including all + check({"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float))) + + # include: + check( + {"PERIODIC_LOG_DELTA": {"include": ["downloader/"]}}, + lambda k, v: isinstance(v, (int, float)) and "downloader/" in k, + ) + + # include multiple + check( + {"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}}, + lambda k, v: isinstance(v, (int, float)) + and ("downloader/" in k or "scheduler/" in k), + ) + + # exclude + check( + {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}}, + lambda k, v: isinstance(v, (int, float)) and "downloader/" not in k, + ) + + # exclude multiple + check( + {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}}, + lambda k, v: isinstance(v, (int, float)) + and ("downloader/" not in k and "scheduler/" not in k), + ) + + # include exclude combined + check( + {"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}}, + lambda k, v: isinstance(v, (int, float)) + and ("downloader/" in k and "scheduler/" not in k), + ) From 315861c31d1bb1b2430883267b6973604b34a147 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Tue, 27 Jun 2023 09:24:03 +0300 Subject: [PATCH 1041/2083] periodic_log: stats filtering updated --- scrapy/extensions/periodic_log.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index b1f5b8894..1023b3cd5 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -127,6 +127,8 @@ class PeriodicLog: for p in exclude: if p in stat_name: return False + if exclude and not include: + return True for p in include: if p in stat_name: return True From 56c38231b49f4f49fe5e215e46ec80c71755d44e Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Tue, 27 Jun 2023 11:48:51 +0300 Subject: [PATCH 1042/2083] periodic_log: tests for logging stats added --- tests/test_extension_periodic_log.py | 55 +++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 1df030548..053613510 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -184,5 +184,58 @@ class TestPeriodicLog(unittest.TestCase): check( {"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: isinstance(v, (int, float)) - and ("downloader/" in k and "scheduler/" not in k), + and ("downloader/" in k and "bytes" not in k), ) + + def test_log_stats(self): + def emulate(settings=None): + ext = extension(settings) + ext.spider_opened(MetaSpider) + ext.set_a() + a = ext.log_crawler_stats() + ext.set_a() + b = ext.log_crawler_stats() + return ext, a, b + + def check(settings: dict, condition: callable): + ext, a, b = emulate(settings) + assert list(a["stats"].keys()) == [ + k for k, v in ext.stats._stats.items() if condition(k, v) + ] + assert list(b["stats"].keys()) == [ + k for k, v in ext.stats._stats.items() if condition(k, v) + ] + + # Including all + check({"PERIODIC_LOG_STATS": True}, lambda k, v: True) + + # include: + check( + {"PERIODIC_LOG_STATS": {"include": ["downloader/"]}}, + lambda k, v: "downloader/" in k, + ) + + # include multiple + check( + {"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}}, + lambda k, v: "downloader/" in k or "scheduler/" in k, + ) + + # exclude + check( + {"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}}, + lambda k, v: "downloader/" not in k, + ) + + # exclude multiple + check( + {"PERIODIC_LOG_STATS": {"exclude": ["downloader/", "scheduler/"]}}, + lambda k, v: "downloader/" not in k and "scheduler/" not in k, + ) + + # include exclude combined + check( + {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}}, + lambda k, v: "downloader/" in k and "bytes" not in k, + ) + # From b6196309cb654e6662197e19de36a9d18a83f12f Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 28 Jun 2023 03:28:49 -0300 Subject: [PATCH 1043/2083] fix: Return value instead of `SettingsAttribute` object when using `pop` method (#5963) --- scrapy/settings/__init__.py | 4 ++-- tests/test_settings/__init__.py | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 8b3bdbabe..cc44d67e8 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -449,12 +449,12 @@ class BaseSettings(MutableMapping): def pop(self, name, default=__default): try: - value = self.attributes[name] + value = self.attributes[name].value except KeyError: if default is self.__default: raise - return SettingsAttribute(default, get_settings_priority("project")) + return default else: self.__delitem__(name) return value diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index bb6dc67fa..5fc825393 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -457,19 +457,15 @@ class SettingsTest(unittest.TestCase): with self.assertRaises(KeyError): settings.pop("DUMMY_CONFIG") - dummy_config = settings.pop("DUMMY_CONFIG", "dummy_value") - - self.assertEqual( - repr(dummy_config), "" - ) - self.assertEqual(dummy_config.value, "dummy_value") + dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value") + self.assertEqual(dummy_config_value, "dummy_value") def test_pop_item_with_immutable_settings(self): settings = Settings( {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} ) - self.assertEqual(settings.pop("DUMMY_CONFIG").value, "dummy_value") + self.assertEqual(settings.pop("DUMMY_CONFIG"), "dummy_value") settings.freeze() From 9612ae3e93239b86cedcd124073de6fff2736e99 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 Jun 2023 12:41:33 +0400 Subject: [PATCH 1044/2083] Remove more code. --- docs/topics/addons.rst | 100 ++++----------------------------- scrapy/addons.py | 124 ++++------------------------------------- tests/test_addons.py | 48 +++++----------- tests/test_crawl.py | 16 ------ 4 files changed, 37 insertions(+), 251 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 6a9590f33..3421864fa 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -17,40 +17,26 @@ Activating and configuring add-ons Add-ons and their configuration live in Scrapy's :class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler` initialization the add-on manager will read a list of enabled add-ons from your -``ADDONS`` setting and their optional configuration from the respective -settings. +``ADDONS`` setting. The ``ADDONS`` setting is a dict in which every key is an addon class or its import path and the value is its priority. -The configuration of an add-on, if necessary at all, is stored as a dictionary -setting whose name is the uppercase add-on name. - -This is an example where two add-ons (in this case with one requiring no -configuration) are enabled/configured in a project's ``settings.py``:: +This is an example where two add-ons are enabled in a project's +``settings.py``:: ADDONS = { 'path.to.someaddon': 0, path.to.someaddon2: 1, } - SOMEADDON = { - 'some_config': True, - } - Writing your own add-ons ======================== -Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*: +Add-ons are (any) Python *objects* that include the following method: -.. attribute:: name - - string with add-on name - - :type: ``str`` - -.. method:: update_settings(config, settings) +.. method:: update_settings(settings) This method is called during the initialization of the :class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks @@ -58,88 +44,26 @@ Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*: :class:`~scrapy.settings.Settings` object as wished, e.g. enable components for this add-on or set required configuration of other extensions. - :param config: Configuration of this add-on - :type config: ``dict`` - :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.Settings` -.. method:: check_configuration(config, crawler) - - This method is called when the :class:`~scrapy.crawler.Crawler` has been - fully initialized, immediately before it starts crawling. You can perform - additional dependency and configuration checks here. - - :param config: Configuration of this add-on - :type config: ``dict`` - - :param crawler: Fully initialized Scrapy crawler - :type crawler: :class:`~scrapy.crawler.Crawler` - - -Add-on base class -================= - -Scrapy comes with a built-in base class for add-ons which provides some -convenience functionality: the add-on configuration can be exposed into -Scrapy's settings via :meth:`~scrapy.addons.Addon.export_config`, configurable -via :attr:`~scrapy.addons.Addon.default_config` and -:attr:`~scrapy.addons.Addon.config_mapping`. - -By default, the base add-on class will expose the add-on configuration into -Scrapy's settings namespace, in upper case. It is -easy to write your own functionality while still being able to use the -convenience functions by overwriting -:meth:`~scrapy.addons.Addon.update_settings`. - -.. module:: scrapy.addons - :noindex: - -.. autoclass:: Addon - :members: - Add-on examples =============== -Set some basic configuration using the :class:`Addon` base class:: +Set some basic configuration:: - from scrapy.addons import Addon - - class MyAddon(Addon): - name = 'myaddon' - - def update_settings(self, config, settings): - super().update_settings(settings) + class MyAddon: + def update_settings(self, settings): settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 settings["DNSCACHE_ENABLED"] = True Check dependencies:: - from scrapy.addons import Addon - - class MyAddon(Addon): - name = 'myaddon' - - def update_settings(self, config, settings): + class MyAddon: + def update_settings(self, settings): try: import boto except ImportError: - raise RuntimeError("myaddon requires the boto library") - super().update_settings(settings) - -Check configuration of fully initialized crawler (see -:ref:`topics-api-crawler`):: - - class MyAddon(object): - name = 'myaddon' - - def update_settings(self, config, settings): - super().update_settings(settings) - settings.set('DNSCACHE_ENABLED', False, priority='addon') - - def check_configuration(self, config, crawler): - if crawler.settings.getbool('DNSCACHE_ENABLED'): - # The spider, some other add-on, or the user messed with the - # DNS cache setting - raise ValueError("myaddon is incompatible with DNS cache") + raise RuntimeError("MyAddon requires the boto library") + ... diff --git a/scrapy/addons.py b/scrapy/addons.py index 523c36e9c..bb4664d8e 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,100 +1,16 @@ -from typing import Any, Dict, Iterator, Mapping, Optional, OrderedDict +from typing import Any, List from scrapy.utils.conf import build_component_list from scrapy.utils.misc import load_object -class Addon(object): - name: str - - default_config = None - """``dict`` with default configuration.""" - - config_mapping = None - """``dict`` with mappings from config names to setting names. The given - setting names will be taken as given, not uppercased. - """ - - def export_config(self, config, settings): - """Export the add-on configuration, all keys in caps, into the settings - object. - - For example, the add-on configuration ``{'key': 'value'}`` will export - the setting ``KEY`` with a value of ``value``. All settings - will be exported with ``addon`` priority (see - :ref:`topics-api-settings`). - - :param config: Add-on configuration to be exposed - :type config: ``dict`` - - :param settings: Settings object into which to export the configuration - :type settings: :class:`~scrapy.settings.Settings` - """ - conf = self.default_config or {} - conf.update(config) - # Since default exported config is case-insensitive (everything will be - # uppercased), make mapped config case-insensitive as well - conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()} - for key, val in conf.items(): - if key.lower() in conf_mapping: - key = conf_mapping[key.lower()] - else: - key = key.upper() - settings.set(key, val, "addon") - - def update_settings(self, config, settings): - """Modifiy `settings` to enable and configure required components. - - :param config: Add-on configuration - :type config: ``dict`` - - :param settings: Crawler settings object - :type settings: :class:`~scrapy.settings.Settings` - """ - self.export_config(config, settings) - - def check_configuration(self, config, crawler): - """Perform post-initialization checks on fully configured `crawler`. - - :param config: Add-on configuration - :type config: ``dict`` - - :param crawler: the fully-initialized crawler - :type crawler: :class:`~scrapy.crawler.Crawler` - """ - pass - - -class AddonManager(Mapping[str, Addon]): - """This class facilitates loading and storing :ref:`topics-addons`. - - You can treat it like a read-only dictionary in which keys correspond to - add-on names and values correspond to the add-on objects. Add-on - configurations are saved in the :attr:`config` dictionary attribute:: - - addons = AddonManager() - # ... load some add-ons here - print(addons.enabled) # prints names of all enabled add-ons - print(addons['TestAddon'].version) # prints version of add-on with name - # 'TestAddon' - print(addons.configs['TestAddon']) # prints configuration of 'TestAddon' - - """ +class AddonManager: + """This class facilitates loading and storing :ref:`topics-addons`.""" def __init__(self) -> None: - self._addons: OrderedDict[str, Addon] = OrderedDict[str, Addon]() - self.configs: Dict[str, Dict[str, Any]] = {} + self.addons: List[Any] = [] - def __getitem__(self, name: str) -> Addon: - return self._addons[name] - - def __iter__(self) -> Iterator[str]: - return iter(self._addons) - - def __len__(self) -> int: - return len(self._addons) - - def add(self, addon: Any, config: Optional[Dict[str, Any]] = None): + def add(self, addon: Any) -> None: """Store an add-on. :param addon: The add-on object (or path) to be stored @@ -107,19 +23,13 @@ class AddonManager(Mapping[str, Addon]): addon = load_object(addon) if isinstance(addon, type): addon = addon() - name = addon.name - if name in self: - raise ValueError(f"Addon '{name}' already loaded") - self._addons[name] = addon - self.configs[name] = config or {} + self.addons.append(addon) - def load_settings(self, settings): + def load_settings(self, settings) -> None: """Load add-ons and configurations from settings object. This will load the addon for every add-on path in the - ``ADDONS`` setting. For each of these add-ons, the configuration will be - read from the dictionary setting whose name matches the uppercase add-on - name. + ``ADDONS`` setting. :param settings: The :class:`~scrapy.settings.Settings` object from \ which to read the add-on configuration @@ -127,9 +37,8 @@ class AddonManager(Mapping[str, Addon]): """ paths = build_component_list(settings["ADDONS"]) addons = [load_object(path) for path in paths] - configs = [settings.getdict(addon.name.upper()) for addon in addons] - for a, c in zip(addons, configs): - self.add(a, c) + for a in addons: + self.add(a) def update_settings(self, settings) -> None: """Call ``update_settings()`` of all held add-ons. @@ -138,14 +47,5 @@ class AddonManager(Mapping[str, Addon]): updated :type settings: :class:`~scrapy.settings.Settings` """ - for name in self: - self[name].update_settings(self.configs[name], settings) - - def check_configuration(self, crawler) -> None: - """Call ``check_configuration()`` of all held add-ons. - - :param crawler: the fully-initialized crawler - :type crawler: :class:`~scrapy.crawler.Crawler` - """ - for name in self: - self[name].check_configuration(self.configs[name], crawler) + for addon in self.addons: + addon.update_settings(settings) diff --git a/tests/test_addons.py b/tests/test_addons.py index ecdc0426c..8ba27236d 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,47 +1,29 @@ import unittest +from typing import Any, Dict, Optional -from scrapy.addons import Addon, AddonManager +from scrapy.addons import AddonManager from scrapy.settings import BaseSettings -class GoodAddon(object): +class GoodAddon: name = "GoodAddon" - def update_settings(self, config, settings): - pass + def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: + super().__init__() + self.config = config or {} - def check_configuration(self, config, crawler): - pass + def update_settings(self, settings): + settings.update(self.config, "addon") class AddonTest(unittest.TestCase): - def setUp(self): - class AddonWithAttributes(Addon): - name = "Test" - - self.testaddon = AddonWithAttributes() - - def test_export_config(self): - settings = BaseSettings() - self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} - self.testaddon.default_config = {"key": 55, "defaultkey": 100} - self.testaddon.export_config( - {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings - ) - self.assertEqual(settings["KEY"], 313) - self.assertEqual(settings["DEFAULTKEY"], 100) - self.assertEqual(settings["OTHERKEY"], True) - self.assertNotIn("MAPPED_key", settings) - self.assertNotIn("MAPPED_KEY", settings) - self.assertEqual(settings["MAPPING_WORKED"], 99) - self.assertEqual(settings.getpriority("KEY"), 15) - def test_update_settings(self): settings = BaseSettings() settings.set("KEY1", "default", priority="default") settings.set("KEY2", "project", priority="project") - addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} - self.testaddon.update_settings(addon_config, settings) + addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"} + testaddon = GoodAddon(addon_config) + testaddon.update_settings(settings) self.assertEqual(settings["KEY1"], "addon") self.assertEqual(settings["KEY2"], "project") self.assertEqual(settings["KEY3"], "addon") @@ -54,8 +36,7 @@ class AddonManagerTest(unittest.TestCase): def test_add(self): manager = AddonManager() manager.add("tests.test_addons.GoodAddon") - self.assertCountEqual(manager, ["GoodAddon"]) - self.assertIsInstance(manager["GoodAddon"], GoodAddon) + self.assertIsInstance(manager.addons[0], GoodAddon) def test_load_settings(self): settings = BaseSettings() @@ -66,7 +47,4 @@ class AddonManagerTest(unittest.TestCase): settings.set("GOODADDON", {"key": "val2"}) manager = AddonManager() manager.load_settings(settings) - self.assertCountEqual(manager, ["GoodAddon"]) - self.assertIsInstance(manager["GoodAddon"], GoodAddon) - self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) - self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") + self.assertIsInstance(manager.addons[0], GoodAddon) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d844a645f..ca9084294 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -13,7 +13,6 @@ from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy import signals -from scrapy.addons import Addon, AddonManager from scrapy.crawler import CrawlerRunner from scrapy.exceptions import StopDownload from scrapy.http import Request @@ -408,21 +407,6 @@ with multiples lines self._assert_retried(log) self.assertIn("Got response 200", str(log)) - @defer.inlineCallbacks - def test_abort_on_addon_failed_check(self): - class FailedCheckAddon(Addon): - name = "FailedCheckAddon" - - def check_configuration(self, config, crawler): - raise ValueError - - addonmgr = AddonManager() - addonmgr.add(FailedCheckAddon()) - crawler = get_crawler(SimpleSpider) - crawler.addons = addonmgr - with self.assertRaises(ValueError): - yield crawler.crawl() - class CrawlSpiderTestCase(TestCase): def setUp(self): From f1ed5598f4d312337a9dd76e0c1a5856cabea8bc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 Jun 2023 19:13:46 +0400 Subject: [PATCH 1045/2083] Remove the check_configuration call. --- scrapy/crawler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index a0cb368ed..7a26cd2e4 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -133,7 +133,6 @@ class Crawler: try: self.spider = self._create_spider(*args, **kwargs) self.engine = self._create_engine() - self.addons.check_configuration(self) start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) yield defer.maybeDeferred(self.engine.start) From 6fd94fdcb3d633015372f8834308b7a0dcde5bf7 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Wed, 28 Jun 2023 19:55:33 +0300 Subject: [PATCH 1046/2083] periodic_log: tests updated (errors fixed) --- tests/test_extension_periodic_log.py | 56 +++------------------------- 1 file changed, 6 insertions(+), 50 deletions(-) diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 053613510..242e390b3 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -2,7 +2,6 @@ import datetime import unittest from scrapy.crawler import Crawler -from scrapy.exceptions import NotConfigured from scrapy.extensions.periodic_log import PeriodicLog from .spiders import MetaSpider @@ -85,61 +84,16 @@ class TestPeriodicLog(unittest.TestCase): assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) - # Raise not configured if not set by settings - with self.assertRaises(NotConfigured): - extension() - - # Regular use cases: - assert extension( - { - "PERIODIC_LOG_STATS": { - "include": [ - "downloader/", - "scheduler/", - "log_count/", - "item_scraped_count/", - ], - "exclude": ["scheduler/"], - } - } - ) - - assert extension( - { - "PERIODIC_LOG_DELTA": {"include": ["downloader/"]}, - "PERIODIC_LOG_TIMING_ENABLED": True, - } - ) - - assert extension( - { - "PERIODIC_LOG_TIMING_ENABLED": True, - } - ) - - assert extension( - { - "PERIODIC_LOG_STATS": { - "include": [ - "downloader/", - "scheduler/", - "log_count/", - "item_scraped_count/", - ], - "exclude": ["scheduler/"], - }, - "PERIODIC_LOG_DELTA": {"include": ["downloader/"]}, - } - ) - def test_log_delta(self): def emulate(settings=None): + spider = MetaSpider() ext = extension(settings) - ext.spider_opened(MetaSpider) + ext.spider_opened(spider) ext.set_a() a = ext.log_delta() ext.set_a() b = ext.log_delta() + ext.spider_closed(spider, reason="finished") return ext, a, b def check(settings: dict, condition: callable): @@ -189,12 +143,14 @@ class TestPeriodicLog(unittest.TestCase): def test_log_stats(self): def emulate(settings=None): + spider = MetaSpider() ext = extension(settings) - ext.spider_opened(MetaSpider) + ext.spider_opened(spider) ext.set_a() a = ext.log_crawler_stats() ext.set_a() b = ext.log_crawler_stats() + ext.spider_closed(spider, reason="finished") return ext, a, b def check(settings: dict, condition: callable): From c92c9af075217c2296af364a4d9ea55463dbe7f0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Jun 2023 15:04:46 +0400 Subject: [PATCH 1047/2083] Add create_instance support to addons. --- docs/topics/addons.rst | 30 +++++++++++++++++++++++++- scrapy/addons.py | 12 +++++++---- scrapy/crawler.py | 2 +- tests/test_addons.py | 48 +++++++++++++++++++++++++++++------------- 4 files changed, 71 insertions(+), 21 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 3421864fa..f1cc070ad 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -34,7 +34,7 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are (any) Python *objects* that include the following method: +Add-ons are (any) Python objects that include the following method: .. method:: update_settings(settings) @@ -47,6 +47,20 @@ Add-ons are (any) Python *objects* that include the following method: :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.Settings` +They can also have the following method: + +.. classmethod:: from_crawler(cls, crawler) + :noindex: + + If present, this class method is called to create an addon instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the addon. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for pipeline to + access them and hook its functionality into Scrapy. + + :param crawler: The crawler that uses this addon + :type crawler: :class:`~scrapy.crawler.Crawler` + Add-on examples =============== @@ -67,3 +81,17 @@ Check dependencies:: except ImportError: raise RuntimeError("MyAddon requires the boto library") ... + +Access the crawler instance:: + + class MyAddon: + def __init__(self, crawler) -> None: + super().__init__() + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler: Crawler): + return cls(crawler) + + def update_settings(self, settings): + ... diff --git a/scrapy/addons.py b/scrapy/addons.py index bb4664d8e..ba33f1865 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,13 +1,17 @@ -from typing import Any, List +from typing import TYPE_CHECKING, Any, List from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_object +from scrapy.utils.misc import create_instance, load_object + +if TYPE_CHECKING: + from scrapy.crawler import Crawler class AddonManager: """This class facilitates loading and storing :ref:`topics-addons`.""" - def __init__(self) -> None: + def __init__(self, crawler: "Crawler") -> None: + self.crawler: "Crawler" = crawler self.addons: List[Any] = [] def add(self, addon: Any) -> None: @@ -22,7 +26,7 @@ class AddonManager: if isinstance(addon, (type, str)): addon = load_object(addon) if isinstance(addon, type): - addon = addon() + addon = create_instance(addon, settings=None, crawler=self.crawler) self.addons.append(addon) def load_settings(self, settings) -> None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 7a26cd2e4..12256440b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -69,7 +69,7 @@ class Crawler: self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self.addons: AddonManager = AddonManager() + self.addons: AddonManager = AddonManager(self) self.addons.load_settings(self.settings) self.addons.update_settings(self.settings) diff --git a/tests/test_addons.py b/tests/test_addons.py index 8ba27236d..d52665869 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,13 +1,12 @@ import unittest from typing import Any, Dict, Optional -from scrapy.addons import AddonManager +from scrapy.crawler import Crawler from scrapy.settings import BaseSettings +from scrapy.utils.test import get_crawler class GoodAddon: - name = "GoodAddon" - def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: super().__init__() self.config = config or {} @@ -16,6 +15,20 @@ class GoodAddon: settings.update(self.config, "addon") +class CreateInstanceAddon: + def __init__(self, crawler: Crawler) -> None: + super().__init__() + self.crawler = crawler + self.config = crawler.settings.getdict("MYADDON") + + @classmethod + def from_crawler(cls, crawler: Crawler): + return cls(crawler) + + def update_settings(self, settings): + settings.update(self.config, "addon") + + class AddonTest(unittest.TestCase): def test_update_settings(self): settings = BaseSettings() @@ -30,21 +43,26 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - def setUp(self): - self.manager = AddonManager() - def test_add(self): - manager = AddonManager() + crawler = get_crawler() + manager = crawler.addons manager.add("tests.test_addons.GoodAddon") self.assertIsInstance(manager.addons[0], GoodAddon) def test_load_settings(self): - settings = BaseSettings() - settings.set( - "ADDONS", - {"tests.test_addons.GoodAddon": 0}, - ) - settings.set("GOODADDON", {"key": "val2"}) - manager = AddonManager() - manager.load_settings(settings) + settings_dict = { + "ADDONS": {"tests.test_addons.GoodAddon": 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons self.assertIsInstance(manager.addons[0], GoodAddon) + + def test_create_instance(self): + settings_dict = { + "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, + "MYADDON": {"MYADDON_KEY": "val"}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertIsInstance(manager.addons[0], CreateInstanceAddon) + self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") From d5f74c72247e8fd4775e380569e0b809ad1cc6b5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Jun 2023 20:43:46 +0400 Subject: [PATCH 1048/2083] Log the enabled addons. --- scrapy/addons.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scrapy/addons.py b/scrapy/addons.py index ba33f1865..612c7effc 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,3 +1,4 @@ +import logging from typing import TYPE_CHECKING, Any, List from scrapy.utils.conf import build_component_list @@ -6,6 +7,8 @@ from scrapy.utils.misc import create_instance, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler +logger = logging.getLogger(__name__) + class AddonManager: """This class facilitates loading and storing :ref:`topics-addons`.""" @@ -43,6 +46,13 @@ class AddonManager: addons = [load_object(path) for path in paths] for a in addons: self.add(a) + logger.info( + "Enabled addons:\n%(addons)s", + { + "addons": addons, + }, + extra={"crawler": self.crawler}, + ) def update_settings(self, settings) -> None: """Call ``update_settings()`` of all held add-ons. From 7ce3d8f98ad6b28d138757bdc5655d8c91ffd042 Mon Sep 17 00:00:00 2001 From: Anderson Carlos Ferreira da Silva Date: Wed, 5 Jul 2023 16:56:34 +0900 Subject: [PATCH 1049/2083] removing hard code entries --- scrapy/utils/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index ab1b8e3ee..652b74759 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -12,8 +12,8 @@ DATADIR_CFG_SECTION = "datadir" def inside_project(): - scrapy_module = os.environ.get("SCRAPY_SETTINGS_MODULE") - if scrapy_module is not None: + scrapy_module = os.environ.get(ENVVAR) + if scrapy_module: try: import_module(scrapy_module) except ImportError as exc: From a2264d3b8b70455f0ed481a9e76abc96056bc546 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Jul 2023 18:57:27 +0400 Subject: [PATCH 1050/2083] Improve docs about setting settings in addons. --- docs/topics/addons.rst | 10 +++++++++- docs/topics/api.rst | 1 + docs/topics/settings.rst | 15 +++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index f1cc070ad..901a8bf8f 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -61,6 +61,14 @@ They can also have the following method: :param crawler: The crawler that uses this addon :type crawler: :class:`~scrapy.crawler.Crawler` +The settings set by the addon should use the ``addon`` priority (see +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`). This +allows users to override these settings in the project or spider configuration. +This is not possible with settings that are mutable objects, such as the dict +that is a value of :setting:`ITEM_PIPELINES`. In these cases you can provide an +addon-specific setting that governs whether the addon will modify +:setting:`ITEM_PIPELINES`. + Add-on examples =============== @@ -70,7 +78,7 @@ Set some basic configuration:: class MyAddon: def update_settings(self, settings): settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 - settings["DNSCACHE_ENABLED"] = True + settings.set("DNSCACHE_ENABLED", True, "addon") Check dependencies:: diff --git a/docs/topics/api.rst b/docs/topics/api.rst index bb46b2b7d..d1a5497fb 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -137,6 +137,7 @@ Settings API SETTINGS_PRIORITIES = { "default": 0, "command": 10, + "addon": 15, "project": 20, "spider": 30, "cmdline": 40, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 143002360..139e0a35f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -40,8 +40,9 @@ precedence: 1. Command line options (most precedence) 2. Settings per-spider 3. Project settings module - 4. Default settings per-command - 5. Default global settings (less precedence) + 4. Settings set by addons + 5. Default settings per-command + 6. Default global settings (less precedence) The population of these settings sources is taken care of internally, but a manual handling is possible using API calls. See the @@ -89,7 +90,13 @@ project, it's where most of your custom settings will be populated. For a standard Scrapy project, this means you'll be adding or changing the settings in the ``settings.py`` file created for your project. -4. Default settings per-command +4. Settings set by addons +------------------------- + +:ref:`Addons ` can modify settings. They should do this with +this priority, though this is not enforced. + +5. Default settings per-command ------------------------------- Each :doc:`Scrapy tool ` command can have its own default @@ -97,7 +104,7 @@ settings, which override the global default settings. Those custom command settings are specified in the ``default_settings`` attribute of the command class. -5. Default global settings +6. Default global settings -------------------------- The global defaults are located in the ``scrapy.settings.default_settings`` From 93962ebefc89e40aea71ef80954b7dce1545ef56 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Jul 2023 22:09:02 +0400 Subject: [PATCH 1051/2083] Bump typing package versions. --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 223ba4258..fe7df9782 100644 --- a/tox.ini +++ b/tox.ini @@ -33,13 +33,13 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.2.0 + mypy==1.4.1 types-attrs==19.1.0 types-lxml==2023.3.28 - types-Pillow==9.5.0.2 - types-Pygments==2.15.0.0 - types-pyOpenSSL==23.1.0.2 - types-setuptools==67.7.0.1 + types-Pillow==10.0.0.1 + types-Pygments==2.15.0.1 + types-pyOpenSSL==23.2.0.1 + types-setuptools==68.0.0.1 commands = mypy --show-error-codes {posargs: scrapy tests} From 187e8f9a2d2fbf2c84fb5b39c25e7e7fc155ced0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jul 2023 00:01:36 +0400 Subject: [PATCH 1052/2083] Typing for scrapy/settings/__init__.py. --- scrapy/crawler.py | 2 +- scrapy/settings/__init__.py | 166 ++++++++++++++++++++++---------- scrapy/utils/conf.py | 1 + tests/test_settings/__init__.py | 21 ++++ tox.ini | 1 + 5 files changed, 140 insertions(+), 51 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 69ff07bb7..192541dd0 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -55,7 +55,7 @@ class Crawler: def __init__( self, spidercls: Type[Spider], - settings: Union[None, dict, Settings] = None, + settings: Union[None, Dict[str, Any], Settings] = None, init_reactor: bool = False, ): if isinstance(spidercls, Spider): diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index cc44d67e8..658c27f0a 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,12 +1,41 @@ +from __future__ import annotations + import copy import json -from collections.abc import MutableMapping from importlib import import_module from pprint import pformat +from types import ModuleType +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + Iterator, + List, + Mapping, + MutableMapping, + Optional, + Tuple, + Union, + cast, +) from scrapy.settings import default_settings -SETTINGS_PRIORITIES = { +# The key types are restricted in BaseSettings._get_key() to ones supported by JSON, +# see https://github.com/scrapy/scrapy/issues/5383. +_SettingsKeyT = Union[bool, float, int, str, None] + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + # https://github.com/python/typing/issues/445#issuecomment-1131458824 + from _typeshed import SupportsItems + from typing_extensions import Self + + _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None] + + +SETTINGS_PRIORITIES: Dict[str, int] = { "default": 0, "command": 10, "project": 20, @@ -15,7 +44,7 @@ SETTINGS_PRIORITIES = { } -def get_settings_priority(priority): +def get_settings_priority(priority: Union[int, str]) -> int: """ Small helper function that looks up a given string priority in the :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its @@ -34,14 +63,15 @@ class SettingsAttribute: for settings configuration, not this one. """ - def __init__(self, value, priority): - self.value = value + def __init__(self, value: Any, priority: int): + self.value: Any = value + self.priority: int if isinstance(self.value, BaseSettings): self.priority = max(self.value.maxpriority(), priority) else: self.priority = priority - def set(self, value, priority): + def set(self, value: Any, priority: int) -> None: """Sets value if priority is higher or equal than current priority.""" if priority >= self.priority: if isinstance(self.value, BaseSettings): @@ -49,11 +79,11 @@ class SettingsAttribute: self.value = value self.priority = priority - def __repr__(self): + def __repr__(self) -> str: return f"" -class BaseSettings(MutableMapping): +class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ Instances of this class behave like dictionaries, but store priorities along with their ``(key, value)`` pairs, and can be frozen (i.e. marked @@ -77,21 +107,23 @@ class BaseSettings(MutableMapping): __default = object() - def __init__(self, values=None, priority="project"): - self.frozen = False - self.attributes = {} + def __init__( + self, values: _SettingsInputT = None, priority: Union[int, str] = "project" + ): + self.frozen: bool = False + self.attributes: dict[_SettingsKeyT, SettingsAttribute] = {} if values: self.update(values, priority) - def __getitem__(self, opt_name): + def __getitem__(self, opt_name: _SettingsKeyT) -> Any: if opt_name not in self: return None return self.attributes[opt_name].value - def __contains__(self, name): + def __contains__(self, name: Any) -> bool: return name in self.attributes - def get(self, name, default=None): + def get(self, name: _SettingsKeyT, default: Any = None) -> Any: """ Get a setting value without affecting its original type. @@ -103,7 +135,7 @@ class BaseSettings(MutableMapping): """ return self[name] if self[name] is not None else default - def getbool(self, name, default=False): + def getbool(self, name: _SettingsKeyT, default: bool = False) -> bool: """ Get a setting value as a boolean. @@ -133,7 +165,7 @@ class BaseSettings(MutableMapping): "'True'/'False' and 'true'/'false'" ) - def getint(self, name, default=0): + def getint(self, name: _SettingsKeyT, default: int = 0) -> int: """ Get a setting value as an int. @@ -145,7 +177,7 @@ class BaseSettings(MutableMapping): """ return int(self.get(name, default)) - def getfloat(self, name, default=0.0): + def getfloat(self, name: _SettingsKeyT, default: float = 0.0) -> float: """ Get a setting value as a float. @@ -157,7 +189,9 @@ class BaseSettings(MutableMapping): """ return float(self.get(name, default)) - def getlist(self, name, default=None): + def getlist( + self, name: _SettingsKeyT, default: Optional[List[Any]] = None + ) -> List[Any]: """ Get a setting value as a list. If the setting original type is a list, a copy of it will be returned. If it's a string it will be split by ",". @@ -176,7 +210,9 @@ class BaseSettings(MutableMapping): value = value.split(",") return list(value) - def getdict(self, name, default=None): + def getdict( + self, name: _SettingsKeyT, default: Optional[Dict[Any, Any]] = None + ) -> Dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a dictionary, a copy of it will be returned. If it is a string it will be @@ -197,7 +233,11 @@ class BaseSettings(MutableMapping): value = json.loads(value) return dict(value) - def getdictorlist(self, name, default=None): + def getdictorlist( + self, + name: _SettingsKeyT, + default: Union[Dict[Any, Any], List[Any], None] = None, + ) -> Union[Dict[Any, Any], List[Any]]: """Get a setting value as either a :class:`dict` or a :class:`list`. If the setting is already a dict or a list, a copy of it will be @@ -224,24 +264,29 @@ class BaseSettings(MutableMapping): return {} if isinstance(value, str): try: - return json.loads(value) + value_loaded = json.loads(value) + assert isinstance(value_loaded, (dict, list)) + return value_loaded except ValueError: return value.split(",") + assert isinstance(value, (dict, list)) return copy.deepcopy(value) - def getwithbase(self, name): + def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings": """Get a composition of a dictionary-like setting and its `_BASE` counterpart. :param name: name of the dictionary-like setting :type name: str """ + if not isinstance(name, str): + raise ValueError(f"Base setting key must be a string, got {name}") compbs = BaseSettings() compbs.update(self[name + "_BASE"]) compbs.update(self[name]) return compbs - def getpriority(self, name): + def getpriority(self, name: _SettingsKeyT) -> Optional[int]: """ Return the current numerical priority value of a setting, or ``None`` if the given ``name`` does not exist. @@ -253,7 +298,7 @@ class BaseSettings(MutableMapping): return None return self.attributes[name].priority - def maxpriority(self): + def maxpriority(self) -> int: """ Return the numerical value of the highest priority present throughout all settings, or the numerical value for ``default`` from @@ -261,13 +306,15 @@ class BaseSettings(MutableMapping): stored. """ if len(self) > 0: - return max(self.getpriority(name) for name in self) + return max(cast(int, self.getpriority(name)) for name in self) return get_settings_priority("default") - def __setitem__(self, name, value): + def __setitem__(self, name: _SettingsKeyT, value: Any) -> None: self.set(name, value) - def set(self, name, value, priority="project"): + def set( + self, name: _SettingsKeyT, value: Any, priority: Union[int, str] = "project" + ) -> None: """ Store a key/value attribute with a given priority. @@ -295,17 +342,26 @@ class BaseSettings(MutableMapping): else: self.attributes[name].set(value, priority) - def setdefault(self, name, default=None, priority="project"): + def setdefault( + self, + name: _SettingsKeyT, + default: Any = None, + priority: Union[int, str] = "project", + ) -> Any: if name not in self: self.set(name, default, priority) return default return self.attributes[name].value - def setdict(self, values, priority="project"): + def setdict( + self, values: _SettingsInputT, priority: Union[int, str] = "project" + ) -> None: self.update(values, priority) - def setmodule(self, module, priority="project"): + def setmodule( + self, module: Union[ModuleType, str], priority: Union[int, str] = "project" + ) -> None: """ Store settings from a module with a given priority. @@ -327,7 +383,8 @@ class BaseSettings(MutableMapping): if key.isupper(): self.set(key, getattr(module, key), priority) - def update(self, values, priority="project"): + # BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports + def update(self, values: _SettingsInputT, priority: Union[int, str] = "project") -> None: # type: ignore[override] """ Store key/value pairs with a given priority. @@ -351,30 +408,34 @@ class BaseSettings(MutableMapping): """ self._assert_mutability() if isinstance(values, str): - values = json.loads(values) + values = cast(dict, json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): - self.set(name, value, values.getpriority(name)) + self.set(name, value, cast(int, values.getpriority(name))) else: for name, value in values.items(): self.set(name, value, priority) - def delete(self, name, priority="project"): + def delete( + self, name: _SettingsKeyT, priority: Union[int, str] = "project" + ) -> None: + if name not in self: + raise KeyError(name) self._assert_mutability() priority = get_settings_priority(priority) - if priority >= self.getpriority(name): + if priority >= cast(int, self.getpriority(name)): del self.attributes[name] - def __delitem__(self, name): + def __delitem__(self, name: _SettingsKeyT) -> None: self._assert_mutability() del self.attributes[name] - def _assert_mutability(self): + def _assert_mutability(self) -> None: if self.frozen: raise TypeError("Trying to modify an immutable Settings object") - def copy(self): + def copy(self) -> "Self": """ Make a deep copy of current settings. @@ -386,7 +447,7 @@ class BaseSettings(MutableMapping): """ return copy.deepcopy(self) - def freeze(self): + def freeze(self) -> None: """ Disable further changes to the current settings. @@ -396,7 +457,7 @@ class BaseSettings(MutableMapping): """ self.frozen = True - def frozencopy(self): + def frozencopy(self) -> "Self": """ Return an immutable copy of the current settings. @@ -406,26 +467,26 @@ class BaseSettings(MutableMapping): copy.freeze() return copy - def __iter__(self): + def __iter__(self) -> Iterator[_SettingsKeyT]: return iter(self.attributes) - def __len__(self): + def __len__(self) -> int: return len(self.attributes) - def _to_dict(self): + def _to_dict(self) -> Dict[_SettingsKeyT, Any]: return { self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items() } - def _get_key(self, key_value): + def _get_key(self, key_value: Any) -> _SettingsKeyT: return ( key_value if isinstance(key_value, (bool, float, int, str, type(None))) else str(key_value) ) - def copy_to_dict(self): + def copy_to_dict(self) -> Dict[_SettingsKeyT, Any]: """ Make a copy of current settings and convert to a dict. @@ -441,13 +502,14 @@ class BaseSettings(MutableMapping): settings = self.copy() return settings._to_dict() - def _repr_pretty_(self, p, cycle): + # https://ipython.readthedocs.io/en/stable/config/integrating.html#pretty-printing + def _repr_pretty_(self, p: Any, cycle: bool) -> None: if cycle: p.text(repr(self)) else: p.text(pformat(self.copy_to_dict())) - def pop(self, name, default=__default): + def pop(self, name: _SettingsKeyT, default: Any = __default) -> Any: try: value = self.attributes[name].value except KeyError: @@ -471,7 +533,9 @@ class Settings(BaseSettings): described on :ref:`topics-settings-ref` already populated. """ - def __init__(self, values=None, priority="project"): + def __init__( + self, values: _SettingsInputT = None, priority: Union[int, str] = "project" + ): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user @@ -485,14 +549,16 @@ class Settings(BaseSettings): self.update(values, priority) -def iter_default_settings(): +def iter_default_settings() -> Iterable[Tuple[str, Any]]: """Return the default settings as an iterator of (name, value) tuples""" for name in dir(default_settings): if name.isupper(): yield name, getattr(default_settings, name) -def overridden_settings(settings): +def overridden_settings( + settings: Mapping[_SettingsKeyT, Any] +) -> Iterable[Tuple[str, Any]]: """Return a dict of the settings that have been overridden""" for name, defvalue in iter_default_settings(): value = settings[name] diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 3ade1d105..0608527ae 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -28,6 +28,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): compbs = BaseSettings() for k, v in compdict.items(): prio = compdict.getpriority(k) + assert prio is not None if compbs.getpriority(convert(k)) == prio: raise ValueError( f"Some paths in {list(compdict.keys())!r} " diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 5fc825393..db000233e 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,6 +1,8 @@ import unittest from unittest import mock +import pytest + from scrapy.settings import ( SETTINGS_PRIORITIES, BaseSettings, @@ -199,6 +201,21 @@ class BaseSettingsTest(unittest.TestCase): settings.update({"key_lowprio": 3}, priority=20) self.assertEqual(settings["key_lowprio"], 1) + @pytest.mark.xfail( + raises=TypeError, reason="BaseSettings.update doesn't support kwargs input" + ) + def test_update_kwargs(self): + settings = BaseSettings({"key": 0}) + settings.update(key=1) + + @pytest.mark.xfail( + raises=AttributeError, + reason="BaseSettings.update doesn't support iterable input", + ) + def test_update_iterable(self): + settings = BaseSettings({"key": 0}) + settings.update([("key", 1)]) + def test_update_jsonstring(self): settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})}) settings.update('{"number": 1, "newnumber": 2}') @@ -217,6 +234,10 @@ class BaseSettingsTest(unittest.TestCase): self.assertIn("key_highprio", settings) del settings["key_highprio"] self.assertNotIn("key_highprio", settings) + with self.assertRaises(KeyError): + settings.delete("notkey") + with self.assertRaises(KeyError): + del settings["notkey"] def test_get(self): test_configuration = { diff --git a/tox.ini b/tox.ini index fe7df9782..1aeb94215 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,7 @@ install_command = basepython = python3 deps = mypy==1.4.1 + typing-extensions==4.7.1 types-attrs==19.1.0 types-lxml==2023.3.28 types-Pillow==10.0.0.1 From 043a24410b2eeed554e8ce3a73b1e78fa53fa4d6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jul 2023 00:09:24 +0400 Subject: [PATCH 1053/2083] Disable pylint for broken code. --- tests/test_settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index db000233e..e7799737f 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -206,7 +206,7 @@ class BaseSettingsTest(unittest.TestCase): ) def test_update_kwargs(self): settings = BaseSettings({"key": 0}) - settings.update(key=1) + settings.update(key=1) # pylint: disable=unexpected-keyword-arg @pytest.mark.xfail( raises=AttributeError, From cdda8ad46dc064229b5d875fb7685fdb32ebfb3f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jul 2023 21:05:58 +0400 Subject: [PATCH 1054/2083] Add docs about fallbacks in addons. --- docs/topics/addons.rst | 89 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 901a8bf8f..cc96207bd 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -62,12 +62,53 @@ They can also have the following method: :type crawler: :class:`~scrapy.crawler.Crawler` The settings set by the addon should use the ``addon`` priority (see -:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`). This -allows users to override these settings in the project or spider configuration. -This is not possible with settings that are mutable objects, such as the dict -that is a value of :setting:`ITEM_PIPELINES`. In these cases you can provide an -addon-specific setting that governs whether the addon will modify -:setting:`ITEM_PIPELINES`. +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: + + class MyAddon: + def update_settings(self, settings): + settings.set("DNSCACHE_ENABLED", True, "addon") + +This allows users to override these settings in the project or spider +configuration. This is not possible with settings that are mutable objects, +such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases +you can provide an addon-specific setting that governs whether the addon will +modify :setting:`ITEM_PIPELINES`:: + + class MyAddon: + def update_settings(self, settings): + if settings.getbool("MYADDON_ENABLE_PIPELINE"): + settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 + +Fallbacks +--------- + +Some components provided by addons need to fallback to "default" +implementations, e.g. a custom download handler needs to send the request that +it doesn't handle via the default download handler, or a stats collector that +includes some additional processing but otherwise uses the default stats +collector. And it's possible that a project needs to use several custom +components of the same type, e.g. two custom download handlers that support +different kinds of custom requests and still need to use the default download +handler for other requests. To make such use cases easier to configure, we +recommend that such custom components should be written in the following way: + +1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the + default Scrapy one (e.g. + ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead + be able to load the class of the fallback component from a special setting + (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use + it. +2. The addons that include these components should read the current value of + the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their + ``update_settings()`` methods, save that value into the fallback setting + (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default + setting to the component provided byt the addon (e.g. + ``MyDownloadHandler``). If the fallback setting is already set by the user, + they shouldn't change it. +3. This way, if there are several addons that want to modify the same setting, + all of them will fallback to the component from the previous one and then to + the Scrapy default. The order of that depends on the priority order in the + ``ADDONS`` setting. Add-on examples @@ -103,3 +144,39 @@ Access the crawler instance:: def update_settings(self, settings): ... + +Use a fallback component:: + + from scrapy.core.downloader.handlers.http import HTTPDownloadHandler + + + fallback_setting = "MY_FALLBACK_DOWNLOAD_HANDLER" + + + class MyHandler: + lazy = False + + def __init__(self, settings, crawler): + dhcls = load_object(settings.get(fallback_setting)) + self._fallback_handler = create_instance( + dhcls, + settings=None, + crawler=crawler, + ) + + def download_request(self, request, spider): + if request.meta.get("my_params"): + # handle the request + ... + else: + return self._fallback_handler.download_request(request, spider) + + + class MyAddon: + def update_settings(self, settings): + if not settings.get(fallback_setting): + settings.set( + fallback_setting, + settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + "addon", + ) From db86f91789d25e45c43f964fff31d3016a451a1e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jul 2023 23:17:11 +0400 Subject: [PATCH 1055/2083] Unbreak isort breakage. --- scrapy/settings/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 658c27f0a..b0adb5ba8 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -27,9 +27,10 @@ from scrapy.settings import default_settings _SettingsKeyT = Union[bool, float, int, str, None] if TYPE_CHECKING: - # typing.Self requires Python 3.11 # https://github.com/python/typing/issues/445#issuecomment-1131458824 from _typeshed import SupportsItems + + # typing.Self requires Python 3.11 from typing_extensions import Self _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None] From 3f5bbe3a8fababac6dfcfcf1a22aa7f7dba199ba Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Tue, 18 Jul 2023 18:30:21 +0800 Subject: [PATCH 1056/2083] introduce CLOSESPIDER_TIMEOUT_NO_ITEM in CloseSpider --- docs/topics/extensions.rst | 13 ++++++++++ scrapy/extensions/closespider.py | 41 ++++++++++++++++++++++++++++++++ tests/test_closespider.py | 12 +++++++++- 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 96e0216b8..8d4749ab3 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -258,6 +258,7 @@ The conditions for closing a spider can be configured through the following settings: * :setting:`CLOSESPIDER_TIMEOUT` +* :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM` * :setting:`CLOSESPIDER_ITEMCOUNT` * :setting:`CLOSESPIDER_PAGECOUNT` * :setting:`CLOSESPIDER_ERRORCOUNT` @@ -280,6 +281,18 @@ more than that number of second, it will be automatically closed with the reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by timeout. +.. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM + +CLOSESPIDER_TIMEOUT_NO_ITEM +""""""""""""""""""""""""""" + +Default: ``0`` + +An integer which specifies a number of seconds. If the spider has not produced +any items in the last number of seconds, it will be closed with the reason +``closespider_timeout_no_item``. If zero (or non set), spiders won't be closed +regardless if it hasn't produced any items. + .. setting:: CLOSESPIDER_ITEMCOUNT CLOSESPIDER_ITEMCOUNT diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index bb6f832f2..456470efd 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -4,11 +4,14 @@ conditions are met. See documentation in docs/topics/extensions.rst """ +import logging from collections import defaultdict from scrapy import signals from scrapy.exceptions import NotConfigured +logger = logging.getLogger(__name__) + class CloseSpider: def __init__(self, crawler): @@ -19,6 +22,7 @@ class CloseSpider: "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"), + "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"), } if not any(self.close_on.values()): @@ -34,6 +38,15 @@ class CloseSpider: crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) if self.close_on.get("itemcount"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) + if self.close_on.get("timeout_no_item"): + self.timeout_no_item = self.close_on["timeout_no_item"] + self.items_in_period = 0 + crawler.signals.connect( + self.spider_opened_no_item, signal=signals.spider_opened + ) + crawler.signals.connect( + self.item_scraped_no_item, signal=signals.item_scraped + ) crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod @@ -69,3 +82,31 @@ class CloseSpider: task = getattr(self, "task", False) if task and task.active(): task.cancel() + + task_no_item = getattr(self, "task_no_item", False) + if task_no_item.running: + task_no_item.stop() + + def spider_opened_no_item(self, spider): + from twisted.internet import task + + self.task_no_item = task.LoopingCall(self._count_items_produced, spider) + self.task_no_item.start(self.timeout_no_item, now=False) + + logger.info( + f"Spider will stop when no items are produced after " + f"{self.timeout_no_item} seconds." + ) + + def item_scraped_no_item(self, item, spider): + self.items_in_period += 1 + + def _count_items_produced(self, spider): + if self.items_in_period >= 1: + self.items_in_period = 0 + else: + logger.info( + f"Closing spider since no items were produced in the last " + f"{self.timeout_no_item} seconds." + ) + self.crawler.engine.close_spider(spider, "closespider_timeout_no_item") diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 9b39187d5..259a1a4ad 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -3,7 +3,7 @@ from twisted.trial.unittest import TestCase from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider +from tests.spiders import DelaySpider, ErrorSpider, FollowAllSpider, ItemSpider class TestCloseSpider(TestCase): @@ -54,3 +54,13 @@ class TestCloseSpider(TestCase): self.assertEqual(reason, "closespider_timeout") total_seconds = crawler.stats.get_value("elapsed_time_seconds") self.assertTrue(total_seconds >= close_on) + + @defer.inlineCallbacks + def test_closespider_timeout_no_item(self): + timeout = 1 + crawler = get_crawler(DelaySpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) + yield crawler.crawl(n=3, total=10, mockserver=self.mockserver) + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_timeout_no_item") + total_seconds = crawler.stats.get_value("elapsed_time_seconds") + self.assertTrue(total_seconds >= timeout) From 2f787a27dc1f7551b98d322b6b93b3cb8bad4e2e Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Tue, 18 Jul 2023 20:49:33 +0800 Subject: [PATCH 1057/2083] fix conditional on task_no_item --- scrapy/extensions/closespider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 456470efd..4307b4170 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -84,7 +84,7 @@ class CloseSpider: task.cancel() task_no_item = getattr(self, "task_no_item", False) - if task_no_item.running: + if task_no_item and task_no_item.running: task_no_item.stop() def spider_opened_no_item(self, spider): From 368ab29ffc32b13d440f8adefdf06431b7ba7c32 Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Tue, 18 Jul 2023 22:29:15 +0800 Subject: [PATCH 1058/2083] improve tests by having SlowSpider --- tests/spiders.py | 16 ++++++++++++++++ tests/test_closespider.py | 6 +++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index 6ff48f471..f29dea2a1 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -77,6 +77,22 @@ class DelaySpider(MetaSpider): self.t2_err = time.time() +class SlowSpider(DelaySpider): + name = "slow" + + def start_requests(self): + # 1st response is fast + url = self.mockserver.url("/delay?n=0&b=0") + yield Request(url, callback=self.parse, errback=self.errback) + + # 2nd response is slow + url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") + yield Request(url, callback=self.parse, errback=self.errback) + + def parse(self, response): + yield Item() + + class SimpleSpider(MetaSpider): name = "simple" diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 259a1a4ad..38ede70e4 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -3,7 +3,7 @@ from twisted.trial.unittest import TestCase from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import DelaySpider, ErrorSpider, FollowAllSpider, ItemSpider +from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider, SlowSpider class TestCloseSpider(TestCase): @@ -58,8 +58,8 @@ class TestCloseSpider(TestCase): @defer.inlineCallbacks def test_closespider_timeout_no_item(self): timeout = 1 - crawler = get_crawler(DelaySpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) - yield crawler.crawl(n=3, total=10, mockserver=self.mockserver) + crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) + yield crawler.crawl(n=3, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] self.assertEqual(reason, "closespider_timeout_no_item") total_seconds = crawler.stats.get_value("elapsed_time_seconds") From 5c34f34ecbc18c0829cd04a621b1d8239c25642b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 18 Jul 2023 19:50:08 +0400 Subject: [PATCH 1059/2083] Make AddonManager.add() private. --- scrapy/addons.py | 20 ++++++-------------- tests/test_addons.py | 6 ------ 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 612c7effc..812892613 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -17,15 +17,8 @@ class AddonManager: self.crawler: "Crawler" = crawler self.addons: List[Any] = [] - def add(self, addon: Any) -> None: - """Store an add-on. - - :param addon: The add-on object (or path) to be stored - :type addon: Python object, class or ``str`` - - :param config: The add-on configuration dictionary - :type config: ``dict`` - """ + def _add(self, addon: Any) -> None: + """Store an add-on.""" if isinstance(addon, (type, str)): addon = load_object(addon) if isinstance(addon, type): @@ -33,7 +26,7 @@ class AddonManager: self.addons.append(addon) def load_settings(self, settings) -> None: - """Load add-ons and configurations from settings object. + """Load add-ons and configurations from a settings object. This will load the addon for every add-on path in the ``ADDONS`` setting. @@ -42,10 +35,9 @@ class AddonManager: which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - paths = build_component_list(settings["ADDONS"]) - addons = [load_object(path) for path in paths] - for a in addons: - self.add(a) + addons = build_component_list(settings["ADDONS"]) + for addon in build_component_list(settings["ADDONS"]): + self._add(addon) logger.info( "Enabled addons:\n%(addons)s", { diff --git a/tests/test_addons.py b/tests/test_addons.py index d52665869..95377d718 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -43,12 +43,6 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - def test_add(self): - crawler = get_crawler() - manager = crawler.addons - manager.add("tests.test_addons.GoodAddon") - self.assertIsInstance(manager.addons[0], GoodAddon) - def test_load_settings(self): settings_dict = { "ADDONS": {"tests.test_addons.GoodAddon": 0}, From 90dae3ee60b1f5ab5ea83d82ac8c00f1be54723e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 18 Jul 2023 19:52:27 +0400 Subject: [PATCH 1060/2083] Doc fixes. --- docs/topics/addons.rst | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index cc96207bd..c432c64d2 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -34,7 +34,7 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are (any) Python objects that include the following method: +Add-ons are Python classes that include the following method: .. method:: update_settings(settings) @@ -54,9 +54,9 @@ They can also have the following method: If present, this class method is called to create an addon instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the addon. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for pipeline to - access them and hook its functionality into Scrapy. + of the addon. The crawler object provides access to all Scrapy core + components like settings and signals; it is a way for the addon to access + them and hook its functionality into Scrapy. :param crawler: The crawler that uses this addon :type crawler: :class:`~scrapy.crawler.Crawler` @@ -82,7 +82,7 @@ modify :setting:`ITEM_PIPELINES`:: Fallbacks --------- -Some components provided by addons need to fallback to "default" +Some components provided by addons need to fall back to "default" implementations, e.g. a custom download handler needs to send the request that it doesn't handle via the default download handler, or a stats collector that includes some additional processing but otherwise uses the default stats @@ -102,7 +102,7 @@ recommend that such custom components should be written in the following way: the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their ``update_settings()`` methods, save that value into the fallback setting (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default - setting to the component provided byt the addon (e.g. + setting to the component provided by the addon (e.g. ``MyDownloadHandler``). If the fallback setting is already set by the user, they shouldn't change it. 3. This way, if there are several addons that want to modify the same setting, @@ -114,14 +114,18 @@ recommend that such custom components should be written in the following way: Add-on examples =============== -Set some basic configuration:: +Set some basic configuration: + +.. code-block:: python class MyAddon: def update_settings(self, settings): settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 settings.set("DNSCACHE_ENABLED", True, "addon") -Check dependencies:: +Check dependencies: + +.. code-block:: python class MyAddon: def update_settings(self, settings): @@ -131,7 +135,9 @@ Check dependencies:: raise RuntimeError("MyAddon requires the boto library") ... -Access the crawler instance:: +Access the crawler instance: + +.. code-block:: python class MyAddon: def __init__(self, crawler) -> None: @@ -145,7 +151,9 @@ Access the crawler instance:: def update_settings(self, settings): ... -Use a fallback component:: +Use a fallback component: + +.. code-block:: python from scrapy.core.downloader.handlers.http import HTTPDownloadHandler From 0a25a300cfc8281c355a6cfcd8605418347ed324 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 18 Jul 2023 20:29:42 +0400 Subject: [PATCH 1061/2083] Fix docs. --- docs/topics/addons.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index c432c64d2..36c7e0fa0 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -145,7 +145,7 @@ Access the crawler instance: self.crawler = crawler @classmethod - def from_crawler(cls, crawler: Crawler): + def from_crawler(cls, crawler): return cls(crawler) def update_settings(self, settings): From 005c8cc5f00f41ad7d836eccf45dbaf39c03ebca Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 19 Jul 2023 13:15:35 +0400 Subject: [PATCH 1062/2083] Unify the "add-on" spelling. --- docs/topics/addons.rst | 22 +++++++++++----------- docs/topics/settings.rst | 8 ++++---- scrapy/addons.py | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 36c7e0fa0..8733f9bde 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -19,7 +19,7 @@ Add-ons and their configuration live in Scrapy's initialization the add-on manager will read a list of enabled add-ons from your ``ADDONS`` setting. -The ``ADDONS`` setting is a dict in which every key is an addon class or its +The ``ADDONS`` setting is a dict in which every key is an add-on class or its import path and the value is its priority. This is an example where two add-ons are enabled in a project's @@ -52,16 +52,16 @@ They can also have the following method: .. classmethod:: from_crawler(cls, crawler) :noindex: - If present, this class method is called to create an addon instance + If present, this class method is called to create an add-on instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the addon. The crawler object provides access to all Scrapy core - components like settings and signals; it is a way for the addon to access + of the add-on. The crawler object provides access to all Scrapy core + components like settings and signals; it is a way for the add-on to access them and hook its functionality into Scrapy. - :param crawler: The crawler that uses this addon + :param crawler: The crawler that uses this add-on :type crawler: :class:`~scrapy.crawler.Crawler` -The settings set by the addon should use the ``addon`` priority (see +The settings set by the add-on should use the ``addon`` priority (see :ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: class MyAddon: @@ -71,7 +71,7 @@ The settings set by the addon should use the ``addon`` priority (see This allows users to override these settings in the project or spider configuration. This is not possible with settings that are mutable objects, such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases -you can provide an addon-specific setting that governs whether the addon will +you can provide an add-on-specific setting that governs whether the add-on will modify :setting:`ITEM_PIPELINES`:: class MyAddon: @@ -82,7 +82,7 @@ modify :setting:`ITEM_PIPELINES`:: Fallbacks --------- -Some components provided by addons need to fall back to "default" +Some components provided by add-ons need to fall back to "default" implementations, e.g. a custom download handler needs to send the request that it doesn't handle via the default download handler, or a stats collector that includes some additional processing but otherwise uses the default stats @@ -98,14 +98,14 @@ recommend that such custom components should be written in the following way: be able to load the class of the fallback component from a special setting (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use it. -2. The addons that include these components should read the current value of +2. The add-ons that include these components should read the current value of the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their ``update_settings()`` methods, save that value into the fallback setting (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default - setting to the component provided by the addon (e.g. + setting to the component provided by the add-on (e.g. ``MyDownloadHandler``). If the fallback setting is already set by the user, they shouldn't change it. -3. This way, if there are several addons that want to modify the same setting, +3. This way, if there are several add-ons that want to modify the same setting, all of them will fallback to the component from the previous one and then to the Scrapy default. The order of that depends on the priority order in the ``ADDONS`` setting. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 139e0a35f..602ab587d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -40,7 +40,7 @@ precedence: 1. Command line options (most precedence) 2. Settings per-spider 3. Project settings module - 4. Settings set by addons + 4. Settings set by add-ons 5. Default settings per-command 6. Default global settings (less precedence) @@ -90,10 +90,10 @@ project, it's where most of your custom settings will be populated. For a standard Scrapy project, this means you'll be adding or changing the settings in the ``settings.py`` file created for your project. -4. Settings set by addons -------------------------- +4. Settings set by add-ons +-------------------------- -:ref:`Addons ` can modify settings. They should do this with +:ref:`Add-ons ` can modify settings. They should do this with this priority, though this is not enforced. 5. Default settings per-command diff --git a/scrapy/addons.py b/scrapy/addons.py index 812892613..e72c5da98 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -28,7 +28,7 @@ class AddonManager: def load_settings(self, settings) -> None: """Load add-ons and configurations from a settings object. - This will load the addon for every add-on path in the + This will load the add-on for every add-on path in the ``ADDONS`` setting. :param settings: The :class:`~scrapy.settings.Settings` object from \ From 583df9f7d063ac0f48eafccdd463f3195a084d9b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 22 Jul 2023 17:44:37 +0400 Subject: [PATCH 1063/2083] Simplify skipping uvloop tests. --- conftest.py | 16 ++++++++++++++ pytest.ini | 1 + tests/test_commands.py | 14 +----------- tests/test_crawler.py | 50 ++++-------------------------------------- 4 files changed, 22 insertions(+), 59 deletions(-) diff --git a/conftest.py b/conftest.py index e1d4b1213..fa5193470 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,10 @@ +import platform +import sys from pathlib import Path import pytest +from twisted import version as twisted_version +from twisted.python.versions import Version from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor @@ -73,6 +77,18 @@ def only_not_asyncio(request, reactor_pytest): pytest.skip("This test is only run without --reactor=asyncio") +@pytest.fixture(autouse=True) +def requires_uvloop(request): + if not request.node.get_closest_marker("requires_uvloop"): + return + if sys.implementation.name == "pypy": + pytest.skip("uvloop does not support pypy properly") + if platform.system() == "Windows": + pytest.skip("uvloop does not support Windows") + if twisted_version == Version("twisted", 21, 2, 0): + pytest.skip("https://twistedmatrix.com/trac/ticket/10106") + + def pytest_configure(config): if config.getoption("--reactor") == "asyncio": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/pytest.ini b/pytest.ini index 866f0c950..16983be5e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,6 +20,7 @@ addopts = markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed + requires_uvloop: marks tests as only enabled when uvloop is known to be working filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated diff --git a/tests/test_commands.py b/tests/test_commands.py index 014f50e92..03d768d1a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -18,8 +18,6 @@ from typing import Dict, Generator, Optional, Union from unittest import skipIf from pytest import mark -from twisted import version as twisted_version -from twisted.python.versions import Version from twisted.trial import unittest import scrapy @@ -802,17 +800,7 @@ class MySpider(scrapy.Spider): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log ) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_custom_asyncio_loop_enabled_true(self): log = self.get_log( self.debug_log_spider, diff --git a/tests/test_crawler.py b/tests/test_crawler.py index d54a2cb7e..68e58144b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -8,9 +8,7 @@ from pathlib import Path from packaging.version import parse as parse_version from pytest import mark, raises -from twisted import version as twisted_version from twisted.internet import defer -from twisted.python.versions import Version from twisted.trial import unittest from w3lib import __version__ as w3lib_version @@ -466,17 +464,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): log, ) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") self.assertIn("Spider closed (finished)", log) @@ -485,17 +473,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") self.assertIn("Spider closed (finished)", log) @@ -505,17 +483,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_asyncio_enabled_reactor_same_loop(self): log = self.run_script("asyncio_enabled_reactor_same_loop.py") self.assertIn("Spider closed (finished)", log) @@ -524,17 +492,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_asyncio_enabled_reactor_different_loop(self): log = self.run_script("asyncio_enabled_reactor_different_loop.py") self.assertNotIn("Spider closed (finished)", log) From e058a05763b45086edaf8b52f067648c0c50ae21 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 22 Jul 2023 17:51:13 +0400 Subject: [PATCH 1064/2083] Skip tests that don't work on Python 3.12. --- conftest.py | 2 ++ tests/requirements.txt | 10 ++++++---- tests/test_feedexport.py | 3 +++ tests/test_pipeline_files.py | 5 +++++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/conftest.py b/conftest.py index fa5193470..635935748 100644 --- a/conftest.py +++ b/conftest.py @@ -87,6 +87,8 @@ def requires_uvloop(request): pytest.skip("uvloop does not support Windows") if twisted_version == Version("twisted", 21, 2, 0): pytest.skip("https://twistedmatrix.com/trac/ticket/10106") + if sys.version_info >= (3, 12): + pytest.skip("uvloop doesn't support Python 3.12 yet") def pytest_configure(config): diff --git a/tests/requirements.txt b/tests/requirements.txt index 618949795..37186f3a7 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,15 +1,17 @@ # Tests requirements attrs -pyftpdlib +# https://github.com/giampaolo/pyftpdlib/issues/560 +pyftpdlib; python_version < "3.12" pytest pytest-cov==4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures -uvloop; platform_system != "Windows" +# uvloop currently doesn't build on 3.12 +uvloop; platform_system != "Windows" and python_version < "3.12" -# optional for shell wrapper tests -bpython +# bpython requires greenlet which currently doesn't build on 3.12 +bpython; python_version < "3.12" # optional for shell wrapper tests brotli # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8df86dbd7..eace59d37 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -125,6 +125,9 @@ class FileFeedStorageTest(unittest.TestCase): path.unlink() +@pytest.mark.skipif( + sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" +) class FTPFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 87f3a0295..fe7b26740 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,6 +1,7 @@ import dataclasses import os import random +import sys import time from datetime import datetime from io import BytesIO @@ -11,6 +12,7 @@ from unittest import mock from urllib.parse import urlparse import attr +import pytest from itemadapter import ItemAdapter from twisted.internet import defer from twisted.trial import unittest @@ -641,6 +643,9 @@ class TestGCSFilesStore(unittest.TestCase): store.bucket.get_blob.assert_called_with(expected_blob_path) +@pytest.mark.skipif( + sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" +) class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): From a346732275b425e4fbebc3bdf133df961528df87 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 22 Jul 2023 17:54:55 +0400 Subject: [PATCH 1065/2083] Skip more non-test files during discovery. --- conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conftest.py b/conftest.py index 635935748..2bfa46f5a 100644 --- a/conftest.py +++ b/conftest.py @@ -18,6 +18,10 @@ def _py_files(folder): collect_ignore = [ # not a test, but looks like a test "scrapy/utils/testsite.py", + "tests/ftpserver.py", + "tests/mockserver.py", + "tests/pipelines.py", + "tests/spiders.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess From 21b6dc5f9fbd2607d6f4df20256dcdd9d5b9dae4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 22 Jul 2023 17:55:32 +0400 Subject: [PATCH 1066/2083] Add 3.12 CI jobs. --- .github/workflows/tests-ubuntu.yml | 12 +++++++++++- setup.py | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 39e3b0af7..54b3fbaa2 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -48,6 +48,16 @@ jobs: env: TOXENV: botocore + - python-version: "3.12.0-beta.4" + env: + TOXENV: py + - python-version: "3.12.0-beta.4" + env: + TOXENV: asyncio + - python-version: "3.12.0-beta.4" + env: + TOXENV: extra-deps + steps: - uses: actions/checkout@v3 @@ -57,7 +67,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || contains(matrix.python-version, '3.12.0') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/setup.py b/setup.py index 1f214571b..405633f55 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ setup( "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Internet :: WWW/HTTP", From 53f8570786fbc7c90bc9990a22279d90a52025b3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 22 Jul 2023 18:46:44 +0400 Subject: [PATCH 1067/2083] Add support for the new entry_points() interface. --- scrapy/cmdline.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index efc9b36ea..6580ba9ce 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -48,7 +48,11 @@ def _get_commands_from_module(module, inproject): def _get_commands_from_entry_points(inproject, group="scrapy.commands"): cmds = {} - for entry_point in entry_points().get(group, {}): + if sys.version_info >= (3, 10): + eps = entry_points(group=group) + else: + eps = entry_points().get(group, ()) + for entry_point in eps: obj = entry_point.load() if inspect.isclass(obj): cmds[entry_point.name] = obj() From 5d91ea12d642503da95df43cf033eeeda3db1553 Mon Sep 17 00:00:00 2001 From: Adnan Awan Date: Sat, 22 Jul 2023 23:13:40 +0500 Subject: [PATCH 1068/2083] Fix JsonItemExporter puts lone comma in the output if encoder fails (#5952) * Partial fix for #3090 - only addresses JSON feeds. * Adding test case for #3090 to Json Exporter * Changing the deliberate-fail JSON example to a complex * Further tightening JsonItemExporter behaviour to prevent corruption. Based on Mikhail's observation that to_bytes can fail also, leading to the same dangling comma as the failure to encode to JSON. Added a new test case to avoid reversion. * [scrapy] JsonItemExporter puts lone comma in the output if encoder fails - Add initial changes from cathal's PR - https://github.com/scrapy/scrapy/issues/3090 * [scrapy] JsonItemExporter puts lone comma in the output if encoder fails - Handle exception not to add empty item. - https://github.com/scrapy/scrapy/issues/3090 * [scrapy] JsonItemExporter puts lone comma in the output if encoder fails - Add comment for handling the exception - https://github.com/scrapy/scrapy/issues/3090 * [scrapy] JsonItemExporter puts lone comma in the output if encoder fails - Remove unused import - https://github.com/scrapy/scrapy/issues/3090 * [scrapy] JsonItemExporter puts lone comma in the output if encoder fails - Fix invalid json issue - https://github.com/scrapy/scrapy/issues/3090 * [scrapy] JsonItemExporter puts lone comma in the output if encoder fails - Perform CR changes - https://github.com/scrapy/scrapy/issues/3090 --------- Co-authored-by: Cathal Garvey --- scrapy/exporters.py | 17 ++++++++++------- tests/test_exporters.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 4538c9ee1..8254ea63e 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -133,6 +133,13 @@ class JsonItemExporter(BaseItemExporter): if self.indent is not None: self.file.write(b"\n") + def _add_comma_after_first(self): + if self.first_item: + self.first_item = False + else: + self.file.write(b",") + self._beautify_newline() + def start_exporting(self): self.file.write(b"[") self._beautify_newline() @@ -142,14 +149,10 @@ class JsonItemExporter(BaseItemExporter): self.file.write(b"]") def export_item(self, item): - if self.first_item: - self.first_item = False - else: - self.file.write(b",") - self._beautify_newline() itemdict = dict(self._get_serialized_fields(item)) - data = self.encoder.encode(itemdict) - self.file.write(to_bytes(data, self.encoding)) + data = to_bytes(self.encoder.encode(itemdict), self.encoding) + self._add_comma_after_first() + self.file.write(data) class XmlItemExporter(BaseItemExporter): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 63bebcf7a..cb24ddd8e 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -599,6 +599,20 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): def test_two_dict_items(self): self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) + def test_two_items_with_failure_between(self): + i1 = TestItem(name="Joseph\xa3", age="22") + i2 = TestItem( + name="Maria", age=1j + ) # Invalid datetimes didn't consistently fail between Python versions + i3 = TestItem(name="Jesus", age="44") + self.ie.start_exporting() + self.ie.export_item(i1) + self.assertRaises(TypeError, self.ie.export_item, i2) + self.ie.export_item(i3) + self.ie.finish_exporting() + exported = json.loads(to_unicode(self.output.getvalue())) + self.assertEqual(exported, [dict(i1), dict(i3)]) + def test_nested_item(self): i1 = self.item_class(name="Joseph\xa3", age="22") i2 = self.item_class(name="Maria", age=i1) @@ -637,6 +651,24 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertEqual(exported, [item]) +class JsonItemExporterToBytesTest(BaseItemExporterTest): + def _get_exporter(self, **kwargs): + kwargs["encoding"] = "latin" + return JsonItemExporter(self.output, **kwargs) + + def test_two_items_with_failure_between(self): + i1 = TestItem(name="Joseph", age="22") + i2 = TestItem(name="\u263a", age="11") + i3 = TestItem(name="Jesus", age="44") + self.ie.start_exporting() + self.ie.export_item(i1) + self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2) + self.ie.export_item(i3) + self.ie.finish_exporting() + exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin")) + self.assertEqual(exported, [dict(i1), dict(i3)]) + + class JsonItemExporterDataclassTest(JsonItemExporterTest): item_class = TestDataClass custom_field_item_class = CustomFieldDataclass From 5e1582491bd3226afc69eb12287b951e78bfc4ba Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 22 Jul 2023 23:19:25 +0400 Subject: [PATCH 1069/2083] mypy --show-error-codes is the default now. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1aeb94215..ef7dd5854 100644 --- a/tox.ini +++ b/tox.ini @@ -42,7 +42,7 @@ deps = types-pyOpenSSL==23.2.0.1 types-setuptools==68.0.0.1 commands = - mypy --show-error-codes {posargs: scrapy tests} + mypy {posargs: scrapy tests} [testenv:pre-commit] basepython = python3 From 7522aeed357d90ed95ee10d3b5a506f1b1049d1f Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 24 Jul 2023 04:53:41 -0300 Subject: [PATCH 1070/2083] fix: -O/-o option does not work with absolute paths on Windows (#5971) --- scrapy/extensions/feedexport.py | 6 ++-- scrapy/utils/conf.py | 3 +- tests/test_commands.py | 58 +++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 6e391d3dc..c81f396cb 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -9,7 +9,7 @@ import re import sys import warnings from datetime import datetime -from pathlib import Path +from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile from typing import IO, Any, Callable, List, Optional, Tuple, Union from urllib.parse import unquote, urlparse @@ -615,7 +615,7 @@ class FeedExporter: def _storage_supported(self, uri, feed_options): scheme = urlparse(uri).scheme - if scheme in self.storages: + if scheme in self.storages or PureWindowsPath(uri).drive: try: self._get_storage(uri, feed_options) return True @@ -633,7 +633,7 @@ class FeedExporter: It supports not passing the *feed_options* parameters to classes that do not support it, and issuing a deprecation warning instead. """ - feedcls = self.storages[urlparse(uri).scheme] + feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) crawler = getattr(self, "crawler", None) def build_instance(builder, *preargs): diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 3ade1d105..05d43e456 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -202,7 +202,8 @@ def feed_process_params_from_cli( for element in output: try: feed_uri, feed_format = element.rsplit(":", 1) - except ValueError: + check_valid_format(feed_format) + except (ValueError, UsageError): feed_uri = element feed_format = Path(element).suffix.replace(".", "") else: diff --git a/tests/test_commands.py b/tests/test_commands.py index 03d768d1a..b1d7be628 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -907,6 +907,64 @@ class MySpider(scrapy.Spider): log = self.get_log(spider_code, args=args) self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log) + @skipIf(platform.system() == "Windows", reason="Linux only") + def test_absolute_path_linux(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}/output1.json:json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json", + log, + ) + + args = ["-o", f"{temp_dir}/output2.json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json", + log, + ) + + @skipIf(platform.system() != "Windows", reason="Windows only") + def test_absolute_path_windows(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}\\output1.json:json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json", + log, + ) + + args = ["-o", f"{temp_dir}\\output2.json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json", + log, + ) + @skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): From 9a1bf40c2f7ac0b5fdd4810336afddbfeb925981 Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Thu, 20 Jul 2023 17:03:51 +0800 Subject: [PATCH 1071/2083] expose AWS_REGION_NAME in S3FeedStorage --- docs/topics/feed-exports.rst | 4 +++- scrapy/extensions/feedexport.py | 5 ++++ tests/test_feedexport.py | 41 +++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2218f45e7..700775e4b 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -210,10 +210,12 @@ passed through the following settings: .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys -You can also define a custom ACL and custom endpoint for exported feeds using this setting: +You can also define a custom ACL, custom endpoint, and region name for exported +feeds using these settings: - :setting:`FEED_STORAGE_S3_ACL` - :setting:`AWS_ENDPOINT_URL` +- :setting:`AWS_REGION_NAME` The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c81f396cb..84724640b 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -169,6 +169,7 @@ class S3FeedStorage(BlockingFeedStorage): secret_key=None, acl=None, endpoint_url=None, + region_name=None, *, feed_options=None, session_token=None, @@ -183,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url + self.region_name = region_name if IS_BOTO3_AVAILABLE: import boto3.session @@ -195,6 +197,7 @@ class S3FeedStorage(BlockingFeedStorage): aws_secret_access_key=self.secret_key, aws_session_token=self.session_token, endpoint_url=self.endpoint_url, + region_name=self.region_name, ) else: warnings.warn( @@ -213,6 +216,7 @@ class S3FeedStorage(BlockingFeedStorage): aws_secret_access_key=self.secret_key, aws_session_token=self.session_token, endpoint_url=self.endpoint_url, + region_name=self.region_name, ) if feed_options and feed_options.get("overwrite", True) is False: @@ -232,6 +236,7 @@ class S3FeedStorage(BlockingFeedStorage): session_token=crawler.settings["AWS_SESSION_TOKEN"], acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None, endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None, + region_name=crawler.settings["AWS_REGION_NAME"] or None, feed_options=feed_options, ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 96bb688ab..9e4fb53bf 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -349,6 +349,19 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, "https://example.com") + def test_init_with_region_name(self): + region_name = "ap-east-1" + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + region_name=region_name, + ) + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.region_name, region_name) + self.assertEqual(storage.s3_client._client_config.region_name, region_name) + def test_from_crawler_without_acl(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -377,6 +390,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, None) + def test_without_region_name(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.s3_client._client_config.region_name, "us-east-1") + def test_from_crawler_with_acl(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -404,6 +431,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, "https://example.com") + def test_from_crawler_with_region_name(self): + region_name = "ap-east-1" + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_REGION_NAME": region_name, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.region_name, region_name) + self.assertEqual(storage.s3_client._client_config.region_name, region_name) + @defer.inlineCallbacks def test_store_without_acl(self): storage = S3FeedStorage( From a689fe5baf35e50d47270e9a402b1d67b6e95ffe Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Mon, 24 Jul 2023 13:08:39 +0800 Subject: [PATCH 1072/2083] move region_name param as kwargs --- scrapy/extensions/feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 84724640b..4687c7382 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -169,10 +169,10 @@ class S3FeedStorage(BlockingFeedStorage): secret_key=None, acl=None, endpoint_url=None, - region_name=None, *, feed_options=None, session_token=None, + region_name=None, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") From 3ba2dc4d682c459ca3fa4419a68ae18b52a47642 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 24 Jul 2023 17:49:04 +0400 Subject: [PATCH 1073/2083] Fixes for addon docs. --- docs/topics/addons.rst | 17 ++++++++--------- docs/topics/api.rst | 11 ----------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 8733f9bde..f02399671 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -14,10 +14,8 @@ developers. Activating and configuring add-ons ================================== -Add-ons and their configuration live in Scrapy's -:class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler` -initialization the add-on manager will read a list of enabled add-ons from your -``ADDONS`` setting. +During :class:`~scrapy.crawler.Crawler` initialization, the list of enabled +add-ons is read from your ``ADDONS`` setting. The ``ADDONS`` setting is a dict in which every key is an add-on class or its import path and the value is its priority. @@ -27,7 +25,7 @@ This is an example where two add-ons are enabled in a project's ADDONS = { 'path.to.someaddon': 0, - path.to.someaddon2: 1, + SomeAddonClass: 1, } @@ -158,14 +156,14 @@ Use a fallback component: from scrapy.core.downloader.handlers.http import HTTPDownloadHandler - fallback_setting = "MY_FALLBACK_DOWNLOAD_HANDLER" + FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" class MyHandler: lazy = False def __init__(self, settings, crawler): - dhcls = load_object(settings.get(fallback_setting)) + dhcls = load_object(settings.get(FALLBACK_SETTING)) self._fallback_handler = create_instance( dhcls, settings=None, @@ -182,9 +180,10 @@ Use a fallback component: class MyAddon: def update_settings(self, settings): - if not settings.get(fallback_setting): + if not settings.get(FALLBACK_SETTING): settings.set( - fallback_setting, + FALLBACK_SETTING, settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "addon", ) + settings["DOWNLOAD_HANDLERS"]["http"] = "path.to.MyHandler" diff --git a/docs/topics/api.rst b/docs/topics/api.rst index d1a5497fb..16c28405c 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -155,17 +155,6 @@ Settings API .. autoclass:: BaseSettings :members: -.. _topics-api-addonmanager: - -AddonManager API -================ - -.. module:: scrapy.addons - :synopsis: Add-on manager - -.. autoclass:: AddonManager - :members: - .. _topics-api-spiderloader: SpiderLoader API From c4f0aa4fdfff802c153b1eeda711957d063aed60 Mon Sep 17 00:00:00 2001 From: freddiewanah Date: Wed, 26 Jul 2023 21:09:03 +1000 Subject: [PATCH 1074/2083] Refactor test cases to improve unit test quality (#5986) --- tests/test_crawler.py | 8 ++++---- tests/test_downloader_handlers.py | 2 +- tests/test_downloadermiddleware_httpcompression.py | 6 +++--- tests/test_pipeline_files.py | 9 +++++++-- tests/test_pipeline_images.py | 10 ++++++++-- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 68e58144b..f99606ccf 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -228,14 +228,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def test_crawler_runner_bootstrap_successful(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) - self.assertEqual(runner.bootstrap_failed, False) + self.assertFalse(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) yield runner.crawl(NoRequestsSpider) - self.assertEqual(runner.bootstrap_failed, False) + self.assertFalse(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_bootstrap_failed(self): @@ -248,7 +248,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): else: self.fail("Exception should be raised from spider") - self.assertEqual(runner.bootstrap_failed, True) + self.assertTrue(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_bootstrap_failed_for_several(self): @@ -263,7 +263,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): yield runner.crawl(NoRequestsSpider) - self.assertEqual(runner.bootstrap_failed, True) + self.assertTrue(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 9731b62c4..8459408ff 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -748,7 +748,7 @@ class Http11MockServerTestCase(unittest.TestCase): yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response failure = crawler.spider.meta.get("failure") - self.assertTrue(failure is None) + self.assertIsNone(failure) reason = crawler.spider.meta["close_reason"] self.assertTrue(reason, "finished") diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index fac5588ff..9dad056de 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -229,7 +229,7 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) self.assertStatsEqual("httpcompression/response_count", 1) - self.assertStatsEqual("httpcompression/response_bytes", 104) + self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_force_recalculate_encoding(self): headers = { @@ -254,7 +254,7 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) self.assertStatsEqual("httpcompression/response_count", 1) - self.assertStatsEqual("httpcompression/response_bytes", 104) + self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_no_content_type_header(self): headers = { @@ -277,7 +277,7 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) self.assertStatsEqual("httpcompression/response_count", 1) - self.assertStatsEqual("httpcompression/response_bytes", 104) + self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_gzipped_contenttype(self): response = self._getresponse("gzip") diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index fe7b26740..bf96f17b6 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -468,8 +468,13 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline = UserDefinedFilesPipeline.from_settings( Settings({"FILES_STORE": self.tempdir}) ) - self.assertEqual(pipeline.files_result_field, "this") - self.assertEqual(pipeline.files_urls_field, "that") + self.assertEqual( + pipeline.files_result_field, + UserDefinedFilesPipeline.DEFAULT_FILES_RESULT_FIELD, + ) + self.assertEqual( + pipeline.files_urls_field, UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD + ) def test_user_defined_subclass_default_key_names(self): """Test situation when user defines subclass of FilesPipeline, diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 6fd8e6308..8924875d1 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -598,8 +598,14 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline = UserDefinedImagePipeline.from_settings( Settings({"IMAGES_STORE": self.tempdir}) ) - self.assertEqual(pipeline.images_result_field, "something_else") - self.assertEqual(pipeline.images_urls_field, "something") + self.assertEqual( + pipeline.images_result_field, + UserDefinedImagePipeline.DEFAULT_IMAGES_RESULT_FIELD, + ) + self.assertEqual( + pipeline.images_urls_field, + UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD, + ) def test_user_defined_subclass_default_key_names(self): """Test situation when user defines subclass of ImagePipeline, From d8c5b415597f9a58e5bf10ab1d8838bd7f730949 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 31 Jul 2023 19:09:21 +0400 Subject: [PATCH 1075/2083] Add more addon tests. --- docs/topics/addons.rst | 2 +- tests/test_addons.py | 107 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 96 insertions(+), 13 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index f02399671..edb1e7883 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -186,4 +186,4 @@ Use a fallback component: settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "addon", ) - settings["DOWNLOAD_HANDLERS"]["http"] = "path.to.MyHandler" + settings["DOWNLOAD_HANDLERS"]["https"] = "path.to.MyHandler" diff --git a/tests/test_addons.py b/tests/test_addons.py index 95377d718..4156762d3 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,18 +1,24 @@ +import itertools import unittest -from typing import Any, Dict, Optional +from typing import Any, Dict -from scrapy.crawler import Crawler -from scrapy.settings import BaseSettings +from scrapy import Spider +from scrapy.crawler import Crawler, CrawlerRunner +from scrapy.settings import BaseSettings, Settings from scrapy.utils.test import get_crawler -class GoodAddon: - def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: - super().__init__() - self.config = config or {} - +class SimpleAddon: def update_settings(self, settings): - settings.update(self.config, "addon") + pass + + +def get_addon_cls(config: Dict[str, Any]) -> type: + class AddonWithConfig: + def update_settings(self, settings: BaseSettings): + settings.update(config, priority="addon") + + return AddonWithConfig class CreateInstanceAddon: @@ -35,7 +41,7 @@ class AddonTest(unittest.TestCase): settings.set("KEY1", "default", priority="default") settings.set("KEY2", "project", priority="project") addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"} - testaddon = GoodAddon(addon_config) + testaddon = get_addon_cls(addon_config)() testaddon.update_settings(settings) self.assertEqual(settings["KEY1"], "addon") self.assertEqual(settings["KEY2"], "project") @@ -45,11 +51,27 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): def test_load_settings(self): settings_dict = { - "ADDONS": {"tests.test_addons.GoodAddon": 0}, + "ADDONS": {"tests.test_addons.SimpleAddon": 0}, } crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons - self.assertIsInstance(manager.addons[0], GoodAddon) + self.assertIsInstance(manager.addons[0], SimpleAddon) + + def test_load_settings_order(self): + # Get three addons with different settings + addonlist = [] + for i in range(3): + addon = get_addon_cls({"KEY1": i}) + addon.number = i + addonlist.append(addon) + # Test for every possible ordering + for ordered_addons in itertools.permutations(addonlist): + expected_order = [a.number for a in ordered_addons] + settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} + crawler = get_crawler(settings_dict=settings) + manager = crawler.addons + self.assertEqual([a.number for a in manager.addons], expected_order) + self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) def test_create_instance(self): settings_dict = { @@ -60,3 +82,64 @@ class AddonManagerTest(unittest.TestCase): manager = crawler.addons self.assertIsInstance(manager.addons[0], CreateInstanceAddon) self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") + + def test_settings_priority(self): + config = { + "KEY": 15, # priority=addon + } + settings_dict = { + "ADDONS": {get_addon_cls(config): 1}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual(crawler.settings.getint("KEY"), 15) + + settings = Settings(settings_dict) + settings.set("KEY", 0, priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(Spider) + self.assertEqual(crawler.settings.getint("KEY"), 15) + + settings_dict = { + "KEY": 20, # priority=project + "ADDONS": {get_addon_cls(config): 1}, + } + settings = Settings(settings_dict) + settings.set("KEY", 0, priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(Spider) + self.assertEqual(crawler.settings.getint("KEY"), 20) + + def test_fallback_workflow(self): + FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" + + class AddonWithFallback: + def update_settings(self, settings): + if not settings.get(FALLBACK_SETTING): + settings.set( + FALLBACK_SETTING, + settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + "addon", + ) + settings["DOWNLOAD_HANDLERS"]["https"] = "AddonHandler" + + settings_dict = { + "ADDONS": {AddonWithFallback: 1}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + ) + self.assertEqual( + crawler.settings.get(FALLBACK_SETTING), + "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + ) + + settings_dict = { + "ADDONS": {AddonWithFallback: 1}, + "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + ) + self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") From b67a81b81dd7c2a1b2017482551b3d818026f04f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 31 Jul 2023 20:25:56 +0400 Subject: [PATCH 1076/2083] Use the MyHandler class directly. --- docs/topics/addons.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index edb1e7883..8dddb7c91 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -186,4 +186,4 @@ Use a fallback component: settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "addon", ) - settings["DOWNLOAD_HANDLERS"]["https"] = "path.to.MyHandler" + settings["DOWNLOAD_HANDLERS"]["https"] = MyHandler From 41a4a163e3716bf0c14160373844bcded0e38abd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 31 Jul 2023 21:09:18 +0400 Subject: [PATCH 1077/2083] Handle NotConfigured in add-ons. --- docs/topics/addons.rst | 6 +++++- scrapy/addons.py | 26 +++++++++++--------------- scrapy/crawler.py | 1 - tests/test_addons.py | 13 +++++++++++++ 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 8dddb7c91..32c085079 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -77,6 +77,10 @@ modify :setting:`ITEM_PIPELINES`:: if settings.getbool("MYADDON_ENABLE_PIPELINE"): settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 +If the ``update_settings`` method raises +:exc:`scrapy.exceptions.NotConfigured`, the add-on will not be skipped. This +makes it easy to enable an add-on only when some conditions are met. + Fallbacks --------- @@ -130,7 +134,7 @@ Check dependencies: try: import boto except ImportError: - raise RuntimeError("MyAddon requires the boto library") + raise NotConfigured("MyAddon requires the boto library") ... Access the crawler instance: diff --git a/scrapy/addons.py b/scrapy/addons.py index e72c5da98..072143107 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,6 +1,8 @@ import logging from typing import TYPE_CHECKING, Any, List +from scrapy.exceptions import NotConfigured +from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.misc import create_instance, load_object @@ -17,19 +19,23 @@ class AddonManager: self.crawler: "Crawler" = crawler self.addons: List[Any] = [] - def _add(self, addon: Any) -> None: + def _add(self, addon: Any, settings: Settings) -> None: """Store an add-on.""" if isinstance(addon, (type, str)): addon = load_object(addon) if isinstance(addon, type): addon = create_instance(addon, settings=None, crawler=self.crawler) - self.addons.append(addon) + try: + addon.update_settings(settings) + self.addons.append(addon) + except NotConfigured: + pass - def load_settings(self, settings) -> None: + def load_settings(self, settings: Settings) -> None: """Load add-ons and configurations from a settings object. This will load the add-on for every add-on path in the - ``ADDONS`` setting. + ``ADDONS`` setting and execute their ``update_settings`` methods. :param settings: The :class:`~scrapy.settings.Settings` object from \ which to read the add-on configuration @@ -37,7 +43,7 @@ class AddonManager: """ addons = build_component_list(settings["ADDONS"]) for addon in build_component_list(settings["ADDONS"]): - self._add(addon) + self._add(addon, settings) logger.info( "Enabled addons:\n%(addons)s", { @@ -45,13 +51,3 @@ class AddonManager: }, extra={"crawler": self.crawler}, ) - - def update_settings(self, settings) -> None: - """Call ``update_settings()`` of all held add-ons. - - :param settings: The :class:`~scrapy.settings.Settings` object to be \ - updated - :type settings: :class:`~scrapy.settings.Settings` - """ - for addon in self.addons: - addon.update_settings(settings) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 12256440b..bf69cee26 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -71,7 +71,6 @@ class Crawler: self.addons: AddonManager = AddonManager(self) self.addons.load_settings(self.settings) - self.addons.update_settings(self.settings) self.signals: SignalManager = SignalManager(self) diff --git a/tests/test_addons.py b/tests/test_addons.py index 4156762d3..5d053ed52 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -4,6 +4,7 @@ from typing import Any, Dict from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner +from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings from scrapy.utils.test import get_crawler @@ -57,6 +58,18 @@ class AddonManagerTest(unittest.TestCase): manager = crawler.addons self.assertIsInstance(manager.addons[0], SimpleAddon) + def test_notconfigured(self): + class NotConfiguredAddon: + def update_settings(self, settings): + raise NotConfigured() + + settings_dict = { + "ADDONS": {NotConfiguredAddon: 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertFalse(manager.addons) + def test_load_settings_order(self): # Get three addons with different settings addonlist = [] From cf55eb05f59d7003bfb1491198e46853976dfb9e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Aug 2023 13:30:56 +0400 Subject: [PATCH 1078/2083] Fix a typo. --- docs/topics/addons.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 32c085079..1bf2172bd 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -78,8 +78,8 @@ modify :setting:`ITEM_PIPELINES`:: settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 If the ``update_settings`` method raises -:exc:`scrapy.exceptions.NotConfigured`, the add-on will not be skipped. This -makes it easy to enable an add-on only when some conditions are met. +:exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes +it easy to enable an add-on only when some conditions are met. Fallbacks --------- From 7fdeb5c5c100fd94fd0fe2b8abc0c3fc0ba8c54d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Aug 2023 16:58:18 +0400 Subject: [PATCH 1079/2083] Improve NotConfigured handling in add-ons. --- scrapy/addons.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 072143107..cd0cfb7de 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -19,18 +19,6 @@ class AddonManager: self.crawler: "Crawler" = crawler self.addons: List[Any] = [] - def _add(self, addon: Any, settings: Settings) -> None: - """Store an add-on.""" - if isinstance(addon, (type, str)): - addon = load_object(addon) - if isinstance(addon, type): - addon = create_instance(addon, settings=None, crawler=self.crawler) - try: - addon.update_settings(settings) - self.addons.append(addon) - except NotConfigured: - pass - def load_settings(self, settings: Settings) -> None: """Load add-ons and configurations from a settings object. @@ -41,13 +29,29 @@ class AddonManager: which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - addons = build_component_list(settings["ADDONS"]) - for addon in build_component_list(settings["ADDONS"]): - self._add(addon, settings) + enabled = [] + for clspath in build_component_list(settings["ADDONS"]): + try: + addoncls = load_object(clspath) + addon = create_instance( + addoncls, settings=settings, crawler=self.crawler + ) + addon.update_settings(settings) + self.addons.append(addon) + except NotConfigured as e: + if e.args: + clsname = ( + clspath.split(".")[-1] if isinstance(clspath, str) else clspath + ) + logger.warning( + "Disabled %(clsname)s: %(eargs)s", + {"clsname": clsname, "eargs": e.args[0]}, + extra={"crawler": self.crawler}, + ) logger.info( "Enabled addons:\n%(addons)s", { - "addons": addons, + "addons": enabled, }, extra={"crawler": self.crawler}, ) From f803ad63f32b488b3b89a81c6069d3575910e28e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Aug 2023 17:23:09 +0400 Subject: [PATCH 1080/2083] Fix a typing issue. --- scrapy/addons.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index cd0cfb7de..f1ddfd211 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -29,7 +29,7 @@ class AddonManager: which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - enabled = [] + enabled: List[Any] = [] for clspath in build_component_list(settings["ADDONS"]): try: addoncls = load_object(clspath) From e58b8078f0d51148c1866d1da3f6e36609b5e2a5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Aug 2023 20:20:15 +0400 Subject: [PATCH 1081/2083] Improve NotConfigured handling when clspath is a class. --- scrapy/addons.py | 7 ++----- scrapy/middleware.py | 5 ++--- tests/test_middleware.py | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index f1ddfd211..02dd4fde8 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -40,12 +40,9 @@ class AddonManager: self.addons.append(addon) except NotConfigured as e: if e.args: - clsname = ( - clspath.split(".")[-1] if isinstance(clspath, str) else clspath - ) logger.warning( - "Disabled %(clsname)s: %(eargs)s", - {"clsname": clsname, "eargs": e.args[0]}, + "Disabled %(clspath)s: %(eargs)s", + {"clspath": clspath, "eargs": e.args[0]}, extra={"crawler": self.crawler}, ) logger.info( diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 03e92b565..04b838d2d 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -46,10 +46,9 @@ class MiddlewareManager: enabled.append(clspath) except NotConfigured as e: if e.args: - clsname = clspath.split(".")[-1] logger.warning( - "Disabled %(clsname)s: %(eargs)s", - {"clsname": clsname, "eargs": e.args[0]}, + "Disabled %(clspath)s: %(eargs)s", + {"clspath": clspath, "eargs": e.args[0]}, extra={"crawler": crawler}, ) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 00ff746ee..a42c7b3d1 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -39,7 +39,7 @@ class MOff: pass def __init__(self): - raise NotConfigured + raise NotConfigured("foo") class TestMiddlewareManager(MiddlewareManager): From 72462a53e2cfcac3ec6068fc94fec657c73e157c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 12:32:53 +0400 Subject: [PATCH 1082/2083] Add more docs for update_settings(). --- docs/topics/settings.rst | 20 ++++++++++++++++++-- docs/topics/spiders.rst | 17 +++++++++++------ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 219509c1e..d0f2acd89 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -66,8 +66,8 @@ Example:: ---------------------- Spiders (See the :ref:`topics-spiders` chapter for reference) can define their -own settings that will take precedence and override the project ones. They can -do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute: +own settings that will take precedence and override the project ones. One way +to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: .. code-block:: python @@ -81,6 +81,22 @@ do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute: "SOME_SETTING": "some value", } +It's often better to provide a :meth:`~scrapy.Spider.update_settings` instead, +and settings set there should use the "spider" priority explicitly: + +.. code-block:: python + + import scrapy + + + class MySpider(scrapy.Spider): + name = "myspider" + + @classmethod + def update_settings(cls, settings): + settings.set("SOME_SETTING", "some value", priority="spider") + super().update_settings(settings) + 3. Project settings module -------------------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 0f6c2b1ba..97b525bd6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -151,13 +151,18 @@ scrapy.Spider and is called during initialization of a spider instance. It takes a :class:`~scrapy.settings.Settings` object as a parameter and - can add or update the spider's configuration values. This method is a class method, - meaning that it is called on the :class:`~scrapy.Spider` class and allows all instances - of the spider to share the same configuration. + can add or update the spider's configuration values. This method is a + class method, meaning that it is called on the :class:`~scrapy.Spider` + class and allows all instances of the spider to share the same + configuration. - One of the main advantages of ``update_settings()`` is that it allows - you to dynamically add, remove or change settings based on other settings - or other external factors. + While per-spider settings can be set in + :attr:`~scrapy.Spider.custom_settings`, using ``update_settings()`` + allows you to dynamically add, remove or change settings based on other + settings, spider attributes or other factors and use setting priorities + other than ``'spider'``. Also, it's easy to extend ``update_settings()`` + in a subclass by overriding it, while doing the same with + :attr:`~scrapy.Spider.custom_settings` is hard or impossible. For example, suppose a spider needs to modify :setting:`FEEDS`: From 9f9a2292e08cb0944e1e88e64cef1f4b17486ec8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 16:21:06 +0400 Subject: [PATCH 1083/2083] Deprecate the custom attribute of build_component_list(). (#5993) --- scrapy/utils/conf.py | 14 ++++++--- tests/test_utils_conf.py | 64 +++++++++++++++++++--------------------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 43a8b65a5..1889f7571 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -50,11 +50,17 @@ def build_component_list(compdict, custom=None, convert=update_classpath): "please provide a real number or None instead" ) - if isinstance(custom, (list, tuple)): - _check_components(custom) - return type(custom)(convert(c) for c in custom) - if custom is not None: + warnings.warn( + "The 'custom' attribute of build_component_list() is deprecated. " + "Please merge its value into 'compdict' manually or change your " + "code to use Settings.getwithbase().", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if isinstance(custom, (list, tuple)): + _check_components(custom) + return type(custom)(convert(c) for c in custom) compdict.update(custom) _validate_values(compdict) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 78ed9a7c9..dc3f01d57 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,6 +1,8 @@ import unittest import warnings +import pytest + from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( @@ -21,44 +23,45 @@ class BuildComponentListTest(unittest.TestCase): def test_backward_compatible_build_dict(self): base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None} custom = {"two": None, "three": 8, "four": 4} - self.assertEqual( - build_component_list(base, custom, convert=lambda x: x), - ["one", "four", "five", "three"], - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list(base, custom, convert=lambda x: x), + ["one", "four", "five", "three"], + ) def test_return_list(self): custom = ["a", "b", "c"] - self.assertEqual( - build_component_list(None, custom, convert=lambda x: x), custom - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list(None, custom, convert=lambda x: x), custom + ) def test_map_dict(self): custom = {"one": 1, "two": 2, "three": 3} - self.assertEqual( - build_component_list({}, custom, convert=lambda x: x.upper()), - ["ONE", "TWO", "THREE"], - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list({}, custom, convert=lambda x: x.upper()), + ["ONE", "TWO", "THREE"], + ) def test_map_list(self): custom = ["a", "b", "c"] - self.assertEqual( - build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"] - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"] + ) def test_duplicate_components_in_dict(self): duplicate_dict = {"one": 1, "two": 2, "ONE": 4} - self.assertRaises( - ValueError, - build_component_list, - {}, - duplicate_dict, - convert=lambda x: x.lower(), - ) + with self.assertRaises(ValueError): + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + build_component_list({}, duplicate_dict, convert=lambda x: x.lower()) def test_duplicate_components_in_list(self): duplicate_list = ["a", "b", "a"] with self.assertRaises(ValueError) as cm: - build_component_list(None, duplicate_list, convert=lambda x: x) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + build_component_list(None, duplicate_list, convert=lambda x: x) self.assertIn(str(duplicate_list), str(cm.exception)) def test_duplicate_components_in_basesettings(self): @@ -76,9 +79,8 @@ class BuildComponentListTest(unittest.TestCase): ) # Same priority raises ValueError duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20) - self.assertRaises( - ValueError, build_component_list, duplicate_bs, convert=lambda x: x.lower() - ) + with self.assertRaises(ValueError): + build_component_list(duplicate_bs, convert=lambda x: x.lower()) def test_valid_numbers(self): # work well with None and numeric values @@ -92,15 +94,9 @@ class BuildComponentListTest(unittest.TestCase): self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"]) # raise exception for invalid values d = {"one": "5"} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": "1.0"} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": [1, 2, 3]} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": {"a": "a", "b": 2}} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": "lorem ipsum"} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + with self.assertRaises(ValueError): + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + build_component_list({}, d, convert=lambda x: x) class UtilsConfTestCase(unittest.TestCase): From af1be835e4a6c14634acb382568935c1a7e10445 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 19:46:16 +0400 Subject: [PATCH 1084/2083] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/settings.rst | 4 ++-- docs/topics/spiders.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d0f2acd89..0963f835f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -81,7 +81,7 @@ to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: "SOME_SETTING": "some value", } -It's often better to provide a :meth:`~scrapy.Spider.update_settings` instead, +It's often better to implement :meth:`~scrapy.Spider.update_settings` instead, and settings set there should use the "spider" priority explicitly: .. code-block:: python @@ -94,8 +94,8 @@ and settings set there should use the "spider" priority explicitly: @classmethod def update_settings(cls, settings): - settings.set("SOME_SETTING", "some value", priority="spider") super().update_settings(settings) + settings.set("SOME_SETTING", "some value", priority="spider") 3. Project settings module -------------------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 97b525bd6..5c3bf6e72 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -162,7 +162,7 @@ scrapy.Spider settings, spider attributes or other factors and use setting priorities other than ``'spider'``. Also, it's easy to extend ``update_settings()`` in a subclass by overriding it, while doing the same with - :attr:`~scrapy.Spider.custom_settings` is hard or impossible. + :attr:`~scrapy.Spider.custom_settings` can be hard. For example, suppose a spider needs to modify :setting:`FEEDS`: @@ -182,8 +182,8 @@ scrapy.Spider @classmethod def update_settings(cls, settings): - settings.setdefault("FEEDS", {}).update(cls.custom_feed) super().update_settings(settings) + settings.setdefault("FEEDS", {}).update(cls.custom_feed) .. method:: start_requests() From b9c32a0cfd13dea2a59d20735f33ceff18daac97 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 3 Aug 2023 12:06:55 -0300 Subject: [PATCH 1085/2083] Remove functions deprecated in 2.6.0 (#5996) --- scrapy/extensions/feedexport.py | 9 --------- scrapy/squeues.py | 30 ------------------------------ scrapy/utils/reqser.py | 27 --------------------------- tests/test_feedexport.py | 5 +---- tests/test_request_dict.py | 28 ---------------------------- 5 files changed, 1 insertion(+), 98 deletions(-) delete mode 100644 scrapy/utils/reqser.py diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4687c7382..c8022ff57 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -672,21 +672,12 @@ class FeedExporter: params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") params["batch_id"] = slot.batch_id + 1 if slot is not None else 1 - original_params = params.copy() uripar_function = ( load_object(uri_params_function) if uri_params_function else lambda params, _: params ) new_params = uripar_function(params, spider) - if new_params is None or original_params != params: - warnings.warn( - "Modifying the params dictionary in-place in the function defined in " - "the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS " - "setting is deprecated. The function must return a new dictionary " - "instead.", - category=ScrapyDeprecationWarning, - ) return new_params if new_params is not None else params def _load_filter(self, feed_options): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 6afe0d636..f665ad88c 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -10,7 +10,6 @@ from typing import Union from queuelib import queue -from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.request import request_from_dict @@ -143,32 +142,3 @@ MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDisk MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) - - -# deprecated queue classes -_subclass_warn_message = "{cls} inherits from deprecated class {old}" -_instance_warn_message = "{cls} is deprecated" -PickleFifoDiskQueueNonRequest = create_deprecated_class( - name="PickleFifoDiskQueueNonRequest", - new_class=_PickleFifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) -PickleLifoDiskQueueNonRequest = create_deprecated_class( - name="PickleLifoDiskQueueNonRequest", - new_class=_PickleLifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) -MarshalFifoDiskQueueNonRequest = create_deprecated_class( - name="MarshalFifoDiskQueueNonRequest", - new_class=_MarshalFifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) -MarshalLifoDiskQueueNonRequest = create_deprecated_class( - name="MarshalLifoDiskQueueNonRequest", - new_class=_MarshalLifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py deleted file mode 100644 index 15705db83..000000000 --- a/scrapy/utils/reqser.py +++ /dev/null @@ -1,27 +0,0 @@ -import warnings -from typing import Optional - -import scrapy -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.request import request_from_dict as _from_dict - -warnings.warn( - ( - "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" - " and/or scrapy.utils.request.request_from_dict instead" - ), - category=ScrapyDeprecationWarning, - stacklevel=2, -) - - -def request_to_dict( - request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None -) -> dict: - return request.to_dict(spider=spider) - - -def request_from_dict( - d: dict, spider: Optional["scrapy.Spider"] = None -) -> "scrapy.Request": - return _from_dict(d, spider=spider) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 9e4fb53bf..46bd5733a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -3067,10 +3067,7 @@ class URIParamsTest: spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with pytest.warns( - ScrapyDeprecationWarning, match="Modifying the params dictionary in-place" - ): - feed_exporter.open_spider(spider) + feed_exporter.open_spider(spider) self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 8665a9205..7312eb036 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,10 +1,6 @@ -import sys import unittest -import warnings -from contextlib import suppress from scrapy import Request, Spider -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import FormRequest, JsonRequest from scrapy.utils.request import request_from_dict @@ -162,30 +158,6 @@ class RequestSerializationTest(unittest.TestCase): self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo")) -class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest): - def _assert_serializes_ok(self, request, spider=None): - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - with suppress(KeyError): - del sys.modules[ - "scrapy.utils.reqser" - ] # delete module to reset the deprecation warning - - from scrapy.utils.reqser import request_from_dict as _from_dict - from scrapy.utils.reqser import request_to_dict as _to_dict - - request_copy = _from_dict(_to_dict(request, spider), spider) - self._assert_same_request(request, request_copy) - - self.assertEqual(len(caught), 1) - self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) - self.assertEqual( - "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" - " and/or scrapy.utils.request.request_from_dict instead", - str(caught[0].message), - ) - - class TestSpiderMixin: def __mixin_callback(self, response): pass From 8a0a9e6d3e5398b884e92cb9336efe3fd19677d6 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 3 Aug 2023 18:31:12 -0300 Subject: [PATCH 1086/2083] Enable Python 3.11 on Windows CI --- .github/workflows/tests-windows.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 5bcf74d5e..c8d1928d7 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -20,13 +20,12 @@ jobs: - python-version: "3.10" env: TOXENV: asyncio -# no binary package for lxml for 3.11 yet -# - python-version: "3.11" -# env: -# TOXENV: py -# - python-version: "3.11" -# env: -# TOXENV: asyncio + - python-version: "3.11" + env: + TOXENV: py + - python-version: "3.11" + env: + TOXENV: asyncio steps: - uses: actions/checkout@v3 From 09c63a178bf28a66b564562125497721bea371b3 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Fri, 4 Aug 2023 02:53:04 -0300 Subject: [PATCH 1087/2083] Remove the deprecated spider parameter from the engine (#5998) --- scrapy/core/engine.py | 93 ++++++++----------------------------------- tests/test_engine.py | 87 +--------------------------------------- 2 files changed, 18 insertions(+), 162 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 3e5a281b2..dad384ddc 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -5,7 +5,6 @@ For more information see docs/topics/architecture.rst """ import logging -import warnings from time import time from typing import ( TYPE_CHECKING, @@ -14,7 +13,6 @@ from typing import ( Generator, Iterable, Iterator, - List, Optional, Set, Type, @@ -29,7 +27,7 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper -from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning +from scrapy.exceptions import CloseSpider, DontCloseSpider from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter from scrapy.settings import BaseSettings, Settings @@ -213,7 +211,7 @@ class ExecutionEngine: if request is None: return None - d = self._download(request, self.spider) + d = self._download(request) d.addBoth(self._handle_downloader_output, request) d.addErrback( lambda f: logger.info( @@ -266,13 +264,7 @@ class ExecutionEngine: ) return d - def spider_is_idle(self, spider: Optional[Spider] = None) -> bool: - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) + def spider_is_idle(self) -> bool: if self.slot is None: raise RuntimeError("Engine slot not assigned") if not self.scraper.slot.is_idle(): # type: ignore[union-attr] @@ -285,18 +277,8 @@ class ExecutionEngine: return False return True - def crawl(self, request: Request, spider: Optional[Spider] = None) -> None: + def crawl(self, request: Request) -> None: """Inject the request into the spider <-> downloader pipeline""" - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if spider is not self.spider: - raise RuntimeError( - f"The spider {spider.name!r} does not match the open spider" - ) if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") self._schedule_request(request, self.spider) @@ -311,39 +293,24 @@ class ExecutionEngine: signals.request_dropped, request=request, spider=spider ) - def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred: + def download(self, request: Request) -> Deferred: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to ExecutionEngine.download is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if spider is not self.spider: - logger.warning( - "The spider '%s' does not match the open spider", spider.name - ) if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - return self._download(request, spider).addBoth( - self._downloaded, request, spider - ) + return self._download(request).addBoth(self._downloaded, request) def _downloaded( - self, result: Union[Response, Request], request: Request, spider: Spider + self, result: Union[Response, Request], request: Request ) -> Union[Deferred, Response]: assert self.slot is not None # typing self.slot.remove_request(request) - return self.download(result, spider) if isinstance(result, Request) else result + return self.download(result) if isinstance(result, Request) else result - def _download(self, request: Request, spider: Optional[Spider]) -> Deferred: + def _download(self, request: Request) -> Deferred: assert self.slot is not None # typing self.slot.add_request(request) - if spider is None: - spider = self.spider - def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: if not isinstance(result, (Response, Request)): raise TypeError( @@ -352,15 +319,17 @@ class ExecutionEngine: if isinstance(result, Response): if result.request is None: result.request = request - assert spider is not None - logkws = self.logformatter.crawled(result.request, result, spider) + assert self.spider is not None + logkws = self.logformatter.crawled(result.request, result, self.spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.spider} + ) self.signals.send_catch_log( signal=signals.response_received, response=result, request=result.request, - spider=spider, + spider=self.spider, ) return result @@ -369,8 +338,8 @@ class ExecutionEngine: self.slot.nextcall.schedule() return _ - assert spider is not None - dwld = self.downloader.fetch(request, spider) + assert self.spider is not None + dwld = self.downloader.fetch(request, self.spider) dwld.addCallbacks(_on_success) dwld.addBoth(_on_complete) return dwld @@ -485,31 +454,3 @@ class ExecutionEngine: dfd.addBoth(lambda _: self._spider_closed_callback(spider)) return dfd - - @property - def open_spiders(self) -> List[Spider]: - warnings.warn( - "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return [self.spider] if self.spider is not None else [] - - def has_capacity(self) -> bool: - warnings.warn( - "ExecutionEngine.has_capacity is deprecated", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return not bool(self.slot) - - def schedule(self, request: Request, spider: Spider) -> None: - warnings.warn( - "ExecutionEngine.schedule is deprecated, please use " - "ExecutionEngine.crawl or ExecutionEngine.download instead", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if self.slot is None: - raise RuntimeError("Engine slot not assigned") - self._schedule_request(request, spider) diff --git a/tests/test_engine.py b/tests/test_engine.py index 410eba921..8d7afb6a1 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -20,7 +20,6 @@ from threading import Timer from urllib.parse import urlparse import attr -import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer, reactor @@ -29,7 +28,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning +from scrapy.exceptions import CloseSpider from scrapy.http import Request from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor @@ -436,90 +435,6 @@ class EngineTest(unittest.TestCase): finally: yield e.stop() - @defer.inlineCallbacks - def test_close_spiders_downloader(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="ExecutionEngine.open_spiders is deprecated, " - "please use ExecutionEngine.spider instead", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - self.assertEqual(len(e.open_spiders), 1) - yield e.close() - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_close_engine_spiders_downloader(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="ExecutionEngine.open_spiders is deprecated, " - "please use ExecutionEngine.spider instead", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - e.start() - self.assertTrue(e.running) - yield e.close() - self.assertFalse(e.running) - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_crawl_deprecated_spider_arg(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Passing a 'spider' argument to " - "ExecutionEngine.crawl is deprecated", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.crawl(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_download_deprecated_spider_arg(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Passing a 'spider' argument to " - "ExecutionEngine.download is deprecated", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.download(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_deprecated_schedule(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="ExecutionEngine.schedule is deprecated, please use " - "ExecutionEngine.crawl or ExecutionEngine.download instead", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.schedule(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_deprecated_has_capacity(self): - with pytest.warns( - ScrapyDeprecationWarning, match="ExecutionEngine.has_capacity is deprecated" - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - self.assertTrue(e.has_capacity()) - spider = TestSpider() - yield e.open_spider(spider, []) - self.assertFalse(e.has_capacity()) - e.start() - yield e.close() - self.assertTrue(e.has_capacity()) - def test_short_timeout(self): args = ( sys.executable, From 72de48be6d351cc504ec901e39bf649d84214665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 4 Aug 2023 09:56:30 +0200 Subject: [PATCH 1088/2083] asyncio: cover accidental bad reactor installation, sort sections, reword the Windows section --- docs/topics/asyncio.rst | 100 ++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7713b1af1..07baea071 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -27,54 +27,43 @@ reactor manually. You can do that using install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') -.. _using-custom-loops: +.. _asyncio-preinstalled-reactor: -Using custom asyncio loops -========================== +Handling a pre-installed reactor +================================ -You can also use custom asyncio event loops with the asyncio reactor. Set the -:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to -use it instead of the default asyncio event loop. +``twisted.internet.reactor`` and some other Twisted imports install the default +Twisted reactor as a side effect. Once a Twisted reactor is installed, it is +not possible to switch to a different reactor at run time. + +If you :ref:`configure the asyncio Twisted reactor ` and, at +run time, Scrapy complains that a different reactor is already installed, +chances are you have some such imports in your code. + +You can usually fix the issue by moving those offending module-level Twisted +imports to the method or function definitions where they are used. For example, +if you have something like: + +.. code-block:: python + + from twisted.internet import reactor -.. _asyncio-windows: + def my_function(): + reactor.callLater(...) -Windows-specific notes -====================== +Switch to something like: -The Windows implementation of :mod:`asyncio` can use two event loop -implementations: +.. code-block:: python -- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required - when using Twisted. + def my_function(): + from twisted.internet import reactor -- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work - with Twisted. + reactor.callLater(...) -So on Python 3.8+ the event loop class needs to be changed. - -.. versionchanged:: 2.6.0 - The event loop class is changed automatically when you change the - :setting:`TWISTED_REACTOR` setting or call - :func:`~scrapy.utils.reactor.install_reactor`. - -To change the event loop class manually, call the following code before -installing the reactor:: - - import asyncio - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - -You can put this in the same function that installs the reactor, if you do that -yourself, or in some code that runs before the reactor is installed, e.g. -``settings.py``. - -.. note:: Other libraries you use may require - :class:`~asyncio.ProactorEventLoop`, e.g. because it supports - subprocesses (this is the case with `playwright`_), so you cannot use - them together with Scrapy on Windows (but you should be able to use - them on WSL or native Linux). - -.. _playwright: https://github.com/microsoft/playwright-python +Alternatively, you can try to :ref:`manually install the asyncio reactor +`, with :func:`~scrapy.utils.reactor.install_reactor`, before +those imports happen. .. _asyncio-await-dfd: @@ -122,3 +111,36 @@ example: f"TWISTED_REACTOR setting. See the asyncio documentation " f"of Scrapy for more information." ) + + +.. _asyncio-windows: + +Windows-specific notes +====================== + +The Windows implementation of :mod:`asyncio` can use two event loop +implementations, :class:`~asyncio.ProactorEventLoop` (default) and +:class:`~asyncio.SelectorEventLoop`. However, only +:class:`~asyncio.SelectorEventLoop` works with Twisted. + +Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop` +automatically when you change the :setting:`TWISTED_REACTOR` setting or call +:func:`~scrapy.utils.reactor.install_reactor`. + +.. note:: Other libraries you use may require + :class:`~asyncio.ProactorEventLoop`, e.g. because it supports + subprocesses (this is the case with `playwright`_), so you cannot use + them together with Scrapy on Windows (but you should be able to use + them on WSL or native Linux). + +.. _playwright: https://github.com/microsoft/playwright-python + + +.. _using-custom-loops: + +Using custom asyncio loops +========================== + +You can also use custom asyncio event loops with the asyncio reactor. Set the +:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event +loop class to use it instead of the default asyncio event loop. From 8b6a50a935b3dd822f7647c828c17f5409db541f Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 4 Aug 2023 11:12:07 +0300 Subject: [PATCH 1089/2083] periodic_log: docs added --- docs/topics/extensions.rst | 101 +++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 96e0216b8..ae94c55a4 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -383,3 +383,104 @@ For more info see `Debugging in Python`_. This extension only works on POSIX-compliant platforms (i.e. not Windows). .. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/ + +Periodic log extension +~~~~~~~~~~~~~~~~~~~~~~ + +.. class:: PeriodicLog + +Extension provides extended stats data periodically in addition to basic data from Log Stats and Core Stats extensions (as JSON compatible dictionary) like: :: + + 2023-08-04 02:30:57 [scrapy.extensions.logstats] INFO: Crawled 976 pages (at 162 pages/min), scraped 925 items (at 161 items/min) + 2023-08-04 02:30:57 [scrapy.extensions.periodic_log] INFO: { + "delta": { + "downloader/request_bytes": 55582, + "downloader/request_count": 162, + "downloader/request_method_count/GET": 162, + "downloader/response_bytes": 618133, + "downloader/response_count": 162, + "downloader/response_status_count/200": 162, + "item_scraped_count": 161 + }, + "stats": { + "downloader/request_bytes": 338243, + "downloader/request_count": 992, + "downloader/request_method_count/GET": 992, + "downloader/response_bytes": 3836736, + "downloader/response_count": 976, + "downloader/response_status_count/200": 976, + "item_scraped_count": 925, + "log_count/INFO": 21, + "log_count/WARNING": 1, + "scheduler/dequeued": 992, + "scheduler/dequeued/memory": 992, + "scheduler/enqueued": 1050, + "scheduler/enqueued/memory": 1050 + }, + "time": { + "elapsed": 360.008903, + "log_interval": 60.0, + "log_interval_real": 60.006694, + "start_time": "2023-08-03 23:24:57", + "utcnow": "2023-08-03 23:30:57" + } + } + +``"delta"`` section shows numeric difference in stats values between current and previous log entry with period of ``LOGSTATS_INTERVAL`` (60 seconds by default). Its applicable for stats with values types ``int`` and ``float``. +Stats values displayed in this section configured by :setting:`PERIODIC_LOG_DELTA` setting. + +``"stats"`` section shows stats values as is at the moment of current period. +Stats values displayed in this section configured by :setting:`PERIODIC_LOG_STATS` setting. + +``"time"`` This extension produce log entries on startup, periodically, and on end of crawl. As final log entry produced earlier than ``LOGSTATS_INTERVAL`` value - detailed timing data required for more precise stats. + +Configured by :setting:`PERIODIC_LOG_TIMING_ENABLED` + + +Example extension configuration: + +.. code-block:: python + + custom_settings = { + "LOG_LEVEL": "INFO", + "PERIODIC_LOG_STATS": { + "include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count/"], + }, + "PERIODIC_LOG_DELTA": {"include": ["downloader/"]}, + "PERIODIC_LOG_TIMING_ENABLED": True, + "EXTENSIONS": { + "scrapy.extensions.periodic_log.PeriodicLog": 0, + }, + } + +.. setting:: PERIODIC_LOG_DELTA + +PERIODIC_LOG_DELTA +"""""""""""""""""" + +Default: ``None`` + +* ``"PERIODIC_LOG_DELTA": True`` - show deltas for all ``int`` and ``float`` stats values. +* ``"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}`` - include stats deltas for stats with names that have listed substrings in stats names. +* ``"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}`` - include all stats deltas except stats with listed substrings in stats names. + +.. setting:: PERIODIC_LOG_STATS + +PERIODIC_LOG_STATS +"""""""""""""""""" + +Default: ``None`` + +* ``"PERIODIC_LOG_STATS": True`` - show all available stats keys/values +* ``"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}`` - include stats for keys that have listed substrings in stats names. +* ``"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}`` - include all stats deltas except stats with listed substrings in stats names. + + +.. setting:: PERIODIC_LOG_TIMING_ENABLED + +PERIODIC_LOG_TIMING_ENABLED +""""""""""""""""""""""""""" + +Default: ``None`` + +``"PERIODIC_LOG_TIMING_ENABLED": True`` - enables logging of timing data \ No newline at end of file From e9b088f1fb430009e4ea4e5b5a3836b4b94364eb Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 4 Aug 2023 12:49:22 +0300 Subject: [PATCH 1090/2083] periodic_log: typing --- tests/test_extension_periodic_log.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 242e390b3..80f5c3177 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,4 +1,5 @@ import datetime +import typing import unittest from scrapy.crawler import Crawler @@ -96,7 +97,7 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict, condition: callable): + def check(settings: dict, condition: typing.Callable): ext, a, b = emulate(settings) assert list(a["delta"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) @@ -153,7 +154,7 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict, condition: callable): + def check(settings: dict, condition: typing.Callable): ext, a, b = emulate(settings) assert list(a["stats"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) From c14a0a9d5d0443165e1581a039803dde648b9ee6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 4 Aug 2023 16:36:27 +0400 Subject: [PATCH 1091/2083] Add release notes for 2.10.0. --- docs/news.rst | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c7ad11862..ae51d05dc 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,176 @@ Release notes ============= +.. _release-2.10.0: + +Scrapy 2.10.0 (YYYY-MM-DD) +-------------------------- + +Highlights: + +- Added Python 3.12 support, dropped Python 3.7 support. + +- The add-ons framework that simplifies configuring 3rd-party components that + support it. + +- Exceptions to retry can now be configured. + +- Many fixes and improvements for feed exports. + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for Python 3.7. (:issue:`5953`) + +- Added support for the upcoming Python 3.12. (:issue:`5984`) + +- Minimum versions increased for these dependencies: + + - lxml_: 4.3.0 → 4.4.1 + + - cryptography_: 3.4.6 → 36.0.0 + +- ``pkg_resources`` is no longer used. (:issue:`5956`, :issue:`5958`) + +- boto3_ is now recommended for exporting to S3 instead of botocore_. + (:issue:`5833`). + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The value of the :setting:`FEED_STORE_EMPTY` is now ``True`` instead of + ``False``. In earlier Scrapy versions empty files were created even when + this setting was ``False`` (which was a bug that is now fixed), so the new + default should keep the old behavior. (:issue:`872`, :issue:`5847`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, + returning ``None`` or modifying the ``params`` input parameter, deprecated + in Scrapy 2.6, is no longer supported. (:issue:`5994`, :issue:`5996`) + +- The ``scrapy.utils.reqser`` module, deprecated in Scrapy 2.6, is removed. + (:issue:`5994`, :issue:`5996`) + +- The ``scrapy.squeues`` classes ``PickleFifoDiskQueueNonRequest``, + ``PickleLifoDiskQueueNonRequest``, ``MarshalFifoDiskQueueNonRequest``, + and ``MarshalLifoDiskQueueNonRequest``, deprecated in + Scrapy 2.6, are removed. (:issue:`5994`, :issue:`5996`) + +- The property ``open_spiders`` and the methods ``has_capacity`` and + ``schedule`` of :class:`scrapy.core.engine.ExecutionEngine`, + deprecated in Scrapy 2.6, are removed. (:issue:`5994`, :issue:`5998`) + +- Passing a ``spider`` argument to the + :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`, + :meth:`~scrapy.core.engine.ExecutionEngine.crawl` and + :meth:`~scrapy.core.engine.ExecutionEngine.download` methods of + :class:`scrapy.core.engine.ExecutionEngine`, deprecated in Scrapy 2.6, is + no longer supported. (:issue:`5994`, :issue:`5998`) + +Deprecations +~~~~~~~~~~~~ + +- :class:`scrapy.utils.datatypes.CaselessDict` is deprecated, use + :class:`scrapy.utils.datatypes.CaseInsensitiveDict` instead. + (:issue:`5146`) + +- Passing the ``custom`` argument to + :func:`scrapy.utils.conf.build_component_list` is deprecated, it was used + in the past to merge ``FOO`` and ``FOO_BASE`` setting values but now Scrapy + uses :func:`scrapy.settings.BaseSettings.getwithbase` to do the same. + Code that uses this argument and cannot be switched to ``getwithbase()`` + can be switched to merging the values explicitly. (:issue:`5726`, + :issue:`5923`) + +New features +~~~~~~~~~~~~ + +- Added support for :ref:`Scrapy add-ons `. (:issue:`5950`) + +- Added the :setting:`RETRY_EXCEPTIONS` setting that configures which + exceptions will be retried by + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`. + (:issue:`2701`, :issue:`5929`) + +- Added the possiiblity to close the spider if no items were produced in the + specified time, configured by :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`. + (:issue:`5979`) + +- Added support for the :setting:`AWS_REGION_NAME` setting to feed exports. + (:issue:`5980`) + +- Added support for using :class:`pathlib.Path` objects that refer to + absolute Windows paths in the :setting:`FEEDS` setting. (:issue:`5939`) + +Bug fixes +~~~~~~~~~ + +- Fixed creating empty feeds even with ``FEED_STORE_EMPTY=False``. + (:issue:`872`, :issue:`5847`) + +- Fixed using absolute Windows paths when specifying output files. + (:issue:`5969`, :issue:`5971`) + +- Fixed problems with uploading large files to S3 by switching to multipart + uploads (requires boto3_). (:issue:`960`, :issue:`5735`, :issue:`5833`) + +- Fixed the JSON exporter writing extra commas when some exceptions occur. + (:issue:`3090`, :issue:`5952`) + +- Fixed the "read of closed file" error in the CSV exporter. (:issue:`5043`, + :issue:`5705`) + +- Fixed an error when a component added by the class object throws + :exc:`~scrapy.exceptions.NotConfigured` with a message. (:issue:`5950`, + :issue:`5992`) + +- Added the missing :meth:`scrapy.settings.BaseSettings.pop` method. + (:issue:`5959`, :issue:`5960`, :issue:`5963`) + +- Added :class:`~scrapy.utils.datatypes.CaseInsensitiveDict` as a replacement + for :class:`~scrapy.utils.datatypes.CaselessDict` that fixes some API + inconsistencies. (:issue:`5146`) + +Documentation +~~~~~~~~~~~~~ + +- Documented :meth:`scrapy.Spider.update_settings`. (:issue:`5745`, + :issue:`5846`) + +- Documented possible problems with early Twisted reactor installation and + their solutions. (:issue:`5981`, :issue:`6000`) + +- Added examples of making additional requests in callbacks. (:issue:`5927`) + +- Improved the feed export docs. (:issue:`5579`, :issue:`5931`) + +- Clarified the docs about request objects on redirection. (:issue:`5707`, + :issue:`5937`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added support for running tests against the installed Scrapy version. + (:issue:`4914`, :issue:`5949`) + +- Extended typing hints. (:issue:`5925`, :issue:`5977`) + +- Fixed the ``test_utils_asyncio.AsyncioTest.test_set_asyncio_event_loop`` + test. (:issue:`5951`) + +- Fixed the ``test_feedexport.BatchDeliveriesTest.test_batch_path_differ`` + test on Windows. (:issue:`5847`) + +- Enabled CI runs for Python 3.11 on Windows. (:issue:`5999`) + +- Simplified skipping tests that depend on ``uvloop``. (:issue:`5984`) + +- Fixed the ``extra-deps-pinned`` tox env. (:issue:`5948`) + +- Implemented cleanups. (:issue:`5965`, :issue:`5986`) + .. _release-2.9.0: Scrapy 2.9.0 (2023-05-08) @@ -5748,6 +5918,7 @@ First release of Scrapy. .. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 +.. _boto3: https://github.com/boto/boto3 .. _botocore: https://github.com/boto/botocore .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ From 7fe4c0c9f7e58d8099c6ec47560c2ffd181bebb8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 4 Aug 2023 16:56:39 +0400 Subject: [PATCH 1092/2083] Update docs/news.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index ae51d05dc..80ad0fa45 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,7 +12,7 @@ Highlights: - Added Python 3.12 support, dropped Python 3.7 support. -- The add-ons framework that simplifies configuring 3rd-party components that +- The new add-ons framework simplifies configuring 3rd-party components that support it. - Exceptions to retry can now be configured. From 022ef0f86b2a5b09cd1044d6a381f025e821ccf3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 4 Aug 2023 17:05:09 +0400 Subject: [PATCH 1093/2083] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/news.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 80ad0fa45..940093e58 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -34,16 +34,17 @@ Modified requirements - ``pkg_resources`` is no longer used. (:issue:`5956`, :issue:`5958`) -- boto3_ is now recommended for exporting to S3 instead of botocore_. +- boto3_ is now recommended instead of botocore_ for exporting to S3. (:issue:`5833`). Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- The value of the :setting:`FEED_STORE_EMPTY` is now ``True`` instead of - ``False``. In earlier Scrapy versions empty files were created even when - this setting was ``False`` (which was a bug that is now fixed), so the new - default should keep the old behavior. (:issue:`872`, :issue:`5847`) +- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` + instead of ``False``. In earlier Scrapy versions empty files were created + even when this setting was ``False`` (which was a bug that is now fixed), + so the new default should keep the old behavior. (:issue:`872`, + :issue:`5847`) Deprecation removals ~~~~~~~~~~~~~~~~~~~~ From 88327c7c58928b5e1e07921ca055c8579c197234 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 4 Aug 2023 17:23:30 +0400 Subject: [PATCH 1094/2083] =?UTF-8?q?Bump=20version:=202.9.0=20=E2=86=92?= =?UTF-8?q?=202.10.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a00b7cfb3..53e873427 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.9.0 +current_version = 2.10.0 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index 940093e58..c55c0b222 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.10.0: -Scrapy 2.10.0 (YYYY-MM-DD) +Scrapy 2.10.0 (2023-08-04) -------------------------- Highlights: diff --git a/scrapy/VERSION b/scrapy/VERSION index c8e38b614..10c2c0c3d 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.9.0 +2.10.0 From d31829b72f6238a92a42c0990953d5056e8f5778 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Jul 2023 23:13:15 +0400 Subject: [PATCH 1095/2083] More typing for scrapy/utils. --- scrapy/exceptions.py | 3 +- scrapy/extensions/feedexport.py | 25 +++++++++++------ scrapy/utils/conf.py | 49 ++++++++++++++++++++++----------- scrapy/utils/ftp.py | 15 ++++++++-- scrapy/utils/job.py | 2 +- scrapy/utils/ossignal.py | 3 +- scrapy/utils/project.py | 8 +++--- scrapy/utils/sitemap.py | 14 ++++++---- scrapy/utils/ssl.py | 7 ++--- scrapy/utils/trackref.py | 19 +++++++++---- 10 files changed, 93 insertions(+), 52 deletions(-) diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index fedd02805..6e83e4a00 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -4,6 +4,7 @@ Scrapy core exceptions These exceptions are documented in docs/topics/exceptions.rst. Please don't add new exceptions here without documenting them there. """ +from typing import Any # Internal @@ -77,7 +78,7 @@ class NotSupported(Exception): class UsageError(Exception): """To indicate a command-line usage error""" - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any): self.print_help = kw.pop("print_help", True) super().__init__(*a, **kw) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c8022ff57..2bbcaf3ad 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,7 +11,7 @@ import warnings from datetime import datetime from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, List, Optional, Tuple, Union +from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads @@ -282,15 +282,22 @@ class GCSFeedStorage(BlockingFeedStorage): class FTPFeedStorage(BlockingFeedStorage): - def __init__(self, uri, use_active_mode=False, *, feed_options=None): + def __init__( + self, + uri: str, + use_active_mode: bool = False, + *, + feed_options: Optional[Dict[str, Any]] = None, + ): u = urlparse(uri) - self.host = u.hostname - self.port = int(u.port or "21") - self.username = u.username - self.password = unquote(u.password or "") - self.path = u.path - self.use_active_mode = use_active_mode - self.overwrite = not feed_options or feed_options.get("overwrite", True) + assert u.hostname + self.host: str = u.hostname + self.port: int = int(u.port or "21") + self.username: str = u.username or "" + self.password: str = unquote(u.password or "") + self.path: str = u.path + self.use_active_mode: bool = use_active_mode + self.overwrite: bool = not feed_options or feed_options.get("overwrite", True) @classmethod def from_crawler(cls, crawler, uri, *, feed_options=None): diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 1889f7571..641dfa4a2 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -5,7 +5,18 @@ import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import ( + Any, + Callable, + Collection, + Dict, + Iterable, + List, + Mapping, + MutableMapping, + Optional, + Union, +) from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings @@ -13,17 +24,21 @@ from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values -def build_component_list(compdict, custom=None, convert=update_classpath): +def build_component_list( + compdict: MutableMapping[Any, Any], + custom: Any = None, + convert: Callable[[Any], Any] = update_classpath, +) -> List[Any]: """Compose a component list from a { class: order } dictionary.""" - def _check_components(complist): + def _check_components(complist: Collection[Any]) -> None: if len({convert(c) for c in complist}) != len(complist): raise ValueError( f"Some paths in {complist!r} convert to the same object, " "please update your settings" ) - def _map_keys(compdict): + def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, Dict[Any, Any]]: if isinstance(compdict, BaseSettings): compbs = BaseSettings() for k, v in compdict.items(): @@ -41,7 +56,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): _check_components(compdict) return {convert(k): v for k, v in compdict.items()} - def _validate_values(compdict): + def _validate_values(compdict: Mapping[Any, Any]) -> None: """Fail if a value in the components dict is not a real number or None.""" for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): @@ -60,7 +75,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): ) if isinstance(custom, (list, tuple)): _check_components(custom) - return type(custom)(convert(c) for c in custom) + return type(custom)(convert(c) for c in custom) # type: ignore[return-value] compdict.update(custom) _validate_values(compdict) @@ -68,7 +83,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] -def arglist_to_dict(arglist): +def arglist_to_dict(arglist: List[str]) -> Dict[str, str]: """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a dict """ @@ -91,7 +106,7 @@ def closest_scrapy_cfg( return closest_scrapy_cfg(path.parent, path) -def init_env(project="default", set_syspath=True): +def init_env(project: str = "default", set_syspath: bool = True) -> None: """Initialize environment to use command-line tool from inside a project dir. This sets the Scrapy settings module and modifies the Python path to be able to locate the project module. @@ -106,7 +121,7 @@ def init_env(project="default", set_syspath=True): sys.path.append(projdir) -def get_config(use_closest=True): +def get_config(use_closest: bool = True) -> ConfigParser: """Get Scrapy config file as a ConfigParser""" sources = get_sources(use_closest) cfg = ConfigParser() @@ -114,7 +129,7 @@ def get_config(use_closest=True): return cfg -def get_sources(use_closest=True) -> List[str]: +def get_sources(use_closest: bool = True) -> List[str]: xdg_config_home = ( os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser() ) @@ -129,7 +144,9 @@ def get_sources(use_closest=True) -> List[str]: return sources -def feed_complete_default_values_from_settings(feed, settings): +def feed_complete_default_values_from_settings( + feed: Dict[str, Any], settings: BaseSettings +) -> Dict[str, Any]: out = feed.copy() out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT")) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) @@ -145,21 +162,21 @@ def feed_complete_default_values_from_settings(feed, settings): def feed_process_params_from_cli( - settings, + settings: BaseSettings, output: List[str], - output_format=None, + output_format: Optional[str] = None, overwrite_output: Optional[List[str]] = None, -): +) -> Dict[str, Dict[str, Any]]: """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary suitable to be used as the FEEDS setting. """ - valid_output_formats = without_none_values( + valid_output_formats: Iterable[str] = without_none_values( settings.getwithbase("FEED_EXPORTERS") ).keys() - def check_valid_format(output_format): + def check_valid_format(output_format: str) -> None: if output_format not in valid_output_formats: raise UsageError( f"Unrecognized output format '{output_format}'. " diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 6bf6e9195..c77681a53 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,9 +1,10 @@ import posixpath from ftplib import FTP, error_perm from posixpath import dirname +from typing import IO -def ftp_makedirs_cwd(ftp, path, first_call=True): +def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None: """Set the current directory of the FTP connection given in the ``ftp`` argument (as a ftplib.FTP object), creating all parent directories if they don't exist. The ftplib.FTP object must be already connected and logged in. @@ -18,8 +19,16 @@ def ftp_makedirs_cwd(ftp, path, first_call=True): def ftp_store_file( - *, path, file, host, port, username, password, use_active_mode=False, overwrite=True -): + *, + path: str, + file: IO, + host: str, + port: int, + username: str, + password: str, + use_active_mode: bool = False, + overwrite: bool = True, +) -> None: """Opens a FTP connection with passed credentials,sets current directory to the directory extracted from given path, then uploads the file to server """ diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 858affc03..c49f7d758 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -5,7 +5,7 @@ from scrapy.settings import BaseSettings def job_dir(settings: BaseSettings) -> Optional[str]: - path = settings["JOBDIR"] + path: str = settings["JOBDIR"] if path and not Path(path).exists(): Path(path).mkdir(parents=True) return path diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 7646264a8..f835a2221 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,4 +1,5 @@ import signal +from typing import Callable signal_names = {} for signame in dir(signal): @@ -8,7 +9,7 @@ for signame in dir(signal): signal_names[signum] = signame -def install_shutdown_handlers(function, override_sigint=True): +def install_shutdown_handlers(function: Callable, override_sigint: bool = True) -> None: """Install the given function as a signal handler for all common shutdown signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the SIGINT handler won't be install if there is already a handler in place diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 652b74759..a2c224b90 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -11,7 +11,7 @@ ENVVAR = "SCRAPY_SETTINGS_MODULE" DATADIR_CFG_SECTION = "datadir" -def inside_project(): +def inside_project() -> bool: scrapy_module = os.environ.get(ENVVAR) if scrapy_module: try: @@ -25,7 +25,7 @@ def inside_project(): return bool(closest_scrapy_cfg()) -def project_data_dir(project="default") -> str: +def project_data_dir(project: str = "default") -> str: """Return the current project data dir, creating it if it doesn't exist""" if not inside_project(): raise NotConfigured("Not inside a project") @@ -44,7 +44,7 @@ def project_data_dir(project="default") -> str: return str(d) -def data_path(path: str, createdir=False) -> str: +def data_path(path: str, createdir: bool = False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. @@ -60,7 +60,7 @@ def data_path(path: str, createdir=False) -> str: return str(path_obj) -def get_project_settings(): +def get_project_settings() -> Settings: if ENVVAR not in os.environ: project = os.environ.get("SCRAPY_PROJECT", "default") init_env(project) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 2622c2775..3d2ecc9a7 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,7 +4,7 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ - +from typing import Any, Dict, Generator, Iterator, Optional from urllib.parse import urljoin import lxml.etree @@ -14,7 +14,7 @@ class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" - def __init__(self, xmltext): + def __init__(self, xmltext: str): xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) @@ -22,9 +22,9 @@ class Sitemap: rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt - def __iter__(self): + def __iter__(self) -> Iterator[Dict[str, Any]]: for elem in self._root.getchildren(): - d = {} + d: Dict[str, Any] = {} for el in elem.getchildren(): tag = el.tag name = tag.split("}", 1)[1] if "}" in tag else tag @@ -39,11 +39,13 @@ class Sitemap: yield d -def sitemap_urls_from_robots(robots_text, base_url=None): +def sitemap_urls_from_robots( + robots_text: str, base_url: Optional[str] = None +) -> Generator[str, Any, None]: """Return an iterator over all sitemap urls contained in the given robots.txt file """ for line in robots_text.splitlines(): if line.lstrip().lower().startswith("sitemap:"): url = line.split(":", 1)[1].strip() - yield urljoin(base_url, url) + yield urljoin(base_url or "", url) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 03ae4ba9e..d520ef809 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, cast +from typing import Any, Optional import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL @@ -58,9 +58,6 @@ def get_temp_key_info(ssl_object: Any) -> Optional[str]: def get_openssl_version() -> str: - # https://github.com/python/typeshed/issues/10024 - system_openssl_bytes = cast( - bytes, OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) - ) + system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) system_openssl = system_openssl_bytes.decode("ascii", errors="replace") return f"{OpenSSL.version.__version__} ({system_openssl})" diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 01b980c93..9ff9a273f 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -12,9 +12,14 @@ alias to object in that case). from collections import defaultdict from operator import itemgetter from time import time -from typing import DefaultDict +from typing import TYPE_CHECKING, Any, DefaultDict, Iterable from weakref import WeakKeyDictionary +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + NoneType = type(None) live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) @@ -24,13 +29,14 @@ class object_ref: __slots__ = () - def __new__(cls, *args, **kwargs): + def __new__(cls, *args: Any, **kwargs: Any) -> "Self": obj = object.__new__(cls) live_refs[cls][obj] = time() return obj -def format_live_refs(ignore=NoneType): +# using Any as it's hard to type type(None) +def format_live_refs(ignore: Any = NoneType) -> str: """Return a tabular representation of tracked objects""" s = "Live References\n\n" now = time() @@ -44,12 +50,12 @@ def format_live_refs(ignore=NoneType): return s -def print_live_refs(*a, **kw): +def print_live_refs(*a: Any, **kw: Any) -> None: """Print tracked objects""" print(format_live_refs(*a, **kw)) -def get_oldest(class_name): +def get_oldest(class_name: str) -> Any: """Get the oldest object for a specific class name""" for cls, wdict in live_refs.items(): if cls.__name__ == class_name: @@ -58,8 +64,9 @@ def get_oldest(class_name): return min(wdict.items(), key=itemgetter(1))[0] -def iter_all(class_name): +def iter_all(class_name: str) -> Iterable[Any]: """Iterate over all objects of the same class by its class name""" for cls, wdict in live_refs.items(): if cls.__name__ == class_name: return wdict.keys() + return [] From d015329d759dda72586b856ee92d787dc730f06e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 31 Jul 2023 01:54:28 +0400 Subject: [PATCH 1096/2083] Add more typing for scrapy/utils/log.py. --- scrapy/utils/log.py | 60 +++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2ce4725f4..2013bfc43 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,8 +1,11 @@ +from __future__ import annotations + import logging import sys import warnings from logging.config import dictConfig -from typing import Tuple +from types import TracebackType +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -12,13 +15,25 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.versions import scrapy_components_versions +if TYPE_CHECKING: + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) -def failure_to_exc_info(failure: Failure): +def failure_to_exc_info( + failure: Failure, +) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]: """Extract exc_info from Failure instances""" if isinstance(failure, Failure): - return (failure.type, failure.value, failure.getTracebackObject()) + assert failure.type + assert failure.value + return ( + failure.type, + failure.value, + cast(Optional[TracebackType], failure.getTracebackObject()), + ) + return None class TopLevelFormatter(logging.Filter): @@ -33,10 +48,10 @@ class TopLevelFormatter(logging.Filter): ``loggers`` list where it should act. """ - def __init__(self, loggers=None): - self.loggers = loggers or [] + def __init__(self, loggers: Optional[List[str]] = None): + self.loggers: List[str] = loggers or [] - def filter(self, record): + def filter(self, record: logging.LogRecord) -> bool: if any(record.name.startswith(logger + ".") for logger in self.loggers): record.name = record.name.split(".", 1)[0] return True @@ -62,7 +77,9 @@ DEFAULT_LOGGING = { } -def configure_logging(settings=None, install_root_handler=True): +def configure_logging( + settings: Union[Settings, dict, None] = None, install_root_handler: bool = True +) -> None: """ Initialize logging defaults for Scrapy. @@ -99,13 +116,13 @@ def configure_logging(settings=None, install_root_handler=True): settings = Settings(settings) if settings.getbool("LOG_STDOUT"): - sys.stdout = StreamLogger(logging.getLogger("stdout")) + sys.stdout = StreamLogger(logging.getLogger("stdout")) # type: ignore[assignment] if install_root_handler: install_scrapy_root_handler(settings) -def install_scrapy_root_handler(settings): +def install_scrapy_root_handler(settings: Settings) -> None: global _scrapy_root_handler if ( @@ -118,16 +135,17 @@ def install_scrapy_root_handler(settings): logging.root.addHandler(_scrapy_root_handler) -def get_scrapy_root_handler(): +def get_scrapy_root_handler() -> Optional[logging.Handler]: return _scrapy_root_handler -_scrapy_root_handler = None +_scrapy_root_handler: Optional[logging.Handler] = None -def _get_handler(settings): +def _get_handler(settings: Settings) -> logging.Handler: """Return a log handler object according to settings""" filename = settings.get("LOG_FILE") + handler: logging.Handler if filename: mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w" encoding = settings.get("LOG_ENCODING") @@ -181,16 +199,16 @@ class StreamLogger: https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ """ - def __init__(self, logger, log_level=logging.INFO): - self.logger = logger - self.log_level = log_level - self.linebuf = "" + def __init__(self, logger: logging.Logger, log_level: int = logging.INFO): + self.logger: logging.Logger = logger + self.log_level: int = log_level + self.linebuf: str = "" - def write(self, buf): + def write(self, buf: str) -> None: for line in buf.rstrip().splitlines(): self.logger.log(self.log_level, line.rstrip()) - def flush(self): + def flush(self) -> None: for h in self.logger.handlers: h.flush() @@ -198,11 +216,11 @@ class StreamLogger: class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" - def __init__(self, crawler, *args, **kwargs): + def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) - self.crawler = crawler + self.crawler: Crawler = crawler - def emit(self, record): + def emit(self, record: logging.LogRecord) -> None: sname = f"log_count/{record.levelname}" self.crawler.stats.inc_value(sname) From c43798cb9bee99ccf96047f3dfcc6debb65973d4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 22:15:37 +0400 Subject: [PATCH 1097/2083] More typing for scrapy/utils/defer.py and scrapy/utils/spider.py. --- scrapy/spiderloader.py | 9 +++++---- scrapy/spiders/__init__.py | 2 +- scrapy/utils/defer.py | 24 ++++++++++++++++++++---- scrapy/utils/spider.py | 31 +++++++++++++++++++++++++++---- 4 files changed, 53 insertions(+), 13 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index ea5a26e77..13d6f9f87 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,11 +1,12 @@ import traceback import warnings from collections import defaultdict +from types import ModuleType from typing import DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer -from scrapy import Spider +from scrapy import Request, Spider from scrapy.interfaces import ISpiderLoader from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules @@ -45,12 +46,12 @@ class SpiderLoader: category=UserWarning, ) - def _load_spiders(self, module): + def _load_spiders(self, module: ModuleType) -> None: for spcls in iter_spider_classes(module): self._found[spcls.name].append((module.__name__, spcls.__name__)) self._spiders[spcls.name] = spcls - def _load_all_spiders(self): + def _load_all_spiders(self) -> None: for name in self.spider_modules: try: for module in walk_modules(name): @@ -81,7 +82,7 @@ class SpiderLoader: except KeyError: raise KeyError(f"Spider not found: {spider_name}") - def find_by_request(self, request): + def find_by_request(self, request: Request) -> List[str]: """ Return the list of spider names that can handle the given request. """ diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 3502f8b27..388439f4f 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -82,7 +82,7 @@ class Spider(object_ref): settings.setdict(cls.custom_settings or {}, priority="spider") @classmethod - def handles_request(cls, request): + def handles_request(cls, request: Request) -> bool: return url_is_from_spider(request.url, cls) @staticmethod diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 03f026ce9..bf3c5ef5b 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -5,11 +5,13 @@ import asyncio import inspect from asyncio import Future from functools import wraps +from types import CoroutineType from typing import ( Any, AsyncGenerator, AsyncIterable, AsyncIterator, + Awaitable, Callable, Coroutine, Dict, @@ -19,8 +21,10 @@ from typing import ( List, Optional, Tuple, + TypeVar, Union, cast, + overload, ) from twisted.internet import defer @@ -186,9 +190,7 @@ class _AsyncCooperatorAdapter(Iterator): def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. - self.anext_deferred = cast( - Deferred, deferred_from_coro(self.aiterator.__anext__()) - ) + self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) def __next__(self) -> Deferred: @@ -297,7 +299,21 @@ async def aiter_errback( errback(failure.Failure(), *a, **kw) -def deferred_from_coro(o: Any) -> Any: +_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future]) +_T = TypeVar("_T") + + +@overload +def deferred_from_coro(o: _CT) -> Deferred: + ... + + +@overload +def deferred_from_coro(o: _T) -> _T: + ... + + +def deferred_from_coro(o: _T) -> Union[Deferred, _T]: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, Deferred): return o diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 86449eeb2..3228eda49 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -1,14 +1,33 @@ +from __future__ import annotations + import inspect import logging +from types import ModuleType +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterable, + Generator, + Iterable, + Optional, + Type, + Union, +) +from twisted.internet.defer import Deferred + +from scrapy import Request from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter +if TYPE_CHECKING: + from scrapy.spiderloader import SpiderLoader + logger = logging.getLogger(__name__) -def iterate_spider_output(result): +def iterate_spider_output(result: Any) -> Union[Iterable, AsyncIterable, Deferred]: if inspect.isasyncgen(result): return result if inspect.iscoroutine(result): @@ -18,7 +37,7 @@ def iterate_spider_output(result): return arg_to_iter(deferred_from_coro(result)) -def iter_spider_classes(module): +def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ @@ -37,8 +56,12 @@ def iter_spider_classes(module): def spidercls_for_request( - spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False -): + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Optional[Type[Spider]] = None, + log_none: bool = False, + log_multiple: bool = False, +) -> Optional[Type[Spider]]: """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using From d1f87e4f088c0757dd833d2a0841be0b830deb57 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 23:11:15 +0400 Subject: [PATCH 1098/2083] More typing for scrapy/utils/iterators.py. --- scrapy/http/response/text.py | 6 +- scrapy/selector/unified.py | 18 ++++-- scrapy/utils/iterators.py | 106 ++++++++++++++++++++++++++-------- tests/test_utils_iterators.py | 7 ++- 4 files changed, 103 insertions(+), 34 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 5289f014a..f228e11c1 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst import json from contextlib import suppress -from typing import Generator, Tuple +from typing import Generator, Optional, Tuple from urllib.parse import urljoin import parsel @@ -37,7 +37,7 @@ class TextResponse(Response): def __init__(self, *args, **kwargs): self._encoding = kwargs.pop("encoding", None) self._cached_benc = None - self._cached_ubody = None + self._cached_ubody: Optional[str] = None self._cached_selector = None super().__init__(*args, **kwargs) @@ -82,7 +82,7 @@ class TextResponse(Response): return self._cached_decoded_json @property - def text(self): + def text(self) -> str: """Body as unicode""" # access self.encoding before _cached_ubody to make sure # _body_inferred_encoding is called diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index caff79e9c..863fb6032 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -1,10 +1,11 @@ """ XPath selectors based on lxml """ +from typing import Any, Optional, Type, Union from parsel import Selector as _ParselSelector -from scrapy.http import HtmlResponse, XmlResponse +from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref @@ -13,14 +14,14 @@ __all__ = ["Selector", "SelectorList"] _NOT_SET = object() -def _st(response, st): +def _st(response: Optional[TextResponse], st: Optional[str]) -> str: if st is None: return "xml" if isinstance(response, XmlResponse) else "html" return st -def _response_from_text(text, st): - rt = XmlResponse if st == "xml" else HtmlResponse +def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse: + rt: Type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) @@ -65,7 +66,14 @@ class Selector(_ParselSelector, object_ref): __slots__ = ["response"] selectorlist_cls = SelectorList - def __init__(self, response=None, text=None, type=None, root=_NOT_SET, **kwargs): + def __init__( + self, + response: Optional[TextResponse] = None, + text: Optional[str] = None, + type: Optional[str] = None, + root: Optional[Any] = _NOT_SET, + **kwargs: Any, + ): if response is not None and text is not None: raise ValueError( f"{self.__class__.__name__}.__init__() received " diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 170055d5e..58850b843 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,16 +1,37 @@ +from __future__ import annotations + import csv import logging import re from io import StringIO +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + Iterable, + List, + Literal, + Optional, + Union, + cast, + overload, +) from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode +if TYPE_CHECKING: + from lxml._types import SupportsReadClose + logger = logging.getLogger(__name__) -def xmliter(obj, nodename): +def xmliter( + obj: Union[Response, str, bytes], nodename: str +) -> Generator[Selector, Any, None]: """Return a iterator of Selector's over all nodes of a XML document, given the name of the node to iterate. Useful for parsing XML feeds. @@ -27,20 +48,22 @@ def xmliter(obj, nodename): NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) text = _body_or_str(obj) - document_header = re.search(DOCUMENT_HEADER_RE, text) - document_header = document_header.group().strip() if document_header else "" + document_header_match = re.search(DOCUMENT_HEADER_RE, text) + document_header = ( + document_header_match.group().strip() if document_header_match else "" + ) header_end_idx = re_rsearch(HEADER_END_RE, text) header_end = text[header_end_idx[1] :].strip() if header_end_idx else "" - namespaces = {} + namespaces: Dict[str, str] = {} if header_end: for tagname in reversed(re.findall(END_TAG_RE, header_end)): + assert header_end_idx tag = re.search( rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S ) if tag: - namespaces.update( - reversed(x) for x in re.findall(NAMESPACE_RE, tag.group()) - ) + for x in re.findall(NAMESPACE_RE, tag.group()): + namespaces[x[1]] = x[0] r = re.compile(rf"<{nodename_patt}[\s>].*?", re.DOTALL) for match in r.finditer(text): @@ -54,12 +77,19 @@ def xmliter(obj, nodename): yield Selector(text=nodetext, type="xml") -def xmliter_lxml(obj, nodename, namespace=None, prefix="x"): +def xmliter_lxml( + obj: Union[TextResponse, str, bytes], + nodename: str, + namespace: Optional[str] = None, + prefix: str = "x", +) -> Generator[Selector, Any, None]: from lxml import etree reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename - iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) + iterable = etree.iterparse( + cast(SupportsReadClose[bytes], reader), tag=tag, encoding=reader.encoding + ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: nodetext = etree.tostring(node, encoding="unicode") @@ -71,30 +101,39 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix="x"): class _StreamReader: - def __init__(self, obj): - self._ptr = 0 - if isinstance(obj, Response): + def __init__(self, obj: Union[TextResponse, str, bytes]): + self._ptr: int = 0 + self._text: Union[str, bytes] + if isinstance(obj, TextResponse): self._text, self.encoding = obj.body, obj.encoding else: self._text, self.encoding = obj, "utf-8" - self._is_unicode = isinstance(self._text, str) + self._is_unicode: bool = isinstance(self._text, str) - def read(self, n=65535): - self.read = self._read_unicode if self._is_unicode else self._read_string + def read(self, n: int = 65535) -> bytes: + self.read: Callable[[int], bytes] = ( # type: ignore[method-assign] + self._read_unicode if self._is_unicode else self._read_string + ) return self.read(n).lstrip() - def _read_string(self, n=65535): + def _read_string(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n self._ptr = e - return self._text[s:e] + return cast(bytes, self._text)[s:e] - def _read_unicode(self, n=65535): + def _read_unicode(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n self._ptr = e - return self._text[s:e].encode("utf-8") + return cast(str, self._text)[s:e].encode("utf-8") -def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): +def csviter( + obj: Union[Response, str, bytes], + delimiter: Optional[str] = None, + headers: Optional[List[str]] = None, + encoding: Optional[str] = None, + quotechar: Optional[str] = None, +) -> Generator[Dict[str, str], Any, None]: """Returns an iterator of dictionaries from the given csv object obj can be: @@ -112,12 +151,12 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8" - def row_to_unicode(row_): + def row_to_unicode(row_: Iterable) -> List[str]: return [to_unicode(field, encoding) for field in row_] lines = StringIO(_body_or_str(obj, unicode=True)) - kwargs = {} + kwargs: Dict[str, Any] = {} if delimiter: kwargs["delimiter"] = delimiter if quotechar: @@ -147,7 +186,24 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): yield dict(zip(headers, row)) -def _body_or_str(obj, unicode=True): +@overload +def _body_or_str(obj: Union[Response, str, bytes]) -> str: + ... + + +@overload +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: + ... + + +@overload +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes: + ... + + +def _body_or_str( + obj: Union[Response, str, bytes], unicode: bool = True +) -> Union[str, bytes]: expected_types = (Response, str, bytes) if not isinstance(obj, expected_types): expected_types_str = " or ".join(t.__name__ for t in expected_types) @@ -156,10 +212,10 @@ def _body_or_str(obj, unicode=True): ) if isinstance(obj, Response): if not unicode: - return obj.body + return cast(bytes, obj.body) if isinstance(obj, TextResponse): return obj.text - return obj.body.decode("utf-8") + return cast(bytes, obj.body).decode("utf-8") if isinstance(obj, str): return obj if unicode else obj.encode("utf-8") return obj.decode("utf-8") if unicode else obj diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 3598fa0bb..5dfd7e7ac 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,13 +1,18 @@ +from typing import Callable, Iterable, Union + from pytest import mark from twisted.trial import unittest +from scrapy import Selector from scrapy.http import Response, TextResponse, XmlResponse from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata class XmliterTestCase(unittest.TestCase): - xmliter = staticmethod(xmliter) + xmliter: Callable[ + [Union[TextResponse, str, bytes], str], Iterable[Selector] + ] = staticmethod(xmliter) def test_xmliter(self): body = b""" From 9fe662d856a6b2496379585143aa2f4d023039f8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 23:42:59 +0400 Subject: [PATCH 1099/2083] Add typing for scrapy/utils/testproc.py. --- scrapy/utils/testproc.py | 45 ++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 5f9bdef37..5f7a7db14 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -1,7 +1,13 @@ +from __future__ import annotations + import os import sys +from typing import Iterable, Optional, Tuple, cast -from twisted.internet import defer, protocol +from twisted.internet.defer import Deferred +from twisted.internet.error import ProcessTerminated +from twisted.internet.protocol import ProcessProtocol +from twisted.python.failure import Failure class ProcessTest: @@ -9,7 +15,12 @@ class ProcessTest: prefix = [sys.executable, "-m", "scrapy.cmdline"] cwd = os.getcwd() # trial chdirs to temp dir - def execute(self, args, check_code=True, settings=None): + def execute( + self, + args: Iterable[str], + check_code: bool = True, + settings: Optional[str] = None, + ) -> Deferred: from twisted.internet import reactor env = os.environ.copy() @@ -21,29 +32,31 @@ class ProcessTest: reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) return pp.deferred - def _process_finished(self, pp, cmd, check_code): + def _process_finished( + self, pp: TestProcessProtocol, cmd: str, check_code: bool + ) -> Tuple[int, bytes, bytes]: if pp.exitcode and check_code: msg = f"process {cmd} exit with code {pp.exitcode}" - msg += f"\n>>> stdout <<<\n{pp.out}" + msg += f"\n>>> stdout <<<\n{pp.out.decode()}" msg += "\n" - msg += f"\n>>> stderr <<<\n{pp.err}" + msg += f"\n>>> stderr <<<\n{pp.err.decode()}" raise RuntimeError(msg) - return pp.exitcode, pp.out, pp.err + return cast(int, pp.exitcode), pp.out, pp.err -class TestProcessProtocol(protocol.ProcessProtocol): - def __init__(self): - self.deferred = defer.Deferred() - self.out = b"" - self.err = b"" - self.exitcode = None +class TestProcessProtocol(ProcessProtocol): + def __init__(self) -> None: + self.deferred: Deferred = Deferred() + self.out: bytes = b"" + self.err: bytes = b"" + self.exitcode: Optional[int] = None - def outReceived(self, data): + def outReceived(self, data: bytes) -> None: self.out += data - def errReceived(self, data): + def errReceived(self, data: bytes) -> None: self.err += data - def processEnded(self, status): - self.exitcode = status.value.exitCode + def processEnded(self, status: Failure) -> None: + self.exitcode = cast(ProcessTerminated, status.value).exitCode self.deferred.callback(self) From 66bad1150cf0a72b39d1af8a63d7b4eb7c1f42fe Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 3 Aug 2023 00:08:28 +0400 Subject: [PATCH 1100/2083] Add more typing for scrapy/utils/signal.py. --- scrapy/utils/signal.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 9e7ddd827..21a12a19e 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -23,7 +23,10 @@ logger = logging.getLogger(__name__) def send_catch_log( - signal=Any, sender=Anonymous, *arguments, **named + signal: TypingAny = Any, + sender: TypingAny = Anonymous, + *arguments: TypingAny, + **named: TypingAny ) -> List[Tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. @@ -65,13 +68,18 @@ def send_catch_log( return responses -def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): +def send_catch_log_deferred( + signal: TypingAny = Any, + sender: TypingAny = Anonymous, + *arguments: TypingAny, + **named: TypingAny +) -> Deferred: """Like send_catch_log but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were fired. """ - def logerror(failure, recv): + def logerror(failure: Failure, recv: Any) -> Failure: if dont_log is None or not isinstance(failure.value, dont_log): logger.error( "Error caught on signal handler: %(receiver)s", @@ -96,7 +104,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): return d -def disconnect_all(signal=Any, sender=Any): +def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: """Disconnect all signal handlers. Useful for cleaning up after running tests """ From 518e56046e45d38c505db9e2bc00677ad08ac58c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 6 Aug 2023 17:28:34 +0400 Subject: [PATCH 1101/2083] Check for async callbacks in contracts. --- scrapy/contracts/__init__.py | 13 ++++++++++--- tests/test_contracts.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 86098edca..1ec2a0234 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -2,7 +2,8 @@ import re import sys from functools import wraps from inspect import getmembers -from typing import Dict +from types import CoroutineType +from typing import AsyncGenerator, Dict from unittest import TestCase from scrapy.http import Request @@ -37,7 +38,10 @@ class Contract: else: results.addSuccess(self.testcase_pre) finally: - return list(iterate_spider_output(cb(response, **cb_kwargs))) + cb_result = cb(response, **cb_kwargs) + if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + raise TypeError("Contracts don't support async callbacks") + return list(iterate_spider_output(cb_result)) request.callback = wrapper @@ -49,7 +53,10 @@ class Contract: @wraps(cb) def wrapper(response, **cb_kwargs): - output = list(iterate_spider_output(cb(response, **cb_kwargs))) + cb_result = cb(response, **cb_kwargs) + if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + raise TypeError("Contracts don't support async callbacks") + output = list(iterate_spider_output(cb_result)) try: results.startTest(self.testcase_post) self.post_process(output) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 813927fc5..1459e0b5f 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -63,6 +63,13 @@ class TestSpider(Spider): """ return Request("http://scrapy.org", callback=self.returns_item) + async def returns_request_async(self, response): + """async method which returns request + @url http://scrapy.org + @returns requests 1 + """ + return Request("http://scrapy.org", callback=self.returns_item) + def returns_item(self, response): """method which returns item @url http://scrapy.org @@ -337,6 +344,14 @@ class ContractsManagerTest(unittest.TestCase): request.callback(response) self.should_fail() + def test_returns_async(self): + spider = TestSpider() + response = ResponseMock() + + request = self.conman.from_method(spider.returns_request_async, self.results) + request.callback(response) + self.should_error() + def test_scrapes(self): spider = TestSpider() response = ResponseMock() From e2adec629b63e9d7735efd58b4353dcfe7ab2863 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 6 Aug 2023 17:31:11 +0400 Subject: [PATCH 1102/2083] Fix regressions in typing. --- scrapy/commands/__init__.py | 4 +-- scrapy/commands/fetch.py | 8 +++-- scrapy/commands/shell.py | 8 +++-- scrapy/http/response/text.py | 12 +++++--- scrapy/spiderloader.py | 2 +- scrapy/utils/ossignal.py | 14 +++++++-- scrapy/utils/spider.py | 60 ++++++++++++++++++++++++++++++++++-- 7 files changed, 91 insertions(+), 17 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9baee3a48..2aa569cdd 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -4,7 +4,7 @@ Base class for Scrapy commands import argparse import os from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from twisted.python import failure @@ -116,7 +116,7 @@ class ScrapyCommand: if opts.pdb: failure.startDebugMode() - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: """ Entry point for running commands """ diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 1359e445f..cdb7ad4ae 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,7 +1,10 @@ import sys +from argparse import Namespace +from typing import List, Type from w3lib.url import is_url +from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.http import Request @@ -57,7 +60,7 @@ class Command(ScrapyCommand): def _print_bytes(self, bytes_): sys.stdout.buffer.write(bytes_ + b"\n") - def run(self, args, opts): + def run(self, args: List[str], opts: Namespace) -> None: if len(args) != 1 or not is_url(args[0]): raise UsageError() request = Request( @@ -73,7 +76,8 @@ class Command(ScrapyCommand): else: request.meta["handle_httpstatus_all"] = True - spidercls = DefaultSpider + spidercls: Type[Spider] = DefaultSpider + assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: spidercls = spider_loader.load(opts.spider) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 63c23d04c..0a5e61f7a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -3,8 +3,11 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ +from argparse import Namespace from threading import Thread +from typing import List, Type +from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.shell import Shell @@ -54,15 +57,16 @@ class Command(ScrapyCommand): """ pass - def run(self, args, opts): + def run(self, args: List[str], opts: Namespace) -> None: url = args[0] if args else None if url: # first argument may be a local file url = guess_scheme(url) + assert self.crawler_process spider_loader = self.crawler_process.spider_loader - spidercls = DefaultSpider + spidercls: Type[Spider] = DefaultSpider if opts.spider: spidercls = spider_loader.load(opts.spider) elif url: diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index f228e11c1..7fc54b5d3 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -4,10 +4,11 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ +from __future__ import annotations import json from contextlib import suppress -from typing import Generator, Optional, Tuple +from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple from urllib.parse import urljoin import parsel @@ -25,6 +26,9 @@ from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url +if TYPE_CHECKING: + from scrapy.selector import Selector + _NONE = object() @@ -34,11 +38,11 @@ class TextResponse(Response): attributes: Tuple[str, ...] = Response.attributes + ("encoding",) - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): self._encoding = kwargs.pop("encoding", None) - self._cached_benc = None + self._cached_benc: Optional[str] = None self._cached_ubody: Optional[str] = None - self._cached_selector = None + self._cached_selector: Optional[Selector] = None super().__init__(*args, **kwargs) def _set_url(self, url): diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 13d6f9f87..f6bb93ddc 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -27,7 +27,7 @@ class SpiderLoader: self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list) self._load_all_spiders() - def _check_name_duplicates(self): + def _check_name_duplicates(self) -> None: dupes = [] for name, locations in self._found.items(): dupes.extend( diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index f835a2221..2334ea792 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,7 +1,13 @@ import signal -from typing import Callable +from types import FrameType +from typing import Any, Callable, Dict, Optional, Union -signal_names = {} +# copy of _HANDLER from typeshed/stdlib/signal.pyi +SignalHandlerT = Union[ + Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None +] + +signal_names: Dict[int, str] = {} for signame in dir(signal): if signame.startswith("SIG") and not signame.startswith("SIG_"): signum = getattr(signal, signame) @@ -9,7 +15,9 @@ for signame in dir(signal): signal_names[signum] = signame -def install_shutdown_handlers(function: Callable, override_sigint: bool = True) -> None: +def install_shutdown_handlers( + function: SignalHandlerT, override_sigint: bool = True +) -> None: """Install the given function as a signal handler for all common shutdown signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the SIGINT handler won't be install if there is already a handler in place diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 3228eda49..704df8657 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,16 +2,19 @@ from __future__ import annotations import inspect import logging -from types import ModuleType +from types import CoroutineType, ModuleType from typing import ( TYPE_CHECKING, Any, - AsyncIterable, + AsyncGenerator, Generator, Iterable, + Literal, Optional, Type, + TypeVar, Union, + overload, ) from twisted.internet.defer import Deferred @@ -26,8 +29,26 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_T = TypeVar("_T") -def iterate_spider_output(result: Any) -> Union[Iterable, AsyncIterable, Deferred]: + +# https://stackoverflow.com/questions/60222982 +@overload +def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ignore[misc] + ... + + +@overload +def iterate_spider_output(result: CoroutineType) -> Deferred: + ... + + +@overload +def iterate_spider_output(result: _T) -> Iterable: + ... + + +def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: if inspect.isasyncgen(result): return result if inspect.iscoroutine(result): @@ -55,6 +76,39 @@ def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None yield obj +@overload +def spidercls_for_request( + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Type[Spider], + log_none: bool = ..., + log_multiple: bool = ..., +) -> Type[Spider]: + ... + + +@overload +def spidercls_for_request( + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Literal[None], + log_none: bool = ..., + log_multiple: bool = ..., +) -> Optional[Type[Spider]]: + ... + + +@overload +def spidercls_for_request( + spider_loader: SpiderLoader, + request: Request, + *, + log_none: bool = ..., + log_multiple: bool = ..., +) -> Optional[Type[Spider]]: + ... + + def spidercls_for_request( spider_loader: SpiderLoader, request: Request, From f5f593e5f5e2b0c216e9d6fd41f4260f70c74d34 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 6 Aug 2023 17:46:28 +0400 Subject: [PATCH 1103/2083] Remove a workaround for a w3lib typing bug. --- scrapy/utils/response.py | 3 +-- tox.ini | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 794678c48..c540d6278 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -42,8 +42,7 @@ def get_meta_refresh( """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] - # a w3lib typing bug here, fixed in https://github.com/scrapy/w3lib/pull/211 - _metaref_cache[response] = html.get_meta_refresh( # type: ignore[assignment] + _metaref_cache[response] = html.get_meta_refresh( text, response.url, response.encoding, ignore_tags=ignore_tags ) return _metaref_cache[response] diff --git a/tox.ini b/tox.ini index ef7dd5854..8b2d207c7 100644 --- a/tox.ini +++ b/tox.ini @@ -41,6 +41,8 @@ deps = types-Pygments==2.15.0.1 types-pyOpenSSL==23.2.0.1 types-setuptools==68.0.0.1 + # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 + w3lib >= 2.1.2 commands = mypy {posargs: scrapy tests} From 471281d29e5c7b8e293f59ac3c3330ecda687d19 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 6 Aug 2023 23:05:02 +0400 Subject: [PATCH 1104/2083] Fixes for scrapy/utils/iterators.py typing. --- scrapy/utils/iterators.py | 11 ++++++----- tests/test_utils_iterators.py | 7 +------ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 58850b843..40af68dec 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import csv import logging import re @@ -78,7 +76,7 @@ def xmliter( def xmliter_lxml( - obj: Union[TextResponse, str, bytes], + obj: Union[Response, str, bytes], nodename: str, namespace: Optional[str] = None, prefix: str = "x", @@ -87,8 +85,9 @@ def xmliter_lxml( reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename + # technically, etree.iterparse only needs .read() AFAICS, but this is how it's typed iterable = etree.iterparse( - cast(SupportsReadClose[bytes], reader), tag=tag, encoding=reader.encoding + cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: @@ -101,11 +100,13 @@ def xmliter_lxml( class _StreamReader: - def __init__(self, obj: Union[TextResponse, str, bytes]): + def __init__(self, obj: Union[Response, str, bytes]): self._ptr: int = 0 self._text: Union[str, bytes] if isinstance(obj, TextResponse): self._text, self.encoding = obj.body, obj.encoding + elif isinstance(obj, Response): + self._text, self.encoding = obj.body, "utf-8" else: self._text, self.encoding = obj, "utf-8" self._is_unicode: bool = isinstance(self._text, str) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 5dfd7e7ac..3598fa0bb 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,18 +1,13 @@ -from typing import Callable, Iterable, Union - from pytest import mark from twisted.trial import unittest -from scrapy import Selector from scrapy.http import Response, TextResponse, XmlResponse from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata class XmliterTestCase(unittest.TestCase): - xmliter: Callable[ - [Union[TextResponse, str, bytes], str], Iterable[Selector] - ] = staticmethod(xmliter) + xmliter = staticmethod(xmliter) def test_xmliter(self): body = b""" From 644a71bfd40be3ce8a465ad49891d34c47516f56 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:17:52 +0400 Subject: [PATCH 1105/2083] Use ftp:// URLs in FTP tests. --- tests/test_feedexport.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 46bd5733a..42fa25b1d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2978,8 +2978,8 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): def test_init(self): settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptions}, + "FEED_URI": "ftp://localhost/foo", + "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptions}, } with pytest.warns( ScrapyDeprecationWarning, @@ -3000,8 +3000,8 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): def test_from_crawler(self): settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptionsWithFromCrawler}, + "FEED_URI": "ftp://localhost/foo", + "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptionsWithFromCrawler}, } with pytest.warns( ScrapyDeprecationWarning, From 23af21491d526072873ab7ec3d1429560861964f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:21:06 +0400 Subject: [PATCH 1106/2083] Move definitions around to woark around a pypy3.8 bug. --- scrapy/utils/log.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2013bfc43..0d17f6153 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -122,6 +122,9 @@ def configure_logging( install_scrapy_root_handler(settings) +_scrapy_root_handler: Optional[logging.Handler] = None + + def install_scrapy_root_handler(settings: Settings) -> None: global _scrapy_root_handler @@ -139,9 +142,6 @@ def get_scrapy_root_handler() -> Optional[logging.Handler]: return _scrapy_root_handler -_scrapy_root_handler: Optional[logging.Handler] = None - - def _get_handler(settings: Settings) -> logging.Handler: """Return a log handler object according to settings""" filename = settings.get("LOG_FILE") From 110d5fffb4eb034cb876f5339eb2c95b89d8630b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 12:57:48 +0400 Subject: [PATCH 1107/2083] Update tool versions. (#6002) --- .pre-commit-config.yaml | 8 ++++---- tox.ini | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31e9ed1ad..5998ebef8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,11 +5,11 @@ repos: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -17,8 +17,8 @@ repos: hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs - rev: 1.13.0 + rev: 1.15.0 hooks: - id: blacken-docs additional_dependencies: - - black==23.3.0 + - black==23.7.0 diff --git a/tox.ini b/tox.ini index ef7dd5854..b22ef404d 100644 --- a/tox.ini +++ b/tox.ini @@ -37,10 +37,10 @@ deps = typing-extensions==4.7.1 types-attrs==19.1.0 types-lxml==2023.3.28 - types-Pillow==10.0.0.1 - types-Pygments==2.15.0.1 - types-pyOpenSSL==23.2.0.1 - types-setuptools==68.0.0.1 + types-Pillow==10.0.0.2 + types-Pygments==2.15.0.2 + types-pyOpenSSL==23.2.0.2 + types-setuptools==68.0.0.3 commands = mypy {posargs: scrapy tests} @@ -55,7 +55,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.17.2 + pylint==2.17.5 commands = pylint conftest.py docs extras scrapy setup.py tests From 53539483c32dec537064628c0c6e407eb8ce1c2f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 15:13:20 +0400 Subject: [PATCH 1108/2083] Refactor _StreamReader.read(). --- scrapy/utils/iterators.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 40af68dec..baf92681a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -110,12 +110,17 @@ class _StreamReader: else: self._text, self.encoding = obj, "utf-8" self._is_unicode: bool = isinstance(self._text, str) + self._is_first_read: bool = True def read(self, n: int = 65535) -> bytes: - self.read: Callable[[int], bytes] = ( # type: ignore[method-assign] + method: Callable[[int], bytes] = ( self._read_unicode if self._is_unicode else self._read_string ) - return self.read(n).lstrip() + result = method(n) + if self._is_first_read: + self._is_first_read = False + result = result.lstrip() + return result def _read_string(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n From 8050257c1495fe405767dcd37c4db0f6f29c38aa Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 9 Aug 2023 23:17:32 +0400 Subject: [PATCH 1109/2083] Small cleanup. --- scrapy/extensions/feedexport.py | 3 ++- scrapy/utils/iterators.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 2bbcaf3ad..c1b77f4fb 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -290,7 +290,8 @@ class FTPFeedStorage(BlockingFeedStorage): feed_options: Optional[Dict[str, Any]] = None, ): u = urlparse(uri) - assert u.hostname + if not u.hostname: + raise ValueError(f"Got a storage URI without a hostname: {uri}") self.host: str = u.hostname self.port: int = int(u.port or "21") self.username: str = u.username or "" diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index baf92681a..03d779afb 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -85,7 +85,6 @@ def xmliter_lxml( reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename - # technically, etree.iterparse only needs .read() AFAICS, but this is how it's typed iterable = etree.iterparse( cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding ) From 084a9ba0768c34e2c0b82e13c7182da33460a19b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 6 Aug 2023 23:51:24 +0400 Subject: [PATCH 1110/2083] Full typing for scrapy/crawler.py and scrapy/spiders/__init__.py. --- scrapy/crawler.py | 77 +++++++++++++++++++++++--------------- scrapy/logformatter.py | 13 ++++++- scrapy/middleware.py | 29 ++++++++++++-- scrapy/spiderloader.py | 10 ++++- scrapy/spiders/__init__.py | 37 ++++++++++-------- 5 files changed, 114 insertions(+), 52 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index c5b3e1903..531798893 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,9 +4,14 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union -from twisted.internet import defer +from twisted.internet.defer import ( + Deferred, + DeferredList, + inlineCallbacks, + maybeDeferred, +) from zope.interface.exceptions import DoesNotImplement try: @@ -24,7 +29,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.logformatter import LogFormatter -from scrapy.settings import Settings, overridden_settings +from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager from scrapy.spiderloader import SpiderLoader from scrapy.statscollectors import StatsCollector @@ -123,8 +128,8 @@ class Crawler: self.spider: Optional[Spider] = None self.engine: Optional[ExecutionEngine] = None - @defer.inlineCallbacks - def crawl(self, *args, **kwargs): + @inlineCallbacks + def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: if self.crawling: raise RuntimeError("Crawling already taking place") self.crawling = True @@ -134,26 +139,27 @@ class Crawler: self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) - yield defer.maybeDeferred(self.engine.start) + yield maybeDeferred(self.engine.start) except Exception: self.crawling = False if self.engine is not None: yield self.engine.close() raise - def _create_spider(self, *args, **kwargs): + def _create_spider(self, *args: Any, **kwargs: Any) -> Spider: return self.spidercls.from_crawler(self, *args, **kwargs) - def _create_engine(self): + def _create_engine(self) -> ExecutionEngine: return ExecutionEngine(self, lambda _: self.stop()) - @defer.inlineCallbacks - def stop(self): + @inlineCallbacks + def stop(self) -> Generator[Deferred, Any, None]: """Starts a graceful stop of the crawler and returns a deferred that is fired when the crawler is stopped.""" if self.crawling: self.crawling = False - yield defer.maybeDeferred(self.engine.stop) + assert self.engine + yield maybeDeferred(self.engine.stop) class CrawlerRunner: @@ -176,10 +182,10 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings) -> SpiderLoader: + def _get_spider_loader(settings: BaseSettings) -> SpiderLoader: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") - loader_cls = load_object(cls_path) + loader_cls: Type[SpiderLoader] = load_object(cls_path) excs = ( (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement ) @@ -201,11 +207,11 @@ class CrawlerRunner: self.settings = settings self.spider_loader = self._get_spider_loader(settings) self._crawlers: Set[Crawler] = set() - self._active: Set[defer.Deferred] = set() + self._active: Set[Deferred] = set() self.bootstrap_failed = False @property - def spiders(self): + def spiders(self) -> SpiderLoader: warnings.warn( "CrawlerRunner.spiders attribute is renamed to " "CrawlerRunner.spider_loader.", @@ -214,7 +220,12 @@ class CrawlerRunner: ) return self.spider_loader - def crawl(self, crawler_or_spidercls, *args, **kwargs): + def crawl( + self, + crawler_or_spidercls: Union[Type[Spider], str, Crawler], + *args: Any, + **kwargs: Any, + ) -> Deferred: """ Run a crawler with the provided arguments. @@ -244,12 +255,12 @@ class CrawlerRunner: crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl(self, crawler, *args, **kwargs): + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred: self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) - def _done(result): + def _done(result: Any) -> Any: self.crawlers.discard(crawler) self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) @@ -284,16 +295,16 @@ class CrawlerRunner: spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) - def stop(self): + def stop(self) -> Deferred: """ Stops simultaneously all the crawling jobs taking place. Returns a deferred that is fired when they all have ended. """ - return defer.DeferredList([c.stop() for c in list(self.crawlers)]) + return DeferredList([c.stop() for c in list(self.crawlers)]) - @defer.inlineCallbacks - def join(self): + @inlineCallbacks + def join(self) -> Generator[Deferred, Any, None]: """ join() @@ -301,7 +312,7 @@ class CrawlerRunner: completed their executions. """ while self._active: - yield defer.DeferredList(self._active) + yield DeferredList(self._active) class CrawlerProcess(CrawlerRunner): @@ -328,13 +339,17 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None, install_root_handler=True): + def __init__( + self, + settings: Union[Dict[str, Any], Settings, None] = None, + install_root_handler: bool = True, + ): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) self._initialized_reactor = False - def _signal_shutdown(self, signum, _): + def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor install_shutdown_handlers(self._signal_kill) @@ -345,7 +360,7 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._graceful_stop_reactor) - def _signal_kill(self, signum, _): + def _signal_kill(self, signum: int, _: Any) -> None: from twisted.internet import reactor install_shutdown_handlers(signal.SIG_IGN) @@ -355,14 +370,16 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._stop_reactor) - def _create_crawler(self, spidercls): + def _create_crawler(self, spidercls: Union[Type[Spider], str]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True return Crawler(spidercls, self.settings, init_reactor=init_reactor) - def start(self, stop_after_crawl=True, install_signal_handlers=True): + def start( + self, stop_after_crawl: bool = True, install_signal_handlers: bool = True + ) -> None: """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache @@ -396,12 +413,12 @@ class CrawlerProcess(CrawlerRunner): reactor.addSystemEventTrigger("before", "shutdown", self.stop) reactor.run(installSignalHandlers=False) # blocking call - def _graceful_stop_reactor(self): + def _graceful_stop_reactor(self) -> Deferred: d = self.stop() d.addBoth(self._stop_reactor) return d - def _stop_reactor(self, _=None): + def _stop_reactor(self, _: Any = None) -> None: from twisted.internet import reactor try: diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 7cb379b46..600da0d40 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import logging import os -from typing import Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Union from twisted.python.failure import Failure @@ -8,6 +10,13 @@ from scrapy import Request, Spider from scrapy.http import Response from scrapy.utils.request import referer_str +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s" DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s" CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" @@ -161,5 +170,5 @@ class LogFormatter: } @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls() diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 04b838d2d..090588130 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,21 @@ +from __future__ import annotations + import logging import pprint from collections import defaultdict, deque -from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Deque, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) from twisted.internet.defer import Deferred @@ -11,6 +25,13 @@ from scrapy.settings import Settings from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import create_instance, load_object +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) @@ -34,7 +55,9 @@ class MiddlewareManager: raise NotImplementedError @classmethod - def from_settings(cls, settings: Settings, crawler=None): + def from_settings( + cls, settings: Settings, crawler: Optional[Crawler] = None + ) -> Self: mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] @@ -63,7 +86,7 @@ class MiddlewareManager: return cls(*middlewares) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls.from_settings(crawler.settings, crawler) def _add_middleware(self, mw: Any) -> None: diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index f6bb93ddc..cd60fce9d 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,8 +1,10 @@ +from __future__ import annotations + import traceback import warnings from collections import defaultdict from types import ModuleType -from typing import DefaultDict, Dict, List, Tuple, Type +from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer @@ -12,6 +14,10 @@ from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + @implementer(ISpiderLoader) class SpiderLoader: @@ -69,7 +75,7 @@ class SpiderLoader: self._check_name_duplicates() @classmethod - def from_settings(cls, settings): + def from_settings(cls, settings: BaseSettings) -> Self: return cls(settings) def load(self, spider_name: str) -> Type[Spider]: diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 388439f4f..e16d71727 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -6,15 +6,21 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast + +from twisted.internet.defer import Deferred from scrapy import signals -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings class Spider(object_ref): @@ -25,21 +31,21 @@ class Spider(object_ref): name: str custom_settings: Optional[dict] = None - def __init__(self, name=None, **kwargs): + def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: self.name = name elif not getattr(self, "name", None): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) if not hasattr(self, "start_urls"): - self.start_urls = [] + self.start_urls: List[str] = [] @property - def logger(self): + def logger(self) -> logging.LoggerAdapter: logger = logging.getLogger(self.name) return logging.LoggerAdapter(logger, {"spider": self}) - def log(self, message, level=logging.DEBUG, **kw): + def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None: """Log the given message at the given log level This helper wraps a log call to the logger within the spider, but you @@ -49,17 +55,17 @@ class Spider(object_ref): self.logger.log(level, message, **kw) @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = cls(*args, **kwargs) spider._set_crawler(crawler) return spider - def _set_crawler(self, crawler: Crawler): + def _set_crawler(self, crawler: Crawler) -> None: self.crawler = crawler self.settings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - def start_requests(self): + def start_requests(self) -> Iterable[Request]: if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " @@ -69,16 +75,16 @@ class Spider(object_ref): for url in self.start_urls: yield Request(url, dont_filter=True) - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: return self.parse(response, **kwargs) - def parse(self, response, **kwargs): + def parse(self, response: Response, **kwargs: Any) -> Any: raise NotImplementedError( f"{self.__class__.__name__}.parse callback is not defined" ) @classmethod - def update_settings(cls, settings): + def update_settings(cls, settings: BaseSettings) -> None: settings.setdict(cls.custom_settings or {}, priority="spider") @classmethod @@ -86,12 +92,13 @@ class Spider(object_ref): return url_is_from_spider(request.url, cls) @staticmethod - def close(spider, reason): + def close(spider: Spider, reason: str) -> Union[Deferred, None]: closed = getattr(spider, "closed", None) if callable(closed): - return closed(reason) + return cast(Union[Deferred, None], closed(reason)) + return None - def __repr__(self): + def __repr__(self) -> str: return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>" From dc6e142096e77a99e5340f36309f40e70f4145cd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:35:44 +0400 Subject: [PATCH 1111/2083] Full typing for scrapy/spiderloader.py. --- scrapy/spiderloader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index cd60fce9d..d855c962c 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -27,8 +27,8 @@ class SpiderLoader: """ def __init__(self, settings: BaseSettings): - self.spider_modules = settings.getlist("SPIDER_MODULES") - self.warn_only = settings.getbool("SPIDER_LOADER_WARN_ONLY") + self.spider_modules: List[str] = settings.getlist("SPIDER_MODULES") + self.warn_only: bool = settings.getbool("SPIDER_LOADER_WARN_ONLY") self._spiders: Dict[str, Type[Spider]] = {} self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list) self._load_all_spiders() @@ -96,7 +96,7 @@ class SpiderLoader: name for name, cls in self._spiders.items() if cls.handles_request(request) ] - def list(self): + def list(self) -> List[str]: """ Return a list with the names of all spiders available in the project. """ From 89503ae3f1228e9406428a19bc6c49c1f8e9e19b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:41:11 +0400 Subject: [PATCH 1112/2083] Full typing for scrapy/dupefilters.py. --- scrapy/dupefilters.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index d796e5cbb..bc912268c 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import logging from pathlib import Path -from typing import Optional, Set, Type, TypeVar +from typing import TYPE_CHECKING, Optional, Set from warnings import warn from twisted.internet.defer import Deferred @@ -12,14 +14,16 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.job import job_dir from scrapy.utils.request import RequestFingerprinter, referer_str -BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler class BaseDupeFilter: @classmethod - def from_settings( - cls: Type[BaseDupeFilterTV], settings: BaseSettings - ) -> BaseDupeFilterTV: + def from_settings(cls, settings: BaseSettings) -> Self: return cls() def request_seen(self, request: Request) -> bool: @@ -36,9 +40,6 @@ class BaseDupeFilter: pass -RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter") - - class RFPDupeFilter(BaseDupeFilter): """Request Fingerprint duplicates filter""" @@ -47,10 +48,12 @@ class RFPDupeFilter(BaseDupeFilter): path: Optional[str] = None, debug: bool = False, *, - fingerprinter=None, + fingerprinter: Optional[RequestFingerprinter] = None, ) -> None: self.file = None - self.fingerprinter = fingerprinter or RequestFingerprinter() + self.fingerprinter: RequestFingerprinter = ( + fingerprinter or RequestFingerprinter() + ) self.fingerprints: Set[str] = set() self.logdupes = True self.debug = debug @@ -62,8 +65,11 @@ class RFPDupeFilter(BaseDupeFilter): @classmethod def from_settings( - cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None - ) -> RFPDupeFilterTV: + cls, + settings: BaseSettings, + *, + fingerprinter: Optional[RequestFingerprinter] = None, + ) -> Self: debug = settings.getbool("DUPEFILTER_DEBUG") try: return cls(job_dir(settings), debug, fingerprinter=fingerprinter) @@ -75,11 +81,11 @@ class RFPDupeFilter(BaseDupeFilter): ScrapyDeprecationWarning, ) result = cls(job_dir(settings), debug) - result.fingerprinter = fingerprinter + result.fingerprinter = fingerprinter or RequestFingerprinter() return result @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: try: return cls.from_settings( crawler.settings, From 9960c62b871a4b76d72c57abdead553b6e7178e1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:42:06 +0400 Subject: [PATCH 1113/2083] Full typing for scrapy/logformatter.py. --- scrapy/core/scraper.py | 1 + scrapy/logformatter.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index a85f6a661..a54929712 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -364,6 +364,7 @@ class Scraper: spider=spider, exception=output.value, ) + assert ex logkws = self.logformatter.item_error(item, ex, response, spider) logger.log( *logformatter_adapter(logkws), diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 600da0d40..9b05e1153 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -114,7 +114,7 @@ class LogFormatter: } def item_error( - self, item: Any, exception, response: Response, spider: Spider + self, item: Any, exception: BaseException, response: Response, spider: Spider ) -> dict: """Logs a message when an item causes an error while it is passing through the item pipeline. From c5885fc13b7d44ddaf403e8ccdd8dccb14082eee Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:44:46 +0400 Subject: [PATCH 1114/2083] Full typing for scrapy/exceptions.py. --- scrapy/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 6e83e4a00..6d188c489 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -52,7 +52,7 @@ class StopDownload(Exception): should be handled by the request errback. Note that 'fail' is a keyword-only argument. """ - def __init__(self, *, fail=True): + def __init__(self, *, fail: bool = True): super().__init__() self.fail = fail From 7dca18e2e7e92e7dcc0fb8ae769f82287dd64386 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:45:16 +0400 Subject: [PATCH 1115/2083] Full typing for scrapy/responsetypes.py. --- scrapy/responsetypes.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 58884f21a..9e411d4aa 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -5,6 +5,7 @@ based on different criteria. from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data +from typing import Dict, Mapping, Optional, Type, Union from scrapy.http import Response from scrapy.utils.misc import load_object @@ -29,15 +30,19 @@ class ResponseTypes: "text/*": "scrapy.http.TextResponse", } - def __init__(self): - self.classes = {} - self.mimetypes = MimeTypes() - mimedata = get_data("scrapy", "mime.types").decode("utf8") - self.mimetypes.readfp(StringIO(mimedata)) + def __init__(self) -> None: + self.classes: Dict[str, Type[Response]] = {} + self.mimetypes: MimeTypes = MimeTypes() + mimedata = get_data("scrapy", "mime.types") + if not mimedata: + raise ValueError( + "The mime.types file is not found in the Scrapy installation" + ) + self.mimetypes.readfp(StringIO(mimedata.decode("utf8"))) for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) - def from_mimetype(self, mimetype): + def from_mimetype(self, mimetype: str) -> Type[Response]: """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response @@ -46,7 +51,9 @@ class ResponseTypes: basetype = f"{mimetype.split('/')[0]}/*" return self.classes.get(basetype, Response) - def from_content_type(self, content_type, content_encoding=None): + def from_content_type( + self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = None + ) -> Type[Response]: """Return the most appropriate Response class from an HTTP Content-Type header""" if content_encoding: @@ -56,7 +63,9 @@ class ResponseTypes: ) return self.from_mimetype(mimetype) - def from_content_disposition(self, content_disposition): + def from_content_disposition( + self, content_disposition: Union[str, bytes] + ) -> Type[Response]: try: filename = ( to_unicode(content_disposition, encoding="latin-1", errors="replace") @@ -68,7 +77,7 @@ class ResponseTypes: except IndexError: return Response - def from_headers(self, headers): + def from_headers(self, headers: Mapping[bytes, bytes]) -> Type[Response]: """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response @@ -81,14 +90,14 @@ class ResponseTypes: cls = self.from_content_disposition(headers[b"Content-Disposition"]) return cls - def from_filename(self, filename): + def from_filename(self, filename: str) -> Type[Response]: """Return the most appropriate Response class from a file name""" mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) return Response - def from_body(self, body): + def from_body(self, body: bytes) -> Type[Response]: """Try to guess the appropriate response based on the body content. This method is a bit magic and could be improved in the future, but it's not meant to be used except for special cases where response types @@ -106,7 +115,13 @@ class ResponseTypes: return self.from_mimetype("text/html") return self.from_mimetype("text") - def from_args(self, headers=None, url=None, filename=None, body=None): + def from_args( + self, + headers: Optional[Mapping[bytes, bytes]] = None, + url: Optional[str] = None, + filename: Optional[str] = None, + body: Optional[bytes] = None, + ) -> Type[Response]: """Guess the most appropriate Response class based on the given arguments.""" cls = Response From e6d919497d08f5c1ddcdd0c210fcc17bd78f2fe9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:52:30 +0400 Subject: [PATCH 1116/2083] More typing for scrapy/core/scheduler.py. --- scrapy/core/scheduler.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 3fb0bbaff..70b6dc8a1 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,8 +1,10 @@ +from __future__ import annotations + import json import logging from abc import abstractmethod from pathlib import Path -from typing import Any, Optional, Type, TypeVar, cast +from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast from twisted.internet.defer import Deferred @@ -14,6 +16,11 @@ from scrapy.statscollectors import StatsCollector from scrapy.utils.job import job_dir from scrapy.utils.misc import create_instance, load_object +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) @@ -54,7 +61,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ @classmethod - def from_crawler(cls, crawler: Crawler): + def from_crawler(cls, crawler: Crawler) -> Self: """ Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument. """ @@ -325,6 +332,7 @@ class Scheduler(BaseScheduler): def _dq(self): """Create a new priority queue instance, with disk storage""" + assert self.dqdir state = self._read_dqs_state(self.dqdir) q = create_instance( self.pqclass, From b50268c100c6ad70ea81f0606aedcb720a12692b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:55:40 +0400 Subject: [PATCH 1117/2083] Full typing for scrapy/link.py. --- scrapy/link.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/scrapy/link.py b/scrapy/link.py index 704649731..0868ae5ef 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ +from typing import Any class Link: @@ -26,16 +27,20 @@ class Link: __slots__ = ["url", "text", "fragment", "nofollow"] - def __init__(self, url, text="", fragment="", nofollow=False): + def __init__( + self, url: str, text: str = "", fragment: str = "", nofollow: bool = False + ): if not isinstance(url, str): got = url.__class__.__name__ raise TypeError(f"Link urls must be str objects, got {got}") - self.url = url - self.text = text - self.fragment = fragment - self.nofollow = nofollow + self.url: str = url + self.text: str = text + self.fragment: str = fragment + self.nofollow: bool = nofollow - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Link): + raise NotImplementedError return ( self.url == other.url and self.text == other.text @@ -43,12 +48,12 @@ class Link: and self.nofollow == other.nofollow ) - def __hash__(self): + def __hash__(self) -> int: return ( hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow) ) - def __repr__(self): + def __repr__(self) -> str: return ( f"Link(url={self.url!r}, text={self.text!r}, " f"fragment={self.fragment!r}, nofollow={self.nofollow!r})" From 4a090d951a721881d0c434ebf4207e78a4eadfe1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 10 Aug 2023 02:36:42 -0300 Subject: [PATCH 1118/2083] Remove deprecated PythonItemExporter.binary (#6007) --- scrapy/exporters.py | 14 +------------- tests/test_exporters.py | 12 +----------- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 8254ea63e..f85f1dad8 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -7,13 +7,11 @@ import io import marshal import pickle import pprint -import warnings from collections.abc import Mapping from xml.sax.saxutils import XMLGenerator from itemadapter import ItemAdapter, is_item -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -330,13 +328,7 @@ class PythonItemExporter(BaseItemExporter): """ def _configure(self, options, dont_fail=False): - self.binary = options.pop("binary", True) super()._configure(options, dont_fail) - if self.binary: - warnings.warn( - "PythonItemExporter will drop support for binary export in the future", - ScrapyDeprecationWarning, - ) if not self.encoding: self.encoding = "utf-8" @@ -351,18 +343,14 @@ class PythonItemExporter(BaseItemExporter): return dict(self._serialize_item(value)) if is_listlike(value): return [self._serialize_value(v) for v in value] - encode_func = to_bytes if self.binary else to_unicode if isinstance(value, (str, bytes)): - return encode_func(value, encoding=self.encoding) + return to_unicode(value, encoding=self.encoding) return value def _serialize_item(self, item): for key, value in ItemAdapter(item).items(): - key = to_bytes(key) if self.binary else key yield key, self._serialize_value(value) def export_item(self, item): result = dict(self._get_serialized_fields(item)) - if self.binary: - result = dict(self._serialize_item(result)) return result diff --git a/tests/test_exporters.py b/tests/test_exporters.py index cb24ddd8e..f4e82705a 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -7,12 +7,10 @@ import tempfile import unittest from datetime import datetime from io import BytesIO -from warnings import catch_warnings, filterwarnings import lxml.etree from itemadapter import ItemAdapter -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exporters import ( BaseItemExporter, CsvItemExporter, @@ -143,7 +141,7 @@ class BaseItemExporterDataclassTest(BaseItemExporterTest): class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): - return PythonItemExporter(binary=False, **kwargs) + return PythonItemExporter(**kwargs) def test_invalid_option(self): with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"): @@ -198,14 +196,6 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported["age"][0]), dict) self.assertEqual(type(exported["age"][0]["age"][0]), dict) - def test_export_binary(self): - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - exporter = PythonItemExporter(binary=True) - value = self.item_class(name="John\xa3", age="22") - expected = {b"name": b"John\xc2\xa3", b"age": b"22"} - self.assertEqual(expected, exporter.export_item(value)) - def test_nonstring_types_item(self): item = self._get_nonstring_types_item() ie = self._get_exporter() From 9e74748fca94e6b5c2c70346cd8789d80bc04507 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 10 Aug 2023 08:48:43 -0300 Subject: [PATCH 1119/2083] Remove extra spider parameter in item pipeline docs (#6009) --- docs/topics/item-pipeline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index bc26bbebe..a5f6e07b8 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -215,7 +215,7 @@ item. screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url, callback=NO_CALLBACK) response = await maybe_deferred_to_future( - spider.crawler.engine.download(request, spider) + spider.crawler.engine.download(request) ) if response.status != 200: From f05657e54245eff9f912914014c51b37b0bc6b34 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 10 Aug 2023 21:31:10 +0300 Subject: [PATCH 1120/2083] periodic_log: interval check moved --- scrapy/extensions/periodic_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 1023b3cd5..214f55777 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -37,6 +37,8 @@ class PeriodicLog: @classmethod def from_crawler(cls, crawler): interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + if not interval: + raise NotConfigured try: ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS") except (TypeError, ValueError): @@ -57,8 +59,6 @@ class PeriodicLog: ext_timing_enabled = crawler.settings.getbool( "PERIODIC_LOG_TIMING_ENABLED", False ) - if not interval: - raise NotConfigured if not (ext_stats or ext_delta or ext_timing_enabled): raise NotConfigured o = cls( From 736a4b615c08cc7e610ddb18f5ac782f0aedca61 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 10 Aug 2023 21:34:35 +0300 Subject: [PATCH 1121/2083] Update scrapy/settings/default_settings.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/settings/default_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9660e0bcd..f74b07c20 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -242,7 +242,7 @@ NEWSPIDER_MODULE = "" PERIODIC_LOG_DELTA = None PERIODIC_LOG_STATS = None -PERIODIC_LOG_TIMING_ENABLED = None +PERIODIC_LOG_TIMING_ENABLED = False RANDOMIZE_DOWNLOAD_DELAY = True From 9df67a554e542b3ec8f92491c3840fca5ea182d2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 10 Aug 2023 22:53:22 +0400 Subject: [PATCH 1122/2083] Add RequestFingerprinterProtocol. --- scrapy/dupefilters.py | 12 ++++++++---- scrapy/utils/request.py | 6 ++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index bc912268c..d2639104b 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -12,7 +12,11 @@ from scrapy.settings import BaseSettings from scrapy.spiders import Spider from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.job import job_dir -from scrapy.utils.request import RequestFingerprinter, referer_str +from scrapy.utils.request import ( + RequestFingerprinter, + RequestFingerprinterProtocol, + referer_str, +) if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -48,10 +52,10 @@ class RFPDupeFilter(BaseDupeFilter): path: Optional[str] = None, debug: bool = False, *, - fingerprinter: Optional[RequestFingerprinter] = None, + fingerprinter: Optional[RequestFingerprinterProtocol] = None, ) -> None: self.file = None - self.fingerprinter: RequestFingerprinter = ( + self.fingerprinter: RequestFingerprinterProtocol = ( fingerprinter or RequestFingerprinter() ) self.fingerprints: Set[str] = set() @@ -68,7 +72,7 @@ class RFPDupeFilter(BaseDupeFilter): cls, settings: BaseSettings, *, - fingerprinter: Optional[RequestFingerprinter] = None, + fingerprinter: Optional[RequestFingerprinterProtocol] = None, ) -> Self: debug = settings.getbool("DUPEFILTER_DEBUG") try: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 6c7f3b345..24fcbd85e 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -14,6 +14,7 @@ from typing import ( Iterable, List, Optional, + Protocol, Tuple, Type, Union, @@ -230,6 +231,11 @@ def fingerprint( return cache[cache_key] +class RequestFingerprinterProtocol(Protocol): + def fingerprint(self, request: Request) -> bytes: + ... + + class RequestFingerprinter: """Default fingerprinter. From 44b15c3004d950021460293c391f2283f2418726 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 10 Aug 2023 23:01:54 +0400 Subject: [PATCH 1123/2083] Remove typing for CrawlerRunner.spider_loader. --- scrapy/crawler.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 531798893..ee845a831 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,7 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast from twisted.internet.defer import ( Deferred, @@ -31,7 +31,6 @@ from scrapy.interfaces import ISpiderLoader from scrapy.logformatter import LogFormatter from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager -from scrapy.spiderloader import SpiderLoader from scrapy.statscollectors import StatsCollector from scrapy.utils.log import ( LogCounterHandler, @@ -182,10 +181,10 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings: BaseSettings) -> SpiderLoader: + def _get_spider_loader(settings: BaseSettings): """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") - loader_cls: Type[SpiderLoader] = load_object(cls_path) + loader_cls = load_object(cls_path) excs = ( (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement ) @@ -211,7 +210,7 @@ class CrawlerRunner: self.bootstrap_failed = False @property - def spiders(self) -> SpiderLoader: + def spiders(self): warnings.warn( "CrawlerRunner.spiders attribute is renamed to " "CrawlerRunner.spider_loader.", @@ -293,7 +292,8 @@ class CrawlerRunner: def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings) + # temporary cast until self.spider_loader is typed + return Crawler(cast(Type[Spider], spidercls), self.settings) def stop(self) -> Deferred: """ @@ -375,7 +375,10 @@ class CrawlerProcess(CrawlerRunner): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True - return Crawler(spidercls, self.settings, init_reactor=init_reactor) + # temporary cast until self.spider_loader is typed + return Crawler( + cast(Type[Spider], spidercls), self.settings, init_reactor=init_reactor + ) def start( self, stop_after_crawl: bool = True, install_signal_handlers: bool = True From 34d050cfe5aec94a15f00950fcecbc49cb470fb8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 11 Aug 2023 12:41:05 +0400 Subject: [PATCH 1124/2083] Remove deprecated CrawlerRunner.spiders. (#6010) --- scrapy/crawler.py | 10 ---------- tests/test_crawler.py | 11 ----------- 2 files changed, 21 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index c5b3e1903..bc0ab02df 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -204,16 +204,6 @@ class CrawlerRunner: self._active: Set[defer.Deferred] = set() self.bootstrap_failed = False - @property - def spiders(self): - warnings.warn( - "CrawlerRunner.spiders attribute is renamed to " - "CrawlerRunner.spider_loader.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return self.spider_loader - def crawl(self, crawler_or_spidercls, *args, **kwargs): """ Run a crawler with the provided arguments. diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f99606ccf..4c5c48e6d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -20,7 +20,6 @@ from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler -from scrapy.utils.misc import load_object from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer, get_mockserver_env @@ -182,16 +181,6 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): runner = CrawlerRunner() self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_deprecated_attribute_spiders(self): - with warnings.catch_warnings(record=True) as w: - runner = CrawlerRunner(Settings()) - spiders = runner.spiders - self.assertEqual(len(w), 1) - self.assertIn("CrawlerRunner.spiders", str(w[0].message)) - self.assertIn("CrawlerRunner.spider_loader", str(w[0].message)) - sl_cls = load_object(runner.settings["SPIDER_LOADER_CLASS"]) - self.assertIsInstance(spiders, sl_cls) - class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_dict(self): From 2f094a7a5ca080ec6b43c1db4e0a70a385a0f756 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 11 Aug 2023 18:48:14 +0300 Subject: [PATCH 1125/2083] Update docs/topics/extensions.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/extensions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index ae94c55a4..e280224b1 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -481,6 +481,6 @@ Default: ``None`` PERIODIC_LOG_TIMING_ENABLED """"""""""""""""""""""""""" -Default: ``None`` +Default: ``False`` -``"PERIODIC_LOG_TIMING_ENABLED": True`` - enables logging of timing data \ No newline at end of file +``True`` enables logging of timing data (i.e. the ``"time"`` section). \ No newline at end of file From e6bd9829bddba9d3301d8c883e80632944a19e79 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 11 Aug 2023 18:48:31 +0300 Subject: [PATCH 1126/2083] Update docs/topics/extensions.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/extensions.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index e280224b1..439e3e06e 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -471,9 +471,9 @@ PERIODIC_LOG_STATS Default: ``None`` -* ``"PERIODIC_LOG_STATS": True`` - show all available stats keys/values -* ``"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}`` - include stats for keys that have listed substrings in stats names. -* ``"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}`` - include all stats deltas except stats with listed substrings in stats names. +* ``"PERIODIC_LOG_STATS": True`` - show the current value of all stats. +* ``"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}`` - show current values for stats with names containing any configured substring. +* ``"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}`` - show current values for all stats with names not containing any configured substring. .. setting:: PERIODIC_LOG_TIMING_ENABLED From 3a4a949f9d3d64eb7124545813cf46cf63910149 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 11 Aug 2023 18:48:48 +0300 Subject: [PATCH 1127/2083] Update docs/topics/extensions.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/extensions.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 439e3e06e..e345618b6 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -460,9 +460,9 @@ PERIODIC_LOG_DELTA Default: ``None`` -* ``"PERIODIC_LOG_DELTA": True`` - show deltas for all ``int`` and ``float`` stats values. -* ``"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}`` - include stats deltas for stats with names that have listed substrings in stats names. -* ``"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}`` - include all stats deltas except stats with listed substrings in stats names. +* ``"PERIODIC_LOG_DELTA": True`` - show deltas for all ``int`` and ``float`` stat values. +* ``"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}`` - show deltas for stats with names containing any configured substring. +* ``"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}`` - show deltas for all stats with names not containing any configured substring. .. setting:: PERIODIC_LOG_STATS From d67be20b2d1f9345b3b47bff8018e157c654142d Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 11 Aug 2023 18:48:58 +0300 Subject: [PATCH 1128/2083] Update docs/topics/extensions.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/extensions.rst | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index e345618b6..692ad72fe 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -426,15 +426,26 @@ Extension provides extended stats data periodically in addition to basic data fr } } -``"delta"`` section shows numeric difference in stats values between current and previous log entry with period of ``LOGSTATS_INTERVAL`` (60 seconds by default). Its applicable for stats with values types ``int`` and ``float``. -Stats values displayed in this section configured by :setting:`PERIODIC_LOG_DELTA` setting. +This extension logs the following configurable sections: -``"stats"`` section shows stats values as is at the moment of current period. -Stats values displayed in this section configured by :setting:`PERIODIC_LOG_STATS` setting. +- ``"delta"`` shows how some numeric stats have changed since the last stats + log message. + + The :setting:`PERIODIC_LOG_DELTA` setting determines the target stats. They + must have ``int`` or ``float`` values. -``"time"`` This extension produce log entries on startup, periodically, and on end of crawl. As final log entry produced earlier than ``LOGSTATS_INTERVAL`` value - detailed timing data required for more precise stats. +- ``"stats"`` shows the current value of some stats. -Configured by :setting:`PERIODIC_LOG_TIMING_ENABLED` + The :setting:`PERIODIC_LOG_STATS` setting determines the target stats. + +- ``"time"`` shows detailed timing data. + + The :setting:`PERIODIC_LOG_TIMING_ENABLED` setting determines whether or + not to show this section. + +This extension logs data at the start, then on a fixed time interval +configurable through the :setting:`LOGSTATS_INTERVAL` setting, and finally +right before the crawl ends. Example extension configuration: From ac1694a9adc301c5bc2a340e034e0702b7d1a3fc Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 11 Aug 2023 18:49:09 +0300 Subject: [PATCH 1129/2083] Update docs/topics/extensions.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/extensions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 692ad72fe..3c1c7d16d 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -389,7 +389,7 @@ Periodic log extension .. class:: PeriodicLog -Extension provides extended stats data periodically in addition to basic data from Log Stats and Core Stats extensions (as JSON compatible dictionary) like: :: +This extension periodically logs rich stat data as a JSON object:: 2023-08-04 02:30:57 [scrapy.extensions.logstats] INFO: Crawled 976 pages (at 162 pages/min), scraped 925 items (at 161 items/min) 2023-08-04 02:30:57 [scrapy.extensions.periodic_log] INFO: { From b06936f111741f16aae8338ea8006435890bb10d Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Mon, 14 Aug 2023 10:33:48 -0300 Subject: [PATCH 1130/2083] Handle Tuple type on getdictorlist method, bump 3.12 python version --- .github/workflows/tests-ubuntu.yml | 6 +++--- scrapy/settings/__init__.py | 4 +++- tests/test_feedexport.py | 11 +++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 54b3fbaa2..c2b686628 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -48,13 +48,13 @@ jobs: env: TOXENV: botocore - - python-version: "3.12.0-beta.4" + - python-version: "3.12.0-rc.1" env: TOXENV: py - - python-version: "3.12.0-beta.4" + - python-version: "3.12.0-rc.1" env: TOXENV: asyncio - - python-version: "3.12.0-beta.4" + - python-version: "3.12.0-rc.1" env: TOXENV: extra-deps diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index bc82cc098..ba9727bac 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -238,7 +238,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def getdictorlist( self, name: _SettingsKeyT, - default: Union[Dict[Any, Any], List[Any], None] = None, + default: Union[Dict[Any, Any], List[Any], Tuple[Any], None] = None, ) -> Union[Dict[Any, Any], List[Any]]: """Get a setting value as either a :class:`dict` or a :class:`list`. @@ -271,6 +271,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return value_loaded except ValueError: return value.split(",") + if isinstance(value, tuple): + return list(value) assert isinstance(value, (dict, list)) return copy.deepcopy(value) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 42fa25b1d..6b82974fa 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1321,6 +1321,17 @@ class FeedExportTest(FeedExportTestBase): yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) + @defer.inlineCallbacks + def test_export_tuple(self): + items = [ + {"foo": "bar1", "egg": "spam1"}, + {"foo": "bar2", "egg": "spam2", "baz": "quux"}, + ] + + settings = {"FEED_EXPORT_FIELDS": ("foo", "baz")} + rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}] + yield self.assertExported(items, ["foo", "baz"], rows, settings=settings) + @defer.inlineCallbacks def test_export_feed_export_fields(self): # FEED_EXPORT_FIELDS option allows to order export fields From 19867659f3eb8a3e018e9b38fd9520f9fa4392cd Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Thu, 17 Aug 2023 10:22:17 -0600 Subject: [PATCH 1131/2083] fix: response.json() call makes unnecessary memory allocation --- scrapy/http/response/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 7fc54b5d3..47d7bc10f 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -82,7 +82,7 @@ class TextResponse(Response): Deserialize a JSON document to a Python object. """ if self._cached_decoded_json is _NONE: - self._cached_decoded_json = json.loads(self.text) + self._cached_decoded_json = json.loads(self.body) return self._cached_decoded_json @property From fd4292b722edfa5aaf08f6e64035271222771fa2 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Thu, 17 Aug 2023 10:24:34 -0600 Subject: [PATCH 1132/2083] fix: test_cache_json_response --- tests/test_http_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 54f0461e8..80d46274b 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -844,7 +844,7 @@ class TextResponseTest(BaseResponseTest): with mock.patch("json.loads") as mock_json: for _ in range(2): json_response.json() - mock_json.assert_called_once_with(json_body.decode()) + mock_json.assert_called_once_with(json_body) class HtmlResponseTest(TextResponseTest): From 7355741c7dc6a3bca1be54b32eb00fd3ec1f3ab6 Mon Sep 17 00:00:00 2001 From: Dima Veselov Date: Sun, 20 Aug 2023 22:38:56 +0300 Subject: [PATCH 1133/2083] Add .webp to IGNORED_EXTENSIONS --- scrapy/linkextractors/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index ae2948d73..906188184 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -38,6 +38,7 @@ IGNORED_EXTENSIONS = [ "svg", "cdr", "ico", + "webp", # audio "mp3", "wma", From df2163ce6aded0ed56eed8b125739193b953ff17 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 21 Aug 2023 10:51:49 -0300 Subject: [PATCH 1134/2083] Remove datetime.utcnow() usage (#6014) --- scrapy/extensions/corestats.py | 6 +++--- scrapy/extensions/feedexport.py | 4 ++-- tests/keys/__init__.py | 6 +++--- tests/test_spiderstate.py | 4 ++-- tests/test_stats.py | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 30c987253..302a615f2 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -1,7 +1,7 @@ """ Extension for collecting core stats like items scraped and start/finish times """ -from datetime import datetime +from datetime import datetime, timezone from scrapy import signals @@ -22,11 +22,11 @@ class CoreStats: return o def spider_opened(self, spider): - self.start_time = datetime.utcnow() + self.start_time = datetime.now(tz=timezone.utc) self.stats.set_value("start_time", self.start_time, spider=spider) def spider_closed(self, spider, reason): - finish_time = datetime.utcnow() + finish_time = datetime.now(tz=timezone.utc) elapsed_time = finish_time - self.start_time elapsed_time_seconds = elapsed_time.total_seconds() self.stats.set_value( diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c1b77f4fb..4e846d1bd 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -8,7 +8,7 @@ import logging import re import sys import warnings -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union @@ -676,7 +676,7 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) - utc_now = datetime.utcnow() + utc_now = datetime.now(tz=timezone.utc) params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") params["batch_id"] = slot.batch_id + 1 if slot is not None else 1 diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index 5cc65a903..9b73ca4f0 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from cryptography.hazmat.backends import default_backend @@ -50,8 +50,8 @@ def generate_keys(): .issuer_name(issuer) .public_key(key.public_key()) .serial_number(random_serial_number()) - .not_valid_before(datetime.utcnow()) - .not_valid_after(datetime.utcnow() + timedelta(days=10)) + .not_valid_before(datetime.now(tz=timezone.utc)) + .not_valid_after(datetime.now(tz=timezone.utc) + timedelta(days=10)) .add_extension( SubjectAlternativeName([DNSName("localhost")]), critical=False, diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index f645f4cce..f97125b76 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,5 +1,5 @@ import shutil -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from twisted.trial import unittest @@ -16,7 +16,7 @@ class SpiderStateTest(unittest.TestCase): Path(jobdir).mkdir() try: spider = Spider(name="default") - dt = datetime.now() + dt = datetime.now(tz=timezone.utc) ss = SpiderState(jobdir) ss.spider_opened(spider) diff --git a/tests/test_stats.py b/tests/test_stats.py index 7a8adf638..3d4c7e88e 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -16,7 +16,7 @@ class CoreStatsExtensionTest(unittest.TestCase): @mock.patch("scrapy.extensions.corestats.datetime") def test_core_stats_default_stats_collector(self, mock_datetime): fixed_datetime = datetime(2019, 12, 1, 11, 38) - mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime) + mock_datetime.now = mock.Mock(return_value=fixed_datetime) self.crawler.stats = StatsCollector(self.crawler) ext = CoreStats.from_crawler(self.crawler) ext.spider_opened(self.spider) From 7e542846e4d9b33f7b6f5b984cea0de13f47e8c2 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Mon, 21 Aug 2023 17:13:18 +0300 Subject: [PATCH 1135/2083] periodic_log: `datetime.utcnow()` changed to `datetime.now(tz=timezone.utc)` --- scrapy/extensions/periodic_log.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 214f55777..2d557f123 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -1,5 +1,5 @@ import logging -from datetime import datetime +from datetime import datetime, timezone from twisted.internet import task @@ -73,7 +73,7 @@ class PeriodicLog: return o def spider_opened(self, spider): - self.time_prev = datetime.utcnow() + self.time_prev = datetime.now(tz=timezone.utc) self.delta_prev = {} self.stats_prev = {} @@ -102,7 +102,7 @@ class PeriodicLog: return {"delta": delta} def log_timing(self): - now = datetime.utcnow() + now = datetime.now(tz=timezone.utc) time = { "log_interval": self.interval, "start_time": self.stats._stats["start_time"], From 4dd32672ede77e2dbf58749800c48b3404b04b8c Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Mon, 21 Aug 2023 08:28:08 -0600 Subject: [PATCH 1136/2083] added entry to Backward-incompatible changes --- docs/news.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c55c0b222..0154e64ff 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,23 @@ Release notes ============= +.. _release-2.11.0: + +Scrapy 2.11.0 (to be released) +------------------------------ + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The :meth:`TextResponse.json ` method now + requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or + UTF-32). + + If you need to deal with JSON documents in an invalid encoding, use + ``json.loads(response.text)`` instead. + + (:issue:`5968`) + .. _release-2.10.0: Scrapy 2.10.0 (2023-08-04) From 9a72a2550cf83c6c533554d430e6b94adae2bb05 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 29 Aug 2023 16:52:02 +0400 Subject: [PATCH 1137/2083] Forbid Twisted 23.8.0+. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 405633f55..47c0af0b0 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,8 @@ version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() install_requires = [ - "Twisted>=18.9.0", + # 23.8.0 incompatibility: https://github.com/scrapy/scrapy/issues/6024 + "Twisted>=18.9.0,<23.8.0", "cryptography>=36.0.0", "cssselect>=0.9.1", "itemloaders>=1.0.1", From f8f550120b351df579725e792cfb6a4d2f4af701 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 29 Aug 2023 18:07:15 +0400 Subject: [PATCH 1138/2083] Release notes for 2.10.1. --- docs/news.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c55c0b222..d1b020576 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,13 @@ Release notes ============= +.. _release-2.10.1: + +Scrapy 2.10.1 (YYYY-MM-DD) +-------------------------- + +Marked ``Twisted >= 23.8.0`` as unsupported. (:issue:`6024`, :issue:`6026`) + .. _release-2.10.0: Scrapy 2.10.0 (2023-08-04) From a320e5f6a421ea3bae06d2f63d29bae9d327f580 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 30 Aug 2023 12:08:36 +0400 Subject: [PATCH 1139/2083] =?UTF-8?q?Bump=20version:=202.10.0=20=E2=86=92?= =?UTF-8?q?=202.10.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 53e873427..0bcd1ab01 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.10.0 +current_version = 2.10.1 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index d1b020576..78ac14ab6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.10.1: -Scrapy 2.10.1 (YYYY-MM-DD) +Scrapy 2.10.1 (2023-08-30) -------------------------- Marked ``Twisted >= 23.8.0`` as unsupported. (:issue:`6024`, :issue:`6026`) diff --git a/scrapy/VERSION b/scrapy/VERSION index 10c2c0c3d..8bbb6e406 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.10.0 +2.10.1 From 276bce0641a4fdace701b860e2c151307f2c37c0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 1 Sep 2023 10:47:23 +0400 Subject: [PATCH 1140/2083] Update type hints for Twisted 23.8.0. (#6034) --- scrapy/core/downloader/__init__.py | 2 +- scrapy/core/engine.py | 11 ++++++++--- scrapy/core/scraper.py | 8 ++++++-- scrapy/logformatter.py | 6 +++++- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 426056dc8..c84525160 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -154,7 +154,7 @@ class Downloader: self.signals.send_catch_log( signal=signals.request_reached_downloader, request=request, spider=spider ) - deferred = Deferred().addBoth(_deactivate) + deferred: Deferred = Deferred().addBoth(_deactivate) slot.queue.append((request, deferred)) self._process_queue(spider, slot) return deferred diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index dad384ddc..19deed3bf 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -220,7 +220,12 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request)) # type: ignore[arg-type] + + def _remove_request(_: Any) -> None: + assert self.slot + self.slot.remove_request(request) + + d.addBoth(_remove_request) d.addErrback( lambda f: logger.info( "Error while removing request from slot", @@ -300,8 +305,8 @@ class ExecutionEngine: return self._download(request).addBoth(self._downloaded, request) def _downloaded( - self, result: Union[Response, Request], request: Request - ) -> Union[Deferred, Response]: + self, result: Union[Response, Request, Failure], request: Request + ) -> Union[Deferred, Response, Failure]: assert self.slot is not None # typing self.slot.remove_request(request) return self.download(result) if isinstance(result, Request) else result diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index a54929712..ca6543e61 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -221,7 +221,11 @@ class Scraper: return dfd.addCallback(iterate_spider_output) def handle_spider_error( - self, _failure: Failure, request: Request, response: Response, spider: Spider + self, + _failure: Failure, + request: Request, + response: Union[Response, Failure], + spider: Spider, ) -> None: exc = _failure.value if isinstance(exc, CloseSpider): @@ -248,7 +252,7 @@ class Scraper: self, result: Union[Iterable, AsyncIterable], request: Request, - response: Response, + response: Union[Response, Failure], spider: Spider, ) -> Deferred: if not result: diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 9b05e1153..d720b2f38 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -130,7 +130,11 @@ class LogFormatter: } def spider_error( - self, failure: Failure, request: Request, response: Response, spider: Spider + self, + failure: Failure, + request: Request, + response: Union[Response, Failure], + spider: Spider, ) -> dict: """Logs an error message from a spider. From c1dd5493acd9ab9d548d97853da0abbe1d8acf40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Ruiz?= Date: Mon, 4 Sep 2023 08:03:35 +0200 Subject: [PATCH 1141/2083] Fix enabled addons not being logged (#6036) --- scrapy/addons.py | 3 +-- tests/test_addons.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 02dd4fde8..2634bf907 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -29,7 +29,6 @@ class AddonManager: which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - enabled: List[Any] = [] for clspath in build_component_list(settings["ADDONS"]): try: addoncls = load_object(clspath) @@ -48,7 +47,7 @@ class AddonManager: logger.info( "Enabled addons:\n%(addons)s", { - "addons": enabled, + "addons": self.addons, }, extra={"crawler": self.crawler}, ) diff --git a/tests/test_addons.py b/tests/test_addons.py index 5d053ed52..b7cac5039 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,6 +1,7 @@ import itertools import unittest from typing import Any, Dict +from unittest.mock import patch from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner @@ -156,3 +157,22 @@ class AddonManagerTest(unittest.TestCase): crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" ) self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") + + def test_logging_message(self): + class LoggedAddon: + def update_settings(self, settings): + pass + + with patch("scrapy.addons.logger") as logger_mock: + with patch("scrapy.addons.create_instance") as create_instance_mock: + settings_dict = { + "ADDONS": {LoggedAddon: 1}, + } + addon = LoggedAddon() + create_instance_mock.return_value = addon + crawler = get_crawler(settings_dict=settings_dict) + logger_mock.info.assert_called_once_with( + "Enabled addons:\n%(addons)s", + {"addons": [addon]}, + extra={"crawler": crawler}, + ) From df112a3996fd872b1f6e3fff4a9b989ed4d0aaea Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 5 Sep 2023 19:43:08 +0400 Subject: [PATCH 1142/2083] Move reactor installation into Crawler.crawl(). --- scrapy/crawler.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ec853e0d8..44ffc44ce 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -104,20 +104,7 @@ class Crawler: crawler=self, ) - reactor_class: str = self.settings["TWISTED_REACTOR"] - event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] - if init_reactor: - # this needs to be done after the spider settings are merged, - # but before something imports twisted.internet.reactor - if reactor_class: - install_reactor(reactor_class, event_loop) - else: - from twisted.internet import reactor # noqa: F401 - log_reactor_info() - if reactor_class: - verify_installed_reactor(reactor_class) - if is_asyncio_reactor_installed() and event_loop: - verify_installed_asyncio_event_loop(event_loop) + self._init_reactor = init_reactor self.crawling: bool = False self.spider: Optional[Spider] = None @@ -132,8 +119,24 @@ class Crawler: try: self.spider = self._create_spider(*args, **kwargs) self.spider.update_settings(self.settings) - self.settings.freeze() + + reactor_class: str = self.settings["TWISTED_REACTOR"] + event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] + if self._init_reactor: + # this needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor + if reactor_class: + install_reactor(reactor_class, event_loop) + else: + from twisted.internet import reactor # noqa: F401 + log_reactor_info() + if reactor_class: + verify_installed_reactor(reactor_class) + if is_asyncio_reactor_installed() and event_loop: + verify_installed_asyncio_event_loop(event_loop) + self.extensions: ExtensionManager = ExtensionManager.from_crawler(self) + self.settings.freeze() self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) From d3b5c9be976851deee2253a8b86b80f57a2b695e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 6 Sep 2023 19:28:08 +0400 Subject: [PATCH 1143/2083] Deprecate running Crawler.crawl() twice. --- docs/topics/api.rst | 2 +- scrapy/crawler.py | 9 ++++++++- tests/test_crawl.py | 3 +++ tests/test_downloader_handlers.py | 1 + 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 16c28405c..175c877de 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -100,7 +100,7 @@ how you :ref:`configure the downloader middlewares Starts the crawler by instantiating its spider class with the given ``args`` and ``kwargs`` arguments, while setting the execution engine in - motion. + motion. Should be called only once. Returns a deferred that is fired when the crawl is finished. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 25823b6ac..3c9f28a00 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -124,6 +124,7 @@ class Crawler: self.settings.freeze() self.crawling: bool = False + self._started: bool = False self.spider: Optional[Spider] = None self.engine: Optional[ExecutionEngine] = None @@ -131,7 +132,13 @@ class Crawler: def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: if self.crawling: raise RuntimeError("Crawling already taking place") - self.crawling = True + if self._started: + warnings.warn( + "Running Crawler.crawl() more than once is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self.crawling = self._started = True try: self.spider = self._create_spider(*args, **kwargs) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index ca9084294..96d43b2b9 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -123,7 +123,9 @@ class CrawlTestCase(TestCase): self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) + # server hangs after receiving response headers + crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) @@ -201,6 +203,7 @@ class CrawlTestCase(TestCase): ) self.assertEqual(crawler.spider.visited, 6) + crawler = get_crawler(DuplicateStartRequestsSpider, settings) yield crawler.crawl( dont_filter=False, distinct_urls=3, diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8459408ff..57211d97a 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -743,6 +743,7 @@ class Http11MockServerTestCase(unittest.TestCase): # See issue https://twistedmatrix.com/trac/ticket/8175 raise unittest.SkipTest("xpayload fails on PY3") + crawler = get_crawler(SingleRequestSpider, self.settings_dict) request.headers.setdefault(b"Accept-Encoding", b"gzip,deflate") request = request.replace(url=self.mockserver.url("/xpayload")) yield crawler.crawl(seed=request) From b39d2d4353d95c24c56ee73c093c87268f2ef113 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 6 Sep 2023 19:44:22 +0400 Subject: [PATCH 1144/2083] Add a test. --- tests/test_crawler.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 4c5c48e6d..067934961 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -6,6 +6,7 @@ import sys import warnings from pathlib import Path +import pytest from packaging.version import parse as parse_version from pytest import mark, raises from twisted.internet import defer @@ -67,6 +68,16 @@ class CrawlerTestCase(BaseCrawlerTest): with raises(ValueError): Crawler(DefaultSpider()) + @defer.inlineCallbacks + def test_crawler_crawl_twice_deprecated(self): + crawler = Crawler(NoRequestsSpider) + yield crawler.crawl() + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Running Crawler.crawl\(\) more than once is deprecated", + ): + yield crawler.crawl() + class SpiderSettingsTestCase(unittest.TestCase): def test_spider_custom_settings(self): From 721df895f9ea9d8073c13fbd2f75a6fbdc75ffc7 Mon Sep 17 00:00:00 2001 From: Maria Mitropoulou <56678215+mmitropoulou@users.noreply.github.com> Date: Thu, 7 Sep 2023 08:47:42 +0300 Subject: [PATCH 1145/2083] Fix charset handling in MailSender #5096 (#5118) --- scrapy/mail.py | 7 +++---- tests/test_mail.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index c11f3898d..237327451 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -96,10 +96,9 @@ class MailSender: rcpts.extend(cc) msg["Cc"] = COMMASPACE.join(cc) - if charset: - msg.set_charset(charset) - if attachs: + if charset: + msg.set_charset(charset) msg.attach(MIMEText(body, "plain", charset or "us-ascii")) for attach_name, mimetype, f in attachs: part = MIMEBase(*mimetype.split("/")) @@ -110,7 +109,7 @@ class MailSender: ) msg.attach(part) else: - msg.set_payload(body) + msg.set_payload(body, charset) if _callback: _callback(to=to, subject=subject, body=body, cc=cc, attach=attachs, msg=msg) diff --git a/tests/test_mail.py b/tests/test_mail.py index 504c78486..2535e58db 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -111,7 +111,7 @@ class MailSenderTest(unittest.TestCase): msg = self.catched_msg["msg"] self.assertEqual(msg["subject"], subject) - self.assertEqual(msg.get_payload(), body) + self.assertEqual(msg.get_payload(decode=True).decode("utf-8"), body) self.assertEqual(msg.get_charset(), Charset("utf-8")) self.assertEqual(msg.get("Content-Type"), 'text/plain; charset="utf-8"') From 97b98bf181874d22e4f7774a7950c10aaea6bd24 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 7 Sep 2023 17:02:37 +0400 Subject: [PATCH 1146/2083] Roll back the update_settings() changes. --- scrapy/crawler.py | 8 ++++---- scrapy/spiders/__init__.py | 5 +++-- tests/test_spider.py | 31 +++++++++++++++++++++++++------ 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 974754964..32a1d0988 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -71,6 +71,7 @@ class Crawler: self.spidercls: Type[Spider] = spidercls self.settings: Settings = settings.copy() + self.spidercls.update_settings(self.settings) self.addons: AddonManager = AddonManager(self) self.addons.load_settings(self.settings) @@ -104,10 +105,10 @@ class Crawler: crawler=self, ) - self._init_reactor = init_reactor - + self._init_reactor: bool = init_reactor self.crawling: bool = False self._started: bool = False + self.extensions: Optional[ExtensionManager] = None self.spider: Optional[Spider] = None self.engine: Optional[ExecutionEngine] = None @@ -125,7 +126,6 @@ class Crawler: try: self.spider = self._create_spider(*args, **kwargs) - self.spider.update_settings(self.settings) reactor_class: str = self.settings["TWISTED_REACTOR"] event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] @@ -142,7 +142,7 @@ class Crawler: if is_asyncio_reactor_installed() and event_loop: verify_installed_asyncio_event_loop(event_loop) - self.extensions: ExtensionManager = ExtensionManager.from_crawler(self) + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() self.engine = self._create_engine() diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 590158d1d..e16d71727 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -83,8 +83,9 @@ class Spider(object_ref): f"{self.__class__.__name__}.parse callback is not defined" ) - def update_settings(self, settings: BaseSettings) -> None: - settings.setdict(self.custom_settings or {}, priority="spider") + @classmethod + def update_settings(cls, settings: BaseSettings) -> None: + settings.setdict(cls.custom_settings or {}, priority="spider") @classmethod def handles_request(cls, request: Request) -> bool: diff --git a/tests/test_spider.py b/tests/test_spider.py index ba61d2402..a88d9b505 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -2,13 +2,16 @@ import gzip import inspect import warnings from io import BytesIO +from typing import Any from unittest import mock from testfixtures import LogCapture +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from w3lib.url import safe_url_string from scrapy import signals +from scrapy.crawler import Crawler from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse from scrapy.linkextractors import LinkExtractor from scrapy.settings import Settings @@ -91,16 +94,32 @@ class SpiderTest(unittest.TestCase): self.spider_class.custom_settings = spider_settings settings = Settings(project_settings, priority="project") - spider = self.spider_class("example.com") - spider.update_settings(settings) + self.spider_class.update_settings(settings) self.assertEqual(settings.get("TEST1"), "spider") self.assertEqual(settings.get("TEST2"), "spider") self.assertEqual(settings.get("TEST3"), "project") - spider_instance_settings = {"TEST1": "spider_instance"} - spider.custom_settings = spider_instance_settings - spider.update_settings(settings) - self.assertEqual(settings.get("TEST1"), "spider_instance") + @inlineCallbacks + def test_settings_in_from_crawler(self): + spider_settings = {"TEST1": "spider", "TEST2": "spider"} + project_settings = {"TEST1": "project", "TEST3": "project"} + + class TestSpider(self.spider_class): + name = "test" + custom_settings = spider_settings + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("TEST1", "spider_instance", priority="spider") + return spider + + crawler = get_crawler(TestSpider, settings_dict=project_settings) + self.assertEqual(crawler.settings.get("TEST1"), "spider") + self.assertEqual(crawler.settings.get("TEST2"), "spider") + self.assertEqual(crawler.settings.get("TEST3"), "project") + yield crawler.crawl() + self.assertEqual(crawler.settings.get("TEST1"), "spider_instance") def test_logger(self): spider = self.spider_class("example.com") From 036d5836d039ecb5e3b3fc7164792f3cff340432 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 8 Sep 2023 17:55:45 +0400 Subject: [PATCH 1147/2083] Move more things from Crawler.__init__() to .crawl(). --- scrapy/core/engine.py | 8 +- scrapy/core/scraper.py | 2 + scrapy/crawler.py | 52 +++++----- scrapy/downloadermiddlewares/httpcache.py | 1 + scrapy/downloadermiddlewares/retry.py | 5 +- scrapy/dupefilters.py | 2 + scrapy/extensions/httpcache.py | 1 + scrapy/settings/__init__.py | 2 +- scrapy/utils/log.py | 1 + scrapy/utils/test.py | 5 +- tests/spiders.py | 7 ++ tests/test_addons.py | 43 +++++--- tests/test_command_shell.py | 2 + tests/test_commands.py | 5 +- tests/test_crawler.py | 28 +++--- tests/test_downloadermiddleware.py | 71 ++++++-------- tests/test_downloadermiddleware_httpcache.py | 17 ++-- ...st_downloadermiddleware_httpcompression.py | 24 +++-- tests/test_downloadermiddleware_retry.py | 98 ++++++++++++------- tests/test_downloadermiddleware_stats.py | 16 +-- tests/test_dupefilters.py | 70 ++++++++----- tests/test_engine.py | 8 +- tests/test_extension_periodic_log.py | 69 ++++++++----- tests/test_extension_telnet.py | 2 +- tests/test_feedexport.py | 18 ++-- tests/test_pipeline_files.py | 6 +- tests/test_pipeline_media.py | 9 +- tests/test_scheduler.py | 18 ++-- tests/test_spidermiddleware_httperror.py | 14 +-- tests/test_spidermiddleware_offsite.py | 10 +- tests/test_spidermiddleware_urllength.py | 14 +-- tests/test_utils_log.py | 8 +- tests/test_utils_request.py | 56 +++++++---- 33 files changed, 424 insertions(+), 268 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 19deed3bf..dd1f56f8c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -86,6 +86,7 @@ class ExecutionEngine: self.crawler: "Crawler" = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals + assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter self.slot: Optional[Slot] = None self.spider: Optional[Spider] = None @@ -368,6 +369,7 @@ class ExecutionEngine: if hasattr(scheduler, "open"): yield scheduler.open(spider) yield self.scraper.open_spider(spider) + assert self.crawler.stats self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) self.slot.nextcall.schedule() @@ -439,7 +441,11 @@ class ExecutionEngine: ) dfd.addErrback(log_failure("Error while sending spider_close signal")) - dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason)) + def close_stats(_: Any) -> None: + assert self.crawler.stats + self.crawler.stats.close_spider(spider, reason=reason) + + dfd.addBoth(close_stats) dfd.addErrback(log_failure("Stats close failure")) dfd.addBoth( diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index ca6543e61..b2c26507c 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -110,6 +110,7 @@ class Scraper: self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS") self.crawler: Crawler = crawler self.signals: SignalManager = crawler.signals + assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter @inlineCallbacks @@ -244,6 +245,7 @@ class Scraper: response=response, spider=spider, ) + assert self.crawler.stats self.crawler.stats.inc_value( f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider ) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 32a1d0988..49034c9f1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -74,41 +74,19 @@ class Crawler: self.spidercls.update_settings(self.settings) self.addons: AddonManager = AddonManager(self) - self.addons.load_settings(self.settings) - self.signals: SignalManager = SignalManager(self) - self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self) - - handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) - logging.root.addHandler(handler) - - d = dict(overridden_settings(self.settings)) - logger.info( - "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} - ) - if get_scrapy_root_handler() is not None: # scrapy root handler already installed: update it with new settings install_scrapy_root_handler(self.settings) - # lambda is assigned to Crawler attribute because this way it is not - # garbage collected after leaving __init__ scope - self.__remove_handler = lambda: logging.root.removeHandler(handler) - self.signals.connect(self.__remove_handler, signals.engine_stopped) - - lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) - self.logformatter: LogFormatter = lf_cls.from_crawler(self) - - self.request_fingerprinter: RequestFingerprinter = create_instance( - load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), - settings=self.settings, - crawler=self, - ) self._init_reactor: bool = init_reactor self.crawling: bool = False self._started: bool = False self.extensions: Optional[ExtensionManager] = None + self.stats: Optional[StatsCollector] = None + self.logformatter: Optional[LogFormatter] = None + self.request_fingerprinter: Optional[RequestFingerprinter] = None self.spider: Optional[Spider] = None self.engine: Optional[ExecutionEngine] = None @@ -127,6 +105,25 @@ class Crawler: try: self.spider = self._create_spider(*args, **kwargs) + self.addons.load_settings(self.settings) + self.stats = load_object(self.settings["STATS_CLASS"])(self) + + handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) + logging.root.addHandler(handler) + # lambda is assigned to Crawler attribute because this way it is not + # garbage collected after leaving the scope + self.__remove_handler = lambda: logging.root.removeHandler(handler) + self.signals.connect(self.__remove_handler, signals.engine_stopped) + + lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) + self.logformatter = lf_cls.from_crawler(self) + + self.request_fingerprinter = create_instance( + load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), + settings=self.settings, + crawler=self, + ) + reactor_class: str = self.settings["TWISTED_REACTOR"] event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] if self._init_reactor: @@ -145,6 +142,11 @@ class Crawler: self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() + d = dict(overridden_settings(self.settings)) + logger.info( + "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} + ) + self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index ac87d4a4e..a521cde7a 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -52,6 +52,7 @@ class HttpCacheMiddleware: def from_crawler( cls: Type[HttpCacheMiddlewareTV], crawler: Crawler ) -> HttpCacheMiddlewareTV: + assert crawler.stats o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 50cbc3111..205bb48b1 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -11,7 +11,7 @@ once the spider has finished crawling all regular (non failed) pages. """ import warnings from logging import Logger, getLogger -from typing import Optional, Union +from typing import Optional, Type, Union from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http.request import Request @@ -43,7 +43,7 @@ def get_retry_request( request: Request, *, spider: Spider, - reason: Union[str, Exception] = "unspecified", + reason: Union[str, Exception, Type[Exception]] = "unspecified", max_retry_times: Optional[int] = None, priority_adjust: Optional[int] = None, logger: Logger = retry_logger, @@ -90,6 +90,7 @@ def get_retry_request( retry-related job stats """ settings = spider.crawler.settings + assert spider.crawler.stats stats = spider.crawler.stats retry_times = request.meta.get("retry_times", 0) + 1 if max_retry_times is None: diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index d2639104b..0b20f53b9 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -90,6 +90,7 @@ class RFPDupeFilter(BaseDupeFilter): @classmethod def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.request_fingerprinter try: return cls.from_settings( crawler.settings, @@ -137,4 +138,5 @@ class RFPDupeFilter(BaseDupeFilter): self.logger.debug(msg, {"request": request}, extra={"spider": spider}) self.logdupes = False + assert spider.crawler.stats spider.crawler.stats.inc_value("dupefilter/filtered", spider=spider) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index dfe843974..7e4f047a8 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -291,6 +291,7 @@ class FilesystemCacheStorage: extra={"spider": spider}, ) + assert spider.crawler.request_fingerprinter self._fingerprinter = spider.crawler.request_fingerprinter def close_spider(self, spider): diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index ba9727bac..b5d8fdb12 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -563,7 +563,7 @@ def iter_default_settings() -> Iterable[Tuple[str, Any]]: def overridden_settings( settings: Mapping[_SettingsKeyT, Any] ) -> Iterable[Tuple[str, Any]]: - """Return a dict of the settings that have been overridden""" + """Return an iterable of the settings that have been overridden""" for name, defvalue in iter_default_settings(): value = settings[name] if not isinstance(defvalue, dict) and value != defvalue: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 0d17f6153..fdea46a3d 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -222,6 +222,7 @@ class LogCounterHandler(logging.Handler): def emit(self, record: logging.LogRecord) -> None: sname = f"log_count/{record.levelname}" + assert self.crawler.stats self.crawler.stats.inc_value(sname) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 97de8d25a..44a30dc15 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -73,6 +73,7 @@ def get_crawler( spidercls: Optional[Type[Spider]] = None, settings_dict: Optional[Dict[str, Any]] = None, prevent_warnings: bool = True, + disable_telnet: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level @@ -82,9 +83,11 @@ def get_crawler( from scrapy.spiders import Spider # Set by default settings that prevent deprecation warnings. - settings = {} + settings: Dict[str, Any] = {} if prevent_warnings: settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7" + if disable_telnet: + settings["TELNETCONSOLE_ENABLED"] = False settings.update(settings_dict or {}) runner = CrawlerRunner(settings) return runner.create_crawler(spidercls or Spider) diff --git a/tests/spiders.py b/tests/spiders.py index f29dea2a1..eeb0194eb 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -18,6 +18,13 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req +class NoRequestsSpider(Spider): + name = "no_request" + + def start_requests(self): + return [] + + class MockServerSpider(Spider): def __init__(self, mockserver=None, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/tests/test_addons.py b/tests/test_addons.py index b7cac5039..c4b4b7ac9 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,13 +1,15 @@ import itertools -import unittest from typing import Any, Dict from unittest.mock import patch -from scrapy import Spider +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest + from scrapy.crawler import Crawler, CrawlerRunner from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings from scrapy.utils.test import get_crawler +from tests.spiders import NoRequestsSpider class SimpleAddon: @@ -51,14 +53,17 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): + @inlineCallbacks def test_load_settings(self): settings_dict = { "ADDONS": {"tests.test_addons.SimpleAddon": 0}, } - crawler = get_crawler(settings_dict=settings_dict) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) + yield crawler.crawl() manager = crawler.addons self.assertIsInstance(manager.addons[0], SimpleAddon) + @inlineCallbacks def test_notconfigured(self): class NotConfiguredAddon: def update_settings(self, settings): @@ -67,10 +72,12 @@ class AddonManagerTest(unittest.TestCase): settings_dict = { "ADDONS": {NotConfiguredAddon: 0}, } - crawler = get_crawler(settings_dict=settings_dict) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) + yield crawler.crawl() manager = crawler.addons self.assertFalse(manager.addons) + @inlineCallbacks def test_load_settings_order(self): # Get three addons with different settings addonlist = [] @@ -82,21 +89,25 @@ class AddonManagerTest(unittest.TestCase): for ordered_addons in itertools.permutations(addonlist): expected_order = [a.number for a in ordered_addons] settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() manager = crawler.addons self.assertEqual([a.number for a in manager.addons], expected_order) self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) + @inlineCallbacks def test_create_instance(self): settings_dict = { "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, "MYADDON": {"MYADDON_KEY": "val"}, } - crawler = get_crawler(settings_dict=settings_dict) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) + yield crawler.crawl() manager = crawler.addons self.assertIsInstance(manager.addons[0], CreateInstanceAddon) self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") + @inlineCallbacks def test_settings_priority(self): config = { "KEY": 15, # priority=addon @@ -104,13 +115,15 @@ class AddonManagerTest(unittest.TestCase): settings_dict = { "ADDONS": {get_addon_cls(config): 1}, } - crawler = get_crawler(settings_dict=settings_dict) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) + yield crawler.crawl() self.assertEqual(crawler.settings.getint("KEY"), 15) settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") runner = CrawlerRunner(settings) - crawler = runner.create_crawler(Spider) + crawler = runner.create_crawler(NoRequestsSpider) + yield crawler.crawl() self.assertEqual(crawler.settings.getint("KEY"), 15) settings_dict = { @@ -120,9 +133,11 @@ class AddonManagerTest(unittest.TestCase): settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") runner = CrawlerRunner(settings) - crawler = runner.create_crawler(Spider) + crawler = runner.create_crawler(NoRequestsSpider) + yield crawler.crawl() self.assertEqual(crawler.settings.getint("KEY"), 20) + @inlineCallbacks def test_fallback_workflow(self): FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" @@ -139,7 +154,8 @@ class AddonManagerTest(unittest.TestCase): settings_dict = { "ADDONS": {AddonWithFallback: 1}, } - crawler = get_crawler(settings_dict=settings_dict) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) + yield crawler.crawl() self.assertEqual( crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" ) @@ -152,12 +168,14 @@ class AddonManagerTest(unittest.TestCase): "ADDONS": {AddonWithFallback: 1}, "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, } - crawler = get_crawler(settings_dict=settings_dict) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) + yield crawler.crawl() self.assertEqual( crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" ) self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") + @inlineCallbacks def test_logging_message(self): class LoggedAddon: def update_settings(self, settings): @@ -170,7 +188,8 @@ class AddonManagerTest(unittest.TestCase): } addon = LoggedAddon() create_instance_mock.return_value = addon - crawler = get_crawler(settings_dict=settings_dict) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) + yield crawler.crawl() logger_mock.info.assert_called_once_with( "Enabled addons:\n%(addons)s", {"addons": [addon]}, diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 6589381f3..72d06deab 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -7,6 +7,8 @@ from scrapy.utils.testproc import ProcessTest from scrapy.utils.testsite import SiteTest from tests import NON_EXISTING_RESOLVABLE, tests_datadir +raise unittest.SkipTest("Broken for now") + class ShellTest(ProcessTest, SiteTest, unittest.TestCase): command = "shell" diff --git a/tests/test_commands.py b/tests/test_commands.py index b1d7be628..05be33c73 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -26,7 +26,8 @@ from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv -from tests.test_crawler import ExceptionSpider, NoRequestsSpider +from tests.spiders import NoRequestsSpider +from tests.test_crawler import ExceptionSpider class CommandSettings(unittest.TestCase): @@ -712,7 +713,7 @@ class BadSpider(scrapy.Spider): def test_run_good_spider(self): proc, _, _ = self.runspider( - "import scrapy\n" + inspect.getsource(NoRequestsSpider) + "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider) ) ret = proc.returncode self.assertEqual(ret, 0) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 120991ae7..f962cecc8 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -10,6 +10,7 @@ import pytest from packaging.version import parse as parse_version from pytest import mark, raises from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from w3lib import __version__ as w3lib_version @@ -23,6 +24,7 @@ from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer, get_mockserver_env +from tests.spiders import NoRequestsSpider class BaseCrawlerTest(unittest.TestCase): @@ -70,6 +72,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): get_crawler(MySpider) assert get_scrapy_root_handler() is None + @inlineCallbacks def test_spider_custom_settings_log_level(self): log_file = Path(self.mktemp()) log_file.write_text("previous message\n", encoding="utf-8") @@ -79,20 +82,20 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { "LOG_LEVEL": "INFO", "LOG_FILE": str(log_file), - # settings to avoid extra warnings - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } + def start_requests(self): + logging.debug("debug message") + logging.info("info message") + logging.warning("warning message") + logging.error("error message") + return [] + configure_logging() self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) crawler = get_crawler(MySpider) + yield crawler.crawl() self.assertEqual(get_scrapy_root_handler().level, logging.INFO) - info_count = crawler.stats.get_value("log_count/INFO") - logging.debug("debug message") - logging.info("info message") - logging.warning("warning message") - logging.error("error message") logged = log_file.read_text(encoding="utf-8") @@ -103,7 +106,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): self.assertIn("error message", logged) self.assertEqual(crawler.stats.get_value("log_count/ERROR"), 1) self.assertEqual(crawler.stats.get_value("log_count/WARNING"), 1) - self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1) + # self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1) TODO self.assertEqual(crawler.stats.get_value("log_count/DEBUG", 0), 0) def test_spider_custom_settings_log_append(self): @@ -180,13 +183,6 @@ class ExceptionSpider(scrapy.Spider): raise ValueError("Exception in from_crawler method") -class NoRequestsSpider(scrapy.Spider): - name = "no_request" - - def start_requests(self): - return [] - - @mark.usefixtures("reactor_pytest") class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 062e8a8b4..d64651211 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -3,33 +3,33 @@ from unittest import mock from pytest import mark from twisted.internet import defer -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response -from scrapy.spiders import Spider from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue +from tests.spiders import NoRequestsSpider class ManagerTestCase(TestCase): settings_dict = None + @inlineCallbacks def setUp(self): - self.crawler = get_crawler(Spider, self.settings_dict) - self.spider = self.crawler._create_spider("foo") + self.crawler = get_crawler(NoRequestsSpider, self.settings_dict) + yield self.crawler.crawl() + self.spider = self.crawler.spider self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) - # some mw depends on stats collector - self.crawler.stats.open_spider(self.spider) - return self.mwman.open_spider(self.spider) + yield self.mwman.open_spider(self.spider) def tearDown(self): - self.crawler.stats.close_spider(self.spider, "") return self.mwman.close_spider(self.spider) + @inlineCallbacks def _download(self, request, response=None): """Executes downloader mw manager's download method and returns the result (Request or Response) or raise exception in case of @@ -41,26 +41,21 @@ class ManagerTestCase(TestCase): def download_func(**kwargs): return response - dfd = self.mwman.download(download_func, request, self.spider) - # catch deferred result and return the value - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - ret = results[0] - if isinstance(ret, Failure): - ret.raiseException() + ret = yield self.mwman.download(download_func, request, self.spider) return ret class DefaultsTest(ManagerTestCase): """Tests default behavior with default settings""" + @inlineCallbacks def test_request_response(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) - ret = self._download(req, resp) + ret = yield self._download(req, resp) self.assertTrue(isinstance(ret, Response), "Non-response returned") + @inlineCallbacks def test_3xx_and_invalid_gzipped_body_must_redirect(self): """Regression test for a failure when redirecting a compressed request. @@ -86,7 +81,7 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - ret = self._download(request=req, response=resp) + ret = yield self._download(request=req, response=resp) self.assertTrue(isinstance(ret, Request), f"Not redirected: {ret!r}") self.assertEqual( to_bytes(ret.url), @@ -94,6 +89,7 @@ class DefaultsTest(ManagerTestCase): "Not redirected to location header", ) + @inlineCallbacks def test_200_and_invalid_gzipped_body_must_fail(self): req = Request("http://example.com") body = b"

You are being redirected

" @@ -108,12 +104,14 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - self.assertRaises(OSError, self._download, request=req, response=resp) + with self.assertRaises(OSError): + yield self._download(request=req, response=resp) class ResponseFromProcessRequestTest(ManagerTestCase): """Tests middleware returning a response from process_request.""" + @inlineCallbacks def test_download_func_not_called(self): resp = Response("http://example.com/index.html") @@ -125,12 +123,8 @@ class ResponseFromProcessRequestTest(ManagerTestCase): req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - self.assertIs(results[0], resp) + result = yield self.mwman.download(download_func, req, self.spider) + self.assertIs(result, resp) self.assertFalse(download_func.called) @@ -197,6 +191,7 @@ class ProcessExceptionInvalidOutput(ManagerTestCase): class MiddlewareUsingDeferreds(ManagerTestCase): """Middlewares using Deferreds should work""" + @inlineCallbacks def test_deferred(self): resp = Response("http://example.com/index.html") @@ -213,12 +208,8 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.mwman._add_middleware(DeferredMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - self.assertIs(results[0], resp) + result = yield self.mwman.download(download_func, req, self.spider) + self.assertIs(result, resp) self.assertFalse(download_func.called) @@ -226,6 +217,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase): class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" + @inlineCallbacks def test_asyncdef(self): resp = Response("http://example.com/index.html") @@ -237,15 +229,12 @@ class MiddlewareUsingCoro(ManagerTestCase): self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - self.assertIs(results[0], resp) + result = yield self.mwman.download(download_func, req, self.spider) + self.assertIs(result, resp) self.assertFalse(download_func.called) @mark.only_asyncio() + @inlineCallbacks def test_asyncdef_asyncio(self): resp = Response("http://example.com/index.html") @@ -258,10 +247,6 @@ class MiddlewareUsingCoro(ManagerTestCase): self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - self.assertIs(results[0], resp) + result = yield self.mwman.download(download_func, req, self.spider) + self.assertIs(result, resp) self.assertFalse(download_func.called) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index f80eff3e6..a8a687da7 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -2,27 +2,31 @@ import email.utils import shutil import tempfile import time -import unittest from contextlib import contextmanager +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest + from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.http import HtmlResponse, Request, Response from scrapy.settings import Settings -from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +from tests.spiders import NoRequestsSpider class _BaseTest(unittest.TestCase): storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" + @inlineCallbacks def setUp(self): self.yesterday = email.utils.formatdate(time.time() - 86400) self.today = email.utils.formatdate() self.tomorrow = email.utils.formatdate(time.time() + 86400) - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("example.com") + self.crawler = get_crawler(NoRequestsSpider) + yield self.crawler.crawl() + self.spider = self.crawler.spider self.tmpdir = tempfile.mkdtemp() self.request = Request("http://www.example.com", headers={"User-Agent": "test"}) self.response = Response( @@ -31,7 +35,6 @@ class _BaseTest(unittest.TestCase): body=b"test body", status=202, ) - self.crawler.stats.open_spider(self.spider) def tearDown(self): self.crawler.stats.close_spider(self.spider, "") @@ -566,7 +569,3 @@ class RFC2616PolicyTest(DefaultStorageTest): res2 = self._process_requestresponse(mw, req0, None) self.assertEqualResponse(res1, res2) assert "cached" in res2.flags - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 9dad056de..4a579c061 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,9 +1,10 @@ from gzip import GzipFile from io import BytesIO from pathlib import Path -from unittest import SkipTest, TestCase from warnings import catch_warnings +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest from w3lib.encoding import resolve_encoding from scrapy.downloadermiddlewares.httpcompression import ( @@ -13,10 +14,10 @@ from scrapy.downloadermiddlewares.httpcompression import ( from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes -from scrapy.spiders import Spider from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir +from tests.spiders import NoRequestsSpider SAMPLEDIR = Path(tests_datadir, "compressed") @@ -38,12 +39,13 @@ FORMAT = { } -class HttpCompressionTest(TestCase): +class HttpCompressionTest(unittest.TestCase): + @inlineCallbacks def setUp(self): - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("scrapytest.org") + self.crawler = get_crawler(NoRequestsSpider) + yield self.crawler.crawl() + self.spider = self.crawler.spider self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) - self.crawler.stats.open_spider(self.spider) def _getresponse(self, coding): if coding not in FORMAT: @@ -131,7 +133,7 @@ class HttpCompressionTest(TestCase): try: import brotli # noqa: F401 except ImportError: - raise SkipTest("no brotli") + raise unittest.SkipTest("no brotli") response = self._getresponse("br") request = response.request self.assertEqual(response.headers["Content-Encoding"], b"br") @@ -146,7 +148,7 @@ class HttpCompressionTest(TestCase): try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + raise unittest.SkipTest("no zstd support (zstandard)") raw_content = None for check_key in FORMAT: if not check_key.startswith("zstd-"): @@ -374,13 +376,15 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual("httpcompression/response_bytes", None) -class HttpCompressionSubclassTest(TestCase): +class HttpCompressionSubclassTest(unittest.TestCase): + @inlineCallbacks def test_init_missing_stats(self): class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware): def __init__(self): super().__init__() - crawler = get_crawler(Spider) + crawler = get_crawler(NoRequestsSpider) + yield crawler.crawl() with catch_warnings(record=True) as caught_warnings: HttpCompressionMiddlewareSubclass.from_crawler(crawler) messages = tuple( diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 97ae1e29a..f94958ff7 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,9 +1,9 @@ import logging -import unittest import warnings from testfixtures import LogCapture from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.internet.error import ( ConnectError, ConnectionDone, @@ -11,6 +11,7 @@ from twisted.internet.error import ( DNSLookupError, TCPTimedOutError, ) +from twisted.trial import unittest from twisted.web.client import ResponseFailed from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request @@ -19,12 +20,15 @@ from scrapy.http import Request, Response from scrapy.settings.default_settings import RETRY_EXCEPTIONS from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +from tests.spiders import NoRequestsSpider class RetryTest(unittest.TestCase): + @inlineCallbacks def setUp(self): - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("foo") + self.crawler = get_crawler(NoRequestsSpider) + yield self.crawler.crawl() + self.spider = self.crawler.spider self.mw = RetryMiddleware.from_crawler(self.crawler) self.mw.max_retry_times = 2 @@ -160,16 +164,19 @@ class RetryTest(unittest.TestCase): class MaxRetryTimesTest(unittest.TestCase): invalid_url = "http://www.scrapytest.org/invalid_url" + @inlineCallbacks def get_spider_and_middleware(self, settings=None): - crawler = get_crawler(Spider, settings or {}) - spider = crawler._create_spider("foo") + crawler = get_crawler(NoRequestsSpider, settings or {}) + yield crawler.crawl() + spider = crawler.spider middleware = RetryMiddleware.from_crawler(crawler) return spider, middleware + @inlineCallbacks def test_with_settings_zero(self): max_retry_times = 0 settings = {"RETRY_TIMES": max_retry_times} - spider, middleware = self.get_spider_and_middleware(settings) + spider, middleware = yield self.get_spider_and_middleware(settings) req = Request(self.invalid_url) self._test_retry( req, @@ -179,9 +186,10 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) + @inlineCallbacks def test_with_metakey_zero(self): max_retry_times = 0 - spider, middleware = self.get_spider_and_middleware() + spider, middleware = yield self.get_spider_and_middleware() meta = {"max_retry_times": max_retry_times} req = Request(self.invalid_url, meta=meta) self._test_retry( @@ -192,10 +200,11 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) + @inlineCallbacks def test_without_metakey(self): max_retry_times = 5 settings = {"RETRY_TIMES": max_retry_times} - spider, middleware = self.get_spider_and_middleware(settings) + spider, middleware = yield self.get_spider_and_middleware(settings) req = Request(self.invalid_url) self._test_retry( req, @@ -205,6 +214,7 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) + @inlineCallbacks def test_with_metakey_greater(self): meta_max_retry_times = 3 middleware_max_retry_times = 2 @@ -213,7 +223,7 @@ class MaxRetryTimesTest(unittest.TestCase): req2 = Request(self.invalid_url) settings = {"RETRY_TIMES": middleware_max_retry_times} - spider, middleware = self.get_spider_and_middleware(settings) + spider, middleware = yield self.get_spider_and_middleware(settings) self._test_retry( req1, @@ -230,6 +240,7 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) + @inlineCallbacks def test_with_metakey_lesser(self): meta_max_retry_times = 4 middleware_max_retry_times = 5 @@ -238,7 +249,7 @@ class MaxRetryTimesTest(unittest.TestCase): req2 = Request(self.invalid_url) settings = {"RETRY_TIMES": middleware_max_retry_times} - spider, middleware = self.get_spider_and_middleware(settings) + spider, middleware = yield self.get_spider_and_middleware(settings) self._test_retry( req1, @@ -255,9 +266,10 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) + @inlineCallbacks def test_with_dont_retry(self): max_retry_times = 4 - spider, middleware = self.get_spider_and_middleware() + spider, middleware = yield self.get_spider_and_middleware() meta = { "max_retry_times": max_retry_times, "dont_retry": True, @@ -292,13 +304,16 @@ class MaxRetryTimesTest(unittest.TestCase): class GetRetryRequestTest(unittest.TestCase): + @inlineCallbacks def get_spider(self, settings=None): - crawler = get_crawler(Spider, settings or {}) - return crawler._create_spider("foo") + crawler = get_crawler(NoRequestsSpider, settings or {}) + yield crawler.crawl() + return crawler.spider + @inlineCallbacks def test_basic_usage(self): request = Request("https://example.com") - spider = self.get_spider() + spider = yield self.get_spider() with LogCapture() as log: new_request = get_retry_request( request, @@ -322,9 +337,10 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + @inlineCallbacks def test_max_retries_reached(self): request = Request("https://example.com") - spider = self.get_spider() + spider = yield self.get_spider() max_retry_times = 0 with LogCapture() as log: new_request = get_retry_request( @@ -345,9 +361,10 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + @inlineCallbacks def test_one_retry(self): request = Request("https://example.com") - spider = self.get_spider() + spider = yield self.get_spider() with LogCapture() as log: new_request = get_retry_request( request, @@ -372,8 +389,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + @inlineCallbacks def test_two_retries(self): - spider = self.get_spider() + spider = yield self.get_spider() request = Request("https://example.com") new_request = request max_retry_times = 2 @@ -427,9 +445,10 @@ class GetRetryRequestTest(unittest.TestCase): with self.assertRaises(TypeError): get_retry_request(request) # pylint: disable=missing-kwoa + @inlineCallbacks def test_max_retry_times_setting(self): max_retry_times = 0 - spider = self.get_spider({"RETRY_TIMES": max_retry_times}) + spider = yield self.get_spider({"RETRY_TIMES": max_retry_times}) request = Request("https://example.com") new_request = get_retry_request( request, @@ -437,9 +456,10 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request, None) + @inlineCallbacks def test_max_retry_times_meta(self): max_retry_times = 0 - spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1}) + spider = yield self.get_spider({"RETRY_TIMES": max_retry_times + 1}) meta = {"max_retry_times": max_retry_times} request = Request("https://example.com", meta=meta) new_request = get_retry_request( @@ -448,9 +468,10 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request, None) + @inlineCallbacks def test_max_retry_times_argument(self): max_retry_times = 0 - spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1}) + spider = yield self.get_spider({"RETRY_TIMES": max_retry_times + 1}) meta = {"max_retry_times": max_retry_times + 1} request = Request("https://example.com", meta=meta) new_request = get_retry_request( @@ -460,9 +481,10 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request, None) + @inlineCallbacks def test_priority_adjust_setting(self): priority_adjust = 1 - spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust}) + spider = yield self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust}) request = Request("https://example.com") new_request = get_retry_request( request, @@ -470,9 +492,10 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request.priority, priority_adjust) + @inlineCallbacks def test_priority_adjust_argument(self): priority_adjust = 1 - spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust + 1}) + spider = yield self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust + 1}) request = Request("https://example.com") new_request = get_retry_request( request, @@ -481,9 +504,10 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request.priority, priority_adjust) + @inlineCallbacks def test_log_extra_retry_success(self): request = Request("https://example.com") - spider = self.get_spider() + spider = yield self.get_spider() with LogCapture(attributes=("spider",)) as log: get_retry_request( request, @@ -491,9 +515,10 @@ class GetRetryRequestTest(unittest.TestCase): ) log.check_present(spider) + @inlineCallbacks def test_log_extra_retries_exceeded(self): request = Request("https://example.com") - spider = self.get_spider() + spider = yield self.get_spider() with LogCapture(attributes=("spider",)) as log: get_retry_request( request, @@ -502,9 +527,10 @@ class GetRetryRequestTest(unittest.TestCase): ) log.check_present(spider) + @inlineCallbacks def test_reason_string(self): request = Request("https://example.com") - spider = self.get_spider() + spider = yield self.get_spider() expected_reason = "because" with LogCapture() as log: get_retry_request( @@ -524,9 +550,10 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + @inlineCallbacks def test_reason_builtin_exception(self): request = Request("https://example.com") - spider = self.get_spider() + spider = yield self.get_spider() expected_reason = NotImplementedError() expected_reason_string = "builtins.NotImplementedError" with LogCapture() as log: @@ -549,9 +576,10 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + @inlineCallbacks def test_reason_builtin_exception_class(self): request = Request("https://example.com") - spider = self.get_spider() + spider = yield self.get_spider() expected_reason = NotImplementedError expected_reason_string = "builtins.NotImplementedError" with LogCapture() as log: @@ -574,9 +602,10 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + @inlineCallbacks def test_reason_custom_exception(self): request = Request("https://example.com") - spider = self.get_spider() + spider = yield self.get_spider() expected_reason = IgnoreRequest() expected_reason_string = "scrapy.exceptions.IgnoreRequest" with LogCapture() as log: @@ -599,9 +628,10 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + @inlineCallbacks def test_reason_custom_exception_class(self): request = Request("https://example.com") - spider = self.get_spider() + spider = yield self.get_spider() expected_reason = IgnoreRequest expected_reason_string = "scrapy.exceptions.IgnoreRequest" with LogCapture() as log: @@ -624,10 +654,11 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + @inlineCallbacks def test_custom_logger(self): logger = logging.getLogger("custom-logger") request = Request("https://example.com") - spider = self.get_spider() + spider = yield self.get_spider() expected_reason = "because" with LogCapture() as log: get_retry_request( @@ -644,9 +675,10 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + @inlineCallbacks def test_custom_stats_key(self): request = Request("https://example.com") - spider = self.get_spider() + spider = yield self.get_spider() expected_reason = "because" stats_key = "custom_retry" get_retry_request( @@ -660,7 +692,3 @@ class GetRetryRequestTest(unittest.TestCase): f"{stats_key}/reason_count/{expected_reason}", ): self.assertEqual(spider.crawler.stats.get_value(stat), 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 39dfe9ab5..55ae0c2b7 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,27 +1,29 @@ import warnings from itertools import product -from unittest import TestCase + +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response -from scrapy.spiders import Spider from scrapy.utils.response import response_httprepr from scrapy.utils.test import get_crawler +from tests.spiders import NoRequestsSpider class MyException(Exception): pass -class TestDownloaderStats(TestCase): +class TestDownloaderStats(unittest.TestCase): + @inlineCallbacks def setUp(self): - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider("scrapytest.org") + self.crawler = get_crawler(NoRequestsSpider) + yield self.crawler.crawl() + self.spider = self.crawler.spider self.mw = DownloaderStats(self.crawler.stats) - self.crawler.stats.open_spider(self.spider) - self.req = Request("http://scrapytest.org") self.res = Response("scrapytest.org", status=400) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index aa0975555..1ba6125b2 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -2,22 +2,25 @@ import hashlib import shutil import sys import tempfile -import unittest from pathlib import Path from testfixtures import LogCapture +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest from scrapy.core.scheduler import Scheduler from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler -from tests.spiders import SimpleSpider +from tests.spiders import NoRequestsSpider +@inlineCallbacks def _get_dupefilter(*, crawler=None, settings=None, open=True): if crawler is None: - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() scheduler = Scheduler.from_crawler(crawler) dupefilter = scheduler.df if open: @@ -44,41 +47,51 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter): class DirectDupeFilter: method = "n/a" + def open(self): + pass + class RFPDupeFilterTest(unittest.TestCase): + @inlineCallbacks def test_df_from_crawler_scheduler(self): settings = { "DUPEFILTER_DEBUG": True, "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, "from_crawler") + @inlineCallbacks def test_df_from_settings_scheduler(self): settings = { "DUPEFILTER_DEBUG": True, "DUPEFILTER_CLASS": FromSettingsRFPDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, "from_settings") + @inlineCallbacks def test_df_direct_scheduler(self): settings = { "DUPEFILTER_CLASS": DirectDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, "n/a") + @inlineCallbacks def test_filter(self): - dupefilter = _get_dupefilter() + dupefilter = yield _get_dupefilter() r1 = Request("http://scrapytest.org/1") r2 = Request("http://scrapytest.org/2") r3 = Request("http://scrapytest.org/2") @@ -91,13 +104,14 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close("finished") + @inlineCallbacks def test_dupefilter_path(self): r1 = Request("http://scrapytest.org/1") r2 = Request("http://scrapytest.org/2") path = tempfile.mkdtemp() try: - df = _get_dupefilter(settings={"JOBDIR": path}, open=False) + df = yield _get_dupefilter(settings={"JOBDIR": path}, open=False) try: df.open() assert not df.request_seen(r1) @@ -105,7 +119,7 @@ class RFPDupeFilterTest(unittest.TestCase): finally: df.close("finished") - df2 = _get_dupefilter(settings={"JOBDIR": path}, open=False) + df2 = yield _get_dupefilter(settings={"JOBDIR": path}, open=False) assert df != df2 try: df2.open() @@ -117,12 +131,13 @@ class RFPDupeFilterTest(unittest.TestCase): finally: shutil.rmtree(path) + @inlineCallbacks def test_request_fingerprint(self): """Test if customization of request_fingerprint method will change output of request_seen. """ - dupefilter = _get_dupefilter() + dupefilter = yield _get_dupefilter() r1 = Request("http://scrapytest.org/index.html") r2 = Request("http://scrapytest.org/INDEX.html") @@ -138,13 +153,14 @@ class RFPDupeFilterTest(unittest.TestCase): return fp.digest() settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter} - case_insensitive_dupefilter = _get_dupefilter(settings=settings) + case_insensitive_dupefilter = yield _get_dupefilter(settings=settings) assert not case_insensitive_dupefilter.request_seen(r1) assert case_insensitive_dupefilter.request_seen(r2) case_insensitive_dupefilter.close("finished") + @inlineCallbacks def test_seenreq_newlines(self): """Checks against adding duplicate \r to line endings on Windows platforms.""" @@ -152,7 +168,8 @@ class RFPDupeFilterTest(unittest.TestCase): r1 = Request("http://scrapytest.org/1") path = tempfile.mkdtemp() - crawler = get_crawler(settings_dict={"JOBDIR": path}) + crawler = get_crawler(NoRequestsSpider, settings_dict={"JOBDIR": path}) + yield crawler.crawl() try: scheduler = Scheduler.from_crawler(crawler) df = scheduler.df @@ -161,7 +178,10 @@ class RFPDupeFilterTest(unittest.TestCase): df.close("finished") with Path(path, "requests.seen").open("rb") as seen_file: - line = next(seen_file).decode() + try: + line = next(seen_file).decode() + except StopIteration: + return assert not line.endswith("\r\r\n") if sys.platform == "win32": assert line.endswith("\r\n") @@ -171,6 +191,7 @@ class RFPDupeFilterTest(unittest.TestCase): finally: shutil.rmtree(path) + @inlineCallbacks def test_log(self): with LogCapture() as log: settings = { @@ -178,9 +199,10 @@ class RFPDupeFilterTest(unittest.TestCase): "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(SimpleSpider, settings_dict=settings) - spider = SimpleSpider.from_crawler(crawler) - dupefilter = _get_dupefilter(crawler=crawler) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() + spider = crawler.spider + dupefilter = yield _get_dupefilter(crawler=crawler) r1 = Request("http://scrapytest.org/index.html") r2 = Request("http://scrapytest.org/index.html") @@ -200,6 +222,7 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close("finished") + @inlineCallbacks def test_log_debug(self): with LogCapture() as log: settings = { @@ -207,9 +230,10 @@ class RFPDupeFilterTest(unittest.TestCase): "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(SimpleSpider, settings_dict=settings) - spider = SimpleSpider.from_crawler(crawler) - dupefilter = _get_dupefilter(crawler=crawler) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() + spider = crawler.spider + dupefilter = yield _get_dupefilter(crawler=crawler) r1 = Request("http://scrapytest.org/index.html") r2 = Request( @@ -239,15 +263,17 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close("finished") + @inlineCallbacks def test_log_debug_default_dupefilter(self): with LogCapture() as log: settings = { "DUPEFILTER_DEBUG": True, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(SimpleSpider, settings_dict=settings) - spider = SimpleSpider.from_crawler(crawler) - dupefilter = _get_dupefilter(crawler=crawler) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() + spider = crawler.spider + dupefilter = yield _get_dupefilter(crawler=crawler) r1 = Request("http://scrapytest.org/index.html") r2 = Request( diff --git a/tests/test_engine.py b/tests/test_engine.py index 8d7afb6a1..5deae5146 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -420,12 +420,16 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_close_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + crawler = get_crawler(TestSpider) + yield crawler.crawl() + e = ExecutionEngine(crawler, lambda _: None) yield e.close() @defer.inlineCallbacks def test_start_already_running_exception(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + crawler = get_crawler(TestSpider) + yield crawler.crawl() + e = ExecutionEngine(crawler, lambda _: None) yield e.open_spider(TestSpider(), []) e.start() try: diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 80f5c3177..ce8452b5f 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,6 +1,8 @@ import datetime import typing -import unittest + +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest from scrapy.crawler import Crawler from scrapy.extensions.periodic_log import PeriodicLog @@ -59,36 +61,44 @@ class TestExtPeriodicLog(PeriodicLog): self.stats._stats = stats_dump_2 +@inlineCallbacks def extension(settings=None): - return TestExtPeriodicLog.from_crawler( - Crawler( - MetaSpider, - settings=settings, - ) + crawler = Crawler( + MetaSpider, + settings=settings, ) + yield crawler.crawl() + return TestExtPeriodicLog.from_crawler(crawler) class TestPeriodicLog(unittest.TestCase): + @inlineCallbacks def test_extension_enabled(self): # Expected that settings for this extension loaded succesfully # And on certain conditions - extension raising NotConfigured # "PERIODIC_LOG_STATS": True -> set to {"enabled": True} # due to TypeError exception from settings.getdict - assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60}) + assert (yield extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60})) # "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True} # due to JSONDecodeError(ValueError) exception from settings.getdict - assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60}) + assert ( + yield extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60}) + ) # The ame for PERIODIC_LOG_DELTA: - assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) - assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) + assert (yield extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})) + assert ( + yield extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) + ) + @inlineCallbacks def test_log_delta(self): + @inlineCallbacks def emulate(settings=None): spider = MetaSpider() - ext = extension(settings) + ext = yield extension(settings) ext.spider_opened(spider) ext.set_a() a = ext.log_delta() @@ -97,8 +107,9 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b + @inlineCallbacks def check(settings: dict, condition: typing.Callable): - ext, a, b = emulate(settings) + ext, a, b = yield emulate(settings) assert list(a["delta"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) ] @@ -107,45 +118,49 @@ class TestPeriodicLog(unittest.TestCase): ] # Including all - check({"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float))) + yield check( + {"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float)) + ) # include: - check( + yield check( {"PERIODIC_LOG_DELTA": {"include": ["downloader/"]}}, lambda k, v: isinstance(v, (int, float)) and "downloader/" in k, ) # include multiple - check( + yield check( {"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}}, lambda k, v: isinstance(v, (int, float)) and ("downloader/" in k or "scheduler/" in k), ) # exclude - check( + yield check( {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}}, lambda k, v: isinstance(v, (int, float)) and "downloader/" not in k, ) # exclude multiple - check( + yield check( {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}}, lambda k, v: isinstance(v, (int, float)) and ("downloader/" not in k and "scheduler/" not in k), ) # include exclude combined - check( + yield check( {"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: isinstance(v, (int, float)) and ("downloader/" in k and "bytes" not in k), ) + @inlineCallbacks def test_log_stats(self): + @inlineCallbacks def emulate(settings=None): spider = MetaSpider() - ext = extension(settings) + ext = yield extension(settings) ext.spider_opened(spider) ext.set_a() a = ext.log_crawler_stats() @@ -154,8 +169,9 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b + @inlineCallbacks def check(settings: dict, condition: typing.Callable): - ext, a, b = emulate(settings) + ext, a, b = yield emulate(settings) assert list(a["stats"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) ] @@ -164,35 +180,34 @@ class TestPeriodicLog(unittest.TestCase): ] # Including all - check({"PERIODIC_LOG_STATS": True}, lambda k, v: True) + yield check({"PERIODIC_LOG_STATS": True}, lambda k, v: True) # include: - check( + yield check( {"PERIODIC_LOG_STATS": {"include": ["downloader/"]}}, lambda k, v: "downloader/" in k, ) # include multiple - check( + yield check( {"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}}, lambda k, v: "downloader/" in k or "scheduler/" in k, ) # exclude - check( + yield check( {"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}}, lambda k, v: "downloader/" not in k, ) # exclude multiple - check( + yield check( {"PERIODIC_LOG_STATS": {"exclude": ["downloader/", "scheduler/"]}}, lambda k, v: "downloader/" not in k and "scheduler/" not in k, ) # include exclude combined - check( + yield check( {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: "downloader/" in k and "bytes" not in k, ) - # diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 9fd680e9f..6f35510b7 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -9,7 +9,7 @@ from scrapy.utils.test import get_crawler class TelnetExtensionTest(unittest.TestCase): def _get_console_and_portal(self, settings=None): - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(settings_dict=settings, disable_telnet=False) console = TelnetConsole(crawler) # This function has some side effects we don't need for this test diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 6b82974fa..875abdb1f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -26,6 +26,7 @@ import lxml.etree import pytest from testfixtures import LogCapture from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from w3lib.url import file_uri_to_path, path_to_file_uri from zope.interface import implementer @@ -51,7 +52,7 @@ from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto from tests.mockserver import MockFTPServer, MockServer -from tests.spiders import ItemSpider +from tests.spiders import ItemSpider, NoRequestsSpider def path_to_url(path): @@ -2747,13 +2748,14 @@ class FeedExporterSignalsTest(unittest.TestCase): d.callback(None) return d + @inlineCallbacks def run_signaled_feed_exporter( self, feed_exporter_signal_handler, feed_slot_signal_handler ): - crawler = get_crawler(settings_dict=self.settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=self.settings) + yield crawler.crawl() feed_exporter = FeedExporter.from_crawler(crawler) - spider = scrapy.Spider("default") - spider.crawler = crawler + spider = crawler.spider crawler.signals.connect( feed_exporter_signal_handler, signal=signals.feed_exporter_closed, @@ -2764,24 +2766,26 @@ class FeedExporterSignalsTest(unittest.TestCase): feed_exporter.open_spider(spider) for item in self.items: feed_exporter.item_scraped(item, spider) - defer.ensureDeferred(feed_exporter.close_spider(spider)) + yield defer.ensureDeferred(feed_exporter.close_spider(spider)) + @inlineCallbacks def test_feed_exporter_signals_sent(self): self.feed_exporter_closed_received = False self.feed_slot_closed_received = False - self.run_signaled_feed_exporter( + yield self.run_signaled_feed_exporter( self.feed_exporter_closed_signal_handler, self.feed_slot_closed_signal_handler, ) self.assertTrue(self.feed_slot_closed_received) self.assertTrue(self.feed_exporter_closed_received) + @inlineCallbacks def test_feed_exporter_signals_sent_deferred(self): self.feed_exporter_closed_received = False self.feed_slot_closed_received = False - self.run_signaled_feed_exporter( + yield self.run_signaled_feed_exporter( self.feed_exporter_closed_signal_handler_deferred, self.feed_slot_closed_signal_handler_deferred, ) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index bf96f17b6..0d769f29b 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -15,6 +15,7 @@ import attr import pytest from itemadapter import ItemAdapter from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from scrapy.http import Request, Response @@ -35,15 +36,18 @@ from scrapy.utils.test import ( skip_if_no_boto, ) from tests.mockserver import MockFTPServer +from tests.spiders import NoRequestsSpider from .test_pipeline_media import _mocked_download_func class FilesPipelineTestCase(unittest.TestCase): + @inlineCallbacks def setUp(self): self.tempdir = mkdtemp() settings_dict = {"FILES_STORE": self.tempdir} - crawler = get_crawler(spidercls=None, settings_dict=settings_dict) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) + yield crawler.crawl() self.pipeline = FilesPipeline.from_crawler(crawler) self.pipeline.download_func = _mocked_download_func self.pipeline.open_spider(None) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d655eb128..07ead2e83 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -14,11 +14,11 @@ from scrapy.pipelines.files import FileException from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings -from scrapy.spiders import Spider from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler +from tests.spiders import NoRequestsSpider try: from PIL import Image # noqa: imported just to check for the import error @@ -40,10 +40,11 @@ class BaseMediaPipelineTestCase(unittest.TestCase): pipeline_class = MediaPipeline settings = None + @inlineCallbacks def setUp(self): - spider_cls = Spider - self.spider = spider_cls("media.com") - crawler = get_crawler(spider_cls, self.settings) + crawler = get_crawler(NoRequestsSpider, self.settings) + yield crawler.crawl() + self.spider = crawler.spider self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe.download_func = _mocked_download_func self.pipe.open_spider(self.spider) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index bfb370373..54c77eb68 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,10 +1,10 @@ import collections import shutil import tempfile -import unittest from twisted.internet import defer -from twisted.trial.unittest import TestCase +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest from scrapy.core.downloader import Downloader from scrapy.core.scheduler import Scheduler @@ -12,8 +12,10 @@ from scrapy.crawler import Crawler from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver import MockServer +from tests.spiders import NoRequestsSpider MockEngine = collections.namedtuple("MockEngine", ["downloader"]) MockSlot = collections.namedtuple("MockSlot", ["active"]) @@ -54,6 +56,7 @@ class MockCrawler(Crawler): ) super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) + self.stats = load_object(self.settings["STATS_CLASS"])(self) class SchedulerHandler: @@ -307,7 +310,7 @@ class StartUrlsSpider(Spider): pass -class TestIntegrationWithDownloaderAwareInMemory(TestCase): +class TestIntegrationWithDownloaderAwareInMemory(unittest.TestCase): def setUp(self): self.crawler = get_crawler( spidercls=StartUrlsSpider, @@ -334,16 +337,19 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase): class TestIncompatibility(unittest.TestCase): + @inlineCallbacks def _incompatible(self): settings = dict( SCHEDULER_PRIORITY_QUEUE="scrapy.pqueues.DownloaderAwarePriorityQueue", CONCURRENT_REQUESTS_PER_IP=1, ) - crawler = get_crawler(Spider, settings) + crawler = get_crawler(NoRequestsSpider, settings) + yield crawler.crawl() + spider = crawler.spider scheduler = Scheduler.from_crawler(crawler) - spider = Spider(name="spider") scheduler.open(spider) + @inlineCallbacks def test_incompatibility(self): with self.assertRaises(ValueError): - self._incompatible() + yield self._incompatible() diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 1d5a887cc..2fcb5b364 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,9 +1,9 @@ import logging -from unittest import TestCase from testfixtures import LogCapture from twisted.internet import defer -from twisted.trial.unittest import TestCase as TrialTestCase +from twisted.internet.defer import inlineCallbacks +from twisted.trial.unittest import TestCase from scrapy.http import Request, Response from scrapy.settings import Settings @@ -11,7 +11,7 @@ from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import MockServerSpider +from tests.spiders import MockServerSpider, NoRequestsSpider class _HttpErrorSpider(MockServerSpider): @@ -59,9 +59,11 @@ def _responses(request, status_codes): class TestHttpErrorMiddleware(TestCase): + @inlineCallbacks def setUp(self): - crawler = get_crawler(Spider) - self.spider = Spider.from_crawler(crawler, name="foo") + crawler = get_crawler(NoRequestsSpider) + yield crawler.crawl() + self.spider = crawler.spider self.mw = HttpErrorMiddleware(Settings({})) self.req = Request("http://scrapytest.org") self.res200, self.res404 = _responses(self.req, [200, 404]) @@ -171,7 +173,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.assertIsNone(mw.process_spider_input(res402, self.spider)) -class TestHttpErrorMiddlewareIntegrational(TrialTestCase): +class TestHttpErrorMiddlewareIntegrational(TestCase): def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index ea45b7698..ed00c0a6b 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -1,17 +1,21 @@ import warnings -from unittest import TestCase from urllib.parse import urlparse +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest + from scrapy.http import Request, Response from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, PortWarning, URLWarning from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestOffsiteMiddleware(TestCase): +class TestOffsiteMiddleware(unittest.TestCase): + @inlineCallbacks def setUp(self): crawler = get_crawler(Spider) - self.spider = crawler._create_spider(**self._get_spiderargs()) + yield crawler.crawl(**self._get_spiderargs()) + self.spider = crawler.spider self.mw = OffsiteMiddleware.from_crawler(crawler) self.mw.spider_opened(self.spider) diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 9111e4c82..a3c284484 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,21 +1,23 @@ -from unittest import TestCase - from testfixtures import LogCapture +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware -from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +from tests.spiders import NoRequestsSpider -class TestUrlLengthMiddleware(TestCase): +class TestUrlLengthMiddleware(unittest.TestCase): + @inlineCallbacks def setUp(self): self.maxlength = 25 settings = Settings({"URLLENGTH_LIMIT": self.maxlength}) - crawler = get_crawler(Spider) - self.spider = crawler._create_spider("foo") + crawler = get_crawler(NoRequestsSpider) + yield crawler.crawl() + self.spider = crawler.spider self.stats = crawler.stats self.mw = UrlLengthMiddleware.from_settings(settings) diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index eae744df5..2117e65b1 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,9 +1,10 @@ import logging import sys -import unittest from testfixtures import LogCapture +from twisted.internet.defer import inlineCallbacks from twisted.python.failure import Failure +from twisted.trial import unittest from scrapy.extensions import telnet from scrapy.utils.log import ( @@ -13,6 +14,7 @@ from scrapy.utils.log import ( failure_to_exc_info, ) from scrapy.utils.test import get_crawler +from tests.spiders import NoRequestsSpider class FailureToExcInfoTest(unittest.TestCase): @@ -60,6 +62,7 @@ class TopLevelFormatterTest(unittest.TestCase): class LogCounterHandlerTest(unittest.TestCase): + @inlineCallbacks def setUp(self): settings = {"LOG_LEVEL": "WARNING"} if not telnet.TWISTED_CONCH_AVAILABLE: @@ -68,7 +71,8 @@ class LogCounterHandlerTest(unittest.TestCase): self.logger = logging.getLogger("test") self.logger.setLevel(logging.NOTSET) self.logger.propagate = False - self.crawler = get_crawler(settings_dict=settings) + self.crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield self.crawler.crawl() self.handler = LogCounterHandler(self.crawler) self.logger.addHandler(self.handler) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index e6d1abe3f..9ca9faa0c 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,11 +1,12 @@ import json -import unittest import warnings from hashlib import sha1 from typing import Dict, Mapping, Optional, Tuple, Union from weakref import WeakKeyDictionary import pytest +from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest from w3lib.url import canonicalize_url from scrapy.http import Request @@ -22,6 +23,7 @@ from scrapy.utils.request import ( request_to_curl, ) from scrapy.utils.test import get_crawler +from tests.spiders import NoRequestsSpider class UtilsRequestTest(unittest.TestCase): @@ -449,15 +451,18 @@ class BackwardCompatibilityTestCase(unittest.TestCase): ) self.assertEqual(fp, old_fp) + @inlineCallbacks def test_component_backward_compatibility(self): for request_object in REQUEST_OBJECTS_TO_TEST: with warnings.catch_warnings(): warnings.simplefilter("ignore") - crawler = get_crawler(prevent_warnings=False) + crawler = get_crawler(NoRequestsSpider, prevent_warnings=False) + yield crawler.crawl() fp = crawler.request_fingerprinter.fingerprint(request_object) old_fp = request_fingerprint_2_6(request_object) self.assertEqual(fp.hex(), old_fp) + @inlineCallbacks def test_custom_component_backward_compatibility(self): """Tests that the backward-compatible request fingerprinting class featured in the documentation is indeed backward compatible and does not cause a @@ -480,7 +485,8 @@ class BackwardCompatibilityTestCase(unittest.TestCase): settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() fp = crawler.request_fingerprinter.fingerprint(request_object) old_fp = request_fingerprint_2_6(request_object) self.assertEqual(fp.hex(), old_fp) @@ -488,9 +494,11 @@ class BackwardCompatibilityTestCase(unittest.TestCase): class RequestFingerprinterTestCase(unittest.TestCase): + @inlineCallbacks def test_default_implementation(self): with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(prevent_warnings=False) + crawler = get_crawler(NoRequestsSpider, prevent_warnings=False) + yield crawler.crawl() request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), @@ -498,12 +506,14 @@ class RequestFingerprinterTestCase(unittest.TestCase): ) self.assertTrue(logged_warnings) + @inlineCallbacks def test_deprecated_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6", } with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), @@ -511,12 +521,14 @@ class RequestFingerprinterTestCase(unittest.TestCase): ) self.assertTrue(logged_warnings) + @inlineCallbacks def test_recommended_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), @@ -524,15 +536,18 @@ class RequestFingerprinterTestCase(unittest.TestCase): ) self.assertFalse(logged_warnings) + @inlineCallbacks def test_unknown_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5", } with self.assertRaises(ValueError): - get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() class CustomRequestFingerprinterTestCase(unittest.TestCase): + @inlineCallbacks def test_include_headers(self): class RequestFingerprinter: def fingerprint(self, request): @@ -541,7 +556,8 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() r1 = Request("http://www.example.com", headers={"X-ID": "1"}) fp1 = crawler.request_fingerprinter.fingerprint(r1) @@ -549,6 +565,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp2 = crawler.request_fingerprinter.fingerprint(r2) self.assertNotEqual(fp1, fp2) + @inlineCallbacks def test_dont_canonicalize(self): class RequestFingerprinter: cache = WeakKeyDictionary() @@ -563,7 +580,8 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() r1 = Request("http://www.example.com?a=1&a=2") fp1 = crawler.request_fingerprinter.fingerprint(r1) @@ -571,6 +589,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp2 = crawler.request_fingerprinter.fingerprint(r2) self.assertNotEqual(fp1, fp2) + @inlineCallbacks def test_meta(self): class RequestFingerprinter: def fingerprint(self, request): @@ -581,7 +600,8 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() r1 = Request("http://www.example.com") fp1 = crawler.request_fingerprinter.fingerprint(r1) @@ -596,6 +616,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): self.assertNotEqual(fp2, fp4) self.assertEqual(fp2, fp3) + @inlineCallbacks def test_from_crawler(self): class RequestFingerprinter: @classmethod @@ -612,12 +633,14 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) self.assertEqual(fingerprint, settings["FINGERPRINT"]) + @inlineCallbacks def test_from_settings(self): class RequestFingerprinter: @classmethod @@ -634,12 +657,14 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) self.assertEqual(fingerprint, settings["FINGERPRINT"]) + @inlineCallbacks def test_from_crawler_and_settings(self): class RequestFingerprinter: # This method is ignored due to the presence of from_crawler @@ -661,7 +686,8 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings) + yield crawler.crawl() request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) @@ -728,7 +754,3 @@ class RequestToCurlTest(unittest.TestCase): " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" ) self._test_request(request_object, expected_curl_command) - - -if __name__ == "__main__": - unittest.main() From 6629a61dd98735994cce2fcfba63cad46e5d3683 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 8 Sep 2023 20:15:49 +0400 Subject: [PATCH 1148/2083] Fix one more testcase. --- tests/test_pipeline_media.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 07ead2e83..8a13ea552 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -431,12 +431,14 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): skip = skip_pillow + @inlineCallbacks def setUp(self): settings_dict = { "IMAGES_STORE": "store-uri", "IMAGES_THUMBS": {"small": (50, 50)}, } - crawler = get_crawler(spidercls=None, settings_dict=settings_dict) + crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) + yield crawler.crawl() self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler) self.pipe.download_func = _mocked_download_func self.pipe.open_spider(None) From bb15c93a2bbd7daaee4a02d2d6f6b52cea4b3313 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 12 Sep 2023 14:44:16 +0400 Subject: [PATCH 1149/2083] Add Crawler._load_settings(). --- scrapy/commands/shell.py | 1 + scrapy/crawler.py | 92 +++++++++-------- scrapy/utils/test.py | 12 ++- tests/spiders.py | 7 -- tests/test_addons.py | 44 +++------ tests/test_command_shell.py | 2 - tests/test_commands.py | 5 +- tests/test_crawler.py | 28 +++--- tests/test_downloadermiddleware.py | 71 ++++++++------ tests/test_downloadermiddleware_httpcache.py | 17 ++-- ...st_downloadermiddleware_httpcompression.py | 24 ++--- tests/test_downloadermiddleware_retry.py | 98 +++++++------------ tests/test_downloadermiddleware_stats.py | 16 ++- tests/test_dupefilters.py | 70 +++++-------- tests/test_engine.py | 8 +- tests/test_extension_periodic_log.py | 66 +++++-------- tests/test_extension_telnet.py | 2 +- tests/test_feedexport.py | 18 ++-- tests/test_pipeline_files.py | 6 +- tests/test_pipeline_media.py | 13 +-- tests/test_scheduler.py | 16 ++- tests/test_spider.py | 2 +- tests/test_spidermiddleware_httperror.py | 14 ++- tests/test_spidermiddleware_offsite.py | 10 +- tests/test_spidermiddleware_urllength.py | 14 ++- tests/test_utils_log.py | 8 +- tests/test_utils_request.py | 56 ++++------- 27 files changed, 295 insertions(+), 425 deletions(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 0a5e61f7a..71f43365d 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -77,6 +77,7 @@ class Command(ScrapyCommand): # The crawler is created this way since the Shell manually handles the # crawling engine, so the set up in the crawl method won't work crawler = self.crawler_process._create_crawler(spidercls) + crawler._load_settings() # The Shell class needs a persistent engine in the crawler crawler.engine = crawler._create_engine() crawler.engine.start() diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 49034c9f1..67e44541d 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -82,6 +82,7 @@ class Crawler: self._init_reactor: bool = init_reactor self.crawling: bool = False + self._settings_loaded: bool = False self._started: bool = False self.extensions: Optional[ExtensionManager] = None self.stats: Optional[StatsCollector] = None @@ -90,6 +91,53 @@ class Crawler: self.spider: Optional[Spider] = None self.engine: Optional[ExecutionEngine] = None + def _load_settings(self) -> None: + if self._settings_loaded: + return + self._settings_loaded = True + + self.addons.load_settings(self.settings) + self.stats = load_object(self.settings["STATS_CLASS"])(self) + + handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) + logging.root.addHandler(handler) + # lambda is assigned to Crawler attribute because this way it is not + # garbage collected after leaving the scope + self.__remove_handler = lambda: logging.root.removeHandler(handler) + self.signals.connect(self.__remove_handler, signals.engine_stopped) + + lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) + self.logformatter = lf_cls.from_crawler(self) + + self.request_fingerprinter = create_instance( + load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), + settings=self.settings, + crawler=self, + ) + + reactor_class: str = self.settings["TWISTED_REACTOR"] + event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] + if self._init_reactor: + # this needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor + if reactor_class: + install_reactor(reactor_class, event_loop) + else: + from twisted.internet import reactor # noqa: F401 + log_reactor_info() + if reactor_class: + verify_installed_reactor(reactor_class) + if is_asyncio_reactor_installed() and event_loop: + verify_installed_asyncio_event_loop(event_loop) + + self.extensions = ExtensionManager.from_crawler(self) + self.settings.freeze() + + d = dict(overridden_settings(self.settings)) + logger.info( + "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} + ) + @inlineCallbacks def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: if self.crawling: @@ -104,49 +152,7 @@ class Crawler: try: self.spider = self._create_spider(*args, **kwargs) - - self.addons.load_settings(self.settings) - self.stats = load_object(self.settings["STATS_CLASS"])(self) - - handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) - logging.root.addHandler(handler) - # lambda is assigned to Crawler attribute because this way it is not - # garbage collected after leaving the scope - self.__remove_handler = lambda: logging.root.removeHandler(handler) - self.signals.connect(self.__remove_handler, signals.engine_stopped) - - lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) - self.logformatter = lf_cls.from_crawler(self) - - self.request_fingerprinter = create_instance( - load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), - settings=self.settings, - crawler=self, - ) - - reactor_class: str = self.settings["TWISTED_REACTOR"] - event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] - if self._init_reactor: - # this needs to be done after the spider settings are merged, - # but before something imports twisted.internet.reactor - if reactor_class: - install_reactor(reactor_class, event_loop) - else: - from twisted.internet import reactor # noqa: F401 - log_reactor_info() - if reactor_class: - verify_installed_reactor(reactor_class) - if is_asyncio_reactor_installed() and event_loop: - verify_installed_asyncio_event_loop(event_loop) - - self.extensions = ExtensionManager.from_crawler(self) - self.settings.freeze() - - d = dict(overridden_settings(self.settings)) - logger.info( - "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} - ) - + self._load_settings() self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 44a30dc15..9397e78b9 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -69,28 +69,30 @@ def get_ftp_content_and_delete( return b"".join(ftp_data) +class TestSpider(Spider): + name = "test" + + def get_crawler( spidercls: Optional[Type[Spider]] = None, settings_dict: Optional[Dict[str, Any]] = None, prevent_warnings: bool = True, - disable_telnet: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. """ from scrapy.crawler import CrawlerRunner - from scrapy.spiders import Spider # Set by default settings that prevent deprecation warnings. settings: Dict[str, Any] = {} if prevent_warnings: settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7" - if disable_telnet: - settings["TELNETCONSOLE_ENABLED"] = False settings.update(settings_dict or {}) runner = CrawlerRunner(settings) - return runner.create_crawler(spidercls or Spider) + crawler = runner.create_crawler(spidercls or TestSpider) + crawler._load_settings() + return crawler def get_pythonpath() -> str: diff --git a/tests/spiders.py b/tests/spiders.py index eeb0194eb..f29dea2a1 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -18,13 +18,6 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req -class NoRequestsSpider(Spider): - name = "no_request" - - def start_requests(self): - return [] - - class MockServerSpider(Spider): def __init__(self, mockserver=None, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/tests/test_addons.py b/tests/test_addons.py index c4b4b7ac9..aa1b760c2 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,15 +1,13 @@ import itertools +import unittest from typing import Any, Dict from unittest.mock import patch -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest - +from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider class SimpleAddon: @@ -53,17 +51,14 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - @inlineCallbacks def test_load_settings(self): settings_dict = { "ADDONS": {"tests.test_addons.SimpleAddon": 0}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons self.assertIsInstance(manager.addons[0], SimpleAddon) - @inlineCallbacks def test_notconfigured(self): class NotConfiguredAddon: def update_settings(self, settings): @@ -72,12 +67,10 @@ class AddonManagerTest(unittest.TestCase): settings_dict = { "ADDONS": {NotConfiguredAddon: 0}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons self.assertFalse(manager.addons) - @inlineCallbacks def test_load_settings_order(self): # Get three addons with different settings addonlist = [] @@ -89,25 +82,21 @@ class AddonManagerTest(unittest.TestCase): for ordered_addons in itertools.permutations(addonlist): expected_order = [a.number for a in ordered_addons] settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) manager = crawler.addons self.assertEqual([a.number for a in manager.addons], expected_order) self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) - @inlineCallbacks def test_create_instance(self): settings_dict = { "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, "MYADDON": {"MYADDON_KEY": "val"}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons self.assertIsInstance(manager.addons[0], CreateInstanceAddon) self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") - @inlineCallbacks def test_settings_priority(self): config = { "KEY": 15, # priority=addon @@ -115,15 +104,14 @@ class AddonManagerTest(unittest.TestCase): settings_dict = { "ADDONS": {get_addon_cls(config): 1}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) self.assertEqual(crawler.settings.getint("KEY"), 15) settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") runner = CrawlerRunner(settings) - crawler = runner.create_crawler(NoRequestsSpider) - yield crawler.crawl() + crawler = runner.create_crawler(Spider) + crawler._load_settings() self.assertEqual(crawler.settings.getint("KEY"), 15) settings_dict = { @@ -133,11 +121,9 @@ class AddonManagerTest(unittest.TestCase): settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") runner = CrawlerRunner(settings) - crawler = runner.create_crawler(NoRequestsSpider) - yield crawler.crawl() + crawler = runner.create_crawler(Spider) self.assertEqual(crawler.settings.getint("KEY"), 20) - @inlineCallbacks def test_fallback_workflow(self): FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" @@ -154,8 +140,7 @@ class AddonManagerTest(unittest.TestCase): settings_dict = { "ADDONS": {AddonWithFallback: 1}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) self.assertEqual( crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" ) @@ -168,14 +153,12 @@ class AddonManagerTest(unittest.TestCase): "ADDONS": {AddonWithFallback: 1}, "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) self.assertEqual( crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" ) self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") - @inlineCallbacks def test_logging_message(self): class LoggedAddon: def update_settings(self, settings): @@ -188,8 +171,7 @@ class AddonManagerTest(unittest.TestCase): } addon = LoggedAddon() create_instance_mock.return_value = addon - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) logger_mock.info.assert_called_once_with( "Enabled addons:\n%(addons)s", {"addons": [addon]}, diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 72d06deab..6589381f3 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -7,8 +7,6 @@ from scrapy.utils.testproc import ProcessTest from scrapy.utils.testsite import SiteTest from tests import NON_EXISTING_RESOLVABLE, tests_datadir -raise unittest.SkipTest("Broken for now") - class ShellTest(ProcessTest, SiteTest, unittest.TestCase): command = "shell" diff --git a/tests/test_commands.py b/tests/test_commands.py index 05be33c73..b1d7be628 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -26,8 +26,7 @@ from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv -from tests.spiders import NoRequestsSpider -from tests.test_crawler import ExceptionSpider +from tests.test_crawler import ExceptionSpider, NoRequestsSpider class CommandSettings(unittest.TestCase): @@ -713,7 +712,7 @@ class BadSpider(scrapy.Spider): def test_run_good_spider(self): proc, _, _ = self.runspider( - "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider) + "import scrapy\n" + inspect.getsource(NoRequestsSpider) ) ret = proc.returncode self.assertEqual(ret, 0) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f962cecc8..120991ae7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -10,7 +10,6 @@ import pytest from packaging.version import parse as parse_version from pytest import mark, raises from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from w3lib import __version__ as w3lib_version @@ -24,7 +23,6 @@ from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer, get_mockserver_env -from tests.spiders import NoRequestsSpider class BaseCrawlerTest(unittest.TestCase): @@ -72,7 +70,6 @@ class CrawlerLoggingTestCase(unittest.TestCase): get_crawler(MySpider) assert get_scrapy_root_handler() is None - @inlineCallbacks def test_spider_custom_settings_log_level(self): log_file = Path(self.mktemp()) log_file.write_text("previous message\n", encoding="utf-8") @@ -82,20 +79,20 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { "LOG_LEVEL": "INFO", "LOG_FILE": str(log_file), + # settings to avoid extra warnings + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } - def start_requests(self): - logging.debug("debug message") - logging.info("info message") - logging.warning("warning message") - logging.error("error message") - return [] - configure_logging() self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) crawler = get_crawler(MySpider) - yield crawler.crawl() self.assertEqual(get_scrapy_root_handler().level, logging.INFO) + info_count = crawler.stats.get_value("log_count/INFO") + logging.debug("debug message") + logging.info("info message") + logging.warning("warning message") + logging.error("error message") logged = log_file.read_text(encoding="utf-8") @@ -106,7 +103,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): self.assertIn("error message", logged) self.assertEqual(crawler.stats.get_value("log_count/ERROR"), 1) self.assertEqual(crawler.stats.get_value("log_count/WARNING"), 1) - # self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1) TODO + self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1) self.assertEqual(crawler.stats.get_value("log_count/DEBUG", 0), 0) def test_spider_custom_settings_log_append(self): @@ -183,6 +180,13 @@ class ExceptionSpider(scrapy.Spider): raise ValueError("Exception in from_crawler method") +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + def start_requests(self): + return [] + + @mark.usefixtures("reactor_pytest") class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index d64651211..062e8a8b4 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -3,33 +3,33 @@ from unittest import mock from pytest import mark from twisted.internet import defer -from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.internet.defer import Deferred from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue -from tests.spiders import NoRequestsSpider class ManagerTestCase(TestCase): settings_dict = None - @inlineCallbacks def setUp(self): - self.crawler = get_crawler(NoRequestsSpider, self.settings_dict) - yield self.crawler.crawl() - self.spider = self.crawler.spider + self.crawler = get_crawler(Spider, self.settings_dict) + self.spider = self.crawler._create_spider("foo") self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) - yield self.mwman.open_spider(self.spider) + # some mw depends on stats collector + self.crawler.stats.open_spider(self.spider) + return self.mwman.open_spider(self.spider) def tearDown(self): + self.crawler.stats.close_spider(self.spider, "") return self.mwman.close_spider(self.spider) - @inlineCallbacks def _download(self, request, response=None): """Executes downloader mw manager's download method and returns the result (Request or Response) or raise exception in case of @@ -41,21 +41,26 @@ class ManagerTestCase(TestCase): def download_func(**kwargs): return response - ret = yield self.mwman.download(download_func, request, self.spider) + dfd = self.mwman.download(download_func, request, self.spider) + # catch deferred result and return the value + results = [] + dfd.addBoth(results.append) + self._wait(dfd) + ret = results[0] + if isinstance(ret, Failure): + ret.raiseException() return ret class DefaultsTest(ManagerTestCase): """Tests default behavior with default settings""" - @inlineCallbacks def test_request_response(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) - ret = yield self._download(req, resp) + ret = self._download(req, resp) self.assertTrue(isinstance(ret, Response), "Non-response returned") - @inlineCallbacks def test_3xx_and_invalid_gzipped_body_must_redirect(self): """Regression test for a failure when redirecting a compressed request. @@ -81,7 +86,7 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - ret = yield self._download(request=req, response=resp) + ret = self._download(request=req, response=resp) self.assertTrue(isinstance(ret, Request), f"Not redirected: {ret!r}") self.assertEqual( to_bytes(ret.url), @@ -89,7 +94,6 @@ class DefaultsTest(ManagerTestCase): "Not redirected to location header", ) - @inlineCallbacks def test_200_and_invalid_gzipped_body_must_fail(self): req = Request("http://example.com") body = b"

You are being redirected

" @@ -104,14 +108,12 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - with self.assertRaises(OSError): - yield self._download(request=req, response=resp) + self.assertRaises(OSError, self._download, request=req, response=resp) class ResponseFromProcessRequestTest(ManagerTestCase): """Tests middleware returning a response from process_request.""" - @inlineCallbacks def test_download_func_not_called(self): resp = Response("http://example.com/index.html") @@ -123,8 +125,12 @@ class ResponseFromProcessRequestTest(ManagerTestCase): req = Request("http://example.com/index.html") download_func = mock.MagicMock() - result = yield self.mwman.download(download_func, req, self.spider) - self.assertIs(result, resp) + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self._wait(dfd) + + self.assertIs(results[0], resp) self.assertFalse(download_func.called) @@ -191,7 +197,6 @@ class ProcessExceptionInvalidOutput(ManagerTestCase): class MiddlewareUsingDeferreds(ManagerTestCase): """Middlewares using Deferreds should work""" - @inlineCallbacks def test_deferred(self): resp = Response("http://example.com/index.html") @@ -208,8 +213,12 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.mwman._add_middleware(DeferredMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - result = yield self.mwman.download(download_func, req, self.spider) - self.assertIs(result, resp) + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self._wait(dfd) + + self.assertIs(results[0], resp) self.assertFalse(download_func.called) @@ -217,7 +226,6 @@ class MiddlewareUsingDeferreds(ManagerTestCase): class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" - @inlineCallbacks def test_asyncdef(self): resp = Response("http://example.com/index.html") @@ -229,12 +237,15 @@ class MiddlewareUsingCoro(ManagerTestCase): self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - result = yield self.mwman.download(download_func, req, self.spider) - self.assertIs(result, resp) + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self._wait(dfd) + + self.assertIs(results[0], resp) self.assertFalse(download_func.called) @mark.only_asyncio() - @inlineCallbacks def test_asyncdef_asyncio(self): resp = Response("http://example.com/index.html") @@ -247,6 +258,10 @@ class MiddlewareUsingCoro(ManagerTestCase): self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - result = yield self.mwman.download(download_func, req, self.spider) - self.assertIs(result, resp) + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self._wait(dfd) + + self.assertIs(results[0], resp) self.assertFalse(download_func.called) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index a8a687da7..f80eff3e6 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -2,31 +2,27 @@ import email.utils import shutil import tempfile import time +import unittest from contextlib import contextmanager -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest - from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.http import HtmlResponse, Request, Response from scrapy.settings import Settings +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider class _BaseTest(unittest.TestCase): storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" - @inlineCallbacks def setUp(self): self.yesterday = email.utils.formatdate(time.time() - 86400) self.today = email.utils.formatdate() self.tomorrow = email.utils.formatdate(time.time() + 86400) - self.crawler = get_crawler(NoRequestsSpider) - yield self.crawler.crawl() - self.spider = self.crawler.spider + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider("example.com") self.tmpdir = tempfile.mkdtemp() self.request = Request("http://www.example.com", headers={"User-Agent": "test"}) self.response = Response( @@ -35,6 +31,7 @@ class _BaseTest(unittest.TestCase): body=b"test body", status=202, ) + self.crawler.stats.open_spider(self.spider) def tearDown(self): self.crawler.stats.close_spider(self.spider, "") @@ -569,3 +566,7 @@ class RFC2616PolicyTest(DefaultStorageTest): res2 = self._process_requestresponse(mw, req0, None) self.assertEqualResponse(res1, res2) assert "cached" in res2.flags + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 4a579c061..9dad056de 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,10 +1,9 @@ from gzip import GzipFile from io import BytesIO from pathlib import Path +from unittest import SkipTest, TestCase from warnings import catch_warnings -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest from w3lib.encoding import resolve_encoding from scrapy.downloadermiddlewares.httpcompression import ( @@ -14,10 +13,10 @@ from scrapy.downloadermiddlewares.httpcompression import ( from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes +from scrapy.spiders import Spider from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir -from tests.spiders import NoRequestsSpider SAMPLEDIR = Path(tests_datadir, "compressed") @@ -39,13 +38,12 @@ FORMAT = { } -class HttpCompressionTest(unittest.TestCase): - @inlineCallbacks +class HttpCompressionTest(TestCase): def setUp(self): - self.crawler = get_crawler(NoRequestsSpider) - yield self.crawler.crawl() - self.spider = self.crawler.spider + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider("scrapytest.org") self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) + self.crawler.stats.open_spider(self.spider) def _getresponse(self, coding): if coding not in FORMAT: @@ -133,7 +131,7 @@ class HttpCompressionTest(unittest.TestCase): try: import brotli # noqa: F401 except ImportError: - raise unittest.SkipTest("no brotli") + raise SkipTest("no brotli") response = self._getresponse("br") request = response.request self.assertEqual(response.headers["Content-Encoding"], b"br") @@ -148,7 +146,7 @@ class HttpCompressionTest(unittest.TestCase): try: import zstandard # noqa: F401 except ImportError: - raise unittest.SkipTest("no zstd support (zstandard)") + raise SkipTest("no zstd support (zstandard)") raw_content = None for check_key in FORMAT: if not check_key.startswith("zstd-"): @@ -376,15 +374,13 @@ class HttpCompressionTest(unittest.TestCase): self.assertStatsEqual("httpcompression/response_bytes", None) -class HttpCompressionSubclassTest(unittest.TestCase): - @inlineCallbacks +class HttpCompressionSubclassTest(TestCase): def test_init_missing_stats(self): class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware): def __init__(self): super().__init__() - crawler = get_crawler(NoRequestsSpider) - yield crawler.crawl() + crawler = get_crawler(Spider) with catch_warnings(record=True) as caught_warnings: HttpCompressionMiddlewareSubclass.from_crawler(crawler) messages = tuple( diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index f94958ff7..97ae1e29a 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,9 +1,9 @@ import logging +import unittest import warnings from testfixtures import LogCapture from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from twisted.internet.error import ( ConnectError, ConnectionDone, @@ -11,7 +11,6 @@ from twisted.internet.error import ( DNSLookupError, TCPTimedOutError, ) -from twisted.trial import unittest from twisted.web.client import ResponseFailed from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request @@ -20,15 +19,12 @@ from scrapy.http import Request, Response from scrapy.settings.default_settings import RETRY_EXCEPTIONS from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider class RetryTest(unittest.TestCase): - @inlineCallbacks def setUp(self): - self.crawler = get_crawler(NoRequestsSpider) - yield self.crawler.crawl() - self.spider = self.crawler.spider + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider("foo") self.mw = RetryMiddleware.from_crawler(self.crawler) self.mw.max_retry_times = 2 @@ -164,19 +160,16 @@ class RetryTest(unittest.TestCase): class MaxRetryTimesTest(unittest.TestCase): invalid_url = "http://www.scrapytest.org/invalid_url" - @inlineCallbacks def get_spider_and_middleware(self, settings=None): - crawler = get_crawler(NoRequestsSpider, settings or {}) - yield crawler.crawl() - spider = crawler.spider + crawler = get_crawler(Spider, settings or {}) + spider = crawler._create_spider("foo") middleware = RetryMiddleware.from_crawler(crawler) return spider, middleware - @inlineCallbacks def test_with_settings_zero(self): max_retry_times = 0 settings = {"RETRY_TIMES": max_retry_times} - spider, middleware = yield self.get_spider_and_middleware(settings) + spider, middleware = self.get_spider_and_middleware(settings) req = Request(self.invalid_url) self._test_retry( req, @@ -186,10 +179,9 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) - @inlineCallbacks def test_with_metakey_zero(self): max_retry_times = 0 - spider, middleware = yield self.get_spider_and_middleware() + spider, middleware = self.get_spider_and_middleware() meta = {"max_retry_times": max_retry_times} req = Request(self.invalid_url, meta=meta) self._test_retry( @@ -200,11 +192,10 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) - @inlineCallbacks def test_without_metakey(self): max_retry_times = 5 settings = {"RETRY_TIMES": max_retry_times} - spider, middleware = yield self.get_spider_and_middleware(settings) + spider, middleware = self.get_spider_and_middleware(settings) req = Request(self.invalid_url) self._test_retry( req, @@ -214,7 +205,6 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) - @inlineCallbacks def test_with_metakey_greater(self): meta_max_retry_times = 3 middleware_max_retry_times = 2 @@ -223,7 +213,7 @@ class MaxRetryTimesTest(unittest.TestCase): req2 = Request(self.invalid_url) settings = {"RETRY_TIMES": middleware_max_retry_times} - spider, middleware = yield self.get_spider_and_middleware(settings) + spider, middleware = self.get_spider_and_middleware(settings) self._test_retry( req1, @@ -240,7 +230,6 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) - @inlineCallbacks def test_with_metakey_lesser(self): meta_max_retry_times = 4 middleware_max_retry_times = 5 @@ -249,7 +238,7 @@ class MaxRetryTimesTest(unittest.TestCase): req2 = Request(self.invalid_url) settings = {"RETRY_TIMES": middleware_max_retry_times} - spider, middleware = yield self.get_spider_and_middleware(settings) + spider, middleware = self.get_spider_and_middleware(settings) self._test_retry( req1, @@ -266,10 +255,9 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) - @inlineCallbacks def test_with_dont_retry(self): max_retry_times = 4 - spider, middleware = yield self.get_spider_and_middleware() + spider, middleware = self.get_spider_and_middleware() meta = { "max_retry_times": max_retry_times, "dont_retry": True, @@ -304,16 +292,13 @@ class MaxRetryTimesTest(unittest.TestCase): class GetRetryRequestTest(unittest.TestCase): - @inlineCallbacks def get_spider(self, settings=None): - crawler = get_crawler(NoRequestsSpider, settings or {}) - yield crawler.crawl() - return crawler.spider + crawler = get_crawler(Spider, settings or {}) + return crawler._create_spider("foo") - @inlineCallbacks def test_basic_usage(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() with LogCapture() as log: new_request = get_retry_request( request, @@ -337,10 +322,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_max_retries_reached(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() max_retry_times = 0 with LogCapture() as log: new_request = get_retry_request( @@ -361,10 +345,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_one_retry(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() with LogCapture() as log: new_request = get_retry_request( request, @@ -389,9 +372,8 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_two_retries(self): - spider = yield self.get_spider() + spider = self.get_spider() request = Request("https://example.com") new_request = request max_retry_times = 2 @@ -445,10 +427,9 @@ class GetRetryRequestTest(unittest.TestCase): with self.assertRaises(TypeError): get_retry_request(request) # pylint: disable=missing-kwoa - @inlineCallbacks def test_max_retry_times_setting(self): max_retry_times = 0 - spider = yield self.get_spider({"RETRY_TIMES": max_retry_times}) + spider = self.get_spider({"RETRY_TIMES": max_retry_times}) request = Request("https://example.com") new_request = get_retry_request( request, @@ -456,10 +437,9 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request, None) - @inlineCallbacks def test_max_retry_times_meta(self): max_retry_times = 0 - spider = yield self.get_spider({"RETRY_TIMES": max_retry_times + 1}) + spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1}) meta = {"max_retry_times": max_retry_times} request = Request("https://example.com", meta=meta) new_request = get_retry_request( @@ -468,10 +448,9 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request, None) - @inlineCallbacks def test_max_retry_times_argument(self): max_retry_times = 0 - spider = yield self.get_spider({"RETRY_TIMES": max_retry_times + 1}) + spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1}) meta = {"max_retry_times": max_retry_times + 1} request = Request("https://example.com", meta=meta) new_request = get_retry_request( @@ -481,10 +460,9 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request, None) - @inlineCallbacks def test_priority_adjust_setting(self): priority_adjust = 1 - spider = yield self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust}) + spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust}) request = Request("https://example.com") new_request = get_retry_request( request, @@ -492,10 +470,9 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request.priority, priority_adjust) - @inlineCallbacks def test_priority_adjust_argument(self): priority_adjust = 1 - spider = yield self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust + 1}) + spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust + 1}) request = Request("https://example.com") new_request = get_retry_request( request, @@ -504,10 +481,9 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request.priority, priority_adjust) - @inlineCallbacks def test_log_extra_retry_success(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() with LogCapture(attributes=("spider",)) as log: get_retry_request( request, @@ -515,10 +491,9 @@ class GetRetryRequestTest(unittest.TestCase): ) log.check_present(spider) - @inlineCallbacks def test_log_extra_retries_exceeded(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() with LogCapture(attributes=("spider",)) as log: get_retry_request( request, @@ -527,10 +502,9 @@ class GetRetryRequestTest(unittest.TestCase): ) log.check_present(spider) - @inlineCallbacks def test_reason_string(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = "because" with LogCapture() as log: get_retry_request( @@ -550,10 +524,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_reason_builtin_exception(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = NotImplementedError() expected_reason_string = "builtins.NotImplementedError" with LogCapture() as log: @@ -576,10 +549,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_reason_builtin_exception_class(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = NotImplementedError expected_reason_string = "builtins.NotImplementedError" with LogCapture() as log: @@ -602,10 +574,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_reason_custom_exception(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = IgnoreRequest() expected_reason_string = "scrapy.exceptions.IgnoreRequest" with LogCapture() as log: @@ -628,10 +599,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_reason_custom_exception_class(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = IgnoreRequest expected_reason_string = "scrapy.exceptions.IgnoreRequest" with LogCapture() as log: @@ -654,11 +624,10 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_custom_logger(self): logger = logging.getLogger("custom-logger") request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = "because" with LogCapture() as log: get_retry_request( @@ -675,10 +644,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_custom_stats_key(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = "because" stats_key = "custom_retry" get_retry_request( @@ -692,3 +660,7 @@ class GetRetryRequestTest(unittest.TestCase): f"{stats_key}/reason_count/{expected_reason}", ): self.assertEqual(spider.crawler.stats.get_value(stat), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 55ae0c2b7..39dfe9ab5 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,29 +1,27 @@ import warnings from itertools import product - -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest +from unittest import TestCase from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils.response import response_httprepr from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider class MyException(Exception): pass -class TestDownloaderStats(unittest.TestCase): - @inlineCallbacks +class TestDownloaderStats(TestCase): def setUp(self): - self.crawler = get_crawler(NoRequestsSpider) - yield self.crawler.crawl() - self.spider = self.crawler.spider + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider("scrapytest.org") self.mw = DownloaderStats(self.crawler.stats) + self.crawler.stats.open_spider(self.spider) + self.req = Request("http://scrapytest.org") self.res = Response("scrapytest.org", status=400) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 1ba6125b2..aa0975555 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -2,25 +2,22 @@ import hashlib import shutil import sys import tempfile +import unittest from pathlib import Path from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest from scrapy.core.scheduler import Scheduler from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider +from tests.spiders import SimpleSpider -@inlineCallbacks def _get_dupefilter(*, crawler=None, settings=None, open=True): if crawler is None: - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) dupefilter = scheduler.df if open: @@ -47,51 +44,41 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter): class DirectDupeFilter: method = "n/a" - def open(self): - pass - class RFPDupeFilterTest(unittest.TestCase): - @inlineCallbacks def test_df_from_crawler_scheduler(self): settings = { "DUPEFILTER_DEBUG": True, "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, "from_crawler") - @inlineCallbacks def test_df_from_settings_scheduler(self): settings = { "DUPEFILTER_DEBUG": True, "DUPEFILTER_CLASS": FromSettingsRFPDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, "from_settings") - @inlineCallbacks def test_df_direct_scheduler(self): settings = { "DUPEFILTER_CLASS": DirectDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, "n/a") - @inlineCallbacks def test_filter(self): - dupefilter = yield _get_dupefilter() + dupefilter = _get_dupefilter() r1 = Request("http://scrapytest.org/1") r2 = Request("http://scrapytest.org/2") r3 = Request("http://scrapytest.org/2") @@ -104,14 +91,13 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close("finished") - @inlineCallbacks def test_dupefilter_path(self): r1 = Request("http://scrapytest.org/1") r2 = Request("http://scrapytest.org/2") path = tempfile.mkdtemp() try: - df = yield _get_dupefilter(settings={"JOBDIR": path}, open=False) + df = _get_dupefilter(settings={"JOBDIR": path}, open=False) try: df.open() assert not df.request_seen(r1) @@ -119,7 +105,7 @@ class RFPDupeFilterTest(unittest.TestCase): finally: df.close("finished") - df2 = yield _get_dupefilter(settings={"JOBDIR": path}, open=False) + df2 = _get_dupefilter(settings={"JOBDIR": path}, open=False) assert df != df2 try: df2.open() @@ -131,13 +117,12 @@ class RFPDupeFilterTest(unittest.TestCase): finally: shutil.rmtree(path) - @inlineCallbacks def test_request_fingerprint(self): """Test if customization of request_fingerprint method will change output of request_seen. """ - dupefilter = yield _get_dupefilter() + dupefilter = _get_dupefilter() r1 = Request("http://scrapytest.org/index.html") r2 = Request("http://scrapytest.org/INDEX.html") @@ -153,14 +138,13 @@ class RFPDupeFilterTest(unittest.TestCase): return fp.digest() settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter} - case_insensitive_dupefilter = yield _get_dupefilter(settings=settings) + case_insensitive_dupefilter = _get_dupefilter(settings=settings) assert not case_insensitive_dupefilter.request_seen(r1) assert case_insensitive_dupefilter.request_seen(r2) case_insensitive_dupefilter.close("finished") - @inlineCallbacks def test_seenreq_newlines(self): """Checks against adding duplicate \r to line endings on Windows platforms.""" @@ -168,8 +152,7 @@ class RFPDupeFilterTest(unittest.TestCase): r1 = Request("http://scrapytest.org/1") path = tempfile.mkdtemp() - crawler = get_crawler(NoRequestsSpider, settings_dict={"JOBDIR": path}) - yield crawler.crawl() + crawler = get_crawler(settings_dict={"JOBDIR": path}) try: scheduler = Scheduler.from_crawler(crawler) df = scheduler.df @@ -178,10 +161,7 @@ class RFPDupeFilterTest(unittest.TestCase): df.close("finished") with Path(path, "requests.seen").open("rb") as seen_file: - try: - line = next(seen_file).decode() - except StopIteration: - return + line = next(seen_file).decode() assert not line.endswith("\r\r\n") if sys.platform == "win32": assert line.endswith("\r\n") @@ -191,7 +171,6 @@ class RFPDupeFilterTest(unittest.TestCase): finally: shutil.rmtree(path) - @inlineCallbacks def test_log(self): with LogCapture() as log: settings = { @@ -199,10 +178,9 @@ class RFPDupeFilterTest(unittest.TestCase): "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() - spider = crawler.spider - dupefilter = yield _get_dupefilter(crawler=crawler) + crawler = get_crawler(SimpleSpider, settings_dict=settings) + spider = SimpleSpider.from_crawler(crawler) + dupefilter = _get_dupefilter(crawler=crawler) r1 = Request("http://scrapytest.org/index.html") r2 = Request("http://scrapytest.org/index.html") @@ -222,7 +200,6 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close("finished") - @inlineCallbacks def test_log_debug(self): with LogCapture() as log: settings = { @@ -230,10 +207,9 @@ class RFPDupeFilterTest(unittest.TestCase): "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() - spider = crawler.spider - dupefilter = yield _get_dupefilter(crawler=crawler) + crawler = get_crawler(SimpleSpider, settings_dict=settings) + spider = SimpleSpider.from_crawler(crawler) + dupefilter = _get_dupefilter(crawler=crawler) r1 = Request("http://scrapytest.org/index.html") r2 = Request( @@ -263,17 +239,15 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close("finished") - @inlineCallbacks def test_log_debug_default_dupefilter(self): with LogCapture() as log: settings = { "DUPEFILTER_DEBUG": True, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() - spider = crawler.spider - dupefilter = yield _get_dupefilter(crawler=crawler) + crawler = get_crawler(SimpleSpider, settings_dict=settings) + spider = SimpleSpider.from_crawler(crawler) + dupefilter = _get_dupefilter(crawler=crawler) r1 = Request("http://scrapytest.org/index.html") r2 = Request( diff --git a/tests/test_engine.py b/tests/test_engine.py index 5deae5146..8d7afb6a1 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -420,16 +420,12 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_close_downloader(self): - crawler = get_crawler(TestSpider) - yield crawler.crawl() - e = ExecutionEngine(crawler, lambda _: None) + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.close() @defer.inlineCallbacks def test_start_already_running_exception(self): - crawler = get_crawler(TestSpider) - yield crawler.crawl() - e = ExecutionEngine(crawler, lambda _: None) + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() try: diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index ce8452b5f..6e5fb0325 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,8 +1,6 @@ import datetime import typing - -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest +import unittest from scrapy.crawler import Crawler from scrapy.extensions.periodic_log import PeriodicLog @@ -61,44 +59,33 @@ class TestExtPeriodicLog(PeriodicLog): self.stats._stats = stats_dump_2 -@inlineCallbacks def extension(settings=None): - crawler = Crawler( - MetaSpider, - settings=settings, - ) - yield crawler.crawl() + crawler = Crawler(MetaSpider, settings=settings) + crawler._load_settings() return TestExtPeriodicLog.from_crawler(crawler) class TestPeriodicLog(unittest.TestCase): - @inlineCallbacks def test_extension_enabled(self): # Expected that settings for this extension loaded succesfully # And on certain conditions - extension raising NotConfigured # "PERIODIC_LOG_STATS": True -> set to {"enabled": True} # due to TypeError exception from settings.getdict - assert (yield extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60})) + assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60}) # "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True} # due to JSONDecodeError(ValueError) exception from settings.getdict - assert ( - yield extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60}) - ) + assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60}) # The ame for PERIODIC_LOG_DELTA: - assert (yield extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})) - assert ( - yield extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) - ) + assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) + assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) - @inlineCallbacks def test_log_delta(self): - @inlineCallbacks def emulate(settings=None): spider = MetaSpider() - ext = yield extension(settings) + ext = extension(settings) ext.spider_opened(spider) ext.set_a() a = ext.log_delta() @@ -107,9 +94,8 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - @inlineCallbacks def check(settings: dict, condition: typing.Callable): - ext, a, b = yield emulate(settings) + ext, a, b = emulate(settings) assert list(a["delta"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) ] @@ -118,49 +104,45 @@ class TestPeriodicLog(unittest.TestCase): ] # Including all - yield check( - {"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float)) - ) + check({"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float))) # include: - yield check( + check( {"PERIODIC_LOG_DELTA": {"include": ["downloader/"]}}, lambda k, v: isinstance(v, (int, float)) and "downloader/" in k, ) # include multiple - yield check( + check( {"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}}, lambda k, v: isinstance(v, (int, float)) and ("downloader/" in k or "scheduler/" in k), ) # exclude - yield check( + check( {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}}, lambda k, v: isinstance(v, (int, float)) and "downloader/" not in k, ) # exclude multiple - yield check( + check( {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}}, lambda k, v: isinstance(v, (int, float)) and ("downloader/" not in k and "scheduler/" not in k), ) # include exclude combined - yield check( + check( {"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: isinstance(v, (int, float)) and ("downloader/" in k and "bytes" not in k), ) - @inlineCallbacks def test_log_stats(self): - @inlineCallbacks def emulate(settings=None): spider = MetaSpider() - ext = yield extension(settings) + ext = extension(settings) ext.spider_opened(spider) ext.set_a() a = ext.log_crawler_stats() @@ -169,9 +151,8 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - @inlineCallbacks def check(settings: dict, condition: typing.Callable): - ext, a, b = yield emulate(settings) + ext, a, b = emulate(settings) assert list(a["stats"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) ] @@ -180,34 +161,35 @@ class TestPeriodicLog(unittest.TestCase): ] # Including all - yield check({"PERIODIC_LOG_STATS": True}, lambda k, v: True) + check({"PERIODIC_LOG_STATS": True}, lambda k, v: True) # include: - yield check( + check( {"PERIODIC_LOG_STATS": {"include": ["downloader/"]}}, lambda k, v: "downloader/" in k, ) # include multiple - yield check( + check( {"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}}, lambda k, v: "downloader/" in k or "scheduler/" in k, ) # exclude - yield check( + check( {"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}}, lambda k, v: "downloader/" not in k, ) # exclude multiple - yield check( + check( {"PERIODIC_LOG_STATS": {"exclude": ["downloader/", "scheduler/"]}}, lambda k, v: "downloader/" not in k and "scheduler/" not in k, ) # include exclude combined - yield check( + check( {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: "downloader/" in k and "bytes" not in k, ) + # diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 6f35510b7..9fd680e9f 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -9,7 +9,7 @@ from scrapy.utils.test import get_crawler class TelnetExtensionTest(unittest.TestCase): def _get_console_and_portal(self, settings=None): - crawler = get_crawler(settings_dict=settings, disable_telnet=False) + crawler = get_crawler(settings_dict=settings) console = TelnetConsole(crawler) # This function has some side effects we don't need for this test diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 875abdb1f..6b82974fa 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -26,7 +26,6 @@ import lxml.etree import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from w3lib.url import file_uri_to_path, path_to_file_uri from zope.interface import implementer @@ -52,7 +51,7 @@ from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto from tests.mockserver import MockFTPServer, MockServer -from tests.spiders import ItemSpider, NoRequestsSpider +from tests.spiders import ItemSpider def path_to_url(path): @@ -2748,14 +2747,13 @@ class FeedExporterSignalsTest(unittest.TestCase): d.callback(None) return d - @inlineCallbacks def run_signaled_feed_exporter( self, feed_exporter_signal_handler, feed_slot_signal_handler ): - crawler = get_crawler(NoRequestsSpider, settings_dict=self.settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=self.settings) feed_exporter = FeedExporter.from_crawler(crawler) - spider = crawler.spider + spider = scrapy.Spider("default") + spider.crawler = crawler crawler.signals.connect( feed_exporter_signal_handler, signal=signals.feed_exporter_closed, @@ -2766,26 +2764,24 @@ class FeedExporterSignalsTest(unittest.TestCase): feed_exporter.open_spider(spider) for item in self.items: feed_exporter.item_scraped(item, spider) - yield defer.ensureDeferred(feed_exporter.close_spider(spider)) + defer.ensureDeferred(feed_exporter.close_spider(spider)) - @inlineCallbacks def test_feed_exporter_signals_sent(self): self.feed_exporter_closed_received = False self.feed_slot_closed_received = False - yield self.run_signaled_feed_exporter( + self.run_signaled_feed_exporter( self.feed_exporter_closed_signal_handler, self.feed_slot_closed_signal_handler, ) self.assertTrue(self.feed_slot_closed_received) self.assertTrue(self.feed_exporter_closed_received) - @inlineCallbacks def test_feed_exporter_signals_sent_deferred(self): self.feed_exporter_closed_received = False self.feed_slot_closed_received = False - yield self.run_signaled_feed_exporter( + self.run_signaled_feed_exporter( self.feed_exporter_closed_signal_handler_deferred, self.feed_slot_closed_signal_handler_deferred, ) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 0d769f29b..bf96f17b6 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -15,7 +15,6 @@ import attr import pytest from itemadapter import ItemAdapter from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from scrapy.http import Request, Response @@ -36,18 +35,15 @@ from scrapy.utils.test import ( skip_if_no_boto, ) from tests.mockserver import MockFTPServer -from tests.spiders import NoRequestsSpider from .test_pipeline_media import _mocked_download_func class FilesPipelineTestCase(unittest.TestCase): - @inlineCallbacks def setUp(self): self.tempdir = mkdtemp() settings_dict = {"FILES_STORE": self.tempdir} - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(spidercls=None, settings_dict=settings_dict) self.pipeline = FilesPipeline.from_crawler(crawler) self.pipeline.download_func = _mocked_download_func self.pipeline.open_spider(None) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 8a13ea552..d655eb128 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -14,11 +14,11 @@ from scrapy.pipelines.files import FileException from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings +from scrapy.spiders import Spider from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider try: from PIL import Image # noqa: imported just to check for the import error @@ -40,11 +40,10 @@ class BaseMediaPipelineTestCase(unittest.TestCase): pipeline_class = MediaPipeline settings = None - @inlineCallbacks def setUp(self): - crawler = get_crawler(NoRequestsSpider, self.settings) - yield crawler.crawl() - self.spider = crawler.spider + spider_cls = Spider + self.spider = spider_cls("media.com") + crawler = get_crawler(spider_cls, self.settings) self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe.download_func = _mocked_download_func self.pipe.open_spider(self.spider) @@ -431,14 +430,12 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): skip = skip_pillow - @inlineCallbacks def setUp(self): settings_dict = { "IMAGES_STORE": "store-uri", "IMAGES_THUMBS": {"small": (50, 50)}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(spidercls=None, settings_dict=settings_dict) self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler) self.pipe.download_func = _mocked_download_func self.pipe.open_spider(None) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 54c77eb68..ef9b360c4 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,10 +1,10 @@ import collections import shutil import tempfile +import unittest from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest +from twisted.trial.unittest import TestCase from scrapy.core.downloader import Downloader from scrapy.core.scheduler import Scheduler @@ -15,7 +15,6 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import NoRequestsSpider MockEngine = collections.namedtuple("MockEngine", ["downloader"]) MockSlot = collections.namedtuple("MockSlot", ["active"]) @@ -310,7 +309,7 @@ class StartUrlsSpider(Spider): pass -class TestIntegrationWithDownloaderAwareInMemory(unittest.TestCase): +class TestIntegrationWithDownloaderAwareInMemory(TestCase): def setUp(self): self.crawler = get_crawler( spidercls=StartUrlsSpider, @@ -337,19 +336,16 @@ class TestIntegrationWithDownloaderAwareInMemory(unittest.TestCase): class TestIncompatibility(unittest.TestCase): - @inlineCallbacks def _incompatible(self): settings = dict( SCHEDULER_PRIORITY_QUEUE="scrapy.pqueues.DownloaderAwarePriorityQueue", CONCURRENT_REQUESTS_PER_IP=1, ) - crawler = get_crawler(NoRequestsSpider, settings) - yield crawler.crawl() - spider = crawler.spider + crawler = get_crawler(Spider, settings) scheduler = Scheduler.from_crawler(crawler) + spider = Spider(name="spider") scheduler.open(spider) - @inlineCallbacks def test_incompatibility(self): with self.assertRaises(ValueError): - yield self._incompatible() + self._incompatible() diff --git a/tests/test_spider.py b/tests/test_spider.py index a88d9b505..00da3d485 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -114,7 +114,7 @@ class SpiderTest(unittest.TestCase): spider.settings.set("TEST1", "spider_instance", priority="spider") return spider - crawler = get_crawler(TestSpider, settings_dict=project_settings) + crawler = Crawler(TestSpider, project_settings) self.assertEqual(crawler.settings.get("TEST1"), "spider") self.assertEqual(crawler.settings.get("TEST2"), "spider") self.assertEqual(crawler.settings.get("TEST3"), "project") diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 2fcb5b364..1d5a887cc 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,9 +1,9 @@ import logging +from unittest import TestCase from testfixtures import LogCapture from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks -from twisted.trial.unittest import TestCase +from twisted.trial.unittest import TestCase as TrialTestCase from scrapy.http import Request, Response from scrapy.settings import Settings @@ -11,7 +11,7 @@ from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import MockServerSpider, NoRequestsSpider +from tests.spiders import MockServerSpider class _HttpErrorSpider(MockServerSpider): @@ -59,11 +59,9 @@ def _responses(request, status_codes): class TestHttpErrorMiddleware(TestCase): - @inlineCallbacks def setUp(self): - crawler = get_crawler(NoRequestsSpider) - yield crawler.crawl() - self.spider = crawler.spider + crawler = get_crawler(Spider) + self.spider = Spider.from_crawler(crawler, name="foo") self.mw = HttpErrorMiddleware(Settings({})) self.req = Request("http://scrapytest.org") self.res200, self.res404 = _responses(self.req, [200, 404]) @@ -173,7 +171,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.assertIsNone(mw.process_spider_input(res402, self.spider)) -class TestHttpErrorMiddlewareIntegrational(TestCase): +class TestHttpErrorMiddlewareIntegrational(TrialTestCase): def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index ed00c0a6b..ea45b7698 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -1,21 +1,17 @@ import warnings +from unittest import TestCase from urllib.parse import urlparse -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest - from scrapy.http import Request, Response from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, PortWarning, URLWarning from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestOffsiteMiddleware(unittest.TestCase): - @inlineCallbacks +class TestOffsiteMiddleware(TestCase): def setUp(self): crawler = get_crawler(Spider) - yield crawler.crawl(**self._get_spiderargs()) - self.spider = crawler.spider + self.spider = crawler._create_spider(**self._get_spiderargs()) self.mw = OffsiteMiddleware.from_crawler(crawler) self.mw.spider_opened(self.spider) diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index a3c284484..9111e4c82 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,23 +1,21 @@ +from unittest import TestCase + from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider -class TestUrlLengthMiddleware(unittest.TestCase): - @inlineCallbacks +class TestUrlLengthMiddleware(TestCase): def setUp(self): self.maxlength = 25 settings = Settings({"URLLENGTH_LIMIT": self.maxlength}) - crawler = get_crawler(NoRequestsSpider) - yield crawler.crawl() - self.spider = crawler.spider + crawler = get_crawler(Spider) + self.spider = crawler._create_spider("foo") self.stats = crawler.stats self.mw = UrlLengthMiddleware.from_settings(settings) diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 2117e65b1..eae744df5 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,10 +1,9 @@ import logging import sys +import unittest from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks from twisted.python.failure import Failure -from twisted.trial import unittest from scrapy.extensions import telnet from scrapy.utils.log import ( @@ -14,7 +13,6 @@ from scrapy.utils.log import ( failure_to_exc_info, ) from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider class FailureToExcInfoTest(unittest.TestCase): @@ -62,7 +60,6 @@ class TopLevelFormatterTest(unittest.TestCase): class LogCounterHandlerTest(unittest.TestCase): - @inlineCallbacks def setUp(self): settings = {"LOG_LEVEL": "WARNING"} if not telnet.TWISTED_CONCH_AVAILABLE: @@ -71,8 +68,7 @@ class LogCounterHandlerTest(unittest.TestCase): self.logger = logging.getLogger("test") self.logger.setLevel(logging.NOTSET) self.logger.propagate = False - self.crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield self.crawler.crawl() + self.crawler = get_crawler(settings_dict=settings) self.handler = LogCounterHandler(self.crawler) self.logger.addHandler(self.handler) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 9ca9faa0c..e6d1abe3f 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,12 +1,11 @@ import json +import unittest import warnings from hashlib import sha1 from typing import Dict, Mapping, Optional, Tuple, Union from weakref import WeakKeyDictionary import pytest -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest from w3lib.url import canonicalize_url from scrapy.http import Request @@ -23,7 +22,6 @@ from scrapy.utils.request import ( request_to_curl, ) from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider class UtilsRequestTest(unittest.TestCase): @@ -451,18 +449,15 @@ class BackwardCompatibilityTestCase(unittest.TestCase): ) self.assertEqual(fp, old_fp) - @inlineCallbacks def test_component_backward_compatibility(self): for request_object in REQUEST_OBJECTS_TO_TEST: with warnings.catch_warnings(): warnings.simplefilter("ignore") - crawler = get_crawler(NoRequestsSpider, prevent_warnings=False) - yield crawler.crawl() + crawler = get_crawler(prevent_warnings=False) fp = crawler.request_fingerprinter.fingerprint(request_object) old_fp = request_fingerprint_2_6(request_object) self.assertEqual(fp.hex(), old_fp) - @inlineCallbacks def test_custom_component_backward_compatibility(self): """Tests that the backward-compatible request fingerprinting class featured in the documentation is indeed backward compatible and does not cause a @@ -485,8 +480,7 @@ class BackwardCompatibilityTestCase(unittest.TestCase): settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) fp = crawler.request_fingerprinter.fingerprint(request_object) old_fp = request_fingerprint_2_6(request_object) self.assertEqual(fp.hex(), old_fp) @@ -494,11 +488,9 @@ class BackwardCompatibilityTestCase(unittest.TestCase): class RequestFingerprinterTestCase(unittest.TestCase): - @inlineCallbacks def test_default_implementation(self): with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(NoRequestsSpider, prevent_warnings=False) - yield crawler.crawl() + crawler = get_crawler(prevent_warnings=False) request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), @@ -506,14 +498,12 @@ class RequestFingerprinterTestCase(unittest.TestCase): ) self.assertTrue(logged_warnings) - @inlineCallbacks def test_deprecated_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6", } with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), @@ -521,14 +511,12 @@ class RequestFingerprinterTestCase(unittest.TestCase): ) self.assertTrue(logged_warnings) - @inlineCallbacks def test_recommended_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), @@ -536,18 +524,15 @@ class RequestFingerprinterTestCase(unittest.TestCase): ) self.assertFalse(logged_warnings) - @inlineCallbacks def test_unknown_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5", } with self.assertRaises(ValueError): - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + get_crawler(settings_dict=settings) class CustomRequestFingerprinterTestCase(unittest.TestCase): - @inlineCallbacks def test_include_headers(self): class RequestFingerprinter: def fingerprint(self, request): @@ -556,8 +541,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) r1 = Request("http://www.example.com", headers={"X-ID": "1"}) fp1 = crawler.request_fingerprinter.fingerprint(r1) @@ -565,7 +549,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp2 = crawler.request_fingerprinter.fingerprint(r2) self.assertNotEqual(fp1, fp2) - @inlineCallbacks def test_dont_canonicalize(self): class RequestFingerprinter: cache = WeakKeyDictionary() @@ -580,8 +563,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) r1 = Request("http://www.example.com?a=1&a=2") fp1 = crawler.request_fingerprinter.fingerprint(r1) @@ -589,7 +571,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp2 = crawler.request_fingerprinter.fingerprint(r2) self.assertNotEqual(fp1, fp2) - @inlineCallbacks def test_meta(self): class RequestFingerprinter: def fingerprint(self, request): @@ -600,8 +581,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) r1 = Request("http://www.example.com") fp1 = crawler.request_fingerprinter.fingerprint(r1) @@ -616,7 +596,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): self.assertNotEqual(fp2, fp4) self.assertEqual(fp2, fp3) - @inlineCallbacks def test_from_crawler(self): class RequestFingerprinter: @classmethod @@ -633,14 +612,12 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) self.assertEqual(fingerprint, settings["FINGERPRINT"]) - @inlineCallbacks def test_from_settings(self): class RequestFingerprinter: @classmethod @@ -657,14 +634,12 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) self.assertEqual(fingerprint, settings["FINGERPRINT"]) - @inlineCallbacks def test_from_crawler_and_settings(self): class RequestFingerprinter: # This method is ignored due to the presence of from_crawler @@ -686,8 +661,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) @@ -754,3 +728,7 @@ class RequestToCurlTest(unittest.TestCase): " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" ) self._test_request(request_object, expected_curl_command) + + +if __name__ == "__main__": + unittest.main() From 1a0572ad02446c0dacc682e503baffc2b5e67e98 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 12 Sep 2023 16:46:35 +0400 Subject: [PATCH 1150/2083] Reinstall the log handler after loading per-spider settings. --- scrapy/crawler.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 67e44541d..90f5e7918 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -72,18 +72,16 @@ class Crawler: self.spidercls: Type[Spider] = spidercls self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) + self._update_root_log_handler() self.addons: AddonManager = AddonManager(self) self.signals: SignalManager = SignalManager(self) - if get_scrapy_root_handler() is not None: - # scrapy root handler already installed: update it with new settings - install_scrapy_root_handler(self.settings) - self._init_reactor: bool = init_reactor self.crawling: bool = False self._settings_loaded: bool = False self._started: bool = False + self.extensions: Optional[ExtensionManager] = None self.stats: Optional[StatsCollector] = None self.logformatter: Optional[LogFormatter] = None @@ -91,6 +89,11 @@ class Crawler: self.spider: Optional[Spider] = None self.engine: Optional[ExecutionEngine] = None + def _update_root_log_handler(self) -> None: + if get_scrapy_root_handler() is not None: + # scrapy root handler already installed: update it with new settings + install_scrapy_root_handler(self.settings) + def _load_settings(self) -> None: if self._settings_loaded: return @@ -153,6 +156,7 @@ class Crawler: try: self.spider = self._create_spider(*args, **kwargs) self._load_settings() + self._update_root_log_handler() self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) From e26bf4f918befe0fe0625d240ea4a54294923a62 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 12 Sep 2023 18:18:25 +0400 Subject: [PATCH 1151/2083] Pin brotli for PyPy tests (#6045) --- tests/requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 37186f3a7..3ea7f3333 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -12,7 +12,9 @@ uvloop; platform_system != "Windows" and python_version < "3.12" # bpython requires greenlet which currently doesn't build on 3.12 bpython; python_version < "3.12" # optional for shell wrapper tests -brotli # optional for HTTP compress downloader middleware tests +brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests +# 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 +brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" From 77f39be4073cdce9e1b52ea0ee846881b13e6f23 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 12 Sep 2023 20:16:26 +0400 Subject: [PATCH 1152/2083] Add integration tests for modifying settings via args. --- tests/CrawlerProcess/args_settings.py | 24 ++++++++++++++++++++++++ tests/test_commands.py | 22 ++++++++++++++++++++++ tests/test_crawler.py | 5 +++++ 3 files changed, 51 insertions(+) create mode 100644 tests/CrawlerProcess/args_settings.py diff --git a/tests/CrawlerProcess/args_settings.py b/tests/CrawlerProcess/args_settings.py new file mode 100644 index 000000000..a46a8806b --- /dev/null +++ b/tests/CrawlerProcess/args_settings.py @@ -0,0 +1,24 @@ +from typing import Any + +import scrapy +from scrapy.crawler import Crawler, CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("FOO", kwargs.get("foo")) + return spider + + def start_requests(self): + self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") + return [] + + +process = CrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider, foo=42) +process.start() diff --git a/tests/test_commands.py b/tests/test_commands.py index b1d7be628..36f800850 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -965,6 +965,28 @@ class MySpider(scrapy.Spider): log, ) + def test_args_change_settings(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("FOO", kwargs.get("foo")) + return spider + + def start_requests(self): + self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") + return [] +""" + args = ["-a", "foo=42"] + log = self.get_log(spider_code, args=args) + self.assertIn("Spider closed (finished)", log) + self.assertIn("The value of FOO is 42", log) + @skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 120991ae7..bfae6c690 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -481,6 +481,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertNotIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) + def test_args_change_settings(self): + log = self.run_script("args_settings.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("The value of FOO is 42", log) + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" From a3f22046efaf661bd7d463decb65eea871b7a1d5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 12 Sep 2023 20:41:36 +0400 Subject: [PATCH 1153/2083] Document changing settings in Spider.from_crawler(). --- docs/topics/settings.rst | 20 ++++++++++++++++++++ docs/topics/spiders.rst | 8 ++++++++ 2 files changed, 28 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e1936eb5b..65823e071 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -98,6 +98,26 @@ and settings set there should use the "spider" priority explicitly: super().update_settings(settings) settings.set("SOME_SETTING", "some value", priority="spider") +.. versionadded:: VERSION + +It's also possible to modify the settings in the +:meth:`~scrapy.Spider.from_crawler` method, e.g. based on :ref:`spider +arguments ` or other logic: + +.. code-block:: python + + import scrapy + + + class MySpider(scrapy.Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("SOME_SETTING", kwargs["value"], priority="spider") + return spider + 3. Project settings module -------------------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 5c3bf6e72..4ed9b8dc3 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -136,6 +136,14 @@ scrapy.Spider attributes in the new instance so they can be accessed later inside the spider's code. + .. versionchanged:: VERSION + + The settings available in this method can now be modified, which is + handy if you want to modify them based on arguments. As a + consequence, the settings available in this method aren't the final + values as they can be modified later by e.g. :ref:`add-ons + `. + :param crawler: crawler to which the spider will be bound :type crawler: :class:`~scrapy.crawler.Crawler` instance From da39fbd2709edc737059c68c5f70da19b89bbaa2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 12 Sep 2023 22:07:10 +0400 Subject: [PATCH 1154/2083] Update tool versions (#6046) --- .github/workflows/tests-ubuntu.yml | 6 +++--- .pre-commit-config.yaml | 6 +++--- tox.ini | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index c2b686628..5ff92a571 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -48,13 +48,13 @@ jobs: env: TOXENV: botocore - - python-version: "3.12.0-rc.1" + - python-version: "3.12.0-rc.2" env: TOXENV: py - - python-version: "3.12.0-rc.1" + - python-version: "3.12.0-rc.2" env: TOXENV: asyncio - - python-version: "3.12.0-rc.1" + - python-version: "3.12.0-rc.2" env: TOXENV: extra-deps diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5998ebef8..0cff5cc73 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 23.7.0 + rev: 23.9.1 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -17,8 +17,8 @@ repos: hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs - rev: 1.15.0 + rev: 1.16.0 hooks: - id: blacken-docs additional_dependencies: - - black==23.7.0 + - black==23.9.1 diff --git a/tox.ini b/tox.ini index 3ed8b6f63..9c2522a43 100644 --- a/tox.ini +++ b/tox.ini @@ -33,14 +33,14 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.4.1 + mypy==1.5.1 typing-extensions==4.7.1 types-attrs==19.1.0 types-lxml==2023.3.28 - types-Pillow==10.0.0.2 - types-Pygments==2.15.0.2 + types-Pillow==10.0.0.3 + types-Pygments==2.16.0.0 types-pyOpenSSL==23.2.0.2 - types-setuptools==68.0.0.3 + types-setuptools==68.2.0.0 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands = @@ -65,7 +65,7 @@ commands = basepython = python3 deps = twine==4.0.2 - build==0.10.0 + build==1.0.3 commands = python -m build --sdist twine check dist/* From 61e6bfc023e580dbcd601b6a694122ef2534039b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Sep 2023 12:35:15 +0400 Subject: [PATCH 1155/2083] Docs improvements. --- docs/topics/settings.rst | 3 ++- docs/topics/spiders.rst | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 65823e071..d3fe6bbe2 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -115,7 +115,8 @@ arguments ` or other logic: @classmethod def from_crawler(cls, crawler, *args, **kwargs): spider = super().from_crawler(crawler, *args, **kwargs) - spider.settings.set("SOME_SETTING", kwargs["value"], priority="spider") + if "value" in kwargs: + spider.settings.set("SOME_SETTING", kwargs["value"], priority="spider") return spider 3. Project settings module diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 4ed9b8dc3..d9cbbe35a 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -138,11 +138,10 @@ scrapy.Spider .. versionchanged:: VERSION - The settings available in this method can now be modified, which is + The settings in ``crawler.settings`` can now be modified, which is handy if you want to modify them based on arguments. As a - consequence, the settings available in this method aren't the final - values as they can be modified later by e.g. :ref:`add-ons - `. + consequence, these settings aren't the final values as they can be + modified later by e.g. :ref:`add-ons `. :param crawler: crawler to which the spider will be bound :type crawler: :class:`~scrapy.crawler.Crawler` instance From 028a56b9a2e090bde761a0487a8504a3b263eb97 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Sep 2023 14:17:41 +0400 Subject: [PATCH 1156/2083] Improve and simplify tests. --- tests/test_addons.py | 21 +++++++++++++++++++++ tests/test_crawl.py | 40 ++-------------------------------------- tests/test_crawler.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 38 deletions(-) diff --git a/tests/test_addons.py b/tests/test_addons.py index aa1b760c2..8375a6495 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -3,6 +3,8 @@ import unittest from typing import Any, Dict from unittest.mock import patch +from twisted.internet.defer import inlineCallbacks + from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner from scrapy.exceptions import NotConfigured @@ -177,3 +179,22 @@ class AddonManagerTest(unittest.TestCase): {"addons": [addon]}, extra={"crawler": crawler}, ) + + @inlineCallbacks + def test_enable_addon_in_spider(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + addon_config = {"KEY": "addon"} + addon_cls = get_addon_cls(addon_config) + spider.settings.set("ADDONS", {addon_cls: 1}, priority="spider") + return spider + + runner = CrawlerRunner({"KEY": "project"}) + crawler = runner.create_crawler(MySpider) + self.assertEqual(crawler.settings.get("KEY"), "project") + yield crawler.crawl() + self.assertEqual(crawler.settings.get("KEY"), "addon") diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 496ab77a5..96d43b2b9 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,7 +1,6 @@ import json import logging import unittest -import warnings from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse @@ -14,14 +13,11 @@ from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy import signals -from scrapy.crawler import Crawler, CrawlerRunner -from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload -from scrapy.extensions.throttle import AutoThrottle +from scrapy.crawler import CrawlerRunner +from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.http.response import Response -from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import MockServer @@ -414,38 +410,6 @@ with multiples lines self._assert_retried(log) self.assertIn("Got response 200", str(log)) - @defer.inlineCallbacks - def test_populate_spidercls_settings(self): - spider_settings = { - "TEST1": "spider", - "TEST2": "spider", - "AUTOTHROTTLE_ENABLED": True, - } - project_settings = {"TEST1": "project", "TEST3": "project"} - - class CustomSettingsSpider(DefaultSpider): - custom_settings = spider_settings - - def parse(self, response): - return - - settings = Settings() - settings.setdict(project_settings, priority="project") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - crawler = Crawler(CustomSettingsSpider, settings) - yield crawler.crawl() - - self.assertEqual(crawler.settings.get("TEST1"), "spider") - self.assertEqual(crawler.settings.get("TEST2"), "spider") - self.assertEqual(crawler.settings.get("TEST3"), "project") - - enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] - self.assertIn(AutoThrottle, enabled_exts) - - self.assertFalse(settings.frozen) - self.assertTrue(crawler.settings.frozen) - class CrawlSpiderTestCase(TestCase): def setUp(self): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index bfae6c690..08149725c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -17,6 +17,7 @@ import scrapy from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions import telnet +from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler @@ -32,6 +33,25 @@ class BaseCrawlerTest(unittest.TestCase): class CrawlerTestCase(BaseCrawlerTest): + def test_populate_spidercls_settings(self): + spider_settings = {"TEST1": "spider", "TEST2": "spider"} + project_settings = {"TEST1": "project", "TEST3": "project"} + + class CustomSettingsSpider(DefaultSpider): + custom_settings = spider_settings + + settings = Settings() + settings.setdict(project_settings, priority="project") + crawler = Crawler(CustomSettingsSpider, settings) + crawler._load_settings() + + self.assertEqual(crawler.settings.get("TEST1"), "spider") + self.assertEqual(crawler.settings.get("TEST2"), "spider") + self.assertEqual(crawler.settings.get("TEST3"), "project") + + self.assertFalse(settings.frozen) + self.assertTrue(crawler.settings.frozen) + def test_crawler_accepts_dict(self): crawler = get_crawler(DefaultSpider, {"foo": "bar"}) self.assertEqual(crawler.settings["foo"], "bar") @@ -58,6 +78,17 @@ class CrawlerTestCase(BaseCrawlerTest): yield crawler.crawl() +class SpiderSettingsTestCase(unittest.TestCase): + def test_spider_custom_settings(self): + class MySpider(scrapy.Spider): + name = "spider" + custom_settings = {"AUTOTHROTTLE_ENABLED": True} + + crawler = get_crawler(MySpider) + enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] + self.assertIn(AutoThrottle, enabled_exts) + + class CrawlerLoggingTestCase(unittest.TestCase): def test_no_root_handler_installed(self): handler = get_scrapy_root_handler() From 619140717fd74aff0c8119217f4d11f688d4dac6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Sep 2023 15:33:51 +0400 Subject: [PATCH 1157/2083] Fix the new addon test. --- tests/test_addons.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_addons.py b/tests/test_addons.py index 8375a6495..68e91c655 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,9 +1,9 @@ import itertools -import unittest from typing import Any, Dict from unittest.mock import patch from twisted.internet.defer import inlineCallbacks +from twisted.trial import unittest from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner @@ -193,8 +193,10 @@ class AddonManagerTest(unittest.TestCase): spider.settings.set("ADDONS", {addon_cls: 1}, priority="spider") return spider - runner = CrawlerRunner({"KEY": "project"}) + settings = Settings() + settings.set("KEY", "default", priority="default") + runner = CrawlerRunner(settings) crawler = runner.create_crawler(MySpider) - self.assertEqual(crawler.settings.get("KEY"), "project") + self.assertEqual(crawler.settings.get("KEY"), "default") yield crawler.crawl() self.assertEqual(crawler.settings.get("KEY"), "addon") From 37562163393cc145171b802699c84467781c701b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Sep 2023 15:47:22 +0400 Subject: [PATCH 1158/2083] Rename methods. --- scrapy/addons.py | 4 ++-- scrapy/commands/shell.py | 2 +- scrapy/crawler.py | 6 +++--- scrapy/utils/test.py | 2 +- tests/test_addons.py | 2 +- tests/test_crawler.py | 2 +- tests/test_extension_periodic_log.py | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 2634bf907..389a3cdde 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -19,8 +19,8 @@ class AddonManager: self.crawler: "Crawler" = crawler self.addons: List[Any] = [] - def load_settings(self, settings: Settings) -> None: - """Load add-ons and configurations from a settings object. + def apply_settings(self, settings: Settings) -> None: + """Load add-ons and configurations from a settings object and apply them. This will load the add-on for every add-on path in the ``ADDONS`` setting and execute their ``update_settings`` methods. diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 71f43365d..12e37babc 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -77,7 +77,7 @@ class Command(ScrapyCommand): # The crawler is created this way since the Shell manually handles the # crawling engine, so the set up in the crawl method won't work crawler = self.crawler_process._create_crawler(spidercls) - crawler._load_settings() + crawler._apply_settings() # The Shell class needs a persistent engine in the crawler crawler.engine = crawler._create_engine() crawler.engine.start() diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 90f5e7918..ee4d6fd59 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -94,12 +94,12 @@ class Crawler: # scrapy root handler already installed: update it with new settings install_scrapy_root_handler(self.settings) - def _load_settings(self) -> None: + def _apply_settings(self) -> None: if self._settings_loaded: return self._settings_loaded = True - self.addons.load_settings(self.settings) + self.addons.apply_settings(self.settings) self.stats = load_object(self.settings["STATS_CLASS"])(self) handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) @@ -155,7 +155,7 @@ class Crawler: try: self.spider = self._create_spider(*args, **kwargs) - self._load_settings() + self._apply_settings() self._update_root_log_handler() self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 9397e78b9..709e0b00d 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -91,7 +91,7 @@ def get_crawler( settings.update(settings_dict or {}) runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or TestSpider) - crawler._load_settings() + crawler._apply_settings() return crawler diff --git a/tests/test_addons.py b/tests/test_addons.py index 68e91c655..0f4f2e5b8 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -113,7 +113,7 @@ class AddonManagerTest(unittest.TestCase): settings.set("KEY", 0, priority="default") runner = CrawlerRunner(settings) crawler = runner.create_crawler(Spider) - crawler._load_settings() + crawler._apply_settings() self.assertEqual(crawler.settings.getint("KEY"), 15) settings_dict = { diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 08149725c..2b141e894 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -43,7 +43,7 @@ class CrawlerTestCase(BaseCrawlerTest): settings = Settings() settings.setdict(project_settings, priority="project") crawler = Crawler(CustomSettingsSpider, settings) - crawler._load_settings() + crawler._apply_settings() self.assertEqual(crawler.settings.get("TEST1"), "spider") self.assertEqual(crawler.settings.get("TEST2"), "spider") diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 6e5fb0325..502ada6be 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -61,7 +61,7 @@ class TestExtPeriodicLog(PeriodicLog): def extension(settings=None): crawler = Crawler(MetaSpider, settings=settings) - crawler._load_settings() + crawler._apply_settings() return TestExtPeriodicLog.from_crawler(crawler) From 61ef37a59447ad7ee107f7462c51ca77bdb27367 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Sep 2023 15:47:41 +0400 Subject: [PATCH 1159/2083] Docs improvements. --- docs/topics/settings.rst | 6 ++++-- docs/topics/spiders.rst | 9 +++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d3fe6bbe2..3006fb8b1 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -115,8 +115,10 @@ arguments ` or other logic: @classmethod def from_crawler(cls, crawler, *args, **kwargs): spider = super().from_crawler(crawler, *args, **kwargs) - if "value" in kwargs: - spider.settings.set("SOME_SETTING", kwargs["value"], priority="spider") + if "some_argument" in kwargs: + spider.settings.set( + "SOME_SETTING", kwargs["some_argument"], priority="spider" + ) return spider 3. Project settings module diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index d9cbbe35a..3197daf65 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -138,10 +138,11 @@ scrapy.Spider .. versionchanged:: VERSION - The settings in ``crawler.settings`` can now be modified, which is - handy if you want to modify them based on arguments. As a - consequence, these settings aren't the final values as they can be - modified later by e.g. :ref:`add-ons `. + The settings in ``crawler.settings`` can now be modified in this + method, which is handy if you want to modify them based on + arguments. As a consequence, these settings aren't the final values + as they can be modified later by e.g. :ref:`add-ons + `. The settings become final when the spider starts. :param crawler: crawler to which the spider will be bound :type crawler: :class:`~scrapy.crawler.Crawler` instance From ac201d310b812c53465ad6a27b033f22206dae4b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Sep 2023 16:17:52 +0400 Subject: [PATCH 1160/2083] Small improvements, --- docs/topics/spiders.rst | 3 ++- scrapy/addons.py | 2 +- scrapy/crawler.py | 6 ++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 3197daf65..1ca7eda7b 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -142,7 +142,8 @@ scrapy.Spider method, which is handy if you want to modify them based on arguments. As a consequence, these settings aren't the final values as they can be modified later by e.g. :ref:`add-ons - `. The settings become final when the spider starts. + `. The final settings are available in the + :meth:`start_requests` method and later. :param crawler: crawler to which the spider will be bound :type crawler: :class:`~scrapy.crawler.Crawler` instance diff --git a/scrapy/addons.py b/scrapy/addons.py index 389a3cdde..9060d4f3f 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -19,7 +19,7 @@ class AddonManager: self.crawler: "Crawler" = crawler self.addons: List[Any] = [] - def apply_settings(self, settings: Settings) -> None: + def load_settings(self, settings: Settings) -> None: """Load add-ons and configurations from a settings object and apply them. This will load the add-on for every add-on path in the diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ee4d6fd59..22fd65be7 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -79,7 +79,6 @@ class Crawler: self._init_reactor: bool = init_reactor self.crawling: bool = False - self._settings_loaded: bool = False self._started: bool = False self.extensions: Optional[ExtensionManager] = None @@ -95,11 +94,10 @@ class Crawler: install_scrapy_root_handler(self.settings) def _apply_settings(self) -> None: - if self._settings_loaded: + if self.settings.frozen: return - self._settings_loaded = True - self.addons.apply_settings(self.settings) + self.addons.load_settings(self.settings) self.stats = load_object(self.settings["STATS_CLASS"])(self) handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) From be0e33af92d1c84cdca6068e54fc736bb524342f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Sep 2023 16:59:54 +0400 Subject: [PATCH 1161/2083] Don't rely on unhandled exception logging in the expected to fail scripts. --- .../asyncio_enabled_reactor_different_loop.py | 3 ++- .../reactor_default_twisted_reactor_select.py | 3 ++- .../reactor_select_subclass_twisted_reactor_select.py | 3 ++- .../twisted_reactor_custom_settings_conflict.py | 6 ++++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index 34ef00143..6fe88b011 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -24,5 +24,6 @@ process = CrawlerProcess( "ASYNCIO_EVENT_LOOP": "uvloop.Loop", } ) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(lambda failure: failure.printTraceback()) process.start() diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index 744b4ecb5..091e9d424 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -17,5 +17,6 @@ process = CrawlerProcess( } ) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(lambda failure: failure.printTraceback()) process.start() diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index a8f707841..5615e7a68 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -26,5 +26,6 @@ process = CrawlerProcess( } ) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(lambda failure: failure.printTraceback()) process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py index 19cc08be6..7074a7389 100644 --- a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -17,6 +17,8 @@ class AsyncioReactorSpider(scrapy.Spider): process = CrawlerProcess() -process.crawl(SelectReactorSpider) -process.crawl(AsyncioReactorSpider) +d1 = process.crawl(SelectReactorSpider) +d1.addErrback(lambda failure: failure.printTraceback()) +d2 = process.crawl(AsyncioReactorSpider) +d2.addErrback(lambda failure: failure.printTraceback()) process.start() From 6428356584c05e1a5074f54ad7f6722cc8a69fc7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Sep 2023 18:20:42 +0400 Subject: [PATCH 1162/2083] Use twisted.python.log instead of failure.printTraceback(). --- .../asyncio_enabled_reactor_different_loop.py | 3 ++- .../reactor_default_twisted_reactor_select.py | 3 ++- .../reactor_select_subclass_twisted_reactor_select.py | 3 ++- .../twisted_reactor_custom_settings_conflict.py | 6 ++++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index 6fe88b011..9dc8ce46b 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -2,6 +2,7 @@ import asyncio import sys from twisted.internet import asyncioreactor +from twisted.python import log if sys.version_info >= (3, 8) and sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) @@ -25,5 +26,5 @@ process = CrawlerProcess( } ) d = process.crawl(NoRequestsSpider) -d.addErrback(lambda failure: failure.printTraceback()) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index 091e9d424..eee808c32 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -1,4 +1,5 @@ from twisted.internet import reactor # noqa: F401 +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -18,5 +19,5 @@ process = CrawlerProcess( ) d = process.crawl(NoRequestsSpider) -d.addErrback(lambda failure: failure.printTraceback()) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 5615e7a68..38ca4c4f1 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -1,5 +1,6 @@ from twisted.internet.main import installReactor from twisted.internet.selectreactor import SelectReactor +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -27,5 +28,5 @@ process = CrawlerProcess( ) d = process.crawl(NoRequestsSpider) -d.addErrback(lambda failure: failure.printTraceback()) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py index 7074a7389..d71014b34 100644 --- a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -1,3 +1,5 @@ +from twisted.python import log + import scrapy from scrapy.crawler import CrawlerProcess @@ -18,7 +20,7 @@ class AsyncioReactorSpider(scrapy.Spider): process = CrawlerProcess() d1 = process.crawl(SelectReactorSpider) -d1.addErrback(lambda failure: failure.printTraceback()) +d1.addErrback(log.err) d2 = process.crawl(AsyncioReactorSpider) -d2.addErrback(lambda failure: failure.printTraceback()) +d2.addErrback(log.err) process.start() From 8dd48a08e4e5dfab80b1d715fb88c4df946fd797 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Sep 2023 20:46:32 +0400 Subject: [PATCH 1163/2083] Move PeriodicLog docs from Debugging to General purpose. --- docs/topics/extensions.rst | 110 +++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 0286581c0..f7b2f3799 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -350,52 +350,8 @@ full list of parameters, including examples on how to instantiate .. module:: scrapy.extensions.debug :synopsis: Extensions for debugging Scrapy -Debugging extensions --------------------- - -Stack trace dump extension -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. class:: StackTraceDump - -Dumps information about the running process when a `SIGQUIT`_ or `SIGUSR2`_ -signal is received. The information dumped is the following: - -1. engine status (using ``scrapy.utils.engine.get_engine_status()``) -2. live references (see :ref:`topics-leaks-trackrefs`) -3. stack trace of all threads - -After the stack trace and engine status is dumped, the Scrapy process continues -running normally. - -This extension only works on POSIX-compliant platforms (i.e. not Windows), -because the `SIGQUIT`_ and `SIGUSR2`_ signals are not available on Windows. - -There are at least two ways to send Scrapy the `SIGQUIT`_ signal: - -1. By pressing Ctrl-\ while a Scrapy process is running (Linux only?) -2. By running this command (assuming ```` is the process id of the Scrapy - process):: - - kill -QUIT - -.. _SIGUSR2: https://en.wikipedia.org/wiki/SIGUSR1_and_SIGUSR2 -.. _SIGQUIT: https://en.wikipedia.org/wiki/SIGQUIT - -Debugger extension -~~~~~~~~~~~~~~~~~~ - -.. class:: Debugger - -Invokes a :doc:`Python debugger ` inside a running Scrapy process when a `SIGUSR2`_ -signal is received. After the debugger is exited, the Scrapy process continues -running normally. - -For more info see `Debugging in Python`_. - -This extension only works on POSIX-compliant platforms (i.e. not Windows). - -.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/ +.. module:: scrapy.extensions.periodic_log + :synopsis: Periodic stats logging Periodic log extension ~~~~~~~~~~~~~~~~~~~~~~ @@ -441,10 +397,10 @@ This extension periodically logs rich stat data as a JSON object:: This extension logs the following configurable sections: -- ``"delta"`` shows how some numeric stats have changed since the last stats +- ``"delta"`` shows how some numeric stats have changed since the last stats log message. - - The :setting:`PERIODIC_LOG_DELTA` setting determines the target stats. They + + The :setting:`PERIODIC_LOG_DELTA` setting determines the target stats. They must have ``int`` or ``float`` values. - ``"stats"`` shows the current value of some stats. @@ -453,11 +409,11 @@ This extension logs the following configurable sections: - ``"time"`` shows detailed timing data. - The :setting:`PERIODIC_LOG_TIMING_ENABLED` setting determines whether or + The :setting:`PERIODIC_LOG_TIMING_ENABLED` setting determines whether or not to show this section. -This extension logs data at the start, then on a fixed time interval -configurable through the :setting:`LOGSTATS_INTERVAL` setting, and finally +This extension logs data at the start, then on a fixed time interval +configurable through the :setting:`LOGSTATS_INTERVAL` setting, and finally right before the crawl ends. @@ -507,4 +463,52 @@ PERIODIC_LOG_TIMING_ENABLED Default: ``False`` -``True`` enables logging of timing data (i.e. the ``"time"`` section). \ No newline at end of file +``True`` enables logging of timing data (i.e. the ``"time"`` section). + + +Debugging extensions +-------------------- + +Stack trace dump extension +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. class:: StackTraceDump + +Dumps information about the running process when a `SIGQUIT`_ or `SIGUSR2`_ +signal is received. The information dumped is the following: + +1. engine status (using ``scrapy.utils.engine.get_engine_status()``) +2. live references (see :ref:`topics-leaks-trackrefs`) +3. stack trace of all threads + +After the stack trace and engine status is dumped, the Scrapy process continues +running normally. + +This extension only works on POSIX-compliant platforms (i.e. not Windows), +because the `SIGQUIT`_ and `SIGUSR2`_ signals are not available on Windows. + +There are at least two ways to send Scrapy the `SIGQUIT`_ signal: + +1. By pressing Ctrl-\ while a Scrapy process is running (Linux only?) +2. By running this command (assuming ```` is the process id of the Scrapy + process):: + + kill -QUIT + +.. _SIGUSR2: https://en.wikipedia.org/wiki/SIGUSR1_and_SIGUSR2 +.. _SIGQUIT: https://en.wikipedia.org/wiki/SIGQUIT + +Debugger extension +~~~~~~~~~~~~~~~~~~ + +.. class:: Debugger + +Invokes a :doc:`Python debugger ` inside a running Scrapy process when a `SIGUSR2`_ +signal is received. After the debugger is exited, the Scrapy process continues +running normally. + +For more info see `Debugging in Python`_. + +This extension only works on POSIX-compliant platforms (i.e. not Windows). + +.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/ From f96a3ed5f0e5dcc2bf6849219248c3da2fb85c72 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Sep 2023 20:46:55 +0400 Subject: [PATCH 1164/2083] Cover up to cddb8c15d in the release notes. --- docs/news.rst | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 9e758f05d..2237697c7 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,6 +8,13 @@ Release notes Scrapy 2.11.0 (to be released) ------------------------------ +Highlights: + +- + +- Periodic stats logging. + + Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -20,6 +27,50 @@ Backward-incompatible changes (:issue:`5968`) +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed the binary export mode of + :class:`~scrapy.exporters.PythonItemExporter`, deprecated in Scrapy 1.1.0. + (:issue:`6006`, :issue:`6007`) + +- Removed the ``CrawlerRunner.spiders`` attribute, deprecated in Scrapy + 1.0.0, use :attr:`CrawlerRunner.spider_loader + ` instead. (:issue:`6010`) + +New features +~~~~~~~~~~~~ + +- Added the :class:`~scrapy.extensions.periodic_log.PeriodicLog` extension + which can be enabled to log stats and/or their differences periodically. + (:issue:`5926`) + +- Links to ``.webp`` files are now ignored by :ref:`link extractors + `. (:issue:`6021`) + +Bug fixes +~~~~~~~~~ + +- :meth:`scrapy.settings.BaseSettings.getdictorlist`, used to parse + :setting:`FEED_EXPORT_FIELDS`, now handles tuple values. (:issue:`6011`, + :issue:`6013`) + +- Calls to ``datetime.utcnow()``, no longer recommended to be used, have been + replaced with calls to ``datetime.now()`` with a timezone. (:issue:`6014`) + +Documentation +~~~~~~~~~~~~~ + +- Updated a deprecated function call in a pipeline example. (:issue:`6008`, + :issue:`6009`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Extended typing hints. (:issue:`6003`, :issue:`6005`) + +- Other CI and pre-commit improvements. (:issue:`6002`, :issue:`6013`) + .. _release-2.10.1: Scrapy 2.10.1 (2023-08-30) From 922ff5738448205c5b7e5ced533bb8820b168480 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 14 Sep 2023 15:51:00 +0400 Subject: [PATCH 1165/2083] Improve the backwards compatibility for RetryMiddleware.EXCEPTIONS_TO_RETRY. --- scrapy/downloadermiddlewares/retry.py | 27 ++++++++++++++++-------- tests/test_downloadermiddleware_retry.py | 17 ++++++++++++++- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 205bb48b1..af590be47 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -23,12 +23,13 @@ from scrapy.utils.response import response_status_message retry_logger = getLogger(__name__) +DEPRECATED_ATTRIBUTE = "EXCEPTIONS_TO_RETRY" -class BackwardsCompatibilityMetaclass(type): - @property - def EXCEPTIONS_TO_RETRY(cls): + +def backwards_compatibility_getattr(self, name): + if name == DEPRECATED_ATTRIBUTE: warnings.warn( - "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " + f"Attribute RetryMiddleware.{DEPRECATED_ATTRIBUTE} is deprecated. " "Use the RETRY_EXCEPTIONS setting instead.", ScrapyDeprecationWarning, stacklevel=2, @@ -37,6 +38,13 @@ class BackwardsCompatibilityMetaclass(type): load_object(x) if isinstance(x, str) else x for x in Settings().getlist("RETRY_EXCEPTIONS") ) + raise AttributeError( + f"{self.__class__.__name__!r} object has no attribute {name!r}" + ) + + +class BackwardsCompatibilityMetaclass(type): + __getattr__ = backwards_compatibility_getattr def get_retry_request( @@ -137,15 +145,14 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): ) self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") - if not hasattr( - self, "EXCEPTIONS_TO_RETRY" - ): # If EXCEPTIONS_TO_RETRY is not "overriden" + try: + self.exceptions_to_retry = self.__getattribute__(DEPRECATED_ATTRIBUTE) + except AttributeError: + # If EXCEPTIONS_TO_RETRY is not "overridden" self.exceptions_to_retry = tuple( load_object(x) if isinstance(x, str) else x for x in settings.getlist("RETRY_EXCEPTIONS") ) - else: - self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY @classmethod def from_crawler(cls, crawler): @@ -175,3 +182,5 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): max_retry_times=max_retry_times, priority_adjust=priority_adjust, ) + + __getattr__ = backwards_compatibility_getattr diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 97ae1e29a..661175840 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -122,7 +122,7 @@ class RetryTest(unittest.TestCase): req = Request(f"http://www.scrapytest.org/{exc.__name__}") self._test_retry_exception(req, exc("foo"), mw) - def test_exception_to_retry_customMiddleware(self): + def test_exception_to_retry_custom_middleware(self): exc = ValueError with warnings.catch_warnings(record=True) as warns: @@ -138,6 +138,21 @@ class RetryTest(unittest.TestCase): assert isinstance(req, Request) self.assertEqual(req.meta["retry_times"], 1) + def test_exception_to_retry_custom_middleware_self(self): + class MyRetryMiddleware(RetryMiddleware): + def process_exception(self, request, exception, spider): + if isinstance(exception, self.EXCEPTIONS_TO_RETRY): + return self._retry(request, exception, spider) + + exc = OSError + mw2 = MyRetryMiddleware.from_crawler(self.crawler) + req = Request(f"http://www.scrapytest.org/{exc.__name__}") + with warnings.catch_warnings(record=True) as warns: + req = mw2.process_exception(req, exc("foo"), self.spider) + assert isinstance(req, Request) + self.assertEqual(req.meta["retry_times"], 1) + self.assertEqual(len(warns), 1) + def _test_retry_exception(self, req, exception, mw=None): if mw is None: mw = self.mw From 800c1f112e883070d97aa679b4bae10f618633a9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 15 Sep 2023 13:13:06 +0400 Subject: [PATCH 1166/2083] Remove the constant. --- scrapy/downloadermiddlewares/retry.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index af590be47..380623cea 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -23,13 +23,11 @@ from scrapy.utils.response import response_status_message retry_logger = getLogger(__name__) -DEPRECATED_ATTRIBUTE = "EXCEPTIONS_TO_RETRY" - def backwards_compatibility_getattr(self, name): - if name == DEPRECATED_ATTRIBUTE: + if name == "EXCEPTIONS_TO_RETRY": warnings.warn( - f"Attribute RetryMiddleware.{DEPRECATED_ATTRIBUTE} is deprecated. " + "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " "Use the RETRY_EXCEPTIONS setting instead.", ScrapyDeprecationWarning, stacklevel=2, @@ -146,7 +144,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") try: - self.exceptions_to_retry = self.__getattribute__(DEPRECATED_ATTRIBUTE) + self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY") except AttributeError: # If EXCEPTIONS_TO_RETRY is not "overridden" self.exceptions_to_retry = tuple( From c2346b4a95e51ec1d3a255e19b49042b4598a02d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 15 Sep 2023 19:15:05 +0400 Subject: [PATCH 1167/2083] Update the release notes up to current master. --- docs/news.rst | 72 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 2237697c7..7e26299c7 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,7 +10,9 @@ Scrapy 2.11.0 (to be released) Highlights: -- +- Spiders can now modify :ref:`settings ` in their + :meth:`~scrapy.Spider.from_crawler` methods, e.g. based on :ref:`spider + arguments `. - Periodic stats logging. @@ -18,14 +20,25 @@ Highlights: Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- Most of the initialization of :class:`scrapy.crawler.Crawler` instances is + now done in :meth:`~scrapy.crawler.Crawler.crawl`, so the state of + instances before that method is called is now different compared to older + Scrapy versions. We do not recommend using the + :class:`~scrapy.crawler.Crawler` instances before + :meth:`~scrapy.crawler.Crawler.crawl` is called. (:issue:`6038`) + +- :meth:`scrapy.Spider.from_crawler` is now called before the initialization + of various components previously initialized in + :meth:`scrapy.crawler.Crawler.__init__` and before the settings are + finalized and frozen. This change was needed to allow changing the settings + in :meth:`scrapy.Spider.from_crawler`. If you want to access the final + setting values in the spider code as early as possible you can do this in + :meth:`~scrapy.Spider.start_requests`. (:issue:`6038`) + - The :meth:`TextResponse.json ` method now requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or - UTF-32). - - If you need to deal with JSON documents in an invalid encoding, use - ``json.loads(response.text)`` instead. - - (:issue:`5968`) + UTF-32). If you need to deal with JSON documents in an invalid encoding, + use ``json.loads(response.text)`` instead. (:issue:`6016`) Deprecation removals ~~~~~~~~~~~~~~~~~~~~ @@ -38,19 +51,55 @@ Deprecation removals 1.0.0, use :attr:`CrawlerRunner.spider_loader ` instead. (:issue:`6010`) + .. note:: If you are using this Scrapy version on Scrapy Cloud with a stack + that includes an older Scrapy version and get a "TypeError: + Unexpected options: binary" error, you may need to add + ``scrapinghub-entrypoint-scrapy > 0.14.0`` to your project + requirements or switch to a stack that includes Scrapy 2.11. + +Deprecations +~~~~~~~~~~~~ + +- Running :meth:`~scrapy.crawler.Crawler.crawl` more than once on the same + :class:`scrapy.crawler.Crawler` instance is now deprecated. (:issue:`1587`, + :issue:`6040`) + New features ~~~~~~~~~~~~ +- Changed the :class:`scrapy.crawler.Crawler` initialization order, so that + most of the initialization that previously happened in + :meth:`~scrapy.crawler.Crawler.__init__` now happens in + :meth:`~scrapy.crawler.Crawler.crawl` after the spider instance is created. + This allows spider instances to modify settings in their + :meth:`~scrapy.Spider.from_crawler` methods, e.g. based on :ref:`spider + arguments `. (:issue:`1305`, :issue:`1580`, :issue:`2392`, + :issue:`3663`, :issue:`6038`) + - Added the :class:`~scrapy.extensions.periodic_log.PeriodicLog` extension which can be enabled to log stats and/or their differences periodically. (:issue:`5926`) +- Optimized the memory usage in :meth:`TextResponse.json + ` by removing unnecessary body decoding. + (:issue:`5968`, :issue:`6016`) + - Links to ``.webp`` files are now ignored by :ref:`link extractors `. (:issue:`6021`) Bug fixes ~~~~~~~~~ +- Fixed logging enabled add-ons. (:issue:`6036`) + +- Fixed :class:`~scrapy.mail.MailSender` producing invalid message bodies + when the ``charset`` argument is passed to + :meth:`~scrapy.mail.MailSender.send`. (:issue:`5096`, :issue:`5118`) + +- Fixed an exception when accessing ``self.EXCEPTIONS_TO_RETRY`` from a + subclass of :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`. + (:issue:`6049`, :issue:`6050`) + - :meth:`scrapy.settings.BaseSettings.getdictorlist`, used to parse :setting:`FEED_EXPORT_FIELDS`, now handles tuple values. (:issue:`6011`, :issue:`6013`) @@ -67,9 +116,14 @@ Documentation Quality assurance ~~~~~~~~~~~~~~~~~ -- Extended typing hints. (:issue:`6003`, :issue:`6005`) +- Extended typing hints. (:issue:`6003`, :issue:`6005`, :issue:`6031`, + :issue:`6034`) -- Other CI and pre-commit improvements. (:issue:`6002`, :issue:`6013`) +- Pinned brotli_ to 1.0.9 for the PyPy tests as 1.1.0 breaks them. + (:issue:`6044`, :issue:`6045`) + +- Other CI and pre-commit improvements. (:issue:`6002`, :issue:`6013`, + :issue:`6046`) .. _release-2.10.1: From 2fa768399a27aca615bccfc7c466758a968f10fe Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 15 Sep 2023 19:19:42 +0400 Subject: [PATCH 1168/2083] Replace the VERSION vars. --- docs/topics/settings.rst | 2 +- docs/topics/spiders.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 3006fb8b1..7cdfb8768 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -98,7 +98,7 @@ and settings set there should use the "spider" priority explicitly: super().update_settings(settings) settings.set("SOME_SETTING", "some value", priority="spider") -.. versionadded:: VERSION +.. versionadded:: 2.11 It's also possible to modify the settings in the :meth:`~scrapy.Spider.from_crawler` method, e.g. based on :ref:`spider diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 1ca7eda7b..20452d558 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -136,7 +136,7 @@ scrapy.Spider attributes in the new instance so they can be accessed later inside the spider's code. - .. versionchanged:: VERSION + .. versionchanged:: 2.11 The settings in ``crawler.settings`` can now be modified in this method, which is handy if you want to modify them based on From 528911da85f871fd0f7546d4d16bbea556793d4b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 18 Sep 2023 14:35:28 +0400 Subject: [PATCH 1169/2083] Fix/reword the release notes. --- docs/news.rst | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 7e26299c7..0566ff28e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -14,7 +14,7 @@ Highlights: :meth:`~scrapy.Spider.from_crawler` methods, e.g. based on :ref:`spider arguments `. -- Periodic stats logging. +- Periodic logging of stats. Backward-incompatible changes @@ -47,16 +47,16 @@ Deprecation removals :class:`~scrapy.exporters.PythonItemExporter`, deprecated in Scrapy 1.1.0. (:issue:`6006`, :issue:`6007`) -- Removed the ``CrawlerRunner.spiders`` attribute, deprecated in Scrapy - 1.0.0, use :attr:`CrawlerRunner.spider_loader - ` instead. (:issue:`6010`) - .. note:: If you are using this Scrapy version on Scrapy Cloud with a stack that includes an older Scrapy version and get a "TypeError: Unexpected options: binary" error, you may need to add - ``scrapinghub-entrypoint-scrapy > 0.14.0`` to your project + ``scrapinghub-entrypoint-scrapy >= 0.14.1`` to your project requirements or switch to a stack that includes Scrapy 2.11. +- Removed the ``CrawlerRunner.spiders`` attribute, deprecated in Scrapy + 1.0.0, use :attr:`CrawlerRunner.spider_loader + ` instead. (:issue:`6010`) + Deprecations ~~~~~~~~~~~~ @@ -67,12 +67,8 @@ Deprecations New features ~~~~~~~~~~~~ -- Changed the :class:`scrapy.crawler.Crawler` initialization order, so that - most of the initialization that previously happened in - :meth:`~scrapy.crawler.Crawler.__init__` now happens in - :meth:`~scrapy.crawler.Crawler.crawl` after the spider instance is created. - This allows spider instances to modify settings in their - :meth:`~scrapy.Spider.from_crawler` methods, e.g. based on :ref:`spider +- Spiders can now modify settings in their + :meth:`~scrapy.Spider.from_crawler` method, e.g. based on :ref:`spider arguments `. (:issue:`1305`, :issue:`1580`, :issue:`2392`, :issue:`3663`, :issue:`6038`) From f2fb4760d2e98cc92ab00a84bcf20cd8a6de3799 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 18 Sep 2023 15:24:28 +0400 Subject: [PATCH 1170/2083] =?UTF-8?q?Bump=20version:=202.10.1=20=E2=86=92?= =?UTF-8?q?=202.11.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 4 ++-- scrapy/VERSION | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0bcd1ab01..f76bf783d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.10.1 +current_version = 2.11.0 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index 0566ff28e..fc3cfd9e8 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,8 +5,8 @@ Release notes .. _release-2.11.0: -Scrapy 2.11.0 (to be released) ------------------------------- +Scrapy 2.11.0 (2023-09-18) +-------------------------- Highlights: diff --git a/scrapy/VERSION b/scrapy/VERSION index 8bbb6e406..46b81d815 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.10.1 +2.11.0 From d5cc469ca93cf9ee726aba1620893b308b2da5a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 21 Sep 2023 13:06:12 +0200 Subject: [PATCH 1171/2083] Add py.typed --- MANIFEST.in | 1 + scrapy/py.typed | 0 2 files changed, 1 insertion(+) create mode 100644 scrapy/py.typed diff --git a/MANIFEST.in b/MANIFEST.in index ae7db51fa..4920dc0c3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,6 +7,7 @@ include NEWS include scrapy/VERSION include scrapy/mime.types +include scrapy/py.typed include codecov.yml include conftest.py diff --git a/scrapy/py.typed b/scrapy/py.typed new file mode 100644 index 000000000..e69de29bb From e84fb6d5cb15a36022dbb9ef27f9924383d317fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 21 Sep 2023 13:39:39 +0200 Subject: [PATCH 1172/2083] Add package_data={"scrapy": "py.typed"} to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 47c0af0b0..aff51b1cd 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ setup( license="BSD", packages=find_packages(exclude=("tests", "tests.*")), include_package_data=True, + package_data={"scrapy": "py.typed"}, zip_safe=False, entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]}, classifiers=[ From 11bdc3df590d461634865027d6911cc5f8116cfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 21 Sep 2023 13:40:40 +0200 Subject: [PATCH 1173/2083] Revert "Add package_data={"scrapy": "py.typed"} to setup.py" This reverts commit e84fb6d5cb15a36022dbb9ef27f9924383d317fc. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index aff51b1cd..47c0af0b0 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,6 @@ setup( license="BSD", packages=find_packages(exclude=("tests", "tests.*")), include_package_data=True, - package_data={"scrapy": "py.typed"}, zip_safe=False, entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]}, classifiers=[ From 908da8ba829fd0d8a2738625e84051435305b67f Mon Sep 17 00:00:00 2001 From: Yash nagarkar <116726926+yash08123@users.noreply.github.com> Date: Fri, 22 Sep 2023 13:42:20 +0530 Subject: [PATCH 1174/2083] Cover the removal of is_botocore on the release notes (#6061) --- docs/news.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index fc3cfd9e8..c5b75aae2 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -469,6 +469,10 @@ Deprecation removals has now been removed. (:issue:`5719`) +- The ``scrapy.utils.boto.is_botocore()`` function, deprecated in Scrapy 2.4, + has now been removed. + (:issue:`5719`) + Deprecations ~~~~~~~~~~~~ From 720f351a3eea5e5bfa83a6eaf50210cd1fa43992 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 24 Sep 2023 18:12:52 +0400 Subject: [PATCH 1175/2083] Refactor installing signals. --- scrapy/crawler.py | 8 +++++--- scrapy/utils/ossignal.py | 9 +++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 22fd65be7..15aec2757 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -416,15 +416,17 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) - if install_signal_handlers: - install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) reactor.addSystemEventTrigger("before", "shutdown", self.stop) - reactor.run(installSignalHandlers=False) # blocking call + if install_signal_handlers: + reactor.addSystemEventTrigger( + "after", "startup", install_shutdown_handlers, self._signal_shutdown + ) + reactor.run() # blocking call def _graceful_stop_reactor(self) -> Deferred: d = self.stop() diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 2334ea792..db9a71273 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -19,13 +19,10 @@ def install_shutdown_handlers( function: SignalHandlerT, override_sigint: bool = True ) -> None: """Install the given function as a signal handler for all common shutdown - signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the - SIGINT handler won't be install if there is already a handler in place - (e.g. Pdb) + signals (such as SIGINT, SIGTERM, etc). If ``override_sigint`` is ``False`` the + SIGINT handler won't be installed if there is already a handler in place + (e.g. Pdb) """ - from twisted.internet import reactor - - reactor._handleSignals() signal.signal(signal.SIGTERM, function) if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: signal.signal(signal.SIGINT, function) From 4b14215f83996a18d34c95bec953ef9cfba05245 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 24 Sep 2023 18:17:43 +0400 Subject: [PATCH 1176/2083] Remove the Twisted version restriction. --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 47c0af0b0..405633f55 100644 --- a/setup.py +++ b/setup.py @@ -6,8 +6,7 @@ version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() install_requires = [ - # 23.8.0 incompatibility: https://github.com/scrapy/scrapy/issues/6024 - "Twisted>=18.9.0,<23.8.0", + "Twisted>=18.9.0", "cryptography>=36.0.0", "cssselect>=0.9.1", "itemloaders>=1.0.1", From 197781e3af51cd46ae7761938afa474c40c36b76 Mon Sep 17 00:00:00 2001 From: Yash nagarkar <116726926+yash08123@users.noreply.github.com> Date: Fri, 22 Sep 2023 13:42:20 +0530 Subject: [PATCH 1177/2083] Cover the removal of is_botocore on the release notes (#6061) --- docs/news.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index fc3cfd9e8..c5b75aae2 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -469,6 +469,10 @@ Deprecation removals has now been removed. (:issue:`5719`) +- The ``scrapy.utils.boto.is_botocore()`` function, deprecated in Scrapy 2.4, + has now been removed. + (:issue:`5719`) + Deprecations ~~~~~~~~~~~~ From 0c6440a427a6f3ed2920de38f97d15d692833851 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 24 Sep 2023 19:25:59 +0400 Subject: [PATCH 1178/2083] Fix additional typing errors with new Twisted. --- scrapy/utils/testproc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 5f7a7db14..0688e014b 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,7 +2,7 @@ from __future__ import annotations import os import sys -from typing import Iterable, Optional, Tuple, cast +from typing import Iterable, List, Optional, Tuple, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated @@ -26,14 +26,15 @@ class ProcessTest: env = os.environ.copy() if settings is not None: env["SCRAPY_SETTINGS_MODULE"] = settings + assert self.command cmd = self.prefix + [self.command] + list(args) pp = TestProcessProtocol() - pp.deferred.addBoth(self._process_finished, cmd, check_code) + pp.deferred.addCallback(self._process_finished, cmd, check_code) reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) return pp.deferred def _process_finished( - self, pp: TestProcessProtocol, cmd: str, check_code: bool + self, pp: TestProcessProtocol, cmd: List[str], check_code: bool ) -> Tuple[int, bytes, bytes]: if pp.exitcode and check_code: msg = f"process {cmd} exit with code {pp.exitcode}" From 0630e4aaa10c3fb8c79c2542a229f5c0632cddde Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 24 Sep 2023 19:36:29 +0400 Subject: [PATCH 1179/2083] Fix `scrapy shell`. --- scrapy/crawler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 15aec2757..6f54e62e9 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -404,8 +404,8 @@ class CrawlerProcess(CrawlerRunner): :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished - :param bool install_signal_handlers: whether to install the shutdown - handlers (default: True) + :param bool install_signal_handlers: whether to install the OS signal + handlers from Twisted and Scrapy (default: True) """ from twisted.internet import reactor @@ -426,7 +426,7 @@ class CrawlerProcess(CrawlerRunner): reactor.addSystemEventTrigger( "after", "startup", install_shutdown_handlers, self._signal_shutdown ) - reactor.run() # blocking call + reactor.run(installSignalHandlers=install_signal_handlers) # blocking call def _graceful_stop_reactor(self) -> Deferred: d = self.stop() From d19e315b0b07b070f45cf3c082d4dfe0e6a87186 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 24 Sep 2023 21:51:19 +0400 Subject: [PATCH 1180/2083] Add an interactive test for `scrapy shell`. --- tests/requirements.txt | 1 + tests/test_command_shell.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/tests/requirements.txt b/tests/requirements.txt index 3ea7f3333..dc004f3f1 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,6 @@ # Tests requirements attrs +pexpect >= 4.8.0 # https://github.com/giampaolo/pyftpdlib/issues/560 pyftpdlib; python_version < "3.12" pytest diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 6589381f3..7d87eb62c 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,11 +1,15 @@ +import sys +from io import BytesIO from pathlib import Path +from pexpect.popen_spawn import PopenSpawn from twisted.internet import defer from twisted.trial import unittest from scrapy.utils.testproc import ProcessTest from scrapy.utils.testsite import SiteTest from tests import NON_EXISTING_RESOLVABLE, tests_datadir +from tests.mockserver import MockServer class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @@ -133,3 +137,25 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] _, _, err = yield self.execute(args, check_code=True) self.assertNotIn(b"RuntimeError: There is no current event loop in thread", err) + + +class InteractiveShellTest(unittest.TestCase): + def test_fetch(self): + args = ( + sys.executable, + "-m", + "scrapy.cmdline", + "shell", + ) + logfile = BytesIO() + p = PopenSpawn(args, timeout=5) + p.logfile_read = logfile + p.expect_exact("Available Scrapy objects") + with MockServer() as mockserver: + p.sendline(f"fetch('{mockserver.url('/')}')") + p.sendline("type(response)") + p.expect_exact("HtmlResponse") + p.sendeof() + p.wait() + logfile.seek(0) + self.assertNotIn("Traceback", logfile.read().decode()) From 4b5fb9b5a6c7738aee22b5080fd96588feea39a5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 24 Sep 2023 22:52:17 +0400 Subject: [PATCH 1181/2083] Add a test for SIGTERM handling. --- tests/CrawlerProcess/sleeping.py | 24 ++++++++++++++++++++++++ tests/test_crawler.py | 31 +++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 tests/CrawlerProcess/sleeping.py diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py new file mode 100644 index 000000000..b46f7ee2d --- /dev/null +++ b/tests/CrawlerProcess/sleeping.py @@ -0,0 +1,24 @@ +from twisted.internet.defer import Deferred + +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.utils.defer import maybe_deferred_to_future + + +class SleepingSpider(scrapy.Spider): + name = "sleeping" + + start_urls = ["data:,;"] + + async def parse(self, response): + from twisted.internet import reactor + + d = Deferred() + reactor.callLater(2, d.callback, None) + await maybe_deferred_to_future(d) + + +process = CrawlerProcess(settings={}) + +process.crawl(SleepingSpider) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 2b141e894..f0a308d95 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,13 +1,16 @@ import logging import os import platform +import signal import subprocess import sys import warnings from pathlib import Path +from typing import List import pytest from packaging.version import parse as parse_version +from pexpect.popen_spawn import PopenSpawn from pytest import mark, raises from twisted.internet import defer from twisted.trial import unittest @@ -289,9 +292,12 @@ class ScriptRunnerMixin: script_dir: Path cwd = os.getcwd() - def run_script(self, script_name: str, *script_args): + def get_script_args(self, script_name: str, *script_args: str) -> List[str]: script_path = self.script_dir / script_name - args = [sys.executable, str(script_path)] + list(script_args) + return [sys.executable, str(script_path)] + list(script_args) + + def run_script(self, script_name: str, *script_args: str) -> str: + args = self.get_script_args(script_name, *script_args) p = subprocess.Popen( args, env=get_mockserver_env(), @@ -517,6 +523,27 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("The value of FOO is 42", log) + def test_shutdown_graceful(self): + args = self.get_script_args("sleeping.py") + p = PopenSpawn(args, timeout=5) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(signal.SIGTERM) + p.expect_exact("shutting down gracefully") + p.expect_exact("Spider closed (shutdown)") + p.wait() + + def test_shutdown_forced(self): + args = self.get_script_args("sleeping.py") + p = PopenSpawn(args, timeout=5) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(signal.SIGTERM) + p.expect_exact("shutting down gracefully") + p.kill(signal.SIGTERM) + p.expect_exact("forcing unclean shutdown") + p.wait() + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" From eb5e2e79ba8d2c492737a6e2ab8e0d4de2bd1a6e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 24 Sep 2023 23:31:46 +0400 Subject: [PATCH 1182/2083] Use SIGINT instead of SIGTERM to support Windows. --- tests/test_crawler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f0a308d95..b43a5826c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -528,7 +528,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") - p.kill(signal.SIGTERM) + p.kill(signal.SIGINT) p.expect_exact("shutting down gracefully") p.expect_exact("Spider closed (shutdown)") p.wait() @@ -538,9 +538,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") - p.kill(signal.SIGTERM) + p.kill(signal.SIGINT) p.expect_exact("shutting down gracefully") - p.kill(signal.SIGTERM) + p.kill(signal.SIGINT) p.expect_exact("forcing unclean shutdown") p.wait() From 4c98d6068aa467f2ce88cfaf5b6ea52ae5aaf640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 25 Sep 2023 10:18:35 +0200 Subject: [PATCH 1183/2083] Add a template for question/help issues --- .github/ISSUE_TEMPLATE/question.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/question.md diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 000000000..63cae77e7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,13 @@ +--- +name: Question / Help +about: Ask a question about Scrapy or ask for help with your Scrapy code. +--- + +Thanks for taking an interest in Scrapy! + +The Scrapy GitHub issue tracker is not meant for questions or help. Please ask +for help in the [Scrapy community resources](https://scrapy.org/community/) +instead. + +The GitHub issue tracker's purpose is to deal with bug reports and feature +requests for the project itself. From 83fff6c9511fdde1024d8cd33d99314ddf4eb2ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20Schir=C3=A9n?= Date: Thu, 28 Sep 2023 23:50:14 +0200 Subject: [PATCH 1184/2083] Add dotx to exclude for documents --- scrapy/linkextractors/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 906188184..492c455fd 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -73,6 +73,7 @@ IGNORED_EXTENSIONS = [ "pps", "doc", "docx", + "dotx", "odt", "ods", "odg", From da6e75d00a8a64fea0073816d59eef2ae3b4c672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20Schir=C3=A9n?= Date: Fri, 29 Sep 2023 09:44:04 +0200 Subject: [PATCH 1185/2083] Added more common MS Office file extensions --- scrapy/linkextractors/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 492c455fd..6b8be909e 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -67,12 +67,21 @@ IGNORED_EXTENSIONS = [ "webm", # office suites "xls", + "xlsm", "xlsx", + "xltm", + "xltx", + "potm", + "potx", "ppt", + "pptm", "pptx", "pps", "doc", + "docb", + "docm", "docx", + "dotm", "dotx", "odt", "ods", From fe5ef0a80a0008788e35c5b9d3c016e40ddf610f Mon Sep 17 00:00:00 2001 From: Aryan <1111aryantiwari@gmail.com> Date: Sun, 1 Oct 2023 11:47:51 +0530 Subject: [PATCH 1186/2083] Fixed the dont_merge_cookies example in docs --- docs/topics/request-response.rst | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 41df51589..8f204fb13 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -115,21 +115,18 @@ Request objects cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. - To create a request that does not send stored cookies and does not - store received cookies, set the ``dont_merge_cookies`` key to ``True`` - in :attr:`request.meta `. - - Example of a request that sends manually-defined cookies and ignores - cookie storage: + Example of a request that sends manually-defined cookies: .. code-block:: python Request( url="http://www.example.com", cookies={"currency": "USD", "country": "UY"}, - meta={"dont_merge_cookies": True}, ) + It is recommended not to set the ``dont_merge_cookies`` key to ``True`` + in :attr:`request.meta ` as it causes custom cookies to be ignored. + For more info see :ref:`cookies-mw`. .. caution:: Cookies set via the ``Cookie`` header are not considered by the From 884840e3a30da24067825f97d3003d4c677b854e Mon Sep 17 00:00:00 2001 From: Andrew Armbruster Date: Sun, 1 Oct 2023 12:12:24 +0200 Subject: [PATCH 1187/2083] Use concurrency groups to limit duplicate jobs Leverage concurrency groups along with cancel-in-progress to favor running the most recent job. Concurrency groups are on a per workflow, per branch/tag basis. So, pushing newer updates to a branch, e.g. as part of a PR, should cancel any in progress runs of workflows that have been retriggered. See: - https://docs.github.com/en/enterprise-cloud@latest/actions/using-workflows/workflow-syntax-for-github-actions#concurrency - https://docs.github.com/en/enterprise-cloud@latest/actions/learn-github-actions/contexts#github-context --- .github/workflows/checks.yml | 4 ++++ .github/workflows/publish.yml | 4 ++++ .github/workflows/tests-macos.yml | 4 ++++ .github/workflows/tests-ubuntu.yml | 4 ++++ .github/workflows/tests-windows.yml | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ee0cb4b1e..afa713032 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,6 +1,10 @@ name: Checks on: [push, pull_request] +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: checks: runs-on: ubuntu-latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 22b8996b6..ec4d8fb32 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,6 +4,10 @@ on: tags: - '[0-9]+.[0-9]+.[0-9]+' +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: publish: runs-on: ubuntu-latest diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 3044a1af3..47392ff88 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,6 +1,10 @@ name: macOS on: [push, pull_request] +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: tests: runs-on: macos-11 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 5ff92a571..84e2fdc36 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,6 +1,10 @@ name: Ubuntu on: [push, pull_request] +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: tests: runs-on: ubuntu-latest diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index c8d1928d7..9949dbae8 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,6 +1,10 @@ name: Windows on: [push, pull_request] +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: tests: runs-on: windows-latest From e146c3a2fc9059101f27112fc5e797f89e642fb5 Mon Sep 17 00:00:00 2001 From: Aryan <1111aryantiwari@gmail.com> Date: Mon, 2 Oct 2023 15:36:29 +0530 Subject: [PATCH 1188/2083] removed the entire example for dont_merge_cookies --- docs/topics/request-response.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8f204fb13..d02af2a10 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -115,15 +115,6 @@ Request objects cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. - Example of a request that sends manually-defined cookies: - - .. code-block:: python - - Request( - url="http://www.example.com", - cookies={"currency": "USD", "country": "UY"}, - ) - It is recommended not to set the ``dont_merge_cookies`` key to ``True`` in :attr:`request.meta ` as it causes custom cookies to be ignored. From ef1ed4fab7315dce1163930b16b0b898e1bbf5a6 Mon Sep 17 00:00:00 2001 From: kokobhara <146670393+kokobhara@users.noreply.github.com> Date: Mon, 2 Oct 2023 15:44:05 +0530 Subject: [PATCH 1189/2083] Cover PythonItemExporter backwaird-incompatible changes in 2.11 (#6081) --- docs/news.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c5b75aae2..fd8fa3ea3 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -40,6 +40,9 @@ Backward-incompatible changes UTF-32). If you need to deal with JSON documents in an invalid encoding, use ``json.loads(response.text)`` instead. (:issue:`6016`) +- :class:`~scrapy.exporters.PythonItemExporter` used the binary output by + default but it no longer does. (:issue:`6006`, :issue:`6007`) + Deprecation removals ~~~~~~~~~~~~~~~~~~~~ From 8dc72dfc4d5a651e034a956a03d8e0a5f4c8a94d Mon Sep 17 00:00:00 2001 From: kokobhara <146670393+kokobhara@users.noreply.github.com> Date: Mon, 2 Oct 2023 15:44:05 +0530 Subject: [PATCH 1190/2083] Cover PythonItemExporter backwaird-incompatible changes in 2.11 (#6081) --- docs/news.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c5b75aae2..fd8fa3ea3 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -40,6 +40,9 @@ Backward-incompatible changes UTF-32). If you need to deal with JSON documents in an invalid encoding, use ``json.loads(response.text)`` instead. (:issue:`6016`) +- :class:`~scrapy.exporters.PythonItemExporter` used the binary output by + default but it no longer does. (:issue:`6006`, :issue:`6007`) + Deprecation removals ~~~~~~~~~~~~~~~~~~~~ From 42adbb21043f75bec5f513eb3b7ed86c988f7562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 2 Oct 2023 14:43:30 +0200 Subject: [PATCH 1191/2083] Update docs/topics/request-response.rst --- docs/topics/request-response.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d02af2a10..adf3d0f4a 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -115,8 +115,9 @@ Request objects cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. - It is recommended not to set the ``dont_merge_cookies`` key to ``True`` - in :attr:`request.meta ` as it causes custom cookies to be ignored. + Note that setting the :reqmeta:`dont_merge_cookies` key to ``True`` in + :attr:`request.meta ` causes custom cookies to be + ignored. For more info see :ref:`cookies-mw`. From 1ed9ed4f923ca40e0b43a72bff9e90d38db3a3bb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 2 Oct 2023 21:33:02 +0400 Subject: [PATCH 1192/2083] Require pyftpdlib that supports Python 3.12. --- tests/requirements.txt | 3 +-- tests/test_feedexport.py | 3 --- tests/test_pipeline_files.py | 5 ----- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 3ea7f3333..7178fb5b8 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,7 +1,6 @@ # Tests requirements attrs -# https://github.com/giampaolo/pyftpdlib/issues/560 -pyftpdlib; python_version < "3.12" +pyftpdlib >= 1.5.8 pytest pytest-cov==4.0.0 pytest-xdist diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 6b82974fa..56967c0d5 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -125,9 +125,6 @@ class FileFeedStorageTest(unittest.TestCase): path.unlink() -@pytest.mark.skipif( - sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" -) class FTPFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index bf96f17b6..468751446 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,7 +1,6 @@ import dataclasses import os import random -import sys import time from datetime import datetime from io import BytesIO @@ -12,7 +11,6 @@ from unittest import mock from urllib.parse import urlparse import attr -import pytest from itemadapter import ItemAdapter from twisted.internet import defer from twisted.trial import unittest @@ -648,9 +646,6 @@ class TestGCSFilesStore(unittest.TestCase): store.bucket.get_blob.assert_called_with(expected_blob_path) -@pytest.mark.skipif( - sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" -) class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): From 60d5f391c41996bb3003f8a30c634d59f0eecf7f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 2 Oct 2023 21:35:45 +0400 Subject: [PATCH 1193/2083] Use the 3.12 release, enable it for all OSes. --- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 20 +++++++++----------- .github/workflows/tests-windows.yml | 8 ++++---- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 47392ff88..c23b437d2 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 84e2fdc36..80b597dc2 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -21,7 +21,10 @@ jobs: - python-version: "3.11" env: TOXENV: py - - python-version: "3.11" + - python-version: "3.12" + env: + TOXENV: py + - python-version: "3.12" env: TOXENV: asyncio - python-version: pypy3.9 @@ -45,22 +48,17 @@ jobs: env: TOXENV: botocore-pinned - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: extra-deps - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: botocore - - python-version: "3.12.0-rc.2" - env: - TOXENV: py - - python-version: "3.12.0-rc.2" + # keep until uvloop supports 3.12 + - python-version: "3.11" env: TOXENV: asyncio - - python-version: "3.12.0-rc.2" - env: - TOXENV: extra-deps steps: - uses: actions/checkout@v3 @@ -71,7 +69,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || contains(matrix.python-version, '3.12.0') + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 9949dbae8..955924349 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -21,13 +21,13 @@ jobs: - python-version: "3.10" env: TOXENV: py - - python-version: "3.10" - env: - TOXENV: asyncio - python-version: "3.11" env: TOXENV: py - - python-version: "3.11" + - python-version: "3.12" + env: + TOXENV: py + - python-version: "3.12" env: TOXENV: asyncio From 9ae8d97d81fd0b660603473a3d0601f9d1aa77f5 Mon Sep 17 00:00:00 2001 From: Klaus Rettinghaus Date: Tue, 3 Oct 2023 13:04:18 +0200 Subject: [PATCH 1194/2083] Update GitHub Actions checkout action to version 4 (#6084) --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 2 +- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- .github/workflows/tests-windows.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index afa713032..0ceb53848 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -26,7 +26,7 @@ jobs: TOXENV: twinecheck steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -42,5 +42,5 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ec4d8fb32..dd778fc64 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,7 @@ jobs: publish: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: 3.11 diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 47392ff88..d0d7c248a 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -14,7 +14,7 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 84e2fdc36..b6d5cd27a 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -63,7 +63,7 @@ jobs: TOXENV: extra-deps steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 9949dbae8..eaf4df41b 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -32,7 +32,7 @@ jobs: TOXENV: asyncio steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 From fb4debda04cff4e33daf62d95c71ab2919c37c07 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Oct 2023 16:47:03 +0400 Subject: [PATCH 1195/2083] Use path_to_file_uri to fix the test on Windows. --- tests/test_downloader_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 57211d97a..f12243e1d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -127,7 +127,7 @@ class FileTestCase(unittest.TestCase): return self.download_request(request, Spider("foo")).addCallback(_test) def test_non_existent(self): - request = Request(f"file://{self.mktemp()}") + request = Request(path_to_file_uri(self.mktemp())) d = self.download_request(request, Spider("foo")) return self.assertFailure(d, OSError) From a338873e3acbb6024de2c371392dc5beaa73376d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Oct 2023 16:50:52 +0400 Subject: [PATCH 1196/2083] Re-enable bpython on Python 3.12. --- tests/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 7178fb5b8..c07fda2d6 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,8 +9,7 @@ testfixtures # uvloop currently doesn't build on 3.12 uvloop; platform_system != "Windows" and python_version < "3.12" -# bpython requires greenlet which currently doesn't build on 3.12 -bpython; python_version < "3.12" # optional for shell wrapper tests +bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests # 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests From c65567988da2f6dd8ad894cf0cf57f2c074be10f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 6 Oct 2023 13:27:02 +0400 Subject: [PATCH 1197/2083] Change supported PyPy versions to 3.9 and 3.10. (#6087) * Change support PyPy versions to 3.9 and 3.10. * Update the RTD URL for coverage. * Move pypy3-pinned back to pypy3.8. --- .github/workflows/tests-ubuntu.yml | 5 ++++- docs/conf.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 7ac0305f5..a307eb337 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -30,6 +30,9 @@ jobs: - python-version: pypy3.9 env: TOXENV: pypy3 + - python-version: pypy3.10 + env: + TOXENV: pypy3 # pinned deps - python-version: 3.8.17 @@ -69,7 +72,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') + if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/docs/conf.py b/docs/conf.py index 38ca81932..9ca0f817a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -276,7 +276,7 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { "attrs": ("https://www.attrs.org/en/stable/", None), - "coverage": ("https://coverage.readthedocs.io/en/stable", None), + "coverage": ("https://coverage.readthedocs.io/en/latest", None), "cryptography": ("https://cryptography.io/en/latest/", None), "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), From 2f436c05e088b063b70f000a638409772df27138 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 8 Oct 2023 22:55:05 +0400 Subject: [PATCH 1198/2083] Use SIGBREAK in Windows tests. --- tests/test_crawler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b43a5826c..60b92377d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -524,23 +524,25 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("The value of FOO is 42", log) def test_shutdown_graceful(self): + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK args = self.get_script_args("sleeping.py") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") - p.kill(signal.SIGINT) + p.kill(sig) p.expect_exact("shutting down gracefully") p.expect_exact("Spider closed (shutdown)") p.wait() def test_shutdown_forced(self): + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK args = self.get_script_args("sleeping.py") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") - p.kill(signal.SIGINT) + p.kill(sig) p.expect_exact("shutting down gracefully") - p.kill(signal.SIGINT) + p.kill(sig) p.expect_exact("forcing unclean shutdown") p.wait() From 029a56384dcac0a52fe40f1bebaaa53eda1cc902 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 9 Oct 2023 00:16:22 +0400 Subject: [PATCH 1199/2083] Increase the timeout. --- tests/CrawlerProcess/sleeping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py index b46f7ee2d..420d9d328 100644 --- a/tests/CrawlerProcess/sleeping.py +++ b/tests/CrawlerProcess/sleeping.py @@ -14,7 +14,7 @@ class SleepingSpider(scrapy.Spider): from twisted.internet import reactor d = Deferred() - reactor.callLater(2, d.callback, None) + reactor.callLater(3, d.callback, None) await maybe_deferred_to_future(d) From 064256b059cf792ba0c1983c64ae19204df3785e Mon Sep 17 00:00:00 2001 From: Sandesh Pyakurel <82999440+Sandesh-Pyakurel@users.noreply.github.com> Date: Tue, 17 Oct 2023 22:24:39 +0545 Subject: [PATCH 1200/2083] Typo fixed in news.rst file (#6094) --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index fd8fa3ea3..5db37969c 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -324,7 +324,7 @@ Deprecations New features ~~~~~~~~~~~~ -- Settings correponding to :setting:`DOWNLOAD_DELAY`, +- Settings corresponding to :setting:`DOWNLOAD_DELAY`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) From 5807970a22b9781e291a623c9a713d3124265295 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 17 Oct 2023 21:07:13 +0400 Subject: [PATCH 1201/2083] Add typing hints to CaselessDict and Headers. (#6097) --- scrapy/core/http2/stream.py | 4 +- scrapy/http/headers.py | 93 ++++++++++++++++++++--------- scrapy/http/request/__init__.py | 16 ++++- scrapy/http/response/__init__.py | 4 +- scrapy/http/response/text.py | 8 +-- scrapy/utils/datatypes.py | 56 +++++++++++------ tests/test_http2_client_protocol.py | 16 +++-- 7 files changed, 137 insertions(+), 60 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 6c6ed6f9b..39d5921f4 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -228,8 +228,8 @@ class Stream: content_length_name = self._request.headers.normkey(b"Content-Length") for name, values in self._request.headers.items(): - for value in values: - value = str(value, "utf-8") + for value_bytes in values: + value = str(value_bytes, "utf-8") if name == content_length_name: if value != content_length: logger.warning( diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 822597c84..21eb9fb73 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,41 +1,73 @@ +from __future__ import annotations + from collections.abc import Mapping +from typing import ( + TYPE_CHECKING, + Any, + AnyStr, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) from w3lib.http import headers_dict_to_raw from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + +_RawValueT = Union[bytes, str, int] + + +# isn't fully compatible typing-wise with either dict or CaselessDict, +# but it needs refactoring anyway, see also https://github.com/scrapy/scrapy/pull/5146 class Headers(CaselessDict): """Case insensitive http headers dictionary""" - def __init__(self, seq=None, encoding="utf-8"): - self.encoding = encoding + def __init__( + self, + seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + encoding: str = "utf-8", + ): + self.encoding: str = encoding super().__init__(seq) - def update(self, seq): + def update( # type: ignore[override] + self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]] + ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq - iseq = {} + iseq: Dict[bytes, List[bytes]] = {} for k, v in seq: iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) super().update(iseq) - def normkey(self, key): + def normkey(self, key: AnyStr) -> bytes: # type: ignore[override] """Normalize key to bytes""" return self._tobytes(key.title()) - def normvalue(self, value): + def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> List[bytes]: """Normalize values to bytes""" + _value: Iterable[_RawValueT] if value is None: - value = [] + _value = [] elif isinstance(value, (str, bytes)): - value = [value] - elif not hasattr(value, "__iter__"): - value = [value] + _value = [value] + elif hasattr(value, "__iter__"): + _value = value + else: + _value = [value] - return [self._tobytes(x) for x in value] + return [self._tobytes(x) for x in _value] - def _tobytes(self, x): + def _tobytes(self, x: _RawValueT) -> bytes: if isinstance(x, bytes): return x if isinstance(x, str): @@ -44,49 +76,52 @@ class Headers(CaselessDict): return str(x).encode(self.encoding) raise TypeError(f"Unsupported value type: {type(x)}") - def __getitem__(self, key): + def __getitem__(self, key: AnyStr) -> Optional[bytes]: try: - return super().__getitem__(key)[-1] + return cast(List[bytes], super().__getitem__(key))[-1] except IndexError: return None - def get(self, key, def_val=None): + def get(self, key: AnyStr, def_val: Any = None) -> Optional[bytes]: try: - return super().get(key, def_val)[-1] + return cast(List[bytes], super().get(key, def_val))[-1] except IndexError: return None - def getlist(self, key, def_val=None): + def getlist(self, key: AnyStr, def_val: Any = None) -> List[bytes]: try: - return super().__getitem__(key) + return cast(List[bytes], super().__getitem__(key)) except KeyError: if def_val is not None: return self.normvalue(def_val) return [] - def setlist(self, key, list_): + def setlist(self, key: AnyStr, list_: Iterable[_RawValueT]) -> None: self[key] = list_ - def setlistdefault(self, key, default_list=()): + def setlistdefault( + self, key: AnyStr, default_list: Iterable[_RawValueT] = () + ) -> Any: return self.setdefault(key, default_list) - def appendlist(self, key, value): + def appendlist(self, key: AnyStr, value: Iterable[_RawValueT]) -> None: lst = self.getlist(key) lst.extend(self.normvalue(value)) self[key] = lst - def items(self): + def items(self) -> Iterable[Tuple[bytes, List[bytes]]]: # type: ignore[override] return ((k, self.getlist(k)) for k in self.keys()) - def values(self): + def values(self) -> List[Optional[bytes]]: # type: ignore[override] return [self[k] for k in self.keys()] - def to_string(self): - return headers_dict_to_raw(self) + def to_string(self) -> bytes: + # cast() can be removed if the headers_dict_to_raw() hint is improved + return cast(bytes, headers_dict_to_raw(self)) - def to_unicode_dict(self): - """Return headers as a CaselessDict with unicode keys - and unicode values. Multiple values are joined with ','. + def to_unicode_dict(self) -> CaseInsensitiveDict: + """Return headers as a CaseInsensitiveDict with str keys + and str values. Multiple values are joined with ','. """ return CaseInsensitiveDict( ( @@ -96,7 +131,7 @@ class Headers(CaselessDict): for key, value in self.items() ) - def __copy__(self): + def __copy__(self) -> Self: return self.__class__(self) copy = __copy__ diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 9ba6ddf20..0b443c7d0 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -5,7 +5,19 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ import inspect -from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union +from typing import ( + Any, + AnyStr, + Callable, + Iterable, + List, + Mapping, + Optional, + Tuple, + Type, + TypeVar, + Union, +) from w3lib.url import safe_url_string @@ -77,7 +89,7 @@ class Request(object_ref): url: str, callback: Optional[Callable] = None, method: str = "GET", - headers: Optional[dict] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[Union[dict, List[dict]]] = None, meta: Optional[dict] = None, diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index a82ed834a..82274fc3a 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,7 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ -from typing import Generator, Tuple +from typing import Any, AnyStr, Generator, Iterable, Mapping, Tuple, Union from urllib.parse import urljoin from scrapy.exceptions import NotSupported @@ -42,7 +42,7 @@ class Response(object_ref): self, url: str, status=200, - headers=None, + headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body=b"", flags=None, request=None, diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 47d7bc10f..98ae1f307 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,7 +8,7 @@ from __future__ import annotations import json from contextlib import suppress -from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple +from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple, cast from urllib.parse import urljoin import parsel @@ -102,14 +102,14 @@ class TextResponse(Response): return urljoin(get_base_url(self), url) @memoizemethod_noargs - def _headers_encoding(self): - content_type = self.headers.get(b"Content-Type", b"") + def _headers_encoding(self) -> Optional[str]: + content_type = cast(bytes, self.headers.get(b"Content-Type", b"")) return http_content_type_encoding(to_unicode(content_type, encoding="latin-1")) def _body_inferred_encoding(self): if self._cached_benc is None: content_type = to_unicode( - self.headers.get(b"Content-Type", b""), encoding="latin-1" + cast(bytes, self.headers.get(b"Content-Type", b"")), encoding="latin-1" ) benc, ubody = html_to_unicode( content_type, diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index d5b9544cc..0ba2fe4e2 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -5,14 +5,32 @@ Python Standard Library. This module must not depend on any module outside the Standard Library. """ +from __future__ import annotations + import collections import warnings import weakref from collections.abc import Mapping -from typing import Any, AnyStr, Optional, OrderedDict, Sequence, TypeVar +from typing import ( + TYPE_CHECKING, + Any, + AnyStr, + Iterable, + Optional, + OrderedDict, + Sequence, + Tuple, + TypeVar, + Union, +) from scrapy.exceptions import ScrapyDeprecationWarning +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + _KT = TypeVar("_KT") _VT = TypeVar("_VT") @@ -20,7 +38,7 @@ _VT = TypeVar("_VT") class CaselessDict(dict): __slots__ = () - def __new__(cls, *args, **kwargs): + def __new__(cls, *args: Any, **kwargs: Any) -> Self: from scrapy.http.headers import Headers if issubclass(cls, CaselessDict) and not issubclass(cls, Headers): @@ -32,54 +50,58 @@ class CaselessDict(dict): ) return super().__new__(cls, *args, **kwargs) - def __init__(self, seq=None): + def __init__( + self, + seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + ): super().__init__() if seq: self.update(seq) - def __getitem__(self, key): + def __getitem__(self, key: AnyStr) -> Any: return dict.__getitem__(self, self.normkey(key)) - def __setitem__(self, key, value): + def __setitem__(self, key: AnyStr, value: Any) -> None: dict.__setitem__(self, self.normkey(key), self.normvalue(value)) - def __delitem__(self, key): + def __delitem__(self, key: AnyStr) -> None: dict.__delitem__(self, self.normkey(key)) - def __contains__(self, key): + def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] return dict.__contains__(self, self.normkey(key)) has_key = __contains__ - def __copy__(self): + def __copy__(self) -> Self: return self.__class__(self) copy = __copy__ - def normkey(self, key): + def normkey(self, key: AnyStr) -> AnyStr: """Method to normalize dictionary key access""" return key.lower() - def normvalue(self, value): + def normvalue(self, value: Any) -> Any: """Method to normalize values prior to be set""" return value - def get(self, key, def_val=None): + def get(self, key: AnyStr, def_val: Any = None) -> Any: return dict.get(self, self.normkey(key), self.normvalue(def_val)) - def setdefault(self, key, def_val=None): - return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) + def setdefault(self, key: AnyStr, def_val: Any = None) -> Any: + return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # type: ignore[arg-type] - def update(self, seq): + # doesn't fully implement MutableMapping.update() + def update(self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]]) -> None: # type: ignore[override] seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super().update(iseq) @classmethod - def fromkeys(cls, keys, value=None): - return cls((k, value) for k in keys) + def fromkeys(cls, keys: Iterable[AnyStr], value: Any = None) -> Self: # type: ignore[override] + return cls((k, value) for k in keys) # type: ignore[misc] - def pop(self, key, *args): + def pop(self, key: AnyStr, *args: Any) -> Any: return dict.pop(self, self.normkey(key), *args) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 4709007a8..deb35a579 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -275,7 +275,9 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.body, expected_body) self.assertEqual(response.request, request) - content_length = int(response.headers.get("Content-Length")) + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) self.assertEqual(len(response.body), content_length) d = self.make_request(request) @@ -320,11 +322,15 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, expected_status) self.assertEqual(response.request, request) - content_length = int(response.headers.get("Content-Length")) + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) self.assertEqual(len(response.body), content_length) # Parse the body - content_encoding = str(response.headers[b"Content-Encoding"], "utf-8") + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") body = json.loads(str(response.body, content_encoding)) self.assertIn("request-body", body) self.assertIn("extra-data", body) @@ -562,7 +568,9 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(self.get_url(f"/query-params?{urlencode(params)}")) def assert_query_params(response: Response): - content_encoding = str(response.headers[b"Content-Encoding"], "utf-8") + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") data = json.loads(str(response.body, content_encoding)) self.assertEqual(data, params) From 991121fa91aee4d428ae09e75427d4e91970a41b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 17 Oct 2023 21:24:44 +0400 Subject: [PATCH 1202/2083] Re-enable uvloop tests on 3.12 (#6098) --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 2 +- .github/workflows/tests-ubuntu.yml | 5 ----- scrapy/contracts/__init__.py | 6 ++++-- tests/requirements.txt | 3 +-- tox.ini | 2 +- 6 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 0ceb53848..d6fc0f6c5 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: pylint - python-version: 3.8 @@ -21,7 +21,7 @@ jobs: - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: twinecheck diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index dd778fc64..affaa32a5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - run: | pip install --upgrade build twine python -m build diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index a307eb337..f50a4d104 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -58,11 +58,6 @@ jobs: env: TOXENV: botocore - # keep until uvloop supports 3.12 - - python-version: "3.11" - env: - TOXENV: asyncio - steps: - uses: actions/checkout@v4 diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 1ec2a0234..2d9ddd89a 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -41,7 +41,9 @@ class Contract: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") - return list(iterate_spider_output(cb_result)) + return list( # pylint: disable=return-in-finally + iterate_spider_output(cb_result) + ) request.callback = wrapper @@ -68,7 +70,7 @@ class Contract: else: results.addSuccess(self.testcase_post) finally: - return output + return output # pylint: disable=return-in-finally request.callback = wrapper diff --git a/tests/requirements.txt b/tests/requirements.txt index d4bfead40..5b75674f5 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -7,8 +7,7 @@ pytest-cov==4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures -# uvloop currently doesn't build on 3.12 -uvloop; platform_system != "Windows" and python_version < "3.12" +uvloop; platform_system != "Windows" bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests diff --git a/tox.ini b/tox.ini index 9c2522a43..381da9773 100644 --- a/tox.ini +++ b/tox.ini @@ -57,7 +57,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.17.5 + pylint==3.0.1 commands = pylint conftest.py docs extras scrapy setup.py tests From 2cb1e10c764175a9d7deee387feafaee0dfaa70b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 17 Oct 2023 23:56:54 +0400 Subject: [PATCH 1203/2083] Make shutdown tests more robust. --- tests/CrawlerProcess/sleeping.py | 2 +- tests/test_crawler.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py index 420d9d328..45479ea4f 100644 --- a/tests/CrawlerProcess/sleeping.py +++ b/tests/CrawlerProcess/sleeping.py @@ -14,7 +14,7 @@ class SleepingSpider(scrapy.Spider): from twisted.internet import reactor d = Deferred() - reactor.callLater(3, d.callback, None) + reactor.callLater(int(self.sleep), d.callback, None) await maybe_deferred_to_future(d) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 60b92377d..5c11ca6e0 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -525,7 +525,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_shutdown_graceful(self): sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py") + args = self.get_script_args("sleeping.py", "-a", "sleep=3") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") @@ -536,7 +536,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_shutdown_forced(self): sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py") + args = self.get_script_args("sleeping.py", "-a", "sleep=10") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") From 7a5cefbcfa1c7b7b0207263bf0d3d6b5e7dcc49a Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 17 Oct 2023 17:49:22 -0300 Subject: [PATCH 1204/2083] Remove deprecated scrapy.downloadermiddlewares.decompression --- scrapy/downloadermiddlewares/decompression.py | 94 ------------------- ...test_downloadermiddleware_decompression.py | 53 ----------- 2 files changed, 147 deletions(-) delete mode 100644 scrapy/downloadermiddlewares/decompression.py delete mode 100644 tests/test_downloadermiddleware_decompression.py diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py deleted file mode 100644 index 3b8702419..000000000 --- a/scrapy/downloadermiddlewares/decompression.py +++ /dev/null @@ -1,94 +0,0 @@ -""" This module implements the DecompressionMiddleware which tries to recognise -and extract the potentially compressed responses that may arrive. -""" - -import bz2 -import gzip -import logging -import tarfile -import zipfile -from io import BytesIO -from tempfile import mktemp -from warnings import warn - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.responsetypes import responsetypes - -warn( - "scrapy.downloadermiddlewares.decompression is deprecated", - ScrapyDeprecationWarning, - stacklevel=2, -) - - -logger = logging.getLogger(__name__) - - -class DecompressionMiddleware: - """This middleware tries to recognise and extract the possibly compressed - responses that may arrive.""" - - def __init__(self): - self._formats = { - "tar": self._is_tar, - "zip": self._is_zip, - "gz": self._is_gzip, - "bz2": self._is_bzip2, - } - - def _is_tar(self, response): - archive = BytesIO(response.body) - try: - tar_file = tarfile.open(name=mktemp(), fileobj=archive) - except tarfile.ReadError: - return - - body = tar_file.extractfile(tar_file.members[0]).read() - respcls = responsetypes.from_args(filename=tar_file.members[0].name, body=body) - return response.replace(body=body, cls=respcls) - - def _is_zip(self, response): - archive = BytesIO(response.body) - try: - zip_file = zipfile.ZipFile(archive) - except zipfile.BadZipFile: - return - - namelist = zip_file.namelist() - body = zip_file.read(namelist[0]) - respcls = responsetypes.from_args(filename=namelist[0], body=body) - return response.replace(body=body, cls=respcls) - - def _is_gzip(self, response): - archive = BytesIO(response.body) - try: - body = gzip.GzipFile(fileobj=archive).read() - except OSError: - return - - respcls = responsetypes.from_args(body=body) - return response.replace(body=body, cls=respcls) - - def _is_bzip2(self, response): - try: - body = bz2.decompress(response.body) - except OSError: - return - - respcls = responsetypes.from_args(body=body) - return response.replace(body=body, cls=respcls) - - def process_response(self, request, response, spider): - if not response.body: - return response - - for fmt, func in self._formats.items(): - new_response = func(response) - if new_response: - logger.debug( - "Decompressed response with format: %(responsefmt)s", - {"responsefmt": fmt}, - extra={"spider": spider}, - ) - return new_response - return response diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py deleted file mode 100644 index 95739414e..000000000 --- a/tests/test_downloadermiddleware_decompression.py +++ /dev/null @@ -1,53 +0,0 @@ -from unittest import TestCase, main - -from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware -from scrapy.http import Response, XmlResponse -from scrapy.spiders import Spider -from scrapy.utils.test import assert_samelines -from tests import get_testdata - - -def _test_data(formats): - uncompressed_body = get_testdata("compressed", "feed-sample1.xml") - test_responses = {} - for format in formats: - body = get_testdata("compressed", "feed-sample1." + format) - test_responses[format] = Response("http://foo.com/bar", body=body) - return uncompressed_body, test_responses - - -class DecompressionMiddlewareTest(TestCase): - test_formats = ["tar", "xml.bz2", "xml.gz", "zip"] - uncompressed_body, test_responses = _test_data(test_formats) - - def setUp(self): - self.mw = DecompressionMiddleware() - self.spider = Spider("foo") - - def test_known_compression_formats(self): - for fmt in self.test_formats: - rsp = self.test_responses[fmt] - new = self.mw.process_response(None, rsp, self.spider) - error_msg = f"Failed {fmt}, response type {type(new).__name__}" - assert isinstance(new, XmlResponse), error_msg - assert_samelines(self, new.body, self.uncompressed_body, fmt) - - def test_plain_response(self): - rsp = Response(url="http://test.com", body=self.uncompressed_body) - new = self.mw.process_response(None, rsp, self.spider) - assert new is rsp - assert_samelines(self, new.body, rsp.body) - - def test_empty_response(self): - rsp = Response(url="http://test.com", body=b"") - new = self.mw.process_response(None, rsp, self.spider) - assert new is rsp - assert not rsp.body - assert not new.body - - def tearDown(self): - del self.mw - - -if __name__ == "__main__": - main() From bdb4abcc7a0c5b1d452557bc74087c7dbb3ff06f Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 17 Oct 2023 17:52:18 -0300 Subject: [PATCH 1205/2083] Remove the deprecation warning from CrawlerRunner._get_spider_loader() --- scrapy/crawler.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 6f54e62e9..1d3a11208 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -12,7 +12,6 @@ from twisted.internet.defer import ( inlineCallbacks, maybeDeferred, ) -from zope.interface.exceptions import DoesNotImplement try: # zope >= 5.0 only supports MultipleInvalid @@ -205,19 +204,7 @@ class CrawlerRunner: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) - excs = ( - (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement - ) - try: - verifyClass(ISpiderLoader, loader_cls) - except excs: - warnings.warn( - "SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does " - "not fully implement scrapy.interfaces.ISpiderLoader interface. " - "Please add all missing methods to avoid unexpected runtime errors.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) + verifyClass(ISpiderLoader, loader_cls) return loader_cls.from_settings(settings.frozencopy()) def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): From 83f500a352c038fe6afb7b04f0de00e20fe1b887 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 17 Oct 2023 18:14:08 -0300 Subject: [PATCH 1206/2083] Remove scrapy.pipelines.media.MediaPipeline._make_compatible --- scrapy/pipelines/media.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 401029439..8cc4df855 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -44,9 +44,6 @@ class MediaPipeline: self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False) self._handle_statuses(self.allow_redirects) - # Check if deprecated methods are being used and make them compatible - self._make_compatible() - def _handle_statuses(self, allow_redirects): self.handle_httpstatus_list = None if allow_redirects: @@ -126,23 +123,6 @@ class MediaPipeline: ) return dfd.addBoth(lambda _: wad) # it must return wad at last - def _make_compatible(self): - """Make overridable methods of MediaPipeline and subclasses backwards compatible""" - methods = [ - "file_path", - "thumb_path", - "media_to_download", - "media_downloaded", - "file_downloaded", - "image_downloaded", - "get_images", - ] - - for method_name in methods: - method = getattr(self, method_name, None) - if callable(method): - setattr(self, method_name, self._compatible(method)) - def _compatible(self, func): """Wrapper for overridable methods to allow backwards compatibility""" self._check_signature(func) From 85c57778b5ac701f30721c4aa968efc8474c6e1e Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 17 Oct 2023 18:15:20 -0300 Subject: [PATCH 1207/2083] Remove deprecated JSONRequest --- scrapy/http/request/__init__.py | 2 +- scrapy/http/request/json_request.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 0b443c7d0..06a52f5fd 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -200,7 +200,7 @@ class Request(object_ref): ``ignore_unknown_options=False``. .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request` - subclasses, such as :class:`~scrapy.http.JSONRequest`, or + subclasses, such as :class:`~scrapy.http.JsonRequest`, or :class:`~scrapy.http.XmlRpcRequest`, as well as having :ref:`downloader middlewares ` and diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 510c903db..f52c0401d 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -11,7 +11,6 @@ import warnings from typing import Optional, Tuple from scrapy.http.request import Request -from scrapy.utils.deprecate import create_deprecated_class class JsonRequest(Request): @@ -58,6 +57,3 @@ class JsonRequest(Request): def _dumps(self, data: dict) -> str: """Convert to JSON""" return json.dumps(data, **self._dumps_kwargs) - - -JSONRequest = create_deprecated_class("JSONRequest", JsonRequest) From aa0a4258264716ad5f99d9fca7c64bde8cf3576b Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 17 Oct 2023 18:20:12 -0300 Subject: [PATCH 1208/2083] Remove deprecated code moved to itemloaders --- scrapy/loader/common.py | 21 ------------------ scrapy/loader/processors.py | 20 ----------------- scrapy/utils/misc.py | 38 --------------------------------- tests/test_loader_deprecated.py | 23 -------------------- 4 files changed, 102 deletions(-) delete mode 100644 scrapy/loader/common.py delete mode 100644 scrapy/loader/processors.py diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py deleted file mode 100644 index 3e8644e0c..000000000 --- a/scrapy/loader/common.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Common functions used in Item Loaders code""" - -import warnings - -from itemloaders import common - -from scrapy.utils.deprecate import ScrapyDeprecationWarning - - -def wrap_loader_context(function, context): - """Wrap functions that receive loader_context to contain the context - "pre-loaded" and expose a interface that receives only one argument - """ - warnings.warn( - "scrapy.loader.common.wrap_loader_context has moved to a new library." - "Please update your reference to itemloaders.common.wrap_loader_context", - ScrapyDeprecationWarning, - stacklevel=2, - ) - - return common.wrap_loader_context(function, context) diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py deleted file mode 100644 index b82c6d5c7..000000000 --- a/scrapy/loader/processors.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -This module provides some commonly used processors for Item Loaders. - -See documentation in docs/topics/loaders.rst -""" -from itemloaders import processors - -from scrapy.utils.deprecate import create_deprecated_class - -MapCompose = create_deprecated_class("MapCompose", processors.MapCompose) - -Compose = create_deprecated_class("Compose", processors.Compose) - -TakeFirst = create_deprecated_class("TakeFirst", processors.TakeFirst) - -Identity = create_deprecated_class("Identity", processors.Identity) - -SelectJmes = create_deprecated_class("SelectJmes", processors.SelectJmes) - -Join = create_deprecated_class("Join", processors.Join) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b3c28da92..a9364bea2 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -21,17 +21,12 @@ from typing import ( Iterable, List, Optional, - Pattern, Union, cast, ) -from w3lib.html import replace_entities - from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache -from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.python import flatten, to_unicode if TYPE_CHECKING: from scrapy import Spider @@ -108,39 +103,6 @@ def walk_modules(path: str) -> List[ModuleType]: return mods -def extract_regex( - regex: Union[str, Pattern], text: str, encoding: str = "utf-8" -) -> List[str]: - """Extract a list of unicode strings from the given text/encoding using the following policies: - - * if the regex contains a named group called "extract" that will be returned - * if the regex contains multiple numbered groups, all those will be returned (flattened) - * if the regex doesn't contain any group the entire regex matching is returned - """ - warnings.warn( - "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - - if isinstance(regex, str): - regex = re.compile(regex, re.UNICODE) - - try: - # named group - strings = [regex.search(text).group("extract")] # type: ignore[union-attr] - except Exception: - # full regex or numbered groups - strings = regex.findall(text) - strings = flatten(strings) - - if isinstance(text, str): - return [replace_entities(s, keep=["lt", "amp"]) for s in strings] - return [ - replace_entities(to_unicode(s, encoding), keep=["lt", "amp"]) for s in strings - ] - - def md5sum(file: IO) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 3eae87599..d7f773d5c 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -4,7 +4,6 @@ Once we remove the references from scrapy, we can remove these tests. """ import unittest -import warnings from functools import partial from itemloaders.processors import ( @@ -18,9 +17,6 @@ from itemloaders.processors import ( from scrapy.item import Field, Item from scrapy.loader import ItemLoader -from scrapy.loader.common import wrap_loader_context -from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.misc import extract_regex # test items @@ -722,24 +718,5 @@ class FunctionProcessorTestCase(unittest.TestCase): self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) -class DeprecatedUtilityFunctionsTestCase(unittest.TestCase): - def test_deprecated_wrap_loader_context(self): - def function(*args): - return None - - with warnings.catch_warnings(record=True) as w: - wrap_loader_context(function, context={}) - - assert len(w) == 1 - assert issubclass(w[0].category, ScrapyDeprecationWarning) - - def test_deprecated_extract_regex(self): - with warnings.catch_warnings(record=True) as w: - extract_regex(r"\w+", "this is a test") - - assert len(w) == 1 - assert issubclass(w[0].category, ScrapyDeprecationWarning) - - if __name__ == "__main__": unittest.main() From ad4e8b64d4343e01e944c9698afaa29802d557a9 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 17 Oct 2023 18:58:27 -0300 Subject: [PATCH 1209/2083] fix test_spider_manager_verify_interface test --- tests/test_crawler.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 60b92377d..a71b9c2b7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -15,6 +15,7 @@ from pytest import mark, raises from twisted.internet import defer from twisted.trial import unittest from w3lib import __version__ as w3lib_version +from zope.interface.exceptions import MultipleInvalid import scrapy from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner @@ -179,11 +180,7 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, } ) - with warnings.catch_warnings(record=True) as w: - self.assertRaises(AttributeError, CrawlerRunner, settings) - self.assertEqual(len(w), 1) - self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message)) - self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message)) + self.assertRaises(MultipleInvalid, CrawlerRunner, settings) def test_crawler_runner_accepts_dict(self): runner = CrawlerRunner({"foo": "bar"}) From 2de95f1fc01fa8835f85c2554faba6bb536e71f8 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 17 Oct 2023 19:44:12 -0300 Subject: [PATCH 1210/2083] Remove leftover from _make_compatible removal --- scrapy/pipelines/media.py | 14 ------ tests/test_pipeline_media.py | 97 ------------------------------------ 2 files changed, 111 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 8cc4df855..34ab18404 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -123,20 +123,6 @@ class MediaPipeline: ) return dfd.addBoth(lambda _: wad) # it must return wad at last - def _compatible(self, func): - """Wrapper for overridable methods to allow backwards compatibility""" - self._check_signature(func) - - @functools.wraps(func) - def wrapper(*args, **kwargs): - if self._expects_item[func.__name__]: - return func(*args, **kwargs) - - kwargs.pop("item", None) - return func(*args, **kwargs) - - return wrapper - def _check_signature(self, func): sig = signature(func) self._expects_item[func.__name__] = True diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d655eb128..820484565 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -15,7 +15,6 @@ from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.spiders import Spider -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler @@ -427,102 +426,6 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): return super().image_downloaded(response, request, info) -class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): - skip = skip_pillow - - def setUp(self): - settings_dict = { - "IMAGES_STORE": "store-uri", - "IMAGES_THUMBS": {"small": (50, 50)}, - } - crawler = get_crawler(spidercls=None, settings_dict=settings_dict) - self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler) - self.pipe.download_func = _mocked_download_func - self.pipe.open_spider(None) - self.item = dict(image_urls=["http://picsum.photos/id/1014/200/300"], images=[]) - - def _assert_method_called_with_warnings(self, method, message, warnings): - self.assertIn(method, self.pipe._mockcalled) - warningShown = False - for warning in warnings: - if ( - warning["message"] == message - and warning["category"] == ScrapyDeprecationWarning - ): - warningShown = True - self.assertTrue(warningShown) - - @inlineCallbacks - def test_media_to_download_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "media_to_download(self, request, info) is deprecated, " - "please use media_to_download(self, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("media_to_download", message, warnings) - - @inlineCallbacks - def test_media_downloaded_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "media_downloaded(self, response, request, info) is deprecated, " - "please use media_downloaded(self, response, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("media_downloaded", message, warnings) - - @inlineCallbacks - def test_file_downloaded_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "file_downloaded(self, response, request, info) is deprecated, " - "please use file_downloaded(self, response, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("file_downloaded", message, warnings) - - @inlineCallbacks - def test_file_path_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "file_path(self, request, response=None, info=None) is deprecated, " - "please use file_path(self, request, response=None, info=None, *, item=None)" - ) - self._assert_method_called_with_warnings("file_path", message, warnings) - - @inlineCallbacks - def test_thumb_path_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, " - "please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)" - ) - self._assert_method_called_with_warnings("thumb_path", message, warnings) - - @inlineCallbacks - def test_get_images_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "get_images(self, response, request, info) is deprecated, " - "please use get_images(self, response, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("get_images", message, warnings) - - @inlineCallbacks - def test_image_downloaded_called(self): - yield self.pipe.process_item(self.item, None) - warnings = self.flushWarnings([MediaPipeline._compatible]) - message = ( - "image_downloaded(self, response, request, info) is deprecated, " - "please use image_downloaded(self, response, request, info, *, item=None)" - ) - self._assert_method_called_with_warnings("image_downloaded", message, warnings) - - class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(settings=Settings(settings)) From 4ad727f0b566f2e50587e7bd2c9c691f10700daf Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 17 Oct 2023 19:47:01 -0300 Subject: [PATCH 1211/2083] Remove _check_signature --- scrapy/pipelines/media.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 34ab18404..75532034a 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,8 +1,6 @@ import functools import logging from collections import defaultdict -from inspect import signature -from warnings import warn from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure @@ -11,7 +9,6 @@ from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import defer_result, mustbe_deferred -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter @@ -123,21 +120,6 @@ class MediaPipeline: ) return dfd.addBoth(lambda _: wad) # it must return wad at last - def _check_signature(self, func): - sig = signature(func) - self._expects_item[func.__name__] = True - - if "item" not in sig.parameters: - old_params = str(sig)[1:-1] - new_params = old_params + ", *, item=None" - warn( - f"{func.__name__}(self, {old_params}) is deprecated, " - f"please use {func.__name__}(self, {new_params})", - ScrapyDeprecationWarning, - stacklevel=2, - ) - self._expects_item[func.__name__] = False - def _modify_media_request(self, request): if self.handle_httpstatus_list: request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list From a2b9351f04f0b9f81edbf3c3ebdf2ad1d5ba847e Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 17 Oct 2023 20:20:38 -0300 Subject: [PATCH 1212/2083] Remove support for using HttpAuthMiddleware without http_auth_domain --- scrapy/downloadermiddlewares/httpauth.py | 20 +------------ tests/test_downloadermiddleware_httpauth.py | 33 +-------------------- 2 files changed, 2 insertions(+), 51 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index de5a81388..5228db786 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -3,13 +3,10 @@ HTTP basic auth downloader middleware See documentation in docs/topics/downloader-middleware.rst """ -import warnings from w3lib.http import basic_auth_header from scrapy import signals -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.url import url_is_from_any_domain @@ -28,25 +25,10 @@ class HttpAuthMiddleware: pwd = getattr(spider, "http_pass", "") if usr or pwd: self.auth = basic_auth_header(usr, pwd) - if not hasattr(spider, "http_auth_domain"): - warnings.warn( - "Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security " - "problems if the spider makes requests to several different domains. http_auth_domain " - "will be set to the domain of the first request, please set it to the correct value " - "explicitly.", - category=ScrapyDeprecationWarning, - ) - self.domain_unset = True - else: - self.domain = spider.http_auth_domain - self.domain_unset = False + self.domain = spider.http_auth_domain def process_request(self, request, spider): auth = getattr(self, "auth", None) if auth and b"Authorization" not in request.headers: - domain = urlparse_cached(request).hostname - if self.domain_unset: - self.domain = domain - self.domain_unset = False if not self.domain or url_is_from_any_domain(request.url, [self.domain]): request.headers[b"Authorization"] = auth diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 6b79234d0..fc110e6cc 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,10 +1,8 @@ import unittest -import pytest from w3lib.http import basic_auth_header from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider @@ -31,39 +29,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): self.spider = TestSpiderLegacy("foo") def test_auth(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Using HttpAuthMiddleware without http_auth_domain is deprecated", - ): + with self.assertRaises(AttributeError): mw = HttpAuthMiddleware() mw.spider_opened(self.spider) - # initial request, sets the domain and sends the header - req = Request("http://example.com/") - assert mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) - - # subsequent request to the same domain, should send the header - req = Request("http://example.com/") - assert mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) - - # subsequent request to a different domain, shouldn't send the header - req = Request("http://example-noauth.com/") - assert mw.process_request(req, self.spider) is None - self.assertNotIn("Authorization", req.headers) - - def test_auth_already_set(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Using HttpAuthMiddleware without http_auth_domain is deprecated", - ): - mw = HttpAuthMiddleware() - mw.spider_opened(self.spider) - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) - assert mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Digest 123") - class HttpAuthMiddlewareTest(unittest.TestCase): def setUp(self): From 3db438127cb4e13f02d16322b3a88a84ca4655db Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 17 Oct 2023 21:38:38 -0300 Subject: [PATCH 1213/2083] Remove support for HttpCompressionMiddleware subclasses without stats --- .../downloadermiddlewares/httpcompression.py | 15 +------------ ...st_downloadermiddleware_httpcompression.py | 21 +++---------------- 2 files changed, 4 insertions(+), 32 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index ead426951..7b1d3f829 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,11 +1,9 @@ import io -import warnings import zlib from scrapy.exceptions import NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip ACCEPTED_ENCODINGS = [b"gzip", b"deflate"] @@ -36,18 +34,7 @@ class HttpCompressionMiddleware: def from_crawler(cls, crawler): if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured - try: - return cls(stats=crawler.stats) - except TypeError: - warnings.warn( - "HttpCompressionMiddleware subclasses must either modify " - "their '__init__' method to support a 'stats' parameter or " - "reimplement the 'from_crawler' method.", - ScrapyDeprecationWarning, - ) - result = cls() - result.stats = crawler.stats - return result + return cls(stats=crawler.stats) def process_request(self, request, spider): request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 9dad056de..fcbe50e2b 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -2,7 +2,6 @@ from gzip import GzipFile from io import BytesIO from pathlib import Path from unittest import SkipTest, TestCase -from warnings import catch_warnings from w3lib.encoding import resolve_encoding @@ -10,7 +9,7 @@ from scrapy.downloadermiddlewares.httpcompression import ( ACCEPTED_ENCODINGS, HttpCompressionMiddleware, ) -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider @@ -381,20 +380,6 @@ class HttpCompressionSubclassTest(TestCase): super().__init__() crawler = get_crawler(Spider) - with catch_warnings(record=True) as caught_warnings: + + with self.assertRaises(TypeError): HttpCompressionMiddlewareSubclass.from_crawler(crawler) - messages = tuple( - str(warning.message) - for warning in caught_warnings - if warning.category is ScrapyDeprecationWarning - ) - self.assertEqual( - messages, - ( - ( - "HttpCompressionMiddleware subclasses must either modify " - "their '__init__' method to support a 'stats' parameter " - "or reimplement the 'from_crawler' method." - ), - ), - ) From 644ab3af48a16f746f843a8390ff9e1593dc5ce0 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 17 Oct 2023 22:39:09 -0300 Subject: [PATCH 1214/2083] Remove support for feed storage backends without feed_options --- scrapy/extensions/feedexport.py | 14 +--- tests/test_feedexport.py | 115 +++----------------------------- 2 files changed, 11 insertions(+), 118 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4e846d1bd..fadbbb582 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -29,7 +29,7 @@ from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.python import get_func_args, without_none_values +from scrapy.utils.python import without_none_values logger = logging.getLogger(__name__) @@ -42,17 +42,7 @@ except ImportError: def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): - argument_names = get_func_args(builder) - if "feed_options" in argument_names: - kwargs["feed_options"] = feed_options - else: - warnings.warn( - f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove this " - "warning. This parameter will become mandatory in a future " - "version of Scrapy.", - category=ScrapyDeprecationWarning, - ) + kwargs["feed_options"] = feed_options return builder(*preargs, uri, *args, **kwargs) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 56967c0d5..62fcebde1 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2850,20 +2850,8 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): "FEED_URI": "file:///tmp/foobar", "FEED_STORAGES": {"file": StdoutFeedStorageWithoutFeedOptions}, } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - with pytest.warns( - ScrapyDeprecationWarning, - match="StdoutFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) + with pytest.raises(TypeError): + get_crawler(settings_dict=settings_dict) class FileFeedStorageWithoutFeedOptions(FileFeedStorage): @@ -2872,10 +2860,6 @@ class FileFeedStorageWithoutFeedOptions(FileFeedStorage): class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): - """Make sure that any feed exporter created by users before the - introduction of the ``feed_options`` parameter continues to work as - expected, and simply issues a warning.""" - maxDiff = None def test_init(self): @@ -2884,20 +2868,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): "FEED_URI": f"file:///{temp.name}", "FEED_STORAGES": {"file": FileFeedStorageWithoutFeedOptions}, } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - spider = scrapy.Spider("default") - - with pytest.warns( - ScrapyDeprecationWarning, - match="FileFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) + with self.assertRaises(TypeError): + get_crawler(settings_dict=settings_dict) class S3FeedStorageWithoutFeedOptions(S3FeedStorage): @@ -2912,10 +2884,6 @@ class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): - """Make sure that any feed exporter created by users before the - introduction of the ``feed_options`` parameter continues to work as - expected, and simply issues a warning.""" - maxDiff = None def setUp(self): @@ -2936,34 +2904,15 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): spider = scrapy.Spider("default") spider.crawler = crawler - with pytest.warns( - ScrapyDeprecationWarning, - match="S3FeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) + feed_exporter.open_spider(spider) def test_from_crawler(self): settings_dict = { "FEED_URI": "file:///tmp/foobar", "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptionsWithFromCrawler}, } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - spider.crawler = crawler - - with pytest.warns( - ScrapyDeprecationWarning, - match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) + with pytest.raises(TypeError): + get_crawler(settings_dict=settings_dict) class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage): @@ -2971,17 +2920,7 @@ class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage): super().__init__(uri) -class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage): - @classmethod - def from_crawler(cls, crawler, uri): - return super().from_crawler(crawler, uri) - - class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): - """Make sure that any feed exporter created by users before the - introduction of the ``feed_options`` parameter continues to work as - expected, and simply issues a warning.""" - maxDiff = None def test_init(self): @@ -2989,44 +2928,8 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): "FEED_URI": "ftp://localhost/foo", "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptions}, } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - spider.crawler = crawler - - with pytest.warns( - ScrapyDeprecationWarning, - match="FTPFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) - - def test_from_crawler(self): - settings_dict = { - "FEED_URI": "ftp://localhost/foo", - "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptionsWithFromCrawler}, - } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - spider.crawler = crawler - - with pytest.warns( - ScrapyDeprecationWarning, - match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " - "the 'feed_options' keyword argument.", - ): - feed_exporter.open_spider(spider) + with pytest.raises(TypeError): + get_crawler(settings_dict=settings_dict) class URIParamsTest: From 0956b764657b9f1d414e196a339988d3951c49dc Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 17 Oct 2023 22:44:15 -0300 Subject: [PATCH 1215/2083] Remove support for dupefilters without a fingerprinter --- scrapy/dupefilters.py | 35 +++++------------------------------ 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 0b20f53b9..dd2420e98 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -3,14 +3,12 @@ from __future__ import annotations import logging from pathlib import Path from typing import TYPE_CHECKING, Optional, Set -from warnings import warn from twisted.internet.defer import Deferred from scrapy.http.request import Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.job import job_dir from scrapy.utils.request import ( RequestFingerprinter, @@ -75,38 +73,15 @@ class RFPDupeFilter(BaseDupeFilter): fingerprinter: Optional[RequestFingerprinterProtocol] = None, ) -> Self: debug = settings.getbool("DUPEFILTER_DEBUG") - try: - return cls(job_dir(settings), debug, fingerprinter=fingerprinter) - except TypeError: - warn( - "RFPDupeFilter subclasses must either modify their '__init__' " - "method to support a 'fingerprinter' parameter or reimplement " - "the 'from_settings' class method.", - ScrapyDeprecationWarning, - ) - result = cls(job_dir(settings), debug) - result.fingerprinter = fingerprinter or RequestFingerprinter() - return result + return cls(job_dir(settings), debug, fingerprinter=fingerprinter) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: assert crawler.request_fingerprinter - try: - return cls.from_settings( - crawler.settings, - fingerprinter=crawler.request_fingerprinter, - ) - except TypeError: - warn( - "RFPDupeFilter subclasses must either modify their overridden " - "'__init__' method and 'from_settings' class method to " - "support a 'fingerprinter' parameter, or reimplement the " - "'from_crawler' class method.", - ScrapyDeprecationWarning, - ) - result = cls.from_settings(crawler.settings) - result.fingerprinter = crawler.request_fingerprinter - return result + return cls.from_settings( + crawler.settings, + fingerprinter=crawler.request_fingerprinter, + ) def request_seen(self, request: Request) -> bool: fp = self.request_fingerprint(request) From 39ee8d1ee2decce995c4dafc5da5a6fb55f478c9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 18 Oct 2023 11:10:21 +0400 Subject: [PATCH 1216/2083] Deprecate ReppyRobotParser (#6099) --- docs/topics/downloader-middleware.rst | 3 ++- scrapy/robotstxt.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index a8e5b23bf..1abbc4968 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1039,8 +1039,8 @@ RobotsTxtMiddleware * :ref:`Protego ` (default) * :ref:`RobotFileParser ` - * :ref:`Reppy ` * :ref:`Robotexclusionrulesparser ` + * :ref:`Reppy ` (deprecated) You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` setting. Or you can also :ref:`implement support for a new parser `. @@ -1133,6 +1133,7 @@ In order to use this parser: .. warning:: `Upstream issue #122 `_ prevents reppy usage in Python 3.9+. + Because of this the Reppy parser is deprecated. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 604b5e314..5c5ac4e41 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -1,7 +1,9 @@ import logging import sys from abc import ABCMeta, abstractmethod +from warnings import warn +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) @@ -79,6 +81,7 @@ class PythonRobotParser(RobotParser): class ReppyRobotParser(RobotParser): def __init__(self, robotstxt_body, spider): + warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2) from reppy.robots import Robots self.spider = spider From 38dbd4399361d6e3ac14cda40f4aee8d04204346 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 18 Oct 2023 06:29:47 -0300 Subject: [PATCH 1217/2083] Remove tests not necessary anymore --- ...st_downloadermiddleware_httpcompression.py | 12 --- tests/test_feedexport.py | 97 ------------------- 2 files changed, 109 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index fcbe50e2b..a96b710f3 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -371,15 +371,3 @@ class HttpCompressionTest(TestCase): self.assertEqual(response.body, b"") self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) - - -class HttpCompressionSubclassTest(TestCase): - def test_init_missing_stats(self): - class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware): - def __init__(self): - super().__init__() - - crawler = get_crawler(Spider) - - with self.assertRaises(TypeError): - HttpCompressionMiddlewareSubclass.from_crawler(crawler) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 62fcebde1..89169fd7c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2835,103 +2835,6 @@ class FeedExportInitTest(unittest.TestCase): self.assertIsInstance(exporter, FeedExporter) -class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage): - def __init__(self, uri): - super().__init__(uri) - - -class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): - """Make sure that any feed exporter created by users before the - introduction of the ``feed_options`` parameter continues to work as - expected, and simply issues a warning.""" - - def test_init(self): - settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": StdoutFeedStorageWithoutFeedOptions}, - } - with pytest.raises(TypeError): - get_crawler(settings_dict=settings_dict) - - -class FileFeedStorageWithoutFeedOptions(FileFeedStorage): - def __init__(self, uri): - super().__init__(uri) - - -class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): - maxDiff = None - - def test_init(self): - with tempfile.NamedTemporaryFile() as temp: - settings_dict = { - "FEED_URI": f"file:///{temp.name}", - "FEED_STORAGES": {"file": FileFeedStorageWithoutFeedOptions}, - } - with self.assertRaises(TypeError): - get_crawler(settings_dict=settings_dict) - - -class S3FeedStorageWithoutFeedOptions(S3FeedStorage): - def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs): - super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs) - - -class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): - @classmethod - def from_crawler(cls, crawler, uri): - return super().from_crawler(crawler, uri) - - -class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): - maxDiff = None - - def setUp(self): - skip_if_no_boto() - - def test_init(self): - settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptions}, - } - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) - - spider = scrapy.Spider("default") - spider.crawler = crawler - - feed_exporter.open_spider(spider) - - def test_from_crawler(self): - settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": S3FeedStorageWithoutFeedOptionsWithFromCrawler}, - } - with pytest.raises(TypeError): - get_crawler(settings_dict=settings_dict) - - -class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage): - def __init__(self, uri, use_active_mode=False): - super().__init__(uri) - - -class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): - maxDiff = None - - def test_init(self): - settings_dict = { - "FEED_URI": "ftp://localhost/foo", - "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptions}, - } - with pytest.raises(TypeError): - get_crawler(settings_dict=settings_dict) - - class URIParamsTest: spider_name = "uri_params_spider" deprecated_options = False From cc9c415bf3312e2c23288dfeb5f977a00874d0a2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 18 Oct 2023 18:21:34 +0400 Subject: [PATCH 1218/2083] Disable ipython for InteractiveShellTest. --- tests/test_command_shell.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7d87eb62c..7918d94b2 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,3 +1,4 @@ +import os import sys from io import BytesIO from pathlib import Path @@ -147,8 +148,10 @@ class InteractiveShellTest(unittest.TestCase): "scrapy.cmdline", "shell", ) + env = os.environ.copy() + env["SCRAPY_PYTHON_SHELL"] = "python" logfile = BytesIO() - p = PopenSpawn(args, timeout=5) + p = PopenSpawn(args, env=env, timeout=5) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") with MockServer() as mockserver: From 6b0c18e921c046d1c14106c629243e96d6cd60f0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 19 Oct 2023 23:13:47 +0400 Subject: [PATCH 1219/2083] Improve signal sending in test_shutdown_forced. --- tests/test_crawler.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 5c11ca6e0..0a7f9bac8 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -534,7 +534,10 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.expect_exact("Spider closed (shutdown)") p.wait() + @defer.inlineCallbacks def test_shutdown_forced(self): + from twisted.internet import reactor + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK args = self.get_script_args("sleeping.py", "-a", "sleep=10") p = PopenSpawn(args, timeout=5) @@ -542,6 +545,10 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.expect_exact("Crawled (200)") p.kill(sig) p.expect_exact("shutting down gracefully") + # sending the second signal too fast often causes problems + d = defer.Deferred() + reactor.callLater(0.1, d.callback, None) + yield d p.kill(sig) p.expect_exact("forcing unclean shutdown") p.wait() From 1f797d0fdb2615af31a125e343ebbae664d4b238 Mon Sep 17 00:00:00 2001 From: Chenwei Niu <61908960+Chenwei-Niu@users.noreply.github.com> Date: Mon, 30 Oct 2023 19:59:11 +1100 Subject: [PATCH 1220/2083] Removed some deprecated functions and functionalities (#6116) --- scrapy/utils/log.py | 13 +--------- scrapy/utils/response.py | 20 --------------- tests/test_downloadermiddleware_stats.py | 23 ----------------- tests/test_utils_response.py | 32 ------------------------ 4 files changed, 1 insertion(+), 87 deletions(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index fdea46a3d..2a38f151a 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -2,7 +2,6 @@ from __future__ import annotations import logging import sys -import warnings from logging.config import dictConfig from types import TracebackType from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast @@ -11,7 +10,6 @@ from twisted.python import log as twisted_log from twisted.python.failure import Failure import scrapy -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.versions import scrapy_components_versions @@ -232,18 +230,9 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: and adapts it into a tuple of positional arguments for logger.log calls, handling backward compatibility as well. """ - if not {"level", "msg", "args"} <= set(logkws): - warnings.warn("Missing keys in LogFormatter method", ScrapyDeprecationWarning) - - if "format" in logkws: - warnings.warn( - "`format` key in LogFormatter methods has been " - "deprecated, use `msg` instead", - ScrapyDeprecationWarning, - ) level = logkws.get("level", logging.INFO) - message = logkws.get("format", logkws.get("msg")) + message = logkws.get("msg") or "" # NOTE: This also handles 'args' being an empty dict, that case doesn't # play well in logger.log calls args = logkws if not logkws.get("args") else logkws["args"] diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c540d6278..77d54aff9 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -14,7 +14,6 @@ from w3lib import html import scrapy from scrapy.http.response import Response -from scrapy.utils.decorators import deprecated from scrapy.utils.python import to_bytes, to_unicode _baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() @@ -55,25 +54,6 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str: return f"{status_int} {to_unicode(message)}" -@deprecated -def response_httprepr(response: Response) -> bytes: - """Return raw HTTP representation (as bytes) of the given response. This - is provided only for reference, since it's not the exact stream of bytes - that was received (that's not exposed by Twisted). - """ - values = [ - b"HTTP/1.1 ", - to_bytes(str(response.status)), - b" ", - to_bytes(http.RESPONSES.get(response.status, b"")), - b"\r\n", - ] - if response.headers: - values.extend([response.headers.to_string(), b"\r\n"]) - values.extend([b"\r\n", response.body]) - return b"".join(values) - - def open_in_browser( response: Union[ "scrapy.http.response.html.HtmlResponse", diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 39dfe9ab5..5b7181848 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,12 +1,8 @@ -import warnings -from itertools import product from unittest import TestCase from scrapy.downloadermiddlewares.stats import DownloaderStats -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.spiders import Spider -from scrapy.utils.response import response_httprepr from scrapy.utils.test import get_crawler @@ -40,25 +36,6 @@ class TestDownloaderStats(TestCase): self.mw.process_response(self.req, self.res, self.spider) self.assertStatsEqual("downloader/response_count", 1) - def test_response_len(self): - body = (b"", b"not_empty") # empty/notempty body - headers = ( - {}, - {"lang": "en"}, - {"lang": "en", "User-Agent": "scrapy"}, - ) # 0 headers, 1h and 2h - test_responses = [ # form test responses with all combinations of body/headers - Response(url="scrapytest.org", status=200, body=r[0], headers=r[1]) - for r in product(body, headers) - ] - for test_response in test_responses: - self.crawler.stats.set_value("downloader/response_bytes", 0) - self.mw.process_response(self.req, test_response, self.spider) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - resp_size = len(response_httprepr(test_response)) - self.assertStatsEqual("downloader/response_bytes", resp_size) - def test_process_exception(self): self.mw.process_exception(self.req, MyException(), self.spider) self.assertStatsEqual("downloader/exception_count", 1) diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 80e15a60f..661fb47a3 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,16 +1,13 @@ import unittest -import warnings from pathlib import Path from urllib.parse import urlparse -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import ( get_base_url, get_meta_refresh, open_in_browser, - response_httprepr, response_status_message, ) @@ -20,35 +17,6 @@ __doctests__ = ["scrapy.utils.response"] class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") - def test_response_httprepr(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - - r1 = Response("http://www.example.com") - self.assertEqual(response_httprepr(r1), b"HTTP/1.1 200 OK\r\n\r\n") - - r1 = Response( - "http://www.example.com", - status=404, - headers={"Content-type": "text/html"}, - body=b"Some body", - ) - self.assertEqual( - response_httprepr(r1), - b"HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body", - ) - - r1 = Response( - "http://www.example.com", - status=6666, - headers={"Content-type": "text/html"}, - body=b"Some body", - ) - self.assertEqual( - response_httprepr(r1), - b"HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body", - ) - def test_open_in_browser(self): url = "http:///www.example.com/some/page.html" body = b" test page test body " From 8dff9633d0b80aff0c6f481e395755f3c8985f27 Mon Sep 17 00:00:00 2001 From: andy53 Date: Mon, 30 Oct 2023 21:40:26 -0600 Subject: [PATCH 1221/2083] added_extensions --- scrapy/linkextractors/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 6b8be909e..3774430a7 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -96,6 +96,16 @@ IGNORED_EXTENSIONS = [ "dmg", "iso", "apk", + "jar", + "sh", + "rb", + "js", + "hta", + "bat", + "cpl", + "msi", + "msp", + "py", ] From 04024f1e796f99e77dda544396722fb9203f1d49 Mon Sep 17 00:00:00 2001 From: nihilisticneuralnet <138315505+nihilisticneuralnet@users.noreply.github.com> Date: Tue, 31 Oct 2023 19:07:26 +0530 Subject: [PATCH 1222/2083] [Solved] JOBDIR= None for when Scheduler initializes disk queue even if JOBDIR is empty string (#6124) Co-authored-by: John Doe --- docs/topics/settings.rst | 2 +- scrapy/core/scheduler.py | 2 +- scrapy/settings/default_settings.py | 2 ++ scrapy/utils/job.py | 6 ++++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7cdfb8768..06f8481ba 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1120,7 +1120,7 @@ modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. JOBDIR ------ -Default: ``''`` +Default: ``None`` A string indicating the directory for storing the state of a crawl when :ref:`pausing and resuming crawls `. diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 70b6dc8a1..17c95f1ea 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -352,7 +352,7 @@ class Scheduler(BaseScheduler): def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: """Return a folder name to keep disk queue state at""" - if jobdir is not None: + if jobdir: dqdir = Path(jobdir, "requests.queue") if not dqdir.exists(): dqdir.mkdir(parents=True) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index fa06e5ee8..d6b3585e2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -206,6 +206,8 @@ ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager" ITEM_PIPELINES = {} ITEM_PIPELINES_BASE = {} +JOBDIR = None + LOG_ENABLED = True LOG_ENCODING = "utf-8" LOG_FORMATTER = "scrapy.logformatter.LogFormatter" diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index c49f7d758..e230e4235 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -5,7 +5,9 @@ from scrapy.settings import BaseSettings def job_dir(settings: BaseSettings) -> Optional[str]: - path: str = settings["JOBDIR"] - if path and not Path(path).exists(): + path: Optional[str] = settings["JOBDIR"] + if not path: + return None + if not Path(path).exists(): Path(path).mkdir(parents=True) return path From 732557e6988dd3d1df26f973e3ae651f7bcbdc5b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 31 Oct 2023 22:34:43 +0400 Subject: [PATCH 1223/2083] Full typing for scrapy/http/request. --- scrapy/http/common.py | 7 +++-- scrapy/http/request/__init__.py | 47 ++++++++++++++++------------- scrapy/http/request/form.py | 27 +++++++++++------ scrapy/http/request/json_request.py | 8 +++-- scrapy/http/request/rpc.py | 4 +-- 5 files changed, 55 insertions(+), 38 deletions(-) diff --git a/scrapy/http/common.py b/scrapy/http/common.py index bc8861574..a3d9d5b81 100644 --- a/scrapy/http/common.py +++ b/scrapy/http/common.py @@ -1,5 +1,8 @@ -def obsolete_setter(setter, attrname): - def newsetter(self, value): +from typing import Any, Callable, NoReturn + + +def obsolete_setter(setter: Callable, attrname: str) -> Callable[[Any, Any], NoReturn]: + def newsetter(self: Any, value: Any) -> NoReturn: c = self.__class__.__name__ msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead" raise AttributeError(msg) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 06a52f5fd..7c9a4ba95 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -9,14 +9,17 @@ from typing import ( Any, AnyStr, Callable, + Dict, Iterable, List, Mapping, + NoReturn, Optional, Tuple, Type, TypeVar, Union, + cast, ) from w3lib.url import safe_url_string @@ -32,7 +35,7 @@ from scrapy.utils.url import escape_ajax RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") -def NO_CALLBACK(*args, **kwargs): +def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: """When assigned to the ``callback`` parameter of :class:`~scrapy.http.Request`, it indicates that the request is not meant to have a spider callback at all. @@ -92,21 +95,21 @@ class Request(object_ref): headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[Union[dict, List[dict]]] = None, - meta: Optional[dict] = None, + meta: Optional[Dict[str, Any]] = None, encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable] = None, flags: Optional[List[str]] = None, - cb_kwargs: Optional[dict] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, ) -> None: - self._encoding = encoding # this one has to be set first - self.method = str(method).upper() + self._encoding: str = encoding # this one has to be set first + self.method: str = str(method).upper() self._set_url(url) self._set_body(body) if not isinstance(priority, int): raise TypeError(f"Request priority not an integer: {priority!r}") - self.priority = priority + self.priority: int = priority if not (callable(callback) or callback is None): raise TypeError( @@ -114,25 +117,27 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") - self.callback = callback - self.errback = errback + self.callback: Optional[Callable] = callback + self.errback: Optional[Callable] = errback - self.cookies = cookies or {} - self.headers = Headers(headers or {}, encoding=encoding) - self.dont_filter = dont_filter + self.cookies: Union[dict, List[dict]] = cookies or {} + self.headers: Headers = Headers(headers or {}, encoding=encoding) + self.dont_filter: bool = dont_filter - self._meta = dict(meta) if meta else None - self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None - self.flags = [] if flags is None else list(flags) + self._meta: Optional[Dict[str, Any]] = dict(meta) if meta else None + self._cb_kwargs: Optional[Dict[str, Any]] = ( + dict(cb_kwargs) if cb_kwargs else None + ) + self.flags: List[str] = [] if flags is None else list(flags) @property - def cb_kwargs(self) -> dict: + def cb_kwargs(self) -> Dict[str, Any]: if self._cb_kwargs is None: self._cb_kwargs = {} return self._cb_kwargs @property - def meta(self) -> dict: + def meta(self) -> Dict[str, Any]: if self._meta is None: self._meta = {} return self._meta @@ -174,19 +179,19 @@ class Request(object_ref): def copy(self) -> "Request": return self.replace() - def replace(self, *args, **kwargs) -> "Request": + def replace(self, *args: Any, **kwargs: Any) -> "Request": """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop("cls", self.__class__) - return cls(*args, **kwargs) + return cast(Request, cls(*args, **kwargs)) @classmethod def from_curl( cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, - **kwargs, + **kwargs: Any, ) -> RequestTypeVar: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the @@ -219,7 +224,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict: + def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. @@ -244,7 +249,7 @@ class Request(object_ref): return d -def _find_method(obj, func): +def _find_method(obj: Any, func: Callable) -> str: """Helper function for Request.to_dict""" # Only instance methods contain ``__func__`` if obj and hasattr(func, "__func__"): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 8da779b7c..0f80a0ab7 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,7 +5,9 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from lxml.html import ( @@ -24,7 +26,10 @@ from scrapy.http.response.text import TextResponse from scrapy.utils.python import is_listlike, to_bytes from scrapy.utils.response import get_base_url -FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + FormdataKVType = Tuple[str, Union[str, Iterable[str]]] FormdataType = Optional[Union[dict, List[FormdataKVType]]] @@ -33,7 +38,9 @@ FormdataType = Optional[Union[dict, List[FormdataKVType]]] class FormRequest(Request): valid_form_methods = ["GET", "POST"] - def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None: + def __init__( + self, *args: Any, formdata: FormdataType = None, **kwargs: Any + ) -> None: if formdata and kwargs.get("method") is None: kwargs["method"] = "POST" @@ -54,7 +61,7 @@ class FormRequest(Request): @classmethod def from_response( - cls: Type[FormRequestTypeVar], + cls, response: TextResponse, formname: Optional[str] = None, formid: Optional[str] = None, @@ -64,8 +71,8 @@ class FormRequest(Request): dont_click: bool = False, formxpath: Optional[str] = None, formcss: Optional[str] = None, - **kwargs, - ) -> FormRequestTypeVar: + **kwargs: Any, + ) -> Self: kwargs.setdefault("encoding", response.encoding) if formcss is not None: @@ -121,12 +128,12 @@ def _get_form( if formname is not None: f = root.xpath(f'//form[@name="{formname}"]') if f: - return f[0] + return cast(FormElement, f[0]) if formid is not None: f = root.xpath(f'//form[@id="{formid}"]') if f: - return f[0] + return cast(FormElement, f[0]) # Get form element from xpath, if not found, go up if formxpath is not None: @@ -135,7 +142,7 @@ def _get_form( el = nodes[0] while True: if el.tag == "form": - return el + return cast(FormElement, el) el = el.getparent() if el is None: break @@ -147,7 +154,7 @@ def _get_form( except IndexError: raise IndexError(f"Form number {formnumber} not found in {response}") else: - return form + return cast(FormElement, form) def _get_inputs( diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index f52c0401d..1dd9e6c87 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings -from typing import Optional, Tuple +from typing import Any, Optional, Tuple from scrapy.http.request import Request @@ -16,7 +16,9 @@ from scrapy.http.request import Request class JsonRequest(Request): attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) - def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None: + def __init__( + self, *args: Any, dumps_kwargs: Optional[dict] = None, **kwargs: Any + ) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault("sort_keys", True) self._dumps_kwargs = dumps_kwargs @@ -42,7 +44,7 @@ class JsonRequest(Request): def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args, **kwargs) -> Request: + def replace(self, *args: Any, **kwargs: Any) -> Request: body_passed = kwargs.get("body", None) is not None data = kwargs.pop("data", None) data_passed = data is not None diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 43692923b..bde860a66 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -5,7 +5,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ import xmlrpc.client as xmlrpclib -from typing import Optional +from typing import Any, Optional from scrapy.http.request import Request from scrapy.utils.python import get_func_args @@ -14,7 +14,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - def __init__(self, *args, encoding: Optional[str] = None, **kwargs): + def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any): if "body" not in kwargs and "params" in kwargs: kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) kwargs["body"] = xmlrpclib.dumps(**kw) From 4cb2fc2c3b9deba49ae3a32400819ed95ea262c9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 31 Oct 2023 22:55:39 +0400 Subject: [PATCH 1224/2083] Update typing package versions. --- scrapy/utils/ssl.py | 2 +- tox.ini | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index d520ef809..e74769c65 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,6 +1,6 @@ from typing import Any, Optional -import OpenSSL._util as pyOpenSSLutil +import OpenSSL._util as pyOpenSSLutil # type: ignore[import-untyped] import OpenSSL.SSL import OpenSSL.version from OpenSSL.crypto import X509Name diff --git a/tox.ini b/tox.ini index 381da9773..932c0b805 100644 --- a/tox.ini +++ b/tox.ini @@ -33,13 +33,13 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.5.1 - typing-extensions==4.7.1 + mypy==1.6.1 + typing-extensions==4.8.0 types-attrs==19.1.0 - types-lxml==2023.3.28 - types-Pillow==10.0.0.3 + types-lxml==2023.10.21 + types-Pillow==10.1.0.0 types-Pygments==2.16.0.0 - types-pyOpenSSL==23.2.0.2 + types-pyOpenSSL==23.3.0.0 types-setuptools==68.2.0.0 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 From 01d9d28324cbd5c7edff0d25ba00d8491400698a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 31 Oct 2023 23:14:00 +0400 Subject: [PATCH 1225/2083] Full typing for scrapy/http/response/__init__.py. --- scrapy/core/downloader/handlers/datauri.py | 9 +- scrapy/core/scraper.py | 1 + scrapy/http/response/__init__.py | 142 ++++++++++++--------- scrapy/http/response/text.py | 16 +-- tests/test_http2_client_protocol.py | 1 + 5 files changed, 98 insertions(+), 71 deletions(-) diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index 8b78c53c1..a7ae56a85 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -1,6 +1,9 @@ +from typing import Any, Dict + from w3lib.url import parse_data_uri -from scrapy.http import TextResponse +from scrapy import Request, Spider +from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers @@ -9,11 +12,11 @@ class DataURIDownloadHandler: lazy = False @defers - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Response: uri = parse_data_uri(request.url) respcls = responsetypes.from_mimetype(uri.media_type) - resp_kwargs = {} + resp_kwargs: Dict[str, Any] = {} if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text": charset = uri.media_type_parameters.get("charset") resp_kwargs["encoding"] = charset diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index b2c26507c..8fb16b8a9 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -206,6 +206,7 @@ class Scraper: if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request + assert result.request callback = result.request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) dfd = defer_succeed(result) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 82274fc3a..8e9237dad 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,9 +4,28 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ -from typing import Any, AnyStr, Generator, Iterable, Mapping, Tuple, Union +from __future__ import annotations + +from ipaddress import IPv4Address, IPv6Address +from typing import ( + TYPE_CHECKING, + Any, + AnyStr, + Callable, + Dict, + Generator, + Iterable, + List, + Mapping, + Optional, + Tuple, + Union, + cast, +) from urllib.parse import urljoin +from twisted.internet.ssl import Certificate + from scrapy.exceptions import NotSupported from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers @@ -14,6 +33,9 @@ from scrapy.http.request import Request from scrapy.link import Link from scrapy.utils.trackref import object_ref +if TYPE_CHECKING: + from scrapy.selector import SelectorList + class Response(object_ref): """An object that represents an HTTP response, which is usually @@ -41,29 +63,29 @@ class Response(object_ref): def __init__( self, url: str, - status=200, + status: int = 200, headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, - body=b"", - flags=None, - request=None, - certificate=None, - ip_address=None, - protocol=None, + body: bytes = b"", + flags: Optional[List[str]] = None, + request: Optional[Request] = None, + certificate: Optional[Certificate] = None, + ip_address: Union[IPv4Address, IPv6Address, None] = None, + protocol: Optional[str] = None, ): - self.headers = Headers(headers or {}) - self.status = int(status) + self.headers: Headers = Headers(headers or {}) + self.status: int = int(status) self._set_body(body) self._set_url(url) - self.request = request - self.flags = [] if flags is None else list(flags) - self.certificate = certificate - self.ip_address = ip_address - self.protocol = protocol + self.request: Optional[Request] = request + self.flags: List[str] = [] if flags is None else list(flags) + self.certificate: Optional[Certificate] = certificate + self.ip_address: Union[IPv4Address, IPv6Address, None] = ip_address + self.protocol: Optional[str] = protocol @property - def cb_kwargs(self): + def cb_kwargs(self) -> Dict[str, Any]: try: - return self.request.cb_kwargs + return self.request.cb_kwargs # type: ignore[union-attr] except AttributeError: raise AttributeError( "Response.cb_kwargs not available, this response " @@ -71,21 +93,21 @@ class Response(object_ref): ) @property - def meta(self): + def meta(self) -> Dict[str, Any]: try: - return self.request.meta + return self.request.meta # type: ignore[union-attr] except AttributeError: raise AttributeError( "Response.meta not available, this response " "is not tied to any request" ) - def _get_url(self): + def _get_url(self) -> str: return self._url - def _set_url(self, url: str): + def _set_url(self, url: str) -> None: if isinstance(url, str): - self._url = url + self._url: str = url else: raise TypeError( f"{type(self).__name__} url must be str, " f"got {type(url).__name__}" @@ -93,10 +115,10 @@ class Response(object_ref): url = property(_get_url, obsolete_setter(_set_url, "url")) - def _get_body(self): + def _get_body(self) -> bytes: return self._body - def _set_body(self, body): + def _set_body(self, body: Optional[bytes]) -> None: if body is None: self._body = b"" elif not isinstance(body, bytes): @@ -110,45 +132,45 @@ class Response(object_ref): body = property(_get_body, obsolete_setter(_set_body, "body")) - def __repr__(self): + def __repr__(self) -> str: return f"<{self.status} {self.url}>" - def copy(self): + def copy(self) -> Response: """Return a copy of this Response""" return self.replace() - def replace(self, *args, **kwargs): + def replace(self, *args: Any, **kwargs: Any) -> Response: """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop("cls", self.__class__) - return cls(*args, **kwargs) + return cast(Response, cls(*args, **kwargs)) - def urljoin(self, url): + def urljoin(self, url: str) -> str: """Join this Response's url with a possible relative url to form an absolute interpretation of the latter.""" - return urljoin(self.url, url) + return urljoin(cast(str, self.url), url) @property - def text(self): + def text(self) -> str: """For subclasses of TextResponse, this will return the body as str """ raise AttributeError("Response content isn't text") - def css(self, *a, **kw): + def css(self, *a: Any, **kw: Any) -> SelectorList: """Shortcut method implemented only by responses whose content is text (subclasses of TextResponse). """ raise NotSupported("Response content isn't text") - def jmespath(self, *a, **kw): + def jmespath(self, *a: Any, **kw: Any) -> SelectorList: """Shortcut method implemented only by responses whose content is text (subclasses of TextResponse). """ raise NotSupported("Response content isn't text") - def xpath(self, *a, **kw): + def xpath(self, *a: Any, **kw: Any) -> SelectorList: """Shortcut method implemented only by responses whose content is text (subclasses of TextResponse). """ @@ -156,19 +178,19 @@ class Response(object_ref): def follow( self, - url, - callback=None, - method="GET", - headers=None, - body=None, - cookies=None, - meta=None, - encoding="utf-8", - priority=0, - dont_filter=False, - errback=None, - cb_kwargs=None, - flags=None, + url: Union[str, Link], + callback: Optional[Callable] = None, + method: str = "GET", + headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]] = None, + meta: Optional[Dict[str, Any]] = None, + encoding: str = "utf-8", + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, + flags: Optional[List[str]] = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -207,19 +229,19 @@ class Response(object_ref): def follow_all( self, - urls, - callback=None, - method="GET", - headers=None, - body=None, - cookies=None, - meta=None, - encoding="utf-8", - priority=0, - dont_filter=False, - errback=None, - cb_kwargs=None, - flags=None, + urls: Iterable[Union[str, Link]], + callback: Optional[Callable] = None, + method: str = "GET", + headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]] = None, + meta: Optional[Dict[str, Any]] = None, + encoding: str = "utf-8", + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, + flags: Optional[List[str]] = None, ) -> Generator[Request, None, None]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 98ae1f307..21a93fbd2 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -27,7 +27,7 @@ from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: - from scrapy.selector import Selector + from scrapy.selector import Selector, SelectorList _NONE = object() @@ -138,26 +138,26 @@ class TextResponse(Response): return read_bom(self.body)[0] @property - def selector(self): + def selector(self) -> Selector: from scrapy.selector import Selector if self._cached_selector is None: self._cached_selector = Selector(self) return self._cached_selector - def jmespath(self, query, **kwargs): + def jmespath(self, query: str, **kwargs: Any) -> SelectorList: if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined] raise AttributeError( "Please install parsel >= 1.8.1 to get jmespath support" ) - return self.selector.jmespath(query, **kwargs) # type: ignore[attr-defined] + return cast(SelectorList, self.selector.jmespath(query, **kwargs)) # type: ignore[attr-defined] - def xpath(self, query, **kwargs): - return self.selector.xpath(query, **kwargs) + def xpath(self, query: str, **kwargs: Any) -> SelectorList: + return cast(SelectorList, self.selector.xpath(query, **kwargs)) - def css(self, query): - return self.selector.css(query) + def css(self, query: str) -> SelectorList: + return cast(SelectorList, self.selector.css(query)) def follow( self, diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index deb35a579..8fdf3d56f 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -600,6 +600,7 @@ class Https2ClientProtocolTestCase(TestCase): def assert_metadata(response: Response): self.assertEqual(response.request, request) self.assertIsInstance(response.certificate, Certificate) + assert response.certificate # typing self.assertIsNotNone(response.certificate.original) self.assertEqual( response.certificate.getIssuer(), self.client_certificate.getIssuer() From 24f21e96b99bac1ba991fe02c8823782833ca298 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Nov 2023 01:53:25 +0400 Subject: [PATCH 1226/2083] Full typing for scrapy/http/response/text.py. --- scrapy/http/response/__init__.py | 6 +- scrapy/http/response/text.py | 113 +++++++++++++++++-------------- 2 files changed, 68 insertions(+), 51 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 8e9237dad..61010f14f 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -185,7 +185,7 @@ class Response(object_ref): body: Optional[Union[bytes, str]] = None, cookies: Optional[Union[dict, List[dict]]] = None, meta: Optional[Dict[str, Any]] = None, - encoding: str = "utf-8", + encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable] = None, @@ -205,6 +205,8 @@ class Response(object_ref): .. versionadded:: 2.0 The *flags* parameter. """ + if encoding is None: + raise ValueError("encoding can't be None") if isinstance(url, Link): url = url.url elif url is None: @@ -236,7 +238,7 @@ class Response(object_ref): body: Optional[Union[bytes, str]] = None, cookies: Optional[Union[dict, List[dict]]] = None, meta: Optional[Dict[str, Any]] = None, - encoding: str = "utf-8", + encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable] = None, diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 21a93fbd2..6596d8a5c 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,7 +8,21 @@ from __future__ import annotations import json from contextlib import suppress -from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple, cast +from typing import ( + TYPE_CHECKING, + Any, + AnyStr, + Callable, + Dict, + Generator, + Iterable, + List, + Mapping, + Optional, + Tuple, + Union, + cast, +) from urllib.parse import urljoin import parsel @@ -23,6 +37,7 @@ from w3lib.html import strip_html5_whitespace from scrapy.http import Request from scrapy.http.response import Response +from scrapy.link import Link from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url @@ -39,20 +54,14 @@ class TextResponse(Response): attributes: Tuple[str, ...] = Response.attributes + ("encoding",) def __init__(self, *args: Any, **kwargs: Any): - self._encoding = kwargs.pop("encoding", None) + self._encoding: Optional[str] = kwargs.pop("encoding", None) self._cached_benc: Optional[str] = None self._cached_ubody: Optional[str] = None self._cached_selector: Optional[Selector] = None super().__init__(*args, **kwargs) - def _set_url(self, url): - if isinstance(url, str): - self._url = to_unicode(url, self.encoding) - else: - super()._set_url(url) - - def _set_body(self, body): - self._body = b"" # used by encoding detection + def _set_body(self, body: Union[str, bytes, None]) -> None: + self._body: bytes = b"" # used by encoding detection if isinstance(body, str): if self._encoding is None: raise TypeError( @@ -64,10 +73,10 @@ class TextResponse(Response): super()._set_body(body) @property - def encoding(self): + def encoding(self) -> str: return self._declared_encoding() or self._body_inferred_encoding() - def _declared_encoding(self): + def _declared_encoding(self) -> Optional[str]: return ( self._encoding or self._bom_encoding() @@ -75,7 +84,7 @@ class TextResponse(Response): or self._body_declared_encoding() ) - def json(self): + def json(self) -> Any: """ .. versionadded:: 2.2 @@ -96,7 +105,7 @@ class TextResponse(Response): self._cached_ubody = html_to_unicode(charset, self.body)[1] return self._cached_ubody - def urljoin(self, url): + def urljoin(self, url: str) -> str: """Join this Response's url with a possible relative url to form an absolute interpretation of the latter.""" return urljoin(get_base_url(self), url) @@ -106,7 +115,7 @@ class TextResponse(Response): content_type = cast(bytes, self.headers.get(b"Content-Type", b"")) return http_content_type_encoding(to_unicode(content_type, encoding="latin-1")) - def _body_inferred_encoding(self): + def _body_inferred_encoding(self) -> str: if self._cached_benc is None: content_type = to_unicode( cast(bytes, self.headers.get(b"Content-Type", b"")), encoding="latin-1" @@ -121,20 +130,21 @@ class TextResponse(Response): self._cached_ubody = ubody return self._cached_benc - def _auto_detect_fun(self, text): + def _auto_detect_fun(self, text: bytes) -> Optional[str]: for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"): try: text.decode(enc) except UnicodeError: continue return resolve_encoding(enc) + return None @memoizemethod_noargs - def _body_declared_encoding(self): + def _body_declared_encoding(self) -> Optional[str]: return html_body_declared_encoding(self.body) @memoizemethod_noargs - def _bom_encoding(self): + def _bom_encoding(self) -> Optional[str]: return read_bom(self.body)[0] @property @@ -146,6 +156,8 @@ class TextResponse(Response): return self._cached_selector def jmespath(self, query: str, **kwargs: Any) -> SelectorList: + from scrapy.selector import SelectorList + if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined] raise AttributeError( "Please install parsel >= 1.8.1 to get jmespath support" @@ -154,26 +166,30 @@ class TextResponse(Response): return cast(SelectorList, self.selector.jmespath(query, **kwargs)) # type: ignore[attr-defined] def xpath(self, query: str, **kwargs: Any) -> SelectorList: + from scrapy.selector import SelectorList + return cast(SelectorList, self.selector.xpath(query, **kwargs)) def css(self, query: str) -> SelectorList: + from scrapy.selector import SelectorList + return cast(SelectorList, self.selector.css(query)) def follow( self, - url, - callback=None, - method="GET", - headers=None, - body=None, - cookies=None, - meta=None, - encoding=None, - priority=0, - dont_filter=False, - errback=None, - cb_kwargs=None, - flags=None, + url: Union[str, Link, parsel.Selector], + callback: Optional[Callable] = None, + method: str = "GET", + headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]] = None, + meta: Optional[Dict[str, Any]] = None, + encoding: Optional[str] = None, + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, + flags: Optional[List[str]] = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -214,21 +230,21 @@ class TextResponse(Response): def follow_all( self, - urls=None, - callback=None, - method="GET", - headers=None, - body=None, - cookies=None, - meta=None, - encoding=None, - priority=0, - dont_filter=False, - errback=None, - cb_kwargs=None, - flags=None, - css=None, - xpath=None, + urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None, + callback: Optional[Callable] = None, + method: str = "GET", + headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]] = None, + meta: Optional[Dict[str, Any]] = None, + encoding: Optional[str] = None, + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, + flags: Optional[List[str]] = None, + css: Optional[str] = None, + xpath: Optional[str] = None, ) -> Generator[Request, None, None]: """ A generator that produces :class:`~.Request` instances to follow all @@ -270,7 +286,7 @@ class TextResponse(Response): with suppress(_InvalidSelector): urls.append(_url_from_selector(sel)) return super().follow_all( - urls=urls, + urls=cast(Iterable[Union[str, Link]], urls), callback=callback, method=method, headers=headers, @@ -292,8 +308,7 @@ class _InvalidSelector(ValueError): """ -def _url_from_selector(sel): - # type: (parsel.Selector) -> str +def _url_from_selector(sel: parsel.Selector) -> str: if isinstance(sel.root, str): # e.g. ::attr(href) result return strip_html5_whitespace(sel.root) From e54dc598999e913526fc6aa5cad26ee0e7ca6140 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 2 Nov 2023 23:40:14 +0400 Subject: [PATCH 1227/2083] Full typing for scrapy/spidermiddlewares. --- scrapy/spidermiddlewares/depth.py | 41 ++++++++--- scrapy/spidermiddlewares/httperror.py | 33 +++++++-- scrapy/spidermiddlewares/offsite.py | 45 ++++++++---- scrapy/spidermiddlewares/referer.py | 101 +++++++++++++++++--------- scrapy/spidermiddlewares/urllength.py | 34 ++++++--- 5 files changed, 178 insertions(+), 76 deletions(-) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index eadc7c6ab..1e96654e2 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -4,46 +4,67 @@ Depth Spider Middleware See documentation in docs/topics/spider-middleware.rst """ -import logging +from __future__ import annotations -from scrapy.http import Request +import logging +from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable + +from scrapy import Spider +from scrapy.crawler import Crawler +from scrapy.http import Request, Response +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class DepthMiddleware: - def __init__(self, maxdepth, stats, verbose_stats=False, prio=1): + def __init__( + self, + maxdepth: int, + stats: StatsCollector, + verbose_stats: bool = False, + prio: int = 1, + ): self.maxdepth = maxdepth self.stats = stats self.verbose_stats = verbose_stats self.prio = prio @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: settings = crawler.settings maxdepth = settings.getint("DEPTH_LIMIT") verbose = settings.getbool("DEPTH_STATS_VERBOSE") prio = settings.getint("DEPTH_PRIORITY") + assert crawler.stats return cls(maxdepth, crawler.stats, verbose, prio) - def process_spider_output(self, response, result, spider): + def process_spider_output( + self, response: Response, result: Iterable[Any], spider: Spider + ) -> Iterable[Any]: self._init_depth(response, spider) - return (r for r in result or () if self._filter(r, response, spider)) + return (r for r in result if self._filter(r, response, spider)) - async def process_spider_output_async(self, response, result, spider): + async def process_spider_output_async( + self, response: Response, result: AsyncIterable[Any], spider: Spider + ) -> AsyncIterable[Any]: self._init_depth(response, spider) - async for r in result or (): + async for r in result: if self._filter(r, response, spider): yield r - def _init_depth(self, response, spider): + def _init_depth(self, response: Response, spider: Spider) -> None: # base case (depth=0) if "depth" not in response.meta: response.meta["depth"] = 0 if self.verbose_stats: self.stats.inc_value("request_depth_count/0", spider=spider) - def _filter(self, request, response, spider): + def _filter(self, request: Any, response: Response, spider: Spider) -> bool: if not isinstance(request, Request): return True depth = response.meta["depth"] + 1 diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 0d3e5fe0b..94450b35b 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -3,9 +3,20 @@ HttpError Spider Middleware See documentation in docs/topics/spider-middleware.rst """ -import logging +from __future__ import annotations +import logging +from typing import TYPE_CHECKING, Any, Iterable, List, Optional + +from scrapy import Spider +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest +from scrapy.http import Response +from scrapy.settings import BaseSettings + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) @@ -13,21 +24,23 @@ logger = logging.getLogger(__name__) class HttpError(IgnoreRequest): """A non-200 response was filtered""" - def __init__(self, response, *args, **kwargs): + def __init__(self, response: Response, *args: Any, **kwargs: Any): self.response = response super().__init__(*args, **kwargs) class HttpErrorMiddleware: @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def __init__(self, settings): - self.handle_httpstatus_all = settings.getbool("HTTPERROR_ALLOW_ALL") - self.handle_httpstatus_list = settings.getlist("HTTPERROR_ALLOWED_CODES") + def __init__(self, settings: BaseSettings): + self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL") + self.handle_httpstatus_list: List[int] = settings.getlist( + "HTTPERROR_ALLOWED_CODES" + ) - def process_spider_input(self, response, spider): + def process_spider_input(self, response: Response, spider: Spider) -> None: if 200 <= response.status < 300: # common case return meta = response.meta @@ -45,8 +58,11 @@ class HttpErrorMiddleware: return raise HttpError(response, "Ignoring non-200 response") - def process_spider_exception(self, response, exception, spider): + def process_spider_exception( + self, response: Response, exception: Exception, spider: Spider + ) -> Optional[Iterable[Any]]: if isinstance(exception, HttpError): + assert spider.crawler.stats spider.crawler.stats.inc_value("httperror/response_ignored_count") spider.crawler.stats.inc_value( f"httperror/response_ignored_status_count/{response.status}" @@ -57,3 +73,4 @@ class HttpErrorMiddleware: extra={"spider": spider}, ) return [] + return None diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 1a48926b3..a5214702d 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -3,36 +3,51 @@ Offsite Spider Middleware See documentation in docs/topics/spider-middleware.rst """ +from __future__ import annotations + import logging import re import warnings +from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set -from scrapy import signals -from scrapy.http import Request +from scrapy import Spider, signals +from scrapy.crawler import Crawler +from scrapy.http import Request, Response +from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) class OffsiteMiddleware: - def __init__(self, stats): - self.stats = stats + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def process_spider_output(self, response, result, spider): - return (r for r in result or () if self._filter(r, spider)) + def process_spider_output( + self, response: Response, result: Iterable[Any], spider: Spider + ) -> Iterable[Any]: + return (r for r in result if self._filter(r, spider)) - async def process_spider_output_async(self, response, result, spider): - async for r in result or (): + async def process_spider_output_async( + self, response: Response, result: AsyncIterable[Any], spider: Spider + ) -> AsyncIterable[Any]: + async for r in result: if self._filter(r, spider): yield r - def _filter(self, request, spider) -> bool: + def _filter(self, request: Any, spider: Spider) -> bool: if not isinstance(request, Request): return True if request.dont_filter or self.should_follow(request, spider): @@ -49,13 +64,13 @@ class OffsiteMiddleware: self.stats.inc_value("offsite/filtered", spider=spider) return False - def should_follow(self, request, spider): + def should_follow(self, request: Request, spider: Spider) -> bool: regex = self.host_regex # hostname can be None for wrong urls (like javascript links) host = urlparse_cached(request).hostname or "" return bool(regex.search(host)) - def get_host_regex(self, spider): + def get_host_regex(self, spider: Spider) -> re.Pattern[str]: """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, "allowed_domains", None) if not allowed_domains: @@ -83,9 +98,9 @@ class OffsiteMiddleware: regex = rf'^(.*\.)?({"|".join(domains)})$' return re.compile(regex) - def spider_opened(self, spider): - self.host_regex = self.get_host_regex(spider) - self.domains_seen = set() + def spider_opened(self, spider: Spider) -> None: + self.host_regex: re.Pattern[str] = self.get_host_regex(spider) + self.domains_seen: Set[str] = set() class URLWarning(Warning): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index fd91e658b..a29e0ebb5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,20 +2,39 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ +from __future__ import annotations + import warnings -from typing import Tuple +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterable, + Dict, + Iterable, + Optional, + Tuple, + Type, + Union, + cast, +) from urllib.parse import urlparse from w3lib.url import safe_url_string -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response +from scrapy.settings import BaseSettings from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url -LOCAL_SCHEMES = ( +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + +LOCAL_SCHEMES: Tuple[str, ...] = ( "about", "blob", "data", @@ -37,18 +56,20 @@ class ReferrerPolicy: NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES name: str - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: raise NotImplementedError() - def stripped_referrer(self, url): + def stripped_referrer(self, url: str) -> Optional[str]: if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: return self.strip_url(url) + return None - def origin_referrer(self, url): + def origin_referrer(self, url: str) -> Optional[str]: if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: return self.origin(url) + return None - def strip_url(self, url, origin_only=False): + def strip_url(self, url: str, origin_only: bool = False) -> Optional[str]: """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -72,18 +93,18 @@ class ReferrerPolicy: origin_only=origin_only, ) - def origin(self, url): + def origin(self, url: str) -> Optional[str]: """Return serialized origin (scheme, host, path) for a request or response URL.""" return self.strip_url(url, origin_only=True) - def potentially_trustworthy(self, url): + def potentially_trustworthy(self, url: str) -> bool: # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy parsed_url = urlparse(url) if parsed_url.scheme in ("data",): return False return self.tls_protected(url) - def tls_protected(self, url): + def tls_protected(self, url: str) -> bool: return urlparse(url).scheme in ("https", "ftps") @@ -98,7 +119,7 @@ class NoReferrerPolicy(ReferrerPolicy): name: str = POLICY_NO_REFERRER - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: return None @@ -119,9 +140,10 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: if not self.tls_protected(response_url) or self.tls_protected(request_url): return self.stripped_referrer(response_url) + return None class SameOriginPolicy(ReferrerPolicy): @@ -137,9 +159,10 @@ class SameOriginPolicy(ReferrerPolicy): name: str = POLICY_SAME_ORIGIN - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: if self.origin(response_url) == self.origin(request_url): return self.stripped_referrer(response_url) + return None class OriginPolicy(ReferrerPolicy): @@ -154,7 +177,7 @@ class OriginPolicy(ReferrerPolicy): name: str = POLICY_ORIGIN - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: return self.origin_referrer(response_url) @@ -174,13 +197,14 @@ class StrictOriginPolicy(ReferrerPolicy): name: str = POLICY_STRICT_ORIGIN - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) or not self.tls_protected(response_url) ): return self.origin_referrer(response_url) + return None class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -197,7 +221,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) @@ -224,7 +248,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) @@ -234,6 +258,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): or not self.tls_protected(response_url) ): return self.origin_referrer(response_url) + return None class UnsafeUrlPolicy(ReferrerPolicy): @@ -252,7 +277,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): name: str = POLICY_UNSAFE_URL - def referrer(self, response_url, request_url): + def referrer(self, response_url: str, request_url: str) -> Optional[str]: return self.stripped_referrer(response_url) @@ -267,7 +292,7 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): name: str = POLICY_SCRAPY_DEFAULT -_policy_classes = { +_policy_classes: Dict[str, Type[ReferrerPolicy]] = { p.name: p for p in ( NoReferrerPolicy, @@ -286,14 +311,16 @@ _policy_classes = { _policy_classes[""] = NoReferrerWhenDowngradePolicy -def _load_policy_class(policy, warning_only=False): +def _load_policy_class( + policy: str, warning_only: bool = False +) -> Optional[Type[ReferrerPolicy]]: """ Expect a string for the path to the policy class, otherwise try to interpret the string as a standard value from https://www.w3.org/TR/referrer-policy/#referrer-policies """ try: - return load_object(policy) + return cast(Type[ReferrerPolicy], load_object(policy)) except ValueError: try: return _policy_classes[policy.lower()] @@ -307,13 +334,15 @@ def _load_policy_class(policy, warning_only=False): class RefererMiddleware: - def __init__(self, settings=None): - self.default_policy = DefaultReferrerPolicy + def __init__(self, settings: Optional[BaseSettings] = None): + self.default_policy: Type[ReferrerPolicy] = DefaultReferrerPolicy if settings is not None: - self.default_policy = _load_policy_class(settings.get("REFERRER_POLICY")) + settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) + assert settings_policy + self.default_policy = settings_policy @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("REFERER_ENABLED"): raise NotConfigured mw = cls(crawler.settings) @@ -323,7 +352,9 @@ class RefererMiddleware: return mw - def policy(self, resp_or_url, request): + def policy( + self, resp_or_url: Union[Response, str], request: Request + ) -> ReferrerPolicy: """ Determine Referrer-Policy to use from a parent Response (or URL), and a Request to be sent. @@ -348,21 +379,25 @@ class RefererMiddleware: cls = _load_policy_class(policy_name, warning_only=True) return cls() if cls else self.default_policy() - def process_spider_output(self, response, result, spider): - return (self._set_referer(r, response) for r in result or ()) + def process_spider_output( + self, response: Response, result: Iterable[Any], spider: Spider + ) -> Iterable[Any]: + return (self._set_referer(r, response) for r in result) - async def process_spider_output_async(self, response, result, spider): - async for r in result or (): + async def process_spider_output_async( + self, response: Response, result: AsyncIterable[Any], spider: Spider + ) -> AsyncIterable[Any]: + async for r in result: yield self._set_referer(r, response) - def _set_referer(self, r, response): + def _set_referer(self, r: Any, response: Response) -> Any: if isinstance(r, Request): referrer = self.policy(response, r).referrer(response.url, r.url) if referrer is not None: r.headers.setdefault("Referer", referrer) return r - def request_scheduled(self, request, spider): + def request_scheduled(self, request: Request, spider: Spider) -> None: # check redirected request to patch "Referer" header if necessary redirected_urls = request.meta.get("redirect_urls", []) if redirected_urls: @@ -378,7 +413,7 @@ class RefererMiddleware: policy_referrer = self.policy(parent_url, request).referrer( parent_url, request.url ) - if policy_referrer != request_referrer: + if policy_referrer != request_referrer.decode("latin1"): if policy_referrer is None: request.headers.pop("Referer") else: diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index f6d92e53a..e2aa554a7 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -4,40 +4,54 @@ Url Length Spider Middleware See documentation in docs/topics/spider-middleware.rst """ -import logging +from __future__ import annotations +import logging +from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable + +from scrapy import Spider from scrapy.exceptions import NotConfigured -from scrapy.http import Request +from scrapy.http import Request, Response +from scrapy.settings import BaseSettings + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class UrlLengthMiddleware: - def __init__(self, maxlength): - self.maxlength = maxlength + def __init__(self, maxlength: int): + self.maxlength: int = maxlength @classmethod - def from_settings(cls, settings): + def from_settings(cls, settings: BaseSettings) -> Self: maxlength = settings.getint("URLLENGTH_LIMIT") if not maxlength: raise NotConfigured return cls(maxlength) - def process_spider_output(self, response, result, spider): - return (r for r in result or () if self._filter(r, spider)) + def process_spider_output( + self, response: Response, result: Iterable[Any], spider: Spider + ) -> Iterable[Any]: + return (r for r in result if self._filter(r, spider)) - async def process_spider_output_async(self, response, result, spider): - async for r in result or (): + async def process_spider_output_async( + self, response: Response, result: AsyncIterable[Any], spider: Spider + ) -> AsyncIterable[Any]: + async for r in result: if self._filter(r, spider): yield r - def _filter(self, request, spider): + def _filter(self, request: Any, spider: Spider) -> bool: if isinstance(request, Request) and len(request.url) > self.maxlength: logger.info( "Ignoring link (url length > %(maxlength)d): %(url)s ", {"maxlength": self.maxlength, "url": request.url}, extra={"spider": spider}, ) + assert spider.crawler.stats spider.crawler.stats.inc_value( "urllength/request_ignored_count", spider=spider ) From dda6feb935b77bf4c7e02b5b00ec6fc0fa6f97ee Mon Sep 17 00:00:00 2001 From: Jeesang Kim Date: Fri, 3 Nov 2023 17:24:25 +0900 Subject: [PATCH 1228/2083] Improve assert readability (#6132) --- tests/test_linkextractors.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index e1ec19601..c90065e67 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -235,20 +235,20 @@ class Base: url2 = "http://evenmorestuff.com/uglystuff/index" lx = self.extractor_cls(allow=(r"stuff1",)) - self.assertEqual(lx.matches(url1), True) - self.assertEqual(lx.matches(url2), False) + self.assertTrue(lx.matches(url1)) + self.assertFalse(lx.matches(url2)) lx = self.extractor_cls(deny=(r"uglystuff",)) - self.assertEqual(lx.matches(url1), True) - self.assertEqual(lx.matches(url2), False) + self.assertTrue(lx.matches(url1)) + self.assertFalse(lx.matches(url2)) lx = self.extractor_cls(allow_domains=("evenmorestuff.com",)) - self.assertEqual(lx.matches(url1), False) - self.assertEqual(lx.matches(url2), True) + self.assertFalse(lx.matches(url1)) + self.assertTrue(lx.matches(url2)) lx = self.extractor_cls(deny_domains=("lotsofstuff.com",)) - self.assertEqual(lx.matches(url1), False) - self.assertEqual(lx.matches(url2), True) + self.assertFalse(lx.matches(url1)) + self.assertTrue(lx.matches(url2)) lx = self.extractor_cls( allow=["blah1"], @@ -256,10 +256,10 @@ class Base: allow_domains=["blah1.com"], deny_domains=["blah2.com"], ) - self.assertEqual(lx.matches("http://blah1.com/blah1"), True) - self.assertEqual(lx.matches("http://blah1.com/blah2"), False) - self.assertEqual(lx.matches("http://blah2.com/blah1"), False) - self.assertEqual(lx.matches("http://blah2.com/blah2"), False) + self.assertTrue(lx.matches("http://blah1.com/blah1")) + self.assertFalse(lx.matches("http://blah1.com/blah2")) + self.assertFalse(lx.matches("http://blah2.com/blah1")) + self.assertFalse(lx.matches("http://blah2.com/blah2")) def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',)) From 6587556af9ba92cd1f73ea2ed172a513bf80e6a5 Mon Sep 17 00:00:00 2001 From: Jessica Allman-LaPorte Date: Fri, 3 Nov 2023 05:02:18 -0400 Subject: [PATCH 1229/2083] Make shell switching more clear in the tutorial (#6128) --- docs/intro/tutorial.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 19a76fc16..8ea98f29b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -493,7 +493,15 @@ in the callback, as you can see below: "tags": quote.css("div.tags a.tag::text").getall(), } -If you run this spider, it will output the extracted data with the log:: +To run this spider, exit the scrapy shell by entering:: + + quit() + +Then, run:: + + scrapy crawl quotes + +Now, it should output the extracted data with the log:: 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.â€'} From 2ac3ef73e6208aced93fa538184f1db7f14d125c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 3 Nov 2023 20:12:18 +0400 Subject: [PATCH 1230/2083] Remove obsolete setters for body and url on Request and Response. --- scrapy/http/common.py | 10 ---------- scrapy/http/request/__init__.py | 11 ++++------- scrapy/http/response/__init__.py | 13 +++++-------- scrapy/utils/iterators.py | 4 ++-- 4 files changed, 11 insertions(+), 27 deletions(-) delete mode 100644 scrapy/http/common.py diff --git a/scrapy/http/common.py b/scrapy/http/common.py deleted file mode 100644 index a3d9d5b81..000000000 --- a/scrapy/http/common.py +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Any, Callable, NoReturn - - -def obsolete_setter(setter: Callable, attrname: str) -> Callable[[Any, Any], NoReturn]: - def newsetter(self: Any, value: Any) -> NoReturn: - c = self.__class__.__name__ - msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead" - raise AttributeError(msg) - - return newsetter diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 7c9a4ba95..a1c5a5e51 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -25,7 +25,6 @@ from typing import ( from w3lib.url import safe_url_string import scrapy -from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes @@ -142,7 +141,8 @@ class Request(object_ref): self._meta = {} return self._meta - def _get_url(self) -> str: + @property + def url(self) -> str: return self._url def _set_url(self, url: str) -> None: @@ -159,16 +159,13 @@ class Request(object_ref): ): raise ValueError(f"Missing scheme in request url: {self._url}") - url = property(_get_url, obsolete_setter(_set_url, "url")) - - def _get_body(self) -> bytes: + @property + def body(self) -> bytes: return self._body def _set_body(self, body: Optional[Union[str, bytes]]) -> None: self._body = b"" if body is None else to_bytes(body, self.encoding) - body = property(_get_body, obsolete_setter(_set_body, "body")) - @property def encoding(self) -> str: return self._encoding diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 61010f14f..6eae3e8b3 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -27,7 +27,6 @@ from urllib.parse import urljoin from twisted.internet.ssl import Certificate from scrapy.exceptions import NotSupported -from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers from scrapy.http.request import Request from scrapy.link import Link @@ -102,7 +101,8 @@ class Response(object_ref): "is not tied to any request" ) - def _get_url(self) -> str: + @property + def url(self) -> str: return self._url def _set_url(self, url: str) -> None: @@ -113,9 +113,8 @@ class Response(object_ref): f"{type(self).__name__} url must be str, " f"got {type(url).__name__}" ) - url = property(_get_url, obsolete_setter(_set_url, "url")) - - def _get_body(self) -> bytes: + @property + def body(self) -> bytes: return self._body def _set_body(self, body: Optional[bytes]) -> None: @@ -130,8 +129,6 @@ class Response(object_ref): else: self._body = body - body = property(_get_body, obsolete_setter(_set_body, "body")) - def __repr__(self) -> str: return f"<{self.status} {self.url}>" @@ -149,7 +146,7 @@ class Response(object_ref): def urljoin(self, url: str) -> str: """Join this Response's url with a possible relative url to form an absolute interpretation of the latter.""" - return urljoin(cast(str, self.url), url) + return urljoin(self.url, url) @property def text(self) -> str: diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 03d779afb..55362efdf 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -217,10 +217,10 @@ def _body_or_str( ) if isinstance(obj, Response): if not unicode: - return cast(bytes, obj.body) + return obj.body if isinstance(obj, TextResponse): return obj.text - return cast(bytes, obj.body).decode("utf-8") + return obj.body.decode("utf-8") if isinstance(obj, str): return obj if unicode else obj.encode("utf-8") return obj.decode("utf-8") if unicode else obj From eafe828484d95f7e73a475c1c5d23492616ebc07 Mon Sep 17 00:00:00 2001 From: cakemd Date: Mon, 6 Nov 2023 09:37:18 +0200 Subject: [PATCH 1231/2083] scrapy.utils.data_path type hint change (#6133) --- scrapy/utils/project.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index a2c224b90..de3c8eaf9 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,7 +1,9 @@ import os import warnings from importlib import import_module +from os import PathLike from pathlib import Path +from typing import Union from scrapy.exceptions import NotConfigured from scrapy.settings import Settings @@ -44,7 +46,7 @@ def project_data_dir(project: str = "default") -> str: return str(d) -def data_path(path: str, createdir: bool = False) -> str: +def data_path(path: Union[str, PathLike], createdir: bool = False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. From 7c27c22a987b7bb113d6275bf9646b74afffe552 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 6 Nov 2023 11:52:14 +0400 Subject: [PATCH 1232/2083] Full typing for scrapy/downloadermiddlewares (#6129) --- scrapy/downloadermiddlewares/ajaxcrawl.py | 28 +++++-- scrapy/downloadermiddlewares/cookies.py | 62 +++++++++++---- .../downloadermiddlewares/defaultheaders.py | 21 ++++- .../downloadermiddlewares/downloadtimeout.py | 24 ++++-- scrapy/downloadermiddlewares/httpauth.py | 23 ++++-- scrapy/downloadermiddlewares/httpcache.py | 28 ++++--- .../downloadermiddlewares/httpcompression.py | 27 +++++-- scrapy/downloadermiddlewares/httpproxy.py | 45 +++++++---- scrapy/downloadermiddlewares/redirect.py | 51 ++++++++---- scrapy/downloadermiddlewares/retry.py | 36 ++++++--- scrapy/downloadermiddlewares/robotstxt.py | 77 +++++++++++++------ scrapy/downloadermiddlewares/stats.py | 37 +++++++-- scrapy/downloadermiddlewares/useragent.py | 23 ++++-- scrapy/http/cookies.py | 10 ++- scrapy/robotstxt.py | 17 +++- setup.cfg | 3 - 16 files changed, 373 insertions(+), 139 deletions(-) diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 04ae719de..0e757e4be 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -1,10 +1,20 @@ +from __future__ import annotations + import logging import re +from typing import TYPE_CHECKING, Union from w3lib import html +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import HtmlResponse +from scrapy.http import HtmlResponse, Response +from scrapy.settings import BaseSettings + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) @@ -15,7 +25,7 @@ class AjaxCrawlMiddleware: For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. """ - def __init__(self, settings): + def __init__(self, settings: BaseSettings): if not settings.getbool("AJAXCRAWL_ENABLED"): raise NotConfigured @@ -23,13 +33,15 @@ class AjaxCrawlMiddleware: # middleware parses first 4k. 4k turns out to be insufficient # for this middleware, and parsing 100k could be slow. # We use something in between (32K) by default. - self.lookup_bytes = settings.getint("AJAXCRAWL_MAXSIZE", 32768) + self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if not isinstance(response, HtmlResponse) or response.status != 200: return response @@ -54,7 +66,7 @@ class AjaxCrawlMiddleware: ajax_crawl_request.meta["ajax_crawlable"] = True return ajax_crawl_request - def _has_ajax_crawlable_variant(self, response): + def _has_ajax_crawlable_variant(self, response: Response) -> bool: """ Return True if a page without hash fragment could be "AJAX crawlable" according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. @@ -64,12 +76,12 @@ class AjaxCrawlMiddleware: # XXX: move it to w3lib? -_ajax_crawlable_re = re.compile( +_ajax_crawlable_re: re.Pattern[str] = re.compile( r'' ) -def _has_ajaxcrawlable_meta(text): +def _has_ajaxcrawlable_meta(text: str) -> bool: """ >>> _has_ajaxcrawlable_meta('') True diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 6495157d7..85781efd6 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,21 +1,41 @@ +from __future__ import annotations + import logging from collections import defaultdict +from http.cookiejar import Cookie +from typing import ( + TYPE_CHECKING, + Any, + DefaultDict, + Dict, + Iterable, + Optional, + Sequence, + Union, +) from tldextract import TLDExtract +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) _split_domain = TLDExtract(include_psl_private_domains=True) -def _is_public_domain(domain): +def _is_public_domain(domain: str) -> bool: parts = _split_domain(domain) return not parts.domain @@ -23,23 +43,27 @@ def _is_public_domain(domain): class CookiesMiddleware: """This middleware enables working with sites that need cookies""" - def __init__(self, debug=False): - self.jars = defaultdict(CookieJar) - self.debug = debug + def __init__(self, debug: bool = False): + self.jars: DefaultDict[Any, CookieJar] = defaultdict(CookieJar) + self.debug: bool = debug @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COOKIES_ENABLED"): raise NotConfigured return cls(crawler.settings.getbool("COOKIES_DEBUG")) - def _process_cookies(self, cookies, *, jar, request): + def _process_cookies( + self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request + ) -> None: for cookie in cookies: cookie_domain = cookie.domain if cookie_domain.startswith("."): cookie_domain = cookie_domain[1:] - request_domain = urlparse_cached(request).hostname.lower() + hostname = urlparse_cached(request).hostname + assert hostname is not None + request_domain = hostname.lower() if cookie_domain and _is_public_domain(cookie_domain): if cookie_domain != request_domain: @@ -48,9 +72,11 @@ class CookiesMiddleware: jar.set_cookie_if_ok(cookie, request) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: if request.meta.get("dont_merge_cookies", False): - return + return None cookiejarkey = request.meta.get("cookiejar") jar = self.jars[cookiejarkey] @@ -61,8 +87,11 @@ class CookiesMiddleware: request.headers.pop("Cookie", None) jar.add_cookie_header(request) self._debug_cookie(request, spider) + return None - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if request.meta.get("dont_merge_cookies", False): return response @@ -76,7 +105,7 @@ class CookiesMiddleware: return response - def _debug_cookie(self, request, spider): + def _debug_cookie(self, request: Request, spider: Spider) -> None: if self.debug: cl = [ to_unicode(c, errors="replace") @@ -87,7 +116,7 @@ class CookiesMiddleware: msg = f"Sending cookies to: {request}\n{cookies}" logger.debug(msg, extra={"spider": spider}) - def _debug_set_cookie(self, response, spider): + def _debug_set_cookie(self, response: Response, spider: Spider) -> None: if self.debug: cl = [ to_unicode(c, errors="replace") @@ -98,7 +127,7 @@ class CookiesMiddleware: msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={"spider": spider}) - def _format_cookie(self, cookie, request): + def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]: """ Given a dict consisting of cookie components, return its string representation. Decode from bytes if necessary. @@ -109,7 +138,7 @@ class CookiesMiddleware: if key in ("name", "value"): msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" logger.warning(msg) - return + return None continue if isinstance(cookie[key], (bool, float, int, str)): decoded[key] = str(cookie[key]) @@ -129,12 +158,15 @@ class CookiesMiddleware: cookie_str += f"; {key.capitalize()}={value}" return cookie_str - def _get_request_cookies(self, jar, request): + def _get_request_cookies( + self, jar: CookieJar, request: Request + ) -> Sequence[Cookie]: """ Extract cookies from the Request.cookies attribute """ if not request.cookies: return [] + cookies: Iterable[Dict[str, Any]] if isinstance(request.cookies, dict): cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) else: diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index cdacc7368..8aec37cf1 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -3,19 +3,32 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ +from __future__ import annotations +from typing import TYPE_CHECKING, Iterable, Tuple, Union + +from scrapy import Request, Spider +from scrapy.crawler import Crawler +from scrapy.http import Response from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class DefaultHeadersMiddleware: - def __init__(self, headers): - self._headers = headers + def __init__(self, headers: Iterable[Tuple[str, str]]): + self._headers: Iterable[Tuple[str, str]] = headers @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"]) return cls(headers.items()) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: for k, v in self._headers: request.headers.setdefault(k, v) + return None diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index a926ecf56..1c904c05b 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -3,23 +3,35 @@ Download timeout middleware See documentation in docs/topics/downloader-middleware.rst """ +from __future__ import annotations -from scrapy import signals +from typing import TYPE_CHECKING, Union + +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class DownloadTimeoutMiddleware: - def __init__(self, timeout=180): - self._timeout = timeout + def __init__(self, timeout: float = 180): + self._timeout: float = timeout @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: o = cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT")) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self._timeout = getattr(spider, "download_timeout", self._timeout) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: if self._timeout: request.meta.setdefault("download_timeout", self._timeout) + return None diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 5228db786..63490a37a 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -4,31 +4,44 @@ HTTP basic auth downloader middleware See documentation in docs/topics/downloader-middleware.rst """ +from __future__ import annotations + +from typing import TYPE_CHECKING, Union + from w3lib.http import basic_auth_header -from scrapy import signals +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler +from scrapy.http import Response from scrapy.utils.url import url_is_from_any_domain +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class HttpAuthMiddleware: """Set Basic HTTP Authorization header (http_user and http_pass spider class attributes)""" @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: o = cls() crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: usr = getattr(spider, "http_user", "") pwd = getattr(spider, "http_pass", "") if usr or pwd: self.auth = basic_auth_header(usr, pwd) - self.domain = spider.http_auth_domain + self.domain = spider.http_auth_domain # type: ignore[attr-defined] - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: auth = getattr(self, "auth", None) if auth and b"Authorization" not in request.headers: if not self.domain or url_is_from_any_domain(request.url, [self.domain]): request.headers[b"Authorization"] = auth + return None diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index a521cde7a..971473403 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from email.utils import formatdate -from typing import Optional, Type, TypeVar +from typing import TYPE_CHECKING, Optional, Union from twisted.internet import defer from twisted.internet.error import ( @@ -23,7 +25,9 @@ from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object -HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class HttpCacheMiddleware: @@ -49,9 +53,7 @@ class HttpCacheMiddleware: self.stats = stats @classmethod - def from_crawler( - cls: Type[HttpCacheMiddlewareTV], crawler: Crawler - ) -> HttpCacheMiddlewareTV: + def from_crawler(cls, crawler: Crawler) -> Self: assert crawler.stats o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) @@ -64,7 +66,9 @@ class HttpCacheMiddleware: def spider_closed(self, spider: Spider) -> None: self.storage.close_spider(spider) - def process_request(self, request: Request, spider: Spider) -> Optional[Response]: + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: if request.meta.get("dont_cache", False): return None @@ -74,7 +78,9 @@ class HttpCacheMiddleware: return None # Look for cached response and check if expired - cachedresponse = self.storage.retrieve_response(spider, request) + cachedresponse: Optional[Response] = self.storage.retrieve_response( + spider, request + ) if cachedresponse is None: self.stats.inc_value("httpcache/miss", spider=spider) if self.ignore_missing: @@ -96,7 +102,7 @@ class HttpCacheMiddleware: def process_response( self, request: Request, response: Response, spider: Spider - ) -> Response: + ) -> Union[Request, Response]: if request.meta.get("dont_cache", False): return response @@ -111,7 +117,7 @@ class HttpCacheMiddleware: response.headers["Date"] = formatdate(usegmt=True) # Do not validate first-hand responses - cachedresponse = request.meta.pop("cached_response", None) + cachedresponse: Optional[Response] = request.meta.pop("cached_response", None) if cachedresponse is None: self.stats.inc_value("httpcache/firsthand", spider=spider) self._cache_response(spider, response, request, cachedresponse) @@ -127,8 +133,8 @@ class HttpCacheMiddleware: def process_exception( self, request: Request, exception: Exception, spider: Spider - ) -> Optional[Response]: - cachedresponse = request.meta.pop("cached_response", None) + ) -> Union[Request, Response, None]: + cachedresponse: Optional[Response] = request.meta.pop("cached_response", None) if cachedresponse is not None and isinstance( exception, self.DOWNLOAD_EXCEPTIONS ): diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 7b1d3f829..56a58a750 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,12 +1,22 @@ +from __future__ import annotations + import io import zlib +from typing import TYPE_CHECKING, List, Optional, Union +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes +from scrapy.statscollectors import StatsCollector from scrapy.utils.gz import gunzip -ACCEPTED_ENCODINGS = [b"gzip", b"deflate"] +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + +ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] try: import brotli @@ -27,19 +37,24 @@ class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, stats=None): + def __init__(self, stats: Optional[StatsCollector] = None): self.stats = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured return cls(stats=crawler.stats) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) + return None - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if request.method == "HEAD": return response if isinstance(response, Response): @@ -70,7 +85,7 @@ class HttpCompressionMiddleware: return response - def _decode(self, body, encoding): + def _decode(self, body: bytes, encoding: bytes) -> bytes: if encoding == b"gzip" or encoding == b"x-gzip": body = gunzip(body) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index f74d84b69..335896ac1 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,16 +1,30 @@ -import base64 -from urllib.parse import unquote, urlunparse -from urllib.request import _parse_proxy, getproxies, proxy_bypass +from __future__ import annotations +import base64 +from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union +from urllib.parse import unquote, urlunparse +from urllib.request import ( # type: ignore[attr-defined] + _parse_proxy, + getproxies, + proxy_bypass, +) + +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class HttpProxyMiddleware: - def __init__(self, auth_encoding="latin-1"): - self.auth_encoding = auth_encoding - self.proxies = {} + def __init__(self, auth_encoding: Optional[str] = "latin-1"): + self.auth_encoding: Optional[str] = auth_encoding + self.proxies: Dict[str, Tuple[Optional[bytes], str]] = {} for type_, url in getproxies().items(): try: self.proxies[type_] = self._get_proxy(url, type_) @@ -20,19 +34,19 @@ class HttpProxyMiddleware: continue @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("HTTPPROXY_ENABLED"): raise NotConfigured - auth_encoding = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") + auth_encoding: Optional[str] = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") return cls(auth_encoding) - def _basic_auth_header(self, username, password): + def _basic_auth_header(self, username: str, password: str) -> bytes: user_pass = to_bytes( f"{unquote(username)}:{unquote(password)}", encoding=self.auth_encoding ) return base64.b64encode(user_pass) - def _get_proxy(self, url, orig_type): + def _get_proxy(self, url: str, orig_type: str) -> Tuple[Optional[bytes], str]: proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) @@ -43,7 +57,9 @@ class HttpProxyMiddleware: return creds, proxy_url - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: creds, proxy_url = None, None if "proxy" in request.meta: if request.meta["proxy"] is not None: @@ -54,13 +70,16 @@ class HttpProxyMiddleware: if ( # 'no_proxy' is only supported by http schemes scheme not in ("http", "https") - or not proxy_bypass(parsed.hostname) + or (parsed.hostname and not proxy_bypass(parsed.hostname)) ) and scheme in self.proxies: creds, proxy_url = self.proxies[scheme] self._set_proxy_and_creds(request, proxy_url, creds) + return None - def _set_proxy_and_creds(self, request, proxy_url, creds): + def _set_proxy_and_creds( + self, request: Request, proxy_url: Optional[str], creds: Optional[bytes] + ) -> None: if proxy_url: request.meta["proxy"] = proxy_url elif request.meta.get("proxy") is not None: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 65f1d2224..814b1a561 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,17 +1,29 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Any, List, Union, cast from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http import HtmlResponse +from scrapy.http import HtmlResponse, Response +from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) -def _build_redirect_request(source_request, *, url, **kwargs): +def _build_redirect_request( + source_request: Request, *, url: str, **kwargs: Any +) -> Request: redirect_request = source_request.replace( url=url, **kwargs, @@ -26,20 +38,22 @@ def _build_redirect_request(source_request, *, url, **kwargs): class BaseRedirectMiddleware: - enabled_setting = "REDIRECT_ENABLED" + enabled_setting: str = "REDIRECT_ENABLED" - def __init__(self, settings): + def __init__(self, settings: BaseSettings): if not settings.getbool(self.enabled_setting): raise NotConfigured - self.max_redirect_times = settings.getint("REDIRECT_MAX_TIMES") - self.priority_adjust = settings.getint("REDIRECT_PRIORITY_ADJUST") + self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES") + self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST") @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def _redirect(self, redirected, request, spider, reason): + def _redirect( + self, redirected: Request, request: Request, spider: Spider, reason: Any + ) -> Request: ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times) redirects = request.meta.get("redirect_times", 0) + 1 @@ -67,7 +81,9 @@ class BaseRedirectMiddleware: ) raise IgnoreRequest("max redirections reached") - def _redirect_request_using_get(self, request, redirect_url): + def _redirect_request_using_get( + self, request: Request, redirect_url: str + ) -> Request: redirect_request = _build_redirect_request( request, url=redirect_url, @@ -85,7 +101,9 @@ class RedirectMiddleware(BaseRedirectMiddleware): and meta-refresh html tag. """ - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if ( request.meta.get("dont_redirect", False) or response.status in getattr(spider, "handle_httpstatus_list", []) @@ -98,6 +116,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): if "Location" not in response.headers or response.status not in allowed_status: return response + assert response.headers["Location"] is not None location = safe_url_string(response.headers["Location"]) if response.headers["Location"].startswith(b"//"): request_scheme = urlparse(request.url).scheme @@ -116,12 +135,14 @@ class RedirectMiddleware(BaseRedirectMiddleware): class MetaRefreshMiddleware(BaseRedirectMiddleware): enabled_setting = "METAREFRESH_ENABLED" - def __init__(self, settings): + def __init__(self, settings: BaseSettings): super().__init__(settings) - self._ignore_tags = settings.getlist("METAREFRESH_IGNORE_TAGS") - self._maxdelay = settings.getint("METAREFRESH_MAXDELAY") + self._ignore_tags: List[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") + self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY") - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if ( request.meta.get("dont_redirect", False) or request.method == "HEAD" @@ -130,7 +151,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): return response interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags) - if url and interval < self._maxdelay: + if url and cast(float, interval) < self._maxdelay: redirected = self._redirect_request_using_get(request, url) return self._redirect(redirected, request, spider, "meta refresh") diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 380623cea..3c494de78 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,22 +9,30 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ +from __future__ import annotations + import warnings from logging import Logger, getLogger -from typing import Optional, Type, Union +from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.http import Response from scrapy.http.request import Request -from scrapy.settings import Settings +from scrapy.settings import BaseSettings, Settings from scrapy.spiders import Spider from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + retry_logger = getLogger(__name__) -def backwards_compatibility_getattr(self, name): +def backwards_compatibility_getattr(self: Any, name: str) -> Tuple[Any, ...]: if name == "EXCEPTIONS_TO_RETRY": warnings.warn( "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " @@ -54,7 +62,7 @@ def get_retry_request( priority_adjust: Optional[int] = None, logger: Logger = retry_logger, stats_base_key: str = "retry", -): +) -> Optional[Request]: """ Returns a new :class:`~scrapy.Request` object to retry the specified request, or ``None`` if retries of the specified request have been @@ -134,7 +142,7 @@ def get_retry_request( class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): - def __init__(self, settings): + def __init__(self, settings: BaseSettings): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured self.max_retry_times = settings.getint("RETRY_TIMES") @@ -153,10 +161,12 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): ) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: if request.meta.get("dont_retry", False): return response if response.status in self.retry_http_codes: @@ -164,13 +174,21 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): return self._retry(request, reason, spider) or response return response - def process_exception(self, request, exception, spider): + def process_exception( + self, request: Request, exception: Exception, spider: Spider + ) -> Union[Request, Response, None]: if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( "dont_retry", False ): return self._retry(request, exception, spider) + return None - def _retry(self, request, reason, spider): + def _retry( + self, + request: Request, + reason: Union[str, Exception, Type[Exception]], + spider: Spider, + ) -> Optional[Request]: max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) priority_adjust = request.meta.get("priority_adjust", self.priority_adjust) return get_retry_request( diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6cab27c5a..6a0ecb7bf 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -4,65 +4,87 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. """ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Any, Dict, Optional, Union from twisted.internet.defer import Deferred, maybeDeferred +from twisted.python.failure import Failure +from scrapy import Spider +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK +from scrapy.robotstxt import RobotParser from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) class RobotsTxtMiddleware: - DOWNLOAD_PRIORITY = 1000 + DOWNLOAD_PRIORITY: int = 1000 - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured - self._default_useragent = crawler.settings.get("USER_AGENT", "Scrapy") - self._robotstxt_useragent = crawler.settings.get("ROBOTSTXT_USER_AGENT", None) - self.crawler = crawler - self._parsers = {} - self._parserimpl = load_object(crawler.settings.get("ROBOTSTXT_PARSER")) + self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy") + self._robotstxt_useragent: Optional[str] = crawler.settings.get( + "ROBOTSTXT_USER_AGENT", None + ) + self.crawler: Crawler = crawler + self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {} + self._parserimpl: RobotParser = load_object( + crawler.settings.get("ROBOTSTXT_PARSER") + ) # check if parser dependencies are met, this should throw an error otherwise. self._parserimpl.from_crawler(self.crawler, b"") @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def process_request(self, request, spider): + def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]: if request.meta.get("dont_obey_robotstxt"): - return + return None if request.url.startswith("data:") or request.url.startswith("file:"): - return - d = maybeDeferred(self.robot_parser, request, spider) + return None + d: Deferred = maybeDeferred(self.robot_parser, request, spider) d.addCallback(self.process_request_2, request, spider) return d - def process_request_2(self, rp, request, spider): + def process_request_2( + self, rp: Optional[RobotParser], request: Request, spider: Spider + ) -> None: if rp is None: return - useragent = self._robotstxt_useragent + useragent: Union[str, bytes, None] = self._robotstxt_useragent if not useragent: useragent = request.headers.get(b"User-Agent", self._default_useragent) + assert useragent is not None if not rp.allowed(request.url, useragent): logger.debug( "Forbidden by robots.txt: %(request)s", {"request": request}, extra={"spider": spider}, ) + assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/forbidden") raise IgnoreRequest("Forbidden by robots.txt") - def robot_parser(self, request, spider): + def robot_parser( + self, request: Request, spider: Spider + ) -> Union[RobotParser, Deferred, None]: url = urlparse_cached(request) netloc = url.netloc @@ -75,24 +97,27 @@ class RobotsTxtMiddleware: meta={"dont_obey_robotstxt": True}, callback=NO_CALLBACK, ) + assert self.crawler.engine + assert self.crawler.stats dfd = self.crawler.engine.download(robotsreq) dfd.addCallback(self._parse_robots, netloc, spider) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) self.crawler.stats.inc_value("robotstxt/request_count") - if isinstance(self._parsers[netloc], Deferred): - d = Deferred() + parser = self._parsers[netloc] + if isinstance(parser, Deferred): + d: Deferred = Deferred() - def cb(result): + def cb(result: Any) -> Any: d.callback(result) return result - self._parsers[netloc].addCallback(cb) + parser.addCallback(cb) return d - return self._parsers[netloc] + return parser - def _logerror(self, failure, request, spider): + def _logerror(self, failure: Failure, request: Request, spider: Spider) -> Failure: if failure.type is not IgnoreRequest: logger.error( "Error downloading %(request)s: %(f_exception)s", @@ -102,20 +127,24 @@ class RobotsTxtMiddleware: ) return failure - def _parse_robots(self, response, netloc, spider): + def _parse_robots(self, response: Response, netloc: str, spider: Spider) -> None: + assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/response_count") self.crawler.stats.inc_value( f"robotstxt/response_status_count/{response.status}" ) rp = self._parserimpl.from_crawler(self.crawler, response.body) rp_dfd = self._parsers[netloc] + assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = rp rp_dfd.callback(rp) - def _robots_error(self, failure, netloc): + def _robots_error(self, failure: Failure, netloc: str) -> None: if failure.type is not IgnoreRequest: key = f"robotstxt/exception_count/{failure.type}" + assert self.crawler.stats self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] + assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = None rp_dfd.callback(None) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index a0f62e262..df30e8ca4 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,11 +1,23 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Dict, Union + from twisted.web import http +from scrapy import Request, Spider +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response +from scrapy.statscollectors import StatsCollector from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self -def get_header_size(headers): + +def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int: size = 0 for key, value in headers.items(): if isinstance(value, (list, tuple)): @@ -14,30 +26,36 @@ def get_header_size(headers): return size + len(b"\r\n") * (len(headers.keys()) - 1) -def get_status_size(response_status): +def get_status_size(response_status: int) -> int: return len(to_bytes(http.RESPONSES.get(response_status, b""))) + 15 # resp.status + b"\r\n" + b"HTTP/1.1 <100-599> " class DownloaderStats: - def __init__(self, stats): - self.stats = stats + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("DOWNLOADER_STATS"): raise NotConfigured + assert crawler.stats return cls(crawler.stats) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: self.stats.inc_value("downloader/request_count", spider=spider) self.stats.inc_value( f"downloader/request_method_count/{request.method}", spider=spider ) reqlen = len(request_httprepr(request)) self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider) + return None - def process_response(self, request, response, spider): + def process_response( + self, request: Request, response: Response, spider: Spider + ) -> Union[Request, Response]: self.stats.inc_value("downloader/response_count", spider=spider) self.stats.inc_value( f"downloader/response_status_count/{response.status}", spider=spider @@ -52,9 +70,12 @@ class DownloaderStats: self.stats.inc_value("downloader/response_bytes", reslen, spider=spider) return response - def process_exception(self, request, exception, spider): + def process_exception( + self, request: Request, exception: Exception, spider: Spider + ) -> Union[Request, Response, None]: ex_class = global_object_name(exception.__class__) self.stats.inc_value("downloader/exception_count", spider=spider) self.stats.inc_value( f"downloader/exception_type_count/{ex_class}", spider=spider ) + return None diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 856a275ab..92f1ec897 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -1,23 +1,36 @@ """Set User-Agent header per spider or use a default value from settings""" -from scrapy import signals +from __future__ import annotations + +from typing import TYPE_CHECKING, Union + +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class UserAgentMiddleware: """This middleware allows spiders to override the user_agent""" - def __init__(self, user_agent="Scrapy"): + def __init__(self, user_agent: str = "Scrapy"): self.user_agent = user_agent @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: o = cls(crawler.settings["USER_AGENT"]) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self.user_agent = getattr(spider, "user_agent", self.user_agent) - def process_request(self, request, spider): + def process_request( + self, request: Request, spider: Spider + ) -> Union[Request, Response, None]: if self.user_agent: request.headers.setdefault(b"User-Agent", self.user_agent) + return None diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index a5329ad51..2595f328e 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,8 +1,12 @@ import re import time +from http.cookiejar import Cookie from http.cookiejar import CookieJar as _CookieJar from http.cookiejar import DefaultCookiePolicy +from typing import Sequence +from scrapy import Request +from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -24,7 +28,7 @@ class CookieJar: wrsp = WrappedResponse(response) return self.jar.extract_cookies(wrsp, wreq) - def add_cookie_header(self, request): + def add_cookie_header(self, request: Request) -> None: wreq = WrappedRequest(request) self.policy._now = self.jar._now = int(time.time()) @@ -75,7 +79,7 @@ class CookieJar: def set_policy(self, pol): return self.jar.set_policy(pol) - def make_cookies(self, response, request): + def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]: wreq = WrappedRequest(request) wrsp = WrappedResponse(response) return self.jar.make_cookies(wrsp, wreq) @@ -83,7 +87,7 @@ class CookieJar: def set_cookie(self, cookie): self.jar.set_cookie(cookie) - def set_cookie_if_ok(self, cookie, request): + def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None: self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 5c5ac4e41..6ea2bfd97 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -1,11 +1,20 @@ +from __future__ import annotations + import logging import sys from abc import ABCMeta, abstractmethod +from typing import TYPE_CHECKING, Union from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) @@ -31,7 +40,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): class RobotParser(metaclass=ABCMeta): @classmethod @abstractmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: """Parse the content of a robots.txt_ file as bytes. This must be a class method. It must return a new instance of the parser backend. @@ -44,14 +53,14 @@ class RobotParser(metaclass=ABCMeta): pass @abstractmethod - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. :param url: Absolute URL - :type url: str + :type url: str or bytes :param user_agent: User agent - :type user_agent: str + :type user_agent: str or bytes """ pass diff --git a/setup.cfg b/setup.cfg index db79c5821..d23549f10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,9 +16,6 @@ follow_imports = skip # FIXME: remove the following sections once the issues are solved -[mypy-scrapy.downloadermiddlewares.httpproxy] -ignore_errors = True - [mypy-scrapy.interfaces] ignore_errors = True From b4acf5c827b710fac83fa09fc5832ac76041e5c8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 7 Nov 2023 12:34:35 +0400 Subject: [PATCH 1233/2083] Fix and remove most of the entries from the mypy ignore list (#6137) --- scrapy/contracts/__init__.py | 4 +- scrapy/loader/__init__.py | 2 +- scrapy/pipelines/images.py | 3 +- scrapy/utils/testproc.py | 2 +- setup.cfg | 73 ++---------------------- tests/mocks/dummydbm.py | 3 +- tests/test_exporters.py | 9 +-- tests/test_http_request.py | 5 +- tests/test_linkextractors.py | 3 +- tests/test_loader.py | 3 +- tests/test_pipeline_crawl.py | 4 +- tests/test_pipeline_files.py | 13 ++--- tests/test_pipeline_images.py | 16 +++--- tests/test_request_cb_kwargs.py | 4 +- tests/test_scheduler.py | 5 +- tests/test_spidermiddleware_httperror.py | 3 +- tests/test_spidermiddleware_referer.py | 31 +++++----- tests/test_utils_serialize.py | 2 +- tests/test_utils_url.py | 4 +- 19 files changed, 67 insertions(+), 122 deletions(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 2d9ddd89a..d46eb7c51 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -3,7 +3,7 @@ import sys from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import AsyncGenerator, Dict +from typing import AsyncGenerator, Dict, Optional, Type from unittest import TestCase from scrapy.http import Request @@ -14,7 +14,7 @@ from scrapy.utils.spider import iterate_spider_output class Contract: """Abstract class for contracts""" - request_cls = None + request_cls: Optional[Type[Request]] = None def __init__(self, method, *args): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 91337b949..1042a3d48 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -78,7 +78,7 @@ class ItemLoader(itemloaders.ItemLoader): read-only. """ - default_item_class = Item + default_item_class: type = Item default_selector_class = Selector def __init__(self, item=None, selector=None, response=None, parent=None, **context): diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 9d18144ee..1bd9832a8 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -8,6 +8,7 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO +from typing import Dict, Tuple from itemadapter import ItemAdapter @@ -48,7 +49,7 @@ class ImagesPipeline(FilesPipeline): MIN_WIDTH = 0 MIN_HEIGHT = 0 EXPIRES = 90 - THUMBS = {} + THUMBS: Dict[str, Tuple[int, int]] = {} DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 0688e014b..3bdffcaa7 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -11,7 +11,7 @@ from twisted.python.failure import Failure class ProcessTest: - command = None + command: Optional[str] = None prefix = [sys.executable, "-m", "scrapy.cmdline"] cwd = os.getcwd() # trial chdirs to temp dir diff --git a/setup.cfg b/setup.cfg index d23549f10..151e784c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,82 +8,17 @@ universal=1 ignore_missing_imports = true # Interface classes are hard to support + [mypy-twisted.internet.interfaces] follow_imports = skip +[mypy-scrapy.interfaces] +ignore_errors = True + [mypy-twisted.internet.reactor] follow_imports = skip # FIXME: remove the following sections once the issues are solved -[mypy-scrapy.interfaces] -ignore_errors = True - -[mypy-scrapy.pipelines.images] -ignore_errors = True - [mypy-scrapy.settings.default_settings] ignore_errors = True - -[mypy-tests.mocks.dummydbm] -ignore_errors = True - -[mypy-tests.test_command_fetch] -ignore_errors = True - -[mypy-tests.test_command_parse] -ignore_errors = True - -[mypy-tests.test_command_shell] -ignore_errors = True - -[mypy-tests.test_command_version] -ignore_errors = True - -[mypy-tests.test_contracts] -ignore_errors = True - -[mypy-tests.test_downloader_handlers] -ignore_errors = True - -[mypy-tests.test_exporters] -ignore_errors = True - -[mypy-tests.test_http_request] -ignore_errors = True - -[mypy-tests.test_linkextractors] -ignore_errors = True - -[mypy-tests.test_loader] -ignore_errors = True - -[mypy-tests.test_loader_deprecated] -ignore_errors = True - -[mypy-tests.test_pipeline_crawl] -ignore_errors = True - -[mypy-tests.test_pipeline_files] -ignore_errors = True - -[mypy-tests.test_pipeline_images] -ignore_errors = True - -[mypy-tests.test_request_cb_kwargs] -ignore_errors = True - -[mypy-tests.test_scheduler] -ignore_errors = True - -[mypy-tests.test_spidermiddleware_httperror] -ignore_errors = True - -[mypy-tests.test_spidermiddleware_referer] -ignore_errors = True - -[mypy-tests.test_utils_serialize] -ignore_errors = True - -[mypy-tests.test_utils_url] -ignore_errors = True diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index e77f53666..2869ff8f7 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -1,5 +1,6 @@ """DBM-like dummy module""" import collections +from typing import Any, DefaultDict class DummyDB(dict): @@ -12,7 +13,7 @@ class DummyDB(dict): error = KeyError -_DATABASES = collections.defaultdict(DummyDB) +_DATABASES: DefaultDict[Any, DummyDB] = collections.defaultdict(DummyDB) def open(file, flag="r", mode=0o666): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index f4e82705a..c11913365 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -7,6 +7,7 @@ import tempfile import unittest from datetime import datetime from io import BytesIO +from typing import Any import lxml.etree from itemadapter import ItemAdapter @@ -53,8 +54,8 @@ class CustomFieldDataclass: class BaseItemExporterTest(unittest.TestCase): - item_class = TestItem - custom_field_item_class = CustomFieldItem + item_class: type = TestItem + custom_field_item_class: type = CustomFieldItem def setUp(self): self.i = self.item_class(name="John\xa3", age="22") @@ -517,7 +518,7 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest): class JsonLinesItemExporterTest(BaseItemExporterTest): - _expected_nested = { + _expected_nested: Any = { "name": "Jesus", "age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}}, } @@ -665,7 +666,7 @@ class JsonItemExporterDataclassTest(JsonItemExporterTest): class CustomExporterItemTest(unittest.TestCase): - item_class = TestItem + item_class: type = TestItem def setUp(self): if self.item_class is None: diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 2bd68f846..6dc9ec8b7 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,6 +3,7 @@ import re import unittest import warnings import xmlrpc.client +from typing import Any, Dict, List from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes, urlparse @@ -21,8 +22,8 @@ from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): request_class = Request default_method = "GET" - default_headers = {} - default_meta = {} + default_headers: Dict[bytes, List[bytes]] = {} + default_meta: Dict[str, Any] = {} def test_init(self): # Request requires url in the __init__ method diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index c90065e67..18e9608c1 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,6 +1,7 @@ import pickle import re import unittest +from typing import Optional from packaging.version import Version from pytest import mark @@ -15,7 +16,7 @@ from tests import get_testdata # a hack to skip base class tests in pytest class Base: class LinkExtractorTestCase(unittest.TestCase): - extractor_cls = None + extractor_cls: Optional[type] = None def setUp(self): body = get_testdata("link_extractor", "linkextractor.html") diff --git a/tests/test_loader.py b/tests/test_loader.py index 0dc2de172..b0b7f8723 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,5 +1,6 @@ import dataclasses import unittest +from typing import Optional import attr from itemadapter import ItemAdapter @@ -87,7 +88,7 @@ class BasicItemLoaderTest(unittest.TestCase): class InitializationTestMixin: - item_class = None + item_class: Optional[type] = None def test_keep_single_value(self): """Loaded item should contain values from the initial item""" diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index cb9464949..ed8483483 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,5 +1,6 @@ import shutil from pathlib import Path +from typing import Optional, Set from testfixtures import LogCapture from twisted.internet import defer @@ -54,7 +55,7 @@ class FileDownloadCrawlTestCase(TestCase): store_setting_key = "FILES_STORE" media_key = "files" media_urls_key = "file_urls" - expected_checksums = { + expected_checksums: Optional[Set[str]] = { "5547178b89448faf0015a13f904c936e", "c2281c83670e31d8aaab7cb642b824db", "ed3f6538dc15d4d9179dae57319edc5f", @@ -193,6 +194,7 @@ class FileDownloadCrawlTestCase(TestCase): ) +skip_pillow: Optional[str] try: from PIL import Image # noqa: imported just to check for the import error except ImportError: diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 468751446..e7000e314 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -7,6 +7,7 @@ from io import BytesIO from pathlib import Path from shutil import rmtree from tempfile import mkdtemp +from typing import Dict, List from unittest import mock from urllib.parse import urlparse @@ -308,11 +309,11 @@ class FilesPipelineTestCaseFieldsDataClass( class FilesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - file_urls = attr.ib(default=lambda: []) - files = attr.ib(default=lambda: []) + file_urls: List[str] = attr.ib(default=lambda: []) + files: List[Dict[str, str]] = attr.ib(default=lambda: []) # overridden fields - custom_file_urls = attr.ib(default=lambda: []) - custom_files = attr.ib(default=lambda: []) + custom_file_urls: List[str] = attr.ib(default=lambda: []) + custom_files: List[Dict[str, str]] = attr.ib(default=lambda: []) class FilesPipelineTestCaseFieldsAttrsItem( @@ -690,7 +691,3 @@ def _prepare_request_object(item_url, flags=None): item_url, meta={"response": Response(item_url, status=200, body=b"data", flags=flags)}, ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 8924875d1..2e2e06b89 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -5,6 +5,7 @@ import random import warnings from shutil import rmtree from tempfile import mkdtemp +from typing import Dict, List, Optional from unittest.mock import patch import attr @@ -18,6 +19,7 @@ from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop from scrapy.settings import Settings from scrapy.utils.python import to_bytes +skip_pillow: Optional[str] try: from PIL import Image except ImportError: @@ -26,7 +28,7 @@ except ImportError: ) else: encoders = {"jpeg_encoder", "jpeg_decoder"} - if not encoders.issubset(set(Image.core.__dict__)): + if not encoders.issubset(set(Image.core.__dict__)): # type: ignore[attr-defined] skip_pillow = "Missing JPEG encoders" else: skip_pillow = None @@ -404,11 +406,11 @@ class ImagesPipelineTestCaseFieldsDataClass( class ImagesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - image_urls = attr.ib(default=lambda: []) - images = attr.ib(default=lambda: []) + image_urls: List[str] = attr.ib(default=lambda: []) + images: List[Dict[str, str]] = attr.ib(default=lambda: []) # overridden fields - custom_image_urls = attr.ib(default=lambda: []) - custom_images = attr.ib(default=lambda: []) + custom_image_urls: List[str] = attr.ib(default=lambda: []) + custom_images: List[Dict[str, str]] = attr.ib(default=lambda: []) class ImagesPipelineTestCaseFieldsAttrsItem( @@ -646,7 +648,3 @@ def _create_image(format, *a, **kw): Image.new(*a, **kw).save(buf, format) buf.seek(0) return Image.open(buf), buf - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index adce6cda7..7299972f6 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,3 +1,5 @@ +from typing import List + from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -62,7 +64,7 @@ class KeywordArgumentsSpider(MockServerSpider): }, } - checks = [] + checks: List[bool] = [] def start_requests(self): data = {"key": "value", "number": 123, "callback": "some_callback"} diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index ef9b360c4..f8465a5ff 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2,6 +2,7 @@ import collections import shutil import tempfile import unittest +from typing import Optional from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -59,7 +60,7 @@ class MockCrawler(Crawler): class SchedulerHandler: - priority_queue_cls = None + priority_queue_cls: Optional[str] = None jobdir = None def create_scheduler(self): @@ -253,7 +254,7 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): class DownloaderAwareSchedulerTestMixin: - priority_queue_cls = "scrapy.pqueues.DownloaderAwarePriorityQueue" + priority_queue_cls: Optional[str] = "scrapy.pqueues.DownloaderAwarePriorityQueue" reopen = False def test_logic(self): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 1d5a887cc..044455415 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,4 +1,5 @@ import logging +from typing import Set from unittest import TestCase from testfixtures import LogCapture @@ -16,7 +17,7 @@ from tests.spiders import MockServerSpider class _HttpErrorSpider(MockServerSpider): name = "httperror" - bypass_status_codes = set() + bypass_status_codes: Set[int] = set() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 2f0dda269..afffa87fb 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,4 +1,5 @@ import warnings +from typing import Any, Dict, List, Optional, Tuple from unittest import TestCase from urllib.parse import urlparse @@ -31,10 +32,10 @@ from scrapy.spiders import Spider class TestRefererMiddleware(TestCase): - req_meta = {} - resp_headers = {} - settings = {} - scenarii = [ + req_meta: Dict[str, Any] = {} + resp_headers: Dict[str, str] = {} + settings: Dict[str, Any] = {} + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] @@ -64,7 +65,7 @@ class MixinDefault: with some additional filtering of s3:// """ - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ ("https://example.com/", "https://scrapy.org/", b"https://example.com/"), ("http://example.com/", "http://scrapy.org/", b"http://example.com/"), ("http://example.com/", "https://scrapy.org/", b"http://example.com/"), @@ -85,7 +86,7 @@ class MixinDefault: class MixinNoReferrer: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ ("https://example.com/page.html", "https://example.com/", None), ("http://www.example.com/", "https://scrapy.org/", None), ("http://www.example.com/", "http://scrapy.org/", None), @@ -95,7 +96,7 @@ class MixinNoReferrer: class MixinNoReferrerWhenDowngrade: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # TLS to TLS: send non-empty referrer ( "https://example.com/page.html", @@ -177,7 +178,7 @@ class MixinNoReferrerWhenDowngrade: class MixinSameOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -246,7 +247,7 @@ class MixinSameOrigin: class MixinOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) ( "https://example.com/page.html", @@ -270,7 +271,7 @@ class MixinOrigin: class MixinStrictOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades ( "https://example.com/page.html", @@ -298,7 +299,7 @@ class MixinStrictOrigin: class MixinOriginWhenCrossOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -405,7 +406,7 @@ class MixinOriginWhenCrossOrigin: class MixinStrictOriginWhenCrossOrigin: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -517,7 +518,7 @@ class MixinStrictOriginWhenCrossOrigin: class MixinUnsafeUrl: - scenarii = [ + scenarii: List[Tuple[str, str, Optional[bytes]]] = [ # TLS to TLS: send referrer ( "https://example.com/sekrit.html", @@ -920,7 +921,9 @@ class TestPolicyHeaderPrecedence004( class TestReferrerOnRedirect(TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} - scenarii = [ + scenarii: List[ + Tuple[str, str, Tuple[Tuple[int, str], ...], Optional[bytes], Optional[bytes]] + ] = [ # type: ignore[assignment] ( "http://scrapytest.org/1", # parent "http://scrapytest.org/2", # target diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index 5cdcc7f7c..055db4e5b 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -58,7 +58,7 @@ class JsonEncoderTestCase(unittest.TestCase): self.assertIn(r.url, rs) self.assertIn(str(r.status), rs) - def test_encode_dataclass_item(self): + def test_encode_dataclass_item(self) -> None: @dataclasses.dataclass class TestDataClass: name: str diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 65522f0fd..59a95b0e2 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -364,7 +364,7 @@ for k, args in enumerate( setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation -for k, args in enumerate( +for k, skip_args in enumerate( [ ( r"C:\absolute\path\to\a\file.html", @@ -374,7 +374,7 @@ for k, args in enumerate( ], start=1, ): - t_method = create_skipped_scheme_t(args) + t_method = create_skipped_scheme_t(skip_args) t_method.__name__ = f"test_uri_skipped_{k:03}" setattr(GuessSchemeTest, t_method.__name__, t_method) From a6cee787dd45fabba3f39dbb1752baeef649f5b7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 11 Nov 2023 20:00:12 +0400 Subject: [PATCH 1234/2083] Improve type hints for copy() and replace() in Request and Response. --- .../downloadermiddlewares/httpcompression.py | 6 +-- scrapy/downloadermiddlewares/redirect.py | 1 + scrapy/http/request/__init__.py | 37 +++++++++++++++---- scrapy/http/request/json_request.py | 26 +++++++++++-- scrapy/http/response/__init__.py | 31 +++++++++++++--- 5 files changed, 81 insertions(+), 20 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 56a58a750..d44eb933a 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -2,7 +2,7 @@ from __future__ import annotations import io import zlib -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from scrapy import Request, Spider from scrapy.crawler import Crawler @@ -74,12 +74,12 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs = dict(cls=respcls, body=decoded_body) + kwargs: Dict[str, Any] = dict(body=decoded_body) if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable kwargs["encoding"] = None - response = response.replace(**kwargs) + response = response.replace(cls=respcls, **kwargs) if not content_encoding: del response.headers["Content-Encoding"] diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 814b1a561..7b1401ac8 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -27,6 +27,7 @@ def _build_redirect_request( redirect_request = source_request.replace( url=url, **kwargs, + cls=None, cookies=None, ) if "Cookie" in redirect_request.headers: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a1c5a5e51..4effc2178 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,8 +4,11 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +from __future__ import annotations + import inspect from typing import ( + TYPE_CHECKING, Any, AnyStr, Callable, @@ -19,7 +22,7 @@ from typing import ( Type, TypeVar, Union, - cast, + overload, ) from w3lib.url import safe_url_string @@ -31,6 +34,11 @@ from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") @@ -173,23 +181,36 @@ class Request(object_ref): def __repr__(self) -> str: return f"<{self.method} {self.url}>" - def copy(self) -> "Request": + def copy(self) -> Self: return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> "Request": + @overload + def replace( + self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + ) -> RequestTypeVar: + ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: + ... + + def replace( + self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + ) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) - cls = kwargs.pop("cls", self.__class__) - return cast(Request, cls(*args, **kwargs)) + if cls is None: + cls = self.__class__ + return cls(*args, **kwargs) @classmethod def from_curl( - cls: Type[RequestTypeVar], + cls, curl_command: str, ignore_unknown_options: bool = True, **kwargs: Any, - ) -> RequestTypeVar: + ) -> Self: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same @@ -221,7 +242,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]: + def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 1dd9e6c87..5c09835e4 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -5,12 +5,18 @@ This module implements the JsonRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +from __future__ import annotations + import copy import json import warnings -from typing import Any, Optional, Tuple +from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, overload -from scrapy.http.request import Request +from scrapy.http.request import Request, RequestTypeVar + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class JsonRequest(Request): @@ -44,7 +50,19 @@ class JsonRequest(Request): def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args: Any, **kwargs: Any) -> Request: + @overload + def replace( + self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + ) -> RequestTypeVar: + ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: + ... + + def replace( + self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + ) -> Request: body_passed = kwargs.get("body", None) is not None data = kwargs.pop("data", None) data_passed = data is not None @@ -54,7 +72,7 @@ class JsonRequest(Request): elif not body_passed and data_passed: kwargs["body"] = self._dumps(data) - return super().replace(*args, **kwargs) + return super().replace(*args, cls=cls, **kwargs) def _dumps(self, data: dict) -> str: """Convert to JSON""" diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 6eae3e8b3..e889a6460 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -19,8 +19,10 @@ from typing import ( Mapping, Optional, Tuple, + Type, + TypeVar, Union, - cast, + overload, ) from urllib.parse import urljoin @@ -33,9 +35,15 @@ from scrapy.link import Link from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + from scrapy.selector import SelectorList +ResponseTypeVar = TypeVar("ResponseTypeVar", bound="Response") + + class Response(object_ref): """An object that represents an HTTP response, which is usually downloaded (by the Downloader) and fed to the Spiders for processing. @@ -132,16 +140,29 @@ class Response(object_ref): def __repr__(self) -> str: return f"<{self.status} {self.url}>" - def copy(self) -> Response: + def copy(self) -> Self: """Return a copy of this Response""" return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> Response: + @overload + def replace( + self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any + ) -> ResponseTypeVar: + ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: + ... + + def replace( + self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any + ) -> Response: """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) - cls = kwargs.pop("cls", self.__class__) - return cast(Response, cls(*args, **kwargs)) + if cls is None: + cls = self.__class__ + return cls(*args, **kwargs) def urljoin(self, url: str) -> str: """Join this Response's url with a possible relative url to form an From 5d55e4f56b77168b961db15e0f03d608fad69e7d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 12 Nov 2023 20:15:06 +0400 Subject: [PATCH 1235/2083] Add mypy tests. --- tests_typing/test_http_request.mypy-testing | 66 ++++++++++++++++++++ tests_typing/test_http_response.mypy-testing | 45 +++++++++++++ tox.ini | 8 +++ 3 files changed, 119 insertions(+) create mode 100644 tests_typing/test_http_request.mypy-testing create mode 100644 tests_typing/test_http_response.mypy-testing diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing new file mode 100644 index 000000000..a306b15fe --- /dev/null +++ b/tests_typing/test_http_request.mypy-testing @@ -0,0 +1,66 @@ +import pytest + +from scrapy import Request +from scrapy.http import JsonRequest + + +class MyRequest(Request): + pass + + +class MyRequest2(Request): + pass + + +@pytest.mark.mypy_testing +def mypy_test_headers(): + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Request("data:,", headers=None) + Request("data:,", headers={}) + Request("data:,", headers=[]) + Request("data:,", headers={"foo": "bar"}) + Request("data:,", headers={b"foo": "bar"}) + Request("data:,", headers={"foo": b"bar"}) + Request("data:,", headers=[("foo", "bar")]) + Request("data:,", headers=[(b"foo", "bar")]) + Request("data:,", headers=[("foo", b"bar")]) + + +@pytest.mark.mypy_testing +def mypy_test_copy(): + req = Request("data:,") + reveal_type(req) # R: scrapy.http.request.Request + req_copy = req.copy() + reveal_type(req_copy) # R: scrapy.http.request.Request + + req = MyRequest("data:,") + reveal_type(req) # R: __main__.MyRequest + req_copy = req.copy() + reveal_type(req_copy) # R: __main__.MyRequest + + +@pytest.mark.mypy_testing +def mypy_test_replace(): + req = Request("data:,") + reveal_type(req) # R: scrapy.http.request.Request + req_copy = req.replace(body=b"a") + reveal_type(req_copy) # R: scrapy.http.request.Request + + req = MyRequest("data:,") + reveal_type(req) # R: __main__.MyRequest + req_copy = req.replace(body=b"a") + reveal_type(req_copy) # R: __main__.MyRequest + req_copy2 = req.replace(body=b"a", cls=MyRequest2) + reveal_type(req_copy2) # R: __main__.MyRequest2 + + +@pytest.mark.mypy_testing +def mypy_test_jsonrequest_copy_replace(): + req = JsonRequest("data:,") + reveal_type(req) # R: scrapy.http.request.json_request.JsonRequest + req_copy = req.copy() + reveal_type(req_copy) # R: scrapy.http.request.json_request.JsonRequest + req_copy = req.replace(body=b"a") + reveal_type(req_copy) # R: scrapy.http.request.json_request.JsonRequest + req_copy_my = req.replace(body=b"a", cls=MyRequest) + reveal_type(req_copy_my) # R: __main__.MyRequest diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing new file mode 100644 index 000000000..66ac6ad1d --- /dev/null +++ b/tests_typing/test_http_response.mypy-testing @@ -0,0 +1,45 @@ +import pytest + +from scrapy.http import HtmlResponse, Response, TextResponse + + +@pytest.mark.mypy_testing +def mypy_test_headers(): + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Response("data:,", headers=None) + Response("data:,", headers={}) + Response("data:,", headers=[]) + Response("data:,", headers={"foo": "bar"}) + Response("data:,", headers={b"foo": "bar"}) + Response("data:,", headers={"foo": b"bar"}) + Response("data:,", headers=[("foo", "bar")]) + Response("data:,", headers=[(b"foo", "bar")]) + Response("data:,", headers=[("foo", b"bar")]) + + +@pytest.mark.mypy_testing +def mypy_test_copy(): + resp = Response("data:,") + reveal_type(resp) # R: scrapy.http.response.Response + resp_copy = resp.copy() + reveal_type(resp_copy) # R: scrapy.http.response.Response + + resp = HtmlResponse("data:,") + reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse + resp_copy = resp.copy() + reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse + + +@pytest.mark.mypy_testing +def mypy_test_replace(): + resp = Response("data:,") + reveal_type(resp) # R: scrapy.http.response.Response + resp_copy = resp.replace(body=b"a") + reveal_type(resp_copy) # R: scrapy.http.response.Response + + resp = HtmlResponse("data:,") + reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse + resp_copy = resp.replace(body=b"a") + reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse + resp_copy2 = resp.replace(body=b"a", cls=TextResponse) + reveal_type(resp_copy2) # R: scrapy.http.response.text.TextResponse diff --git a/tox.ini b/tox.ini index 932c0b805..c3fa54339 100644 --- a/tox.ini +++ b/tox.ini @@ -46,6 +46,14 @@ deps = commands = mypy {posargs: scrapy tests} +[testenv:typing-tests] +deps = + {[testenv]deps} + {[testenv:typing]deps} + pytest-mypy-testing==0.1.1 +commands = + pytest {posargs: tests_typing} + [testenv:pre-commit] basepython = python3 deps = From 204d6e180a7c8bc59f188230fb001339a5a43476 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 12 Nov 2023 20:47:52 +0400 Subject: [PATCH 1236/2083] Enable typing-tests in CI. --- .github/workflows/checks.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d6fc0f6c5..ed1629b67 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -18,6 +18,9 @@ jobs: - python-version: 3.8 env: TOXENV: typing + - python-version: 3.8 + env: + TOXENV: typing-tests - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs From 8776b4a6fb64e87c7baf96ae256e04a09246e360 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 12 Nov 2023 20:52:29 +0400 Subject: [PATCH 1237/2083] Fix env deps for typing-tests. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c3fa54339..21ac4c3ff 100644 --- a/tox.ini +++ b/tox.ini @@ -48,7 +48,7 @@ commands = [testenv:typing-tests] deps = - {[testenv]deps} + -rtests/requirements.txt {[testenv:typing]deps} pytest-mypy-testing==0.1.1 commands = From 492584ec07e2b41f80db86c84215208e04e10d0f Mon Sep 17 00:00:00 2001 From: Kiran <75929997+Kiran1689@users.noreply.github.com> Date: Tue, 14 Nov 2023 00:43:10 +0530 Subject: [PATCH 1238/2083] Updated README.rst (#6144) --- README.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 1918850d6..14adff648 100644 --- a/README.rst +++ b/README.rst @@ -17,9 +17,10 @@ Scrapy :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - :alt: macOS +.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + .. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + .. :alt: macOS + .. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows @@ -41,7 +42,7 @@ Scrapy Overview ======== -Scrapy is a fast high-level web crawling and web scraping framework, used to +Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. @@ -110,4 +111,4 @@ See https://scrapy.org/companies/ for a list. Commercial Support ================== -See https://scrapy.org/support/ for details. +See https://scrapy.org/support/ for details. \ No newline at end of file From db5a73f7bb44704b1751a3d005f53cbcd9846415 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Nov 2023 12:02:39 +0400 Subject: [PATCH 1239/2083] Update the expected mypy output to match the old Python one. --- tests_typing/test_http_request.mypy-testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index a306b15fe..636e6895f 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -14,7 +14,7 @@ class MyRequest2(Request): @pytest.mark.mypy_testing def mypy_test_headers(): - Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" Request("data:,", headers=None) Request("data:,", headers={}) Request("data:,", headers=[]) From ebdea4037a38bb207f90658b9380fda7a2e3e825 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Nov 2023 12:31:31 +0400 Subject: [PATCH 1240/2083] Update another output line. --- tests_typing/test_http_response.mypy-testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index 66ac6ad1d..2e58b4fbc 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -5,7 +5,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse @pytest.mark.mypy_testing def mypy_test_headers(): - Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" Response("data:,", headers=None) Response("data:,", headers={}) Response("data:,", headers=[]) From 5fccf370b87378fe2db6bdd52b98c1e2a951df3b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Nov 2023 15:38:13 +0100 Subject: [PATCH 1241/2083] Update the RTD URL for coverage --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 38ca81932..9ca0f817a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -276,7 +276,7 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { "attrs": ("https://www.attrs.org/en/stable/", None), - "coverage": ("https://coverage.readthedocs.io/en/stable", None), + "coverage": ("https://coverage.readthedocs.io/en/latest", None), "cryptography": ("https://cryptography.io/en/latest/", None), "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), From 080fecd8900b6b1f94e8e143e90338279ba8d6e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Nov 2023 15:39:30 +0100 Subject: [PATCH 1242/2083] Drop the Authorization header on cross-domain redirect --- docs/news.rst | 27 ++++++++++++++++++ scrapy/downloadermiddlewares/redirect.py | 10 +++++-- tests/test_downloadermiddleware_redirect.py | 31 +++++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index fd8fa3ea3..b19ec2e99 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,20 @@ Release notes ============= +.. _release-2.11.1: + +Scrapy 2.11.1 (unreleased) +-------------------------- + +**Security bug fix:** + +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + + .. _release-2.11.0: Scrapy 2.11.0 (2023-09-18) @@ -2869,6 +2883,19 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.4: + +Scrapy 1.8.4 (unreleased) +------------------------- + +**Security bug fix:** + +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + .. _release-1.8.3: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 65f1d2224..3176ed930 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -17,11 +17,17 @@ def _build_redirect_request(source_request, *, url, **kwargs): **kwargs, cookies=None, ) - if "Cookie" in redirect_request.headers: + has_cookie_header = "Cookie" in redirect_request.headers + has_authorization_header = "Authorization" in redirect_request.headers + if has_cookie_header or has_authorization_header: source_request_netloc = urlparse_cached(source_request).netloc redirect_request_netloc = urlparse_cached(redirect_request).netloc if source_request_netloc != redirect_request_netloc: - del redirect_request.headers["Cookie"] + if has_cookie_header: + del redirect_request.headers["Cookie"] + # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name + if has_authorization_header: + del redirect_request.headers["Authorization"] return redirect_request diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index dc15b672c..10b8ca9af 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -247,6 +247,37 @@ class RedirectMiddlewareTest(unittest.TestCase): perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o" self.assertEqual(perc_encoded_utf8_url, req_result.url) + def test_cross_domain_header_dropping(self): + safe_headers = {"A": "B"} + original_request = Request( + "https://example.com", + headers={"Cookie": "a=b", "Authorization": "a", **safe_headers}, + ) + + internal_response = Response( + "https://example.com", + headers={"Location": "https://example.com/a"}, + status=301, + ) + internal_redirect_request = self.mw.process_response( + original_request, internal_response, self.spider + ) + self.assertIsInstance(internal_redirect_request, Request) + self.assertEqual(original_request.headers, internal_redirect_request.headers) + + external_response = Response( + "https://example.com", + headers={"Location": "https://example.org/a"}, + status=301, + ) + external_redirect_request = self.mw.process_response( + original_request, external_response, self.spider + ) + self.assertIsInstance(external_redirect_request, Request) + self.assertEqual( + safe_headers, external_redirect_request.headers.to_unicode_dict() + ) + class MetaRefreshMiddlewareTest(unittest.TestCase): def setUp(self): From 603aa4924afc740f5cc41ca40c8eeca4b17bbe2e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Nov 2023 19:51:04 +0400 Subject: [PATCH 1243/2083] Improve the docs about Crawler attributes and settings initialization. --- docs/news.rst | 6 ++++-- docs/topics/spiders.rst | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 5db37969c..65d9c5181 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -32,8 +32,10 @@ Backward-incompatible changes :meth:`scrapy.crawler.Crawler.__init__` and before the settings are finalized and frozen. This change was needed to allow changing the settings in :meth:`scrapy.Spider.from_crawler`. If you want to access the final - setting values in the spider code as early as possible you can do this in - :meth:`~scrapy.Spider.start_requests`. (:issue:`6038`) + setting values and the initialized :class:`~scrapy.crawler.Crawler` + attributes in the spider code as early as possible you can do this in + :meth:`~scrapy.Spider.start_requests` or in a handler of the + :signal:`engine_started` signal. (:issue:`6038`) - The :meth:`TextResponse.json ` method now requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 20452d558..30677fe74 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -142,8 +142,14 @@ scrapy.Spider method, which is handy if you want to modify them based on arguments. As a consequence, these settings aren't the final values as they can be modified later by e.g. :ref:`add-ons - `. The final settings are available in the - :meth:`start_requests` method and later. + `. For the same reason, most of the + :class:`~scrapy.crawler.Crawler` attributes aren't initialized at + this point. + + The final settings and the initialized + :class:`~scrapy.crawler.Crawler` attributes are available in the + :meth:`start_requests` method, handlers of the + :signal:`engine_started` signal and later. :param crawler: crawler to which the spider will be bound :type crawler: :class:`~scrapy.crawler.Crawler` instance From 75e99c75b3c6219df50546877e02f7bbb37324c3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Nov 2023 19:51:04 +0400 Subject: [PATCH 1244/2083] Improve the docs about Crawler attributes and settings initialization. --- docs/news.rst | 6 ++++-- docs/topics/spiders.rst | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index fd8fa3ea3..0c202639e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -32,8 +32,10 @@ Backward-incompatible changes :meth:`scrapy.crawler.Crawler.__init__` and before the settings are finalized and frozen. This change was needed to allow changing the settings in :meth:`scrapy.Spider.from_crawler`. If you want to access the final - setting values in the spider code as early as possible you can do this in - :meth:`~scrapy.Spider.start_requests`. (:issue:`6038`) + setting values and the initialized :class:`~scrapy.crawler.Crawler` + attributes in the spider code as early as possible you can do this in + :meth:`~scrapy.Spider.start_requests` or in a handler of the + :signal:`engine_started` signal. (:issue:`6038`) - The :meth:`TextResponse.json ` method now requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 20452d558..30677fe74 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -142,8 +142,14 @@ scrapy.Spider method, which is handy if you want to modify them based on arguments. As a consequence, these settings aren't the final values as they can be modified later by e.g. :ref:`add-ons - `. The final settings are available in the - :meth:`start_requests` method and later. + `. For the same reason, most of the + :class:`~scrapy.crawler.Crawler` attributes aren't initialized at + this point. + + The final settings and the initialized + :class:`~scrapy.crawler.Crawler` attributes are available in the + :meth:`start_requests` method, handlers of the + :signal:`engine_started` signal and later. :param crawler: crawler to which the spider will be bound :type crawler: :class:`~scrapy.crawler.Crawler` instance From ffbf943e9d0fed636174fab34b2d957b95ee8800 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 2 Oct 2023 20:40:25 +0400 Subject: [PATCH 1245/2083] Merge pull request #6077 from 11-aryan/11-aryan --- docs/topics/request-response.rst | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 41df51589..adf3d0f4a 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -115,20 +115,9 @@ Request objects cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. - To create a request that does not send stored cookies and does not - store received cookies, set the ``dont_merge_cookies`` key to ``True`` - in :attr:`request.meta `. - - Example of a request that sends manually-defined cookies and ignores - cookie storage: - - .. code-block:: python - - Request( - url="http://www.example.com", - cookies={"currency": "USD", "country": "UY"}, - meta={"dont_merge_cookies": True}, - ) + Note that setting the :reqmeta:`dont_merge_cookies` key to ``True`` in + :attr:`request.meta ` causes custom cookies to be + ignored. For more info see :ref:`cookies-mw`. From 59cfdeaa5c83ca1e65be7220296366c135b7676c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Oct 2023 20:20:12 +0400 Subject: [PATCH 1246/2083] Merge pull request #6083 from wRAR/py3.12-release Adapt to the Python 3.12 final release --- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 20 +++++++++----------- .github/workflows/tests-windows.yml | 8 ++++---- tests/requirements.txt | 6 ++---- tests/test_downloader_handlers.py | 2 +- tests/test_feedexport.py | 3 --- tests/test_pipeline_files.py | 5 ----- 7 files changed, 17 insertions(+), 29 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 3044a1af3..aa9b3851d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 5ff92a571..62b5f123a 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -17,7 +17,10 @@ jobs: - python-version: "3.11" env: TOXENV: py - - python-version: "3.11" + - python-version: "3.12" + env: + TOXENV: py + - python-version: "3.12" env: TOXENV: asyncio - python-version: pypy3.9 @@ -41,22 +44,17 @@ jobs: env: TOXENV: botocore-pinned - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: extra-deps - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: botocore - - python-version: "3.12.0-rc.2" - env: - TOXENV: py - - python-version: "3.12.0-rc.2" + # keep until uvloop supports 3.12 + - python-version: "3.11" env: TOXENV: asyncio - - python-version: "3.12.0-rc.2" - env: - TOXENV: extra-deps steps: - uses: actions/checkout@v3 @@ -67,7 +65,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || contains(matrix.python-version, '3.12.0') + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index c8d1928d7..48e0bea76 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -17,13 +17,13 @@ jobs: - python-version: "3.10" env: TOXENV: py - - python-version: "3.10" - env: - TOXENV: asyncio - python-version: "3.11" env: TOXENV: py - - python-version: "3.11" + - python-version: "3.12" + env: + TOXENV: py + - python-version: "3.12" env: TOXENV: asyncio diff --git a/tests/requirements.txt b/tests/requirements.txt index 3ea7f3333..c07fda2d6 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,7 +1,6 @@ # Tests requirements attrs -# https://github.com/giampaolo/pyftpdlib/issues/560 -pyftpdlib; python_version < "3.12" +pyftpdlib >= 1.5.8 pytest pytest-cov==4.0.0 pytest-xdist @@ -10,8 +9,7 @@ testfixtures # uvloop currently doesn't build on 3.12 uvloop; platform_system != "Windows" and python_version < "3.12" -# bpython requires greenlet which currently doesn't build on 3.12 -bpython; python_version < "3.12" # optional for shell wrapper tests +bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests # 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 57211d97a..f12243e1d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -127,7 +127,7 @@ class FileTestCase(unittest.TestCase): return self.download_request(request, Spider("foo")).addCallback(_test) def test_non_existent(self): - request = Request(f"file://{self.mktemp()}") + request = Request(path_to_file_uri(self.mktemp())) d = self.download_request(request, Spider("foo")) return self.assertFailure(d, OSError) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 6b82974fa..56967c0d5 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -125,9 +125,6 @@ class FileFeedStorageTest(unittest.TestCase): path.unlink() -@pytest.mark.skipif( - sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" -) class FTPFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index bf96f17b6..468751446 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,7 +1,6 @@ import dataclasses import os import random -import sys import time from datetime import datetime from io import BytesIO @@ -12,7 +11,6 @@ from unittest import mock from urllib.parse import urlparse import attr -import pytest from itemadapter import ItemAdapter from twisted.internet import defer from twisted.trial import unittest @@ -648,9 +646,6 @@ class TestGCSFilesStore(unittest.TestCase): store.bucket.get_blob.assert_called_with(expected_blob_path) -@pytest.mark.skipif( - sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" -) class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): From 538192916f496eb21846d797a6feff5c05f501cf Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 17 Oct 2023 17:08:23 +0400 Subject: [PATCH 1247/2083] Merge pull request #6064 from wRAR/signals-proper Refactor installing signals. --- scrapy/crawler.py | 12 +++++++----- scrapy/utils/ossignal.py | 9 +++------ scrapy/utils/testproc.py | 7 ++++--- setup.py | 3 +-- tests/CrawlerProcess/sleeping.py | 24 +++++++++++++++++++++++ tests/requirements.txt | 1 + tests/test_command_shell.py | 26 +++++++++++++++++++++++++ tests/test_crawler.py | 33 ++++++++++++++++++++++++++++++-- 8 files changed, 97 insertions(+), 18 deletions(-) create mode 100644 tests/CrawlerProcess/sleeping.py diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 22fd65be7..6f54e62e9 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -404,8 +404,8 @@ class CrawlerProcess(CrawlerRunner): :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished - :param bool install_signal_handlers: whether to install the shutdown - handlers (default: True) + :param bool install_signal_handlers: whether to install the OS signal + handlers from Twisted and Scrapy (default: True) """ from twisted.internet import reactor @@ -416,15 +416,17 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) - if install_signal_handlers: - install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) reactor.addSystemEventTrigger("before", "shutdown", self.stop) - reactor.run(installSignalHandlers=False) # blocking call + if install_signal_handlers: + reactor.addSystemEventTrigger( + "after", "startup", install_shutdown_handlers, self._signal_shutdown + ) + reactor.run(installSignalHandlers=install_signal_handlers) # blocking call def _graceful_stop_reactor(self) -> Deferred: d = self.stop() diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 2334ea792..db9a71273 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -19,13 +19,10 @@ def install_shutdown_handlers( function: SignalHandlerT, override_sigint: bool = True ) -> None: """Install the given function as a signal handler for all common shutdown - signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the - SIGINT handler won't be install if there is already a handler in place - (e.g. Pdb) + signals (such as SIGINT, SIGTERM, etc). If ``override_sigint`` is ``False`` the + SIGINT handler won't be installed if there is already a handler in place + (e.g. Pdb) """ - from twisted.internet import reactor - - reactor._handleSignals() signal.signal(signal.SIGTERM, function) if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: signal.signal(signal.SIGINT, function) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 5f7a7db14..0688e014b 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,7 +2,7 @@ from __future__ import annotations import os import sys -from typing import Iterable, Optional, Tuple, cast +from typing import Iterable, List, Optional, Tuple, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated @@ -26,14 +26,15 @@ class ProcessTest: env = os.environ.copy() if settings is not None: env["SCRAPY_SETTINGS_MODULE"] = settings + assert self.command cmd = self.prefix + [self.command] + list(args) pp = TestProcessProtocol() - pp.deferred.addBoth(self._process_finished, cmd, check_code) + pp.deferred.addCallback(self._process_finished, cmd, check_code) reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) return pp.deferred def _process_finished( - self, pp: TestProcessProtocol, cmd: str, check_code: bool + self, pp: TestProcessProtocol, cmd: List[str], check_code: bool ) -> Tuple[int, bytes, bytes]: if pp.exitcode and check_code: msg = f"process {cmd} exit with code {pp.exitcode}" diff --git a/setup.py b/setup.py index 47c0af0b0..405633f55 100644 --- a/setup.py +++ b/setup.py @@ -6,8 +6,7 @@ version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() install_requires = [ - # 23.8.0 incompatibility: https://github.com/scrapy/scrapy/issues/6024 - "Twisted>=18.9.0,<23.8.0", + "Twisted>=18.9.0", "cryptography>=36.0.0", "cssselect>=0.9.1", "itemloaders>=1.0.1", diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py new file mode 100644 index 000000000..420d9d328 --- /dev/null +++ b/tests/CrawlerProcess/sleeping.py @@ -0,0 +1,24 @@ +from twisted.internet.defer import Deferred + +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.utils.defer import maybe_deferred_to_future + + +class SleepingSpider(scrapy.Spider): + name = "sleeping" + + start_urls = ["data:,;"] + + async def parse(self, response): + from twisted.internet import reactor + + d = Deferred() + reactor.callLater(3, d.callback, None) + await maybe_deferred_to_future(d) + + +process = CrawlerProcess(settings={}) + +process.crawl(SleepingSpider) +process.start() diff --git a/tests/requirements.txt b/tests/requirements.txt index c07fda2d6..d4bfead40 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,6 @@ # Tests requirements attrs +pexpect >= 4.8.0 pyftpdlib >= 1.5.8 pytest pytest-cov==4.0.0 diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 6589381f3..7d87eb62c 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,11 +1,15 @@ +import sys +from io import BytesIO from pathlib import Path +from pexpect.popen_spawn import PopenSpawn from twisted.internet import defer from twisted.trial import unittest from scrapy.utils.testproc import ProcessTest from scrapy.utils.testsite import SiteTest from tests import NON_EXISTING_RESOLVABLE, tests_datadir +from tests.mockserver import MockServer class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @@ -133,3 +137,25 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] _, _, err = yield self.execute(args, check_code=True) self.assertNotIn(b"RuntimeError: There is no current event loop in thread", err) + + +class InteractiveShellTest(unittest.TestCase): + def test_fetch(self): + args = ( + sys.executable, + "-m", + "scrapy.cmdline", + "shell", + ) + logfile = BytesIO() + p = PopenSpawn(args, timeout=5) + p.logfile_read = logfile + p.expect_exact("Available Scrapy objects") + with MockServer() as mockserver: + p.sendline(f"fetch('{mockserver.url('/')}')") + p.sendline("type(response)") + p.expect_exact("HtmlResponse") + p.sendeof() + p.wait() + logfile.seek(0) + self.assertNotIn("Traceback", logfile.read().decode()) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 2b141e894..60b92377d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,13 +1,16 @@ import logging import os import platform +import signal import subprocess import sys import warnings from pathlib import Path +from typing import List import pytest from packaging.version import parse as parse_version +from pexpect.popen_spawn import PopenSpawn from pytest import mark, raises from twisted.internet import defer from twisted.trial import unittest @@ -289,9 +292,12 @@ class ScriptRunnerMixin: script_dir: Path cwd = os.getcwd() - def run_script(self, script_name: str, *script_args): + def get_script_args(self, script_name: str, *script_args: str) -> List[str]: script_path = self.script_dir / script_name - args = [sys.executable, str(script_path)] + list(script_args) + return [sys.executable, str(script_path)] + list(script_args) + + def run_script(self, script_name: str, *script_args: str) -> str: + args = self.get_script_args(script_name, *script_args) p = subprocess.Popen( args, env=get_mockserver_env(), @@ -517,6 +523,29 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("The value of FOO is 42", log) + def test_shutdown_graceful(self): + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK + args = self.get_script_args("sleeping.py") + p = PopenSpawn(args, timeout=5) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(sig) + p.expect_exact("shutting down gracefully") + p.expect_exact("Spider closed (shutdown)") + p.wait() + + def test_shutdown_forced(self): + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK + args = self.get_script_args("sleeping.py") + p = PopenSpawn(args, timeout=5) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(sig) + p.expect_exact("shutting down gracefully") + p.kill(sig) + p.expect_exact("forcing unclean shutdown") + p.wait() + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" From 5e4fb0bc5fc066136b171ce488599b1ddd64c83a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 17 Oct 2023 21:24:44 +0400 Subject: [PATCH 1248/2083] Re-enable uvloop tests on 3.12 (#6098) --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 2 +- .github/workflows/tests-ubuntu.yml | 5 ----- scrapy/contracts/__init__.py | 6 ++++-- tests/requirements.txt | 3 +-- tox.ini | 2 +- 6 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ee0cb4b1e..f91055ba5 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -8,7 +8,7 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: pylint - python-version: 3.8 @@ -17,7 +17,7 @@ jobs: - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: twinecheck diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 22b8996b6..095793299 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - run: | pip install --upgrade build twine python -m build diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 62b5f123a..c883f958c 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -51,11 +51,6 @@ jobs: env: TOXENV: botocore - # keep until uvloop supports 3.12 - - python-version: "3.11" - env: - TOXENV: asyncio - steps: - uses: actions/checkout@v3 diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 1ec2a0234..2d9ddd89a 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -41,7 +41,9 @@ class Contract: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") - return list(iterate_spider_output(cb_result)) + return list( # pylint: disable=return-in-finally + iterate_spider_output(cb_result) + ) request.callback = wrapper @@ -68,7 +70,7 @@ class Contract: else: results.addSuccess(self.testcase_post) finally: - return output + return output # pylint: disable=return-in-finally request.callback = wrapper diff --git a/tests/requirements.txt b/tests/requirements.txt index d4bfead40..5b75674f5 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -7,8 +7,7 @@ pytest-cov==4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures -# uvloop currently doesn't build on 3.12 -uvloop; platform_system != "Windows" and python_version < "3.12" +uvloop; platform_system != "Windows" bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests diff --git a/tox.ini b/tox.ini index 9c2522a43..381da9773 100644 --- a/tox.ini +++ b/tox.ini @@ -57,7 +57,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.17.5 + pylint==3.0.1 commands = pylint conftest.py docs extras scrapy setup.py tests From 1045856a50d379d145e514ec9c7aeeed231aefd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 30 Oct 2023 09:35:29 +0100 Subject: [PATCH 1249/2083] Merge pull request #6112 from wRAR/test-shutdown-forced Make shutdown tests more robust. --- tests/CrawlerProcess/sleeping.py | 2 +- tests/test_command_shell.py | 5 ++++- tests/test_crawler.py | 11 +++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py index 420d9d328..45479ea4f 100644 --- a/tests/CrawlerProcess/sleeping.py +++ b/tests/CrawlerProcess/sleeping.py @@ -14,7 +14,7 @@ class SleepingSpider(scrapy.Spider): from twisted.internet import reactor d = Deferred() - reactor.callLater(3, d.callback, None) + reactor.callLater(int(self.sleep), d.callback, None) await maybe_deferred_to_future(d) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7d87eb62c..7918d94b2 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,3 +1,4 @@ +import os import sys from io import BytesIO from pathlib import Path @@ -147,8 +148,10 @@ class InteractiveShellTest(unittest.TestCase): "scrapy.cmdline", "shell", ) + env = os.environ.copy() + env["SCRAPY_PYTHON_SHELL"] = "python" logfile = BytesIO() - p = PopenSpawn(args, timeout=5) + p = PopenSpawn(args, env=env, timeout=5) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") with MockServer() as mockserver: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 60b92377d..0a7f9bac8 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -525,7 +525,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_shutdown_graceful(self): sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py") + args = self.get_script_args("sleeping.py", "-a", "sleep=3") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") @@ -534,14 +534,21 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.expect_exact("Spider closed (shutdown)") p.wait() + @defer.inlineCallbacks def test_shutdown_forced(self): + from twisted.internet import reactor + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py") + args = self.get_script_args("sleeping.py", "-a", "sleep=10") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") p.kill(sig) p.expect_exact("shutting down gracefully") + # sending the second signal too fast often causes problems + d = defer.Deferred() + reactor.callLater(0.1, d.callback, None) + yield d p.kill(sig) p.expect_exact("forcing unclean shutdown") p.wait() From 150f9d6d888970d5f164387761989aba59e830c0 Mon Sep 17 00:00:00 2001 From: Jessica Allman-LaPorte Date: Fri, 3 Nov 2023 05:02:18 -0400 Subject: [PATCH 1250/2083] Make shell switching more clear in the tutorial (#6128) --- docs/intro/tutorial.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 19a76fc16..8ea98f29b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -493,7 +493,15 @@ in the callback, as you can see below: "tags": quote.css("div.tags a.tag::text").getall(), } -If you run this spider, it will output the extracted data with the log:: +To run this spider, exit the scrapy shell by entering:: + + quit() + +Then, run:: + + scrapy crawl quotes + +Now, it should output the extracted data with the log:: 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.â€'} From 49b284ab8508d3400582781343ea8171980b1e70 Mon Sep 17 00:00:00 2001 From: Kiran <75929997+Kiran1689@users.noreply.github.com> Date: Tue, 14 Nov 2023 00:43:10 +0530 Subject: [PATCH 1251/2083] Updated README.rst (#6144) --- README.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 1918850d6..14adff648 100644 --- a/README.rst +++ b/README.rst @@ -17,9 +17,10 @@ Scrapy :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - :alt: macOS +.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + .. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + .. :alt: macOS + .. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows @@ -41,7 +42,7 @@ Scrapy Overview ======== -Scrapy is a fast high-level web crawling and web scraping framework, used to +Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. @@ -110,4 +111,4 @@ See https://scrapy.org/companies/ for a list. Commercial Support ================== -See https://scrapy.org/support/ for details. +See https://scrapy.org/support/ for details. \ No newline at end of file From 6969041c5f6891a0298d7e68ece762adee1bb222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 15:52:00 +0100 Subject: [PATCH 1252/2083] Protect against gzip bombs --- .../downloadermiddlewares/httpcompression.py | 26 +++++++++++----- scrapy/utils/_compression.py | 2 ++ scrapy/utils/gz.py | 14 +++++++-- tests/sample_data/compressed/bomb-gzip.bin | Bin 0 -> 27988 bytes ...st_downloadermiddleware_httpcompression.py | 29 ++++++++++++++++-- 5 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 scrapy/utils/_compression.py create mode 100644 tests/sample_data/compressed/bomb-gzip.bin diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index ead426951..5dd67ea87 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -2,9 +2,10 @@ import io import warnings import zlib -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes +from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip @@ -29,24 +30,26 @@ class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, stats=None): - self.stats = stats + def __init__(self, crawler=None): + self.stats = crawler.stats + self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured try: - return cls(stats=crawler.stats) + return cls(crawler=crawler) except TypeError: warnings.warn( "HttpCompressionMiddleware subclasses must either modify " - "their '__init__' method to support a 'stats' parameter or " - "reimplement the 'from_crawler' method.", + "their '__init__' method to support a 'crawler' parameter or " + "reimplement their 'from_crawler' method.", ScrapyDeprecationWarning, ) result = cls() result.stats = crawler.stats + result._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") return result def process_request(self, request, spider): @@ -59,7 +62,14 @@ class HttpCompressionMiddleware: content_encoding = response.headers.getlist("Content-Encoding") if content_encoding: encoding = content_encoding.pop() - decoded_body = self._decode(response.body, encoding.lower()) + try: + decoded_body = self._decode(response.body, encoding.lower()) + except _DecompressionMaxSizeExceeded: + raise IgnoreRequest( + f"Ignored response {response} because its body " + f"({len(response.body)}B) exceeded DOWNLOAD_MAXSIZE " + f"({self._max_size}B) during decompression." + ) if self.stats: self.stats.inc_value( "httpcompression/response_bytes", @@ -85,7 +95,7 @@ class HttpCompressionMiddleware: def _decode(self, body, encoding): if encoding == b"gzip" or encoding == b"x-gzip": - body = gunzip(body) + body = gunzip(body, max_size=self._max_size) if encoding == b"deflate": try: diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py new file mode 100644 index 000000000..e726a70f5 --- /dev/null +++ b/scrapy/utils/_compression.py @@ -0,0 +1,2 @@ +class _DecompressionMaxSizeExceeded(ValueError): + pass diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index c7f74030e..cd5059a5c 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -5,8 +5,10 @@ from typing import List from scrapy.http import Response +from ._compression import _DecompressionMaxSizeExceeded -def gunzip(data: bytes) -> bytes: + +def gunzip(data: bytes, max_size: int = 0) -> bytes: """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. @@ -14,10 +16,10 @@ def gunzip(data: bytes) -> bytes: f = GzipFile(fileobj=BytesIO(data)) output_list: List[bytes] = [] chunk = b"." + decompressed_size = 0 while chunk: try: chunk = f.read1(8196) - output_list.append(chunk) except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error @@ -25,6 +27,14 @@ def gunzip(data: bytes) -> bytes: if output_list: break raise + decompressed_size += len(chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size}B) exceed the specified maximum " + f"({max_size}B)." + ) + output_list.append(chunk) return b"".join(output_list) diff --git a/tests/sample_data/compressed/bomb-gzip.bin b/tests/sample_data/compressed/bomb-gzip.bin new file mode 100644 index 0000000000000000000000000000000000000000..64aa0c3696cc1c6d86d218c70635d88f2035ec9e GIT binary patch literal 27988 zcmeIyElY!86b9f&oiK|G(Y#;~27Xl0-yq1UE0YN_<|{r6TM$G+ga1HfWkC=@i}}T- zAQQ0;tA;J=f*|@M2A1oDJDzYj_mw}*X4m_rN*LQrYP)-t7`Kz1`EpV#FVq|L(0ja} zI9SSs+qBrtti6t3Pxsob#`n?W)Wc%`Y$l!UY&@@AZN0t3&;4p=`TZgaH}D5)fC3Vd zkc1>8Aqh!HLK2dYgd`*(2}wvo5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8Aqh!H zLK2dYgd`*(2}wvo5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8Aqh!HLK2d2h!PI& z=1wx8 S;eSfl{TO{ZZ^qTjoA3)j<4WiN literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 9dad056de..f834e78f5 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -10,7 +10,7 @@ from scrapy.downloadermiddlewares.httpcompression import ( ACCEPTED_ENCODINGS, HttpCompressionMiddleware, ) -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider @@ -35,12 +35,24 @@ FORMAT = { "html-zstd-streaming-no-content-size.bin", "zstd", ), + **{ + f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id) + for format_id in ( + # "br", + "gzip", # 27 988 → 11 511 612 + # "deflate", + # "zstd", + ) + }, } class HttpCompressionTest(TestCase): def setUp(self): - self.crawler = get_crawler(Spider) + settings = { + "DOWNLOAD_MAXSIZE": 10_000_000, # For compression bomb tests. + } + self.crawler = get_crawler(Spider, settings_dict=settings) self.spider = self.crawler._create_spider("scrapytest.org") self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) self.crawler.stats.open_spider(self.spider) @@ -373,6 +385,19 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) + def _test_compression_bomb(self, compression_id): + response = self._getresponse(f"bomb-{compression_id}") + self.assertRaises( + IgnoreRequest, + self.mw.process_response, + response.request, + response, + self.spider, + ) + + def test_compression_bomb_gzip(self): + self._test_compression_bomb("gzip") + class HttpCompressionSubclassTest(TestCase): def test_init_missing_stats(self): From 0bf29a7b1b9b6a641c486780b7b0fa455577bf39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 16:10:50 +0100 Subject: [PATCH 1253/2083] Update test expectations --- scrapy/downloadermiddlewares/httpcompression.py | 4 +++- .../test_downloadermiddleware_httpcompression.py | 16 ++-------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 5dd67ea87..0fec05a14 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -30,7 +30,9 @@ class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, crawler=None): + def __init__(self, *, crawler=None): + if not crawler: + return self.stats = crawler.stats self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index f834e78f5..f5dedd28d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -127,18 +127,6 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) - def test_process_response_gzip_no_stats(self): - mw = HttpCompressionMiddleware() - response = self._getresponse("gzip") - request = response.request - - self.assertEqual(response.headers["Content-Encoding"], b"gzip") - newresponse = mw.process_response(request, response, self.spider) - self.assertEqual(mw.stats, None) - assert newresponse is not response - assert newresponse.body.startswith(b" Date: Wed, 22 Nov 2023 17:12:43 +0100 Subject: [PATCH 1254/2083] Protect against deflate bombs --- .../downloadermiddlewares/httpcompression.py | 20 +++------ scrapy/utils/_compression.py | 39 ++++++++++++++++++ scrapy/utils/gz.py | 2 +- tests/sample_data/compressed/bomb-deflate.bin | Bin 0 -> 27968 bytes ...st_downloadermiddleware_httpcompression.py | 5 ++- 5 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 tests/sample_data/compressed/bomb-deflate.bin diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 0fec05a14..8cec87c47 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,11 +1,10 @@ import io import warnings -import zlib from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.utils._compression import _DecompressionMaxSizeExceeded +from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip @@ -97,23 +96,14 @@ class HttpCompressionMiddleware: def _decode(self, body, encoding): if encoding == b"gzip" or encoding == b"x-gzip": - body = gunzip(body, max_size=self._max_size) - + return gunzip(body, max_size=self._max_size) if encoding == b"deflate": - try: - body = zlib.decompress(body) - except zlib.error: - # ugly hack to work with raw deflate content that may - # be sent by microsoft servers. For more information, see: - # http://carsten.codimi.de/gzip.yaws/ - # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx - # http://www.gzip.org/zlib/zlib_faq.html#faq38 - body = zlib.decompress(body, -15) + return _inflate(body, max_size=self._max_size) if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: - body = brotli.decompress(body) + return brotli.decompress(body) if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: # Using its streaming API since its simple API could handle only cases # where there is content size data embedded in the frame reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body)) - body = reader.read() + return reader.read() return body diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index e726a70f5..34bf2e4f7 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -1,2 +1,41 @@ +import zlib +from io import BytesIO +from typing import List + + class _DecompressionMaxSizeExceeded(ValueError): pass + + +def _inflate(data: bytes, *, max_size: int = 0) -> bytes: + decompressor = zlib.decompressobj() + raw_decompressor = zlib.decompressobj(wbits=-15) + input_stream = BytesIO(data) + output_list: List[bytes] = [] + output_chunk = b"." + decompressed_size = 0 + CHUNK_SIZE = 8196 + while output_chunk: + input_chunk = input_stream.read(CHUNK_SIZE) + try: + output_chunk = decompressor.decompress(input_chunk) + except zlib.error: + if decompressor != raw_decompressor: + # ugly hack to work with raw deflate content that may + # be sent by microsoft servers. For more information, see: + # http://carsten.codimi.de/gzip.yaws/ + # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx + # http://www.gzip.org/zlib/zlib_faq.html#faq38 + decompressor = raw_decompressor + output_chunk = decompressor.decompress(input_chunk) + else: + raise + decompressed_size += len(output_chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size}B) exceed the specified maximum " + f"({max_size}B)." + ) + output_list.append(output_chunk) + return b"".join(output_list) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index cd5059a5c..548134721 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -8,7 +8,7 @@ from scrapy.http import Response from ._compression import _DecompressionMaxSizeExceeded -def gunzip(data: bytes, max_size: int = 0) -> bytes: +def gunzip(data: bytes, *, max_size: int = 0) -> bytes: """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. diff --git a/tests/sample_data/compressed/bomb-deflate.bin b/tests/sample_data/compressed/bomb-deflate.bin new file mode 100644 index 0000000000000000000000000000000000000000..3598aca0777ec2511a721e9655cabb8c21c658bd GIT binary patch literal 27968 zcmeI&u}VS#6b9f+=-?6}`2-Gb=^8GEdrPzhZ7q%$M35jzaB^}JadC@=dVsh@OD>AI zMF>rSC{CdeWcdhg3f~#dd^nu*O@FmB>%Sy!^U2^bI{%2BjpGkTvtGCQb9b0}%gx*A zC^NVm7VfVnqwxEtJUIF4gqj_=18;x=5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8 zAqh!HLK2dYgd`*(2}wvo5|VI-C0ssb8?oTOioa2%K7C$JY75N{+<`Yh0SQS+LK2dY zgd`*(2}wvo5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8Aqh#iPZGYjN(Y-T;OY9R z_8O#jIJamt;d0?};d0?}5|WUFBqSjTNk~Exl8}TXBq0e&NJ0{lkc1>8Aqh#im4waX I@bhBz2V-bE-2eap literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index f5dedd28d..3af8202cc 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -39,8 +39,8 @@ FORMAT = { f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id) for format_id in ( # "br", + "deflate", # 27 968 → 11 511 612 "gzip", # 27 988 → 11 511 612 - # "deflate", # "zstd", ) }, @@ -383,6 +383,9 @@ class HttpCompressionTest(TestCase): self.spider, ) + def test_compression_bomb_deflate(self): + self._test_compression_bomb("deflate") + def test_compression_bomb_gzip(self): self._test_compression_bomb("gzip") From fba167c5e1f356bcc452e95e92199f1a15135c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 17:32:09 +0100 Subject: [PATCH 1255/2083] Protect against brotli bombs --- .../downloadermiddlewares/httpcompression.py | 14 +++++----- scrapy/utils/_compression.py | 26 +++++++++++++++++++ tests/sample_data/compressed/bomb-br.bin | 2 ++ ...st_downloadermiddleware_httpcompression.py | 9 ++++++- 4 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 tests/sample_data/compressed/bomb-br.bin diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 8cec87c47..91748e57e 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -4,25 +4,25 @@ import warnings from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate +from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate, _unbrotli from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip ACCEPTED_ENCODINGS = [b"gzip", b"deflate"] try: - import brotli - - ACCEPTED_ENCODINGS.append(b"br") + import brotli # noqa: F401 except ImportError: pass +else: + ACCEPTED_ENCODINGS.append(b"br") try: import zstandard - - ACCEPTED_ENCODINGS.append(b"zstd") except ImportError: pass +else: + ACCEPTED_ENCODINGS.append(b"zstd") class HttpCompressionMiddleware: @@ -100,7 +100,7 @@ class HttpCompressionMiddleware: if encoding == b"deflate": return _inflate(body, max_size=self._max_size) if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: - return brotli.decompress(body) + return _unbrotli(body, max_size=self._max_size) if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: # Using its streaming API since its simple API could handle only cases # where there is content size data embedded in the frame diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 34bf2e4f7..9a32ce4f0 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -2,6 +2,11 @@ import zlib from io import BytesIO from typing import List +try: + import brotli +except ImportError: + pass + class _DecompressionMaxSizeExceeded(ValueError): pass @@ -39,3 +44,24 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes: ) output_list.append(output_chunk) return b"".join(output_list) + + +def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: + decompressor = brotli.Decompressor() + input_stream = BytesIO(data) + output_list: List[bytes] = [] + output_chunk = b"." + decompressed_size = 0 + CHUNK_SIZE = 8196 + while output_chunk: + input_chunk = input_stream.read(CHUNK_SIZE) + output_chunk = decompressor.process(input_chunk) + decompressed_size += len(output_chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size}B) exceed the specified maximum " + f"({max_size}B)." + ) + output_list.append(output_chunk) + return b"".join(output_list) diff --git a/tests/sample_data/compressed/bomb-br.bin b/tests/sample_data/compressed/bomb-br.bin new file mode 100644 index 000000000..50059866f --- /dev/null +++ b/tests/sample_data/compressed/bomb-br.bin @@ -0,0 +1,2 @@ +Ÿ;§¯ø”nÞªVp SmoY2·å +ï(ä)-д=_o \ No newline at end of file diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 3af8202cc..8858916bc 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -38,7 +38,7 @@ FORMAT = { **{ f"bomb-{format_id}": (f"bomb-{format_id}.bin", format_id) for format_id in ( - # "br", + "br", # 34 → 11 511 612 "deflate", # 27 968 → 11 511 612 "gzip", # 27 988 → 11 511 612 # "zstd", @@ -383,6 +383,13 @@ class HttpCompressionTest(TestCase): self.spider, ) + def test_compression_bomb_br(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_compression_bomb("br") + def test_compression_bomb_deflate(self): self._test_compression_bomb("deflate") From 9cc870387745f45f744ceef6ed226eefcce0e066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 17:53:00 +0100 Subject: [PATCH 1256/2083] Protect against zstandard bombs --- .../downloadermiddlewares/httpcompression.py | 15 ++++++----- scrapy/utils/_compression.py | 25 ++++++++++++++++++ tests/sample_data/compressed/bomb-zstd.bin | Bin 0 -> 1096 bytes ...st_downloadermiddleware_httpcompression.py | 5 +++- 4 files changed, 37 insertions(+), 8 deletions(-) create mode 100644 tests/sample_data/compressed/bomb-zstd.bin diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 91748e57e..8ee1d95a6 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,10 +1,14 @@ -import io import warnings from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.utils._compression import _DecompressionMaxSizeExceeded, _inflate, _unbrotli +from scrapy.utils._compression import ( + _DecompressionMaxSizeExceeded, + _inflate, + _unbrotli, + _unzstd, +) from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip @@ -18,7 +22,7 @@ else: ACCEPTED_ENCODINGS.append(b"br") try: - import zstandard + import zstandard # noqa: F401 except ImportError: pass else: @@ -102,8 +106,5 @@ class HttpCompressionMiddleware: if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: return _unbrotli(body, max_size=self._max_size) if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: - # Using its streaming API since its simple API could handle only cases - # where there is content size data embedded in the frame - reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body)) - return reader.read() + return _unzstd(body, max_size=self._max_size) return body diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 9a32ce4f0..93aa254b2 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -7,6 +7,11 @@ try: except ImportError: pass +try: + import zstandard +except ImportError: + pass + class _DecompressionMaxSizeExceeded(ValueError): pass @@ -65,3 +70,23 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: ) output_list.append(output_chunk) return b"".join(output_list) + + +def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: + decompressor = zstandard.ZstdDecompressor() + stream_reader = decompressor.stream_reader(BytesIO(data)) + output_list: List[bytes] = [] + output_chunk = b"." + decompressed_size = 0 + CHUNK_SIZE = 8196 + while output_chunk: + output_chunk = stream_reader.read(CHUNK_SIZE) + decompressed_size += len(output_chunk) + if max_size and decompressed_size > max_size: + raise _DecompressionMaxSizeExceeded( + f"The number of bytes decompressed so far " + f"({decompressed_size}B) exceed the specified maximum " + f"({max_size}B)." + ) + output_list.append(output_chunk) + return b"".join(output_list) diff --git a/tests/sample_data/compressed/bomb-zstd.bin b/tests/sample_data/compressed/bomb-zstd.bin new file mode 100644 index 0000000000000000000000000000000000000000..4b0efa8a41c88a38dffe7cea9e4a6726bdb137c4 GIT binary patch literal 1096 zcmdPcs{gko!e;q;1{Fqz2O$}m#R@=_sF0kWTTql*T%4Jor;wDNo219Z$k6h?-fpfB z0|SQwBg3EnmI6#5b|Mlx7l~br#Lh!vBdZBP5+5~lG&~1uT5@4vU|?kU`+vT_!f29* VWPM*?)(2)~i{-##pxebr9RRD(6-595 literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 8858916bc..7babd1318 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -41,7 +41,7 @@ FORMAT = { "br", # 34 → 11 511 612 "deflate", # 27 968 → 11 511 612 "gzip", # 27 988 → 11 511 612 - # "zstd", + "zstd", # 1 096 → 11 511 612 ) }, } @@ -396,6 +396,9 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_gzip(self): self._test_compression_bomb("gzip") + def test_compression_bomb_zstd(self): + self._test_compression_bomb("zstd") + class HttpCompressionSubclassTest(TestCase): def test_init_missing_stats(self): From 3fda2fe103dafa8d4d48b21c63b2321ccec9c378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 18:34:37 +0100 Subject: [PATCH 1257/2083] Protect against gzip bomb sitemaps --- scrapy/spiders/sitemap.py | 8 +++++++- tests/test_spider.py | 15 +++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index aaf75a519..cc8b13cc3 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -3,6 +3,7 @@ import re from scrapy.http import Request, XmlResponse from scrapy.spiders import Spider +from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots @@ -71,7 +72,12 @@ class SitemapSpider(Spider): if isinstance(response, XmlResponse): return response.body if gzip_magic_number(response): - return gunzip(response.body) + try: + return gunzip( + response.body, max_size=self.settings.getint("DOWNLOAD_MAXSIZE") + ) + except _DecompressionMaxSizeExceeded: + return None # actual gzipped sitemap files are decompressed above ; # if we are here (response body is not gzipped) # and have a response for .xml.gz, diff --git a/tests/test_spider.py b/tests/test_spider.py index 00da3d485..875ff5454 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -2,6 +2,7 @@ import gzip import inspect import warnings from io import BytesIO +from pathlib import Path from typing import Any from unittest import mock @@ -25,7 +26,7 @@ from scrapy.spiders import ( ) from scrapy.spiders.init import InitSpider from scrapy.utils.test import get_crawler -from tests import get_testdata +from tests import get_testdata, tests_datadir class SpiderTest(unittest.TestCase): @@ -489,7 +490,8 @@ class SitemapSpiderTest(SpiderTest): GZBODY = f.getvalue() def assertSitemapBody(self, response, body): - spider = self.spider_class("example.com") + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") self.assertEqual(spider._get_sitemap_body(response), body) def test_get_sitemap_body(self): @@ -692,6 +694,15 @@ Sitemap: /sitemap-relative-url.xml ["http://www.example.com/sitemap2.xml"], ) + def test_compression_bomb(self): + settings = {"DOWNLOAD_MAXSIZE": 10_000_000} + crawler = get_crawler(settings_dict=settings) + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + response = Response(url="https://example.com", body=body) + self.assertIsNone(spider._get_sitemap_body(response)) + class DeprecationTest(unittest.TestCase): def test_crawl_spider(self): From e0b66c021ae20cdcc24e4bb02ffae56005d3a073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 22 Nov 2023 19:03:24 +0100 Subject: [PATCH 1258/2083] Mind Spider.download_maxsize and Request.meta['download_maxsize'] --- .../downloadermiddlewares/httpcompression.py | 30 ++++-- scrapy/spiders/sitemap.py | 10 +- ...st_downloadermiddleware_httpcompression.py | 99 ++++++++++++++++--- tests/test_spider.py | 35 ++++++- 4 files changed, 143 insertions(+), 31 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 8ee1d95a6..e6463307e 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,5 +1,6 @@ import warnings +from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes @@ -38,6 +39,7 @@ class HttpCompressionMiddleware: return self.stats = crawler.stats self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + crawler.signals.connect(self.open_spider, signals.spider_opened) @classmethod def from_crawler(cls, crawler): @@ -52,10 +54,15 @@ class HttpCompressionMiddleware: "reimplement their 'from_crawler' method.", ScrapyDeprecationWarning, ) - result = cls() - result.stats = crawler.stats - result._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") - return result + spider = cls() + spider.stats = crawler.stats + spider._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + crawler.signals.connect(spider.open_spider, signals.spider_opened) + return spider + + def open_spider(self, spider): + if hasattr(spider, "download_maxsize"): + self._max_size = spider.download_maxsize def process_request(self, request, spider): request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) @@ -67,8 +74,11 @@ class HttpCompressionMiddleware: content_encoding = response.headers.getlist("Content-Encoding") if content_encoding: encoding = content_encoding.pop() + max_size = request.meta.get("download_maxsize", self._max_size) try: - decoded_body = self._decode(response.body, encoding.lower()) + decoded_body = self._decode( + response.body, encoding.lower(), max_size + ) except _DecompressionMaxSizeExceeded: raise IgnoreRequest( f"Ignored response {response} because its body " @@ -98,13 +108,13 @@ class HttpCompressionMiddleware: return response - def _decode(self, body, encoding): + def _decode(self, body, encoding, max_size): if encoding == b"gzip" or encoding == b"x-gzip": - return gunzip(body, max_size=self._max_size) + return gunzip(body, max_size=max_size) if encoding == b"deflate": - return _inflate(body, max_size=self._max_size) + return _inflate(body, max_size=max_size) if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: - return _unbrotli(body, max_size=self._max_size) + return _unbrotli(body, max_size=max_size) if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: - return _unzstd(body, max_size=self._max_size) + return _unzstd(body, max_size=max_size) return body diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index cc8b13cc3..3bca3f5c2 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -72,10 +72,14 @@ class SitemapSpider(Spider): if isinstance(response, XmlResponse): return response.body if gzip_magic_number(response): + max_size = response.meta.get( + "download_maxsize", + getattr( + self, "download_maxsize", self.settings.getint("DOWNLOAD_MAXSIZE") + ), + ) try: - return gunzip( - response.body, max_size=self.settings.getint("DOWNLOAD_MAXSIZE") - ) + return gunzip(response.body, max_size=max_size) except _DecompressionMaxSizeExceeded: return None # actual gzipped sitemap files are decompressed above ; diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 7babd1318..6d71ba71e 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -49,10 +49,7 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - settings = { - "DOWNLOAD_MAXSIZE": 10_000_000, # For compression bomb tests. - } - self.crawler = get_crawler(Spider, settings_dict=settings) + self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("scrapytest.org") self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) self.crawler.stats.open_spider(self.spider) @@ -373,31 +370,103 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) - def _test_compression_bomb(self, compression_id): + def _test_compression_bomb_setting(self, compression_id): + settings = {"DOWNLOAD_MAXSIZE": 10_000_000} + crawler = get_crawler(Spider, settings_dict=settings) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(f"bomb-{compression_id}") self.assertRaises( IgnoreRequest, - self.mw.process_response, + mw.process_response, response.request, response, - self.spider, + spider, ) - def test_compression_bomb_br(self): + def test_compression_bomb_setting_br(self): try: import brotli # noqa: F401 except ImportError: raise SkipTest("no brotli") - self._test_compression_bomb("br") + self._test_compression_bomb_setting("br") - def test_compression_bomb_deflate(self): - self._test_compression_bomb("deflate") + def test_compression_bomb_setting_deflate(self): + self._test_compression_bomb_setting("deflate") - def test_compression_bomb_gzip(self): - self._test_compression_bomb("gzip") + def test_compression_bomb_setting_gzip(self): + self._test_compression_bomb_setting("gzip") - def test_compression_bomb_zstd(self): - self._test_compression_bomb("zstd") + def test_compression_bomb_setting_zstd(self): + self._test_compression_bomb_setting("zstd") + + def _test_compression_bomb_spider_attr(self, compression_id): + class DownloadMaxSizeSpider(Spider): + download_maxsize = 10_000_000 + + crawler = get_crawler(DownloadMaxSizeSpider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + + response = self._getresponse(f"bomb-{compression_id}") + self.assertRaises( + IgnoreRequest, + mw.process_response, + response.request, + response, + spider, + ) + + def test_compression_bomb_spider_attr_br(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_compression_bomb_spider_attr("br") + + def test_compression_bomb_spider_attr_deflate(self): + self._test_compression_bomb_spider_attr("deflate") + + def test_compression_bomb_spider_attr_gzip(self): + self._test_compression_bomb_spider_attr("gzip") + + def test_compression_bomb_spider_attr_zstd(self): + self._test_compression_bomb_spider_attr("zstd") + + def _test_compression_bomb_request_meta(self, compression_id): + crawler = get_crawler(Spider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + + response = self._getresponse(f"bomb-{compression_id}") + response.meta["download_maxsize"] = 10_000_000 + self.assertRaises( + IgnoreRequest, + mw.process_response, + response.request, + response, + spider, + ) + + def test_compression_bomb_request_meta_br(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_compression_bomb_request_meta("br") + + def test_compression_bomb_request_meta_deflate(self): + self._test_compression_bomb_request_meta("deflate") + + def test_compression_bomb_request_meta_gzip(self): + self._test_compression_bomb_request_meta("gzip") + + def test_compression_bomb_request_meta_zstd(self): + self._test_compression_bomb_request_meta("zstd") class HttpCompressionSubclassTest(TestCase): diff --git a/tests/test_spider.py b/tests/test_spider.py index 875ff5454..e8480ceb4 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -509,6 +509,7 @@ class SitemapSpiderTest(SpiderTest): url="http://www.example.com/sitemap", body=self.GZBODY, headers={"content-type": "application/gzip"}, + request=Request("http://www.example.com/sitemap"), ) self.assertSitemapBody(r, self.BODY) @@ -517,7 +518,11 @@ class SitemapSpiderTest(SpiderTest): self.assertSitemapBody(r, self.BODY) def test_get_sitemap_body_xml_url_compressed(self): - r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY) + r = Response( + url="http://www.example.com/sitemap.xml.gz", + body=self.GZBODY, + request=Request("http://www.example.com/sitemap"), + ) self.assertSitemapBody(r, self.BODY) # .xml.gz but body decoded by HttpCompression middleware already @@ -694,13 +699,37 @@ Sitemap: /sitemap-relative-url.xml ["http://www.example.com/sitemap2.xml"], ) - def test_compression_bomb(self): + def test_compression_bomb_setting(self): settings = {"DOWNLOAD_MAXSIZE": 10_000_000} crawler = get_crawler(settings_dict=settings) spider = self.spider_class.from_crawler(crawler, "example.com") body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") body = body_path.read_bytes() - response = Response(url="https://example.com", body=body) + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + self.assertIsNone(spider._get_sitemap_body(response)) + + def test_compression_bomb_spider_attr(self): + class DownloadMaxSizeSpider(self.spider_class): + download_maxsize = 10_000_000 + + crawler = get_crawler() + spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + self.assertIsNone(spider._get_sitemap_body(response)) + + def test_compression_bomb_request_meta(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_maxsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) self.assertIsNone(spider._get_sitemap_body(response)) From 1087bb7b2eab28543bf9ba13149adb7acd4a3675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 23 Nov 2023 09:11:14 +0100 Subject: [PATCH 1259/2083] Update the docs --- docs/topics/request-response.rst | 1 + docs/topics/settings.rst | 36 +++++++++++++++++--------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index adf3d0f4a..2d1227cf8 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -702,6 +702,7 @@ Those are: * :reqmeta:`download_fail_on_dataloss` * :reqmeta:`download_latency` * :reqmeta:`download_maxsize` +* :reqmeta:`download_warnsize` * :reqmeta:`download_timeout` * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * ``ftp_user`` (See :setting:`FTP_USER` for more info) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7cdfb8768..eb24b834a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -873,40 +873,42 @@ The amount of time (in secs) that the downloader will wait before timing out. Request.meta key. .. setting:: DOWNLOAD_MAXSIZE +.. reqmeta:: download_maxsize DOWNLOAD_MAXSIZE ---------------- -Default: ``1073741824`` (1024MB) +Default: ``1073741824`` (1 GiB) -The maximum response size (in bytes) that downloader will download. +The maximum response body size (in bytes) allowed. Bigger responses are +aborted and ignored. -If you want to disable it set to 0. +This applies both before and after compression. If decompressing a response +body would exceed this limit, decompression is aborted and the response is +ignored. -.. reqmeta:: download_maxsize +Use ``0`` to disable this limit. -.. note:: - - This size can be set per spider using :attr:`download_maxsize` - spider attribute and per-request using :reqmeta:`download_maxsize` - Request.meta key. +This limit can be set per spider using the :attr:`download_maxsize` spider +attribute and per request using the :reqmeta:`download_maxsize` Request.meta +key. .. setting:: DOWNLOAD_WARNSIZE +.. reqmeta:: download_warnsize DOWNLOAD_WARNSIZE ----------------- -Default: ``33554432`` (32MB) +Default: ``33554432`` (32 MiB) -The response size (in bytes) that downloader will start to warn. +If the size of a response exceeds this value, before or after compression, a +warning will be logged about it. -If you want to disable it set to 0. +Use ``0`` to disable this limit. -.. note:: - - This size can be set per spider using :attr:`download_warnsize` - spider attribute and per-request using :reqmeta:`download_warnsize` - Request.meta key. +This limit can be set per spider using the :attr:`download_warnsize` spider +attribute and per request using the :reqmeta:`download_warnsize` Request.meta +key. .. setting:: DOWNLOAD_FAIL_ON_DATALOSS From 03d9866518ab43844ba0309394529240f4cf115e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 23 Nov 2023 10:26:47 +0100 Subject: [PATCH 1260/2083] Also use DOWNLOAD_WARNSIZE for decompressions --- .../downloadermiddlewares/httpcompression.py | 18 ++- scrapy/spiders/sitemap.py | 35 ++++- scrapy/utils/_compression.py | 12 +- scrapy/utils/gz.py | 4 +- ...st_downloadermiddleware_httpcompression.py | 130 ++++++++++++++++++ tests/test_spider.py | 78 +++++++++++ 6 files changed, 260 insertions(+), 17 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index e6463307e..95bc1849d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,4 +1,5 @@ import warnings +from logging import getLogger from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -13,6 +14,8 @@ from scrapy.utils._compression import ( from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip +logger = getLogger(__name__) + ACCEPTED_ENCODINGS = [b"gzip", b"deflate"] try: @@ -39,6 +42,7 @@ class HttpCompressionMiddleware: return self.stats = crawler.stats self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") crawler.signals.connect(self.open_spider, signals.spider_opened) @classmethod @@ -57,12 +61,15 @@ class HttpCompressionMiddleware: spider = cls() spider.stats = crawler.stats spider._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + spider._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") crawler.signals.connect(spider.open_spider, signals.spider_opened) return spider def open_spider(self, spider): if hasattr(spider, "download_maxsize"): self._max_size = spider.download_maxsize + if hasattr(spider, "download_warnsize"): + self._warn_size = spider.download_warnsize def process_request(self, request, spider): request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) @@ -75,6 +82,7 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() max_size = request.meta.get("download_maxsize", self._max_size) + warn_size = request.meta.get("download_warnsize", self._warn_size) try: decoded_body = self._decode( response.body, encoding.lower(), max_size @@ -82,8 +90,14 @@ class HttpCompressionMiddleware: except _DecompressionMaxSizeExceeded: raise IgnoreRequest( f"Ignored response {response} because its body " - f"({len(response.body)}B) exceeded DOWNLOAD_MAXSIZE " - f"({self._max_size}B) during decompression." + f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE " + f"({self._max_size} B) during decompression." + ) + if len(response.body) < warn_size and len(decoded_body) >= warn_size: + logger.warning( + f"{response} body size after decompression " + f"({len(decoded_body)} B) is larger than the " + f"download warning size ({warn_size} B)." ) if self.stats: self.stats.inc_value( diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 3bca3f5c2..0574f0ccb 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,5 +1,6 @@ import logging import re +from typing import TYPE_CHECKING, Any from scrapy.http import Request, XmlResponse from scrapy.spiders import Spider @@ -7,6 +8,12 @@ from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) @@ -16,6 +23,17 @@ class SitemapSpider(Spider): sitemap_follow = [""] sitemap_alternate_links = False + @classmethod + def from_crawler(cls, crawler: "Crawler", *args: Any, **kwargs: Any) -> "Self": + spider = super().from_crawler(crawler, *args, **kwargs) + spider._max_size = getattr( + spider, "download_maxsize", spider.settings.getint("DOWNLOAD_MAXSIZE") + ) + spider._warn_size = getattr( + spider, "download_warnsize", spider.settings.getint("DOWNLOAD_WARNSIZE") + ) + return spider + def __init__(self, *a, **kw): super().__init__(*a, **kw) self._cbs = [] @@ -72,16 +90,19 @@ class SitemapSpider(Spider): if isinstance(response, XmlResponse): return response.body if gzip_magic_number(response): - max_size = response.meta.get( - "download_maxsize", - getattr( - self, "download_maxsize", self.settings.getint("DOWNLOAD_MAXSIZE") - ), - ) + uncompressed_size = len(response.body) + max_size = response.meta.get("download_maxsize", self._max_size) + warn_size = response.meta.get("download_warnsize", self._warn_size) try: - return gunzip(response.body, max_size=max_size) + body = gunzip(response.body, max_size=max_size) except _DecompressionMaxSizeExceeded: return None + if uncompressed_size < warn_size and len(body) >= warn_size: + logger.warning( + f"{response} body size after decompression ({len(body)} B) " + f"is larger than the download warning size ({warn_size} B)." + ) + return body # actual gzipped sitemap files are decompressed above ; # if we are here (response body is not gzipped) # and have a response for .xml.gz, diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 93aa254b2..a70f6c275 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -44,8 +44,8 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes: if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( f"The number of bytes decompressed so far " - f"({decompressed_size}B) exceed the specified maximum " - f"({max_size}B)." + f"({decompressed_size} B) exceed the specified maximum " + f"({max_size} B)." ) output_list.append(output_chunk) return b"".join(output_list) @@ -65,8 +65,8 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( f"The number of bytes decompressed so far " - f"({decompressed_size}B) exceed the specified maximum " - f"({max_size}B)." + f"({decompressed_size} B) exceed the specified maximum " + f"({max_size} B)." ) output_list.append(output_chunk) return b"".join(output_list) @@ -85,8 +85,8 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( f"The number of bytes decompressed so far " - f"({decompressed_size}B) exceed the specified maximum " - f"({max_size}B)." + f"({decompressed_size} B) exceed the specified maximum " + f"({max_size} B)." ) output_list.append(output_chunk) return b"".join(output_list) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 548134721..e5cf68d62 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -31,8 +31,8 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( f"The number of bytes decompressed so far " - f"({decompressed_size}B) exceed the specified maximum " - f"({max_size}B)." + f"({decompressed_size} B) exceed the specified maximum " + f"({max_size} B)." ) output_list.append(chunk) return b"".join(output_list) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 6d71ba71e..f74fff218 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,9 +1,11 @@ from gzip import GzipFile from io import BytesIO +from logging import WARNING from pathlib import Path from unittest import SkipTest, TestCase from warnings import catch_warnings +from testfixtures import LogCapture from w3lib.encoding import resolve_encoding from scrapy.downloadermiddlewares.httpcompression import ( @@ -468,6 +470,134 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_request_meta_zstd(self): self._test_compression_bomb_request_meta("zstd") + def _test_download_warnsize_setting(self, compression_id): + settings = {"DOWNLOAD_WARNSIZE": 10_000_000} + crawler = get_crawler(Spider, settings_dict=settings) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(f"bomb-{compression_id}") + + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + mw.process_response(response.request, response, spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "<200 http://scrapytest.org/> body size after " + "decompression (11511612 B) is larger than the download " + "warning size (10000000 B)." + ), + ), + ) + + def test_download_warnsize_setting_br(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_download_warnsize_setting("br") + + def test_download_warnsize_setting_deflate(self): + self._test_download_warnsize_setting("deflate") + + def test_download_warnsize_setting_gzip(self): + self._test_download_warnsize_setting("gzip") + + def test_download_warnsize_setting_zstd(self): + self._test_download_warnsize_setting("zstd") + + def _test_download_warnsize_spider_attr(self, compression_id): + class DownloadWarnSizeSpider(Spider): + download_warnsize = 10_000_000 + + crawler = get_crawler(DownloadWarnSizeSpider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(f"bomb-{compression_id}") + + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + mw.process_response(response.request, response, spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "<200 http://scrapytest.org/> body size after " + "decompression (11511612 B) is larger than the download " + "warning size (10000000 B)." + ), + ), + ) + + def test_download_warnsize_spider_attr_br(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_download_warnsize_spider_attr("br") + + def test_download_warnsize_spider_attr_deflate(self): + self._test_download_warnsize_spider_attr("deflate") + + def test_download_warnsize_spider_attr_gzip(self): + self._test_download_warnsize_spider_attr("gzip") + + def test_download_warnsize_spider_attr_zstd(self): + self._test_download_warnsize_spider_attr("zstd") + + def _test_download_warnsize_request_meta(self, compression_id): + crawler = get_crawler(Spider) + spider = crawler._create_spider("scrapytest.org") + mw = HttpCompressionMiddleware.from_crawler(crawler) + mw.open_spider(spider) + response = self._getresponse(f"bomb-{compression_id}") + response.meta["download_warnsize"] = 10_000_000 + + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + mw.process_response(response.request, response, spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "<200 http://scrapytest.org/> body size after " + "decompression (11511612 B) is larger than the download " + "warning size (10000000 B)." + ), + ), + ) + + def test_download_warnsize_request_meta_br(self): + try: + import brotli # noqa: F401 + except ImportError: + raise SkipTest("no brotli") + self._test_download_warnsize_request_meta("br") + + def test_download_warnsize_request_meta_deflate(self): + self._test_download_warnsize_request_meta("deflate") + + def test_download_warnsize_request_meta_gzip(self): + self._test_download_warnsize_request_meta("gzip") + + def test_download_warnsize_request_meta_zstd(self): + self._test_download_warnsize_request_meta("zstd") + class HttpCompressionSubclassTest(TestCase): def test_init_missing_stats(self): diff --git a/tests/test_spider.py b/tests/test_spider.py index e8480ceb4..3f595cc93 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -2,6 +2,7 @@ import gzip import inspect import warnings from io import BytesIO +from logging import WARNING from pathlib import Path from typing import Any from unittest import mock @@ -732,6 +733,83 @@ Sitemap: /sitemap-relative-url.xml response = Response(url="https://example.com", body=body, request=request) self.assertIsNone(spider._get_sitemap_body(response)) + def test_download_warnsize_setting(self): + settings = {"DOWNLOAD_WARNSIZE": 10_000_000} + crawler = get_crawler(settings_dict=settings) + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + def test_download_warnsize_spider_attr(self): + class DownloadWarnSizeSpider(self.spider_class): + download_warnsize = 10_000_000 + + crawler = get_crawler() + spider = DownloadWarnSizeSpider.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_warnsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + def test_download_warnsize_request_meta(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_warnsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + class DeprecationTest(unittest.TestCase): def test_crawl_spider(self): From b53ed52a22470adbe269f5a7dcc67a0da369eaf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 23 Nov 2023 11:36:45 +0100 Subject: [PATCH 1261/2083] Update the release notes --- docs/news.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 0c202639e..c4081b99b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,20 @@ Release notes ============= +.. _release-2.11.1: + +Scrapy 2.11.1 (unreleased) +-------------------------- + +**Security bug fix:** + +- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply + to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security + advisory`_ for more information. + + .. _7j7m-v7m3-jqm7 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-7j7m-v7m3-jqm7 + + .. _release-2.11.0: Scrapy 2.11.0 (2023-09-18) @@ -2871,6 +2885,17 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.4: + +Scrapy 1.8.4 (unreleased) +------------------------- + +**Security bug fix:** + +- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply + to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security + advisory`_ for more information. + .. _release-1.8.3: From cf80e5670e8317858b9f60008a5d2d97b4988da0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 23 Nov 2023 12:07:15 +0100 Subject: [PATCH 1262/2083] Solve linting and typing issues --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- scrapy/spiders/sitemap.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 95bc1849d..6c8b659bd 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -93,7 +93,7 @@ class HttpCompressionMiddleware: f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE " f"({self._max_size} B) during decompression." ) - if len(response.body) < warn_size and len(decoded_body) >= warn_size: + if len(response.body) < warn_size <= len(decoded_body): logger.warning( f"{response} body size after decompression " f"({len(decoded_body)} B) is larger than the " diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 0574f0ccb..386aa6a6e 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -22,6 +22,8 @@ class SitemapSpider(Spider): sitemap_rules = [("", "parse")] sitemap_follow = [""] sitemap_alternate_links = False + _max_size: int + _warn_size: int @classmethod def from_crawler(cls, crawler: "Crawler", *args: Any, **kwargs: Any) -> "Self": @@ -97,7 +99,7 @@ class SitemapSpider(Spider): body = gunzip(response.body, max_size=max_size) except _DecompressionMaxSizeExceeded: return None - if uncompressed_size < warn_size and len(body) >= warn_size: + if uncompressed_size < warn_size <= len(body): logger.warning( f"{response} body size after decompression ({len(body)} B) " f"is larger than the download warning size ({warn_size} B)." From 8e25f8c157e53c9f5df51e950fed18becfe7797d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 23 Nov 2023 14:12:59 +0100 Subject: [PATCH 1263/2083] Fix bad message --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 6c8b659bd..f03294d65 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -91,7 +91,7 @@ class HttpCompressionMiddleware: raise IgnoreRequest( f"Ignored response {response} because its body " f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE " - f"({self._max_size} B) during decompression." + f"({max_size} B) during decompression." ) if len(response.body) < warn_size <= len(decoded_body): logger.warning( From 09ce0ef52681e2e8344b6848b28bc222016362ef Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 23 Nov 2023 10:50:46 -0300 Subject: [PATCH 1264/2083] Remove test_download_gzip_response test --- tests/test_downloader_handlers.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index f12243e1d..924ece6f9 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -726,33 +726,6 @@ class Http11MockServerTestCase(unittest.TestCase): reason = crawler.spider.meta["close_reason"] self.assertTrue(reason, "finished") - @defer.inlineCallbacks - def test_download_gzip_response(self): - crawler = get_crawler(SingleRequestSpider, self.settings_dict) - body = b"1" * 100 # PayloadResource requires body length to be 100 - request = Request( - self.mockserver.url("/payload"), - method="POST", - body=body, - meta={"download_maxsize": 50}, - ) - yield crawler.crawl(seed=request) - failure = crawler.spider.meta["failure"] - # download_maxsize < 100, hence the CancelledError - self.assertIsInstance(failure.value, defer.CancelledError) - - # See issue https://twistedmatrix.com/trac/ticket/8175 - raise unittest.SkipTest("xpayload fails on PY3") - crawler = get_crawler(SingleRequestSpider, self.settings_dict) - request.headers.setdefault(b"Accept-Encoding", b"gzip,deflate") - request = request.replace(url=self.mockserver.url("/xpayload")) - yield crawler.crawl(seed=request) - # download_maxsize = 50 is enough for the gzipped response - failure = crawler.spider.meta.get("failure") - self.assertIsNone(failure) - reason = crawler.spider.meta["close_reason"] - self.assertTrue(reason, "finished") - class UriResource(resource.Resource): """Return the full uri that was requested""" From 5f2827efe7b069e514a87bd0f7a9589f49f0a97a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 24 Nov 2023 10:02:56 +0100 Subject: [PATCH 1265/2083] Make HttpCompressionMiddleware changes backward-comaptible --- scrapy/downloadermiddlewares/httpcompression.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index f03294d65..1a3f6962a 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -37,8 +37,10 @@ class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, *, crawler=None): + def __init__(self, stats=None, *, crawler=None): if not crawler: + if stats: + self.stats = stats return self.stats = crawler.stats self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") From 6c278e1862c5453ec45a8f6a5c2472710895cad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 24 Nov 2023 10:15:21 +0100 Subject: [PATCH 1266/2083] =?UTF-8?q?List[bytes]=20=E2=86=92=20BytesIO?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/utils/_compression.py | 22 ++++++++++++---------- scrapy/utils/gz.py | 12 ++++++------ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index a70f6c275..b17fd7881 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -1,6 +1,5 @@ import zlib from io import BytesIO -from typing import List try: import brotli @@ -21,7 +20,7 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes: decompressor = zlib.decompressobj() raw_decompressor = zlib.decompressobj(wbits=-15) input_stream = BytesIO(data) - output_list: List[bytes] = [] + output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 CHUNK_SIZE = 8196 @@ -47,14 +46,15 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes: f"({decompressed_size} B) exceed the specified maximum " f"({max_size} B)." ) - output_list.append(output_chunk) - return b"".join(output_list) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: decompressor = brotli.Decompressor() input_stream = BytesIO(data) - output_list: List[bytes] = [] + output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 CHUNK_SIZE = 8196 @@ -68,14 +68,15 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: f"({decompressed_size} B) exceed the specified maximum " f"({max_size} B)." ) - output_list.append(output_chunk) - return b"".join(output_list) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: decompressor = zstandard.ZstdDecompressor() stream_reader = decompressor.stream_reader(BytesIO(data)) - output_list: List[bytes] = [] + output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 CHUNK_SIZE = 8196 @@ -88,5 +89,6 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: f"({decompressed_size} B) exceed the specified maximum " f"({max_size} B)." ) - output_list.append(output_chunk) - return b"".join(output_list) + output_stream.write(output_chunk) + output_stream.seek(0) + return output_stream.read() diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index e5cf68d62..5d23e8f05 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,7 +1,6 @@ import struct from gzip import GzipFile from io import BytesIO -from typing import List from scrapy.http import Response @@ -14,7 +13,7 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output_list: List[bytes] = [] + output_stream = BytesIO() chunk = b"." decompressed_size = 0 while chunk: @@ -23,8 +22,8 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output_list is empty - if output_list: + # some pages are quite small so output_stream is empty + if output_stream: break raise decompressed_size += len(chunk) @@ -34,8 +33,9 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: f"({decompressed_size} B) exceed the specified maximum " f"({max_size} B)." ) - output_list.append(chunk) - return b"".join(output_list) + output_stream.write(chunk) + output_stream.seek(0) + return output_stream.read() def gzip_magic_number(response: Response) -> bool: From 62398e424c0a3d66d0bdd3908e47284cbe797ef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 24 Nov 2023 10:20:14 +0100 Subject: [PATCH 1267/2083] =?UTF-8?q?CHUNK=5FSIZE:=208=20KiB=20=E2=86=92?= =?UTF-8?q?=2032=20KiB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/utils/_compression.py | 12 ++++++------ scrapy/utils/gz.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index b17fd7881..5610595d3 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -12,6 +12,9 @@ except ImportError: pass +_CHUNK_SIZE = 65536 # 64 KiB + + class _DecompressionMaxSizeExceeded(ValueError): pass @@ -23,9 +26,8 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes: output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 - CHUNK_SIZE = 8196 while output_chunk: - input_chunk = input_stream.read(CHUNK_SIZE) + input_chunk = input_stream.read(_CHUNK_SIZE) try: output_chunk = decompressor.decompress(input_chunk) except zlib.error: @@ -57,9 +59,8 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 - CHUNK_SIZE = 8196 while output_chunk: - input_chunk = input_stream.read(CHUNK_SIZE) + input_chunk = input_stream.read(_CHUNK_SIZE) output_chunk = decompressor.process(input_chunk) decompressed_size += len(output_chunk) if max_size and decompressed_size > max_size: @@ -79,9 +80,8 @@ def _unzstd(data: bytes, *, max_size: int = 0) -> bytes: output_stream = BytesIO() output_chunk = b"." decompressed_size = 0 - CHUNK_SIZE = 8196 while output_chunk: - output_chunk = stream_reader.read(CHUNK_SIZE) + output_chunk = stream_reader.read(_CHUNK_SIZE) decompressed_size += len(output_chunk) if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 5d23e8f05..cf7316e82 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -4,7 +4,7 @@ from io import BytesIO from scrapy.http import Response -from ._compression import _DecompressionMaxSizeExceeded +from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded def gunzip(data: bytes, *, max_size: int = 0) -> bytes: @@ -18,7 +18,7 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: decompressed_size = 0 while chunk: try: - chunk = f.read1(8196) + chunk = f.read1(_CHUNK_SIZE) except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error From 8a73c6c90c5984292d39a0a9d4be86e792d81cb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 24 Nov 2023 10:25:01 +0100 Subject: [PATCH 1268/2083] Fix HttpCompressionMiddleware backward compatibility --- scrapy/downloadermiddlewares/httpcompression.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 1a3f6962a..58ca1017f 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -39,8 +39,9 @@ class HttpCompressionMiddleware: def __init__(self, stats=None, *, crawler=None): if not crawler: - if stats: - self.stats = stats + self.stats = stats + self._max_size = 1073741824 + self._warn_size = 33554432 return self.stats = crawler.stats self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") From a113208a0643263fdd7198238bf9213f9148bc1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 24 Nov 2023 11:35:15 +0100 Subject: [PATCH 1269/2083] Fix BytesIO non-emptiness check --- scrapy/utils/gz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index cf7316e82..2e487d88b 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -23,7 +23,7 @@ def gunzip(data: bytes, *, max_size: int = 0) -> bytes: # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output_stream is empty - if output_stream: + if output_stream.getbuffer().nbytes > 0: break raise decompressed_size += len(chunk) From c947f51077e1ab246f30764cc4cc7a1cc5835d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 29 Nov 2023 11:54:08 +0100 Subject: [PATCH 1270/2083] Set an arbitrary upper limit on ReDoS-vulnerable regexps --- docs/news.rst | 28 ++++++++++++++++++++++++++++ scrapy/utils/iterators.py | 12 +++++++----- scrapy/utils/response.py | 4 ++-- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index fd8fa3ea3..121cc0322 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,22 @@ Release notes ============= +.. _release-2.11.1: + +Scrapy 2.11.1 (unreleased) +-------------------------- + +**Security bug fix:** + +- The regular expressions of the ``iternodes`` node iterator of + :class:`~scrapy.spiders.XMLFeedSpider` are no longer susceptible to a + `ReDoS attack`_. Please, see the `cc65-xxvf-f7r9 security + advisory`_ for more information. + + .. _ReDoS attack: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS + .. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9 + + .. _release-2.11.0: Scrapy 2.11.0 (2023-09-18) @@ -2869,6 +2885,18 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.4: + +Scrapy 1.8.4 (unreleased) +------------------------- + +**Security bug fix:** + +- The regular expressions of the ``iternodes`` node iterator of + :class:`~scrapy.spiders.XMLFeedSpider` are no longer susceptible to a + `ReDoS attack`_. Please, see the `cc65-xxvf-f7r9 security + advisory`_ for more information. + .. _release-1.8.3: diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 03d779afb..6b89334e0 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -40,10 +40,10 @@ def xmliter( """ nodename_patt = re.escape(nodename) - DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) + DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]{1,1024}>\s*", re.S) HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S) - END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S) - NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) + END_TAG_RE = re.compile(r"<\s*/([^\s>]{1,1024})\s*>", re.S) + NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]{,1024})=[^>\s]+)", re.S) text = _body_or_str(obj) document_header_match = re.search(DOCUMENT_HEADER_RE, text) @@ -57,13 +57,15 @@ def xmliter( for tagname in reversed(re.findall(END_TAG_RE, header_end)): assert header_end_idx tag = re.search( - rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S + rf"<\s*{tagname}.{{,1024}}?xmlns[:=][^>]{{,1024}}>", + text[: header_end_idx[1]], + re.S, ) if tag: for x in re.findall(NAMESPACE_RE, tag.group()): namespaces[x[1]] = x[0] - r = re.compile(rf"<{nodename_patt}[\s>].*?", re.DOTALL) + r = re.compile(rf"<{nodename_patt}[\s>].{{,1024}}?", re.DOTALL) for match in r.finditer(text): nodetext = ( document_header diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c540d6278..b0e106c5e 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -91,8 +91,8 @@ def open_in_browser( if isinstance(response, HtmlResponse): if b"' - body = re.sub(b"", b"", body, flags=re.DOTALL) - body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) + body = re.sub(b"", b"", body, flags=re.DOTALL) + body = re.sub(rb"(|\s.{,1024}?>))", to_bytes(repl), body) ext = ".html" elif isinstance(response, TextResponse): ext = ".txt" From eb8b2c5197f3dd8570bad1945b23886ee57d045f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 29 Nov 2023 12:13:04 +0100 Subject: [PATCH 1271/2083] Mention open_in_browser in the release notes --- docs/news.rst | 16 ++++++++-------- docs/topics/debug.rst | 18 +++--------------- scrapy/utils/response.py | 17 +++++++++++++++-- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 121cc0322..2bbe833a5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,10 +10,10 @@ Scrapy 2.11.1 (unreleased) **Security bug fix:** -- The regular expressions of the ``iternodes`` node iterator of - :class:`~scrapy.spiders.XMLFeedSpider` are no longer susceptible to a - `ReDoS attack`_. Please, see the `cc65-xxvf-f7r9 security - advisory`_ for more information. +- Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the + ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider` and + the :func:`~scrapy.utils.response.open_in_browser` function. Please, see + the `cc65-xxvf-f7r9 security advisory`_ for more information. .. _ReDoS attack: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS .. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9 @@ -2892,10 +2892,10 @@ Scrapy 1.8.4 (unreleased) **Security bug fix:** -- The regular expressions of the ``iternodes`` node iterator of - :class:`~scrapy.spiders.XMLFeedSpider` are no longer susceptible to a - `ReDoS attack`_. Please, see the `cc65-xxvf-f7r9 security - advisory`_ for more information. +- Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the + ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider` and + the :func:`~scrapy.utils.response.open_in_browser` function. Please, see + the `cc65-xxvf-f7r9 security advisory`_ for more information. .. _release-1.8.3: diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 49c5b0410..988e37bbd 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -125,25 +125,15 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see See also: :ref:`topics-shell-inspect-response`. + Open in browser =============== Sometimes you just want to see how a certain response looks in a browser, you -can use the ``open_in_browser`` function for that. Here is an example of how -you would use it: +can use the :func:`~scrapy.utils.response.open_in_browser` function for that: -.. code-block:: python +.. autofunction:: scrapy.utils.response.open_in_browser - from scrapy.utils.response import open_in_browser - - - def parse_details(self, response): - if "item name" not in response.body: - open_in_browser(response) - -``open_in_browser`` will open a browser with the response received by Scrapy at -that point, adjusting the `base tag`_ so that images and styles are displayed -properly. Logging ======= @@ -163,8 +153,6 @@ available in all future runs should they be necessary again: For more information, check the :ref:`topics-logging` section. -.. _base tag: https://www.w3schools.com/tags/tag_base.asp - .. _debug-vscode: Visual Studio Code diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b0e106c5e..8401d4ed1 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -81,8 +81,21 @@ def open_in_browser( ], _openfunc: Callable[[str], Any] = webbrowser.open, ) -> Any: - """Open the given response in a local web browser, populating the - tag for external links to work + """Open *response* in a local web browser, adjusting the `base tag`_ for + external links to work, e.g. so that images and styles are displayed. + + .. _base tag: https://www.w3schools.com/tags/tag_base.asp + + For example: + + .. code-block:: python + + from scrapy.utils.response import open_in_browser + + + def parse_details(self, response): + if "item name" not in response.body: + open_in_browser(response) """ from scrapy.http import HtmlResponse, TextResponse From 40b3efbbee3919e8f6900daeaed5e14183cabfd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 29 Nov 2023 12:47:04 +0100 Subject: [PATCH 1272/2083] Remove open_in_browser from the 1.8.4 release notes --- docs/news.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 2bbe833a5..c14815d06 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -2893,9 +2893,8 @@ Scrapy 1.8.4 (unreleased) **Security bug fix:** - Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the - ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider` and - the :func:`~scrapy.utils.response.open_in_browser` function. Please, see - the `cc65-xxvf-f7r9 security advisory`_ for more information. + ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider`. + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. .. _release-1.8.3: From 731f7495563591f1b76b1b2ae83f07d2239b7897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Nov 2023 10:54:09 +0100 Subject: [PATCH 1273/2083] Extend Request.meta documentation (#5565) --- docs/topics/request-response.rst | 47 ++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index adf3d0f4a..8edf710bc 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -193,18 +193,47 @@ Request objects :meth:`replace`. .. attribute:: Request.meta + :value: {} - A dict that contains arbitrary metadata for this request. This dict is - empty for new Requests, and is usually populated by different Scrapy - components (extensions, middlewares, etc). So the data contained in this - dict depends on the extensions you have enabled. + A dictionary of arbitrary metadata for the request. - See :ref:`topics-request-meta` for a list of special meta keys - recognized by Scrapy. + You may extend request metadata as you see fit. - This dict is :doc:`shallow copied ` when the request is - cloned using the ``copy()`` or ``replace()`` methods, and can also be - accessed, in your spider, from the ``response.meta`` attribute. + Request metadata can also be accessed through the + :attr:`~scrapy.http.Response.meta` attribute of a response. + + To pass data from one spider callback to another, consider using + :attr:`cb_kwargs` instead. However, request metadata may be the right + choice in certain scenarios, such as to maintain some debugging data + across all follow-up requests (e.g. the source URL). + + A common use of request metadata is to define request-specific + parameters for Scrapy components (extensions, middlewares, etc.). For + example, if you set ``dont_retry`` to ``True``, + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` will never + retry that request, even if it fails. See :ref:`topics-request-meta`. + + You may also use request metadata in your custom Scrapy components, for + example, to keep request state information relevant to your component. + For example, + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses the + ``retry_times`` metadata key to keep track of how many times a request + has been retried so far. + + Copying all the metadata of a previous request into a new, follow-up + request in a spider callback is a bad practice, because request + metadata may include metadata set by Scrapy components that is not + meant to be copied into other requests. For example, copying the + ``retry_times`` metadata key into follow-up requests can lower the + amount of retries allowed for those follow-up requests. + + You should only copy all request metadata from one request to another + if the new request is meant to replace the old request, as is often the + case when returning a request from a :ref:`downloader middleware + ` method. + + Also mind that the :meth:`copy` and :meth:`replace` request methods + :doc:`shallow-copy ` request metadata. .. attribute:: Request.cb_kwargs From 70ba3a0868dd6d9b996beba7ff5e8ec62773b206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Nov 2023 11:01:22 +0100 Subject: [PATCH 1274/2083] =?UTF-8?q?SPM=20=E2=86=92=20Zyte=20API=20(#6163?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/practices.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index f64da22d8..b1b8c9e9c 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -288,9 +288,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * use a pool of rotating IPs. For example, the free `Tor project`_ or paid services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. -* use a highly distributed downloader that circumvents bans internally, so you - can just focus on parsing clean pages. One example of such downloaders is - `Zyte Smart Proxy Manager`_ +* use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy + plugin `__ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. @@ -301,4 +300,4 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _Common Crawl: https://commoncrawl.org/ .. _testspiders: https://github.com/scrapinghub/testspiders .. _scrapoxy: https://scrapoxy.io/ -.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ +.. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html From c66b51770637d3d72347ab41a201f672618aeaf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Dec 2023 10:36:27 +0100 Subject: [PATCH 1275/2083] Add Python 3.13 alpha to CI --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 2 +- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 9 ++++++--- .github/workflows/tests-windows.yml | 5 ++++- conftest.py | 2 -- setup.py | 1 + 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d6fc0f6c5..7a380a7a5 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.12" + - python-version: "3.13.0-alpha.2" env: TOXENV: pylint - python-version: 3.8 @@ -21,7 +21,7 @@ jobs: - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs - - python-version: "3.12" + - python-version: "3.13.0-alpha.2" env: TOXENV: twinecheck diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index affaa32a5..456c0ffdd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: 3.12 + python-version: "3.13.0-alpha.2" - run: | pip install --upgrade build twine python -m build diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 252176464..6b110b5d7 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13.0-alpha.2"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f50a4d104..fd08247e4 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -24,7 +24,10 @@ jobs: - python-version: "3.12" env: TOXENV: py - - python-version: "3.12" + - python-version: "3.13.0-alpha.2" + env: + TOXENV: py + - python-version: "3.13.0-alpha.2" env: TOXENV: asyncio - python-version: pypy3.9 @@ -51,10 +54,10 @@ jobs: env: TOXENV: botocore-pinned - - python-version: "3.12" + - python-version: "3.13.0-alpha.2" env: TOXENV: extra-deps - - python-version: "3.12" + - python-version: "3.13.0-alpha.2" env: TOXENV: botocore diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 757d62285..be082393e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -27,7 +27,10 @@ jobs: - python-version: "3.12" env: TOXENV: py - - python-version: "3.12" + - python-version: "3.13.0-alpha.2" + env: + TOXENV: py + - python-version: "3.13.0-alpha.2" env: TOXENV: asyncio diff --git a/conftest.py b/conftest.py index 2bfa46f5a..68921f119 100644 --- a/conftest.py +++ b/conftest.py @@ -91,8 +91,6 @@ def requires_uvloop(request): pytest.skip("uvloop does not support Windows") if twisted_version == Version("twisted", 21, 2, 0): pytest.skip("https://twistedmatrix.com/trac/ticket/10106") - if sys.version_info >= (3, 12): - pytest.skip("uvloop doesn't support Python 3.12 yet") def pytest_configure(config): diff --git a/setup.py b/setup.py index 405633f55..d6ba4765e 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,7 @@ setup( "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Internet :: WWW/HTTP", From bb74badd1bd66c59a63268c52342507c76d290b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 11 Dec 2023 17:39:55 +0100 Subject: [PATCH 1276/2083] =?UTF-8?q?spider=20=E2=86=92=20mw?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/downloadermiddlewares/httpcompression.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 58ca1017f..816be25a1 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -61,12 +61,12 @@ class HttpCompressionMiddleware: "reimplement their 'from_crawler' method.", ScrapyDeprecationWarning, ) - spider = cls() - spider.stats = crawler.stats - spider._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") - spider._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") - crawler.signals.connect(spider.open_spider, signals.spider_opened) - return spider + mw = cls() + mw.stats = crawler.stats + mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") + mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") + crawler.signals.connect(mw.open_spider, signals.spider_opened) + return mw def open_spider(self, spider): if hasattr(spider, "download_maxsize"): From 1533b69032e2fb5e495e88a3fed57c0d98502612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 13 Dec 2023 12:01:35 +0100 Subject: [PATCH 1277/2083] Test and address ReDoS attack vectors for open_in_browser --- scrapy/utils/response.py | 6 +++--- tests/test_utils_response.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 8401d4ed1..4369e6439 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -103,9 +103,9 @@ def open_in_browser( body = response.body if isinstance(response, HtmlResponse): if b"' - body = re.sub(b"", b"", body, flags=re.DOTALL) - body = re.sub(rb"(|\s.{,1024}?>))", to_bytes(repl), body) + repl = rf'\0' + body = re.sub(b"(?s)|$)", b"", body) + body = re.sub(rb"]*?>)", to_bytes(repl), body, count=1) ext = ".html" elif isinstance(response, TextResponse): ext = ".txt" diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 80e15a60f..942584d92 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,10 +1,12 @@ import unittest import warnings from pathlib import Path +from time import process_time from urllib.parse import urlparse from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response, TextResponse +from scrapy.settings.default_settings import DOWNLOAD_MAXSIZE from scrapy.utils.python import to_bytes from scrapy.utils.response import ( get_base_url, @@ -198,3 +200,37 @@ class ResponseUtilsTest(unittest.TestCase): assert open_in_browser( r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" + + def test_open_in_browser_redos_comment(self): + MAX_CPU_TIME = 30 + + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for // (old pattern to remove comments). + body = b"->" + + response = HtmlResponse("https://example.com", body=body) + + start_time = process_time() + + open_in_browser(response, lambda url: True) + + end_time = process_time() + self.assertLess(end_time - start_time, MAX_CPU_TIME) + + def test_open_in_browser_redos_head(self): + MAX_CPU_TIME = 15 + + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for /(|\s.*?>))/ (old pattern to find the head element). + body = b" Date: Tue, 17 Oct 2023 17:49:22 -0300 Subject: [PATCH 1278/2083] Remove deprecated scrapy.downloadermiddlewares.decompression --- scrapy/downloadermiddlewares/decompression.py | 94 ------------------- ...test_downloadermiddleware_decompression.py | 53 ----------- 2 files changed, 147 deletions(-) delete mode 100644 scrapy/downloadermiddlewares/decompression.py delete mode 100644 tests/test_downloadermiddleware_decompression.py diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py deleted file mode 100644 index 3b8702419..000000000 --- a/scrapy/downloadermiddlewares/decompression.py +++ /dev/null @@ -1,94 +0,0 @@ -""" This module implements the DecompressionMiddleware which tries to recognise -and extract the potentially compressed responses that may arrive. -""" - -import bz2 -import gzip -import logging -import tarfile -import zipfile -from io import BytesIO -from tempfile import mktemp -from warnings import warn - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.responsetypes import responsetypes - -warn( - "scrapy.downloadermiddlewares.decompression is deprecated", - ScrapyDeprecationWarning, - stacklevel=2, -) - - -logger = logging.getLogger(__name__) - - -class DecompressionMiddleware: - """This middleware tries to recognise and extract the possibly compressed - responses that may arrive.""" - - def __init__(self): - self._formats = { - "tar": self._is_tar, - "zip": self._is_zip, - "gz": self._is_gzip, - "bz2": self._is_bzip2, - } - - def _is_tar(self, response): - archive = BytesIO(response.body) - try: - tar_file = tarfile.open(name=mktemp(), fileobj=archive) - except tarfile.ReadError: - return - - body = tar_file.extractfile(tar_file.members[0]).read() - respcls = responsetypes.from_args(filename=tar_file.members[0].name, body=body) - return response.replace(body=body, cls=respcls) - - def _is_zip(self, response): - archive = BytesIO(response.body) - try: - zip_file = zipfile.ZipFile(archive) - except zipfile.BadZipFile: - return - - namelist = zip_file.namelist() - body = zip_file.read(namelist[0]) - respcls = responsetypes.from_args(filename=namelist[0], body=body) - return response.replace(body=body, cls=respcls) - - def _is_gzip(self, response): - archive = BytesIO(response.body) - try: - body = gzip.GzipFile(fileobj=archive).read() - except OSError: - return - - respcls = responsetypes.from_args(body=body) - return response.replace(body=body, cls=respcls) - - def _is_bzip2(self, response): - try: - body = bz2.decompress(response.body) - except OSError: - return - - respcls = responsetypes.from_args(body=body) - return response.replace(body=body, cls=respcls) - - def process_response(self, request, response, spider): - if not response.body: - return response - - for fmt, func in self._formats.items(): - new_response = func(response) - if new_response: - logger.debug( - "Decompressed response with format: %(responsefmt)s", - {"responsefmt": fmt}, - extra={"spider": spider}, - ) - return new_response - return response diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py deleted file mode 100644 index 95739414e..000000000 --- a/tests/test_downloadermiddleware_decompression.py +++ /dev/null @@ -1,53 +0,0 @@ -from unittest import TestCase, main - -from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware -from scrapy.http import Response, XmlResponse -from scrapy.spiders import Spider -from scrapy.utils.test import assert_samelines -from tests import get_testdata - - -def _test_data(formats): - uncompressed_body = get_testdata("compressed", "feed-sample1.xml") - test_responses = {} - for format in formats: - body = get_testdata("compressed", "feed-sample1." + format) - test_responses[format] = Response("http://foo.com/bar", body=body) - return uncompressed_body, test_responses - - -class DecompressionMiddlewareTest(TestCase): - test_formats = ["tar", "xml.bz2", "xml.gz", "zip"] - uncompressed_body, test_responses = _test_data(test_formats) - - def setUp(self): - self.mw = DecompressionMiddleware() - self.spider = Spider("foo") - - def test_known_compression_formats(self): - for fmt in self.test_formats: - rsp = self.test_responses[fmt] - new = self.mw.process_response(None, rsp, self.spider) - error_msg = f"Failed {fmt}, response type {type(new).__name__}" - assert isinstance(new, XmlResponse), error_msg - assert_samelines(self, new.body, self.uncompressed_body, fmt) - - def test_plain_response(self): - rsp = Response(url="http://test.com", body=self.uncompressed_body) - new = self.mw.process_response(None, rsp, self.spider) - assert new is rsp - assert_samelines(self, new.body, rsp.body) - - def test_empty_response(self): - rsp = Response(url="http://test.com", body=b"") - new = self.mw.process_response(None, rsp, self.spider) - assert new is rsp - assert not rsp.body - assert not new.body - - def tearDown(self): - del self.mw - - -if __name__ == "__main__": - main() From 12b10a7a6427c43968cc18d98a3ed3c6366eeabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 13 Dec 2023 13:35:05 +0100 Subject: [PATCH 1279/2083] Cover scrapy.downloadermiddlewares.decompression in the release notes --- docs/news.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c4081b99b..a12bda53f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -16,6 +16,10 @@ Scrapy 2.11.1 (unreleased) .. _7j7m-v7m3-jqm7 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-7j7m-v7m3-jqm7 +- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, the + deprecated ``scrapy.downloadermiddlewares.decompression`` module has been + removed. + .. _release-2.11.0: @@ -2896,6 +2900,10 @@ Scrapy 1.8.4 (unreleased) to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security advisory`_ for more information. +- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, use of the + ``scrapy.downloadermiddlewares.decompression`` module is discouraged and + will trigger a warning. + .. _release-1.8.3: From 4f72b49f975a406784779dc19ede31364f92235f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 10:06:13 +0100 Subject: [PATCH 1280/2083] Fix namespaces nodename support for xmliter_lxml --- scrapy/utils/iterators.py | 23 +++++++++++++++++++++-- tests/test_utils_iterators.py | 5 ----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 6b89334e0..9c53ab524 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -12,11 +12,14 @@ from typing import ( List, Literal, Optional, + Tuple, Union, cast, overload, ) +from lxml import etree + from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode @@ -77,15 +80,31 @@ def xmliter( yield Selector(text=nodetext, type="xml") +def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: + if ":" not in element_name: + return element_name, None, None + reader: "SupportsReadClose[bytes]" = _StreamReader(data) + node_prefix, element_name = element_name.split(":", maxsplit=1) + ns_iterator = etree.iterparse( + reader, encoding=reader.encoding, events=("start-ns",) + ) + for event, (_prefix, _namespace) in ns_iterator: + if _prefix != node_prefix: + continue + return element_name, _prefix, _namespace + return f"{node_prefix}:{element_name}", None, None + + def xmliter_lxml( obj: Union[Response, str, bytes], nodename: str, namespace: Optional[str] = None, prefix: str = "x", ) -> Generator[Selector, Any, None]: - from lxml import etree + if not namespace: + nodename, prefix, namespace = _resolve_xml_namespace(nodename, obj) - reader = _StreamReader(obj) + reader: "SupportsReadClose[bytes]" = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 3598fa0bb..24f03155b 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,4 +1,3 @@ -from pytest import mark from twisted.trial import unittest from scrapy.http import Response, TextResponse, XmlResponse @@ -247,10 +246,6 @@ class XmliterTestCase(unittest.TestCase): class LxmlXmliterTestCase(XmliterTestCase): xmliter = staticmethod(xmliter_lxml) - @mark.xfail(reason="known bug of the current implementation") - def test_xmliter_namespaced_nodename(self): - super().test_xmliter_namespaced_nodename() - def test_xmliter_iterate_namespace(self): body = b""" From d50f436a73ef13fca8d3d9c302ae1a48e984a4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 10:08:45 +0100 Subject: [PATCH 1281/2083] Enable huge_tree for xmliter_lxml --- scrapy/utils/iterators.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 9c53ab524..1c51c0c6a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -86,7 +86,10 @@ def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: reader: "SupportsReadClose[bytes]" = _StreamReader(data) node_prefix, element_name = element_name.split(":", maxsplit=1) ns_iterator = etree.iterparse( - reader, encoding=reader.encoding, events=("start-ns",) + reader, + encoding=reader.encoding, + events=("start-ns",), + huge_tree=True, ) for event, (_prefix, _namespace) in ns_iterator: if _prefix != node_prefix: @@ -107,7 +110,10 @@ def xmliter_lxml( reader: "SupportsReadClose[bytes]" = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( - cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding + reader, + tag=tag, + encoding=reader.encoding, + huge_tree=True, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: From 9655b0b8eb4bbc66b0fe540a19265b6342cf371b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 10:19:47 +0100 Subject: [PATCH 1282/2083] Mark slow tests, with their own tox env and CI job --- .github/workflows/tests-ubuntu.yml | 6 ++++ pytest.ini | 2 ++ tests/test_utils_response.py | 50 +++++++++++++++++------------- tox.ini | 6 ++++ 4 files changed, 42 insertions(+), 22 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 5ff92a571..7562cf22b 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -47,6 +47,9 @@ jobs: - python-version: "3.11" env: TOXENV: botocore + - python-version: "3.11" + env: + TOXENV: slow - python-version: "3.12.0-rc.2" env: @@ -57,6 +60,9 @@ jobs: - python-version: "3.12.0-rc.2" env: TOXENV: extra-deps + - python-version: "3.12.0-rc.2" + env: + TOXENV: slow steps: - uses: actions/checkout@v3 diff --git a/pytest.ini b/pytest.ini index 16983be5e..877fbcd1d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -17,10 +17,12 @@ addopts = --ignore=docs/topics/stats.rst --ignore=docs/topics/telnetconsole.rst --ignore=docs/utils + -m 'not slow' markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed requires_uvloop: marks tests as only enabled when uvloop is known to be working + slow: marks tests as slow, not executed by default filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 942584d92..93b9bacaf 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -4,6 +4,8 @@ from pathlib import Path from time import process_time from urllib.parse import urlparse +import pytest + from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.settings.default_settings import DOWNLOAD_MAXSIZE @@ -201,36 +203,40 @@ class ResponseUtilsTest(unittest.TestCase): r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" - def test_open_in_browser_redos_comment(self): - MAX_CPU_TIME = 30 - # Exploit input from - # https://makenowjust-labs.github.io/recheck/playground/ - # for // (old pattern to remove comments). - body = b"->" +@pytest.mark.slow +def test_open_in_browser_redos_comment(): + MAX_CPU_TIME = 30 - response = HtmlResponse("https://example.com", body=body) + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for // (old pattern to remove comments). + body = b"->" - start_time = process_time() + response = HtmlResponse("https://example.com", body=body) - open_in_browser(response, lambda url: True) + start_time = process_time() - end_time = process_time() - self.assertLess(end_time - start_time, MAX_CPU_TIME) + open_in_browser(response, lambda url: True) - def test_open_in_browser_redos_head(self): - MAX_CPU_TIME = 15 + end_time = process_time() + assert (end_time - start_time) < MAX_CPU_TIME - # Exploit input from - # https://makenowjust-labs.github.io/recheck/playground/ - # for /(|\s.*?>))/ (old pattern to find the head element). - body = b"|\s.*?>))/ (old pattern to find the head element). + body = b" Date: Fri, 15 Dec 2023 10:23:24 +0100 Subject: [PATCH 1283/2083] Restore the implementation of xmliter --- scrapy/utils/iterators.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 1c51c0c6a..b67029433 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -43,10 +43,10 @@ def xmliter( """ nodename_patt = re.escape(nodename) - DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]{1,1024}>\s*", re.S) + DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S) - END_TAG_RE = re.compile(r"<\s*/([^\s>]{1,1024})\s*>", re.S) - NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]{,1024})=[^>\s]+)", re.S) + END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S) + NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) text = _body_or_str(obj) document_header_match = re.search(DOCUMENT_HEADER_RE, text) @@ -60,15 +60,13 @@ def xmliter( for tagname in reversed(re.findall(END_TAG_RE, header_end)): assert header_end_idx tag = re.search( - rf"<\s*{tagname}.{{,1024}}?xmlns[:=][^>]{{,1024}}>", - text[: header_end_idx[1]], - re.S, + rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S ) if tag: for x in re.findall(NAMESPACE_RE, tag.group()): namespaces[x[1]] = x[0] - r = re.compile(rf"<{nodename_patt}[\s>].{{,1024}}?", re.DOTALL) + r = re.compile(rf"<{nodename_patt}[\s>].*?", re.DOTALL) for match in r.finditer(text): nodetext = ( document_header From 150d96764b5a455c75315596ca8ba5ded0f416dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 11:42:55 +0100 Subject: [PATCH 1284/2083] Deprecate xmliter in favor of xmliter_lxml --- docs/faq.rst | 10 ++++--- docs/news.rst | 32 +++++++++++++++++----- scrapy/spiders/feed.py | 4 +-- scrapy/utils/iterators.py | 23 ++++++++++++++-- tests/test_utils_iterators.py | 35 +++++++++++++++++++++--- tests/test_utils_response.py | 50 +++++++++++++++++------------------ 6 files changed, 110 insertions(+), 44 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 20dd814df..657802fd3 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -297,9 +297,13 @@ build the DOM of the entire feed in memory, and this can be quite slow and consume a lot of memory. In order to avoid parsing all the entire feed at once in memory, you can use -the functions ``xmliter`` and ``csviter`` from ``scrapy.utils.iterators`` -module. In fact, this is what the feed spiders (see :ref:`topics-spiders`) use -under the cover. +the :func:`~scrapy.utils.iterators.xmliter_lxml` and +:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what +:class:`~scrapy.spiders.XMLFeedSpider` uses. + +.. autofunction:: scrapy.utils.iterators.xmliter_lxml + +.. autofunction:: scrapy.utils.iterators.csviter Does Scrapy manage cookies automatically? ----------------------------------------- diff --git a/docs/news.rst b/docs/news.rst index c14815d06..57b99e94f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,12 +10,23 @@ Scrapy 2.11.1 (unreleased) **Security bug fix:** -- Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the - ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider` and - the :func:`~scrapy.utils.response.open_in_browser` function. Please, see - the `cc65-xxvf-f7r9 security advisory`_ for more information. +- Addressed `ReDoS vulnerabilities`_: - .. _ReDoS attack: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS + - ``scrapy.utils.iterators.xmliter`` is now deprecated in favor of + :func:`~scrapy.utils.iterators.xmliter_lxml`, which + :class:`~scrapy.spiders.XMLFeedSpider` now uses. + + To minimize the impact of this change on existing code, + :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating + the node namespace with a prefix in the node name, and big files with + highly nested trees. + + - Fixed regular expressions in the implementation of the + :func:`~scrapy.utils.response.open_in_browser` function. + + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. + + .. _ReDoS vulnerabilities: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS .. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9 @@ -2892,8 +2903,15 @@ Scrapy 1.8.4 (unreleased) **Security bug fix:** -- Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the - ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider`. +- Due to its `ReDoS vulnerabilities`_, ``scrapy.utils.iterators.xmliter`` is + now deprecated in favor of :func:`~scrapy.utils.iterators.xmliter_lxml`, + which :class:`~scrapy.spiders.XMLFeedSpider` now uses. + + To minimize the impact of this change on existing code, + :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating + the node namespace as a prefix in the node name, and big files with highly + nested trees when using lxml 4.2 or later. + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 6afadc577..42675c76a 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst from scrapy.exceptions import NotConfigured, NotSupported from scrapy.selector import Selector from scrapy.spiders import Spider -from scrapy.utils.iterators import csviter, xmliter +from scrapy.utils.iterators import csviter, xmliter_lxml from scrapy.utils.spider import iterate_spider_output @@ -84,7 +84,7 @@ class XMLFeedSpider(Spider): return self.parse_nodes(response, nodes) def _iternodes(self, response): - for node in xmliter(response, self.itertag): + for node in xmliter_lxml(response, self.itertag): self._register_namespaces(node) yield node diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index b67029433..7574e377a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -17,9 +17,12 @@ from typing import ( cast, overload, ) +from warnings import warn from lxml import etree +from packaging.version import Version +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode @@ -29,6 +32,12 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_LXML_VERSION = Version(etree.__version__) +_LXML_HUGE_TREE_VERSION = Version("4.2") +_ITERPARSE_KWARGS = {} +if _LXML_VERSION >= _LXML_HUGE_TREE_VERSION: + _ITERPARSE_KWARGS["huge_tree"] = True + def xmliter( obj: Union[Response, str, bytes], nodename: str @@ -41,6 +50,16 @@ def xmliter( - a unicode string - a string encoded as utf-8 """ + warn( + ( + "xmliter is deprecated and its use strongly discouraged because " + "it is vulnerable to ReDoS attacks. Use xmliter_lxml instead. See " + "https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9" + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + nodename_patt = re.escape(nodename) DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) @@ -87,7 +106,7 @@ def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: reader, encoding=reader.encoding, events=("start-ns",), - huge_tree=True, + **_ITERPARSE_KWARGS, ) for event, (_prefix, _namespace) in ns_iterator: if _prefix != node_prefix: @@ -111,7 +130,7 @@ def xmliter_lxml( reader, tag=tag, encoding=reader.encoding, - huge_tree=True, + **_ITERPARSE_KWARGS, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 24f03155b..505cc276c 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,13 +1,14 @@ +import pytest from twisted.trial import unittest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse, XmlResponse from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata -class XmliterTestCase(unittest.TestCase): - xmliter = staticmethod(xmliter) - +class XmliterBaseTestCase: + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter(self): body = b""" @@ -39,6 +40,7 @@ class XmliterTestCase(unittest.TestCase): attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])] ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unusual_node(self): body = b""" @@ -52,6 +54,7 @@ class XmliterTestCase(unittest.TestCase): ] self.assertEqual(nodenames, [["matchme..."]]) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 body = """ @@ -111,6 +114,7 @@ class XmliterTestCase(unittest.TestCase): [("26", ["-"], ["80"]), ("21", ["Ab"], ["76"]), ("27", ["A"], ["27"])], ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_text(self): body = ( '' @@ -122,6 +126,7 @@ class XmliterTestCase(unittest.TestCase): [["one"], ["two"]], ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaces(self): body = b""" @@ -161,6 +166,7 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath("id/text()").getall(), []) self.assertEqual(node.xpath("price/text()").getall(), []) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename(self): body = b""" @@ -189,6 +195,7 @@ class XmliterTestCase(unittest.TestCase): ["http://www.mydummycompany.com/images/item1.jpg"], ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename_missing(self): body = b""" @@ -213,6 +220,7 @@ class XmliterTestCase(unittest.TestCase): with self.assertRaises(StopIteration): next(my_iter) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_exception(self): body = ( '' @@ -225,10 +233,12 @@ class XmliterTestCase(unittest.TestCase): self.assertRaises(StopIteration, next, iter) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") self.assertRaises(TypeError, next, i) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_encoding(self): body = ( b'\n' @@ -243,7 +253,24 @@ class XmliterTestCase(unittest.TestCase): ) -class LxmlXmliterTestCase(XmliterTestCase): +class XmliterTestCase(XmliterBaseTestCase, unittest.TestCase): + xmliter = staticmethod(xmliter) + + def test_deprecation(self): + body = b""" + + + + + """ + with pytest.warns( + ScrapyDeprecationWarning, + match="xmliter", + ): + next(self.xmliter(body, "product")) + + +class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): xmliter = staticmethod(xmliter_lxml) def test_xmliter_iterate_namespace(self): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 93b9bacaf..1dbe187bf 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -203,40 +203,38 @@ class ResponseUtilsTest(unittest.TestCase): r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" + @pytest.mark.slow + def test_open_in_browser_redos_comment(self): + MAX_CPU_TIME = 30 -@pytest.mark.slow -def test_open_in_browser_redos_comment(): - MAX_CPU_TIME = 30 + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for // (old pattern to remove comments). + body = b"->" - # Exploit input from - # https://makenowjust-labs.github.io/recheck/playground/ - # for // (old pattern to remove comments). - body = b"->" + response = HtmlResponse("https://example.com", body=body) - response = HtmlResponse("https://example.com", body=body) + start_time = process_time() - start_time = process_time() + open_in_browser(response, lambda url: True) - open_in_browser(response, lambda url: True) + end_time = process_time() + self.assertLess(end_time - start_time, MAX_CPU_TIME) - end_time = process_time() - assert (end_time - start_time) < MAX_CPU_TIME + @pytest.mark.slow + def test_open_in_browser_redos_head(self): + MAX_CPU_TIME = 15 + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for /(|\s.*?>))/ (old pattern to find the head element). + body = b"|\s.*?>))/ (old pattern to find the head element). - body = b" Date: Fri, 15 Dec 2023 11:49:22 +0100 Subject: [PATCH 1285/2083] Minor naming changes --- scrapy/utils/iterators.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 7574e377a..f239630f4 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -101,18 +101,18 @@ def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: if ":" not in element_name: return element_name, None, None reader: "SupportsReadClose[bytes]" = _StreamReader(data) - node_prefix, element_name = element_name.split(":", maxsplit=1) + input_prefix, element_name = element_name.split(":", maxsplit=1) ns_iterator = etree.iterparse( reader, encoding=reader.encoding, events=("start-ns",), **_ITERPARSE_KWARGS, ) - for event, (_prefix, _namespace) in ns_iterator: - if _prefix != node_prefix: + for event, (prefix, namespace) in ns_iterator: + if prefix != input_prefix: continue - return element_name, _prefix, _namespace - return f"{node_prefix}:{element_name}", None, None + return element_name, prefix, namespace + return f"{input_prefix}:{element_name}", None, None def xmliter_lxml( From a49c8762dd163b60cc73c4486a662471cfa7ac7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 12:14:53 +0100 Subject: [PATCH 1286/2083] Avoid calling iterparse twice --- scrapy/utils/iterators.py | 42 +++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index f239630f4..8610e9b77 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -12,7 +12,6 @@ from typing import ( List, Literal, Optional, - Tuple, Union, cast, overload, @@ -97,43 +96,38 @@ def xmliter( yield Selector(text=nodetext, type="xml") -def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: - if ":" not in element_name: - return element_name, None, None - reader: "SupportsReadClose[bytes]" = _StreamReader(data) - input_prefix, element_name = element_name.split(":", maxsplit=1) - ns_iterator = etree.iterparse( - reader, - encoding=reader.encoding, - events=("start-ns",), - **_ITERPARSE_KWARGS, - ) - for event, (prefix, namespace) in ns_iterator: - if prefix != input_prefix: - continue - return element_name, prefix, namespace - return f"{input_prefix}:{element_name}", None, None - - def xmliter_lxml( obj: Union[Response, str, bytes], nodename: str, namespace: Optional[str] = None, prefix: str = "x", ) -> Generator[Selector, Any, None]: - if not namespace: - nodename, prefix, namespace = _resolve_xml_namespace(nodename, obj) - reader: "SupportsReadClose[bytes]" = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( reader, - tag=tag, encoding=reader.encoding, + events=("end", "start-ns"), **_ITERPARSE_KWARGS, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) - for _, node in iterable: + needs_namespace_resolution = not namespace and ":" in nodename + if needs_namespace_resolution: + prefix, nodename = nodename.split(":", maxsplit=1) + for event, data in iterable: + if event == "start-ns": + if needs_namespace_resolution: + _prefix, _namespace = data + if _prefix != prefix: + continue + namespace = _namespace + needs_namespace_resolution = False + selxpath = f"//{prefix}:{nodename}" + tag = f"{{{namespace}}}{nodename}" + continue + node = data + if node.tag != tag: + continue nodetext = etree.tostring(node, encoding="unicode") node.clear() xs = Selector(text=nodetext, type="xml") From ce9d290eff8b5992023ffa5e833881b83a0669c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 12:23:26 +0100 Subject: [PATCH 1287/2083] Remove the lxml version check for huge_tree on xmliter_lxml iterparse supports the option since lxml 2.2.1, it was the HTML parser that only got it in 4.2 --- docs/news.rst | 2 +- scrapy/utils/iterators.py | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 525ddbf40..fab8b6f20 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -19,7 +19,7 @@ Scrapy 2.11.1 (unreleased) To minimize the impact of this change on existing code, :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating the node namespace with a prefix in the node name, and big files with - highly nested trees. + highly nested trees when using libxml2 2.7+. - Fixed regular expressions in the implementation of the :func:`~scrapy.utils.response.open_in_browser` function. diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 8610e9b77..b6abe2e0c 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -19,7 +19,6 @@ from typing import ( from warnings import warn from lxml import etree -from packaging.version import Version from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse @@ -31,12 +30,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -_LXML_VERSION = Version(etree.__version__) -_LXML_HUGE_TREE_VERSION = Version("4.2") -_ITERPARSE_KWARGS = {} -if _LXML_VERSION >= _LXML_HUGE_TREE_VERSION: - _ITERPARSE_KWARGS["huge_tree"] = True - def xmliter( obj: Union[Response, str, bytes], nodename: str @@ -108,7 +101,7 @@ def xmliter_lxml( reader, encoding=reader.encoding, events=("end", "start-ns"), - **_ITERPARSE_KWARGS, + huge_tree=True, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) needs_namespace_resolution = not namespace and ":" in nodename From bc138ef8e958f4bac5a4413d40566efc2b59acfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 12:24:04 +0100 Subject: [PATCH 1288/2083] Minor release notes fix --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index fab8b6f20..f346d1239 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -2912,7 +2912,7 @@ Scrapy 1.8.4 (unreleased) To minimize the impact of this change on existing code, :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating the node namespace as a prefix in the node name, and big files with highly - nested trees when using lxml 4.2 or later. + nested trees when using libxml2 2.7+. Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. From c7c7a488b950806888691f58dda0b06478b98c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 13:18:23 +0100 Subject: [PATCH 1289/2083] Fix typing issues --- scrapy/utils/iterators.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index b6abe2e0c..ab48e525f 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -95,10 +95,10 @@ def xmliter_lxml( namespace: Optional[str] = None, prefix: str = "x", ) -> Generator[Selector, Any, None]: - reader: "SupportsReadClose[bytes]" = _StreamReader(obj) + reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( - reader, + cast("SupportsReadClose[bytes]", reader), encoding=reader.encoding, events=("end", "start-ns"), huge_tree=True, @@ -109,6 +109,7 @@ def xmliter_lxml( prefix, nodename = nodename.split(":", maxsplit=1) for event, data in iterable: if event == "start-ns": + assert isinstance(data, tuple) if needs_namespace_resolution: _prefix, _namespace = data if _prefix != prefix: @@ -118,6 +119,7 @@ def xmliter_lxml( selxpath = f"//{prefix}:{nodename}" tag = f"{{{namespace}}}{nodename}" continue + assert isinstance(data, etree._Element) node = data if node.tag != tag: continue From 27781a85e738052e0441c81d773b3ec124194594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 13:52:12 +0100 Subject: [PATCH 1290/2083] Fix bad closing tags in XMLFeedSpider tests --- tests/test_spider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 00da3d485..5c4007d87 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -149,10 +149,10 @@ class XMLFeedSpiderTest(SpiderTest): body = b""" - http://www.example.com/Special-Offers.html2009-08-16 + http://www.example.com/Special-Offers.html2009-08-16 - http://www.example.com/2009-08-16 + http://www.example.com/2009-08-16 """ response = XmlResponse(url="http://example.com/sitemap.xml", body=body) From c67f73069570dd6dbe8427f55d6f9e65af07132a Mon Sep 17 00:00:00 2001 From: Swati Anshu <97234193+sa2415@users.noreply.github.com> Date: Mon, 18 Dec 2023 05:51:02 -0500 Subject: [PATCH 1291/2083] =?UTF-8?q?create=5Finstance=20=E2=86=92=20build?= =?UTF-8?q?=5Ffrom=5Fcrawler,=20build=5Ffrom=5Fsettings=20(#6169)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/addons.py | 6 +- scrapy/core/downloader/contextfactory.py | 16 ++--- scrapy/core/downloader/handlers/__init__.py | 9 ++- scrapy/core/downloader/handlers/http10.py | 9 ++- scrapy/core/downloader/handlers/s3.py | 9 ++- scrapy/core/engine.py | 6 +- scrapy/core/scheduler.py | 14 ++-- scrapy/crawler.py | 9 ++- scrapy/extensions/feedexport.py | 4 +- scrapy/middleware.py | 7 +- scrapy/pqueues.py | 5 +- scrapy/utils/misc.py | 47 +++++++++++++ tests/test_addons.py | 6 +- tests/test_downloader_handlers.py | 53 ++++++--------- tests/test_downloader_handlers_http2.py | 6 +- tests/test_settings/__init__.py | 4 +- tests/test_utils_misc/__init__.py | 74 +++++++++++++++++++++ tests/test_webclient.py | 10 +-- 18 files changed, 197 insertions(+), 97 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 9060d4f3f..65d7a0310 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, List from scrapy.exceptions import NotConfigured from scrapy.settings import Settings from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler @@ -32,9 +32,7 @@ class AddonManager: for clspath in build_component_list(settings["ADDONS"]): try: addoncls = load_object(clspath) - addon = create_instance( - addoncls, settings=settings, crawler=self.crawler - ) + addon = build_from_crawler(addoncls, self.crawler) addon.update_settings(settings) self.addons.append(addon) except NotConfigured as e: diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 909cc273f..dba4d8cdc 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -20,7 +20,7 @@ from scrapy.core.downloader.tls import ( openssl_methods, ) from scrapy.settings import BaseSettings -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from twisted.internet._sslverify import ClientTLSOptions @@ -165,18 +165,16 @@ def load_context_factory_from_settings(settings, crawler): context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]) # try method-aware context factory try: - context_factory = create_instance( - objcls=context_factory_cls, - settings=settings, - crawler=crawler, + context_factory = build_from_crawler( + context_factory_cls, + crawler, method=ssl_method, ) except TypeError: # use context factory defaults - context_factory = create_instance( - objcls=context_factory_cls, - settings=settings, - crawler=crawler, + context_factory = build_from_crawler( + context_factory_cls, + crawler, ) msg = ( f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept " diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 6a211aafa..416669b7f 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -9,7 +9,7 @@ from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, NotSupported from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: @@ -55,10 +55,9 @@ class DownloadHandlers: dhcls = load_object(path) if skip_lazy and getattr(dhcls, "lazy", True): return None - dh = create_instance( - objcls=dhcls, - settings=self._crawler.settings, - crawler=self._crawler, + dh = build_from_crawler( + dhcls, + self._crawler, ) except NotConfigured as ex: self._notconfigured[scheme] = str(ex) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 6c1dac4a5..b6ac7a251 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,6 +1,6 @@ """Download handlers for http and https schemes """ -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode @@ -30,10 +30,9 @@ class HTTP10DownloadHandler: host, port = to_unicode(factory.host), factory.port if factory.scheme == b"https": - client_context_factory = create_instance( - objcls=self.ClientContextFactory, - settings=self._settings, - crawler=self._crawler, + client_context_factory = build_from_crawler( + self.ClientContextFactory, + self._crawler, ) return reactor.connectSSL(host, port, factory, client_context_factory) return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 81d8e8115..1f7533759 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,7 +2,7 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_crawler class S3DownloadHandler: @@ -50,10 +50,9 @@ class S3DownloadHandler: ) ) - _http_handler = create_instance( - objcls=httpdownloadhandler, - settings=settings, - crawler=crawler, + _http_handler = build_from_crawler( + httpdownloadhandler, + crawler, ) self._download_http = _http_handler.download_request diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index dd1f56f8c..545cd401f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -34,7 +34,7 @@ from scrapy.settings import BaseSettings, Settings from scrapy.signalmanager import SignalManager from scrapy.spiders import Spider from scrapy.utils.log import failure_to_exc_info, logformatter_adapter -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: @@ -358,9 +358,7 @@ class ExecutionEngine: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) nextcall = CallLaterOnce(self._next_request) - scheduler = create_instance( - self.scheduler_cls, settings=None, crawler=self.crawler - ) + scheduler = build_from_crawler(self.scheduler_cls, self.crawler) start_requests = yield self.scraper.spidermw.process_start_requests( start_requests, spider ) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 17c95f1ea..f41b83a67 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -14,7 +14,7 @@ from scrapy.http.request import Request from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.job import job_dir -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -202,7 +202,7 @@ class Scheduler(BaseScheduler): """ dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"]) return cls( - dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler), + dupefilter=build_from_crawler(dupefilter_cls, crawler), jobdir=job_dir(crawler.settings), dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]), mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]), @@ -322,10 +322,9 @@ class Scheduler(BaseScheduler): def _mq(self): """Create a new priority queue instance, with in-memory storage""" - return create_instance( + return build_from_crawler( self.pqclass, - settings=None, - crawler=self.crawler, + self.crawler, downstream_queue_cls=self.mqclass, key="", ) @@ -334,10 +333,9 @@ class Scheduler(BaseScheduler): """Create a new priority queue instance, with disk storage""" assert self.dqdir state = self._read_dqs_state(self.dqdir) - q = create_instance( + q = build_from_crawler( self.pqclass, - settings=None, - crawler=self.crawler, + self.crawler, downstream_queue_cls=self.dqclass, key=self.dqdir, startprios=state, diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1d3a11208..844d5f759 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -39,7 +39,7 @@ from scrapy.utils.log import ( log_reactor_info, log_scrapy_info, ) -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.reactor import ( install_reactor, @@ -109,10 +109,9 @@ class Crawler: lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) self.logformatter = lf_cls.from_crawler(self) - self.request_fingerprinter = create_instance( + self.request_fingerprinter = build_from_crawler( load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), - settings=self.settings, - crawler=self, + self, ) reactor_class: str = self.settings["TWISTED_REACTOR"] @@ -404,7 +403,7 @@ class CrawlerProcess(CrawlerRunner): d.addBoth(self._stop_reactor) resolver_class = load_object(self.settings["DNS_RESOLVER"]) - resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) + resolver = build_from_crawler(resolver_class, self, reactor=reactor) resolver.install_on_reactor() tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fadbbb582..e5e363b52 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -28,7 +28,7 @@ from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values logger = logging.getLogger(__name__) @@ -371,7 +371,7 @@ class FeedSlot: self._exporting = True def _get_instance(self, objcls, *args, **kwargs): - return create_instance(objcls, self.settings, self.crawler, *args, **kwargs) + return build_from_crawler(objcls, self.crawler, *args, **kwargs) def _get_exporter(self, file, format, *args, **kwargs): return self._get_instance(self.exporters[format], file, *args, **kwargs) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 090588130..f60c726f9 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -23,7 +23,7 @@ from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.settings import Settings from scrapy.utils.defer import process_chain, process_parallel -from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -64,7 +64,10 @@ class MiddlewareManager: for clspath in mwlist: try: mwcls = load_object(clspath) - mw = create_instance(mwcls, settings, crawler) + if crawler is not None: + mw = build_from_crawler(mwcls, crawler) + else: + mw = build_from_settings(mwcls, settings) middlewares.append(mw) enabled.append(clspath) except NotConfigured as e: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 62a9af477..b62d2fe58 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,7 +1,7 @@ import hashlib import logging -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_crawler logger = logging.getLogger(__name__) @@ -72,9 +72,8 @@ class ScrapyPriorityQueue: self.curprio = min(startprios) def qfactory(self, key): - return create_instance( + return build_from_crawler( self.downstream_queue_cls, - None, self.crawler, self.key + "/" + str(key), ) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index a9364bea2..b38190cb3 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -25,6 +25,7 @@ from typing import ( cast, ) +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache @@ -142,6 +143,13 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an extension has not been implemented correctly). """ + warnings.warn( + "The create_instance() function is deprecated. " + "Please use build_from_crawler() or build_from_settings() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if settings is None: if crawler is None: raise ValueError("Specify at least one of settings and crawler.") @@ -160,6 +168,45 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): return instance +def build_from_crawler(objcls, crawler, /, *args, **kwargs): + """Construct a class instance using its ``from_crawler`` constructor. + + ``*args`` and ``**kwargs`` are forwarded to the constructor. + + Raises ``TypeError`` if the resulting instance is ``None``. + """ + if hasattr(objcls, "from_crawler"): + instance = objcls.from_crawler(crawler, *args, **kwargs) + method_name = "from_crawler" + elif hasattr(objcls, "from_settings"): + instance = objcls.from_settings(crawler.settings, *args, **kwargs) + method_name = "from_settings" + else: + instance = objcls(*args, **kwargs) + method_name = "__new__" + if instance is None: + raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") + return instance + + +def build_from_settings(objcls, settings, /, *args, **kwargs): + """Construct a class instance using its ``from_settings`` constructor. + + ``*args`` and ``**kwargs`` are forwarded to the constructor. + + Raises ``TypeError`` if the resulting instance is ``None``. + """ + if hasattr(objcls, "from_settings"): + instance = objcls.from_settings(settings, *args, **kwargs) + method_name = "from_settings" + else: + instance = objcls(*args, **kwargs) + method_name = "__new__" + if instance is None: + raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") + return instance + + @contextmanager def set_environ(**kwargs: str) -> Generator[None, Any, None]: """Temporarily set environment variables inside the context manager and diff --git a/tests/test_addons.py b/tests/test_addons.py index 0f4f2e5b8..f1b01bc5c 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -89,7 +89,7 @@ class AddonManagerTest(unittest.TestCase): self.assertEqual([a.number for a in manager.addons], expected_order) self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) - def test_create_instance(self): + def test_build_from_crawler(self): settings_dict = { "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, "MYADDON": {"MYADDON_KEY": "val"}, @@ -167,12 +167,12 @@ class AddonManagerTest(unittest.TestCase): pass with patch("scrapy.addons.logger") as logger_mock: - with patch("scrapy.addons.create_instance") as create_instance_mock: + with patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock: settings_dict = { "ADDONS": {LoggedAddon: 1}, } addon = LoggedAddon() - create_instance_mock.return_value = addon + build_from_crawler_mock.return_value = addon crawler = get_crawler(settings_dict=settings_dict) logger_mock.info.assert_called_once_with( "Enabled addons:\n%(addons)s", diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 924ece6f9..dd07d33f1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -29,7 +29,7 @@ from scrapy.http import Headers, HtmlResponse, Request from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto from tests import NON_EXISTING_RESOLVABLE @@ -109,7 +109,7 @@ class FileTestCase(unittest.TestCase): # add a special char to check that they are handled correctly self.tmpname = Path(self.mktemp() + "^") Path(self.tmpname).write_text("0123456789", encoding="utf-8") - handler = create_instance(FileDownloadHandler, None, get_crawler()) + handler = build_from_crawler(FileDownloadHandler, get_crawler()) self.download_request = handler.download_request def tearDown(self): @@ -257,8 +257,8 @@ class HttpTestCase(unittest.TestCase): else: self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) self.portno = self.port.getHost().port - self.download_handler = create_instance( - self.download_handler_cls, None, get_crawler() + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() ) self.download_request = self.download_handler.download_request @@ -557,7 +557,7 @@ class Http11TestCase(HttpTestCase): def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) - download_handler = create_instance(self.download_handler_cls, None, crawler) + download_handler = build_from_crawler(self.download_handler_cls, crawler) request = Request(self.getURL(url)) d = download_handler.download_request(request, Spider("foo")) d.addCallback(lambda r: r.flags) @@ -590,7 +590,7 @@ class Https11TestCase(Http11TestCase): crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} ) - download_handler = create_instance(self.download_handler_cls, None, crawler) + download_handler = build_from_crawler(self.download_handler_cls, crawler) try: with LogCapture() as log_capture: request = Request(self.getURL("file")) @@ -669,9 +669,7 @@ class Https11CustomCiphers(unittest.TestCase): crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"} ) - self.download_handler = create_instance( - self.download_handler_cls, None, crawler - ) + self.download_handler = build_from_crawler(self.download_handler_cls, crawler) self.download_request = self.download_handler.download_request @defer.inlineCallbacks @@ -751,8 +749,8 @@ class HttpProxyTestCase(unittest.TestCase): wrapper = WrappingFactory(site) self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") self.portno = self.port.getHost().port - self.download_handler = create_instance( - self.download_handler_cls, None, get_crawler() + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() ) self.download_request = self.download_handler.download_request @@ -830,10 +828,9 @@ class S3AnonTestCase(unittest.TestCase): def setUp(self): skip_if_no_boto() crawler = get_crawler() - self.s3reqh = create_instance( - objcls=S3DownloadHandler, - settings=None, - crawler=crawler, + self.s3reqh = build_from_crawler( + S3DownloadHandler, + crawler, httpdownloadhandler=HttpDownloadHandlerMock, # anon=True, # implicit ) @@ -861,10 +858,9 @@ class S3TestCase(unittest.TestCase): def setUp(self): skip_if_no_boto() crawler = get_crawler() - s3reqh = create_instance( - objcls=S3DownloadHandler, - settings=None, - crawler=crawler, + s3reqh = build_from_crawler( + S3DownloadHandler, + crawler, aws_access_key_id=self.AWS_ACCESS_KEY_ID, aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, httpdownloadhandler=HttpDownloadHandlerMock, @@ -889,10 +885,9 @@ class S3TestCase(unittest.TestCase): def test_extra_kw(self): try: crawler = get_crawler() - create_instance( - objcls=S3DownloadHandler, - settings=None, - crawler=crawler, + build_from_crawler( + S3DownloadHandler, + crawler, extra_kw=True, ) except Exception as e: @@ -1039,9 +1034,7 @@ class BaseFTPTestCase(unittest.TestCase): self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port crawler = get_crawler() - self.download_handler = create_instance( - FTPDownloadHandler, crawler.settings, crawler - ) + self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) self.addCleanup(self.port.stopListening) def tearDown(self): @@ -1185,9 +1178,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase): self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port crawler = get_crawler() - self.download_handler = create_instance( - FTPDownloadHandler, crawler.settings, crawler - ) + self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) self.addCleanup(self.port.stopListening) def tearDown(self): @@ -1197,9 +1188,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase): class DataURITestCase(unittest.TestCase): def setUp(self): crawler = get_crawler() - self.download_handler = create_instance( - DataURIDownloadHandler, crawler.settings, crawler - ) + self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler) self.download_request = self.download_handler.download_request self.spider = Spider("foo") diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 31fa1430d..322075043 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -11,7 +11,7 @@ from twisted.web.http import H2_ENABLED from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory from tests.test_downloader_handlers import ( @@ -240,8 +240,8 @@ class Https2ProxyTestCase(Http11ProxyTestCase): interface=self.host, ) self.portno = self.port.getHost().port - self.download_handler = create_instance( - self.download_handler_cls, None, get_crawler() + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() ) self.download_request = self.download_handler.download_request diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index e7799737f..3fde5e8c5 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -440,7 +440,7 @@ class SettingsTest(unittest.TestCase): def test_passing_objects_as_values(self): from scrapy.core.downloader.handlers.file import FileDownloadHandler - from scrapy.utils.misc import create_instance + from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler class TestPipeline: @@ -468,7 +468,7 @@ class SettingsTest(unittest.TestCase): myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp") self.assertEqual(myhandler, FileDownloadHandler) - myhandler_instance = create_instance(myhandler, None, get_crawler()) + myhandler_instance = build_from_crawler(myhandler, get_crawler()) self.assertIsInstance(myhandler_instance, FileDownloadHandler) self.assertTrue(hasattr(myhandler_instance, "download_request")) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 69793ee75..ee3314d8e 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -7,6 +7,8 @@ from unittest import mock from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, + build_from_crawler, + build_from_settings, create_instance, load_object, rel_has_nofollow, @@ -153,6 +155,78 @@ class UtilsMiscTestCase(unittest.TestCase): with self.assertRaises(TypeError): create_instance(m, settings, None) + def test_build_from_crawler(self): + settings = mock.MagicMock() + crawler = mock.MagicMock(spec_set=["settings"]) + args = (True, 100.0) + kwargs = {"key": "val"} + + def _test_with_crawler(mock, settings, crawler): + build_from_crawler(mock, crawler, *args, **kwargs) + if hasattr(mock, "from_crawler"): + mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) + if hasattr(mock, "from_settings"): + self.assertEqual(mock.from_settings.call_count, 0) + self.assertEqual(mock.call_count, 0) + elif hasattr(mock, "from_settings"): + mock.from_settings.assert_called_once_with(settings, *args, **kwargs) + self.assertEqual(mock.call_count, 0) + else: + mock.assert_called_once_with(*args, **kwargs) + + # Check usage of correct constructor using three mocks: + # 1. with no alternative constructors + # 2. with from_crawler() constructor + # 3. with from_settings() and from_crawler() constructor + spec_sets = ( + ["__qualname__"], + ["__qualname__", "from_crawler"], + ["__qualname__", "from_settings", "from_crawler"], + ) + for specs in spec_sets: + m = mock.MagicMock(spec_set=specs) + _test_with_crawler(m, settings, crawler) + m.reset_mock() + + # Check adoption of crawler + m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"]) + m.from_crawler.return_value = None + with self.assertRaises(TypeError): + build_from_crawler(m, crawler, *args, **kwargs) + + def test_build_from_settings(self): + settings = mock.MagicMock() + args = (True, 100.0) + kwargs = {"key": "val"} + + def _test_with_settings(mock, settings): + build_from_settings(mock, settings, *args, **kwargs) + if hasattr(mock, "from_settings"): + mock.from_settings.assert_called_once_with(settings, *args, **kwargs) + self.assertEqual(mock.call_count, 0) + else: + mock.assert_called_once_with(*args, **kwargs) + + # Check usage of correct constructor using three mocks: + # 1. with no alternative constructors + # 2. with from_settings() constructor + # 3. with from_settings() and from_crawler() constructor + spec_sets = ( + ["__qualname__"], + ["__qualname__", "from_settings"], + ["__qualname__", "from_settings", "from_crawler"], + ) + for specs in spec_sets: + m = mock.MagicMock(spec_set=specs) + _test_with_settings(m, settings) + m.reset_mock() + + # Check adoption of crawler settings + m = mock.MagicMock(spec_set=["__qualname__", "from_settings"]) + m.from_settings.return_value = None + with self.assertRaises(TypeError): + build_from_settings(m, settings, *args, **kwargs) + def test_set_environ(self): assert os.environ.get("some_test_environ") is None with set_environ(some_test_environ="test_value"): diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 0042fe8f0..d4b6ba15b 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -24,7 +24,7 @@ from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from scrapy.http import Headers, Request from scrapy.settings import Settings -from scrapy.utils.misc import create_instance +from scrapy.utils.misc import build_from_settings from scrapy.utils.python import to_bytes, to_unicode from tests.mockserver import ( BrokenDownloadResource, @@ -470,8 +470,8 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): def testPayload(self): s = "0123456789" * 10 settings = Settings({"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers}) - client_context_factory = create_instance( - ScrapyClientContextFactory, settings=settings, crawler=None + client_context_factory = build_from_settings( + ScrapyClientContextFactory, settings ) return getPage( self.getURL("payload"), body=s, contextFactory=client_context_factory @@ -482,8 +482,8 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): settings = Settings( {"DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384"} ) - client_context_factory = create_instance( - ScrapyClientContextFactory, settings=settings, crawler=None + client_context_factory = build_from_settings( + ScrapyClientContextFactory, settings ) d = getPage( self.getURL("payload"), body=s, contextFactory=client_context_factory From 1864f48e9e3752e4cb7e24c22b2d51b727e4086c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 20 Dec 2023 12:47:18 +0100 Subject: [PATCH 1292/2083] =?UTF-8?q?Link=20to=20Zyte=E2=80=99s=20export?= =?UTF-8?q?=20guides?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/feed-exports.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 700775e4b..f64bbac06 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -13,6 +13,11 @@ Scrapy provides this functionality out of the box with the Feed Exports, which allows you to generate feeds with the scraped items, using multiple serialization formats and storage backends. +This page provides detailed documentation for all feed export features. If you +are looking for a step-by-step guide, check out `Zyte’s export guides`_. + +.. _Zyte’s export guides: https://docs.zyte.com/web-scraping/guides/export/index.html#exporting-scraped-data + .. _topics-feed-format: Serialization formats From 1fab844f7dd5fe622899c41ad8a0d28dd27c5089 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 20 Dec 2023 15:57:51 +0400 Subject: [PATCH 1293/2083] Pin the Python version for typing-tests. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 21ac4c3ff..f0788c0af 100644 --- a/tox.ini +++ b/tox.ini @@ -47,6 +47,7 @@ commands = mypy {posargs: scrapy tests} [testenv:typing-tests] +basepython = python3.8 deps = -rtests/requirements.txt {[testenv:typing]deps} From a72394a388a8c41ab07f4511b096d85e6de168fe Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 20 Dec 2023 16:14:53 +0400 Subject: [PATCH 1294/2083] Add tests for replace() with kwargs. --- tests_typing/test_http_request.mypy-testing | 14 ++++++++++++++ tests_typing/test_http_response.mypy-testing | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index 636e6895f..665db9088 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -1,3 +1,5 @@ +from typing import Any, Dict + import pytest from scrapy import Request @@ -33,6 +35,9 @@ def mypy_test_copy(): req_copy = req.copy() reveal_type(req_copy) # R: scrapy.http.request.Request + +@pytest.mark.mypy_testing +def mypy_test_copy_subclass(): req = MyRequest("data:,") reveal_type(req) # R: __main__.MyRequest req_copy = req.copy() @@ -45,13 +50,22 @@ def mypy_test_replace(): reveal_type(req) # R: scrapy.http.request.Request req_copy = req.replace(body=b"a") reveal_type(req_copy) # R: scrapy.http.request.Request + kwargs: Dict[str, Any] = {} + req_copy2 = req.replace(body=b"a", **kwargs) + reveal_type(req_copy2) # R: Any + +@pytest.mark.mypy_testing +def mypy_test_replace_subclass(): req = MyRequest("data:,") reveal_type(req) # R: __main__.MyRequest req_copy = req.replace(body=b"a") reveal_type(req_copy) # R: __main__.MyRequest req_copy2 = req.replace(body=b"a", cls=MyRequest2) reveal_type(req_copy2) # R: __main__.MyRequest2 + kwargs: Dict[str, Any] = {} + req_copy3 = req.replace(body=b"a", cls=MyRequest2, **kwargs) + reveal_type(req_copy3) # R: __main__.MyRequest2 @pytest.mark.mypy_testing diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index 2e58b4fbc..d58ac1027 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -1,3 +1,5 @@ +from typing import Any, Dict + import pytest from scrapy.http import HtmlResponse, Response, TextResponse @@ -24,6 +26,9 @@ def mypy_test_copy(): resp_copy = resp.copy() reveal_type(resp_copy) # R: scrapy.http.response.Response + +@pytest.mark.mypy_testing +def mypy_test_copy_subclass(): resp = HtmlResponse("data:,") reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse resp_copy = resp.copy() @@ -36,10 +41,19 @@ def mypy_test_replace(): reveal_type(resp) # R: scrapy.http.response.Response resp_copy = resp.replace(body=b"a") reveal_type(resp_copy) # R: scrapy.http.response.Response + kwargs: Dict[str, Any] = {} + resp_copy2 = resp.replace(body=b"a", **kwargs) + reveal_type(resp_copy2) # R: Any + +@pytest.mark.mypy_testing +def mypy_test_replace_subclass(): resp = HtmlResponse("data:,") reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse resp_copy = resp.replace(body=b"a") reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse resp_copy2 = resp.replace(body=b"a", cls=TextResponse) reveal_type(resp_copy2) # R: scrapy.http.response.text.TextResponse + kwargs: Dict[str, Any] = {} + resp_copy3 = resp.replace(body=b"a", cls=TextResponse, **kwargs) + reveal_type(resp_copy3) # R: scrapy.http.response.text.TextResponse From f56b5fc39ef3b322b8d0ad17fb424440bd79da0b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 20 Dec 2023 16:19:11 +0400 Subject: [PATCH 1295/2083] Bump typing deps. --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index f0788c0af..25b30d759 100644 --- a/tox.ini +++ b/tox.ini @@ -33,14 +33,14 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.6.1 - typing-extensions==4.8.0 + mypy==1.7.1 + typing-extensions==4.9.0 types-attrs==19.1.0 types-lxml==2023.10.21 - types-Pillow==10.1.0.0 - types-Pygments==2.16.0.0 + types-Pillow==10.1.0.2 + types-Pygments==2.17.0.0 types-pyOpenSSL==23.3.0.0 - types-setuptools==68.2.0.0 + types-setuptools==69.0.0.0 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands = From b095dd218fe64f2541079d691e3c2c68d2e03ff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Nov 2023 10:54:09 +0100 Subject: [PATCH 1296/2083] Extend Request.meta documentation (#5565) --- docs/topics/request-response.rst | 47 ++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index adf3d0f4a..8edf710bc 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -193,18 +193,47 @@ Request objects :meth:`replace`. .. attribute:: Request.meta + :value: {} - A dict that contains arbitrary metadata for this request. This dict is - empty for new Requests, and is usually populated by different Scrapy - components (extensions, middlewares, etc). So the data contained in this - dict depends on the extensions you have enabled. + A dictionary of arbitrary metadata for the request. - See :ref:`topics-request-meta` for a list of special meta keys - recognized by Scrapy. + You may extend request metadata as you see fit. - This dict is :doc:`shallow copied ` when the request is - cloned using the ``copy()`` or ``replace()`` methods, and can also be - accessed, in your spider, from the ``response.meta`` attribute. + Request metadata can also be accessed through the + :attr:`~scrapy.http.Response.meta` attribute of a response. + + To pass data from one spider callback to another, consider using + :attr:`cb_kwargs` instead. However, request metadata may be the right + choice in certain scenarios, such as to maintain some debugging data + across all follow-up requests (e.g. the source URL). + + A common use of request metadata is to define request-specific + parameters for Scrapy components (extensions, middlewares, etc.). For + example, if you set ``dont_retry`` to ``True``, + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` will never + retry that request, even if it fails. See :ref:`topics-request-meta`. + + You may also use request metadata in your custom Scrapy components, for + example, to keep request state information relevant to your component. + For example, + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses the + ``retry_times`` metadata key to keep track of how many times a request + has been retried so far. + + Copying all the metadata of a previous request into a new, follow-up + request in a spider callback is a bad practice, because request + metadata may include metadata set by Scrapy components that is not + meant to be copied into other requests. For example, copying the + ``retry_times`` metadata key into follow-up requests can lower the + amount of retries allowed for those follow-up requests. + + You should only copy all request metadata from one request to another + if the new request is meant to replace the old request, as is often the + case when returning a request from a :ref:`downloader middleware + ` method. + + Also mind that the :meth:`copy` and :meth:`replace` request methods + :doc:`shallow-copy ` request metadata. .. attribute:: Request.cb_kwargs From 369712ee50f7438c2863359f053cb1b221a42169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Nov 2023 11:01:22 +0100 Subject: [PATCH 1297/2083] =?UTF-8?q?SPM=20=E2=86=92=20Zyte=20API=20(#6163?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/practices.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index f64da22d8..b1b8c9e9c 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -288,9 +288,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * use a pool of rotating IPs. For example, the free `Tor project`_ or paid services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. -* use a highly distributed downloader that circumvents bans internally, so you - can just focus on parsing clean pages. One example of such downloaders is - `Zyte Smart Proxy Manager`_ +* use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy + plugin `__ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. @@ -301,4 +300,4 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _Common Crawl: https://commoncrawl.org/ .. _testspiders: https://github.com/scrapinghub/testspiders .. _scrapoxy: https://scrapoxy.io/ -.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ +.. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html From 48a9a58ff27d24910b55a0ad5e6b014589c71115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 20 Dec 2023 12:47:18 +0100 Subject: [PATCH 1298/2083] =?UTF-8?q?Link=20to=20Zyte=E2=80=99s=20export?= =?UTF-8?q?=20guides?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/feed-exports.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 700775e4b..f64bbac06 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -13,6 +13,11 @@ Scrapy provides this functionality out of the box with the Feed Exports, which allows you to generate feeds with the scraped items, using multiple serialization formats and storage backends. +This page provides detailed documentation for all feed export features. If you +are looking for a step-by-step guide, check out `Zyte’s export guides`_. + +.. _Zyte’s export guides: https://docs.zyte.com/web-scraping/guides/export/index.html#exporting-scraped-data + .. _topics-feed-format: Serialization formats From d25cfe5315c9c0346776529a6e14ffb405913d2e Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Thu, 21 Dec 2023 03:36:21 -0600 Subject: [PATCH 1299/2083] Add JsonResponse (#6174) --- docs/topics/request-response.rst | 10 ++++++++++ scrapy/http/__init__.py | 1 + scrapy/http/response/json.py | 12 ++++++++++++ scrapy/responsetypes.py | 6 +++--- tests/test_responsetypes.py | 14 +++++++++++--- 5 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 scrapy/http/response/json.py diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8edf710bc..9d64eee45 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1357,3 +1357,13 @@ XmlResponse objects line. See :attr:`TextResponse.encoding`. .. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241 + +JsonResponse objects +-------------------- + +.. class:: JsonResponse(url[, ...]) + + The :class:`JsonResponse` class is a subclass of :class:`TextResponse` + that is used when the response has a `JSON MIME type + `_ in its `Content-Type` + header. diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index ac3946302..d0b726bad 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -12,5 +12,6 @@ from scrapy.http.request.json_request import JsonRequest from scrapy.http.request.rpc import XmlRpcRequest from scrapy.http.response import Response from scrapy.http.response.html import HtmlResponse +from scrapy.http.response.json import JsonResponse from scrapy.http.response.text import TextResponse from scrapy.http.response.xml import XmlResponse diff --git a/scrapy/http/response/json.py b/scrapy/http/response/json.py new file mode 100644 index 000000000..219691094 --- /dev/null +++ b/scrapy/http/response/json.py @@ -0,0 +1,12 @@ +""" +This module implements the JsonResponse class that is used when the response +has a JSON MIME type in its Content-Type header. + +See documentation in docs/topics/request-response.rst +""" + +from scrapy.http.response.text import TextResponse + + +class JsonResponse(TextResponse): + pass diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 9e411d4aa..0d127d851 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -21,9 +21,9 @@ class ResponseTypes: "application/xhtml+xml": "scrapy.http.HtmlResponse", "application/vnd.wap.xhtml+xml": "scrapy.http.HtmlResponse", "application/xml": "scrapy.http.XmlResponse", - "application/json": "scrapy.http.TextResponse", - "application/x-json": "scrapy.http.TextResponse", - "application/json-amazonui-streaming": "scrapy.http.TextResponse", + "application/json": "scrapy.http.JsonResponse", + "application/x-json": "scrapy.http.JsonResponse", + "application/json-amazonui-streaming": "scrapy.http.JsonResponse", "application/javascript": "scrapy.http.TextResponse", "application/x-javascript": "scrapy.http.TextResponse", "text/xml": "scrapy.http.XmlResponse", diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 6e1ed82f0..713a83d52 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -1,6 +1,13 @@ import unittest -from scrapy.http import Headers, HtmlResponse, Response, TextResponse, XmlResponse +from scrapy.http import ( + Headers, + HtmlResponse, + JsonResponse, + Response, + TextResponse, + XmlResponse, +) from scrapy.responsetypes import responsetypes @@ -40,8 +47,9 @@ class ResponseTypesTest(unittest.TestCase): ("application/vnd.wap.xhtml+xml; charset=utf-8", HtmlResponse), ("application/xml; charset=UTF-8", XmlResponse), ("application/octet-stream", Response), - ("application/x-json; encoding=UTF8;charset=UTF-8", TextResponse), - ("application/json-amazonui-streaming;charset=UTF-8", TextResponse), + ("application/json; encoding=UTF8;charset=UTF-8", JsonResponse), + ("application/x-json; encoding=UTF8;charset=UTF-8", JsonResponse), + ("application/json-amazonui-streaming;charset=UTF-8", JsonResponse), (b"application/x-download; filename=\x80dummy.txt", Response), ] for source, cls in mappings: From 0e78acb65798a1eb4e55a472232e77d55e6c7cd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 21 Dec 2023 21:01:07 +0100 Subject: [PATCH 1300/2083] MediaPipeline: log media_to_download errors before stripping them (#5068) --- scrapy/pipelines/media.py | 10 +++++----- tests/test_pipeline_crawl.py | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 75532034a..fc156ab41 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -112,14 +112,14 @@ class MediaPipeline: info.downloading.add(fp) dfd = mustbe_deferred(self.media_to_download, request, info, item=item) dfd.addCallback(self._check_media_to_download, request, info, item=item) + dfd.addErrback(self._log_exception) dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) - dfd.addErrback( - lambda f: logger.error( - f.value, exc_info=failure_to_exc_info(f), extra={"spider": info.spider} - ) - ) return dfd.addBoth(lambda _: wad) # it must return wad at last + def _log_exception(self, result): + logger.exception(result) + return result + def _modify_media_request(self, request): if self.handle_httpstatus_list: request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index ed8483483..c41ab483f 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -9,6 +9,7 @@ from w3lib.url import add_or_replace_parameter from scrapy import signals from scrapy.crawler import CrawlerRunner +from scrapy.utils.misc import load_object from tests.mockserver import MockServer from tests.spiders import SimpleSpider @@ -193,6 +194,29 @@ class FileDownloadCrawlTestCase(TestCase): crawler.stats.get_value("downloader/response_status_count/302"), 3 ) + @defer.inlineCallbacks + def test_download_media_file_path_error(self): + cls = load_object(self.pipeline_class) + + class ExceptionRaisingMediaPipeline(cls): + def file_path(self, request, response=None, info=None, *, item=None): + return 1 / 0 + + settings = { + **self.settings, + "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1}, + } + runner = CrawlerRunner(settings) + crawler = self._create_crawler(MediaDownloadSpider, runner=runner) + with LogCapture() as log: + yield crawler.crawl( + self.mockserver.url("/files/images/"), + media_key=self.media_key, + media_urls_key=self.media_urls_key, + mockserver=self.mockserver, + ) + self.assertIn("ZeroDivisionError", str(log)) + skip_pillow: Optional[str] try: From 2534a28ef032ae03e567859a498307b07ad34f64 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 25 Dec 2023 15:03:08 +0400 Subject: [PATCH 1301/2083] Bump mypy. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 25b30d759..8996b12a4 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.7.1 + mypy==1.8.0 typing-extensions==4.9.0 types-attrs==19.1.0 types-lxml==2023.10.21 From 34e01a8a933cc24e1daf2c3a0cc024f37e3614f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 28 Dec 2023 12:25:01 +0100 Subject: [PATCH 1302/2083] Update quotes.toscrape.com page copies (#6190) --- docs/_tests/quotes.html | 2 +- docs/_tests/quotes1.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html index 71aff8847..f4002ecd1 100644 --- a/docs/_tests/quotes.html +++ b/docs/_tests/quotes.html @@ -273,7 +273,7 @@ Quotes by: GoodReads.com

- Made with ⤠by Scrapinghub + Made with ⤠by Zyte

diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html index 71aff8847..f4002ecd1 100644 --- a/docs/_tests/quotes1.html +++ b/docs/_tests/quotes1.html @@ -273,7 +273,7 @@ Quotes by: GoodReads.com

- Made with ⤠by Scrapinghub + Made with ⤠by Zyte

From 19022849428573a9bdf5f3217638d6e3c86d1796 Mon Sep 17 00:00:00 2001 From: Chan Sau Yee <15137352+y26805@users.noreply.github.com> Date: Fri, 29 Dec 2023 20:32:51 +0900 Subject: [PATCH 1303/2083] Update black reference in docs (#6192) --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 2b3249601..d728338da 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -178,7 +178,7 @@ Scrapy: * We use `black `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also - run black manually with ``tox -e black``. + run black manually with ``tox -e pre-commit``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. From 40e623b2768598e36c4f367bd166b36fffceb3f6 Mon Sep 17 00:00:00 2001 From: Chan Sau Yee <15137352+y26805@users.noreply.github.com> Date: Fri, 29 Dec 2023 20:33:37 +0900 Subject: [PATCH 1304/2083] Add type hints (#6191) --- scrapy/pipelines/files.py | 4 +++- scrapy/pipelines/images.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 5c09ab37e..1990ba825 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -340,7 +340,9 @@ class FilesPipeline(MediaPipeline): DEFAULT_FILES_URLS_FIELD = "file_urls" DEFAULT_FILES_RESULT_FIELD = "files" - def __init__(self, store_uri, download_func=None, settings=None): + def __init__( + self, store_uri: Union[str, PathLike], download_func=None, settings=None + ): store_uri = _to_string(store_uri) if not store_uri: raise NotConfigured diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 1bd9832a8..02c4b1361 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -8,7 +8,8 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import Dict, Tuple +from os import PathLike +from typing import Dict, Tuple, Union from itemadapter import ItemAdapter @@ -53,7 +54,9 @@ class ImagesPipeline(FilesPipeline): DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" - def __init__(self, store_uri, download_func=None, settings=None): + def __init__( + self, store_uri: Union[str, PathLike], download_func=None, settings=None + ): try: from PIL import Image From badc7c5be9dcfaf7c8acbb87fa530f0477ec3c35 Mon Sep 17 00:00:00 2001 From: Chan Sau Yee <15137352+y26805@users.noreply.github.com> Date: Fri, 29 Dec 2023 20:32:51 +0900 Subject: [PATCH 1305/2083] Update black reference in docs (#6192) --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 2b3249601..d728338da 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -178,7 +178,7 @@ Scrapy: * We use `black `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also - run black manually with ``tox -e black``. + run black manually with ``tox -e pre-commit``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. From 6127f7d27824de1f9847f7bb07f9755c955d9c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 28 Dec 2023 12:25:01 +0100 Subject: [PATCH 1306/2083] Update quotes.toscrape.com page copies (#6190) --- docs/_tests/quotes.html | 2 +- docs/_tests/quotes1.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html index 71aff8847..f4002ecd1 100644 --- a/docs/_tests/quotes.html +++ b/docs/_tests/quotes.html @@ -273,7 +273,7 @@ Quotes by: GoodReads.com

- Made with ⤠by Scrapinghub + Made with ⤠by Zyte

diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html index 71aff8847..f4002ecd1 100644 --- a/docs/_tests/quotes1.html +++ b/docs/_tests/quotes1.html @@ -273,7 +273,7 @@ Quotes by: GoodReads.com

- Made with ⤠by Scrapinghub + Made with ⤠by Zyte

From c7b2b097b18c0b30d06cea4b803d5d42aca98715 Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Fri, 5 Jan 2024 10:50:45 +0100 Subject: [PATCH 1307/2083] fix(typo): correct `successfully` --- tests/test_extension_periodic_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 502ada6be..b7312bbcd 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -67,7 +67,7 @@ def extension(settings=None): class TestPeriodicLog(unittest.TestCase): def test_extension_enabled(self): - # Expected that settings for this extension loaded succesfully + # Expected that settings for this extension loaded successfully # And on certain conditions - extension raising NotConfigured # "PERIODIC_LOG_STATS": True -> set to {"enabled": True} From 0d445a3224ecb0abfdf56a93e181692d3b5e4a6b Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Fri, 5 Jan 2024 12:30:10 +0100 Subject: [PATCH 1308/2083] refactor(yield): use `yield from` syntax --- scrapy/core/spidermw.py | 3 +-- scrapy/spiders/crawl.py | 3 +-- scrapy/spiders/feed.py | 6 ++---- scrapy/spiders/sitemap.py | 3 +-- scrapy/utils/python.py | 3 +-- scrapy/utils/request.py | 3 +-- tests/spiders.py | 3 +-- tests/test_feedexport.py | 6 ++---- tests/test_spider.py | 3 +-- tests/test_spidermiddleware.py | 9 +++------ tests/test_utils_defer.py | 3 +-- 11 files changed, 15 insertions(+), 30 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index dcf1a6dbc..031a0be36 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -103,8 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager): ) -> Union[Generator, AsyncGenerator]: def process_sync(iterable: Iterable) -> Generator: try: - for r in iterable: - yield r + yield from iterable except Exception as ex: exception_result = self._process_spider_exception( response, spider, Failure(ex), exception_processor_index diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 31e845716..ebb4f5984 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -131,8 +131,7 @@ class CrawlSpider(Spider): def _handle_failure(self, failure, errback): if errback: results = errback(failure) or () - for request_or_item in iterate_spider_output(results): - yield request_or_item + yield from iterate_spider_output(results) def _compile_rules(self): self._rules = [] diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 6afadc577..47827e442 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -58,8 +58,7 @@ class XMLFeedSpider(Spider): for selector in nodes: ret = iterate_spider_output(self.parse_node(response, selector)) - for result_item in self.process_results(response, ret): - yield result_item + yield from self.process_results(response, ret) def _parse(self, response, **kwargs): if not hasattr(self, "parse_node"): @@ -133,8 +132,7 @@ class CSVFeedSpider(Spider): response, self.delimiter, self.headers, quotechar=self.quotechar ): ret = iterate_spider_output(self.parse_row(response, row)) - for result_item in self.process_results(response, ret): - yield result_item + yield from self.process_results(response, ret) def _parse(self, response, **kwargs): if not hasattr(self, "parse_row"): diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index aaf75a519..974665fe0 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -33,8 +33,7 @@ class SitemapSpider(Spider): attributes, for example, you can filter locs with lastmod greater than a given date (see docs). """ - for entry in entries: - yield entry + yield from entries def _parse_sitemap(self, response): if response.url.endswith("/robots.txt"): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 0b5dc324f..68ca96b69 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -57,8 +57,7 @@ def iflatten(x: Iterable) -> Iterable: Similar to ``.flatten()``, but returns iterator instead""" for el in x: if is_listlike(el): - for el_ in iflatten(el): - yield el_ + yield from iflatten(el) else: yield el diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 24fcbd85e..cea1bc727 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -44,8 +44,7 @@ def _serialize_headers( for header in headers: if header in request.headers: yield header - for value in request.headers.getlist(header): - yield value + yield from request.headers.getlist(header) def request_fingerprint( diff --git a/tests/spiders.py b/tests/spiders.py index f29dea2a1..3df153a12 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -301,8 +301,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): def parse(self, response): self.seedsseen.append(response.meta.get("seed")) - for req in super().parse(response): - yield req + yield from super().parse(response) class SingleRequestSpider(MetaSpider): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 89169fd7c..c7d955bc7 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -673,8 +673,7 @@ class FeedExportTestBase(ABC, unittest.TestCase): name = "testspider" def parse(self, response): - for item in items: - yield item + yield from items data = yield self.run_and_export(TestSpider, settings) return data @@ -2696,8 +2695,7 @@ class BatchDeliveriesTest(FeedExportTestBase): name = "testspider" def parse(self, response): - for item in items: - yield item + yield from items with MockServer() as server: TestSpider.start_urls = [server.url("/")] diff --git a/tests/test_spider.py b/tests/test_spider.py index 00da3d485..9ce40f921 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -317,8 +317,7 @@ class CrawlSpiderTest(SpiderTest): rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) def dummy_process_links(self, links): - for link in links: - yield link + yield from links spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index d167adbb7..38ca8d950 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -170,8 +170,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): class ProcessSpiderOutputSimpleMiddleware: def process_spider_output(self, response, result, spider): - for r in result: - yield r + yield from result class ProcessSpiderOutputAsyncGenMiddleware: @@ -182,8 +181,7 @@ class ProcessSpiderOutputAsyncGenMiddleware: class ProcessSpiderOutputUniversalMiddleware: def process_spider_output(self, response, result, spider): - for r in result: - yield r + yield from result async def process_spider_output_async(self, response, result, spider): async for r in result: @@ -324,8 +322,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): class ProcessStartRequestsSimpleMiddleware: def process_start_requests(self, start_requests, spider): - for r in start_requests: - yield r + yield from start_requests class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index bb0ebc2a4..a7d54b565 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -107,8 +107,7 @@ class DeferUtilsTest(unittest.TestCase): class IterErrbackTest(unittest.TestCase): def test_iter_errback_good(self): def itergood(): - for x in range(10): - yield x + yield from range(10) errors = [] out = list(iter_errback(itergood(), errors.append)) From 42c481cb4a12a81e88923930e21a661eee967a5f Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Fri, 5 Jan 2024 12:36:36 +0100 Subject: [PATCH 1309/2083] refactor(): use `OSError` exception https://docs.astral.sh/ruff/rules/os-error-alias/ --- scrapy/pipelines/files.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 1990ba825..73064ad10 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -8,7 +8,6 @@ import functools import hashlib import logging import mimetypes -import os import time from collections import defaultdict from contextlib import suppress @@ -66,7 +65,7 @@ class FSFilesStore: absolute_path = self._get_filesystem_path(path) try: last_modified = absolute_path.stat().st_mtime - except os.error: + except OSError: return {} with absolute_path.open("rb") as f: From 745b8412f6ec02605c27d295b2be9d71b624cf70 Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Fri, 5 Jan 2024 14:53:51 +0100 Subject: [PATCH 1310/2083] fix(flake8): lint errors E226 missing whitespace around arithmetic operator E201 whitespace after '{' --- scrapy/extensions/memusage.py | 6 +++--- tests/test_utils_iterators.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 221967bda..ca766c938 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -128,9 +128,9 @@ class MemoryUsage: def _send_report(self, rcpts, subject): """send notification mail with some additional useful info""" stats = self.crawler.stats - s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n" - s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n" - s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n" + s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n" + s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n" + s += f"Current memory usage : {self.get_virtual_size() / 1024 / 1024}M\r\n" s += ( "ENGINE STATUS ------------------------------------------------------- \r\n" diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 3598fa0bb..4a0c34d82 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -524,6 +524,6 @@ class TestHelper(unittest.TestCase): def _assert_type_and_value(self, a, b, obj): self.assertTrue( - type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}" + type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}" ) self.assertEqual(a, b) From 68fccb1d58f291f70e864fd8ecd167887bda4112 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 6 Jan 2024 01:35:56 +0400 Subject: [PATCH 1311/2083] Fix and re-enable newer mitmproxy usage in tests. --- tests/test_proxy_connect.py | 3 ++- tox.ini | 8 ++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index dc0a82086..46d42e9f6 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -31,6 +31,7 @@ sys.exit(mitmdump()) self.proc = Popen( [ sys.executable, + "-u", "-c", script, "--listen-host", @@ -46,7 +47,7 @@ sys.exit(mitmdump()) stdout=PIPE, ) line = self.proc.stdout.readline().decode("utf-8") - host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1) + host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1) address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}" return address diff --git a/tox.ini b/tox.ini index 932c0b805..d9dcacc01 100644 --- a/tox.ini +++ b/tox.ini @@ -11,11 +11,7 @@ minversion = 1.7.0 deps = -rtests/requirements.txt # mitmproxy does not support PyPy - # Python 3.9+ requires mitmproxy >= 5.3.0 - # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 - #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' - # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 - mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' + mitmproxy; implementation_name != 'pypy' passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -87,7 +83,7 @@ deps = lxml==4.4.1 -rtests/requirements.txt - # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies + # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment setenv = _SCRAPY_PINNED=true From c2baf4d0dad5f656e51dce6e00118fbc0419d0db Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 12 Jan 2024 18:30:41 +0400 Subject: [PATCH 1312/2083] Remove a defer.returnValue call. --- tests/test_feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c7d955bc7..277555608 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2299,7 +2299,7 @@ class BatchDeliveriesTest(FeedExportTestBase): content[feed["format"]].append(file.read_bytes()) finally: self.tearDown() - defer.returnValue(content) + return content @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): From fa0c598096de6e26a7b22e7d53cf8c073f96f3a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 15 Jan 2024 13:14:02 +0100 Subject: [PATCH 1313/2083] Add component getters to Crawler (#6181) --- scrapy/crawler.py | 42 +++++ tests/test_crawler.py | 388 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 419 insertions(+), 11 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 844d5f759..1db9ace28 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -178,6 +178,48 @@ class Crawler: assert self.engine yield maybeDeferred(self.engine.stop) + @staticmethod + def _get_component(component_class, components): + for component in components: + if isinstance(component, component_class): + return component + return None + + def get_addon(self, cls): + return self._get_component(cls, self.addons.addons) + + def get_downloader_middleware(self, cls): + if not self.engine: + raise RuntimeError( + "Crawler.get_downloader_middleware() can only be called after " + "the crawl engine has been created." + ) + return self._get_component(cls, self.engine.downloader.middleware.middlewares) + + def get_extension(self, cls): + if not self.extensions: + raise RuntimeError( + "Crawler.get_extension() can only be called after the " + "extension manager has been created." + ) + return self._get_component(cls, self.extensions.middlewares) + + def get_item_pipeline(self, cls): + if not self.engine: + raise RuntimeError( + "Crawler.get_item_pipeline() can only be called after the " + "crawl engine has been created." + ) + return self._get_component(cls, self.engine.scraper.itemproc.middlewares) + + def get_spider_middleware(self, cls): + if not self.engine: + raise RuntimeError( + "Crawler.get_spider_middleware() can only be called after the " + "crawl engine has been created." + ) + return self._get_component(cls, self.engine.scraper.spidermw.middlewares) + class CrawlerRunner: """ diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 92bd5f38f..989208694 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -12,12 +12,13 @@ import pytest from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn from pytest import mark, raises -from twisted.internet import defer +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.trial import unittest from w3lib import __version__ as w3lib_version from zope.interface.exceptions import MultipleInvalid import scrapy +from scrapy import Spider from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions import telnet @@ -29,6 +30,19 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer, get_mockserver_env +# To prevent warnings. +BASE_SETTINGS = { + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", +} + + +def get_raw_crawler(spidercls=None, settings_dict=None): + """get_crawler alternative that only calls the __init__ method of the + crawler.""" + settings = Settings() + settings.setdict(settings_dict or {}) + return Crawler(spidercls or DefaultSpider, settings) + class BaseCrawlerTest(unittest.TestCase): def assertOptionIsDefault(self, settings, key): @@ -39,7 +53,7 @@ class BaseCrawlerTest(unittest.TestCase): class CrawlerTestCase(BaseCrawlerTest): def test_populate_spidercls_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {"TEST1": "project", "TEST3": "project"} + project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"} class CustomSettingsSpider(DefaultSpider): custom_settings = spider_settings @@ -71,9 +85,9 @@ class CrawlerTestCase(BaseCrawlerTest): with raises(ValueError): Crawler(DefaultSpider()) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_crawl_twice_deprecated(self): - crawler = Crawler(NoRequestsSpider) + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) yield crawler.crawl() with pytest.warns( ScrapyDeprecationWarning, @@ -81,6 +95,358 @@ class CrawlerTestCase(BaseCrawlerTest): ): yield crawler.crawl() + def test_get_addon(self): + class ParentAddon: + pass + + class TrackingAddon(ParentAddon): + instances = [] + + def __init__(self): + TrackingAddon.instances.append(self) + + def update_settings(self, settings): + pass + + settings = { + **BASE_SETTINGS, + "ADDONS": { + TrackingAddon: 0, + }, + } + crawler = get_crawler(settings_dict=settings) + self.assertEqual(len(TrackingAddon.instances), 1) + expected = TrackingAddon.instances[-1] + + addon = crawler.get_addon(TrackingAddon) + self.assertEqual(addon, expected) + + addon = crawler.get_addon(DefaultSpider) + self.assertIsNone(addon) + + addon = crawler.get_addon(ParentAddon) + self.assertEqual(addon, expected) + + class ChildAddon(TrackingAddon): + pass + + addon = crawler.get_addon(ChildAddon) + self.assertIsNone(addon) + + @inlineCallbacks + def test_get_downloader_middleware(self): + class ParentDownloaderMiddleware: + pass + + class TrackingDownloaderMiddleware(ParentDownloaderMiddleware): + instances = [] + + def __init__(self): + TrackingDownloaderMiddleware.instances.append(self) + + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) + + def __init__(self, crawler): + self.crawler = crawler + + def start_requests(self): + MySpider.result = crawler.get_downloader_middleware(MySpider.cls) + return + yield + + settings = { + **BASE_SETTINGS, + "DOWNLOADER_MIDDLEWARES": { + TrackingDownloaderMiddleware: 0, + }, + } + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = TrackingDownloaderMiddleware + yield crawler.crawl() + self.assertEqual(len(TrackingDownloaderMiddleware.instances), 1) + self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1]) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = DefaultSpider + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ParentDownloaderMiddleware + yield crawler.crawl() + self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1]) + + class ChildDownloaderMiddleware(TrackingDownloaderMiddleware): + pass + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ChildDownloaderMiddleware + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + def test_get_downloader_middleware_not_crawling(self): + crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) + self.assertRaises( + RuntimeError, crawler.get_downloader_middleware, DefaultSpider + ) + + @inlineCallbacks + def test_get_downloader_middleware_no_engine(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + try: + crawler.get_downloader_middleware(DefaultSpider) + except Exception as e: + MySpider.result = e + raise + + crawler = get_raw_crawler(MySpider, BASE_SETTINGS) + with raises(RuntimeError): + yield crawler.crawl() + + @inlineCallbacks + def test_get_extension(self): + class ParentExtension: + pass + + class TrackingExtension(ParentExtension): + instances = [] + + def __init__(self): + TrackingExtension.instances.append(self) + + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) + + def __init__(self, crawler): + self.crawler = crawler + + def start_requests(self): + MySpider.result = crawler.get_extension(MySpider.cls) + return + yield + + settings = { + **BASE_SETTINGS, + "EXTENSIONS": { + TrackingExtension: 0, + }, + } + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = TrackingExtension + yield crawler.crawl() + self.assertEqual(len(TrackingExtension.instances), 1) + self.assertEqual(MySpider.result, TrackingExtension.instances[-1]) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = DefaultSpider + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ParentExtension + yield crawler.crawl() + self.assertEqual(MySpider.result, TrackingExtension.instances[-1]) + + class ChildExtension(TrackingExtension): + pass + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ChildExtension + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + def test_get_extension_not_crawling(self): + crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) + self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider) + + @inlineCallbacks + def test_get_extension_no_engine(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + try: + crawler.get_extension(DefaultSpider) + except Exception as e: + MySpider.result = e + raise + + crawler = get_raw_crawler(MySpider, BASE_SETTINGS) + with raises(RuntimeError): + yield crawler.crawl() + + @inlineCallbacks + def test_get_item_pipeline(self): + class ParentItemPipeline: + pass + + class TrackingItemPipeline(ParentItemPipeline): + instances = [] + + def __init__(self): + TrackingItemPipeline.instances.append(self) + + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) + + def __init__(self, crawler): + self.crawler = crawler + + def start_requests(self): + MySpider.result = crawler.get_item_pipeline(MySpider.cls) + return + yield + + settings = { + **BASE_SETTINGS, + "ITEM_PIPELINES": { + TrackingItemPipeline: 0, + }, + } + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = TrackingItemPipeline + yield crawler.crawl() + self.assertEqual(len(TrackingItemPipeline.instances), 1) + self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1]) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = DefaultSpider + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ParentItemPipeline + yield crawler.crawl() + self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1]) + + class ChildItemPipeline(TrackingItemPipeline): + pass + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ChildItemPipeline + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + def test_get_item_pipeline_not_crawling(self): + crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) + self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider) + + @inlineCallbacks + def test_get_item_pipeline_no_engine(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + try: + crawler.get_item_pipeline(DefaultSpider) + except Exception as e: + MySpider.result = e + raise + + crawler = get_raw_crawler(MySpider, BASE_SETTINGS) + with raises(RuntimeError): + yield crawler.crawl() + + @inlineCallbacks + def test_get_spider_middleware(self): + class ParentSpiderMiddleware: + pass + + class TrackingSpiderMiddleware(ParentSpiderMiddleware): + instances = [] + + def __init__(self): + TrackingSpiderMiddleware.instances.append(self) + + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) + + def __init__(self, crawler): + self.crawler = crawler + + def start_requests(self): + MySpider.result = crawler.get_spider_middleware(MySpider.cls) + return + yield + + settings = { + **BASE_SETTINGS, + "SPIDER_MIDDLEWARES": { + TrackingSpiderMiddleware: 0, + }, + } + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = TrackingSpiderMiddleware + yield crawler.crawl() + self.assertEqual(len(TrackingSpiderMiddleware.instances), 1) + self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1]) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = DefaultSpider + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ParentSpiderMiddleware + yield crawler.crawl() + self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1]) + + class ChildSpiderMiddleware(TrackingSpiderMiddleware): + pass + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ChildSpiderMiddleware + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + def test_get_spider_middleware_not_crawling(self): + crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) + self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider) + + @inlineCallbacks + def test_get_spider_middleware_no_engine(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + try: + crawler.get_spider_middleware(DefaultSpider) + except Exception as e: + MySpider.result = e + raise + + crawler = get_raw_crawler(MySpider, BASE_SETTINGS) + with raises(RuntimeError): + yield crawler.crawl() + class SpiderSettingsTestCase(unittest.TestCase): def test_spider_custom_settings(self): @@ -223,20 +589,20 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): return CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"}) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_bootstrap_successful(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) self.assertFalse(runner.bootstrap_failed) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) yield runner.crawl(NoRequestsSpider) self.assertFalse(runner.bootstrap_failed) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_bootstrap_failed(self): runner = self._runner() @@ -249,7 +615,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertTrue(runner.bootstrap_failed) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_bootstrap_failed_for_several(self): runner = self._runner() @@ -264,7 +630,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertTrue(runner.bootstrap_failed) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == "asyncio": CrawlerRunner( @@ -531,7 +897,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.expect_exact("Spider closed (shutdown)") p.wait() - @defer.inlineCallbacks + @inlineCallbacks def test_shutdown_forced(self): from twisted.internet import reactor @@ -543,7 +909,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.kill(sig) p.expect_exact("shutting down gracefully") # sending the second signal too fast often causes problems - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.callback, None) yield d p.kill(sig) From e8dadb959219afea1d3a3f67ce03ac3c7a51520c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 15 Jan 2024 13:37:03 +0100 Subject: [PATCH 1314/2083] scrapy parse: fix the signature of callbacks from the CLI (#6182) --- scrapy/commands/parse.py | 59 ++++++++++++++++++++----------------- tests/test_command_check.py | 4 +-- tests/test_command_parse.py | 18 ++++++++++- 3 files changed, 51 insertions(+), 30 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ac937e464..c9f8586d3 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,3 +1,4 @@ +import functools import inspect import json import logging @@ -251,39 +252,40 @@ class Command(BaseRunSpiderCommand): return scraped_data + def _get_callback(self, *, spider, opts, response=None): + cb = None + if response: + cb = response.meta["_callback"] + if not cb: + if opts.callback: + cb = opts.callback + elif response and opts.rules and self.first_response == response: + cb = self.get_callback_from_rules(spider, response) + if not cb: + raise ValueError( + f"Cannot find a rule that matches {response.url!r} in spider: " + f"{spider.name}" + ) + else: + cb = "parse" + + if not callable(cb): + cb_method = getattr(spider, cb, None) + if callable(cb_method): + cb = cb_method + else: + raise ValueError( + f"Cannot find callback {cb!r} in spider: {spider.name}" + ) + return cb + def prepare_request(self, spider, request, opts): def callback(response, **cb_kwargs): # memorize first request if not self.first_response: self.first_response = response - # determine real callback - cb = response.meta["_callback"] - if not cb: - if opts.callback: - cb = opts.callback - elif opts.rules and self.first_response == response: - cb = self.get_callback_from_rules(spider, response) - - if not cb: - logger.error( - "Cannot find a rule that matches %(url)r in spider: %(spider)s", - {"url": response.url, "spider": spider.name}, - ) - return - else: - cb = "parse" - - if not callable(cb): - cb_method = getattr(spider, cb, None) - if callable(cb_method): - cb = cb_method - else: - logger.error( - "Cannot find callback %(callback)r in spider: %(spider)s", - {"callback": cb, "spider": spider.name}, - ) - return + cb = self._get_callback(spider=spider, opts=opts, response=response) # parse items and requests depth = response.meta["_depth"] @@ -303,6 +305,9 @@ class Command(BaseRunSpiderCommand): request.meta["_depth"] = 1 request.meta["_callback"] = request.callback + if not request.callback and not opts.rules: + cb = self._get_callback(spider=spider, opts=opts) + functools.update_wrapper(callback, cb) request.callback = callback return request diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 129ef0121..592494aba 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -16,11 +16,11 @@ import scrapy class CheckSpider(scrapy.Spider): name = '{self.spider_name}' - start_urls = ['http://toscrape.com'] + start_urls = ['data:,'] def parse(self, response, **cb_kwargs): \"\"\" - @url http://toscrape.com + @url data:, {contracts} \"\"\" {parse_def} diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 037333c03..9356d6b79 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -78,9 +78,21 @@ class AsyncDefAsyncioGenExcSpider(scrapy.Spider): if i > 5: raise ValueError("Stopping the processing") +class CallbackSignatureDownloaderMiddleware: + def process_request(self, request, spider): + from inspect import signature + spider.logger.debug(f"request.callback signature: {{signature(request.callback)}}") + + class MySpider(scrapy.Spider): name = '{self.spider_name}' + custom_settings = {{ + "DOWNLOADER_MIDDLEWARES": {{ + CallbackSignatureDownloaderMiddleware: 0, + }} + }} + def parse(self, response): if getattr(self, 'test_arg', None): self.logger.debug('It Works!') @@ -220,7 +232,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + log = _textmode(stderr) + self.assertIn("DEBUG: It Works!", log) + self.assertIn( + "DEBUG: request.callback signature: (response, foo=None, key=None)", log + ) @defer.inlineCallbacks def test_request_without_meta(self): From d5233bb57f35c54b0c03981dc59c7328ca44cb9a Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Mon, 15 Jan 2024 14:11:33 +0100 Subject: [PATCH 1315/2083] chore(docs): update `sphinx` dependencies (#6200) --- docs/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 9f9aef711..5f683d34c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -sphinx==5.0.2 -sphinx-hoverxref==1.1.1 -sphinx-notfound-page==0.8 -sphinx-rtd-theme==1.0.0 +sphinx==6.2.1 +sphinx-hoverxref==1.3.0 +sphinx-notfound-page==1.0.0 +sphinx-rtd-theme==2.0.0 From 09a7efef7c75558c9ea198a00fc11ab26fb16ce5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 12 Jan 2024 18:30:41 +0400 Subject: [PATCH 1316/2083] Remove a defer.returnValue call. --- tests/test_feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 56967c0d5..ae5810fb8 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2300,7 +2300,7 @@ class BatchDeliveriesTest(FeedExportTestBase): content[feed["format"]].append(file.read_bytes()) finally: self.tearDown() - defer.returnValue(content) + return content @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): From 88285e75b6b7a22689208c2d321a1eda60b64003 Mon Sep 17 00:00:00 2001 From: Kevin Toms Date: Wed, 17 Jan 2024 10:05:22 -0500 Subject: [PATCH 1317/2083] Add FAQ on making a blank request --- docs/faq.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 20dd814df..9df4490d4 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -405,6 +405,25 @@ or :class:`~scrapy.signals.headers_received` signals and raising a :ref:`topics-stop-response-download` topic for additional information and examples. +.. _faq-blank-request: + +How can I make a blank request? +------------------------------- + +.. code-block:: python + + from scrapy import Request + + yield Request( + url="data:,", + callback=self.your_call_back, + ) + +In this case, the URL is set to a data URI scheme. Data URLs allow you to include data +in-line in web pages as if they were external resources. The "data:" scheme with an empty +content (",") essentially creates a request to a data URL without any specific content. + + Running ``runspider`` I get ``error: No spider found in file: `` -------------------------------------------------------------------------- From 46f94ec9cb0f480999f018ceab4a5751abaf180e Mon Sep 17 00:00:00 2001 From: Kevin Toms Date: Wed, 17 Jan 2024 15:49:51 -0500 Subject: [PATCH 1318/2083] Fix test Wrap the yield line in a function to prevent throwing error when the code snippet is executed --- docs/faq.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 9df4490d4..0282fc6e2 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -414,10 +414,11 @@ How can I make a blank request? from scrapy import Request - yield Request( - url="data:,", - callback=self.your_call_back, - ) + def make_blank_request(your_call_back): + yield Request( + url="data:,", + callback=your_call_back, + ) In this case, the URL is set to a data URI scheme. Data URLs allow you to include data in-line in web pages as if they were external resources. The "data:" scheme with an empty From 9074c16497bde04f5d561df1d584abe2cc73f183 Mon Sep 17 00:00:00 2001 From: Kevin Toms Date: Thu, 18 Jan 2024 09:36:25 -0500 Subject: [PATCH 1319/2083] make suggestion --- docs/faq.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 0282fc6e2..2113b0964 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -414,11 +414,8 @@ How can I make a blank request? from scrapy import Request - def make_blank_request(your_call_back): - yield Request( - url="data:,", - callback=your_call_back, - ) + + blank_request = Request("data:,") In this case, the URL is set to a data URI scheme. Data URLs allow you to include data in-line in web pages as if they were external resources. The "data:" scheme with an empty From 2487e3cc035e490777b921badc84c11b9fb77b20 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 10:05:50 -0300 Subject: [PATCH 1320/2083] Cleanup deprecated fingerprint code in scrapy.utils.request --- scrapy/settings/default_settings.py | 2 +- scrapy/utils/request.py | 143 +-------------- tests/test_utils_request.py | 262 +--------------------------- 3 files changed, 6 insertions(+), 401 deletions(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d6b3585e2..02494bad0 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -260,7 +260,7 @@ REFERER_ENABLED = True REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" -REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.6" +REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index cea1bc727..b230cd214 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -5,7 +5,6 @@ scrapy.http.Request objects import hashlib import json -import warnings from typing import ( TYPE_CHECKING, Any, @@ -26,7 +25,6 @@ from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url from scrapy import Request, Spider -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode @@ -34,9 +32,6 @@ from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: from scrapy.crawler import Crawler -_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" -_deprecated_fingerprint_cache = WeakKeyDictionary() - def _serialize_headers( headers: Iterable[bytes], request: Request @@ -47,120 +42,6 @@ def _serialize_headers( yield from request.headers.getlist(header) -def request_fingerprint( - request: Request, - include_headers: Optional[Iterable[Union[bytes, str]]] = None, - keep_fragments: bool = False, -) -> str: - """ - Return the request fingerprint as an hexadecimal string. - - The request fingerprint is a hash that uniquely identifies the resource the - request points to. For example, take the following two urls: - - http://www.example.com/query?id=111&cat=222 - http://www.example.com/query?cat=222&id=111 - - Even though those are two different URLs both point to the same resource - and are equivalent (i.e. they should return the same response). - - Another example are cookies used to store session ids. Suppose the - following page is only accessible to authenticated users: - - http://www.example.com/members/offers.html - - Lots of sites use a cookie to store the session id, which adds a random - component to the HTTP Request and thus should be ignored when calculating - the fingerprint. - - For this reason, request headers are ignored by default when calculating - the fingerprint. If you want to include specific headers use the - include_headers argument, which is a list of Request headers to include. - - Also, servers usually ignore fragments in urls when handling requests, - so they are also ignored by default when calculating the fingerprint. - If you want to include them, set the keep_fragments argument to True - (for instance when handling requests with a headless browser). - """ - if include_headers or keep_fragments: - message = ( - "Call to deprecated function " - "scrapy.utils.request.request_fingerprint().\n" - "\n" - "If you are using this function in a Scrapy component because you " - "need a non-default fingerprinting algorithm, and you are OK " - "with that non-default fingerprinting algorithm being used by " - "all Scrapy components and not just the one calling this " - "function, use crawler.request_fingerprinter.fingerprint() " - "instead in your Scrapy component (you can get the crawler " - "object from the 'from_crawler' class method), and use the " - "'REQUEST_FINGERPRINTER_CLASS' setting to configure your " - "non-default fingerprinting algorithm.\n" - "\n" - "Otherwise, consider using the " - "scrapy.utils.request.fingerprint() function instead.\n" - "\n" - "If you switch to 'fingerprint()', or assign the " - "'REQUEST_FINGERPRINTER_CLASS' setting a class that uses " - "'fingerprint()', the generated fingerprints will not only be " - "bytes instead of a string, but they will also be different from " - "those generated by 'request_fingerprint()'. Before you switch, " - "make sure that you understand the consequences of this (e.g. " - "cache invalidation) and are OK with them; otherwise, consider " - "implementing your own function which returns the same " - "fingerprints as the deprecated 'request_fingerprint()' function." - ) - else: - message = ( - "Call to deprecated function " - "scrapy.utils.request.request_fingerprint().\n" - "\n" - "If you are using this function in a Scrapy component, and you " - "are OK with users of your component changing the fingerprinting " - "algorithm through settings, use " - "crawler.request_fingerprinter.fingerprint() instead in your " - "Scrapy component (you can get the crawler object from the " - "'from_crawler' class method).\n" - "\n" - "Otherwise, consider using the " - "scrapy.utils.request.fingerprint() function instead.\n" - "\n" - "Either way, the resulting fingerprints will be returned as " - "bytes, not as a string, and they will also be different from " - "those generated by 'request_fingerprint()'. Before you switch, " - "make sure that you understand the consequences of this (e.g. " - "cache invalidation) and are OK with them; otherwise, consider " - "implementing your own function which returns the same " - "fingerprints as the deprecated 'request_fingerprint()' function." - ) - warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) - processed_include_headers: Optional[Tuple[bytes, ...]] = None - if include_headers: - processed_include_headers = tuple( - to_bytes(h.lower()) for h in sorted(include_headers) - ) - cache = _deprecated_fingerprint_cache.setdefault(request, {}) - cache_key = (processed_include_headers, keep_fragments) - if cache_key not in cache: - fp = hashlib.sha1() - fp.update(to_bytes(request.method)) - fp.update( - to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)) - ) - fp.update(request.body or b"") - if processed_include_headers: - for part in _serialize_headers(processed_include_headers, request): - fp.update(part) - cache[cache_key] = fp.hexdigest() - return cache[cache_key] - - -def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - return bytes.fromhex(request_fingerprint(*args, **kwargs)) - - _fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" _fingerprint_cache = WeakKeyDictionary() @@ -258,32 +139,14 @@ class RequestFingerprinter: "REQUEST_FINGERPRINTER_IMPLEMENTATION" ) else: - implementation = "2.6" - if implementation == "2.6": - message = ( - "'2.6' is a deprecated value for the " - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n" - "\n" - "It is also the default value. In other words, it is normal " - "to get this warning if you have not defined a value for the " - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so " - "for backward compatibility reasons, but it will change in a " - "future version of Scrapy.\n" - "\n" - "See the documentation of the " - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for " - "information on how to handle this deprecation." - ) - warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) - self._fingerprint = _request_fingerprint_as_bytes - elif implementation == "2.7": + implementation = "2.7" + if implementation == "2.7": self._fingerprint = fingerprint else: raise ValueError( f"Got an invalid value on setting " f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': " - f"{implementation!r}. Valid values are '2.6' (deprecated) " - f"and '2.7'." + f"{implementation!r}. Valid value is '2.7'." ) def fingerprint(self, request: Request) -> bytes: diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index e6d1abe3f..f6bc9ba6f 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,23 +1,15 @@ import json import unittest -import warnings from hashlib import sha1 -from typing import Dict, Mapping, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union from weakref import WeakKeyDictionary -import pytest -from w3lib.url import canonicalize_url - from scrapy.http import Request -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.python import to_bytes from scrapy.utils.request import ( - _deprecated_fingerprint_cache, _fingerprint_cache, - _request_fingerprint_as_bytes, fingerprint, request_authenticate, - request_fingerprint, request_httprepr, request_to_curl, ) @@ -233,168 +225,6 @@ class FingerprintTest(unittest.TestCase): self.assertEqual(actual, expected) -class RequestFingerprintTest(FingerprintTest): - function = staticmethod(request_fingerprint) - cache = _deprecated_fingerprint_cache - known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( - ( - Request("http://example.org"), - "b2e5245ef826fd9576c93bd6e392fce3133fab62", - {}, - ), - ( - Request("https://example.org"), - "bd10a0a89ea32cdee77917320f1309b0da87e892", - {}, - ), - ( - Request("https://example.org?a"), - "2fb7d48ae02f04b749f40caa969c0bc3c43204ce", - {}, - ), - ( - Request("https://example.org?a=b"), - "42e5fe149b147476e3f67ad0670c57b4cc57856a", - {}, - ), - ( - Request("https://example.org?a=b&a"), - "d23a9787cb56c6375c2cae4453c5a8c634526942", - {}, - ), - ( - Request("https://example.org?a=b&a=c"), - "9a18a7a8552a9182b7f1e05d33876409e421e5c5", - {}, - ), - ( - Request("https://example.org", method="POST"), - "ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6", - {}, - ), - ( - Request("https://example.org", body=b"a"), - "4bb136e54e715a4ea7a9dd1101831765d33f2d60", - {}, - ), - ( - Request("https://example.org", method="POST", body=b"a"), - "6c6595374a304b293be762f7b7be3f54e9947c65", - {}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "bd10a0a89ea32cdee77917320f1309b0da87e892", - {}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "515b633cb3ca502a33a9d8c890e889ec1e425e65", - {"include_headers": ["A"]}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "505c96e7da675920dfef58725e8c957dfdb38f47", - {"keep_fragments": True}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da", - {"include_headers": ["A"], "keep_fragments": True}, - ), - ( - Request("https://example.org/ab"), - "4e2870fee58582d6f81755e9b8fdefe3cba0c951", - {}, - ), - ( - Request("https://example.org/a", body=b"b"), - "4e2870fee58582d6f81755e9b8fdefe3cba0c951", - {}, - ), - ) - - def setUp(self) -> None: - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - - def tearDown(self) -> None: - warnings.simplefilter("default", ScrapyDeprecationWarning) - - @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True) - def test_part_separation(self): - super().test_part_separation() - - -class RequestFingerprintDeprecationTest(unittest.TestCase): - def test_deprecation_default_parameters(self): - with pytest.warns(ScrapyDeprecationWarning) as warnings: - request_fingerprint(Request("http://www.example.com")) - messages = [str(warning.message) for warning in warnings] - self.assertTrue( - any("Call to deprecated function" in message for message in messages) - ) - self.assertFalse(any("non-default" in message for message in messages)) - - def test_deprecation_non_default_parameters(self): - with pytest.warns(ScrapyDeprecationWarning) as warnings: - request_fingerprint(Request("http://www.example.com"), keep_fragments=True) - messages = [str(warning.message) for warning in warnings] - self.assertTrue( - any("Call to deprecated function" in message for message in messages) - ) - self.assertTrue(any("non-default" in message for message in messages)) - - -class RequestFingerprintAsBytesTest(FingerprintTest): - function = staticmethod(_request_fingerprint_as_bytes) - cache = _deprecated_fingerprint_cache - known_hashes = RequestFingerprintTest.known_hashes - - def test_caching(self): - r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") - self.assertEqual( - self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key]) - ) - - @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True) - def test_part_separation(self): - super().test_part_separation() - - def test_hashes(self): - actual = [ - self.function(request, **kwargs) for request, _, kwargs in self.known_hashes - ] - expected = [ - bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes - ] - self.assertEqual(actual, expected) - - -_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary() - - -def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False): - if include_headers: - include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) - cache = _fingerprint_cache_2_6.setdefault(request, {}) - cache_key = (include_headers, keep_fragments) - if cache_key not in cache: - fp = sha1() - fp.update(to_bytes(request.method)) - fp.update( - to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)) - ) - fp.update(request.body or b"") - if include_headers: - for hdr in include_headers: - if hdr in request.headers: - fp.update(hdr) - for v in request.headers.getlist(hdr): - fp.update(v) - cache[cache_key] = fp.hexdigest() - return cache[cache_key] - - REQUEST_OBJECTS_TO_TEST = ( Request("http://www.example.com/"), Request("http://www.example.com/query?id=111&cat=222"), @@ -424,105 +254,17 @@ REQUEST_OBJECTS_TO_TEST = ( ) -class BackwardCompatibilityTestCase(unittest.TestCase): - def test_function_backward_compatibility(self): - include_headers_to_test = ( - None, - ["Accept-Language"], - ["accept-language", "sessionid"], - ["SESSIONID", "Accept-Language"], - ) - for request_object in REQUEST_OBJECTS_TO_TEST: - for include_headers in include_headers_to_test: - for keep_fragments in (False, True): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - fp = request_fingerprint( - request_object, - include_headers=include_headers, - keep_fragments=keep_fragments, - ) - old_fp = request_fingerprint_2_6( - request_object, - include_headers=include_headers, - keep_fragments=keep_fragments, - ) - self.assertEqual(fp, old_fp) - - def test_component_backward_compatibility(self): - for request_object in REQUEST_OBJECTS_TO_TEST: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - crawler = get_crawler(prevent_warnings=False) - fp = crawler.request_fingerprinter.fingerprint(request_object) - old_fp = request_fingerprint_2_6(request_object) - self.assertEqual(fp.hex(), old_fp) - - def test_custom_component_backward_compatibility(self): - """Tests that the backward-compatible request fingerprinting class featured - in the documentation is indeed backward compatible and does not cause a - warning to be logged.""" - - class RequestFingerprinter: - cache = WeakKeyDictionary() - - def fingerprint(self, request): - if request not in self.cache: - fp = sha1() - fp.update(to_bytes(request.method)) - fp.update(to_bytes(canonicalize_url(request.url))) - fp.update(request.body or b"") - self.cache[request] = fp.digest() - return self.cache[request] - - for request_object in REQUEST_OBJECTS_TO_TEST: - with warnings.catch_warnings() as logged_warnings: - settings = { - "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, - } - crawler = get_crawler(settings_dict=settings) - fp = crawler.request_fingerprinter.fingerprint(request_object) - old_fp = request_fingerprint_2_6(request_object) - self.assertEqual(fp.hex(), old_fp) - self.assertFalse(logged_warnings) - - class RequestFingerprinterTestCase(unittest.TestCase): def test_default_implementation(self): - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(prevent_warnings=False) - request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - _request_fingerprint_as_bytes(request), - ) - self.assertTrue(logged_warnings) - - def test_deprecated_implementation(self): - settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6", - } - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(settings_dict=settings) - request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - _request_fingerprint_as_bytes(request), - ) - self.assertTrue(logged_warnings) - - def test_recommended_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(settings_dict=settings) request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), fingerprint(request), ) - self.assertFalse(logged_warnings) def test_unknown_implementation(self): settings = { From 019443dd5761afd5b4f7bba5948508554f1cb5f1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 10:08:07 -0300 Subject: [PATCH 1321/2083] Remove settings from default implementation test --- tests/test_utils_request.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index f6bc9ba6f..68f6eb045 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -256,10 +256,7 @@ REQUEST_OBJECTS_TO_TEST = ( class RequestFingerprinterTestCase(unittest.TestCase): def test_default_implementation(self): - settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - } - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler() request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), From bacaf0db7ac8b3b424af41d24e12e89a3a15b004 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 10:14:48 -0300 Subject: [PATCH 1322/2083] Update documentation --- docs/topics/request-response.rst | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 9d64eee45..6dbcb4584 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -477,20 +477,12 @@ REQUEST_FINGERPRINTER_IMPLEMENTATION .. versionadded:: 2.7 -Default: ``'2.6'`` +Default: ``'2.7'`` Determines which request fingerprinting algorithm is used by the default request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`). -Possible values are: - -- ``'2.6'`` (default) - - This implementation uses the same request fingerprinting algorithm as - Scrapy 2.6 and earlier versions. - - Even though this is the default value for backward compatibility reasons, - it is a deprecated value. +Possible value is: - ``'2.7'`` @@ -500,29 +492,13 @@ Possible values are: New projects should use this value. The :command:`startproject` command sets this value in the generated ``settings.py`` file. -If you are using the default value (``'2.6'``) for this setting, and you are -using Scrapy components where changing the request fingerprinting algorithm -would cause undesired results, you need to carefully decide when to change the -value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS` -setting to a custom request fingerprinter class that implements the 2.6 request -fingerprinting algorithm and does not log this warning ( -:ref:`2.6-request-fingerprinter` includes an example implementation of such a -class). - Scenarios where changing the request fingerprinting algorithm may cause undesired results include, for example, using the HTTP cache middleware (see :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`). Changing the request fingerprinting algorithm would invalidate the current cache, requiring you to redownload all requests again. -Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in -your settings to switch already to the request fingerprinting implementation -that will be the only request fingerprinting implementation available in a -future version of Scrapy, and remove the deprecation warning triggered by using -the default value (``'2.6'``). - -.. _2.6-request-fingerprinter: .. _custom-request-fingerprinter: Writing your own request fingerprinter From 24634f1bb236f72a1a7b73900f8e3b524f7717b5 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 12:29:43 -0300 Subject: [PATCH 1323/2083] Attend PR comments --- docs/topics/request-response.rst | 30 ------------------- scrapy/settings/default_settings.py | 2 +- .../templates/project/module/settings.py.tmpl | 1 - scrapy/utils/request.py | 16 ++++++++-- scrapy/utils/test.py | 3 -- tests/test_utils_request.py | 14 +++++++++ 6 files changed, 28 insertions(+), 38 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 6dbcb4584..67fc0c6e9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -469,36 +469,6 @@ import path. .. autoclass:: scrapy.utils.request.RequestFingerprinter - -.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION - -REQUEST_FINGERPRINTER_IMPLEMENTATION -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 2.7 - -Default: ``'2.7'`` - -Determines which request fingerprinting algorithm is used by the default -request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`). - -Possible value is: - -- ``'2.7'`` - - This implementation was introduced in Scrapy 2.7 to fix an issue of the - previous implementation. - - New projects should use this value. The :command:`startproject` command - sets this value in the generated ``settings.py`` file. - -Scenarios where changing the request fingerprinting algorithm may cause -undesired results include, for example, using the HTTP cache middleware (see -:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`). -Changing the request fingerprinting algorithm would invalidate the current -cache, requiring you to redownload all requests again. - - .. _custom-request-fingerprinter: Writing your own request fingerprinter diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 02494bad0..49ab1b5ef 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -260,7 +260,7 @@ REFERER_ENABLED = True REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" -REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" +REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL" RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index ecb1e5e5c..b4779e555 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -88,6 +88,5 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # Set settings whose default value is deprecated to a future-proof value -REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index b230cd214..068e5bdcb 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -5,6 +5,7 @@ scrapy.http.Request objects import hashlib import json +import warnings from typing import ( TYPE_CHECKING, Any, @@ -25,6 +26,7 @@ from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url from scrapy import Request, Spider +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode @@ -139,14 +141,22 @@ class RequestFingerprinter: "REQUEST_FINGERPRINTER_IMPLEMENTATION" ) else: - implementation = "2.7" - if implementation == "2.7": + implementation = "SENTINEL" + + if implementation == "SENTINEL": + self._fingerprint = fingerprint + elif implementation == "2.7": + message = ( + "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n" + "And it will be removed in future version of Scrapy." + ) + warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) self._fingerprint = fingerprint else: raise ValueError( f"Got an invalid value on setting " f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': " - f"{implementation!r}. Valid value is '2.7'." + f"{implementation!r}. Valid values are '2.7' and 'SENTINEL'." ) def fingerprint(self, request: Request) -> bytes: diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 709e0b00d..c6a31cacf 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -76,7 +76,6 @@ class TestSpider(Spider): def get_crawler( spidercls: Optional[Type[Spider]] = None, settings_dict: Optional[Dict[str, Any]] = None, - prevent_warnings: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level @@ -86,8 +85,6 @@ def get_crawler( # Set by default settings that prevent deprecation warnings. settings: Dict[str, Any] = {} - if prevent_warnings: - settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7" settings.update(settings_dict or {}) runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or TestSpider) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 68f6eb045..c0c44875e 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,5 +1,6 @@ import json import unittest +import warnings from hashlib import sha1 from typing import Dict, Optional, Tuple, Union from weakref import WeakKeyDictionary @@ -263,6 +264,19 @@ class RequestFingerprinterTestCase(unittest.TestCase): fingerprint(request), ) + def test_deprecated_implementation(self): + settings = { + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } + with warnings.catch_warnings(record=True) as logged_warnings: + crawler = get_crawler(settings_dict=settings) + request = Request("https://example.com") + self.assertEqual( + crawler.request_fingerprinter.fingerprint(request), + fingerprint(request), + ) + self.assertTrue(logged_warnings) + def test_unknown_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5", From 7001193c802029612542ab7a30fa8c0e147a1894 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 12:53:08 -0300 Subject: [PATCH 1324/2083] Simplify the logic --- scrapy/utils/request.py | 12 ++---------- scrapy/utils/test.py | 3 +++ tests/test_utils_request.py | 7 ------- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 068e5bdcb..db0b44cf4 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -143,21 +143,13 @@ class RequestFingerprinter: else: implementation = "SENTINEL" - if implementation == "SENTINEL": - self._fingerprint = fingerprint - elif implementation == "2.7": + if implementation != "SENTINEL": message = ( "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n" "And it will be removed in future version of Scrapy." ) warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) - self._fingerprint = fingerprint - else: - raise ValueError( - f"Got an invalid value on setting " - f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': " - f"{implementation!r}. Valid values are '2.7' and 'SENTINEL'." - ) + self._fingerprint = fingerprint def fingerprint(self, request: Request) -> bytes: return self._fingerprint(request) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index c6a31cacf..9234ec2ea 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -76,6 +76,7 @@ class TestSpider(Spider): def get_crawler( spidercls: Optional[Type[Spider]] = None, settings_dict: Optional[Dict[str, Any]] = None, + prevent_warnings: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level @@ -85,6 +86,8 @@ def get_crawler( # Set by default settings that prevent deprecation warnings. settings: Dict[str, Any] = {} + if prevent_warnings: + pass settings.update(settings_dict or {}) runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or TestSpider) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index c0c44875e..633077eec 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -277,13 +277,6 @@ class RequestFingerprinterTestCase(unittest.TestCase): ) self.assertTrue(logged_warnings) - def test_unknown_implementation(self): - settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5", - } - with self.assertRaises(ValueError): - get_crawler(settings_dict=settings) - class CustomRequestFingerprinterTestCase(unittest.TestCase): def test_include_headers(self): From 53ccf0016d99e54adec6f98236cce37ede835f63 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 13:18:10 -0300 Subject: [PATCH 1325/2083] Remove empty statement --- scrapy/utils/test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 9234ec2ea..7a8c5c859 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -86,8 +86,6 @@ def get_crawler( # Set by default settings that prevent deprecation warnings. settings: Dict[str, Any] = {} - if prevent_warnings: - pass settings.update(settings_dict or {}) runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or TestSpider) From c5dad41190551578c2973c34520952f26f75dc7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 2 Feb 2024 14:03:16 +0100 Subject: [PATCH 1326/2083] Speed up tests, remove comments without regexps --- .github/workflows/tests-ubuntu.yml | 3 -- scrapy/utils/response.py | 14 +++++++- tests/test_utils_response.py | 51 ++++++++++++++++++++++++++---- tox.ini | 6 ---- 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 388ba9572..338c99584 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -50,9 +50,6 @@ jobs: - python-version: "3.12" env: TOXENV: botocore - - python-version: "3.12" - env: - TOXENV: slow steps: - uses: actions/checkout@v3 diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 4369e6439..fabfb1167 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -74,6 +74,18 @@ def response_httprepr(response: Response) -> bytes: return b"".join(values) +def _remove_html_comments(body): + start = body.find(b"", start + 1) + if end == -1: + return body[:start] + else: + body = body[:start] + body[end + 3 :] + start = body.find(b"|$)", b"", body) body = re.sub(rb"]*?>)", to_bytes(repl), body, count=1) ext = ".html" elif isinstance(response, TextResponse): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 1dbe187bf..db3c31b89 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -8,9 +8,9 @@ import pytest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response, TextResponse -from scrapy.settings.default_settings import DOWNLOAD_MAXSIZE from scrapy.utils.python import to_bytes from scrapy.utils.response import ( + _remove_html_comments, get_base_url, get_meta_refresh, open_in_browser, @@ -203,14 +203,13 @@ class ResponseUtilsTest(unittest.TestCase): r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" - @pytest.mark.slow def test_open_in_browser_redos_comment(self): - MAX_CPU_TIME = 30 + MAX_CPU_TIME = 0.001 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ # for // (old pattern to remove comments). - body = b"->" + body = b"->" response = HtmlResponse("https://example.com", body=body) @@ -221,14 +220,13 @@ class ResponseUtilsTest(unittest.TestCase): end_time = process_time() self.assertLess(end_time - start_time, MAX_CPU_TIME) - @pytest.mark.slow def test_open_in_browser_redos_head(self): - MAX_CPU_TIME = 15 + MAX_CPU_TIME = 0.001 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ # for /(|\s.*?>))/ (old pattern to find the head element). - body = b"b", + b"ab", + ), + ( + b"ac", + b"ac", + ), + ( + b"acccd", + b"acd", + ), + ( + b"ad", + b"ad", + ), + ), +) +def test_remove_html_comments(input_body, output_body): + assert ( + _remove_html_comments(input_body) == output_body + ), f"{_remove_html_comments(input_body)=} == {output_body=}" diff --git a/tox.ini b/tox.ini index e87d6a175..381da9773 100644 --- a/tox.ini +++ b/tox.ini @@ -221,9 +221,3 @@ setenv = {[pinned]setenv} commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} - - -[testenv:slow] -basepython = python3 -commands = - {[testenv]commands} -m 'slow' From 810aaa637da12a1f393291eb2b13aa0c8a163efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 2 Feb 2024 14:04:28 +0100 Subject: [PATCH 1327/2083] Undo an unintended change --- .github/workflows/tests-ubuntu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 338c99584..c883f958c 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -50,6 +50,7 @@ jobs: - python-version: "3.12" env: TOXENV: botocore + steps: - uses: actions/checkout@v3 From 5e5a92026e43023b80f7733844a2703c3f966009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 2 Feb 2024 14:06:45 +0100 Subject: [PATCH 1328/2083] Remove slow leftovers --- pytest.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 877fbcd1d..16983be5e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -17,12 +17,10 @@ addopts = --ignore=docs/topics/stats.rst --ignore=docs/topics/telnetconsole.rst --ignore=docs/utils - -m 'not slow' markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed requires_uvloop: marks tests as only enabled when uvloop is known to be working - slow: marks tests as slow, not executed by default filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated From 1c9d308accd38a91ffa92e3aff8912cd792070eb Mon Sep 17 00:00:00 2001 From: Andy <128531452+Andy-W-Developer@users.noreply.github.com> Date: Tue, 6 Feb 2024 00:52:01 +1300 Subject: [PATCH 1329/2083] Cover the deprecation and removal of response_httprepr in the release notes (#6216) --- docs/news.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 65d9c5181..d90e32560 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -62,6 +62,9 @@ Deprecation removals 1.0.0, use :attr:`CrawlerRunner.spider_loader ` instead. (:issue:`6010`) +- The :func:`scrapy.utils.response.response_httprepr` function, deprecated in + Scrapy 2.6.0, has now been removed. (:issue:`6111`) + Deprecations ~~~~~~~~~~~~ @@ -1157,6 +1160,9 @@ Deprecations Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider` first to set the :class:`~scrapy.Spider` object. +- :func:`scrapy.utils.response.response_httprepr` is now deprecated. + (:issue:`4972`) + New features ~~~~~~~~~~~~ From a55e933c11899997757bd4107738f9472d1d3c2e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Feb 2024 20:08:40 +0400 Subject: [PATCH 1330/2083] Release notes for 2.11.1 (#6150) --- docs/news.rst | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 0c202639e..c26cef22c 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,64 @@ Release notes ============= +.. _release-2.11.1: + +Scrapy 2.11.1 (YYYY-MM-DD) +-------------------------- + +Highlights: + +- Support for Twisted >= 23.8.0. + +- Documentation improvements. + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- The Twisted dependency is no longer restricted to < 23.8.0. (:issue:`6024`, + :issue:`6064`, :issue:`6142`) + +Bug fixes +~~~~~~~~~ + +- The OS signal handling code was refactored to no longer use private Twisted + functions. (:issue:`6024`, :issue:`6064`, :issue:`6112`) + +Documentation +~~~~~~~~~~~~~ + +- Improved documentation for :class:`~scrapy.crawler.Crawler` initialization + changes made in the 2.11.0 release. (:issue:`6057`, :issue:`6147`) + +- Extended documentation for :attr:`Request.meta `. + (:issue:`5565`) + +- Fixed the :reqmeta:`dont_merge_cookies` documentation. (:issue:`5936`, + :issue:`6077`) + +- Added a link to Zyte's export guides to the :ref:`feed exports + ` documentation. (:issue:`6183`) + +- Added a missing note about backward-incompatible changes in + :class:`~scrapy.exporters.PythonItemExporter` to the 2.11.0 release notes. + (:issue:`6060`, :issue:`6081`) + +- Added a missing note about removing the deprecated + ``scrapy.utils.boto.is_botocore()`` function to the 2.8.0 release notes. + (:issue:`6056`, :issue:`6061`) + +- Other documentation improvements. (:issue:`6128`, :issue:`6144`, + :issue:`6163`, :issue:`6190`, :issue:`6192`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added Python 3.12 to the CI configuration, re-enabled tests that were + disabled when the pre-release support was added. (:issue:`5985`, + :issue:`6083`, :issue:`6098`) + +- Fixed a test issue on PyPy 7.3.14. (:issue:`6204`, :issue:`6205`) + .. _release-2.11.0: Scrapy 2.11.0 (2023-09-18) From 6b88b3346c393f07c4e4481405c3fd1ab4cc58a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 14 Feb 2024 18:16:40 +0100 Subject: [PATCH 1331/2083] Set the release date of versions 2.11.1 and 1.8.4 --- docs/news.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 16e7e79a7..518632a5b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.11.1: -Scrapy 2.11.1 (unreleased) +Scrapy 2.11.1 (2024-02-14) -------------------------- Highlights: @@ -2972,7 +2972,7 @@ affect subclasses: .. _release-1.8.4: -Scrapy 1.8.4 (unreleased) +Scrapy 1.8.4 (2024-02-14) ------------------------- **Security bug fixes:** From 502addc717b6b971425a9385359a382b8d0187a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 14 Feb 2024 18:17:48 +0100 Subject: [PATCH 1332/2083] =?UTF-8?q?Bump=20version:=202.11.0=20=E2=86=92?= =?UTF-8?q?=202.11.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f76bf783d..6ce6e2a59 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.11.0 +current_version = 2.11.1 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 46b81d815..6ceb272ee 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.11.0 +2.11.1 From 2f1d345e74d19e33016f9e69fcda0bda9afb568d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 14 Feb 2024 18:59:01 +0100 Subject: [PATCH 1333/2083] Solve test issues --- ...st_downloadermiddleware_httpcompression.py | 24 +++++++++++++++++++ tests/test_utils_response.py | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index f74fff218..9deb81c37 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -402,6 +402,10 @@ class HttpCompressionTest(TestCase): self._test_compression_bomb_setting("gzip") def test_compression_bomb_setting_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") self._test_compression_bomb_setting("zstd") def _test_compression_bomb_spider_attr(self, compression_id): @@ -436,6 +440,10 @@ class HttpCompressionTest(TestCase): self._test_compression_bomb_spider_attr("gzip") def test_compression_bomb_spider_attr_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") self._test_compression_bomb_spider_attr("zstd") def _test_compression_bomb_request_meta(self, compression_id): @@ -468,6 +476,10 @@ class HttpCompressionTest(TestCase): self._test_compression_bomb_request_meta("gzip") def test_compression_bomb_request_meta_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") self._test_compression_bomb_request_meta("zstd") def _test_download_warnsize_setting(self, compression_id): @@ -510,6 +522,10 @@ class HttpCompressionTest(TestCase): self._test_download_warnsize_setting("gzip") def test_download_warnsize_setting_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") self._test_download_warnsize_setting("zstd") def _test_download_warnsize_spider_attr(self, compression_id): @@ -554,6 +570,10 @@ class HttpCompressionTest(TestCase): self._test_download_warnsize_spider_attr("gzip") def test_download_warnsize_spider_attr_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") self._test_download_warnsize_spider_attr("zstd") def _test_download_warnsize_request_meta(self, compression_id): @@ -596,6 +616,10 @@ class HttpCompressionTest(TestCase): self._test_download_warnsize_request_meta("gzip") def test_download_warnsize_request_meta_zstd(self): + try: + import zstandard # noqa: F401 + except ImportError: + raise SkipTest("no zstd support (zstandard)") self._test_download_warnsize_request_meta("zstd") diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index db3c31b89..37ef89e76 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -204,7 +204,7 @@ class ResponseUtilsTest(unittest.TestCase): ), "Inject unique base url with conditional comment" def test_open_in_browser_redos_comment(self): - MAX_CPU_TIME = 0.001 + MAX_CPU_TIME = 0.02 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ @@ -221,7 +221,7 @@ class ResponseUtilsTest(unittest.TestCase): self.assertLess(end_time - start_time, MAX_CPU_TIME) def test_open_in_browser_redos_head(self): - MAX_CPU_TIME = 0.001 + MAX_CPU_TIME = 0.02 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ From bccb4cf18ba38c8bf09d61d19e0ffabaf15554b1 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 14 Feb 2024 12:29:29 -0600 Subject: [PATCH 1334/2083] fix: LxmlLinkExtractor unique_list missing key --- scrapy/linkextractors/lxmlhtml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 23cbd0116..98781ba7f 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -248,5 +248,5 @@ class LxmlLinkExtractor: links = self._extract_links(doc, response.url, response.encoding, base_url) all_links.extend(self._process_links(links)) if self.link_extractor.unique: - return unique_list(all_links) + return unique_list(all_links, key=self.link_extractor.link_key) return all_links From 660e3b19532c50eac7d135549e594b7f98285184 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Thu, 15 Feb 2024 16:55:08 -0600 Subject: [PATCH 1335/2083] update: docs/topics/items.rst --- docs/topics/items.rst | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 3c38ac2dc..97ed7a900 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -399,12 +399,7 @@ In code that receives an item, such as methods of :ref:`item pipelines `, it is a good practice to use the :class:`~itemadapter.ItemAdapter` class and the :func:`~itemadapter.is_item` function to write code that works for -any :ref:`supported item type `: - -.. autoclass:: itemadapter.ItemAdapter - -.. autofunction:: itemadapter.is_item - +any supported item type. Other classes related to items ============================== From 3e7b704c08aacd51a8a7589e32c73c7754582eed Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Thu, 15 Feb 2024 16:57:44 -0600 Subject: [PATCH 1336/2083] update: docs/topics/selectors.rst --- docs/topics/selectors.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 4a64d530b..c841400b6 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -1032,11 +1032,6 @@ whereas the CSS lookup is translated into XPath and thus runs more efficiently, so performance-wise its uses are limited to situations that are not easily described with CSS selectors. -Parsel also simplifies adding your own XPath extensions. - -.. autofunction:: parsel.xpathfuncs.set_xpathfunc - - .. _topics-selectors-ref: Built-in Selectors reference From 9bb973dc54766a0f8d10eca0947d11f195c1a1be Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Fri, 16 Feb 2024 19:25:38 +0800 Subject: [PATCH 1337/2083] Refactor LogStats extension to log IPM and RPM to stats on spider_close (#4111) --- scrapy/extensions/logstats.py | 44 +++++++++++++++++++------ tests/test_logstats.py | 62 +++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 10 deletions(-) create mode 100644 tests/test_logstats.py diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 78874a6db..9f63e9c4b 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -9,7 +9,10 @@ logger = logging.getLogger(__name__) class LogStats: - """Log basic scraping stats periodically""" + """Log basic scraping stats periodically like: + * RPM - Requests per Minute + * IPM - Items per Minute + """ def __init__(self, stats, interval=60.0): self.stats = stats @@ -35,24 +38,45 @@ class LogStats: self.task.start(self.interval) def log(self, spider): - items = self.stats.get_value("item_scraped_count", 0) - pages = self.stats.get_value("response_received_count", 0) - irate = (items - self.itemsprev) * self.multiplier - prate = (pages - self.pagesprev) * self.multiplier - self.pagesprev, self.itemsprev = pages, items + self.calculate_stats() msg = ( "Crawled %(pages)d pages (at %(pagerate)d pages/min), " "scraped %(items)d items (at %(itemrate)d items/min)" ) log_args = { - "pages": pages, - "pagerate": prate, - "items": items, - "itemrate": irate, + "pages": self.pages, + "pagerate": self.prate, + "items": self.items, + "itemrate": self.irate, } logger.info(msg, log_args, extra={"spider": spider}) + def calculate_stats(self): + self.items = self.stats.get_value("item_scraped_count", 0) + self.pages = self.stats.get_value("response_received_count", 0) + self.irate = (self.items - self.itemsprev) * self.multiplier + self.prate = (self.pages - self.pagesprev) * self.multiplier + self.pagesprev, self.itemsprev = self.pages, self.items + def spider_closed(self, spider, reason): if self.task and self.task.running: self.task.stop() + + rpm_final, ipm_final = self.calculate_final_stats(spider) + self.stats.set_value("responses_per_minute", rpm_final) + self.stats.set_value("items_per_minute", ipm_final) + + def calculate_final_stats(self, spider): + start_time = self.stats.get_value("start_time") + finished_time = self.stats.get_value("finished_time") + + if not start_time or not finished_time: + return None, None + + mins_elapsed = (finished_time - start_time).seconds / 60 + + items = self.stats.get_value("item_scraped_count", 0) + pages = self.stats.get_value("response_received_count", 0) + + return (pages / mins_elapsed), (items / mins_elapsed) diff --git a/tests/test_logstats.py b/tests/test_logstats.py new file mode 100644 index 000000000..d87285df7 --- /dev/null +++ b/tests/test_logstats.py @@ -0,0 +1,62 @@ +import unittest +from datetime import datetime + +from scrapy.extensions.logstats import LogStats +from scrapy.utils.test import get_crawler +from tests.spiders import SimpleSpider + + +class TestLogStats(unittest.TestCase): + def setUp(self): + self.crawler = get_crawler(SimpleSpider) + self.spider = self.crawler._create_spider("spidey") + self.stats = self.crawler.stats + + self.stats.set_value("response_received_count", 4802) + self.stats.set_value("item_scraped_count", 3201) + + def test_stats_calculations(self): + logstats = LogStats.from_crawler(self.crawler) + + with self.assertRaises(AttributeError): + logstats.pagesprev + logstats.itemsprev + + logstats.spider_opened(self.spider) + self.assertEqual(logstats.pagesprev, 4802) + self.assertEqual(logstats.itemsprev, 3201) + + logstats.calculate_stats() + self.assertEqual(logstats.items, 3201) + self.assertEqual(logstats.pages, 4802) + self.assertEqual(logstats.irate, 0.0) + self.assertEqual(logstats.prate, 0.0) + self.assertEqual(logstats.pagesprev, 4802) + self.assertEqual(logstats.itemsprev, 3201) + + # Simulate what happens after a minute + self.stats.set_value("response_received_count", 5187) + self.stats.set_value("item_scraped_count", 3492) + logstats.calculate_stats() + self.assertEqual(logstats.items, 3492) + self.assertEqual(logstats.pages, 5187) + self.assertEqual(logstats.irate, 291.0) + self.assertEqual(logstats.prate, 385.0) + self.assertEqual(logstats.pagesprev, 5187) + self.assertEqual(logstats.itemsprev, 3492) + + # Simulate when spider closes after running for 30 mins + self.stats.set_value("start_time", datetime.fromtimestamp(1655100172)) + self.stats.set_value("finished_time", datetime.fromtimestamp(1655101972)) + logstats.spider_closed(self.spider, "test reason") + self.assertEqual(self.stats.get_value("responses_per_minute"), 172.9) + self.assertEqual(self.stats.get_value("items_per_minute"), 116.4) + + def test_stats_calculations_no_time(self): + """The stat values should be None since the start and finish time are + not available. + """ + logstats = LogStats.from_crawler(self.crawler) + logstats.spider_closed(self.spider, "test reason") + self.assertIsNone(self.stats.get_value("responses_per_minute")) + self.assertIsNone(self.stats.get_value("items_per_minute")) From 36f72877ba8863a7fc39383e79f478400f6c09e9 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 16 Feb 2024 10:39:16 -0600 Subject: [PATCH 1338/2083] update: docs/topics/selectors.rst --- docs/topics/selectors.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index c841400b6..e32fc2b70 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -1032,6 +1032,9 @@ whereas the CSS lookup is translated into XPath and thus runs more efficiently, so performance-wise its uses are limited to situations that are not easily described with CSS selectors. +Parsel also simplifies adding your own XPath extensions with +:func:`~parsel.xpathfuncs.set_xpathfunc`. + .. _topics-selectors-ref: Built-in Selectors reference From 5e51417a485f296354e9639f85fb0b51a4a3e533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 16 Feb 2024 20:10:52 +0100 Subject: [PATCH 1339/2083] Add tests, fix canonicalize passing --- scrapy/linkextractors/lxmlhtml.py | 2 +- tests/test_linkextractors.py | 112 ++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 98781ba7f..7abdaaec4 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -153,7 +153,7 @@ class LxmlLinkExtractor: unique=unique, process=process_value, strip=strip, - canonicalized=canonicalize, + canonicalized=not canonicalize, ) self.allow_res = [ x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 18e9608c1..f23b8988e 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -745,6 +745,118 @@ class Base: lx = self.extractor_cls() self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) + def test_link_extractor_aggregation(self): + """When a parameter like restrict_css is used, the underlying + implementation calls its internal link extractor once per selector + matching the specified restrictions, and then aggregates the + extracted links. + + Test that aggregation respects the unique and canonicalize + parameters. + """ + # unique=True (default), canonicalize=False (default) + lx = self.extractor_cls(restrict_css=("div",)) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + self.assertEqual( + actual, + [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/b?b=2&a=1", text="b2"), + ], + ) + + # unique=True (default), canonicalize=True + lx = self.extractor_cls(restrict_css=("div",), canonicalize=True) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + self.assertEqual( + actual, + [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + ], + ) + + # unique=False, canonicalize=False (default) + lx = self.extractor_cls(restrict_css=("div",), unique=False) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + self.assertEqual( + actual, + [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/a", text="a2"), + Link(url="https://example.com/b?b=2&a=1", text="b2"), + ], + ) + + # unique=False, canonicalize=True + lx = self.extractor_cls( + restrict_css=("div",), unique=False, canonicalize=True + ) + response = HtmlResponse( + "https://example.com", + body=b""" +
+ a1 + b1 +
+
+ a2 + b2 +
+ """, + ) + actual = lx.extract_links(response) + self.assertEqual( + actual, + [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/a", text="a2"), + Link(url="https://example.com/b?a=1&b=2", text="b2"), + ], + ) + class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = LxmlLinkExtractor From c4e4b9b56e7fe10c5e7472b152dd47253a97af5b Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 20 Feb 2024 14:50:16 +0500 Subject: [PATCH 1340/2083] Add a SECURITY.md file (#6051) --- .bumpversion.cfg | 4 ++++ SECURITY.md | 12 ++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 SECURITY.md diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6ce6e2a59..968a34d96 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -5,3 +5,7 @@ tag = True tag_name = {new_version} [bumpversion:file:scrapy/VERSION] + +[bumpversion:file:SECURITY.md] +parse = (?P\d+)\.(?P\d+)\.x +serialize = {major}.{minor}.x diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..51305d95e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,12 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 2.11.x | :white_check_mark: | +| < 2.11.x | :x: | + +## Reporting a Vulnerability + +Please report the vulnerability using https://github.com/scrapy/scrapy/security/advisories/new. From ee1189512f652fae72f013c9d4759976b8b69994 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Tue, 20 Feb 2024 08:47:29 -0300 Subject: [PATCH 1341/2083] Replace urlparse with urlparse_cached where possible (#6229) --- docs/topics/media-pipeline.rst | 8 ++++---- scrapy/core/http2/stream.py | 6 +++--- scrapy/downloadermiddlewares/redirect.py | 4 ++-- tests/CrawlerRunner/ip_address.py | 3 ++- tests/test_http_cookies.py | 10 +++++----- tests/test_http_request.py | 11 ++++++----- tests/test_scheduler_base.py | 5 +++-- 7 files changed, 25 insertions(+), 22 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index da0587aa4..c96dd0f99 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -532,14 +532,14 @@ See here the methods that you can override in your custom Files Pipeline: .. code-block:: python from pathlib import PurePosixPath - from urllib.parse import urlparse + from scrapy.utils.httpobj import urlparse_cached from scrapy.pipelines.files import FilesPipeline class MyFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return "files/" + PurePosixPath(urlparse(request.url).path).name + return "files/" + PurePosixPath(urlparse_cached(request).path).name Similarly, you can use the ``item`` to determine the file path based on some item property. @@ -690,14 +690,14 @@ See here the methods that you can override in your custom Images Pipeline: .. code-block:: python from pathlib import PurePosixPath - from urllib.parse import urlparse + from scrapy.utils.httpobj import urlparse_cached from scrapy.pipelines.images import ImagesPipeline class MyImagesPipeline(ImagesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return "files/" + PurePosixPath(urlparse(request.url).path).name + return "files/" + PurePosixPath(urlparse_cached(request).path).name Similarly, you can use the ``item`` to determine the file path based on some item property. diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 39d5921f4..0f282d83d 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -2,7 +2,6 @@ import logging from enum import Enum from io import BytesIO from typing import TYPE_CHECKING, Dict, List, Optional, Tuple -from urllib.parse import urlparse from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -15,6 +14,7 @@ from twisted.web.client import ResponseFailed from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes +from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol @@ -185,7 +185,7 @@ class Stream: def check_request_url(self) -> bool: # Make sure that we are sending the request to the correct URL - url = urlparse(self._request.url) + url = urlparse_cached(self._request) return ( url.netloc == str(self._protocol.metadata["uri"].host, "utf-8") or url.netloc == str(self._protocol.metadata["uri"].netloc, "utf-8") @@ -194,7 +194,7 @@ class Stream: ) def _get_request_headers(self) -> List[Tuple[str, str]]: - url = urlparse(self._request.url) + url = urlparse_cached(self._request) path = url.path if url.query: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 83afdf7d7..24089afea 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, List, Union, cast -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin from w3lib.url import safe_url_string @@ -125,7 +125,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): assert response.headers["Location"] is not None location = safe_url_string(response.headers["Location"]) if response.headers["Location"].startswith(b"//"): - request_scheme = urlparse(request.url).scheme + request_scheme = urlparse_cached(request).scheme location = request_scheme + "://" + location.lstrip("/") redirected_url = urljoin(request.url, location) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 23260ab0d..5bf7512bc 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -9,6 +9,7 @@ from twisted.python.runtime import platform from scrapy import Request, Spider from scrapy.crawler import CrawlerRunner +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import configure_logging from tests.mockserver import MockDNSServer, MockServer @@ -30,7 +31,7 @@ class LocalhostSpider(Spider): yield Request(self.url) def parse(self, response): - netloc = urlparse(response.url).netloc + netloc = urlparse_cached(response).netloc host = netloc.split(":")[0] self.logger.info(f"Host: {host}") self.logger.info(f"Type: {type(response.ip_address)}") diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 9e43b72b0..8b5554914 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,8 +1,8 @@ from unittest import TestCase -from urllib.parse import urlparse from scrapy.http import Request, Response from scrapy.http.cookies import WrappedRequest, WrappedResponse +from scrapy.utils.httpobj import urlparse_cached class WrappedRequestTest(TestCase): @@ -17,12 +17,12 @@ class WrappedRequestTest(TestCase): self.assertEqual(self.wrapped.full_url, self.request.url) def test_get_host(self): - self.assertEqual(self.wrapped.get_host(), urlparse(self.request.url).netloc) - self.assertEqual(self.wrapped.host, urlparse(self.request.url).netloc) + self.assertEqual(self.wrapped.get_host(), urlparse_cached(self.request).netloc) + self.assertEqual(self.wrapped.host, urlparse_cached(self.request).netloc) def test_get_type(self): - self.assertEqual(self.wrapped.get_type(), urlparse(self.request.url).scheme) - self.assertEqual(self.wrapped.type, urlparse(self.request.url).scheme) + self.assertEqual(self.wrapped.get_type(), urlparse_cached(self.request).scheme) + self.assertEqual(self.wrapped.type, urlparse_cached(self.request).scheme) def test_is_unverifiable(self): self.assertFalse(self.wrapped.is_unverifiable()) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6dc9ec8b7..04fcaa231 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -5,7 +5,7 @@ import warnings import xmlrpc.client from typing import Any, Dict, List from unittest import mock -from urllib.parse import parse_qs, unquote_to_bytes, urlparse +from urllib.parse import parse_qs, unquote_to_bytes from scrapy.http import ( FormRequest, @@ -16,6 +16,7 @@ from scrapy.http import ( XmlRpcRequest, ) from scrapy.http.request import NO_CALLBACK +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode @@ -617,8 +618,8 @@ class FormRequestTest(RequestTest): method="GET", formdata=(("foo", "bar"), ("foo", "baz")), ) - self.assertEqual(urlparse(req.url).hostname, "www.example.com") - self.assertEqual(urlparse(req.url).query, "foo=bar&foo=baz") + self.assertEqual(urlparse_cached(req).hostname, "www.example.com") + self.assertEqual(urlparse_cached(req).query, "foo=bar&foo=baz") def test_from_response_override_duplicate_form_key(self): response = _buildresponse( @@ -666,8 +667,8 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) self.assertEqual(r1.method, "GET") - self.assertEqual(urlparse(r1.url).hostname, "www.example.com") - self.assertEqual(urlparse(r1.url).path, "/this/get.php") + self.assertEqual(urlparse_cached(r1).hostname, "www.example.com") + self.assertEqual(urlparse_cached(r1).path, "/this/get.php") fs = _qs(r1) self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 76ca777a8..5db2e4e50 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,6 +1,6 @@ from typing import Dict, Optional from unittest import TestCase -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin from testfixtures import LogCapture from twisted.internet import defer @@ -9,6 +9,7 @@ from twisted.trial.unittest import TestCase as TwistedTestCase from scrapy.core.scheduler import BaseScheduler from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.request import fingerprint from scrapy.utils.test import get_crawler from tests.mockserver import MockServer @@ -57,7 +58,7 @@ class TestSpider(Spider): self.start_urls = map(mockserver.url, PATHS) def parse(self, response): - return {"path": urlparse(response.url).path} + return {"path": urlparse_cached(response).path} class InterfaceCheckMixin: From f096f17fa4ac1307fa1c81ae082bb52e9f86653a Mon Sep 17 00:00:00 2001 From: Elias Ram Date: Tue, 20 Feb 2024 20:32:02 +0100 Subject: [PATCH 1342/2083] test #6 added tests for check command --- tests/test_command_check.py | 55 +++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 592494aba..d503628b8 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -1,3 +1,8 @@ +import sys +from io import StringIO +from unittest.mock import Mock, PropertyMock, patch + +from scrapy.commands.check import Command from tests.test_commands import CommandTest @@ -94,3 +99,53 @@ class CheckSpider(scrapy.Spider): raise Exception('SCRAPY_CHECK not set') """ self._test_contract(parse_def=parse_def) + + @patch("scrapy.commands.check.ContractsManager") + def test_run_with_opts_list_prints_spider(self, cm_cls_mock): + output = StringIO() + sys.stdout = output + cmd = Command() + cmd.settings = Mock(getwithbase=Mock(return_value={})) + cm_cls_mock.return_value = cm_mock = Mock() + spider_loader_mock = Mock() + cmd.crawler_process = Mock(spider_loader=spider_loader_mock) + spider_name = "FakeSpider" + spider_cls_mock = Mock() + type(spider_cls_mock).name = PropertyMock(return_value=spider_name) + spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[ + x + ] + tested_methods = ["fakeMethod1", "fakeMethod2"] + cm_mock.tested_methods_from_spidercls.side_effect = lambda x: { + spider_cls_mock: tested_methods + }[x] + + cmd.run([spider_name], Mock(list=True)) + + self.assertEqual( + "FakeSpider\n * fakeMethod1\n * fakeMethod2\n", output.getvalue() + ) + sys.stdout = sys.__stdout__ + + @patch("scrapy.commands.check.ContractsManager") + def test_run_without_opts_list_does_not_crawl_spider_with_no_tested_methods( + self, cm_cls_mock + ): + cmd = Command() + cmd.settings = Mock(getwithbase=Mock(return_value={})) + cm_cls_mock.return_value = cm_mock = Mock() + spider_loader_mock = Mock() + cmd.crawler_process = Mock(spider_loader=spider_loader_mock) + spider_name = "FakeSpider" + spider_cls_mock = Mock() + spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[ + x + ] + tested_methods = [] + cm_mock.tested_methods_from_spidercls.side_effect = lambda x: { + spider_cls_mock: tested_methods + }[x] + + cmd.run([spider_name], Mock(list=False)) + + cmd.crawler_process.crawl.assert_not_called() From e8e6d28479a0479361cd3de0fb854c243ed684b1 Mon Sep 17 00:00:00 2001 From: Elias Ram Date: Tue, 20 Feb 2024 20:41:18 +0100 Subject: [PATCH 1343/2083] test #8 added tests for LxmlLinkExtractor --- tests/test_linkextractors.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 18e9608c1..66a30c635 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -2,6 +2,7 @@ import pickle import re import unittest from typing import Optional +from unittest.mock import Mock from packaging.version import Version from pytest import mark @@ -851,3 +852,36 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ), ], ) + + def test_link_allowed_is_false_with_empty_url(self): + mock_link = Mock() + mock_link.url = "" + expected = False + + actual = LxmlLinkExtractor()._link_allowed(mock_link) + + self.assertEqual(expected, actual) + + def test_link_allowed_is_false_with_bad_url_prefix(self): + mock_link = Mock() + mock_link.url = "htp://should_be_http.com" + expected = False + + actual = LxmlLinkExtractor()._link_allowed(mock_link) + + self.assertEqual(expected, actual) + + def test_link_allowed_is_false_with_missing_url_prefix(self): + mock_link = Mock() + mock_link.url = "should_have_prefix.com" + expected = False + + actual = LxmlLinkExtractor()._link_allowed(mock_link) + + self.assertEqual(expected, actual) + + def test_link_allowed_raises_with_none_url(self): + mock_link = Mock() + mock_link.url = None + + self.assertRaises(AttributeError, LxmlLinkExtractor()._link_allowed, mock_link) From e27d320c3cde3c965e61a674e8880043943cf17a Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Tue, 20 Feb 2024 23:58:39 +0100 Subject: [PATCH 1344/2083] test #3 Increased branch coverage for form.py --- tests/test_http_request.py | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6dc9ec8b7..510ae74ba 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1642,6 +1642,62 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["ensure_ascii"], True) self.assertEqual(kwargs["allow_nan"], True) + def test_form_response_with_invalid_formdata_type_error(self): + """Test that a form response with invalid form data throws a type error""" + response = _buildresponse( + """ + + + + """ + ) + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=123) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_form_response_with_custom_invalid_formdata_value_error(self): + """Test that a form response with invalid form data throws a value error""" + response = _buildresponse( + """ +
+ +
+ """ + ) + + class CustomFormdata: + def __iter__(self): + raise ValueError("Custom iteration error for testing") + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=CustomFormdata()) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_get_form_with_xpath_no_form_parent(self): + """Test that _get_from raised a ValueError when an XPath selects an element + not nested within a
and no parent is found""" + response = _buildresponse( + """ +
+

This paragraph is not inside a form.

+
+ + +
+ """ + ) + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') + + self.assertIn("No
element found with", str(context.exception)) + def tearDown(self): warnings.resetwarnings() super().tearDown() From e2a0c85f1167c7c32219eaf095c1818b2c74702c Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:21:24 +0100 Subject: [PATCH 1345/2083] doc #3 Clarified test comments --- tests/test_http_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 510ae74ba..95e4a7be0 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1643,7 +1643,7 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["allow_nan"], True) def test_form_response_with_invalid_formdata_type_error(self): - """Test that a form response with invalid form data throws a type error""" + """Test that a ValueError is raised for non-iterable and non-dict formdata input""" response = _buildresponse( """ @@ -1659,7 +1659,7 @@ class JsonRequestTest(RequestTest): ) def test_form_response_with_custom_invalid_formdata_value_error(self): - """Test that a form response with invalid form data throws a value error""" + """Test that a ValueError is raised for fault-inducing iterable formdata input""" response = _buildresponse( """ From 12b4417c56d8aa76cbe3a36c026962612453ee6e Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:30:50 +0100 Subject: [PATCH 1346/2083] test #22 Improve json_request.py coverage --- tests/test_http_request.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6dc9ec8b7..d1c435468 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1642,6 +1642,26 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["ensure_ascii"], True) self.assertEqual(kwargs["allow_nan"], True) + def test_replacement_both_body_and_data_warns(self): + """Test that we can get a warning if both body and data are passed for branch coverage""" + body1 = None + body2 = b"body" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) + + with mock.patch("warnings.warn") as mock_warn: + r1.replace(data=data2, body=body2) + mock_warn.assert_called_once() + (warning_message,), _ = mock_warn.call_args + self.assertIn( + "Both body and data passed. data will be ignored", warning_message + ) + def tearDown(self): warnings.resetwarnings() super().tearDown() From bc036542a82ec054dd6a36b4b928a9bc6ae48e63 Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:35:45 +0100 Subject: [PATCH 1347/2083] refactor #3 Moved tests to FormRequestTest --- tests/test_http_request.py | 112 ++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 95e4a7be0..39afc5fd1 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1426,6 +1426,62 @@ class FormRequestTest(RequestTest): r = self.request_class.from_response(response) self.assertEqual(r.method, expected) + def test_form_response_with_invalid_formdata_type_error(self): + """Test that a ValueError is raised for non-iterable and non-dict formdata input""" + response = _buildresponse( + """ + + +
+ """ + ) + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=123) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_form_response_with_custom_invalid_formdata_value_error(self): + """Test that a ValueError is raised for fault-inducing iterable formdata input""" + response = _buildresponse( + """ +
+ +
+ """ + ) + + class CustomFormdata: + def __iter__(self): + raise ValueError("Custom iteration error for testing") + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=CustomFormdata()) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_get_form_with_xpath_no_form_parent(self): + """Test that _get_from raised a ValueError when an XPath selects an element + not nested within a
and no parent is found""" + response = _buildresponse( + """ +
+

This paragraph is not inside a form.

+
+ + +
+ """ + ) + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') + + self.assertIn("No
element found with", str(context.exception)) + def _buildresponse(body, **kwargs): kwargs.setdefault("body", body) @@ -1642,62 +1698,6 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["ensure_ascii"], True) self.assertEqual(kwargs["allow_nan"], True) - def test_form_response_with_invalid_formdata_type_error(self): - """Test that a ValueError is raised for non-iterable and non-dict formdata input""" - response = _buildresponse( - """ - - -
- """ - ) - with self.assertRaises(ValueError) as context: - FormRequest.from_response(response, formdata=123) - - self.assertIn( - "formdata should be a dict or iterable of tuples", str(context.exception) - ) - - def test_form_response_with_custom_invalid_formdata_value_error(self): - """Test that a ValueError is raised for fault-inducing iterable formdata input""" - response = _buildresponse( - """ -
- -
- """ - ) - - class CustomFormdata: - def __iter__(self): - raise ValueError("Custom iteration error for testing") - - with self.assertRaises(ValueError) as context: - FormRequest.from_response(response, formdata=CustomFormdata()) - - self.assertIn( - "formdata should be a dict or iterable of tuples", str(context.exception) - ) - - def test_get_form_with_xpath_no_form_parent(self): - """Test that _get_from raised a ValueError when an XPath selects an element - not nested within a
and no parent is found""" - response = _buildresponse( - """ -
-

This paragraph is not inside a form.

-
- - -
- """ - ) - - with self.assertRaises(ValueError) as context: - FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') - - self.assertIn("No
element found with", str(context.exception)) - def tearDown(self): warnings.resetwarnings() super().tearDown() From 6fc78270427c41e401a01a46551d27dd4ddf846c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 21 Feb 2024 14:27:42 +0100 Subject: [PATCH 1348/2083] Do not close the underlying file from compression plugins (#6239) --- docs/topics/feed-exports.rst | 8 +++++++- scrapy/extensions/postprocessing.py | 3 --- tests/test_feedexport.py | 2 ++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index f64bbac06..922b765db 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -390,7 +390,13 @@ Each plugin is a class that must implement the following methods: .. method:: close(self) - Close the target file object. + Clean up the plugin. + + For example, you might want to close a file wrapper that you might have + used to compress data written into the file received in the ``__init__`` + method. + + .. warning:: Do not close the file from the ``__init__`` method. To pass a parameter to your plugin, use :ref:`feed options `. You can then access those parameters from the ``__init__`` method of your plugin. diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 79e3b1656..17969c5b0 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -42,7 +42,6 @@ class GzipPlugin: def close(self) -> None: self.gzipfile.close() - self.file.close() class Bz2Plugin: @@ -69,7 +68,6 @@ class Bz2Plugin: def close(self) -> None: self.bz2file.close() - self.file.close() class LZMAPlugin: @@ -111,7 +109,6 @@ class LZMAPlugin: def close(self) -> None: self.lzmafile.close() - self.file.close() # io.IOBase is subclassed here, so that exporters can use the PostProcessingManager diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 277555608..d7560b5ff 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1731,6 +1731,7 @@ class FeedExportTest(FeedExportTestBase): def store(self, file): Storage.store_file = file + Storage.file_was_closed = file.closed file.close() settings = { @@ -1746,6 +1747,7 @@ class FeedExportTest(FeedExportTestBase): } yield self.exported_no_data(settings) self.assertIs(Storage.open_file, Storage.store_file) + self.assertFalse(Storage.file_was_closed) class FeedPostProcessedExportsTest(FeedExportTestBase): From f19045403a44011a62162d73fedacf5038af098f Mon Sep 17 00:00:00 2001 From: Elias Ram Date: Wed, 21 Feb 2024 16:03:51 +0100 Subject: [PATCH 1349/2083] test #8 made tests cleaner --- tests/test_linkextractors.py | 34 ++++++---------------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 66a30c635..55ea9eed2 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -2,7 +2,6 @@ import pickle import re import unittest from typing import Optional -from unittest.mock import Mock from packaging.version import Version from pytest import mark @@ -854,34 +853,13 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ) def test_link_allowed_is_false_with_empty_url(self): - mock_link = Mock() - mock_link.url = "" - expected = False - - actual = LxmlLinkExtractor()._link_allowed(mock_link) - - self.assertEqual(expected, actual) + bad_link = Link("") + self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) def test_link_allowed_is_false_with_bad_url_prefix(self): - mock_link = Mock() - mock_link.url = "htp://should_be_http.com" - expected = False - - actual = LxmlLinkExtractor()._link_allowed(mock_link) - - self.assertEqual(expected, actual) + bad_link = Link("htp://should_be_http.example") + self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) def test_link_allowed_is_false_with_missing_url_prefix(self): - mock_link = Mock() - mock_link.url = "should_have_prefix.com" - expected = False - - actual = LxmlLinkExtractor()._link_allowed(mock_link) - - self.assertEqual(expected, actual) - - def test_link_allowed_raises_with_none_url(self): - mock_link = Mock() - mock_link.url = None - - self.assertRaises(AttributeError, LxmlLinkExtractor()._link_allowed, mock_link) + bad_link = Link("should_have_prefix.example") + self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) From b7a7ae7dbbaddd9d14b50c1257370af51e1ac1b5 Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:04:45 +0100 Subject: [PATCH 1350/2083] refactor #22 Change comment and warning catching --- tests/test_http_request.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index d1c435468..a45293695 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1643,7 +1643,7 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["allow_nan"], True) def test_replacement_both_body_and_data_warns(self): - """Test that we can get a warning if both body and data are passed for branch coverage""" + """Test that we get a warning if both body and data are passed""" body1 = None body2 = b"body" data1 = { @@ -1654,12 +1654,11 @@ class JsonRequestTest(RequestTest): } r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) - with mock.patch("warnings.warn") as mock_warn: + with warnings.catch_warnings(record=True) as _warnings: r1.replace(data=data2, body=body2) - mock_warn.assert_called_once() - (warning_message,), _ = mock_warn.call_args self.assertIn( - "Both body and data passed. data will be ignored", warning_message + "Both body and data passed. data will be ignored", + str(_warnings[0].message), ) def tearDown(self): From 877398a3dee8e92300ef52177b986be16a55f277 Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:13:57 +0100 Subject: [PATCH 1351/2083] refactor #3 Remove inner class in form test --- tests/test_http_request.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 39afc5fd1..71d442a81 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1452,12 +1452,8 @@ class FormRequestTest(RequestTest): """ ) - class CustomFormdata: - def __iter__(self): - raise ValueError("Custom iteration error for testing") - with self.assertRaises(ValueError) as context: - FormRequest.from_response(response, formdata=CustomFormdata()) + FormRequest.from_response(response, formdata=("a",)) self.assertIn( "formdata should be a dict or iterable of tuples", str(context.exception) From c513e7d6e58efc99c5dfc8ad135dc23df528c5fd Mon Sep 17 00:00:00 2001 From: Can Kupeli Date: Wed, 21 Feb 2024 23:52:35 +0100 Subject: [PATCH 1352/2083] implemented tests for branch coverage for function printSummary --- tests/test_command_check.py | 52 +++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 592494aba..70098a4e7 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -1,3 +1,6 @@ +from unittest.mock import Mock, call, patch + +from scrapy.commands.check import TextTestResult from tests.test_commands import CommandTest @@ -94,3 +97,52 @@ class CheckSpider(scrapy.Spider): raise Exception('SCRAPY_CHECK not set') """ self._test_contract(parse_def=parse_def) + + def test_printSummary_with_unsuccessful_test_result_without_errors_and_without_failures( + self, + ): + result = TextTestResult(Mock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [] + result.errors = [] + result.unexpectedSuccesses = ["a", "b"] + with patch.object(result.stream, "write") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_has_calls([call("FAILED"), call("\n")]) + + def test_printSummary_with_unsuccessful_test_result_with_only_failures(self): + result = TextTestResult(Mock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [(self, "failure")] + result.errors = [] + with patch.object(result.stream, "writeln") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_called_with(" (failures=1)") + + def test_printSummary_with_unsuccessful_test_result_with_only_errors(self): + result = TextTestResult(Mock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [] + result.errors = [(self, "error")] + with patch.object(result.stream, "writeln") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_called_with(" (errors=1)") + + def test_printSummary_with_unsuccessful_test_result_with_both_failures_and_errors( + self, + ): + result = TextTestResult(Mock(), descriptions=False, verbosity=1) + start_time = 1.0 + stop_time = 2.0 + result.testsRun = 5 + result.failures = [(self, "failure")] + result.errors = [(self, "error")] + with patch.object(result.stream, "writeln") as mock_write: + result.printSummary(start_time, stop_time) + mock_write.assert_called_with(" (failures=1, errors=1)") From e208f82076182e71a4eb8470eb527363158f4b0c Mon Sep 17 00:00:00 2001 From: vishesh10 Date: Thu, 22 Feb 2024 16:46:24 +0530 Subject: [PATCH 1353/2083] Add support for multiple-compressed responses (#6063) --- .../downloadermiddlewares/httpcompression.py | 34 ++++++++-- .../compressed/html-gzip-deflate-gzip.bin | Bin 0 -> 8014 bytes .../compressed/html-gzip-deflate.bin | Bin 0 -> 7991 bytes ...st_downloadermiddleware_httpcompression.py | 58 ++++++++++++++++++ 4 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 tests/sample_data/compressed/html-gzip-deflate-gzip.bin create mode 100644 tests/sample_data/compressed/html-gzip-deflate.bin diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 709333948..1e340abb6 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,7 @@ from __future__ import annotations import warnings +from itertools import chain from logging import getLogger from typing import TYPE_CHECKING, List, Optional, Union @@ -102,18 +103,18 @@ class HttpCompressionMiddleware: if isinstance(response, Response): content_encoding = response.headers.getlist("Content-Encoding") if content_encoding: - encoding = content_encoding.pop() max_size = request.meta.get("download_maxsize", self._max_size) warn_size = request.meta.get("download_warnsize", self._warn_size) try: - decoded_body = self._decode( - response.body, encoding.lower(), max_size + decoded_body, content_encoding = self._handle_encoding( + response.body, content_encoding, max_size ) except _DecompressionMaxSizeExceeded: raise IgnoreRequest( f"Ignored response {response} because its body " - f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE " - f"({max_size} B) during decompression." + f"({len(response.body)} B compressed) exceeded " + f"DOWNLOAD_MAXSIZE ({max_size} B) during " + f"decompression." ) if len(response.body) < warn_size <= len(decoded_body): logger.warning( @@ -121,6 +122,7 @@ class HttpCompressionMiddleware: f"({len(decoded_body)} B) is larger than the " f"download warning size ({warn_size} B)." ) + response.headers["Content-Encoding"] = content_encoding if self.stats: self.stats.inc_value( "httpcompression/response_bytes", @@ -144,6 +146,28 @@ class HttpCompressionMiddleware: return response + def _handle_encoding(self, body, content_encoding, max_size): + to_decode, to_keep = self._split_encodings(content_encoding) + for encoding in to_decode: + body = self._decode(body, encoding, max_size) + return body, to_keep + + def _split_encodings(self, content_encoding): + to_keep = [ + encoding.strip().lower() + for encoding in chain.from_iterable( + encodings.split(b",") for encodings in content_encoding + ) + ] + to_decode = [] + while to_keep: + encoding = to_keep.pop() + if encoding not in ACCEPTED_ENCODINGS: + to_keep.append(encoding) + return to_decode, to_keep + to_decode.append(encoding) + return to_decode, to_keep + def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes: if encoding == b"gzip" or encoding == b"x-gzip": return gunzip(body, max_size=max_size) diff --git a/tests/sample_data/compressed/html-gzip-deflate-gzip.bin b/tests/sample_data/compressed/html-gzip-deflate-gzip.bin new file mode 100644 index 0000000000000000000000000000000000000000..d66f4c5a03df80085208d8990df59e1702c63b50 GIT binary patch literal 8014 zcmV-UAF<#ciwFP!000000|7T5$l!RKHC1(7lmGWskY!}j3`dRLA?64iIN z(Bm6+BEw8}=7FI~PdU-NoI$sYi==Cu_J|W)>riui_5yjZAmBi- za14I9JL=!~z9y>ZG1)S5g}nXRqrXC3vsTsJxlC(lv|L*QFL5BGB|t7A5yV z8)A3oNJKx+@#>nN_hRjk)tc|^i3jX>&B*BY@$Ev^66`9i!wQBVG74zW%9*8cbkc6?RKdAV?G-_trm>*sW}ar*a&5LpX`354B2 zJVft*L#}$lU@P1H7kM35$fXWsR*Qqj@tWcB+nQ6xzO@V ze?a^36_WPKCXE`Ns`Negn^k~p+;;T*8$ie;1bp6St z(w-Ipec=t_y1GQ*KveU{lv3?#F=k0kbRY>0A4F~1xD=?@M3uRBHJka-ks|i%Zhnt8W>_q2-r0 z^-u!V5+n|%pa!682{6Q>_06P`h@o0wBYJVSaF=L&RRKcrX_*L8RPz(U)gdu0GGdDR zmc_)V1l1k>bRUH7XWSo5PdMZvi$h1BEM-6_=In`%3tnn}lq#Bg^$gCISUqXVBb1=* zd=tDfApw0l+JMM+?{BO)BF&g7SF7Y^v$HLN3tN#W8E16ETYD2oYs&_?H2}6vR3UK( z_{)kAF7wb}OL;&Fv4dgPP)f^a3N7o&phv&nA!Y*U!Q7&zuaeqQZM719B2RW`lsx3o zn}CT^%CI5x5-raR$+spIROp~2V919=AR9CgHJ-+tBW9%yxo|vdE`0z*Kq8?Yy!_nCd0S zh&z!TM(H1d`T$FT@GA2%irjKTzDys+x{4~d>X-f>r5ug~b{GdfW>#P(#egxpc%qf6 z>DP)e>+TYNEZc`qf&?IRFykM(p%xoY;UW7o?z&cs=h!}l(6Jr)`SWdK%$PEnW3qZN(FbDwU!XPf{z^VG<~FX z&xfzFGk;zP0l}tQw4^=M2HBeW_}kj`4nrkCy!EuEN`yGtlr9ASNb>n1Y8?{hyUM@n!6o3f z6SG?oNxgcFz!&||FlnvZyt!%0ht!C`uw6er024X2R##+YlH+!Jlk?0>v5gEL1%8^X z#)~KRTHSOzRcp-VW^=~w(7#UAm9^kQx;Vr2PGE?Fi@jVzteIk@LTaV7eJLKzoOm^= zPO$GM*vFVAZU!MA4#ARF2CfGRScR@PcD$ zx534=Hc-ue8xs{+-&UcPpC)Xd7P2{PfTKW?JyEN9_`8l>uA(3n5>A+qtPXu`_XUVQ zSssHW4u@`d^A|e*kKaHzuJTpiGM-8+Ly1c<2iC3JFUo< zuG0h_{fJ>0^G(Byut|Y%JNI)2#|sQRp@sAHH!(o6B>-kztNl)s#BD}o67^i9h602L z{~(>?#Z}a6wPtl!Tjv=t0QYO=sSl_g)aK1@h0XK2QUM>Bq8mX8455VR*bA`qLcE{P zM=BU97HPKOP!*(gOI%nFIMs&%*7TC1QXyPlr!Br|VtiyOO^K>`nL%uB-=Xf-N=YvA zinAzm&>c!yr$(75NdAB;6JUoc<4hR$PCiEiLK4#-XXaeqq`;M74STk+$|gl@#TbsT z#1dxlq1DSMz+?M2UO`98lP=Pz75o=3rSO~9friQqcBbNCTAjSQS6nH$OTla{J~mI8 zOpmodeX0B7fihrZv|maZ?9PR=U{+SH8C)Sl+Y#C*^iBvRu}u;!hnbWd*Hx)Tu`HOM zpUC<>_^F2Vzj7;UJsc)}GSxslTdO>3RL8-k77`?Shp`IDGp?Wdh{L469oaO;TUu@%0*;vkQn?|LgUZR| zJe2~e7&)3r#>c=AgZ8%M%8nW>k}Mg<)`AFoN#oKU=p_!*+2@p3Xb%dplB7c6#m^F^ zJVQQCvP+cc#vyt!(Qj zKjP3Qeu8C3XBwpx(-o(>uH$xE#F%SBX{0P~fp#%uib4XImbHj=Ouiy|w#+_)3nwU9 z?5uu@tThe(elbD&;ORsRtBxK!DNJFn5ZE(U86;msFc&>QK`mydl9MQdHFgNK>8sXa zv@MeNjEsL}qsm5UR>g7L?ljS^nA3SIHTN^E_RnREelLy@BWanbG zquU4T#~@qxdWf@1^+a{AB@u`3=0~=CRJK)7#IEy#nH@7?UUhTzK$}MJpb^ABP%5Ij zeyxZ=Q?VVUj2q`eTk7^z$5IzT(`v2hR{h8uiWx+0IY-wX5YPI@DHNT$RJ+Z-4#@T@#!%X`l_lvrgE4Q>mq6fs`- zQ7EgmGApdZRm8D;n2*b)*`R7UB^_0^4*|O6d*eZcO^iioi|TN;?fXLNS(GU>@ALV< zOzmx4P+Q!J`I<6umbbnlFN+XJ<38VdIe!3dHSCX)tCWekKgVA0xc0cP8n<2zCq7r$_&$>#yt9zxNF&WewuA9^hBUskUknYcv^aAdSlL~f@qSeOj>tl zVEbAjyk$2R1oR-)T=4d+VlRSjUk3G+hNh@*o6aZ)?S0D5mer{6QycM9yAffGQ z_VfUD341ZN6LUGQYHkxKjMO(-%?t3jeq9#h`Z1?@3ws{3yVFxHzwrI1@#QmG5?T?+ zWe&>tDBWs5H5zyh=5c6GdF9-uF_%_A9hj$#cHxkcnxlmfpP$c**KPEpW0H~ineFyY zip9_NO-^rd@+Oq-OJxNMwHMm<$;d^yBBwL|p^?v{WPtrA`x8ZGOECmwOIY;NFG+by z*}{S=lWfcneHg3#RF^R3UctdXf+7_Hil z6d-11AO0ks`-HCq9g(iP!1M31F>XJUY8eXcUucF_sfX>x2&+L~Kz@>3Tj#Dd5|E8C z!<QWOGi~F~SlKb3yxK$ymy9d}zWl`l#B5^3K`vs+c}trIdW$lDBndDt@2D zAAMtE7xhRUTw*`pAYj%+w8T`suGak0TPL=(UI-C=oz|u!rVBXp`0dGpB-Si#RhMN! zN#F3di4856Utsk=PRy-c8(&Yetg~DcF>F{alA!UE-UGbm35gyo?ME3=LZtoedinK} z6hGfNW|0!~!P=Y(Hfb9Rrsa?tjUZx|fK6pV5+eD0P$K-ABaiq#O107{&BfC~vH7m{M zEC2n~Nqh5o@x<3SMxK1^wLTzxrAByM(n>h?42QnZb`}w21xROuo`^hOdWvF$Qh{9Q zuUVQKHFUQ|cPmodPVCV>dHRw@%=A7S({K@6t9j}1jnLDs(f|zo z3`P&G#bJ_kPC8u~c~e~-VMN95EkcP*^IyNYgJlT>P(9T7xV23Qk31N3wYT{-^xSRE z-rC&nn)djd=@^j1ma_7W{g+F!!pfFtyYJ7&&acShdj6RrC(azS1Aca`2Ha;IU!Nfk zvc|%Ey0hC$z1r_)=lD+cE|Ibe(Y4t|zsOZJv|kr^n26bl`3s&!o`n6q4vW3NJb4j! z%;VdBEn!t8X6XBI#fIGOHL-%w{H-s@{yke0RhYnZ)B5&M@b;AWAVAzR6@Po)Oth!Jcvi1Eobj`;5XZY}OZ_ zlnNN&{hQE{J|j%T9HpVJVMTq+=L_|;jY@jbEU0Dae0XnZGhk)CR^a`&lb3}-6`qfD zZ4E#J{ai|AvuO5gy?GADo|;yexy_EptN~b8j5h}^V0N15{~ZeKMWs3+hCyF(2JfW~ zGrMcxT|fH6=X5l_rE)(3FaHslsKQ+Q)E4SQHVNePoJfs)VDdO!qXRU4jK31&1mJGq z3c`x?vVs=|8V;E~P95$V4U$%ReJ7K79qpcwB$o71pyfVeo>e;9TH}b>ptL4rm!DLG z;+pZ)_acn`lI&_=*Qy?F^+*B#lJSWhhoD$$4B%6e5IVhmNh+JbfN5}+>_?r1*G@F>u%?zVnIk-zqoT}p!1WZdl~ft~!*?-K1S@Pm>jaowvWTzM3M z3BO3Q?cBgXz-ghNjex$)Dyy}b^}|<#e!ZPG_y=}x|A_)rabLSx-L3Fveo_CZ*(l&Y zD-PmNR`wb_gMTng=Jr(dN|pQ^@_U)=5Ad>nB1uM>`*TdVu_VwMedMxb*nyPzg`whV zkp!u@25{G8YUy=6CQ1wNN)g&6{F=>QtTR>ZABWAWo}llHD)P_;%+YYq&gY@8@cO_s zgsbW7fYdvsb3{I${S2^W_|oxnh~E+hdmPVOOZg(**+rdNR_}~QcPu)+`Bfzp9$76y z6T+np{EKgEE;^}z?xk*7^H=Rw_Ys&}O=pgIx*((K=_d(n<;Q-?yDPf$immw+a9f!H z&u^NUNJVA#re0KVsW!U&hkgFmIcdB0{_ z#Bl%6wRy!kdZG*EcO>Zk^tvU+>`RLvYZ(XNwi&{ID`}XlJ5jhWcb(`*Lva1(KUTBV zhfaSma5(B6EQ06UYGaH6pkq)+gpv0f;?D?Z<(D_BQI+_A-n$!-Rx+%bNmE5C^N6sZ z?!#B=k!8R0OqYJba7(V4f z>$L&v0%`mTRKf(w9UN05)wyYYrqA$yrzw6Dy=dAet5K{Gj|r9|XeeqX^HPspIr%#f z-Aw!@o>~Pv&;!?>3M34&iT~68Cnva*Wr1jYc~RHe=R+*6RqTW*oNBxbKRqZb9+QW% z+~M1SJh1f^$Bx-%y(xrahiea@_nRT6?LEfN`chIKuKs7I=8dB3HQ$^*Vq_U4V;+(w z&xG)L+F8d!V#+C|rY7Nm0t0<2|A#65JFnvXdz1cP70V_MJz=!LJ0Xg_V*T+sRYv;R zk6NoD&MlCTj4_HDj!_nEA@WQ$2I`K>-@yWCZO+TFwM`)3rU>GemRv%4iXAX)Dm$R- za*9iw5l!Q?@hHkwSSVMuBT)PUJq%QA%gVpgTKg&{h&TAP(v%`-=@0$|LgRkkMv*{8b~W^|czkMAmIW zmlU}Cp)R`JX^iwkhun`=j|IJMt*!gDfG2X2|n?_@z@CA|)6 z3`06Xrc9BdI_ki=huE7)wOrRUO1{@QOe>+%_%A2EQQWFNd=WDq<&a z+_E-HUS;ZI-9l`!iYI!%9(=|zhQ77j+{T2b5cw5!bbDUf=X&-JWBK8lV86803mq~1R=iDU`^3$m7bIF+9 z1>vegA@&$FPKA@#v43`Eu62Rt3%EZq4I{GSipQk~{<(ySVwZW*v7nY8DCIaGya}hV zb^LoCY$}$>rHhKM+9xQUJjFN)ix3ju;lz;|4_8E@aq8(rEQ`u5(1-rvY8O*hm9TM) z?v19F-4lL0hz@HJwCY-)YAfY8wL@DPP!sFt@}Zl+BIlB3eakrVV!u0lfz`qaC+TFa zo}g~k#v4}w^-$=`d?OmxcKMV^k9F?V$tm<+F`dJ-+9d2#zYW^bZ}=`Ae|+H{TIepN zl-j37Gk^rkAH_*WbxV5sq#3&T_Y;W{Hh&k5?X!wpJ5ztb&+T^4JdK=dGS%b;M3lok z@2Yrle?&a;Kd8JjC2RWrb~TWLyojo%L=OInJrbKQS=DL^zTULK{mimaP~o>9B`!G9 z1rZ|Y`F3|MTRC2aZin8z6E*r5Y@{V=g`>eKzeWSR-;Hm$VN7-k9`}SlnRez0?PW_HBspUFdbM}- z`4+T860?A9h}Ey4oA{2x3_H41NP%T%chNw{(GIq9EQ*Y9xGp-=#g3S%a<~fBOt0%KB-SuxU8W zdyDjj-LJI#r2m8#xQ%lF^Upn6O;t$R)PT|G8FgluVbv@@`NnC&62)cDzqs-1ki!e2 zQLpvFTNk`&ItA-gK|IUd6W$WE`OV2^VtdeoRF$(4HPoZ{lnn~If%mEl(;L+Y<6f`68G1^B8dYC&;MZOJ$>6i> zf)IOGg!+1{gSn9E?Bai$VjF=Y#bq>rVp4u97H}lC1h(a+7xTRT1R8SZT zjjA;JE2E<+!gMD)?ge)hV97F(Hu!bu2fU$dP(%Q$vFOokK4LJ5o~bmuh}Tv7!&50^ z2RR8~I!V7qCEjqF?N_u*)FCy>W4a0ARS>d}F&eizQK56ySvR6MAU@p&Rt5gNuE;ok z+%4f_^-8>64gPq2)FWXPIDL;JVuG2lZBs)hSEL#BEj;puW0iGW@UO~-99A96VP4=A z7vEFbl7a>Z20k~mk2MEdGaQ{L>X)uj`mt@oggAT5M?lu3kE=B-y^pm{{fl6rkLNJ7 zKt^x8KUavhaoJ{Qng2|8e0|)w2p2y6!scm*n_g9g=;I|AEpsSX7>LxdbZi~UQ;pL$t#qTaOdBB3>-Lu`E&L;goo6E^}JZg0%i1 zzRKR8DoM>^WCz;RRey$cp8V!!MKh-uH3Z4v=PJU%cenZLe%~3n_3%XWvv!dd!_z1K Q142u<+9f5~j5i+u01a&HYybcN literal 0 HcmV?d00001 diff --git a/tests/sample_data/compressed/html-gzip-deflate.bin b/tests/sample_data/compressed/html-gzip-deflate.bin new file mode 100644 index 0000000000000000000000000000000000000000..5066842ef7d137786f43682c99c63e24eae261ec GIT binary patch literal 7991 zcmV-7AIRW%oHbQ-T$BIzRgh+cATb04X;45K35n4uFiN_IbaxEt?(QBT-6bI%gMrdW zNC-&B1is_*d;R`>&U5a$aqfN7MqxiOdBgVM33^wKK@!z>x6tDob|S-J=M>3F*C}#i zpg=;RAY+OFu42tH27|;~He|t~fdY4cUpv;8DWh17mkiDKZ8oR6255#$`;^y|kJ#;? zxZmCH0IwUPfE&w;W6mC=J92Aq@D5>dKjnXAi8OEWb_!_iS#~fwGCbtuyjp-AtQ6Qh!9x|h6#k-K|Dn7e?zW%!eA@g{ug;2 zSIDIfWLArV$MKrs@!OhH#^cL0WYl^0UI%=kR9k=B_A)Qm$E~I3!7;!$o1qk8bai$0 z(9m&vV&{Fq&xAfh} zCx_;QKOonFU+l7@A9jL(uMon{A@W9RA`k8t>d`GQ2X~JEpZi1Q=MQO|SK8vM+5Rqb zNB)|77G#M1rOm7EfR2YdZP>Ajwtq{<%8JqQr+_AJuiL*C-u)d>S!0bV?h74vhZC@3 z=mNR;@3eoAl-w)#i(S#AN(b?X)64iR@2lOVp%xs(en(=nItgp9 zteho0ETa9jxz^azT6KFto8yn`)^wyQe)r2mxqRX9r0n;eZthlqzi)lf$gpawo<-4_ zWs@Ru?>0`==(vF)ue8i7)q!`hc=KVs(4C}xBq~%nyIW($VR27?5HdSAd$p+PG+Q<3 z=JX?E5BbG)ru1mgZQ`p<*2uyA&2`VF8!ev;n7CS9nkbTpoDqag%nX{F-B*ajmZ_E} zzAMc%WH8aFop{dxa@|W+WG@82m*hw`(NKl)veJk1wX8NmWJ8o0yIn~1i|&iVSO4r6 zhg*`A_INbYkH?l5sY&xufWq||LC>6Om9^Nn9Za7uH?qn{#^c%s->hatVnbxHDVago zvvYQfHn3dM0Lrh!Y|4ihp`{x%M@qkdR4w;u5)PNxb4msS?e+R8M%i zx|G#;|7Sj>+Kr(nNvl=s#h7&K(^Q)&bFZS6yX0k5*0Z+GlJo|jkF3i4$;^OQ57^5= z(o-G30nwx@Q5gl<(wK&8d+RxS>(ZVU0e#^O;<~y-;6POK z$dpp;YB6R>O>`g$4Ie~p+PD;`*F=@Mcjah1+f^y4DG$KRv4gK@@*42rs>OBK-tJk# zBo0IOuH@KtboRk*yOW*Pu;72iq4OfT!*m6U0IviN6^JWD09zwx?v-QI^BnyiyP^*g zk*m=WcxE50YsMf3%ODSkfHa8!#d4t`9g9oS+^cUH#i8YwHuX>f))FKRr=SL)YY8yK zqV>(Bl8B*NU?X~Qw{Vwed{qHL@oAX|QdIL3!qp)$Eiz(?`mjtgFDf0Qbkd-V*?mRLP$$|IDZ?0gfvG9dwdI@*B9ckgek zI3mrMDOaoHX0x*`f(u)bC>dvT!&`e3NNdXmxitW`O;jOq2l&g15H9o3U`u&G3bBJ> z*HB8!XbLUs$)HES-XUfJ>A~EhrmvFPQf;*oej-nHXp}tU(VKvYQ_8R*^Aau349T}9 z6;$Y;Bw)ygL?9b95H+60oFitX4Y_bUYc72NL_jIPp-ix({Hqu5WWBCKa0J%=WL4po z_59n&8e0u^R4PobS19xvc?i?yj1BpKWtr@ZO6|O|N0{m*$cQ_U9Y*ONg8BeUf$%Ex zGK$=CL%vKO#=43sx9XSvAEg|Q1a=q)K4w;6CdGg;yLh6Ns_EB?G3)LUe=OUFPl5y> zbTM8LgD@UoQj|SD`V1>%D(IVWwuNEWsDO0RdliTp;sd1$<>lv6pwe66%xrI~f1Y6I zld8TzRFtsfVAg~R7eJzpoSC7ilS&15o3)k@yn>G$@HBm-be|jwv^!mBhuF5Ldwqin+(Ejek3mwEyKHNNTa*p-HyOc_60M+ zaCzS!OmznYn@;+HVl+_cx)fJtzwTb{g>KDC&4>kLHi|*)Ehrrf9Rb0nTePG-)CSp_ z`uN-0^$tTNK)m&|rb>i3+LSH?|48!rA!;2G=DW(j>cJ)8wiB~k5J|myjldWE(J*PP z+q}7H%7@g5z_49EJpdCqwN_VTWs>7|dz16bOtFm&9|eAzt;UNd_FCO^J5_7U=4Nxo z?$EzZ)s?m2L%KM_^-f@jf{VReLado$q(W+?w0$Wa&762Os!p))C)mfBCT<2H9}dBi zR|c*J3Rs1%H+HG@lRIb;>$kzhwKh=Aej5`NSl?Em zmY*hUpBAz?Y=EOcl08wYdHB1IU9O@a6%tOEkgN`UZTAI;KUp4wB@ZFu1!;_7t`2cm zwHX^~U1`AuLc#`lD?;bD3af$7>?>0C^s>S`E0j~d|B$UF(yGz|dMKp=i#7A+Eatv0 z+Y@oB!$ceSO?)(1$=G2M#J}2v6p4R9z1o+ruH!j!9y_hbm#)(U9{q@681qfTjIc?8 za69*N2FD8wJfVg2^*1p&3R;9@OT|ZiUVBx>5ljn4%j&2@Iix=-3Oe^g_I!&qpd4Di&$B;ZPN%bxT}W z4>;9_0oL@AqEaDTU#Bg;X<~e2Dou&1d6_|MZr`Eq)=Eh(@`|%4bkH42S*J#sC`kT* zD-&RcE8|QU_f9@X140thA7|!V-lV{lVGVn>vC1YzY{eLku*4E(@uAhrD8OU;H(o(U z%abnBs1^JdFQxFC)`5n~40fjCVOpKMx>sB&xJ$uoEj~6+nM{whKz*tE`iR4%za7~$$6H!%9RiM-1X8&nl7q_0gEe*twdt$YVze!jDP!O7)T)*T{@n^Bs`>R0NXRS0ODLK^4 zQZ0+L9I3`&^y=k5k(1>-dmlV!_*)w-t5wh_*^>v{=w#<&wxinz>&GBl_j-u4O7%o_ zuO$(O@8(Cgd{nkoQN*tEf|(sNVqSG~^+20O@SqXIKTs;7x_+&QKvS_Dri>fsLtEZT#Eb{4E8b2D$k6=_^BS(=abXFVCRhuW!R+ttjZes(H`C{g>|LktN7X2U zH}sx~x?Dzg;m><8L&a^R&eRVqJj;|X8M-iQO};NXDLa_^tO3RcU*0#Ye4l;VDXzm+ z!7=(y%fq)?kGKd}lSCz4Oz>&1Q;*g}RFU2H1G5Iz)bCJTu@uwiusx{ahA8f zA}@;&NaH@=dO3dpZZ+(WlB<-7xj)BV@3{83uo|~s4deWO@RJ}YoA_s~?5Z92w)t)+ zt~g*Nk>n`}lyzawX;Hn;?NLGNLG;PyZ6LhBou#BapR?V*KwCT|i6Uq$J zpvFAFMZ=2322km{z&X(1v@KYP{Q@astGvdgr!uIq4b_shiwi9zXuWD`+ zD2&uMS@@Lc#dNanGl7tgyg z2?~~d)P82jl<>(=w_blhtVD-x?dg5@uk)?Xw5*Y-UKp*~juaqfW*`0}p8JHa1RasC zyTJ4Bu`zBxlxi6Y?O$kyR;h>W#t5rHUqF76TwCX^H4>1GF~girA&aurTV!)i*fGKq z4s${KW64;`aeQdPGWw|6hVstY@~W6VV5O9N-jcU_LK}p~6w}}lcmtSD@KTgc8 zT^nCdv#hgR6ftaAE|Q?}limZo<_U=&EbT`bQ9`8s?Rxq3lN3MSIcAX(^ugMB=T>94 z{U+4FQEyUwVpSgHnN9>91Lu8#Q#dMt3Vx+)nJ# zK6(0*M$Gg+9Mf58F^D( z9brVp?kz%zO!HsAxr1d11W-NH`M9-B36DG&bhWqnHT2wV&fePG@0#}boaq>l!XNM!(2aHMCzBc$kRUiTMkjMV^HHy$*}LzdU&ncg*A4el1~DBxdOQam9w* z?lrN3(EP10$o@TB6IGbNbkq9wQ1JGY_#i;wV74aUvQ6}*qT1xj22WOE$VHbf)e)Yr*=2gk%Lvj9F(8Ly?r%OLTOq6~U8;Jr;x z|G9We8Si~}H%!JH#=gm9ES?eBOu?RV1Ouf-Kl_ZrEo{~opOgw1;QgD>kv=0##2lre zuVF=f%;yXBw2ex7(k!TD>3n!^X)|DDy;k7;x09EJK^2~lbZrel1N~e|WwU7ZY`u97 z$DW#2n7Pf4$E*QZSBy6YE?{<==>HuG>_w$IA%;O;aR%?D4l}!J;9WoZ!{>A~zNKi8U84guevH2o;{@Pt;0nTu^s<5%1{w~TJx(3& z8V!OsS>3JhXMR!tsM#psKPwL6P*(ODJ%fKROy>4f z^h%Zd9P)da><{p=ej-Umnfr50xUnSA8hzxlW!Qm~_=Tb3YLNt~xCU_7WNPVkJSIvD z@JbQdCH$JrU#v4#?H`BDtDd0mj4JZb1ggv5Y~{y(%DXGN^NOwc6mVOa0ncxmnMg%t_NHD`aH%%B z{D*!1*5q&a7hdmARB!x*=>UdQ*P)Hf{}CqEa)NB*OLU7$r%)1!WIPj~+L7?cLrjGu z-%A~N^rNr#EVW#8sKpuBe9^_E5DV5m^=j1^cA471qN-ZcF>}AH;#I4BnTG;jvw5cR z>AinxCK^Sn1P{D2G;?=Em#$&ri{mXXMwko4GA66iR0qDAGI7s_({ql_ur<5` z`O5FIlm5Xp^U1L=NoA5ZD_yb1>txaQI?o)gSU2yDell%OI$gCBnE7FftFhq_HH7gK zvQ8#vJb+$q2nyuNyjXTh#O%D99&e^X)xkCtVWSJQ78pL|LhH2w>jG)~3RJ=b$sHV1 zBh|TSex}dxf2S#a6TN8KC#zAc5swL$BWNgUCi7B{T{-zX5Zz4tCZ1XaJJ18yp9&-l zvWfrG|0gH7lVyQueR)yW+UG+ou2t-WDV%D&3_m?6D;|@FvfSa@fjqGF7RQd+X1ytd zV~1-Gp!b_0rtLk(&-zkQAFlpqr{;~K>own;K4N4UBx4?uCeMWMdfHjXLSo7(rluz0 zfdT`4D*uNm{yVSY{d<%CU=_5Rv9(Pg-=+xSmX=&Xd5Rq{Y$`jT>vD=qoDogqwDBm)Rahuj zwIfjc13e5>Y|F~O(^~r~CWtrqwbGO#Y9(Tc9r@j2MgIauWSK=mJfWs$vB*exer}OH z|7tJNF**>7XChJ*bL;$dp~@rjPop4H@^oMQ!K3h!_~eG<*+*|y6uLoHxlpo>n={v~tv z4BZfGvX2QN7D*)5KQCpwzi^g&;gNUVi&_gBnP7pt#> zZf=OA8ZjCtbTSq&t#o~}gMoVqhdA3dExN#$;_qZZVkNx}Y79dK@d%MZHt6!bA4~h+&xgOLmAz@KQ)oN|wNLRk|Dk2|ziV(Uw5Bj?;D5%SZh>~qPO-38&QLm~DUG){$+*Rg+g zX0COC<_ow#F%2WK$fb*lui7Ul zo;<}k3X2dD-{HiO8V^@QqH*f!L@bNSEzpPl;c6FCR+X@EjP8x5mfaJ6JBSWz5wz-B zpK2@RH?>1s8c-AK=klSOz#`|8W_`;z@?yU`e1X-%3Mc7guAZQ7)y5lF0rgPm%X}jm z)^_=nNso2z)yXOJUNN1+wAv)>Q@;(`(r@@K9)En{9$M%wrIgyIMKgc|%OAx_M|Dej z`lK1U`S%lv5;lJqjqS6FTsu>L!O!h>&peHsYckd321Jy@JnyP_aeqWS@;|7&GbL;K z{&qExgS?2UrbG_@iaipWFIm-U3clX7!Trp#QBdKxA0;k0(ghJB>G^hdE?YTXhHi)6 zy%RP17i^>@X@#S~DZfSoyx)y)xM56o3Lf`_Kbdys3GHP`FFGe^NcJoR1<`HjQWFdH zFaUDNKc>nA!3wN1+YY;@T_n}L!`!|HomkMhsQL^18i*iBaA3)Ur=i?~E)K%yeyw1& zMWw*6ubnD=zMM@csBC~{lV4(DUl2%VDZNK+NhCR9`Fgc?^!XOFLlU!qZHU#cpqu!P z!VEjQR7iny@WSXC$;@tStk;l`SA*b?S zP+Xv44!3lHWTGJ6PHH54ecz=$m05$X9)J4`Bg*<|m#}F#&3lXVhTX5U{G|Vc7PyUb z0Q1j1T1{0*+SGv2=oxipm|@i{Kl#RK!V<-0&%e0w>yX0>qEWB)!dn--XgUS!R6#t; z-4ostwE4}+XJUKMgH)BX5jE7K_~YGksK)T!bE_E>ch=;8LLNy8-3wcydeP{pA08b~ z%1vMzE|d)lyMgzr3)36b2;*L_z8QK-f*Ms{bKuuq{K??6?1B(`SA_a{tb@6b>g?iw zn_?S*BgJJjfMQa9E97+z@bA@{rZ@A+42VK83%I3ONmNi63yrEY`zxcPDZ+FoJMIN{ z6=2CSkv8~s=m)%^Y*0i1tFh?OY(8QziJqx6yNK6S`@>TyV+T12U^+>^MkU^Gn(bG# zOVlAX%450-;#Cl`kTDv!I#Ho>)mb;9I3PaV237_ByspSNe%vkLWA#eBUJd?uebggi z6*zs5BVvM?ux(RACs(8y^({Q|hGUg=T=1{Th8$KM%VA#N6c^u9+LD3>2nIekw2w6h zTQeM;DC(E4QTnlM!h|?`%tt`hq>rmLEWMAlPW_8uppWM;v_M90ygyfnwsF~JXqo>^ zcYJ-^xCj?M{leyHhnrqih3MlY7%g)sSQv=Zv2<)5%2SQxP=Im(-$gIq_-1-Vz^C05 zFlr+*WGx-rB}25sBU_IXw<2CFy|FA&<>8cgY%X(J(So%8Aim1ppDIbsVq^!})K!0m tb)Nj@W<@in7&QdR;O8pB!FRX$>we!Ex%Kcw^s{!67Q@pg{{uoxxY|2R=d%C+ literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 9deb81c37..ae5569d0a 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -27,6 +27,8 @@ FORMAT = { "x-gzip": ("html-gzip.bin", "gzip"), "rawdeflate": ("html-rawdeflate.bin", "deflate"), "zlibdeflate": ("html-zlibdeflate.bin", "deflate"), + "gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"), + "gzip-deflate-gzip": ("html-gzip-deflate-gzip.bin", "gzip, deflate, gzip"), "br": ("html-br.bin", "br"), # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin "zstd-static-content-size": ("html-zstd-static-content-size.bin", "zstd"), @@ -205,6 +207,62 @@ class HttpCompressionTest(TestCase): assert newresponse is not response self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"uuencode"]) + def test_multi_compression_single_header(self): + response = self._getresponse("gzip-deflate") + request = response.request + newresponse = self.mw.process_response(request, response, self.spider) + assert newresponse is not response + assert "Content-Encoding" not in newresponse.headers + assert newresponse.body.startswith(b" Date: Mon, 26 Feb 2024 10:53:06 -0800 Subject: [PATCH 1354/2083] Remove usage of deprecated mktemp (#5285) --- .bandit.yml | 1 - tests/test_commands.py | 5 ++--- tests/test_downloader_handlers.py | 28 ++++++++++++---------------- tests/test_http2_client_protocol.py | 4 ++-- tests/test_pipeline_crawl.py | 4 ++-- tests/test_spiderloader/__init__.py | 4 ++-- tests/test_spiderstate.py | 5 ++--- tests/test_squeues_request.py | 2 +- tests/test_webclient.py | 7 +++---- 9 files changed, 26 insertions(+), 34 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 2aae8a0aa..8c6a08e1b 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -4,7 +4,6 @@ skips: - B105 - B301 - B303 -- B306 - B307 - B311 - B320 diff --git a/tests/test_commands.py b/tests/test_commands.py index 36f800850..2f36baa87 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -6,13 +6,12 @@ import platform import re import subprocess import sys -import tempfile from contextlib import contextmanager from itertools import chain from pathlib import Path from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION -from tempfile import mkdtemp +from tempfile import TemporaryFile, mkdtemp from threading import Timer from typing import Dict, Generator, Optional, Union from unittest import skipIf @@ -82,7 +81,7 @@ class ProjectTest(unittest.TestCase): rmtree(self.temp_path) def call(self, *new_args, **kwargs): - with tempfile.TemporaryFile() as out: + with TemporaryFile() as out: args = (sys.executable, "-m", "scrapy.cmdline") + new_args return subprocess.call( args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index dd07d33f1..d3fd63847 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -2,8 +2,8 @@ import contextlib import os import shutil import sys -import tempfile from pathlib import Path +from tempfile import mkdtemp, mkstemp from typing import Optional, Type from unittest import SkipTest, mock @@ -107,13 +107,14 @@ class LoadTestCase(unittest.TestCase): class FileTestCase(unittest.TestCase): def setUp(self): # add a special char to check that they are handled correctly - self.tmpname = Path(self.mktemp() + "^") + self.fd, self.tmpname = mkstemp(suffix="^") Path(self.tmpname).write_text("0123456789", encoding="utf-8") handler = build_from_crawler(FileDownloadHandler, get_crawler()) self.download_request = handler.download_request def tearDown(self): - self.tmpname.unlink() + os.close(self.fd) + os.remove(self.tmpname) def test_download(self): def _test(response): @@ -122,12 +123,12 @@ class FileTestCase(unittest.TestCase): self.assertEqual(response.body, b"0123456789") self.assertEqual(response.protocol, None) - request = Request(path_to_file_uri(str(self.tmpname))) + request = Request(path_to_file_uri(self.tmpname)) assert request.url.upper().endswith("%5E") return self.download_request(request, Spider("foo")).addCallback(_test) def test_non_existent(self): - request = Request(path_to_file_uri(self.mktemp())) + request = Request(path_to_file_uri(mkdtemp())) d = self.download_request(request, Spider("foo")) return self.assertFailure(d, OSError) @@ -224,8 +225,7 @@ class HttpTestCase(unittest.TestCase): certfile = "keys/localhost.crt" def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) @@ -651,8 +651,7 @@ class Https11CustomCiphers(unittest.TestCase): certfile = "keys/localhost.crt" def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) self.site = server.Site(r, timeout=None) @@ -1015,8 +1014,7 @@ class BaseFTPTestCase(unittest.TestCase): from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dirs and test file - self.directory = Path(self.mktemp()) - self.directory.mkdir() + self.directory = Path(mkdtemp()) userdir = self.directory / self.username userdir.mkdir() for filename, content in self.test_files: @@ -1092,7 +1090,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): - f, local_fname = tempfile.mkstemp() + f, local_fname = mkstemp() fname_bytes = to_bytes(local_fname) local_fname = Path(local_fname) os.close(f) @@ -1113,7 +1111,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def _test_response_class(self, filename, response_class): - f, local_fname = tempfile.mkstemp() + f, local_fname = mkstemp() local_fname = Path(local_fname) os.close(f) meta = {} @@ -1163,9 +1161,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase): from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dir and test file - self.directory = Path(self.mktemp()) - self.directory.mkdir() - + self.directory = Path(mkdtemp()) for filename, content in self.test_files: (self.directory / filename).write_bytes(content) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 8fdf3d56f..995c02a1a 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,6 +5,7 @@ import shutil import string from ipaddress import IPv4Address from pathlib import Path +from tempfile import mkdtemp from typing import Dict from unittest import mock, skipIf from urllib.parse import urlencode @@ -185,8 +186,7 @@ class Https2ClientProtocolTestCase(TestCase): certificate_file = Path(__file__).parent / "keys" / "localhost.crt" def _init_resource(self): - self.temp_directory = self.mktemp() - Path(self.temp_directory).mkdir() + self.temp_directory = mkdtemp() r = File(self.temp_directory) r.putChild(b"get-data-html-small", GetDataHtmlSmall()) r.putChild(b"get-data-html-large", GetDataHtmlLarge()) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index c41ab483f..be9811980 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,5 +1,6 @@ import shutil from pathlib import Path +from tempfile import mkdtemp from typing import Optional, Set from testfixtures import LogCapture @@ -67,8 +68,7 @@ class FileDownloadCrawlTestCase(TestCase): self.mockserver.__enter__() # prepare a directory for storing files - self.tmpmediastore = Path(self.mktemp()) - self.tmpmediastore.mkdir() + self.tmpmediastore = Path(mkdtemp()) self.settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", "ITEM_PIPELINES": {self.pipeline_class: 1}, diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 04025d30d..f950739f2 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -3,6 +3,7 @@ import sys import tempfile import warnings from pathlib import Path +from tempfile import mkdtemp from twisted.trial import unittest from zope.interface.verify import verifyObject @@ -139,8 +140,7 @@ class SpiderLoaderTest(unittest.TestCase): class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): orig_spiders_dir = module_dir / "test_spiders" - self.tmpdir = Path(self.mktemp()) - self.tmpdir.mkdir() + self.tmpdir = Path(mkdtemp()) self.spiders_dir = self.tmpdir / "test_spiders_xxx" _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(str(self.tmpdir)) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index f97125b76..59d18d92e 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,6 +1,6 @@ import shutil from datetime import datetime, timezone -from pathlib import Path +from tempfile import mkdtemp from twisted.trial import unittest @@ -12,8 +12,7 @@ from scrapy.utils.test import get_crawler class SpiderStateTest(unittest.TestCase): def test_store_load(self): - jobdir = self.mktemp() - Path(jobdir).mkdir() + jobdir = mkdtemp() try: spider = Spider(name="default") dt = datetime.now(tz=timezone.utc) diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index b444c32b7..499ca46b8 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -25,7 +25,7 @@ class BaseQueueTestCase(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") self.qpath = self.tempfilename() - self.qdir = self.mkdtemp() + self.qdir = tempfile.mkdtemp() self.crawler = get_crawler(Spider) def tearDown(self): diff --git a/tests/test_webclient.py b/tests/test_webclient.py index d4b6ba15b..53558814d 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -4,6 +4,7 @@ Tests borrowed from the twisted.web.client tests. """ import shutil from pathlib import Path +from tempfile import mkdtemp import OpenSSL.SSL from twisted.internet import defer, reactor @@ -274,8 +275,7 @@ class WebClientTestCase(unittest.TestCase): return reactor.listenTCP(0, site, interface="127.0.0.1") def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) @@ -440,8 +440,7 @@ class WebClientSSLTestCase(unittest.TestCase): return f"https://127.0.0.1:{self.portno}/{path}" def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) r.putChild(b"payload", PayloadResource()) From 2d46b4acf5855faaf2d6baa36615f08bd8aefccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 27 Feb 2024 09:28:02 +0100 Subject: [PATCH 1355/2083] Complete coverage for the AutoThrottle extension (#6245) --- docs/topics/autothrottle.rst | 2 +- scrapy/extensions/throttle.py | 5 + tests/test_extension_throttle.py | 340 +++++++++++++++++++++++++++++++ 3 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 tests/test_extension_throttle.py diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 8e6aae65c..5370d77b3 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -131,7 +131,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote -websites. +websites. It must be higher than ``0.0``. By default, AutoThrottle adjusts the delay to send a single concurrent request to each of the remote websites. Set this option to diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 396800775..d217c7a69 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -16,6 +16,11 @@ class AutoThrottle: self.target_concurrency = crawler.settings.getfloat( "AUTOTHROTTLE_TARGET_CONCURRENCY" ) + if self.target_concurrency <= 0.0: + raise NotConfigured( + f"AUTOTHROTTLE_TARGET_CONCURRENCY " + f"({self.target_concurrency!r}) must be higher than 0." + ) crawler.signals.connect(self._spider_opened, signal=signals.spider_opened) crawler.signals.connect( self._response_downloaded, signal=signals.response_downloaded diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py new file mode 100644 index 000000000..dae4ea966 --- /dev/null +++ b/tests/test_extension_throttle.py @@ -0,0 +1,340 @@ +from logging import INFO +from unittest.mock import Mock + +import pytest + +from scrapy import Request, Spider +from scrapy.exceptions import NotConfigured +from scrapy.extensions.throttle import AutoThrottle +from scrapy.http.response import Response +from scrapy.settings.default_settings import ( + AUTOTHROTTLE_MAX_DELAY, + AUTOTHROTTLE_START_DELAY, + DOWNLOAD_DELAY, +) +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler as _get_crawler + +UNSET = object() + + +class TestSpider(Spider): + name = "test" + + +def get_crawler(settings=None, spidercls=None): + settings = settings or {} + settings["AUTOTHROTTLE_ENABLED"] = True + return _get_crawler(settings_dict=settings, spidercls=spidercls) + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + (UNSET, False), + (False, False), + (True, True), + ), +) +def test_enabled(value, expected): + settings = {} + if value is not UNSET: + settings["AUTOTHROTTLE_ENABLED"] = value + crawler = _get_crawler(settings_dict=settings) + if expected: + build_from_crawler(AutoThrottle, crawler) + else: + with pytest.raises(NotConfigured): + build_from_crawler(AutoThrottle, crawler) + + +@pytest.mark.parametrize( + "value", + ( + 0.0, + -1.0, + ), +) +def test_target_concurrency_invalid(value): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": value} + crawler = get_crawler(settings) + with pytest.raises(NotConfigured): + build_from_crawler(AutoThrottle, crawler) + + +@pytest.mark.parametrize( + ("spider", "setting", "expected"), + ( + (UNSET, UNSET, DOWNLOAD_DELAY), + (1.0, UNSET, 1.0), + (UNSET, 1.0, 1.0), + (1.0, 2.0, 1.0), + (3.0, 2.0, 3.0), + ), +) +def test_mindelay_definition(spider, setting, expected): + settings = {} + if setting is not UNSET: + settings["DOWNLOAD_DELAY"] = setting + + class _TestSpider(Spider): + name = "test" + + if spider is not UNSET: + _TestSpider.download_delay = spider + + crawler = get_crawler(settings, _TestSpider) + at = build_from_crawler(AutoThrottle, crawler) + at._spider_opened(_TestSpider()) + assert at.mindelay == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + (UNSET, AUTOTHROTTLE_MAX_DELAY), + (1.0, 1.0), + ), +) +def test_maxdelay_definition(value, expected): + settings = {} + if value is not UNSET: + settings["AUTOTHROTTLE_MAX_DELAY"] = value + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + at._spider_opened(TestSpider()) + assert at.maxdelay == expected + + +@pytest.mark.parametrize( + ("min_spider", "min_setting", "start_setting", "expected"), + ( + (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), + (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0), + ( + AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 1.0, + ), + ( + AUTOTHROTTLE_START_DELAY + 2.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, + ), + ( + AUTOTHROTTLE_START_DELAY + 1.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 2.0, + AUTOTHROTTLE_START_DELAY + 2.0, + ), + ), +) +def test_startdelay_definition(min_spider, min_setting, start_setting, expected): + settings = {} + if min_setting is not UNSET: + settings["DOWNLOAD_DELAY"] = min_setting + if start_setting is not UNSET: + settings["AUTOTHROTTLE_START_DELAY"] = start_setting + + class _TestSpider(Spider): + name = "test" + + if min_spider is not UNSET: + _TestSpider.download_delay = min_spider + + crawler = get_crawler(settings, _TestSpider) + at = build_from_crawler(AutoThrottle, crawler) + spider = _TestSpider() + at._spider_opened(spider) + assert spider.download_delay == expected + + +@pytest.mark.parametrize( + ("meta", "slot"), + ( + ({}, None), + ({"download_latency": 1.0}, None), + ({"download_slot": "foo"}, None), + ({"download_slot": "foo"}, "foo"), + ({"download_latency": 1.0, "download_slot": "foo"}, None), + ), +) +def test_skipped(meta, slot): + crawler = get_crawler() + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + request = Request("https://example.com", meta=meta) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + if slot is not None: + crawler.engine.downloader.slots[slot] = object() + at._adjust_delay = None # Raise exception if called. + + at._response_downloaded(None, request, spider) + + +@pytest.mark.parametrize( + ("download_latency", "target_concurrency", "slot_delay", "expected"), + ( + (2.0, 2.0, 1.0, 1.0), + (1.0, 2.0, 1.0, 0.75), + (4.0, 2.0, 1.0, 2.0), + (2.0, 1.0, 1.0, 2.0), + (2.0, 4.0, 1.0, 0.75), + (2.0, 2.0, 0.5, 1.0), + (2.0, 2.0, 2.0, 1.5), + ), +) +def test_adjustment(download_latency, target_concurrency, slot_delay, expected): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +@pytest.mark.parametrize( + ("mindelay", "maxdelay", "expected"), + ( + (0.5, 2.0, 1.0), + (0.25, 0.5, 0.5), + (2.0, 4.0, 2.0), + ), +) +def test_adjustment_limits(mindelay, maxdelay, expected): + download_latency, target_concurrency, slot_delay = (2.0, 2.0, 1.0) + # expected adjustment without limits with these values: 1.0 + settings = { + "AUTOTHROTTLE_MAX_DELAY": maxdelay, + "AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency, + "DOWNLOAD_DELAY": mindelay, + } + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +@pytest.mark.parametrize( + ("download_latency", "target_concurrency", "slot_delay", "expected"), + ( + (2.0, 2.0, 1.0, 1.0), + (1.0, 2.0, 1.0, 1.0), # Instead of 0.75 + (4.0, 2.0, 1.0, 2.0), + ), +) +def test_adjustment_bad_response( + download_latency, target_concurrency, slot_delay, expected +): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, status=400) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +def test_debug(caplog): + settings = {"AUTOTHROTTLE_DEBUG": True} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": 1.0, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, body=b"foo") + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = 2.0 + slot.transferring = (None, None) + crawler.engine.downloader.slots["foo"] = slot + + caplog.clear() + with caplog.at_level(INFO): + at._response_downloaded(response, request, spider) + + assert caplog.record_tuples == [ + ( + "scrapy.extensions.throttle", + INFO, + "slot: foo | conc: 2 | delay: 1500 ms (-500) | latency: 1000 ms | size: 3 bytes", + ), + ] + + +def test_debug_disabled(caplog): + crawler = get_crawler() + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": 1.0, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, body=b"foo") + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = 2.0 + slot.transferring = (None, None) + crawler.engine.downloader.slots["foo"] = slot + + caplog.clear() + with caplog.at_level(INFO): + at._response_downloaded(response, request, spider) + + assert caplog.record_tuples == [] From d87f949526470fc4847c99f58e1dcc40d4e9ed00 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 27 Feb 2024 17:08:13 -0300 Subject: [PATCH 1356/2083] Use defusedxml.xmlrpc --- scrapy/http/request/rpc.py | 4 ++++ setup.py | 1 + 2 files changed, 5 insertions(+) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index bde860a66..2bf5ba4b6 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -7,9 +7,13 @@ See documentation in docs/topics/request-response.rst import xmlrpc.client as xmlrpclib from typing import Any, Optional +from defusedxml import xmlrpc + from scrapy.http.request import Request from scrapy.utils.python import get_func_args +xmlrpc.monkey_patch() + DUMPS_ARGS = get_func_args(xmlrpclib.dumps) diff --git a/setup.py b/setup.py index 405633f55..2d6d26b0c 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ install_requires = [ "packaging", "tldextract", "lxml>=4.4.1", + "defusedxml>=0.7.1", ] extras_require = { ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], From 7f945ad6db728234987629b2299750abee5c1781 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 27 Feb 2024 17:39:29 -0300 Subject: [PATCH 1357/2083] Import defusedxml.xmlrpc using alias --- scrapy/http/request/rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 2bf5ba4b6..5a2107f76 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -7,12 +7,12 @@ See documentation in docs/topics/request-response.rst import xmlrpc.client as xmlrpclib from typing import Any, Optional -from defusedxml import xmlrpc +import defusedxml.xmlrpc as xml_rpc from scrapy.http.request import Request from scrapy.utils.python import get_func_args -xmlrpc.monkey_patch() +xml_rpc.monkey_patch() DUMPS_ARGS = get_func_args(xmlrpclib.dumps) From 008ebb65fc2f0e7cfe9fa43d7ac938a94b3098fb Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 27 Feb 2024 18:10:28 -0300 Subject: [PATCH 1358/2083] Change immport style --- scrapy/http/request/rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 5a2107f76..84b433990 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -7,12 +7,12 @@ See documentation in docs/topics/request-response.rst import xmlrpc.client as xmlrpclib from typing import Any, Optional -import defusedxml.xmlrpc as xml_rpc +import defusedxml.xmlrpc from scrapy.http.request import Request from scrapy.utils.python import get_func_args -xml_rpc.monkey_patch() +defusedxml.xmlrpc.monkey_patch() DUMPS_ARGS = get_func_args(xmlrpclib.dumps) From 282767f23b2e71969bea3bd5492abde88d2054c6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 19:49:06 +0500 Subject: [PATCH 1359/2083] Bump black. --- .flake8 | 2 +- .pre-commit-config.yaml | 4 ++-- docs/topics/addons.rst | 3 +-- scrapy/commands/__init__.py | 1 + scrapy/commands/shell.py | 1 + scrapy/core/downloader/handlers/__init__.py | 6 +++--- scrapy/core/downloader/handlers/http10.py | 1 + scrapy/core/downloader/middleware.py | 1 + scrapy/core/engine.py | 1 + scrapy/core/http2/stream.py | 12 ++++++------ scrapy/core/scraper.py | 1 + scrapy/core/spidermw.py | 1 + scrapy/downloadermiddlewares/defaultheaders.py | 1 + scrapy/downloadermiddlewares/downloadtimeout.py | 1 + scrapy/downloadermiddlewares/retry.py | 1 + scrapy/exceptions.py | 1 + scrapy/extension.py | 1 + scrapy/extensions/corestats.py | 1 + scrapy/extensions/memusage.py | 1 + scrapy/extensions/postprocessing.py | 1 + scrapy/http/request/__init__.py | 17 +++++++++++------ scrapy/http/request/rpc.py | 1 + scrapy/http/response/__init__.py | 1 + scrapy/http/response/text.py | 1 + scrapy/link.py | 1 + scrapy/linkextractors/__init__.py | 1 + scrapy/linkextractors/lxmlhtml.py | 1 + scrapy/loader/__init__.py | 1 + scrapy/mail.py | 1 + scrapy/pipelines/__init__.py | 1 + scrapy/pipelines/files.py | 1 + scrapy/pipelines/images.py | 1 + scrapy/responsetypes.py | 1 + scrapy/selector/unified.py | 1 + scrapy/settings/__init__.py | 1 - scrapy/shell.py | 1 + scrapy/spidermiddlewares/httperror.py | 1 + scrapy/spidermiddlewares/offsite.py | 1 + scrapy/spidermiddlewares/referer.py | 1 + scrapy/spiders/__init__.py | 1 + scrapy/spiders/feed.py | 1 + scrapy/statscollectors.py | 1 + scrapy/utils/defer.py | 7 +++---- scrapy/utils/deprecate.py | 6 ++---- scrapy/utils/iterators.py | 11 +++++------ scrapy/utils/misc.py | 1 + scrapy/utils/python.py | 7 +++---- scrapy/utils/request.py | 7 ++++--- scrapy/utils/response.py | 7 ++++--- scrapy/utils/signal.py | 1 + scrapy/utils/sitemap.py | 1 + scrapy/utils/spider.py | 15 +++++---------- scrapy/utils/url.py | 1 + tests/mocks/dummydbm.py | 1 + tests/spiders.py | 1 + tests/test_exporters.py | 4 +++- tests/test_pipeline_media.py | 6 +++--- tests/test_responsetypes.py | 5 ++++- tests/test_webclient.py | 1 + 59 files changed, 102 insertions(+), 60 deletions(-) diff --git a/.flake8 b/.flake8 index 544d72956..62ccad9cf 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] max-line-length = 119 -ignore = W503, E203 +ignore = E203, E501, E701, E704, W503 exclude = docs/conf.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0cff5cc73..83bc65b67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 23.9.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -21,4 +21,4 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==23.9.1 + - black==24.2.0 diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 1bf2172bd..d2fc41003 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -150,8 +150,7 @@ Access the crawler instance: def from_crawler(cls, crawler): return cls(crawler) - def update_settings(self, settings): - ... + def update_settings(self, settings): ... Use a fallback component: diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 2aa569cdd..27993710e 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -1,6 +1,7 @@ """ Base class for Scrapy commands """ + import argparse import os from pathlib import Path diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 12e37babc..f72a23c6a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -3,6 +3,7 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ + from argparse import Namespace from threading import Thread from typing import List, Type diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 416669b7f..ade51ca63 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -21,9 +21,9 @@ logger = logging.getLogger(__name__) class DownloadHandlers: def __init__(self, crawler: "Crawler"): self._crawler: "Crawler" = crawler - self._schemes: Dict[ - str, Union[str, Callable] - ] = {} # stores acceptable schemes on instancing + self._schemes: Dict[str, Union[str, Callable]] = ( + {} + ) # stores acceptable schemes on instancing self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers handlers: Dict[str, Union[str, Callable]] = without_none_values( diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index b6ac7a251..d168c2b2e 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,5 +1,6 @@ """Download handlers for http and https schemes """ + from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index dca13c01e..52ebe4e22 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,6 +3,7 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ + from typing import Any, Callable, Generator, List, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 545cd401f..2db085081 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spider. For more information see docs/topics/architecture.rst """ + import logging from time import time from typing import ( diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 0f282d83d..4132fc385 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -111,17 +111,17 @@ class Stream: # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated self.metadata: Dict = { - "request_content_length": 0 - if self._request.body is None - else len(self._request.body), + "request_content_length": ( + 0 if self._request.body is None else len(self._request.body) + ), # Flag to keep track whether the stream has initiated the request "request_sent": False, # Flag to track whether we have logged about exceeding download warnsize "reached_warnsize": False, # Each time we send a data frame, we will decrease value by the amount send. - "remaining_content_length": 0 - if self._request.body is None - else len(self._request.body), + "remaining_content_length": ( + 0 if self._request.body is None else len(self._request.body) + ), # Flag to keep track whether client (self) have closed this stream "stream_closed_local": False, # Flag to keep track whether the server has closed the stream diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8fb16b8a9..272841e01 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,5 +1,6 @@ """This module implements the Scraper component which parses responses and extracts information from them""" + from __future__ import annotations import logging diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 031a0be36..1ccfd08a2 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,6 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ + import logging from inspect import isasyncgenfunction, iscoroutine from itertools import islice diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 8aec37cf1..58fd415b9 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -3,6 +3,7 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ + from __future__ import annotations from typing import TYPE_CHECKING, Iterable, Tuple, Union diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index 1c904c05b..fd7c03a38 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -3,6 +3,7 @@ Download timeout middleware See documentation in docs/topics/downloader-middleware.rst """ + from __future__ import annotations from typing import TYPE_CHECKING, Union diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 3c494de78..46587a898 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,6 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ + from __future__ import annotations import warnings diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 6d188c489..e7ecdbe0c 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -4,6 +4,7 @@ Scrapy core exceptions These exceptions are documented in docs/topics/exceptions.rst. Please don't add new exceptions here without documenting them there. """ + from typing import Any # Internal diff --git a/scrapy/extension.py b/scrapy/extension.py index 4e365cfa1..6be14450c 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -3,6 +3,7 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ + from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 302a615f2..717c249d9 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -1,6 +1,7 @@ """ Extension for collecting core stats like items scraped and start/finish times """ + from datetime import datetime, timezone from scrapy import signals diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index ca766c938..4d4501c44 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -3,6 +3,7 @@ MemoryUsage extension See documentation in docs/topics/extensions.rst """ + import logging import socket import sys diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 17969c5b0..f8b59827b 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -1,6 +1,7 @@ """ Extension for processing data before they are exported to feeds. """ + from bz2 import BZ2File from gzip import GzipFile from io import IOBase diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a1c5a5e51..6269ee86a 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,6 +4,7 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ + import inspect from typing import ( Any, @@ -231,12 +232,16 @@ class Request(object_ref): """ d = { "url": self.url, # urls are safe (safe_string_url) - "callback": _find_method(spider, self.callback) - if callable(self.callback) - else self.callback, - "errback": _find_method(spider, self.errback) - if callable(self.errback) - else self.errback, + "callback": ( + _find_method(spider, self.callback) + if callable(self.callback) + else self.callback + ), + "errback": ( + _find_method(spider, self.errback) + if callable(self.errback) + else self.errback + ), "headers": dict(self.headers), } for attr in self.attributes: diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 84b433990..e20e7c438 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -4,6 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ + import xmlrpc.client as xmlrpclib from typing import Any, Optional diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 6eae3e8b3..d73dfce4b 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,6 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ + from __future__ import annotations from ipaddress import IPv4Address, IPv6Address diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6596d8a5c..2816610fb 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -4,6 +4,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ + from __future__ import annotations import json diff --git a/scrapy/link.py b/scrapy/link.py index 0868ae5ef..4bdbc1823 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ + from typing import Any diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 3774430a7..73a63651c 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -5,6 +5,7 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ + import re # common file extensions that are not followed if they occur in links diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 23cbd0116..d76db20ba 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,6 +1,7 @@ """ Link extractor based on lxml.html """ + import logging import operator from functools import partial diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 1042a3d48..529fa279e 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -3,6 +3,7 @@ Item Loader See documentation in docs/topics/loaders.rst """ + import itemloaders from scrapy.item import Item diff --git a/scrapy/mail.py b/scrapy/mail.py index 237327451..4b18b6003 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -3,6 +3,7 @@ Mail sending helpers See documentation in docs/topics/email.rst """ + import logging from email import encoders as Encoders from email.mime.base import MIMEBase diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index c97d71fb6..f9544d329 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -3,6 +3,7 @@ Item pipeline See documentation in docs/item-pipeline.rst """ + from typing import Any, List from twisted.internet.defer import Deferred diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 73064ad10..1d7625299 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -3,6 +3,7 @@ Files Pipeline See documentation in topics/media-pipeline.rst """ + import base64 import functools import hashlib diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 02c4b1361..8169583f8 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -3,6 +3,7 @@ Images Pipeline See documentation in topics/media-pipeline.rst """ + import functools import hashlib import warnings diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 0d127d851..702e50536 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -2,6 +2,7 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ + from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 863fb6032..75d5e9fbd 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -1,6 +1,7 @@ """ XPath selectors based on lxml """ + from typing import Any, Optional, Type, Union from parsel import Selector as _ParselSelector diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b5d8fdb12..d270a72f4 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -58,7 +58,6 @@ def get_settings_priority(priority: Union[int, str]) -> int: class SettingsAttribute: - """Class for storing data related to settings attributes. This class is intended for internal usage, you should try Settings class diff --git a/scrapy/shell.py b/scrapy/shell.py index bb3b1461c..05909977a 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -3,6 +3,7 @@ See documentation in docs/topics/shell.rst """ + import os import signal diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 94450b35b..35c869a75 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -3,6 +3,7 @@ HttpError Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index a5214702d..dd2fccfcb 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -3,6 +3,7 @@ Offsite Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a29e0ebb5..a0b6851e5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,6 +2,7 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ + from __future__ import annotations import warnings diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e16d71727..72c2aaba7 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -3,6 +3,7 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 599af7360..5caf8c79e 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -4,6 +4,7 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ + from scrapy.exceptions import NotConfigured, NotSupported from scrapy.selector import Selector from scrapy.spiders import Spider diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 15193aac5..ab571a3ab 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -1,6 +1,7 @@ """ Scrapy extension for collecting scraping stats """ + import logging import pprint from typing import TYPE_CHECKING, Any, Dict, Optional diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bf3c5ef5b..c391db9fd 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -1,6 +1,7 @@ """ Helper functions for dealing with Twisted deferreds """ + import asyncio import inspect from asyncio import Future @@ -304,13 +305,11 @@ _T = TypeVar("_T") @overload -def deferred_from_coro(o: _CT) -> Deferred: - ... +def deferred_from_coro(o: _CT) -> Deferred: ... @overload -def deferred_from_coro(o: _T) -> _T: - ... +def deferred_from_coro(o: _T) -> _T: ... def deferred_from_coro(o: _T) -> Union[Deferred, _T]: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index ea577c44a..e0f2ac763 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -138,13 +138,11 @@ DEPRECATION_RULES: List[Tuple[str, str]] = [] @overload -def update_classpath(path: str) -> str: - ... +def update_classpath(path: str) -> str: ... @overload -def update_classpath(path: Any) -> Any: - ... +def update_classpath(path: Any) -> Any: ... def update_classpath(path: Any) -> Any: diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c56be5ea2..93a2ba7a1 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -225,18 +225,17 @@ def csviter( @overload -def _body_or_str(obj: Union[Response, str, bytes]) -> str: - ... +def _body_or_str(obj: Union[Response, str, bytes]) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: - ... +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes: - ... +def _body_or_str( + obj: Union[Response, str, bytes], unicode: Literal[False] +) -> bytes: ... def _body_or_str( diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b38190cb3..7b43760a8 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,4 +1,5 @@ """Helper functions which don't fit anywhere else""" + import ast import hashlib import inspect diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 68ca96b69..7b408c49c 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,6 +1,7 @@ """ This module contains essential stuff that should've come with Python itself ;) """ + import collections.abc import gc import inspect @@ -285,13 +286,11 @@ def equal_attributes( @overload -def without_none_values(iterable: Mapping) -> dict: - ... +def without_none_values(iterable: Mapping) -> dict: ... @overload -def without_none_values(iterable: Iterable) -> Iterable: - ... +def without_none_values(iterable: Iterable) -> Iterable: ... def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index db0b44cf4..e99d1eeb5 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -44,7 +44,9 @@ def _serialize_headers( yield from request.headers.getlist(header) -_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" +_fingerprint_cache: ( + "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" +) _fingerprint_cache = WeakKeyDictionary() @@ -114,8 +116,7 @@ def fingerprint( class RequestFingerprinterProtocol(Protocol): - def fingerprint(self, request: Request) -> bytes: - ... + def fingerprint(self, request: Request) -> bytes: ... class RequestFingerprinter: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 33cd692bf..63a484b42 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -2,6 +2,7 @@ This module provides some useful functions for working with scrapy.http.Response objects """ + import os import re import tempfile @@ -29,9 +30,9 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: return _baseurl_cache[response] -_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = ( - WeakKeyDictionary() -) +_metaref_cache: ( + "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" +) = WeakKeyDictionary() def get_meta_refresh( diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 21a12a19e..a25100c03 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,4 +1,5 @@ """Helper functions for working with signals""" + import collections.abc import logging from typing import Any as TypingAny diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 3d2ecc9a7..8bf941eb2 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,6 +4,7 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ + from typing import Any, Dict, Generator, Iterator, Optional from urllib.parse import urljoin diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 704df8657..855bc8f87 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -39,13 +39,11 @@ def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ig @overload -def iterate_spider_output(result: CoroutineType) -> Deferred: - ... +def iterate_spider_output(result: CoroutineType) -> Deferred: ... @overload -def iterate_spider_output(result: _T) -> Iterable: - ... +def iterate_spider_output(result: _T) -> Iterable: ... def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: @@ -83,8 +81,7 @@ def spidercls_for_request( default_spidercls: Type[Spider], log_none: bool = ..., log_multiple: bool = ..., -) -> Type[Spider]: - ... +) -> Type[Spider]: ... @overload @@ -94,8 +91,7 @@ def spidercls_for_request( default_spidercls: Literal[None], log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: - ... +) -> Optional[Type[Spider]]: ... @overload @@ -105,8 +101,7 @@ def spidercls_for_request( *, log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: - ... +) -> Optional[Type[Spider]]: ... def spidercls_for_request( diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 22b4197f9..9d97cb12f 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -5,6 +5,7 @@ library. Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ + import re from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 2869ff8f7..bde3de228 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -1,4 +1,5 @@ """DBM-like dummy module""" + import collections from typing import Any, DefaultDict diff --git a/tests/spiders.py b/tests/spiders.py index 3df153a12..94969db99 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -1,6 +1,7 @@ """ Some spiders used for testing and benchmarking """ + import asyncio import time from urllib.parse import urlencode diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c11913365..59b724495 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -121,7 +121,9 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(name, "John\xa3") ie = self._get_exporter(fields_to_export={"name": "å稱"}) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [("å稱", "John\xa3")]) + self.assertEqual( + list(ie._get_serialized_fields(self.i)), [("å稱", "John\xa3")] + ) def test_field_custom_serializer(self): i = self.custom_field_item_class(name="John\xa3", age="22") diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 820484565..d477b59be 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -22,9 +22,9 @@ from scrapy.utils.test import get_crawler try: from PIL import Image # noqa: imported just to check for the import error except ImportError: - skip_pillow: Optional[ - str - ] = "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + skip_pillow: Optional[str] = ( + "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + ) else: skip_pillow = None diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 713a83d52..2633cca5b 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -33,7 +33,10 @@ class ResponseTypesTest(unittest.TestCase): ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response), ("attachment;filename=data高.doc".encode("gbk"), Response), ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse), - ("attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), XmlResponse), + ( + "attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), + XmlResponse, + ), ] for source, cls in mappings: retcls = responsetypes.from_content_disposition(source) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 53558814d..cce119001 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -2,6 +2,7 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ + import shutil from pathlib import Path from tempfile import mkdtemp From 6e5918345b8eb10674e11d9c4c5db8c9028be674 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 19:50:54 +0500 Subject: [PATCH 1360/2083] Bump bandit, flake8 and isort. --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 83bc65b67..a911d4cfe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,11 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 + rev: 1.7.7 hooks: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 - repo: https://github.com/psf/black.git @@ -13,7 +13,7 @@ repos: hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs From 68104b9f48802d1ecc1c892c61aaa197500435b5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 20:21:27 +0500 Subject: [PATCH 1361/2083] Update .bandit.yml, add problem names. --- .bandit.yml | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 8c6a08e1b..6e8331c0f 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,20 +1,19 @@ skips: -- B101 -- B113 # https://github.com/PyCQA/bandit/issues/1010 -- B105 -- B301 -- B303 -- B307 -- B311 -- B320 -- B321 -- B324 -- B402 # https://github.com/scrapy/scrapy/issues/4180 -- B403 -- B404 -- B406 -- B410 -- B503 -- B603 -- B605 +- B101 # assert_used +- B105 # hardcoded_password_string +- B301 # pickle +- B307 # eval +- B311 # random +- B320 # xml_bad_etree +- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 +- B324 # hashlib "Use of weak SHA1 hash for security" +- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 +- B403 # import_pickle +- B404 # import_subprocess +- B406 # import_xml_sax +- B410 # import_lxml +- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 +- B503 # ssl_with_bad_defaults +- B603 # subprocess_without_shell_equals_true +- B605 # start_process_with_a_shell exclude_dirs: ['tests'] From d2c05d9d96394e111ead1aa097402cdbc18c0859 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 20:43:57 +0500 Subject: [PATCH 1362/2083] Bump mypy and type stubs. --- scrapy/utils/spider.py | 3 +-- scrapy/utils/ssl.py | 2 +- tox.ini | 14 +++++++------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 855bc8f87..cbbb01d85 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -34,8 +34,7 @@ _T = TypeVar("_T") # https://stackoverflow.com/questions/60222982 @overload -def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ignore[misc] - ... +def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap] @overload diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index e74769c65..d520ef809 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,6 +1,6 @@ from typing import Any, Optional -import OpenSSL._util as pyOpenSSLutil # type: ignore[import-untyped] +import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL import OpenSSL.version from OpenSSL.crypto import X509Name diff --git a/tox.ini b/tox.ini index 359ff0f73..e787c7bf3 100644 --- a/tox.ini +++ b/tox.ini @@ -29,14 +29,14 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.6.1 - typing-extensions==4.8.0 + mypy==1.8.0 + typing-extensions==4.10.0 types-attrs==19.1.0 - types-lxml==2023.10.21 - types-Pillow==10.1.0.0 - types-Pygments==2.16.0.0 - types-pyOpenSSL==23.3.0.0 - types-setuptools==68.2.0.0 + types-lxml==2024.2.9 + types-Pillow==10.2.0.20240213 + types-Pygments==2.17.0.20240106 + types-pyOpenSSL==24.0.0.20240130 + types-setuptools==69.1.0.20240223 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands = From 4f9dd998dcf01c003bc2a053b6bd78091d12ecd5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 22:01:36 +0500 Subject: [PATCH 1363/2083] Bump pylint, cleanup the ignored tags. --- docs/conf.py | 2 +- pylintrc | 17 +----------- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/http/headers.py | 4 ++- scrapy/utils/ossignal.py | 6 ++++- scrapy/utils/signal.py | 5 +++- tests/test_commands.py | 27 ------------------- tests/test_item.py | 4 ++- tests/test_linkextractors.py | 3 --- tests/test_loader_deprecated.py | 6 ++--- tests/test_settings/__init__.py | 2 +- tests/test_utils_datatypes.py | 2 +- tests/test_utils_signal.py | 6 ----- tests/test_utils_spider.py | 2 +- tox.ini | 2 +- 15 files changed, 25 insertions(+), 65 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9ca0f817a..399078010 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -227,7 +227,7 @@ latex_documents = [ # A list of regular expressions that match URIs that should not be checked when # doing a linkcheck build. linkcheck_ignore = [ - "http://localhost:\d+", + r"http://localhost:\d+", "http://hg.scrapy.org", "http://directory.google.com/", ] diff --git a/pylintrc b/pylintrc index c8654b8d3..78004e78a 100644 --- a/pylintrc +++ b/pylintrc @@ -4,21 +4,14 @@ jobs=1 # >1 hides results [MESSAGES CONTROL] disable=abstract-method, - anomalous-backslash-in-string, arguments-differ, arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, - bad-mcs-classmethod-argument, bare-except, broad-except, broad-exception-raised, c-extension-no-member, - catching-non-exception, - cell-var-from-loop, - comparison-with-callable, - consider-using-dict-items, - consider-using-in, consider-using-with, cyclic-import, dangerous-default-value, @@ -32,7 +25,6 @@ disable=abstract-method, implicit-str-concat, import-error, import-outside-toplevel, - import-self, inconsistent-return-statements, inherit-non-class, invalid-name, @@ -44,7 +36,6 @@ disable=abstract-method, logging-fstring-interpolation, logging-not-lazy, lost-exception, - method-hidden, missing-docstring, no-else-raise, no-else-return, @@ -52,7 +43,7 @@ disable=abstract-method, no-method-argument, no-name-in-module, no-self-argument, - no-value-for-parameter, + no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268 not-callable, pointless-exception-statement, pointless-statement, @@ -77,14 +68,10 @@ disable=abstract-method, too-many-public-methods, too-many-return-statements, unbalanced-tuple-unpacking, - undefined-variable, - undefined-loop-variable, - unexpected-special-method-signature, unnecessary-comprehension, unnecessary-dunder-call, unnecessary-pass, unreachable, - unsubscriptable-object, unused-argument, unused-import, unused-private-member, @@ -92,8 +79,6 @@ disable=abstract-method, unused-wildcard-import, use-dict-literal, used-before-assignment, - useless-object-inheritance, # Required for Python 2 support useless-return, - useless-super-delegation, wildcard-import, wrong-import-position diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 1e340abb6..f0ad24f72 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -169,7 +169,7 @@ class HttpCompressionMiddleware: return to_decode, to_keep def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes: - if encoding == b"gzip" or encoding == b"x-gzip": + if encoding in {b"gzip", b"x-gzip"}: return gunzip(body, max_size=max_size) if encoding == b"deflate": return _inflate(body, max_size=max_size) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 21eb9fb73..73aee7178 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -113,7 +113,9 @@ class Headers(CaselessDict): return ((k, self.getlist(k)) for k in self.keys()) def values(self) -> List[Optional[bytes]]: # type: ignore[override] - return [self[k] for k in self.keys()] + return [ + self[k] for k in self.keys() # pylint: disable=consider-using-dict-items + ] def to_string(self) -> bytes: # cast() can be removed if the headers_dict_to_raw() hint is improved diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index db9a71273..5985a847e 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -24,7 +24,11 @@ def install_shutdown_handlers( (e.g. Pdb) """ signal.signal(signal.SIGTERM, function) - if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: + if ( + signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable + == signal.default_int_handler + or override_sigint + ): signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows if hasattr(signal, "SIGBREAK"): diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index a25100c03..89cfbd2ec 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -98,7 +98,10 @@ def send_catch_log_deferred( robustApply, receiver, signal=signal, sender=sender, *arguments, **named ) d.addErrback(logerror, receiver) - d.addBoth(lambda result: (receiver, result)) + # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html + d.addBoth( + lambda result: (receiver, result) # pylint: disable=cell-var-from-loop + ) dfds.append(d) d = DeferredList(dfds) d.addCallback(lambda out: [x[1] for x in out]) diff --git a/tests/test_commands.py b/tests/test_commands.py index 2f36baa87..febad21da 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -991,38 +991,11 @@ class MySpider(scrapy.Spider): class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = "myspider.pyw" - def setUp(self): - super().setUp() - def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) - def test_run_good_spider(self): - super().test_run_good_spider() - - def test_runspider(self): - super().test_runspider() - - def test_runspider_dnscache_disabled(self): - super().test_runspider_dnscache_disabled() - - def test_runspider_log_level(self): - super().test_runspider_log_level() - - def test_runspider_log_short_names(self): - super().test_runspider_log_short_names() - - def test_runspider_no_spider_found(self): - super().test_runspider_no_spider_found() - - def test_output(self): - super().test_output() - - def test_overwrite_output(self): - super().test_overwrite_output() - def test_runspider_unable_to_load(self): raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") diff --git a/tests/test_item.py b/tests/test_item.py index ce2b4fd15..daf5d4f59 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -290,7 +290,9 @@ class ItemMetaTest(unittest.TestCase): class ItemMetaClassCellRegression(unittest.TestCase): def test_item_meta_classcell_regression(self): class MyItem(Item, metaclass=ItemMeta): - def __init__(self, *args, **kwargs): + def __init__( + self, *args, **kwargs + ): # pylint: disable=useless-parent-delegation # This call to super() trigger the __classcell__ propagation # requirement. When not done properly raises an error: # TypeError: __class__ set to diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 55ea9eed2..6b4df90d8 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -818,9 +818,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ], ) - def test_restrict_xpaths_with_html_entities(self): - super().test_restrict_xpaths_with_html_entities() - @mark.skipif( Version(w3lib_version) < Version("2.0.0"), reason=( diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index d7f773d5c..99cdf88d9 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -678,11 +678,11 @@ class SelectJmesTestCase(unittest.TestCase): } def test_output(self): - for tl in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[tl] + for k, v in self.test_list_equals.items(): + expr, test_list, expected = v test = SelectJmes(expr)(test_list) self.assertEqual( - test, expected, msg=f'test "{tl}" got {test} expected {expected}' + test, expected, msg=f'test "{k}" got {test} expected {expected}' ) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 3fde5e8c5..9ee248538 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -426,7 +426,7 @@ class SettingsTest(unittest.TestCase): mydict = settings.get("TEST_DICT") self.assertIsInstance(mydict, BaseSettings) self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") + self.assertEqual(mydict["key"], "val") # pylint: disable=unsubscriptable-object self.assertEqual(mydict.getpriority("key"), 0) @mock.patch("scrapy.settings.default_settings", default_settings) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 9e5f88f48..be5c6de81 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -353,7 +353,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): for i, r in enumerate(refs): self.assertIn(r, cache) self.assertEqual(cache[r], i) - del r # delete reference to the last object in the list + del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable # delete half of the objects, make sure that is reflected in the cache for _ in range(max // 2): diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 65b99e0c4..60232f10b 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -75,9 +75,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): await defer.succeed(42) return "OK" - def test_send_catch_log(self): - return super().test_send_catch_log() - @mark.only_asyncio() class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): @@ -87,9 +84,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): await asyncio.sleep(0.2) return await get_from_asyncio_queue("OK") - def test_send_catch_log(self): - return super().test_send_catch_log() - class SendCatchLogTest2(unittest.TestCase): def test_error_logged_if_deferred_not_supported(self): diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 460ae40c3..dd1d26448 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -26,7 +26,7 @@ class UtilsSpidersTestCase(unittest.TestCase): self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o]) def test_iter_spider_classes(self): - import tests.test_utils_spider + import tests.test_utils_spider # pylint: disable=import-self it = iter_spider_classes(tests.test_utils_spider) self.assertEqual(set(it), {MySpider1, MySpider2}) diff --git a/tox.ini b/tox.ini index e787c7bf3..4ed9b3bd7 100644 --- a/tox.ini +++ b/tox.ini @@ -53,7 +53,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==3.0.1 + pylint==3.1.0 commands = pylint conftest.py docs extras scrapy setup.py tests From 63acd0720970c87450fdbcb9aa6967118c9c1cf2 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 28 Feb 2024 16:14:08 -0300 Subject: [PATCH 1364/2083] Fix and re-enable unnecessary-comprehension and use-dict-literal pylint tags --- pylintrc | 2 -- scrapy/downloadermiddlewares/httpcompression.py | 2 +- scrapy/spiders/crawl.py | 2 +- scrapy/utils/python.py | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pylintrc b/pylintrc index 78004e78a..c60e4e16a 100644 --- a/pylintrc +++ b/pylintrc @@ -68,7 +68,6 @@ disable=abstract-method, too-many-public-methods, too-many-return-statements, unbalanced-tuple-unpacking, - unnecessary-comprehension, unnecessary-dunder-call, unnecessary-pass, unreachable, @@ -77,7 +76,6 @@ disable=abstract-method, unused-private-member, unused-variable, unused-wildcard-import, - use-dict-literal, used-before-assignment, useless-return, wildcard-import, diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index f0ad24f72..aa3abe853 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -135,7 +135,7 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs = dict(cls=respcls, body=decoded_body) + kwargs = {"cls": respcls, "body": decoded_body} if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index ebb4f5984..2a3913da5 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -85,7 +85,7 @@ class CrawlSpider(Spider): url=link.url, callback=self._callback, errback=self._errback, - meta=dict(rule=rule_index, link_text=link.text), + meta={"rule": rule_index, "link_text": link.text}, ) def _requests_to_follow(self, response): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7b408c49c..1e7364e49 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -162,7 +162,7 @@ def re_rsearch( pattern = re.compile(pattern) for chunk, offset in _chunk_iter(): - matches = [match for match in pattern.finditer(chunk)] + matches = list(pattern.finditer(chunk)) if matches: start, end = matches[-1].span() return offset + start, offset + end From 26a16f2c43dc96fe33d0b0fc8846402e9ae97e9a Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 28 Feb 2024 16:36:19 -0300 Subject: [PATCH 1365/2083] Fix tests --- tests/test_crawl.py | 10 ++--- tests/test_downloadermiddleware_cookies.py | 2 +- tests/test_downloadermiddleware_httpauth.py | 4 +- tests/test_exporters.py | 24 +++++------ tests/test_linkextractors.py | 38 ++++++++-------- tests/test_loader_deprecated.py | 20 ++++----- tests/test_mail.py | 2 +- tests/test_pipeline_crawl.py | 2 +- tests/test_pipeline_files.py | 2 +- tests/test_pipeline_images.py | 18 ++++---- tests/test_pipeline_media.py | 48 ++++++++++----------- tests/test_scheduler.py | 26 +++++------ tests/test_spidermiddleware_offsite.py | 17 +++++--- tests/test_utils_iterators.py | 16 +++---- tests/test_utils_template.py | 2 +- 15 files changed, 117 insertions(+), 114 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 96d43b2b9..6cde4ed8c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -76,11 +76,11 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def _test_delay(self, total, delay, randomize=False): - crawl_kwargs = dict( - maxlatency=delay * 2, - mockserver=self.mockserver, - total=total, - ) + crawl_kwargs = { + "maxlatency": delay * 2, + "mockserver": self.mockserver, + "total": total, + } tolerance = 1 - (0.6 if randomize else 0.2) settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize} diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 4a81a638e..425fabcc7 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -320,7 +320,7 @@ class CookiesMiddlewareTest(TestCase): @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): - DEFAULT_REQUEST_HEADERS = dict(Cookie="default=value; asdf=qwerty") + DEFAULT_REQUEST_HEADERS = {"Cookie": "default=value; asdf=qwerty"} mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) # overwrite with values from 'cookies' request argument req1 = Request("http://example.org", cookies={"default": "something"}) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index fc110e6cc..500af6536 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -59,7 +59,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase): self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) def test_auth_already_set(self): - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) + req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers["Authorization"], b"Digest 123") @@ -79,6 +79,6 @@ class HttpAuthAnyMiddlewareTest(unittest.TestCase): self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) def test_auth_already_set(self): - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) + req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers["Authorization"], b"Digest 123") diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 59b724495..fa9389044 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -152,7 +152,7 @@ class PythonItemExporterTest(BaseItemExporterTest): def test_nested_item(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=i1) + i2 = {"name": "Maria", "age": i1} i3 = self.item_class(name="Jesus", age=i2) ie = self._get_exporter() exported = ie.export_item(i3) @@ -185,7 +185,7 @@ class PythonItemExporterTest(BaseItemExporterTest): def test_export_item_dict_list(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=[i1]) + i2 = {"name": "Maria", "age": [i1]} i3 = self.item_class(name="Jesus", age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) @@ -373,7 +373,7 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_join_multivalue_not_strings(self): self.assertExportResult( - item=dict(name="John", friends=[4, 8]), + item={"name": "John", "friends": [4, 8]}, include_headers_line=False, expected='"[4, 8]",John\r\n', ) @@ -388,14 +388,14 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_errors_default(self): with self.assertRaises(UnicodeEncodeError): self.assertExportResult( - item=dict(text="W\u0275\u200Brd"), + item={"text": "W\u0275\u200Brd"}, expected=None, encoding="windows-1251", ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item=dict(text="W\u0275\u200Brd"), + item={"text": "W\u0275\u200Brd"}, include_headers_line=False, expected="Wɵ​rd\r\n", encoding="windows-1251", @@ -455,8 +455,8 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_item(self): - i1 = dict(name="foo\xa3hoo", age="22") - i2 = dict(name="bar", age=i1) + i1 = {"name": "foo\xa3hoo", "age": "22"} + i2 = {"name": "bar", "age": i1} i3 = self.item_class(name="buz", age=i2) self.assertExportResult( @@ -478,8 +478,8 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_list_item(self): - i1 = dict(name="foo") - i2 = dict(name="bar", v2={"egg": ["spam"]}) + i1 = {"name": "foo"} + i2 = {"name": "bar", "v2": {"egg": ["spam"]}} i3 = self.item_class(name="buz", age=[i1, i2]) self.assertExportResult( @@ -534,7 +534,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): def test_nested_item(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=i1) + i2 = {"name": "Maria", "age": i1} i3 = self.item_class(name="Jesus", age=i2) self.ie.start_exporting() self.ie.export_item(i3) @@ -622,9 +622,9 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertEqual(exported, [expected]) def test_nested_dict_item(self): - i1 = dict(name="Joseph\xa3", age="22") + i1 = {"name": "Joseph\xa3", "age": "22"} i2 = self.item_class(name="Maria", age=i1) - i3 = dict(name="Jesus", age=i2) + i3 = {"name": "Jesus", "age": i2} self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 6b4df90d8..217c7a299 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -37,7 +37,7 @@ class Base: page4_url = "http://example.com/page%204.html" self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -55,7 +55,7 @@ class Base: def test_extract_filter_allow(self): lx = self.extractor_cls(allow=("sample",)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -70,7 +70,7 @@ class Base: def test_extract_filter_allow_with_duplicates(self): lx = self.extractor_cls(allow=("sample",), unique=False) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -93,7 +93,7 @@ class Base: def test_extract_filter_allow_with_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=False, canonicalize=True) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -116,7 +116,7 @@ class Base: def test_extract_filter_allow_no_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=True, canonicalize=True) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -127,7 +127,7 @@ class Base: def test_extract_filter_allow_and_deny(self): lx = self.extractor_cls(allow=("sample",), deny=("3",)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -137,7 +137,7 @@ class Base: def test_extract_filter_allowed_domains(self): lx = self.extractor_cls(allow_domains=("google.com",)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://www.google.com/something", text=""), ], @@ -148,7 +148,7 @@ class Base: lx = self.extractor_cls(allow="sample") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -162,7 +162,7 @@ class Base: lx = self.extractor_cls(allow="sample", deny="3") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -171,7 +171,7 @@ class Base: lx = self.extractor_cls(allow_domains="google.com") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://www.google.com/something", text=""), ], @@ -179,7 +179,7 @@ class Base: lx = self.extractor_cls(deny_domains="example.com") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://www.google.com/something", text=""), ], @@ -265,7 +265,7 @@ class Base: def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -337,7 +337,7 @@ class Base: restrict_css=("#subwrapper + a",), ) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -705,7 +705,7 @@ class Base: response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item1.html", @@ -758,7 +758,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item1.html", text="Item 1", nofollow=False @@ -779,7 +779,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Simple text inclusion test lx = self.extractor_cls(restrict_text="dog") self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item2.html", @@ -791,7 +791,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Unique regex test lx = self.extractor_cls(restrict_text=r"of.*dog") self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item2.html", @@ -803,7 +803,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Multiple regex test lx = self.extractor_cls(restrict_text=[r"of.*dog", r"of.*cat"]) self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item1.html", @@ -834,7 +834,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item2.html", diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 99cdf88d9..528efa142 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -565,37 +565,37 @@ class NoInputReprocessingFromDictTest(unittest.TestCase): """ def test_avoid_reprocessing_with_initial_values_single(self): - il = NoInputReprocessingDictLoader(item=dict(title="foo")) + il = NoInputReprocessingDictLoader(item={"title": "foo"}) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="foo")) + self.assertEqual(il_loaded, {"title": "foo"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} ) def test_avoid_reprocessing_with_initial_values_list(self): - il = NoInputReprocessingDictLoader(item=dict(title=["foo", "bar"])) + il = NoInputReprocessingDictLoader(item={"title": ["foo", "bar"]}) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="foo")) + self.assertEqual(il_loaded, {"title": "foo"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} ) def test_avoid_reprocessing_without_initial_values_single(self): il = NoInputReprocessingDictLoader() il.add_value("title", "foo") il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="FOO")) + self.assertEqual(il_loaded, {"title": "FOO"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} ) def test_avoid_reprocessing_without_initial_values_list(self): il = NoInputReprocessingDictLoader() il.add_value("title", ["foo", "bar"]) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="FOO")) + self.assertEqual(il_loaded, {"title": "FOO"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} ) diff --git a/tests/test_mail.py b/tests/test_mail.py index 2535e58db..ff1505397 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -91,7 +91,7 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(attach.get_payload(decode=True), b"content") def _catch_mail_sent(self, **kwargs): - self.catched_msg = dict(**kwargs) + self.catched_msg = {**kwargs} def test_send_utf8(self): subject = "sübjèçt" diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index be9811980..5a9a217ce 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -140,7 +140,7 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(logs.count(file_dl_failure), 3) # check that no files were written to the media store - self.assertEqual([x for x in self.tmpmediastore.iterdir()], []) + self.assertEqual(list(self.tmpmediastore.iterdir()), []) @defer.inlineCallbacks def test_download_media(self): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e7000e314..0babde4d9 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -221,7 +221,7 @@ class FilesPipelineTestCase(unittest.TestCase): file_path = CustomFilesPipeline.from_settings( Settings({"FILES_STORE": self.tempdir}) ).file_path - item = dict(path="path-to-store-file") + item = {"path": "path-to-store-file"} request = Request("http://example.com") self.assertEqual(file_path(request, item=item), "full/path-to-store-file") diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 2e2e06b89..18a2454b3 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -132,7 +132,7 @@ class ImagesPipelineTestCase(unittest.TestCase): thumb_path = CustomImagesPipeline.from_settings( Settings({"IMAGES_STORE": self.tempdir}) ).thumb_path - item = dict(path="path-to-store-file") + item = {"path": "path-to-store-file"} request = Request("http://example.com") self.assertEqual( thumb_path(request, "small", item=item), "thumb/small/path-to-store-file" @@ -433,14 +433,14 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): ] # This should match what is defined in ImagesPipeline. - default_pipeline_settings = dict( - MIN_WIDTH=0, - MIN_HEIGHT=0, - EXPIRES=90, - THUMBS={}, - IMAGES_URLS_FIELD="image_urls", - IMAGES_RESULT_FIELD="images", - ) + default_pipeline_settings = { + "MIN_WIDTH": 0, + "MIN_HEIGHT": 0, + "EXPIRES": 90, + "THUMBS": {}, + "IMAGES_URLS_FIELD": "image_urls", + "IMAGES_RESULT_FIELD": "images", + } def setUp(self): self.tempdir = mkdtemp() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d477b59be..d4dde4a40 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -59,7 +59,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert self.pipe.media_to_download(request, self.info) is None def test_default_get_media_requests(self): - item = dict(name="name") + item = {"name": "name"} assert self.pipe.get_media_requests(item, self.info) is None def test_default_media_downloaded(self): @@ -73,7 +73,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert self.pipe.media_failed(fail, request, self.info) is fail def test_default_item_completed(self): - item = dict(name="name") + item = {"name": "name"} assert self.pipe.item_completed([], item, self.info) is item # Check that failures are logged by default @@ -98,7 +98,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): @inlineCallbacks def test_default_process_item(self): - item = dict(name="name") + item = {"name": "name"} new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item @@ -226,11 +226,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): rsp = Response("http://url1") req = Request( "http://url1", - meta=dict(response=rsp), + meta={"response": rsp}, callback=self._callback, errback=self._errback, ) - item = dict(requests=req) + item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, rsp)]) self.assertEqual( @@ -250,11 +250,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): fail = Failure(Exception()) req = Request( "http://url1", - meta=dict(response=fail), + meta={"response": fail}, callback=self._callback, errback=self._errback, ) - item = dict(requests=req) + item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(False, fail)]) self.assertEqual( @@ -272,10 +272,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def test_mix_of_success_and_failure(self): self.pipe.LOG_FAILED_RESULTS = False rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) + req1 = Request("http://url1", meta={"response": rsp1}) fail = Failure(Exception()) - req2 = Request("http://url2", meta=dict(response=fail)) - item = dict(requests=[req1, req2]) + req2 = Request("http://url2", meta={"response": fail}) + item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)]) m = self.pipe._mockcalled @@ -294,7 +294,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def test_get_media_requests(self): # returns single Request (without callback) req = Request("http://url") - item = dict(requests=req) # pass a single item + item = {"requests": req} # pass a single item new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item self.assertIn(self.fingerprint(req), self.info.downloaded) @@ -302,7 +302,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): # returns iterable of Requests req1 = Request("http://url1") req2 = Request("http://url2") - item = dict(requests=iter([req1, req2])) + item = {"requests": iter([req1, req2])} new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item assert self.fingerprint(req1) in self.info.downloaded @@ -311,17 +311,17 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): @inlineCallbacks def test_results_are_cached_across_multiple_items(self): rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) - item = dict(requests=req1) + req1 = Request("http://url1", meta={"response": rsp1}) + item = {"requests": req1} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(new_item["results"], [(True, rsp1)]) # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same req2 = Request( - req1.url, meta=dict(response=Response("http://donot.download.me")) + req1.url, meta={"response": Response("http://donot.download.me")} ) - item = dict(requests=req2) + item = {"requests": req2} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) @@ -330,11 +330,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): @inlineCallbacks def test_results_are_cached_for_requests_of_single_item(self): rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) + req1 = Request("http://url1", meta={"response": rsp1}) req2 = Request( - req1.url, meta=dict(response=Response("http://donot.download.me")) + req1.url, meta={"response": Response("http://donot.download.me")} ) - item = dict(requests=[req1, req2]) + item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) @@ -359,16 +359,16 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def rsp2_func(): self.fail("it must cache rsp1 result and must not try to redownload") - req1 = Request("http://url", meta=dict(response=rsp1_func)) - req2 = Request(req1.url, meta=dict(response=rsp2_func)) - item = dict(requests=[req1, req2]) + req1 = Request("http://url", meta={"response": rsp1_func}) + req2 = Request(req1.url, meta={"response": rsp2_func}) + item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) @inlineCallbacks def test_use_media_to_download_result(self): - req = Request("http://url", meta=dict(result="ITSME", response=self.fail)) - item = dict(requests=req) + req = Request("http://url", meta={"result": "ITSME", "response": self.fail}) + item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, "ITSME")]) self.assertEqual( diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index f8465a5ff..37099dae6 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -45,15 +45,15 @@ class MockDownloader: class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): - settings = dict( - SCHEDULER_DEBUG=False, - SCHEDULER_DISK_QUEUE="scrapy.squeues.PickleLifoDiskQueue", - SCHEDULER_MEMORY_QUEUE="scrapy.squeues.LifoMemoryQueue", - SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, - JOBDIR=jobdir, - DUPEFILTER_CLASS="scrapy.dupefilters.BaseDupeFilter", - REQUEST_FINGERPRINTER_IMPLEMENTATION="2.7", - ) + settings = { + "SCHEDULER_DEBUG": False, + "SCHEDULER_DISK_QUEUE": "scrapy.squeues.PickleLifoDiskQueue", + "SCHEDULER_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue", + "SCHEDULER_PRIORITY_QUEUE": priority_queue_cls, + "JOBDIR": jobdir, + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) self.stats = load_object(self.settings["STATS_CLASS"])(self) @@ -338,10 +338,10 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase): class TestIncompatibility(unittest.TestCase): def _incompatible(self): - settings = dict( - SCHEDULER_PRIORITY_QUEUE="scrapy.pqueues.DownloaderAwarePriorityQueue", - CONCURRENT_REQUESTS_PER_IP=1, - ) + settings = { + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue", + "CONCURRENT_REQUESTS_PER_IP": 1, + } crawler = get_crawler(Spider, settings) scheduler = Scheduler.from_crawler(crawler) spider = Spider(name="spider") diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index ea45b7698..837f1c2c8 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -16,10 +16,10 @@ class TestOffsiteMiddleware(TestCase): self.mw.spider_opened(self.spider) def _get_spiderargs(self): - return dict( - name="foo", - allowed_domains=["scrapytest.org", "scrapy.org", "scrapy.test.org"], - ) + return { + "name": "foo", + "allowed_domains": ["scrapytest.org", "scrapy.org", "scrapy.test.org"], + } def test_process_spider_output(self): res = Response("http://scrapytest.org") @@ -50,7 +50,7 @@ class TestOffsiteMiddleware(TestCase): class TestOffsiteMiddleware2(TestOffsiteMiddleware): def _get_spiderargs(self): - return dict(name="foo", allowed_domains=None) + return {"name": "foo", "allowed_domains": None} def test_process_spider_output(self): res = Response("http://scrapytest.org") @@ -61,13 +61,16 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): class TestOffsiteMiddleware3(TestOffsiteMiddleware2): def _get_spiderargs(self): - return dict(name="foo") + return {"name": "foo"} class TestOffsiteMiddleware4(TestOffsiteMiddleware3): def _get_spiderargs(self): bad_hostname = urlparse("http:////scrapytest.org").hostname - return dict(name="foo", allowed_domains=["scrapytest.org", None, bad_hostname]) + return { + "name": "foo", + "allowed_domains": ["scrapytest.org", None, bad_hostname], + } def test_process_spider_output(self): res = Response("http://scrapytest.org") diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ee22e6675..ec377bb19 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -355,7 +355,7 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - result = [row for row in csv] + result = list(csv) self.assertEqual( result, [ @@ -377,7 +377,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter="\t") self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -394,7 +394,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv1 = csviter(response1, quotechar="'") self.assertEqual( - [row for row in csv1], + list(csv1), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -407,7 +407,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv2 = csviter(response2, delimiter="|", quotechar="'") self.assertEqual( - [row for row in csv2], + list(csv2), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -422,7 +422,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual( - [row for row in csv], + list(csv), [ {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, { @@ -441,7 +441,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter="\t") self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -458,7 +458,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, headers=[h.decode("utf-8") for h in headers]) self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -475,7 +475,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index cbe80e157..fc42c0d2f 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -16,7 +16,7 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): rmtree(self.tmp_path) def test_simple_render(self): - context = dict(project_name="proj", name="spi", classname="TheSpider") + context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"} template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" From 706eb8d4275be993867122e5e41c31321488309e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Feb 2024 14:33:55 +0500 Subject: [PATCH 1366/2083] Fix a merge error. --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index aebdfb3e4..2352be0fe 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -135,7 +135,7 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs: Dict[str, Any] = {"cls": respcls, "body": decoded_body} + kwargs: Dict[str, Any] = {"body": decoded_body} if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable From 2169810414a700fcbfe33eafe1e85e46e7f62413 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 29 Feb 2024 06:41:14 -0300 Subject: [PATCH 1367/2083] fix: Proxy tests don't use custom certificate authority --- tests/test_proxy_connect.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 46d42e9f6..93f006c76 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -27,7 +27,7 @@ from mitmproxy.tools.main import mitmdump sys.argv[0] = "mitmdump" sys.exit(mitmdump()) """ - cert_path = Path(__file__).parent.resolve() / "keys" / "mitmproxy-ca.pem" + cert_path = Path(__file__).parent.resolve() / "keys" self.proc = Popen( [ sys.executable, @@ -40,8 +40,8 @@ sys.exit(mitmdump()) "0", "--proxyauth", f"{self.auth_user}:{self.auth_pass}", - "--certs", - str(cert_path), + "--set", + f"confdir={cert_path}", "--ssl-insecure", ], stdout=PIPE, From 2bfd9a2257c79ae56955e95b46f2bc7b23e1eabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 29 Feb 2024 11:11:42 +0100 Subject: [PATCH 1368/2083] bandit: allow-list false positives --- .bandit.yml | 11 +---------- scrapy/commands/bench.py | 6 ++++-- scrapy/commands/edit.py | 2 +- scrapy/commands/genspider.py | 2 +- scrapy/core/downloader/__init__.py | 2 +- scrapy/exporters.py | 4 ++-- scrapy/extensions/httpcache.py | 6 +++--- scrapy/extensions/spiderstate.py | 4 ++-- scrapy/settings/default_settings.py | 2 +- scrapy/shell.py | 2 +- scrapy/squeues.py | 2 +- scrapy/utils/benchserver.py | 2 +- scrapy/utils/engine.py | 2 +- 13 files changed, 20 insertions(+), 27 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 6e8331c0f..4fcd75c57 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,19 +1,10 @@ skips: -- B101 # assert_used -- B105 # hardcoded_password_string -- B301 # pickle -- B307 # eval -- B311 # random +- B101 # assert_used, needed for mypy - B320 # xml_bad_etree - B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 - B324 # hashlib "Use of weak SHA1 hash for security" - B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B403 # import_pickle -- B404 # import_subprocess -- B406 # import_xml_sax - B410 # import_lxml - B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 - B503 # ssl_with_bad_defaults -- B603 # subprocess_without_shell_equals_true -- B605 # start_process_with_a_shell exclude_dirs: ['tests'] diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index e1ccdc451..aaf5a439f 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,4 +1,4 @@ -import subprocess +import subprocess # nosec import sys import time from urllib.parse import urlencode @@ -29,7 +29,9 @@ class _BenchServer: from scrapy.utils.test import get_testenv pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] - self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv()) + self.proc = subprocess.Popen( + pargs, stdout=subprocess.PIPE, env=get_testenv() + ) # nosec self.proc.stdout.readline() def __exit__(self, exc_type, exc_value, traceback): diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 03a8ed5c7..e85d2c9ec 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -37,4 +37,4 @@ class Command(ScrapyCommand): sfile = sys.modules[spidercls.__module__].__file__ sfile = sfile.replace(".pyc", ".py") - self.exitcode = os.system(f'{editor} "{sfile}"') + self.exitcode = os.system(f'{editor} "{sfile}"') # nosec diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 68cbe8ff6..567ebcdc0 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -113,7 +113,7 @@ class Command(ScrapyCommand): if template_file: self._genspider(module, name, url, opts.template, template_file) if opts.edit: - self.exitcode = os.system(f'scrapy edit "{name}"') + self.exitcode = os.system(f'scrapy edit "{name}"') # nosec def _genspider(self, module, name, url, template_name, template_file): """Generate the spider module, based on the given template""" diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index c84525160..666282856 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -40,7 +40,7 @@ class Slot: def download_delay(self) -> float: if self.randomize_delay: - return random.uniform(0.5 * self.delay, 1.5 * self.delay) + return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosec return self.delay def close(self) -> None: diff --git a/scrapy/exporters.py b/scrapy/exporters.py index f85f1dad8..79fd4e56f 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -5,10 +5,10 @@ Item Exporters are used to export/serialize items into different formats. import csv import io import marshal -import pickle +import pickle # nosec import pprint from collections.abc import Mapping -from xml.sax.saxutils import XMLGenerator +from xml.sax.saxutils import XMLGenerator # nosec from itemadapter import ItemAdapter, is_item diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 7e4f047a8..335728502 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,6 +1,6 @@ import gzip import logging -import pickle +import pickle # nosec from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path @@ -274,7 +274,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return # expired - return pickle.loads(db[f"{key}_data"]) + return pickle.loads(db[f"{key}_data"]) # nosec class FilesystemCacheStorage: @@ -352,7 +352,7 @@ class FilesystemCacheStorage: if 0 < self.expiration_secs < time() - mtime: return # expired with self._open(metapath, "rb") as f: - return pickle.load(f) + return pickle.load(f) # nosec def parse_cachecontrol(header): diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 929a3be70..43359401b 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,4 +1,4 @@ -import pickle +import pickle # nosec from pathlib import Path from scrapy import signals @@ -31,7 +31,7 @@ class SpiderState: def spider_opened(self, spider): if self.jobdir and Path(self.statefn).exists(): with Path(self.statefn).open("rb") as f: - spider.state = pickle.load(f) + spider.state = pickle.load(f) # nosec else: spider.state = {} diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 49ab1b5ef..2b3d95a0e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -177,7 +177,7 @@ FILES_STORE_S3_ACL = "private" FILES_STORE_GCS_ACL = "" FTP_USER = "anonymous" -FTP_PASSWORD = "guest" +FTP_PASSWORD = "guest" # nosec FTP_PASSIVE_MODE = True GCS_PROJECT_ID = None diff --git a/scrapy/shell.py b/scrapy/shell.py index 05909977a..63ea33892 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -50,7 +50,7 @@ class Shell: else: self.populate_vars() if self.code: - print(eval(self.code, globals(), self.vars)) + print(eval(self.code, globals(), self.vars)) # nosec else: """ Detect interactive shell setting in scrapy.cfg diff --git a/scrapy/squeues.py b/scrapy/squeues.py index f665ad88c..e20f60f06 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -3,7 +3,7 @@ Scheduler queues """ import marshal -import pickle +import pickle # nosec from os import PathLike from pathlib import Path from typing import Union diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 38884a9f0..f6f704d4b 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -14,7 +14,7 @@ class Root(Resource): def render(self, request): total = _getarg(request, b"total", 100, int) show = _getarg(request, b"show", 10, int) - nlist = [random.randint(1, total) for _ in range(show)] + nlist = [random.randint(1, total) for _ in range(show)] # nosec request.write(b"") args = request.args.copy() for nl in nlist: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index a5f2a8c6e..0b2722663 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -30,7 +30,7 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: checks: List[Tuple[str, Any]] = [] for test in tests: try: - checks += [(test, eval(test))] + checks += [(test, eval(test))] # nosec except Exception as e: checks += [(test, f"{type(e).__name__} (exception)")] From 31cbbb57584fe2a7c42d30acf2aa4707039457b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 29 Feb 2024 11:31:39 +0100 Subject: [PATCH 1369/2083] bandit: ignore md5 usage for download slot names --- scrapy/pqueues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index b62d2fe58..593667f1f 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -20,7 +20,7 @@ def _path_safe(text): pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part - unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() + unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # nosec return "-".join([pathable_slot, unique_slot]) From 032e6a091a27b406aa48293f752d4782f8cac159 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Feb 2024 16:24:52 +0500 Subject: [PATCH 1370/2083] Reformat the new changes with new black. --- scrapy/http/request/json_request.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 5c09835e4..59b11c692 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -53,12 +53,10 @@ class JsonRequest(Request): @overload def replace( self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any - ) -> RequestTypeVar: - ... + ) -> RequestTypeVar: ... @overload - def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: - ... + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any From 1311e7db05204fe2cae7d1c5caf8b0ffe9371cd0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Feb 2024 16:31:06 +0500 Subject: [PATCH 1371/2083] Regenerate the expired mitmproxy-ca.pem. --- tests/keys/mitmproxy-ca.pem | 93 ++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 48 deletions(-) diff --git a/tests/keys/mitmproxy-ca.pem b/tests/keys/mitmproxy-ca.pem index cdef75f99..61a690cc8 100644 --- a/tests/keys/mitmproxy-ca.pem +++ b/tests/keys/mitmproxy-ca.pem @@ -1,50 +1,47 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCYp6U4G9YWITYB -/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7ZDiT -NUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMgz1BJ -u6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniIggUH -JrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE6HFB -eIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/0zls -90iyQ3E/AgMBAAECggEBAJA1dyAdM85uC04vKVNUJM1GDp0xS+0syBReJaKRI3nJ -epoCj+RqxGag1pdaYLI0G84NTPqECz9LOyLdqpPgEfKRIxWlf9oWmSnfnXskArd8 -VfVcWYl6tEPv1TToTZIBmCbYLBFVbLxG/GrbK6uokdhUsqbdXwEKok2IEaSTRlDn -v8BVXte00d9VEKKpmI6EY3f45uPQPHuJNcitP2HGW1mT/C6XoZR6wj+VvoRgUGQT -I7PuktbYpQlLV+oX0uZz9frPGhjydUq0Jti5v3QAJEb+7D0cKrkZW+7fYDx4YkRU -oDiuWEyO2kfpff52Qxs+xUXMiAyw6/8+TamKoAi1TIECgYEAyAzoztW6W4CjL2au -/hN5VmbAvuBxq1m1G5KgXM1myX9V2CgH6OKwzJQNSCEfKMNOjqxB99T7C3tMCjgG -gmbUzylTeciQFF+crrl2Rn/6qZS9dCo1hagb3K5eXMhLXoP425Y4sypNPPqULhPn -YrUDFNAf89rRLqP1KMPLZ+uO7EECgYEAw1lWPxGV+X85iQxYN9xoX85htfJSBXTf -dLirQ4bkykOxSA6ZzFuhDO/G373Q1rze4tmEO790uOCeaiXGgeWC1A+2PMO957i5 -9FqhDIkmerfdIttdEUMM9rQwuTcLnixGZkT5GHDzjtNinaIVB+pv7twRAESqN9dC -QXh7IF7g/X8CgYBMhQOX+hCqZ24D95cAAJrs/ajEWj2geVPZFCDa3oZulJJVeBpu -bieKWScra9/rS6mE0Ub6cTEFl0fisMNspcDI7NnNP3Y9FMVt3+rp1JIgw5AkGvEW -CtN9egUGIGcT5A8Qj0lo3slkhcSgS2S6UNq431MZh51z5askyJ/JREULAQKBgFrR -OatwfYzUfOcd+hVePpfr1rlDwqYOw6P8BoMKP2tZNR4Oy6maH7Fn98kk8eYjQGuu -PC+avqUEqCEpFrRlAwGbnFl7ltoXozvatmyhhmYe/Iur+ASCa5B2DQDOenQ6mTAK -eNPIDzMjSwGFzMk1UHx3it/ZDFmRlZfibzuJYIf5AoGBAIaPHk4qadK/XpcD4Wwx -BOsDEIz27DGWdwWfd5r3EcV4zX/wNzH0G1Z8eydNjUqKzufMZgFwpcTu0Evesl1/ -B8kC8sLHxQoG5SvBu4dBxMwKIU9O9uFnX5SUYZUDpCtUYyZ+GtGom41Jwg5ENrwy -HzPh2taMnCA0h1fNLFFBkw88 ------END PRIVATE KEY----- +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAuq7ujTpHoSwQn9/hFJT837jU/T7xLuyXjkAEfL7uVDuPWSdF +AJy+QJsuL6INMKMlxLUb1RRzxQgAmtYN1dIEbTPplffCNbfYm3FXg0mZlxg1UBg9 +rE1bPwuz/B+M76S35EIiKQlpaCFErLQi5oyhw9FIBvYLZxxgfeDfDPiXQlBvtHix +n9TFqNoLNZkAX+auh2Wj2SSjM74pBQWsuVZLkF5CAwYuMQkpEplCV/QHNX3ZeNdQ +YNFvpA7CxENa3sTZyHpeTmWoOdJXgJFveAWL4ZhSvkSd0HDuPPJp4JckETNESa9B +qOwFfj36SM+5dRiCwiuzwAQ+oaFjisMXuRbVFQIDAQABAoIBABqFaJmCupNgnboA +xcq1QdmMuiGCNCRs8zj/ykNoopYv4fUR+aEVI4gtI5obxRDwVJjF+/7BCZNnyCI3 +H78NN5jGA7zM9nfINwsaRor9xUasZ0KKNxTH5pslz/uVBeIzvfY9GPpIfoOPGmEI +tF6Zgw+9JyTqBoOvCdxIOpfupxqB8TQ0z4UbFUuBiEkGuJ+o8C1rX8Wze0JUl0qG +BOwhQtaCn/yrm+dTXZ5XaelJY5mcwgFy22Jiynmm6TbLhyZlACd2Q/MGak7o1TJL +QgvvGMlcVrK7MZ3TJN+wzwWfwAAjXnT3Xvd5pD5yunZJoNe8YyFOCMlh0swNG5Zt +0tGeX78CgYEA+m6gYGKTNWFnqlqMZRfGTqiqVZeVQKftcLdA1dkscnffRP9bvKOW +9TbgzoGHiyZnjZBDFTUuy67El8RXIMsxYy3GYuGRdUSLS63Fr2af4pBQIYvW6OmG +UZlcAP6ZAhUzn409XGlXaac3F30hFeKdC62+V5ZMnfPlVhHRCoKaaqsCgYEAvtV4 +FuU5sFKyhKPPV3rzaNZtL0swvtBIuODH1oAWhPNySQvCu+45W0EOOAPPpsYP4wGX +G+otOSp4RLdlVXNhkh1rpJzeK77KZ5ZY+ShkuHD+uL/iRARwl/gh2Ve1aqUrm1LE +9ldchmQGvLalN9HalzeW//xHA3X9SF4Vo16Dvz8CgYABeZlUOABp9hLoO/RLvCIc +4H1wV543bUXGvi2RlN/gJLiZ7W8a41PGSfZ1AOpNdYJyoQDkJRYLeRILWsqwlMHL +tb9PYci7ihXP8kwRxmb2rKbsK6iuYoG6BU83akh4bKuLKwfLfYtYQfXfG4uQV29Z +XEKcvXPiEkethBlZGH/UVQKBgCa9Pvum3OcmYob6mgSwOOl3XgLTyLlzns+pEehB +aFDk+rZJZOaxnYMg2boVS/oXCvKSSBKqnzOTo4aPlEqceZonzspD7fYDbSNKKhWq +VYf7qDno+g3EuPagsH5mh7V2gjutub4oTegaNiPpD/Ec8Lrx1f1xQRk6wogGUW4w +qZ4RAoGBAPg1LezV8mlesF5mhj+KubYP4l1Zf9geAeQprjDbFsA0BEAS2KsWgmwR +Ye1fmek7jDjCPLQ4Amq030mLJuQGEM3cZPqjKX2sBZ8fQcgw7pWJWMvKMTBA4Aah +zQx1KXwHJANMWq/0QSFDq/LGJ2OYMlV2F0tH3P5Kp7ZASTyc78ux +-----END RSA PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIDoTCCAomgAwIBAgIGDodLQx9+MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM -CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTIwMDgxMjE3MDMyNloX -DTIzMDgxNDE3MDMyNlowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt -aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYp6U4G9YW -ITYB/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7 -ZDiTNUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMg -z1BJu6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniI -ggUHJrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE -6HFBeIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/ -0zls90iyQ3E/AgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC -AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD -BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG -CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC -AQYwHQYDVR0OBBYEFBCsLPpFz3l9rOOfGmfs+VRc3jhJMA0GCSqGSIb3DQEBCwUA -A4IBAQADTpA15na6U5qqDCe0rr39fkS1/dY804Xnz7g/L3AsxPE1KOMijuJa8sKd -kKwba1173FwMupfK39zY8jUxL8Qprdi92RO6CpoFUsL/icpA///lYhzUSqt32qwe -gRNW3mtYBimOk6KH1NOfQnJolWpJh+g1OEsitQKEeKwIn5Hz+8/yS5tbwLgdnMlY -1/it1H70JSdE7nfJueqN4cFfBsm6XaHZzacJJmN7WP88fd+zztnSQsBFbLlnjnqj -envCDIwCrMywKNMqEBMwmBEGSAF47fVNYj6KzDAtMvBdDkYaHWpBf4tnFfk6v0wj -wiKjdLjCmJgjGAQjRw5VYJ8JI0XO +MIIDNTCCAh2gAwIBAgIUcGDiCmOuhfxMGFS/otcGGFkOSAEwDQYJKoZIhvcNAQEL +BQAwKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAltaXRtcHJveHkwHhcN +MjQwMjI3MTMwNTQ4WhcNMzQwMjI2MTMwNTQ4WjAoMRIwEAYDVQQDDAltaXRtcHJv +eHkxEjAQBgNVBAoMCW1pdG1wcm94eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBALqu7o06R6EsEJ/f4RSU/N+41P0+8S7sl45ABHy+7lQ7j1knRQCcvkCb +Li+iDTCjJcS1G9UUc8UIAJrWDdXSBG0z6ZX3wjW32JtxV4NJmZcYNVAYPaxNWz8L +s/wfjO+kt+RCIikJaWghRKy0IuaMocPRSAb2C2ccYH3g3wz4l0JQb7R4sZ/Uxaja +CzWZAF/mrodlo9kkozO+KQUFrLlWS5BeQgMGLjEJKRKZQlf0BzV92XjXUGDRb6QO +wsRDWt7E2ch6Xk5lqDnSV4CRb3gFi+GYUr5EndBw7jzyaeCXJBEzREmvQajsBX49 ++kjPuXUYgsIrs8AEPqGhY4rDF7kW1RUCAwEAAaNXMFUwDwYDVR0TAQH/BAUwAwEB +/zATBgNVHSUEDDAKBggrBgEFBQcDATAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FOjFT0G7itqsrCij2InhRSfB0sEkMA0GCSqGSIb3DQEBCwUAA4IBAQCVMa5/xlH4 +GUbrWNMdxr9LL7Dh+vK0wYCfAsc/kO2zCq8iVt/MaqVLel/bKcQhvE5RZHvyep13 +x7378OfCqqHkDDDNroWIvij84ZtMUaM53tF13G/ZGOlNsoLNynWs9IVVvqGKsH7o +/buJ1RNArI/0irF0UD7qrMmo1p6SYanZhqdh2PphNy9NS3FsfrfnuWvf+/TRp9Ts +L8058B0p/LIL0OB5trYFircC3iKSOuRl0ERD2ufgSqsSVEYm1mc6UIxv+d1iFD+Q +8CRUF88icQXrec1TCbhh0CfdDxz+FYSTnW0DR0L75coa/CBmRxAjnrkLoXRr3Y1d +sTjU4zDdBcBw -----END CERTIFICATE----- From 4cd94aa668c60f92b1d9f4e5cf27752e1fe9c9cd Mon Sep 17 00:00:00 2001 From: "Yuri H. Galvao" Date: Fri, 1 Mar 2024 04:07:38 -0600 Subject: [PATCH 1372/2083] Restore brotlipy support (#6261) --- scrapy/utils/_compression.py | 32 +++++++++++++++++++++++++++++++- tox.ini | 1 + 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 5610595d3..14531df3f 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -1,10 +1,40 @@ import zlib from io import BytesIO +from warnings import warn + +from scrapy.exceptions import ScrapyDeprecationWarning try: import brotli except ImportError: pass +else: + try: + brotli.Decompressor.process + except AttributeError: + + warn( + ( + "You have brotlipy installed, and Scrapy will use it, but " + "Scrapy support for brotlipy is deprecated and will stop " + "working in a future version of Scrapy. brotlipy itself is " + "deprecated, it has been superseded by brotlicffi (not " + "currently supported by Scrapy). Please, uninstall brotlipy " + "and install brotli instead. brotlipy has the same import " + "name as brotli, so keeping both installed is strongly " + "discouraged." + ), + ScrapyDeprecationWarning, + ) + + def _brotli_decompress(decompressor, data): + return decompressor.decompress(data) + + else: + + def _brotli_decompress(decompressor, data): + return decompressor.process(data) + try: import zstandard @@ -61,7 +91,7 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: decompressed_size = 0 while output_chunk: input_chunk = input_stream.read(_CHUNK_SIZE) - output_chunk = decompressor.process(input_chunk) + output_chunk = _brotli_decompress(decompressor, input_chunk) decompressed_size += len(output_chunk) if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( diff --git a/tox.ini b/tox.ini index 4ed9b3bd7..237aa489c 100644 --- a/tox.ini +++ b/tox.ini @@ -135,6 +135,7 @@ deps = google-cloud-storage==1.29.0 Pillow==7.1.0 robotexclusionrulesparser==1.6.2 + brotlipy install_command = {[pinned]install_command} setenv = {[pinned]setenv} From aa1bf6907964f0281264052cabc28197c5d28107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Mar 2024 12:48:00 +0100 Subject: [PATCH 1373/2083] Mark hashlib usages as not intended for security (#6264) --- .bandit.yml | 1 - scrapy/pipelines/files.py | 28 ++++++++++++++++++++++------ scrapy/pipelines/images.py | 9 ++++----- scrapy/utils/misc.py | 10 +++++++++- scrapy/utils/request.py | 2 +- 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 4fcd75c57..db2fbb84c 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -2,7 +2,6 @@ skips: - B101 # assert_used, needed for mypy - B320 # xml_bad_etree - B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B324 # hashlib "Use of weak SHA1 hash for security" - B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 - B410 # import_lxml - B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 1d7625299..d04218089 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -16,7 +16,7 @@ from ftplib import FTP from io import BytesIO from os import PathLike from pathlib import Path -from typing import DefaultDict, Optional, Set, Union +from typing import IO, DefaultDict, Optional, Set, Union from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -31,7 +31,6 @@ from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str @@ -42,6 +41,23 @@ def _to_string(path: Union[str, PathLike]) -> str: return str(path) # convert a Path object to string +def _md5sum(file: IO) -> str: + """Calculate the md5 checksum of a file-like object without reading its + whole content in memory. + + >>> from io import BytesIO + >>> _md5sum(BytesIO(b'file content to hash')) + '784406af91dd5a54fbb9c84c2236595a' + """ + m = hashlib.md5() # nosec + while True: + d = file.read(8096) + if not d: + break + m.update(d) + return m.hexdigest() + + class FileException(Exception): """General media error exception""" @@ -70,7 +86,7 @@ class FSFilesStore: return {} with absolute_path.open("rb") as f: - checksum = md5sum(f) + checksum = _md5sum(f) return {"last_modified": last_modified, "checksum": checksum} @@ -299,7 +315,7 @@ class FTPFilesStore: ftp.set_pasv(False) file_path = f"{self.basedir}/{path}" last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) - m = hashlib.md5() + m = hashlib.md5() # nosec ftp.retrbinary(f"RETR {file_path}", m.update) return {"last_modified": last_modified, "checksum": m.hexdigest()} # The file doesn't exist @@ -531,7 +547,7 @@ class FilesPipeline(MediaPipeline): def file_downloaded(self, response, request, info, *, item=None): path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) - checksum = md5sum(buf) + checksum = _md5sum(buf) buf.seek(0) self.store.persist_file(path, buf, info) return checksum @@ -542,7 +558,7 @@ class FilesPipeline(MediaPipeline): return item def file_path(self, request, response=None, info=None, *, item=None): - media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec media_ext = Path(request.url).suffix # Handles empty and wild extensions by trying to guess the # mime type then extension or default to empty string otherwise diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 8169583f8..137aa7a9a 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -17,11 +17,10 @@ from itemadapter import ItemAdapter from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.files import FileException, FilesPipeline +from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings -from scrapy.utils.misc import md5sum from scrapy.utils.python import get_func_args, to_bytes @@ -128,7 +127,7 @@ class ImagesPipeline(FilesPipeline): for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) - checksum = md5sum(buf) + checksum = _md5sum(buf) width, height = image.size self.store.persist_file( path, @@ -228,9 +227,9 @@ class ImagesPipeline(FilesPipeline): return item def file_path(self, request, response=None, info=None, *, item=None): - image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"full/{image_guid}.jpg" def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None): - thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"thumbs/{thumb_id}/{thumb_guid}.jpg" diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 7b43760a8..7f83d06fb 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -113,7 +113,15 @@ def md5sum(file: IO) -> str: >>> md5sum(BytesIO(b'file content to hash')) '784406af91dd5a54fbb9c84c2236595a' """ - m = hashlib.md5() + warnings.warn( + ( + "The scrapy.utils.misc.md5sum function is deprecated, and will be " + "removed in a future version of Scrapy." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + m = hashlib.md5() # nosec while True: d = file.read(8096) if not d: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index e99d1eeb5..1f07d58eb 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -111,7 +111,7 @@ def fingerprint( "headers": headers, } fingerprint_json = json.dumps(fingerprint_data, sort_keys=True) - cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() + cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() # nosec return cache[cache_key] From bf149356fc6e519e92fb55150a60b40b14e45ae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Mar 2024 16:02:03 +0100 Subject: [PATCH 1374/2083] Bandit: allow-list lxml usages (#6265) --- .bandit.yml | 2 -- scrapy/http/request/form.py | 17 ++++++----------- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/selector/unified.py | 3 ++- scrapy/utils/_compression.py | 1 - scrapy/utils/iterators.py | 5 +++-- scrapy/utils/sitemap.py | 4 ++-- scrapy/utils/versions.py | 2 +- 8 files changed, 15 insertions(+), 21 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index db2fbb84c..b7f1817e0 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,9 +1,7 @@ skips: - B101 # assert_used, needed for mypy -- B320 # xml_bad_etree - B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 - B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B410 # import_lxml - B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 - B503 # ssl_with_bad_defaults exclude_dirs: ['tests'] diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 0f80a0ab7..3206d79cd 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -10,21 +10,16 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit -from lxml.html import ( - FormElement, - HTMLParser, - InputElement, - MultipleSelectOptions, - SelectElement, - TextareaElement, -) -from parsel.selector import create_root_node +from lxml.html import FormElement # nosec +from lxml.html import InputElement # nosec +from lxml.html import MultipleSelectOptions # nosec +from lxml.html import SelectElement # nosec +from lxml.html import TextareaElement # nosec from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request from scrapy.http.response.text import TextResponse from scrapy.utils.python import is_listlike, to_bytes -from scrapy.utils.response import get_base_url if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -120,7 +115,7 @@ def _get_form( formxpath: Optional[str], ) -> FormElement: """Find the wanted form element within the given response.""" - root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response)) + root = response.selector.root forms = root.xpath("//form") if not forms: raise ValueError(f"No element found in {response}") diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index d76db20ba..55bc0fc43 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -7,7 +7,7 @@ import operator from functools import partial from urllib.parse import urljoin, urlparse -from lxml import etree +from lxml import etree # nosec from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 75d5e9fbd..aa9581fcd 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -8,6 +8,7 @@ from parsel import Selector as _ParselSelector from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.utils.python import to_bytes +from scrapy.utils.response import get_base_url from scrapy.utils.trackref import object_ref __all__ = ["Selector", "SelectorList"] @@ -88,7 +89,7 @@ class Selector(_ParselSelector, object_ref): if response is not None: text = response.text - kwargs.setdefault("base_url", response.url) + kwargs.setdefault("base_url", get_base_url(response)) self.response = response diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 14531df3f..7c40d0a02 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -12,7 +12,6 @@ else: try: brotli.Decompressor.process except AttributeError: - warn( ( "You have brotlipy installed, and Scrapy will use it, but " diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 93a2ba7a1..49493e9c6 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -18,7 +18,7 @@ from typing import ( ) from warnings import warn -from lxml import etree +from lxml import etree # nosec from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse @@ -26,7 +26,7 @@ from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode if TYPE_CHECKING: - from lxml._types import SupportsReadClose + from lxml._types import SupportsReadClose # nosec logger = logging.getLogger(__name__) @@ -101,6 +101,7 @@ def xmliter_lxml( cast("SupportsReadClose[bytes]", reader), encoding=reader.encoding, events=("end", "start-ns"), + resolve_entities=False, huge_tree=True, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 8bf941eb2..7dcee3a2f 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -8,7 +8,7 @@ SitemapSpider, its API is subject to change without notice. from typing import Any, Dict, Generator, Iterator, Optional from urllib.parse import urljoin -import lxml.etree +import lxml.etree # nosec class Sitemap: @@ -19,7 +19,7 @@ class Sitemap: xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) - self._root = lxml.etree.fromstring(xmltext, parser=xmlp) + self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # nosec rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 9b637bdb0..42e5e9be4 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -4,7 +4,7 @@ from typing import List, Tuple import cryptography import cssselect -import lxml.etree +import lxml.etree # nosec import parsel import twisted import w3lib From 6b75d8f3b3107957f3ae381ce3882ac3778f34c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 5 Mar 2024 22:23:48 +0500 Subject: [PATCH 1375/2083] Bump pytest-mypy-testing. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c43bd73d1..7192b6808 100644 --- a/tox.ini +++ b/tox.ini @@ -47,7 +47,7 @@ basepython = python3.8 deps = -rtests/requirements.txt {[testenv:typing]deps} - pytest-mypy-testing==0.1.1 + pytest-mypy-testing==0.1.3 commands = pytest {posargs: tests_typing} From cab1016bb6f719b15043f65502c63fbaa191df36 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 20:24:17 -0300 Subject: [PATCH 1376/2083] Add brotlicffi support --- scrapy/utils/_compression.py | 2 ++ ...st_downloadermiddleware_httpcompression.py | 35 +++++++++++++++---- tox.ini | 1 + 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 7c40d0a02..7896f4c01 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -6,6 +6,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning try: import brotli +except ImportError: + import brotlicffi as brotli except ImportError: pass else: diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index ae5569d0a..7c36f748e 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -130,7 +130,10 @@ class HttpCompressionTest(TestCase): def test_process_response_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") response = self._getresponse("br") @@ -448,7 +451,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_setting_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_setting("br") @@ -486,7 +492,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_spider_attr_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_spider_attr("br") @@ -522,7 +531,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_request_meta_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_request_meta("br") @@ -568,7 +580,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_setting_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_setting("br") @@ -616,7 +631,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_spider_attr_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_spider_attr("br") @@ -662,7 +680,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_request_meta_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_request_meta("br") diff --git a/tox.ini b/tox.ini index 237aa489c..6b804b78c 100644 --- a/tox.ini +++ b/tox.ini @@ -162,6 +162,7 @@ basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} PyPyDispatcher==2.1.0 + brotlicffi commands = pytest --durations=10 scrapy tests install_command = {[pinned]install_command} From 3421823dce94a693ee86915110d899d8da6f3e9f Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 20:26:23 -0300 Subject: [PATCH 1377/2083] Nested try-except block --- scrapy/utils/_compression.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 7896f4c01..477573588 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -5,9 +5,10 @@ from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning try: - import brotli -except ImportError: - import brotlicffi as brotli + try: + import brotli + except ImportError: + import brotlicffi as brotli except ImportError: pass else: From a52429ae08ec15d70f7f3e2079d32933bb639d6b Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 21:12:25 -0300 Subject: [PATCH 1378/2083] Update disclaimer --- scrapy/utils/_compression.py | 4 ++-- tox.ini | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 477573588..4b3fd342d 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -20,8 +20,8 @@ else: "You have brotlipy installed, and Scrapy will use it, but " "Scrapy support for brotlipy is deprecated and will stop " "working in a future version of Scrapy. brotlipy itself is " - "deprecated, it has been superseded by brotlicffi (not " - "currently supported by Scrapy). Please, uninstall brotlipy " + "deprecated, it has been superseded by brotlicffi " + "Please, uninstall brotlipy " "and install brotli instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " "discouraged." diff --git a/tox.ini b/tox.ini index 6b804b78c..9cf3c92ad 100644 --- a/tox.ini +++ b/tox.ini @@ -124,7 +124,7 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - brotli + brotli; implementation_name != 'pypy' zstandard [testenv:extra-deps-pinned] From 16864ea602ebc3d1a764aaf6c101a5f20ff57bee Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 21:18:24 -0300 Subject: [PATCH 1379/2083] Remove PyPy condition --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9cf3c92ad..6b804b78c 100644 --- a/tox.ini +++ b/tox.ini @@ -124,7 +124,7 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - brotli; implementation_name != 'pypy' + brotli zstandard [testenv:extra-deps-pinned] From 532cd2eabd8b280e64a1087c49b8f5eb5f05530f Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 21:30:20 -0300 Subject: [PATCH 1380/2083] Use brotlicffi for PyPy --- tests/requirements.txt | 3 +-- tox.ini | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 5b75674f5..ca5f6ddbd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -11,8 +11,7 @@ uvloop; platform_system != "Windows" bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -# 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 -brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests +brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tox.ini b/tox.ini index 6b804b78c..237aa489c 100644 --- a/tox.ini +++ b/tox.ini @@ -162,7 +162,6 @@ basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} PyPyDispatcher==2.1.0 - brotlicffi commands = pytest --durations=10 scrapy tests install_command = {[pinned]install_command} From 7f1fbdba3cc6f118cbf11285ed26e488f854aed1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 22:11:11 -0300 Subject: [PATCH 1381/2083] Check brotlicffi for ACCEPTED_ENCODINGS --- scrapy/downloadermiddlewares/httpcompression.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index aa3abe853..0e5e215ac 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -29,7 +29,10 @@ logger = getLogger(__name__) ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: pass else: From 7be919138d84ec00feb80a78e13721dff998c10c Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 6 Mar 2024 05:49:31 -0300 Subject: [PATCH 1382/2083] Update scrapy/utils/_compression.py Co-authored-by: Andrey Rakhmatullin --- scrapy/utils/_compression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 4b3fd342d..349fd9ac0 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -20,7 +20,7 @@ else: "You have brotlipy installed, and Scrapy will use it, but " "Scrapy support for brotlipy is deprecated and will stop " "working in a future version of Scrapy. brotlipy itself is " - "deprecated, it has been superseded by brotlicffi " + "deprecated, it has been superseded by brotlicffi. " "Please, uninstall brotlipy " "and install brotli instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " From 99f7165c63a8a2dba72090f65ba1093d476d669a Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 6 Mar 2024 09:02:01 -0300 Subject: [PATCH 1383/2083] Update scrapy/utils/_compression.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/_compression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 349fd9ac0..84c255c28 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -22,7 +22,7 @@ else: "working in a future version of Scrapy. brotlipy itself is " "deprecated, it has been superseded by brotlicffi. " "Please, uninstall brotlipy " - "and install brotli instead. brotlipy has the same import " + "and install brotli or brotlicffi instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " "discouraged." ), From 6ecc9e0a34be6317d1b35a3ca1fc13cb98129732 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 6 Mar 2024 17:21:08 +0500 Subject: [PATCH 1384/2083] Add typing for scrapy/commands (#6268) --- scrapy/commands/__init__.py | 48 +++++++----- scrapy/commands/bench.py | 18 +++-- scrapy/commands/check.py | 18 +++-- scrapy/commands/crawl.py | 14 +++- scrapy/commands/edit.py | 14 ++-- scrapy/commands/fetch.py | 23 +++--- scrapy/commands/genspider.py | 31 +++++--- scrapy/commands/list.py | 8 +- scrapy/commands/parse.py | 128 +++++++++++++++++++++++--------- scrapy/commands/runspider.py | 12 +-- scrapy/commands/settings.py | 13 ++-- scrapy/commands/shell.py | 19 ++--- scrapy/commands/startproject.py | 22 +++--- scrapy/commands/version.py | 13 ++-- scrapy/commands/view.py | 15 +++- scrapy/shell.py | 70 +++++++++++------ scrapy/utils/console.py | 48 ++++++++---- scrapy/utils/response.py | 26 +++---- 18 files changed, 355 insertions(+), 185 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 27993710e..9fe803d3c 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -3,61 +3,62 @@ Base class for Scrapy commands """ import argparse +import builtins import os from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterable, List, Optional from twisted.python import failure -from scrapy.crawler import CrawlerProcess +from scrapy.crawler import Crawler, CrawlerProcess from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli class ScrapyCommand: - requires_project = False + requires_project: bool = False crawler_process: Optional[CrawlerProcess] = None # default settings to be used for this command instead of global defaults default_settings: Dict[str, Any] = {} - exitcode = 0 + exitcode: int = 0 def __init__(self) -> None: self.settings: Any = None # set in scrapy.cmdline - def set_crawler(self, crawler): + def set_crawler(self, crawler: Crawler) -> None: if hasattr(self, "_crawler"): raise RuntimeError("crawler already set") - self._crawler = crawler + self._crawler: Crawler = crawler - def syntax(self): + def syntax(self) -> str: """ Command syntax (preferably one-line). Do not include command name. """ return "" - def short_desc(self): + def short_desc(self) -> str: """ A short description of the command """ return "" - def long_desc(self): + def long_desc(self) -> str: """A long description of the command. Return short description when not available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() - def help(self): + def help(self) -> str: """An extensive help for the command. It will be shown when using the "help" command. It can contain newlines since no post-formatting will be applied to its contents. """ return self.long_desc() - def add_options(self, parser): + def add_options(self, parser: argparse.ArgumentParser) -> None: """ Populate option parse with options available for this command """ @@ -92,7 +93,7 @@ class ScrapyCommand: ) group.add_argument("--pdb", action="store_true", help="enable pdb on failure") - def process_options(self, args, opts): + def process_options(self, args: List[str], opts: argparse.Namespace) -> None: try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: @@ -129,8 +130,8 @@ class BaseRunSpiderCommand(ScrapyCommand): Common class used to share functionality between the crawl, parse and runspider commands """ - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-a", dest="spargs", @@ -162,8 +163,8 @@ class BaseRunSpiderCommand(ScrapyCommand): help="format to use for dumping items", ) - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) + def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + super().process_options(args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: @@ -183,7 +184,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): Help Formatter for scrapy command line help messages. """ - def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): + def __init__( + self, + prog: str, + indent_increment: int = 2, + max_help_position: int = 24, + width: Optional[int] = None, + ): super().__init__( prog, indent_increment=indent_increment, @@ -191,11 +198,12 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): width=width, ) - def _join_parts(self, part_strings): - parts = self.format_part_strings(part_strings) + def _join_parts(self, part_strings: Iterable[str]) -> str: + # scrapy.commands.list shadows builtins.list + parts = self.format_part_strings(builtins.list(part_strings)) return super()._join_parts(parts) - def format_part_strings(self, part_strings): + def format_part_strings(self, part_strings: List[str]) -> List[str]: """ Underline and title case command line help message headers. """ diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index aaf5a439f..2e6bb5d86 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,10 +1,14 @@ +import argparse import subprocess # nosec import sys import time +from typing import Any, Iterable, List from urllib.parse import urlencode import scrapy +from scrapy import Request from scrapy.commands import ScrapyCommand +from scrapy.http import Response from scrapy.linkextractors import LinkExtractor @@ -15,26 +19,28 @@ class Command(ScrapyCommand): "CLOSESPIDER_TIMEOUT": 10, } - def short_desc(self): + def short_desc(self) -> str: return "Run quick benchmark test" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: with _BenchServer(): + assert self.crawler_process self.crawler_process.crawl(_BenchSpider, total=100000) self.crawler_process.start() class _BenchServer: - def __enter__(self): + def __enter__(self) -> None: from scrapy.utils.test import get_testenv pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] self.proc = subprocess.Popen( pargs, stdout=subprocess.PIPE, env=get_testenv() ) # nosec + assert self.proc.stdout self.proc.stdout.readline() - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type, exc_value, traceback) -> None: self.proc.kill() self.proc.wait() time.sleep(0.2) @@ -49,11 +55,11 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - def start_requests(self): + def start_requests(self) -> Iterable[Request]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" return [scrapy.Request(url, dont_filter=True)] - def parse(self, response): + def parse(self, response: Response) -> Any: # type: ignore[override] for link in self.link_extractor.extract_links(response): yield scrapy.Request(link.url, callback=self.parse) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index de54ca4d3..22c8abf7a 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,5 +1,7 @@ +import argparse import time from collections import defaultdict +from typing import List from unittest import TextTestResult as _TextTestResult from unittest import TextTestRunner @@ -10,9 +12,10 @@ from scrapy.utils.misc import load_object, set_environ class TextTestResult(_TextTestResult): - def printSummary(self, start, stop): + def printSummary(self, start: float, stop: float) -> None: write = self.stream.write - writeln = self.stream.writeln + # _WritelnDecorator isn't implemented in typeshed yet + writeln = self.stream.writeln # type: ignore[attr-defined] run = self.testsRun plural = "s" if run != 1 else "" @@ -42,14 +45,14 @@ class Command(ScrapyCommand): requires_project = True default_settings = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Check spider contracts" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-l", "--list", @@ -66,7 +69,7 @@ class Command(ScrapyCommand): help="print contract tests for all spiders", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: # load contracts contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) conman = ContractsManager(load_object(c) for c in contracts) @@ -76,6 +79,7 @@ class Command(ScrapyCommand): # contract requests contract_reqs = defaultdict(list) + assert self.crawler_process spider_loader = self.crawler_process.spider_loader with set_environ(SCRAPY_CHECK="true"): diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 2f0f1c7b9..6e023af81 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,3 +1,8 @@ +import argparse +from typing import List, cast + +from twisted.python.failure import Failure + from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -5,13 +10,13 @@ from scrapy.exceptions import UsageError class Command(BaseRunSpiderCommand): requires_project = True - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Run a spider" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) < 1: raise UsageError() elif len(args) > 1: @@ -20,10 +25,11 @@ class Command(BaseRunSpiderCommand): ) spname = args[0] + assert self.crawler_process crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) if getattr(crawl_defer, "result", None) is not None and issubclass( - crawl_defer.result.type, Exception + cast(Failure, crawl_defer.result).type, Exception ): self.exitcode = 1 else: diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index e85d2c9ec..04012bee8 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,5 +1,7 @@ +import argparse import os import sys +from typing import List from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError @@ -9,32 +11,34 @@ class Command(ScrapyCommand): requires_project = True default_settings = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "" - def short_desc(self): + def short_desc(self) -> str: return "Edit spider" - def long_desc(self): + def long_desc(self) -> str: return ( "Edit a spider using the editor defined in the EDITOR environment" " variable or else the EDITOR setting" ) - def _err(self, msg): + def _err(self, msg: str) -> None: sys.stderr.write(msg + os.linesep) self.exitcode = 1 - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() editor = self.settings["EDITOR"] + assert self.crawler_process try: spidercls = self.crawler_process.spider_loader.load(args[0]) except KeyError: return self._err(f"Spider not found: {args[0]}") sfile = sys.modules[spidercls.__module__].__file__ + assert sfile sfile = sfile.replace(".pyc", ".py") self.exitcode = os.system(f'{editor} "{sfile}"') # nosec diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index cdb7ad4ae..1acf2d26f 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,13 +1,13 @@ import sys -from argparse import Namespace -from typing import List, Type +from argparse import ArgumentParser, Namespace +from typing import Dict, List, Type from w3lib.url import is_url from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import DefaultSpider, spidercls_for_request @@ -15,20 +15,20 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request class Command(ScrapyCommand): requires_project = False - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Fetch a URL using the Scrapy downloader" - def long_desc(self): + def long_desc(self) -> str: return ( "Fetch a URL using the Scrapy downloader and print its content" " to stdout. You may want to use --nolog to disable logging" ) - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: ArgumentParser) -> None: + super().add_options(parser) parser.add_argument("--spider", dest="spider", help="use this spider") parser.add_argument( "--headers", @@ -44,20 +44,21 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def _print_headers(self, headers, prefix): + def _print_headers(self, headers: Dict[bytes, List[bytes]], prefix: bytes) -> None: for key, values in headers.items(): for value in values: self._print_bytes(prefix + b" " + key + b": " + value) - def _print_response(self, response, opts): + def _print_response(self, response: Response, opts: Namespace) -> None: if opts.headers: + assert response.request self._print_headers(response.request.headers, b">") print(">") self._print_headers(response.headers, b"<") else: self._print_bytes(response.body) - def _print_bytes(self, bytes_): + def _print_bytes(self, bytes_: bytes) -> None: sys.stdout.buffer.write(bytes_ + b"\n") def run(self, args: List[str], opts: Namespace) -> None: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 567ebcdc0..2649fb23d 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,9 +1,10 @@ +import argparse import os import shutil import string from importlib import import_module from pathlib import Path -from typing import Optional, cast +from typing import List, Optional, Union, cast from urllib.parse import urlparse import scrapy @@ -12,7 +13,7 @@ from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase -def sanitize_module_name(module_name): +def sanitize_module_name(module_name: str) -> str: """Sanitize the given module name, by replacing dashes and points with underscores and prefixing it with a letter if it doesn't start with one @@ -23,7 +24,7 @@ def sanitize_module_name(module_name): return module_name -def extract_domain(url): +def extract_domain(url: str) -> str: """Extract domain name from URL string""" o = urlparse(url) if o.scheme == "" and o.netloc == "": @@ -31,7 +32,7 @@ def extract_domain(url): return o.netloc -def verify_url_scheme(url): +def verify_url_scheme(url: str) -> str: """Check url for scheme and insert https if none found.""" parsed = urlparse(url) if parsed.scheme == "" and parsed.netloc == "": @@ -43,14 +44,14 @@ class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Generate new spider using pre-defined templates" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-l", "--list", @@ -86,7 +87,7 @@ class Command(ScrapyCommand): help="If the spider already exists, overwrite it with the template", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if opts.list: self._list_templates() return @@ -115,7 +116,14 @@ class Command(ScrapyCommand): if opts.edit: self.exitcode = os.system(f'scrapy edit "{name}"') # nosec - def _genspider(self, module, name, url, template_name, template_file): + def _genspider( + self, + module: str, + name: str, + url: str, + template_name: str, + template_file: Union[str, os.PathLike], + ) -> None: """Generate the spider module, based on the given template""" capitalized_module = "".join(s.capitalize() for s in module.split("_")) domain = extract_domain(url) @@ -130,6 +138,7 @@ class Command(ScrapyCommand): } if self.settings.get("NEWSPIDER_MODULE"): spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) + assert spiders_module.__file__ spiders_dir = Path(spiders_module.__file__).parent.resolve() else: spiders_module = None @@ -152,7 +161,7 @@ class Command(ScrapyCommand): print('Use "scrapy genspider --list" to see all available templates.') return None - def _list_templates(self): + def _list_templates(self) -> None: print("Available templates:") for file in sorted(Path(self.templates_dir).iterdir()): if file.suffix == ".tmpl": diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 2f5032360..dcc51a694 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,3 +1,6 @@ +import argparse +from typing import List + from scrapy.commands import ScrapyCommand @@ -5,9 +8,10 @@ class Command(ScrapyCommand): requires_project = True default_settings = {"LOG_ENABLED": False} - def short_desc(self): + def short_desc(self) -> str: return "List available spiders" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: + assert self.crawler_process for s in sorted(self.crawler_process.spider_loader.list()): print(s) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index c9f8586d3..2453c0d39 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,16 +1,32 @@ +import argparse import functools import inspect import json import logging -from typing import Dict +from types import CoroutineType +from typing import ( + Any, + AsyncGenerator, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + TypeVar, + Union, + overload, +) from itemadapter import ItemAdapter, is_item -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.python.failure import Failure from w3lib.url import is_url from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError -from scrapy.http import Request +from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils import display from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import aiter_errback, deferred_from_coro @@ -20,24 +36,26 @@ from scrapy.utils.spider import spidercls_for_request logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Command(BaseRunSpiderCommand): requires_project = True spider = None - items: Dict[int, list] = {} - requests: Dict[int, list] = {} + items: Dict[int, List[Any]] = {} + requests: Dict[int, List[Request]] = {} first_response = None - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Parse URL (using its spider) and print the results" - def add_options(self, parser): - BaseRunSpiderCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--spider", dest="spider", @@ -106,7 +124,7 @@ class Command(BaseRunSpiderCommand): ) @property - def max_level(self): + def max_level(self) -> int: max_items, max_requests = 0, 0 if self.items: max_items = max(self.items) @@ -114,13 +132,21 @@ class Command(BaseRunSpiderCommand): max_requests = max(self.requests) return max(max_items, max_requests) - def handle_exception(self, _failure): + def handle_exception(self, _failure: Failure) -> None: logger.error( "An error is caught while iterating the async iterable", exc_info=failure_to_exc_info(_failure), ) - def iterate_spider_output(self, result): + @overload + def iterate_spider_output( + self, result: Union[AsyncGenerator, CoroutineType] + ) -> Deferred: ... + + @overload + def iterate_spider_output(self, result: _T) -> Iterable: ... + + def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]: if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) @@ -133,15 +159,15 @@ class Command(BaseRunSpiderCommand): return d return arg_to_iter(deferred_from_coro(result)) - def add_items(self, lvl, new_items): + def add_items(self, lvl: int, new_items: List[Any]) -> None: old_items = self.items.get(lvl, []) self.items[lvl] = old_items + new_items - def add_requests(self, lvl, new_reqs): + def add_requests(self, lvl: int, new_reqs: List[Request]) -> None: old_reqs = self.requests.get(lvl, []) self.requests[lvl] = old_reqs + new_reqs - def print_items(self, lvl=None, colour=True): + def print_items(self, lvl: Optional[int] = None, colour: bool = True) -> None: if lvl is None: items = [item for lst in self.items.values() for item in lst] else: @@ -150,7 +176,7 @@ class Command(BaseRunSpiderCommand): print("# Scraped Items ", "-" * 60) display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour) - def print_requests(self, lvl=None, colour=True): + def print_requests(self, lvl: Optional[int] = None, colour: bool = True) -> None: if lvl is None: if self.requests: requests = self.requests[max(self.requests)] @@ -162,7 +188,7 @@ class Command(BaseRunSpiderCommand): print("# Requests ", "-" * 65) display.pprint(requests, colorize=colour) - def print_results(self, opts): + def print_results(self, opts: argparse.Namespace) -> None: colour = not opts.nocolour if opts.verbose: @@ -179,7 +205,14 @@ class Command(BaseRunSpiderCommand): if not opts.nolinks: self.print_requests(colour=colour) - def _get_items_and_requests(self, spider_output, opts, depth, spider, callback): + def _get_items_and_requests( + self, + spider_output: Iterable[Any], + opts: argparse.Namespace, + depth: int, + spider: Spider, + callback: Callable, + ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]: items, requests = [], [] for x in spider_output: if is_item(x): @@ -188,14 +221,21 @@ class Command(BaseRunSpiderCommand): requests.append(x) return items, requests, opts, depth, spider, callback - def run_callback(self, response, callback, cb_kwargs=None): + def run_callback( + self, + response: Response, + callback: Callable, + cb_kwargs: Optional[Dict[str, Any]] = None, + ) -> Deferred: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) return d - def get_callback_from_rules(self, spider, response): + def get_callback_from_rules( + self, spider: Spider, response: Response + ) -> Union[Callable, str, None]: if getattr(spider, "rules", None): - for rule in spider.rules: + for rule in spider.rules: # type: ignore[attr-defined] if rule.link_extractor.matches(response.url): return rule.callback or "parse" else: @@ -204,8 +244,10 @@ class Command(BaseRunSpiderCommand): "please specify a callback to use for parsing", {"spider": spider.name}, ) + return None - def set_spidercls(self, url, opts): + def set_spidercls(self, url: str, opts: argparse.Namespace) -> None: + assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: try: @@ -219,13 +261,14 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - def _start_requests(spider): + def _start_requests(spider: Spider) -> Iterable[Request]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: self.spidercls.start_requests = _start_requests - def start_parsing(self, url, opts): + def start_parsing(self, url: str, opts: argparse.Namespace) -> None: + assert self.crawler_process self.crawler_process.crawl(self.spidercls, **opts.spargs) self.pcrawler = list(self.crawler_process.crawlers)[0] self.crawler_process.start() @@ -233,7 +276,12 @@ class Command(BaseRunSpiderCommand): if not self.first_response: logger.error("No response downloaded for: %(url)s", {"url": url}) - def scraped_data(self, args): + def scraped_data( + self, + args: Tuple[ + List[Any], List[Request], argparse.Namespace, int, Spider, Callable + ], + ) -> List[Any]: items, requests, opts, depth, spider, callback = args if opts.pipelines: itemproc = self.pcrawler.engine.scraper.itemproc @@ -252,8 +300,14 @@ class Command(BaseRunSpiderCommand): return scraped_data - def _get_callback(self, *, spider, opts, response=None): - cb = None + def _get_callback( + self, + *, + spider: Spider, + opts: argparse.Namespace, + response: Optional[Response] = None, + ) -> Callable: + cb: Union[str, Callable, None] = None if response: cb = response.meta["_callback"] if not cb: @@ -270,6 +324,7 @@ class Command(BaseRunSpiderCommand): cb = "parse" if not callable(cb): + assert cb is not None cb_method = getattr(spider, cb, None) if callable(cb_method): cb = cb_method @@ -277,10 +332,13 @@ class Command(BaseRunSpiderCommand): raise ValueError( f"Cannot find callback {cb!r} in spider: {spider.name}" ) + assert callable(cb) return cb - def prepare_request(self, spider, request, opts): - def callback(response, **cb_kwargs): + def prepare_request( + self, spider: Spider, request: Request, opts: argparse.Namespace + ) -> Request: + def callback(response: Response, **cb_kwargs: Any) -> Deferred: # memorize first request if not self.first_response: self.first_response = response @@ -288,7 +346,7 @@ class Command(BaseRunSpiderCommand): cb = self._get_callback(spider=spider, opts=opts, response=response) # parse items and requests - depth = response.meta["_depth"] + depth: int = response.meta["_depth"] d = self.run_callback(response, cb, cb_kwargs) d.addCallback(self._get_items_and_requests, opts, depth, spider, callback) @@ -311,13 +369,13 @@ class Command(BaseRunSpiderCommand): request.callback = callback return request - def process_options(self, args, opts): - BaseRunSpiderCommand.process_options(self, args, opts) + def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + super().process_options(args, opts) self.process_request_meta(opts) self.process_request_cb_kwargs(opts) - def process_request_meta(self, opts): + def process_request_meta(self, opts: argparse.Namespace) -> None: if opts.meta: try: opts.meta = json.loads(opts.meta) @@ -328,7 +386,7 @@ class Command(BaseRunSpiderCommand): print_help=False, ) - def process_request_cb_kwargs(self, opts): + def process_request_cb_kwargs(self, opts: argparse.Namespace) -> None: if opts.cbkwargs: try: opts.cbkwargs = json.loads(opts.cbkwargs) @@ -339,7 +397,7 @@ class Command(BaseRunSpiderCommand): print_help=False, ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: # parse arguments if not len(args) == 1 or not is_url(args[0]): raise UsageError() diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 58ed89a81..77850e7b5 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,9 +1,10 @@ +import argparse import sys from importlib import import_module from os import PathLike from pathlib import Path from types import ModuleType -from typing import Union +from typing import List, Union from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -27,16 +28,16 @@ class Command(BaseRunSpiderCommand): requires_project = False default_settings = {"SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Run a self-contained spider (without creating a project)" - def long_desc(self): + def long_desc(self) -> str: return "Run the spider defined in the given file" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() filename = Path(args[0]) @@ -51,6 +52,7 @@ class Command(BaseRunSpiderCommand): raise UsageError(f"No spider found in file: {filename}\n") spidercls = spclasses.pop() + assert self.crawler_process self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 318187204..dbda73b44 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,4 +1,6 @@ +import argparse import json +from typing import List from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings @@ -8,14 +10,14 @@ class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return "[options]" - def short_desc(self): + def short_desc(self) -> str: return "Get settings values" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--get", dest="get", metavar="SETTING", help="print raw setting value" ) @@ -44,7 +46,8 @@ class Command(ScrapyCommand): help="print setting value, interpreted as a list", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: + assert self.crawler_process settings = self.crawler_process.settings if opts.get: s = settings.get(opts.get) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index f72a23c6a..668c95a7b 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -4,9 +4,9 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -from argparse import Namespace +from argparse import ArgumentParser, Namespace from threading import Thread -from typing import List, Type +from typing import Any, Dict, List, Type from scrapy import Spider from scrapy.commands import ScrapyCommand @@ -24,20 +24,20 @@ class Command(ScrapyCommand): "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", } - def syntax(self): + def syntax(self) -> str: return "[url|file]" - def short_desc(self): + def short_desc(self) -> str: return "Interactive scraping console" - def long_desc(self): + def long_desc(self) -> str: return ( "Interactive console for scraping the given url or file. " "Use ./file.html syntax or full path for local file." ) - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-c", dest="code", @@ -52,7 +52,7 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def update_vars(self, vars): + def update_vars(self, vars: Dict[str, Any]) -> None: """You can use this function to update the Scrapy objects that will be available in the shell """ @@ -88,7 +88,8 @@ class Command(ScrapyCommand): shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) shell.start(url=url, redirect=not opts.no_redirect) - def _start_crawler_thread(self): + def _start_crawler_thread(self) -> None: + assert self.crawler_process t = Thread( target=self.crawler_process.start, kwargs={"stop_after_crawl": False, "install_signal_handlers": False}, diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index fde609c6f..58c1aa28f 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,3 +1,4 @@ +import argparse import os import re import string @@ -5,13 +6,14 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION +from typing import List, Tuple, Union import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase -TEMPLATES_TO_RENDER = ( +TEMPLATES_TO_RENDER: Tuple[Tuple[str, ...], ...] = ( ("scrapy.cfg",), ("${project_name}", "settings.py.tmpl"), ("${project_name}", "items.py.tmpl"), @@ -22,7 +24,7 @@ TEMPLATES_TO_RENDER = ( IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") -def _make_writable(path): +def _make_writable(path: Union[str, os.PathLike]) -> None: current_permissions = os.stat(path).st_mode os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) @@ -31,14 +33,14 @@ class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return " [project_dir]" - def short_desc(self): + def short_desc(self) -> str: return "Create new project" - def _is_valid_name(self, project_name): - def _module_exists(module_name): + def _is_valid_name(self, project_name: str) -> bool: + def _module_exists(module_name: str) -> bool: spec = find_spec(module_name) return spec is not None and spec.loader is not None @@ -53,7 +55,7 @@ class Command(ScrapyCommand): return True return False - def _copytree(self, src: Path, dst: Path): + def _copytree(self, src: Path, dst: Path) -> None: """ Since the original function always creates the directory, to resolve the issue a new function had to be created. It's a simple copy and @@ -84,7 +86,7 @@ class Command(ScrapyCommand): copystat(src, dst) _make_writable(dst) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) not in (1, 2): raise UsageError() @@ -105,7 +107,9 @@ class Command(ScrapyCommand): return self._copytree(Path(self.templates_dir), project_dir.resolve()) - move(project_dir / "module", project_dir / project_name) + # On 3.8 shutil.move doesn't fully support Path args, but it supports our use case + # See https://bugs.python.org/issue32689 + move(project_dir / "module", project_dir / project_name) # type: ignore[arg-type] for paths in TEMPLATES_TO_RENDER: tplfile = Path( project_dir, diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 47582866b..f057e8544 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,3 +1,6 @@ +import argparse +from typing import List + import scrapy from scrapy.commands import ScrapyCommand from scrapy.utils.versions import scrapy_components_versions @@ -6,14 +9,14 @@ from scrapy.utils.versions import scrapy_components_versions class Command(ScrapyCommand): default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return "[-v]" - def short_desc(self): + def short_desc(self) -> str: return "Print Scrapy version" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--verbose", "-v", @@ -22,7 +25,7 @@ class Command(ScrapyCommand): help="also display twisted/python/platform info (useful for bug reports)", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index ebdfa10a8..21679e3aa 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,21 +1,28 @@ import argparse +import logging from scrapy.commands import fetch +from scrapy.http import Response, TextResponse from scrapy.utils.response import open_in_browser +logger = logging.getLogger(__name__) + class Command(fetch.Command): - def short_desc(self): + def short_desc(self) -> str: return "Open URL in browser, as seen by Scrapy" - def long_desc(self): + def long_desc(self) -> str: return ( "Fetch a URL using the Scrapy downloader and show its contents in a browser" ) - def add_options(self, parser): + def add_options(self, parser: argparse.ArgumentParser) -> None: super().add_options(parser) parser.add_argument("--headers", help=argparse.SUPPRESS) - def _print_response(self, response, opts): + def _print_response(self, response: Response, opts: argparse.Namespace) -> None: + if not isinstance(response, TextResponse): + logger.error("Cannot view a non-text response.") + return open_in_browser(response) diff --git a/scrapy/shell.py b/scrapy/shell.py index 63ea33892..fac42e8a2 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst import os import signal +from typing import Any, Callable, Dict, Optional, Tuple, Union from itemadapter import is_item from twisted.internet import defer, threads @@ -26,18 +27,32 @@ from scrapy.utils.response import open_in_browser class Shell: - relevant_classes = (Crawler, Spider, Request, Response, Settings) + relevant_classes: Tuple[type, ...] = (Crawler, Spider, Request, Response, Settings) - def __init__(self, crawler, update_vars=None, code=None): - self.crawler = crawler - self.update_vars = update_vars or (lambda x: None) - self.item_class = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) - self.spider = None - self.inthread = not threadable.isInIOThread() - self.code = code - self.vars = {} + def __init__( + self, + crawler: Crawler, + update_vars: Optional[Callable[[Dict[str, Any]], None]] = None, + code: Optional[str] = None, + ): + self.crawler: Crawler = crawler + self.update_vars: Callable[[Dict[str, Any]], None] = update_vars or ( + lambda x: None + ) + self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) + self.spider: Optional[Spider] = None + self.inthread: bool = not threadable.isInIOThread() + self.code: Optional[str] = code + self.vars: Dict[str, Any] = {} - def start(self, url=None, request=None, response=None, spider=None, redirect=True): + def start( + self, + url: Optional[str] = None, + request: Optional[Request] = None, + response: Optional[Response] = None, + spider: Optional[Spider] = None, + redirect: bool = True, + ) -> None: # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: @@ -77,7 +92,7 @@ class Shell: self.vars, shells=shells, banner=self.vars.pop("banner", "") ) - def _schedule(self, request, spider): + def _schedule(self, request: Request, spider: Optional[Spider]) -> defer.Deferred: if is_asyncio_reactor_installed(): # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] @@ -85,10 +100,11 @@ class Shell: spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) + assert self.crawler.engine self.crawler.engine.crawl(request) return d - def _open_spider(self, request, spider): + def _open_spider(self, request: Request, spider: Optional[Spider]) -> Spider: if self.spider: return self.spider @@ -96,11 +112,18 @@ class Shell: spider = self.crawler.spider or self.crawler._create_spider() self.crawler.spider = spider + assert self.crawler.engine self.crawler.engine.open_spider(spider, close_if_idle=False) self.spider = spider return spider - def fetch(self, request_or_url, spider=None, redirect=True, **kwargs): + def fetch( + self, + request_or_url: Union[Request, str], + spider: Optional[Spider] = None, + redirect: bool = True, + **kwargs: Any, + ) -> None: from twisted.internet import reactor if isinstance(request_or_url, Request): @@ -123,7 +146,12 @@ class Shell: pass self.populate_vars(response, request, spider) - def populate_vars(self, response=None, request=None, spider=None): + def populate_vars( + self, + response: Optional[Response] = None, + request: Optional[Request] = None, + spider: Optional[Spider] = None, + ) -> None: import scrapy self.vars["scrapy"] = scrapy @@ -141,10 +169,10 @@ class Shell: if not self.code: self.vars["banner"] = self.get_help() - def print_help(self): + def print_help(self) -> None: print(self.get_help()) - def get_help(self): + def get_help(self) -> str: b = [] b.append("Available Scrapy objects:") b.append( @@ -168,11 +196,11 @@ class Shell: return "\n".join(f"[s] {line}" for line in b) - def _is_relevant(self, value): + def _is_relevant(self, value: Any) -> bool: return isinstance(value, self.relevant_classes) or is_item(value) -def inspect_response(response, spider): +def inspect_response(response: Response, spider: Spider) -> None: """Open a shell to inspect the given response""" # Shell.start removes the SIGINT handler, so save it and re-add it after # the shell has closed @@ -181,7 +209,7 @@ def inspect_response(response, spider): signal.signal(signal.SIGINT, sigint_handler) -def _request_deferred(request): +def _request_deferred(request: Request) -> defer.Deferred: """Wrap a request inside a Deferred. This function is harmful, do not use it until you know what you are doing. @@ -195,12 +223,12 @@ def _request_deferred(request): request_callback = request.callback request_errback = request.errback - def _restore_callbacks(result): + def _restore_callbacks(result: Any) -> Any: request.callback = request_callback request.errback = request_errback return result - d = defer.Deferred() + d: defer.Deferred = defer.Deferred() d.addBoth(_restore_callbacks) if request.callback: d.addCallbacks(request.callback, request.errback) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 100f040bb..bf1803115 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,17 +1,27 @@ from functools import wraps +from typing import Any, Callable, Dict, Iterable, Optional + +EmbedFuncT = Callable[..., None] +KnownShellsT = Dict[str, Callable[..., EmbedFuncT]] -def _embed_ipython_shell(namespace={}, banner=""): +def _embed_ipython_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start an IPython Shell""" try: from IPython.terminal.embed import InteractiveShellEmbed from IPython.terminal.ipapp import load_default_config except ImportError: - from IPython.frontend.terminal.embed import InteractiveShellEmbed - from IPython.frontend.terminal.ipapp import load_default_config + from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] + InteractiveShellEmbed, + ) + from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] + load_default_config, + ) @wraps(_embed_ipython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: config = load_default_config() # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports @@ -26,30 +36,36 @@ def _embed_ipython_shell(namespace={}, banner=""): return wrapper -def _embed_bpython_shell(namespace={}, banner=""): +def _embed_bpython_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start a bpython shell""" import bpython @wraps(_embed_bpython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: bpython.embed(locals_=namespace, banner=banner) return wrapper -def _embed_ptpython_shell(namespace={}, banner=""): +def _embed_ptpython_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start a ptpython shell""" import ptpython.repl @wraps(_embed_ptpython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: print(banner) ptpython.repl.embed(locals=namespace) return wrapper -def _embed_standard_shell(namespace={}, banner=""): +def _embed_standard_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start a standard python shell""" import code @@ -63,13 +79,13 @@ def _embed_standard_shell(namespace={}, banner=""): readline.parse_and_bind("tab:complete") @wraps(_embed_standard_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: code.interact(banner=banner, local=namespace) return wrapper -DEFAULT_PYTHON_SHELLS = { +DEFAULT_PYTHON_SHELLS: KnownShellsT = { "ptpython": _embed_ptpython_shell, "ipython": _embed_ipython_shell, "bpython": _embed_bpython_shell, @@ -77,7 +93,9 @@ DEFAULT_PYTHON_SHELLS = { } -def get_shell_embed_func(shells=None, known_shells=None): +def get_shell_embed_func( + shells: Optional[Iterable[str]] = None, known_shells: Optional[KnownShellsT] = None +) -> Any: """Return the first acceptable shell-embed function from a given list of shell names. """ @@ -95,7 +113,11 @@ def get_shell_embed_func(shells=None, known_shells=None): continue -def start_python_console(namespace=None, banner="", shells=None): +def start_python_console( + namespace: Optional[Dict[str, Any]] = None, + banner: str = "", + shells: Optional[Iterable[str]] = None, +) -> None: """Start Python console bound to the given namespace. Readline support and tab completion will be used on Unix, if available. """ diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 63a484b42..a0b06f75c 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,24 +3,27 @@ This module provides some useful functions for working with scrapy.http.Response objects """ +from __future__ import annotations + import os import re import tempfile import webbrowser -from typing import Any, Callable, Iterable, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Iterable, Tuple, Union from weakref import WeakKeyDictionary from twisted.web import http from w3lib import html -import scrapy -from scrapy.http.response import Response from scrapy.utils.python import to_bytes, to_unicode -_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() +if TYPE_CHECKING: + from scrapy.http import Response, TextResponse + +_baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary() -def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: +def get_base_url(response: TextResponse) -> str: """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] @@ -30,13 +33,13 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: return _baseurl_cache[response] -_metaref_cache: ( - "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" -) = WeakKeyDictionary() +_metaref_cache: WeakKeyDictionary[ + Response, Union[Tuple[None, None], Tuple[float, str]] +] = WeakKeyDictionary() def get_meta_refresh( - response: "scrapy.http.response.text.TextResponse", + response: TextResponse, ignore_tags: Iterable[str] = ("script", "noscript"), ) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refresh parameter from the given response""" @@ -68,10 +71,7 @@ def _remove_html_comments(body): def open_in_browser( - response: Union[ - "scrapy.http.response.html.HtmlResponse", - "scrapy.http.response.text.TextResponse", - ], + response: TextResponse, _openfunc: Callable[[str], Any] = webbrowser.open, ) -> Any: """Open *response* in a local web browser, adjusting the `base tag`_ for From 8985a04bd1328cd6156a7c33a5db74ad8c81802f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 8 Mar 2024 10:57:03 +0500 Subject: [PATCH 1385/2083] Full typing for scrapy/exporters.py. (#6275) --- scrapy/exporters.py | 150 ++++++++++++++++++-------------- scrapy/extensions/feedexport.py | 11 +-- 2 files changed, 89 insertions(+), 72 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 79fd4e56f..fb4998099 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -3,16 +3,18 @@ Item Exporters are used to export/serialize items into different formats. """ import csv -import io import marshal import pickle # nosec import pprint -from collections.abc import Mapping +from io import BytesIO, TextIOWrapper +from json import JSONEncoder +from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union from xml.sax.saxutils import XMLGenerator # nosec +from xml.sax.xmlreader import AttributesImpl # nosec from itemadapter import ItemAdapter, is_item -from scrapy.item import Item +from scrapy.item import Field, Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -29,36 +31,42 @@ __all__ = [ class BaseItemExporter: - def __init__(self, *, dont_fail=False, **kwargs): - self._kwargs = kwargs + def __init__(self, *, dont_fail: bool = False, **kwargs: Any): + self._kwargs: Dict[str, Any] = kwargs self._configure(kwargs, dont_fail=dont_fail) - def _configure(self, options, dont_fail=False): + def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: """Configure the exporter by popping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) """ - self.encoding = options.pop("encoding", None) - self.fields_to_export = options.pop("fields_to_export", None) - self.export_empty_fields = options.pop("export_empty_fields", False) - self.indent = options.pop("indent", None) + self.encoding: Optional[str] = options.pop("encoding", None) + self.fields_to_export: Union[Mapping[str, str], Iterable[str], None] = ( + options.pop("fields_to_export", None) + ) + self.export_empty_fields: bool = options.pop("export_empty_fields", False) + self.indent: Optional[int] = options.pop("indent", None) if not dont_fail and options: raise TypeError(f"Unexpected options: {', '.join(options.keys())}") - def export_item(self, item): + def export_item(self, item: Any) -> None: raise NotImplementedError - def serialize_field(self, field, name, value): - serializer = field.get("serializer", lambda x: x) + def serialize_field( + self, field: Union[Mapping[str, Any], Field], name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x) return serializer(value) - def start_exporting(self): + def start_exporting(self) -> None: pass - def finish_exporting(self): + def finish_exporting(self) -> None: pass - def _get_serialized_fields(self, item, default_value=None, include_empty=None): + def _get_serialized_fields( + self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None + ) -> Iterable[Tuple[str, Any]]: """Return the fields to export as an iterable of tuples (name, serialized_value) """ @@ -100,22 +108,22 @@ class BaseItemExporter: class JsonLinesItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(dont_fail=True, **kwargs) - self.file = file + self.file: BytesIO = file self._kwargs.setdefault("ensure_ascii", not self.encoding) - self.encoder = ScrapyJSONEncoder(**self._kwargs) + self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs) - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) data = self.encoder.encode(itemdict) + "\n" self.file.write(to_bytes(data, self.encoding)) class JsonItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(dont_fail=True, **kwargs) - self.file = file + self.file: BytesIO = file # there is a small difference between the behaviour or JsonItemExporter.indent # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent # the addition of newlines everywhere @@ -127,26 +135,26 @@ class JsonItemExporter(BaseItemExporter): self.encoder = ScrapyJSONEncoder(**self._kwargs) self.first_item = True - def _beautify_newline(self): + def _beautify_newline(self) -> None: if self.indent is not None: self.file.write(b"\n") - def _add_comma_after_first(self): + def _add_comma_after_first(self) -> None: if self.first_item: self.first_item = False else: self.file.write(b",") self._beautify_newline() - def start_exporting(self): + def start_exporting(self) -> None: self.file.write(b"[") self._beautify_newline() - def finish_exporting(self): + def finish_exporting(self) -> None: self._beautify_newline() self.file.write(b"]") - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) data = to_bytes(self.encoder.encode(itemdict), self.encoding) self._add_comma_after_first() @@ -154,7 +162,7 @@ class JsonItemExporter(BaseItemExporter): class XmlItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): self.item_element = kwargs.pop("item_element", "item") self.root_element = kwargs.pop("root_element", "items") super().__init__(**kwargs) @@ -162,22 +170,22 @@ class XmlItemExporter(BaseItemExporter): self.encoding = "utf-8" self.xg = XMLGenerator(file, encoding=self.encoding) - def _beautify_newline(self, new_item=False): + def _beautify_newline(self, new_item: bool = False) -> None: if self.indent is not None and (self.indent > 0 or new_item): self.xg.characters("\n") - def _beautify_indent(self, depth=1): + def _beautify_indent(self, depth: int = 1) -> None: if self.indent: self.xg.characters(" " * self.indent * depth) - def start_exporting(self): + def start_exporting(self) -> None: self.xg.startDocument() - self.xg.startElement(self.root_element, {}) + self.xg.startElement(self.root_element, AttributesImpl({})) self._beautify_newline(new_item=True) - def export_item(self, item): + def export_item(self, item: Any) -> None: self._beautify_indent(depth=1) - self.xg.startElement(self.item_element, {}) + self.xg.startElement(self.item_element, AttributesImpl({})) self._beautify_newline() for name, value in self._get_serialized_fields(item, default_value=""): self._export_xml_field(name, value, depth=2) @@ -185,13 +193,13 @@ class XmlItemExporter(BaseItemExporter): self.xg.endElement(self.item_element) self._beautify_newline(new_item=True) - def finish_exporting(self): + def finish_exporting(self) -> None: self.xg.endElement(self.root_element) self.xg.endDocument() - def _export_xml_field(self, name, serialized_value, depth): + def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None: self._beautify_indent(depth=depth) - self.xg.startElement(name, {}) + self.xg.startElement(name, AttributesImpl({})) if hasattr(serialized_value, "items"): self._beautify_newline() for subname, value in serialized_value.items(): @@ -213,17 +221,17 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): def __init__( self, - file, - include_headers_line=True, - join_multivalued=",", - errors=None, - **kwargs, + file: BytesIO, + include_headers_line: bool = True, + join_multivalued: str = ",", + errors: Optional[str] = None, + **kwargs: Any, ): super().__init__(dont_fail=True, **kwargs) if not self.encoding: self.encoding = "utf-8" self.include_headers_line = include_headers_line - self.stream = io.TextIOWrapper( + self.stream = TextIOWrapper( file, line_buffering=False, write_through=True, @@ -235,11 +243,13 @@ class CsvItemExporter(BaseItemExporter): self._headers_not_written = True self._join_multivalued = join_multivalued - def serialize_field(self, field, name, value): - serializer = field.get("serializer", self._join_if_needed) + def serialize_field( + self, field: Union[Mapping[str, Any], Field], name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get("serializer", self._join_if_needed) return serializer(value) - def _join_if_needed(self, value): + def _join_if_needed(self, value: Any) -> Any: if isinstance(value, (list, tuple)): try: return self._join_multivalued.join(value) @@ -247,7 +257,7 @@ class CsvItemExporter(BaseItemExporter): pass return value - def export_item(self, item): + def export_item(self, item: Any) -> None: if self._headers_not_written: self._headers_not_written = False self._write_headers_and_set_fields_to_export(item) @@ -256,36 +266,38 @@ class CsvItemExporter(BaseItemExporter): values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) - def finish_exporting(self): + def finish_exporting(self) -> None: self.stream.detach() # Avoid closing the wrapped file. - def _build_row(self, values): + def _build_row(self, values: Iterable[Any]) -> Iterable[Any]: for s in values: try: yield to_unicode(s, self.encoding) except TypeError: yield s - def _write_headers_and_set_fields_to_export(self, item): + def _write_headers_and_set_fields_to_export(self, item: Any) -> None: if self.include_headers_line: if not self.fields_to_export: # use declared field names, or keys if the item is a dict self.fields_to_export = ItemAdapter(item).field_names() + fields: Iterable[str] if isinstance(self.fields_to_export, Mapping): fields = self.fields_to_export.values() else: + assert self.fields_to_export fields = self.fields_to_export row = list(self._build_row(fields)) self.csv_writer.writerow(row) class PickleItemExporter(BaseItemExporter): - def __init__(self, file, protocol=4, **kwargs): + def __init__(self, file: BytesIO, protocol: int = 4, **kwargs: Any): super().__init__(**kwargs) - self.file = file - self.protocol = protocol + self.file: BytesIO = file + self.protocol: int = protocol - def export_item(self, item): + def export_item(self, item: Any) -> None: d = dict(self._get_serialized_fields(item)) pickle.dump(d, self.file, self.protocol) @@ -299,20 +311,20 @@ class MarshalItemExporter(BaseItemExporter): opened in binary mode, a :class:`~io.BytesIO` object, etc) """ - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(**kwargs) - self.file = file + self.file: BytesIO = file - def export_item(self, item): + def export_item(self, item: Any) -> None: marshal.dump(dict(self._get_serialized_fields(item)), self.file) class PprintItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(**kwargs) - self.file = file + self.file: BytesIO = file - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) self.file.write(to_bytes(pprint.pformat(itemdict) + "\n")) @@ -327,16 +339,20 @@ class PythonItemExporter(BaseItemExporter): .. _msgpack: https://pypi.org/project/msgpack/ """ - def _configure(self, options, dont_fail=False): + def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: super()._configure(options, dont_fail) if not self.encoding: self.encoding = "utf-8" - def serialize_field(self, field, name, value): - serializer = field.get("serializer", self._serialize_value) + def serialize_field( + self, field: Union[Mapping[str, Any], Field], name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get( + "serializer", self._serialize_value + ) return serializer(value) - def _serialize_value(self, value): + def _serialize_value(self, value: Any) -> Any: if isinstance(value, Item): return self.export_item(value) if is_item(value): @@ -347,10 +363,10 @@ class PythonItemExporter(BaseItemExporter): return to_unicode(value, encoding=self.encoding) return value - def _serialize_item(self, item): + def _serialize_item(self, item: Any) -> Iterable[Tuple[Union[str, bytes], Any]]: for key, value in ItemAdapter(item).items(): yield key, self._serialize_value(value) - def export_item(self, item): - result = dict(self._get_serialized_fields(item)) + def export_item(self, item: Any) -> Dict[Union[str, bytes], Any]: # type: ignore[override] + result: Dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item)) return result diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e5e363b52..3b0dd804e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,7 +11,7 @@ import warnings from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union +from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Type, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads @@ -21,6 +21,7 @@ from zope.interface import Interface, implementer from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exporters import BaseItemExporter from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings @@ -324,12 +325,12 @@ class FeedSlot: filter, feed_options, spider, - exporters, + exporters: Dict[str, Type[BaseItemExporter]], settings, crawler, ): self.file = None - self.exporter = None + self.exporter: Optional[BaseItemExporter] = None self.storage = storage # feed params self.batch_id = batch_id @@ -341,7 +342,7 @@ class FeedSlot: # exporter params self.feed_options = feed_options self.spider = spider - self.exporters = exporters + self.exporters: Dict[str, Type[BaseItemExporter]] = exporters self.settings = settings self.crawler = crawler # flags @@ -373,7 +374,7 @@ class FeedSlot: def _get_instance(self, objcls, *args, **kwargs): return build_from_crawler(objcls, self.crawler, *args, **kwargs) - def _get_exporter(self, file, format, *args, **kwargs): + def _get_exporter(self, file, format, *args, **kwargs) -> BaseItemExporter: return self._get_instance(self.exporters[format], file, *args, **kwargs) def finish_exporting(self): From 421e08dd4a4d5ed2acf3cd0fca4c7bfa5a6d3eb2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 8 Mar 2024 10:59:56 +0500 Subject: [PATCH 1386/2083] Full typing for scrapy/extensions, part 1. (#6276) --- scrapy/extensions/closespider.py | 52 +++++++++++++++++++++----------- scrapy/extensions/corestats.py | 33 +++++++++++++------- scrapy/extensions/debug.py | 25 ++++++++++----- scrapy/extensions/logstats.py | 50 ++++++++++++++++++------------ scrapy/extensions/memdebug.py | 22 ++++++++++---- scrapy/extensions/memusage.py | 51 ++++++++++++++++++++----------- scrapy/mail.py | 12 ++++++-- scrapy/utils/engine.py | 13 ++++---- 8 files changed, 171 insertions(+), 87 deletions(-) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 4307b4170..812b3553c 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -4,20 +4,31 @@ conditions are met. See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + import logging from collections import defaultdict +from typing import TYPE_CHECKING, Any, DefaultDict, Dict -from scrapy import signals +from twisted.python.failure import Failure + +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class CloseSpider: - def __init__(self, crawler): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler - self.close_on = { + self.close_on: Dict[str, Any] = { "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"), "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), @@ -28,7 +39,7 @@ class CloseSpider: if not any(self.close_on.values()): raise NotConfigured - self.counter = defaultdict(int) + self.counter: DefaultDict[str, int] = defaultdict(int) if self.close_on.get("errorcount"): crawler.signals.connect(self.error_count, signal=signals.spider_error) @@ -39,8 +50,8 @@ class CloseSpider: if self.close_on.get("itemcount"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) if self.close_on.get("timeout_no_item"): - self.timeout_no_item = self.close_on["timeout_no_item"] - self.items_in_period = 0 + self.timeout_no_item: int = self.close_on["timeout_no_item"] + self.items_in_period: int = 0 crawler.signals.connect( self.spider_opened_no_item, signal=signals.spider_opened ) @@ -50,22 +61,25 @@ class CloseSpider: crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def error_count(self, failure, response, spider): + def error_count(self, failure: Failure, response: Response, spider: Spider) -> None: self.counter["errorcount"] += 1 if self.counter["errorcount"] == self.close_on["errorcount"]: + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_errorcount") - def page_count(self, response, request, spider): + def page_count(self, response: Response, request: Request, spider: Spider) -> None: self.counter["pagecount"] += 1 if self.counter["pagecount"] == self.close_on["pagecount"]: + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_pagecount") - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: from twisted.internet import reactor + assert self.crawler.engine self.task = reactor.callLater( self.close_on["timeout"], self.crawler.engine.close_spider, @@ -73,21 +87,22 @@ class CloseSpider: reason="closespider_timeout", ) - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: self.counter["itemcount"] += 1 if self.counter["itemcount"] == self.close_on["itemcount"]: + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_itemcount") - def spider_closed(self, spider): - task = getattr(self, "task", False) + def spider_closed(self, spider: Spider) -> None: + task = getattr(self, "task", None) if task and task.active(): task.cancel() - task_no_item = getattr(self, "task_no_item", False) + task_no_item = getattr(self, "task_no_item", None) if task_no_item and task_no_item.running: task_no_item.stop() - def spider_opened_no_item(self, spider): + def spider_opened_no_item(self, spider: Spider) -> None: from twisted.internet import task self.task_no_item = task.LoopingCall(self._count_items_produced, spider) @@ -98,10 +113,10 @@ class CloseSpider: f"{self.timeout_no_item} seconds." ) - def item_scraped_no_item(self, item, spider): + def item_scraped_no_item(self, item: Any, spider: Spider) -> None: self.items_in_period += 1 - def _count_items_produced(self, spider): + def _count_items_produced(self, spider: Spider) -> None: if self.items_in_period >= 1: self.items_in_period = 0 else: @@ -109,4 +124,5 @@ class CloseSpider: f"Closing spider since no items were produced in the last " f"{self.timeout_no_item} seconds." ) + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_timeout_no_item") diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 717c249d9..f3ac19623 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -2,18 +2,28 @@ Extension for collecting core stats like items scraped and start/finish times """ -from datetime import datetime, timezone +from __future__ import annotations -from scrapy import signals +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional + +from scrapy import Spider, signals +from scrapy.crawler import Crawler +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class CoreStats: - def __init__(self, stats): - self.stats = stats - self.start_time = None + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats + self.start_time: Optional[datetime] = None @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) @@ -22,11 +32,12 @@ class CoreStats: crawler.signals.connect(o.response_received, signal=signals.response_received) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self.start_time = datetime.now(tz=timezone.utc) self.stats.set_value("start_time", self.start_time, spider=spider) - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: + assert self.start_time is not None finish_time = datetime.now(tz=timezone.utc) elapsed_time = finish_time - self.start_time elapsed_time_seconds = elapsed_time.total_seconds() @@ -36,13 +47,13 @@ class CoreStats: self.stats.set_value("finish_time", finish_time, spider=spider) self.stats.set_value("finish_reason", reason, spider=spider) - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: self.stats.inc_value("item_scraped_count", spider=spider) - def response_received(self, spider): + def response_received(self, spider: Spider) -> None: self.stats.inc_value("response_received_count", spider=spider) - def item_dropped(self, item, spider, exception): + def item_dropped(self, item: Any, spider: Spider, exception: BaseException) -> None: reason = exception.__class__.__name__ self.stats.inc_value("item_dropped_count", spider=spider) self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider) diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 1b6c7777f..26726b662 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -4,22 +4,31 @@ Extensions for debugging Scrapy See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + import logging import signal import sys import threading import traceback from pdb import Pdb +from types import FrameType +from typing import TYPE_CHECKING, Optional +from scrapy.crawler import Crawler from scrapy.utils.engine import format_engine_status from scrapy.utils.trackref import format_live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) class StackTraceDump: - def __init__(self, crawler=None): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler try: signal.signal(signal.SIGUSR2, self.dump_stacktrace) signal.signal(signal.SIGQUIT, self.dump_stacktrace) @@ -28,10 +37,11 @@ class StackTraceDump: pass @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def dump_stacktrace(self, signum, frame): + def dump_stacktrace(self, signum: int, frame: Optional[FrameType]) -> None: + assert self.crawler.engine log_args = { "stackdumps": self._thread_stacks(), "enginestatus": format_engine_status(self.crawler.engine), @@ -44,7 +54,7 @@ class StackTraceDump: extra={"crawler": self.crawler}, ) - def _thread_stacks(self): + def _thread_stacks(self) -> str: id2name = dict((th.ident, th.name) for th in threading.enumerate()) dumps = "" for id_, frame in sys._current_frames().items(): @@ -55,12 +65,13 @@ class StackTraceDump: class Debugger: - def __init__(self): + def __init__(self) -> None: try: signal.signal(signal.SIGUSR2, self._enter_debugger) except AttributeError: # win32 platforms don't support SIGUSR signals pass - def _enter_debugger(self, signum, frame): + def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None: + assert frame Pdb().set_trace(frame.f_back) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 9f63e9c4b..2388afa75 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -1,9 +1,18 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Optional, Tuple, Union from twisted.internet import task -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) @@ -14,30 +23,31 @@ class LogStats: * IPM - Items per Minute """ - def __init__(self, stats, interval=60.0): - self.stats = stats - self.interval = interval - self.multiplier = 60.0 / self.interval - self.task = None + def __init__(self, stats: StatsCollector, interval: float = 60.0): + self.stats: StatsCollector = stats + self.interval: float = interval + self.multiplier: float = 60.0 / self.interval + self.task: Optional[task.LoopingCall] = None @classmethod - def from_crawler(cls, crawler): - interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + def from_crawler(cls, crawler: Crawler) -> Self: + interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured + assert crawler.stats o = cls(crawler.stats, interval) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): - self.pagesprev = 0 - self.itemsprev = 0 + def spider_opened(self, spider: Spider) -> None: + self.pagesprev: int = 0 + self.itemsprev: int = 0 self.task = task.LoopingCall(self.log, spider) self.task.start(self.interval) - def log(self, spider): + def log(self, spider: Spider) -> None: self.calculate_stats() msg = ( @@ -52,14 +62,14 @@ class LogStats: } logger.info(msg, log_args, extra={"spider": spider}) - def calculate_stats(self): - self.items = self.stats.get_value("item_scraped_count", 0) - self.pages = self.stats.get_value("response_received_count", 0) - self.irate = (self.items - self.itemsprev) * self.multiplier - self.prate = (self.pages - self.pagesprev) * self.multiplier + def calculate_stats(self) -> None: + self.items: int = self.stats.get_value("item_scraped_count", 0) + self.pages: int = self.stats.get_value("response_received_count", 0) + self.irate: float = (self.items - self.itemsprev) * self.multiplier + self.prate: float = (self.pages - self.pagesprev) * self.multiplier self.pagesprev, self.itemsprev = self.pages, self.items - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: if self.task and self.task.running: self.task.stop() @@ -67,7 +77,9 @@ class LogStats: self.stats.set_value("responses_per_minute", rpm_final) self.stats.set_value("items_per_minute", ipm_final) - def calculate_final_stats(self, spider): + def calculate_final_stats( + self, spider: Spider + ) -> Union[Tuple[None, None], Tuple[float, float]]: start_time = self.stats.get_value("start_time") finished_time = self.stats.get_value("finished_time") diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 03ede0681..f304e1bf2 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -4,26 +4,36 @@ MemoryDebugger extension See documentation in docs/topics/extensions.rst """ -import gc +from __future__ import annotations -from scrapy import signals +import gc +from typing import TYPE_CHECKING + +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.statscollectors import StatsCollector from scrapy.utils.trackref import live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class MemoryDebugger: - def __init__(self, stats): - self.stats = stats + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("MEMDEBUG_ENABLED"): raise NotConfigured + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: gc.collect() self.stats.set_value( "memdebug/gc_garbage_count", len(gc.garbage), spider=spider diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 4d4501c44..9de06b24d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -4,24 +4,32 @@ MemoryUsage extension See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + import logging import socket import sys from importlib import import_module from pprint import pformat +from typing import TYPE_CHECKING, List from twisted.internet import task from scrapy import signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender from scrapy.utils.engine import get_engine_status +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) class MemoryUsage: - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("MEMUSAGE_ENABLED"): raise NotConfigured try: @@ -30,32 +38,33 @@ class MemoryUsage: except ImportError: raise NotConfigured - self.crawler = crawler - self.warned = False - self.notify_mails = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") - self.limit = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 - self.warning = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 - self.check_interval = crawler.settings.getfloat( + self.crawler: Crawler = crawler + self.warned: bool = False + self.notify_mails: List[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") + self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 + self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 + self.check_interval: float = crawler.settings.getfloat( "MEMUSAGE_CHECK_INTERVAL_SECONDS" ) - self.mail = MailSender.from_settings(crawler.settings) + self.mail: MailSender = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def get_virtual_size(self): - size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss + def get_virtual_size(self) -> int: + size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss if sys.platform != "darwin": # on macOS ru_maxrss is in bytes, on Linux it is in KB size *= 1024 return size - def engine_started(self): + def engine_started(self) -> None: + assert self.crawler.stats self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) - self.tasks = [] + self.tasks: List[task.LoopingCall] = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) @@ -68,15 +77,18 @@ class MemoryUsage: self.tasks.append(tsk) tsk.start(self.check_interval, now=True) - def engine_stopped(self): + def engine_stopped(self) -> None: for tsk in self.tasks: if tsk.running: tsk.stop() - def update(self): + def update(self) -> None: + assert self.crawler.stats self.crawler.stats.max_value("memusage/max", self.get_virtual_size()) - def _check_limit(self): + def _check_limit(self) -> None: + assert self.crawler.engine + assert self.crawler.stats peak_mem_usage = self.get_virtual_size() if peak_mem_usage > self.limit: self.crawler.stats.set_value("memusage/limit_reached", 1) @@ -106,9 +118,10 @@ class MemoryUsage: {"virtualsize": peak_mem_usage / 1024 / 1024}, ) - def _check_warning(self): + def _check_warning(self) -> None: if self.warned: # warn only once return + assert self.crawler.stats if self.get_virtual_size() > self.warning: self.crawler.stats.set_value("memusage/warning_reached", 1) mem = self.warning / 1024 / 1024 @@ -126,8 +139,10 @@ class MemoryUsage: self.crawler.stats.set_value("memusage/warning_notified", 1) self.warned = True - def _send_report(self, rcpts, subject): + def _send_report(self, rcpts: List[str], subject: str) -> None: """send notification mail with some additional useful info""" + assert self.crawler.engine + assert self.crawler.stats stats = self.crawler.stats s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n" s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n" diff --git a/scrapy/mail.py b/scrapy/mail.py index 4b18b6003..dce33fcdf 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -4,6 +4,8 @@ Mail sending helpers See documentation in docs/topics/email.rst """ +from __future__ import annotations + import logging from email import encoders as Encoders from email.mime.base import MIMEBase @@ -12,14 +14,20 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO +from typing import TYPE_CHECKING from twisted import version as twisted_version from twisted.internet import defer, ssl from twisted.python.versions import Version +from scrapy.settings import BaseSettings from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) @@ -56,7 +64,7 @@ class MailSender: self.debug = debug @classmethod - def from_settings(cls, settings): + def from_settings(cls, settings: BaseSettings) -> Self: return cls( smtphost=settings["MAIL_HOST"], mailfrom=settings["MAIL_FROM"], @@ -203,7 +211,7 @@ class MailSender: to_addrs, msg, d, - **factory_keywords + **factory_keywords, ) factory.noisy = False return factory diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 0b2722663..fdcf484d4 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -1,14 +1,15 @@ """Some debugging functions for working with the Scrapy engine""" +from __future__ import annotations + # used in global tests code from time import time # noqa: F401 -from typing import TYPE_CHECKING, Any, List, Tuple +from typing import Any, List, Tuple -if TYPE_CHECKING: - from scrapy.core.engine import ExecutionEngine +from scrapy.core.engine import ExecutionEngine -def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: +def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: """Return a report of the current engine status""" tests = [ "time()-engine.start_time", @@ -37,7 +38,7 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: return checks -def format_engine_status(engine: "ExecutionEngine") -> str: +def format_engine_status(engine: ExecutionEngine) -> str: checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: @@ -47,5 +48,5 @@ def format_engine_status(engine: "ExecutionEngine") -> str: return s -def print_engine_status(engine: "ExecutionEngine") -> None: +def print_engine_status(engine: ExecutionEngine) -> None: print(format_engine_status(engine)) From 6e84648c0717642b069249225857019a87de54b9 Mon Sep 17 00:00:00 2001 From: pengqiseven <134899215+pengqiseven@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:03:06 +0800 Subject: [PATCH 1387/2083] Fix some comments (#6285) Signed-off-by: pengqiseven Co-authored-by: pengqiseven --- sep/sep-018.rst | 2 +- tests/test_utils_python.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sep/sep-018.rst b/sep/sep-018.rst index 9ac62c090..13ab501ed 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -140,7 +140,7 @@ Example: The data flow with Spider Middleware v2 is as follows: -1. When a response arrives from the engine, it it passed through all the spider +1. When a response arrives from the engine, it is passed through all the spider middlewares (in descending order). The result of each middleware ``process_response`` is kept and then returned along with the spider callback result diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 80d2e8da1..1d1d19146 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -175,7 +175,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertTrue(equal_attributes(a, b, ["x", "y"])) a.y = 1 - # differente attributes + # different attributes self.assertFalse(equal_attributes(a, b, ["x", "y"])) # test callable From 642af407049a5ce8e76b7999c68333670c8c8622 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 11 Mar 2024 14:09:09 +0500 Subject: [PATCH 1388/2083] Full typing for scrapy/extensions, part 2. (#6279) --- docs/topics/telnetconsole.rst | 4 +- scrapy/extensions/periodic_log.py | 89 +++++++++++++++++------------ scrapy/extensions/postprocessing.py | 4 +- scrapy/extensions/spiderstate.py | 26 ++++++--- scrapy/extensions/statsmailer.py | 31 +++++++--- scrapy/extensions/telnet.py | 37 +++++++----- scrapy/extensions/throttle.py | 49 ++++++++++------ scrapy/mail.py | 12 ++-- scrapy/utils/reactor.py | 8 +-- 9 files changed, 165 insertions(+), 95 deletions(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 832829b75..0e4a8fa6c 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -172,8 +172,8 @@ TELNETCONSOLE_PORT Default: ``[6023, 6073]`` -The port range to use for the telnet console. If set to ``None`` or ``0``, a -dynamically assigned port is used. +The port range to use for the telnet console. If set to ``None``, a dynamically +assigned port is used. .. setting:: TELNETCONSOLE_HOST diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 2d557f123..9567f948a 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -1,12 +1,22 @@ +from __future__ import annotations + import logging from datetime import datetime, timezone +from json import JSONEncoder +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from twisted.internet import task -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.statscollectors import StatsCollector from scrapy.utils.serialize import ScrapyJSONEncoder +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) @@ -15,32 +25,34 @@ class PeriodicLog: def __init__( self, - stats, - interval=60.0, - ext_stats={}, - ext_delta={}, - ext_timing_enabled=False, + stats: StatsCollector, + interval: float = 60.0, + ext_stats: Dict[str, Any] = {}, + ext_delta: Dict[str, Any] = {}, + ext_timing_enabled: bool = False, ): - self.stats = stats - self.interval = interval - self.multiplier = 60.0 / self.interval - self.task = None - self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4) - self.ext_stats_enabled = bool(ext_stats) - self.ext_stats_include = ext_stats.get("include", []) - self.ext_stats_exclude = ext_stats.get("exclude", []) - self.ext_delta_enabled = bool(ext_delta) - self.ext_delta_include = ext_delta.get("include", []) - self.ext_delta_exclude = ext_delta.get("exclude", []) - self.ext_timing_enabled = ext_timing_enabled + self.stats: StatsCollector = stats + self.interval: float = interval + self.multiplier: float = 60.0 / self.interval + self.task: Optional[task.LoopingCall] = None + self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) + self.ext_stats_enabled: bool = bool(ext_stats) + self.ext_stats_include: List[str] = ext_stats.get("include", []) + self.ext_stats_exclude: List[str] = ext_stats.get("exclude", []) + self.ext_delta_enabled: bool = bool(ext_delta) + self.ext_delta_include: List[str] = ext_delta.get("include", []) + self.ext_delta_exclude: List[str] = ext_delta.get("exclude", []) + self.ext_timing_enabled: bool = ext_timing_enabled @classmethod - def from_crawler(cls, crawler): - interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + def from_crawler(cls, crawler: Crawler) -> Self: + interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured try: - ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS") + ext_stats: Optional[Dict[str, Any]] = crawler.settings.getdict( + "PERIODIC_LOG_STATS" + ) except (TypeError, ValueError): ext_stats = ( {"enabled": True} @@ -48,7 +60,9 @@ class PeriodicLog: else None ) try: - ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA") + ext_delta: Optional[Dict[str, Any]] = crawler.settings.getdict( + "PERIODIC_LOG_DELTA" + ) except (TypeError, ValueError): ext_delta = ( {"enabled": True} @@ -56,11 +70,14 @@ class PeriodicLog: else None ) - ext_timing_enabled = crawler.settings.getbool( + ext_timing_enabled: bool = crawler.settings.getbool( "PERIODIC_LOG_TIMING_ENABLED", False ) if not (ext_stats or ext_delta or ext_timing_enabled): raise NotConfigured + assert crawler.stats + assert ext_stats is not None + assert ext_delta is not None o = cls( crawler.stats, interval, @@ -72,16 +89,16 @@ class PeriodicLog: crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): - self.time_prev = datetime.now(tz=timezone.utc) - self.delta_prev = {} - self.stats_prev = {} + def spider_opened(self, spider: Spider) -> None: + self.time_prev: datetime = datetime.now(tz=timezone.utc) + self.delta_prev: Dict[str, Union[int, float]] = {} + self.stats_prev: Dict[str, Union[int, float]] = {} self.task = task.LoopingCall(self.log) self.task.start(self.interval) - def log(self): - data = {} + def log(self) -> None: + data: Dict[str, Any] = {} if self.ext_timing_enabled: data.update(self.log_timing()) if self.ext_delta_enabled: @@ -90,8 +107,8 @@ class PeriodicLog: data.update(self.log_crawler_stats()) logger.info(self.encoder.encode(data)) - def log_delta(self): - num_stats = { + def log_delta(self) -> Dict[str, Any]: + num_stats: Dict[str, Union[int, float]] = { k: v for k, v in self.stats._stats.items() if isinstance(v, (int, float)) @@ -101,7 +118,7 @@ class PeriodicLog: self.delta_prev = num_stats return {"delta": delta} - def log_timing(self): + def log_timing(self) -> Dict[str, Any]: now = datetime.now(tz=timezone.utc) time = { "log_interval": self.interval, @@ -113,7 +130,7 @@ class PeriodicLog: self.time_prev = now return {"time": time} - def log_crawler_stats(self): + def log_crawler_stats(self) -> Dict[str, Any]: stats = { k: v for k, v in self.stats._stats.items() @@ -121,7 +138,9 @@ class PeriodicLog: } return {"stats": stats} - def param_allowed(self, stat_name, include, exclude): + def param_allowed( + self, stat_name: str, include: List[str], exclude: List[str] + ) -> bool: if not include and not exclude: return True for p in exclude: @@ -134,7 +153,7 @@ class PeriodicLog: return True return False - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: self.log() if self.task and self.task.running: self.task.stop() diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index f8b59827b..7ffbd8bc3 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -6,7 +6,7 @@ from bz2 import BZ2File from gzip import GzipFile from io import IOBase from lzma import LZMAFile -from typing import Any, BinaryIO, Dict, List +from typing import Any, BinaryIO, Dict, List, cast from scrapy.utils.misc import load_object @@ -142,7 +142,7 @@ class PostProcessingManager(IOBase): :return: returns number of bytes written :rtype: int """ - return self.head_plugin.write(data) + return cast(int, self.head_plugin.write(data)) def tell(self) -> int: return self.file.tell() diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 43359401b..c6eb20277 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,19 +1,27 @@ +from __future__ import annotations + import pickle # nosec from pathlib import Path +from typing import TYPE_CHECKING, Optional -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class SpiderState: """Store and load spider state during a scraping job""" - def __init__(self, jobdir=None): - self.jobdir = jobdir + def __init__(self, jobdir: Optional[str] = None): + self.jobdir: Optional[str] = jobdir @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: jobdir = job_dir(crawler.settings) if not jobdir: raise NotConfigured @@ -23,18 +31,20 @@ class SpiderState: crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened) return obj - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> None: if self.jobdir: with Path(self.statefn).open("wb") as f: + assert hasattr(spider, "state") # set in spider_opened pickle.dump(spider.state, f, protocol=4) - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: if self.jobdir and Path(self.statefn).exists(): with Path(self.statefn).open("rb") as f: - spider.state = pickle.load(f) # nosec + spider.state = pickle.load(f) # type: ignore[attr-defined] # nosec else: - spider.state = {} + spider.state = {} # type: ignore[attr-defined] @property def statefn(self) -> str: + assert self.jobdir return str(Path(self.jobdir, "spider.state")) diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 58610c25e..20b8f910c 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -4,28 +4,41 @@ StatsMailer extension sends an email when a spider finishes scraping. Use STATSMAILER_RCPTS setting to enable and give the recipient mail address """ -from scrapy import signals +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Optional + +from twisted.internet.defer import Deferred + +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class StatsMailer: - def __init__(self, stats, recipients, mail): - self.stats = stats - self.recipients = recipients - self.mail = mail + def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender): + self.stats: StatsCollector = stats + self.recipients: List[str] = recipients + self.mail: MailSender = mail @classmethod - def from_crawler(cls, crawler): - recipients = crawler.settings.getlist("STATSMAILER_RCPTS") + def from_crawler(cls, crawler: Crawler) -> Self: + recipients: List[str] = crawler.settings.getlist("STATSMAILER_RCPTS") if not recipients: raise NotConfigured - mail = MailSender.from_settings(crawler.settings) + mail: MailSender = MailSender.from_settings(crawler.settings) + assert crawler.stats o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> Optional[Deferred]: spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index c92b7f5fe..00c69434c 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -4,13 +4,17 @@ Scrapy Telnet Console extension See documentation in docs/topics/telnetconsole.rst """ +from __future__ import annotations + import binascii import logging import os import pprint import traceback +from typing import TYPE_CHECKING, Any, Dict, List from twisted.internet import protocol +from twisted.internet.tcp import Port try: from twisted.conch import manhole, telnet @@ -22,12 +26,16 @@ except (ImportError, SyntaxError): TWISTED_CONCH_AVAILABLE = False from scrapy import signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import defers from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp from scrapy.utils.trackref import print_live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) # signal to update telnet variables @@ -36,7 +44,7 @@ update_telnet_vars = object() class TelnetConsole(protocol.ServerFactory): - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: @@ -44,14 +52,14 @@ class TelnetConsole(protocol.ServerFactory): "TELNETCONSOLE_ENABLED setting is True but required twisted " "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK ) - self.crawler = crawler - self.noisy = False - self.portrange = [ + self.crawler: Crawler = crawler + self.noisy: bool = False + self.portrange: List[int] = [ int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT") ] - self.host = crawler.settings["TELNETCONSOLE_HOST"] - self.username = crawler.settings["TELNETCONSOLE_USERNAME"] - self.password = crawler.settings["TELNETCONSOLE_PASSWORD"] + self.host: str = crawler.settings["TELNETCONSOLE_HOST"] + self.username: str = crawler.settings["TELNETCONSOLE_USERNAME"] + self.password: str = crawler.settings["TELNETCONSOLE_PASSWORD"] if not self.password: self.password = binascii.hexlify(os.urandom(8)).decode("utf8") @@ -61,11 +69,11 @@ class TelnetConsole(protocol.ServerFactory): self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def start_listening(self): - self.port = listen_tcp(self.portrange, self.host, self) + def start_listening(self) -> None: + self.port: Port = listen_tcp(self.portrange, self.host, self) h = self.port.getHost() logger.info( "Telnet console listening on %(host)s:%(port)d", @@ -73,10 +81,10 @@ class TelnetConsole(protocol.ServerFactory): extra={"crawler": self.crawler}, ) - def stop_listening(self): + def stop_listening(self) -> None: self.port.stopListening() - def protocol(self): + def protocol(self) -> telnet.TelnetTransport: # type: ignore[override] class Portal: """An implementation of IPortal""" @@ -95,9 +103,10 @@ class TelnetConsole(protocol.ServerFactory): return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal()) - def _get_telnet_vars(self): + def _get_telnet_vars(self) -> Dict[str, Any]: # Note: if you add entries here also update topics/telnetconsole.rst - telnet_vars = { + assert self.crawler.engine + telnet_vars: Dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, "slot": self.crawler.engine.slot, diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index d217c7a69..bf4e6bb63 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -1,19 +1,29 @@ -import logging +from __future__ import annotations -from scrapy import signals +import logging +from typing import TYPE_CHECKING, Optional, Tuple + +from scrapy import Request, Spider, signals +from scrapy.core.downloader import Slot +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class AutoThrottle: - def __init__(self, crawler): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"): raise NotConfigured - self.debug = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") - self.target_concurrency = crawler.settings.getfloat( + self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") + self.target_concurrency: float = crawler.settings.getfloat( "AUTOTHROTTLE_TARGET_CONCURRENCY" ) if self.target_concurrency <= 0.0: @@ -27,27 +37,29 @@ class AutoThrottle: ) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def _spider_opened(self, spider): + def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) + spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] - def _min_delay(self, spider): + def _min_delay(self, spider: Spider) -> float: s = self.crawler.settings return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY")) - def _max_delay(self, spider): + def _max_delay(self, spider: Spider) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") - def _start_delay(self, spider): + def _start_delay(self, spider: Spider) -> float: return max( self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") ) - def _response_downloaded(self, response, request, spider): + def _response_downloaded( + self, response: Response, request: Request, spider: Spider + ) -> None: key, slot = self._get_slot(request, spider) latency = request.meta.get("download_latency") if latency is None or slot is None: @@ -74,11 +86,16 @@ class AutoThrottle: extra={"spider": spider}, ) - def _get_slot(self, request, spider): - key = request.meta.get("download_slot") + def _get_slot( + self, request: Request, spider: Spider + ) -> Tuple[Optional[str], Optional[Slot]]: + key: Optional[str] = request.meta.get("download_slot") + if key is None: + return None, None + assert self.crawler.engine return key, self.crawler.engine.downloader.slots.get(key) - def _adjust_delay(self, slot, latency, response): + def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None: """Define delay adjustment policy""" # If a server needs `latency` seconds to respond then diff --git a/scrapy/mail.py b/scrapy/mail.py index dce33fcdf..7cb5ef454 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -14,10 +14,11 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from twisted import version as twisted_version -from twisted.internet import defer, ssl +from twisted.internet import ssl +from twisted.internet.defer import Deferred from twisted.python.versions import Version from scrapy.settings import BaseSettings @@ -85,9 +86,10 @@ class MailSender: mimetype="text/plain", charset=None, _callback=None, - ): + ) -> Optional[Deferred]: from twisted.internet import reactor + msg: MIMEBase if attachs: msg = MIMEMultipart() else: @@ -134,7 +136,7 @@ class MailSender: "mailattachs": len(attachs), }, ) - return + return None dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8")) dfd.addCallbacks( @@ -178,7 +180,7 @@ class MailSender: from twisted.internet import reactor msg = BytesIO(msg) - d = defer.Deferred() + d = Deferred() factory = self._create_sender_factory(to_addrs, msg, d) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ad3d1d8bc..6cde49bfe 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -2,17 +2,19 @@ import asyncio import sys from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress -from typing import Any, Callable, Dict, Optional, Sequence, Type +from typing import Any, Callable, Dict, List, Optional, Sequence, Type from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error from twisted.internet.base import DelayedCall +from twisted.internet.protocol import ServerFactory +from twisted.internet.tcp import Port from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object -def listen_tcp(portrange, host, factory): +def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor @@ -20,8 +22,6 @@ def listen_tcp(portrange, host, factory): raise ValueError(f"invalid portrange: {portrange}") if not portrange: return reactor.listenTCP(0, factory, interface=host) - if not hasattr(portrange, "__iter__"): - return reactor.listenTCP(portrange, factory, interface=host) if len(portrange) == 1: return reactor.listenTCP(portrange[0], factory, interface=host) for x in range(portrange[0], portrange[1] + 1): From ab5ea32ffd9cbea22d0fb10ece5258cea207dd61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 11 Mar 2024 23:26:19 +0100 Subject: [PATCH 1389/2083] Fix WindowsRunSpiderCommandTest skip outside Windows for older Twisted. --- tests/test_commands.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index febad21da..ff308c5ac 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -987,10 +987,14 @@ class MySpider(scrapy.Spider): self.assertIn("The value of FOO is 42", log) -@skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = "myspider.pyw" + def setUp(self): + # https://github.com/scrapy/scrapy/issues/6286 + if platform.system() != "Windows": + raise unittest.SkipTest("Windows required for .pyw files") + def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") self.assertIn("start_requests", log) From 188d9a8bb363ab3ff37dbb6020354afbc72ec02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 11 Mar 2024 23:28:57 +0100 Subject: [PATCH 1390/2083] Remove unnecessary comment --- tests/test_commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index ff308c5ac..ae8289ba7 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -991,7 +991,6 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = "myspider.pyw" def setUp(self): - # https://github.com/scrapy/scrapy/issues/6286 if platform.system() != "Windows": raise unittest.SkipTest("Windows required for .pyw files") From e72de11f55dc5f37d449385f20d2ce4c504914d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 12 Mar 2024 09:29:10 +0100 Subject: [PATCH 1391/2083] Add super --- tests/test_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index ae8289ba7..b9d468c66 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -993,6 +993,7 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): def setUp(self): if platform.system() != "Windows": raise unittest.SkipTest("Windows required for .pyw files") + return super().setUp() def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") From d7581c6b41e97fc09c011b089cf34ddd62f41876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 12 Mar 2024 09:44:29 +0100 Subject: [PATCH 1392/2083] Allow disabling the AutoThrottle extension for a given slot (#6246) --- docs/topics/autothrottle.rst | 12 +++++++++++ docs/topics/settings.rst | 12 +++++++++-- scrapy/core/downloader/__init__.py | 19 ++++++++++++++---- scrapy/extensions/throttle.py | 2 +- tests/test_core_downloader.py | 3 ++- tests/test_downloaderslotssettings.py | 29 ++++++++++++++++++++++++++- tests/test_extension_throttle.py | 19 ++++++++++-------- 7 files changed, 79 insertions(+), 17 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 5370d77b3..8a13b8976 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -47,6 +47,18 @@ effect, but there are some important differences: AutoThrottle doesn't have these issues. +Disabling throttling on a downloader slot +========================================= + +It is possible to disable AutoThrottle for a specific download slot at run time +by setting its ``throttle`` attribute to ``False``, e.g. using +:setting:`DOWNLOAD_SLOTS`. + +Note, however, that AutoThrottle still determines the starting delay of every +slot by setting the ``download_delay`` attribute on the running spider. You +might want to set a custom value for the ``delay`` attribute of the slot, e.g. +using :setting:`DOWNLOAD_SLOTS`. + Throttling algorithm ==================== diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 439aedc18..2bd9cf1ed 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -835,7 +835,7 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2: .. setting:: DOWNLOAD_SLOTS DOWNLOAD_SLOTS ----------------- +-------------- Default: ``{}`` @@ -844,7 +844,12 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: .. code-block:: python DOWNLOAD_SLOTS = { - "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, + "quotes.toscrape.com": { + "concurrency": 1, + "delay": 2, + "randomize_delay": False, + "throttle": False, + }, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, } @@ -856,6 +861,9 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` + There is no global setting for ``throttle``, whose default value is + ``None``. + .. setting:: DOWNLOAD_TIMEOUT diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 666282856..ecd3e8b56 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -2,7 +2,7 @@ import random from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast +from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast from twisted.internet import task from twisted.internet.defer import Deferred @@ -24,10 +24,18 @@ if TYPE_CHECKING: class Slot: """Downloader slot""" - def __init__(self, concurrency: int, delay: float, randomize_delay: bool): + def __init__( + self, + concurrency: int, + delay: float, + randomize_delay: bool, + *, + throttle: Optional[bool] = None, + ): self.concurrency: int = concurrency self.delay: float = delay self.randomize_delay: bool = randomize_delay + self.throttle = throttle self.active: Set[Request] = set() self.queue: Deque[Tuple[Request, Deferred]] = deque() @@ -52,13 +60,15 @@ class Slot: return ( f"{cls_name}(concurrency={self.concurrency!r}, " f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r})" + f"randomize_delay={self.randomize_delay!r}, " + f"throttle={self.throttle!r})" ) def __str__(self) -> str: return ( f"" @@ -127,7 +137,8 @@ class Downloader: slot_settings.get("delay", delay), ) randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) - new_slot = Slot(conc, delay, randomize_delay) + throttle = slot_settings.get("throttle", None) + new_slot = Slot(conc, delay, randomize_delay, throttle=throttle) self.slots[key] = new_slot return key, self.slots[key] diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index bf4e6bb63..217e61a81 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -62,7 +62,7 @@ class AutoThrottle: ) -> None: key, slot = self._get_slot(request, spider) latency = request.meta.get("download_latency") - if latency is None or slot is None: + if latency is None or slot is None or slot.throttle is False: return olddelay = slot.delay diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 9a6e9e4ff..81cff4947 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -7,5 +7,6 @@ class SlotTest(unittest.TestCase): def test_repr(self): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) self.assertEqual( - repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)" + repr(slot), + "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)", ) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9d4072d19..ea8c5b4f0 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -3,8 +3,10 @@ import time from twisted.internet import defer from twisted.trial.unittest import TestCase +from scrapy import Request +from scrapy.core.downloader import Downloader, Slot from scrapy.crawler import CrawlerRunner -from scrapy.http import Request +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import MetaSpider @@ -20,6 +22,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): "concurrency": 1, "delay": 2, "randomize_delay": False, + "throttle": False, }, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, }, @@ -70,3 +73,27 @@ class CrawlTestCase(TestCase): } self.assertTrue(max(list(error_delta.values())) < tolerance) + + +def test_params(): + params = { + "concurrency": 1, + "delay": 2, + "randomize_delay": False, + "throttle": False, + } + settings = { + "DOWNLOAD_SLOTS": { + "example.com": params, + }, + } + crawler = get_crawler(settings_dict=settings) + downloader = Downloader(crawler) + downloader._slot_gc_loop.stop() # Prevent an unclean reactor. + request = Request("https://example.com") + _, actual = downloader._get_slot(request, spider=None) + expected = Slot(**params) + for param in params: + assert getattr(expected, param) == getattr( + actual, param + ), f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}" diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index dae4ea966..722a05c26 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -157,16 +157,17 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) @pytest.mark.parametrize( - ("meta", "slot"), + ("meta", "slot", "throttle"), ( - ({}, None), - ({"download_latency": 1.0}, None), - ({"download_slot": "foo"}, None), - ({"download_slot": "foo"}, "foo"), - ({"download_latency": 1.0, "download_slot": "foo"}, None), + ({}, None, None), + ({"download_latency": 1.0}, None, None), + ({"download_slot": "foo"}, None, None), + ({"download_slot": "foo"}, "foo", None), + ({"download_latency": 1.0, "download_slot": "foo"}, None, None), + ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False), ), ) -def test_skipped(meta, slot): +def test_skipped(meta, slot, throttle): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) spider = TestSpider() @@ -177,7 +178,9 @@ def test_skipped(meta, slot): crawler.engine.downloader = Mock() crawler.engine.downloader.slots = {} if slot is not None: - crawler.engine.downloader.slots[slot] = object() + _slot = Mock() + _slot.throttle = throttle + crawler.engine.downloader.slots[slot] = _slot at._adjust_delay = None # Raise exception if called. at._response_downloaded(None, request, spider) From 4460d3ed9631f8409c24f78f1abb36345967b5d5 Mon Sep 17 00:00:00 2001 From: Lucas Belo <144740771+lucas-belo@users.noreply.github.com> Date: Wed, 13 Mar 2024 03:22:48 -0300 Subject: [PATCH 1393/2083] Remove tests/requirements.txt and refactor extra deps (#6272) Co-authored-by: lucasbelo777 --- conftest.py | 18 ++++++------------ tests/requirements.txt | 17 ----------------- tox.ini | 32 ++++++++++++++++++++++++++++---- 3 files changed, 34 insertions(+), 33 deletions(-) delete mode 100644 tests/requirements.txt diff --git a/conftest.py b/conftest.py index 2bfa46f5a..2ab3dffd4 100644 --- a/conftest.py +++ b/conftest.py @@ -1,10 +1,6 @@ -import platform -import sys from pathlib import Path import pytest -from twisted import version as twisted_version -from twisted.python.versions import Version from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor @@ -85,14 +81,12 @@ def only_not_asyncio(request, reactor_pytest): def requires_uvloop(request): if not request.node.get_closest_marker("requires_uvloop"): return - if sys.implementation.name == "pypy": - pytest.skip("uvloop does not support pypy properly") - if platform.system() == "Windows": - pytest.skip("uvloop does not support Windows") - if twisted_version == Version("twisted", 21, 2, 0): - pytest.skip("https://twistedmatrix.com/trac/ticket/10106") - if sys.version_info >= (3, 12): - pytest.skip("uvloop doesn't support Python 3.12 yet") + try: + import uvloop + + del uvloop + except ImportError: + pytest.skip("uvloop is not installed") def pytest_configure(config): diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index ca5f6ddbd..000000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Tests requirements -attrs -pexpect >= 4.8.0 -pyftpdlib >= 1.5.8 -pytest -pytest-cov==4.0.0 -pytest-xdist -sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 -testfixtures -uvloop; platform_system != "Windows" - -bpython # optional for shell wrapper tests -brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests -zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -ipython -pywin32; sys_platform == "win32" diff --git a/tox.ini b/tox.ini index 237aa489c..b5effb527 100644 --- a/tox.ini +++ b/tox.ini @@ -7,9 +7,23 @@ envlist = pre-commit,pylint,typing,py minversion = 1.7.0 +[test-requirements] +deps = + attrs + pexpect >= 4.8.0 + pyftpdlib >= 1.5.8 + pygments + pytest + pytest-cov==4.0.0 + pytest-xdist + sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 + testfixtures + pywin32; sys_platform == "win32" + [testenv] deps = - -rtests/requirements.txt + {[test-requirements]deps} + # mitmproxy does not support PyPy mitmproxy; implementation_name != 'pypy' passenv = @@ -81,7 +95,7 @@ deps = w3lib==1.17.0 zope.interface==5.1.0 lxml==4.4.1 - -rtests/requirements.txt + {[test-requirements]deps} # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment @@ -124,8 +138,12 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - brotli - zstandard + uvloop; platform_system != "Windows" + bpython # optional for shell wrapper tests + brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests + zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + ipython [testenv:extra-deps-pinned] basepython = python3.8 @@ -136,6 +154,12 @@ deps = Pillow==7.1.0 robotexclusionrulesparser==1.6.2 brotlipy + uvloop==0.14.0; platform_system != "Windows" + bpython==0.7.1 + zstandard==0.1; implementation_name != 'pypy' + ipython==2.0.0 + brotli==0.5.2; implementation_name != 'pypy' + brotlicffi==0.8.0; implementation_name == 'pypy' install_command = {[pinned]install_command} setenv = {[pinned]setenv} From 8d917c0b55cbeebd284f7446ccc0f90af60729ac Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 14 Mar 2024 17:44:57 +0500 Subject: [PATCH 1394/2083] Run black. --- tests/test_command_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index effafae54..b0f1cd38a 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -1,6 +1,6 @@ import sys from io import StringIO -from unittest.mock import Mock, call, PropertyMock, patch +from unittest.mock import Mock, PropertyMock, call, patch from scrapy.commands.check import Command, TextTestResult from tests.test_commands import CommandTest From f7bf3f726e3f19bf68b5e7e460f116850896eb42 Mon Sep 17 00:00:00 2001 From: igeni Date: Mon, 1 Apr 2024 16:37:23 +0300 Subject: [PATCH 1395/2083] modified string's concat to f-strings (#6296) --- tests/test_selector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_selector.py b/tests/test_selector.py index 85527bba9..1b5f3f018 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -80,10 +80,10 @@ class SelectorTestCase(unittest.TestCase): meta = ( '' ) - head = "" + meta + "" + head = f"{meta}" body_content = '\xa3' - body = "" + body_content + "" - html = "" + head + body + "" + body = f"{body_content}" + html = f"{head}{body}" encoding = "utf-8" html_utf8 = html.encode(encoding) From 7b37dcd80d3783d3a21ff524e572019ff2b9e0ed Mon Sep 17 00:00:00 2001 From: Lorenzo Verardo Date: Thu, 4 Apr 2024 12:22:50 +0200 Subject: [PATCH 1396/2083] Handle robots.txt files not UTF-8 encoded --- scrapy/robotstxt.py | 2 +- tests/test_robotstxt_interface.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 6ea2bfd97..ad06137e2 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -23,7 +23,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): if to_native_str_type: robotstxt_body = to_unicode(robotstxt_body) else: - robotstxt_body = robotstxt_body.decode("utf-8") + robotstxt_body = robotstxt_body.decode("utf-8", errors="ignore") except UnicodeDecodeError: # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. # Switch to 'allow all' state. diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index d7a923085..6ad30deed 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,5 +1,7 @@ from twisted.trial import unittest +from scrapy.robotstxt import decode_robotstxt + def reppy_available(): # check if reppy parser is installed @@ -141,6 +143,25 @@ class BaseRobotParserTest: ) +class DecodeRobotsTxtTest(unittest.TestCase): + def test_native_string_conversion(self): + robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + decoded_content = decode_robotstxt( + robotstxt_body, spider=None, to_native_str_type=True + ) + self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + + def test_decode_utf8(self): + robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + decoded_content = decode_robotstxt(robotstxt_body, spider=None) + self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + + def test_decode_non_utf8(self): + robotstxt_body = b"User-agent: *\n\xFFDisallow: /\n" + decoded_content = decode_robotstxt(robotstxt_body, spider=None) + self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + + class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import PythonRobotParser From 48c5a8c98f545e35708a580e724d0b8e1ada5e6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Ruiz?= Date: Mon, 8 Apr 2024 11:47:53 +0200 Subject: [PATCH 1397/2083] Fix WrappedRequest.get_header raising TypeError if default is None (#6310) --- scrapy/http/cookies.py | 3 ++- tests/test_http_cookies.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 2595f328e..72855bad5 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -170,7 +170,8 @@ class WrappedRequest: return name in self.request.headers def get_header(self, name, default=None): - return to_unicode(self.request.headers.get(name, default), errors="replace") + value = self.request.headers.get(name, default) + return to_unicode(value, errors="replace") if value is not None else None def header_items(self): return [ diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 8b5554914..932644320 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -43,6 +43,13 @@ class WrappedRequestTest(TestCase): def test_get_header(self): self.assertEqual(self.wrapped.get_header("content-type"), "text/html") self.assertEqual(self.wrapped.get_header("xxxxx", "def"), "def") + self.assertEqual(self.wrapped.get_header("xxxxx"), None) + wrapped = WrappedRequest( + Request( + "http://www.example.com/page.html", headers={"empty-binary-header": b""} + ) + ) + self.assertEqual(wrapped.get_header("empty-binary-header"), "") def test_header_items(self): self.assertEqual(self.wrapped.header_items(), [("Content-Type", ["text/html"])]) From 1d11ea3a54607b436f9a88f07911902a4882f0e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 11 Apr 2024 12:19:32 +0200 Subject: [PATCH 1398/2083] Update practices.rst --- docs/topics/practices.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index b1b8c9e9c..cd359b147 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -289,7 +289,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites: services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. * use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy - plugin `__ + plugin `__ and additional + features, like `AI web scraping `__ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. From 5f67c01d1d2cc8a0104361b0323d755e22ed93dc Mon Sep 17 00:00:00 2001 From: TechVest <166724172+TechVest@users.noreply.github.com> Date: Wed, 17 Apr 2024 16:56:26 +0800 Subject: [PATCH 1399/2083] chore: fix some typos in comments (#6317) Signed-off-by: TechVest --- .git-blame-ignore-revs | 4 ++-- docs/topics/broad-crawls.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index dbcebfa0a..a9fc3dd68 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,7 +1,7 @@ # .git-blame-ignore-revs # adding black formatter to all the code e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d -# re applying black to the code with default line length +# reapplying black to the code with default line length 303f0a70fcf8067adf0a909c2096a5009162383a -# reaplying black again and removing line length on pre-commit black config +# reapplying black again and removing line length on pre-commit black config c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962 \ No newline at end of file diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 8be89feb2..750aae554 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -116,7 +116,7 @@ Reduce log level When doing broad crawls you are often only interested in the crawl rates you get and any errors found. These stats are reported by Scrapy when using the ``INFO`` log level. In order to save CPU (and log storage requirements) you -should not use ``DEBUG`` log level when preforming large broad crawls in +should not use ``DEBUG`` log level when performing large broad crawls in production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. From b1fe97dc6c8509d58b29c61cf7801eeee1b409a9 Mon Sep 17 00:00:00 2001 From: kokobhara <146670393+kokobhara@users.noreply.github.com> Date: Wed, 17 Apr 2024 16:44:57 +0530 Subject: [PATCH 1400/2083] Fix test expectations (#6316) --- tests/test_utils_python.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 1d1d19146..4c60deafe 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -239,8 +239,11 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"]) if platform.python_implementation() == "CPython": - # doesn't work on CPython: https://bugs.python.org/issue42785 - self.assertEqual(get_func_args(operator.itemgetter(2)), []) + # This didn't work on older versions of CPython: https://github.com/python/cpython/issues/86951 + self.assertIn( + get_func_args(operator.itemgetter(2), stripself=True), + [[], ["args", "kwargs"]], + ) elif platform.python_implementation() == "PyPy": self.assertEqual( get_func_args(operator.itemgetter(2), stripself=True), ["obj"] From a166e9739962ec7cca8a655e5f20a18a1bce7d14 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 12:39:22 +0500 Subject: [PATCH 1401/2083] Remove the auto-generated copyright years from the docs footer. (#6322) --- docs/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 399078010..dcd2c9a3a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,7 +10,6 @@ # serve to show the default. import sys -from datetime import datetime from pathlib import Path # If your extensions are in another directory, add it here. If the directory @@ -48,7 +47,7 @@ master_doc = "index" # General information about the project. project = "Scrapy" -copyright = f"2008–{datetime.now().year}, Scrapy developers" +copyright = "Scrapy developers" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 57acad3c38602f4399c307c2c002f9eddde97cbc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 12:43:45 +0500 Subject: [PATCH 1402/2083] Full typing for scrapy/extensions, part 3. (#6325) --- scrapy/extensions/feedexport.py | 331 +++++++++++++++++++--------- scrapy/extensions/httpcache.py | 153 ++++++++----- scrapy/extensions/postprocessing.py | 4 +- scrapy/utils/iterators.py | 6 +- tox.ini | 14 +- 5 files changed, 328 insertions(+), 180 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3b0dd804e..97f39afe7 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -4,6 +4,8 @@ Feed Exports extension See documentation in docs/topics/feed-exports.rst """ +from __future__ import annotations + import logging import re import sys @@ -11,18 +13,36 @@ import warnings from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Type, Union +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Protocol, + Tuple, + Type, + TypeVar, + Union, + cast, +) from urllib.parse import unquote, urlparse -from twisted.internet import defer, threads -from twisted.internet.defer import DeferredList +from twisted.internet import threads +from twisted.internet.defer import Deferred, DeferredList, maybeDeferred +from twisted.python.failure import Failure from w3lib.url import file_uri_to_path from zope.interface import Interface, implementer from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import BaseItemExporter from scrapy.extensions.postprocessing import PostProcessingManager +from scrapy.settings import BaseSettings, Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import maybe_deferred_to_future @@ -32,6 +52,12 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + from _typeshed import OpenBinaryMode + + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) try: @@ -41,8 +67,19 @@ try: except ImportError: IS_BOTO3_AVAILABLE = False +UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]] -def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): +_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") + + +def build_storage( + builder: Callable[..., _StorageT], + uri: str, + *args: Any, + feed_options: Optional[Dict[str, Any]] = None, + preargs: Iterable[Any] = (), + **kwargs: Any, +) -> _StorageT: kwargs["feed_options"] = feed_options return builder(*preargs, uri, *args, **kwargs) @@ -56,10 +93,10 @@ class ItemFilter: :type feed_options: dict """ - feed_options: Optional[dict] - item_classes: Tuple + feed_options: Optional[Dict[str, Any]] + item_classes: Tuple[type, ...] - def __init__(self, feed_options: Optional[dict]) -> None: + def __init__(self, feed_options: Optional[Dict[str, Any]]) -> None: self.feed_options = feed_options if feed_options is not None: self.item_classes = tuple( @@ -98,28 +135,49 @@ class IFeedStorage(Interface): """Store the given file stream""" +class FeedStorageProtocol(Protocol): + """Reimplementation of ``IFeedStorage`` that can be used in type hints.""" + + def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + """Initialize the storage with the parameters given in the URI and the + feed-specific options (see :setting:`FEEDS`)""" + + def open(self, spider: Spider) -> IO[bytes]: + """Open the storage for the given spider. It must return a file-like + object that will be used for the exporters""" + + def store(self, file: IO[bytes]) -> Optional[Deferred]: + """Store the given file stream""" + + @implementer(IFeedStorage) class BlockingFeedStorage: - def open(self, spider): + def open(self, spider: Spider) -> IO[bytes]: path = spider.crawler.settings["FEED_TEMPDIR"] if path and not Path(path).is_dir(): raise OSError("Not a Directory: " + str(path)) return NamedTemporaryFile(prefix="feed-", dir=path) - def store(self, file): + def store(self, file: IO[bytes]) -> Optional[Deferred]: return threads.deferToThread(self._store_in_thread, file) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: raise NotImplementedError @implementer(IFeedStorage) class StdoutFeedStorage: - def __init__(self, uri, _stdout=None, *, feed_options=None): + def __init__( + self, + uri: str, + _stdout: Optional[IO[bytes]] = None, + *, + feed_options: Optional[Dict[str, Any]] = None, + ): if not _stdout: _stdout = sys.stdout.buffer - self._stdout = _stdout + self._stdout: IO[bytes] = _stdout if feed_options and feed_options.get("overwrite", False) is True: logger.warning( "Standard output (stdout) storage does not support " @@ -128,54 +186,58 @@ class StdoutFeedStorage: "it to False." ) - def open(self, spider): + def open(self, spider: Spider) -> IO[bytes]: return self._stdout - def store(self, file): + def store(self, file: IO[bytes]) -> Optional[Deferred]: pass @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri, *, feed_options=None): - self.path = file_uri_to_path(uri) + def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + self.path: str = file_uri_to_path(uri) feed_options = feed_options or {} - self.write_mode = "wb" if feed_options.get("overwrite", False) else "ab" + self.write_mode: OpenBinaryMode = ( + "wb" if feed_options.get("overwrite", False) else "ab" + ) - def open(self, spider) -> IO[Any]: + def open(self, spider: Spider) -> IO[bytes]: dirname = Path(self.path).parent if dirname and not dirname.exists(): dirname.mkdir(parents=True) return Path(self.path).open(self.write_mode) - def store(self, file): + def store(self, file: IO[bytes]) -> Optional[Deferred]: file.close() + return None class S3FeedStorage(BlockingFeedStorage): def __init__( self, - uri, - access_key=None, - secret_key=None, - acl=None, - endpoint_url=None, + uri: str, + access_key: Optional[str] = None, + secret_key: Optional[str] = None, + acl: Optional[str] = None, + endpoint_url: Optional[str] = None, *, - feed_options=None, - session_token=None, - region_name=None, + feed_options: Optional[Dict[str, Any]] = None, + session_token: Optional[str] = None, + region_name: Optional[str] = None, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") u = urlparse(uri) - self.bucketname = u.hostname - self.access_key = u.username or access_key - self.secret_key = u.password or secret_key - self.session_token = session_token - self.keyname = u.path[1:] # remove first "/" - self.acl = acl - self.endpoint_url = endpoint_url - self.region_name = region_name + assert u.hostname + self.bucketname: str = u.hostname + self.access_key: Optional[str] = u.username or access_key + self.secret_key: Optional[str] = u.password or secret_key + self.session_token: Optional[str] = session_token + self.keyname: str = u.path[1:] # remove first "/" + self.acl: Optional[str] = acl + self.endpoint_url: Optional[str] = endpoint_url + self.region_name: Optional[str] = region_name if IS_BOTO3_AVAILABLE: import boto3.session @@ -218,7 +280,13 @@ class S3FeedStorage(BlockingFeedStorage): ) @classmethod - def from_crawler(cls, crawler, uri, *, feed_options=None): + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: Optional[Dict[str, Any]] = None, + ) -> Self: return build_storage( cls, uri, @@ -231,8 +299,9 @@ class S3FeedStorage(BlockingFeedStorage): feed_options=feed_options, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) + kwargs: Dict[str, Any] if IS_BOTO3_AVAILABLE: kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} self.s3_client.upload_fileobj( @@ -247,22 +316,23 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri, project_id, acl): - self.project_id = project_id - self.acl = acl + def __init__(self, uri: str, project_id: Optional[str], acl: Optional[str]): + self.project_id: Optional[str] = project_id + self.acl: Optional[str] = acl u = urlparse(uri) - self.bucket_name = u.hostname - self.blob_name = u.path[1:] # remove first "/" + assert u.hostname + self.bucket_name: str = u.hostname + self.blob_name: str = u.path[1:] # remove first "/" @classmethod - def from_crawler(cls, crawler, uri): + def from_crawler(cls, crawler: Crawler, uri: str) -> Self: return cls( uri, crawler.settings["GCS_PROJECT_ID"], crawler.settings["FEED_STORAGE_GCS_ACL"] or None, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) from google.cloud.storage import Client @@ -292,7 +362,13 @@ class FTPFeedStorage(BlockingFeedStorage): self.overwrite: bool = not feed_options or feed_options.get("overwrite", True) @classmethod - def from_crawler(cls, crawler, uri, *, feed_options=None): + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: Optional[Dict[str, Any]] = None, + ) -> Self: return build_storage( cls, uri, @@ -300,7 +376,7 @@ class FTPFeedStorage(BlockingFeedStorage): feed_options=feed_options, ) - def _store_in_thread(self, file): + def _store_in_thread(self, file: IO[bytes]) -> None: ftp_store_file( path=self.path, file=file, @@ -316,46 +392,51 @@ class FTPFeedStorage(BlockingFeedStorage): class FeedSlot: def __init__( self, - storage, - uri, - format, - store_empty, - batch_id, - uri_template, - filter, - feed_options, - spider, + storage: FeedStorageProtocol, + uri: str, + format: str, + store_empty: bool, + batch_id: int, + uri_template: str, + filter: ItemFilter, + feed_options: Dict[str, Any], + spider: Spider, exporters: Dict[str, Type[BaseItemExporter]], - settings, - crawler, + settings: BaseSettings, + crawler: Crawler, ): - self.file = None + self.file: Optional[IO[bytes]] = None self.exporter: Optional[BaseItemExporter] = None - self.storage = storage + self.storage: FeedStorageProtocol = storage # feed params - self.batch_id = batch_id - self.format = format - self.store_empty = store_empty - self.uri_template = uri_template - self.uri = uri - self.filter = filter + self.batch_id: int = batch_id + self.format: str = format + self.store_empty: bool = store_empty + self.uri_template: str = uri_template + self.uri: str = uri + self.filter: ItemFilter = filter # exporter params - self.feed_options = feed_options - self.spider = spider + self.feed_options: Dict[str, Any] = feed_options + self.spider: Spider = spider self.exporters: Dict[str, Type[BaseItemExporter]] = exporters - self.settings = settings - self.crawler = crawler + self.settings: BaseSettings = settings + self.crawler: Crawler = crawler # flags - self.itemcount = 0 - self._exporting = False - self._fileloaded = False + self.itemcount: int = 0 + self._exporting: bool = False + self._fileloaded: bool = False - def start_exporting(self): + def start_exporting(self) -> None: if not self._fileloaded: self.file = self.storage.open(self.spider) if "postprocessing" in self.feed_options: - self.file = PostProcessingManager( - self.feed_options["postprocessing"], self.file, self.feed_options + self.file = cast( + IO[bytes], + PostProcessingManager( + self.feed_options["postprocessing"], + self.file, + self.feed_options, + ), ) self.exporter = self._get_exporter( file=self.file, @@ -368,17 +449,23 @@ class FeedSlot: self._fileloaded = True if not self._exporting: + assert self.exporter self.exporter.start_exporting() self._exporting = True - def _get_instance(self, objcls, *args, **kwargs): + def _get_instance( + self, objcls: Type[BaseItemExporter], *args: Any, **kwargs: Any + ) -> BaseItemExporter: return build_from_crawler(objcls, self.crawler, *args, **kwargs) - def _get_exporter(self, file, format, *args, **kwargs) -> BaseItemExporter: + def _get_exporter( + self, file: IO[bytes], format: str, *args: Any, **kwargs: Any + ) -> BaseItemExporter: return self._get_instance(self.exporters[format], file, *args, **kwargs) - def finish_exporting(self): + def finish_exporting(self) -> None: if self._exporting: + assert self.exporter self.exporter.finish_exporting() self._exporting = False @@ -390,22 +477,22 @@ _FeedSlot = create_deprecated_class( class FeedExporter: - _pending_deferreds: List[defer.Deferred] = [] + _pending_deferreds: List[Deferred] = [] @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: exporter = cls(crawler) crawler.signals.connect(exporter.open_spider, signals.spider_opened) crawler.signals.connect(exporter.close_spider, signals.spider_closed) crawler.signals.connect(exporter.item_scraped, signals.item_scraped) return exporter - def __init__(self, crawler): - self.crawler = crawler - self.settings = crawler.settings + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler + self.settings: Settings = crawler.settings self.feeds = {} - self.slots = [] - self.filters = {} + self.slots: List[FeedSlot] = [] + self.filters: Dict[str, ItemFilter] = {} if not self.settings["FEEDS"] and not self.settings["FEED_URI"]: raise NotConfigured @@ -437,8 +524,12 @@ class FeedExporter: ) self.filters[uri] = self._load_filter(feed_options) - self.storages = self._load_components("FEED_STORAGES") - self.exporters = self._load_components("FEED_EXPORTERS") + self.storages: Dict[str, Type[FeedStorageProtocol]] = self._load_components( + "FEED_STORAGES" + ) + self.exporters: Dict[str, Type[BaseItemExporter]] = self._load_components( + "FEED_EXPORTERS" + ) for uri, feed_options in self.feeds.items(): if not self._storage_supported(uri, feed_options): raise NotConfigured @@ -447,7 +538,7 @@ class FeedExporter: if not self._exporter_supported(feed_options["format"]): raise NotConfigured - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: for uri, feed_options in self.feeds.items(): uri_params = self._get_uri_params(spider, feed_options["uri_params"]) self.slots.append( @@ -460,7 +551,7 @@ class FeedExporter: ) ) - async def close_spider(self, spider): + async def close_spider(self, spider: Spider) -> None: for slot in self.slots: self._close_slot(slot, spider) @@ -473,8 +564,9 @@ class FeedExporter: self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) ) - def _close_slot(self, slot, spider): - def get_file(slot_): + def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]: + def get_file(slot_: FeedSlot) -> IO[bytes]: + assert slot_.file if isinstance(slot_.file, PostProcessingManager): slot_.file.close() return slot_.file.file @@ -492,7 +584,7 @@ class FeedExporter: return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d = defer.maybeDeferred(slot.storage.store, get_file(slot)) + d: Deferred = maybeDeferred(slot.storage.store, get_file(slot)) d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ @@ -510,20 +602,33 @@ class FeedExporter: return d - def _handle_store_error(self, f, logmsg, spider, slot_type): + def _handle_store_error( + self, f: Failure, logmsg: str, spider: Spider, slot_type: str + ) -> None: logger.error( "Error storing %s", logmsg, exc_info=failure_to_exc_info(f), extra={"spider": spider}, ) + assert self.crawler.stats self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") - def _handle_store_success(self, f, logmsg, spider, slot_type): + def _handle_store_success( + self, f: Failure, logmsg: str, spider: Spider, slot_type: str + ) -> None: logger.info("Stored %s", logmsg, extra={"spider": spider}) + assert self.crawler.stats self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") - def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): + def _start_new_batch( + self, + batch_id: int, + uri: str, + feed_options: Dict[str, Any], + spider: Spider, + uri_template: str, + ) -> FeedSlot: """ Redirect the output data stream to a new file. Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified @@ -546,11 +651,11 @@ class FeedExporter: spider=spider, exporters=self.exporters, settings=self.settings, - crawler=getattr(self, "crawler", None), + crawler=self.crawler, ) return slot - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: slots = [] for slot in self.slots: if not slot.filter.accepts(item): @@ -560,6 +665,7 @@ class FeedExporter: continue slot.start_exporting() + assert slot.exporter slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one @@ -584,7 +690,7 @@ class FeedExporter: slots.append(slot) self.slots = slots - def _load_components(self, setting_prefix): + def _load_components(self, setting_prefix: str) -> Dict[str, Any]: conf = without_none_values(self.settings.getwithbase(setting_prefix)) d = {} for k, v in conf.items(): @@ -594,12 +700,13 @@ class FeedExporter: pass return d - def _exporter_supported(self, format): + def _exporter_supported(self, format: str) -> bool: if format in self.exporters: return True logger.error("Unknown feed format: %(format)s", {"format": format}) + return False - def _settings_are_valid(self): + def _settings_are_valid(self) -> bool: """ If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain %(batch_time)s or %(batch_id)d to distinguish different files of partial output @@ -617,7 +724,7 @@ class FeedExporter: return False return True - def _storage_supported(self, uri, feed_options): + def _storage_supported(self, uri: str, feed_options: Dict[str, Any]) -> bool: scheme = urlparse(uri).scheme if scheme in self.storages or PureWindowsPath(uri).drive: try: @@ -630,8 +737,11 @@ class FeedExporter: ) else: logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme}) + return False - def _get_storage(self, uri, feed_options): + def _get_storage( + self, uri: str, feed_options: Dict[str, Any] + ) -> FeedStorageProtocol: """Fork of create_instance specific to feed storage classes It supports not passing the *feed_options* parameters to classes that @@ -640,11 +750,14 @@ class FeedExporter: feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) crawler = getattr(self, "crawler", None) - def build_instance(builder, *preargs): + def build_instance( + builder: Type[FeedStorageProtocol], *preargs: Any + ) -> FeedStorageProtocol: return build_storage( builder, uri, feed_options=feed_options, preargs=preargs ) + instance: FeedStorageProtocol if crawler and hasattr(feedcls, "from_crawler"): instance = build_instance(feedcls.from_crawler, crawler) method_name = "from_crawler" @@ -661,9 +774,9 @@ class FeedExporter: def _get_uri_params( self, spider: Spider, - uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]], + uri_params_function: Union[str, UriParamsCallableT, None], slot: Optional[FeedSlot] = None, - ) -> dict: + ) -> Dict[str, Any]: params = {} for k in dir(spider): params[k] = getattr(spider, k) @@ -671,7 +784,7 @@ class FeedExporter: params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") params["batch_id"] = slot.batch_id + 1 if slot is not None else 1 - uripar_function = ( + uripar_function: UriParamsCallableT = ( load_object(uri_params_function) if uri_params_function else lambda params, _: params @@ -679,7 +792,9 @@ class FeedExporter: new_params = uripar_function(params, spider) return new_params if new_params is not None else params - def _load_filter(self, feed_options): + def _load_filter(self, feed_options: Dict[str, Any]) -> ItemFilter: # load the item filter if declared else load the default filter class - item_filter_class = load_object(feed_options.get("item_filter", ItemFilter)) + item_filter_class: Type[ItemFilter] = load_object( + feed_options.get("item_filter", ItemFilter) + ) return item_filter_class(feed_options) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 335728502..dd5bce24f 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,10 +1,13 @@ import gzip import logging +import os import pickle # nosec from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path from time import time +from types import ModuleType +from typing import IO, TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict @@ -12,49 +15,65 @@ from w3lib.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.http import Headers, Response from scrapy.http.request import Request from scrapy.responsetypes import responsetypes +from scrapy.settings import BaseSettings from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.request import RequestFingerprinter + +if TYPE_CHECKING: + # typing.Concatenate requires Python 3.10 + from typing_extensions import Concatenate + logger = logging.getLogger(__name__) class DummyPolicy: - def __init__(self, settings): - self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self.ignore_http_codes = [ + def __init__(self, settings: BaseSettings): + self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self.ignore_http_codes: List[int] = [ int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES") ] - def should_cache_request(self, request): + def should_cache_request(self, request: Request) -> bool: return urlparse_cached(request).scheme not in self.ignore_schemes - def should_cache_response(self, response, request): + def should_cache_response(self, response: Response, request: Request) -> bool: return response.status not in self.ignore_http_codes - def is_cached_response_fresh(self, cachedresponse, request): + def is_cached_response_fresh( + self, cachedresponse: Response, request: Request + ) -> bool: return True - def is_cached_response_valid(self, cachedresponse, response, request): + def is_cached_response_valid( + self, cachedresponse: Response, response: Response, request: Request + ) -> bool: return True class RFC2616Policy: MAXAGE = 3600 * 24 * 365 # one year - def __init__(self, settings): - self.always_store = settings.getbool("HTTPCACHE_ALWAYS_STORE") - self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self._cc_parsed = WeakKeyDictionary() - self.ignore_response_cache_controls = [ + def __init__(self, settings: BaseSettings): + self.always_store: bool = settings.getbool("HTTPCACHE_ALWAYS_STORE") + self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self._cc_parsed: WeakKeyDictionary[ + Union[Request, Response], Dict[bytes, Optional[bytes]] + ] = WeakKeyDictionary() + self.ignore_response_cache_controls: List[bytes] = [ to_bytes(cc) for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS") ] - def _parse_cachecontrol(self, r): + def _parse_cachecontrol( + self, r: Union[Request, Response] + ) -> Dict[bytes, Optional[bytes]]: if r not in self._cc_parsed: cch = r.headers.get(b"Cache-Control", b"") + assert cch is not None parsed = parse_cachecontrol(cch) if isinstance(r, Response): for key in self.ignore_response_cache_controls: @@ -62,7 +81,7 @@ class RFC2616Policy: self._cc_parsed[r] = parsed return self._cc_parsed[r] - def should_cache_request(self, request): + def should_cache_request(self, request: Request) -> bool: if urlparse_cached(request).scheme in self.ignore_schemes: return False cc = self._parse_cachecontrol(request) @@ -72,7 +91,7 @@ class RFC2616Policy: # Any other is eligible for caching return True - def should_cache_response(self, response, request): + def should_cache_response(self, response: Response, request: Request) -> bool: # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 # Status code 206 is not included because cache can not deal with partial contents @@ -100,7 +119,9 @@ class RFC2616Policy: # info and can not be revalidated return False - def is_cached_response_fresh(self, cachedresponse, request): + def is_cached_response_fresh( + self, cachedresponse: Response, request: Request + ) -> bool: cc = self._parse_cachecontrol(cachedresponse) ccreq = self._parse_cachecontrol(request) if b"no-cache" in cc or b"no-cache" in ccreq: @@ -141,7 +162,9 @@ class RFC2616Policy: self._set_conditional_validators(request, cachedresponse) return False - def is_cached_response_valid(self, cachedresponse, response, request): + def is_cached_response_valid( + self, cachedresponse: Response, response: Response, request: Request + ) -> bool: # Use the cached response if the new response is a server error, # as long as the old response didn't specify must-revalidate. if response.status >= 500: @@ -152,7 +175,9 @@ class RFC2616Policy: # Use the cached response if the server says it hasn't changed. return response.status == 304 - def _set_conditional_validators(self, request, cachedresponse): + def _set_conditional_validators( + self, request: Request, cachedresponse: Response + ) -> None: if b"Last-Modified" in cachedresponse.headers: request.headers[b"If-Modified-Since"] = cachedresponse.headers[ b"Last-Modified" @@ -161,13 +186,15 @@ class RFC2616Policy: if b"ETag" in cachedresponse.headers: request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"] - def _get_max_age(self, cc): + def _get_max_age(self, cc: Dict[bytes, Optional[bytes]]) -> Optional[int]: try: - return max(0, int(cc[b"max-age"])) + return max(0, int(cc[b"max-age"])) # type: ignore[arg-type] except (KeyError, ValueError): return None - def _compute_freshness_lifetime(self, response, request, now): + def _compute_freshness_lifetime( + self, response: Response, request: Request, now: float + ) -> float: # Reference nsHttpResponseHead::ComputeFreshnessLifetime # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#706 cc = self._parse_cachecontrol(response) @@ -198,10 +225,12 @@ class RFC2616Policy: # Insufficient information to compute freshness lifetime return 0 - def _compute_current_age(self, response, request, now): + def _compute_current_age( + self, response: Response, request: Request, now: float + ) -> float: # Reference nsHttpResponseHead::ComputeCurrentAge # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#658 - currentage = 0 + currentage: float = 0 # If Date header is not set we assume it is a fast connection, and # clock is in sync with the server date = rfc1123_to_epoch(response.headers.get(b"Date")) or now @@ -210,7 +239,7 @@ class RFC2616Policy: if b"Age" in response.headers: try: - age = int(response.headers[b"Age"]) + age = int(response.headers[b"Age"]) # type: ignore[arg-type] currentage = max(currentage, age) except ValueError: pass @@ -219,13 +248,13 @@ class RFC2616Policy: class DbmCacheStorage: - def __init__(self, settings): - self.cachedir = data_path(settings["HTTPCACHE_DIR"], createdir=True) - self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS") - self.dbmodule = import_module(settings["HTTPCACHE_DBM_MODULE"]) - self.db = None + def __init__(self, settings: BaseSettings): + self.cachedir: str = data_path(settings["HTTPCACHE_DIR"], createdir=True) + self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") + self.dbmodule: ModuleType = import_module(settings["HTTPCACHE_DBM_MODULE"]) + self.db: Any = None # the real type is private - def open_spider(self, spider: Spider): + def open_spider(self, spider: Spider) -> None: dbpath = Path(self.cachedir, f"{spider.name}.db") self.db = self.dbmodule.open(str(dbpath), "c") @@ -235,15 +264,16 @@ class DbmCacheStorage: extra={"spider": spider}, ) - self._fingerprinter = spider.crawler.request_fingerprinter + assert spider.crawler.request_fingerprinter + self._fingerprinter: RequestFingerprinter = spider.crawler.request_fingerprinter - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> None: self.db.close() - def retrieve_response(self, spider, request): + def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]: data = self._read_data(spider, request) if data is None: - return # not cached + return None # not cached url = data["url"] status = data["status"] headers = Headers(data["headers"]) @@ -252,7 +282,9 @@ class DbmCacheStorage: response = respcls(url=url, headers=headers, status=status, body=body) return response - def store_response(self, spider, request, response): + def store_response( + self, spider: Spider, request: Request, response: Response + ) -> None: key = self._fingerprinter.fingerprint(request).hex() data = { "status": response.status, @@ -263,28 +295,31 @@ class DbmCacheStorage: self.db[f"{key}_data"] = pickle.dumps(data, protocol=4) self.db[f"{key}_time"] = str(time()) - def _read_data(self, spider, request): + def _read_data(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: key = self._fingerprinter.fingerprint(request).hex() db = self.db tkey = f"{key}_time" if tkey not in db: - return # not found + return None # not found ts = db[tkey] if 0 < self.expiration_secs < time() - float(ts): - return # expired + return None # expired - return pickle.loads(db[f"{key}_data"]) # nosec + return cast(Dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec class FilesystemCacheStorage: - def __init__(self, settings): - self.cachedir = data_path(settings["HTTPCACHE_DIR"]) - self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS") - self.use_gzip = settings.getbool("HTTPCACHE_GZIP") - self._open = gzip.open if self.use_gzip else open + def __init__(self, settings: BaseSettings): + self.cachedir: str = data_path(settings["HTTPCACHE_DIR"]) + self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") + self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") + # https://github.com/python/mypy/issues/10740 + self._open: Callable[Concatenate[Union[str, os.PathLike], str, ...], IO] = ( + gzip.open if self.use_gzip else open # type: ignore[assignment] + ) - def open_spider(self, spider: Spider): + def open_spider(self, spider: Spider) -> None: logger.debug( "Using filesystem cache storage in %(cachedir)s", {"cachedir": self.cachedir}, @@ -294,27 +329,29 @@ class FilesystemCacheStorage: assert spider.crawler.request_fingerprinter self._fingerprinter = spider.crawler.request_fingerprinter - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> None: pass - def retrieve_response(self, spider: Spider, request: Request): + def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]: """Return response if present in cache, or None otherwise.""" metadata = self._read_meta(spider, request) if metadata is None: - return # not cached + return None # not cached rpath = Path(self._get_request_path(spider, request)) with self._open(rpath / "response_body", "rb") as f: body = f.read() with self._open(rpath / "response_headers", "rb") as f: rawheaders = f.read() - url = metadata.get("response_url") + url = metadata["response_url"] status = metadata["status"] headers = Headers(headers_raw_to_dict(rawheaders)) respcls = responsetypes.from_args(headers=headers, url=url, body=body) response = respcls(url=url, headers=headers, status=status, body=body) return response - def store_response(self, spider: Spider, request: Request, response): + def store_response( + self, spider: Spider, request: Request, response: Response + ) -> None: """Store the given response in the cache.""" rpath = Path(self._get_request_path(spider, request)) if not rpath.exists(): @@ -343,19 +380,19 @@ class FilesystemCacheStorage: key = self._fingerprinter.fingerprint(request).hex() return str(Path(self.cachedir, spider.name, key[0:2], key)) - def _read_meta(self, spider: Spider, request: Request): + def _read_meta(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: rpath = Path(self._get_request_path(spider, request)) metapath = rpath / "pickled_meta" if not metapath.exists(): - return # not found + return None # not found mtime = metapath.stat().st_mtime if 0 < self.expiration_secs < time() - mtime: - return # expired + return None # expired with self._open(metapath, "rb") as f: - return pickle.load(f) # nosec + return cast(Dict[str, Any], pickle.load(f)) # nosec -def parse_cachecontrol(header): +def parse_cachecontrol(header: bytes) -> Dict[bytes, Optional[bytes]]: """Parse Cache-Control header https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 @@ -375,9 +412,9 @@ def parse_cachecontrol(header): return directives -def rfc1123_to_epoch(date_str): +def rfc1123_to_epoch(date_str: Union[str, bytes, None]) -> Optional[int]: try: - date_str = to_unicode(date_str, encoding="ascii") - return mktime_tz(parsedate_tz(date_str)) + date_str = to_unicode(date_str, encoding="ascii") # type: ignore[arg-type] + return mktime_tz(parsedate_tz(date_str)) # type: ignore[arg-type] except Exception: return None diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 7ffbd8bc3..ac12ad829 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -6,7 +6,7 @@ from bz2 import BZ2File from gzip import GzipFile from io import IOBase from lzma import LZMAFile -from typing import Any, BinaryIO, Dict, List, cast +from typing import IO, Any, BinaryIO, Dict, List, cast from scrapy.utils.misc import load_object @@ -126,7 +126,7 @@ class PostProcessingManager(IOBase): """ def __init__( - self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any] + self, plugins: List[Any], file: IO[bytes], feed_options: Dict[str, Any] ) -> None: self.plugins = self._load_plugins(plugins) self.file = file diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 49493e9c6..cd6e9d04e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -3,7 +3,6 @@ import logging import re from io import StringIO from typing import ( - TYPE_CHECKING, Any, Callable, Dict, @@ -25,9 +24,6 @@ from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode -if TYPE_CHECKING: - from lxml._types import SupportsReadClose # nosec - logger = logging.getLogger(__name__) @@ -98,7 +94,7 @@ def xmliter_lxml( reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( - cast("SupportsReadClose[bytes]", reader), + reader, encoding=reader.encoding, events=("end", "start-ns"), resolve_entities=False, diff --git a/tox.ini b/tox.ini index b5effb527..d7527bb04 100644 --- a/tox.ini +++ b/tox.ini @@ -43,14 +43,14 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.8.0 - typing-extensions==4.10.0 + mypy==1.10.0 + typing-extensions==4.11.0 types-attrs==19.1.0 - types-lxml==2024.2.9 - types-Pillow==10.2.0.20240213 - types-Pygments==2.17.0.20240106 - types-pyOpenSSL==24.0.0.20240130 - types-setuptools==69.1.0.20240223 + types-lxml==2024.4.14 + types-Pillow==10.2.0.20240423 + types-Pygments==2.17.0.20240310 + types-pyOpenSSL==24.0.0.20240417 + types-setuptools==69.5.0.20240423 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands = From d7da298e0637d105dcec379f6cbb3196e752ae72 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 19:14:59 +0500 Subject: [PATCH 1403/2083] Typing for build_from_*. (#6326) --- scrapy/core/engine.py | 3 ++- scrapy/core/scheduler.py | 2 ++ scrapy/crawler.py | 4 +++- scrapy/utils/misc.py | 28 +++++++++++++++++++--------- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 2db085081..93a0c51bc 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -366,7 +366,8 @@ class ExecutionEngine: self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider if hasattr(scheduler, "open"): - yield scheduler.open(spider) + if d := scheduler.open(spider): + yield d yield self.scraper.open_spider(spider) assert self.crawler.stats self.crawler.stats.open_spider(spider) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index f41b83a67..b2209e53f 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -322,6 +322,7 @@ class Scheduler(BaseScheduler): def _mq(self): """Create a new priority queue instance, with in-memory storage""" + assert self.crawler return build_from_crawler( self.pqclass, self.crawler, @@ -331,6 +332,7 @@ class Scheduler(BaseScheduler): def _dq(self): """Create a new priority queue instance, with disk storage""" + assert self.crawler assert self.dqdir state = self._read_dqs_state(self.dqdir) q = build_from_crawler( diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1db9ace28..ccfe78891 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -445,7 +445,9 @@ class CrawlerProcess(CrawlerRunner): d.addBoth(self._stop_reactor) resolver_class = load_object(self.settings["DNS_RESOLVER"]) - resolver = build_from_crawler(resolver_class, self, reactor=reactor) + # We pass self, which is CrawlerProcess, instead of Crawler here, + # which works because the default resolvers only use crawler.settings. + resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type] resolver.install_on_reactor() tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 7f83d06fb..faf52e44a 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,5 +1,7 @@ """Helper functions which don't fit anywhere else""" +from __future__ import annotations + import ast import hashlib import inspect @@ -22,6 +24,8 @@ from typing import ( Iterable, List, Optional, + Type, + TypeVar, Union, cast, ) @@ -32,9 +36,11 @@ from scrapy.utils.datatypes import LocalWeakReferencedCache if TYPE_CHECKING: from scrapy import Spider - + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes +T = TypeVar("T") def arg_to_iter(arg: Any) -> Iterable[Any]: @@ -177,7 +183,9 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): return instance -def build_from_crawler(objcls, crawler, /, *args, **kwargs): +def build_from_crawler( + objcls: Type[T], crawler: Crawler, /, *args: Any, **kwargs: Any +) -> T: """Construct a class instance using its ``from_crawler`` constructor. ``*args`` and ``**kwargs`` are forwarded to the constructor. @@ -185,20 +193,22 @@ def build_from_crawler(objcls, crawler, /, *args, **kwargs): Raises ``TypeError`` if the resulting instance is ``None``. """ if hasattr(objcls, "from_crawler"): - instance = objcls.from_crawler(crawler, *args, **kwargs) + instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined] method_name = "from_crawler" elif hasattr(objcls, "from_settings"): - instance = objcls.from_settings(crawler.settings, *args, **kwargs) + instance = objcls.from_settings(crawler.settings, *args, **kwargs) # type: ignore[attr-defined] method_name = "from_settings" else: instance = objcls(*args, **kwargs) method_name = "__new__" if instance is None: raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return instance + return cast(T, instance) -def build_from_settings(objcls, settings, /, *args, **kwargs): +def build_from_settings( + objcls: Type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any +) -> T: """Construct a class instance using its ``from_settings`` constructor. ``*args`` and ``**kwargs`` are forwarded to the constructor. @@ -206,14 +216,14 @@ def build_from_settings(objcls, settings, /, *args, **kwargs): Raises ``TypeError`` if the resulting instance is ``None``. """ if hasattr(objcls, "from_settings"): - instance = objcls.from_settings(settings, *args, **kwargs) + instance = objcls.from_settings(settings, *args, **kwargs) # type: ignore[attr-defined] method_name = "from_settings" else: instance = objcls(*args, **kwargs) method_name = "__new__" if instance is None: raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return instance + return cast(T, instance) @contextmanager @@ -290,7 +300,7 @@ def is_generator_with_return_value(callable: Callable) -> bool: return bool(_generator_callbacks_cache[callable]) -def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None: +def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None: """ Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None From 9eea22fb0ca99193b7f38f9f9398b278d64ea977 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 18:59:31 +0500 Subject: [PATCH 1404/2083] Full typing for scrapy/cmdline.py. --- scrapy/cmdline.py | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 6580ba9ce..4df5698a6 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -4,18 +4,22 @@ import inspect import os import sys from importlib.metadata import entry_points +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError +from scrapy.settings import BaseSettings, Settings from scrapy.utils.misc import walk_modules from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect class ScrapyArgumentParser(argparse.ArgumentParser): - def _parse_optional(self, arg_string): + def _parse_optional( + self, arg_string: str + ) -> Optional[Tuple[Optional[argparse.Action], str, Optional[str]]]: # if starts with -: it means that is a parameter not a argument if arg_string[:2] == "-:": return None @@ -23,7 +27,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser): return super()._parse_optional(arg_string) -def _iter_command_classes(module_name): +def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]: # TODO: add `name` attribute to commands and merge this function with # scrapy.utils.spider.iter_spider_classes for module in walk_modules(module_name): @@ -37,8 +41,8 @@ def _iter_command_classes(module_name): yield obj -def _get_commands_from_module(module, inproject): - d = {} +def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyCommand]: + d: Dict[str, ScrapyCommand] = {} for cmd in _iter_command_classes(module): if inproject or not cmd.requires_project: cmdname = cmd.__module__.split(".")[-1] @@ -46,8 +50,10 @@ def _get_commands_from_module(module, inproject): return d -def _get_commands_from_entry_points(inproject, group="scrapy.commands"): - cmds = {} +def _get_commands_from_entry_points( + inproject: bool, group: str = "scrapy.commands" +) -> Dict[str, ScrapyCommand]: + cmds: Dict[str, ScrapyCommand] = {} if sys.version_info >= (3, 10): eps = entry_points(group=group) else: @@ -61,7 +67,9 @@ def _get_commands_from_entry_points(inproject, group="scrapy.commands"): return cmds -def _get_commands_dict(settings, inproject): +def _get_commands_dict( + settings: BaseSettings, inproject: bool +) -> Dict[str, ScrapyCommand]: cmds = _get_commands_from_module("scrapy.commands", inproject) cmds.update(_get_commands_from_entry_points(inproject)) cmds_module = settings["COMMANDS_MODULE"] @@ -70,16 +78,17 @@ def _get_commands_dict(settings, inproject): return cmds -def _pop_command_name(argv): +def _pop_command_name(argv: List[str]) -> Optional[str]: i = 0 for arg in argv[1:]: if not arg.startswith("-"): del argv[i] return arg i += 1 + return None -def _print_header(settings, inproject): +def _print_header(settings: BaseSettings, inproject: bool) -> None: version = scrapy.__version__ if inproject: print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n") @@ -88,7 +97,7 @@ def _print_header(settings, inproject): print(f"Scrapy {version} - no active project\n") -def _print_commands(settings, inproject): +def _print_commands(settings: BaseSettings, inproject: bool) -> None: _print_header(settings, inproject) print("Usage:") print(" scrapy [options] [args]\n") @@ -103,13 +112,17 @@ def _print_commands(settings, inproject): print('Use "scrapy -h" to see more info about a command') -def _print_unknown_command(settings, cmdname, inproject): +def _print_unknown_command( + settings: BaseSettings, cmdname: str, inproject: bool +) -> None: _print_header(settings, inproject) print(f"Unknown command: {cmdname}\n") print('Use "scrapy" to see available commands') -def _run_print_help(parser, func, *a, **kw): +def _run_print_help( + parser: argparse.ArgumentParser, func: Callable, *a: Any, **kw: Any +) -> None: try: func(*a, **kw) except UsageError as e: @@ -120,7 +133,9 @@ def _run_print_help(parser, func, *a, **kw): sys.exit(2) -def execute(argv=None, settings=None): +def execute( + argv: Optional[List[str]] = None, settings: Optional[Settings] = None +) -> None: if argv is None: argv = sys.argv @@ -162,14 +177,16 @@ def execute(argv=None, settings=None): sys.exit(cmd.exitcode) -def _run_command(cmd, args, opts): +def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) -> None: if opts.profile: _run_command_profiled(cmd, args, opts) else: cmd.run(args, opts) -def _run_command_profiled(cmd, args, opts): +def _run_command_profiled( + cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace +) -> None: if opts.profile: sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() From fc1a83e7c42dc5142eb9190fdca887385254a7a6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 19:07:31 +0500 Subject: [PATCH 1405/2083] Full typing for scrapy/item.py. --- scrapy/item.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index d3eb90b7b..e04e994ef 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,14 +4,20 @@ Scrapy Item See documentation in docs/topics/item.rst """ +from __future__ import annotations + from abc import ABCMeta from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat -from typing import Dict +from typing import TYPE_CHECKING, Any, Dict, Iterator, KeysView, NoReturn, Tuple from scrapy.utils.trackref import object_ref +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class Field(dict): """Container of field metadata""" @@ -23,7 +29,9 @@ class ItemMeta(ABCMeta): .. _metaclass: https://realpython.com/python-metaclasses """ - def __new__(mcs, class_name, bases, attrs): + def __new__( + mcs, class_name: str, bases: Tuple[type, ...], attrs: Dict[str, Any] + ) -> ItemMeta: classcell = attrs.pop("__classcell__", None) new_bases = tuple(base._class for base in bases if hasattr(base, "_class")) _class = super().__new__(mcs, "x_" + class_name, new_bases, attrs) @@ -44,7 +52,7 @@ class ItemMeta(ABCMeta): return super().__new__(mcs, class_name, bases, new_attrs) -class Item(MutableMapping, object_ref, metaclass=ItemMeta): +class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): """ Base class for scraped items. @@ -69,51 +77,51 @@ class Item(MutableMapping, object_ref, metaclass=ItemMeta): fields: Dict[str, Field] - def __init__(self, *args, **kwargs): - self._values = {} + def __init__(self, *args: Any, **kwargs: Any): + self._values: Dict[str, Any] = {} if args or kwargs: # avoid creating dict for most common case for k, v in dict(*args, **kwargs).items(): self[k] = v - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: return self._values[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any) -> None: if key in self.fields: self._values[key] = value else: raise KeyError(f"{self.__class__.__name__} does not support field: {key}") - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: del self._values[key] - def __getattr__(self, name): + def __getattr__(self, name: str) -> NoReturn: if name in self.fields: raise AttributeError(f"Use item[{name!r}] to get field value") raise AttributeError(name) - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: Any) -> None: if not name.startswith("_"): raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value") super().__setattr__(name, value) - def __len__(self): + def __len__(self) -> int: return len(self._values) - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._values) __hash__ = object_ref.__hash__ - def keys(self): + def keys(self) -> KeysView[str]: return self._values.keys() - def __repr__(self): + def __repr__(self) -> str: return pformat(dict(self)) - def copy(self): + def copy(self) -> Self: return self.__class__(self) - def deepcopy(self): + def deepcopy(self) -> Self: """Return a :func:`~copy.deepcopy` of this item.""" return deepcopy(self) From 08a265b6ff9bc47774173238b06715154b39e534 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 19:10:00 +0500 Subject: [PATCH 1406/2083] Full typing for scrapy/extension.py. --- scrapy/extension.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/extension.py b/scrapy/extension.py index 6be14450c..8221b675e 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -4,7 +4,10 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ +from typing import Any, List + from scrapy.middleware import MiddlewareManager +from scrapy.settings import Settings from scrapy.utils.conf import build_component_list @@ -12,5 +15,5 @@ class ExtensionManager(MiddlewareManager): component_name = "extension" @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: return build_component_list(settings.getwithbase("EXTENSIONS")) From 38020e0b0481d2b15792757669272d6d3bf4b14f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 20:12:30 +0500 Subject: [PATCH 1407/2083] Full typing for scrapy/mail.py. --- scrapy/mail.py | 98 +++++++++++++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 7cb5ef454..56adba934 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -14,11 +14,23 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from typing import TYPE_CHECKING, Optional +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Optional, + Sequence, + Tuple, + Union, +) from twisted import version as twisted_version from twisted.internet import ssl from twisted.internet.defer import Deferred +from twisted.python.failure import Failure from twisted.python.versions import Version from scrapy.settings import BaseSettings @@ -26,6 +38,9 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes if TYPE_CHECKING: + # imports twisted.internet.reactor + from twisted.mail.smtp import ESMTPSenderFactory + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -37,7 +52,7 @@ logger = logging.getLogger(__name__) COMMASPACE = ", " -def _to_bytes_or_none(text): +def _to_bytes_or_none(text: Union[str, bytes, None]) -> Optional[bytes]: if text is None: return None return to_bytes(text) @@ -46,23 +61,23 @@ def _to_bytes_or_none(text): class MailSender: def __init__( self, - smtphost="localhost", - mailfrom="scrapy@localhost", - smtpuser=None, - smtppass=None, - smtpport=25, - smtptls=False, - smtpssl=False, - debug=False, + smtphost: str = "localhost", + mailfrom: str = "scrapy@localhost", + smtpuser: Optional[str] = None, + smtppass: Optional[str] = None, + smtpport: int = 25, + smtptls: bool = False, + smtpssl: bool = False, + debug: bool = False, ): - self.smtphost = smtphost - self.smtpport = smtpport - self.smtpuser = _to_bytes_or_none(smtpuser) - self.smtppass = _to_bytes_or_none(smtppass) - self.smtptls = smtptls - self.smtpssl = smtpssl - self.mailfrom = mailfrom - self.debug = debug + self.smtphost: str = smtphost + self.smtpport: int = smtpport + self.smtpuser: Optional[bytes] = _to_bytes_or_none(smtpuser) + self.smtppass: Optional[bytes] = _to_bytes_or_none(smtppass) + self.smtptls: bool = smtptls + self.smtpssl: bool = smtpssl + self.mailfrom: str = mailfrom + self.debug: bool = debug @classmethod def from_settings(cls, settings: BaseSettings) -> Self: @@ -78,14 +93,14 @@ class MailSender: def send( self, - to, - subject, - body, - cc=None, - attachs=(), - mimetype="text/plain", - charset=None, - _callback=None, + to: Union[str, List[str]], + subject: str, + body: str, + cc: Union[str, List[str], None] = None, + attachs: Sequence[Tuple[str, str, IO]] = (), + mimetype: str = "text/plain", + charset: Optional[str] = None, + _callback: Optional[Callable[..., None]] = None, ) -> Optional[Deferred]: from twisted.internet import reactor @@ -142,13 +157,15 @@ class MailSender: dfd.addCallbacks( callback=self._sent_ok, errback=self._sent_failed, - callbackArgs=[to, cc, subject, len(attachs)], - errbackArgs=[to, cc, subject, len(attachs)], + callbackArgs=(to, cc, subject, len(attachs)), + errbackArgs=(to, cc, subject, len(attachs)), ) reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd) return dfd - def _sent_ok(self, result, to, cc, subject, nattachs): + def _sent_ok( + self, result: Any, to: List[str], cc: List[str], subject: str, nattachs: int + ) -> None: logger.info( "Mail sent OK: To=%(mailto)s Cc=%(mailcc)s " 'Subject="%(mailsubject)s" Attachs=%(mailattachs)d', @@ -160,7 +177,14 @@ class MailSender: }, ) - def _sent_failed(self, failure, to, cc, subject, nattachs): + def _sent_failed( + self, + failure: Failure, + to: List[str], + cc: List[str], + subject: str, + nattachs: int, + ) -> Failure: errstr = str(failure.value) logger.error( "Unable to send mail: To=%(mailto)s Cc=%(mailcc)s " @@ -176,13 +200,13 @@ class MailSender: ) return failure - def _sendmail(self, to_addrs, msg): + def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred: from twisted.internet import reactor - msg = BytesIO(msg) - d = Deferred() + msg_io = BytesIO(msg) + d: Deferred = Deferred() - factory = self._create_sender_factory(to_addrs, msg, d) + factory = self._create_sender_factory(to_addrs, msg_io, d) if self.smtpssl: reactor.connectSSL( @@ -193,10 +217,12 @@ class MailSender: return d - def _create_sender_factory(self, to_addrs, msg, d): + def _create_sender_factory( + self, to_addrs: List[str], msg: IO, d: Deferred + ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory - factory_keywords = { + factory_keywords: Dict[str, Any] = { "heloFallback": True, "requireAuthentication": False, "requireTransportSecurity": self.smtptls, From 0c8e21b8acfcac2d6d057f3c67b2252d0fa660e3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 20:57:31 +0500 Subject: [PATCH 1408/2083] Full typing for scrapy/pqueues.py. --- scrapy/core/downloader/__init__.py | 2 +- scrapy/pqueues.py | 138 +++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 40 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index ecd3e8b56..f88da41ea 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -143,7 +143,7 @@ class Downloader: return key, self.slots[key] - def _get_slot_key(self, request: Request, spider: Spider) -> str: + def _get_slot_key(self, request: Request, spider: Any) -> str: if self.DOWNLOAD_SLOT in request.meta: return cast(str, request.meta[self.DOWNLOAD_SLOT]) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 593667f1f..213ad590d 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,12 +1,32 @@ +from __future__ import annotations + import hashlib import logging +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + Optional, + Protocol, + Tuple, + Type, + cast, +) +from scrapy import Request +from scrapy.core.downloader import Downloader +from scrapy.crawler import Crawler from scrapy.utils.misc import build_from_crawler +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) -def _path_safe(text): +def _path_safe(text: str) -> str: """ Return a filesystem-safe version of a string ``text`` @@ -24,6 +44,18 @@ def _path_safe(text): return "-".join([pathable_slot, unique_slot]) +class QueueProtocol(Protocol): + """Protocol for downstream queues of ``ScrapyPriorityQueue``.""" + + def push(self, request: Request) -> None: ... + + def pop(self) -> Optional[Request]: ... + + def close(self) -> None: ... + + def __len__(self) -> int: ... + + class ScrapyPriorityQueue: """A priority queue implemented using multiple internal queues (typically, FIFO queues). It uses one internal queue for each priority value. The internal @@ -51,18 +83,30 @@ class ScrapyPriorityQueue: """ @classmethod - def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): + def from_crawler( + cls, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + startprios: Iterable[int] = (), + ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) - def __init__(self, crawler, downstream_queue_cls, key, startprios=()): - self.crawler = crawler - self.downstream_queue_cls = downstream_queue_cls - self.key = key - self.queues = {} - self.curprio = None + def __init__( + self, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + startprios: Iterable[int] = (), + ): + self.crawler: Crawler = crawler + self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.key: str = key + self.queues: Dict[int, QueueProtocol] = {} + self.curprio: Optional[int] = None self.init_prios(startprios) - def init_prios(self, startprios): + def init_prios(self, startprios: Iterable[int]) -> None: if not startprios: return @@ -71,17 +115,17 @@ class ScrapyPriorityQueue: self.curprio = min(startprios) - def qfactory(self, key): + def qfactory(self, key: int) -> QueueProtocol: return build_from_crawler( self.downstream_queue_cls, self.crawler, self.key + "/" + str(key), ) - def priority(self, request): + def priority(self, request: Request) -> int: return -request.priority - def push(self, request): + def push(self, request: Request) -> None: priority = self.priority(request) if priority not in self.queues: self.queues[priority] = self.qfactory(priority) @@ -90,9 +134,9 @@ class ScrapyPriorityQueue: if self.curprio is None or priority < self.curprio: self.curprio = priority - def pop(self): + def pop(self) -> Optional[Request]: if self.curprio is None: - return + return None q = self.queues[self.curprio] m = q.pop() if not q: @@ -102,7 +146,7 @@ class ScrapyPriorityQueue: self.curprio = min(prios) if prios else None return m - def peek(self): + def peek(self) -> Optional[Request]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -112,30 +156,32 @@ class ScrapyPriorityQueue: if self.curprio is None: return None queue = self.queues[self.curprio] - return queue.peek() + # Protocols can't declare optional members + return cast(Request, queue.peek()) # type: ignore[attr-defined] - def close(self): - active = [] + def close(self) -> List[int]: + active: List[int] = [] for p, q in self.queues.items(): active.append(p) q.close() return active - def __len__(self): + def __len__(self) -> int: return sum(len(x) for x in self.queues.values()) if self.queues else 0 class DownloaderInterface: - def __init__(self, crawler): - self.downloader = crawler.engine.downloader + def __init__(self, crawler: Crawler): + assert crawler.engine + self.downloader: Downloader = crawler.engine.downloader - def stats(self, possible_slots): + def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]: return [(self._active_downloads(slot), slot) for slot in possible_slots] - def get_slot_key(self, request): + def get_slot_key(self, request: Request) -> str: return self.downloader._get_slot_key(request, None) - def _active_downloads(self, slot): + def _active_downloads(self, slot: str) -> int: """Return a number of requests in a Downloader for a given slot""" if slot not in self.downloader.slots: return 0 @@ -149,10 +195,22 @@ class DownloaderAwarePriorityQueue: """ @classmethod - def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): + def from_crawler( + cls, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + startprios: Optional[Dict[str, Iterable[int]]] = None, + ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) - def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): + def __init__( + self, + crawler: Crawler, + downstream_queue_cls: Type[QueueProtocol], + key: str, + slot_startprios: Optional[Dict[str, Iterable[int]]] = None, + ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' @@ -169,16 +227,18 @@ class DownloaderAwarePriorityQueue: "queue class can be resumed." ) - self._downloader_interface = DownloaderInterface(crawler) - self.downstream_queue_cls = downstream_queue_cls - self.key = key - self.crawler = crawler + self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) + self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.key: str = key + self.crawler: Crawler = crawler - self.pqueues = {} # slot -> priority queue + self.pqueues: Dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue for slot, startprios in (slot_startprios or {}).items(): self.pqueues[slot] = self.pqfactory(slot, startprios) - def pqfactory(self, slot, startprios=()): + def pqfactory( + self, slot: str, startprios: Iterable[int] = () + ) -> ScrapyPriorityQueue: return ScrapyPriorityQueue( self.crawler, self.downstream_queue_cls, @@ -186,11 +246,11 @@ class DownloaderAwarePriorityQueue: startprios, ) - def pop(self): + def pop(self) -> Optional[Request]: stats = self._downloader_interface.stats(self.pqueues) if not stats: - return + return None slot = min(stats)[1] queue = self.pqueues[slot] @@ -199,14 +259,14 @@ class DownloaderAwarePriorityQueue: del self.pqueues[slot] return request - def push(self, request): + def push(self, request: Request) -> None: slot = self._downloader_interface.get_slot_key(request) if slot not in self.pqueues: self.pqueues[slot] = self.pqfactory(slot) queue = self.pqueues[slot] queue.push(request) - def peek(self): + def peek(self) -> Optional[Request]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -220,13 +280,13 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] return queue.peek() - def close(self): + def close(self) -> Dict[str, List[int]]: active = {slot: queue.close() for slot, queue in self.pqueues.items()} self.pqueues.clear() return active - def __len__(self): + def __len__(self) -> int: return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 - def __contains__(self, slot): + def __contains__(self, slot: str) -> bool: return slot in self.pqueues From 21fa0761818c158ae9fc35b49ea8d2300f0fa510 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 22:41:58 +0500 Subject: [PATCH 1409/2083] Fix MutableMapping import for Python 3.8. --- scrapy/item.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index e04e994ef..2daea64cc 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -7,10 +7,18 @@ See documentation in docs/topics/item.rst from __future__ import annotations from abc import ABCMeta -from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat -from typing import TYPE_CHECKING, Any, Dict, Iterator, KeysView, NoReturn, Tuple +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + KeysView, + MutableMapping, + NoReturn, + Tuple, +) from scrapy.utils.trackref import object_ref From ad35ffdb0da052d0df194ce5dc1ba7e8d823190f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 23:10:03 +0500 Subject: [PATCH 1410/2083] Full typing for scrapy/resolver.py. --- scrapy/resolver.py | 67 +++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index e2e8beff4..ba7cd716b 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,8 +1,12 @@ -from typing import Any +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type from twisted.internet import defer -from twisted.internet.base import ThreadedResolver +from twisted.internet.base import ReactorBase, ThreadedResolver +from twisted.internet.defer import Deferred from twisted.internet.interfaces import ( + IAddress, IHostnameResolver, IHostResolution, IResolutionReceiver, @@ -12,6 +16,12 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + # TODO: cache misses dnscache: LocalCache[str, Any] = LocalCache(10000) @@ -22,65 +32,66 @@ class CachingThreadedResolver(ThreadedResolver): Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests. """ - def __init__(self, reactor, cache_size, timeout): + def __init__(self, reactor: ReactorBase, cache_size: int, timeout: float): super().__init__(reactor) dnscache.limit = cache_size self.timeout = timeout @classmethod - def from_crawler(cls, crawler, reactor): + def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self: if crawler.settings.getbool("DNSCACHE_ENABLED"): cache_size = crawler.settings.getint("DNSCACHE_SIZE") else: cache_size = 0 return cls(reactor, cache_size, crawler.settings.getfloat("DNS_TIMEOUT")) - def install_on_reactor(self): + def install_on_reactor(self) -> None: self.reactor.installResolver(self) - def getHostByName(self, name: str, timeout=None): + def getHostByName(self, name: str, timeout: Sequence[int] = ()) -> Deferred[str]: if name in dnscache: return defer.succeed(dnscache[name]) # in Twisted<=16.6, getHostByName() is always called with # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple), # so the input argument above is simply overridden # to enforce Scrapy's DNS_TIMEOUT setting's value - timeout = (self.timeout,) + # The timeout arg is typed as Sequence[int] but supports floats. + timeout = (self.timeout,) # type: ignore[assignment] d = super().getHostByName(name, timeout) if dnscache.limit: d.addCallback(self._cache_result, name) return d - def _cache_result(self, result, name): + def _cache_result(self, result: Any, name: str) -> Any: dnscache[name] = result return result @implementer(IHostResolution) class HostResolution: - def __init__(self, name): - self.name = name + def __init__(self, name: str): + self.name: str = name - def cancel(self): + def cancel(self) -> None: raise NotImplementedError() @provider(IResolutionReceiver) class _CachingResolutionReceiver: - def __init__(self, resolutionReceiver, hostName): - self.resolutionReceiver = resolutionReceiver - self.hostName = hostName - self.addresses = [] + def __init__(self, resolutionReceiver: IResolutionReceiver, hostName: str): + self.resolutionReceiver: IResolutionReceiver = resolutionReceiver + self.hostName: str = hostName + self.addresses: List[IAddress] = [] - def resolutionBegan(self, resolution): + def resolutionBegan(self, resolution: IHostResolution) -> None: self.resolutionReceiver.resolutionBegan(resolution) self.resolution = resolution - def addressResolved(self, address): + def addressResolved(self, address: IAddress) -> None: self.resolutionReceiver.addressResolved(address) self.addresses.append(address) - def resolutionComplete(self): + def resolutionComplete(self) -> None: self.resolutionReceiver.resolutionComplete() if self.addresses: dnscache[self.hostName] = self.addresses @@ -93,30 +104,30 @@ class CachingHostnameResolver: does not support setting a timeout value for DNS requests. """ - def __init__(self, reactor, cache_size): - self.reactor = reactor - self.original_resolver = reactor.nameResolver + def __init__(self, reactor: ReactorBase, cache_size: int): + self.reactor: ReactorBase = reactor + self.original_resolver: IHostnameResolver = reactor.nameResolver dnscache.limit = cache_size @classmethod - def from_crawler(cls, crawler, reactor): + def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self: if crawler.settings.getbool("DNSCACHE_ENABLED"): cache_size = crawler.settings.getint("DNSCACHE_SIZE") else: cache_size = 0 return cls(reactor, cache_size) - def install_on_reactor(self): + def install_on_reactor(self) -> None: self.reactor.installNameResolver(self) def resolveHostName( self, - resolutionReceiver, + resolutionReceiver: IResolutionReceiver, hostName: str, - portNumber=0, - addressTypes=None, - transportSemantics="TCP", - ): + portNumber: int = 0, + addressTypes: Optional[Sequence[Type[IAddress]]] = None, + transportSemantics: str = "TCP", + ) -> IHostResolution: try: addresses = dnscache[hostName] except KeyError: From b749db92e5b15c974b0d77280c22b63000ad4263 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 23:17:59 +0500 Subject: [PATCH 1411/2083] Full typing for scrapy/robotstxt.py. --- scrapy/robotstxt.py | 65 +++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index ad06137e2..a33f73306 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -3,9 +3,10 @@ from __future__ import annotations import logging import sys from abc import ABCMeta, abstractmethod -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Optional, Union from warnings import warn +from scrapy import Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode @@ -18,12 +19,14 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): +def decode_robotstxt( + robotstxt_body: bytes, spider: Optional[Spider], to_native_str_type: bool = False +) -> str: try: if to_native_str_type: - robotstxt_body = to_unicode(robotstxt_body) + body_decoded = to_unicode(robotstxt_body) else: - robotstxt_body = robotstxt_body.decode("utf-8", errors="ignore") + body_decoded = robotstxt_body.decode("utf-8", errors="ignore") except UnicodeDecodeError: # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. # Switch to 'allow all' state. @@ -33,8 +36,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): exc_info=sys.exc_info(), extra={"spider": spider}, ) - robotstxt_body = "" - return robotstxt_body + body_decoded = "" + return body_decoded class RobotParser(metaclass=ABCMeta): @@ -66,82 +69,80 @@ class RobotParser(metaclass=ABCMeta): class PythonRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): from urllib.robotparser import RobotFileParser - self.spider = spider - robotstxt_body = decode_robotstxt( - robotstxt_body, spider, to_native_str_type=True - ) - self.rp = RobotFileParser() - self.rp.parse(robotstxt_body.splitlines()) + self.spider: Optional[Spider] = spider + body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) + self.rp: RobotFileParser = RobotFileParser() + self.rp.parse(body_decoded.splitlines()) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(user_agent, url) class ReppyRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2) from reppy.robots import Robots - self.spider = spider + self.spider: Optional[Spider] = spider self.rp = Robots.parse("", robotstxt_body) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: return self.rp.allowed(url, user_agent) class RerpRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): from robotexclusionrulesparser import RobotExclusionRulesParser - self.spider = spider - self.rp = RobotExclusionRulesParser() - robotstxt_body = decode_robotstxt(robotstxt_body, spider) - self.rp.parse(robotstxt_body) + self.spider: Optional[Spider] = spider + self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser() + body_decoded = decode_robotstxt(robotstxt_body, spider) + self.rp.parse(body_decoded) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.is_allowed(user_agent, url) class ProtegoRobotParser(RobotParser): - def __init__(self, robotstxt_body, spider): + def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): from protego import Protego - self.spider = spider - robotstxt_body = decode_robotstxt(robotstxt_body, spider) - self.rp = Protego.parse(robotstxt_body) + self.spider: Optional[Spider] = spider + body_decoded = decode_robotstxt(robotstxt_body, spider) + self.rp = Protego.parse(body_decoded) @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider o = cls(robotstxt_body, spider) return o - def allowed(self, url, user_agent): + def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) From 5f7fd2a653407da3eb3e53c853209d9bfb6b275f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 23:39:31 +0500 Subject: [PATCH 1412/2083] Full typing for scrapy/squeues.py. --- scrapy/squeues.py | 58 +++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index e20f60f06..4676b058e 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -2,20 +2,28 @@ Scheduler queues """ +from __future__ import annotations + import marshal import pickle # nosec from os import PathLike from pathlib import Path -from typing import Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union from queuelib import queue +from scrapy import Request +from scrapy.crawler import Crawler from scrapy.utils.request import request_from_dict +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self -def _with_mkdir(queue_class): + +def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: class DirectoriesCreated(queue_class): - def __init__(self, path: Union[str, PathLike], *args, **kwargs): + def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) @@ -24,18 +32,23 @@ def _with_mkdir(queue_class): return DirectoriesCreated -def _serializable_queue(queue_class, serialize, deserialize): +def _serializable_queue( + queue_class: Type[queue.BaseQueue], + serialize: Callable[[Any], bytes], + deserialize: Callable[[bytes], Any], +) -> Type[queue.BaseQueue]: class SerializableQueue(queue_class): - def push(self, obj): + def push(self, obj: Any) -> None: s = serialize(obj) super().push(s) - def pop(self): + def pop(self) -> Optional[Any]: s = super().pop() if s: return deserialize(s) + return None - def peek(self): + def peek(self) -> Optional[Any]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -50,31 +63,36 @@ def _serializable_queue(queue_class, serialize, deserialize): ) from ex if s: return deserialize(s) + return None return SerializableQueue -def _scrapy_serialization_queue(queue_class): +def _scrapy_serialization_queue( + queue_class: Type[queue.BaseQueue], +) -> Type[queue.BaseQueue]: class ScrapyRequestQueue(queue_class): - def __init__(self, crawler, key): + def __init__(self, crawler: Crawler, key: str): self.spider = crawler.spider super().__init__(key) @classmethod - def from_crawler(cls, crawler, key, *args, **kwargs): + def from_crawler( + cls, crawler: Crawler, key: str, *args: Any, **kwargs: Any + ) -> Self: return cls(crawler, key) - def push(self, request): - request = request.to_dict(spider=self.spider) - return super().push(request) + def push(self, request: Request) -> None: + request_dict = request.to_dict(spider=self.spider) + super().push(request_dict) - def pop(self): + def pop(self) -> Optional[Request]: request = super().pop() if not request: return None return request_from_dict(request, spider=self.spider) - def peek(self): + def peek(self) -> Optional[Request]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -89,13 +107,15 @@ def _scrapy_serialization_queue(queue_class): return ScrapyRequestQueue -def _scrapy_non_serialization_queue(queue_class): +def _scrapy_non_serialization_queue( + queue_class: Type[queue.BaseQueue], +) -> Type[queue.BaseQueue]: class ScrapyRequestQueue(queue_class): @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: return cls() - def peek(self): + def peek(self) -> Optional[Any]: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -113,7 +133,7 @@ def _scrapy_non_serialization_queue(queue_class): return ScrapyRequestQueue -def _pickle_serialize(obj): +def _pickle_serialize(obj: Any) -> bytes: try: return pickle.dumps(obj, protocol=4) # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s) From 203fa9667fb69f6251c0a44b14cf0450ce769a32 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 29 Apr 2024 23:47:55 +0500 Subject: [PATCH 1413/2083] Add queue typing to scrapy/core/scheduler.py. --- scrapy/core/scheduler.py | 33 +++++++++++++++++---------------- scrapy/pqueues.py | 3 ++- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index b2209e53f..ab59c0d14 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,13 +4,15 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast +from typing import TYPE_CHECKING, Any, Optional, Type, cast +from queuelib.queue import BaseQueue from twisted.internet.defer import Deferred from scrapy.crawler import Crawler from scrapy.dupefilters import BaseDupeFilter from scrapy.http.request import Request +from scrapy.pqueues import ScrapyPriorityQueue from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.job import job_dir @@ -121,9 +123,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): raise NotImplementedError() -SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler") - - class Scheduler(BaseScheduler): """ Default Scrapy scheduler. This implementation also handles duplication @@ -179,24 +178,24 @@ class Scheduler(BaseScheduler): self, dupefilter: BaseDupeFilter, jobdir: Optional[str] = None, - dqclass=None, - mqclass=None, + dqclass: Optional[Type[BaseQueue]] = None, + mqclass: Optional[Type[BaseQueue]] = None, logunser: bool = False, stats: Optional[StatsCollector] = None, - pqclass=None, + pqclass: Optional[Type[ScrapyPriorityQueue]] = None, crawler: Optional[Crawler] = None, ): self.df: BaseDupeFilter = dupefilter self.dqdir: Optional[str] = self._dqdir(jobdir) - self.pqclass = pqclass - self.dqclass = dqclass - self.mqclass = mqclass + self.pqclass: Optional[Type[ScrapyPriorityQueue]] = pqclass + self.dqclass: Optional[Type[BaseQueue]] = dqclass + self.mqclass: Optional[Type[BaseQueue]] = mqclass self.logunser: bool = logunser self.stats: Optional[StatsCollector] = stats self.crawler: Optional[Crawler] = crawler @classmethod - def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV: + def from_crawler(cls, crawler: Crawler) -> Self: """ Factory method, initializes the scheduler with arguments taken from the crawl settings """ @@ -221,9 +220,9 @@ class Scheduler(BaseScheduler): (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory (3) return the result of the dupefilter's ``open`` method """ - self.spider = spider - self.mqs = self._mq() - self.dqs = self._dq() if self.dqdir else None + self.spider: Spider = spider + self.mqs: ScrapyPriorityQueue = self._mq() + self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None return self.df.open() def close(self, reason: str) -> Optional[Deferred]: @@ -320,9 +319,10 @@ class Scheduler(BaseScheduler): return self.dqs.pop() return None - def _mq(self): + def _mq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with in-memory storage""" assert self.crawler + assert self.pqclass return build_from_crawler( self.pqclass, self.crawler, @@ -330,10 +330,11 @@ class Scheduler(BaseScheduler): key="", ) - def _dq(self): + def _dq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with disk storage""" assert self.crawler assert self.dqdir + assert self.pqclass state = self._read_dqs_state(self.dqdir) q = build_from_crawler( self.pqclass, diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 213ad590d..773825c5e 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -16,13 +16,14 @@ from typing import ( from scrapy import Request from scrapy.core.downloader import Downloader -from scrapy.crawler import Crawler from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) From bd0d4cee885744c7ea38185ec42f0137e7632b79 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 16:12:44 +0500 Subject: [PATCH 1414/2083] Fixes for queuelib. --- scrapy/core/scheduler.py | 4 +++- scrapy/squeues.py | 21 +++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index ab59c0d14..f30a5d9c9 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -6,7 +6,6 @@ from abc import abstractmethod from pathlib import Path from typing import TYPE_CHECKING, Any, Optional, Type, cast -from queuelib.queue import BaseQueue from twisted.internet.defer import Deferred from scrapy.crawler import Crawler @@ -19,6 +18,9 @@ from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: + # requires queuelib >= 1.6.2 + from queuelib.queue import BaseQueue + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 4676b058e..6f80ee388 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -22,7 +22,7 @@ if TYPE_CHECKING: def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: - class DirectoriesCreated(queue_class): + class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): @@ -37,7 +37,7 @@ def _serializable_queue( serialize: Callable[[Any], bytes], deserialize: Callable[[bytes], Any], ) -> Type[queue.BaseQueue]: - class SerializableQueue(queue_class): + class SerializableQueue(queue_class): # type: ignore[valid-type,misc] def push(self, obj: Any) -> None: s = serialize(obj) super().push(s) @@ -71,7 +71,7 @@ def _serializable_queue( def _scrapy_serialization_queue( queue_class: Type[queue.BaseQueue], ) -> Type[queue.BaseQueue]: - class ScrapyRequestQueue(queue_class): + class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] def __init__(self, crawler: Crawler, key: str): self.spider = crawler.spider super().__init__(key) @@ -110,7 +110,7 @@ def _scrapy_serialization_queue( def _scrapy_non_serialization_queue( queue_class: Type[queue.BaseQueue], ) -> Type[queue.BaseQueue]: - class ScrapyRequestQueue(queue_class): + class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] @classmethod def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: return cls() @@ -142,17 +142,18 @@ def _pickle_serialize(obj: Any) -> bytes: raise ValueError(str(e)) from e +# queue.*Queue aren't subclasses of queue.BaseQueue _PickleFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads + _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type] ) _PickleLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads + _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type] ) _MarshalFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads + _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type] ) _MarshalLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads + _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type] ) # public queue classes @@ -160,5 +161,5 @@ PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQu PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue) MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue) MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) -FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) -LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) +FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) # type: ignore[arg-type] +LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) # type: ignore[arg-type] From 1f394306e14ccab9d14ffb9adc64a7c5c08d9af6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 20:37:30 +0500 Subject: [PATCH 1415/2083] Use the Self type hint in from_crawler/from_settings. --- scrapy/core/downloader/contextfactory.py | 7 +++++- scrapy/core/downloader/handlers/ftp.py | 10 +++++++- scrapy/core/downloader/handlers/http10.py | 10 +++++++- scrapy/core/downloader/handlers/http11.py | 10 +++++++- scrapy/core/downloader/handlers/http2.py | 14 +++++------ scrapy/core/downloader/handlers/s3.py | 10 +++++++- scrapy/http/request/__init__.py | 13 ++++++---- scrapy/pipelines/files.py | 20 ++++++++++++---- scrapy/pipelines/images.py | 29 ++++++++++++++++++----- scrapy/pipelines/media.py | 12 ++++++++-- scrapy/spiders/crawl.py | 12 +++++++--- scrapy/spiders/sitemap.py | 4 +++- scrapy/utils/request.py | 9 +++++-- 13 files changed, 124 insertions(+), 36 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index dba4d8cdc..6a82634f1 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import warnings from typing import TYPE_CHECKING, Any, List, Optional @@ -25,6 +27,9 @@ from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from twisted.internet._sslverify import ClientTLSOptions + # typing.Self requires Python 3.11 + from typing_extensions import Self + @implementer(IPolicyForHTTPS) class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): @@ -62,7 +67,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): method: int = SSL.SSLv23_METHOD, *args: Any, **kwargs: Any, - ): + ) -> Self: tls_verbose_logging: bool = settings.getbool( "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" ) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 4081545ce..69add8558 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -28,18 +28,26 @@ In case of status 200 request, response.headers will come with two keys: 'Size' - with size of the downloaded data """ +from __future__ import annotations + import re from io import BytesIO +from typing import TYPE_CHECKING from urllib.parse import unquote from twisted.internet.protocol import ClientCreator, Protocol from twisted.protocols.ftp import CommandFailed, FTPClient +from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class ReceivedDataProtocol(Protocol): def __init__(self, filename=None): @@ -76,7 +84,7 @@ class FTPDownloadHandler: self.passive_mode = settings["FTP_PASSIVE_MODE"] @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) def download_request(self, request, spider): diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index d168c2b2e..256dc36a1 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,9 +1,17 @@ """Download handlers for http and https schemes """ +from __future__ import annotations + +from typing import TYPE_CHECKING + from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class HTTP10DownloadHandler: lazy = False @@ -17,7 +25,7 @@ class HTTP10DownloadHandler: self._crawler = crawler @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler) -> Self: return cls(crawler.settings, crawler) def download_request(self, request, spider): diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c3704de3d..15f8abc64 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,11 +1,14 @@ """Download handlers for http and https schemes""" +from __future__ import annotations + import ipaddress import logging import re from contextlib import suppress from io import BytesIO from time import time +from typing import TYPE_CHECKING from urllib.parse import urldefrag, urlunparse from twisted.internet import defer, protocol, ssl @@ -32,6 +35,11 @@ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) @@ -56,7 +64,7 @@ class HTTP11DownloadHandler: self._disconnect_timeout = 1 @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler) -> Self: return cls(crawler.settings, crawler) def download_request(self, request, spider): diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index b2579362c..e9a6b6fa3 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from time import time -from typing import Optional, Type, TypeVar +from typing import TYPE_CHECKING, Optional from urllib.parse import urldefrag from twisted.internet.base import DelayedCall @@ -16,9 +18,9 @@ from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes -H2DownloadHandlerOrSubclass = TypeVar( - "H2DownloadHandlerOrSubclass", bound="H2DownloadHandler" -) +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class H2DownloadHandler: @@ -31,9 +33,7 @@ class H2DownloadHandler: self._context_factory = load_context_factory_from_settings(settings, crawler) @classmethod - def from_crawler( - cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler - ) -> H2DownloadHandlerOrSubclass: + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) def download_request(self, request: Request, spider: Spider) -> Deferred: diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 1f7533759..99fbb49ce 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,9 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class S3DownloadHandler: def __init__( @@ -57,7 +65,7 @@ class S3DownloadHandler: self._download_http = _http_handler.download_request @classmethod - def from_crawler(cls, crawler, **kwargs): + def from_crawler(cls, crawler, **kwargs) -> Self: return cls(crawler.settings, crawler=crawler, **kwargs) def download_request(self, request, spider): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 6269ee86a..191b3cef4 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -5,8 +5,11 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +from __future__ import annotations + import inspect from typing import ( + TYPE_CHECKING, Any, AnyStr, Callable, @@ -17,8 +20,6 @@ from typing import ( NoReturn, Optional, Tuple, - Type, - TypeVar, Union, cast, ) @@ -32,7 +33,9 @@ from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax -RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: @@ -186,11 +189,11 @@ class Request(object_ref): @classmethod def from_curl( - cls: Type[RequestTypeVar], + cls, curl_command: str, ignore_unknown_options: bool = True, **kwargs: Any, - ) -> RequestTypeVar: + ) -> Self: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index d04218089..d00f44502 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -4,6 +4,8 @@ Files Pipeline See documentation in topics/media-pipeline.rst """ +from __future__ import annotations + import base64 import functools import hashlib @@ -16,7 +18,7 @@ from ftplib import FTP from io import BytesIO from os import PathLike from pathlib import Path -from typing import IO, DefaultDict, Optional, Set, Union +from typing import IO, TYPE_CHECKING, DefaultDict, Optional, Set, Type, Union, cast from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -34,6 +36,10 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) @@ -385,8 +391,8 @@ class FilesPipeline(MediaPipeline): super().__init__(download_func=download_func, settings=settings) @classmethod - def from_settings(cls, settings): - s3store = cls.STORE_SCHEMES["s3"] + def from_settings(cls, settings) -> Self: + s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -396,11 +402,15 @@ class FilesPipeline(MediaPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["FILES_STORE_S3_ACL"] - gcs_store = cls.STORE_SCHEMES["gs"] + gcs_store: Type[GCSFilesStore] = cast( + Type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None - ftp_store = cls.STORE_SCHEMES["ftp"] + ftp_store: Type[FTPFilesStore] = cast( + Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 137aa7a9a..e7ef06fb3 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -4,25 +4,38 @@ Images Pipeline See documentation in topics/media-pipeline.rst """ +from __future__ import annotations + import functools import hashlib import warnings from contextlib import suppress from io import BytesIO from os import PathLike -from typing import Dict, Tuple, Union +from typing import TYPE_CHECKING, Dict, Tuple, Type, Union, cast from itemadapter import ItemAdapter from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum +from scrapy.pipelines.files import ( + FileException, + FilesPipeline, + FTPFilesStore, + GCSFilesStore, + S3FilesStore, + _md5sum, +) # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.python import get_func_args, to_bytes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class NoimagesDrop(DropItem): """Product with no images exception""" @@ -96,8 +109,8 @@ class ImagesPipeline(FilesPipeline): self._deprecated_convert_image = None @classmethod - def from_settings(cls, settings): - s3store = cls.STORE_SCHEMES["s3"] + def from_settings(cls, settings) -> Self: + s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -107,11 +120,15 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["IMAGES_STORE_S3_ACL"] - gcs_store = cls.STORE_SCHEMES["gs"] + gcs_store: Type[GCSFilesStore] = cast( + Type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None - ftp_store = cls.STORE_SCHEMES["ftp"] + ftp_store: Type[FTPFilesStore] = cast( + Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index fc156ab41..fd5e70cb9 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,6 +1,9 @@ +from __future__ import annotations + import functools import logging from collections import defaultdict +from typing import TYPE_CHECKING from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure @@ -12,6 +15,11 @@ from scrapy.utils.defer import defer_result, mustbe_deferred from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) @@ -67,9 +75,9 @@ class MediaPipeline: return formatted_key @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler) -> Self: try: - pipe = cls.from_settings(crawler.settings) + pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined] except AttributeError: pipe = cls() pipe.crawler = crawler diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 2a3913da5..ba8b7b366 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -5,8 +5,10 @@ for scraping typical web sites that requires crawling pages. See documentation in docs/topics/spiders.rst """ +from __future__ import annotations + import copy -from typing import AsyncIterable, Awaitable, Sequence +from typing import TYPE_CHECKING, AsyncIterable, Awaitable, Sequence from scrapy.http import HtmlResponse, Request, Response from scrapy.linkextractors import LinkExtractor @@ -14,6 +16,10 @@ from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + def _identity(x): return x @@ -140,9 +146,9 @@ class CrawlSpider(Spider): self._rules[-1]._compile(self) @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler, *args, **kwargs) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) - spider._follow_links = crawler.settings.getbool( + spider._follow_links = crawler.settings.getbool( # type: ignore[attr-defined] "CRAWLSPIDER_FOLLOW_LINKS", True ) return spider diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index cd83a1464..f0e630c42 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging import re from typing import TYPE_CHECKING, Any @@ -26,7 +28,7 @@ class SitemapSpider(Spider): _warn_size: int @classmethod - def from_crawler(cls, crawler: "Crawler", *args: Any, **kwargs: Any) -> "Self": + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) spider._max_size = getattr( spider, "download_maxsize", spider.settings.getint("DOWNLOAD_MAXSIZE") diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 1f07d58eb..c86f9fe39 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -3,6 +3,8 @@ This module provides some useful functions for working with scrapy.http.Request objects """ +from __future__ import annotations + import hashlib import json import warnings @@ -32,6 +34,9 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + from scrapy.crawler import Crawler @@ -133,10 +138,10 @@ class RequestFingerprinter: """ @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler) -> Self: return cls(crawler) - def __init__(self, crawler: Optional["Crawler"] = None): + def __init__(self, crawler: Optional[Crawler] = None): if crawler: implementation = crawler.settings.get( "REQUEST_FINGERPRINTER_IMPLEMENTATION" From 8a08283580176049cb539423795830b9faea91a9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 5 May 2024 22:32:46 +0500 Subject: [PATCH 1416/2083] Full typing for scrapy/http/cookies.py. --- scrapy/http/cookies.py | 123 +++++++++++++++++++++-------------- scrapy/http/response/text.py | 4 +- 2 files changed, 76 insertions(+), 51 deletions(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 72855bad5..8af89c74f 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,36 +1,56 @@ +from __future__ import annotations + import re import time from http.cookiejar import Cookie from http.cookiejar import CookieJar as _CookieJar -from http.cookiejar import DefaultCookiePolicy -from typing import Sequence +from http.cookiejar import CookiePolicy, DefaultCookiePolicy +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterator, + List, + Optional, + Sequence, + Tuple, + cast, +) from scrapy import Request from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + # Defined in the http.cookiejar module, but undocumented: # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 IPV4_RE = re.compile(r"\.\d+$", re.ASCII) class CookieJar: - def __init__(self, policy=None, check_expired_frequency=10000): - self.policy = policy or DefaultCookiePolicy() - self.jar = _CookieJar(self.policy) - self.jar._cookies_lock = _DummyLock() - self.check_expired_frequency = check_expired_frequency - self.processed = 0 + def __init__( + self, + policy: Optional[CookiePolicy] = None, + check_expired_frequency: int = 10000, + ): + self.policy: CookiePolicy = policy or DefaultCookiePolicy() + self.jar: _CookieJar = _CookieJar(self.policy) + self.jar._cookies_lock = _DummyLock() # type: ignore[attr-defined] + self.check_expired_frequency: int = check_expired_frequency + self.processed: int = 0 - def extract_cookies(self, response, request): + def extract_cookies(self, response: Response, request: Request) -> None: wreq = WrappedRequest(request) wrsp = WrappedResponse(response) - return self.jar.extract_cookies(wrsp, wreq) + self.jar.extract_cookies(wrsp, wreq) # type: ignore[arg-type] def add_cookie_header(self, request: Request) -> None: wreq = WrappedRequest(request) - self.policy._now = self.jar._now = int(time.time()) + self.policy._now = self.jar._now = int(time.time()) # type: ignore[attr-defined] # the cookiejar implementation iterates through all domains # instead we restrict to potential matches on the domain @@ -47,10 +67,10 @@ class CookieJar: cookies = [] for host in hosts: - if host in self.jar._cookies: - cookies += self.jar._cookies_for_domain(host, wreq) + if host in self.jar._cookies: # type: ignore[attr-defined] + cookies += self.jar._cookies_for_domain(host, wreq) # type: ignore[attr-defined] - attrs = self.jar._cookie_attrs(cookies) + attrs = self.jar._cookie_attrs(cookies) # type: ignore[attr-defined] if attrs: if not wreq.has_header("Cookie"): wreq.add_unredirected_header("Cookie", "; ".join(attrs)) @@ -61,37 +81,42 @@ class CookieJar: self.jar.clear_expired_cookies() @property - def _cookies(self): - return self.jar._cookies + def _cookies(self) -> Dict[str, Dict[str, Dict[str, Cookie]]]: + return self.jar._cookies # type: ignore[attr-defined,no-any-return] - def clear_session_cookies(self, *args, **kwargs): - return self.jar.clear_session_cookies(*args, **kwargs) + def clear_session_cookies(self) -> None: + return self.jar.clear_session_cookies() - def clear(self, domain=None, path=None, name=None): - return self.jar.clear(domain, path, name) + def clear( + self, + domain: Optional[str] = None, + path: Optional[str] = None, + name: Optional[str] = None, + ) -> None: + self.jar.clear(domain, path, name) - def __iter__(self): + def __iter__(self) -> Iterator[Cookie]: return iter(self.jar) - def __len__(self): + def __len__(self) -> int: return len(self.jar) - def set_policy(self, pol): - return self.jar.set_policy(pol) + def set_policy(self, pol: CookiePolicy) -> None: + self.jar.set_policy(pol) def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]: wreq = WrappedRequest(request) wrsp = WrappedResponse(response) - return self.jar.make_cookies(wrsp, wreq) + return self.jar.make_cookies(wrsp, wreq) # type: ignore[arg-type] - def set_cookie(self, cookie): + def set_cookie(self, cookie: Cookie) -> None: self.jar.set_cookie(cookie) def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None: - self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) + self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) # type: ignore[arg-type] -def potential_domain_matches(domain): +def potential_domain_matches(domain: str) -> List[str]: """Potential domain matches for a cookie >>> potential_domain_matches('www.example.com') @@ -111,10 +136,10 @@ def potential_domain_matches(domain): class _DummyLock: - def acquire(self): + def acquire(self) -> None: pass - def release(self): + def release(self) -> None: pass @@ -124,19 +149,19 @@ class WrappedRequest: see http://docs.python.org/library/urllib2.html#urllib2.Request """ - def __init__(self, request): + def __init__(self, request: Request): self.request = request - def get_full_url(self): + def get_full_url(self) -> str: return self.request.url - def get_host(self): + def get_host(self) -> str: return urlparse_cached(self.request).netloc - def get_type(self): + def get_type(self) -> str: return urlparse_cached(self.request).scheme - def is_unverifiable(self): + def is_unverifiable(self) -> bool: """Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965. It defaults to False. An unverifiable request is one whose URL the user did not have the @@ -144,36 +169,36 @@ class WrappedRequest: HTML document, and the user had no option to approve the automatic fetching of the image, this should be true. """ - return self.request.meta.get("is_unverifiable", False) + return cast(bool, self.request.meta.get("is_unverifiable", False)) @property - def full_url(self): + def full_url(self) -> str: return self.get_full_url() @property - def host(self): + def host(self) -> str: return self.get_host() @property - def type(self): + def type(self) -> str: return self.get_type() @property - def unverifiable(self): + def unverifiable(self) -> bool: return self.is_unverifiable() @property - def origin_req_host(self): - return urlparse_cached(self.request).hostname + def origin_req_host(self) -> str: + return cast(str, urlparse_cached(self.request).hostname) - def has_header(self, name): + def has_header(self, name: str) -> bool: return name in self.request.headers - def get_header(self, name, default=None): + def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]: value = self.request.headers.get(name, default) return to_unicode(value, errors="replace") if value is not None else None - def header_items(self): + def header_items(self) -> List[Tuple[str, List[str]]]: return [ ( to_unicode(k, errors="replace"), @@ -182,18 +207,18 @@ class WrappedRequest: for k, v in self.request.headers.items() ] - def add_unredirected_header(self, name, value): + def add_unredirected_header(self, name: str, value: str) -> None: self.request.headers.appendlist(name, value) class WrappedResponse: - def __init__(self, response): + def __init__(self, response: Response): self.response = response - def info(self): + def info(self) -> Self: return self - def get_all(self, name, default=None): + def get_all(self, name: str, default: Any = None) -> List[str]: return [ to_unicode(v, errors="replace") for v in self.response.headers.getlist(name) ] diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 2816610fb..522ffc0d5 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -159,12 +159,12 @@ class TextResponse(Response): def jmespath(self, query: str, **kwargs: Any) -> SelectorList: from scrapy.selector import SelectorList - if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined] + if not hasattr(self.selector, "jmespath"): raise AttributeError( "Please install parsel >= 1.8.1 to get jmespath support" ) - return cast(SelectorList, self.selector.jmespath(query, **kwargs)) # type: ignore[attr-defined] + return cast(SelectorList, self.selector.jmespath(query, **kwargs)) def xpath(self, query: str, **kwargs: Any) -> SelectorList: from scrapy.selector import SelectorList From c76dfc383f34514a5a2841d2e32ac4e0c8c751a3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 5 May 2024 22:52:15 +0500 Subject: [PATCH 1417/2083] Full typing for scrapy/linkextractors. --- scrapy/commands/bench.py | 3 +- scrapy/linkextractors/__init__.py | 8 +- scrapy/linkextractors/lxmlhtml.py | 149 +++++++++++++++++------------- 3 files changed, 91 insertions(+), 69 deletions(-) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 2e6bb5d86..7523f3cfe 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -8,7 +8,7 @@ from urllib.parse import urlencode import scrapy from scrapy import Request from scrapy.commands import ScrapyCommand -from scrapy.http import Response +from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor @@ -61,5 +61,6 @@ class _BenchSpider(scrapy.Spider): return [scrapy.Request(url, dont_filter=True)] def parse(self, response: Response) -> Any: # type: ignore[override] + assert isinstance(Response, TextResponse) for link in self.link_extractor.extract_links(response): yield scrapy.Request(link.url, callback=self.parse) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 73a63651c..38dbe8135 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -7,6 +7,7 @@ For more info see docs/topics/link-extractors.rst """ import re +from typing import Iterable # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ @@ -110,14 +111,11 @@ IGNORED_EXTENSIONS = [ ] -_re_type = type(re.compile("", 0)) - - -def _matches(url, regexs): +def _matches(url: str, regexs: Iterable[re.Pattern[str]]) -> bool: return any(r.search(url) for r in regexs) -def _is_valid_url(url): +def _is_valid_url(url: str) -> bool: return url.split("://", 1)[0] in {"http", "https", "file", "ftp"} diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 71c6d08fc..3fa7d1e3a 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -5,21 +5,19 @@ Link extractor based on lxml.html import logging import operator from functools import partial +from typing import Any, Callable, Iterable, List, Optional, Set, Tuple, Union, cast from urllib.parse import urljoin, urlparse from lxml import etree # nosec +from lxml.html import HtmlElement # nosec from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string +from scrapy import Selector +from scrapy.http import TextResponse from scrapy.link import Link -from scrapy.linkextractors import ( - IGNORED_EXTENSIONS, - _is_valid_url, - _matches, - _re_type, - re, -) +from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url @@ -33,41 +31,56 @@ XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" _collect_string_content = etree.XPath("string()") -def _nons(tag): +def _nons(tag: Any) -> Any: if isinstance(tag, str): if tag[0] == "{" and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE: return tag.split("}")[-1] return tag -def _identity(x): +def _identity(x: Any) -> Any: return x -def _canonicalize_link_url(link): +def _canonicalize_link_url(link: Link) -> str: return canonicalize_url(link.url, keep_fragments=True) class LxmlParserLinkExtractor: def __init__( self, - tag="a", - attr="href", - process=None, - unique=False, - strip=True, - canonicalized=False, + tag: Union[str, Callable[[str], bool]] = "a", + attr: Union[str, Callable[[str], bool]] = "href", + process: Optional[Callable[[Any], Any]] = None, + unique: bool = False, + strip: bool = True, + canonicalized: bool = False, ): - self.scan_tag = tag if callable(tag) else partial(operator.eq, tag) - self.scan_attr = attr if callable(attr) else partial(operator.eq, attr) - self.process_attr = process if callable(process) else _identity - self.unique = unique - self.strip = strip - self.link_key = ( - operator.attrgetter("url") if canonicalized else _canonicalize_link_url + # mypy doesn't infer types for operator.* and also for partial() + self.scan_tag: Callable[[str], bool] = ( + tag + if callable(tag) + else cast(Callable[[str], bool], partial(operator.eq, tag)) + ) + self.scan_attr: Callable[[str], bool] = ( + attr + if callable(attr) + else cast(Callable[[str], bool], partial(operator.eq, attr)) + ) + self.process_attr: Callable[[Any], Any] = ( + process if callable(process) else _identity + ) + self.unique: bool = unique + self.strip: bool = strip + self.link_key: Callable[[Link], str] = ( + cast(Callable[[Link], str], operator.attrgetter("url")) + if canonicalized + else _canonicalize_link_url ) - def _iter_links(self, document): + def _iter_links( + self, document: HtmlElement + ) -> Iterable[Tuple[HtmlElement, str, str]]: for el in document.iter(etree.Element): if not self.scan_tag(_nons(el.tag)): continue @@ -75,10 +88,16 @@ class LxmlParserLinkExtractor: for attrib in attribs: if not self.scan_attr(attrib): continue - yield (el, attrib, attribs[attrib]) + yield el, attrib, attribs[attrib] - def _extract_links(self, selector, response_url, response_encoding, base_url): - links = [] + def _extract_links( + self, + selector: Selector, + response_url: str, + response_encoding: str, + base_url: str, + ) -> List[Link]: + links: List[Link] = [] # hacky way to get the underlying lxml parsed document for el, attr, attr_val in self._iter_links(selector.root): # pseudo lxml.html.HtmlElement.make_links_absolute(base_url) @@ -108,44 +127,48 @@ class LxmlParserLinkExtractor: links.append(link) return self._deduplicate_if_needed(links) - def extract_links(self, response): + def extract_links(self, response: TextResponse) -> List[Link]: base_url = get_base_url(response) return self._extract_links( response.selector, response.url, response.encoding, base_url ) - def _process_links(self, links): + def _process_links(self, links: List[Link]) -> List[Link]: """Normalize and filter extracted links The subclass should override it if necessary """ return self._deduplicate_if_needed(links) - def _deduplicate_if_needed(self, links): + def _deduplicate_if_needed(self, links: List[Link]) -> List[Link]: if self.unique: return unique_list(links, key=self.link_key) return links +_RegexT = Union[str, re.Pattern[str]] +_RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]] + + class LxmlLinkExtractor: _csstranslator = HTMLTranslator() def __init__( self, - allow=(), - deny=(), - allow_domains=(), - deny_domains=(), - restrict_xpaths=(), - tags=("a", "area"), - attrs=("href",), - canonicalize=False, - unique=True, - process_value=None, - deny_extensions=None, - restrict_css=(), - strip=True, - restrict_text=None, + allow: _RegexOrSeveralT = (), + deny: _RegexOrSeveralT = (), + allow_domains: Union[str, Iterable[str]] = (), + deny_domains: Union[str, Iterable[str]] = (), + restrict_xpaths: Union[str, Iterable[str]] = (), + tags: Union[str, Iterable[str]] = ("a", "area"), + attrs: Union[str, Iterable[str]] = ("href",), + canonicalize: bool = False, + unique: bool = True, + process_value: Optional[Callable[[Any], Any]] = None, + deny_extensions: Union[str, Iterable[str], None] = None, + restrict_css: Union[str, Iterable[str]] = (), + strip: bool = True, + restrict_text: Optional[_RegexOrSeveralT] = None, ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) self.link_extractor = LxmlParserLinkExtractor( @@ -156,31 +179,31 @@ class LxmlLinkExtractor: strip=strip, canonicalized=not canonicalize, ) - self.allow_res = [ - x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow) - ] - self.deny_res = [ - x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny) - ] + self.allow_res: List[re.Pattern[str]] = self._compile_regexes(allow) + self.deny_res: List[re.Pattern[str]] = self._compile_regexes(deny) - self.allow_domains = set(arg_to_iter(allow_domains)) - self.deny_domains = set(arg_to_iter(deny_domains)) + self.allow_domains: Set[str] = set(arg_to_iter(allow_domains)) + self.deny_domains: Set[str] = set(arg_to_iter(deny_domains)) - self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + self.restrict_xpaths: Tuple[str, ...] = tuple(arg_to_iter(restrict_xpaths)) self.restrict_xpaths += tuple( map(self._csstranslator.css_to_xpath, arg_to_iter(restrict_css)) ) if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS - self.canonicalize = canonicalize - self.deny_extensions = {"." + e for e in arg_to_iter(deny_extensions)} - self.restrict_text = [ - x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(restrict_text) + self.canonicalize: bool = canonicalize + self.deny_extensions: Set[str] = {"." + e for e in arg_to_iter(deny_extensions)} + self.restrict_text: List[re.Pattern[str]] = self._compile_regexes(restrict_text) + + @staticmethod + def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[re.Pattern[str]]: + return [ + x if isinstance(x, re.Pattern) else re.compile(x) + for x in arg_to_iter(value) ] - def _link_allowed(self, link): + def _link_allowed(self, link: Link) -> bool: if not _is_valid_url(link.url): return False if self.allow_res and not _matches(link.url, self.allow_res): @@ -202,7 +225,7 @@ class LxmlLinkExtractor: return False return True - def matches(self, url): + def matches(self, url: str) -> bool: if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): return False if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): @@ -216,7 +239,7 @@ class LxmlLinkExtractor: denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] return any(allowed) and not any(denied) - def _process_links(self, links): + def _process_links(self, links: List[Link]) -> List[Link]: links = [x for x in links if self._link_allowed(x)] if self.canonicalize: for link in links: @@ -224,10 +247,10 @@ class LxmlLinkExtractor: links = self.link_extractor._process_links(links) return links - def _extract_links(self, *args, **kwargs): + def _extract_links(self, *args: Any, **kwargs: Any) -> List[Link]: return self.link_extractor._extract_links(*args, **kwargs) - def extract_links(self, response): + def extract_links(self, response: TextResponse) -> List[Link]: """Returns a list of :class:`~scrapy.link.Link` objects from the specified :class:`response `. From 4b47a5dc32232431494b38f3a1d9ddb3dbaa6247 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 5 May 2024 22:55:21 +0500 Subject: [PATCH 1418/2083] Skip coverage checks for TYPE_CHECKING blocks. --- .coveragerc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.coveragerc b/.coveragerc index ad0ee0f6c..f9ad353d5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,3 +4,9 @@ include = scrapy/* omit = tests/* disable_warnings = include-ignored + +[report] +# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 +exclude_lines = + pragma: no cover + if TYPE_CHECKING: From 40e4a5960477299e36a0f7363db4a3c03576f64e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 5 May 2024 23:00:16 +0500 Subject: [PATCH 1419/2083] Fix Python 3.8. --- scrapy/linkextractors/__init__.py | 4 ++-- scrapy/linkextractors/lxmlhtml.py | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 38dbe8135..d59005edd 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -7,7 +7,7 @@ For more info see docs/topics/link-extractors.rst """ import re -from typing import Iterable +from typing import Iterable, Pattern # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ @@ -111,7 +111,7 @@ IGNORED_EXTENSIONS = [ ] -def _matches(url: str, regexs: Iterable[re.Pattern[str]]) -> bool: +def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool: return any(r.search(url) for r in regexs) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 3fa7d1e3a..33a10cd6c 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -5,7 +5,18 @@ Link extractor based on lxml.html import logging import operator from functools import partial -from typing import Any, Callable, Iterable, List, Optional, Set, Tuple, Union, cast +from typing import ( + Any, + Callable, + Iterable, + List, + Optional, + Pattern, + Set, + Tuple, + Union, + cast, +) from urllib.parse import urljoin, urlparse from lxml import etree # nosec @@ -146,7 +157,7 @@ class LxmlParserLinkExtractor: return links -_RegexT = Union[str, re.Pattern[str]] +_RegexT = Union[str, Pattern[str]] _RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]] @@ -179,8 +190,8 @@ class LxmlLinkExtractor: strip=strip, canonicalized=not canonicalize, ) - self.allow_res: List[re.Pattern[str]] = self._compile_regexes(allow) - self.deny_res: List[re.Pattern[str]] = self._compile_regexes(deny) + self.allow_res: List[Pattern[str]] = self._compile_regexes(allow) + self.deny_res: List[Pattern[str]] = self._compile_regexes(deny) self.allow_domains: Set[str] = set(arg_to_iter(allow_domains)) self.deny_domains: Set[str] = set(arg_to_iter(deny_domains)) @@ -194,10 +205,10 @@ class LxmlLinkExtractor: deny_extensions = IGNORED_EXTENSIONS self.canonicalize: bool = canonicalize self.deny_extensions: Set[str] = {"." + e for e in arg_to_iter(deny_extensions)} - self.restrict_text: List[re.Pattern[str]] = self._compile_regexes(restrict_text) + self.restrict_text: List[Pattern[str]] = self._compile_regexes(restrict_text) @staticmethod - def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[re.Pattern[str]]: + def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[Pattern[str]]: return [ x if isinstance(x, re.Pattern) else re.compile(x) for x in arg_to_iter(value) From aa025d7eacb461ccb0c724584532d402bb400bd1 Mon Sep 17 00:00:00 2001 From: Sanchay Kumar <51812506+kumar-sanchay@users.noreply.github.com> Date: Mon, 6 May 2024 14:59:35 +0530 Subject: [PATCH 1420/2083] Indicate that Selector.type can be json (#6334) --- scrapy/selector/unified.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index aa9581fcd..e852aadc7 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -47,7 +47,7 @@ class Selector(_ParselSelector, object_ref): ``response`` isn't available. Using ``text`` and ``response`` together is undefined behavior. - ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` + ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"json"`` or ``None`` (default). If ``type`` is ``None``, the selector automatically chooses the best type From 2cba7896d26dda51ff2e598300363531ebd328b8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 6 May 2024 14:31:24 +0500 Subject: [PATCH 1421/2083] Small fix for _get_slot_key(). --- scrapy/core/downloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index f88da41ea..98e1af6fb 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -143,7 +143,7 @@ class Downloader: return key, self.slots[key] - def _get_slot_key(self, request: Request, spider: Any) -> str: + def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str: if self.DOWNLOAD_SLOT in request.meta: return cast(str, request.meta[self.DOWNLOAD_SLOT]) From c4d2748ff572adad8150f1bc09b2d52e61d9dfc8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 17:07:19 +0500 Subject: [PATCH 1422/2083] Small typing improvements in scrapy/core/downloader/contextfactory.py. --- scrapy/core/downloader/contextfactory.py | 7 +++++-- scrapy/core/downloader/handlers/http10.py | 2 +- scrapy/core/downloader/handlers/http11.py | 2 +- scrapy/core/downloader/handlers/http2.py | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 6a82634f1..0e77cd2fe 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,6 +21,7 @@ from scrapy.core.downloader.tls import ( ScrapyClientTLSOptions, openssl_methods, ) +from scrapy.crawler import Crawler from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object @@ -102,7 +103,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: - ctx = self.getCertificateOptions().getContext() + ctx: SSL.Context = self.getCertificateOptions().getContext() ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx @@ -165,7 +166,9 @@ class AcceptableProtocolsContextFactory: return options -def load_context_factory_from_settings(settings, crawler): +def load_context_factory_from_settings( + settings: BaseSettings, crawler: Crawler +) -> IPolicyForHTTPS: ssl_method = openssl_methods[settings.get("DOWNLOADER_CLIENT_TLS_METHOD")] context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]) # try method-aware context factory diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 256dc36a1..2507a4231 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: class HTTP10DownloadHandler: lazy = False - def __init__(self, settings, crawler=None): + def __init__(self, settings, crawler): self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"]) self.ClientContextFactory = load_object( settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 15f8abc64..52561f7e6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -46,7 +46,7 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler: lazy = False - def __init__(self, settings, crawler=None): + def __init__(self, settings, crawler): self._crawler = crawler from twisted.internet import reactor diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index e9a6b6fa3..efe45c459 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: class H2DownloadHandler: - def __init__(self, settings: Settings, crawler: Optional[Crawler] = None): + def __init__(self, settings: Settings, crawler: Crawler): self._crawler = crawler from twisted.internet import reactor From ec4d40702227f5486c184de9439da4896121a33b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 17:36:16 +0500 Subject: [PATCH 1423/2083] Full typing for smaller download handlers. --- scrapy/core/downloader/handlers/file.py | 4 +++- scrapy/core/downloader/handlers/http11.py | 6 ++--- scrapy/core/downloader/handlers/http2.py | 3 ++- scrapy/core/downloader/handlers/s3.py | 27 ++++++++++++++--------- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 4824167da..17dd7483b 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -2,6 +2,8 @@ from pathlib import Path from w3lib.url import file_uri_to_path +from scrapy import Request, Spider +from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers @@ -10,7 +12,7 @@ class FileDownloadHandler: lazy = False @defers - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Response: filepath = file_uri_to_path(request.url) body = Path(filepath).read_bytes() respcls = responsetypes.from_args(filename=filepath, body=body) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 52561f7e6..2e7ea559a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -27,11 +27,11 @@ from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer from zope.interface import implementer -from scrapy import signals +from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import StopDownload -from scrapy.http import Headers +from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.python import to_bytes, to_unicode @@ -67,7 +67,7 @@ class HTTP11DownloadHandler: def from_crawler(cls, crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Response: """Return a deferred for the HTTP download""" agent = ScrapyAgent( contextFactory=self._contextFactory, diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index efe45c459..16fc1e3ae 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -8,6 +8,7 @@ from twisted.internet.base import DelayedCall from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI +from twisted.web.iweb import IPolicyForHTTPS from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse @@ -54,7 +55,7 @@ class ScrapyH2Agent: def __init__( self, - context_factory, + context_factory: IPolicyForHTTPS, pool: H2ConnectionPool, connect_timeout: int = 10, bind_address: Optional[bytes] = None, diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 99fbb49ce..c88dd2cdc 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,9 +1,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Optional, Type +from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response +from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler @@ -16,14 +20,14 @@ if TYPE_CHECKING: class S3DownloadHandler: def __init__( self, - settings, + settings: BaseSettings, *, - crawler=None, - aws_access_key_id=None, - aws_secret_access_key=None, - aws_session_token=None, - httpdownloadhandler=HTTPDownloadHandler, - **kw, + crawler: Crawler, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + aws_session_token: Optional[str] = None, + httpdownloadhandler: Type[HTTPDownloadHandler] = HTTPDownloadHandler, + **kw: Any, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") @@ -51,6 +55,8 @@ class S3DownloadHandler: if kw: raise TypeError(f"Unexpected keyword arguments: {kw}") if not self.anon: + assert aws_access_key_id is not None + assert aws_secret_access_key is not None SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"] self._signer = SignerCls( botocore.credentials.Credentials( @@ -65,10 +71,10 @@ class S3DownloadHandler: self._download_http = _http_handler.download_request @classmethod - def from_crawler(cls, crawler, **kwargs) -> Self: + def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self: return cls(crawler.settings, crawler=crawler, **kwargs) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Response: p = urlparse_cached(request) scheme = "https" if request.meta.get("is_secure") else "http" bucket = p.hostname @@ -85,6 +91,7 @@ class S3DownloadHandler: headers=request.headers.to_unicode_dict(), data=request.body, ) + assert self._signer self._signer.add_auth(awsrequest) request = request.replace(url=url, headers=awsrequest.headers.items()) return self._download_http(request, spider) From e8e13ebb78dd1d2db425b285335f31dba7c1fd39 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 18:03:06 +0500 Subject: [PATCH 1424/2083] Full typing for scrapy/core/downloader/handlers/ftp.py. --- scrapy/core/downloader/handlers/ftp.py | 40 +++++++++++++++----------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 69add8558..ed94b2221 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -32,15 +32,19 @@ from __future__ import annotations import re from io import BytesIO -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional from urllib.parse import unquote +from twisted.internet.defer import Deferred from twisted.internet.protocol import ClientCreator, Protocol from twisted.protocols.ftp import CommandFailed, FTPClient +from twisted.python.failure import Failure +from scrapy import Request, Spider from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.responsetypes import responsetypes +from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes @@ -50,20 +54,20 @@ if TYPE_CHECKING: class ReceivedDataProtocol(Protocol): - def __init__(self, filename=None): - self.__filename = filename - self.body = open(filename, "wb") if filename else BytesIO() - self.size = 0 + def __init__(self, filename: Optional[str] = None): + self.__filename: Optional[str] = filename + self.body: BinaryIO = open(filename, "wb") if filename else BytesIO() + self.size: int = 0 - def dataReceived(self, data): + def dataReceived(self, data: bytes) -> None: self.body.write(data) self.size += len(data) @property - def filename(self): + def filename(self) -> Optional[str]: return self.__filename - def close(self): + def close(self) -> None: self.body.close() if self.filename else self.body.seek(0) @@ -73,12 +77,12 @@ _CODE_RE = re.compile(r"\d+") class FTPDownloadHandler: lazy = False - CODE_MAPPING = { + CODE_MAPPING: Dict[str, int] = { "550": 404, "default": 503, } - def __init__(self, settings): + def __init__(self, settings: BaseSettings): self.default_user = settings["FTP_USER"] self.default_password = settings["FTP_PASSWORD"] self.passive_mode = settings["FTP_PASSIVE_MODE"] @@ -87,7 +91,7 @@ class FTPDownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: from twisted.internet import reactor parsed_url = urlparse_cached(request) @@ -99,10 +103,10 @@ class FTPDownloadHandler: creator = ClientCreator( reactor, FTPClient, user, password, passive=passive_mode ) - dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) + dfd: Deferred = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path)) - def gotClient(self, client, request, filepath): + def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred: self.client = client protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) return client.retrieveFile(filepath, protocol).addCallbacks( @@ -112,15 +116,18 @@ class FTPDownloadHandler: errbackArgs=(request,), ) - def _build_response(self, result, request, protocol): + def _build_response( + self, result: Any, request: Request, protocol: ReceivedDataProtocol + ) -> Response: self.result = result protocol.close() headers = {"local filename": protocol.filename or "", "size": protocol.size} body = to_bytes(protocol.filename or protocol.body.read()) respcls = responsetypes.from_args(url=request.url, body=body) - return respcls(url=request.url, status=200, body=body, headers=headers) + # hints for Headers-related types may need to be fixed to not use AnyStr + return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type] - def _failed(self, result, request): + def _failed(self, result: Failure, request: Request) -> Response: message = result.getErrorMessage() if result.type == CommandFailed: m = _CODE_RE.search(message) @@ -130,4 +137,5 @@ class FTPDownloadHandler: return Response( url=request.url, status=httpcode, body=to_bytes(message) ) + assert result.type raise result.type(result.value) From af3e38ab1f3dc7095ae27f572cbde0ea652d7664 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 18:09:25 +0500 Subject: [PATCH 1425/2083] Full typing for scrapy/core/downloader/handlers/http10.py. --- scrapy/core/downloader/handlers/http10.py | 27 +++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 2507a4231..9117cb818 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -3,8 +3,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Type +from twisted.internet.defer import Deferred + +from scrapy import Request, Spider +from scrapy.crawler import Crawler +from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode @@ -12,29 +17,33 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory + from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory class HTTP10DownloadHandler: lazy = False - def __init__(self, settings, crawler): - self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"]) - self.ClientContextFactory = load_object( + def __init__(self, settings: BaseSettings, crawler: Crawler): + self.HTTPClientFactory: Type[ScrapyHTTPClientFactory] = load_object( + settings["DOWNLOADER_HTTPCLIENTFACTORY"] + ) + self.ClientContextFactory: Type[ScrapyClientContextFactory] = load_object( settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] ) - self._settings = settings - self._crawler = crawler + self._settings: BaseSettings = settings + self._crawler: Crawler = crawler @classmethod - def from_crawler(cls, crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: """Return a deferred for the HTTP download""" factory = self.HTTPClientFactory(request) self._connect(factory) return factory.deferred - def _connect(self, factory): + def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred: from twisted.internet import reactor host, port = to_unicode(factory.host), factory.port From 045387e07faba4a18ecff919457d69938e89b710 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 4 May 2024 19:56:23 +0500 Subject: [PATCH 1426/2083] More typing for scrapy/core/downloader/handlers/http11.py. --- scrapy/core/downloader/handlers/http11.py | 286 +++++++++++++--------- scrapy/core/downloader/handlers/s3.py | 5 +- 2 files changed, 169 insertions(+), 122 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 2e7ea559a..40ae1921a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,31 +8,33 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast from urllib.parse import urldefrag, urlunparse -from twisted.internet import defer, protocol, ssl +from twisted.internet import ssl +from twisted.internet.base import ReactorBase +from twisted.internet.defer import CancelledError, Deferred, succeed from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError +from twisted.internet.interfaces import IConsumer +from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure -from twisted.web.client import ( - URI, - Agent, - HTTPConnectionPool, - ResponseDone, - ResponseFailed, -) +from twisted.web.client import URI, Agent, HTTPConnectionPool +from twisted.web.client import Response as TxResponse +from twisted.web.client import ResponseDone, ResponseFailed from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS from zope.interface import implementer from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse +from scrapy.crawler import Crawler from scrapy.exceptions import StopDownload from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes +from scrapy.settings import BaseSettings from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: @@ -46,28 +48,30 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler: lazy = False - def __init__(self, settings, crawler): + def __init__(self, settings: BaseSettings, crawler: Crawler): self._crawler = crawler from twisted.internet import reactor - self._pool = HTTPConnectionPool(reactor, persistent=True) + self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint( "CONCURRENT_REQUESTS_PER_DOMAIN" ) self._pool._factory.noisy = False - self._contextFactory = load_context_factory_from_settings(settings, crawler) - self._default_maxsize = settings.getint("DOWNLOAD_MAXSIZE") - self._default_warnsize = settings.getint("DOWNLOAD_WARNSIZE") - self._fail_on_dataloss = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS") - self._disconnect_timeout = 1 + self._contextFactory: IPolicyForHTTPS = load_context_factory_from_settings( + settings, crawler + ) + self._default_maxsize: int = settings.getint("DOWNLOAD_MAXSIZE") + self._default_warnsize: int = settings.getint("DOWNLOAD_WARNSIZE") + self._fail_on_dataloss: bool = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS") + self._disconnect_timeout: int = 1 @classmethod - def from_crawler(cls, crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider) -> Response: + def download_request(self, request: Request, spider: Spider) -> Deferred: """Return a deferred for the HTTP download""" agent = ScrapyAgent( contextFactory=self._contextFactory, @@ -79,10 +83,10 @@ class HTTP11DownloadHandler: ) return agent.download_request(request) - def close(self): + def close(self) -> Deferred: from twisted.internet import reactor - d = self._pool.closeCachedConnections() + d: Deferred = self._pool.closeCachedConnections() # closeCachedConnections will hang on network or server issues, so # we'll manually timeout the deferred. # @@ -93,7 +97,7 @@ class HTTP11DownloadHandler: # issue a callback after `_disconnect_timeout` seconds. delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, []) - def cancel_delayed_call(result): + def cancel_delayed_call(result: Any) -> Any: if delayed_call.active(): delayed_call.cancel() return result @@ -123,39 +127,41 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def __init__( self, - reactor, - host, - port, - proxyConf, - contextFactory, - timeout=30, - bindAddress=None, + reactor: ReactorBase, + host: str, + port: int, + proxyConf: Tuple[str, int, Optional[bytes]], + contextFactory: IPolicyForHTTPS, + timeout: float = 30, + bindAddress: Optional[Tuple[str, int]] = None, ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) - self._tunnelReadyDeferred = defer.Deferred() - self._tunneledHost = host - self._tunneledPort = port - self._contextFactory = contextFactory - self._connectBuffer = bytearray() + self._tunnelReadyDeferred: Deferred = Deferred() + self._tunneledHost: str = host + self._tunneledPort: int = port + self._contextFactory: IPolicyForHTTPS = contextFactory + self._connectBuffer: bytearray = bytearray() - def requestTunnel(self, protocol): + def requestTunnel(self, protocol: Protocol) -> Protocol: """Asks the proxy to open a tunnel.""" + assert protocol.transport tunnelReq = tunnel_request_data( self._tunneledHost, self._tunneledPort, self._proxyAuthHeader ) protocol.transport.write(tunnelReq) self._protocolDataReceived = protocol.dataReceived - protocol.dataReceived = self.processProxyResponse + protocol.dataReceived = self.processProxyResponse # type: ignore[method-assign] self._protocol = protocol return protocol - def processProxyResponse(self, rcvd_bytes): + def processProxyResponse(self, data: bytes) -> None: """Processes the response from the proxy. If the tunnel is successfully created, notifies the client that we are ready to send requests. If not raises a TunnelError. """ - self._connectBuffer += rcvd_bytes + assert self._protocol.transport + self._connectBuffer += data # make sure that enough (all) bytes are consumed # and that we've got all HTTP headers (ending with a blank line) # from the proxy so that we don't send those bytes to the TLS layer @@ -163,23 +169,24 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): # see https://github.com/scrapy/scrapy/issues/2491 if b"\r\n\r\n" not in self._connectBuffer: return - self._protocol.dataReceived = self._protocolDataReceived + self._protocol.dataReceived = self._protocolDataReceived # type: ignore[method-assign] respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) if respm and int(respm.group("status")) == 200: # set proper Server Name Indication extension - sslOptions = self._contextFactory.creatorForNetloc( + sslOptions = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc] self._tunneledHost, self._tunneledPort ) self._protocol.transport.startTLS(sslOptions, self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) else: + extra: Any if respm: extra = { "status": int(respm.group("status")), "reason": respm.group("reason").strip(), } else: - extra = rcvd_bytes[: self._truncatedLength] + extra = data[: self._truncatedLength] self._tunnelReadyDeferred.errback( TunnelError( "Could not open CONNECT tunnel with proxy " @@ -187,11 +194,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): ) ) - def connectFailed(self, reason): + def connectFailed(self, reason: Failure) -> None: """Propagates the errback to the appropriate deferred.""" self._tunnelReadyDeferred.errback(reason) - def connect(self, protocolFactory): + def connect(self, protocolFactory: Factory) -> Deferred: self._protocolFactory = protocolFactory connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) @@ -199,7 +206,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): return self._tunnelReadyDeferred -def tunnel_request_data(host, port, proxy_auth_header=None): +def tunnel_request_data( + host: str, port: int, proxy_auth_header: Optional[bytes] = None +) -> bytes: r""" Return binary content of a CONNECT request. @@ -230,18 +239,20 @@ class TunnelingAgent(Agent): def __init__( self, - reactor, - proxyConf, - contextFactory=None, - connectTimeout=None, - bindAddress=None, - pool=None, + reactor: ReactorBase, + proxyConf: Tuple[str, int, Optional[bytes]], + contextFactory: Optional[IPolicyForHTTPS] = None, + connectTimeout: Optional[float] = None, + bindAddress: Optional[bytes] = None, + pool: Optional[HTTPConnectionPool] = None, ): + # TODO make this arg required instead + assert contextFactory is not None super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) - self._proxyConf = proxyConf - self._contextFactory = contextFactory + self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf + self._contextFactory: IPolicyForHTTPS = contextFactory - def _getEndpoint(self, uri): + def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint: return TunnelingTCP4ClientEndpoint( reactor=self._reactor, host=uri.host, @@ -253,8 +264,15 @@ class TunnelingAgent(Agent): ) def _requestWithEndpoint( - self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath - ): + self, + key: Any, + endpoint: TCP4ClientEndpoint, + method: bytes, + parsedURI: bytes, + headers: Optional[TxHeaders], + bodyProducer: Optional[IBodyProducer], + requestPath: bytes, + ) -> Deferred: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -272,7 +290,12 @@ class TunnelingAgent(Agent): class ScrapyProxyAgent(Agent): def __init__( - self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None + self, + reactor: ReactorBase, + proxyURI: bytes, + connectTimeout: Optional[float] = None, + bindAddress: Optional[bytes] = None, + pool: Optional[HTTPConnectionPool] = None, ): super().__init__( reactor=reactor, @@ -280,9 +303,15 @@ class ScrapyProxyAgent(Agent): bindAddress=bindAddress, pool=pool, ) - self._proxyURI = URI.fromBytes(proxyURI) + self._proxyURI: URI = URI.fromBytes(proxyURI) - def request(self, method, uri, headers=None, bodyProducer=None): + def request( + self, + method: bytes, + uri: bytes, + headers: Optional[TxHeaders] = None, + bodyProducer: Optional[IBodyProducer] = None, + ) -> Deferred: """ Issue a new request via the configured proxy. """ @@ -306,26 +335,29 @@ class ScrapyAgent: def __init__( self, - contextFactory=None, - connectTimeout=10, - bindAddress=None, - pool=None, - maxsize=0, - warnsize=0, - fail_on_dataloss=True, - crawler=None, + contextFactory: Optional[IPolicyForHTTPS] = None, + connectTimeout: float = 10, + bindAddress: Optional[bytes] = None, + pool: Optional[HTTPConnectionPool] = None, + maxsize: int = 0, + warnsize: int = 0, + fail_on_dataloss: bool = True, + crawler: Optional[Crawler] = None, ): - self._contextFactory = contextFactory - self._connectTimeout = connectTimeout - self._bindAddress = bindAddress - self._pool = pool - self._maxsize = maxsize - self._warnsize = warnsize - self._fail_on_dataloss = fail_on_dataloss - self._txresponse = None - self._crawler = crawler + # TODO make these args required instead + assert contextFactory is not None + assert crawler is not None + self._contextFactory: IPolicyForHTTPS = contextFactory + self._connectTimeout: float = connectTimeout + self._bindAddress: Optional[bytes] = bindAddress + self._pool: Optional[HTTPConnectionPool] = pool + self._maxsize: int = maxsize + self._warnsize: int = warnsize + self._fail_on_dataloss: bool = fail_on_dataloss + self._txresponse: Optional[TxResponse] = None + self._crawler: Crawler = crawler - def _get_agent(self, request, timeout): + def _get_agent(self, request: Request, timeout: float) -> Agent: from twisted.internet import reactor bindaddress = request.meta.get("bindaddress") or self._bindAddress @@ -333,10 +365,10 @@ class ScrapyAgent: if proxy: proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] - proxyHost = to_unicode(proxyHost) + proxyHost_str = to_unicode(proxyHost) if scheme == b"https": proxyAuth = request.headers.get(b"Proxy-Authorization", None) - proxyConf = (proxyHost, proxyPort, proxyAuth) + proxyConf = (proxyHost_str, proxyPort, proxyAuth) return self._TunnelingAgent( reactor=reactor, proxyConf=proxyConf, @@ -346,7 +378,9 @@ class ScrapyAgent: pool=self._pool, ) proxyScheme = proxyScheme or b"http" - proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, "", "", "")) + proxyURI = urlunparse( + (proxyScheme, proxyNetloc, proxyParams, b"", b"", b"") + ) return self._ProxyAgent( reactor=reactor, proxyURI=to_bytes(proxyURI, encoding="ascii"), @@ -363,7 +397,7 @@ class ScrapyAgent: pool=self._pool, ) - def download_request(self, request): + def download_request(self, request: Request) -> Deferred: from twisted.internet import reactor timeout = request.meta.get("download_timeout") or self._connectTimeout @@ -380,7 +414,7 @@ class ScrapyAgent: else: bodyproducer = None start_time = time() - d = agent.request( + d: Deferred = agent.request( method, to_bytes(url, encoding="ascii"), headers, bodyproducer ) # set download latency @@ -393,7 +427,9 @@ class ScrapyAgent: d.addBoth(self._cb_timeout, request, url, timeout) return d - def _cb_timeout(self, result, request, url, timeout): + def _cb_timeout( + self, result: Any, request: Request, url: str, timeout: float + ) -> Any: if self._timeout_cl.active(): self._timeout_cl.cancel() return result @@ -404,19 +440,21 @@ class ScrapyAgent: raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") - def _cb_latency(self, result, request, start_time): + def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any: request.meta["download_latency"] = time() - start_time return result @staticmethod - def _headers_from_twisted_response(response): + def _headers_from_twisted_response(response: TxResponse) -> Headers: headers = Headers() if response.length != UNKNOWN_LENGTH: headers[b"Content-Length"] = str(response.length).encode() headers.update(response.headers.getAllRawHeaders()) return headers - def _cb_bodyready(self, txresponse, request): + def _cb_bodyready( + self, txresponse: TxResponse, request: Request + ) -> Union[Dict[str, Any], Deferred]: headers_received_result = self._crawler.signals.send_catch_log( signal=signals.headers_received, headers=self._headers_from_twisted_response(txresponse), @@ -472,7 +510,7 @@ class ScrapyAgent: logger.warning(warning_msg, warning_args) txresponse._transport.loseConnection() - raise defer.CancelledError(warning_msg % warning_args) + raise CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: logger.warning( @@ -481,11 +519,11 @@ class ScrapyAgent: {"size": expected_size, "warnsize": warnsize, "request": request}, ) - def _cancel(_): + def _cancel(_: Any) -> None: # Abort connection immediately. txresponse._transport._producer.abortConnection() - d = defer.Deferred(_cancel) + d: Deferred = Deferred(_cancel) txresponse.deliverBody( _ResponseReader( finished=d, @@ -503,7 +541,9 @@ class ScrapyAgent: return d - def _cb_bodydone(self, result, request, url): + def _cb_bodydone( + self, result: Dict[str, Any], request: Request, url: str + ) -> Union[Response, Failure]: headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) try: @@ -523,53 +563,57 @@ class ScrapyAgent: ) if result.get("failure"): result["failure"].value.response = response - return result["failure"] + return cast(Failure, result["failure"]) return response @implementer(IBodyProducer) class _RequestBodyProducer: - def __init__(self, body): + def __init__(self, body: bytes): self.body = body self.length = len(body) - def startProducing(self, consumer): + def startProducing(self, consumer: IConsumer) -> Deferred: consumer.write(self.body) - return defer.succeed(None) + return succeed(None) - def pauseProducing(self): + def pauseProducing(self) -> None: pass - def stopProducing(self): + def stopProducing(self) -> None: pass -class _ResponseReader(protocol.Protocol): +class _ResponseReader(Protocol): def __init__( self, - finished, - txresponse, - request, - maxsize, - warnsize, - fail_on_dataloss, - crawler, + finished: Deferred, + txresponse: TxResponse, + request: Request, + maxsize: int, + warnsize: int, + fail_on_dataloss: bool, + crawler: Crawler, ): - self._finished = finished - self._txresponse = txresponse - self._request = request - self._bodybuf = BytesIO() - self._maxsize = maxsize - self._warnsize = warnsize - self._fail_on_dataloss = fail_on_dataloss - self._fail_on_dataloss_warned = False - self._reached_warnsize = False - self._bytes_received = 0 - self._certificate = None - self._ip_address = None - self._crawler = crawler + self._finished: Deferred = finished + self._txresponse: TxResponse = txresponse + self._request: Request = request + self._bodybuf: BytesIO = BytesIO() + self._maxsize: int = maxsize + self._warnsize: int = warnsize + self._fail_on_dataloss: bool = fail_on_dataloss + self._fail_on_dataloss_warned: bool = False + self._reached_warnsize: bool = False + self._bytes_received: int = 0 + self._certificate: Optional[ssl.Certificate] = None + self._ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] = ( + None + ) + self._crawler: Crawler = crawler - def _finish_response(self, flags=None, failure=None): + def _finish_response( + self, flags: Optional[List[str]] = None, failure: Optional[Failure] = None + ) -> None: self._finished.callback( { "txresponse": self._txresponse, @@ -581,7 +625,8 @@ class _ResponseReader(protocol.Protocol): } ) - def connectionMade(self): + def connectionMade(self) -> None: + assert self.transport if self._certificate is None: with suppress(AttributeError): self._certificate = ssl.Certificate( @@ -593,11 +638,12 @@ class _ResponseReader(protocol.Protocol): self.transport._producer.getPeer().host ) - def dataReceived(self, bodyBytes): + def dataReceived(self, bodyBytes: bytes) -> None: # This maybe called several times after cancel was called with buffered data. if self._finished.called: return + assert self.transport self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) @@ -644,7 +690,7 @@ class _ResponseReader(protocol.Protocol): {"warnsize": self._warnsize, "request": self._request}, ) - def connectionLost(self, reason): + def connectionLost(self, reason: Failure = connectionDone) -> None: if self._finished.called: return diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index c88dd2cdc..9a0811a50 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,11 +2,12 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional, Type +from twisted.internet.defer import Deferred + from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached @@ -74,7 +75,7 @@ class S3DownloadHandler: def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self: return cls(crawler.settings, crawler=crawler, **kwargs) - def download_request(self, request: Request, spider: Spider) -> Response: + def download_request(self, request: Request, spider: Spider) -> Deferred: p = urlparse_cached(request) scheme = "https" if request.meta.get("is_secure") else "http" bucket = p.hostname From a828da98c3834ae70a1258278890485803ac7a5c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 6 May 2024 22:34:36 +0500 Subject: [PATCH 1427/2083] Re-run pre-commit. --- scrapy/core/downloader/handlers/http10.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 9117cb818..da9559525 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -20,6 +20,7 @@ if TYPE_CHECKING: from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory + class HTTP10DownloadHandler: lazy = False From 6bbfb537f9f1ba5dd1c51fc860022adb1d326117 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 8 May 2024 00:39:05 +0500 Subject: [PATCH 1428/2083] Update MANIFEST.in. --- MANIFEST.in | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 4920dc0c3..06971e39c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,8 @@ -include README.rst -include AUTHORS -include INSTALL -include LICENSE -include MANIFEST.in +include CODE_OF_CONDUCT.md +include CONTRIBUTING.md +include INSTALL.md include NEWS +include SECURITY.md include scrapy/VERSION include scrapy/mime.types @@ -12,16 +11,13 @@ include scrapy/py.typed include codecov.yml include conftest.py include pytest.ini -include requirements-*.txt include tox.ini recursive-include scrapy/templates * -recursive-include scrapy license.txt recursive-include docs * prune docs/build recursive-include extras * -recursive-include bin * recursive-include tests * global-exclude __pycache__ *.py[cod] From 180bc9bad7aceb3a9e10c1411212914bc32fb721 Mon Sep 17 00:00:00 2001 From: aisha-partha <153170327+aisha-partha@users.noreply.github.com> Date: Wed, 8 May 2024 22:06:46 +0530 Subject: [PATCH 1429/2083] =?UTF-8?q?Closes=20#6342.=20Setting=20METAREFRE?= =?UTF-8?q?SH=5FIGNORE=5FTAGS=20to=20[=E2=80=98noscript=E2=80=99]=20by=20d?= =?UTF-8?q?efault?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/downloader-middleware.rst | 4 ++-- scrapy/settings/default_settings.py | 2 +- tests/test_downloadermiddleware_redirect.py | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1abbc4968..3f90cf2ed 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -876,13 +876,13 @@ Whether the Meta Refresh middleware will be enabled. METAREFRESH_IGNORE_TAGS ^^^^^^^^^^^^^^^^^^^^^^^ -Default: ``[]`` +Default: ``['noscript']`` Meta tags within these tags are ignored. .. versionchanged:: 2.0 The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from - ``['script', 'noscript']`` to ``[]``. + ``[]`` to ``['noscript']``. .. setting:: METAREFRESH_MAXDELAY diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 2b3d95a0e..d7ac7ec35 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -239,7 +239,7 @@ MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_WARNING_MB = 0 METAREFRESH_ENABLED = True -METAREFRESH_IGNORE_TAGS = [] +METAREFRESH_IGNORE_TAGS = ["noscript"] METAREFRESH_MAXDELAY = 100 NEWSPIDER_MODULE = "" diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 10b8ca9af..83ff25982 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -395,9 +395,8 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): """content="0;URL='http://example.org/newpage'">""" ) rsp = HtmlResponse(req.url, body=body.encode()) - req2 = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req2, Request) - self.assertEqual(req2.url, "http://example.org/newpage") + response = self.mw.process_response(req, rsp, self.spider) + assert isinstance(response, Response) def test_ignore_tags_1_x_list(self): """Test that Scrapy 1.x behavior remains possible""" From 3590a1f66b30edfc836b95bbd0a3611eee9371e2 Mon Sep 17 00:00:00 2001 From: aisha-partha <153170327+aisha-partha@users.noreply.github.com> Date: Wed, 8 May 2024 23:23:17 +0530 Subject: [PATCH 1430/2083] Closes #6342. Update documentation on METAREFRESH_IGNORE_TAGS default value change --- docs/topics/downloader-middleware.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3f90cf2ed..2663a3cf3 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -881,6 +881,10 @@ Default: ``['noscript']`` Meta tags within these tags are ignored. .. versionchanged:: 2.0 + The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from + ``['script', 'noscript']`` to ``[]``. + +.. versionchanges:: VERSION The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from ``[]`` to ``['noscript']``. From 2e13a9b8e19ae1c0f595435d7d3ce096c3b29ced Mon Sep 17 00:00:00 2001 From: aisha-partha <153170327+aisha-partha@users.noreply.github.com> Date: Wed, 8 May 2024 23:33:13 +0530 Subject: [PATCH 1431/2083] Update METAREFRESH_IGNORE_TAGS default value in documentation --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 2663a3cf3..01bde772c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -876,7 +876,7 @@ Whether the Meta Refresh middleware will be enabled. METAREFRESH_IGNORE_TAGS ^^^^^^^^^^^^^^^^^^^^^^^ -Default: ``['noscript']`` +Default: ``[]`` Meta tags within these tags are ignored. From fe163d98ea81b4aff82bb9b194c4b0268c44f1db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 8 May 2024 20:41:20 +0200 Subject: [PATCH 1432/2083] Fix typo --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 01bde772c..d4cd062fe 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -884,7 +884,7 @@ Meta tags within these tags are ignored. The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from ``['script', 'noscript']`` to ``[]``. -.. versionchanges:: VERSION +.. versionchanged:: VERSION The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from ``[]`` to ``['noscript']``. From ae7bb849f50af0b91eea4f022d93ad201e545c06 Mon Sep 17 00:00:00 2001 From: Sanchay Kumar <51812506+kumar-sanchay@users.noreply.github.com> Date: Fri, 10 May 2024 15:13:49 +0530 Subject: [PATCH 1433/2083] Make certain args of ScrapyAgent and TunnelingAgent required (#6349) --- scrapy/core/downloader/handlers/http11.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 40ae1921a..5e84be6ba 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -239,15 +239,14 @@ class TunnelingAgent(Agent): def __init__( self, + *, reactor: ReactorBase, proxyConf: Tuple[str, int, Optional[bytes]], - contextFactory: Optional[IPolicyForHTTPS] = None, + contextFactory: IPolicyForHTTPS, connectTimeout: Optional[float] = None, bindAddress: Optional[bytes] = None, pool: Optional[HTTPConnectionPool] = None, ): - # TODO make this arg required instead - assert contextFactory is not None super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf self._contextFactory: IPolicyForHTTPS = contextFactory @@ -335,18 +334,16 @@ class ScrapyAgent: def __init__( self, - contextFactory: Optional[IPolicyForHTTPS] = None, + *, + contextFactory: IPolicyForHTTPS, connectTimeout: float = 10, bindAddress: Optional[bytes] = None, pool: Optional[HTTPConnectionPool] = None, maxsize: int = 0, warnsize: int = 0, fail_on_dataloss: bool = True, - crawler: Optional[Crawler] = None, + crawler: Crawler, ): - # TODO make these args required instead - assert contextFactory is not None - assert crawler is not None self._contextFactory: IPolicyForHTTPS = contextFactory self._connectTimeout: float = connectTimeout self._bindAddress: Optional[bytes] = bindAddress From c9ef5209365bb820ba8f2a3cd9df9fdeca0c9591 Mon Sep 17 00:00:00 2001 From: Sanchay Kumar <51812506+kumar-sanchay@users.noreply.github.com> Date: Fri, 10 May 2024 17:56:45 +0530 Subject: [PATCH 1434/2083] Add Downloader.get_slot_key() without a spider parameter (#6352) --- docs/news.rst | 14 ++++++++++++++ scrapy/core/downloader/__init__.py | 14 ++++++++++++-- scrapy/pqueues.py | 2 +- tests/test_scheduler.py | 6 +++--- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index fafea0bf8..7db4e59a1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,20 @@ Release notes ============= + +.. _release-VERSION: + +Scrapy VERSION (YYYY-MM-DD) +--------------------------- + +Deprecations +~~~~~~~~~~~~ + +- :func:`scrapy.core.downloader.Downloader._get_slot_key` is now deprecated. + Consider using its corresponding public method get_slot_key() instead. + (:issue:`6340`) + + .. _release-2.11.1: Scrapy 2.11.1 (2024-02-14) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 98e1af6fb..0ab3bdb77 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,4 +1,5 @@ import random +import warnings from collections import deque from datetime import datetime from time import time @@ -10,6 +11,7 @@ from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response from scrapy.resolver import dnscache from scrapy.settings import BaseSettings @@ -125,7 +127,7 @@ class Downloader: return len(self.active) >= self.total_concurrency def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]: - key = self._get_slot_key(request, spider) + key = self.get_slot_key(request) if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) conc = ( @@ -143,7 +145,7 @@ class Downloader: return key, self.slots[key] - def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str: + def get_slot_key(self, request: Request) -> str: if self.DOWNLOAD_SLOT in request.meta: return cast(str, request.meta[self.DOWNLOAD_SLOT]) @@ -153,6 +155,14 @@ class Downloader: return key + def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str: + warnings.warn( + "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return self.get_slot_key(request) + def _enqueue_request(self, request: Request, spider: Spider) -> Deferred: key, slot = self._get_slot(request, spider) request.meta[self.DOWNLOAD_SLOT] = key diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 773825c5e..58a47ef0f 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -180,7 +180,7 @@ class DownloaderInterface: return [(self._active_downloads(slot), slot) for slot in possible_slots] def get_slot_key(self, request: Request) -> str: - return self.downloader._get_slot_key(request, None) + return self.downloader.get_slot_key(request) def _active_downloads(self, slot: str) -> int: """Return a number of requests in a Downloader for a given slot""" diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 37099dae6..02b50baa3 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -25,7 +25,7 @@ class MockDownloader: def __init__(self): self.slots = {} - def _get_slot_key(self, request, spider): + def get_slot_key(self, request): if Downloader.DOWNLOAD_SLOT in request.meta: return request.meta[Downloader.DOWNLOAD_SLOT] @@ -273,14 +273,14 @@ class DownloaderAwareSchedulerTestMixin: while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() # pylint: disable=protected-access - slot = downloader._get_slot_key(request, None) + slot = downloader.get_slot_key(request) dequeued_slots.append(slot) downloader.increment(slot) requests.append(request) for request in requests: # pylint: disable=protected-access - slot = downloader._get_slot_key(request, None) + slot = downloader.get_slot_key(request) downloader.decrement(slot) self.assertTrue( From 93f06285309bd46e96fd147bf41e564c94b5bf2b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 13 May 2024 13:55:45 +0400 Subject: [PATCH 1435/2083] Improve typing for Spider.parse(). (#6274) --- .github/workflows/checks.yml | 3 ++ scrapy/commands/bench.py | 2 +- scrapy/spiders/__init__.py | 19 +++++-- tests_typing/test_spiders.mypy-testing | 68 ++++++++++++++++++++++++++ tox.ini | 9 ++++ 5 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 tests_typing/test_spiders.mypy-testing diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d6fc0f6c5..ed1629b67 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -18,6 +18,9 @@ jobs: - python-version: 3.8 env: TOXENV: typing + - python-version: 3.8 + env: + TOXENV: typing-tests - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 7523f3cfe..0c4ebcd23 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -60,7 +60,7 @@ class _BenchSpider(scrapy.Spider): url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" return [scrapy.Request(url, dont_filter=True)] - def parse(self, response: Response) -> Any: # type: ignore[override] + def parse(self, response: Response) -> Any: assert isinstance(Response, TextResponse) for link in self.link_extractor.extract_links(response): yield scrapy.Request(link.url, callback=self.parse) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 72c2aaba7..2416d2a4d 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -17,12 +17,17 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: + from collections.abc import Callable + + # typing.Concatenate requires Python 3.10 # typing.Self requires Python 3.11 - from typing_extensions import Self + from typing_extensions import Concatenate, Self from scrapy.crawler import Crawler from scrapy.settings import BaseSettings + CallbackT = Callable[Concatenate[Response, ...], Any] + class Spider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this @@ -79,10 +84,14 @@ class Spider(object_ref): def _parse(self, response: Response, **kwargs: Any) -> Any: return self.parse(response, **kwargs) - def parse(self, response: Response, **kwargs: Any) -> Any: - raise NotImplementedError( - f"{self.__class__.__name__}.parse callback is not defined" - ) + if TYPE_CHECKING: + parse: CallbackT + else: + + def parse(self, response: Response, **kwargs: Any) -> Any: + raise NotImplementedError( + f"{self.__class__.__name__}.parse callback is not defined" + ) @classmethod def update_settings(cls, settings: BaseSettings) -> None: diff --git a/tests_typing/test_spiders.mypy-testing b/tests_typing/test_spiders.mypy-testing new file mode 100644 index 000000000..162e31d0c --- /dev/null +++ b/tests_typing/test_spiders.mypy-testing @@ -0,0 +1,68 @@ +from typing import Any + +import pytest + +from scrapy.http import HtmlResponse, Response +from scrapy.spiders import Spider + + +class SimpleSpider(Spider): + pass + + +class SameOverrideSpider(Spider): + def parse(self, response: Response, **kwargs: Any) -> Any: + pass + + +class NoKwargsSpider(Spider): + def parse(self, response: Response) -> Any: + pass + + +class SpecificKwargsSpider(Spider): + def parse(self, response: Response, page: int) -> Any: + pass + + +class NarrowOverrideSpider(Spider): + # without type: ignore this produces several note lines in addition to an error line, + # which is unsupported by pytest-mypy-testing + def parse(self, response: HtmlResponse, **kwargs: Any) -> Any: # type: ignore[override] + pass + + +@pytest.mark.mypy_testing +def test_spider_parse() -> None: + spider = Spider() + reveal_type(spider.parse) # R: def (scrapy.http.response.Response, *Any, **Any) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_simple() -> None: + spider = SimpleSpider() + reveal_type(spider.parse) # R: def (scrapy.http.response.Response, *Any, **Any) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_same() -> None: + spider = SameOverrideSpider() + reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, **kwargs: Any) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_no_kwargs() -> None: + spider = NoKwargsSpider() + reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_specific_kwargs() -> None: + spider = SpecificKwargsSpider() + reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, page: builtins.int) -> Any + + +@pytest.mark.mypy_testing +def test_spider_parse_override_narrow() -> None: + spider = NarrowOverrideSpider() + reveal_type(spider.parse) # R: def (response: scrapy.http.response.html.HtmlResponse, **kwargs: Any) -> Any diff --git a/tox.ini b/tox.ini index d7527bb04..ede139756 100644 --- a/tox.ini +++ b/tox.ini @@ -56,6 +56,15 @@ deps = commands = mypy {posargs: scrapy tests} +[testenv:typing-tests] +basepython = python3.8 +deps = + {[test-requirements]deps} + {[testenv:typing]deps} + pytest-mypy-testing==0.1.3 +commands = + pytest {posargs: tests_typing} + [testenv:pre-commit] basepython = python3 deps = From 4ed5c5ae91318768efa338680df67337ed0f67fd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 13 May 2024 14:01:52 +0400 Subject: [PATCH 1436/2083] Use ParamSpec for callables. (#6353) --- scrapy/cmdline.py | 15 ++- scrapy/core/downloader/handlers/ftp.py | 10 +- scrapy/core/engine.py | 2 +- scrapy/core/scraper.py | 10 +- scrapy/core/spidermw.py | 6 +- scrapy/mail.py | 8 +- scrapy/pipelines/files.py | 3 +- scrapy/pipelines/media.py | 27 ++--- scrapy/shell.py | 4 +- scrapy/utils/decorators.py | 29 ++++-- scrapy/utils/defer.py | 135 +++++++++++++++++-------- scrapy/utils/python.py | 23 ++++- scrapy/utils/reactor.py | 33 ++++-- 13 files changed, 204 insertions(+), 101 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 4df5698a6..da0e51386 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import argparse import cProfile import inspect import os import sys from importlib.metadata import entry_points -from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type +from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple, Type import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -15,6 +17,12 @@ from scrapy.utils.misc import walk_modules from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect +if TYPE_CHECKING: + # typing.ParamSpec requires Python 3.10 + from typing_extensions import ParamSpec + + _P = ParamSpec("_P") + class ScrapyArgumentParser(argparse.ArgumentParser): def _parse_optional( @@ -121,7 +129,10 @@ def _print_unknown_command( def _run_print_help( - parser: argparse.ArgumentParser, func: Callable, *a: Any, **kw: Any + parser: argparse.ArgumentParser, + func: Callable[_P, None], + *a: _P.args, + **kw: _P.kwargs, ) -> None: try: func(*a, **kw) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index ed94b2221..77dcf3c38 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -109,12 +109,10 @@ class FTPDownloadHandler: def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred: self.client = client protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) - return client.retrieveFile(filepath, protocol).addCallbacks( - callback=self._build_response, - callbackArgs=(request, protocol), - errback=self._failed, - errbackArgs=(request,), - ) + d = client.retrieveFile(filepath, protocol) + d.addCallback(self._build_response, request, protocol) + d.addErrback(self._failed, request) + return d def _build_response( self, result: Any, request: Request, protocol: ReceivedDataProtocol diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 93a0c51bc..6bf3f3e26 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -347,7 +347,7 @@ class ExecutionEngine: assert self.spider is not None dwld = self.downloader.fetch(request, self.spider) - dwld.addCallbacks(_on_success) + dwld.addCallback(_on_success) dwld.addBoth(_on_complete) return dwld diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 272841e01..566e6628b 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -8,7 +8,6 @@ from collections import deque from typing import ( TYPE_CHECKING, Any, - AsyncGenerator, AsyncIterable, Deque, Generator, @@ -18,6 +17,7 @@ from typing import ( Tuple, Type, Union, + cast, ) from itemadapter import is_item @@ -184,7 +184,9 @@ class Scraper: result, request, spider ) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) - dfd.addCallback(self.handle_spider_output, request, result, spider) + dfd.addCallback( + self.handle_spider_output, request, cast(Response, result), spider + ) return dfd def _scrape2( @@ -256,12 +258,12 @@ class Scraper: self, result: Union[Iterable, AsyncIterable], request: Request, - response: Union[Response, Failure], + response: Response, spider: Spider, ) -> Deferred: if not result: return defer_succeed(None) - it: Union[Generator, AsyncGenerator] + it: Union[Iterable, AsyncIterable] if isinstance(result, AsyncIterable): it = aiter_errback( result, self.handle_spider_error, request, response, spider diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1ccfd08a2..2cef2e1dd 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -303,10 +303,8 @@ class SpiderMiddlewareManager(MiddlewareManager): dfd = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) - dfd.addCallbacks( - callback=deferred_f_from_coro_f(process_callback_output), - errback=process_spider_exception, - ) + dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) + dfd.addErrback(process_spider_exception) return dfd def process_start_requests( diff --git a/scrapy/mail.py b/scrapy/mail.py index 56adba934..fd6302550 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -154,12 +154,8 @@ class MailSender: return None dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8")) - dfd.addCallbacks( - callback=self._sent_ok, - errback=self._sent_failed, - callbackArgs=(to, cc, subject, len(attachs)), - errbackArgs=(to, cc, subject, len(attachs)), - ) + dfd.addCallback(self._sent_ok, to, cc, subject, len(attachs)) + dfd.addErrback(self._sent_failed, to, cc, subject, len(attachs)) reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd) return dfd diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index d00f44502..47457f2a8 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -459,7 +459,8 @@ class FilesPipeline(MediaPipeline): path = self.file_path(request, info=info, item=item) dfd = defer.maybeDeferred(self.store.stat_file, path, info) - dfd.addCallbacks(_onsuccess, lambda _: None) + dfd.addCallback(_onsuccess) + dfd.addErrback(lambda _: None) dfd.addErrback( lambda f: logger.error( self.__class__.__name__ + ".store.stat_file", diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index fd5e70cb9..5f6c5cb07 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -106,10 +106,17 @@ class MediaPipeline: # Return cached result if request was already seen if fp in info.downloaded: - return defer_result(info.downloaded[fp]).addCallbacks(cb, eb) + d = defer_result(info.downloaded[fp]) + d.addCallback(cb) + if eb: + d.addErrback(eb) + return d # Otherwise, wait for result - wad = Deferred().addCallbacks(cb, eb) + wad = Deferred() + wad.addCallback(cb) + if eb: + wad.addErrback(eb) info.waiting[fp].append(wad) # Check if request is downloading right now to avoid doing it twice @@ -140,23 +147,11 @@ class MediaPipeline: if self.download_func: # this ugly code was left only to support tests. TODO: remove dfd = mustbe_deferred(self.download_func, request, info.spider) - dfd.addCallbacks( - callback=self.media_downloaded, - callbackArgs=(request, info), - callbackKeywords={"item": item}, - errback=self.media_failed, - errbackArgs=(request, info), - ) else: self._modify_media_request(request) dfd = self.crawler.engine.download(request) - dfd.addCallbacks( - callback=self.media_downloaded, - callbackArgs=(request, info), - callbackKeywords={"item": item}, - errback=self.media_failed, - errbackArgs=(request, info), - ) + dfd.addCallback(self.media_downloaded, request, info, item=item) + dfd.addErrback(self.media_failed, request, info) return dfd def _cache_result_and_execute_waiters(self, result, fp, info): diff --git a/scrapy/shell.py b/scrapy/shell.py index fac42e8a2..2c22d3d8f 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -231,7 +231,9 @@ def _request_deferred(request: Request) -> defer.Deferred: d: defer.Deferred = defer.Deferred() d.addBoth(_restore_callbacks) if request.callback: - d.addCallbacks(request.callback, request.errback) + d.addCallback(request.callback) + if request.errback: + d.addErrback(request.errback) request.callback, request.errback = d.callback, d.errback return d diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 04186559f..7e82dd519 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -1,21 +1,34 @@ +from __future__ import annotations + import warnings from functools import wraps -from typing import Any, Callable +from typing import TYPE_CHECKING, Any, Callable, TypeVar from twisted.internet import defer, threads from twisted.internet.defer import Deferred from scrapy.exceptions import ScrapyDeprecationWarning +if TYPE_CHECKING: + # typing.ParamSpec requires Python 3.10 + from typing_extensions import ParamSpec -def deprecated(use_instead: Any = None) -> Callable: + _P = ParamSpec("_P") + + +_T = TypeVar("_T") + + +def deprecated( + use_instead: Any = None, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" - def deco(func: Callable) -> Callable: + def deco(func: Callable[_P, _T]) -> Callable[_P, _T]: @wraps(func) - def wrapped(*args: Any, **kwargs: Any) -> Any: + def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> Any: message = f"Call to deprecated function {func.__name__}." if use_instead: message += f" Use {use_instead} instead." @@ -30,23 +43,23 @@ def deprecated(use_instead: Any = None) -> Callable: return deco -def defers(func: Callable) -> Callable[..., Deferred]: +def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: """Decorator to make sure a function always returns a deferred""" @wraps(func) - def wrapped(*a: Any, **kw: Any) -> Deferred: + def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: return defer.maybeDeferred(func, *a, **kw) return wrapped -def inthread(func: Callable) -> Callable[..., Deferred]: +def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: """Decorator to call a function in a thread and return a deferred with the result """ @wraps(func) - def wrapped(*a: Any, **kw: Any) -> Deferred: + def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: return threads.deferToThread(func, *a, **kw) return wrapped diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index c391db9fd..abb7e1726 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -2,21 +2,22 @@ Helper functions for dealing with Twisted deferreds """ +from __future__ import annotations + import asyncio import inspect from asyncio import Future from functools import wraps from types import CoroutineType from typing import ( + TYPE_CHECKING, Any, - AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Callable, Coroutine, Dict, - Generator, Iterable, Iterator, List, @@ -37,6 +38,14 @@ from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed +if TYPE_CHECKING: + # typing.Concatenate and typing.ParamSpec require Python 3.10 + from typing_extensions import Concatenate, ParamSpec + + _P = ParamSpec("_P") + +_T = TypeVar("_T") + def defer_fail(_failure: Failure) -> Deferred: """Same as twisted.internet.defer.fail but delay calling errback until @@ -74,7 +83,31 @@ def defer_result(result: Any) -> Deferred: return defer_succeed(result) -def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred: +@overload +def mustbe_deferred( + f: Callable[_P, Deferred[_T]], *args: _P.args, **kw: _P.kwargs +) -> Deferred[_T]: ... + + +@overload +def mustbe_deferred( + f: Callable[_P, Coroutine[Deferred[Any], Any, _T]], + *args: _P.args, + **kw: _P.kwargs, +) -> Deferred[_T]: ... + + +@overload +def mustbe_deferred( + f: Callable[_P, _T], *args: _P.args, **kw: _P.kwargs +) -> Deferred[_T]: ... + + +def mustbe_deferred( + f: Callable[_P, Union[Deferred[_T], Coroutine[Deferred[Any], Any, _T], _T]], + *args: _P.args, + **kw: _P.kwargs, +) -> Deferred[_T]: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -92,7 +125,11 @@ def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred: def parallel( - iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any + iterable: Iterable[_T], + count: int, + callable: Callable[Concatenate[_T, _P], Any], + *args: _P.args, + **named: _P.kwargs, ) -> Deferred: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -104,7 +141,7 @@ def parallel( return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter(Iterator): +class _AsyncCooperatorAdapter(Iterator[Deferred]): """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more @@ -152,28 +189,30 @@ class _AsyncCooperatorAdapter(Iterator): def __init__( self, - aiterable: AsyncIterable, - callable: Callable, - *callable_args: Any, - **callable_kwargs: Any, + aiterable: AsyncIterable[_T], + callable: Callable[Concatenate[_T, _P], Any], + *callable_args: _P.args, + **callable_kwargs: _P.kwargs, ): - self.aiterator: AsyncIterator = aiterable.__aiter__() - self.callable: Callable = callable + self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() + self.callable: Callable[Concatenate[_T, _P], Any] = callable self.callable_args: Tuple[Any, ...] = callable_args self.callable_kwargs: Dict[str, Any] = callable_kwargs self.finished: bool = False self.waiting_deferreds: List[Deferred] = [] - self.anext_deferred: Optional[Deferred] = None + self.anext_deferred: Optional[Deferred[_T]] = None - def _callback(self, result: Any) -> None: + def _callback(self, result: _T) -> None: # This gets called when the result from aiterator.__anext__() is available. # It calls the callable on it and sends the result to the oldest waiting Deferred # (by chaining if the result is a Deferred too or by firing if not). self.anext_deferred = None - result = self.callable(result, *self.callable_args, **self.callable_kwargs) + callable_result = self.callable( + result, *self.callable_args, **self.callable_kwargs + ) d = self.waiting_deferreds.pop(0) - if isinstance(result, Deferred): - result.chainDeferred(d) + if isinstance(callable_result, Deferred): + callable_result.chainDeferred(d) else: d.callback(None) if self.waiting_deferreds: @@ -207,11 +246,11 @@ class _AsyncCooperatorAdapter(Iterator): def parallel_async( - async_iterable: AsyncIterable, + async_iterable: AsyncIterable[_T], count: int, - callable: Callable, - *args: Any, - **named: Any, + callable: Callable[Concatenate[_T, _P], Any], + *args: _P.args, + **named: _P.kwargs, ) -> Deferred: """Like parallel but for async iterators""" coop = Cooperator() @@ -221,7 +260,10 @@ def parallel_async( def process_chain( - callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any + callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], + input: Any, + *a: _P.args, + **kw: _P.kwargs, ) -> Deferred: """Return a Deferred built by chaining the given callbacks""" d: Deferred = Deferred() @@ -232,23 +274,17 @@ def process_chain( def process_chain_both( - callbacks: Iterable[Callable], - errbacks: Iterable[Callable], + callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], + errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]], input: Any, - *a: Any, - **kw: Any, + *a: _P.args, + **kw: _P.kwargs, ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" d: Deferred = Deferred() for cb, eb in zip(callbacks, errbacks): - d.addCallbacks( - callback=cb, - errback=eb, - callbackArgs=a, - callbackKeywords=kw, - errbackArgs=a, - errbackKeywords=kw, - ) + d.addCallback(cb, *a, **kw) + d.addErrback(eb, *a, **kw) if isinstance(input, failure.Failure): d.errback(input) else: @@ -257,20 +293,27 @@ def process_chain_both( def process_parallel( - callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any + callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], + input: Any, + *a: _P.args, + **kw: _P.kwargs, ) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) - d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) + d.addCallback(lambda r: [x[1] for x in r]) + d.addErrback(lambda f: f.value.subFailure) return d def iter_errback( - iterable: Iterable, errback: Callable, *a: Any, **kw: Any -) -> Generator: + iterable: Iterable[_T], + errback: Callable[Concatenate[Failure, _P], Any], + *a: _P.args, + **kw: _P.kwargs, +) -> Iterable[_T]: """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -285,8 +328,11 @@ def iter_errback( async def aiter_errback( - aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any -) -> AsyncGenerator: + aiterable: AsyncIterable[_T], + errback: Callable[Concatenate[Failure, _P], Any], + *a: _P.args, + **kw: _P.kwargs, +) -> AsyncIterable[_T]: """Wraps an async iterable calling an errback if an error is caught while iterating it. Similar to scrapy.utils.defer.iter_errback() """ @@ -301,7 +347,6 @@ async def aiter_errback( _CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future]) -_T = TypeVar("_T") @overload @@ -327,7 +372,9 @@ def deferred_from_coro(o: _T) -> Union[Deferred, _T]: return o -def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: +def deferred_f_from_coro_f( + coro_f: Callable[_P, Coroutine[Any, Any, _T]] +) -> Callable[_P, Deferred[_T]]: """Converts a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. @@ -335,13 +382,15 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: """ @wraps(coro_f) - def f(*coro_args: Any, **coro_kwargs: Any) -> Any: + def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Any: return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) return f -def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred: +def maybeDeferred_coro( + f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs +) -> Deferred: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 1e7364e49..5d2d490b2 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -2,6 +2,8 @@ This module contains essential stuff that should've come with Python itself ;) """ +from __future__ import annotations + import collections.abc import gc import inspect @@ -11,6 +13,7 @@ import weakref from functools import partial, wraps from itertools import chain from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, AsyncIterable, @@ -25,12 +28,21 @@ from typing import ( Optional, Pattern, Tuple, + TypeVar, Union, overload, ) from scrapy.utils.asyncgen import as_async_generator +if TYPE_CHECKING: + # typing.Concatenate and typing.ParamSpec require Python 3.10 + from typing_extensions import Concatenate, ParamSpec + + _P = ParamSpec("_P") + +_T = TypeVar("_T") + def flatten(x: Iterable) -> list: """flatten(sequence) -> list @@ -169,14 +181,19 @@ def re_rsearch( return None -def memoizemethod_noargs(method: Callable) -> Callable: +_SelfT = TypeVar("_SelfT") + + +def memoizemethod_noargs( + method: Callable[Concatenate[_SelfT, _P], _T] +) -> Callable[Concatenate[_SelfT, _P], _T]: """Decorator to cache the result of a method (without arguments) using a weak reference to its object """ - cache: weakref.WeakKeyDictionary[Any, Any] = weakref.WeakKeyDictionary() + cache: weakref.WeakKeyDictionary[_SelfT, _T] = weakref.WeakKeyDictionary() @wraps(method) - def new_method(self: Any, *args: Any, **kwargs: Any) -> Any: + def new_method(self: _SelfT, *args: _P.args, **kwargs: _P.kwargs) -> _T: if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 6cde49bfe..5af6d22eb 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,8 +1,21 @@ +from __future__ import annotations + import asyncio import sys from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress -from typing import Any, Callable, Dict, List, Optional, Sequence, Type +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generic, + List, + Optional, + Tuple, + Type, + TypeVar, +) from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error @@ -13,6 +26,14 @@ from twisted.internet.tcp import Port from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object +if TYPE_CHECKING: + # typing.ParamSpec requires Python 3.10 + from typing_extensions import ParamSpec + + _P = ParamSpec("_P") + +_T = TypeVar("_T") + def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] """Like reactor.listenTCP but tries different ports in a range.""" @@ -32,14 +53,14 @@ def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port: raise -class CallLaterOnce: +class CallLaterOnce(Generic[_T]): """Schedule a function to be called in the next reactor loop, but only if it hasn't been already scheduled since the last time it ran. """ - def __init__(self, func: Callable, *a: Any, **kw: Any): - self._func: Callable = func - self._a: Sequence[Any] = a + def __init__(self, func: Callable[_P, _T], *a: _P.args, **kw: _P.kwargs): + self._func: Callable[_P, _T] = func + self._a: Tuple[Any, ...] = a self._kw: Dict[str, Any] = kw self._call: Optional[DelayedCall] = None @@ -53,7 +74,7 @@ class CallLaterOnce: if self._call: self._call.cancel() - def __call__(self) -> Any: + def __call__(self) -> _T: self._call = None return self._func(*self._a, **self._kw) From b8e333c8ce78c195e265fa453e7bc0b11c5ca4af Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 13 May 2024 14:07:51 +0400 Subject: [PATCH 1437/2083] Full typing for scrapy/spiders. (#6356) --- scrapy/spiders/__init__.py | 6 +- scrapy/spiders/crawl.py | 138 +++++++++++++++++++++++++------------ scrapy/spiders/feed.py | 48 ++++++++----- scrapy/spiders/init.py | 14 ++-- scrapy/spiders/sitemap.py | 49 ++++++++----- scrapy/utils/sitemap.py | 4 +- 6 files changed, 172 insertions(+), 87 deletions(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 2416d2a4d..370801f28 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -39,7 +39,7 @@ class Spider(object_ref): def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: - self.name = name + self.name: str = name elif not getattr(self, "name", None): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) @@ -67,8 +67,8 @@ class Spider(object_ref): return spider def _set_crawler(self, crawler: Crawler) -> None: - self.crawler = crawler - self.settings = crawler.settings + self.crawler: Crawler = crawler + self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) def start_requests(self) -> Iterable[Request]: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index ba8b7b366..48c830d2a 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,9 +8,27 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import copy -from typing import TYPE_CHECKING, AsyncIterable, Awaitable, Sequence +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterable, + Awaitable, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Set, + TypeVar, + Union, + cast, +) + +from twisted.python.failure import Failure from scrapy.http import HtmlResponse, Request, Response +from scrapy.link import Link from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen @@ -20,20 +38,32 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler -def _identity(x): + +_T = TypeVar("_T") +ProcessLinksT = Callable[[List[Link]], List[Link]] +ProcessRequestT = Callable[[Request, Response], Optional[Request]] + + +def _identity(x: _T) -> _T: return x -def _identity_process_request(request, response): +def _identity_process_request( + request: Request, response: Response +) -> Optional[Request]: return request -def _get_method(method, spider): +def _get_method( + method: Union[Callable, str, None], spider: Spider +) -> Optional[Callable]: if callable(method): return method if isinstance(method, str): return getattr(spider, method, None) + return None _default_link_extractor = LinkExtractor() @@ -42,37 +72,46 @@ _default_link_extractor = LinkExtractor() class Rule: def __init__( self, - link_extractor=None, - callback=None, - cb_kwargs=None, - follow=None, - process_links=None, - process_request=None, - errback=None, + link_extractor: Optional[LinkExtractor] = None, + callback: Union[Callable, str, None] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, + follow: Optional[bool] = None, + process_links: Union[ProcessLinksT, str, None] = None, + process_request: Union[ProcessRequestT, str, None] = None, + errback: Union[Callable[[Failure], Any], str, None] = None, ): - self.link_extractor = link_extractor or _default_link_extractor - self.callback = callback - self.errback = errback - self.cb_kwargs = cb_kwargs or {} - self.process_links = process_links or _identity - self.process_request = process_request or _identity_process_request - self.follow = follow if follow is not None else not callback + self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor + self.callback: Union[Callable, str, None] = callback + self.errback: Union[Callable[[Failure], Any], str, None] = errback + self.cb_kwargs: Dict[str, Any] = cb_kwargs or {} + self.process_links: Union[ProcessLinksT, str] = process_links or _identity + self.process_request: Union[ProcessRequestT, str] = ( + process_request or _identity_process_request + ) + self.follow: bool = follow if follow is not None else not callback - def _compile(self, spider): + def _compile(self, spider: Spider) -> None: + # this replaces method names with methods and we can't express this in type hints self.callback = _get_method(self.callback, spider) - self.errback = _get_method(self.errback, spider) - self.process_links = _get_method(self.process_links, spider) - self.process_request = _get_method(self.process_request, spider) + self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider)) + self.process_links = cast( + ProcessLinksT, _get_method(self.process_links, spider) + ) + self.process_request = cast( + ProcessRequestT, _get_method(self.process_request, spider) + ) class CrawlSpider(Spider): rules: Sequence[Rule] = () + _rules: List[Rule] + _follow_links: bool - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) self._compile_rules() - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: return self._parse_response( response=response, callback=self.parse_start_url, @@ -80,13 +119,13 @@ class CrawlSpider(Spider): follow=True, ) - def parse_start_url(self, response, **kwargs): + def parse_start_url(self, response: Response, **kwargs: Any) -> Any: return [] - def process_results(self, response: Response, results: list): + def process_results(self, response: Response, results: Any) -> Any: return results - def _build_request(self, rule_index, link): + def _build_request(self, rule_index: int, link: Link) -> Request: return Request( url=link.url, callback=self._callback, @@ -94,32 +133,43 @@ class CrawlSpider(Spider): meta={"rule": rule_index, "link_text": link.text}, ) - def _requests_to_follow(self, response): + def _requests_to_follow(self, response: Response) -> Iterable[Optional[Request]]: if not isinstance(response, HtmlResponse): return - seen = set() + seen: Set[Link] = set() for rule_index, rule in enumerate(self._rules): - links = [ + links: List[Link] = [ lnk for lnk in rule.link_extractor.extract_links(response) if lnk not in seen ] - for link in rule.process_links(links): + for link in cast(ProcessLinksT, rule.process_links)(links): seen.add(link) request = self._build_request(rule_index, link) - yield rule.process_request(request, response) + yield cast(ProcessRequestT, rule.process_request)(request, response) - def _callback(self, response, **cb_kwargs): - rule = self._rules[response.meta["rule"]] + def _callback(self, response: Response, **cb_kwargs: Any) -> Any: + rule = self._rules[cast(int, response.meta["rule"])] return self._parse_response( - response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow + response, + cast(Callable, rule.callback), + {**rule.cb_kwargs, **cb_kwargs}, + rule.follow, ) - def _errback(self, failure): - rule = self._rules[failure.request.meta["rule"]] - return self._handle_failure(failure, rule.errback) + def _errback(self, failure: Failure) -> Iterable[Any]: + rule = self._rules[cast(int, failure.request.meta["rule"])] # type: ignore[attr-defined] + return self._handle_failure( + failure, cast(Callable[[Failure], Any], rule.errback) + ) - async def _parse_response(self, response, callback, cb_kwargs, follow=True): + async def _parse_response( + self, + response: Response, + callback: Optional[Callable], + cb_kwargs: Dict[str, Any], + follow: bool = True, + ) -> AsyncIterable[Any]: if callback: cb_res = callback(response, **cb_kwargs) or () if isinstance(cb_res, AsyncIterable): @@ -134,21 +184,23 @@ class CrawlSpider(Spider): for request_or_item in self._requests_to_follow(response): yield request_or_item - def _handle_failure(self, failure, errback): + def _handle_failure( + self, failure: Failure, errback: Optional[Callable[[Failure], Any]] + ) -> Iterable[Any]: if errback: results = errback(failure) or () yield from iterate_spider_output(results) - def _compile_rules(self): + def _compile_rules(self) -> None: self._rules = [] for rule in self.rules: self._rules.append(copy.copy(rule)) self._rules[-1]._compile(self) @classmethod - def from_crawler(cls, crawler, *args, **kwargs) -> Self: + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) - spider._follow_links = crawler.settings.getbool( # type: ignore[attr-defined] + spider._follow_links = crawler.settings.getbool( "CRAWLSPIDER_FOLLOW_LINKS", True ) return spider diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 5caf8c79e..9dd8a5d68 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -5,7 +5,10 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + from scrapy.exceptions import NotConfigured, NotSupported +from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.spiders import Spider from scrapy.utils.iterators import csviter, xmliter_lxml @@ -22,11 +25,13 @@ class XMLFeedSpider(Spider): use iternodes, since it's a faster and cleaner. """ - iterator = "iternodes" - itertag = "item" - namespaces = () + iterator: str = "iternodes" + itertag: str = "item" + namespaces: Sequence[Tuple[str, str]] = () - def process_results(self, response, results): + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: """This overridable method is called for each result (item or request) returned by the spider, and it's intended to perform any last time processing required before returning the results to the framework core, @@ -36,20 +41,20 @@ class XMLFeedSpider(Spider): """ return results - def adapt_response(self, response): + def adapt_response(self, response: Response) -> Response: """You can override this function in order to make any changes you want to into the feed before parsing it. This function must return a response. """ return response - def parse_node(self, response, selector): + def parse_node(self, response: Response, selector: Selector) -> Any: """This method must be overridden with your custom spider functionality""" if hasattr(self, "parse_item"): # backward compatibility return self.parse_item(response, selector) raise NotImplementedError - def parse_nodes(self, response, nodes): + def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any: """This method is called for the nodes matching the provided tag name (itertag). Receives the response and an Selector for each node. Overriding this method is mandatory. Otherwise, you spider won't work. @@ -61,20 +66,25 @@ class XMLFeedSpider(Spider): ret = iterate_spider_output(self.parse_node(response, selector)) yield from self.process_results(response, ret) - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: if not hasattr(self, "parse_node"): raise NotConfigured( "You must define parse_node method in order to scrape this XML feed" ) response = self.adapt_response(response) + nodes: Iterable[Selector] if self.iterator == "iternodes": nodes = self._iternodes(response) elif self.iterator == "xml": + if not isinstance(response, TextResponse): + raise ValueError("Response content isn't text") selector = Selector(response, type="xml") self._register_namespaces(selector) nodes = selector.xpath(f"//{self.itertag}") elif self.iterator == "html": + if not isinstance(response, TextResponse): + raise ValueError("Response content isn't text") selector = Selector(response, type="html") self._register_namespaces(selector) nodes = selector.xpath(f"//{self.itertag}") @@ -83,12 +93,12 @@ class XMLFeedSpider(Spider): return self.parse_nodes(response, nodes) - def _iternodes(self, response): + def _iternodes(self, response: Response) -> Iterable[Selector]: for node in xmliter_lxml(response, self.itertag): self._register_namespaces(node) yield node - def _register_namespaces(self, selector): + def _register_namespaces(self, selector: Selector) -> None: for prefix, uri in self.namespaces: selector.register_namespace(prefix, uri) @@ -102,27 +112,29 @@ class CSVFeedSpider(Spider): and the file's headers. """ - delimiter = ( + delimiter: Optional[str] = ( None # When this is None, python's csv module's default delimiter is used ) - quotechar = ( + quotechar: Optional[str] = ( None # When this is None, python's csv module's default quotechar is used ) - headers = None + headers: Optional[List[str]] = None - def process_results(self, response, results): + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: """This method has the same purpose as the one in XMLFeedSpider""" return results - def adapt_response(self, response): + def adapt_response(self, response: Response) -> Response: """This method has the same purpose as the one in XMLFeedSpider""" return response - def parse_row(self, response, row): + def parse_row(self, response: Response, row: Dict[str, str]) -> Any: """This method must be overridden with your custom spider functionality""" raise NotImplementedError - def parse_rows(self, response): + def parse_rows(self, response: Response) -> Any: """Receives a response and a dict (representing each row) with a key for each provided (or detected) header of the CSV file. This spider also gives the opportunity to override adapt_response and @@ -135,7 +147,7 @@ class CSVFeedSpider(Spider): ret = iterate_spider_output(self.parse_row(response, row)) yield from self.process_results(response, ret) - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: if not hasattr(self, "parse_row"): raise NotConfigured( "You must define parse_row method in order to scrape this CSV feed" diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 3cb215b0f..a0898a0cf 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,3 +1,7 @@ +from typing import Any, Iterable, Optional, cast + +from scrapy import Request +from scrapy.http import Response from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output @@ -5,17 +9,17 @@ from scrapy.utils.spider import iterate_spider_output class InitSpider(Spider): """Base Spider with initialization facilities""" - def start_requests(self): - self._postinit_reqs = super().start_requests() - return iterate_spider_output(self.init_request()) + def start_requests(self) -> Iterable[Request]: + self._postinit_reqs: Iterable[Request] = super().start_requests() + return cast(Iterable[Request], iterate_spider_output(self.init_request())) - def initialized(self, response=None): + def initialized(self, response: Optional[Response] = None) -> Any: """This method must be set as the callback of your last initialization request. See self.init_request() docstring for more info. """ return self.__dict__.pop("_postinit_reqs") - def init_request(self): + def init_request(self) -> Any: """This function should return one initialization request, with the self.initialized method as callback. When the self.initialized method is called this spider is considered initialized. If you need to perform diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index f0e630c42..d082fbfdb 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -2,9 +2,21 @@ from __future__ import annotations import logging import re -from typing import TYPE_CHECKING, Any +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + Union, + cast, +) -from scrapy.http import Request, XmlResponse +from scrapy.http import Request, Response, XmlResponse from scrapy.spiders import Spider from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip, gzip_magic_number @@ -20,10 +32,12 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): - sitemap_urls = () - sitemap_rules = [("", "parse")] - sitemap_follow = [""] - sitemap_alternate_links = False + sitemap_urls: Sequence[str] = () + sitemap_rules: Sequence[ + Tuple[Union[re.Pattern[str], str], Union[str, Callable]] + ] = [("", "parse")] + sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""] + sitemap_alternate_links: bool = False _max_size: int _warn_size: int @@ -38,27 +52,29 @@ class SitemapSpider(Spider): ) return spider - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self._cbs = [] + self._cbs: List[Tuple[re.Pattern[str], Callable]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): - c = getattr(self, c) + c = cast(Callable, getattr(self, c)) self._cbs.append((regex(r), c)) - self._follow = [regex(x) for x in self.sitemap_follow] + self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] - def start_requests(self): + def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) - def sitemap_filter(self, entries): + def sitemap_filter( + self, entries: Iterable[Dict[str, Any]] + ) -> Iterable[Dict[str, Any]]: """This method can be used to filter sitemap entries by their attributes, for example, you can filter locs with lastmod greater than a given date (see docs). """ yield from entries - def _parse_sitemap(self, response): + def _parse_sitemap(self, response: Response) -> Iterable[Request]: if response.url.endswith("/robots.txt"): for url in sitemap_urls_from_robots(response.text, base_url=response.url): yield Request(url, callback=self._parse_sitemap) @@ -86,7 +102,7 @@ class SitemapSpider(Spider): yield Request(loc, callback=c) break - def _get_sitemap_body(self, response): + def _get_sitemap_body(self, response: Response) -> Optional[bytes]: """Return the sitemap body contained in the given response, or None if the response is not a sitemap. """ @@ -117,15 +133,16 @@ class SitemapSpider(Spider): # in other word, here, we have plain XML if response.url.endswith(".xml") or response.url.endswith(".xml.gz"): return response.body + return None -def regex(x): +def regex(x: Union[re.Pattern[str], str]) -> re.Pattern[str]: if isinstance(x, str): return re.compile(x) return x -def iterloc(it, alt=False): +def iterloc(it: Iterable[Dict[str, Any]], alt: bool = False) -> Iterable[str]: for d in it: yield d["loc"] diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 7dcee3a2f..cf429043d 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ -from typing import Any, Dict, Generator, Iterator, Optional +from typing import Any, Dict, Generator, Iterator, Optional, Union from urllib.parse import urljoin import lxml.etree # nosec @@ -15,7 +15,7 @@ class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" - def __init__(self, xmltext: str): + def __init__(self, xmltext: Union[str, bytes]): xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) From b88f22c6c5de4ca8828b2abe860516c2468c4fe3 Mon Sep 17 00:00:00 2001 From: Bagaudin Magomedov <51474632+bloodforcream@users.noreply.github.com> Date: Mon, 13 May 2024 15:33:24 +0300 Subject: [PATCH 1438/2083] Add SpiderLoggerAdapter, change Spider.logger to return SpiderLoggerAdapter (#6324) --- scrapy/spiders/__init__.py | 7 +- scrapy/utils/log.py | 25 ++++- tests/spiders.py | 23 +++++ tests/test_utils_log.py | 185 +++++++++++++++++++++++++++++++++++++ 4 files changed, 237 insertions(+), 3 deletions(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 370801f28..bef041325 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -25,6 +25,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.settings import BaseSettings + from scrapy.utils.log import SpiderLoggerAdapter CallbackT = Callable[Concatenate[Response, ...], Any] @@ -47,9 +48,11 @@ class Spider(object_ref): self.start_urls: List[str] = [] @property - def logger(self) -> logging.LoggerAdapter: + def logger(self) -> SpiderLoggerAdapter: + from scrapy.utils.log import SpiderLoggerAdapter + logger = logging.getLogger(self.name) - return logging.LoggerAdapter(logger, {"spider": self}) + return SpiderLoggerAdapter(logger, {"spider": self}) def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None: """Log the given message at the given log level diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2a38f151a..430a91e95 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -4,7 +4,17 @@ import logging import sys from logging.config import dictConfig from types import TracebackType -from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + List, + MutableMapping, + Optional, + Tuple, + Type, + Union, + cast, +) from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -238,3 +248,16 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: args = logkws if not logkws.get("args") else logkws["args"] return (level, message, args) + + +class SpiderLoggerAdapter(logging.LoggerAdapter): + def process( + self, msg: str, kwargs: MutableMapping[str, Any] + ) -> Tuple[str, MutableMapping[str, Any]]: + """Method that augments logging with additional 'extra' data""" + if isinstance(kwargs.get("extra"), MutableMapping): + kwargs["extra"].update(self.extra) + else: + kwargs["extra"] = self.extra + + return msg, kwargs diff --git a/tests/spiders.py b/tests/spiders.py index 94969db99..ea419afbd 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -4,6 +4,7 @@ Some spiders used for testing and benchmarking import asyncio import time +from typing import Optional from urllib.parse import urlencode from twisted.internet import defer @@ -78,6 +79,28 @@ class DelaySpider(MetaSpider): self.t2_err = time.time() +class LogSpider(MetaSpider): + name = "log_spider" + + def log_debug(self, message: str, extra: Optional[dict] = None): + self.logger.debug(message, extra=extra) + + def log_info(self, message: str, extra: Optional[dict] = None): + self.logger.info(message, extra=extra) + + def log_warning(self, message: str, extra: Optional[dict] = None): + self.logger.warning(message, extra=extra) + + def log_error(self, message: str, extra: Optional[dict] = None): + self.logger.error(message, extra=extra) + + def log_critical(self, message: str, extra: Optional[dict] = None): + self.logger.critical(message, extra=extra) + + def parse(self, response): + pass + + class SlowSpider(DelaySpider): name = "slow" diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index eae744df5..a8d080822 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,18 +1,26 @@ +import json import logging +import re import sys import unittest +from io import StringIO +from typing import Any, Dict, Mapping, MutableMapping +from unittest import TestCase +import pytest from testfixtures import LogCapture from twisted.python.failure import Failure from scrapy.extensions import telnet from scrapy.utils.log import ( LogCounterHandler, + SpiderLoggerAdapter, StreamLogger, TopLevelFormatter, failure_to_exc_info, ) from scrapy.utils.test import get_crawler +from tests.spiders import LogSpider class FailureToExcInfoTest(unittest.TestCase): @@ -106,3 +114,180 @@ class StreamLoggerTest(unittest.TestCase): with LogCapture() as log: print("test log msg") log.check(("test", "ERROR", "test log msg")) + + +@pytest.mark.parametrize( + ("base_extra", "log_extra", "expected_extra"), + ( + ( + {"spider": "test"}, + {"extra": {"log_extra": "info"}}, + {"extra": {"log_extra": "info", "spider": "test"}}, + ), + ( + {"spider": "test"}, + {"extra": None}, + {"extra": {"spider": "test"}}, + ), + ( + {"spider": "test"}, + {"extra": {"spider": "test2"}}, + {"extra": {"spider": "test"}}, + ), + ), +) +def test_spider_logger_adapter_process( + base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: Dict +): + logger = logging.getLogger("test") + spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) + + log_message = "test_log_message" + result_message, result_kwargs = spider_logger_adapter.process( + log_message, log_extra + ) + + assert result_message == log_message + assert result_kwargs == expected_extra + + +class LoggingTestCase(TestCase): + def setUp(self): + self.log_stream = StringIO() + handler = logging.StreamHandler(self.log_stream) + logger = logging.getLogger("log_spider") + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + self.handler = handler + self.logger = logger + self.spider = LogSpider() + + def tearDown(self): + self.logger.removeHandler(self.handler) + + def test_debug_logging(self): + log_message = "Foo message" + self.spider.log_debug(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_info_logging(self): + log_message = "Bar message" + self.spider.log_info(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_warning_logging(self): + log_message = "Baz message" + self.spider.log_warning(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_error_logging(self): + log_message = "Foo bar message" + self.spider.log_error(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + def test_critical_logging(self): + log_message = "Foo bar baz message" + self.spider.log_critical(log_message) + log_contents = self.log_stream.getvalue() + + assert log_contents == f"{log_message}\n" + + +class LoggingWithExtraTestCase(TestCase): + def setUp(self): + self.log_stream = StringIO() + handler = logging.StreamHandler(self.log_stream) + formatter = logging.Formatter( + '{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}' + ) + handler.setFormatter(formatter) + logger = logging.getLogger("log_spider") + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + self.handler = handler + self.logger = logger + self.spider = LogSpider() + self.regex_pattern = re.compile(r"^]+>$") + + def tearDown(self): + self.logger.removeHandler(self.handler) + + def test_debug_logging(self): + log_message = "Foo message" + extra = {"important_info": "foo"} + self.spider.log_debug(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "DEBUG" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_info_logging(self): + log_message = "Bar message" + extra = {"important_info": "bar"} + self.spider.log_info(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "INFO" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_warning_logging(self): + log_message = "Baz message" + extra = {"important_info": "baz"} + self.spider.log_warning(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "WARNING" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_error_logging(self): + log_message = "Foo bar message" + extra = {"important_info": "foo bar"} + self.spider.log_error(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "ERROR" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_critical_logging(self): + log_message = "Foo bar baz message" + extra = {"important_info": "foo bar baz"} + self.spider.log_critical(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "CRITICAL" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] + + def test_overwrite_spider_extra(self): + log_message = "Foo message" + extra = {"important_info": "foo", "spider": "shouldn't change"} + self.spider.log_error(log_message, extra) + log_contents = self.log_stream.getvalue() + log_contents = json.loads(log_contents) + + assert log_contents["levelname"] == "ERROR" + assert log_contents["message"] == log_message + assert self.regex_pattern.match(log_contents["spider"]) + assert log_contents["important_info"] == extra["important_info"] From d2f1e00a6afa46c26a334a24217d5bf605bab9fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 14 May 2024 18:54:11 +0200 Subject: [PATCH 1439/2083] Merge 2.11.2 changes (#6363) --- .bumpversion.cfg | 2 +- docs/faq.rst | 40 +- docs/news.rst | 116 +- docs/topics/benchmarking.rst | 4 +- docs/topics/downloader-middleware.rst | 44 +- docs/topics/settings.rst | 2 +- docs/topics/signals.rst | 11 +- docs/topics/spider-middleware.rst | 40 +- docs/topics/spiders.rst | 3 +- scrapy/VERSION | 2 +- scrapy/core/engine.py | 17 +- scrapy/downloadermiddlewares/httpproxy.py | 19 +- scrapy/downloadermiddlewares/offsite.py | 77 ++ scrapy/downloadermiddlewares/redirect.py | 62 +- scrapy/settings/default_settings.py | 2 +- scrapy/spidermiddlewares/offsite.py | 8 +- scrapy/utils/python.py | 2 +- tests/test_downloadermiddleware.py | 8 +- tests/test_downloadermiddleware_offsite.py | 184 +++ tests/test_downloadermiddleware_redirect.py | 1359 +++++++++++++++---- tests/test_engine.py | 40 +- tests/test_utils_project.py | 21 +- tox.ini | 17 +- 23 files changed, 1726 insertions(+), 354 deletions(-) create mode 100644 scrapy/downloadermiddlewares/offsite.py create mode 100644 tests/test_downloadermiddleware_offsite.py diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 968a34d96..599cd0cff 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.11.1 +current_version = 2.11.2 commit = True tag = True tag_name = {new_version} diff --git a/docs/faq.rst b/docs/faq.rst index 7090f0bcd..d394406e8 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -138,39 +138,37 @@ See previous question. How can I prevent memory errors due to many allowed domains? ------------------------------------------------------------ -If you have a spider with a long list of -:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider -replacing the default -:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware -with a :ref:`custom spider middleware ` that requires -less memory. For example: +If you have a spider with a long list of :attr:`~scrapy.Spider.allowed_domains` +(e.g. 50,000+), consider replacing the default +:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` downloader +middleware with a :ref:`custom downloader middleware +` that requires less memory. For example: - If your domain names are similar enough, use your own regular expression - instead joining the strings in - :attr:`~scrapy.Spider.allowed_domains` into a complex regular - expression. + instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into + a complex regular expression. - If you can `meet the installation requirements`_, use pyre2_ instead of Python’s re_ to compile your URL-filtering regular expression. See :issue:`1908`. -See also other suggestions at `StackOverflow`_. +See also `other suggestions at StackOverflow +`__. .. note:: Remember to disable - :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable - your custom implementation: + :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` when you + enable your custom implementation: .. code-block:: python - SPIDER_MIDDLEWARES = { - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, - "myproject.middlewares.CustomOffsiteMiddleware": 500, + DOWNLOADER_MIDDLEWARES = { + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": None, + "myproject.middlewares.CustomOffsiteMiddleware": 50, } .. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation .. _pyre2: https://github.com/andreasvc/pyre2 .. _re: https://docs.python.org/library/re.html -.. _StackOverflow: https://stackoverflow.com/q/36440681/939364 Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- @@ -206,12 +204,10 @@ I get "Filtered offsite request" messages. How can I fix them? Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a problem, so you may not need to fix them. -Those messages are thrown by the Offsite Spider Middleware, which is a spider -middleware (enabled by default) whose purpose is to filter out requests to -domains outside the ones covered by the spider. - -For more info see: -:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware`. +Those messages are thrown by +:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, which is a +downloader middleware (enabled by default) whose purpose is to filter out +requests to domains outside the ones covered by the spider. What is the recommended way to deploy a Scrapy crawler in production? --------------------------------------------------------------------- diff --git a/docs/news.rst b/docs/news.rst index 7db4e59a1..758b22d80 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,6 @@ Release notes ============= - .. _release-VERSION: Scrapy VERSION (YYYY-MM-DD) @@ -12,11 +11,122 @@ Scrapy VERSION (YYYY-MM-DD) Deprecations ~~~~~~~~~~~~ -- :func:`scrapy.core.downloader.Downloader._get_slot_key` is now deprecated. - Consider using its corresponding public method get_slot_key() instead. +- :meth:`scrapy.core.downloader.Downloader._get_slot_key` is deprecated, use + :meth:`scrapy.core.downloader.Downloader.get_slot_key` instead. (:issue:`6340`) +.. _release-2.11.2: + +Scrapy 2.11.2 (2024-05-14) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Redirects to non-HTTP protocols are no longer followed. Please, see the + `23j4-mw76-5v7h security advisory`_ for more information. (:issue:`457`) + + .. _23j4-mw76-5v7h security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-23j4-mw76-5v7h + +- The ``Authorization`` header is now dropped on redirects to a different + scheme (``http://`` or ``https://``) or port, even if the domain is the + same. Please, see the `4qqq-9vqf-3h3f security advisory`_ for more + information. + + .. _4qqq-9vqf-3h3f security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-4qqq-9vqf-3h3f + +- When using system proxy settings that are different for ``http://`` and + ``https://``, redirects to a different URL scheme will now also trigger the + corresponding change in proxy settings for the redirected request. Please, + see the `jm3v-qxmh-hxwv security advisory`_ for more information. + (:issue:`767`) + + .. _jm3v-qxmh-hxwv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-jm3v-qxmh-hxwv + +- :attr:`Spider.allowed_domains ` is now + enforced for all requests, and not only requests from spider callbacks. + (:issue:`1042`, :issue:`2241`, :issue:`6358`) + +- :func:`~scrapy.utils.iterators.xmliter_lxml` no longer resolves XML + entities. (:issue:`6265`) + +- defusedxml_ is now used to make + :class:`scrapy.http.request.rpc.XmlRpcRequest` more secure. + (:issue:`6250`, :issue:`6251`) + + .. _defusedxml: https://github.com/tiran/defusedxml + +Bug fixes +~~~~~~~~~ + +- Restored support for brotlipy_, which had been dropped in Scrapy 2.11.1 in + favor of brotli_. (:issue:`6261`) + + .. _brotli: https://github.com/google/brotli + + .. note:: brotlipy is deprecated, both in Scrapy and upstream. Use brotli + instead if you can. + +- Make :setting:`METAREFRESH_IGNORE_TAGS` ``["noscript"]`` by default. This + prevents + :class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` from + following redirects that would not be followed by web browsers with + JavaScript enabled. (:issue:`6342`, :issue:`6347`) + +- During :ref:`feed export `, do not close the + underlying file from :ref:`built-in post-processing plugins + `. + (:issue:`5932`, :issue:`6178`, :issue:`6239`) + +- :class:`LinkExtractor ` + now properly applies the ``unique`` and ``canonicalize`` parameters. + (:issue:`3273`, :issue:`6221`) + +- Do not initialize the scheduler disk queue if :setting:`JOBDIR` is an empty + string. (:issue:`6121`, :issue:`6124`) + +- Fix :attr:`Spider.logger ` not logging custom extra + information. (:issue:`6323`, :issue:`6324`) + +- ``robots.txt`` files with a non-UTF-8 encoding no longer prevent parsing + the UTF-8-compatible (e.g. ASCII) parts of the document. + (:issue:`6292`, :issue:`6298`) + +- :meth:`scrapy.http.cookies.WrappedRequest.get_header` no longer raises an + exception if ``default`` is ``None``. + (:issue:`6308`, :issue:`6310`) + +- :class:`~scrapy.selector.Selector` now uses + :func:`scrapy.utils.response.get_base_url` to determine the base URL of a + given :class:`~scrapy.http.Response`. (:issue:`6265`) + +- The :meth:`media_to_download` method of :ref:`media pipelines + ` now logs exceptions before stripping them. + (:issue:`5067`, :issue:`5068`) + +- When passing a callback to the :command:`parse` command, build the callback + callable with the right signature. + (:issue:`6182`) + +Documentation +~~~~~~~~~~~~~ + +- Add a FAQ entry about :ref:`creating blank requests `. + (:issue:`6203`, :issue:`6208`) + +- Document that :attr:`scrapy.selector.Selector.type` can be ``"json"``. + (:issue:`6328`, :issue:`6334`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Make builds reproducible. (:issue:`5019`, :issue:`6322`) + +- Packaging and test fixes. + (:issue:`6286`, :issue:`6290`, :issue:`6312`, :issue:`6316`, :issue:`6344`) + + .. _release-2.11.1: Scrapy 2.11.1 (2024-02-14) diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 0643df6a6..b704e54ed 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -24,7 +24,8 @@ You should see an output like this:: 'scrapy.extensions.telnet.TelnetConsole', 'scrapy.extensions.corestats.CoreStats'] 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares: - ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', + ['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware', + 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', @@ -37,7 +38,6 @@ You should see an output like this:: 'scrapy.downloadermiddlewares.stats.DownloaderStats'] 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares: ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', 'scrapy.spidermiddlewares.referer.RefererMiddleware', 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', 'scrapy.spidermiddlewares.depth.DepthMiddleware'] diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index d4cd062fe..c31f7fe43 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -763,6 +763,44 @@ HttpProxyMiddleware Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. +OffsiteMiddleware +----------------- + +.. module:: scrapy.downloadermiddlewares.offsite + :synopsis: Offsite Middleware + +.. class:: OffsiteMiddleware + + .. versionadded:: 2.11.2 + + Filters out Requests for URLs outside the domains covered by the spider. + + This middleware filters out every request whose host names aren't in the + spider's :attr:`~scrapy.Spider.allowed_domains` attribute. + All subdomains of any domain in the list are also allowed. + E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` + but not ``www2.example.com`` nor ``example.com``. + + When your spider returns a request for a domain not belonging to those + covered by the spider, this middleware will log a debug message similar to + this one:: + + DEBUG: Filtered offsite request to 'offsite.example': + + To avoid filling the log with too much noise, it will only print one of + these messages for each new domain filtered. So, for example, if another + request for ``offsite.example`` is filtered, no log message will be + printed. But if a request for ``other.example`` is filtered, a message + will be printed (but only for the first request filtered). + + If the spider doesn't define an + :attr:`~scrapy.Spider.allowed_domains` attribute, or the + attribute is empty, the offsite middleware will allow all requests. + + If the request has the :attr:`~scrapy.Request.dont_filter` attribute + set, the offsite middleware will allow the request even if its domain is not + listed in allowed domains. + RedirectMiddleware ------------------ @@ -882,7 +920,11 @@ Meta tags within these tags are ignored. .. versionchanged:: 2.0 The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from - ``['script', 'noscript']`` to ``[]``. + ``["script", "noscript"]`` to ``[]``. + +.. versionchanged:: 2.11.2 + The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from + ``[]`` to ``["noscript"]``. .. versionchanged:: VERSION The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2bd9cf1ed..904bd7ecc 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -674,6 +674,7 @@ Default: .. code-block:: python { + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50, "scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100, "scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300, "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350, @@ -1613,7 +1614,6 @@ Default: { "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 9bfd1761c..13e636055 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -343,11 +343,18 @@ request_scheduled .. signal:: request_scheduled .. function:: request_scheduled(request, spider) - Sent when the engine schedules a :class:`~scrapy.Request`, to be - downloaded later. + Sent when the engine is asked to schedule a :class:`~scrapy.Request`, to be + downloaded later, before the request reaches the :ref:`scheduler + `. + + Raise :exc:`~scrapy.exceptions.IgnoreRequest` to drop a request before it + reaches the scheduler. This signal does not support returning deferreds from its handlers. + .. versionadded:: 2.11.2 + Allow dropping requests with :exc:`~scrapy.exceptions.IgnoreRequest`. + :param request: the request that reached the scheduler :type request: :class:`~scrapy.Request` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 3f16efea5..8ddf17a14 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -51,8 +51,8 @@ value. For example, if you want to disable the off-site middleware: .. code-block:: python SPIDER_MIDDLEWARES = { - "myproject.middlewares.CustomSpiderMiddleware": 543, - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, + "scrapy.spidermiddlewares.referer.RefererMiddleware": None, + "myproject.middlewares.CustomRefererSpiderMiddleware": 700, } Finally, keep in mind that some middlewares may need to be enabled through a @@ -313,42 +313,6 @@ Default: ``False`` Pass all responses, regardless of its status code. -OffsiteMiddleware ------------------ - -.. module:: scrapy.spidermiddlewares.offsite - :synopsis: Offsite Spider Middleware - -.. class:: OffsiteMiddleware - - Filters out Requests for URLs outside the domains covered by the spider. - - This middleware filters out every request whose host names aren't in the - spider's :attr:`~scrapy.Spider.allowed_domains` attribute. - All subdomains of any domain in the list are also allowed. - E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` - but not ``www2.example.com`` nor ``example.com``. - - When your spider returns a request for a domain not belonging to those - covered by the spider, this middleware will log a debug message similar to - this one:: - - DEBUG: Filtered offsite request to 'www.othersite.com': - - To avoid filling the log with too much noise, it will only print one of - these messages for each new domain filtered. So, for example, if another - request for ``www.othersite.com`` is filtered, no log message will be - printed. But if a request for ``someothersite.com`` is filtered, a message - will be printed (but only for the first request filtered). - - If the spider doesn't define an - :attr:`~scrapy.Spider.allowed_domains` attribute, or the - attribute is empty, the offsite middleware will allow all requests. - - If the request has the :attr:`~scrapy.Request.dont_filter` attribute - set, the offsite middleware will allow the request even if its domain is not - listed in allowed domains. - RefererMiddleware ----------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 30677fe74..8a0102a51 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -75,7 +75,8 @@ scrapy.Spider An optional list of strings containing domains that this spider is allowed to crawl. Requests for URLs not belonging to the domain names specified in this list (or their subdomains) won't be followed if - :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. + :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is + enabled. Let's say your target url is ``https://www.example.com/1.html``, then add ``'example.com'`` to the list. diff --git a/scrapy/VERSION b/scrapy/VERSION index 6ceb272ee..9e5bb77a3 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.11.1 +2.11.2 diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 6bf3f3e26..4eca03800 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -28,7 +28,7 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper -from scrapy.exceptions import CloseSpider, DontCloseSpider +from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter from scrapy.settings import BaseSettings, Settings @@ -36,6 +36,7 @@ from scrapy.signalmanager import SignalManager from scrapy.spiders import Spider from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.python import global_object_name from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: @@ -292,9 +293,19 @@ class ExecutionEngine: self.slot.nextcall.schedule() # type: ignore[union-attr] def _schedule_request(self, request: Request, spider: Spider) -> None: - self.signals.send_catch_log( - signals.request_scheduled, request=request, spider=spider + request_scheduled_result = self.signals.send_catch_log( + signals.request_scheduled, + request=request, + spider=spider, + dont_log=IgnoreRequest, ) + for handler, result in request_scheduled_result: + if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): + logger.debug( + f"Signal handler {global_object_name(handler)} dropped " + f"request {request} before it reached the scheduler." + ) + return if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( signals.request_dropped, request=request, spider=spider diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 335896ac1..5b56ad449 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -60,26 +60,33 @@ class HttpProxyMiddleware: def process_request( self, request: Request, spider: Spider ) -> Union[Request, Response, None]: - creds, proxy_url = None, None + creds, proxy_url, scheme = None, None, None if "proxy" in request.meta: if request.meta["proxy"] is not None: creds, proxy_url = self._get_proxy(request.meta["proxy"], "") elif self.proxies: parsed = urlparse_cached(request) - scheme = parsed.scheme + _scheme = parsed.scheme if ( # 'no_proxy' is only supported by http schemes - scheme not in ("http", "https") + _scheme not in ("http", "https") or (parsed.hostname and not proxy_bypass(parsed.hostname)) - ) and scheme in self.proxies: + ) and _scheme in self.proxies: + scheme = _scheme creds, proxy_url = self.proxies[scheme] - self._set_proxy_and_creds(request, proxy_url, creds) + self._set_proxy_and_creds(request, proxy_url, creds, scheme) return None def _set_proxy_and_creds( - self, request: Request, proxy_url: Optional[str], creds: Optional[bytes] + self, + request: Request, + proxy_url: Optional[str], + creds: Optional[bytes], + scheme: Optional[str], ) -> None: + if scheme: + request.meta["_scheme_proxy"] = True if proxy_url: request.meta["proxy"] = proxy_url elif request.meta.get("proxy") is not None: diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py new file mode 100644 index 000000000..1e5026925 --- /dev/null +++ b/scrapy/downloadermiddlewares/offsite.py @@ -0,0 +1,77 @@ +import logging +import re +import warnings + +from scrapy import signals +from scrapy.exceptions import IgnoreRequest +from scrapy.utils.httpobj import urlparse_cached + +logger = logging.getLogger(__name__) + + +class OffsiteMiddleware: + @classmethod + def from_crawler(cls, crawler): + o = cls(crawler.stats) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled) + return o + + def __init__(self, stats): + self.stats = stats + self.domains_seen = set() + + def spider_opened(self, spider): + self.host_regex = self.get_host_regex(spider) + + def request_scheduled(self, request, spider): + self.process_request(request, spider) + + def process_request(self, request, spider): + if request.dont_filter or self.should_follow(request, spider): + return None + domain = urlparse_cached(request).hostname + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {"domain": domain, "request": request}, + extra={"spider": spider}, + ) + self.stats.inc_value("offsite/domains", spider=spider) + self.stats.inc_value("offsite/filtered", spider=spider) + raise IgnoreRequest + + def should_follow(self, request, spider): + regex = self.host_regex + # hostname can be None for wrong urls (like javascript links) + host = urlparse_cached(request).hostname or "" + return bool(regex.search(host)) + + def get_host_regex(self, spider): + """Override this method to implement a different offsite policy""" + allowed_domains = getattr(spider, "allowed_domains", None) + if not allowed_domains: + return re.compile("") # allow all by default + url_pattern = re.compile(r"^https?://.*$") + port_pattern = re.compile(r":\d+$") + domains = [] + for domain in allowed_domains: + if domain is None: + continue + if url_pattern.match(domain): + message = ( + "allowed_domains accepts only domains, not URLs. " + f"Ignoring URL entry {domain} in allowed_domains." + ) + warnings.warn(message) + elif port_pattern.search(domain): + message = ( + "allowed_domains accepts only domains without ports. " + f"Ignoring entry {domain} in allowed_domains." + ) + warnings.warn(message) + else: + domains.append(re.escape(domain)) + regex = rf'^(.*\.)?({"|".join(domains)})$' + return re.compile(regex) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 24089afea..aa08827c4 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -29,17 +29,49 @@ def _build_redirect_request( **kwargs, cookies=None, ) + if "_scheme_proxy" in redirect_request.meta: + source_request_scheme = urlparse_cached(source_request).scheme + redirect_request_scheme = urlparse_cached(redirect_request).scheme + if source_request_scheme != redirect_request_scheme: + redirect_request.meta.pop("_scheme_proxy") + redirect_request.meta.pop("proxy", None) + redirect_request.meta.pop("_auth_proxy", None) + redirect_request.headers.pop(b"Proxy-Authorization", None) has_cookie_header = "Cookie" in redirect_request.headers has_authorization_header = "Authorization" in redirect_request.headers if has_cookie_header or has_authorization_header: - source_request_netloc = urlparse_cached(source_request).netloc - redirect_request_netloc = urlparse_cached(redirect_request).netloc - if source_request_netloc != redirect_request_netloc: - if has_cookie_header: - del redirect_request.headers["Cookie"] - # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name - if has_authorization_header: - del redirect_request.headers["Authorization"] + default_ports = {"http": 80, "https": 443} + + parsed_source_request = urlparse_cached(source_request) + source_scheme, source_host, source_port = ( + parsed_source_request.scheme, + parsed_source_request.hostname, + parsed_source_request.port + or default_ports.get(parsed_source_request.scheme), + ) + + parsed_redirect_request = urlparse_cached(redirect_request) + redirect_scheme, redirect_host, redirect_port = ( + parsed_redirect_request.scheme, + parsed_redirect_request.hostname, + parsed_redirect_request.port + or default_ports.get(parsed_redirect_request.scheme), + ) + + if has_cookie_header and ( + redirect_scheme not in {source_scheme, "https"} + or source_host != redirect_host + ): + del redirect_request.headers["Cookie"] + + # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name + if has_authorization_header and ( + source_scheme != redirect_scheme + or source_host != redirect_host + or source_port != redirect_port + ): + del redirect_request.headers["Authorization"] + return redirect_request @@ -129,9 +161,11 @@ class RedirectMiddleware(BaseRedirectMiddleware): location = request_scheme + "://" + location.lstrip("/") redirected_url = urljoin(request.url, location) + redirected = _build_redirect_request(request, url=redirected_url) + if urlparse_cached(redirected).scheme not in {"http", "https"}: + return response if response.status in (301, 307, 308) or request.method == "HEAD": - redirected = _build_redirect_request(request, url=redirected_url) return self._redirect(redirected, request, spider, response.status) redirected = self._redirect_request_using_get(request, redirected_url) @@ -153,12 +187,16 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): request.meta.get("dont_redirect", False) or request.method == "HEAD" or not isinstance(response, HtmlResponse) + or urlparse_cached(request).scheme not in {"http", "https"} ): return response interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags) - if url and cast(float, interval) < self._maxdelay: - redirected = self._redirect_request_using_get(request, url) + if not url: + return response + redirected = self._redirect_request_using_get(request, url) + if urlparse_cached(redirected).scheme not in {"http", "https"}: + return response + if cast(float, interval) < self._maxdelay: return self._redirect(redirected, request, spider, "meta refresh") - return response diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d7ac7ec35..932475fb5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -101,6 +101,7 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50, "scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware": 100, "scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware": 300, "scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware": 350, @@ -301,7 +302,6 @@ SPIDER_MIDDLEWARES = {} SPIDER_MIDDLEWARES_BASE = { # Engine side "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index dd2fccfcb..50c93ac9f 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -13,15 +13,21 @@ from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set from scrapy import Spider, signals from scrapy.crawler import Crawler +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached +warnings.warn( + "The scrapy.spidermiddlewares.offsite module is deprecated, use " + "scrapy.downloadermiddlewares.offsite instead.", + ScrapyDeprecationWarning, +) + if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self - logger = logging.getLogger(__name__) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5d2d490b2..578cde2ac 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -331,7 +331,7 @@ def global_object_name(obj: Any) -> str: >>> global_object_name(Request) 'scrapy.http.request.Request' """ - return f"{obj.__module__}.{obj.__name__}" + return f"{obj.__module__}.{obj.__qualname__}" if hasattr(sys, "pypy_version_info"): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 062e8a8b4..0155c62eb 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -22,13 +22,11 @@ class ManagerTestCase(TestCase): self.crawler = get_crawler(Spider, self.settings_dict) self.spider = self.crawler._create_spider("foo") self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) - # some mw depends on stats collector - self.crawler.stats.open_spider(self.spider) - return self.mwman.open_spider(self.spider) + self.crawler.engine = self.crawler._create_engine() + return self.crawler.engine.open_spider(self.spider, start_requests=()) def tearDown(self): - self.crawler.stats.close_spider(self.spider, "") - return self.mwman.close_spider(self.spider) + return self.crawler.engine.close_spider(self.spider) def _download(self, request, response=None): """Executes downloader mw manager's download method and returns diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py new file mode 100644 index 000000000..d4669f450 --- /dev/null +++ b/tests/test_downloadermiddleware_offsite.py @@ -0,0 +1,184 @@ +import pytest + +from scrapy import Request, Spider +from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.utils.test import get_crawler + +UNSET = object() + + +@pytest.mark.parametrize( + ("allowed_domain", "url", "allowed"), + ( + ("example.com", "http://example.com/1", True), + ("example.com", "http://example.org/1", False), + ("example.com", "http://sub.example.com/1", True), + ("sub.example.com", "http://sub.example.com/1", True), + ("sub.example.com", "http://example.com/1", False), + ("example.com", "http://example.com:8000/1", True), + ("example.com", "http://example.org/example.com", False), + ("example.com", "http://example.org/foo.example.com", False), + ("example.com", "http://example.com.example", False), + ("a.example", "http://nota.example", False), + ("b.a.example", "http://notb.a.example", False), + ), +) +def test_process_request_domain_filtering(allowed_domain, url, allowed): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request(url) + if allowed: + assert mw.process_request(request, spider) is None + else: + with pytest.raises(IgnoreRequest): + mw.process_request(request, spider) + + +@pytest.mark.parametrize( + ("value", "filtered"), + ( + (UNSET, True), + (None, True), + (False, True), + (True, False), + ), +) +def test_process_request_dont_filter(value, filtered): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + kwargs = {} + if value is not UNSET: + kwargs["dont_filter"] = value + request = Request("https://b.example", **kwargs) + if filtered: + with pytest.raises(IgnoreRequest): + mw.process_request(request, spider) + else: + assert mw.process_request(request, spider) is None + + +@pytest.mark.parametrize( + "value", + ( + UNSET, + None, + [], + ), +) +def test_process_request_no_allowed_domains(value): + crawler = get_crawler(Spider) + kwargs = {} + if value is not UNSET: + kwargs["allowed_domains"] = value + spider = crawler._create_spider(name="a", **kwargs) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request("https://example.com") + assert mw.process_request(request, spider) is None + + +def test_process_request_invalid_domains(): + crawler = get_crawler(Spider) + allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] + spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request("https://a.example") + assert mw.process_request(request, spider) is None + for letter in ("b", "c"): + request = Request(f"https://{letter}.example") + with pytest.raises(IgnoreRequest): + mw.process_request(request, spider) + + +@pytest.mark.parametrize( + ("allowed_domain", "url", "allowed"), + ( + ("example.com", "http://example.com/1", True), + ("example.com", "http://example.org/1", False), + ("example.com", "http://sub.example.com/1", True), + ("sub.example.com", "http://sub.example.com/1", True), + ("sub.example.com", "http://example.com/1", False), + ("example.com", "http://example.com:8000/1", True), + ("example.com", "http://example.org/example.com", False), + ("example.com", "http://example.org/foo.example.com", False), + ("example.com", "http://example.com.example", False), + ("a.example", "http://nota.example", False), + ("b.a.example", "http://notb.a.example", False), + ), +) +def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request(url) + if allowed: + assert mw.request_scheduled(request, spider) is None + else: + with pytest.raises(IgnoreRequest): + mw.request_scheduled(request, spider) + + +@pytest.mark.parametrize( + ("value", "filtered"), + ( + (UNSET, True), + (None, True), + (False, True), + (True, False), + ), +) +def test_request_scheduled_dont_filter(value, filtered): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + kwargs = {} + if value is not UNSET: + kwargs["dont_filter"] = value + request = Request("https://b.example", **kwargs) + if filtered: + with pytest.raises(IgnoreRequest): + mw.request_scheduled(request, spider) + else: + assert mw.request_scheduled(request, spider) is None + + +@pytest.mark.parametrize( + "value", + ( + UNSET, + None, + [], + ), +) +def test_request_scheduled_no_allowed_domains(value): + crawler = get_crawler(Spider) + kwargs = {} + if value is not UNSET: + kwargs["allowed_domains"] = value + spider = crawler._create_spider(name="a", **kwargs) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request("https://example.com") + assert mw.request_scheduled(request, spider) is None + + +def test_request_scheduled_invalid_domains(): + crawler = get_crawler(Spider) + allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] + spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + request = Request("https://a.example") + assert mw.request_scheduled(request, spider) is None + for letter in ("b", "c"): + request = Request(f"https://{letter}.example") + with pytest.raises(IgnoreRequest): + mw.request_scheduled(request, spider) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 83ff25982..4bfd34fe2 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,5 +1,9 @@ import unittest +from itertools import chain, product +import pytest + +from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.downloadermiddlewares.redirect import ( MetaRefreshMiddleware, RedirectMiddleware, @@ -7,22 +11,1030 @@ from scrapy.downloadermiddlewares.redirect import ( from scrapy.exceptions import IgnoreRequest from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider +from scrapy.utils.misc import set_environ from scrapy.utils.test import get_crawler -class RedirectMiddlewareTest(unittest.TestCase): +class Base: + class Test(unittest.TestCase): + def test_priority_adjust(self): + req = Request("http://a.com") + rsp = self.get_response(req, "http://a.com/redirected") + req2 = self.mw.process_response(req, rsp, self.spider) + self.assertGreater(req2.priority, req.priority) + + def test_dont_redirect(self): + url = "http://www.example.com/301" + url2 = "http://www.example.com/redirected" + req = Request(url, meta={"dont_redirect": True}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp, self.spider) + assert isinstance(r, Response) + assert r is rsp + + # Test that it redirects when dont_redirect is False + req = Request(url, meta={"dont_redirect": False}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp, self.spider) + assert isinstance(r, Request) + + def test_post(self): + url = "http://www.example.com/302" + url2 = "http://www.example.com/redirected2" + req = Request( + url, + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = self.get_response(req, url2) + + req2 = self.mw.process_response(req, rsp, self.spider) + assert isinstance(req2, Request) + self.assertEqual(req2.url, url2) + self.assertEqual(req2.method, "GET") + assert ( + "Content-Type" not in req2.headers + ), "Content-Type header must not be present in redirected request" + assert ( + "Content-Length" not in req2.headers + ), "Content-Length header must not be present in redirected request" + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + + def test_max_redirect_times(self): + self.mw.max_redirect_times = 1 + req = Request("http://scrapytest.org/302") + rsp = self.get_response(req, "/redirected") + + req = self.mw.process_response(req, rsp, self.spider) + assert isinstance(req, Request) + assert "redirect_times" in req.meta + self.assertEqual(req.meta["redirect_times"], 1) + self.assertRaises( + IgnoreRequest, self.mw.process_response, req, rsp, self.spider + ) + + def test_ttl(self): + self.mw.max_redirect_times = 100 + req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) + rsp = self.get_response(req, "/a") + + req = self.mw.process_response(req, rsp, self.spider) + assert isinstance(req, Request) + self.assertRaises( + IgnoreRequest, self.mw.process_response, req, rsp, self.spider + ) + + def test_redirect_urls(self): + req1 = Request("http://scrapytest.org/first") + rsp1 = self.get_response(req1, "/redirected") + req2 = self.mw.process_response(req1, rsp1, self.spider) + rsp2 = self.get_response(req1, "/redirected2") + req3 = self.mw.process_response(req2, rsp2, self.spider) + + self.assertEqual(req2.url, "http://scrapytest.org/redirected") + self.assertEqual( + req2.meta["redirect_urls"], ["http://scrapytest.org/first"] + ) + self.assertEqual(req3.url, "http://scrapytest.org/redirected2") + self.assertEqual( + req3.meta["redirect_urls"], + ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], + ) + + def test_redirect_reasons(self): + req1 = Request("http://scrapytest.org/first") + rsp1 = self.get_response(req1, "/redirected1") + req2 = self.mw.process_response(req1, rsp1, self.spider) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2, self.spider) + self.assertEqual(req2.meta["redirect_reasons"], [self.reason]) + self.assertEqual(req3.meta["redirect_reasons"], [self.reason, self.reason]) + + def test_cross_origin_header_dropping(self): + safe_headers = {"A": "B"} + cookie_header = {"Cookie": "a=b"} + authorization_header = {"Authorization": "Bearer 123456"} + + original_request = Request( + "https://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers. + internal_response = self.get_response( + original_request, "https://example.com/a" + ) + internal_redirect_request = self.mw.process_response( + original_request, internal_response, self.spider + ) + self.assertIsInstance(internal_redirect_request, Request) + self.assertEqual( + original_request.headers, internal_redirect_request.headers + ) + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers also when the scheme is http. + http_request = Request( + "http://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + http_response = self.get_response(http_request, "http://example.com/a") + http_redirect_request = self.mw.process_response( + http_request, http_response, self.spider + ) + self.assertIsInstance(http_redirect_request, Request) + self.assertEqual(http_request.headers, http_redirect_request.headers) + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_explicit_port_response = self.get_response( + original_request, "https://example.com:443/a" + ) + to_explicit_port_redirect_request = self.mw.process_response( + original_request, to_explicit_port_response, self.spider + ) + self.assertIsInstance(to_explicit_port_redirect_request, Request) + self.assertEqual( + original_request.headers, to_explicit_port_redirect_request.headers + ) + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_implicit_port_response = self.get_response( + original_request, "https://example.com/a" + ) + to_implicit_port_redirect_request = self.mw.process_response( + original_request, to_implicit_port_response, self.spider + ) + self.assertIsInstance(to_implicit_port_redirect_request, Request) + self.assertEqual( + original_request.headers, to_implicit_port_redirect_request.headers + ) + + # A port change drops the Authorization header because the origin + # changes, but keeps the Cookie header because the domain remains the + # same. + different_port_response = self.get_response( + original_request, "https://example.com:8080/a" + ) + different_port_redirect_request = self.mw.process_response( + original_request, different_port_response, self.spider + ) + self.assertIsInstance(different_port_redirect_request, Request) + self.assertEqual( + {**safe_headers, **cookie_header}, + different_port_redirect_request.headers.to_unicode_dict(), + ) + + # A domain change drops both the Authorization and the Cookie header. + external_response = self.get_response( + original_request, "https://example.org/a" + ) + external_redirect_request = self.mw.process_response( + original_request, external_response, self.spider + ) + self.assertIsInstance(external_redirect_request, Request) + self.assertEqual( + safe_headers, external_redirect_request.headers.to_unicode_dict() + ) + + # A scheme upgrade (http → https) drops the Authorization header + # because the origin changes, but keeps the Cookie header because the + # domain remains the same. + upgrade_response = self.get_response(http_request, "https://example.com/a") + upgrade_redirect_request = self.mw.process_response( + http_request, upgrade_response, self.spider + ) + self.assertIsInstance(upgrade_redirect_request, Request) + self.assertEqual( + {**safe_headers, **cookie_header}, + upgrade_redirect_request.headers.to_unicode_dict(), + ) + + # A scheme downgrade (https → http) drops the Authorization header + # because the origin changes, and the Cookie header because its value + # cannot indicate whether the cookies were secure (HTTPS-only) or not. + # + # Note: If the Cookie header is set by the cookie management + # middleware, as recommended in the docs, the dropping of Cookie on + # scheme downgrade is not an issue, because the cookie management + # middleware will add again the Cookie header to the new request if + # appropriate. + downgrade_response = self.get_response( + original_request, "http://example.com/a" + ) + downgrade_redirect_request = self.mw.process_response( + original_request, downgrade_response, self.spider + ) + self.assertIsInstance(downgrade_redirect_request, Request) + self.assertEqual( + safe_headers, + downgrade_redirect_request.headers.to_unicode_dict(), + ) + + def test_meta_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_http_to_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_meta_proxy_https_to_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_proxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request2.meta["proxy"], "https://b.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_proxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request1.meta["proxy"], "https://a.example") + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request3.meta["proxy"], "https://a.example") + + def test_system_proxy_unproxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertNotIn("Proxy-Authorization", request1.headers) + self.assertNotIn("_auth_proxy", request1.meta) + self.assertNotIn("proxy", request1.meta) + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request2.meta["proxy"], "https://b.example") + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + def test_system_proxy_unproxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertNotIn("Proxy-Authorization", request1.headers) + self.assertNotIn("_auth_proxy", request1.meta) + self.assertNotIn("proxy", request1.meta) + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + def test_system_proxy_proxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request1.meta["proxy"], "https://b.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request3.meta["proxy"], "https://b.example") + + def test_system_proxy_proxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request1.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request1.meta["proxy"], "https://b.example") + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic Yjo=") + self.assertEqual(request3.meta["_auth_proxy"], "https://b.example") + self.assertEqual(request3.meta["proxy"], "https://b.example") + + def test_system_proxy_unproxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertNotIn("Proxy-Authorization", request1.headers) + self.assertNotIn("_auth_proxy", request1.meta) + self.assertNotIn("proxy", request1.meta) + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") + self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") + self.assertEqual(request2.meta["proxy"], "https://a.example") + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + def test_system_proxy_unproxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + spider = None + proxy_mw.process_request(request1, spider) + + self.assertNotIn("Proxy-Authorization", request1.headers) + self.assertNotIn("_auth_proxy", request1.meta) + self.assertNotIn("proxy", request1.meta) + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1, spider) + + self.assertIsInstance(request2, Request) + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + proxy_mw.process_request(request2, spider) + + self.assertNotIn("Proxy-Authorization", request2.headers) + self.assertNotIn("_auth_proxy", request2.meta) + self.assertNotIn("proxy", request2.meta) + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2, spider) + + self.assertIsInstance(request3, Request) + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + proxy_mw.process_request(request3, spider) + + self.assertNotIn("Proxy-Authorization", request3.headers) + self.assertNotIn("_auth_proxy", request3.meta) + self.assertNotIn("proxy", request3.meta) + + +class RedirectMiddlewareTest(Base.Test): + mwcls = RedirectMiddleware + reason = 302 + def setUp(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") - self.mw = RedirectMiddleware.from_crawler(self.crawler) + self.mw = self.mwcls.from_crawler(self.crawler) - def test_priority_adjust(self): - req = Request("http://a.com") - rsp = Response( - "http://a.com", headers={"Location": "http://a.com/redirected"}, status=301 - ) - req2 = self.mw.process_response(req, rsp, self.spider) - assert req2.priority > req.priority + def get_response(self, request, location, status=302): + headers = {"Location": location} + return Response(request.url, status=status, headers=headers) def test_redirect_3xx_permanent(self): def _test(method, status=301): @@ -52,51 +1064,6 @@ class RedirectMiddlewareTest(unittest.TestCase): _test("POST", status=308) _test("HEAD", status=308) - def test_dont_redirect(self): - url = "http://www.example.com/301" - url2 = "http://www.example.com/redirected" - req = Request(url, meta={"dont_redirect": True}) - rsp = Response(url, headers={"Location": url2}, status=301) - - r = self.mw.process_response(req, rsp, self.spider) - assert isinstance(r, Response) - assert r is rsp - - # Test that it redirects when dont_redirect is False - req = Request(url, meta={"dont_redirect": False}) - rsp = Response(url2, status=200) - - r = self.mw.process_response(req, rsp, self.spider) - assert isinstance(r, Response) - assert r is rsp - - def test_redirect_302(self): - url = "http://www.example.com/302" - url2 = "http://www.example.com/redirected2" - req = Request( - url, - method="POST", - body="test", - headers={"Content-Type": "text/plain", "Content-length": "4"}, - ) - rsp = Response(url, headers={"Location": url2}, status=302) - - req2 = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, "GET") - assert ( - "Content-Type" not in req2.headers - ), "Content-Type header must not be present in redirected request" - assert ( - "Content-Length" not in req2.headers - ), "Content-Length header must not be present in redirected request" - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - - # response without Location header but with status code is 3XX should be ignored - del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp - def test_redirect_302_head(self): url = "http://www.example.com/302" url2 = "http://www.example.com/redirected2" @@ -108,10 +1075,6 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, url2) self.assertEqual(req2.method, "HEAD") - # response without Location header but with status code is 3XX should be ignored - del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp - def test_redirect_302_relative(self): url = "http://www.example.com/302" url2 = "///i8n.example2.com/302" @@ -124,81 +1087,6 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, url3) self.assertEqual(req2.method, "HEAD") - # response without Location header but with status code is 3XX should be ignored - del rsp.headers["Location"] - assert self.mw.process_response(req, rsp, self.spider) is rsp - - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://scrapytest.org/302") - rsp = Response( - "http://scrapytest.org/302", headers={"Location": "/redirected"}, status=302 - ) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - self.assertEqual(req.meta["redirect_times"], 1) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) - rsp = Response( - "http://www.scrapytest.org/302", - headers={"Location": "/redirected"}, - status=302, - ) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_redirect_urls(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = Response( - "http://scrapytest.org/first", - headers={"Location": "/redirected"}, - status=302, - ) - req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = Response( - "http://scrapytest.org/redirected", - headers={"Location": "/redirected2"}, - status=302, - ) - req3 = self.mw.process_response(req2, rsp2, self.spider) - - self.assertEqual(req2.url, "http://scrapytest.org/redirected") - self.assertEqual(req2.meta["redirect_urls"], ["http://scrapytest.org/first"]) - self.assertEqual(req3.url, "http://scrapytest.org/redirected2") - self.assertEqual( - req3.meta["redirect_urls"], - ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], - ) - - def test_redirect_reasons(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = Response( - "http://scrapytest.org/first", - headers={"Location": "/redirected1"}, - status=301, - ) - req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = Response( - "http://scrapytest.org/redirected1", - headers={"Location": "/redirected2"}, - status=301, - ) - req3 = self.mw.process_response(req2, rsp2, self.spider) - - self.assertEqual(req2.meta["redirect_reasons"], [301]) - self.assertEqual(req3.meta["redirect_reasons"], [301, 301]) - def test_spider_handling(self): smartspider = self.crawler._create_spider("smarty") smartspider.handle_httpstatus_list = [404, 301, 302] @@ -247,53 +1135,84 @@ class RedirectMiddlewareTest(unittest.TestCase): perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o" self.assertEqual(perc_encoded_utf8_url, req_result.url) - def test_cross_domain_header_dropping(self): - safe_headers = {"A": "B"} - original_request = Request( - "https://example.com", - headers={"Cookie": "a=b", "Authorization": "a", **safe_headers}, - ) - - internal_response = Response( - "https://example.com", - headers={"Location": "https://example.com/a"}, - status=301, - ) - internal_redirect_request = self.mw.process_response( - original_request, internal_response, self.spider - ) - self.assertIsInstance(internal_redirect_request, Request) - self.assertEqual(original_request.headers, internal_redirect_request.headers) - - external_response = Response( - "https://example.com", - headers={"Location": "https://example.org/a"}, - status=301, - ) - external_redirect_request = self.mw.process_response( - original_request, external_response, self.spider - ) - self.assertIsInstance(external_redirect_request, Request) - self.assertEqual( - safe_headers, external_redirect_request.headers.to_unicode_dict() - ) + def test_no_location(self): + request = Request("https://example.com") + response = Response(request.url, status=302) + assert self.mw.process_response(request, response, self.spider) is response -class MetaRefreshMiddlewareTest(unittest.TestCase): +SCHEME_PARAMS = ("url", "location", "target") +HTTP_SCHEMES = ("http", "https") +NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo") +REDIRECT_SCHEME_CASES = ( + # http/https → http/https redirects + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + f"{output_scheme}://example.com/b", + ) + for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2) + ), + # http/https → data/file/ftp/s3/foo does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in HTTP_SCHEMES + for output_scheme in NON_HTTP_SCHEMES + ), + # http/https → relative redirects + *( + ( + f"{scheme}://example.com/a", + location, + f"{scheme}://example.com/b", + ) + for scheme in HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + # Note: We do not test data/file/ftp/s3 schemes for the initial URL + # because their download handlers cannot return a status code of 3xx. +) + + +@pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES) +def test_redirect_schemes(url, location, target): + crawler = get_crawler(Spider) + spider = crawler._create_spider("foo") + mw = RedirectMiddleware.from_crawler(crawler) + request = Request(url) + response = Response(url, headers={"Location": location}, status=301) + redirect = mw.process_response(request, response, spider) + if target is None: + assert redirect == response + else: + assert isinstance(redirect, Request) + assert redirect.url == target + + +def meta_refresh_body(url, interval=5): + html = f"""""" + return html.encode("utf-8") + + +class MetaRefreshMiddlewareTest(Base.Test): + mwcls = MetaRefreshMiddleware + reason = "meta refresh" + def setUp(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider("foo") - self.mw = MetaRefreshMiddleware.from_crawler(crawler) + self.mw = self.mwcls.from_crawler(crawler) def _body(self, interval=5, url="http://example.org/newpage"): - html = f"""""" - return html.encode("utf-8") + return meta_refresh_body(url, interval) - def test_priority_adjust(self): - req = Request("http://a.com") - rsp = HtmlResponse(req.url, body=self._body()) - req2 = self.mw.process_response(req, rsp, self.spider) - assert req2.priority > req.priority + def get_response(self, request, location): + return HtmlResponse(request.url, body=self._body(url=location)) def test_meta_refresh(self): req = Request(url="http://example.org") @@ -332,62 +1251,6 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): ), "Content-Length header must not be present in redirected request" assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://scrapytest.org/max") - rsp = HtmlResponse(req.url, body=self._body()) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - self.assertEqual(req.meta["redirect_times"], 1) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) - rsp = HtmlResponse(req.url, body=self._body()) - - req = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req, Request) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) - - def test_redirect_urls(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = HtmlResponse(req1.url, body=self._body(url="/redirected")) - req2 = self.mw.process_response(req1, rsp1, self.spider) - assert isinstance(req2, Request), req2 - rsp2 = HtmlResponse(req2.url, body=self._body(url="/redirected2")) - req3 = self.mw.process_response(req2, rsp2, self.spider) - assert isinstance(req3, Request), req3 - self.assertEqual(req2.url, "http://scrapytest.org/redirected") - self.assertEqual(req2.meta["redirect_urls"], ["http://scrapytest.org/first"]) - self.assertEqual(req3.url, "http://scrapytest.org/redirected2") - self.assertEqual( - req3.meta["redirect_urls"], - ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], - ) - - def test_redirect_reasons(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = HtmlResponse( - "http://scrapytest.org/first", body=self._body(url="/redirected") - ) - req2 = self.mw.process_response(req1, rsp1, self.spider) - rsp2 = HtmlResponse( - "http://scrapytest.org/redirected", body=self._body(url="/redirected1") - ) - req3 = self.mw.process_response(req2, rsp2, self.spider) - - self.assertEqual(req2.meta["redirect_reasons"], ["meta refresh"]) - self.assertEqual( - req3.meta["redirect_reasons"], ["meta refresh", "meta refresh"] - ) - def test_ignore_tags_default(self): req = Request(url="http://example.org") body = ( @@ -413,5 +1276,45 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): assert isinstance(response, Response) +@pytest.mark.parametrize( + SCHEME_PARAMS, + ( + *REDIRECT_SCHEME_CASES, + # data/file/ftp/s3/foo → * does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in NON_HTTP_SCHEMES + for output_scheme in chain(HTTP_SCHEMES, NON_HTTP_SCHEMES) + ), + # data/file/ftp/s3/foo → relative does not redirect + *( + ( + f"{scheme}://example.com/a", + location, + None, + ) + for scheme in NON_HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + ), +) +def test_meta_refresh_schemes(url, location, target): + crawler = get_crawler(Spider) + spider = crawler._create_spider("foo") + mw = MetaRefreshMiddleware.from_crawler(crawler) + request = Request(url) + response = HtmlResponse(url, body=meta_refresh_body(location)) + redirect = mw.process_response(request, response, spider) + if target is None: + assert redirect == response + else: + assert isinstance(redirect, Request) + assert redirect.url == target + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_engine.py b/tests/test_engine.py index 8d7afb6a1..33544e8db 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -15,8 +15,10 @@ import subprocess import sys from collections import defaultdict from dataclasses import dataclass +from logging import DEBUG from pathlib import Path from threading import Timer +from unittest.mock import Mock from urllib.parse import urlparse import attr @@ -27,11 +29,13 @@ from twisted.trial import unittest from twisted.web import server, static, util from scrapy import signals -from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import CloseSpider +from scrapy.core.engine import ExecutionEngine, Slot +from scrapy.core.scheduler import BaseScheduler +from scrapy.exceptions import CloseSpider, IgnoreRequest from scrapy.http import Request from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor +from scrapy.signals import request_scheduled from scrapy.spiders import Spider from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler @@ -467,6 +471,38 @@ class EngineTest(unittest.TestCase): self.assertNotIn(b"Traceback", stderr) +def test_request_scheduled_signal(caplog): + class TestScheduler(BaseScheduler): + def __init__(self): + self.enqueued = [] + + def enqueue_request(self, request: Request) -> bool: + self.enqueued.append(request) + return True + + def signal_handler(request: Request, spider: Spider) -> None: + if "drop" in request.url: + raise IgnoreRequest + + spider = TestSpider() + crawler = get_crawler(spider.__class__) + engine = ExecutionEngine(crawler, lambda _: None) + engine.downloader._slot_gc_loop.stop() + scheduler = TestScheduler() + engine.slot = Slot((), None, Mock(), scheduler) + crawler.signals.connect(signal_handler, request_scheduled) + keep_request = Request("https://keep.example") + engine._schedule_request(keep_request, spider) + drop_request = Request("https://drop.example") + caplog.set_level(DEBUG) + engine._schedule_request(drop_request, spider) + assert scheduler.enqueued == [ + keep_request + ], f"{scheduler.enqueued!r} != [{keep_request!r}]" + assert "dropped request " in caplog.text + crawler.signals.disconnect(signal_handler, request_scheduled) + + if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 90bd350a5..3831f4c21 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -6,6 +6,7 @@ import unittest import warnings from pathlib import Path +from scrapy.utils.misc import set_environ from scrapy.utils.project import data_path, get_project_settings @@ -38,20 +39,6 @@ class ProjectUtilsTest(unittest.TestCase): self.assertEqual(abspath, data_path(abspath)) -@contextlib.contextmanager -def set_env(**update): - modified = set(update.keys()) & set(os.environ.keys()) - update_after = {k: os.environ[k] for k in modified} - remove_after = frozenset(k for k in update if k not in os.environ) - try: - os.environ.update(update) - yield - finally: - os.environ.update(update_after) - for k in remove_after: - os.environ.pop(k) - - class GetProjectSettingsTestCase(unittest.TestCase): def test_valid_envvar(self): value = "tests.test_cmdline.settings" @@ -60,7 +47,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): } with warnings.catch_warnings(): warnings.simplefilter("error") - with set_env(**envvars): + with set_environ(**envvars): settings = get_project_settings() assert settings.get("SETTINGS_MODULE") == value @@ -69,7 +56,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): envvars = { "SCRAPY_FOO": "bar", } - with set_env(**envvars): + with set_environ(**envvars): settings = get_project_settings() assert settings.get("SCRAPY_FOO") is None @@ -80,7 +67,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): "SCRAPY_FOO": "bar", "SCRAPY_SETTINGS_MODULE": value, } - with set_env(**envvars): + with set_environ(**envvars): settings = get_project_settings() assert settings.get("SETTINGS_MODULE") == value assert settings.get("SCRAPY_FOO") is None diff --git a/tox.ini b/tox.ini index ede139756..cde4243f3 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,9 @@ deps = # mitmproxy does not support PyPy mitmproxy; implementation_name != 'pypy' + # https://github.com/pallets/werkzeug/pull/2768 breaks flask, required by + # mitmproxy. + werkzeug < 3; python_version < '3.9' and implementation_name != 'pypy' passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -90,6 +93,7 @@ commands = twine check dist/* [pinned] +basepython = python3.8 deps = cryptography==36.0.0 cssselect==0.9.1 @@ -116,7 +120,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] -basepython = python3.8 +basepython = {[pinned]basepython} deps = {[pinned]deps} PyDispatcher==2.0.5 @@ -126,7 +130,7 @@ setenv = commands = {[pinned]commands} [testenv:windows-pinned] -basepython = python3 +basepython = {[pinned]basepython} deps = {[pinned]deps} PyDispatcher==2.0.5 @@ -155,7 +159,7 @@ deps = ipython [testenv:extra-deps-pinned] -basepython = python3.8 +basepython = {[pinned]basepython} deps = {[pinned]deps} boto3==1.20.0 @@ -179,6 +183,7 @@ commands = {[testenv]commands} --reactor=asyncio [testenv:asyncio-pinned] +basepython = {[pinned]basepython} deps = {[testenv:pinned]deps} commands = {[pinned]commands} --reactor=asyncio install_command = {[pinned]install_command} @@ -191,12 +196,12 @@ commands = pytest {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] -basepython = {[testenv:pypy3]basepython} +basepython = pypy3.8 deps = {[pinned]deps} PyPyDispatcher==2.1.0 commands = - pytest --durations=10 scrapy tests + pytest {posargs:--durations=10 scrapy tests} install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -244,7 +249,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} [testenv:botocore-pinned] -basepython = python3.8 +basepython = {[pinned]basepython} deps = {[pinned]deps} botocore==1.4.87 From 812fd2368f705d033f5f39c152130b12a0fe9b1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 May 2024 11:48:43 +0200 Subject: [PATCH 1440/2083] Allow user-defined secure cookies (#6357) --- docs/topics/request-response.rst | 3 +- scrapy/downloadermiddlewares/cookies.py | 14 ++- scrapy/utils/_compression.py | 9 +- tests/test_downloadermiddleware_cookies.py | 111 ++++++++++++++++++++- 4 files changed, 126 insertions(+), 11 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index eb70ebce8..3c2843bc1 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -94,13 +94,14 @@ Request objects .. code-block:: python request_with_cookies = Request( - url="http://www.example.com", + url="https://www.example.com", cookies=[ { "name": "currency", "value": "USD", "domain": "example.com", "path": "/currency", + "secure": True, }, ], ) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 85781efd6..6ada3b474 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -33,6 +33,7 @@ logger = logging.getLogger(__name__) _split_domain = TLDExtract(include_psl_private_domains=True) +_UNSET = object() def _is_public_domain(domain: str) -> bool: @@ -133,6 +134,7 @@ class CookiesMiddleware: Decode from bytes if necessary. """ decoded = {} + flags = set() for key in ("name", "value", "path", "domain"): if cookie.get(key) is None: if key in ("name", "value"): @@ -152,10 +154,16 @@ class CookiesMiddleware: cookie, ) decoded[key] = cookie[key].decode("latin1", errors="replace") - + for flag in ("secure",): + value = cookie.get(flag, _UNSET) + if value is _UNSET or not value: + continue + flags.add(flag) cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" for key, value in decoded.items(): # path, domain cookie_str += f"; {key.capitalize()}={value}" + for flag in flags: # secure + cookie_str += f"; {flag.capitalize()}" return cookie_str def _get_request_cookies( @@ -168,9 +176,11 @@ class CookiesMiddleware: return [] cookies: Iterable[Dict[str, Any]] if isinstance(request.cookies, dict): - cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) + cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items()) else: cookies = request.cookies + for cookie in cookies: + cookie.setdefault("secure", urlparse_cached(request).scheme == "https") formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) response = Response(request.url, headers={"Set-Cookie": formatted}) return jar.make_cookies(response, request) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 84c255c28..591737b8e 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -20,11 +20,10 @@ else: "You have brotlipy installed, and Scrapy will use it, but " "Scrapy support for brotlipy is deprecated and will stop " "working in a future version of Scrapy. brotlipy itself is " - "deprecated, it has been superseded by brotlicffi. " - "Please, uninstall brotlipy " - "and install brotli or brotlicffi instead. brotlipy has the same import " - "name as brotli, so keeping both installed is strongly " - "discouraged." + "deprecated, it has been superseded by brotlicffi. Please, " + "uninstall brotlipy and install brotli or brotlicffi instead. " + "brotlipy has the same import name as brotli, so keeping both " + "installed is strongly discouraged." ), ScrapyDeprecationWarning, ) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 425fabcc7..1f7e6615c 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -14,6 +14,8 @@ from scrapy.spiders import Spider from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler +UNSET = object() + def _cookie_to_set_cookie_value(cookie): """Given a cookie defined as a dictionary with name and value keys, and @@ -414,19 +416,19 @@ class CookiesMiddlewareTest(TestCase): "scrapy.downloadermiddlewares.cookies", "WARNING", "Invalid cookie found in request :" - " {'value': 'bar'} ('name' is missing)", + " {'value': 'bar', 'secure': False} ('name' is missing)", ), ( "scrapy.downloadermiddlewares.cookies", "WARNING", "Invalid cookie found in request :" - " {'name': 'foo'} ('value' is missing)", + " {'name': 'foo', 'secure': False} ('value' is missing)", ), ( "scrapy.downloadermiddlewares.cookies", "WARNING", "Invalid cookie found in request :" - " {'name': 'foo', 'value': None} ('value' is missing)", + " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)", ), ) self.assertCookieValEqual(req1.headers["Cookie"], "key=value1") @@ -732,3 +734,106 @@ class CookiesMiddlewareTest(TestCase): "co.uk", cookies=True, ) + + def _test_cookie_redirect_scheme_change( + self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3 + ): + """When a redirect causes the URL scheme to change from *from_scheme* + to *to_scheme*, while domain and port remain the same, and given a + cookie on the initial request with its secure attribute set to + *secure*, check if the cookie should be set on the Cookie header of the + initial request (*cookies1*), if it should be kept by the redirect + middleware (*cookies2*), and if it should be present on the Cookie + header in the redirected request (*cookie3*).""" + cookie_kwargs = {} + if secure is not UNSET: + cookie_kwargs["secure"] = secure + input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}] + + request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies) + self.mw.process_request(request1, self.spider) + cookies = request1.headers.get("Cookie") + self.assertEqual(cookies, b"a=b" if cookies1 else None) + + response = Response( + f"{from_scheme}://a.example", + headers={"Location": f"{to_scheme}://a.example"}, + status=301, + ) + self.assertEqual( + self.mw.process_response(request1, response, self.spider), + response, + ) + + request2 = self.redirect_middleware.process_response( + request1, + response, + self.spider, + ) + self.assertIsInstance(request2, Request) + cookies = request2.headers.get("Cookie") + self.assertEqual(cookies, b"a=b" if cookies2 else None) + + self.mw.process_request(request2, self.spider) + cookies = request2.headers.get("Cookie") + self.assertEqual(cookies, b"a=b" if cookies3 else None) + + def test_cookie_redirect_secure_undefined_downgrade(self): + self._test_cookie_redirect_scheme_change( + secure=UNSET, + from_scheme="https", + to_scheme="http", + cookies1=True, + cookies2=True, # xfail, due to a bug in the redirect middleware fixed elsewhere + cookies3=False, + ) + + def test_cookie_redirect_secure_undefined_upgrade(self): + self._test_cookie_redirect_scheme_change( + secure=UNSET, + from_scheme="http", + to_scheme="https", + cookies1=True, + cookies2=True, + cookies3=True, + ) + + def test_cookie_redirect_secure_false_downgrade(self): + self._test_cookie_redirect_scheme_change( + secure=False, + from_scheme="https", + to_scheme="http", + cookies1=True, + cookies2=True, # xfail, due to a bug in the redirect middleware fixed elsewhere + cookies3=True, + ) + + def test_cookie_redirect_secure_false_upgrade(self): + self._test_cookie_redirect_scheme_change( + secure=False, + from_scheme="http", + to_scheme="https", + cookies1=True, + cookies2=True, + cookies3=True, + ) + + def test_cookie_redirect_secure_true_downgrade(self): + self._test_cookie_redirect_scheme_change( + secure=True, + from_scheme="https", + to_scheme="http", + cookies1=True, + cookies2=True, # xfail, due to a bug in the redirect middleware fixed elsewhere + cookies3=False, + ) + + def test_cookie_redirect_secure_true_upgrade(self): + self._test_cookie_redirect_scheme_change( + secure=True, + from_scheme="http", + to_scheme="https", + cookies1=False, + cookies2=False, + cookies3=True, + ) From 631fc65fadb874629787ae5f7fdd876b9ec96a29 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 16 May 2024 18:42:09 +0400 Subject: [PATCH 1441/2083] Update expectations of cookies after redirects. (#6367) --- tests/test_downloadermiddleware_cookies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 1f7e6615c..5eccd396a 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -784,7 +784,7 @@ class CookiesMiddlewareTest(TestCase): from_scheme="https", to_scheme="http", cookies1=True, - cookies2=True, # xfail, due to a bug in the redirect middleware fixed elsewhere + cookies2=False, cookies3=False, ) @@ -804,7 +804,7 @@ class CookiesMiddlewareTest(TestCase): from_scheme="https", to_scheme="http", cookies1=True, - cookies2=True, # xfail, due to a bug in the redirect middleware fixed elsewhere + cookies2=False, cookies3=True, ) @@ -824,7 +824,7 @@ class CookiesMiddlewareTest(TestCase): from_scheme="https", to_scheme="http", cookies1=True, - cookies2=True, # xfail, due to a bug in the redirect middleware fixed elsewhere + cookies2=False, cookies3=False, ) From b99526b740890e63f1d05074b3e358a9ae59b77f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 19 May 2024 15:45:51 +0500 Subject: [PATCH 1442/2083] Full typing for scrapy/contracts. --- scrapy/contracts/__init__.py | 75 ++++++++++++++++++++++++------------ scrapy/contracts/default.py | 19 ++++----- 2 files changed, 60 insertions(+), 34 deletions(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index d46eb7c51..b300b8457 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -3,10 +3,23 @@ import sys from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import AsyncGenerator, Dict, Optional, Type -from unittest import TestCase +from typing import ( + Any, + AsyncGenerator, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + Type, +) +from unittest import TestCase, TestResult -from scrapy.http import Request +from twisted.python.failure import Failure + +from scrapy import Spider +from scrapy.http import Request, Response from scrapy.utils.python import get_spec from scrapy.utils.spider import iterate_spider_output @@ -15,18 +28,20 @@ class Contract: """Abstract class for contracts""" request_cls: Optional[Type[Request]] = None + name: str - def __init__(self, method, *args): + def __init__(self, method: Callable, *args: Any): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") self.testcase_post = _create_testcase(method, f"@{self.name} post-hook") - self.args = args + self.args: Tuple[Any, ...] = args - def add_pre_hook(self, request, results): + def add_pre_hook(self, request: Request, results: TestResult) -> Request: if hasattr(self, "pre_process"): cb = request.callback + assert cb is not None @wraps(cb) - def wrapper(response, **cb_kwargs): + def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]: try: results.startTest(self.testcase_pre) self.pre_process(response) @@ -49,12 +64,13 @@ class Contract: return request - def add_post_hook(self, request, results): + def add_post_hook(self, request: Request, results: TestResult) -> Request: if hasattr(self, "post_process"): cb = request.callback + assert cb is not None @wraps(cb) - def wrapper(response, **cb_kwargs): + def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") @@ -76,18 +92,18 @@ class Contract: return request - def adjust_request_args(self, args): + def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: return args class ContractsManager: - contracts: Dict[str, Contract] = {} + contracts: Dict[str, Type[Contract]] = {} - def __init__(self, contracts): + def __init__(self, contracts: Iterable[Type[Contract]]): for contract in contracts: self.contracts[contract.name] = contract - def tested_methods_from_spidercls(self, spidercls): + def tested_methods_from_spidercls(self, spidercls: Type[Spider]) -> List[str]: is_method = re.compile(r"^\s*@", re.MULTILINE).search methods = [] for key, value in getmembers(spidercls): @@ -96,21 +112,26 @@ class ContractsManager: return methods - def extract_contracts(self, method): - contracts = [] + def extract_contracts(self, method: Callable) -> List[Contract]: + contracts: List[Contract] = [] + assert method.__doc__ is not None for line in method.__doc__.split("\n"): line = line.strip() if line.startswith("@"): - name, args = re.match(r"@(\w+)\s*(.*)", line).groups() + m = re.match(r"@(\w+)\s*(.*)", line) + assert m is not None + name, args = m.groups() args = re.split(r"\s+", args) contracts.append(self.contracts[name](method, *args)) return contracts - def from_spider(self, spider, results): - requests = [] + def from_spider( + self, spider: Spider, results: TestResult + ) -> List[Optional[Request]]: + requests: List[Optional[Request]] = [] for method in self.tested_methods_from_spidercls(type(spider)): bound_method = spider.__getattribute__(method) try: @@ -121,7 +142,7 @@ class ContractsManager: return requests - def from_method(self, method, results): + def from_method(self, method: Callable, results: TestResult) -> Optional[Request]: contracts = self.extract_contracts(method) if contracts: request_cls = Request @@ -154,14 +175,18 @@ class ContractsManager: self._clean_req(request, method, results) return request + return None - def _clean_req(self, request, method, results): + def _clean_req( + self, request: Request, method: Callable, results: TestResult + ) -> None: """stop the request from returning objects and records any errors""" cb = request.callback + assert cb is not None @wraps(cb) - def cb_wrapper(response, **cb_kwargs): + def cb_wrapper(response: Response, **cb_kwargs: Any) -> None: try: output = cb(response, **cb_kwargs) output = list(iterate_spider_output(output)) @@ -169,7 +194,7 @@ class ContractsManager: case = _create_testcase(method, "callback") results.addError(case, sys.exc_info()) - def eb_wrapper(failure): + def eb_wrapper(failure: Failure) -> None: case = _create_testcase(method, "errback") exc_info = failure.type, failure.value, failure.getTracebackObject() results.addError(case, exc_info) @@ -178,11 +203,11 @@ class ContractsManager: request.errback = eb_wrapper -def _create_testcase(method, desc): - spider = method.__self__.name +def _create_testcase(method: Callable, desc: str) -> TestCase: + spider = method.__self__.name # type: ignore[attr-defined] class ContractTestCase(TestCase): - def __str__(_self): + def __str__(_self) -> str: return f"[{spider}] {method.__name__} ({desc})" name = f"{spider}_{method.__name__}" diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index eac702cef..71ca4168a 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,4 +1,5 @@ import json +from typing import Any, Callable, Dict, List, Optional from itemadapter import ItemAdapter, is_item @@ -15,7 +16,7 @@ class UrlContract(Contract): name = "url" - def adjust_request_args(self, args): + def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: args["url"] = self.args[0] return args @@ -29,7 +30,7 @@ class CallbackKeywordArgumentsContract(Contract): name = "cb_kwargs" - def adjust_request_args(self, args): + def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: args["cb_kwargs"] = json.loads(" ".join(self.args)) return args @@ -48,14 +49,14 @@ class ReturnsContract(Contract): """ name = "returns" - object_type_verifiers = { + object_type_verifiers: Dict[Optional[str], Callable[[Any], bool]] = { "request": lambda x: isinstance(x, Request), "requests": lambda x: isinstance(x, Request), "item": is_item, "items": is_item, } - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) if len(self.args) not in [1, 2, 3]: @@ -66,16 +67,16 @@ class ReturnsContract(Contract): self.obj_type_verifier = self.object_type_verifiers[self.obj_name] try: - self.min_bound = int(self.args[1]) + self.min_bound: float = int(self.args[1]) except IndexError: self.min_bound = 1 try: - self.max_bound = int(self.args[2]) + self.max_bound: float = int(self.args[2]) except IndexError: self.max_bound = float("inf") - def post_process(self, output): + def post_process(self, output: List[Any]) -> None: occurrences = 0 for x in output: if self.obj_type_verifier(x): @@ -85,7 +86,7 @@ class ReturnsContract(Contract): if not assertion: if self.min_bound == self.max_bound: - expected = self.min_bound + expected = str(self.min_bound) else: expected = f"{self.min_bound}..{self.max_bound}" @@ -101,7 +102,7 @@ class ScrapesContract(Contract): name = "scrapes" - def post_process(self, output): + def post_process(self, output: List[Any]) -> None: for x in output: if is_item(x): missing = [arg for arg in self.args if arg not in ItemAdapter(x)] From 534a66e9548142a71b118c82e328a08c6e0350b4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 22 May 2024 13:16:00 +0500 Subject: [PATCH 1443/2083] Bump 3.13 to beta1. --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 2 +- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 8 ++++---- .github/workflows/tests-windows.yml | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 4c0400cde..46fd15415 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.13.0-alpha.2" + - python-version: "3.13.0-beta.1" env: TOXENV: pylint - python-version: 3.8 @@ -24,7 +24,7 @@ jobs: - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs - - python-version: "3.13.0-alpha.2" + - python-version: "3.13.0-beta.1" env: TOXENV: twinecheck diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 456c0ffdd..ad94ae9cd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: "3.13.0-alpha.2" + python-version: "3.13.0-beta.1" - run: | pip install --upgrade build twine python -m build diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 6b110b5d7..18890239c 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13.0-alpha.2"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13.0-beta.1"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index fd08247e4..121b5271a 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -24,10 +24,10 @@ jobs: - python-version: "3.12" env: TOXENV: py - - python-version: "3.13.0-alpha.2" + - python-version: "3.13.0-beta.1" env: TOXENV: py - - python-version: "3.13.0-alpha.2" + - python-version: "3.13.0-beta.1" env: TOXENV: asyncio - python-version: pypy3.9 @@ -54,10 +54,10 @@ jobs: env: TOXENV: botocore-pinned - - python-version: "3.13.0-alpha.2" + - python-version: "3.13.0-beta.1" env: TOXENV: extra-deps - - python-version: "3.13.0-alpha.2" + - python-version: "3.13.0-beta.1" env: TOXENV: botocore diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index be082393e..e23c3e67d 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -27,10 +27,10 @@ jobs: - python-version: "3.12" env: TOXENV: py - - python-version: "3.13.0-alpha.2" + - python-version: "3.13.0-beta.1" env: TOXENV: py - - python-version: "3.13.0-alpha.2" + - python-version: "3.13.0-beta.1" env: TOXENV: asyncio From b6d3d9076fe4c089a278e0a18bd05f1c0796418f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 22 May 2024 13:20:48 +0500 Subject: [PATCH 1444/2083] Help with building lxml on 3.13beta1. --- .github/workflows/tests-ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 121b5271a..7ea58b7df 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -70,7 +70,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') + if: contains(matrix.python-version, 'pypy') || contains(matrix.python-version, 'beta') || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev From 04bc1e6e2a51e874cc6d676ccb111b0793e2776e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 22 May 2024 13:24:35 +0500 Subject: [PATCH 1445/2083] Skip zstandard on 3.13. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index cde4243f3..37a27ae83 100644 --- a/tox.ini +++ b/tox.ini @@ -155,7 +155,7 @@ deps = bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests - zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + zstandard; implementation_name != 'pypy' and python_version < '3.13' # optional for HTTP compress downloader middleware tests ipython [testenv:extra-deps-pinned] From e676cd3ce0d488f56498b766912725066bcee4d9 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 22 May 2024 07:55:53 -0300 Subject: [PATCH 1446/2083] docs: Remove top-level reactor imports from CrawlerProces/CrawlerRunner examples --- docs/topics/practices.rst | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cd359b147..7731180fe 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -92,7 +92,6 @@ reactor after ``MySpider`` has finished running. .. code-block:: python - from twisted.internet import reactor import scrapy from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging @@ -107,6 +106,33 @@ reactor after ``MySpider`` has finished running. runner = CrawlerRunner() d = runner.crawl(MySpider) + from twisted.internet import reactor + + d.addBoth(lambda _: reactor.stop()) + reactor.run() # the script will block here until the crawling is finished + +Same example but using a non-default reactor, is only necessary call ``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. + +.. code-block:: python + + import scrapy + from scrapy.crawler import CrawlerRunner + from scrapy.utils.log import configure_logging + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + from scrapy.utils.reactor import install_reactor + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + runner = CrawlerRunner() + d = runner.crawl(MySpider) + from twisted.internet import reactor + d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished @@ -151,7 +177,6 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: .. code-block:: python import scrapy - from twisted.internet import reactor from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings @@ -173,6 +198,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() + from twisted.internet import reactor + d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until all crawling jobs are finished @@ -181,7 +208,7 @@ Same example but running the spiders sequentially by chaining the deferreds: .. code-block:: python - from twisted.internet import reactor, defer + from twisted.internet import defer from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings @@ -209,6 +236,8 @@ Same example but running the spiders sequentially by chaining the deferreds: reactor.stop() + from twisted.internet import reactor + crawl() reactor.run() # the script will block here until the last crawl call is finished From 8210fae25a9d812447df617155001b9861e0d834 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 22 May 2024 18:50:50 -0300 Subject: [PATCH 1447/2083] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 7731180fe..710be7aa2 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -111,7 +111,9 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished -Same example but using a non-default reactor, is only necessary call ``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. +Same example but using a non-default reactor, it's only necessary call +``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` + already does this automatically. .. code-block:: python From dc6a495fee41949d50178b9e46d6f41e83425ca2 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 22 May 2024 18:51:02 -0300 Subject: [PATCH 1448/2083] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 710be7aa2..cec098012 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -106,6 +106,7 @@ reactor after ``MySpider`` has finished running. runner = CrawlerRunner() d = runner.crawl(MySpider) + from twisted.internet import reactor d.addBoth(lambda _: reactor.stop()) From 3f66b66e3f645393dbb263a1ec7ab04bdabd74b4 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 22 May 2024 22:01:55 -0300 Subject: [PATCH 1449/2083] fix: checks --- docs/topics/practices.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cec098012..aa81ceea5 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -113,8 +113,7 @@ reactor after ``MySpider`` has finished running. reactor.run() # the script will block here until the crawling is finished Same example but using a non-default reactor, it's only necessary call -``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` - already does this automatically. +``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. .. code-block:: python From e143dc795228424fa98cb40e17b9993617ae61ae Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 22 May 2024 22:26:31 -0300 Subject: [PATCH 1450/2083] Update tests-macos.yml --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 252176464..95016146e 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,4 +1,4 @@ -name: macOS +name: macOS. on: [push, pull_request] concurrency: From 9d5a0d287b69a69fe34cbe3130438fb36f1f3441 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 22 May 2024 22:27:07 -0300 Subject: [PATCH 1451/2083] Retrigger CI --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 95016146e..252176464 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,4 +1,4 @@ -name: macOS. +name: macOS on: [push, pull_request] concurrency: From 17e623cf0cfb5c695c43ceb069026d44cb28ca21 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 23 May 2024 07:00:24 -0300 Subject: [PATCH 1452/2083] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index aa81ceea5..64b3b6e81 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -128,6 +128,7 @@ Same example but using a non-default reactor, it's only necessary call configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + from scrapy.utils.reactor import install_reactor install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") From 8ec67ca230a69effb2e0442fb2a6c06cd6c92adf Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 23 May 2024 07:00:35 -0300 Subject: [PATCH 1453/2083] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 64b3b6e81..ee484e63f 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -134,6 +134,7 @@ Same example but using a non-default reactor, it's only necessary call install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") runner = CrawlerRunner() d = runner.crawl(MySpider) + from twisted.internet import reactor d.addBoth(lambda _: reactor.stop()) From 62c89aaf056687091235bb846ac565f7c801c359 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 23 May 2024 07:00:45 -0300 Subject: [PATCH 1454/2083] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index ee484e63f..1500011e7 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -202,6 +202,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() + from twisted.internet import reactor d.addBoth(lambda _: reactor.stop()) From 2facdd4fb08ec3edaf1752047dd86d5b565621a1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Sun, 26 May 2024 19:55:54 -0300 Subject: [PATCH 1455/2083] Add change reactor test to CrawlerRunner --- .flake8 | 1 + scrapy/crawler.py | 2 ++ tests/CrawlerRunner/change_reactor.py | 31 +++++++++++++++++++++++++++ tests/test_crawler.py | 8 +++++++ 4 files changed, 42 insertions(+) create mode 100644 tests/CrawlerRunner/change_reactor.py diff --git a/.flake8 b/.flake8 index 62ccad9cf..0e43b9b56 100644 --- a/.flake8 +++ b/.flake8 @@ -9,6 +9,7 @@ exclude = per-file-ignores = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file + tests/CrawlerRunner/change_reactor.py:E402 # F401: Module imported but unused scrapy/__init__.py:E402 scrapy/core/downloader/handlers/http.py:F401 diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ccfe78891..4fe5987a7 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -129,6 +129,8 @@ class Crawler: if is_asyncio_reactor_installed() and event_loop: verify_installed_asyncio_event_loop(event_loop) + log_reactor_info() + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py new file mode 100644 index 000000000..b20aa0c7c --- /dev/null +++ b/tests/CrawlerRunner/change_reactor.py @@ -0,0 +1,31 @@ +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + def start_requests(self): + return [] + + +configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"}) + + +from scrapy.utils.reactor import install_reactor + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +runner = CrawlerRunner() + +d = runner.crawl(NoRequestsSpider) + +from twisted.internet import reactor + +d.addBoth(callback=lambda _: reactor.stop()) +reactor.run() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 989208694..791ea1faa 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -926,3 +926,11 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("INFO: Host: not.a.real.domain", log) self.assertIn("INFO: Type: ", log) self.assertIn("INFO: IP address: 127.0.0.1", log) + + def test_change_default_reactor(self): + log = self.run_script("change_reactor.py") + self.assertIn( + "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", + log, + ) + self.assertIn("DEBUG: Using asyncio event loop", log) From 6cd085785028d97393f26e6fee22e6c03e5c90a8 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Sun, 26 May 2024 19:57:16 -0300 Subject: [PATCH 1456/2083] Move path --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 0e43b9b56..cf1a96476 100644 --- a/.flake8 +++ b/.flake8 @@ -9,7 +9,6 @@ exclude = per-file-ignores = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file - tests/CrawlerRunner/change_reactor.py:E402 # F401: Module imported but unused scrapy/__init__.py:E402 scrapy/core/downloader/handlers/http.py:F401 @@ -17,6 +16,7 @@ per-file-ignores = scrapy/linkextractors/__init__.py:E402,F401 scrapy/selector/__init__.py:F401 scrapy/spiders/__init__.py:E402,F401 + tests/CrawlerRunner/change_reactor.py:E402 # Issues pending a review: scrapy/utils/url.py:F403,F405 From 9ba4dd311dd9d2a5341ee9c0c6d1b50eb44ac406 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 May 2024 12:27:49 +0400 Subject: [PATCH 1457/2083] Install typing stubs for boto3 and botocore. (#6370) --- scrapy/core/downloader/handlers/s3.py | 3 ++- scrapy/extensions/feedexport.py | 11 +++++++---- tox.ini | 13 ++++++++----- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 9a0811a50..1a3d36f45 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -59,7 +59,8 @@ class S3DownloadHandler: assert aws_access_key_id is not None assert aws_secret_access_key is not None SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"] - self._signer = SignerCls( + # botocore.auth.BaseSigner doesn't have an __init__() with args, only subclasses do + self._signer = SignerCls( # type: ignore[call-arg] botocore.credentials.Credentials( aws_access_key_id, aws_secret_access_key, aws_session_token ) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 97f39afe7..3c2bb5593 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -238,13 +238,16 @@ class S3FeedStorage(BlockingFeedStorage): self.acl: Optional[str] = acl self.endpoint_url: Optional[str] = endpoint_url self.region_name: Optional[str] = region_name + # It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client, + # there seems to be no good way to infer it statically. + self.s3_client: Any if IS_BOTO3_AVAILABLE: import boto3.session - session = boto3.session.Session() + boto3_session = boto3.session.Session() - self.s3_client = session.client( + self.s3_client = boto3_session.client( "s3", aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, @@ -261,9 +264,9 @@ class S3FeedStorage(BlockingFeedStorage): import botocore.session - session = botocore.session.get_session() + botocore_session = botocore.session.get_session() - self.s3_client = session.create_client( + self.s3_client = botocore_session.create_client( "s3", aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, diff --git a/tox.ini b/tox.ini index cde4243f3..5a5e80496 100644 --- a/tox.ini +++ b/tox.ini @@ -48,12 +48,15 @@ basepython = python3 deps = mypy==1.10.0 typing-extensions==4.11.0 - types-attrs==19.1.0 types-lxml==2024.4.14 - types-Pillow==10.2.0.20240423 - types-Pygments==2.17.0.20240310 - types-pyOpenSSL==24.0.0.20240417 - types-setuptools==69.5.0.20240423 + types-Pygments==2.18.0.20240506 + types-pyOpenSSL==24.1.0.20240425 + types-setuptools==69.5.0.20240518 + botocore-stubs==1.34.94 + boto3-stubs[s3]==1.34.108 + attrs >= 18.2.0 + Pillow >= 10.3.0 + pytest >= 8.2.0 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands = From 986d1ee1dd5b2efba0f787af8ee510450b4af3b4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 May 2024 12:37:19 +0400 Subject: [PATCH 1458/2083] Move CI from the decommissioned macos-11 to macos-latest. (#6372) --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 252176464..a297f494c 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ concurrency: jobs: tests: - runs-on: macos-11 + runs-on: macos-latest strategy: fail-fast: false matrix: From cadb0dd707fc54670cb0eab06f0af65dcaa99354 Mon Sep 17 00:00:00 2001 From: Sanchay Kumar <51812506+kumar-sanchay@users.noreply.github.com> Date: Tue, 28 May 2024 14:12:58 +0530 Subject: [PATCH 1459/2083] Fix overridable methods in MediaPipeline (#6368) --- scrapy/pipelines/media.py | 26 +++--- tests/test_pipeline_media.py | 173 ++++++++++++----------------------- 2 files changed, 73 insertions(+), 126 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 5f6c5cb07..25e00b0ea 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -2,6 +2,7 @@ from __future__ import annotations import functools import logging +from abc import ABC, abstractmethod from collections import defaultdict from typing import TYPE_CHECKING @@ -27,7 +28,7 @@ def _DUMMY_CALLBACK(response): return response -class MediaPipeline: +class MediaPipeline(ABC): LOG_FAILED_RESULTS = True class SpiderInfo: @@ -55,14 +56,6 @@ class MediaPipeline: self.handle_httpstatus_list = SequenceExclude(range(300, 400)) def _key_for_pipe(self, key, base_class_name=None, settings=None): - """ - >>> MediaPipeline()._key_for_pipe("IMAGES") - 'IMAGES' - >>> class MyPipe(MediaPipeline): - ... pass - >>> MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline") - 'MYPIPE_IMAGES' - """ class_name = self.__class__.__name__ formatted_key = f"{class_name.upper()}_{key}" if ( @@ -192,21 +185,25 @@ class MediaPipeline: defer_result(result).chainDeferred(wad) # Overridable Interface + @abstractmethod def media_to_download(self, request, info, *, item=None): """Check request before starting download""" - pass + raise NotImplementedError() + @abstractmethod def get_media_requests(self, item, info): """Returns the media requests to download""" - pass + raise NotImplementedError() + @abstractmethod def media_downloaded(self, response, request, info, *, item=None): """Handler for success downloads""" - return response + raise NotImplementedError() + @abstractmethod def media_failed(self, failure, request, info): """Handler for failed downloads""" - return failure + raise NotImplementedError() def item_completed(self, results, item, info): """Called per item when all media requests has been processed""" @@ -221,6 +218,7 @@ class MediaPipeline: ) return item + @abstractmethod def file_path(self, request, response=None, info=None, *, item=None): """Returns the path where downloaded media should be stored""" - pass + raise NotImplementedError() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d4dde4a40..763453551 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,4 +1,3 @@ -import io from typing import Optional from testfixtures import LogCapture @@ -11,7 +10,6 @@ from scrapy import signals from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException -from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.spiders import Spider @@ -35,8 +33,26 @@ def _mocked_download_func(request, info): return response() if callable(response) else response +class UserDefinedPipeline(MediaPipeline): + + def media_to_download(self, request, info, *, item=None): + pass + + def get_media_requests(self, item, info): + pass + + def media_downloaded(self, response, request, info, *, item=None): + return {} + + def media_failed(self, failure, request, info): + return failure + + def file_path(self, request, response=None, info=None, *, item=None): + return "" + + class BaseMediaPipelineTestCase(unittest.TestCase): - pipeline_class = MediaPipeline + pipeline_class = UserDefinedPipeline settings = None def setUp(self): @@ -54,54 +70,6 @@ class BaseMediaPipelineTestCase(unittest.TestCase): if not name.startswith("_"): disconnect_all(signal) - def test_default_media_to_download(self): - request = Request("http://url") - assert self.pipe.media_to_download(request, self.info) is None - - def test_default_get_media_requests(self): - item = {"name": "name"} - assert self.pipe.get_media_requests(item, self.info) is None - - def test_default_media_downloaded(self): - request = Request("http://url") - response = Response("http://url", body=b"") - assert self.pipe.media_downloaded(response, request, self.info) is response - - def test_default_media_failed(self): - request = Request("http://url") - fail = Failure(Exception()) - assert self.pipe.media_failed(fail, request, self.info) is fail - - def test_default_item_completed(self): - item = {"name": "name"} - assert self.pipe.item_completed([], item, self.info) is item - - # Check that failures are logged by default - fail = Failure(Exception()) - results = [(True, 1), (False, fail)] - - with LogCapture() as log: - new_item = self.pipe.item_completed(results, item, self.info) - - assert new_item is item - assert len(log.records) == 1 - record = log.records[0] - assert record.levelname == "ERROR" - self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) - - # disable failure logging and check again - self.pipe.LOG_FAILED_RESULTS = False - with LogCapture() as log: - new_item = self.pipe.item_completed(results, item, self.info) - assert new_item is item - assert len(log.records) == 0 - - @inlineCallbacks - def test_default_process_item(self): - item = {"name": "name"} - new_item = yield self.pipe.process_item(item, self.spider) - assert new_item is item - def test_modify_media_request(self): request = Request("http://url") self.pipe._modify_media_request(request) @@ -175,8 +143,38 @@ class BaseMediaPipelineTestCase(unittest.TestCase): context = getattr(info.downloaded[fp].value, "__context__", None) self.assertIsNone(context) + def test_default_item_completed(self): + item = {"name": "name"} + assert self.pipe.item_completed([], item, self.info) is item -class MockedMediaPipeline(MediaPipeline): + # Check that failures are logged by default + fail = Failure(Exception()) + results = [(True, 1), (False, fail)] + + with LogCapture() as log: + new_item = self.pipe.item_completed(results, item, self.info) + + assert new_item is item + assert len(log.records) == 1 + record = log.records[0] + assert record.levelname == "ERROR" + self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) + + # disable failure logging and check again + self.pipe.LOG_FAILED_RESULTS = False + with LogCapture() as log: + new_item = self.pipe.item_completed(results, item, self.info) + assert new_item is item + assert len(log.records) == 0 + + @inlineCallbacks + def test_default_process_item(self): + item = {"name": "name"} + new_item = yield self.pipe.process_item(item, self.spider) + assert new_item is item + + +class MockedMediaPipeline(UserDefinedPipeline): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._mockcalled = [] @@ -232,7 +230,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ) item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, rsp)]) + self.assertEqual(new_item["results"], [(True, {})]) self.assertEqual( self.pipe._mockcalled, [ @@ -277,7 +275,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req2 = Request("http://url2", meta={"response": fail}) item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)]) + self.assertEqual(new_item["results"], [(True, {}), (False, fail)]) m = self.pipe._mockcalled # only once self.assertEqual(m[0], "get_media_requests") # first hook called @@ -315,7 +313,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = {"requests": req1} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) - self.assertEqual(new_item["results"], [(True, rsp1)]) + self.assertEqual(new_item["results"], [(True, {})]) # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same req2 = Request( @@ -325,7 +323,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) - self.assertEqual(new_item["results"], [(True, rsp1)]) + self.assertEqual(new_item["results"], [(True, {})]) @inlineCallbacks def test_results_are_cached_for_requests_of_single_item(self): @@ -337,7 +335,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) - self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) + self.assertEqual(new_item["results"], [(True, {}), (True, {})]) @inlineCallbacks def test_wait_if_request_is_downloading(self): @@ -363,7 +361,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req2 = Request(req1.url, meta={"response": rsp2_func}) item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) + self.assertEqual(new_item["results"], [(True, {}), (True, {})]) @inlineCallbacks def test_use_media_to_download_result(self): @@ -376,57 +374,15 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ["get_media_requests", "media_to_download", "item_completed"], ) - -class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._mockcalled = [] - - def get_media_requests(self, item, info): - item_url = item["image_urls"][0] - output_img = io.BytesIO() - img = Image.new("RGB", (60, 30), color="red") - img.save(output_img, format="JPEG") - return Request( - item_url, - meta={ - "response": Response(item_url, status=200, body=output_img.getvalue()) - }, + def test_key_for_pipe(self): + self.assertEqual( + self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline"), + "MOCKEDMEDIAPIPELINE_IMAGES", ) - def inc_stats(self, *args, **kwargs): - return True - - def media_to_download(self, request, info): - self._mockcalled.append("media_to_download") - return super().media_to_download(request, info) - - def media_downloaded(self, response, request, info): - self._mockcalled.append("media_downloaded") - return super().media_downloaded(response, request, info) - - def file_downloaded(self, response, request, info): - self._mockcalled.append("file_downloaded") - return super().file_downloaded(response, request, info) - - def file_path(self, request, response=None, info=None): - self._mockcalled.append("file_path") - return super().file_path(request, response, info) - - def thumb_path(self, request, thumb_id, response=None, info=None): - self._mockcalled.append("thumb_path") - return super().thumb_path(request, thumb_id, response, info) - - def get_images(self, response, request, info): - self._mockcalled.append("get_images") - return super().get_images(response, request, info) - - def image_downloaded(self, response, request, info): - self._mockcalled.append("image_downloaded") - return super().image_downloaded(response, request, info) - class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): + def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(settings=Settings(settings)) request = Request("http://url") @@ -452,18 +408,11 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): else: self.assertNotIn(status, request.meta["handle_httpstatus_list"]) - def test_standard_setting(self): - self._assert_request_no3xx(MediaPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) - def test_subclass_standard_setting(self): - class UserDefinedPipeline(MediaPipeline): - pass self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) def test_subclass_specific_setting(self): - class UserDefinedPipeline(MediaPipeline): - pass self._assert_request_no3xx( UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True} From d9b5538e3c758d9835fd97a0a60ab2dff810e984 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 May 2024 14:04:58 +0500 Subject: [PATCH 1460/2083] Bump twinecheck deps. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 37a27ae83..615164714 100644 --- a/tox.ini +++ b/tox.ini @@ -86,8 +86,8 @@ commands = [testenv:twinecheck] basepython = python3 deps = - twine==4.0.2 - build==1.0.3 + twine==5.1.0 + build==1.2.1 commands = python -m build --sdist twine check dist/* From 42347de53f8704c835cd0c25290245d73355d6f5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 May 2024 14:30:35 +0500 Subject: [PATCH 1461/2083] Install pre-release cffi on 3.13. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 615164714..8e38112e5 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,7 @@ deps = sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures pywin32; sys_platform == "win32" + cffi >= 1.17.0rc1; python_version >= '3.13' [testenv] deps = From e6e9fd75db251c274df37b0493e0305473fa6536 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 May 2024 14:06:04 +0500 Subject: [PATCH 1462/2083] Skip mitmproxy and Pillow on 3.13. --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 8e38112e5..737baec84 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,8 @@ deps = {[test-requirements]deps} # mitmproxy does not support PyPy - mitmproxy; implementation_name != 'pypy' + # mitmproxy requires zstandard which is not yet available on 3.13 + mitmproxy; implementation_name != 'pypy' and python_version < '3.13' # https://github.com/pallets/werkzeug/pull/2768 breaks flask, required by # mitmproxy. werkzeug < 3; python_version < '3.9' and implementation_name != 'pypy' @@ -150,7 +151,7 @@ deps = # restrictions in their deps, so we need to pin old markupsafe here too. markupsafe < 2.1.0 robotexclusionrulesparser - Pillow + Pillow; python_version < '3.13' Twisted[http2] uvloop; platform_system != "Windows" bpython # optional for shell wrapper tests From 5755e224d5f29b1d6b8b9caca1e208e84ae00822 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 May 2024 14:50:09 +0500 Subject: [PATCH 1463/2083] Help with building lxml on 3.13beta1 for checks too. --- .github/workflows/checks.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 46fd15415..d60e259ba 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -36,6 +36,12 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install system libraries + if: contains(matrix.python-version, 'beta') + run: | + sudo apt-get update + sudo apt-get install libxml2-dev libxslt-dev + - name: Run check env: ${{ matrix.env }} run: | From 1be8aee09c3fe532d42be9b22cae914f29c11f2e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 28 May 2024 14:56:23 +0500 Subject: [PATCH 1464/2083] Skip uvloop and bpython on 3.13. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 737baec84..5c2f34c6e 100644 --- a/tox.ini +++ b/tox.ini @@ -153,8 +153,8 @@ deps = robotexclusionrulesparser Pillow; python_version < '3.13' Twisted[http2] - uvloop; platform_system != "Windows" - bpython # optional for shell wrapper tests + uvloop; platform_system != "Windows" and python_version < '3.13' + bpython; python_version < '3.13' # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' and python_version < '3.13' # optional for HTTP compress downloader middleware tests From 0d58af86971ba54bfc1feffa88fc564ffba650f4 Mon Sep 17 00:00:00 2001 From: Fabian Schneebauer Date: Wed, 29 May 2024 10:59:32 +0200 Subject: [PATCH 1465/2083] Add support for multiple referer policy tokens. --- scrapy/spidermiddlewares/referer.py | 20 ++++++----- tests/test_spidermiddleware_referer.py | 47 ++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a0b6851e5..7706c8c15 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -323,15 +323,17 @@ def _load_policy_class( try: return cast(Type[ReferrerPolicy], load_object(policy)) except ValueError: - try: - return _policy_classes[policy.lower()] - except KeyError: - msg = f"Could not load referrer policy {policy!r}" - if not warning_only: - raise RuntimeError(msg) - else: - warnings.warn(msg, RuntimeWarning) - return None + tokens = [token.strip() for token in policy.lower().split(",")] + for token in tokens[::-1]: + if token in _policy_classes: + return _policy_classes[token] + + msg = f"Could not load referrer policy {policy!r}" + if not warning_only: + raise RuntimeError(msg) + else: + warnings.warn(msg, RuntimeWarning) + return None class RefererMiddleware: diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index afffa87fb..5797edfbd 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -884,6 +884,53 @@ class TestSettingsPolicyByName(TestCase): with self.assertRaises(RuntimeError): RefererMiddleware(settings) + def test_multiple_policy_tokens(self): + # test parsing without space(s) after the comma + settings1 = Settings( + { + "REFERRER_POLICY": ",".join( + [ + "some-custom-unknown-policy", + POLICY_SAME_ORIGIN, + POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, + "another-custom-unknown-policy", + ] + ) + } + ) + mw1 = RefererMiddleware(settings1) + self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy) + + # test parsing with space(s) after the comma + settings2 = Settings( + { + "REFERRER_POLICY": ", ".join( + [ + POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, + "another-custom-unknown-policy", + POLICY_UNSAFE_URL, + ] + ) + } + ) + mw2 = RefererMiddleware(settings2) + self.assertEqual(mw2.default_policy, UnsafeUrlPolicy) + + def test_multiple_policy_tokens_all_invalid(self): + settings = Settings( + { + "REFERRER_POLICY": ",".join( + [ + "some-custom-unknown-policy", + "another-custom-unknown-policy", + "yet-another-custom-unknown-policy", + ] + ) + } + ) + with self.assertRaises(RuntimeError): + RefererMiddleware(settings) + class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"} From 62a028b99dc73b8ddddd37f986780a5a070f2938 Mon Sep 17 00:00:00 2001 From: Fabian Schneebauer <67049088+0xdeb@users.noreply.github.com> Date: Wed, 29 May 2024 13:19:27 +0200 Subject: [PATCH 1466/2083] Add spec link to scrapy/spidermiddlewares/referer.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/spidermiddlewares/referer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 7706c8c15..8af0bdf5b 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -324,6 +324,7 @@ def _load_policy_class( return cast(Type[ReferrerPolicy], load_object(policy)) except ValueError: tokens = [token.strip() for token in policy.lower().split(",")] + # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header for token in tokens[::-1]: if token in _policy_classes: return _policy_classes[token] From b4293e8f9efac5046f92e4ebfd744be443b858b0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 10:50:36 +0400 Subject: [PATCH 1467/2083] Misc typing improvements. (#6384) --- scrapy/core/http2/agent.py | 4 +-- scrapy/core/http2/protocol.py | 8 +++-- scrapy/core/http2/stream.py | 4 +-- .../downloadermiddlewares/httpcompression.py | 14 +++++---- scrapy/downloadermiddlewares/offsite.py | 30 ++++++++++++------- scrapy/loader/__init__.py | 12 +++++++- scrapy/utils/benchserver.py | 12 ++++---- scrapy/utils/curl.py | 23 +++++++++----- scrapy/utils/datatypes.py | 2 +- scrapy/utils/request.py | 2 +- scrapy/utils/response.py | 2 +- scrapy/utils/testsite.py | 4 +-- 12 files changed, 78 insertions(+), 39 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 215ea9716..935af2214 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -119,7 +119,7 @@ class H2Agent: self._reactor, self._context_factory, connect_timeout, bind_address ) - def get_endpoint(self, uri: URI): + def get_endpoint(self, uri: URI) -> HostnameEndpoint: return self.endpoint_factory.endpointForURI(uri) def get_key(self, uri: URI) -> Tuple: @@ -161,7 +161,7 @@ class ScrapyProxyH2Agent(H2Agent): ) self._proxy_uri = proxy_uri - def get_endpoint(self, uri: URI): + def get_endpoint(self, uri: URI) -> HostnameEndpoint: return self.endpoint_factory.endpointForURI(self._proxy_uri) def get_key(self, uri: URI) -> Tuple: diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index bc8da50d7..8898b8118 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -22,7 +22,11 @@ from h2.events import ( from h2.exceptions import FrameTooLargeError, H2Error from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory +from twisted.internet.interfaces import ( + IAddress, + IHandshakeListener, + IProtocolNegotiationFactory, +) from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin @@ -431,7 +435,7 @@ class H2ClientFactory(Factory): self.settings = settings self.conn_lost_deferred = conn_lost_deferred - def buildProtocol(self, addr) -> H2ClientProtocol: + def buildProtocol(self, addr: IAddress) -> H2ClientProtocol: return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) def acceptableProtocols(self) -> List[bytes]: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 4132fc385..224691078 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,7 +1,7 @@ import logging from enum import Enum from io import BytesIO -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -142,7 +142,7 @@ class Stream: "headers": Headers({}), } - def _cancel(_) -> None: + def _cancel(_: Any) -> None: # Close this stream as gracefully as possible # If the associated request is initiated we reset this stream # else we directly call close() method diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 0e5e215ac..8e170a1c7 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -3,7 +3,7 @@ from __future__ import annotations import warnings from itertools import chain from logging import getLogger -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Tuple, Union from scrapy import Request, Spider, signals from scrapy.crawler import Crawler @@ -149,20 +149,24 @@ class HttpCompressionMiddleware: return response - def _handle_encoding(self, body, content_encoding, max_size): + def _handle_encoding( + self, body: bytes, content_encoding: List[bytes], max_size: int + ) -> Tuple[bytes, List[bytes]]: to_decode, to_keep = self._split_encodings(content_encoding) for encoding in to_decode: body = self._decode(body, encoding, max_size) return body, to_keep - def _split_encodings(self, content_encoding): - to_keep = [ + def _split_encodings( + self, content_encoding: List[bytes] + ) -> Tuple[List[bytes], List[bytes]]: + to_keep: List[bytes] = [ encoding.strip().lower() for encoding in chain.from_iterable( encodings.split(b",") for encodings in content_encoding ) ] - to_decode = [] + to_decode: List[bytes] = [] while to_keep: encoding = to_keep.pop() if encoding not in ACCEPTED_ENCODINGS: diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 1e5026925..bd8dbe329 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -1,33 +1,43 @@ +from __future__ import annotations + import logging import re import warnings +from typing import TYPE_CHECKING, Set -from scrapy import signals +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest +from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) class OffsiteMiddleware: @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled) return o - def __init__(self, stats): + def __init__(self, stats: StatsCollector): self.stats = stats - self.domains_seen = set() + self.domains_seen: Set[str] = set() - def spider_opened(self, spider): - self.host_regex = self.get_host_regex(spider) + def spider_opened(self, spider: Spider) -> None: + self.host_regex: re.Pattern[str] = self.get_host_regex(spider) - def request_scheduled(self, request, spider): + def request_scheduled(self, request: Request, spider: Spider) -> None: self.process_request(request, spider) - def process_request(self, request, spider): + def process_request(self, request: Request, spider: Spider) -> None: if request.dont_filter or self.should_follow(request, spider): return None domain = urlparse_cached(request).hostname @@ -42,13 +52,13 @@ class OffsiteMiddleware: self.stats.inc_value("offsite/filtered", spider=spider) raise IgnoreRequest - def should_follow(self, request, spider): + def should_follow(self, request: Request, spider: Spider) -> bool: regex = self.host_regex # hostname can be None for wrong urls (like javascript links) host = urlparse_cached(request).hostname or "" return bool(regex.search(host)) - def get_host_regex(self, spider): + def get_host_regex(self, spider: Spider) -> re.Pattern[str]: """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, "allowed_domains", None) if not allowed_domains: diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 529fa279e..db0b4820f 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -4,8 +4,11 @@ Item Loader See documentation in docs/topics/loaders.rst """ +from typing import Any, Optional + import itemloaders +from scrapy.http import TextResponse from scrapy.item import Item from scrapy.selector import Selector @@ -82,7 +85,14 @@ class ItemLoader(itemloaders.ItemLoader): default_item_class: type = Item default_selector_class = Selector - def __init__(self, item=None, selector=None, response=None, parent=None, **context): + def __init__( + self, + item: Any = None, + selector: Optional[Selector] = None, + response: Optional[TextResponse] = None, + parent: Optional[itemloaders.ItemLoader] = None, + **context: Any + ): if selector is None and response is not None: try: selector = self.default_selector_class(response) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index f6f704d4b..e9ea51aa1 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -1,21 +1,23 @@ import random +from typing import Any from urllib.parse import urlencode from twisted.web.resource import Resource -from twisted.web.server import Site +from twisted.web.server import Request, Site class Root(Resource): isLeaf = True - def getChild(self, name, request): + def getChild(self, name: str, request: Request) -> Resource: return self - def render(self, request): + def render(self, request: Request) -> bytes: total = _getarg(request, b"total", 100, int) show = _getarg(request, b"show", 10, int) nlist = [random.randint(1, total) for _ in range(show)] # nosec request.write(b"") + assert request.args is not None args = request.args.copy() for nl in nlist: args["n"] = nl @@ -27,7 +29,7 @@ class Root(Resource): return b"" -def _getarg(request, name, default=None, type=str): +def _getarg(request, name: bytes, default: Any = None, type=str): return type(request.args[name][0]) if name in request.args else default @@ -38,7 +40,7 @@ if __name__ == "__main__": factory = Site(root) httpPort = reactor.listenTCP(8998, Site(root)) - def _print_listening(): + def _print_listening() -> None: httpHost = httpPort.getHost() print(f"Bench server at http://{httpHost.host}:{httpHost.port}") diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index f5dbbd64e..c10e48511 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -2,13 +2,20 @@ import argparse import warnings from http.cookies import SimpleCookie from shlex import split +from typing import Any, Dict, List, NoReturn, Optional, Sequence, Tuple, Union from urllib.parse import urlparse from w3lib.http import basic_auth_header class DataAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: Union[str, Sequence[Any], None], + option_string: Optional[str] = None, + ) -> None: value = str(values) if value.startswith("$"): value = value[1:] @@ -16,7 +23,7 @@ class DataAction(argparse.Action): class CurlParser(argparse.ArgumentParser): - def error(self, message): + def error(self, message: str) -> NoReturn: error_msg = f"There was an error parsing the curl command: {message}" raise ValueError(error_msg) @@ -42,9 +49,11 @@ for argument in safe_to_ignore_arguments: curl_parser.add_argument(*argument, action="store_true") -def _parse_headers_and_cookies(parsed_args): - headers = [] - cookies = {} +def _parse_headers_and_cookies( + parsed_args: argparse.Namespace, +) -> Tuple[List[Tuple[str, bytes]], Dict[str, str]]: + headers: List[Tuple[str, bytes]] = [] + cookies: Dict[str, str] = {} for header in parsed_args.headers or (): name, val = header.split(":", 1) name = name.strip() @@ -64,7 +73,7 @@ def _parse_headers_and_cookies(parsed_args): def curl_to_request_kwargs( curl_command: str, ignore_unknown_options: bool = True -) -> dict: +) -> Dict[str, Any]: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command @@ -98,7 +107,7 @@ def curl_to_request_kwargs( method = parsed_args.method or "GET" - result = {"method": method.upper(), "url": url} + result: Dict[str, Any] = {"method": method.upper(), "url": url} headers, cookies = _parse_headers_and_cookies(parsed_args) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 0ba2fe4e2..b2118495f 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -110,7 +110,7 @@ class CaseInsensitiveDict(collections.UserDict): as keys and allows case-insensitive lookups. """ - def __init__(self, *args, **kwargs) -> None: + def __init__(self, *args: Any, **kwargs: Any) -> None: self._keys: dict = {} super().__init__(*args, **kwargs) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index c86f9fe39..5be80ec0f 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -138,7 +138,7 @@ class RequestFingerprinter: """ @classmethod - def from_crawler(cls, crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) def __init__(self, crawler: Optional[Crawler] = None): diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index a0b06f75c..320059b3a 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -58,7 +58,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str: return f"{status_int} {to_unicode(message)}" -def _remove_html_comments(body): +def _remove_html_comments(body: bytes) -> bytes: start = body.find(b"", start + 1) diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index de9ce992a..ca1f68116 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -15,12 +15,12 @@ class SiteTest: super().tearDown() self.site.stopListening() - def url(self, path): + def url(self, path: str) -> str: return urljoin(self.baseurl, path) class NoMetaRefreshRedirect(util.Redirect): - def render(self, request): + def render(self, request: server.Request) -> bytes: content = util.Redirect.render(self, request) return content.replace( b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' From da42e8f124362a5087c50bca7f76dcc573e8194a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 21:11:50 +0500 Subject: [PATCH 1468/2083] Add parameters to typing.Dict. --- scrapy/core/downloader/handlers/__init__.py | 5 +++- scrapy/core/http2/protocol.py | 4 +-- scrapy/core/http2/stream.py | 4 +-- scrapy/extensions/feedexport.py | 4 ++- scrapy/http/request/__init__.py | 4 +-- scrapy/http/request/form.py | 30 ++++++++++++++------- scrapy/http/request/json_request.py | 18 ++++++------- scrapy/http/response/__init__.py | 4 +-- scrapy/http/response/text.py | 4 +-- scrapy/item.py | 2 +- scrapy/logformatter.py | 22 ++++++++++----- scrapy/settings/__init__.py | 2 +- scrapy/spiders/__init__.py | 6 ++--- scrapy/utils/conf.py | 3 ++- scrapy/utils/log.py | 13 ++++++--- scrapy/utils/python.py | 10 ++++--- scrapy/utils/request.py | 2 +- 17 files changed, 85 insertions(+), 52 deletions(-) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index ade51ca63..af5282553 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -27,7 +27,10 @@ class DownloadHandlers: self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers handlers: Dict[str, Union[str, Callable]] = without_none_values( - crawler.settings.getwithbase("DOWNLOAD_HANDLERS") + cast( + Dict[str, Union[str, Callable]], + crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), + ) ) for scheme, clspath in handlers.items(): self._schemes[scheme] = clspath diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 8898b8118..063835b17 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -3,7 +3,7 @@ import itertools import logging from collections import deque from ipaddress import IPv4Address, IPv6Address -from typing import Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -115,7 +115,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # Some meta data of this connection # initialized when connection is successfully made - self.metadata: Dict = { + self.metadata: Dict[str, Any] = { # Peer certificate instance "certificate": None, # Address of the server we are connected to which diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 224691078..7c70e86db 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -110,7 +110,7 @@ class Stream: # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated - self.metadata: Dict = { + self.metadata: Dict[str, Any] = { "request_content_length": ( 0 if self._request.body is None else len(self._request.body) ), @@ -131,7 +131,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: Dict = { + self._response: Dict[str, Any] = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. "body": BytesIO(), diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3c2bb5593..de8a288f6 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -694,7 +694,9 @@ class FeedExporter: self.slots = slots def _load_components(self, setting_prefix: str) -> Dict[str, Any]: - conf = without_none_values(self.settings.getwithbase(setting_prefix)) + conf = without_none_values( + cast(Dict[str, str], self.settings.getwithbase(setting_prefix)) + ) d = {} for k, v in conf.items(): try: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 191b3cef4..dfb1dca89 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -97,7 +97,7 @@ class Request(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, meta: Optional[Dict[str, Any]] = None, encoding: str = "utf-8", priority: int = 0, @@ -123,7 +123,7 @@ class Request(object_ref): self.callback: Optional[Callable] = callback self.errback: Optional[Callable] = errback - self.cookies: Union[dict, List[dict]] = cookies or {} + self.cookies: Union[Dict[str, str], List[Dict[str, str]]] = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) self.dont_filter: bool = dont_filter diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 3206d79cd..ea98ed795 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,7 +7,17 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from lxml.html import FormElement # nosec @@ -26,8 +36,9 @@ if TYPE_CHECKING: from typing_extensions import Self -FormdataKVType = Tuple[str, Union[str, Iterable[str]]] -FormdataType = Optional[Union[dict, List[FormdataKVType]]] +FormdataVType = Union[str, Iterable[str]] +FormdataKVType = Tuple[str, FormdataVType] +FormdataType = Optional[Union[Dict[str, FormdataVType], List[FormdataKVType]]] class FormRequest(Request): @@ -62,7 +73,7 @@ class FormRequest(Request): formid: Optional[str] = None, formnumber: int = 0, formdata: FormdataType = None, - clickdata: Optional[dict] = None, + clickdata: Optional[Dict[str, Union[str, int]]] = None, dont_click: bool = False, formxpath: Optional[str] = None, formcss: Optional[str] = None, @@ -156,7 +167,7 @@ def _get_inputs( form: FormElement, formdata: FormdataType, dont_click: bool, - clickdata: Optional[dict], + clickdata: Optional[Dict[str, Union[str, int]]], ) -> List[FormdataKVType]: """Return a list of key-value pairs for the inputs found in the given form.""" try: @@ -186,10 +197,8 @@ def _get_inputs( if clickable and clickable[0] not in formdata and not clickable[0] is None: values.append(clickable) - if isinstance(formdata, dict): - formdata = formdata.items() # type: ignore[assignment] - - values.extend((k, v) for k, v in formdata if v is not None) + formdata_items = formdata.items() if isinstance(formdata, dict) else formdata + values.extend((k, v) for k, v in formdata_items if v is not None) return values @@ -216,7 +225,7 @@ def _select_value( def _get_clickable( - clickdata: Optional[dict], form: FormElement + clickdata: Optional[Dict[str, Union[str, int]]], form: FormElement ) -> Optional[Tuple[str, str]]: """ Returns the clickable element specified in clickdata, @@ -243,6 +252,7 @@ def _get_clickable( # because that uniquely identifies the element nr = clickdata.get("nr", None) if nr is not None: + assert isinstance(nr, int) try: el = list(form.inputs)[nr] except IndexError: diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 1dd9e6c87..405c0b9d0 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings -from typing import Any, Optional, Tuple +from typing import Any, Dict, Optional, Tuple from scrapy.http.request import Request @@ -17,15 +17,15 @@ class JsonRequest(Request): attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) def __init__( - self, *args: Any, dumps_kwargs: Optional[dict] = None, **kwargs: Any + self, *args: Any, dumps_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any ) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault("sort_keys", True) - self._dumps_kwargs = dumps_kwargs + self._dumps_kwargs: Dict[str, Any] = dumps_kwargs body_passed = kwargs.get("body", None) is not None - data = kwargs.pop("data", None) - data_passed = data is not None + data: Any = kwargs.pop("data", None) + data_passed: bool = data is not None if body_passed and data_passed: warnings.warn("Both body and data passed. data will be ignored") @@ -41,13 +41,13 @@ class JsonRequest(Request): ) @property - def dumps_kwargs(self) -> dict: + def dumps_kwargs(self) -> Dict[str, Any]: return self._dumps_kwargs def replace(self, *args: Any, **kwargs: Any) -> Request: body_passed = kwargs.get("body", None) is not None - data = kwargs.pop("data", None) - data_passed = data is not None + data: Any = kwargs.pop("data", None) + data_passed: bool = data is not None if body_passed and data_passed: warnings.warn("Both body and data passed. data will be ignored") @@ -56,6 +56,6 @@ class JsonRequest(Request): return super().replace(*args, **kwargs) - def _dumps(self, data: dict) -> str: + def _dumps(self, data: Any) -> str: """Convert to JSON""" return json.dumps(data, **self._dumps_kwargs) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index d73dfce4b..14618e5e7 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -181,7 +181,7 @@ class Response(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, @@ -234,7 +234,7 @@ class Response(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 522ffc0d5..a83279ac8 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -183,7 +183,7 @@ class TextResponse(Response): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, @@ -236,7 +236,7 @@ class TextResponse(Response): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[dict, List[dict]]] = None, + cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, diff --git a/scrapy/item.py b/scrapy/item.py index 2daea64cc..3f93809e7 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -27,7 +27,7 @@ if TYPE_CHECKING: from typing_extensions import Self -class Field(dict): +class Field(Dict[str, Any]): """Container of field metadata""" diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index d720b2f38..42a03b560 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging import os -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union from twisted.python.failure import Failure @@ -26,6 +26,12 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s" DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" +class LogFormatterResult(TypedDict): + level: int + msg: str + args: Union[Dict[str, Any], Tuple[Any, ...]] + + class LogFormatter: """Class for generating log messages for different actions. @@ -64,7 +70,9 @@ class LogFormatter: } """ - def crawled(self, request: Request, response: Response, spider: Spider) -> dict: + def crawled( + self, request: Request, response: Response, spider: Spider + ) -> LogFormatterResult: """Logs a message when the crawler finds a webpage.""" request_flags = f" {str(request.flags)}" if request.flags else "" response_flags = f" {str(response.flags)}" if response.flags else "" @@ -84,7 +92,7 @@ class LogFormatter: def scraped( self, item: Any, response: Union[Response, Failure], spider: Spider - ) -> dict: + ) -> LogFormatterResult: """Logs a message when an item is scraped by a spider.""" src: Any if isinstance(response, Failure): @@ -102,7 +110,7 @@ class LogFormatter: def dropped( self, item: Any, exception: BaseException, response: Response, spider: Spider - ) -> dict: + ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { "level": logging.WARNING, @@ -115,7 +123,7 @@ class LogFormatter: def item_error( self, item: Any, exception: BaseException, response: Response, spider: Spider - ) -> dict: + ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing through the item pipeline. @@ -135,7 +143,7 @@ class LogFormatter: request: Request, response: Union[Response, Failure], spider: Spider, - ) -> dict: + ) -> LogFormatterResult: """Logs an error message from a spider. .. versionadded:: 2.0 @@ -155,7 +163,7 @@ class LogFormatter: request: Request, spider: Spider, errmsg: Optional[str] = None, - ) -> dict: + ) -> LogFormatterResult: """Logs a download error message from a spider (typically coming from the engine). diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index d270a72f4..4448b6f4b 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -411,7 +411,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self._assert_mutability() if isinstance(values, str): - values = cast(dict, json.loads(values)) + values = cast(Dict[_SettingsKeyT, Any], json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index bef041325..7b43f04f2 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union, cast from twisted.internet.defer import Deferred @@ -24,7 +24,7 @@ if TYPE_CHECKING: from typing_extensions import Concatenate, Self from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings + from scrapy.settings import BaseSettings, _SettingsKeyT from scrapy.utils.log import SpiderLoggerAdapter CallbackT = Callable[Concatenate[Response, ...], Any] @@ -36,7 +36,7 @@ class Spider(object_ref): """ name: str - custom_settings: Optional[dict] = None + custom_settings: Optional[Dict[_SettingsKeyT, Any]] = None def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 641dfa4a2..c63b69995 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -16,6 +16,7 @@ from typing import ( MutableMapping, Optional, Union, + cast, ) from scrapy.exceptions import ScrapyDeprecationWarning, UsageError @@ -173,7 +174,7 @@ def feed_process_params_from_cli( suitable to be used as the FEEDS setting. """ valid_output_formats: Iterable[str] = without_none_values( - settings.getwithbase("FEED_EXPORTERS") + cast(Dict[str, str], settings.getwithbase("FEED_EXPORTERS")) ).keys() def check_valid_format(output_format: str) -> None: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 430a91e95..cbfd170ed 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -7,6 +7,7 @@ from types import TracebackType from typing import ( TYPE_CHECKING, Any, + Dict, List, MutableMapping, Optional, @@ -20,7 +21,8 @@ from twisted.python import log as twisted_log from twisted.python.failure import Failure import scrapy -from scrapy.settings import Settings +from scrapy.logformatter import LogFormatterResult +from scrapy.settings import Settings, _SettingsKeyT from scrapy.utils.versions import scrapy_components_versions if TYPE_CHECKING: @@ -86,7 +88,8 @@ DEFAULT_LOGGING = { def configure_logging( - settings: Union[Settings, dict, None] = None, install_root_handler: bool = True + settings: Union[Settings, Dict[_SettingsKeyT, Any], None] = None, + install_root_handler: bool = True, ) -> None: """ Initialize logging defaults for Scrapy. @@ -234,7 +237,9 @@ class LogCounterHandler(logging.Handler): self.crawler.stats.inc_value(sname) -def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: +def logformatter_adapter( + logkws: LogFormatterResult, +) -> Tuple[int, str, Union[Dict[str, Any], Tuple[Any, ...]]]: """ Helper that takes the dictionary output from the methods in LogFormatter and adapts it into a tuple of positional arguments for logger.log calls, @@ -245,7 +250,7 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]: message = logkws.get("msg") or "" # NOTE: This also handles 'args' being an empty dict, that case doesn't # play well in logger.log calls - args = logkws if not logkws.get("args") else logkws["args"] + args = cast(Dict[str, Any], logkws) if not logkws.get("args") else logkws["args"] return (level, message, args) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 578cde2ac..0a50f4e1e 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -42,6 +42,8 @@ if TYPE_CHECKING: _P = ParamSpec("_P") _T = TypeVar("_T") +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") def flatten(x: Iterable) -> list: @@ -303,14 +305,16 @@ def equal_attributes( @overload -def without_none_values(iterable: Mapping) -> dict: ... +def without_none_values(iterable: Mapping[_KT, _VT]) -> Dict[_KT, _VT]: ... @overload -def without_none_values(iterable: Iterable) -> Iterable: ... +def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ... -def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: +def without_none_values( + iterable: Union[Mapping[_KT, _VT], Iterable[_KT]] +) -> Union[Dict[_KT, _VT], Iterable[_KT]]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 5be80ec0f..42a6537a8 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -197,7 +197,7 @@ def referer_str(request: Request) -> Optional[str]: return to_unicode(referrer, errors="replace") -def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: +def request_from_dict(d: Dict[str, Any], *, spider: Optional[Spider] = None) -> Request: """Create a :class:`~scrapy.Request` object from a dict. If a spider is given, it will try to resolve the callbacks looking at the From 98c755e5fbc005083a5fde810476f2de610bf912 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 21:20:22 +0500 Subject: [PATCH 1469/2083] Add parameters to typing.List. --- scrapy/core/scheduler.py | 8 ++++---- scrapy/downloadermiddlewares/stats.py | 6 ++++-- scrapy/utils/asyncgen.py | 10 +++++++--- scrapy/utils/python.py | 8 ++++---- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index f30a5d9c9..e3b95e977 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,7 +4,7 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Type, cast +from typing import TYPE_CHECKING, Any, List, Optional, Type, cast from twisted.internet.defer import Deferred @@ -362,13 +362,13 @@ class Scheduler(BaseScheduler): return str(dqdir) return None - def _read_dqs_state(self, dqdir: str) -> list: + def _read_dqs_state(self, dqdir: str) -> List[int]: path = Path(dqdir, "active.json") if not path.exists(): return [] with path.open(encoding="utf-8") as f: - return cast(list, json.load(f)) + return cast(List[int], json.load(f)) - def _write_dqs_state(self, dqdir: str, state: list) -> None: + def _write_dqs_state(self, dqdir: str, state: List[int]) -> None: with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: json.dump(state, f) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index df30e8ca4..444702757 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Union +from typing import TYPE_CHECKING, Dict, List, Tuple, Union from twisted.web import http @@ -17,7 +17,9 @@ if TYPE_CHECKING: from typing_extensions import Self -def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int: +def get_header_size( + headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]] +) -> int: size = 0 for key, value in headers.items(): if isinstance(value, (list, tuple)): diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 0505db343..67c8e1a01 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,14 +1,18 @@ -from typing import AsyncGenerator, AsyncIterable, Iterable, Union +from typing import AsyncGenerator, AsyncIterable, Iterable, List, TypeVar, Union + +_T = TypeVar("_T") -async def collect_asyncgen(result: AsyncIterable) -> list: +async def collect_asyncgen(result: AsyncIterable[_T]) -> List[_T]: results = [] async for x in result: results.append(x) return results -async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator: +async def as_async_generator( + it: Union[Iterable[_T], AsyncIterable[_T]] +) -> AsyncGenerator[_T, None]: """Wraps an iterable (sync or async) into an async generator.""" if isinstance(it, AsyncIterable): async for r in it: diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 0a50f4e1e..3db7acf81 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -46,7 +46,7 @@ _KT = TypeVar("_KT") _VT = TypeVar("_VT") -def flatten(x: Iterable) -> list: +def flatten(x: Iterable[Any]) -> List[Any]: """flatten(sequence) -> list Returns a single, flat list which contains all elements retrieved @@ -66,7 +66,7 @@ def flatten(x: Iterable) -> list: return list(iflatten(x)) -def iflatten(x: Iterable) -> Iterable: +def iflatten(x: Iterable[Any]) -> Iterable[Any]: """iflatten(sequence) -> iterator Similar to ``.flatten()``, but returns iterator instead""" @@ -101,10 +101,10 @@ def is_listlike(x: Any) -> bool: return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) -def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list: +def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> List[_T]: """efficient function to uniquify a list preserving item order""" seen = set() - result = [] + result: List[_T] = [] for item in list_: seenkey = key(item) if seenkey in seen: From 4164e63725dc19bc8585abbfb0e5009f8eceefcc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 21:23:55 +0500 Subject: [PATCH 1470/2083] Add parameters to typing.Tuple. --- scrapy/core/http2/agent.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 935af2214..999764a6e 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -20,6 +20,8 @@ from scrapy.http.request import Request from scrapy.settings import Settings from scrapy.spiders import Spider +ConnectionKeyT = Tuple[bytes, bytes, int] + class H2ConnectionPool: def __init__(self, reactor: ReactorBase, settings: Settings) -> None: @@ -28,13 +30,13 @@ class H2ConnectionPool: # Store a dictionary which is used to get the respective # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port) - self._connections: Dict[Tuple, H2ClientProtocol] = {} + self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {} # Save all requests that arrive before the connection is established - self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} + self._pending_requests: Dict[ConnectionKeyT, Deque[Deferred]] = {} def get_connection( - self, key: Tuple, uri: URI, endpoint: HostnameEndpoint + self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint ) -> Deferred: if key in self._pending_requests: # Received a request while connecting to remote @@ -54,7 +56,7 @@ class H2ConnectionPool: return self._new_connection(key, uri, endpoint) def _new_connection( - self, key: Tuple, uri: URI, endpoint: HostnameEndpoint + self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint ) -> Deferred: self._pending_requests[key] = deque() @@ -69,7 +71,9 @@ class H2ConnectionPool: self._pending_requests[key].append(d) return d - def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol: + def put_connection( + self, conn: H2ClientProtocol, key: ConnectionKeyT + ) -> H2ClientProtocol: self._connections[key] = conn # Now as we have established a proper HTTP/2 connection @@ -81,7 +85,9 @@ class H2ConnectionPool: return conn - def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None: + def _remove_connection( + self, errors: List[BaseException], key: ConnectionKeyT + ) -> None: self._connections.pop(key) # Call the errback of all the pending requests for this connection @@ -122,7 +128,7 @@ class H2Agent: def get_endpoint(self, uri: URI) -> HostnameEndpoint: return self.endpoint_factory.endpointForURI(uri) - def get_key(self, uri: URI) -> Tuple: + def get_key(self, uri: URI) -> ConnectionKeyT: """ Arguments: uri - URI obtained directly from request URL @@ -164,6 +170,6 @@ class ScrapyProxyH2Agent(H2Agent): def get_endpoint(self, uri: URI) -> HostnameEndpoint: return self.endpoint_factory.endpointForURI(self._proxy_uri) - def get_key(self, uri: URI) -> Tuple: + def get_key(self, uri: URI) -> ConnectionKeyT: """We use the proxy uri instead of uri obtained from request url""" - return "http-proxy", self._proxy_uri.host, self._proxy_uri.port + return b"http-proxy", self._proxy_uri.host, self._proxy_uri.port From 70c56faf4847406de6eb3594758c5531610757e8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 21:41:27 +0500 Subject: [PATCH 1471/2083] Add parameters to typing.IO. --- scrapy/extensions/httpcache.py | 9 ++++++--- scrapy/mail.py | 4 ++-- scrapy/pipelines/files.py | 2 +- scrapy/utils/ftp.py | 2 +- scrapy/utils/misc.py | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index dd5bce24f..3f4af42b7 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -315,7 +315,9 @@ class FilesystemCacheStorage: self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") # https://github.com/python/mypy/issues/10740 - self._open: Callable[Concatenate[Union[str, os.PathLike], str, ...], IO] = ( + self._open: Callable[ + Concatenate[Union[str, os.PathLike], str, ...], IO[bytes] + ] = ( gzip.open if self.use_gzip else open # type: ignore[assignment] ) @@ -368,11 +370,12 @@ class FilesystemCacheStorage: with self._open(rpath / "pickled_meta", "wb") as f: pickle.dump(metadata, f, protocol=4) with self._open(rpath / "response_headers", "wb") as f: - f.write(headers_dict_to_raw(response.headers)) + # headers_dict_to_raw() needs a better type hint + f.write(cast(bytes, headers_dict_to_raw(response.headers))) with self._open(rpath / "response_body", "wb") as f: f.write(response.body) with self._open(rpath / "request_headers", "wb") as f: - f.write(headers_dict_to_raw(request.headers)) + f.write(cast(bytes, headers_dict_to_raw(request.headers))) with self._open(rpath / "request_body", "wb") as f: f.write(request.body) diff --git a/scrapy/mail.py b/scrapy/mail.py index fd6302550..f4ce2800c 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -97,7 +97,7 @@ class MailSender: subject: str, body: str, cc: Union[str, List[str], None] = None, - attachs: Sequence[Tuple[str, str, IO]] = (), + attachs: Sequence[Tuple[str, str, IO[Any]]] = (), mimetype: str = "text/plain", charset: Optional[str] = None, _callback: Optional[Callable[..., None]] = None, @@ -214,7 +214,7 @@ class MailSender: return d def _create_sender_factory( - self, to_addrs: List[str], msg: IO, d: Deferred + self, to_addrs: List[str], msg: IO[bytes], d: Deferred ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 47457f2a8..c1ce0939c 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -47,7 +47,7 @@ def _to_string(path: Union[str, PathLike]) -> str: return str(path) # convert a Path object to string -def _md5sum(file: IO) -> str: +def _md5sum(file: IO[bytes]) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index c77681a53..152f3374e 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -21,7 +21,7 @@ def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None: def ftp_store_file( *, path: str, - file: IO, + file: IO[bytes], host: str, port: int, username: str, diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index faf52e44a..b678d1def 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -111,7 +111,7 @@ def walk_modules(path: str) -> List[ModuleType]: return mods -def md5sum(file: IO) -> str: +def md5sum(file: IO[bytes]) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. From 751c91e614b91827dc68cd462b907c4b9d03f071 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 21:57:14 +0500 Subject: [PATCH 1472/2083] Add parameters to misc generics. --- scrapy/core/engine.py | 20 +++++++++++--------- scrapy/core/http2/protocol.py | 4 ++-- scrapy/utils/datatypes.py | 4 ++-- scrapy/utils/python.py | 2 +- scrapy/utils/test.py | 8 +++++--- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4eca03800..4cb4454e3 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -5,6 +5,8 @@ For more information see docs/topics/architecture.rst """ +from __future__ import annotations + import logging from time import time from typing import ( @@ -51,15 +53,15 @@ class Slot: self, start_requests: Iterable[Request], close_if_idle: bool, - nextcall: CallLaterOnce, - scheduler: "BaseScheduler", + nextcall: CallLaterOnce[None], + scheduler: BaseScheduler, ) -> None: self.closing: Optional[Deferred] = None self.inprogress: Set[Request] = set() self.start_requests: Optional[Iterator[Request]] = iter(start_requests) self.close_if_idle: bool = close_if_idle - self.nextcall: CallLaterOnce = nextcall - self.scheduler: "BaseScheduler" = scheduler + self.nextcall: CallLaterOnce[None] = nextcall + self.scheduler: BaseScheduler = scheduler self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule) def add_request(self, request: Request) -> None: @@ -84,8 +86,8 @@ class Slot: class ExecutionEngine: - def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None: - self.crawler: "Crawler" = crawler + def __init__(self, crawler: Crawler, spider_closed_callback: Callable) -> None: + self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals assert crawler.logformatter @@ -94,7 +96,7 @@ class ExecutionEngine: self.spider: Optional[Spider] = None self.running: bool = False self.paused: bool = False - self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class( + self.scheduler_cls: Type[BaseScheduler] = self._get_scheduler_class( crawler.settings ) downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"]) @@ -103,10 +105,10 @@ class ExecutionEngine: self._spider_closed_callback: Callable = spider_closed_callback self.start_time: Optional[float] = None - def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]: + def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler - scheduler_cls: Type = load_object(settings["SCHEDULER"]) + scheduler_cls: Type[BaseScheduler] = load_object(settings["SCHEDULER"]) if not issubclass(scheduler_cls, BaseScheduler): raise TypeError( f"The provided scheduler class ({settings['SCHEDULER']})" diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 063835b17..f2f1cb0b8 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -3,7 +3,7 @@ import itertools import logging from collections import deque from ipaddress import IPv4Address, IPv6Address -from typing import Any, Dict, List, Optional, Union +from typing import Any, Deque, Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -107,7 +107,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): # If requests are received before connection is made we keep # all requests in a pool and send them as the connection is made - self._pending_request_stream_pool: deque = deque() + self._pending_request_stream_pool: Deque[Stream] = deque() # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index b2118495f..d06887610 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -196,8 +196,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): class SequenceExclude: """Object to test if an item is NOT within some sequence.""" - def __init__(self, seq: Sequence): - self.seq: Sequence = seq + def __init__(self, seq: Sequence[Any]): + self.seq: Sequence[Any] = seq def __contains__(self, item: Any) -> bool: return item not in self.seq diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 3db7acf81..fc1eb4f69 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -148,7 +148,7 @@ def to_bytes( def re_rsearch( - pattern: Union[str, Pattern], text: str, chunk_size: int = 1024 + pattern: Union[str, Pattern[str]], text: str, chunk_size: int = 1024 ) -> Optional[Tuple[int, int]]: """ This function does a reverse search in a text using a regular expression diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 7a8c5c859..268d8d4be 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -7,7 +7,7 @@ import os from importlib import import_module from pathlib import Path from posixpath import split -from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type +from typing import Any, Awaitable, Dict, List, Optional, Tuple, Type, TypeVar from unittest import TestCase, mock from twisted.internet.defer import Deferred @@ -17,6 +17,8 @@ from scrapy import Spider from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available +_T = TypeVar("_T") + def assert_gcs_environ() -> None: if "GCS_PROJECT_ID" not in os.environ: @@ -118,8 +120,8 @@ def assert_samelines( testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) -def get_from_asyncio_queue(value: Any) -> Coroutine: - q: asyncio.Queue = asyncio.Queue() +def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: + q: asyncio.Queue[_T] = asyncio.Queue() getter = q.get() q.put_nowait(value) return getter From 859a77ee4243f17f338072e45785383f12516308 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 May 2024 22:23:26 +0500 Subject: [PATCH 1473/2083] Use a TypedDict for the verbose cookie form. --- scrapy/downloadermiddlewares/cookies.py | 25 +++++++++---------------- scrapy/http/request/__init__.py | 20 ++++++++++++++++---- scrapy/http/response/__init__.py | 6 +++--- scrapy/http/response/text.py | 6 +++--- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 6ada3b474..73c2c57fe 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -3,16 +3,7 @@ from __future__ import annotations import logging from collections import defaultdict from http.cookiejar import Cookie -from typing import ( - TYPE_CHECKING, - Any, - DefaultDict, - Dict, - Iterable, - Optional, - Sequence, - Union, -) +from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union from tldextract import TLDExtract @@ -21,6 +12,7 @@ from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar +from scrapy.http.request import VerboseCookie from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -128,7 +120,7 @@ class CookiesMiddleware: msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={"spider": spider}) - def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]: + def _format_cookie(self, cookie: VerboseCookie, request: Request) -> Optional[str]: """ Given a dict consisting of cookie components, return its string representation. Decode from bytes if necessary. @@ -142,18 +134,19 @@ class CookiesMiddleware: logger.warning(msg) return None continue - if isinstance(cookie[key], (bool, float, int, str)): - decoded[key] = str(cookie[key]) + # https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168 + if isinstance(cookie[key], (bool, float, int, str)): # type: ignore[literal-required] + decoded[key] = str(cookie[key]) # type: ignore[literal-required] else: try: - decoded[key] = cookie[key].decode("utf8") + decoded[key] = cookie[key].decode("utf8") # type: ignore[literal-required] except UnicodeDecodeError: logger.warning( "Non UTF-8 encoded cookie found in request %s: %s", request, cookie, ) - decoded[key] = cookie[key].decode("latin1", errors="replace") + decoded[key] = cookie[key].decode("latin1", errors="replace") # type: ignore[literal-required] for flag in ("secure",): value = cookie.get(flag, _UNSET) if value is _UNSET or not value: @@ -174,7 +167,7 @@ class CookiesMiddleware: """ if not request.cookies: return [] - cookies: Iterable[Dict[str, Any]] + cookies: Iterable[VerboseCookie] if isinstance(request.cookies, dict): cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items()) else: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index dfb1dca89..96d0dc515 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -20,6 +20,7 @@ from typing import ( NoReturn, Optional, Tuple, + TypedDict, Union, cast, ) @@ -34,8 +35,19 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from typing_extensions import Self + # typing.NotRequired and typing.Self require Python 3.11 + from typing_extensions import NotRequired, Self + + +class VerboseCookie(TypedDict): + name: str + value: str + domain: NotRequired[str] + path: NotRequired[str] + secure: NotRequired[bool] + + +CookiesT = Union[Dict[str, str], List[VerboseCookie]] def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: @@ -97,7 +109,7 @@ class Request(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: str = "utf-8", priority: int = 0, @@ -123,7 +135,7 @@ class Request(object_ref): self.callback: Optional[Callable] = callback self.errback: Optional[Callable] = errback - self.cookies: Union[Dict[str, str], List[Dict[str, str]]] = cookies or {} + self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) self.dont_filter: bool = dont_filter diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 14618e5e7..166c4de97 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -29,7 +29,7 @@ from twisted.internet.ssl import Certificate from scrapy.exceptions import NotSupported from scrapy.http.headers import Headers -from scrapy.http.request import Request +from scrapy.http.request import CookiesT, Request from scrapy.link import Link from scrapy.utils.trackref import object_ref @@ -181,7 +181,7 @@ class Response(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, @@ -234,7 +234,7 @@ class Response(object_ref): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index a83279ac8..44c36b682 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -36,7 +36,7 @@ from w3lib.encoding import ( ) from w3lib.html import strip_html5_whitespace -from scrapy.http import Request +from scrapy.http.request import CookiesT, Request from scrapy.http.response import Response from scrapy.link import Link from scrapy.utils.python import memoizemethod_noargs, to_unicode @@ -183,7 +183,7 @@ class TextResponse(Response): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, @@ -236,7 +236,7 @@ class TextResponse(Response): method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, - cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None, + cookies: Optional[CookiesT] = None, meta: Optional[Dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, From 019f23e3b75a0a481a4fcc22dc93c867ce424b18 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 2 Jun 2024 18:42:01 +0500 Subject: [PATCH 1474/2083] Add parameters to some of typing.Callable. --- scrapy/core/downloader/handlers/__init__.py | 12 +++++++----- scrapy/core/engine.py | 12 +++++++++--- scrapy/http/request/__init__.py | 2 +- scrapy/utils/misc.py | 8 +++++--- scrapy/utils/python.py | 6 +++--- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index af5282553..5ec5ef6db 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,5 +1,7 @@ """Download handlers for different schemes""" +from __future__ import annotations + import logging from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast @@ -19,16 +21,16 @@ logger = logging.getLogger(__name__) class DownloadHandlers: - def __init__(self, crawler: "Crawler"): - self._crawler: "Crawler" = crawler - self._schemes: Dict[str, Union[str, Callable]] = ( + def __init__(self, crawler: Crawler): + self._crawler: Crawler = crawler + self._schemes: Dict[str, Union[str, Callable[..., Any]]] = ( {} ) # stores acceptable schemes on instancing self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers - handlers: Dict[str, Union[str, Callable]] = without_none_values( + handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values( cast( - Dict[str, Union[str, Callable]], + Dict[str, Union[str, Callable[..., Any]]], crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), ) ) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4cb4454e3..b342ad7a3 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -86,7 +86,11 @@ class Slot: class ExecutionEngine: - def __init__(self, crawler: Crawler, spider_closed_callback: Callable) -> None: + def __init__( + self, + crawler: Crawler, + spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]], + ) -> None: self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals @@ -102,7 +106,9 @@ class ExecutionEngine: downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) - self._spider_closed_callback: Callable = spider_closed_callback + self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = ( + spider_closed_callback + ) self.start_time: Optional[float] = None def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]: @@ -427,7 +433,7 @@ class ExecutionEngine: dfd = self.slot.close() - def log_failure(msg: str) -> Callable: + def log_failure(msg: str) -> Callable[[Failure], None]: def errback(failure: Failure) -> None: logger.error( msg, exc_info=failure_to_exc_info(failure), extra={"spider": spider} diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 96d0dc515..77149333c 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -266,7 +266,7 @@ class Request(object_ref): return d -def _find_method(obj: Any, func: Callable) -> str: +def _find_method(obj: Any, func: Callable[..., Any]) -> str: """Helper function for Request.to_dict""" # Only instance methods contain ``__func__`` if obj and hasattr(func, "__func__"): diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b678d1def..49f36de2d 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -56,7 +56,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]: return [arg] -def load_object(path: Union[str, Callable]) -> Any: +def load_object(path: Union[str, Callable[..., Any]]) -> Any: """Load an object given its absolute object path, and return it. The object can be the import path of a class, function, variable or an @@ -263,7 +263,7 @@ def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: _generator_callbacks_cache = LocalWeakReferencedCache(limit=128) -def is_generator_with_return_value(callable: Callable) -> bool: +def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: """ Returns True if a callable is a generator function which includes a 'return' statement with a value different than None, False otherwise @@ -300,7 +300,9 @@ def is_generator_with_return_value(callable: Callable) -> bool: return bool(_generator_callbacks_cache[callable]) -def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None: +def warn_on_generator_with_return_value( + spider: Spider, callable: Callable[..., Any] +) -> None: """ Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index fc1eb4f69..37a84a350 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -217,7 +217,7 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) -def get_func_args(func: Callable, stripself: bool = False) -> List[str]: +def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str]: """Return the argument name list of a callable object""" if not callable(func): raise TypeError(f"func must be callable, got '{type(func).__name__}'") @@ -247,7 +247,7 @@ def get_func_args(func: Callable, stripself: bool = False) -> List[str]: return args -def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: +def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: """Returns (args, kwargs) tuple for a function >>> import re >>> get_spec(re.match) @@ -285,7 +285,7 @@ def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: def equal_attributes( - obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]] + obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable[[Any], Any]]]] ) -> bool: """Compare two objects attributes""" # not attributes given return False by default From 492c3bce9dfc6cccdad8fc7002db4bec49cfcb35 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 3 Jun 2024 15:28:20 +0400 Subject: [PATCH 1475/2083] Don't run callbacks of requests from get_media_requests(). (#6386) --- scrapy/pipelines/media.py | 10 ---------- tests/test_pipeline_media.py | 7 ------- 2 files changed, 17 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 25e00b0ea..0e374265e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -24,10 +24,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def _DUMMY_CALLBACK(response): - return response - - class MediaPipeline(ABC): LOG_FAILED_RESULTS = True @@ -89,10 +85,6 @@ class MediaPipeline(ABC): def _process_request(self, request, info, item): fp = self._fingerprinter.fingerprint(request) - if not request.callback or request.callback is NO_CALLBACK: - cb = _DUMMY_CALLBACK - else: - cb = request.callback eb = request.errback request.callback = NO_CALLBACK request.errback = None @@ -100,14 +92,12 @@ class MediaPipeline(ABC): # Return cached result if request was already seen if fp in info.downloaded: d = defer_result(info.downloaded[fp]) - d.addCallback(cb) if eb: d.addErrback(eb) return d # Otherwise, wait for result wad = Deferred() - wad.addCallback(cb) if eb: wad.addErrback(eb) info.waiting[fp].append(wad) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 763453551..127775f43 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -211,10 +211,6 @@ class MockedMediaPipeline(UserDefinedPipeline): class MediaPipelineTestCase(BaseMediaPipelineTestCase): pipeline_class = MockedMediaPipeline - def _callback(self, result): - self.pipe._mockcalled.append("request_callback") - return result - def _errback(self, result): self.pipe._mockcalled.append("request_errback") return result @@ -225,7 +221,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req = Request( "http://url1", meta={"response": rsp}, - callback=self._callback, errback=self._errback, ) item = {"requests": req} @@ -237,7 +232,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): "get_media_requests", "media_to_download", "media_downloaded", - "request_callback", "item_completed", ], ) @@ -249,7 +243,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req = Request( "http://url1", meta={"response": fail}, - callback=self._callback, errback=self._errback, ) item = {"requests": req} From e56b425198bfe3e86f2c578e7bc1f2988c7d3ec9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 5 Jun 2024 08:33:45 +0400 Subject: [PATCH 1476/2083] Full typing for scrapy/pipelines. (#6387) --- scrapy/pipelines/__init__.py | 3 +- scrapy/pipelines/files.py | 274 +++++++++++++++++++++++++---------- scrapy/pipelines/images.py | 128 ++++++++++++---- scrapy/pipelines/media.py | 162 ++++++++++++++++----- 4 files changed, 424 insertions(+), 143 deletions(-) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index f9544d329..0cfbc156f 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -10,6 +10,7 @@ from twisted.internet.defer import Deferred from scrapy import Spider from scrapy.middleware import MiddlewareManager +from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_f_from_coro_f @@ -18,7 +19,7 @@ class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" @classmethod - def _get_mwlist_from_settings(cls, settings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: return build_component_list(settings.getwithbase("ITEM_PIPELINES")) def _add_middleware(self, pipe: Any) -> None: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index c1ce0939c..85a8c77da 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -18,16 +18,35 @@ from ftplib import FTP from io import BytesIO from os import PathLike from pathlib import Path -from typing import IO, TYPE_CHECKING, DefaultDict, Optional, Set, Type, Union, cast +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + DefaultDict, + Dict, + List, + NoReturn, + Optional, + Protocol, + Set, + Type, + TypedDict, + Union, + cast, +) from urllib.parse import urlparse from itemadapter import ItemAdapter from twisted.internet import defer, threads +from twisted.internet.defer import Deferred +from twisted.python.failure import Failure +from scrapy import Spider from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict @@ -40,10 +59,11 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + logger = logging.getLogger(__name__) -def _to_string(path: Union[str, PathLike]) -> str: +def _to_string(path: Union[str, PathLike[str]]) -> str: return str(path) # convert a Path object to string @@ -68,23 +88,54 @@ class FileException(Exception): """General media error exception""" +class StatInfo(TypedDict, total=False): + checksum: str + last_modified: float + + +class FilesStoreProtocol(Protocol): + def __init__(self, basedir: str): ... + + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Optional[Deferred[Any]]: ... + + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Union[StatInfo, Deferred[StatInfo]]: ... + + class FSFilesStore: - def __init__(self, basedir: Union[str, PathLike]): + def __init__(self, basedir: Union[str, PathLike[str]]): basedir = _to_string(basedir) if "://" in basedir: basedir = basedir.split("://", 1)[1] - self.basedir = basedir + self.basedir: str = basedir self._mkdir(Path(self.basedir)) - self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set) + self.created_directories: DefaultDict[MediaPipeline.SpiderInfo, Set[str]] = ( + defaultdict(set) + ) def persist_file( - self, path: Union[str, PathLike], buf, info, meta=None, headers=None - ): + self, + path: Union[str, PathLike[str]], + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> None: absolute_path = self._get_filesystem_path(path) self._mkdir(absolute_path.parent, info) absolute_path.write_bytes(buf.getvalue()) - def stat_file(self, path: Union[str, PathLike], info): + def stat_file( + self, path: Union[str, PathLike[str]], info: MediaPipeline.SpiderInfo + ) -> StatInfo: absolute_path = self._get_filesystem_path(path) try: last_modified = absolute_path.stat().st_mtime @@ -96,12 +147,14 @@ class FSFilesStore: return {"last_modified": last_modified, "checksum": checksum} - def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path: + def _get_filesystem_path(self, path: Union[str, PathLike[str]]) -> Path: path_comps = _to_string(path).split("/") return Path(self.basedir, *path_comps) - def _mkdir(self, dirname: Path, domain: Optional[str] = None): - seen = self.created_directories[domain] if domain else set() + def _mkdir( + self, dirname: Path, domain: Optional[MediaPipeline.SpiderInfo] = None + ) -> None: + seen: Set[str] = self.created_directories[domain] if domain else set() if str(dirname) not in seen: if not dirname.exists(): dirname.mkdir(parents=True) @@ -122,7 +175,7 @@ class S3FilesStore: "Cache-Control": "max-age=172800", } - def __init__(self, uri): + def __init__(self, uri: str): if not is_botocore_available(): raise NotConfigured("missing botocore library") import botocore.session @@ -142,8 +195,10 @@ class S3FilesStore: raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") self.bucket, self.prefix = uri[5:].split("/", 1) - def stat_file(self, path, info): - def _onsuccess(boto_key): + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Deferred[StatInfo]: + def _onsuccess(boto_key: Dict[str, Any]) -> StatInfo: checksum = boto_key["ETag"].strip('"') last_modified = boto_key["LastModified"] modified_stamp = time.mktime(last_modified.timetuple()) @@ -151,13 +206,23 @@ class S3FilesStore: return self._get_boto_key(path).addCallback(_onsuccess) - def _get_boto_key(self, path): + def _get_boto_key(self, path: str) -> Deferred[Dict[str, Any]]: key_name = f"{self.prefix}{path}" - return threads.deferToThread( - self.s3_client.head_object, Bucket=self.bucket, Key=key_name + return cast( + "Deferred[Dict[str, Any]]", + threads.deferToThread( + self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined] + ), ) - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Deferred[Any]: """Upload file to S3 storage""" key_name = f"{self.prefix}{path}" buf.seek(0) @@ -165,7 +230,7 @@ class S3FilesStore: if headers: extra.update(self._headers_to_botocore_kwargs(headers)) return threads.deferToThread( - self.s3_client.put_object, + self.s3_client.put_object, # type: ignore[attr-defined] Bucket=self.bucket, Key=key_name, Body=buf, @@ -174,7 +239,7 @@ class S3FilesStore: **extra, ) - def _headers_to_botocore_kwargs(self, headers): + def _headers_to_botocore_kwargs(self, headers: Dict[str, Any]) -> Dict[str, Any]: """Convert headers to botocore keyword arguments.""" # This is required while we need to support both boto and botocore. mapping = CaseInsensitiveDict( @@ -206,7 +271,7 @@ class S3FilesStore: "X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation", } ) - extra = {} + extra: Dict[str, Any] = {} for key, value in headers.items(): try: kwarg = mapping[key] @@ -226,13 +291,13 @@ class GCSFilesStore: # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. POLICY = None - def __init__(self, uri): + def __init__(self, uri: str): from google.cloud import storage client = storage.Client(project=self.GCS_PROJECT_ID) bucket, prefix = uri[5:].split("/", 1) self.bucket = client.bucket(bucket) - self.prefix = prefix + self.prefix: str = prefix permissions = self.bucket.test_iam_permissions( ["storage.objects.get", "storage.objects.create"] ) @@ -248,8 +313,10 @@ class GCSFilesStore: {"bucket": bucket}, ) - def stat_file(self, path, info): - def _onsuccess(blob): + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Deferred[StatInfo]: + def _onsuccess(blob) -> StatInfo: if blob: checksum = base64.b64decode(blob.md5_hash).hex() last_modified = time.mktime(blob.updated.timetuple()) @@ -257,19 +324,29 @@ class GCSFilesStore: return {} blob_path = self._get_blob_path(path) - return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback( - _onsuccess + return cast( + Deferred[StatInfo], + threads.deferToThread(self.bucket.get_blob, blob_path).addCallback( + _onsuccess + ), ) - def _get_content_type(self, headers): + def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str: if headers and "Content-Type" in headers: return headers["Content-Type"] return "application/octet-stream" - def _get_blob_path(self, path): + def _get_blob_path(self, path: str) -> str: return self.prefix + path - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Deferred[Any]: blob_path = self._get_blob_path(path) blob = self.bucket.blob(blob_path) blob.cache_control = self.CACHE_CONTROL @@ -283,22 +360,33 @@ class GCSFilesStore: class FTPFilesStore: - FTP_USERNAME = None - FTP_PASSWORD = None - USE_ACTIVE_MODE = None + FTP_USERNAME: Optional[str] = None + FTP_PASSWORD: Optional[str] = None + USE_ACTIVE_MODE: Optional[bool] = None - def __init__(self, uri): + def __init__(self, uri: str): if not uri.startswith("ftp://"): raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'") u = urlparse(uri) - self.port = u.port - self.host = u.hostname + assert u.port + assert u.hostname + self.port: int = u.port + self.host: str = u.hostname self.port = int(u.port or 21) - self.username = u.username or self.FTP_USERNAME - self.password = u.password or self.FTP_PASSWORD - self.basedir = u.path.rstrip("/") + assert self.FTP_USERNAME + assert self.FTP_PASSWORD + self.username: str = u.username or self.FTP_USERNAME + self.password: str = u.password or self.FTP_PASSWORD + self.basedir: str = u.path.rstrip("/") - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file( + self, + path: str, + buf: BytesIO, + info: MediaPipeline.SpiderInfo, + meta: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, str]] = None, + ) -> Deferred[Any]: path = f"{self.basedir}/{path}" return threads.deferToThread( ftp_store_file, @@ -311,8 +399,10 @@ class FTPFilesStore: use_active_mode=self.USE_ACTIVE_MODE, ) - def stat_file(self, path, info): - def _stat_file(path): + def stat_file( + self, path: str, info: MediaPipeline.SpiderInfo + ) -> Deferred[StatInfo]: + def _stat_file(path: str) -> StatInfo: try: ftp = FTP() ftp.connect(self.host, self.port) @@ -328,7 +418,7 @@ class FTPFilesStore: except Exception: return {} - return threads.deferToThread(_stat_file, path) + return cast("Deferred[StatInfo]", threads.deferToThread(_stat_file, path)) class FilesPipeline(MediaPipeline): @@ -350,20 +440,23 @@ class FilesPipeline(MediaPipeline): """ - MEDIA_NAME = "file" - EXPIRES = 90 - STORE_SCHEMES = { + MEDIA_NAME: str = "file" + EXPIRES: int = 90 + STORE_SCHEMES: Dict[str, Type[FilesStoreProtocol]] = { "": FSFilesStore, "file": FSFilesStore, "s3": S3FilesStore, "gs": GCSFilesStore, "ftp": FTPFilesStore, } - DEFAULT_FILES_URLS_FIELD = "file_urls" - DEFAULT_FILES_RESULT_FIELD = "files" + DEFAULT_FILES_URLS_FIELD: str = "file_urls" + DEFAULT_FILES_RESULT_FIELD: str = "files" def __init__( - self, store_uri: Union[str, PathLike], download_func=None, settings=None + self, + store_uri: Union[str, PathLike[str]], + download_func: Optional[Callable[[Request, Spider], Response]] = None, + settings: Union[Settings, Dict[str, Any], None] = None, ): store_uri = _to_string(store_uri) if not store_uri: @@ -372,26 +465,26 @@ class FilesPipeline(MediaPipeline): if isinstance(settings, dict) or settings is None: settings = Settings(settings) cls_name = "FilesPipeline" - self.store = self._get_store(store_uri) + self.store: FilesStoreProtocol = self._get_store(store_uri) resolve = functools.partial( self._key_for_pipe, base_class_name=cls_name, settings=settings ) - self.expires = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES) + self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES) if not hasattr(self, "FILES_URLS_FIELD"): self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD if not hasattr(self, "FILES_RESULT_FIELD"): self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD - self.files_urls_field = settings.get( + self.files_urls_field: str = settings.get( resolve("FILES_URLS_FIELD"), self.FILES_URLS_FIELD ) - self.files_result_field = settings.get( + self.files_result_field: str = settings.get( resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD ) super().__init__(download_func=download_func, settings=settings) @classmethod - def from_settings(cls, settings) -> Self: + def from_settings(cls, settings: Settings) -> Self: s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] @@ -418,7 +511,7 @@ class FilesPipeline(MediaPipeline): store_uri = settings["FILES_STORE"] return cls(store_uri, settings=settings) - def _get_store(self, uri: str): + def _get_store(self, uri: str) -> FilesStoreProtocol: if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir scheme = "file" else: @@ -426,19 +519,21 @@ class FilesPipeline(MediaPipeline): store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) - def media_to_download(self, request, info, *, item=None): - def _onsuccess(result): + def media_to_download( + self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None + ) -> Deferred[Optional[FileInfo]]: + def _onsuccess(result: StatInfo) -> Optional[FileInfo]: if not result: - return # returning None force download + return None # returning None force download last_modified = result.get("last_modified", None) if not last_modified: - return # returning None force download + return None # returning None force download age_seconds = time.time() - last_modified age_days = age_seconds / 60 / 60 / 24 if age_days > self.expires: - return # returning None force download + return None # returning None force download referer = referer_str(request) logger.debug( @@ -458,19 +553,22 @@ class FilesPipeline(MediaPipeline): } path = self.file_path(request, info=info, item=item) - dfd = defer.maybeDeferred(self.store.stat_file, path, info) - dfd.addCallback(_onsuccess) - dfd.addErrback(lambda _: None) - dfd.addErrback( + # defer.maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type + dfd: Deferred[StatInfo] = defer.maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type] + dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess) + dfd2.addErrback(lambda _: None) + dfd2.addErrback( lambda f: logger.error( self.__class__.__name__ + ".store.stat_file", exc_info=failure_to_exc_info(f), extra={"spider": info.spider}, ) ) - return dfd + return dfd2 - def media_failed(self, failure, request, info): + def media_failed( + self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo + ) -> NoReturn: if not isinstance(failure.value, IgnoreRequest): referer = referer_str(request) logger.warning( @@ -487,7 +585,14 @@ class FilesPipeline(MediaPipeline): raise FileException - def media_downloaded(self, response, request, info, *, item=None): + def media_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> FileInfo: referer = referer_str(request) if response.status != 200: @@ -546,16 +651,26 @@ class FilesPipeline(MediaPipeline): "status": status, } - def inc_stats(self, spider, status): + def inc_stats(self, spider: Spider, status: str) -> None: + assert spider.crawler.stats spider.crawler.stats.inc_value("file_count", spider=spider) spider.crawler.stats.inc_value(f"file_status_count/{status}", spider=spider) # Overridable Interface - def get_media_requests(self, item, info): + def get_media_requests( + self, item: Any, info: MediaPipeline.SpiderInfo + ) -> List[Request]: urls = ItemAdapter(item).get(self.files_urls_field, []) return [Request(u, callback=NO_CALLBACK) for u in urls] - def file_downloaded(self, response, request, info, *, item=None): + def file_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) checksum = _md5sum(buf) @@ -563,12 +678,21 @@ class FilesPipeline(MediaPipeline): self.store.persist_file(path, buf, info) return checksum - def item_completed(self, results, item, info): + def item_completed( + self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None, *, item=None): + def file_path( + self, + request: Request, + response: Optional[Response] = None, + info: Optional[MediaPipeline.SpiderInfo] = None, + *, + item: Any = None, + ) -> str: media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec media_ext = Path(request.url).suffix # Handles empty and wild extensions by trying to guess the @@ -577,5 +701,5 @@ class FilesPipeline(MediaPipeline): media_ext = "" media_type = mimetypes.guess_type(request.url)[0] if media_type: - media_ext = mimetypes.guess_extension(media_type) + media_ext = cast(str, mimetypes.guess_extension(media_type)) return f"full/{media_guid}{media_ext}" diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e7ef06fb3..27a57b17c 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -12,12 +12,25 @@ import warnings from contextlib import suppress from io import BytesIO from os import PathLike -from typing import TYPE_CHECKING, Dict, Tuple, Type, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + Type, + Union, + cast, +) from itemadapter import ItemAdapter +from scrapy import Spider from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import ( FileException, @@ -27,20 +40,20 @@ from scrapy.pipelines.files import ( S3FilesStore, _md5sum, ) - -# TODO: from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.media import FileInfoOrError, MediaPipeline from scrapy.settings import Settings from scrapy.utils.python import get_func_args, to_bytes if TYPE_CHECKING: # typing.Self requires Python 3.11 + from PIL import Image from typing_extensions import Self class NoimagesDrop(DropItem): """Product with no images exception""" - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): warnings.warn( "The NoimagesDrop class is deprecated", category=ScrapyDeprecationWarning, @@ -56,19 +69,22 @@ class ImageException(FileException): class ImagesPipeline(FilesPipeline): """Abstract pipeline that implement the image thumbnail generation logic""" - MEDIA_NAME = "image" + MEDIA_NAME: str = "image" # Uppercase attributes kept for backward compatibility with code that subclasses # ImagesPipeline. They may be overridden by settings. - MIN_WIDTH = 0 - MIN_HEIGHT = 0 - EXPIRES = 90 + MIN_WIDTH: int = 0 + MIN_HEIGHT: int = 0 + EXPIRES: int = 90 THUMBS: Dict[str, Tuple[int, int]] = {} DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" def __init__( - self, store_uri: Union[str, PathLike], download_func=None, settings=None + self, + store_uri: Union[str, PathLike[str]], + download_func: Optional[Callable[[Request, Spider], Response]] = None, + settings: Union[Settings, Dict[str, Any], None] = None, ): try: from PIL import Image @@ -89,27 +105,33 @@ class ImagesPipeline(FilesPipeline): base_class_name="ImagesPipeline", settings=settings, ) - self.expires = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES) + self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES) if not hasattr(self, "IMAGES_RESULT_FIELD"): - self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELD + self.IMAGES_RESULT_FIELD: str = self.DEFAULT_IMAGES_RESULT_FIELD if not hasattr(self, "IMAGES_URLS_FIELD"): - self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD + self.IMAGES_URLS_FIELD: str = self.DEFAULT_IMAGES_URLS_FIELD - self.images_urls_field = settings.get( + self.images_urls_field: str = settings.get( resolve("IMAGES_URLS_FIELD"), self.IMAGES_URLS_FIELD ) - self.images_result_field = settings.get( + self.images_result_field: str = settings.get( resolve("IMAGES_RESULT_FIELD"), self.IMAGES_RESULT_FIELD ) - self.min_width = settings.getint(resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH) - self.min_height = settings.getint(resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT) - self.thumbs = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS) + self.min_width: int = settings.getint( + resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH + ) + self.min_height: int = settings.getint( + resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT + ) + self.thumbs: Dict[str, Tuple[int, int]] = settings.get( + resolve("IMAGES_THUMBS"), self.THUMBS + ) - self._deprecated_convert_image = None + self._deprecated_convert_image: Optional[bool] = None @classmethod - def from_settings(cls, settings) -> Self: + def from_settings(cls, settings: Settings) -> Self: s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] @@ -136,11 +158,25 @@ class ImagesPipeline(FilesPipeline): store_uri = settings["IMAGES_STORE"] return cls(store_uri, settings=settings) - def file_downloaded(self, response, request, info, *, item=None): + def file_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: return self.image_downloaded(response, request, info, item=item) - def image_downloaded(self, response, request, info, *, item=None): - checksum = None + def image_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: + checksum: Optional[str] = None for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) @@ -153,9 +189,17 @@ class ImagesPipeline(FilesPipeline): meta={"width": width, "height": height}, headers={"Content-Type": "image/jpeg"}, ) + assert checksum is not None return checksum - def get_images(self, response, request, info, *, item=None): + def get_images( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> Iterable[Tuple[str, Image.Image, BytesIO]]: path = self.file_path(request, response=response, info=info, item=item) orig_image = self._Image.open(BytesIO(response.body)) @@ -196,7 +240,12 @@ class ImagesPipeline(FilesPipeline): thumb_image, thumb_buf = self.convert_image(image, size, buf) yield thumb_path, thumb_image, thumb_buf - def convert_image(self, image, size=None, response_body=None): + def convert_image( + self, + image: Image.Image, + size: Optional[Tuple[int, int]] = None, + response_body: Optional[BytesIO] = None, + ) -> Tuple[Image.Image, BytesIO]: if response_body is None: warnings.warn( f"{self.__class__.__name__}.convert_image() method called in a deprecated way, " @@ -225,7 +274,7 @@ class ImagesPipeline(FilesPipeline): # when updating the minimum requirements for Pillow. resampling_filter = self._Image.Resampling.LANCZOS except AttributeError: - resampling_filter = self._Image.ANTIALIAS + resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined] image.thumbnail(size, resampling_filter) elif response_body is not None and image.format == "JPEG": return image, response_body @@ -234,19 +283,38 @@ class ImagesPipeline(FilesPipeline): image.save(buf, "JPEG") return image, buf - def get_media_requests(self, item, info): + def get_media_requests( + self, item: Any, info: MediaPipeline.SpiderInfo + ) -> List[Request]: urls = ItemAdapter(item).get(self.images_urls_field, []) return [Request(u, callback=NO_CALLBACK) for u in urls] - def item_completed(self, results, item, info): + def item_completed( + self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None, *, item=None): + def file_path( + self, + request: Request, + response: Optional[Response] = None, + info: Optional[MediaPipeline.SpiderInfo] = None, + *, + item: Any = None, + ) -> str: image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"full/{image_guid}.jpg" - def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None): + def thumb_path( + self, + request: Request, + thumb_id: str, + response: Optional[Response] = None, + info: Optional[MediaPipeline.SpiderInfo] = None, + *, + item: Any = None, + ) -> str: thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"thumbs/{thumb_id}/{thumb_guid}.jpg" diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0e374265e..3e327105e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -4,54 +4,101 @@ import functools import logging from abc import ABC, abstractmethod from collections import defaultdict -from typing import TYPE_CHECKING +from typing import ( + TYPE_CHECKING, + Any, + Callable, + DefaultDict, + Dict, + List, + Literal, + NoReturn, + Optional, + Set, + Tuple, + TypedDict, + TypeVar, + Union, + cast, +) from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure -from scrapy.http.request import NO_CALLBACK +from scrapy import Spider +from scrapy.crawler import Crawler +from scrapy.http import Response +from scrapy.http.request import NO_CALLBACK, Request from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import defer_result, mustbe_deferred from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter +from scrapy.utils.request import RequestFingerprinter if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self +_T = TypeVar("_T") + + +class FileInfo(TypedDict): + url: str + path: str + checksum: Optional[str] + status: str + + +FileInfoOrError = Union[Tuple[Literal[True], FileInfo], Tuple[Literal[False], Failure]] + logger = logging.getLogger(__name__) class MediaPipeline(ABC): - LOG_FAILED_RESULTS = True + crawler: Crawler + _fingerprinter: RequestFingerprinter + + LOG_FAILED_RESULTS: bool = True class SpiderInfo: - def __init__(self, spider): - self.spider = spider - self.downloading = set() - self.downloaded = {} - self.waiting = defaultdict(list) + def __init__(self, spider: Spider): + self.spider: Spider = spider + self.downloading: Set[bytes] = set() + self.downloaded: Dict[bytes, Union[FileInfo, Failure]] = {} + self.waiting: DefaultDict[bytes, List[Deferred[FileInfo]]] = defaultdict( + list + ) - def __init__(self, download_func=None, settings=None): + def __init__( + self, + download_func: Optional[Callable[[Request, Spider], Response]] = None, + settings: Union[Settings, Dict[str, Any], None] = None, + ): self.download_func = download_func - self._expects_item = {} if isinstance(settings, dict) or settings is None: settings = Settings(settings) resolve = functools.partial( self._key_for_pipe, base_class_name="MediaPipeline", settings=settings ) - self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False) + self.allow_redirects: bool = settings.getbool( + resolve("MEDIA_ALLOW_REDIRECTS"), False + ) self._handle_statuses(self.allow_redirects) - def _handle_statuses(self, allow_redirects): + def _handle_statuses(self, allow_redirects: bool) -> None: self.handle_httpstatus_list = None if allow_redirects: self.handle_httpstatus_list = SequenceExclude(range(300, 400)) - def _key_for_pipe(self, key, base_class_name=None, settings=None): + def _key_for_pipe( + self, + key: str, + base_class_name: Optional[str] = None, + settings: Optional[Settings] = None, + ) -> str: class_name = self.__class__.__name__ formatted_key = f"{class_name.upper()}_{key}" if ( @@ -64,26 +111,34 @@ class MediaPipeline(ABC): return formatted_key @classmethod - def from_crawler(cls, crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: + pipe: Self try: pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined] except AttributeError: pipe = cls() pipe.crawler = crawler + assert crawler.request_fingerprinter pipe._fingerprinter = crawler.request_fingerprinter return pipe - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: self.spiderinfo = self.SpiderInfo(spider) - def process_item(self, item, spider): + def process_item( + self, item: Any, spider: Spider + ) -> Deferred[List[FileInfoOrError]]: info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) dlist = [self._process_request(r, info, item) for r in requests] - dfd = DeferredList(dlist, consumeErrors=True) + dfd = cast( + "Deferred[List[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True) + ) return dfd.addCallback(self.item_completed, item, info) - def _process_request(self, request, info, item): + def _process_request( + self, request: Request, info: SpiderInfo, item: Any + ) -> Deferred[FileInfo]: fp = self._fingerprinter.fingerprint(request) eb = request.errback request.callback = NO_CALLBACK @@ -97,7 +152,7 @@ class MediaPipeline(ABC): return d # Otherwise, wait for result - wad = Deferred() + wad: Deferred[FileInfo] = Deferred() if eb: wad.addErrback(eb) info.waiting[fp].append(wad) @@ -108,36 +163,48 @@ class MediaPipeline(ABC): # Download request checking media_to_download hook output first info.downloading.add(fp) - dfd = mustbe_deferred(self.media_to_download, request, info, item=item) - dfd.addCallback(self._check_media_to_download, request, info, item=item) - dfd.addErrback(self._log_exception) - dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) - return dfd.addBoth(lambda _: wad) # it must return wad at last + dfd: Deferred[Optional[FileInfo]] = mustbe_deferred( + self.media_to_download, request, info, item=item + ) + dfd2: Deferred[FileInfo] = dfd.addCallback( + self._check_media_to_download, request, info, item=item + ) + dfd2.addErrback(self._log_exception) + dfd2.addBoth(self._cache_result_and_execute_waiters, fp, info) + return dfd2.addBoth(lambda _: wad) # it must return wad at last - def _log_exception(self, result): + def _log_exception(self, result: Failure) -> Failure: logger.exception(result) return result - def _modify_media_request(self, request): + def _modify_media_request(self, request: Request) -> None: if self.handle_httpstatus_list: request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list else: request.meta["handle_httpstatus_all"] = True - def _check_media_to_download(self, result, request, info, item): + def _check_media_to_download( + self, result: Optional[FileInfo], request: Request, info: SpiderInfo, item: Any + ) -> Union[FileInfo, Deferred[FileInfo]]: if result is not None: return result + dfd: Deferred[Response] if self.download_func: # this ugly code was left only to support tests. TODO: remove dfd = mustbe_deferred(self.download_func, request, info.spider) else: self._modify_media_request(request) + assert self.crawler.engine dfd = self.crawler.engine.download(request) - dfd.addCallback(self.media_downloaded, request, info, item=item) - dfd.addErrback(self.media_failed, request, info) - return dfd + dfd2: Deferred[FileInfo] = dfd.addCallback( + self.media_downloaded, request, info, item=item + ) + dfd2.addErrback(self.media_failed, request, info) + return dfd2 - def _cache_result_and_execute_waiters(self, result, fp, info): + def _cache_result_and_execute_waiters( + self, result: Union[FileInfo, Failure], fp: bytes, info: SpiderInfo + ) -> None: if isinstance(result, Failure): # minimize cached information for failure result.cleanFailure() @@ -176,30 +243,44 @@ class MediaPipeline(ABC): # Overridable Interface @abstractmethod - def media_to_download(self, request, info, *, item=None): + def media_to_download( + self, request: Request, info: SpiderInfo, *, item: Any = None + ) -> Deferred[Optional[FileInfo]]: """Check request before starting download""" raise NotImplementedError() @abstractmethod - def get_media_requests(self, item, info): + def get_media_requests(self, item: Any, info: SpiderInfo) -> List[Request]: """Returns the media requests to download""" raise NotImplementedError() @abstractmethod - def media_downloaded(self, response, request, info, *, item=None): + def media_downloaded( + self, + response: Response, + request: Request, + info: SpiderInfo, + *, + item: Any = None, + ) -> FileInfo: """Handler for success downloads""" raise NotImplementedError() @abstractmethod - def media_failed(self, failure, request, info): + def media_failed( + self, failure: Failure, request: Request, info: SpiderInfo + ) -> NoReturn: """Handler for failed downloads""" raise NotImplementedError() - def item_completed(self, results, item, info): + def item_completed( + self, results: List[FileInfoOrError], item: Any, info: SpiderInfo + ) -> Any: """Called per item when all media requests has been processed""" if self.LOG_FAILED_RESULTS: for ok, value in results: if not ok: + assert isinstance(value, Failure) logger.error( "%(class)s found errors processing %(item)s", {"class": self.__class__.__name__, "item": item}, @@ -209,6 +290,13 @@ class MediaPipeline(ABC): return item @abstractmethod - def file_path(self, request, response=None, info=None, *, item=None): + def file_path( + self, + request: Request, + response: Optional[Response] = None, + info: Optional[SpiderInfo] = None, + *, + item: Any = None, + ) -> str: """Returns the path where downloaded media should be stored""" raise NotImplementedError() From 3f76853bd27d84f53ebaaa97cb819e8a29195a89 Mon Sep 17 00:00:00 2001 From: Suvan Banerjee Date: Wed, 5 Jun 2024 10:04:46 +0530 Subject: [PATCH 1477/2083] Handle AttributeError: 'NoneType' in contract parsing (#6388) --- scrapy/contracts/__init__.py | 3 ++- tests/test_contracts.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index b300b8457..27bc2fcba 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -120,7 +120,8 @@ class ContractsManager: if line.startswith("@"): m = re.match(r"@(\w+)\s*(.*)", line) - assert m is not None + if m is None: + continue name, args = m.groups() args = re.split(r"\s+", args) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 1459e0b5f..c9c12f0d8 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -182,6 +182,19 @@ class TestSpider(Spider): """ pass + def invalid_regex(self, response): + """method with invalid regex + @ Scrapy is awsome + """ + pass + + def invalid_regex_with_valid_contract(self, response): + """method with invalid regex + @ scrapy is awsome + @url http://scrapy.org + """ + pass + class CustomContractSuccessSpider(Spider): name = "custom_contract_success_spider" @@ -385,6 +398,21 @@ class ContractsManagerTest(unittest.TestCase): message = "ContractFail: Missing fields: name, url" assert message in self.results.failures[-1][-1] + def test_regex(self): + spider = TestSpider() + response = ResponseMock() + + # invalid regex + request = self.conman.from_method(spider.invalid_regex, self.results) + self.should_succeed() + + # invalid regex with valid contract + request = self.conman.from_method( + spider.invalid_regex_with_valid_contract, self.results + ) + self.should_succeed() + request.callback(response) + def test_custom_contracts(self): self.conman.from_spider(CustomContractSuccessSpider(), self.results) self.should_succeed() From 2e214210f6707181a863dbceabf2d34e767396cb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 2 Jun 2024 01:48:37 +0500 Subject: [PATCH 1478/2083] Add parameters to iterable generics, replace generators with iterables. --- scrapy/commands/parse.py | 8 ++-- scrapy/core/engine.py | 5 ++- scrapy/core/scraper.py | 6 ++- scrapy/core/spidermw.py | 70 ++++++++++++++++++++------------ scrapy/http/response/__init__.py | 3 +- scrapy/http/response/text.py | 3 +- scrapy/utils/iterators.py | 28 ++++++------- scrapy/utils/misc.py | 6 +-- scrapy/utils/python.py | 32 +++++++-------- scrapy/utils/request.py | 5 +-- scrapy/utils/sitemap.py | 4 +- scrapy/utils/spider.py | 13 +++--- tests/test_commands.py | 4 +- 13 files changed, 103 insertions(+), 84 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 2453c0d39..f916a3e75 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -140,13 +140,13 @@ class Command(BaseRunSpiderCommand): @overload def iterate_spider_output( - self, result: Union[AsyncGenerator, CoroutineType] - ) -> Deferred: ... + self, result: Union[AsyncGenerator[_T, None], CoroutineType[Any, Any, _T]] + ) -> Deferred[_T]: ... @overload - def iterate_spider_output(self, result: _T) -> Iterable: ... + def iterate_spider_output(self, result: _T) -> Iterable[Any]: ... - def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]: + def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]: if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b342ad7a3..dededf99d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -372,7 +372,10 @@ class ExecutionEngine: @inlineCallbacks def open_spider( - self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True + self, + spider: Spider, + start_requests: Iterable[Request] = (), + close_if_idle: bool = True, ) -> Generator[Deferred, Any, None]: if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 566e6628b..3b7492838 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -16,6 +16,7 @@ from typing import ( Set, Tuple, Type, + TypeVar, Union, cast, ) @@ -47,6 +48,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler +_T = TypeVar("_T") QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] @@ -256,14 +258,14 @@ class Scraper: def handle_spider_output( self, - result: Union[Iterable, AsyncIterable], + result: Union[Iterable[_T], AsyncIterable[_T]], request: Request, response: Response, spider: Spider, ) -> Deferred: if not result: return defer_succeed(None) - it: Union[Iterable, AsyncIterable] + it: Union[Iterable[_T], AsyncIterable[_T]] if isinstance(result, AsyncIterable): it = aiter_errback( result, self.handle_spider_error, request, response, spider diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 2cef2e1dd..cb1a93a68 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -9,7 +9,6 @@ from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import ( Any, - AsyncGenerator, AsyncIterable, Callable, Generator, @@ -17,6 +16,7 @@ from typing import ( List, Optional, Tuple, + TypeVar, Union, cast, ) @@ -42,6 +42,7 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain logger = logging.getLogger(__name__) +_T = TypeVar("_T") ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] @@ -98,31 +99,39 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - iterable: Union[Iterable, AsyncIterable], + iterable: Union[Iterable[_T], AsyncIterable[_T]], exception_processor_index: int, - recover_to: Union[MutableChain, MutableAsyncChain], - ) -> Union[Generator, AsyncGenerator]: - def process_sync(iterable: Iterable) -> Generator: + recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]], + ) -> Union[Iterable[_T], AsyncIterable[_T]]: + def process_sync(iterable: Iterable[_T]) -> Iterable[_T]: try: yield from iterable except Exception as ex: - exception_result = self._process_spider_exception( - response, spider, Failure(ex), exception_processor_index + exception_result = cast( + Union[Failure, MutableChain[_T]], + self._process_spider_exception( + response, spider, Failure(ex), exception_processor_index + ), ) if isinstance(exception_result, Failure): raise + assert isinstance(recover_to, MutableChain) recover_to.extend(exception_result) - async def process_async(iterable: AsyncIterable) -> AsyncGenerator: + async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]: try: async for r in iterable: yield r except Exception as ex: - exception_result = self._process_spider_exception( - response, spider, Failure(ex), exception_processor_index + exception_result = cast( + Union[Failure, MutableAsyncChain[_T]], + self._process_spider_exception( + response, spider, Failure(ex), exception_processor_index + ), ) if isinstance(exception_result, Failure): raise + assert isinstance(recover_to, MutableAsyncChain) recover_to.extend(exception_result) if isinstance(iterable, AsyncIterable): @@ -135,7 +144,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, _failure: Failure, start_index: int = 0, - ) -> Union[Failure, MutableChain]: + ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]: exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -151,14 +160,18 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - dfd: Deferred = self._process_spider_output( - response, spider, result, method_index + 1 + dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = ( + self._process_spider_output( + response, spider, result, method_index + 1 + ) ) # _process_spider_output() returns a Deferred only because of downgrading so this can be # simplified when downgrading is removed. if dfd.called: # the result is available immediately if _process_spider_output didn't do downgrading - return cast(MutableChain, dfd.result) + return cast( + Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result + ) # we forbid waiting here because otherwise we would need to return a deferred from # _process_spider_exception too, which complicates the architecture msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" @@ -181,12 +194,12 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Union[Iterable, AsyncIterable], + result: Union[Iterable[_T], AsyncIterable[_T]], start_index: int = 0, - ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]: + ) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - recovered: Union[MutableChain, MutableAsyncChain] + recovered: Union[MutableChain[_T], MutableAsyncChain[_T]] last_result_is_async = isinstance(result, AsyncIterable) if last_result_is_async: recovered = MutableAsyncChain() @@ -237,7 +250,9 @@ class SpiderMiddlewareManager(MiddlewareManager): # might fail directly if the output value is not a generator result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result = self._process_spider_exception( + exception_result: Union[ + Failure, MutableChain[_T], MutableAsyncChain[_T] + ] = self._process_spider_exception( response, spider, Failure(ex), method_index + 1 ) if isinstance(exception_result, Failure): @@ -267,9 +282,12 @@ class SpiderMiddlewareManager(MiddlewareManager): return MutableChain(result, recovered) # type: ignore[arg-type] async def _process_callback_output( - self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] - ) -> Union[MutableChain, MutableAsyncChain]: - recovered: Union[MutableChain, MutableAsyncChain] + self, + response: Response, + spider: Spider, + result: Union[Iterable[_T], AsyncIterable[_T]], + ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]: + recovered: Union[MutableChain[_T], MutableAsyncChain[_T]] if isinstance(result, AsyncIterable): recovered = MutableAsyncChain() else: @@ -293,14 +311,16 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, ) -> Deferred: async def process_callback_output( - result: Union[Iterable, AsyncIterable] - ) -> Union[MutableChain, MutableAsyncChain]: + result: Union[Iterable[_T], AsyncIterable[_T]] + ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]: return await self._process_callback_output(response, spider, result) - def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: + def process_spider_exception( + _failure: Failure, + ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]: return self._process_spider_exception(response, spider, _failure) - dfd = mustbe_deferred( + dfd: Deferred = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 166c4de97..daf193f59 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -14,7 +14,6 @@ from typing import ( AnyStr, Callable, Dict, - Generator, Iterable, List, Mapping, @@ -242,7 +241,7 @@ class Response(object_ref): errback: Optional[Callable] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, - ) -> Generator[Request, None, None]: + ) -> Iterable[Request]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 44c36b682..df4d90829 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -15,7 +15,6 @@ from typing import ( AnyStr, Callable, Dict, - Generator, Iterable, List, Mapping, @@ -246,7 +245,7 @@ class TextResponse(Response): flags: Optional[List[str]] = None, css: Optional[str] = None, xpath: Optional[str] = None, - ) -> Generator[Request, None, None]: + ) -> Iterable[Request]: """ A generator that produces :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index cd6e9d04e..41a842386 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -6,8 +6,7 @@ from typing import ( Any, Callable, Dict, - Generator, - Iterable, + Iterator, List, Literal, Optional, @@ -22,14 +21,12 @@ from lxml import etree # nosec from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse from scrapy.selector import Selector -from scrapy.utils.python import re_rsearch, to_unicode +from scrapy.utils.python import re_rsearch logger = logging.getLogger(__name__) -def xmliter( - obj: Union[Response, str, bytes], nodename: str -) -> Generator[Selector, Any, None]: +def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selector]: """Return a iterator of Selector's over all nodes of a XML document, given the name of the node to iterate. Useful for parsing XML feeds. @@ -90,7 +87,7 @@ def xmliter_lxml( nodename: str, namespace: Optional[str] = None, prefix: str = "x", -) -> Generator[Selector, Any, None]: +) -> Iterator[Selector]: reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( @@ -168,7 +165,7 @@ def csviter( headers: Optional[List[str]] = None, encoding: Optional[str] = None, quotechar: Optional[str] = None, -) -> Generator[Dict[str, str], Any, None]: +) -> Iterator[Dict[str, str]]: """Returns an iterator of dictionaries from the given csv object obj can be: @@ -184,10 +181,13 @@ def csviter( quotechar is the character used to enclosure fields on the given obj. """ - encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8" - - def row_to_unicode(row_: Iterable) -> List[str]: - return [to_unicode(field, encoding) for field in row_] + if encoding is not None: + warn( + "The encoding argument of csviter() is ignored and will be removed" + " in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) lines = StringIO(_body_or_str(obj, unicode=True)) @@ -200,13 +200,11 @@ def csviter( if not headers: try: - row = next(csv_r) + headers = next(csv_r) except StopIteration: return - headers = row_to_unicode(row) for row in csv_r: - row = row_to_unicode(row) if len(row) != len(headers): logger.warning( "ignoring row %(csvlnum)d (length: %(csvrow)d, " diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 49f36de2d..3d11c1035 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -20,8 +20,8 @@ from typing import ( Any, Callable, Deque, - Generator, Iterable, + Iterator, List, Optional, Type, @@ -227,7 +227,7 @@ def build_from_settings( @contextmanager -def set_environ(**kwargs: str) -> Generator[None, Any, None]: +def set_environ(**kwargs: str) -> Iterator[None]: """Temporarily set environment variables inside the context manager and fully restore previous environment afterwards """ @@ -244,7 +244,7 @@ def set_environ(**kwargs: str) -> Generator[None, Any, None]: os.environ[k] = v -def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: +def walk_callable(node: ast.AST) -> Iterable[ast.AST]: """Similar to ``ast.walk``, but walks only function body and skips nested functions defined within the node. """ diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 37a84a350..059d8e04d 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -15,12 +15,10 @@ from itertools import chain from typing import ( TYPE_CHECKING, Any, - AsyncGenerator, AsyncIterable, AsyncIterator, Callable, Dict, - Generator, Iterable, Iterator, List, @@ -163,7 +161,7 @@ def re_rsearch( the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter() -> Generator[Tuple[str, int], Any, None]: + def _chunk_iter() -> Iterable[Tuple[str, int]]: offset = len(text) while True: offset -= chunk_size * 1024 @@ -351,43 +349,45 @@ else: gc.collect() -class MutableChain(Iterable): +class MutableChain(Iterable[_T]): """ Thin wrapper around itertools.chain, allowing to add iterables "in-place" """ - def __init__(self, *args: Iterable): - self.data = chain.from_iterable(args) + def __init__(self, *args: Iterable[_T]): + self.data: Iterator[_T] = chain.from_iterable(args) - def extend(self, *iterables: Iterable) -> None: + def extend(self, *iterables: Iterable[_T]) -> None: self.data = chain(self.data, chain.from_iterable(iterables)) - def __iter__(self) -> Iterator: + def __iter__(self) -> Iterator[_T]: return self - def __next__(self) -> Any: + def __next__(self) -> _T: return next(self.data) -async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator: +async def _async_chain( + *iterables: Union[Iterable[_T], AsyncIterable[_T]] +) -> AsyncIterator[_T]: for it in iterables: async for o in as_async_generator(it): yield o -class MutableAsyncChain(AsyncIterable): +class MutableAsyncChain(AsyncIterable[_T]): """ Similar to MutableChain but for async iterables """ - def __init__(self, *args: Union[Iterable, AsyncIterable]): - self.data = _async_chain(*args) + def __init__(self, *args: Union[Iterable[_T], AsyncIterable[_T]]): + self.data: AsyncIterator[_T] = _async_chain(*args) - def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None: + def extend(self, *iterables: Union[Iterable[_T], AsyncIterable[_T]]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) - def __aiter__(self) -> AsyncIterator: + def __aiter__(self) -> AsyncIterator[_T]: return self - async def __anext__(self) -> Any: + async def __anext__(self) -> _T: return await self.data.__anext__() diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 42a6537a8..45b8008f4 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -12,7 +12,6 @@ from typing import ( TYPE_CHECKING, Any, Dict, - Generator, Iterable, List, Optional, @@ -40,9 +39,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler -def _serialize_headers( - headers: Iterable[bytes], request: Request -) -> Generator[bytes, Any, None]: +def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]: for header in headers: if header in request.headers: yield header diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index cf429043d..7a91afe59 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ -from typing import Any, Dict, Generator, Iterator, Optional, Union +from typing import Any, Dict, Iterable, Iterator, Optional, Union from urllib.parse import urljoin import lxml.etree # nosec @@ -42,7 +42,7 @@ class Sitemap: def sitemap_urls_from_robots( robots_text: str, base_url: Optional[str] = None -) -> Generator[str, Any, None]: +) -> Iterable[str]: """Return an iterator over all sitemap urls contained in the given robots.txt file """ diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index cbbb01d85..b05135c04 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -7,7 +7,6 @@ from typing import ( TYPE_CHECKING, Any, AsyncGenerator, - Generator, Iterable, Literal, Optional, @@ -34,18 +33,20 @@ _T = TypeVar("_T") # https://stackoverflow.com/questions/60222982 @overload -def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap] +def iterate_spider_output(result: AsyncGenerator[_T, None]) -> AsyncGenerator[_T, None]: ... # type: ignore[overload-overlap] @overload -def iterate_spider_output(result: CoroutineType) -> Deferred: ... +def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ... @overload -def iterate_spider_output(result: _T) -> Iterable: ... +def iterate_spider_output(result: _T) -> Iterable[Any]: ... -def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: +def iterate_spider_output( + result: Any, +) -> Union[Iterable[Any], AsyncGenerator[_T, None], Deferred[_T]]: if inspect.isasyncgen(result): return result if inspect.iscoroutine(result): @@ -55,7 +56,7 @@ def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferr return arg_to_iter(deferred_from_coro(result)) -def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]: +def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ diff --git a/tests/test_commands.py b/tests/test_commands.py index b9d468c66..857a56b73 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -13,7 +13,7 @@ from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import TemporaryFile, mkdtemp from threading import Timer -from typing import Dict, Generator, Optional, Union +from typing import Dict, Iterator, Optional, Union from unittest import skipIf from pytest import mark @@ -674,7 +674,7 @@ class BadSpider(scrapy.Spider): """ @contextmanager - def _create_file(self, content, name=None) -> Generator[str, None, None]: + def _create_file(self, content, name=None) -> Iterator[str]: tmpdir = Path(self.mktemp()) tmpdir.mkdir() if name: From de146ad7cef9e3478290be021129979f69fc6d03 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 5 Jun 2024 22:09:19 +0500 Subject: [PATCH 1479/2083] Bump typing deps. --- scrapy/extensions/httpcache.py | 5 ++--- scrapy/http/headers.py | 3 +-- tox.ini | 9 ++++----- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 3f4af42b7..b7219bf07 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -370,12 +370,11 @@ class FilesystemCacheStorage: with self._open(rpath / "pickled_meta", "wb") as f: pickle.dump(metadata, f, protocol=4) with self._open(rpath / "response_headers", "wb") as f: - # headers_dict_to_raw() needs a better type hint - f.write(cast(bytes, headers_dict_to_raw(response.headers))) + f.write(headers_dict_to_raw(response.headers)) with self._open(rpath / "response_body", "wb") as f: f.write(response.body) with self._open(rpath / "request_headers", "wb") as f: - f.write(cast(bytes, headers_dict_to_raw(request.headers))) + f.write(headers_dict_to_raw(request.headers)) with self._open(rpath / "request_body", "wb") as f: f.write(request.body) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 73aee7178..85b9229d3 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -118,8 +118,7 @@ class Headers(CaselessDict): ] def to_string(self) -> bytes: - # cast() can be removed if the headers_dict_to_raw() hint is improved - return cast(bytes, headers_dict_to_raw(self)) + return headers_dict_to_raw(self) def to_unicode_dict(self) -> CaseInsensitiveDict: """Return headers as a CaseInsensitiveDict with str keys diff --git a/tox.ini b/tox.ini index 5a5e80496..023a86c5a 100644 --- a/tox.ini +++ b/tox.ini @@ -47,18 +47,17 @@ install_command = basepython = python3 deps = mypy==1.10.0 - typing-extensions==4.11.0 + typing-extensions==4.12.1 types-lxml==2024.4.14 types-Pygments==2.18.0.20240506 types-pyOpenSSL==24.1.0.20240425 - types-setuptools==69.5.0.20240518 + types-setuptools==70.0.0.20240524 botocore-stubs==1.34.94 - boto3-stubs[s3]==1.34.108 + boto3-stubs[s3]==1.34.119 attrs >= 18.2.0 Pillow >= 10.3.0 pytest >= 8.2.0 - # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 - w3lib >= 2.1.2 + w3lib >= 2.2.0 commands = mypy {posargs: scrapy tests} From 262c10d85bd34732b0c692bdc8d16375d83a178f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 5 Jun 2024 22:11:34 +0500 Subject: [PATCH 1480/2083] Use typing.Coroutine instead of types.CoroutineType. --- scrapy/commands/parse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index f916a3e75..ce6f4dc51 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -3,11 +3,11 @@ import functools import inspect import json import logging -from types import CoroutineType from typing import ( Any, AsyncGenerator, Callable, + Coroutine, Dict, Iterable, List, @@ -140,7 +140,7 @@ class Command(BaseRunSpiderCommand): @overload def iterate_spider_output( - self, result: Union[AsyncGenerator[_T, None], CoroutineType[Any, Any, _T]] + self, result: Union[AsyncGenerator[_T, None], Coroutine[Any, Any, _T]] ) -> Deferred[_T]: ... @overload From 480a11b68bee19162cc0da59e9bed42b29bc9cfe Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 5 Jun 2024 22:48:16 +0500 Subject: [PATCH 1481/2083] Add mssing __future__ imports. --- scrapy/commands/parse.py | 2 ++ scrapy/core/spidermw.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ce6f4dc51..3320a1ee4 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import functools import inspect diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index cb1a93a68..58873f0d9 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,6 +4,8 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +from __future__ import annotations + import logging from inspect import isasyncgenfunction, iscoroutine from itertools import islice From feb0b8f7dcb78c3df012085f00b992a7fac81f7a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 5 Jun 2024 22:57:18 +0500 Subject: [PATCH 1482/2083] Add pyupgrade. --- .pre-commit-config.yaml | 6 ++ scrapy/core/downloader/contextfactory.py | 8 +-- scrapy/downloadermiddlewares/retry.py | 4 +- scrapy/extensions/debug.py | 2 +- scrapy/http/request/__init__.py | 6 +- scrapy/http/request/rpc.py | 2 +- scrapy/settings/__init__.py | 6 +- scrapy/utils/benchserver.py | 4 +- scrapy/utils/request.py | 6 +- tests/mockserver.py | 6 +- tests/test_downloadermiddleware_cookies.py | 4 +- tests/test_downloadermiddleware_redirect.py | 2 +- tests/test_downloadermiddleware_robotstxt.py | 4 +- tests/test_feedexport.py | 58 ++++++++++---------- tests/test_http_response.py | 4 +- tests/test_pipeline_crawl.py | 4 +- tests/test_pipeline_images.py | 2 +- tests/test_responsetypes.py | 2 +- tests/test_robotstxt_interface.py | 30 ++++------ 19 files changed, 74 insertions(+), 86 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a911d4cfe..f76a04ca1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,3 +22,9 @@ repos: - id: blacken-docs additional_dependencies: - black==24.2.0 +- repo: https://github.com/asottile/pyupgrade + rev: v3.15.2 + hooks: + - id: pyupgrade + args: [--py38-plus, --keep-runtime-typing] + exclude: scrapy/__init__.py diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 0e77cd2fe..9f6edb630 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -107,7 +107,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: return ScrapyClientTLSOptions( hostname.decode("ascii"), self.getContext(), @@ -134,7 +134,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: # trustRoot set to platformTrust() will use the platform's root CAs. # # This means that a website like https://www.cacert.org will be rejected @@ -158,8 +158,8 @@ class AcceptableProtocolsContextFactory: self._wrapped_context_factory: Any = context_factory self._acceptable_protocols: List[bytes] = acceptable_protocols - def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": - options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc( + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: + options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc( hostname, port ) _setAcceptableProtocols(options._ctx, self._acceptable_protocols) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 46587a898..0637f09d4 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -147,9 +147,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured self.max_retry_times = settings.getint("RETRY_TIMES") - self.retry_http_codes = set( - int(x) for x in settings.getlist("RETRY_HTTP_CODES") - ) + self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") try: diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 26726b662..a0fc7b99f 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -55,7 +55,7 @@ class StackTraceDump: ) def _thread_stacks(self) -> str: - id2name = dict((th.ident, th.name) for th in threading.enumerate()) + id2name = {th.ident: th.name for th in threading.enumerate()} dumps = "" for id_, frame in sys._current_frames().items(): name = id2name.get(id_, "") diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 77149333c..3da2e111d 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -189,10 +189,10 @@ class Request(object_ref): def __repr__(self) -> str: return f"<{self.method} {self.url}>" - def copy(self) -> "Request": + def copy(self) -> Request: return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> "Request": + def replace(self, *args: Any, **kwargs: Any) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) @@ -237,7 +237,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]: + def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index e20e7c438..096ecd370 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -21,7 +21,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any): if "body" not in kwargs and "params" in kwargs: - kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) + kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs} kwargs["body"] = xmlrpclib.dumps(**kw) # spec defines that requests must use POST method diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 4448b6f4b..ea1db03f1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -275,7 +275,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): assert isinstance(value, (dict, list)) return copy.deepcopy(value) - def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings": + def getwithbase(self, name: _SettingsKeyT) -> BaseSettings: """Get a composition of a dictionary-like setting and its `_BASE` counterpart. @@ -438,7 +438,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): if self.frozen: raise TypeError("Trying to modify an immutable Settings object") - def copy(self) -> "Self": + def copy(self) -> Self: """ Make a deep copy of current settings. @@ -460,7 +460,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self.frozen = True - def frozencopy(self) -> "Self": + def frozencopy(self) -> Self: """ Return an immutable copy of the current settings. diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index e9ea51aa1..550516141 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -22,9 +22,7 @@ class Root(Resource): for nl in nlist: args["n"] = nl argstr = urlencode(args, doseq=True) - request.write( - f"follow {nl}
".encode("utf8") - ) + request.write(f"follow {nl}
".encode()) request.write(b"") return b"" diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 42a6537a8..aa0b90ee8 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -49,9 +49,9 @@ def _serialize_headers( yield from request.headers.getlist(header) -_fingerprint_cache: ( - "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" -) +_fingerprint_cache: WeakKeyDictionary[ + Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes] +] _fingerprint_cache = WeakKeyDictionary() diff --git a/tests/mockserver.py b/tests/mockserver.py index 647b0682e..233f6b934 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -189,10 +189,10 @@ class Raw(LeafResource): class Echo(LeafResource): def render_GET(self, request): output = { - "headers": dict( - (to_unicode(k), [to_unicode(v) for v in vs]) + "headers": { + to_unicode(k): [to_unicode(v) for v in vs] for k, vs in request.requestHeaders.getAllRawHeaders() - ), + }, "body": to_unicode(request.content.read()), } return to_bytes(json.dumps(output)) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 5eccd396a..6e343d035 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -362,7 +362,7 @@ class CookiesMiddlewareTest(TestCase): def test_request_cookies_encoding(self): # 1) UTF8-encoded bytes - req1 = Request("http://example.org", cookies={"a": "á".encode("utf8")}) + req1 = Request("http://example.org", cookies={"a": "á".encode()}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") @@ -379,7 +379,7 @@ class CookiesMiddlewareTest(TestCase): @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes - req1 = Request("http://example.org", headers={"Cookie": "a=á".encode("utf8")}) + req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1") diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 4bfd34fe2..e37da9715 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1125,7 +1125,7 @@ class RedirectMiddlewareTest(Base.Test): def test_utf8_location(self): req = Request("http://scrapytest.org/first") - utf8_location = "/ação".encode("utf-8") # header using UTF-8 encoding + utf8_location = "/ação".encode() # header using UTF-8 encoding resp = Response( "http://scrapytest.org/first", headers={"Location": utf8_location}, diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 26898a6a1..e166cc000 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -40,9 +40,7 @@ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: Disallow: /wiki/Käyttäjä: User-Agent: UnicödeBöt Disallow: /some/randome/page.html -""".encode( - "utf-8" - ) +""".encode() response = TextResponse("http://site.local/robots.txt", body=ROBOTS) def return_response(request): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d7560b5ff..3771df8f1 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1359,13 +1359,13 @@ class FeedExportTest(FeedExportTestBase): items = [dict({"foo": "Test\xd6"})] formats = { - "json": '[{"foo": "Test\\u00d6"}]'.encode("utf-8"), - "jsonlines": '{"foo": "Test\\u00d6"}\n'.encode("utf-8"), + "json": b'[{"foo": "Test\\u00d6"}]', + "jsonlines": b'{"foo": "Test\\u00d6"}\n', "xml": ( '\n' "Test\xd6" - ).encode("utf-8"), - "csv": "foo\r\nTest\xd6\r\n".encode("utf-8"), + ).encode(), + "csv": "foo\r\nTest\xd6\r\n".encode(), } for fmt, expected in formats.items(): @@ -1379,13 +1379,13 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(expected, data[fmt]) formats = { - "json": '[{"foo": "Test\xd6"}]'.encode("latin-1"), - "jsonlines": '{"foo": "Test\xd6"}\n'.encode("latin-1"), + "json": b'[{"foo": "Test\xd6"}]', + "jsonlines": b'{"foo": "Test\xd6"}\n', "xml": ( - '\n' - "Test\xd6" - ).encode("latin-1"), - "csv": "foo\r\nTest\xd6\r\n".encode("latin-1"), + b'\n' + b"Test\xd6" + ), + "csv": b"foo\r\nTest\xd6\r\n", } for fmt, expected in formats.items(): @@ -1404,12 +1404,12 @@ class FeedExportTest(FeedExportTestBase): items = [dict({"foo": "FOO", "bar": "BAR"})] formats = { - "json": '[\n{"bar": "BAR"}\n]'.encode("utf-8"), + "json": b'[\n{"bar": "BAR"}\n]', "xml": ( - '\n' - "\n \n FOO\n \n" - ).encode("latin-1"), - "csv": "bar,foo\r\nBAR,FOO\r\n".encode("utf-8"), + b'\n' + b"\n \n FOO\n \n" + ), + "csv": b"bar,foo\r\nBAR,FOO\r\n", } settings = { @@ -1663,8 +1663,8 @@ class FeedExportTest(FeedExportTestBase): def test_extend_kwargs(self): items = [{"foo": "FOO", "bar": "BAR"}] - expected_with_title_csv = "foo,bar\r\nFOO,BAR\r\n".encode("utf-8") - expected_without_title_csv = "FOO,BAR\r\n".encode("utf-8") + expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n" + expected_without_title_csv = b"FOO,BAR\r\n" test_cases = [ # with title { @@ -2519,22 +2519,22 @@ class BatchDeliveriesTest(FeedExportTestBase): formats = { "json": [ - '[\n{"bar": "BAR"}\n]'.encode("utf-8"), - '[\n{"bar": "BAR1"}\n]'.encode("utf-8"), + b'[\n{"bar": "BAR"}\n]', + b'[\n{"bar": "BAR1"}\n]', ], "xml": [ ( - '\n' - "\n \n FOO\n \n" - ).encode("latin-1"), + b'\n' + b"\n \n FOO\n \n" + ), ( - '\n' - "\n \n FOO1\n \n" - ).encode("latin-1"), + b'\n' + b"\n \n FOO1\n \n" + ), ], "csv": [ - "foo,bar\r\nFOO,BAR\r\n".encode("utf-8"), - "foo,bar\r\nFOO1,BAR1\r\n".encode("utf-8"), + b"foo,bar\r\nFOO,BAR\r\n", + b"foo,bar\r\nFOO1,BAR1\r\n", ], } @@ -2577,8 +2577,8 @@ class BatchDeliveriesTest(FeedExportTestBase): items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})] formats = { "json": [ - '[{"foo": "FOO"}]'.encode("utf-8"), - '[{"foo": "FOO1"}]'.encode("utf-8"), + b'[{"foo": "FOO"}]', + b'[{"foo": "FOO1"}]', ], } settings = { diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 80d46274b..b8a277295 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -728,9 +728,7 @@ class TextResponseTest(BaseResponseTest): resp1 = self.response_class( "http://example.com", encoding="utf8", - body='click me'.encode( - "utf8" - ), + body='click me'.encode(), ) req = self._assert_followed_url( resp1.css("a")[0], diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 5a9a217ce..cd3442dd4 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -107,9 +107,7 @@ class FileDownloadCrawlTestCase(TestCase): # check that the images/files checksums are what we know they should be if self.expected_checksums is not None: - checksums = set( - i["checksum"] for item in items for i in item[self.media_key] - ) + checksums = {i["checksum"] for item in items for i in item[self.media_key]} self.assertEqual(checksums, self.expected_checksums) # check that the image files where actually written to the media store diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 18a2454b3..7d7c78920 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -628,7 +628,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): class NoimagesDropTestCase(unittest.TestCase): def test_deprecation_warning(self): - arg = str() + arg = "" with warnings.catch_warnings(record=True) as w: NoimagesDrop(arg) self.assertEqual(len(w), 1) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 2633cca5b..7be8150fc 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -29,7 +29,7 @@ class ResponseTypesTest(unittest.TestCase): mappings = [ (b'attachment; filename="data.xml"', XmlResponse), (b"attachment; filename=data.xml", XmlResponse), - ("attachment;filename=data£.tar.gz".encode("utf-8"), Response), + ("attachment;filename=data£.tar.gz".encode(), Response), ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response), ("attachment;filename=data高.doc".encode("gbk"), Response), ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse), diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 6ad30deed..28ad910a8 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -36,10 +36,10 @@ class BaseRobotParserTest: def test_allowed(self): robotstxt_robotstxt_body = ( - "User-agent: * \n" - "Disallow: /disallowed \n" - "Allow: /allowed \n" - "Crawl-delay: 10".encode("utf-8") + b"User-agent: * \n" + b"Disallow: /disallowed \n" + b"Allow: /allowed \n" + b"Crawl-delay: 10" ) rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body @@ -48,15 +48,13 @@ class BaseRobotParserTest: self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*")) def test_allowed_wildcards(self): - robotstxt_robotstxt_body = """User-agent: first + robotstxt_robotstxt_body = b"""User-agent: first Disallow: /disallowed/*/end$ User-agent: second Allow: /*allowed Disallow: / - """.encode( - "utf-8" - ) + """ rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) @@ -77,18 +75,14 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second")) def test_length_based_precedence(self): - robotstxt_robotstxt_body = ( - "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8") - ) + robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) self.assertTrue(rp.allowed("https://www.site.local/page", "*")) def test_order_based_precedence(self): - robotstxt_robotstxt_body = ( - "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8") - ) + robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) @@ -123,9 +117,7 @@ class BaseRobotParserTest: Disallow: /wiki/Käyttäjä: User-Agent: UnicödeBöt - Disallow: /some/randome/page.html""".encode( - "utf-8" - ) + Disallow: /some/randome/page.html""".encode() rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) @@ -145,14 +137,14 @@ class BaseRobotParserTest: class DecodeRobotsTxtTest(unittest.TestCase): def test_native_string_conversion(self): - robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + robotstxt_body = b"User-agent: *\nDisallow: /\n" decoded_content = decode_robotstxt( robotstxt_body, spider=None, to_native_str_type=True ) self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") def test_decode_utf8(self): - robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8") + robotstxt_body = b"User-agent: *\nDisallow: /\n" decoded_content = decode_robotstxt(robotstxt_body, spider=None) self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") From 144ff6c756fa58da2bc1a85879aa6f89300030d1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 5 Jun 2024 21:09:10 -0300 Subject: [PATCH 1483/2083] Document missing parts of response.json method --- docs/topics/dynamic-content.rst | 7 +++---- docs/topics/selectors.rst | 8 ++++++++ scrapy/selector/unified.py | 1 + 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index a0f4b4411..a99f1e222 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -115,15 +115,14 @@ Handling different response formats Once you have a response with the desired data, how you extract the desired data from it depends on the type of response: -- If the response is HTML or XML, use :ref:`selectors +- If the response is HTML, XML or JSON, use :ref:`selectors ` as usual. -- If the response is JSON, use :func:`json.loads` to load the desired data from - :attr:`response.text `: +- If the response is JSON, use :func:`response.json()` to load the desired data: .. code-block:: python - data = json.loads(response.text) + data = response.json() If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index e32fc2b70..0aae41cc8 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -1060,6 +1060,12 @@ Selector objects For convenience, this method can be called as ``response.css()`` + .. automethod:: jmespath + + .. note:: + + For convenience, this method can be called as ``response.jmespath()`` + .. automethod:: get See also: :ref:`old-extraction-api` @@ -1092,6 +1098,8 @@ SelectorList objects .. automethod:: css + .. automethod:: jmespath + .. automethod:: getall See also: :ref:`old-extraction-api` diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index e852aadc7..bfddb87cb 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -59,6 +59,7 @@ class Selector(_ParselSelector, object_ref): * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type + * ``"json"`` for :class:`~scrapy.http.TextResponse` type * ``"html"`` for anything else Otherwise, if ``type`` is set, the selector type will be forced and no From 23b1214e901961057bf43a5fb2548b35dfe19b20 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 6 Jun 2024 21:44:07 +0500 Subject: [PATCH 1484/2083] Add a comment about pyupgrade and scrapy/__init__.py. --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f76a04ca1..505b3c57d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,4 +27,5 @@ repos: hooks: - id: pyupgrade args: [--py38-plus, --keep-runtime-typing] + # scrapy/__init__.py has a sys.version_info check we want to keep exclude: scrapy/__init__.py From ed3a7acaf3169ed6b9f9ffbcffed35db63d840f7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Jun 2024 11:19:37 +0500 Subject: [PATCH 1485/2083] Remove the Python version check from scrapy/__init__.py. --- .pre-commit-config.yaml | 2 -- scrapy/__init__.py | 6 ------ 2 files changed, 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 505b3c57d..63da5544d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,5 +27,3 @@ repos: hooks: - id: pyupgrade args: [--py38-plus, --keep-runtime-typing] - # scrapy/__init__.py has a sys.version_info check we want to keep - exclude: scrapy/__init__.py diff --git a/scrapy/__init__.py b/scrapy/__init__.py index cc0e539c4..1c1a5c2cc 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -33,12 +33,6 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("." twisted_version = (_txv.major, _txv.minor, _txv.micro) -# Check minimum required Python version -if sys.version_info < (3, 8): - print(f"Scrapy {__version__} requires Python 3.8+") - sys.exit(1) - - # Ignore noisy twisted deprecation warnings warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted") From ddc98fe91b454a0944a8558daa2000da08921b62 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Jun 2024 13:16:26 +0500 Subject: [PATCH 1486/2083] Deprecate scrapy.utils.defer.process_chain_both(). (#6397) --- scrapy/utils/defer.py | 9 ++++++++- tests/test_utils_defer.py | 14 -------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index abb7e1726..f60b7dde8 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio import inspect +import warnings from asyncio import Future from functools import wraps from types import CoroutineType @@ -35,7 +36,7 @@ from twisted.internet.task import Cooperator from twisted.python import failure from twisted.python.failure import Failure -from scrapy.exceptions import IgnoreRequest +from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: @@ -281,6 +282,12 @@ def process_chain_both( **kw: _P.kwargs, ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" + warnings.warn( + "process_chain_both() is deprecated and will be removed in a future" + " Scrapy version.", + ScrapyDeprecationWarning, + stacklevel=2, + ) d: Deferred = Deferred() for cb, eb in zip(callbacks, errbacks): d.addCallback(cb, *a, **kw) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index a7d54b565..ec0399865 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -14,7 +14,6 @@ from scrapy.utils.defer import ( mustbe_deferred, parallel_async, process_chain, - process_chain_both, process_parallel, ) @@ -80,19 +79,6 @@ class DeferUtilsTest(unittest.TestCase): gotexc = True self.assertTrue(gotexc) - @defer.inlineCallbacks - def test_process_chain_both(self): - x = yield process_chain_both( - [cb_fail, cb2, cb3], [None, eb1, None], "res", "v1", "v2" - ) - self.assertEqual(x, "(cb3 (eb1 TypeError v1 v2) v1 v2)") - - fail = Failure(ZeroDivisionError()) - x = yield process_chain_both( - [eb1, cb2, cb3], [eb1, None, None], fail, "v1", "v2" - ) - self.assertEqual(x, "(cb3 (cb2 (eb1 ZeroDivisionError v1 v2) v1 v2) v1 v2)") - @defer.inlineCallbacks def test_process_parallel(self): x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") From 1282ddf8f77299edf613679c2ee0b606e96808ce Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Jun 2024 13:27:50 +0500 Subject: [PATCH 1487/2083] Add parameters to most Deferred in scrapy/core. (#6395) --- scrapy/core/downloader/__init__.py | 45 ++++++++++---- scrapy/core/downloader/handlers/__init__.py | 38 +++++++++--- scrapy/core/downloader/handlers/ftp.py | 10 ++- scrapy/core/downloader/handlers/http10.py | 3 +- scrapy/core/downloader/handlers/http11.py | 69 ++++++++++++--------- scrapy/core/downloader/handlers/http2.py | 4 +- scrapy/core/downloader/handlers/s3.py | 3 +- scrapy/core/downloader/middleware.py | 23 ++++--- scrapy/core/downloader/webclient.py | 6 +- scrapy/core/engine.py | 55 +++++++++------- scrapy/core/scheduler.py | 8 +-- scrapy/core/scraper.py | 61 +++++++++++------- scrapy/core/spidermw.py | 18 +++--- scrapy/pipelines/__init__.py | 4 +- scrapy/utils/defer.py | 20 +++--- tests/test_downloadermiddleware.py | 2 +- 16 files changed, 237 insertions(+), 132 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 0ab3bdb77..41f729ed9 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,9 +1,22 @@ +from __future__ import annotations + import random import warnings from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast +from typing import ( + TYPE_CHECKING, + Any, + Deque, + Dict, + Optional, + Set, + Tuple, + TypeVar, + Union, + cast, +) from twisted.internet import task from twisted.internet.defer import Deferred @@ -22,6 +35,8 @@ from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from scrapy.crawler import Crawler +_T = TypeVar("_T") + class Slot: """Downloader slot""" @@ -40,7 +55,7 @@ class Slot: self.throttle = throttle self.active: Set[Request] = set() - self.queue: Deque[Tuple[Request, Deferred]] = deque() + self.queue: Deque[Tuple[Request, Deferred[Response]]] = deque() self.transferring: Set[Request] = set() self.lastseen: float = 0 self.latercall = None @@ -93,7 +108,7 @@ def _get_concurrency_delay( class Downloader: DOWNLOAD_SLOT = "download_slot" - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): self.settings: BaseSettings = crawler.settings self.signals: SignalManager = crawler.signals self.slots: Dict[str, Slot] = {} @@ -114,13 +129,17 @@ class Downloader: "DOWNLOAD_SLOTS", {} ) - def fetch(self, request: Request, spider: Spider) -> Deferred: - def _deactivate(response: Response) -> Response: + def fetch( + self, request: Request, spider: Spider + ) -> Deferred[Union[Response, Request]]: + def _deactivate(response: _T) -> _T: self.active.remove(request) return response self.active.add(request) - dfd = self.middleware.download(self._enqueue_request, request, spider) + dfd: Deferred[Union[Response, Request]] = self.middleware.download( + self._enqueue_request, request, spider + ) return dfd.addBoth(_deactivate) def needs_backout(self) -> bool: @@ -163,7 +182,7 @@ class Downloader: ) return self.get_slot_key(request) - def _enqueue_request(self, request: Request, spider: Spider) -> Deferred: + def _enqueue_request(self, request: Request, spider: Spider) -> Deferred[Response]: key, slot = self._get_slot(request, spider) request.meta[self.DOWNLOAD_SLOT] = key @@ -175,7 +194,7 @@ class Downloader: self.signals.send_catch_log( signal=signals.request_reached_downloader, request=request, spider=spider ) - deferred: Deferred = Deferred().addBoth(_deactivate) + deferred: Deferred[Response] = Deferred().addBoth(_deactivate) slot.queue.append((request, deferred)) self._process_queue(spider, slot) return deferred @@ -208,11 +227,15 @@ class Downloader: self._process_queue(spider, slot) break - def _download(self, slot: Slot, request: Request, spider: Spider) -> Deferred: + def _download( + self, slot: Slot, request: Request, spider: Spider + ) -> Deferred[Response]: # The order is very important for the following deferreds. Do not change! # 1. Create the download deferred - dfd = mustbe_deferred(self.handlers.download_request, request, spider) + dfd: Deferred[Response] = mustbe_deferred( + self.handlers.download_request, request, spider + ) # 2. Notify response_downloaded listeners about the recent download # before querying queue for next request @@ -233,7 +256,7 @@ class Downloader: # middleware itself) slot.transferring.add(request) - def finish_transferring(_: Any) -> Any: + def finish_transferring(_: _T) -> _T: slot.transferring.remove(request) self._process_queue(spider, slot) self.signals.send_catch_log( diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 5ec5ef6db..ebc4898b5 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -3,13 +3,25 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + Optional, + Protocol, + Type, + Union, + cast, +) from twisted.internet import defer from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, NotSupported +from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values @@ -20,13 +32,21 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +class DownloadHandlerProtocol(Protocol): + def download_request( + self, request: Request, spider: Spider + ) -> Deferred[Response]: ... + + class DownloadHandlers: def __init__(self, crawler: Crawler): self._crawler: Crawler = crawler self._schemes: Dict[str, Union[str, Callable[..., Any]]] = ( {} ) # stores acceptable schemes on instancing - self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes + self._handlers: Dict[str, DownloadHandlerProtocol] = ( + {} + ) # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values( cast( @@ -40,7 +60,7 @@ class DownloadHandlers: crawler.signals.connect(self._close, signals.engine_stopped) - def _get_handler(self, scheme: str) -> Any: + def _get_handler(self, scheme: str) -> Optional[DownloadHandlerProtocol]: """Lazy-load the downloadhandler for a scheme only on the first request for that scheme. """ @@ -54,10 +74,12 @@ class DownloadHandlers: return self._load_handler(scheme) - def _load_handler(self, scheme: str, skip_lazy: bool = False) -> Any: + def _load_handler( + self, scheme: str, skip_lazy: bool = False + ) -> Optional[DownloadHandlerProtocol]: path = self._schemes[scheme] try: - dhcls = load_object(path) + dhcls: Type[DownloadHandlerProtocol] = load_object(path) if skip_lazy and getattr(dhcls, "lazy", True): return None dh = build_from_crawler( @@ -80,17 +102,17 @@ class DownloadHandlers: self._handlers[scheme] = dh return dh - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: scheme = urlparse_cached(request).scheme handler = self._get_handler(scheme) if not handler: raise NotSupported( f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}" ) - return cast(Deferred, handler.download_request(request, spider)) + return handler.download_request(request, spider) @defer.inlineCallbacks - def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred, Any, None]: + def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred[Any], Any, None]: for dh in self._handlers.values(): if hasattr(dh, "close"): yield dh.close() diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 77dcf3c38..724717ffd 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -91,7 +91,7 @@ class FTPDownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: from twisted.internet import reactor parsed_url = urlparse_cached(request) @@ -103,10 +103,14 @@ class FTPDownloadHandler: creator = ClientCreator( reactor, FTPClient, user, password, passive=passive_mode ) - dfd: Deferred = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) + dfd: Deferred[FTPClient] = creator.connectTCP( + parsed_url.hostname, parsed_url.port or 21 + ) return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path)) - def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred: + def gotClient( + self, client: FTPClient, request: Request, filepath: str + ) -> Deferred[Response]: self.client = client protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) d = client.retrieveFile(filepath, protocol) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index da9559525..3c4e48abb 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -9,6 +9,7 @@ from twisted.internet.defer import Deferred from scrapy import Request, Spider from scrapy.crawler import Crawler +from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode @@ -38,7 +39,7 @@ class HTTP10DownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: """Return a deferred for the HTTP download""" factory = self.HTTPClientFactory(request) self._connect(factory) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 5e84be6ba..e2ad8f59a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar, Union from urllib.parse import urldefrag, urlunparse from twisted.internet import ssl @@ -38,12 +38,22 @@ from scrapy.settings import BaseSettings from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from typing_extensions import Self - + # typing.NotRequired and typing.Self require Python 3.11 + from typing_extensions import NotRequired, Self logger = logging.getLogger(__name__) +_T = TypeVar("_T") + + +class _ResultT(TypedDict): + txresponse: TxResponse + body: bytes + flags: Optional[List[str]] + certificate: Optional[ssl.Certificate] + ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] + failure: NotRequired[Optional[Failure]] + class HTTP11DownloadHandler: lazy = False @@ -71,7 +81,7 @@ class HTTP11DownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: """Return a deferred for the HTTP download""" agent = ScrapyAgent( contextFactory=self._contextFactory, @@ -83,10 +93,10 @@ class HTTP11DownloadHandler: ) return agent.download_request(request) - def close(self) -> Deferred: + def close(self) -> Deferred[None]: from twisted.internet import reactor - d: Deferred = self._pool.closeCachedConnections() + d: Deferred[None] = self._pool.closeCachedConnections() # closeCachedConnections will hang on network or server issues, so # we'll manually timeout the deferred. # @@ -97,7 +107,7 @@ class HTTP11DownloadHandler: # issue a callback after `_disconnect_timeout` seconds. delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, []) - def cancel_delayed_call(result: Any) -> Any: + def cancel_delayed_call(result: _T) -> _T: if delayed_call.active(): delayed_call.cancel() return result @@ -137,7 +147,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) - self._tunnelReadyDeferred: Deferred = Deferred() + self._tunnelReadyDeferred: Deferred[Protocol] = Deferred() self._tunneledHost: str = host self._tunneledPort: int = port self._contextFactory: IPolicyForHTTPS = contextFactory @@ -198,7 +208,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """Propagates the errback to the appropriate deferred.""" self._tunnelReadyDeferred.errback(reason) - def connect(self, protocolFactory: Factory) -> Deferred: + def connect(self, protocolFactory: Factory) -> Deferred[Protocol]: self._protocolFactory = protocolFactory connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) @@ -271,7 +281,7 @@ class TunnelingAgent(Agent): headers: Optional[TxHeaders], bodyProducer: Optional[IBodyProducer], requestPath: bytes, - ) -> Deferred: + ) -> Deferred[TxResponse]: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -310,7 +320,7 @@ class ScrapyProxyAgent(Agent): uri: bytes, headers: Optional[TxHeaders] = None, bodyProducer: Optional[IBodyProducer] = None, - ) -> Deferred: + ) -> Deferred[TxResponse]: """ Issue a new request via the configured proxy. """ @@ -394,7 +404,7 @@ class ScrapyAgent: pool=self._pool, ) - def download_request(self, request: Request) -> Deferred: + def download_request(self, request: Request) -> Deferred[Response]: from twisted.internet import reactor timeout = request.meta.get("download_timeout") or self._connectTimeout @@ -411,22 +421,20 @@ class ScrapyAgent: else: bodyproducer = None start_time = time() - d: Deferred = agent.request( + d: Deferred[TxResponse] = agent.request( method, to_bytes(url, encoding="ascii"), headers, bodyproducer ) # set download latency d.addCallback(self._cb_latency, request, start_time) # response body is ready to be consumed - d.addCallback(self._cb_bodyready, request) - d.addCallback(self._cb_bodydone, request, url) + d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request) + d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, request, url) # check download timeout - self._timeout_cl = reactor.callLater(timeout, d.cancel) - d.addBoth(self._cb_timeout, request, url, timeout) - return d + self._timeout_cl = reactor.callLater(timeout, d3.cancel) + d3.addBoth(self._cb_timeout, request, url, timeout) + return d3 - def _cb_timeout( - self, result: Any, request: Request, url: str, timeout: float - ) -> Any: + def _cb_timeout(self, result: _T, request: Request, url: str, timeout: float) -> _T: if self._timeout_cl.active(): self._timeout_cl.cancel() return result @@ -437,7 +445,7 @@ class ScrapyAgent: raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") - def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any: + def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T: request.meta["download_latency"] = time() - start_time return result @@ -451,7 +459,7 @@ class ScrapyAgent: def _cb_bodyready( self, txresponse: TxResponse, request: Request - ) -> Union[Dict[str, Any], Deferred]: + ) -> Union[_ResultT, Deferred[_ResultT]]: headers_received_result = self._crawler.signals.send_catch_log( signal=signals.headers_received, headers=self._headers_from_twisted_response(txresponse), @@ -520,7 +528,7 @@ class ScrapyAgent: # Abort connection immediately. txresponse._transport._producer.abortConnection() - d: Deferred = Deferred(_cancel) + d: Deferred[_ResultT] = Deferred(_cancel) txresponse.deliverBody( _ResponseReader( finished=d, @@ -539,7 +547,7 @@ class ScrapyAgent: return d def _cb_bodydone( - self, result: Dict[str, Any], request: Request, url: str + self, result: _ResultT, request: Request, url: str ) -> Union[Response, Failure]: headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) @@ -559,8 +567,9 @@ class ScrapyAgent: protocol=protocol, ) if result.get("failure"): + assert result["failure"] result["failure"].value.response = response - return cast(Failure, result["failure"]) + return result["failure"] return response @@ -570,7 +579,7 @@ class _RequestBodyProducer: self.body = body self.length = len(body) - def startProducing(self, consumer: IConsumer) -> Deferred: + def startProducing(self, consumer: IConsumer) -> Deferred[None]: consumer.write(self.body) return succeed(None) @@ -584,7 +593,7 @@ class _RequestBodyProducer: class _ResponseReader(Protocol): def __init__( self, - finished: Deferred, + finished: Deferred[_ResultT], txresponse: TxResponse, request: Request, maxsize: int, @@ -592,7 +601,7 @@ class _ResponseReader(Protocol): fail_on_dataloss: bool, crawler: Crawler, ): - self._finished: Deferred = finished + self._finished: Deferred[_ResultT] = finished self._txresponse: TxResponse = txresponse self._request: Request = request self._bodybuf: BytesIO = BytesIO() diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 16fc1e3ae..2ac4eca86 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -37,7 +37,7 @@ class H2DownloadHandler: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, @@ -98,7 +98,7 @@ class ScrapyH2Agent: pool=self._pool, ) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: from twisted.internet import reactor timeout = request.meta.get("download_timeout") or self._connect_timeout diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 1a3d36f45..0ad340721 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -8,6 +8,7 @@ from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached @@ -76,7 +77,7 @@ class S3DownloadHandler: def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self: return cls(crawler.settings, crawler=crawler, **kwargs) - def download_request(self, request: Request, spider: Spider) -> Deferred: + def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: p = urlparse_cached(request) scheme = "https" if request.meta.get("is_secure") else "http" bucket = p.hostname diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 52ebe4e22..2d8af114f 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -4,6 +4,8 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ +from __future__ import annotations + from typing import Any, Callable, Generator, List, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks @@ -34,10 +36,15 @@ class DownloaderMiddlewareManager(MiddlewareManager): self.methods["process_exception"].appendleft(mw.process_exception) def download( - self, download_func: Callable, request: Request, spider: Spider - ) -> Deferred: + self, + download_func: Callable[[Request, Spider], Deferred[Response]], + request: Request, + spider: Spider, + ) -> Deferred[Union[Response, Request]]: @inlineCallbacks - def process_request(request: Request) -> Generator[Deferred, Any, Any]: + def process_request( + request: Request, + ) -> Generator[Deferred[Any], Any, Union[Response, Request]]: for method in self.methods["process_request"]: method = cast(Callable, method) response = yield deferred_from_coro( @@ -52,12 +59,12 @@ class DownloaderMiddlewareManager(MiddlewareManager): ) if response: return response - return (yield download_func(request=request, spider=spider)) + return (yield download_func(request, spider)) @inlineCallbacks def process_response( response: Union[Response, Request] - ) -> Generator[Deferred, Any, Union[Response, Request]]: + ) -> Generator[Deferred[Any], Any, Union[Response, Request]]: if response is None: raise TypeError("Received None in process_response") elif isinstance(response, Request): @@ -80,7 +87,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): @inlineCallbacks def process_exception( failure: Failure, - ) -> Generator[Deferred, Any, Union[Failure, Response, Request]]: + ) -> Generator[Deferred[Any], Any, Union[Failure, Response, Request]]: exception = failure.value for method in self.methods["process_exception"]: method = cast(Callable, method) @@ -98,7 +105,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response return failure - deferred = mustbe_deferred(process_request, request) + deferred: Deferred[Union[Response, Request]] = mustbe_deferred( + process_request, request + ) deferred.addErrback(process_exception) deferred.addCallback(process_response) return deferred diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index bb1f73805..08a1d7c71 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -8,7 +8,7 @@ from twisted.internet.protocol import ClientFactory from twisted.web.http import HTTPClient from scrapy import Request -from scrapy.http import Headers +from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode @@ -145,7 +145,7 @@ class ScrapyHTTPClientFactory(ClientFactory): self.response_headers: Optional[Headers] = None self.timeout: float = request.meta.get("download_timeout") or timeout self.start_time: float = time() - self.deferred: defer.Deferred = defer.Deferred().addCallback( + self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback( self._build_response, request ) @@ -155,7 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory): # needed to add the callback _waitForDisconnect. # Specifically this avoids the AttributeError exception when # clientConnectionFailed method is called. - self._disconnectedDeferred: defer.Deferred = defer.Deferred() + self._disconnectedDeferred: defer.Deferred[None] = defer.Deferred() self._set_connection_attributes(request) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index dededf99d..4ffec78b9 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -19,6 +19,7 @@ from typing import ( Optional, Set, Type, + TypeVar, Union, cast, ) @@ -43,10 +44,13 @@ from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: from scrapy.core.scheduler import BaseScheduler + from scrapy.core.scraper import _HandleOutputDeferred from scrapy.crawler import Crawler logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Slot: def __init__( @@ -56,7 +60,7 @@ class Slot: nextcall: CallLaterOnce[None], scheduler: BaseScheduler, ) -> None: - self.closing: Optional[Deferred] = None + self.closing: Optional[Deferred[None]] = None self.inprogress: Set[Request] = set() self.start_requests: Optional[Iterator[Request]] = iter(start_requests) self.close_if_idle: bool = close_if_idle @@ -71,7 +75,7 @@ class Slot: self.inprogress.remove(request) self._maybe_fire_closing() - def close(self) -> Deferred: + def close(self) -> Deferred[None]: self.closing = Deferred() self._maybe_fire_closing() return self.closing @@ -123,20 +127,20 @@ class ExecutionEngine: return scheduler_cls @inlineCallbacks - def start(self) -> Generator[Deferred, Any, None]: + def start(self) -> Generator[Deferred[Any], Any, None]: if self.running: raise RuntimeError("Engine already running") self.start_time = time() yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True - self._closewait: Deferred = Deferred() + self._closewait: Deferred[None] = Deferred() yield self._closewait - def stop(self) -> Deferred: + def stop(self) -> Deferred[None]: """Gracefully stop the execution engine""" @inlineCallbacks - def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]: + def _finish_stopping_engine(_: Any) -> Generator[Deferred[Any], Any, None]: yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) self._closewait.callback(None) @@ -151,7 +155,7 @@ class ExecutionEngine: ) return dfd.addBoth(_finish_stopping_engine) - def close(self) -> Deferred: + def close(self) -> Deferred[None]: """ Gracefully close the execution engine. If it has already been started, stop it. In all cases, close the spider and the downloader. @@ -214,7 +218,7 @@ class ExecutionEngine: or self.scraper.slot.needs_backout() ) - def _next_request_from_scheduler(self) -> Optional[Deferred]: + def _next_request_from_scheduler(self) -> Optional[Deferred[None]]: assert self.slot is not None # typing assert self.spider is not None # typing @@ -222,7 +226,7 @@ class ExecutionEngine: if request is None: return None - d = self._download(request) + d: Deferred[Union[Response, Request]] = self._download(request) d.addBoth(self._handle_downloader_output, request) d.addErrback( lambda f: logger.info( @@ -236,8 +240,8 @@ class ExecutionEngine: assert self.slot self.slot.remove_request(request) - d.addBoth(_remove_request) - d.addErrback( + d2: Deferred[None] = d.addBoth(_remove_request) + d2.addErrback( lambda f: logger.info( "Error while removing request from slot", exc_info=failure_to_exc_info(f), @@ -245,19 +249,19 @@ class ExecutionEngine: ) ) slot = self.slot - d.addBoth(lambda _: slot.nextcall.schedule()) - d.addErrback( + d2.addBoth(lambda _: slot.nextcall.schedule()) + d2.addErrback( lambda f: logger.info( "Error while scheduling new request", exc_info=failure_to_exc_info(f), extra={"spider": self.spider}, ) ) - return d + return d2 def _handle_downloader_output( self, result: Union[Request, Response, Failure], request: Request - ) -> Optional[Deferred]: + ) -> Optional[_HandleOutputDeferred]: assert self.spider is not None # typing if not isinstance(result, (Request, Response, Failure)): @@ -319,20 +323,23 @@ class ExecutionEngine: signals.request_dropped, request=request, spider=spider ) - def download(self, request: Request) -> Deferred: + def download(self, request: Request) -> Deferred[Response]: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - return self._download(request).addBoth(self._downloaded, request) + d: Deferred[Union[Response, Request]] = self._download(request) + # Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type + d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[arg-type] + return d2 def _downloaded( self, result: Union[Response, Request, Failure], request: Request - ) -> Union[Deferred, Response, Failure]: + ) -> Union[Deferred[Response], Response, Failure]: assert self.slot is not None # typing self.slot.remove_request(request) return self.download(result) if isinstance(result, Request) else result - def _download(self, request: Request) -> Deferred: + def _download(self, request: Request) -> Deferred[Union[Response, Request]]: assert self.slot is not None # typing self.slot.add_request(request) @@ -359,13 +366,15 @@ class ExecutionEngine: ) return result - def _on_complete(_: Any) -> Any: + def _on_complete(_: _T) -> _T: assert self.slot is not None self.slot.nextcall.schedule() return _ assert self.spider is not None - dwld = self.downloader.fetch(request, self.spider) + dwld: Deferred[Union[Response, Request]] = self.downloader.fetch( + request, self.spider + ) dwld.addCallback(_on_success) dwld.addBoth(_on_complete) return dwld @@ -376,7 +385,7 @@ class ExecutionEngine: spider: Spider, start_requests: Iterable[Request] = (), close_if_idle: bool = True, - ) -> Generator[Deferred, Any, None]: + ) -> Generator[Deferred[Any], Any, None]: if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) @@ -422,7 +431,7 @@ class ExecutionEngine: assert isinstance(ex, CloseSpider) # typing self.close_spider(self.spider, reason=ex.reason) - def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: + def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred[None]: """Close (cancel) spider and clear all its outstanding requests""" if self.slot is None: raise RuntimeError("Engine slot not assigned") diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index e3b95e977..1e586c53a 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -71,7 +71,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ return cls() - def open(self, spider: Spider) -> Optional[Deferred]: + def open(self, spider: Spider) -> Optional[Deferred[None]]: """ Called when the spider is opened by the engine. It receives the spider instance as argument and it's useful to execute initialization code. @@ -81,7 +81,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ pass - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Optional[Deferred[None]]: """ Called when the spider is closed by the engine. It receives the reason why the crawl finished as argument and it's useful to execute cleaning code. @@ -216,7 +216,7 @@ class Scheduler(BaseScheduler): def has_pending_requests(self) -> bool: return len(self) > 0 - def open(self, spider: Spider) -> Optional[Deferred]: + def open(self, spider: Spider) -> Optional[Deferred[None]]: """ (1) initialize the memory queue (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory @@ -227,7 +227,7 @@ class Scheduler(BaseScheduler): self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None return self.df.open() - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Optional[Deferred[None]]: """ (1) dump pending requests to disk if there is a disk queue (2) return the result of the dupefilter's ``close`` method diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 3b7492838..8a9e8f687 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -12,6 +12,7 @@ from typing import ( Deque, Generator, Iterable, + Iterator, Optional, Set, Tuple, @@ -33,6 +34,7 @@ from scrapy.logformatter import LogFormatter from scrapy.pipelines import ItemPipelineManager from scrapy.signalmanager import SignalManager from scrapy.utils.defer import ( + DeferredListResultListT, aiter_errback, defer_fail, defer_succeed, @@ -48,13 +50,18 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler -_T = TypeVar("_T") -QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] - - logger = logging.getLogger(__name__) +_T = TypeVar("_T") +_ParallelResult = DeferredListResultListT[Iterator[Any]] + +if TYPE_CHECKING: + # parameterized Deferreds require Twisted 21.7.0 + _HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] + QueueTuple = Tuple[Union[Response, Failure], Request, _HandleOutputDeferred] + + class Slot: """Scraper slot (one per running spider)""" @@ -66,12 +73,12 @@ class Slot: self.active: Set[Request] = set() self.active_size: int = 0 self.itemproc_size: int = 0 - self.closing: Optional[Deferred] = None + self.closing: Optional[Deferred[Spider]] = None def add_response_request( self, result: Union[Response, Failure], request: Request - ) -> Deferred: - deferred: Deferred = Deferred() + ) -> _HandleOutputDeferred: + deferred: _HandleOutputDeferred = Deferred() self.queue.append((result, request, deferred)) if isinstance(result, Response): self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -117,12 +124,12 @@ class Scraper: self.logformatter: LogFormatter = crawler.logformatter @inlineCallbacks - def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]: + def open_spider(self, spider: Spider) -> Generator[Deferred[Any], Any, None]: """Open the given spider for scraping and allocate resources for it""" self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) yield self.itemproc.open_spider(spider) - def close_spider(self, spider: Spider) -> Deferred: + def close_spider(self, spider: Spider) -> Deferred[Spider]: """Close a spider being scraped and release its resources""" if self.slot is None: raise RuntimeError("Scraper slot not assigned") @@ -142,12 +149,12 @@ class Scraper: def enqueue_scrape( self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + ) -> _HandleOutputDeferred: if self.slot is None: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) - def finish_scraping(_: Any) -> Any: + def finish_scraping(_: _T) -> _T: assert self.slot is not None self.slot.finish_response(result, request) self._check_if_closing(spider) @@ -174,7 +181,7 @@ class Scraper: def _scrape( self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + ) -> _HandleOutputDeferred: """ Handle the downloaded response or failure through the spider callback/errback """ @@ -182,32 +189,35 @@ class Scraper: raise TypeError( f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) - dfd = self._scrape2( + dfd: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = self._scrape2( result, request, spider ) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) - dfd.addCallback( + dfd2: _HandleOutputDeferred = dfd.addCallback( self.handle_spider_output, request, cast(Response, result), spider ) - return dfd + return dfd2 def _scrape2( self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]: """ Handle the different cases of request's result been a Response or a Failure """ if isinstance(result, Response): - return self.spidermw.scrape_response( + # Deferreds are invariant so Mutable*Chain isn't matched to *Iterable + return self.spidermw.scrape_response( # type: ignore[return-value] self.call_spider, result, request, spider ) # else result is a Failure dfd = self.call_spider(result, request, spider) - return dfd.addErrback(self._log_download_errors, result, request, spider) + dfd.addErrback(self._log_download_errors, result, request, spider) + return dfd def call_spider( self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred: + ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]: + dfd: Deferred[Any] if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request @@ -225,7 +235,10 @@ class Scraper: if request.errback: warn_on_generator_with_return_value(spider, request.errback) dfd.addErrback(request.errback) - return dfd.addCallback(iterate_spider_output) + dfd2: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = dfd.addCallback( + iterate_spider_output + ) + return dfd2 def handle_spider_error( self, @@ -262,10 +275,11 @@ class Scraper: request: Request, response: Response, spider: Spider, - ) -> Deferred: + ) -> _HandleOutputDeferred: if not result: return defer_succeed(None) it: Union[Iterable[_T], AsyncIterable[_T]] + dfd: Deferred[_ParallelResult] if isinstance(result, AsyncIterable): it = aiter_errback( result, self.handle_spider_error, request, response, spider @@ -290,11 +304,12 @@ class Scraper: response, spider, ) - return dfd + # returning Deferred[_ParallelResult] instead of Deferred[Union[_ParallelResult, None]] + return dfd # type: ignore[return-value] def _process_spidermw_output( self, output: Any, request: Request, response: Response, spider: Spider - ) -> Optional[Deferred]: + ) -> Optional[Deferred[Any]]: """Process each Request/Item (given in the output parameter) returned from the given spider """ diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 58873f0d9..e792f8ca7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -45,7 +45,9 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] +ScrapeFunc = Callable[ + [Union[Response, Failure], Request, Spider], Union[Iterable[_T], AsyncIterable[_T]] +] def _isiterable(o: Any) -> bool: @@ -80,7 +82,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Any: + ) -> Union[Iterable[_T], AsyncIterable[_T]]: for method in self.methods["process_spider_input"]: method = cast(Callable, method) try: @@ -311,7 +313,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Deferred: + ) -> Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]]: async def process_callback_output( result: Union[Iterable[_T], AsyncIterable[_T]] ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]: @@ -322,12 +324,14 @@ class SpiderMiddlewareManager(MiddlewareManager): ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]: return self._process_spider_exception(response, spider, _failure) - dfd: Deferred = mustbe_deferred( + dfd: Deferred[Union[Iterable[_T], AsyncIterable[_T]]] = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) - dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) - dfd.addErrback(process_spider_exception) - return dfd + dfd2: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = ( + dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) + ) + dfd2.addErrback(process_spider_exception) + return dfd2 def process_start_requests( self, start_requests: Iterable[Request], spider: Spider diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 0cfbc156f..21d649e3c 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -4,6 +4,8 @@ Item pipeline See documentation in docs/item-pipeline.rst """ +from __future__ import annotations + from typing import Any, List from twisted.internet.defer import Deferred @@ -29,5 +31,5 @@ class ItemPipelineManager(MiddlewareManager): deferred_f_from_coro_f(pipe.process_item) ) - def process_item(self, item: Any, spider: Spider) -> Deferred: + def process_item(self, item: Any, spider: Spider) -> Deferred[Any]: return self._process_chain("process_item", item, spider) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index f60b7dde8..ddb68c86b 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -46,6 +46,12 @@ if TYPE_CHECKING: _P = ParamSpec("_P") _T = TypeVar("_T") +_T2 = TypeVar("_T2") + +# copied from twisted.internet.defer +_SelfResultT = TypeVar("_SelfResultT") +_DeferredListResultItemT = Tuple[bool, _SelfResultT] +DeferredListResultListT = List[_DeferredListResultItemT[_SelfResultT]] def defer_fail(_failure: Failure) -> Deferred: @@ -62,7 +68,7 @@ def defer_fail(_failure: Failure) -> Deferred: return d -def defer_succeed(result: Any) -> Deferred: +def defer_succeed(result: _T) -> Deferred[_T]: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -128,10 +134,10 @@ def mustbe_deferred( def parallel( iterable: Iterable[_T], count: int, - callable: Callable[Concatenate[_T, _P], Any], + callable: Callable[Concatenate[_T, _P], _T2], *args: _P.args, **named: _P.kwargs, -) -> Deferred: +) -> Deferred[DeferredListResultListT[Iterator[_T2]]]: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -191,12 +197,12 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): def __init__( self, aiterable: AsyncIterable[_T], - callable: Callable[Concatenate[_T, _P], Any], + callable: Callable[Concatenate[_T, _P], _T2], *callable_args: _P.args, **callable_kwargs: _P.kwargs, ): self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() - self.callable: Callable[Concatenate[_T, _P], Any] = callable + self.callable: Callable[Concatenate[_T, _P], _T2] = callable self.callable_args: Tuple[Any, ...] = callable_args self.callable_kwargs: Dict[str, Any] = callable_kwargs self.finished: bool = False @@ -249,10 +255,10 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): def parallel_async( async_iterable: AsyncIterable[_T], count: int, - callable: Callable[Concatenate[_T, _P], Any], + callable: Callable[Concatenate[_T, _P], _T2], *args: _P.args, **named: _P.kwargs, -) -> Deferred: +) -> Deferred[DeferredListResultListT[Iterator[_T2]]]: """Like parallel but for async iterators""" coop = Cooperator() work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 0155c62eb..dd3f8ceb9 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -36,7 +36,7 @@ class ManagerTestCase(TestCase): if not response: response = Response(request.url) - def download_func(**kwargs): + def download_func(request, spider): return response dfd = self.mwman.download(download_func, request, self.spider) From 365c9e62ad9e99725eb1898cbd2806c63105cd58 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jun 2024 14:37:11 +0500 Subject: [PATCH 1488/2083] Removing empty example reference (#6402) Co-authored-by: Michael Duane Mooring --- docs/topics/link-extractors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 1201c926d..f9744ed16 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -85,7 +85,7 @@ LxmlLinkExtractor :param restrict_xpaths: is an XPath (or list of XPath's) which defines regions inside the response where links should be extracted from. If given, only the text selected by those XPath will be scanned for - links. See examples below. + links. :type restrict_xpaths: str or list :param restrict_css: a CSS selector (or list of selectors) which defines From a364560fadbbc0dd7cca78670bbd9d3c00d4d366 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jun 2024 14:38:10 +0500 Subject: [PATCH 1489/2083] Unpin markupsafe in extra-deps. (#6403) --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index 023a86c5a..d665fc5a5 100644 --- a/tox.ini +++ b/tox.ini @@ -147,9 +147,6 @@ deps = {[testenv]deps} boto3 google-cloud-storage - # Twisted[http2] currently forces old mitmproxy because of h2 version - # restrictions in their deps, so we need to pin old markupsafe here too. - markupsafe < 2.1.0 robotexclusionrulesparser Pillow Twisted[http2] From d13219062500eae1a6d5330ceea3502590cd89cb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 19 Jun 2024 23:26:25 +0500 Subject: [PATCH 1490/2083] flake8-debugger --- .pre-commit-config.yaml | 2 ++ scrapy/extensions/debug.py | 2 +- scrapy/utils/console.py | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 63da5544d..eb3404b7f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,6 +8,8 @@ repos: rev: 7.0.0 hooks: - id: flake8 + additional_dependencies: + - flake8-debugger - repo: https://github.com/psf/black.git rev: 24.2.0 hooks: diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index a0fc7b99f..b360ce48d 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -74,4 +74,4 @@ class Debugger: def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None: assert frame - Pdb().set_trace(frame.f_back) + Pdb().set_trace(frame.f_back) # noqa: T100 diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index bf1803115..328219831 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -10,10 +10,10 @@ def _embed_ipython_shell( ) -> EmbedFuncT: """Start an IPython Shell""" try: - from IPython.terminal.embed import InteractiveShellEmbed + from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100 from IPython.terminal.ipapp import load_default_config except ImportError: - from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] + from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] # noqa: T100 InteractiveShellEmbed, ) from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] From a617e04d2eb89b64f15df7a6a0326bfaf57f8dde Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 19 Jun 2024 23:28:58 +0500 Subject: [PATCH 1491/2083] flake8-string-format --- .flake8 | 8 ++++++-- .pre-commit-config.yaml | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.flake8 b/.flake8 index cf1a96476..b6048c9ee 100644 --- a/.flake8 +++ b/.flake8 @@ -1,8 +1,12 @@ [flake8] max-line-length = 119 -ignore = E203, E501, E701, E704, W503 - +ignore = + E203, E501, E701, E704, W503 + # docstring does contain unindexed parameters + P102 + # other string does contain unindexed parameters + P103 exclude = docs/conf.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb3404b7f..47a3df53d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,7 @@ repos: - id: flake8 additional_dependencies: - flake8-debugger + - flake8-string-format - repo: https://github.com/psf/black.git rev: 24.2.0 hooks: From 1c70d3e60555084b4bec9dfd794adb93b24b2171 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 19 Jun 2024 23:36:36 +0500 Subject: [PATCH 1492/2083] flake8-comprehensions --- .pre-commit-config.yaml | 1 + scrapy/extensions/feedexport.py | 2 +- tests/test_commands.py | 2 +- tests/test_feedexport.py | 10 +++++----- tests/test_loader.py | 6 ++---- tests/test_loader_deprecated.py | 4 ++-- tests/test_scheduler.py | 2 +- tests/test_spider.py | 18 +++++++++--------- 8 files changed, 22 insertions(+), 23 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 47a3df53d..974d397c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,7 @@ repos: hooks: - id: flake8 additional_dependencies: + - flake8-comprehensions - flake8-debugger - flake8-string-format - repo: https://github.com/psf/black.git diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index de8a288f6..941bd4b26 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -104,7 +104,7 @@ class ItemFilter: for item_class in feed_options.get("item_classes") or () ) else: - self.item_classes = tuple() + self.item_classes = () def accepts(self, item: Any) -> bool: """ diff --git a/tests/test_commands.py b/tests/test_commands.py index 857a56b73..d829b1701 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -200,7 +200,7 @@ def get_permissions_dict( path_obj = Path(path) - renamings = renamings or tuple() + renamings = renamings or () permissions_dict = { ".": get_permissions(path_obj), } diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 3771df8f1..253987e15 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1356,7 +1356,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_encoding(self): - items = [dict({"foo": "Test\xd6"})] + items = [{"foo": "Test\xd6"}] formats = { "json": b'[{"foo": "Test\\u00d6"}]', @@ -1401,7 +1401,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): - items = [dict({"foo": "FOO", "bar": "BAR"})] + items = [{"foo": "FOO", "bar": "BAR"}] formats = { "json": b'[\n{"bar": "BAR"}\n]', @@ -2513,8 +2513,8 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): items = [ - dict({"foo": "FOO", "bar": "BAR"}), - dict({"foo": "FOO1", "bar": "BAR1"}), + {"foo": "FOO", "bar": "BAR"}, + {"foo": "FOO1", "bar": "BAR1"}, ] formats = { @@ -2574,7 +2574,7 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): - items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})] + items = [{"foo": "FOO"}, {"foo": "FOO1"}] formats = { "json": [ b'[{"foo": "FOO"}]', diff --git a/tests/test_loader.py b/tests/test_loader.py index b0b7f8723..8db929dcf 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -156,7 +156,7 @@ class InitializationTestMixin: self.assertEqual(il.get_output_value("name"), ["foo"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({"name": ["foo"]})) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]}) def test_get_output_value_list(self): """Getting output value must not remove value from item""" @@ -165,9 +165,7 @@ class InitializationTestMixin: self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), dict({"name": ["foo", "bar"]}) - ) + self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) def test_values_single(self): """Values from initial item must be added to loader._values""" diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 528efa142..0d245bec9 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -526,7 +526,7 @@ class InitializationFromDictTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["foo"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, dict({"name": ["foo"]})) + self.assertEqual(loaded_item, {"name": ["foo"]}) def test_get_output_value_list(self): """Getting output value must not remove value from item""" @@ -535,7 +535,7 @@ class InitializationFromDictTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) loaded_item = il.load_item() self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, dict({"name": ["foo", "bar"]})) + self.assertEqual(loaded_item, {"name": ["foo", "bar"]}) def test_values_single(self): """Values from initial item must be added to loader._values""" diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 02b50baa3..9b7bad4bf 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -284,7 +284,7 @@ class DownloaderAwareSchedulerTestMixin: downloader.decrement(slot) self.assertTrue( - _is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS), dequeued_slots) + _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots) ) self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0) diff --git a/tests/test_spider.py b/tests/test_spider.py index d629d33af..18a863350 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -244,7 +244,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -270,7 +270,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -299,7 +299,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 2) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -324,7 +324,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -352,7 +352,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -383,7 +383,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -413,7 +413,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -445,7 +445,7 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertTrue(all(isinstance(r, Request) for r in output)) self.assertEqual( [r.url for r in output], [ @@ -637,7 +637,7 @@ Sitemap: /sitemap-relative-url.xml class FilteredSitemapSpider(self.spider_class): def sitemap_filter(self, entries): for entry in entries: - alternate_links = entry.get("alternate", tuple()) + alternate_links = entry.get("alternate", ()) for link in alternate_links: if "/deutsch/" in link: entry["loc"] = link From 1ef9c337cad36ac6c80eab86622f8ae9fc8d1075 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 19 Jun 2024 23:57:40 +0500 Subject: [PATCH 1493/2083] flake8-docstrings --- .flake8 | 33 +++++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 1 + tests/test_dupefilters.py | 2 +- tests/test_linkextractors.py | 2 +- 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/.flake8 b/.flake8 index b6048c9ee..222ba7179 100644 --- a/.flake8 +++ b/.flake8 @@ -2,11 +2,44 @@ max-line-length = 119 ignore = + # black disagrees with flake8 about these E203, E501, E701, E704, W503 # docstring does contain unindexed parameters P102 # other string does contain unindexed parameters P103 + # Missing docstring in public module + D100 + # Missing docstring in public class + D101 + # Missing docstring in public method + D102 + # Missing docstring in public function + D103 + # Missing docstring in public package + D104 + # Missing docstring in magic method + D105 + # Missing docstring in public nested class + D106 + # Missing docstring in __init__ + D107 + # One-line docstring should fit on one line with quotes + D200 + # No blank lines allowed after function docstring + D202 + # 1 blank line required between summary line and description + D205 + # Multi-line docstring closing quotes should be on a separate line + D209 + # First line should end with a period + D400 + # First line should be in imperative mood; try rephrasing + D401 + # First line should not be the function's "signature" + D402 + # First word of the first line should be properly capitalized + D403 exclude = docs/conf.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 974d397c8..6b60eff68 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,7 @@ repos: additional_dependencies: - flake8-comprehensions - flake8-debugger + - flake8-docstrings - flake8-string-format - repo: https://github.com/psf/black.git rev: 24.2.0 diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index aa0975555..f617fc027 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -146,7 +146,7 @@ class RFPDupeFilterTest(unittest.TestCase): case_insensitive_dupefilter.close("finished") def test_seenreq_newlines(self): - """Checks against adding duplicate \r to + r"""Checks against adding duplicate \r to line endings on Windows platforms.""" r1 = Request("http://scrapytest.org/1") diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index d9c09a16a..b1043c111 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -186,7 +186,7 @@ class Base: ) def test_nofollow(self): - '''Test the extractor's behaviour for links with rel="nofollow"''' + """Test the extractor's behaviour for links with rel='nofollow'""" html = b"""Page title<title> <body> From 3d8dbd5648406227c9b96736da62046b90c554e5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 20 Jun 2024 00:22:43 +0500 Subject: [PATCH 1494/2083] flake8-bugbear --- .flake8 | 15 +++++++++++++++ .pre-commit-config.yaml | 1 + scrapy/pipelines/media.py | 2 +- scrapy/utils/defer.py | 2 +- scrapy/utils/python.py | 2 +- scrapy/utils/signal.py | 5 ++++- tests/test_cmdline/__init__.py | 2 +- tests/test_command_version.py | 4 ++-- tests/test_commands.py | 2 +- tests/test_downloader_handlers.py | 2 +- tests/test_engine.py | 2 +- tests/test_request_dict.py | 2 +- 12 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.flake8 b/.flake8 index 222ba7179..57117d2cf 100644 --- a/.flake8 +++ b/.flake8 @@ -4,6 +4,21 @@ max-line-length = 119 ignore = # black disagrees with flake8 about these E203, E501, E701, E704, W503 + # Assigning to `os.environ` doesn't clear the environment. + B003 + # Do not use mutable data structures for argument defaults. + B006 + # Loop control variable not used within the loop body. + B007 + # Do not perform function calls in argument defaults. + B008 + # return/continue/break inside finally blocks cause exceptions to be + # silenced. + B012 + # Star-arg unpacking after a keyword argument is strongly discouraged + B026 + # No explicit stacklevel argument found. + B028 # docstring does contain unindexed parameters P102 # other string does contain unindexed parameters diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b60eff68..f70effc5d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,7 @@ repos: hooks: - id: flake8 additional_dependencies: + - flake8-bugbear - flake8-comprehensions - flake8-debugger - flake8-docstrings diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 3e327105e..09e95cf5d 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -234,7 +234,7 @@ class MediaPipeline(ABC): # Exception Chaining (https://www.python.org/dev/peps/pep-3134/). context = getattr(result.value, "__context__", None) if isinstance(context, StopIteration): - setattr(result.value, "__context__", None) + result.value.__context__ = None info.downloading.remove(fp) info.downloaded[fp] = result # cache result diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index ddb68c86b..877eb4388 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -407,7 +407,7 @@ def maybeDeferred_coro( """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) - except: # noqa: E722 + except: # noqa: E722,B001 return defer.fail(failure.Failure(captureVars=Deferred.debug)) if isinstance(result, Deferred): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 059d8e04d..f56950fdd 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -269,7 +269,7 @@ def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: if inspect.isfunction(func) or inspect.ismethod(func): spec = inspect.getfullargspec(func) - elif hasattr(func, "__call__"): + elif hasattr(func, "__call__"): # noqa: B004 spec = inspect.getfullargspec(func.__call__) else: raise TypeError(f"{type(func)} is not callable") diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 89cfbd2ec..bb6d807ee 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -100,7 +100,10 @@ def send_catch_log_deferred( d.addErrback(logerror, receiver) # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html d.addBoth( - lambda result: (receiver, result) # pylint: disable=cell-var-from-loop + lambda result: ( + receiver, # pylint: disable=cell-var-from-loop # noqa: B023 + result, + ) ) dfds.append(d) d = DeferredList(dfds) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 25ded143c..4835e936b 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -20,7 +20,7 @@ class CmdlineTest(unittest.TestCase): self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings" def _execute(self, *new_args, **kwargs): - encoding = getattr(sys.stdout, "encoding") or "utf-8" + encoding = sys.stdout.encoding or "utf-8" args = (sys.executable, "-m", "scrapy.cmdline") + new_args proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs) comm = proc.communicate()[0].strip() diff --git a/tests/test_command_version.py b/tests/test_command_version.py index a52d0d13c..18c1c531c 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -12,7 +12,7 @@ class VersionTest(ProcessTest, unittest.TestCase): @defer.inlineCallbacks def test_output(self): - encoding = getattr(sys.stdout, "encoding") or "utf-8" + encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute([]) self.assertEqual( out.strip().decode(encoding), @@ -21,7 +21,7 @@ class VersionTest(ProcessTest, unittest.TestCase): @defer.inlineCallbacks def test_verbose_output(self): - encoding = getattr(sys.stdout, "encoding") or "utf-8" + encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute(["-v"]) headers = [ line.partition(":")[0].strip() diff --git a/tests/test_commands.py b/tests/test_commands.py index d829b1701..a23b7f4a9 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -101,7 +101,7 @@ class ProjectTest(unittest.TestCase): def kill_proc(): p.kill() p.communicate() - assert False, "Command took too much time to complete" + raise AssertionError("Command took too much time to complete") timer = Timer(15, kill_proc) try: diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index d3fd63847..884491d01 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -892,7 +892,7 @@ class S3TestCase(unittest.TestCase): except Exception as e: self.assertIsInstance(e, (TypeError, NotConfigured)) else: - assert False + raise AssertionError() def test_request_signing1(self): # gets an object from the johnsmith bucket. diff --git a/tests/test_engine.py b/tests/test_engine.py index 33544e8db..86526420f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -459,7 +459,7 @@ class EngineTest(unittest.TestCase): def kill_proc(): p.kill() p.communicate() - assert False, "Command took too much time to complete" + raise AssertionError("Command took too much time to complete") timer = Timer(15, kill_proc) try: diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 7312eb036..d3f416347 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -147,7 +147,7 @@ class RequestSerializationTest(unittest.TestCase): spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) - setattr(spider, "parse", None) + spider.parse = None self.assertRaises(ValueError, r.to_dict, spider=spider) def test_callback_not_available(self): From 13d3b1af470bbe7e82fda51017f0f72cb8eed9dd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 20 Jun 2024 00:42:43 +0500 Subject: [PATCH 1495/2083] Split ignores into blocks. --- .flake8 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.flake8 b/.flake8 index 57117d2cf..be9d83eaf 100644 --- a/.flake8 +++ b/.flake8 @@ -4,6 +4,7 @@ max-line-length = 119 ignore = # black disagrees with flake8 about these E203, E501, E701, E704, W503 + # Assigning to `os.environ` doesn't clear the environment. B003 # Do not use mutable data structures for argument defaults. @@ -19,10 +20,12 @@ ignore = B026 # No explicit stacklevel argument found. B028 + # docstring does contain unindexed parameters P102 # other string does contain unindexed parameters P103 + # Missing docstring in public module D100 # Missing docstring in public class From 326e323e11a7f5fc760250be6eae23d0159f6429 Mon Sep 17 00:00:00 2001 From: mlmsmith <mlmsmith@hotmail.co.uk> Date: Fri, 21 Jun 2024 18:24:10 +0800 Subject: [PATCH 1496/2083] Apply grammar fixes (#6411) --- docs/intro/overview.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 542760b4f..ef1294470 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -44,13 +44,13 @@ https://quotes.toscrape.com, following the pagination: if next_page is not None: yield response.follow(next_page, self.parse) -Put this in a text file, name it to something like ``quotes_spider.py`` +Put this in a text file, name it something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: scrapy runspider quotes_spider.py -o quotes.jsonl When this finishes you will have in the ``quotes.jsonl`` file a list of the -quotes in JSON Lines format, containing text and author, looking like this:: +quotes in JSON Lines format, containing the text and author, which will look like this:: {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"} @@ -72,11 +72,11 @@ using a CSS Selector, yield a Python dict with the extracted quote text and auth look for a link to the next page and schedule another request using the same ``parse`` method as callback. -Here you notice one of the main advantages about Scrapy: requests are +Here you will notice one of the main advantages of Scrapy: requests are :ref:`scheduled and processed asynchronously <topics-architecture>`. This means that Scrapy doesn't need to wait for a request to be finished and processed, it can send another request or do other things in the meantime. This -also means that other requests can keep going even if some request fails or an +also means that other requests can keep going even if a request fails or an error happens while handling it. While this enables you to do very fast crawls (sending multiple concurrent From d08f559600f0bb45b916be158a06e033753d45f5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 25 Jun 2024 13:20:59 +0500 Subject: [PATCH 1497/2083] Add flake8-type-checking. (#6413) --- .flake8 | 4 +++ .pre-commit-config.yaml | 1 + scrapy/addons.py | 9 ++++-- scrapy/cmdline.py | 3 +- scrapy/commands/__init__.py | 8 +++-- scrapy/commands/bench.py | 8 +++-- scrapy/commands/crawl.py | 8 +++-- scrapy/commands/fetch.py | 8 +++-- scrapy/commands/list.py | 8 +++-- scrapy/commands/parse.py | 9 ++++-- scrapy/commands/runspider.py | 12 ++++--- scrapy/commands/shell.py | 8 +++-- scrapy/contracts/__init__.py | 11 +++++-- scrapy/core/downloader/__init__.py | 5 +-- scrapy/core/downloader/contextfactory.py | 5 +-- scrapy/core/downloader/handlers/__init__.py | 6 ++-- scrapy/core/downloader/handlers/datauri.py | 8 +++-- scrapy/core/downloader/handlers/file.py | 9 ++++-- scrapy/core/downloader/handlers/ftp.py | 12 ++++--- scrapy/core/downloader/handlers/http10.py | 12 +++---- scrapy/core/downloader/handlers/http11.py | 11 ++++--- scrapy/core/downloader/handlers/http2.py | 16 +++++----- scrapy/core/downloader/handlers/s3.py | 13 ++++---- scrapy/core/downloader/middleware.py | 11 ++++--- scrapy/core/downloader/webclient.py | 8 +++-- scrapy/core/engine.py | 6 ++-- scrapy/core/http2/agent.py | 18 +++++++---- scrapy/core/http2/protocol.py | 21 ++++++++----- scrapy/core/http2/stream.py | 11 ++++--- scrapy/core/scheduler.py | 16 +++++----- scrapy/core/spidermw.py | 6 +++- scrapy/downloadermiddlewares/ajaxcrawl.py | 8 +++-- scrapy/downloadermiddlewares/cookies.py | 10 +++--- .../downloadermiddlewares/defaultheaders.py | 7 +++-- .../downloadermiddlewares/downloadtimeout.py | 5 +-- scrapy/downloadermiddlewares/httpauth.py | 5 +-- scrapy/downloadermiddlewares/httpcache.py | 13 ++++---- .../downloadermiddlewares/httpcompression.py | 6 ++-- scrapy/downloadermiddlewares/httpproxy.py | 7 +++-- scrapy/downloadermiddlewares/offsite.py | 6 ++-- scrapy/downloadermiddlewares/redirect.py | 8 +++-- scrapy/downloadermiddlewares/retry.py | 10 +++--- scrapy/downloadermiddlewares/robotstxt.py | 10 +++--- scrapy/downloadermiddlewares/stats.py | 9 +++--- scrapy/downloadermiddlewares/useragent.py | 5 +-- scrapy/dupefilters.py | 10 +++--- scrapy/extension.py | 8 +++-- scrapy/extensions/closespider.py | 10 +++--- scrapy/extensions/corestats.py | 5 +-- scrapy/extensions/debug.py | 7 +++-- scrapy/extensions/feedexport.py | 17 +++++----- scrapy/extensions/httpcache.py | 9 ++++-- scrapy/extensions/logstats.py | 6 ++-- scrapy/extensions/memdebug.py | 5 +-- scrapy/extensions/memusage.py | 4 ++- scrapy/extensions/periodic_log.py | 6 ++-- scrapy/extensions/spiderstate.py | 3 +- scrapy/extensions/statsmailer.py | 9 +++--- scrapy/extensions/telnet.py | 5 ++- scrapy/extensions/throttle.py | 8 +++-- scrapy/http/cookies.py | 6 ++-- scrapy/http/request/form.py | 3 +- scrapy/http/response/__init__.py | 7 +++-- scrapy/http/response/text.py | 3 +- scrapy/linkextractors/lxmlhtml.py | 13 ++++++-- scrapy/loader/__init__.py | 10 ++++-- scrapy/logformatter.py | 5 +-- scrapy/mail.py | 6 ++-- scrapy/middleware.py | 8 ++--- scrapy/pipelines/__init__.py | 12 ++++--- scrapy/pipelines/files.py | 31 ++++++++++--------- scrapy/pipelines/images.py | 11 ++++--- scrapy/pipelines/media.py | 10 +++--- scrapy/resolver.py | 3 +- scrapy/robotstxt.py | 3 +- scrapy/settings/__init__.py | 3 +- scrapy/signalmanager.py | 8 +++-- scrapy/spiderloader.py | 8 +++-- scrapy/spidermiddlewares/depth.py | 8 +++-- scrapy/spidermiddlewares/httperror.py | 10 +++--- scrapy/spidermiddlewares/offsite.py | 6 ++-- scrapy/spidermiddlewares/referer.py | 6 ++-- scrapy/spidermiddlewares/urllength.py | 6 ++-- scrapy/spiders/init.py | 8 +++-- scrapy/squeues.py | 8 +++-- scrapy/statscollectors.py | 10 +++--- scrapy/utils/decorators.py | 8 ++--- scrapy/utils/defer.py | 3 +- scrapy/utils/engine.py | 5 +-- scrapy/utils/gz.py | 8 +++-- scrapy/utils/httpobj.py | 10 ++++-- scrapy/utils/job.py | 9 ++++-- scrapy/utils/log.py | 3 +- scrapy/utils/misc.py | 4 ++- scrapy/utils/project.py | 5 +-- scrapy/utils/reactor.py | 8 +++-- scrapy/utils/spider.py | 10 +++--- scrapy/utils/ssl.py | 8 +++-- scrapy/utils/template.py | 8 +++-- scrapy/utils/test.py | 19 ++++++++++-- scrapy/utils/testproc.py | 6 ++-- tests/mockserver.py | 8 +++-- tests/test_feedexport.py | 8 +++-- tests/test_http2_client_protocol.py | 8 +++-- 104 files changed, 562 insertions(+), 300 deletions(-) diff --git a/.flake8 b/.flake8 index be9d83eaf..c4814f13a 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,7 @@ [flake8] max-line-length = 119 +extend-select = TC, TC1 ignore = # black disagrees with flake8 about these E203, E501, E701, E704, W503 @@ -58,6 +59,9 @@ ignore = D402 # First word of the first line should be properly capitalized D403 + + # Annotation in typing.cast() should be a string literal + TC006 exclude = docs/conf.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f70effc5d..38526d720 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,7 @@ repos: - flake8-debugger - flake8-docstrings - flake8-string-format + - flake8-type-checking - repo: https://github.com/psf/black.git rev: 24.2.0 hooks: diff --git a/scrapy/addons.py b/scrapy/addons.py index 65d7a0310..f9ec58cea 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,13 +1,16 @@ +from __future__ import annotations + import logging from typing import TYPE_CHECKING, Any, List from scrapy.exceptions import NotConfigured -from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.settings import Settings + logger = logging.getLogger(__name__) @@ -15,8 +18,8 @@ logger = logging.getLogger(__name__) class AddonManager: """This class facilitates loading and storing :ref:`topics-addons`.""" - def __init__(self, crawler: "Crawler") -> None: - self.crawler: "Crawler" = crawler + def __init__(self, crawler: Crawler) -> None: + self.crawler: Crawler = crawler self.addons: List[Any] = [] def load_settings(self, settings: Settings) -> None: diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index da0e51386..e010b159a 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -12,7 +12,6 @@ import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError -from scrapy.settings import BaseSettings, Settings from scrapy.utils.misc import walk_modules from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect @@ -21,6 +20,8 @@ if TYPE_CHECKING: # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec + from scrapy.settings import BaseSettings, Settings + _P = ParamSpec("_P") diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9fe803d3c..0322390e5 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -2,18 +2,22 @@ Base class for Scrapy commands """ +from __future__ import annotations + import argparse import builtins import os from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional from twisted.python import failure -from scrapy.crawler import Crawler, CrawlerProcess from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +if TYPE_CHECKING: + from scrapy.crawler import Crawler, CrawlerProcess + class ScrapyCommand: requires_project: bool = False diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 0c4ebcd23..f91fec57e 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,16 +1,20 @@ +from __future__ import annotations + import argparse import subprocess # nosec import sys import time -from typing import Any, Iterable, List +from typing import TYPE_CHECKING, Any, Iterable, List from urllib.parse import urlencode import scrapy -from scrapy import Request from scrapy.commands import ScrapyCommand from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor +if TYPE_CHECKING: + from scrapy import Request + class Command(ScrapyCommand): default_settings = { diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 6e023af81..fe1864372 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,11 +1,15 @@ -import argparse -from typing import List, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, List, cast from twisted.python.failure import Failure from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError +if TYPE_CHECKING: + import argparse + class Command(BaseRunSpiderCommand): requires_project = True diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 1acf2d26f..0bdc429da 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,6 +1,7 @@ +from __future__ import annotations + import sys -from argparse import ArgumentParser, Namespace -from typing import Dict, List, Type +from typing import TYPE_CHECKING, Dict, List, Type from w3lib.url import is_url @@ -11,6 +12,9 @@ from scrapy.http import Request, Response from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import DefaultSpider, spidercls_for_request +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index dcc51a694..10330c92a 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,8 +1,12 @@ -import argparse -from typing import List +from __future__ import annotations + +from typing import TYPE_CHECKING, List from scrapy.commands import ScrapyCommand +if TYPE_CHECKING: + import argparse + class Command(ScrapyCommand): requires_project = True diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 3320a1ee4..e6c5e2a47 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -6,6 +6,7 @@ import inspect import json import logging from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, Callable, @@ -22,13 +23,11 @@ from typing import ( from itemadapter import ItemAdapter, is_item from twisted.internet.defer import Deferred, maybeDeferred -from twisted.python.failure import Failure from w3lib.url import is_url from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.http import Request, Response -from scrapy.spiders import Spider from scrapy.utils import display from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import aiter_errback, deferred_from_coro @@ -36,6 +35,12 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter from scrapy.utils.spider import spidercls_for_request +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy.spiders import Spider + + logger = logging.getLogger(__name__) _T = TypeVar("_T") diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 77850e7b5..87acf9a01 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,17 +1,21 @@ +from __future__ import annotations + import argparse import sys from importlib import import_module -from os import PathLike from pathlib import Path -from types import ModuleType -from typing import List, Union +from typing import TYPE_CHECKING, List, Union from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.utils.spider import iter_spider_classes +if TYPE_CHECKING: + from os import PathLike + from types import ModuleType -def _import_file(filepath: Union[str, PathLike]) -> ModuleType: + +def _import_file(filepath: Union[str, PathLike[str]]) -> ModuleType: abspath = Path(filepath).resolve() if abspath.suffix not in (".py", ".pyw"): raise ValueError(f"Not a Python source file: {abspath}") diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 668c95a7b..f03cf997a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -4,9 +4,10 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -from argparse import ArgumentParser, Namespace +from __future__ import annotations + from threading import Thread -from typing import Any, Dict, List, Type +from typing import TYPE_CHECKING, Any, Dict, List, Type from scrapy import Spider from scrapy.commands import ScrapyCommand @@ -15,6 +16,9 @@ from scrapy.shell import Shell from scrapy.utils.spider import DefaultSpider, spidercls_for_request from scrapy.utils.url import guess_scheme +if TYPE_CHECKING: + from argparse import ArgumentParser, Namespace + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 27bc2fcba..440e0dc44 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -1,9 +1,12 @@ +from __future__ import annotations + import re import sys from functools import wraps from inspect import getmembers from types import CoroutineType from typing import ( + TYPE_CHECKING, Any, AsyncGenerator, Callable, @@ -16,13 +19,15 @@ from typing import ( ) from unittest import TestCase, TestResult -from twisted.python.failure import Failure - -from scrapy import Spider from scrapy.http import Request, Response from scrapy.utils.python import get_spec from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy import Spider + class Contract: """Abstract class for contracts""" diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 41f729ed9..6786d7acf 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -25,15 +25,16 @@ from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.http import Response from scrapy.resolver import dnscache -from scrapy.settings import BaseSettings from scrapy.signalmanager import SignalManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + _T = TypeVar("_T") diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 9f6edb630..2b388a9f5 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,8 +21,6 @@ from scrapy.core.downloader.tls import ( ScrapyClientTLSOptions, openssl_methods, ) -from scrapy.crawler import Crawler -from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: @@ -31,6 +29,9 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + @implementer(IPolicyForHTTPS) class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index ebc4898b5..70d356b83 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -17,17 +17,19 @@ from typing import ( ) from twisted.internet import defer -from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, NotSupported -from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from twisted.internet.defer import Deferred + from scrapy.crawler import Crawler + from scrapy.http import Response + logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index a7ae56a85..bf6879521 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -1,12 +1,16 @@ -from typing import Any, Dict +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict from w3lib.url import parse_data_uri -from scrapy import Request, Spider from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers +if TYPE_CHECKING: + from scrapy import Request, Spider + class DataURIDownloadHandler: lazy = False diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 17dd7483b..d55c516f0 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -1,12 +1,17 @@ +from __future__ import annotations + from pathlib import Path +from typing import TYPE_CHECKING from w3lib.url import file_uri_to_path -from scrapy import Request, Spider -from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers +if TYPE_CHECKING: + from scrapy import Request, Spider + from scrapy.http import Response + class FileDownloadHandler: lazy = False diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 724717ffd..69c2d88e1 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -35,23 +35,25 @@ from io import BytesIO from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional from urllib.parse import unquote -from twisted.internet.defer import Deferred from twisted.internet.protocol import ClientCreator, Protocol from twisted.protocols.ftp import CommandFailed, FTPClient -from twisted.python.failure import Failure -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.responsetypes import responsetypes -from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes if TYPE_CHECKING: + from twisted.internet.defer import Deferred + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + class ReceivedDataProtocol(Protocol): def __init__(self, filename: Optional[str] = None): diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 3c4e48abb..98f62efcf 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -5,21 +5,21 @@ from __future__ import annotations from typing import TYPE_CHECKING, Type -from twisted.internet.defer import Deferred - -from scrapy import Request, Spider -from scrapy.crawler import Crawler -from scrapy.http import Response -from scrapy.settings import BaseSettings from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings class HTTP10DownloadHandler: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index e2ad8f59a..c06d90f01 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -12,11 +12,9 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar from urllib.parse import urldefrag, urlunparse from twisted.internet import ssl -from twisted.internet.base import ReactorBase from twisted.internet.defer import CancelledError, Deferred, succeed from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError -from twisted.internet.interfaces import IConsumer from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure from twisted.web.client import URI, Agent, HTTPConnectionPool @@ -30,17 +28,22 @@ from zope.interface import implementer from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse -from scrapy.crawler import Crawler from scrapy.exceptions import StopDownload from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes -from scrapy.settings import BaseSettings from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: + from twisted.internet.base import ReactorBase + from twisted.internet.interfaces import IConsumer + # typing.NotRequired and typing.Self require Python 3.11 from typing_extensions import NotRequired, Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) _T = TypeVar("_T") diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 2ac4eca86..4722c612d 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -4,25 +4,27 @@ from time import time from typing import TYPE_CHECKING, Optional from urllib.parse import urldefrag -from twisted.internet.base import DelayedCall -from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.web.client import URI -from twisted.web.iweb import IPolicyForHTTPS from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent -from scrapy.crawler import Crawler -from scrapy.http import Request, Response -from scrapy.settings import Settings -from scrapy.spiders import Spider from scrapy.utils.python import to_bytes if TYPE_CHECKING: + from twisted.internet.base import DelayedCall + from twisted.internet.defer import Deferred + from twisted.web.iweb import IPolicyForHTTPS + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Request, Response + from scrapy.settings import Settings + from scrapy.spiders import Spider + class H2DownloadHandler: def __init__(self, settings: Settings, crawler: Crawler): diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 0ad340721..edf370193 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,22 +2,23 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional, Type -from twisted.internet.defer import Deferred - -from scrapy import Request, Spider from scrapy.core.downloader.handlers.http import HTTPDownloadHandler -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response -from scrapy.settings import BaseSettings from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + class S3DownloadHandler: def __init__( diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 2d8af114f..0bdb756c8 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -6,19 +6,22 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import Any, Callable, Generator, List, Union, cast +from typing import TYPE_CHECKING, Any, Callable, Generator, List, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks -from twisted.python.failure import Failure -from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager -from scrapy.settings import BaseSettings from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_from_coro, mustbe_deferred +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from scrapy import Spider + from scrapy.settings import BaseSettings + class DownloaderMiddlewareManager(MiddlewareManager): component_name = "downloader middleware" diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 08a1d7c71..99502f0d2 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,18 +1,22 @@ +from __future__ import annotations + import re from time import time -from typing import Optional, Tuple +from typing import TYPE_CHECKING, Optional, Tuple from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from twisted.internet import defer from twisted.internet.protocol import ClientFactory from twisted.web.http import HTTPClient -from scrapy import Request from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from scrapy import Request + def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]: # Assume parsed is urlparse-d from Request.url, diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4ffec78b9..5318cbd64 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -34,9 +34,8 @@ from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter -from scrapy.settings import BaseSettings, Settings +from scrapy.settings import Settings from scrapy.signalmanager import SignalManager -from scrapy.spiders import Spider from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import global_object_name @@ -46,6 +45,9 @@ if TYPE_CHECKING: from scrapy.core.scheduler import BaseScheduler from scrapy.core.scraper import _HandleOutputDeferred from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 999764a6e..d291a5b8a 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,10 +1,10 @@ +from __future__ import annotations + from collections import deque -from typing import Deque, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Deque, Dict, List, Optional, Tuple from twisted.internet import defer -from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred -from twisted.internet.endpoints import HostnameEndpoint from twisted.python.failure import Failure from twisted.web.client import ( URI, @@ -16,9 +16,15 @@ from twisted.web.error import SchemeNotSupported from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol -from scrapy.http.request import Request -from scrapy.settings import Settings -from scrapy.spiders import Spider + +if TYPE_CHECKING: + from twisted.internet.base import ReactorBase + from twisted.internet.endpoints import HostnameEndpoint + + from scrapy.http.request import Request + from scrapy.settings import Settings + from scrapy.spiders import Spider + ConnectionKeyT = Tuple[bytes, bytes, int] diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index f2f1cb0b8..a6809102b 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -1,9 +1,10 @@ +from __future__ import annotations + import ipaddress import itertools import logging from collections import deque -from ipaddress import IPv4Address, IPv6Address -from typing import Any, Deque, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -20,7 +21,6 @@ from h2.events import ( WindowUpdated, ) from h2.exceptions import FrameTooLargeError, H2Error -from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError from twisted.internet.interfaces import ( IAddress, @@ -30,14 +30,21 @@ from twisted.internet.interfaces import ( from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin -from twisted.python.failure import Failure -from twisted.web.client import URI from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.http import Request -from scrapy.settings import Settings -from scrapy.spiders import Spider + +if TYPE_CHECKING: + from ipaddress import IPv4Address, IPv6Address + + from twisted.internet.defer import Deferred + from twisted.python.failure import Failure + from twisted.web.client import URI + + from scrapy.settings import Settings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 7c70e86db..a02fbb328 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging from enum import Enum from io import BytesIO @@ -5,19 +7,20 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError -from hpack import HeaderTuple from twisted.internet.defer import CancelledError, Deferred from twisted.internet.error import ConnectionClosed from twisted.python.failure import Failure from twisted.web.client import ResponseFailed -from scrapy.http import Request from scrapy.http.headers import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: + from hpack import HeaderTuple + from scrapy.core.http2.protocol import H2ClientProtocol + from scrapy.http import Request logger = logging.getLogger(__name__) @@ -87,7 +90,7 @@ class Stream: self, stream_id: int, request: Request, - protocol: "H2ClientProtocol", + protocol: H2ClientProtocol, download_maxsize: int = 0, download_warnsize: int = 0, ) -> None: @@ -99,7 +102,7 @@ class Stream: """ self.stream_id: int = stream_id self._request: Request = request - self._protocol: "H2ClientProtocol" = protocol + self._protocol: H2ClientProtocol = protocol self._download_maxsize = self._request.meta.get( "download_maxsize", download_maxsize diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 1e586c53a..d4286c874 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -6,14 +6,10 @@ from abc import abstractmethod from pathlib import Path from typing import TYPE_CHECKING, Any, List, Optional, Type, cast -from twisted.internet.defer import Deferred +# working around https://github.com/sphinx-doc/sphinx/issues/10400 +from twisted.internet.defer import Deferred # noqa: TC002 -from scrapy.crawler import Crawler -from scrapy.dupefilters import BaseDupeFilter -from scrapy.http.request import Request -from scrapy.pqueues import ScrapyPriorityQueue -from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector +from scrapy.spiders import Spider # noqa: TC001 from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object @@ -24,6 +20,12 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.dupefilters import BaseDupeFilter + from scrapy.http.request import Request + from scrapy.pqueues import ScrapyPriorityQueue + from scrapy.statscollectors import StatsCollector + logger = logging.getLogger(__name__) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index e792f8ca7..37a666605 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -10,6 +10,7 @@ import logging from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import ( + TYPE_CHECKING, Any, AsyncIterable, Callable, @@ -30,7 +31,6 @@ from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager -from scrapy.settings import BaseSettings from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( @@ -41,6 +41,10 @@ from scrapy.utils.defer import ( ) from scrapy.utils.python import MutableAsyncChain, MutableChain +if TYPE_CHECKING: + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 0e757e4be..5fc7f31a3 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -6,16 +6,18 @@ from typing import TYPE_CHECKING, Union from w3lib import html -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse, Response -from scrapy.settings import BaseSettings if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 73c2c57fe..23140d263 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -2,24 +2,26 @@ from __future__ import annotations import logging from collections import defaultdict -from http.cookiejar import Cookie from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union from tldextract import TLDExtract -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar -from scrapy.http.request import VerboseCookie from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from http.cookiejar import Cookie + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http.request import VerboseCookie + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 58fd415b9..49b9fdc05 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -8,15 +8,16 @@ from __future__ import annotations from typing import TYPE_CHECKING, Iterable, Tuple, Union -from scrapy import Request, Spider -from scrapy.crawler import Crawler -from scrapy.http import Response from scrapy.utils.python import without_none_values if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + class DefaultHeadersMiddleware: def __init__(self, headers: Iterable[Tuple[str, str]]): diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index fd7c03a38..ee7a24825 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -9,13 +9,14 @@ from __future__ import annotations from typing import TYPE_CHECKING, Union from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class DownloadTimeoutMiddleware: def __init__(self, timeout: float = 180): diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 63490a37a..39165e155 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -11,14 +11,15 @@ from typing import TYPE_CHECKING, Union from w3lib.http import basic_auth_header from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response from scrapy.utils.url import url_is_from_any_domain if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class HttpAuthMiddleware: """Set Basic HTTP Authorization header diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 971473403..8377a3c1d 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -16,19 +16,20 @@ from twisted.internet.error import ( from twisted.web.client import ResponseFailed from scrapy import signals -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.http.request import Request -from scrapy.http.response import Response -from scrapy.settings import Settings -from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http.request import Request + from scrapy.http.response import Response + from scrapy.settings import Settings + from scrapy.spiders import Spider + from scrapy.statscollectors import StatsCollector + class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = ( diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index f3647e05f..6b0a56f7f 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -6,11 +6,9 @@ from logging import getLogger from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.statscollectors import StatsCollector from scrapy.utils._compression import ( _DecompressionMaxSizeExceeded, _inflate, @@ -24,6 +22,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = getLogger(__name__) ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 5b56ad449..a7af83f7d 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -9,10 +9,7 @@ from urllib.request import ( # type: ignore[attr-defined] proxy_bypass, ) -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes @@ -20,6 +17,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + class HttpProxyMiddleware: def __init__(self, auth_encoding: Optional[str] = "latin-1"): diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index bd8dbe329..6f67e3975 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -6,15 +6,17 @@ import warnings from typing import TYPE_CHECKING, Set from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest -from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 371e2fd3b..53081237c 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -6,11 +6,8 @@ from urllib.parse import urljoin from w3lib.url import safe_url_string -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Response -from scrapy.settings import BaseSettings from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh @@ -18,6 +15,11 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 0637f09d4..8d7b7293c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -16,12 +16,8 @@ import warnings from logging import Logger, getLogger from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Response -from scrapy.http.request import Request from scrapy.settings import BaseSettings, Settings -from scrapy.spiders import Spider from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message @@ -30,6 +26,12 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.http.request import Request + from scrapy.spiders import Spider + + retry_logger = getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6a0ecb7bf..70393576b 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -10,22 +10,24 @@ import logging from typing import TYPE_CHECKING, Any, Dict, Optional, Union from twisted.internet.defer import Deferred, maybeDeferred -from twisted.python.failure import Failure -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.robotstxt import RobotParser from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object if TYPE_CHECKING: + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.robotstxt import RobotParser + logger = logging.getLogger(__name__) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 444702757..0faae7b5a 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -4,11 +4,7 @@ from typing import TYPE_CHECKING, Dict, List, Tuple, Union from twisted.web import http -from scrapy import Request, Spider -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response -from scrapy.statscollectors import StatsCollector from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr @@ -16,6 +12,11 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.statscollectors import StatsCollector + def get_header_size( headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]] diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 92f1ec897..109f1a4d9 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -5,13 +5,14 @@ from __future__ import annotations from typing import TYPE_CHECKING, Union from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler -from scrapy.http import Response if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + class UserAgentMiddleware: """This middleware allows spiders to override the user_agent""" diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index dd2420e98..ffaf783a7 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -4,11 +4,6 @@ import logging from pathlib import Path from typing import TYPE_CHECKING, Optional, Set -from twisted.internet.defer import Deferred - -from scrapy.http.request import Request -from scrapy.settings import BaseSettings -from scrapy.spiders import Spider from scrapy.utils.job import job_dir from scrapy.utils.request import ( RequestFingerprinter, @@ -17,10 +12,15 @@ from scrapy.utils.request import ( ) if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import Request + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider class BaseDupeFilter: diff --git a/scrapy/extension.py b/scrapy/extension.py index 8221b675e..8c81ab356 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -4,12 +4,16 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ -from typing import Any, List +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List from scrapy.middleware import MiddlewareManager -from scrapy.settings import Settings from scrapy.utils.conf import build_component_list +if TYPE_CHECKING: + from scrapy.settings import Settings + class ExtensionManager(MiddlewareManager): component_name = "extension" diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 812b3553c..4627e7f98 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -10,17 +10,19 @@ import logging from collections import defaultdict from typing import TYPE_CHECKING, Any, DefaultDict, Dict -from twisted.python.failure import Failure - from scrapy import Request, Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response if TYPE_CHECKING: + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.http import Response + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index f3ac19623..6ef2d0382 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -8,13 +8,14 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Optional from scrapy import Spider, signals -from scrapy.crawler import Crawler -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class CoreStats: def __init__(self, stats: StatsCollector): diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index b360ce48d..c54871e02 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -12,17 +12,20 @@ import sys import threading import traceback from pdb import Pdb -from types import FrameType from typing import TYPE_CHECKING, Optional -from scrapy.crawler import Crawler from scrapy.utils.engine import format_engine_status from scrapy.utils.trackref import format_live_refs if TYPE_CHECKING: + from types import FrameType + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 941bd4b26..43c2d2815 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -31,18 +31,15 @@ from typing import ( ) from urllib.parse import unquote, urlparse -from twisted.internet import threads from twisted.internet.defer import Deferred, DeferredList, maybeDeferred -from twisted.python.failure import Failure +from twisted.internet.threads import deferToThread from w3lib.url import file_uri_to_path from zope.interface import Interface, implementer from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.exporters import BaseItemExporter from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.settings import BaseSettings, Settings +from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import maybe_deferred_to_future @@ -54,11 +51,14 @@ from scrapy.utils.python import without_none_values if TYPE_CHECKING: from _typeshed import OpenBinaryMode + from twisted.python.failure import Failure # typing.Self requires Python 3.11 from typing_extensions import Self -logger = logging.getLogger(__name__) + from scrapy.crawler import Crawler + from scrapy.exporters import BaseItemExporter + from scrapy.settings import BaseSettings try: import boto3 # noqa: F401 @@ -67,6 +67,9 @@ try: except ImportError: IS_BOTO3_AVAILABLE = False + +logger = logging.getLogger(__name__) + UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]] _StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") @@ -160,7 +163,7 @@ class BlockingFeedStorage: return NamedTemporaryFile(prefix="feed-", dir=path) def store(self, file: IO[bytes]) -> Optional[Deferred]: - return threads.deferToThread(self._store_in_thread, file) + return deferToThread(self._store_in_thread, file) def _store_in_thread(self, file: IO[bytes]) -> None: raise NotImplementedError diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index b7219bf07..448d5f1ab 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import gzip import logging import os @@ -13,10 +15,7 @@ from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.http import Headers, Response -from scrapy.http.request import Request from scrapy.responsetypes import responsetypes -from scrapy.settings import BaseSettings -from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode @@ -26,6 +25,10 @@ if TYPE_CHECKING: # typing.Concatenate requires Python 3.10 from typing_extensions import Concatenate + from scrapy.http.request import Request + from scrapy.settings import BaseSettings + from scrapy.spiders import Spider + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 2388afa75..c4f43482d 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -6,14 +6,16 @@ from typing import TYPE_CHECKING, Optional, Tuple, Union from twisted.internet import task from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index f304e1bf2..3cbbb64e5 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -10,15 +10,16 @@ import gc from typing import TYPE_CHECKING from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.statscollectors import StatsCollector from scrapy.utils.trackref import live_refs if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class MemoryDebugger: def __init__(self, stats: StatsCollector): diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 9de06b24d..25f63ecc6 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -16,7 +16,6 @@ from typing import TYPE_CHECKING, List from twisted.internet import task from scrapy import signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender from scrapy.utils.engine import get_engine_status @@ -25,6 +24,9 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 9567f948a..80c0a3b26 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -8,15 +8,17 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from twisted.internet import task from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.statscollectors import StatsCollector from scrapy.utils.serialize import ScrapyJSONEncoder if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index c6eb20277..567efd7a1 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Optional from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir @@ -13,6 +12,8 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + class SpiderState: """Store and load spider state during a scraping job""" diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 20b8f910c..e43de6f5c 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -8,18 +8,19 @@ from __future__ import annotations from typing import TYPE_CHECKING, List, Optional -from twisted.internet.defer import Deferred - from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + class StatsMailer: def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender): diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 00c69434c..c4e01b3d9 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -26,7 +26,6 @@ except (ImportError, SyntaxError): TWISTED_CONCH_AVAILABLE = False from scrapy import signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import defers from scrapy.utils.engine import print_engine_status @@ -36,6 +35,10 @@ from scrapy.utils.trackref import print_live_refs if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) # signal to update telnet variables diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 217e61a81..6ce9ce63a 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -4,15 +4,17 @@ import logging from typing import TYPE_CHECKING, Optional, Tuple from scrapy import Request, Spider, signals -from scrapy.core.downloader import Slot -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from scrapy.http import Response if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.core.downloader import Slot + from scrapy.crawler import Crawler + from scrapy.http import Response + + logger = logging.getLogger(__name__) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 8af89c74f..cc88a9420 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -17,8 +17,6 @@ from typing import ( cast, ) -from scrapy import Request -from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -26,6 +24,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request + from scrapy.http import Response + + # Defined in the http.cookiejar module, but undocumented: # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 IPV4_RE = re.compile(r"\.\d+$", re.ASCII) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ea98ed795..a8c242e8b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -28,13 +28,14 @@ from lxml.html import TextareaElement # nosec from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request -from scrapy.http.response.text import TextResponse from scrapy.utils.python import is_listlike, to_bytes if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.http.response.text import TextResponse + FormdataVType = Union[str, Iterable[str]] FormdataKVType = Tuple[str, FormdataVType] diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 684439097..ff3581abb 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,7 +7,6 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from ipaddress import IPv4Address, IPv6Address from typing import ( TYPE_CHECKING, Any, @@ -26,8 +25,6 @@ from typing import ( ) from urllib.parse import urljoin -from twisted.internet.ssl import Certificate - from scrapy.exceptions import NotSupported from scrapy.http.headers import Headers from scrapy.http.request import CookiesT, Request @@ -35,6 +32,10 @@ from scrapy.link import Link from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + from ipaddress import IPv4Address, IPv6Address + + from twisted.internet.ssl import Certificate + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index df4d90829..0635f744f 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -35,15 +35,16 @@ from w3lib.encoding import ( ) from w3lib.html import strip_html5_whitespace -from scrapy.http.request import CookiesT, Request from scrapy.http.response import Response from scrapy.link import Link from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: + from scrapy.http.request import CookiesT, Request from scrapy.selector import Selector, SelectorList + _NONE = object() diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 33a10cd6c..d27a132b3 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -2,10 +2,13 @@ Link extractor based on lxml.html """ +from __future__ import annotations + import logging import operator from functools import partial from typing import ( + TYPE_CHECKING, Any, Callable, Iterable, @@ -20,13 +23,10 @@ from typing import ( from urllib.parse import urljoin, urlparse from lxml import etree # nosec -from lxml.html import HtmlElement # nosec from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string -from scrapy import Selector -from scrapy.http import TextResponse from scrapy.link import Link from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re from scrapy.utils.misc import arg_to_iter, rel_has_nofollow @@ -34,6 +34,13 @@ from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain +if TYPE_CHECKING: + from lxml.html import HtmlElement # nosec + + from scrapy import Selector + from scrapy.http import TextResponse + + logger = logging.getLogger(__name__) # from lxml/src/lxml/html/__init__.py diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index db0b4820f..9644cc093 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -4,14 +4,18 @@ Item Loader See documentation in docs/topics/loaders.rst """ -from typing import Any, Optional +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional import itemloaders -from scrapy.http import TextResponse from scrapy.item import Item from scrapy.selector import Selector +if TYPE_CHECKING: + from scrapy.http import TextResponse + class ItemLoader(itemloaders.ItemLoader): """ @@ -91,7 +95,7 @@ class ItemLoader(itemloaders.ItemLoader): selector: Optional[Selector] = None, response: Optional[TextResponse] = None, parent: Optional[itemloaders.ItemLoader] = None, - **context: Any + **context: Any, ): if selector is None and response is not None: try: diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 42a03b560..601209fb0 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -6,8 +6,9 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union from twisted.python.failure import Failure -from scrapy import Request, Spider -from scrapy.http import Response +# working around https://github.com/sphinx-doc/sphinx/issues/10400 +from scrapy import Request, Spider # noqa: TC001 +from scrapy.http import Response # noqa: TC001 from scrapy.utils.request import referer_str if TYPE_CHECKING: diff --git a/scrapy/mail.py b/scrapy/mail.py index f4ce2800c..3ea20e831 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -30,20 +30,22 @@ from typing import ( from twisted import version as twisted_version from twisted.internet import ssl from twisted.internet.defer import Deferred -from twisted.python.failure import Failure from twisted.python.versions import Version -from scrapy.settings import BaseSettings from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes if TYPE_CHECKING: # imports twisted.internet.reactor from twisted.mail.smtp import ESMTPSenderFactory + from twisted.python.failure import Failure # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index f60c726f9..ea5488ba1 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -17,19 +17,19 @@ from typing import ( cast, ) -from twisted.internet.defer import Deferred - -from scrapy import Spider from scrapy.exceptions import NotConfigured -from scrapy.settings import Settings from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider from scrapy.crawler import Crawler + from scrapy.settings import Settings logger = logging.getLogger(__name__) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 21d649e3c..480a5a58c 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -6,16 +6,18 @@ See documentation in docs/item-pipeline.rst from __future__ import annotations -from typing import Any, List +from typing import TYPE_CHECKING, Any, List -from twisted.internet.defer import Deferred - -from scrapy import Spider from scrapy.middleware import MiddlewareManager -from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_f_from_coro_f +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + + from scrapy import Spider + from scrapy.settings import Settings + class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 85a8c77da..1a13aeaf2 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -16,7 +16,6 @@ from collections import defaultdict from contextlib import suppress from ftplib import FTP from io import BytesIO -from os import PathLike from pathlib import Path from typing import ( IO, @@ -38,11 +37,9 @@ from typing import ( from urllib.parse import urlparse from itemadapter import ItemAdapter -from twisted.internet import defer, threads -from twisted.internet.defer import Deferred -from twisted.python.failure import Failure +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.threads import deferToThread -from scrapy import Spider from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -56,9 +53,15 @@ from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str if TYPE_CHECKING: + from os import PathLike + + from twisted.python.failure import Failure + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + logger = logging.getLogger(__name__) @@ -210,7 +213,7 @@ class S3FilesStore: key_name = f"{self.prefix}{path}" return cast( "Deferred[Dict[str, Any]]", - threads.deferToThread( + deferToThread( self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined] ), ) @@ -229,7 +232,7 @@ class S3FilesStore: extra = self._headers_to_botocore_kwargs(self.HEADERS) if headers: extra.update(self._headers_to_botocore_kwargs(headers)) - return threads.deferToThread( + return deferToThread( self.s3_client.put_object, # type: ignore[attr-defined] Bucket=self.bucket, Key=key_name, @@ -326,9 +329,7 @@ class GCSFilesStore: blob_path = self._get_blob_path(path) return cast( Deferred[StatInfo], - threads.deferToThread(self.bucket.get_blob, blob_path).addCallback( - _onsuccess - ), + deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess), ) def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str: @@ -351,7 +352,7 @@ class GCSFilesStore: blob = self.bucket.blob(blob_path) blob.cache_control = self.CACHE_CONTROL blob.metadata = {k: str(v) for k, v in (meta or {}).items()} - return threads.deferToThread( + return deferToThread( blob.upload_from_string, data=buf.getvalue(), content_type=self._get_content_type(headers), @@ -388,7 +389,7 @@ class FTPFilesStore: headers: Optional[Dict[str, str]] = None, ) -> Deferred[Any]: path = f"{self.basedir}/{path}" - return threads.deferToThread( + return deferToThread( ftp_store_file, path=path, file=buf, @@ -418,7 +419,7 @@ class FTPFilesStore: except Exception: return {} - return cast("Deferred[StatInfo]", threads.deferToThread(_stat_file, path)) + return cast("Deferred[StatInfo]", deferToThread(_stat_file, path)) class FilesPipeline(MediaPipeline): @@ -553,8 +554,8 @@ class FilesPipeline(MediaPipeline): } path = self.file_path(request, info=info, item=item) - # defer.maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type - dfd: Deferred[StatInfo] = defer.maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type] + # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type + dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type] dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess) dfd2.addErrback(lambda _: None) dfd2.addErrback( diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 27a57b17c..166f81314 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,7 +11,6 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from os import PathLike from typing import ( TYPE_CHECKING, Any, @@ -28,7 +27,6 @@ from typing import ( from itemadapter import ItemAdapter -from scrapy import Spider from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -40,15 +38,20 @@ from scrapy.pipelines.files import ( S3FilesStore, _md5sum, ) -from scrapy.pipelines.media import FileInfoOrError, MediaPipeline from scrapy.settings import Settings from scrapy.utils.python import get_func_args, to_bytes if TYPE_CHECKING: - # typing.Self requires Python 3.11 + from os import PathLike + from PIL import Image + + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.pipelines.media import FileInfoOrError, MediaPipeline + class NoimagesDrop(DropItem): """Product with no images exception""" diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 09e95cf5d..ea36a9e8a 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -25,21 +25,23 @@ from typing import ( from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure -from scrapy import Spider -from scrapy.crawler import Crawler -from scrapy.http import Response from scrapy.http.request import NO_CALLBACK, Request from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import defer_result, mustbe_deferred from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter -from scrapy.utils.request import RequestFingerprinter if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.utils.request import RequestFingerprinter + + _T = TypeVar("_T") diff --git a/scrapy/resolver.py b/scrapy/resolver.py index ba7cd716b..d5eedf9b1 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type from twisted.internet import defer from twisted.internet.base import ReactorBase, ThreadedResolver -from twisted.internet.defer import Deferred from twisted.internet.interfaces import ( IAddress, IHostnameResolver, @@ -17,6 +16,8 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache if TYPE_CHECKING: + from twisted.internet.defer import Deferred + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index a33f73306..0d282dc37 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -6,7 +6,6 @@ from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING, Optional, Union from warnings import warn -from scrapy import Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode @@ -14,8 +13,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index ea1db03f1..6703c569f 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -4,7 +4,6 @@ import copy import json from importlib import import_module from pprint import pformat -from types import ModuleType from typing import ( TYPE_CHECKING, Any, @@ -27,6 +26,8 @@ from scrapy.settings import default_settings _SettingsKeyT = Union[bool, float, int, str, None] if TYPE_CHECKING: + from types import ModuleType + # https://github.com/python/typing/issues/445#issuecomment-1131458824 from _typeshed import SupportsItems diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index f6df191d8..3d37b8235 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,10 +1,14 @@ -from typing import Any, List, Tuple +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List, Tuple from pydispatch import dispatcher -from twisted.internet.defer import Deferred from scrapy.utils import signal as _signal +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + class SignalManager: def __init__(self, sender: Any = dispatcher.Anonymous): diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index d855c962c..b8fe65668 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,21 +3,23 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from types import ModuleType from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer -from scrapy import Request, Spider from scrapy.interfaces import ISpiderLoader -from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: + from types import ModuleType + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request, Spider + from scrapy.settings import BaseSettings + @implementer(ISpiderLoader) class SpiderLoader: diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 1e96654e2..c5b7f0749 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -9,15 +9,17 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.http import Request, Response -from scrapy.statscollectors import StatsCollector if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 35c869a75..ea1686c25 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -9,16 +9,18 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, Iterable, List, Optional -from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest -from scrapy.http import Response -from scrapy.settings import BaseSettings if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.http import Response + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 50c93ac9f..379c5d0a3 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -12,10 +12,8 @@ import warnings from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response -from scrapy.statscollectors import StatsCollector from scrapy.utils.httpobj import urlparse_cached warnings.warn( @@ -28,6 +26,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.statscollectors import StatsCollector + + logger = logging.getLogger(__name__) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 8af0bdf5b..d35cf8f71 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -23,10 +23,8 @@ from urllib.parse import urlparse from w3lib.url import safe_url_string from scrapy import Spider, signals -from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.settings import BaseSettings from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url @@ -35,6 +33,10 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings + + LOCAL_SCHEMES: Tuple[str, ...] = ( "about", "blob", diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index e2aa554a7..34df54ca7 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -9,15 +9,17 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable -from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.settings import BaseSettings if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Spider + from scrapy.settings import BaseSettings + + logger = logging.getLogger(__name__) diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index a0898a0cf..ce0f1bbaa 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,10 +1,14 @@ -from typing import Any, Iterable, Optional, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Iterable, Optional, cast from scrapy import Request -from scrapy.http import Response from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from scrapy.http import Response + class InitSpider(Spider): """Base Spider with initialization facilities""" diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 6f80ee388..d3e7896c5 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -6,20 +6,22 @@ from __future__ import annotations import marshal import pickle # nosec -from os import PathLike from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union from queuelib import queue -from scrapy import Request -from scrapy.crawler import Crawler from scrapy.utils.request import request_from_dict if TYPE_CHECKING: + from os import PathLike + # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy import Request + from scrapy.crawler import Crawler + def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index ab571a3ab..88e72f366 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -2,15 +2,17 @@ Scrapy extension for collecting scraping stats """ +from __future__ import annotations + import logging import pprint from typing import TYPE_CHECKING, Any, Dict, Optional -from scrapy import Spider - if TYPE_CHECKING: + from scrapy import Spider from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) @@ -18,7 +20,7 @@ StatsT = Dict[str, Any] class StatsCollector: - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): self._dump: bool = crawler.settings.getbool("STATS_DUMP") self._stats: StatsT = {} @@ -67,7 +69,7 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): - def __init__(self, crawler: "Crawler"): + def __init__(self, crawler: Crawler): super().__init__(crawler) self.spider_stats: Dict[str, StatsT] = {} diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 7e82dd519..2240f0b58 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -4,8 +4,8 @@ import warnings from functools import wraps from typing import TYPE_CHECKING, Any, Callable, TypeVar -from twisted.internet import defer, threads -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.threads import deferToThread from scrapy.exceptions import ScrapyDeprecationWarning @@ -48,7 +48,7 @@ def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: @wraps(func) def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: - return defer.maybeDeferred(func, *a, **kw) + return maybeDeferred(func, *a, **kw) return wrapped @@ -60,6 +60,6 @@ def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: @wraps(func) def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: - return threads.deferToThread(func, *a, **kw) + return deferToThread(func, *a, **kw) return wrapped diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 877eb4388..1d578e8a3 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -34,12 +34,13 @@ from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred from twisted.internet.task import Cooperator from twisted.python import failure -from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: + from twisted.python.failure import Failure + # typing.Concatenate and typing.ParamSpec require Python 3.10 from typing_extensions import Concatenate, ParamSpec diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index fdcf484d4..770ee0b1b 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -4,9 +4,10 @@ from __future__ import annotations # used in global tests code from time import time # noqa: F401 -from typing import Any, List, Tuple +from typing import TYPE_CHECKING, Any, List, Tuple -from scrapy.core.engine import ExecutionEngine +if TYPE_CHECKING: + from scrapy.core.engine import ExecutionEngine def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 2e487d88b..85324361c 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,11 +1,15 @@ +from __future__ import annotations + import struct from gzip import GzipFile from io import BytesIO - -from scrapy.http import Response +from typing import TYPE_CHECKING from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded +if TYPE_CHECKING: + from scrapy.http import Response + def gunzip(data: bytes, *, max_size: int = 0) -> bytes: """Gunzip the given data and return as much data as possible. diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index d502e8910..3cf9585ec 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,12 +1,16 @@ """Helper functions for scrapy.http objects (Request, Response)""" -from typing import Union +from __future__ import annotations + +from typing import TYPE_CHECKING, Union from urllib.parse import ParseResult, urlparse from weakref import WeakKeyDictionary -from scrapy.http import Request, Response +if TYPE_CHECKING: + from scrapy.http import Request, Response -_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = ( + +_urlparse_cache: WeakKeyDictionary[Union[Request, Response], ParseResult] = ( WeakKeyDictionary() ) diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index e230e4235..488c7994b 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,7 +1,10 @@ -from pathlib import Path -from typing import Optional +from __future__ import annotations -from scrapy.settings import BaseSettings +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from scrapy.settings import BaseSettings def job_dir(settings: BaseSettings) -> Optional[str]: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index cbfd170ed..439b065a9 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -21,12 +21,13 @@ from twisted.python import log as twisted_log from twisted.python.failure import Failure import scrapy -from scrapy.logformatter import LogFormatterResult from scrapy.settings import Settings, _SettingsKeyT from scrapy.utils.versions import scrapy_components_versions if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.logformatter import LogFormatterResult + logger = logging.getLogger(__name__) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 3d11c1035..3c787e50f 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -13,7 +13,6 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from types import ModuleType from typing import ( IO, TYPE_CHECKING, @@ -35,10 +34,13 @@ from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache if TYPE_CHECKING: + from types import ModuleType + from scrapy import Spider from scrapy.crawler import Crawler from scrapy.settings import BaseSettings + _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes T = TypeVar("T") diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index de3c8eaf9..efb6af299 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,7 +1,8 @@ +from __future__ import annotations + import os import warnings from importlib import import_module -from os import PathLike from pathlib import Path from typing import Union @@ -46,7 +47,7 @@ def project_data_dir(project: str = "default") -> str: return str(d) -def data_path(path: Union[str, PathLike], createdir: bool = False) -> str: +def data_path(path: Union[str, os.PathLike[str]], createdir: bool = False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 5af6d22eb..a627db601 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -2,7 +2,6 @@ from __future__ import annotations import asyncio import sys -from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress from typing import ( TYPE_CHECKING, @@ -20,13 +19,16 @@ from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error from twisted.internet.base import DelayedCall -from twisted.internet.protocol import ServerFactory -from twisted.internet.tcp import Port from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object if TYPE_CHECKING: + from asyncio import AbstractEventLoop, AbstractEventLoopPolicy + + from twisted.internet.protocol import ServerFactory + from twisted.internet.tcp import Port + # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index b05135c04..ce754fad3 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,7 +2,6 @@ from __future__ import annotations import inspect import logging -from types import CoroutineType, ModuleType from typing import ( TYPE_CHECKING, Any, @@ -16,16 +15,19 @@ from typing import ( overload, ) -from twisted.internet.defer import Deferred - -from scrapy import Request from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter if TYPE_CHECKING: + from types import CoroutineType, ModuleType + + from twisted.internet.defer import Deferred + + from scrapy import Request from scrapy.spiderloader import SpiderLoader + logger = logging.getLogger(__name__) _T = TypeVar("_T") diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index d520ef809..95611ebd9 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,12 +1,16 @@ -from typing import Any, Optional +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL import OpenSSL.version -from OpenSSL.crypto import X509Name from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + from OpenSSL.crypto import X509Name + def ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 6b22f3bfa..08f3f2dc9 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,10 +1,14 @@ """Helper functions for working with templates""" +from __future__ import annotations + import re import string -from os import PathLike from pathlib import Path -from typing import Any, Union +from typing import TYPE_CHECKING, Any, Union + +if TYPE_CHECKING: + from os import PathLike def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None: diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 268d8d4be..fe2bfa042 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,21 +2,36 @@ This module contains some assorted functions used in tests """ +from __future__ import annotations + import asyncio import os from importlib import import_module from pathlib import Path from posixpath import split -from typing import Any, Awaitable, Dict, List, Optional, Tuple, Type, TypeVar +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Dict, + List, + Optional, + Tuple, + Type, + TypeVar, +) from unittest import TestCase, mock -from twisted.internet.defer import Deferred from twisted.trial.unittest import SkipTest from scrapy import Spider from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available +if TYPE_CHECKING: + from twisted.internet.defer import Deferred + + _T = TypeVar("_T") diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 3bdffcaa7..8882bfc5f 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,12 +2,14 @@ from __future__ import annotations import os import sys -from typing import Iterable, List, Optional, Tuple, cast +from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated from twisted.internet.protocol import ProcessProtocol -from twisted.python.failure import Failure + +if TYPE_CHECKING: + from twisted.python.failure import Failure class ProcessTest: diff --git a/tests/mockserver.py b/tests/mockserver.py index 233f6b934..6ec46aa3d 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import json import os @@ -7,12 +9,11 @@ from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp -from typing import Dict +from typing import TYPE_CHECKING, Dict from urllib.parse import urlencode from OpenSSL import SSL from twisted.internet import defer, reactor, ssl -from twisted.internet.protocol import ServerFactory from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory @@ -23,6 +24,9 @@ from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from twisted.internet.protocol import ServerFactory + def getarg(request, name, default=None, type=None): if name in request.args: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 253987e15..ea3ed3b05 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import bz2 import csv import gzip @@ -14,10 +16,9 @@ from collections import defaultdict from contextlib import ExitStack from io import BytesIO from logging import getLogger -from os import PathLike from pathlib import Path from string import ascii_letters, digits -from typing import Union +from typing import TYPE_CHECKING, Union from unittest import mock from urllib.parse import quote, urljoin from urllib.request import pathname2url @@ -53,6 +54,9 @@ from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider +if TYPE_CHECKING: + from os import PathLike + def path_to_url(path): return urljoin("file:", pathname2url(str(path))) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 995c02a1a..7ea3fe8c9 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import random import re @@ -6,7 +8,7 @@ import string from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp -from typing import Dict +from typing import TYPE_CHECKING, Dict from unittest import mock, skipIf from urllib.parse import urlencode @@ -20,7 +22,6 @@ from twisted.internet.defer import ( from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint from twisted.internet.error import TimeoutError from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS -from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from twisted.web.client import URI, ResponseFailed from twisted.web.http import H2_ENABLED @@ -33,6 +34,9 @@ from scrapy.settings import Settings from scrapy.spiders import Spider from tests.mockserver import LeafResource, Status, ssl_context_factory +if TYPE_CHECKING: + from twisted.python.failure import Failure + def generate_random_string(size): return "".join(random.choices(string.ascii_uppercase + string.digits, k=size)) From e47110f9a5a16f0628e53e16b9cb5f6a4f9721d3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 26 Jun 2024 13:01:43 +0500 Subject: [PATCH 1498/2083] Add parameteres to most Deferred instances. (#6414) --- scrapy/commands/parse.py | 6 +- scrapy/core/downloader/handlers/http10.py | 3 +- scrapy/core/http2/agent.py | 26 +++++---- scrapy/core/http2/protocol.py | 18 ++++-- scrapy/core/http2/stream.py | 6 +- scrapy/core/scraper.py | 6 +- scrapy/core/spidermw.py | 7 ++- scrapy/crawler.py | 33 +++++++---- scrapy/downloadermiddlewares/robotstxt.py | 26 ++++++--- scrapy/dupefilters.py | 4 +- scrapy/extensions/feedexport.py | 16 ++--- scrapy/extensions/statsmailer.py | 2 +- scrapy/mail.py | 12 ++-- scrapy/middleware.py | 27 ++++++--- scrapy/shell.py | 10 +++- scrapy/signalmanager.py | 4 +- scrapy/spiders/__init__.py | 10 ++-- scrapy/utils/defer.py | 71 ++++++++++++----------- scrapy/utils/signal.py | 24 ++++---- scrapy/utils/test.py | 3 +- scrapy/utils/testproc.py | 4 +- 21 files changed, 190 insertions(+), 128 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index e6c5e2a47..1265aa38e 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -153,7 +153,7 @@ class Command(BaseRunSpiderCommand): @overload def iterate_spider_output(self, result: _T) -> Iterable[Any]: ... - def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]: + def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred[Any]]: if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) @@ -233,7 +233,7 @@ class Command(BaseRunSpiderCommand): response: Response, callback: Callable, cb_kwargs: Optional[Dict[str, Any]] = None, - ) -> Deferred: + ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) return d @@ -345,7 +345,7 @@ class Command(BaseRunSpiderCommand): def prepare_request( self, spider: Spider, request: Request, opts: argparse.Namespace ) -> Request: - def callback(response: Response, **cb_kwargs: Any) -> Deferred: + def callback(response: Response, **cb_kwargs: Any) -> Deferred[List[Any]]: # memorize first request if not self.first_response: self.first_response = response diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 98f62efcf..8d7b0635c 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -10,6 +10,7 @@ from scrapy.utils.python import to_unicode if TYPE_CHECKING: from twisted.internet.defer import Deferred + from twisted.internet.interfaces import IConnector # typing.Self requires Python 3.11 from typing_extensions import Self @@ -45,7 +46,7 @@ class HTTP10DownloadHandler: self._connect(factory) return factory.deferred - def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred: + def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector: from twisted.internet import reactor host, port = to_unicode(factory.host), factory.port diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index d291a5b8a..640fb7129 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: from twisted.internet.base import ReactorBase from twisted.internet.endpoints import HostnameEndpoint - from scrapy.http.request import Request + from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider @@ -39,16 +39,18 @@ class H2ConnectionPool: self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {} # Save all requests that arrive before the connection is established - self._pending_requests: Dict[ConnectionKeyT, Deque[Deferred]] = {} + self._pending_requests: Dict[ + ConnectionKeyT, Deque[Deferred[H2ClientProtocol]] + ] = {} def get_connection( self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint - ) -> Deferred: + ) -> Deferred[H2ClientProtocol]: if key in self._pending_requests: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol # instance - d: Deferred = Deferred() + d: Deferred[H2ClientProtocol] = Deferred() self._pending_requests[key].append(d) return d @@ -63,17 +65,17 @@ class H2ConnectionPool: def _new_connection( self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint - ) -> Deferred: + ) -> Deferred[H2ClientProtocol]: self._pending_requests[key] = deque() - conn_lost_deferred: Deferred = Deferred() + conn_lost_deferred: Deferred[List[BaseException]] = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) conn_d = endpoint.connect(factory) conn_d.addCallback(self.put_connection, key) - d: Deferred = Deferred() + d: Deferred[H2ClientProtocol] = Deferred() self._pending_requests[key].append(d) return d @@ -141,7 +143,7 @@ class H2Agent: """ return uri.scheme, uri.host, uri.port - def request(self, request: Request, spider: Spider) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred[Response]: uri = URI.fromBytes(bytes(request.url, encoding="utf-8")) try: endpoint = self.get_endpoint(uri) @@ -149,9 +151,11 @@ class H2Agent: return defer.fail(Failure()) key = self.get_key(uri) - d = self._pool.get_connection(key, uri, endpoint) - d.addCallback(lambda conn: conn.request(request, spider)) - return d + d: Deferred[H2ClientProtocol] = self._pool.get_connection(key, uri, endpoint) + d2: Deferred[Response] = d.addCallback( + lambda conn: conn.request(request, spider) + ) + return d2 class ScrapyProxyH2Agent(H2Agent): diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index a6809102b..8aebbaab4 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -33,7 +33,7 @@ from twisted.protocols.policies import TimeoutMixin from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason -from scrapy.http import Request +from scrapy.http import Request, Response if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address @@ -88,7 +88,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): IDLE_TIMEOUT = 240 def __init__( - self, uri: URI, settings: Settings, conn_lost_deferred: Deferred + self, + uri: URI, + settings: Settings, + conn_lost_deferred: Deferred[List[BaseException]], ) -> None: """ Arguments: @@ -99,7 +102,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): conn_lost_deferred -- Deferred fires with the reason: Failure to notify that connection was lost """ - self._conn_lost_deferred = conn_lost_deferred + self._conn_lost_deferred: Deferred[List[BaseException]] = conn_lost_deferred config = H2Configuration(client_side=True, header_encoding="utf-8") self.conn = H2Connection(config=config) @@ -215,14 +218,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin): data = self.conn.data_to_send() self.transport.write(data) - def request(self, request: Request, spider: Spider) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred[Response]: if not isinstance(request, Request): raise TypeError( f"Expected scrapy.http.Request, received {request.__class__.__qualname__}" ) stream = self._new_stream(request, spider) - d = stream.get_response() + d: Deferred[Response] = stream.get_response() # Add the stream to the request pool self._pending_request_stream_pool.append(stream) @@ -436,7 +439,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): @implementer(IProtocolNegotiationFactory) class H2ClientFactory(Factory): def __init__( - self, uri: URI, settings: Settings, conn_lost_deferred: Deferred + self, + uri: URI, + settings: Settings, + conn_lost_deferred: Deferred[List[BaseException]], ) -> None: self.uri = uri self.settings = settings diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a02fbb328..d8b5cc8eb 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: from hpack import HeaderTuple from scrapy.core.http2.protocol import H2ClientProtocol - from scrapy.http import Request + from scrapy.http import Request, Response logger = logging.getLogger(__name__) @@ -154,7 +154,7 @@ class Stream: else: self.close(StreamCloseReason.CANCELLED) - self._deferred_response: Deferred = Deferred(_cancel) + self._deferred_response: Deferred[Response] = Deferred(_cancel) def __repr__(self) -> str: return f"Stream(id={self.stream_id!r})" @@ -180,7 +180,7 @@ class Stream: and not self.metadata["reached_warnsize"] ) - def get_response(self) -> Deferred: + def get_response(self) -> Deferred[Response]: """Simply return a Deferred which fires when response from the asynchronous request is available """ diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8a9e8f687..a7d65e1e3 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -13,6 +13,7 @@ from typing import ( Generator, Iterable, Iterator, + List, Optional, Set, Tuple, @@ -34,7 +35,6 @@ from scrapy.logformatter import LogFormatter from scrapy.pipelines import ItemPipelineManager from scrapy.signalmanager import SignalManager from scrapy.utils.defer import ( - DeferredListResultListT, aiter_errback, defer_fail, defer_succeed, @@ -54,7 +54,7 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -_ParallelResult = DeferredListResultListT[Iterator[Any]] +_ParallelResult = List[Tuple[bool, Iterator[Any]]] if TYPE_CHECKING: # parameterized Deferreds require Twisted 21.7.0 @@ -374,7 +374,7 @@ class Scraper: def _itemproc_finished( self, output: Any, item: Any, response: Response, spider: Spider - ) -> Deferred: + ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing self.slot.itemproc_size -= 1 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 37a666605..c9feac29c 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -302,7 +302,10 @@ class SpiderMiddlewareManager(MiddlewareManager): recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) result = await maybe_deferred_to_future( - self._process_spider_output(response, spider, result) + cast( + "Deferred[Union[Iterable[_T], AsyncIterable[_T]]]", + self._process_spider_output(response, spider, result), + ) ) if isinstance(result, AsyncIterable): return MutableAsyncChain(result, recovered) @@ -339,7 +342,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def process_start_requests( self, start_requests: Iterable[Request], spider: Spider - ) -> Deferred: + ) -> Deferred[Iterable[Request]]: return self._process_chain("process_start_requests", start_requests, spider) # This method is only needed until _async compatibility methods are removed. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 4fe5987a7..877ea5928 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,18 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generator, + Optional, + Set, + Type, + TypeVar, + Union, + cast, +) from twisted.internet.defer import ( Deferred, @@ -54,6 +65,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Crawler: def __init__( @@ -140,7 +153,7 @@ class Crawler: ) @inlineCallbacks - def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: + def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]: if self.crawling: raise RuntimeError("Crawling already taking place") if self._started: @@ -172,7 +185,7 @@ class Crawler: return ExecutionEngine(self, lambda _: self.stop()) @inlineCallbacks - def stop(self) -> Generator[Deferred, Any, None]: + def stop(self) -> Generator[Deferred[Any], Any, None]: """Starts a graceful stop of the crawler and returns a deferred that is fired when the crawler is stopped.""" if self.crawling: @@ -256,7 +269,7 @@ class CrawlerRunner: self.settings = settings self.spider_loader = self._get_spider_loader(settings) self._crawlers: Set[Crawler] = set() - self._active: Set[Deferred] = set() + self._active: Set[Deferred[None]] = set() self.bootstrap_failed = False def crawl( @@ -264,7 +277,7 @@ class CrawlerRunner: crawler_or_spidercls: Union[Type[Spider], str, Crawler], *args: Any, **kwargs: Any, - ) -> Deferred: + ) -> Deferred[None]: """ Run a crawler with the provided arguments. @@ -294,12 +307,12 @@ class CrawlerRunner: crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred: + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred[None]: self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) - def _done(result: Any) -> Any: + def _done(result: _T) -> _T: self.crawlers.discard(crawler) self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) @@ -335,7 +348,7 @@ class CrawlerRunner: # temporary cast until self.spider_loader is typed return Crawler(cast(Type[Spider], spidercls), self.settings) - def stop(self) -> Deferred: + def stop(self) -> Deferred[Any]: """ Stops simultaneously all the crawling jobs taking place. @@ -344,7 +357,7 @@ class CrawlerRunner: return DeferredList([c.stop() for c in list(self.crawlers)]) @inlineCallbacks - def join(self) -> Generator[Deferred, Any, None]: + def join(self) -> Generator[Deferred[Any], Any, None]: """ join() @@ -460,7 +473,7 @@ class CrawlerProcess(CrawlerRunner): ) reactor.run(installSignalHandlers=install_signal_handlers) # blocking call - def _graceful_stop_reactor(self) -> Deferred: + def _graceful_stop_reactor(self) -> Deferred[Any]: d = self.stop() d.addBoth(self._stop_reactor) return d diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 70393576b..73757162f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Dict, Optional, TypeVar, Union from twisted.internet.defer import Deferred, maybeDeferred @@ -31,6 +31,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class RobotsTxtMiddleware: DOWNLOAD_PRIORITY: int = 1000 @@ -43,7 +45,9 @@ class RobotsTxtMiddleware: "ROBOTSTXT_USER_AGENT", None ) self.crawler: Crawler = crawler - self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {} + self._parsers: Dict[ + str, Union[RobotParser, Deferred[Optional[RobotParser]], None] + ] = {} self._parserimpl: RobotParser = load_object( crawler.settings.get("ROBOTSTXT_PARSER") ) @@ -55,14 +59,18 @@ class RobotsTxtMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]: + def process_request( + self, request: Request, spider: Spider + ) -> Optional[Deferred[None]]: if request.meta.get("dont_obey_robotstxt"): return None if request.url.startswith("data:") or request.url.startswith("file:"): return None - d: Deferred = maybeDeferred(self.robot_parser, request, spider) - d.addCallback(self.process_request_2, request, spider) - return d + d: Deferred[Optional[RobotParser]] = maybeDeferred( + self.robot_parser, request, spider # type: ignore[arg-type] + ) + d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider) + return d2 def process_request_2( self, rp: Optional[RobotParser], request: Request, spider: Spider @@ -86,7 +94,7 @@ class RobotsTxtMiddleware: def robot_parser( self, request: Request, spider: Spider - ) -> Union[RobotParser, Deferred, None]: + ) -> Union[RobotParser, Deferred[Optional[RobotParser]], None]: url = urlparse_cached(request) netloc = url.netloc @@ -109,9 +117,9 @@ class RobotsTxtMiddleware: parser = self._parsers[netloc] if isinstance(parser, Deferred): - d: Deferred = Deferred() + d: Deferred[Optional[RobotParser]] = Deferred() - def cb(result: Any) -> Any: + def cb(result: Optional[RobotParser]) -> Optional[RobotParser]: d.callback(result) return result diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index ffaf783a7..40ea48510 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -31,10 +31,10 @@ class BaseDupeFilter: def request_seen(self, request: Request) -> bool: return False - def open(self) -> Optional[Deferred]: + def open(self) -> Optional[Deferred[None]]: pass - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Optional[Deferred[None]]: pass def log(self, request: Request, spider: Spider) -> None: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 43c2d2815..0d7f5bfd4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -149,7 +149,7 @@ class FeedStorageProtocol(Protocol): """Open the storage for the given spider. It must return a file-like object that will be used for the exporters""" - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: """Store the given file stream""" @@ -162,7 +162,7 @@ class BlockingFeedStorage: return NamedTemporaryFile(prefix="feed-", dir=path) - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: return deferToThread(self._store_in_thread, file) def _store_in_thread(self, file: IO[bytes]) -> None: @@ -192,7 +192,7 @@ class StdoutFeedStorage: def open(self, spider: Spider) -> IO[bytes]: return self._stdout - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: pass @@ -211,7 +211,7 @@ class FileFeedStorage: dirname.mkdir(parents=True) return Path(self.path).open(self.write_mode) - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: file.close() return None @@ -483,7 +483,7 @@ _FeedSlot = create_deprecated_class( class FeedExporter: - _pending_deferreds: List[Deferred] = [] + _pending_deferreds: List[Deferred[None]] = [] @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -570,7 +570,7 @@ class FeedExporter: self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) ) - def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]: + def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred[None]]: def get_file(slot_: FeedSlot) -> IO[bytes]: assert slot_.file if isinstance(slot_.file, PostProcessingManager): @@ -590,7 +590,7 @@ class FeedExporter: return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d: Deferred = maybeDeferred(slot.storage.store, get_file(slot)) + d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[arg-type] d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ @@ -621,7 +621,7 @@ class FeedExporter: self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") def _handle_store_success( - self, f: Failure, logmsg: str, spider: Spider, slot_type: str + self, result: Any, logmsg: str, spider: Spider, slot_type: str ) -> None: logger.info("Stored %s", logmsg, extra={"spider": spider}) assert self.crawler.stats diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index e43de6f5c..cad607514 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -39,7 +39,7 @@ class StatsMailer: crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider: Spider) -> Optional[Deferred]: + def spider_closed(self, spider: Spider) -> Optional[Deferred[None]]: spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) diff --git a/scrapy/mail.py b/scrapy/mail.py index 3ea20e831..c020732f9 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -103,7 +103,7 @@ class MailSender: mimetype: str = "text/plain", charset: Optional[str] = None, _callback: Optional[Callable[..., None]] = None, - ) -> Optional[Deferred]: + ) -> Optional[Deferred[None]]: from twisted.internet import reactor msg: MIMEBase @@ -155,7 +155,9 @@ class MailSender: ) return None - dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8")) + dfd: Deferred[Any] = self._sendmail( + rcpts, msg.as_string().encode(charset or "utf-8") + ) dfd.addCallback(self._sent_ok, to, cc, subject, len(attachs)) dfd.addErrback(self._sent_failed, to, cc, subject, len(attachs)) reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd) @@ -198,11 +200,11 @@ class MailSender: ) return failure - def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred: + def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred[Any]: from twisted.internet import reactor msg_io = BytesIO(msg) - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() factory = self._create_sender_factory(to_addrs, msg_io, d) @@ -216,7 +218,7 @@ class MailSender: return d def _create_sender_factory( - self, to_addrs: List[str], msg: IO[bytes], d: Deferred + self, to_addrs: List[str], msg: IO[bytes], d: Deferred[Any] ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory diff --git a/scrapy/middleware.py b/scrapy/middleware.py index ea5488ba1..2296db90e 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -13,6 +13,7 @@ from typing import ( List, Optional, Tuple, + TypeVar, Union, cast, ) @@ -24,16 +25,22 @@ from scrapy.utils.misc import build_from_crawler, build_from_settings, load_obje if TYPE_CHECKING: from twisted.internet.defer import Deferred + # typing.Concatenate and typing.ParamSpec require Python 3.10 # typing.Self requires Python 3.11 - from typing_extensions import Self + from typing_extensions import Concatenate, ParamSpec, Self from scrapy import Spider from scrapy.crawler import Crawler from scrapy.settings import Settings + _P = ParamSpec("_P") + logger = logging.getLogger(__name__) +_T = TypeVar("_T") +_T2 = TypeVar("_T2") + class MiddlewareManager: """Base class for implementing middleware managers""" @@ -98,16 +105,22 @@ class MiddlewareManager: if hasattr(mw, "close_spider"): self.methods["close_spider"].appendleft(mw.close_spider) - def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred: - methods = cast(Iterable[Callable], self.methods[methodname]) + def _process_parallel( + self, methodname: str, obj: _T, *args: Any + ) -> Deferred[List[_T2]]: + methods = cast( + "Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname] + ) return process_parallel(methods, obj, *args) - def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred: - methods = cast(Iterable[Callable], self.methods[methodname]) + def _process_chain(self, methodname: str, obj: _T, *args: Any) -> Deferred[_T]: + methods = cast( + "Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname] + ) return process_chain(methods, obj, *args) - def open_spider(self, spider: Spider) -> Deferred: + def open_spider(self, spider: Spider) -> Deferred[List[None]]: return self._process_parallel("open_spider", spider) - def close_spider(self, spider: Spider) -> Deferred: + def close_spider(self, spider: Spider) -> Deferred[List[None]]: return self._process_parallel("close_spider", spider) diff --git a/scrapy/shell.py b/scrapy/shell.py index 2c22d3d8f..b7e46274f 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -4,6 +4,8 @@ See documentation in docs/topics/shell.rst """ +from __future__ import annotations + import os import signal from typing import Any, Callable, Dict, Optional, Tuple, Union @@ -92,7 +94,9 @@ class Shell: self.vars, shells=shells, banner=self.vars.pop("banner", "") ) - def _schedule(self, request: Request, spider: Optional[Spider]) -> defer.Deferred: + def _schedule( + self, request: Request, spider: Optional[Spider] + ) -> defer.Deferred[Any]: if is_asyncio_reactor_installed(): # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] @@ -209,7 +213,7 @@ def inspect_response(response: Response, spider: Spider) -> None: signal.signal(signal.SIGINT, sigint_handler) -def _request_deferred(request: Request) -> defer.Deferred: +def _request_deferred(request: Request) -> defer.Deferred[Any]: """Wrap a request inside a Deferred. This function is harmful, do not use it until you know what you are doing. @@ -228,7 +232,7 @@ def _request_deferred(request: Request) -> defer.Deferred: request.errback = request_errback return result - d: defer.Deferred = defer.Deferred() + d: defer.Deferred[Any] = defer.Deferred() d.addBoth(_restore_callbacks) if request.callback: d.addCallback(request.callback) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 3d37b8235..b2c6dea5d 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -50,7 +50,9 @@ class SignalManager: kwargs.setdefault("sender", self.sender) return _signal.send_catch_log(signal, **kwargs) - def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred: + def send_catch_log_deferred( + self, signal: Any, **kwargs: Any + ) -> Deferred[List[Tuple[Any, Any]]]: """ Like :meth:`send_catch_log` but supports returning :class:`~twisted.internet.defer.Deferred` objects from signal handlers. diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 7b43f04f2..f0b0c0988 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,9 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union, cast - -from twisted.internet.defer import Deferred +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, cast from scrapy import signals from scrapy.http import Request, Response @@ -19,6 +17,8 @@ from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: from collections.abc import Callable + from twisted.internet.defer import Deferred + # typing.Concatenate requires Python 3.10 # typing.Self requires Python 3.11 from typing_extensions import Concatenate, Self @@ -105,10 +105,10 @@ class Spider(object_ref): return url_is_from_spider(request.url, cls) @staticmethod - def close(spider: Spider, reason: str) -> Union[Deferred, None]: + def close(spider: Spider, reason: str) -> Optional[Deferred[None]]: closed = getattr(spider, "closed", None) if callable(closed): - return cast(Union[Deferred, None], closed(reason)) + return cast("Optional[Deferred[None]]", closed(reason)) return None def __repr__(self) -> str: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 1d578e8a3..c5763a06c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -49,13 +49,8 @@ if TYPE_CHECKING: _T = TypeVar("_T") _T2 = TypeVar("_T2") -# copied from twisted.internet.defer -_SelfResultT = TypeVar("_SelfResultT") -_DeferredListResultItemT = Tuple[bool, _SelfResultT] -DeferredListResultListT = List[_DeferredListResultItemT[_SelfResultT]] - -def defer_fail(_failure: Failure) -> Deferred: +def defer_fail(_failure: Failure) -> Deferred[Any]: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -64,7 +59,7 @@ def defer_fail(_failure: Failure) -> Deferred: """ from twisted.internet import reactor - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() reactor.callLater(0.1, d.errback, _failure) return d @@ -78,12 +73,12 @@ def defer_succeed(result: _T) -> Deferred[_T]: """ from twisted.internet import reactor - d: Deferred = Deferred() + d: Deferred[_T] = Deferred() reactor.callLater(0.1, d.callback, result) return d -def defer_result(result: Any) -> Deferred: +def defer_result(result: Any) -> Deferred[Any]: if isinstance(result, Deferred): return result if isinstance(result, failure.Failure): @@ -138,14 +133,14 @@ def parallel( callable: Callable[Concatenate[_T, _P], _T2], *args: _P.args, **named: _P.kwargs, -) -> Deferred[DeferredListResultListT[Iterator[_T2]]]: +) -> Deferred[List[Tuple[bool, Iterator[_T2]]]]: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. Taken from: https://jcalderone.livejournal.com/24285.html """ coop = Cooperator() - work = (callable(elem, *args, **named) for elem in iterable) + work: Iterator[_T2] = (callable(elem, *args, **named) for elem in iterable) return DeferredList([coop.coiterate(work) for _ in range(count)]) @@ -198,16 +193,16 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): def __init__( self, aiterable: AsyncIterable[_T], - callable: Callable[Concatenate[_T, _P], _T2], + callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], *callable_args: _P.args, **callable_kwargs: _P.kwargs, ): self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() - self.callable: Callable[Concatenate[_T, _P], _T2] = callable + self.callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]] = callable self.callable_args: Tuple[Any, ...] = callable_args self.callable_kwargs: Dict[str, Any] = callable_kwargs self.finished: bool = False - self.waiting_deferreds: List[Deferred] = [] + self.waiting_deferreds: List[Deferred[Any]] = [] self.anext_deferred: Optional[Deferred[_T]] = None def _callback(self, result: _T) -> None: @@ -241,12 +236,12 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) - def __next__(self) -> Deferred: + def __next__(self) -> Deferred[Any]: # This puts a new Deferred into self.waiting_deferreds and returns it. # It also calls __anext__() if needed. if self.finished: raise StopIteration - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() self.waiting_deferreds.append(d) if not self.anext_deferred: self._call_anext() @@ -256,25 +251,29 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): def parallel_async( async_iterable: AsyncIterable[_T], count: int, - callable: Callable[Concatenate[_T, _P], _T2], + callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], *args: _P.args, **named: _P.kwargs, -) -> Deferred[DeferredListResultListT[Iterator[_T2]]]: - """Like parallel but for async iterators""" +) -> Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]]: + """Like ``parallel`` but for async iterators""" coop = Cooperator() - work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) - dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)]) + work: Iterator[Deferred[Any]] = _AsyncCooperatorAdapter( + async_iterable, callable, *args, **named + ) + dl: Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList( + [coop.coiterate(work) for _ in range(count)] + ) return dl def process_chain( - callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], - input: Any, + callbacks: Iterable[Callable[Concatenate[_T, _P], _T]], + input: _T, *a: _P.args, **kw: _P.kwargs, -) -> Deferred: +) -> Deferred[_T]: """Return a Deferred built by chaining the given callbacks""" - d: Deferred = Deferred() + d: Deferred[_T] = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) @@ -307,19 +306,21 @@ def process_chain_both( def process_parallel( - callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], - input: Any, + callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]], + input: _T, *a: _P.args, **kw: _P.kwargs, -) -> Deferred: +) -> Deferred[List[_T2]]: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) - d.addCallback(lambda r: [x[1] for x in r]) - d.addErrback(lambda f: f.value.subFailure) - return d + d: Deferred[List[Tuple[bool, _T2]]] = DeferredList( + dfds, fireOnOneErrback=True, consumeErrors=True + ) + d2: Deferred[List[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) + d2.addErrback(lambda f: f.value.subFailure) + return d2 def iter_errback( @@ -404,7 +405,7 @@ def deferred_f_from_coro_f( def maybeDeferred_coro( f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs -) -> Deferred: +) -> Deferred[Any]: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) @@ -420,7 +421,7 @@ def maybeDeferred_coro( return defer.succeed(result) -def deferred_to_future(d: Deferred) -> Future: +def deferred_to_future(d: Deferred[_T]) -> Future[_T]: """ .. versionadded:: 2.6.0 @@ -442,7 +443,7 @@ def deferred_to_future(d: Deferred) -> Future: return d.asFuture(_get_asyncio_event_loop()) -def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: +def maybe_deferred_to_future(d: Deferred[_T]) -> Union[Deferred[_T], Future[_T]]: """ .. versionadded:: 2.6.0 diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index bb6d807ee..4310c1d56 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,5 +1,7 @@ """Helper functions for working with signals""" +from __future__ import annotations + import collections.abc import logging from typing import Any as TypingAny @@ -27,7 +29,7 @@ def send_catch_log( signal: TypingAny = Any, sender: TypingAny = Anonymous, *arguments: TypingAny, - **named: TypingAny + **named: TypingAny, ) -> List[Tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. @@ -73,8 +75,8 @@ def send_catch_log_deferred( signal: TypingAny = Any, sender: TypingAny = Anonymous, *arguments: TypingAny, - **named: TypingAny -) -> Deferred: + **named: TypingAny, +) -> Deferred[List[Tuple[TypingAny, TypingAny]]]: """Like send_catch_log but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were fired. @@ -92,23 +94,25 @@ def send_catch_log_deferred( dont_log = named.pop("dont_log", None) spider = named.get("spider", None) - dfds = [] + dfds: List[Deferred[Tuple[TypingAny, TypingAny]]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - d = maybeDeferred_coro( + d: Deferred[TypingAny] = maybeDeferred_coro( robustApply, receiver, signal=signal, sender=sender, *arguments, **named ) d.addErrback(logerror, receiver) # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html - d.addBoth( + d2: Deferred[Tuple[TypingAny, TypingAny]] = d.addBoth( lambda result: ( receiver, # pylint: disable=cell-var-from-loop # noqa: B023 result, ) ) - dfds.append(d) - d = DeferredList(dfds) - d.addCallback(lambda out: [x[1] for x in out]) - return d + dfds.append(d2) + dl = DeferredList(dfds) + d3: Deferred[List[Tuple[TypingAny, TypingAny]]] = dl.addCallback( + lambda out: [x[1] for x in out] + ) + return d3 def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index fe2bfa042..30f235592 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -30,6 +30,7 @@ from scrapy.utils.boto import is_botocore_available if TYPE_CHECKING: from twisted.internet.defer import Deferred + from twisted.web.client import Response as TxResponse _T = TypeVar("_T") @@ -159,7 +160,7 @@ def mock_google_cloud_storage() -> Tuple[Any, Any, Any]: return (client_mock, bucket_mock, blob_mock) -def get_web_client_agent_req(url: str) -> Deferred: +def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: from twisted.internet import reactor from twisted.web.client import Agent # imports twisted.internet.reactor diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 8882bfc5f..bb269a9f5 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -22,7 +22,7 @@ class ProcessTest: args: Iterable[str], check_code: bool = True, settings: Optional[str] = None, - ) -> Deferred: + ) -> Deferred[TestProcessProtocol]: from twisted.internet import reactor env = os.environ.copy() @@ -49,7 +49,7 @@ class ProcessTest: class TestProcessProtocol(ProcessProtocol): def __init__(self) -> None: - self.deferred: Deferred = Deferred() + self.deferred: Deferred[TestProcessProtocol] = Deferred() self.out: bytes = b"" self.err: bytes = b"" self.exitcode: Optional[int] = None From 96d6519b25a3d7b02e8efa1180f6f59e5244f977 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 26 Jun 2024 17:43:59 +0500 Subject: [PATCH 1499/2083] Bump twine in twinecheck. (#6416) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d665fc5a5..c325064d9 100644 --- a/tox.ini +++ b/tox.ini @@ -88,7 +88,7 @@ commands = [testenv:twinecheck] basepython = python3 deps = - twine==4.0.2 + twine==5.0.0 build==1.0.3 commands = python -m build --sdist From 41e15e93e7459673e93ff2591462b47b7ae01566 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 26 Jun 2024 17:44:12 +0500 Subject: [PATCH 1500/2083] Remove an obsolete import. (#6415) --- scrapy/crawler.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 877ea5928..ecb0a8150 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -23,13 +23,6 @@ from twisted.internet.defer import ( inlineCallbacks, maybeDeferred, ) - -try: - # zope >= 5.0 only supports MultipleInvalid - from zope.interface.exceptions import MultipleInvalid -except ImportError: - MultipleInvalid = None - from zope.interface.verify import verifyClass from scrapy import Spider, signals From 558b1d11d2f1e3063aba59d444fdb93d42a9ddb9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 1 Jul 2024 12:30:49 +0500 Subject: [PATCH 1501/2083] Use CallbackT for Request.callback. (#6422) --- scrapy/commands/parse.py | 16 ++++++++-------- scrapy/contracts/__init__.py | 7 ++++--- scrapy/http/request/__init__.py | 20 ++++++++++++++------ scrapy/http/response/__init__.py | 12 +++++++----- scrapy/http/response/text.py | 12 +++++++----- scrapy/spiders/__init__.py | 8 ++------ scrapy/spiders/crawl.py | 15 +++++++++------ scrapy/spiders/sitemap.py | 8 ++++---- 8 files changed, 55 insertions(+), 43 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 1265aa38e..fbd200d88 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -9,7 +9,6 @@ from typing import ( TYPE_CHECKING, Any, AsyncGenerator, - Callable, Coroutine, Dict, Iterable, @@ -38,6 +37,7 @@ from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: from twisted.python.failure import Failure + from scrapy.http.request import CallbackT from scrapy.spiders import Spider @@ -218,8 +218,8 @@ class Command(BaseRunSpiderCommand): opts: argparse.Namespace, depth: int, spider: Spider, - callback: Callable, - ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]: + callback: CallbackT, + ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT]: items, requests = [], [] for x in spider_output: if is_item(x): @@ -231,7 +231,7 @@ class Command(BaseRunSpiderCommand): def run_callback( self, response: Response, - callback: Callable, + callback: CallbackT, cb_kwargs: Optional[Dict[str, Any]] = None, ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} @@ -240,7 +240,7 @@ class Command(BaseRunSpiderCommand): def get_callback_from_rules( self, spider: Spider, response: Response - ) -> Union[Callable, str, None]: + ) -> Union[CallbackT, str, None]: if getattr(spider, "rules", None): for rule in spider.rules: # type: ignore[attr-defined] if rule.link_extractor.matches(response.url): @@ -286,7 +286,7 @@ class Command(BaseRunSpiderCommand): def scraped_data( self, args: Tuple[ - List[Any], List[Request], argparse.Namespace, int, Spider, Callable + List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT ], ) -> List[Any]: items, requests, opts, depth, spider, callback = args @@ -313,8 +313,8 @@ class Command(BaseRunSpiderCommand): spider: Spider, opts: argparse.Namespace, response: Optional[Response] = None, - ) -> Callable: - cb: Union[str, Callable, None] = None + ) -> CallbackT: + cb: Union[str, CallbackT, None] = None if response: cb = response.meta["_callback"] if not cb: diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 440e0dc44..a7e129948 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -16,6 +16,7 @@ from typing import ( Optional, Tuple, Type, + cast, ) from unittest import TestCase, TestResult @@ -62,7 +63,7 @@ class Contract: if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") return list( # pylint: disable=return-in-finally - iterate_spider_output(cb_result) + cast(Iterable[Any], iterate_spider_output(cb_result)) ) request.callback = wrapper @@ -79,7 +80,7 @@ class Contract: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") - output = list(iterate_spider_output(cb_result)) + output = list(cast(Iterable[Any], iterate_spider_output(cb_result))) try: results.startTest(self.testcase_post) self.post_process(output) @@ -195,7 +196,7 @@ class ContractsManager: def cb_wrapper(response: Response, **cb_kwargs: Any) -> None: try: output = cb(response, **cb_kwargs) - output = list(iterate_spider_output(output)) + output = list(cast(Iterable[Any], iterate_spider_output(output))) except Exception: case = _create_testcase(method, "callback") results.addError(case, sys.exc_info()) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 04589dd37..9381a6cb3 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -12,7 +12,6 @@ from typing import ( TYPE_CHECKING, Any, AnyStr, - Callable, Dict, Iterable, List, @@ -37,8 +36,17 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax if TYPE_CHECKING: + from collections.abc import Callable + + from twisted.python.failure import Failure + + # typing.Concatenate requires Python 3.10 # typing.NotRequired and typing.Self require Python 3.11 - from typing_extensions import NotRequired, Self + from typing_extensions import Concatenate, NotRequired, Self + + from scrapy.http import Response + + CallbackT = Callable[Concatenate[Response, ...], Any] class VerboseCookie(TypedDict): @@ -110,7 +118,7 @@ class Request(object_ref): def __init__( self, url: str, - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -119,7 +127,7 @@ class Request(object_ref): encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, flags: Optional[List[str]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, ) -> None: @@ -137,8 +145,8 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") - self.callback: Optional[Callable] = callback - self.errback: Optional[Callable] = errback + self.callback: Optional[CallbackT] = callback + self.errback: Optional[Callable[[Failure], Any]] = errback self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index ff3581abb..92e4852b6 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -27,7 +27,7 @@ from urllib.parse import urljoin from scrapy.exceptions import NotSupported from scrapy.http.headers import Headers -from scrapy.http.request import CookiesT, Request +from scrapy.http.request import Request from scrapy.link import Link from scrapy.utils.trackref import object_ref @@ -35,10 +35,12 @@ if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address from twisted.internet.ssl import Certificate + from twisted.python.failure import Failure # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.http.request import CallbackT, CookiesT from scrapy.selector import SelectorList @@ -196,7 +198,7 @@ class Response(object_ref): def follow( self, url: Union[str, Link], - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -205,7 +207,7 @@ class Response(object_ref): encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, ) -> Request: @@ -249,7 +251,7 @@ class Response(object_ref): def follow_all( self, urls: Iterable[Union[str, Link]], - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -258,7 +260,7 @@ class Response(object_ref): encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, ) -> Iterable[Request]: diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 0635f744f..588695002 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -41,7 +41,9 @@ from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: - from scrapy.http.request import CookiesT, Request + from twisted.python.failure import Failure + + from scrapy.http.request import CallbackT, CookiesT, Request from scrapy.selector import Selector, SelectorList @@ -179,7 +181,7 @@ class TextResponse(Response): def follow( self, url: Union[str, Link, parsel.Selector], - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -188,7 +190,7 @@ class TextResponse(Response): encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, ) -> Request: @@ -232,7 +234,7 @@ class TextResponse(Response): def follow_all( self, urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None, - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -241,7 +243,7 @@ class TextResponse(Response): encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, css: Optional[str] = None, diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index f0b0c0988..d977acd26 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -15,20 +15,16 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import Callable - from twisted.internet.defer import Deferred - # typing.Concatenate requires Python 3.10 # typing.Self requires Python 3.11 - from typing_extensions import Concatenate, Self + from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT from scrapy.settings import BaseSettings, _SettingsKeyT from scrapy.utils.log import SpiderLoggerAdapter - CallbackT = Callable[Concatenate[Response, ...], Any] - class Spider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 48c830d2a..2639f14b2 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -39,6 +39,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT _T = TypeVar("_T") @@ -73,7 +74,7 @@ class Rule: def __init__( self, link_extractor: Optional[LinkExtractor] = None, - callback: Union[Callable, str, None] = None, + callback: Union[CallbackT, str, None] = None, cb_kwargs: Optional[Dict[str, Any]] = None, follow: Optional[bool] = None, process_links: Union[ProcessLinksT, str, None] = None, @@ -81,7 +82,7 @@ class Rule: errback: Union[Callable[[Failure], Any], str, None] = None, ): self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor - self.callback: Union[Callable, str, None] = callback + self.callback: Union[CallbackT, str, None] = callback self.errback: Union[Callable[[Failure], Any], str, None] = errback self.cb_kwargs: Dict[str, Any] = cb_kwargs or {} self.process_links: Union[ProcessLinksT, str] = process_links or _identity @@ -92,7 +93,7 @@ class Rule: def _compile(self, spider: Spider) -> None: # this replaces method names with methods and we can't express this in type hints - self.callback = _get_method(self.callback, spider) + self.callback = cast("CallbackT", _get_method(self.callback, spider)) self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider)) self.process_links = cast( ProcessLinksT, _get_method(self.process_links, spider) @@ -122,7 +123,9 @@ class CrawlSpider(Spider): def parse_start_url(self, response: Response, **kwargs: Any) -> Any: return [] - def process_results(self, response: Response, results: Any) -> Any: + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: return results def _build_request(self, rule_index: int, link: Link) -> Request: @@ -152,7 +155,7 @@ class CrawlSpider(Spider): rule = self._rules[cast(int, response.meta["rule"])] return self._parse_response( response, - cast(Callable, rule.callback), + cast("CallbackT", rule.callback), {**rule.cb_kwargs, **cb_kwargs}, rule.follow, ) @@ -166,7 +169,7 @@ class CrawlSpider(Spider): async def _parse_response( self, response: Response, - callback: Optional[Callable], + callback: Optional[CallbackT], cb_kwargs: Dict[str, Any], follow: bool = True, ) -> AsyncIterable[Any]: diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index d082fbfdb..1542ef79c 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -5,7 +5,6 @@ import re from typing import ( TYPE_CHECKING, Any, - Callable, Dict, Iterable, List, @@ -27,6 +26,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT logger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): sitemap_urls: Sequence[str] = () sitemap_rules: Sequence[ - Tuple[Union[re.Pattern[str], str], Union[str, Callable]] + Tuple[Union[re.Pattern[str], str], Union[str, CallbackT]] ] = [("", "parse")] sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""] sitemap_alternate_links: bool = False @@ -54,10 +54,10 @@ class SitemapSpider(Spider): def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self._cbs: List[Tuple[re.Pattern[str], Callable]] = [] + self._cbs: List[Tuple[re.Pattern[str], CallbackT]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): - c = cast(Callable, getattr(self, c)) + c = cast("CallbackT", getattr(self, c)) self._cbs.append((regex(r), c)) self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] From d8ecd28c5557e27f42e00bd1223b457468ea2ea7 Mon Sep 17 00:00:00 2001 From: mlmsmith <mlmsmith@hotmail.co.uk> Date: Thu, 4 Jul 2024 18:16:26 +0800 Subject: [PATCH 1502/2083] Documentation improvements (#6429) --- docs/intro/install.rst | 13 ++++--------- docs/intro/overview.rst | 12 ++++++------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index c90c1d2bf..e6c9a683b 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -37,7 +37,7 @@ Note that sometimes this may require solving compilation issues for some Scrapy dependencies depending on your operating system, so be sure to check the :ref:`intro-install-platform-notes`. -For more detailed and platform specifics instructions, as well as +For more detailed and platform-specific instructions, as well as troubleshooting information, read on. @@ -101,7 +101,7 @@ Windows ------- Though it's possible to install Scrapy on Windows using pip, we recommend you -to install `Anaconda`_ or `Miniconda`_ and use the package from the +install `Anaconda`_ or `Miniconda`_ and use the package from the `conda-forge`_ channel, which will avoid most installation issues. Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: @@ -141,7 +141,7 @@ But it should support older versions of Ubuntu too, like Ubuntu 14.04, albeit with potential issues with TLS connections. **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are -typically too old and slow to catch up with latest Scrapy. +typically too old and slow to catch up with the latest Scrapy release. To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install @@ -170,7 +170,7 @@ macOS Building Scrapy's dependencies requires the presence of a C compiler and development headers. On macOS this is typically provided by Apple’s Xcode -development tools. To install the Xcode command line tools open a terminal +development tools. To install the Xcode command-line tools, open a terminal window and run:: xcode-select --install @@ -200,11 +200,6 @@ solutions: brew install python - * Latest versions of python have ``pip`` bundled with them so you won't need - to install it separately. If this is not the case, upgrade python:: - - brew update; brew upgrade python - * *(Optional)* :ref:`Install Scrapy inside a Python virtual environment <intro-using-virtualenv>`. diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index ef1294470..cd17b1968 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -65,7 +65,7 @@ When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked fo Spider definition inside it and ran it through its crawler engine. The crawl started by making requests to the URLs defined in the ``start_urls`` -attribute (in this case, only the URL for quotes in *humor* category) +attribute (in this case, only the URL for quotes in the *humor* category) and called the default callback method ``parse``, passing the response object as an argument. In the ``parse`` callback, we loop through the quote elements using a CSS Selector, yield a Python dict with the extracted quote text and author, @@ -83,9 +83,9 @@ While this enables you to do very fast crawls (sending multiple concurrent requests at the same time, in a fault-tolerant way) Scrapy also gives you control over the politeness of the crawl through :ref:`a few settings <topics-settings-ref>`. You can do things like setting a download delay between -each request, limiting amount of concurrent requests per domain or per IP, and +each request, limiting the amount of concurrent requests per domain or per IP, and even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries -to figure out these automatically. +to figure these settings out automatically. .. note:: @@ -106,10 +106,10 @@ scraping easy and efficient, such as: * Built-in support for :ref:`selecting and extracting <topics-selectors>` data from HTML/XML sources using extended CSS selectors and XPath expressions, - with helper methods to extract using regular expressions. + with helper methods for extraction using regular expressions. * An :ref:`interactive shell console <topics-shell>` (IPython aware) for trying - out the CSS and XPath expressions to scrape data, very useful when writing or + out the CSS and XPath expressions to scrape data, which is very useful when writing or debugging your spiders. * Built-in support for :ref:`generating feed exports <topics-feed-exports>` in @@ -124,7 +124,7 @@ scraping easy and efficient, such as: well-defined API (middlewares, :ref:`extensions <topics-extensions>`, and :ref:`pipelines <topics-item-pipeline>`). -* Wide range of built-in extensions and middlewares for handling: +* A wide range of built-in extensions and middlewares for handling: - cookies and session handling - HTTP features like compression, authentication, caching From ceedb026f8c8ccb049187baa14202f98b2a3a60c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 9 Jul 2024 11:34:58 +0500 Subject: [PATCH 1503/2083] Remove top-level imports that install the reactor from scrapy.extensions.telnet. (#6432) --- scrapy/extensions/telnet.py | 22 +++++++--------------- tests/test_crawler.py | 4 ---- tests/test_utils_log.py | 4 ---- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index c4e01b3d9..c64a0b417 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -10,21 +10,11 @@ import binascii import logging import os import pprint -import traceback from typing import TYPE_CHECKING, Any, Dict, List from twisted.internet import protocol from twisted.internet.tcp import Port -try: - from twisted.conch import manhole, telnet - from twisted.conch.insults import insults - - TWISTED_CONCH_AVAILABLE = True -except (ImportError, SyntaxError): - _TWISTED_CONCH_TRACEBACK = traceback.format_exc() - TWISTED_CONCH_AVAILABLE = False - from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import defers @@ -33,6 +23,8 @@ from scrapy.utils.reactor import listen_tcp from scrapy.utils.trackref import print_live_refs if TYPE_CHECKING: + from twisted.conch import telnet + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -50,11 +42,7 @@ class TelnetConsole(protocol.ServerFactory): def __init__(self, crawler: Crawler): if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"): raise NotConfigured - if not TWISTED_CONCH_AVAILABLE: - raise NotConfigured( - "TELNETCONSOLE_ENABLED setting is True but required twisted " - "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK - ) + self.crawler: Crawler = crawler self.noisy: bool = False self.portrange: List[int] = [ @@ -88,6 +76,10 @@ class TelnetConsole(protocol.ServerFactory): self.port.stopListening() def protocol(self) -> telnet.TelnetTransport: # type: ignore[override] + # these import twisted.internet.reactor + from twisted.conch import manhole, telnet + from twisted.conch.insults import insults + class Portal: """An implementation of IPortal""" diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 791ea1faa..c87e65758 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -21,7 +21,6 @@ import scrapy from scrapy import Spider from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.extensions import telnet from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader @@ -482,7 +481,6 @@ class CrawlerLoggingTestCase(unittest.TestCase): "LOG_FILE": str(log_file), # settings to avoid extra warnings "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } configure_logging() @@ -516,8 +514,6 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { "LOG_FILE": str(log_file), "LOG_FILE_APPEND": False, - # disable telnet if not available to avoid an extra warning - "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } configure_logging() diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index a8d080822..0f75bdb5c 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -11,7 +11,6 @@ import pytest from testfixtures import LogCapture from twisted.python.failure import Failure -from scrapy.extensions import telnet from scrapy.utils.log import ( LogCounterHandler, SpiderLoggerAdapter, @@ -70,9 +69,6 @@ class TopLevelFormatterTest(unittest.TestCase): class LogCounterHandlerTest(unittest.TestCase): def setUp(self): settings = {"LOG_LEVEL": "WARNING"} - if not telnet.TWISTED_CONCH_AVAILABLE: - # disable it to avoid the extra warning - settings["TELNETCONSOLE_ENABLED"] = False self.logger = logging.getLogger("test") self.logger.setLevel(logging.NOTSET) self.logger.propagate = False From 0b8604bb5d8bffbbd5c78783022965fa2606c131 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Tue, 9 Jul 2024 15:52:49 -0300 Subject: [PATCH 1504/2083] add CLOSESPIDER_PAGECOUNT_NO_ITEM to CloseSpider extension --- scrapy/extensions/closespider.py | 25 +++++++++++++++++++++++++ tests/keys/mitmproxy-dhparam.pem | 14 ++++++++++++++ tests/test_closespider.py | 13 +++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 tests/keys/mitmproxy-dhparam.pem diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 4627e7f98..6ebf98e65 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, DefaultDict, Dict from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.signalmanager import dispatcher if TYPE_CHECKING: from twisted.python.failure import Failure @@ -36,6 +37,9 @@ class CloseSpider: "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"), "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"), + "pagecount_no_item": crawler.settings.getint( + "CLOSESPIDER_PAGECOUNT_NO_ITEM" + ), } if not any(self.close_on.values()): @@ -60,6 +64,19 @@ class CloseSpider: crawler.signals.connect( self.item_scraped_no_item, signal=signals.item_scraped ) + if self.close_on.get("pagecount_no_item"): + if self.page_count not in dispatcher.getReceivers( + signal=signals.response_received + ): + crawler.signals.connect( + self.page_count, signal=signals.response_received + ) + + if self.item_scraped not in dispatcher.getReceivers( + signal=signals.item_scraped + ): + crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) + crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod @@ -74,9 +91,16 @@ class CloseSpider: def page_count(self, response: Response, request: Request, spider: Spider) -> None: self.counter["pagecount"] += 1 + self.counter["pagecount_since_last_item"] += 1 if self.counter["pagecount"] == self.close_on["pagecount"]: assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_pagecount") + if self.close_on["pagecount_no_item"] and ( + self.counter["pagecount_since_last_item"] + >= self.close_on["pagecount_no_item"] + ): + assert self.crawler.engine + self.crawler.engine.close_spider(spider, "closespider_pagecount_no_item") def spider_opened(self, spider: Spider) -> None: from twisted.internet import reactor @@ -91,6 +115,7 @@ class CloseSpider: def item_scraped(self, item: Any, spider: Spider) -> None: self.counter["itemcount"] += 1 + self.counter["pagecount_since_last_item"] = 0 if self.counter["itemcount"] == self.close_on["itemcount"]: assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_itemcount") diff --git a/tests/keys/mitmproxy-dhparam.pem b/tests/keys/mitmproxy-dhparam.pem new file mode 100644 index 000000000..c10121fbf --- /dev/null +++ b/tests/keys/mitmproxy-dhparam.pem @@ -0,0 +1,14 @@ + +-----BEGIN DH PARAMETERS----- +MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3 +O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv +j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ +Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB +chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC +ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq +o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX +IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv +A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8 +6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I +rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI= +-----END DH PARAMETERS----- diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 38ede70e4..caaa9f183 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -34,6 +34,19 @@ class TestCloseSpider(TestCase): pagecount = crawler.stats.get_value("response_received_count") self.assertTrue(pagecount >= close_on) + @defer.inlineCallbacks + def test_closespider_pagecount_no_item(self): + close_on = 5 + crawler = get_crawler( + FollowAllSpider, + {"CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on}, + ) + yield crawler.crawl(mockserver=self.mockserver) + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_pagecount_no_item") + pagecount = crawler.stats.get_value("response_received_count") + self.assertTrue(pagecount >= close_on) + @defer.inlineCallbacks def test_closespider_errorcount(self): close_on = 5 From a44818afeacc25cc5e05705bf8ae5804e0545c89 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Tue, 9 Jul 2024 16:07:55 -0300 Subject: [PATCH 1505/2083] restore mitmproxy-dhparam --- tests/keys/mitmproxy-dhparam.pem | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 tests/keys/mitmproxy-dhparam.pem diff --git a/tests/keys/mitmproxy-dhparam.pem b/tests/keys/mitmproxy-dhparam.pem deleted file mode 100644 index c10121fbf..000000000 --- a/tests/keys/mitmproxy-dhparam.pem +++ /dev/null @@ -1,14 +0,0 @@ - ------BEGIN DH PARAMETERS----- -MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3 -O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv -j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ -Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB -chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC -ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq -o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX -IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv -A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8 -6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I -rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI= ------END DH PARAMETERS----- From d6352f9f66f655f11332fe6c52ed71ebb2e55bf4 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Wed, 10 Jul 2024 11:03:01 -0300 Subject: [PATCH 1506/2083] refactor changes on closespider.py and improve test --- scrapy/extensions/closespider.py | 18 +++--------------- tests/spiders.py | 17 +++++++++++++++++ tests/test_closespider.py | 24 +++++++++++++++++++----- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 6ebf98e65..cef5527b7 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, DefaultDict, Dict from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured -from scrapy.signalmanager import dispatcher if TYPE_CHECKING: from twisted.python.failure import Failure @@ -49,11 +48,11 @@ class CloseSpider: if self.close_on.get("errorcount"): crawler.signals.connect(self.error_count, signal=signals.spider_error) - if self.close_on.get("pagecount"): + if self.close_on.get("pagecount") or self.close_on.get("pagecount_no_item"): crawler.signals.connect(self.page_count, signal=signals.response_received) if self.close_on.get("timeout"): crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) - if self.close_on.get("itemcount"): + if self.close_on.get("itemcount") or self.close_on.get("pagecount_no_item"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) if self.close_on.get("timeout_no_item"): self.timeout_no_item: int = self.close_on["timeout_no_item"] @@ -64,18 +63,6 @@ class CloseSpider: crawler.signals.connect( self.item_scraped_no_item, signal=signals.item_scraped ) - if self.close_on.get("pagecount_no_item"): - if self.page_count not in dispatcher.getReceivers( - signal=signals.response_received - ): - crawler.signals.connect( - self.page_count, signal=signals.response_received - ) - - if self.item_scraped not in dispatcher.getReceivers( - signal=signals.item_scraped - ): - crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @@ -95,6 +82,7 @@ class CloseSpider: if self.counter["pagecount"] == self.close_on["pagecount"]: assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_pagecount") + return if self.close_on["pagecount_no_item"] and ( self.counter["pagecount_since_last_item"] >= self.close_on["pagecount_no_item"] diff --git a/tests/spiders.py b/tests/spiders.py index ea419afbd..2bcec5624 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -283,6 +283,23 @@ class ItemSpider(FollowAllSpider): yield {} +class MaxItemsSpider(ItemSpider): + def __init__(self, max_items=10, *args, **kwargs): + super().__init__(*args, **kwargs) + self.max_items = max_items + self.items_scraped = 0 + + def parse(self, response): + for item_or_req in super().parse(response): + if isinstance(item_or_req, Request): + yield item_or_req + else: + if self.items_scraped >= self.max_items: + continue + self.items_scraped += 1 + yield item_or_req + + class DefaultError(Exception): pass diff --git a/tests/test_closespider.py b/tests/test_closespider.py index caaa9f183..0046b4e29 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -3,7 +3,13 @@ from twisted.trial.unittest import TestCase from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider, SlowSpider +from tests.spiders import ( + ErrorSpider, + FollowAllSpider, + ItemSpider, + MaxItemsSpider, + SlowSpider, +) class TestCloseSpider(TestCase): @@ -37,15 +43,23 @@ class TestCloseSpider(TestCase): @defer.inlineCallbacks def test_closespider_pagecount_no_item(self): close_on = 5 + close_on_pagecount = 20 + max_items = 5 crawler = get_crawler( - FollowAllSpider, - {"CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on}, + MaxItemsSpider, + { + "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on, + "CLOSESPIDER_PAGECOUNT": close_on_pagecount, + }, ) - yield crawler.crawl(mockserver=self.mockserver) + yield crawler.crawl(max_items=max_items, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] self.assertEqual(reason, "closespider_pagecount_no_item") pagecount = crawler.stats.get_value("response_received_count") - self.assertTrue(pagecount >= close_on) + itemcount = crawler.stats.get_value("item_scraped_count") + self.assertEqual(itemcount, max_items) + self.assertLess(pagecount, close_on_pagecount) + self.assertTrue((pagecount - itemcount) >= close_on) @defer.inlineCallbacks def test_closespider_errorcount(self): From 59782d73088e46618d1c042e74ce5197e880536a Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Wed, 10 Jul 2024 11:08:22 -0300 Subject: [PATCH 1507/2083] update docs --- docs/topics/extensions.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index f7b2f3799..a503fd746 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -317,6 +317,18 @@ crawls more than that, the spider will be closed with the reason ``closespider_pagecount``. If zero (or non set), spiders won't be closed by number of crawled responses. +.. setting:: CLOSESPIDER_PAGECOUNT_NO_ITEM + +CLOSESPIDER_PAGECOUNT_NO_ITEM +""""""""""""""""""""" + +Default: ``0`` + +An integer which specifies the maximum number of consecutive responses to crawl without items scraped. If the spider +crawls more consecutive responses than that and no items are scraped in the meantime, the spider will be closed with the reason +``closespider_pagecount_no_item``. If zero (or non set), spiders won't be closed by +number of crawled responses with no items. + .. setting:: CLOSESPIDER_ERRORCOUNT CLOSESPIDER_ERRORCOUNT From 8646d2ec7bc44ef96f5df015e03ff37ceb5554c0 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Wed, 10 Jul 2024 11:44:44 -0300 Subject: [PATCH 1508/2083] fix docs detail --- docs/topics/extensions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index a503fd746..29bcaa0f2 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -320,7 +320,7 @@ number of crawled responses. .. setting:: CLOSESPIDER_PAGECOUNT_NO_ITEM CLOSESPIDER_PAGECOUNT_NO_ITEM -""""""""""""""""""""" +""""""""""""""""""""""""""""" Default: ``0`` From 129dbfa0bf1ad464ab6b50f3dee0da39853de6a1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 11 Jul 2024 12:20:36 +0500 Subject: [PATCH 1509/2083] Bump tool versions. --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 4 ++-- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- .github/workflows/tests-windows.yml | 2 +- .pre-commit-config.yaml | 12 ++++++------ tox.ini | 14 +++++++------- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ed1629b67..1841bda1c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -46,4 +46,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index affaa32a5..03e94f761 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,13 +13,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.12 - run: | pip install --upgrade build twine python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.6.4 + uses: pypa/gh-action-pypi-publish@v1.9.0 with: password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index a297f494c..8ebe7f1db 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f50a4d104..763de9eff 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -62,7 +62,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 757d62285..80d09e7a0 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 38526d720..addad838f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,11 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.7.7 + rev: 1.7.9 hooks: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 7.0.0 + rev: 7.1.0 hooks: - id: flake8 additional_dependencies: @@ -16,7 +16,7 @@ repos: - flake8-string-format - flake8-type-checking - repo: https://github.com/psf/black.git - rev: 24.2.0 + rev: 24.4.2 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -24,13 +24,13 @@ repos: hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs - rev: 1.16.0 + rev: 1.18.0 hooks: - id: blacken-docs additional_dependencies: - - black==24.2.0 + - black==24.4.2 - repo: https://github.com/asottile/pyupgrade - rev: v3.15.2 + rev: v3.16.0 hooks: - id: pyupgrade args: [--py38-plus, --keep-runtime-typing] diff --git a/tox.ini b/tox.ini index c325064d9..29d240031 100644 --- a/tox.ini +++ b/tox.ini @@ -46,14 +46,14 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.10.0 - typing-extensions==4.12.1 + mypy==1.10.1 + typing-extensions==4.12.2 types-lxml==2024.4.14 types-Pygments==2.18.0.20240506 types-pyOpenSSL==24.1.0.20240425 - types-setuptools==70.0.0.20240524 - botocore-stubs==1.34.94 - boto3-stubs[s3]==1.34.119 + types-setuptools==70.3.0.20240710 + botocore-stubs==1.34.143 + boto3-stubs[s3]==1.34.143 attrs >= 18.2.0 Pillow >= 10.3.0 pytest >= 8.2.0 @@ -88,8 +88,8 @@ commands = [testenv:twinecheck] basepython = python3 deps = - twine==5.0.0 - build==1.0.3 + twine==5.1.1 + build==1.2.1 commands = python -m build --sdist twine check dist/* From 435686830cbe86d14aa09c9259157695596be07f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 11 Jul 2024 12:25:13 +0500 Subject: [PATCH 1510/2083] Bump the Python version for RTD. --- .github/workflows/checks.yml | 2 +- .readthedocs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 1841bda1c..2be6a9502 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -21,7 +21,7 @@ jobs: - python-version: 3.8 env: TOXENV: typing-tests - - python-version: "3.11" # Keep in sync with .readthedocs.yml + - python-version: "3.12" # Keep in sync with .readthedocs.yml env: TOXENV: docs - python-version: "3.12" diff --git a/.readthedocs.yml b/.readthedocs.yml index e71d34f3a..0c544df7e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.11" # Keep in sync with .github/workflows/checks.yml + python: "3.12" # Keep in sync with .github/workflows/checks.yml python: install: From 3c9c1a31bcdcced96e87e299689aaa7be8f5bdee Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 11 Jul 2024 12:30:12 +0500 Subject: [PATCH 1511/2083] Bump pylint. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 29d240031..e3dd96425 100644 --- a/tox.ini +++ b/tox.ini @@ -81,7 +81,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==3.1.0 + pylint==3.2.5 commands = pylint conftest.py docs extras scrapy setup.py tests From a40d5281cfb8fdaf7d7edce80d3addbddef897a6 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Thu, 11 Jul 2024 11:14:30 -0300 Subject: [PATCH 1512/2083] improve test_closespider_pagecount_no_item and MaxItemsSpider --- tests/spiders.py | 13 +++++-------- tests/test_closespider.py | 10 ++++------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index 2bcec5624..d1998ca69 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -283,21 +283,18 @@ class ItemSpider(FollowAllSpider): yield {} -class MaxItemsSpider(ItemSpider): +class MaxItemsKeepCrawlingSpider(FollowAllSpider): def __init__(self, max_items=10, *args, **kwargs): super().__init__(*args, **kwargs) self.max_items = max_items self.items_scraped = 0 def parse(self, response): - for item_or_req in super().parse(response): - if isinstance(item_or_req, Request): - yield item_or_req - else: - if self.items_scraped >= self.max_items: - continue + for request in super().parse(response): + yield request + if self.items_scraped < self.max_items: + yield Item() self.items_scraped += 1 - yield item_or_req class DefaultError(Exception): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 0046b4e29..50b483a74 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -7,7 +7,7 @@ from tests.spiders import ( ErrorSpider, FollowAllSpider, ItemSpider, - MaxItemsSpider, + MaxItemsKeepCrawlingSpider, SlowSpider, ) @@ -43,13 +43,11 @@ class TestCloseSpider(TestCase): @defer.inlineCallbacks def test_closespider_pagecount_no_item(self): close_on = 5 - close_on_pagecount = 20 max_items = 5 crawler = get_crawler( - MaxItemsSpider, + MaxItemsKeepCrawlingSpider, { "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on, - "CLOSESPIDER_PAGECOUNT": close_on_pagecount, }, ) yield crawler.crawl(max_items=max_items, mockserver=self.mockserver) @@ -58,8 +56,8 @@ class TestCloseSpider(TestCase): pagecount = crawler.stats.get_value("response_received_count") itemcount = crawler.stats.get_value("item_scraped_count") self.assertEqual(itemcount, max_items) - self.assertLess(pagecount, close_on_pagecount) - self.assertTrue((pagecount - itemcount) >= close_on) + self.assertLessEqual(pagecount, close_on + itemcount) + self.assertGreater(pagecount, itemcount) @defer.inlineCallbacks def test_closespider_errorcount(self): From 5f0fad16f5d86134bcf72964f6e453541031eb06 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Thu, 11 Jul 2024 13:26:22 -0300 Subject: [PATCH 1513/2083] improve test_closespider_pagecount_no_item and corresponding test spider --- tests/spiders.py | 12 ++++++++---- tests/test_closespider.py | 11 ++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index d1998ca69..743811893 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -283,15 +283,19 @@ class ItemSpider(FollowAllSpider): yield {} -class MaxItemsKeepCrawlingSpider(FollowAllSpider): - def __init__(self, max_items=10, *args, **kwargs): +class MaxItemsAndRequestsSpider(FollowAllSpider): + def __init__(self, max_items=10, max_requests=10, *args, **kwargs): super().__init__(*args, **kwargs) self.max_items = max_items - self.items_scraped = 0 + self.max_requests = max_requests def parse(self, response): + self.items_scraped = 0 + self.pages_crawled = 1 # account for the start url for request in super().parse(response): - yield request + if self.pages_crawled < self.max_requests: + yield request + self.pages_crawled += 1 if self.items_scraped < self.max_items: yield Item() self.items_scraped += 1 diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 50b483a74..9810d10fb 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -7,7 +7,7 @@ from tests.spiders import ( ErrorSpider, FollowAllSpider, ItemSpider, - MaxItemsKeepCrawlingSpider, + MaxItemsAndRequestsSpider, SlowSpider, ) @@ -44,20 +44,21 @@ class TestCloseSpider(TestCase): def test_closespider_pagecount_no_item(self): close_on = 5 max_items = 5 + max_requests = close_on + max_items crawler = get_crawler( - MaxItemsKeepCrawlingSpider, + MaxItemsAndRequestsSpider, { "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on, }, ) - yield crawler.crawl(max_items=max_items, mockserver=self.mockserver) + yield crawler.crawl( + max_items=max_items, max_requests=max_requests, mockserver=self.mockserver + ) reason = crawler.spider.meta["close_reason"] self.assertEqual(reason, "closespider_pagecount_no_item") pagecount = crawler.stats.get_value("response_received_count") itemcount = crawler.stats.get_value("item_scraped_count") - self.assertEqual(itemcount, max_items) self.assertLessEqual(pagecount, close_on + itemcount) - self.assertGreater(pagecount, itemcount) @defer.inlineCallbacks def test_closespider_errorcount(self): From 9cdbcb4f63922f09194fab7d211ba297319b5135 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <guillermo.bondonno@zyte.com> Date: Thu, 11 Jul 2024 14:02:24 -0300 Subject: [PATCH 1514/2083] add test_closespider_pagecount_no_item_with_pagecount --- tests/test_closespider.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 9810d10fb..9a837350f 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -60,6 +60,23 @@ class TestCloseSpider(TestCase): itemcount = crawler.stats.get_value("item_scraped_count") self.assertLessEqual(pagecount, close_on + itemcount) + @defer.inlineCallbacks + def test_closespider_pagecount_no_item_with_pagecount(self): + close_on_pagecount_no_item = 5 + close_on_pagecount = 20 + crawler = get_crawler( + FollowAllSpider, + { + "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on_pagecount_no_item, + "CLOSESPIDER_PAGECOUNT": close_on_pagecount, + }, + ) + yield crawler.crawl(mockserver=self.mockserver) + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_pagecount_no_item") + pagecount = crawler.stats.get_value("response_received_count") + self.assertLess(pagecount, close_on_pagecount) + @defer.inlineCallbacks def test_closespider_errorcount(self): close_on = 5 From 026d6065287e882c244d9b90e0c4fa5e873e29fe Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <95530227+guillermo-bondonno@users.noreply.github.com> Date: Fri, 12 Jul 2024 08:09:03 -0300 Subject: [PATCH 1515/2083] clean closespider_pagecount_no_item docs section Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name> --- docs/topics/extensions.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 29bcaa0f2..7b34a19d5 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -324,10 +324,11 @@ CLOSESPIDER_PAGECOUNT_NO_ITEM Default: ``0`` -An integer which specifies the maximum number of consecutive responses to crawl without items scraped. If the spider -crawls more consecutive responses than that and no items are scraped in the meantime, the spider will be closed with the reason -``closespider_pagecount_no_item``. If zero (or non set), spiders won't be closed by -number of crawled responses with no items. +An integer which specifies the maximum number of consecutive responses to crawl +without items scraped. If the spider crawls more consecutive responses than that +and no items are scraped in the meantime, the spider will be closed with the +reason ``closespider_pagecount_no_item``. If zero (or not set), spiders won't be +closed by number of crawled responses with no items. .. setting:: CLOSESPIDER_ERRORCOUNT From e376c0b31a01cedd8a8c5c1ccd423d72ae1fb169 Mon Sep 17 00:00:00 2001 From: mlmsmith <mlmsmith@hotmail.co.uk> Date: Wed, 24 Jul 2024 12:40:01 +0800 Subject: [PATCH 1516/2083] Tutorial edits (#6440) --- docs/intro/tutorial.rst | 53 ++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8ea98f29b..ee6a1184c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -18,11 +18,11 @@ This tutorial will walk you through these tasks: 4. Changing spider to recursively follow links 5. Using spider arguments -Scrapy is written in Python_. If you're new to the language you might want to -start by getting an idea of what the language is like, to get the most out of -Scrapy. +Scrapy is written in Python_. The more you learn about Python, the more you +can get out of Scrapy. -If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource. +If you're already familiar with other languages and want to learn Python quickly, the +`Python Tutorial`_ is a good resource. If you're new to programming and want to start with Python, the following books may be useful to you: @@ -76,10 +76,9 @@ This will create a ``tutorial`` directory with the following contents:: Our first Spider ================ -Spiders are classes that you define and that Scrapy uses to scrape information -from a website (or a group of websites). They must subclass -:class:`~scrapy.Spider` and define the initial requests to make, -optionally how to follow links in the pages, and how to parse the downloaded +Spiders are classes that you define and that Scrapy uses to scrape information from a website +(or a group of websites). They must subclass :class:`~scrapy.Spider` and define the initial +requests to be made, and optionally, how to follow links in pages and parse the downloaded page content to extract data. This is the code for our first Spider. Save it in a file named @@ -138,7 +137,7 @@ To put our spider to work, go to the project's top level directory and run:: scrapy crawl quotes -This command runs the spider with name ``quotes`` that we've just added, that +This command runs the spider named ``quotes`` that we've just added, that will send some requests for the ``quotes.toscrape.com`` domain. You will get an output similar to this:: @@ -169,7 +168,7 @@ Scrapy schedules the :class:`scrapy.Request <scrapy.Request>` objects returned by the ``start_requests`` method of the Spider. Upon receiving a response for each one, it instantiates :class:`~scrapy.http.Response` objects and calls the callback method associated with the request (in this case, the -``parse`` method) passing the response as argument. +``parse`` method) passing the response as an argument. A shortcut to the start_requests method @@ -217,7 +216,7 @@ using the :ref:`Scrapy shell <topics-shell>`. Run:: .. note:: - Remember to always enclose urls in quotes when running Scrapy shell from + Remember to always enclose urls in quotes when running Scrapy shell from the command-line, otherwise urls containing arguments (i.e. ``&`` character) will not work. @@ -257,7 +256,7 @@ object: The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of :class:`~scrapy.Selector` objects that wrap around XML/HTML elements -and allow you to run further queries to fine-grain the selection or extract the +and allow you to run further queries to refine the selection or extract the data. To extract the text from the title above, you can do: @@ -354,12 +353,12 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions: XPath expressions are very powerful, and are the foundation of Scrapy Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You -can see that if you read closely the text representation of the selector -objects in the shell. +can see that if you read the text representation of the selector +objects in the shell closely. While perhaps not as popular as CSS selectors, XPath expressions offer more power because besides navigating the structure, it can also look at the -content. Using XPath, you're able to select things like: *select the link +content. Using XPath, you're able to select things like: *the link that contains the text "Next Page"*. This makes XPath very fitting to the task of scraping, and we encourage you to learn XPath even if you already know how to construct CSS selectors, it will make scraping much easier. @@ -422,7 +421,7 @@ variable, so that we can run our CSS selectors directly on a particular quote: >>> quote = response.css("div.quote")[0] -Now, let's extract ``text``, ``author`` and the ``tags`` from that quote +Now, let's extract the ``text``, ``author`` and ``tags`` from that quote using the ``quote`` object we just created: .. code-block:: pycon @@ -448,7 +447,7 @@ to get all of them: from sys import version_info Having figured out how to extract each bit, we can now iterate over all the -quotes elements and put them together into a Python dictionary: +quote elements and put them together into a Python dictionary: .. code-block:: pycon @@ -465,8 +464,8 @@ quotes elements and put them together into a Python dictionary: Extracting data in our spider ----------------------------- -Let's get back to our spider. Until now, it doesn't extract any data in -particular, just saves the whole HTML page to a local file. Let's integrate the +Let's get back to our spider. Until now, it hasn't extracted any data in +particular, just saving the whole HTML page to a local file. Let's integrate the extraction logic above into our spider. A Scrapy spider typically generates many dictionaries containing the data @@ -529,8 +528,8 @@ using a different serialization format, such as `JSON Lines`_:: scrapy crawl quotes -o quotes.jsonl -The `JSON Lines`_ format is useful because it's stream-like, you can easily -append new records to it. It doesn't have the same problem of JSON when you run +The `JSON Lines`_ format is useful because it's stream-like, so you can easily +append new records to it. It doesn't have the same problem as JSON when you run twice. Also, as each record is a separate line, you can process big files without having to fit everything in memory, there are tools like `JQ`_ to help do that at the command-line. @@ -555,7 +554,7 @@ from https://quotes.toscrape.com, you want quotes from all the pages in the webs Now that you know how to extract data from pages, let's see how to follow links from them. -First thing is to extract the link to the page we want to follow. Examining +The first thing to do is extract the link to the page we want to follow. Examining our page, we can see there is a link to the next page with the following markup: @@ -589,7 +588,7 @@ There is also an ``attrib`` property available >>> response.css("li.next a").attrib["href"] '/page/2/' -Let's see now our spider modified to recursively follow the link to the next +Now let's see our spider, modified to recursively follow the link to the next page, extracting data from it: .. code-block:: python @@ -756,8 +755,8 @@ Another interesting thing this spider demonstrates is that, even if there are many quotes from the same author, we don't need to worry about visiting the same author page multiple times. By default, Scrapy filters out duplicated requests to URLs already visited, avoiding the problem of hitting servers too -much because of a programming mistake. This can be configured by the setting -:setting:`DUPEFILTER_CLASS`. +much because of a programming mistake. This can be configured in the +:setting:`DUPEFILTER_CLASS` setting. Hopefully by now you have a good understanding of how to use the mechanism of following links and callbacks with Scrapy. @@ -824,12 +823,12 @@ Next steps ========== This tutorial covered only the basics of Scrapy, but there's a lot of other -features not mentioned here. Check the :ref:`topics-whatelse` section in +features not mentioned here. Check the :ref:`topics-whatelse` section in the :ref:`intro-overview` chapter for a quick overview of the most important ones. You can continue from the section :ref:`section-basics` to know more about the command-line tool, spiders, selectors and other things the tutorial hasn't covered like -modeling the scraped data. If you prefer to play with an example project, check +modeling the scraped data. If you'd prefer to play with an example project, check the :ref:`intro-examples` section. .. _JSON: https://en.wikipedia.org/wiki/JSON From 03a15ced4f0a4284c75a917fdfb07c44b21f9ff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 31 Jul 2024 11:37:19 +0200 Subject: [PATCH 1517/2083] Do not suggest logging dropped items twice (#6448) Co-authored-by: Kevin Lloyd Bernal <kevinoxy@gmail.com> --- docs/topics/item-pipeline.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index a5f6e07b8..58c922e0d 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -99,7 +99,7 @@ contain a price: adapter["price"] = adapter["price"] * self.vat_factor return item else: - raise DropItem(f"Missing price in {item}") + raise DropItem("Missing price") Write items to a JSON lines file @@ -254,7 +254,7 @@ returns multiples items with the same id: def process_item(self, item, spider): adapter = ItemAdapter(item) if adapter["id"] in self.ids_seen: - raise DropItem(f"Duplicate item found: {item!r}") + raise DropItem(f"Item ID already seen: {adapter['id']}") else: self.ids_seen.add(adapter["id"]) return item From b9ef1326a51140f70325609501265300fdac5e9b Mon Sep 17 00:00:00 2001 From: mlmsmith <mlmsmith@hotmail.co.uk> Date: Thu, 1 Aug 2024 15:29:11 +0800 Subject: [PATCH 1518/2083] Proofread the commands documentation (#6449) --- docs/intro/tutorial.rst | 4 ++-- docs/topics/commands.rst | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index ee6a1184c..dd1efd3b3 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -216,8 +216,8 @@ using the :ref:`Scrapy shell <topics-shell>`. Run:: .. note:: - Remember to always enclose urls in quotes when running Scrapy shell from the - command-line, otherwise urls containing arguments (i.e. ``&`` character) + Remember to always enclose URLs in quotes when running Scrapy shell from the + command line, otherwise URLs containing arguments (i.e. ``&`` character) will not work. On Windows, use double quotes instead:: diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 1d37895c2..6eb4af9bd 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -6,7 +6,7 @@ Command line tool ================= -Scrapy is controlled through the ``scrapy`` command-line tool, to be referred +Scrapy is controlled through the ``scrapy`` command-line tool, to be referred to here as the "Scrapy tool" to differentiate it from the sub-commands, which we just call "commands" or "Scrapy commands". @@ -185,8 +185,8 @@ And you can see all available commands with:: There are two kinds of commands, those that only work from inside a Scrapy project (Project-specific commands) and those that also work without an active -Scrapy project (Global commands), though they may behave slightly different -when running from inside a project (as they would use the project overridden +Scrapy project (Global commands), though they may behave slightly differently +when run from inside a project (as they would use the project overridden settings). Global commands: @@ -236,7 +236,7 @@ genspider .. versionadded:: 2.6.0 The ability to pass a URL instead of a domain. -Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. +Creates a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. Usage example:: @@ -253,7 +253,7 @@ Usage example:: $ scrapy genspider -t crawl scrapyorg scrapy.org Created spider 'scrapyorg' using template 'crawl' -This is just a convenience shortcut command for creating spiders based on +This is just a convenient shortcut command for creating spiders based on pre-defined templates, but certainly not the only way to create spiders. You can just create the spider source code files yourself, instead of using this command. @@ -274,9 +274,9 @@ Supported options: * ``-a NAME=VALUE``: set a spider argument (may be repeated) -* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) +* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout). To define the output format, set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) -* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) +* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) * ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O`` @@ -353,7 +353,7 @@ edit Edit the given spider using the editor defined in the ``EDITOR`` environment variable or (if unset) the :setting:`EDITOR` setting. -This command is provided only as a convenience shortcut for the most common +This command is provided only as a convenient shortcut for the most common case, the developer is of course free to choose any tool or IDE to write and debug spiders. @@ -372,7 +372,7 @@ fetch Downloads the given URL using the Scrapy downloader and writes the contents to standard output. -The interesting thing about this command is that it fetches the page how the +The interesting thing about this command is that it fetches the page the way the spider would download it. For example, if the spider has a ``USER_AGENT`` attribute which overrides the User Agent, it will use that one. From 70756fd57cff61a1806317127f7dfcd0e77bf1f0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 11 Aug 2024 13:57:13 +0500 Subject: [PATCH 1519/2083] Revert cffi and Pillow restrictions. --- tox.ini | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index ee810eae3..4ccaea653 100644 --- a/tox.ini +++ b/tox.ini @@ -19,15 +19,13 @@ deps = sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures pywin32; sys_platform == "win32" - cffi >= 1.17.0rc1; python_version >= '3.13' [testenv] deps = {[test-requirements]deps} # mitmproxy does not support PyPy - # mitmproxy requires zstandard which is not yet available on 3.13 - mitmproxy; implementation_name != 'pypy' and python_version < '3.13' + mitmproxy; implementation_name != 'pypy' # https://github.com/pallets/werkzeug/pull/2768 breaks flask, required by # mitmproxy. werkzeug < 3; python_version < '3.9' and implementation_name != 'pypy' @@ -150,13 +148,13 @@ deps = boto3 google-cloud-storage robotexclusionrulesparser - Pillow; python_version < '3.13' + Pillow Twisted[http2] uvloop; platform_system != "Windows" and python_version < '3.13' bpython; python_version < '3.13' # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests - zstandard; implementation_name != 'pypy' and python_version < '3.13' # optional for HTTP compress downloader middleware tests + zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython [testenv:extra-deps-pinned] From af15bd1dadf74b1314b96b1c3b682b41207a1f52 Mon Sep 17 00:00:00 2001 From: mlmsmith <mlmsmith@hotmail.co.uk> Date: Mon, 19 Aug 2024 19:55:09 +0800 Subject: [PATCH 1520/2083] minor changes to items section of docs (#6462) --- docs/topics/items.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 97ed7a900..f13a7b5b1 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -48,7 +48,7 @@ make it the most feature-complete item type: :class:`Item` objects replicate the standard :class:`dict` API, including its ``__init__`` method. - :class:`Item` allows defining field names, so that: + :class:`Item` allows the defining of field names, so that: - :class:`KeyError` is raised when using undefined field names (i.e. prevents typos going unnoticed) @@ -57,7 +57,7 @@ make it the most feature-complete item type: default even if the first scraped object does not have values for all of them - :class:`Item` also allows defining field metadata, which can be used to + :class:`Item` also allows the defining of field metadata, which can be used to :ref:`customize serialization <topics-exporters-field-serialization>`. :mod:`trackref` tracks :class:`Item` objects to help find memory leaks @@ -94,11 +94,11 @@ Dataclass objects .. versionadded:: 2.2 -:func:`~dataclasses.dataclass` allows defining item classes with field names, +:func:`~dataclasses.dataclass` allows the defining of item classes with field names, so that :ref:`item exporters <topics-exporters>` can export all fields by default even if the first scraped object does not have values for all of them. -Additionally, ``dataclass`` items also allow to: +Additionally, ``dataclass`` items also allow you to: * define the type and default value of each defined field. @@ -126,7 +126,7 @@ attr.s objects .. versionadded:: 2.2 -:func:`attr.s` allows defining item classes with field names, +:func:`attr.s` allows the defining of item classes with field names, so that :ref:`item exporters <topics-exporters>` can export all fields by default even if the first scraped object does not have values for all of them. From c21c4a18509ec9657bfe6e6f99bd913bba7ea41d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 21 Aug 2024 01:06:05 +0500 Subject: [PATCH 1521/2083] Revert uvloop restrictions. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4ccaea653..bd6782ce5 100644 --- a/tox.ini +++ b/tox.ini @@ -150,7 +150,7 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - uvloop; platform_system != "Windows" and python_version < '3.13' + uvloop; platform_system != "Windows" bpython; python_version < '3.13' # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests From 5794071f9679c89ef4ee75e8a627274b2464b65b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 23 Aug 2024 15:48:01 +0500 Subject: [PATCH 1522/2083] Typing fixes and updates. (#6460) --- scrapy/pipelines/media.py | 2 +- scrapy/utils/defer.py | 3 ++- scrapy/utils/log.py | 2 +- scrapy/utils/ssl.py | 2 +- tox.ini | 12 ++++++------ 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index ea36a9e8a..6bd3ed9b4 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -211,7 +211,7 @@ class MediaPipeline(ABC): # minimize cached information for failure result.cleanFailure() result.frames = [] - result.stack = None + result.stack = [] # This code fixes a memory leak by avoiding to keep references to # the Request and Response objects on the Media Pipeline cache. diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index c5763a06c..33ec23cec 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -19,6 +19,7 @@ from typing import ( Callable, Coroutine, Dict, + Generic, Iterable, Iterator, List, @@ -144,7 +145,7 @@ def parallel( return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter(Iterator[Deferred]): +class _AsyncCooperatorAdapter(Iterator[Deferred], Generic[_T]): """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 439b065a9..4a70de6b4 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -128,7 +128,7 @@ def configure_logging( settings = Settings(settings) if settings.getbool("LOG_STDOUT"): - sys.stdout = StreamLogger(logging.getLogger("stdout")) # type: ignore[assignment] + sys.stdout = StreamLogger(logging.getLogger("stdout")) if install_root_handler: install_scrapy_root_handler(settings) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 95611ebd9..2c3a259c1 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -20,7 +20,7 @@ def x509name_to_string(x509name: X509Name) -> str: # from OpenSSL.crypto.X509Name.__repr__ result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512) pyOpenSSLutil.lib.X509_NAME_oneline( - x509name._name, result_buffer, len(result_buffer) # type: ignore[attr-defined] + x509name._name, result_buffer, len(result_buffer) ) return ffi_buf_to_string(result_buffer) diff --git a/tox.ini b/tox.ini index e3dd96425..2d62f1cb7 100644 --- a/tox.ini +++ b/tox.ini @@ -46,16 +46,16 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.10.1 + mypy==1.11.1 typing-extensions==4.12.2 - types-lxml==2024.4.14 + types-lxml==2024.8.7 types-Pygments==2.18.0.20240506 - types-pyOpenSSL==24.1.0.20240425 - types-setuptools==70.3.0.20240710 - botocore-stubs==1.34.143 - boto3-stubs[s3]==1.34.143 + types-setuptools==71.1.0.20240806 + botocore-stubs==1.34.158 + boto3-stubs[s3]==1.34.158 attrs >= 18.2.0 Pillow >= 10.3.0 + pyOpenSSL >= 24.2.1 pytest >= 8.2.0 w3lib >= 2.2.0 commands = From 6ce0342beb1a5b588f353e52fe03d5e0ec84d938 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi <GeorgeA92@users.noreply.github.com> Date: Mon, 26 Aug 2024 20:53:06 +0200 Subject: [PATCH 1523/2083] Allow yielding items from start_requests (#6417) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Georgiy Zatserklianyi <george.zatseklyany@gmail.com> Co-authored-by: Adrián Chaves <adrian@chaves.io> Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name> --- docs/topics/signals.rst | 16 ++++++++------ docs/topics/spider-middleware.rst | 2 +- docs/topics/spiders.rst | 3 ++- scrapy/core/engine.py | 14 ++++++++++-- scrapy/core/scraper.py | 21 ++++++++++++------ scrapy/logformatter.py | 19 ++++++++++++---- tests/spiders.py | 13 +++++++++++ tests/test_crawl.py | 36 +++++++++++++++++++++++++++++++ tests/test_spidermiddleware.py | 9 ++++---- 9 files changed, 109 insertions(+), 24 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 13e636055..b45b12540 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -159,8 +159,9 @@ item_scraped :param spider: the spider which scraped the item :type spider: :class:`~scrapy.Spider` object - :param response: the response from where the item was scraped - :type response: :class:`~scrapy.http.Response` object + :param response: the response from where the item was scraped, or ``None`` + if it was yielded from :meth:`~scrapy.Spider.start_requests`. + :type response: :class:`~scrapy.http.Response` | ``None`` item_dropped ~~~~~~~~~~~~ @@ -179,8 +180,9 @@ item_dropped :param spider: the spider which scraped the item :type spider: :class:`~scrapy.Spider` object - :param response: the response from where the item was dropped - :type response: :class:`~scrapy.http.Response` object + :param response: the response from where the item was dropped, or ``None`` + if it was yielded from :meth:`~scrapy.Spider.start_requests`. + :type response: :class:`~scrapy.http.Response` | ``None`` :param exception: the exception (which must be a :exc:`~scrapy.exceptions.DropItem` subclass) which caused the item @@ -201,8 +203,10 @@ item_error :param item: the item that caused the error in the :ref:`topics-item-pipeline` :type item: :ref:`item object <item-types>` - :param response: the response being processed when the exception was raised - :type response: :class:`~scrapy.http.Response` object + :param response: the response being processed when the exception was + raised, or ``None`` if it was yielded from + :meth:`~scrapy.Spider.start_requests`. + :type response: :class:`~scrapy.http.Response` | ``None`` :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8ddf17a14..8f39bcd53 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -176,7 +176,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`. items). It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.Request` objects. + return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects <topics-items>`. .. note:: When implementing this method in your spider middleware, you should always return an iterable (that follows the input one) and diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 8a0102a51..e1b1c5ad6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -203,7 +203,8 @@ scrapy.Spider .. method:: start_requests() - This method must return an iterable with the first Requests to crawl for + This method must return an iterable with the first Requests to crawl and/or with :ref:`item objects + <topics-items>` for this spider. It is called by Scrapy when the spider is opened for scraping. Scrapy calls it only once, so it is safe to implement :meth:`start_requests` as a generator. diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 5318cbd64..63d84339d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -24,6 +24,7 @@ from typing import ( cast, ) +from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -194,7 +195,7 @@ class ExecutionEngine: if self.slot.start_requests is not None and not self._needs_backout(): try: - request = next(self.slot.start_requests) + request_or_item = next(self.slot.start_requests) except StopIteration: self.slot.start_requests = None except Exception: @@ -205,7 +206,16 @@ class ExecutionEngine: extra={"spider": self.spider}, ) else: - self.crawl(request) + if isinstance(request_or_item, Request): + self.crawl(request_or_item) + elif is_item(request_or_item): + self.scraper.start_itemproc(request_or_item, response=None) + else: + logger.error( + f"Got {request_or_item!r} among start requests. Only " + f"requests and items are supported. It will be " + f"ignored." + ) if self.spider_is_idle() and self.slot.close_if_idle: self._spider_idle() diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index a7d65e1e3..7a51dbeb4 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -313,15 +313,11 @@ class Scraper: """Process each Request/Item (given in the output parameter) returned from the given spider """ - assert self.slot is not None # typing if isinstance(output, Request): assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) elif is_item(output): - self.slot.itemproc_size += 1 - dfd = self.itemproc.process_item(output, spider) - dfd.addBoth(self._itemproc_finished, output, response, spider) - return dfd + return self.start_itemproc(output, response=response) elif output is None: pass else: @@ -333,6 +329,19 @@ class Scraper: ) return None + def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]: + """Send *item* to the item pipelines for processing. + + *response* is the source of the item data. If the item does not come + from response data, e.g. it was hard-coded, set it to ``None``. + """ + assert self.slot is not None # typing + assert self.crawler.spider is not None # typing + self.slot.itemproc_size += 1 + dfd = self.itemproc.process_item(item, self.crawler.spider) + dfd.addBoth(self._itemproc_finished, item, response, self.crawler.spider) + return dfd + def _log_download_errors( self, spider_failure: Failure, @@ -373,7 +382,7 @@ class Scraper: return None def _itemproc_finished( - self, output: Any, item: Any, response: Response, spider: Spider + self, output: Any, item: Any, response: Optional[Response], spider: Spider ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 601209fb0..fea7003e5 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -9,6 +9,7 @@ from twisted.python.failure import Failure # working around https://github.com/sphinx-doc/sphinx/issues/10400 from scrapy import Request, Spider # noqa: TC001 from scrapy.http import Response # noqa: TC001 +from scrapy.utils.python import global_object_name from scrapy.utils.request import referer_str if TYPE_CHECKING: @@ -92,11 +93,13 @@ class LogFormatter: } def scraped( - self, item: Any, response: Union[Response, Failure], spider: Spider + self, item: Any, response: Union[Response, Failure, None], spider: Spider ) -> LogFormatterResult: """Logs a message when an item is scraped by a spider.""" src: Any - if isinstance(response, Failure): + if response is None: + src = f"{global_object_name(spider.__class__)}.start_requests" + elif isinstance(response, Failure): src = response.getErrorMessage() else: src = response @@ -110,7 +113,11 @@ class LogFormatter: } def dropped( - self, item: Any, exception: BaseException, response: Response, spider: Spider + self, + item: Any, + exception: BaseException, + response: Optional[Response], + spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { @@ -123,7 +130,11 @@ class LogFormatter: } def item_error( - self, item: Any, exception: BaseException, response: Response, spider: Spider + self, + item: Any, + exception: BaseException, + response: Optional[Response], + spider: Spider, ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing through the item pipeline. diff --git a/tests/spiders.py b/tests/spiders.py index 743811893..5d5792858 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -346,6 +346,19 @@ class BrokenStartRequestsSpider(FollowAllSpider): yield from super().parse(response) +class StartRequestsItemSpider(FollowAllSpider): + def start_requests(self): + yield {"name": "test item"} + + +class StartRequestsGoodAndBadOutput(FollowAllSpider): + def start_requests(self): + yield {"a": "a"} + yield Request("data:,a") + yield "data:,b" + yield object() + + class SingleRequestSpider(MetaSpider): seed = None callback_func = None diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 6cde4ed8c..125709571 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,5 +1,6 @@ import json import logging +import re import unittest from ipaddress import IPv4Address from socket import gethostbyname @@ -49,6 +50,8 @@ from tests.spiders import ( HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, + StartRequestsGoodAndBadOutput, + StartRequestsItemSpider, ) @@ -184,6 +187,39 @@ class CrawlTestCase(TestCase): self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) + @defer.inlineCallbacks + def test_start_requests_items(self): + with LogCapture("scrapy", level=logging.ERROR) as log: + crawler = get_crawler(StartRequestsItemSpider) + yield crawler.crawl(mockserver=self.mockserver) + + self.assertEqual(len(log.records), 0) + + @defer.inlineCallbacks + def test_start_requests_unsupported_output(self): + with LogCapture("scrapy", level=logging.ERROR) as log: + crawler = get_crawler(StartRequestsGoodAndBadOutput) + yield crawler.crawl(mockserver=self.mockserver) + + self.assertEqual(len(log.records), 2) + self.assertEqual( + log.records[0].msg, + ( + "Got 'data:,b' among start requests. Only requests and items " + "are supported. It will be ignored." + ), + ) + self.assertTrue( + re.match( + ( + r"^Got <object object at 0x[0-9a-fA-F]+> among start " + r"requests\. Only requests and items are supported\. It " + r"will be ignored\.$" + ), + log.records[1].msg, + ) + ) + @defer.inlineCallbacks def test_start_requests_laziness(self): settings = {"CONCURRENT_REQUESTS": 1} diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 38ca8d950..9dbffe353 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,5 +1,5 @@ import collections.abc -from typing import Optional +from typing import Optional, Union from unittest import mock from testfixtures import LogCapture @@ -112,7 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): Should work for process_spider_output and, when it's supported, process_start_requests. """ - ITEM_TYPE: type + ITEM_TYPE: Union[type, tuple] RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod @@ -328,12 +328,13 @@ class ProcessStartRequestsSimpleMiddleware: class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): """process_start_requests tests for simple start_requests""" - ITEM_TYPE = Request + ITEM_TYPE = (Request, dict) MW_SIMPLE = ProcessStartRequestsSimpleMiddleware def _start_requests(self): - for i in range(3): + for i in range(2): yield Request(f"https://example.com/{i}", dont_filter=True) + yield {"name": "test item"} @defer.inlineCallbacks def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): From b85e5a66ede0a2255b335ee4869836e9d30c580a Mon Sep 17 00:00:00 2001 From: Laerte Pereira <laertefbk@gmail.com> Date: Mon, 26 Aug 2024 23:21:09 -0300 Subject: [PATCH 1524/2083] Add support for meta in Spider Contracts --- docs/topics/contracts.rst | 8 +++ scrapy/contracts/default.py | 14 +++++ scrapy/settings/default_settings.py | 1 + tests/test_contracts.py | 79 +++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 2d61026e9..a912ff986 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -46,6 +46,14 @@ This callback is tested using three built-in contracts: @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} +.. class:: MetadataContract + + This contract (``@meta``) sets the :attr:` meta <scrapy.Request.meta>` + attribute for the sample request. It must be a valid JSON dictionary. + :: + + @meta {"arg1": "value1", "arg2": "value2", ...} + .. class:: ReturnsContract This contract (``@returns``) sets lower and upper bounds for the items and diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 71ca4168a..87099b950 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -35,6 +35,20 @@ class CallbackKeywordArgumentsContract(Contract): return args +class MetadataContract(Contract): + """Contract to key metadata arguments for the request. + The value should be JSON-encoded dictionary, e.g.: + + @meta {"arg1": "some value"} + """ + + name = "meta" + + def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + args["meta"] = json.loads(" ".join(self.args)) + return args + + class ReturnsContract(Contract): """Contract to check the output of a callback diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 932475fb5..7ba0128a5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -333,6 +333,7 @@ SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { "scrapy.contracts.default.UrlContract": 1, "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1, + "scrapy.contracts.default.MetadataContract": 1, "scrapy.contracts.default.ReturnsContract": 2, "scrapy.contracts.default.ScrapesContract": 3, } diff --git a/tests/test_contracts.py b/tests/test_contracts.py index c9c12f0d8..d578b3af4 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -8,6 +8,7 @@ from scrapy import FormRequest from scrapy.contracts import Contract, ContractsManager from scrapy.contracts.default import ( CallbackKeywordArgumentsContract, + MetadataContract, ReturnsContract, ScrapesContract, UrlContract, @@ -29,6 +30,10 @@ class ResponseMock: url = "http://scrapy.org" +class ResponseMetaMock(ResponseMock): + meta = None + + class CustomSuccessContract(Contract): name = "custom_success_contract" @@ -195,6 +200,33 @@ class TestSpider(Spider): """ pass + def returns_request_meta(self, response): + """method which returns request + @url https://example.org + @meta {"cookiejar": "session1"} + @returns requests 1 + """ + return Request( + "https://example.org", meta=response.meta, callback=self.returns_item_meta + ) + + def returns_item_meta(self, response): + """method which returns item + @url http://scrapy.org + @meta {"key": "example"} + @returns items 1 1 + """ + return TestItem(name="example", url=response.url) + + def returns_error_missing_meta(self, response): + """method which depends of metadata be defined + + @url http://scrapy.org + @returns items 1 + """ + key = response.meta["key"] + yield {key: "value"} + class CustomContractSuccessSpider(Spider): name = "custom_contract_success_spider" @@ -224,6 +256,7 @@ class ContractsManagerTest(unittest.TestCase): contracts = [ UrlContract, CallbackKeywordArgumentsContract, + MetadataContract, ReturnsContract, ScrapesContract, CustomFormContract, @@ -328,6 +361,52 @@ class ContractsManagerTest(unittest.TestCase): request.callback(response, **request.cb_kwargs) self.should_error() + def test_meta(self): + spider = TestSpider() + + # extract contracts correctly + contracts = self.conman.extract_contracts(spider.returns_request_meta) + self.assertEqual(len(contracts), 3) + self.assertEqual( + frozenset(type(x) for x in contracts), + frozenset([UrlContract, MetadataContract, ReturnsContract]), + ) + + contracts = self.conman.extract_contracts(spider.returns_item_meta) + self.assertEqual(len(contracts), 3) + self.assertEqual( + frozenset(type(x) for x in contracts), + frozenset([UrlContract, MetadataContract, ReturnsContract]), + ) + + response = ResponseMetaMock() + + # returns_request + request = self.conman.from_method(spider.returns_request_meta, self.results) + assert request.meta["cookiejar"] == "session1" + response.meta = request.meta + request.callback(response) + assert response.meta["cookiejar"] == "session1" + self.should_succeed() + + response = ResponseMetaMock() + + # returns_item + request = self.conman.from_method(spider.returns_item_meta, self.results) + assert request.meta["key"] == "example" + response.meta = request.meta + request.callback(ResponseMetaMock) + assert response.meta["key"] == "example" + self.should_succeed() + + response = ResponseMetaMock() + + request = self.conman.from_method( + spider.returns_error_missing_meta, self.results + ) + request.callback(response) + self.should_error() + def test_returns(self): spider = TestSpider() response = ResponseMock() From f68f29dd1361f427be151b09c99068b292275923 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 26 Aug 2024 23:37:57 -0300 Subject: [PATCH 1525/2083] Update docs/topics/contracts.rst --- docs/topics/contracts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index a912ff986..7557dacc0 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -48,7 +48,7 @@ This callback is tested using three built-in contracts: .. class:: MetadataContract - This contract (``@meta``) sets the :attr:` meta <scrapy.Request.meta>` + This contract (``@meta``) sets the :attr:`meta <scrapy.Request.meta>` attribute for the sample request. It must be a valid JSON dictionary. :: From 3c2a9fa262dd3e63acc58c8f0a2f91cf65c33bc4 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <laertefbk@gmail.com> Date: Tue, 27 Aug 2024 07:16:01 -0300 Subject: [PATCH 1526/2083] update docs --- docs/topics/contracts.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index a912ff986..82afa0dc1 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -20,13 +20,13 @@ following example: This function parses a sample response. Some contracts are mingled with this docstring. - @url http://www.amazon.com/s?field-keywords=selfish+gene + @url http://www.example.com/s?field-keywords=selfish+gene @returns items 1 16 @returns requests 0 0 @scrapes Title Author Year Price """ -This callback is tested using three built-in contracts: +You can use the following contracts: .. module:: scrapy.contracts.default From ddbdfeb699a2308ca600781b2d1549cbee62725c Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Tue, 27 Aug 2024 07:24:57 -0300 Subject: [PATCH 1527/2083] Update scrapy/contracts/default.py --- scrapy/contracts/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 87099b950..e7b11d426 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -36,7 +36,7 @@ class CallbackKeywordArgumentsContract(Contract): class MetadataContract(Contract): - """Contract to key metadata arguments for the request. + """Contract to set metadata arguments for the request. The value should be JSON-encoded dictionary, e.g.: @meta {"arg1": "some value"} From 67ab8d4650c1e9212c9508803c7b5265e166cbaa Mon Sep 17 00:00:00 2001 From: Daniel O'Connor <daniel.oconnor@gmail.com> Date: Thu, 29 Aug 2024 04:37:49 +0930 Subject: [PATCH 1528/2083] Refactor genspider slightly so template variables can be overridden (#6470) --- scrapy/commands/genspider.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2649fb23d..6c3713f8f 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -116,6 +116,24 @@ class Command(ScrapyCommand): if opts.edit: self.exitcode = os.system(f'scrapy edit "{name}"') # nosec + def _generate_template_variables( + self, + module: str, + name: str, + url: str, + template_name: str, + ): + capitalized_module = "".join(s.capitalize() for s in module.split("_")) + return { + "project_name": self.settings.get("BOT_NAME"), + "ProjectName": string_camelcase(self.settings.get("BOT_NAME")), + "module": module, + "name": name, + "url": url, + "domain": extract_domain(url), + "classname": f"{capitalized_module}Spider", + } + def _genspider( self, module: str, @@ -125,17 +143,7 @@ class Command(ScrapyCommand): template_file: Union[str, os.PathLike], ) -> None: """Generate the spider module, based on the given template""" - capitalized_module = "".join(s.capitalize() for s in module.split("_")) - domain = extract_domain(url) - tvars = { - "project_name": self.settings.get("BOT_NAME"), - "ProjectName": string_camelcase(self.settings.get("BOT_NAME")), - "module": module, - "name": name, - "url": url, - "domain": domain, - "classname": f"{capitalized_module}Spider", - } + tvars = self._generate_template_variables(module, name, url, template_name) if self.settings.get("NEWSPIDER_MODULE"): spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) assert spiders_module.__file__ From f260f819e0794708868ed447ae154caa74d965f7 Mon Sep 17 00:00:00 2001 From: LucasSD <lucas.stonedrake@gmail.com> Date: Mon, 9 Sep 2024 20:26:02 +0100 Subject: [PATCH 1529/2083] Remove debug log message from _schedule_request method --- scrapy/core/engine.py | 5 ----- tests/test_engine.py | 1 - 2 files changed, 6 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 63d84339d..fd9a5f781 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -39,7 +39,6 @@ from scrapy.settings import Settings from scrapy.signalmanager import SignalManager from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object -from scrapy.utils.python import global_object_name from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: @@ -325,10 +324,6 @@ class ExecutionEngine: ) for handler, result in request_scheduled_result: if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): - logger.debug( - f"Signal handler {global_object_name(handler)} dropped " - f"request {request} before it reached the scheduler." - ) return if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( diff --git a/tests/test_engine.py b/tests/test_engine.py index 86526420f..2ebc0b5e4 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -499,7 +499,6 @@ def test_request_scheduled_signal(caplog): assert scheduler.enqueued == [ keep_request ], f"{scheduler.enqueued!r} != [{keep_request!r}]" - assert "dropped request <GET https://drop.example>" in caplog.text crawler.signals.disconnect(signal_handler, request_scheduled) From b3f562d6a5265a7879a9dde3a3dde231bb3970c7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 12 Sep 2024 14:31:50 +0500 Subject: [PATCH 1530/2083] Revert "Revert uvloop restrictions." This reverts commit c21c4a18509ec9657bfe6e6f99bd913bba7ea41d. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index bd6782ce5..4ccaea653 100644 --- a/tox.ini +++ b/tox.ini @@ -150,7 +150,7 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - uvloop; platform_system != "Windows" + uvloop; platform_system != "Windows" and python_version < '3.13' bpython; python_version < '3.13' # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests From ee9ee2d12d386764044ac7c73f5062548bd1157d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 12 Sep 2024 14:32:32 +0500 Subject: [PATCH 1531/2083] Revert bpython restrictions. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4ccaea653..8f3d23d75 100644 --- a/tox.ini +++ b/tox.ini @@ -151,7 +151,7 @@ deps = Pillow Twisted[http2] uvloop; platform_system != "Windows" and python_version < '3.13' - bpython; python_version < '3.13' # optional for shell wrapper tests + bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests From e139d22db9f3becc0a7e19e79daa1da3bb65383f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 20 Sep 2024 19:28:28 +0500 Subject: [PATCH 1532/2083] Fix expectations for get_func_args() on 3.13. --- tests/test_utils_python.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 4c60deafe..5681ff9a4 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,6 +1,7 @@ import functools import operator import platform +import sys from twisted.trial import unittest @@ -238,16 +239,18 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(str.split, stripself=True), ["sep", "maxsplit"]) self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"]) - if platform.python_implementation() == "CPython": - # This didn't work on older versions of CPython: https://github.com/python/cpython/issues/86951 + if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy": + # the correct and correctly extracted signature + self.assertEqual( + get_func_args(operator.itemgetter(2), stripself=True), ["obj"] + ) + elif platform.python_implementation() == "CPython": + # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature + # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951) self.assertIn( get_func_args(operator.itemgetter(2), stripself=True), [[], ["args", "kwargs"]], ) - elif platform.python_implementation() == "PyPy": - self.assertEqual( - get_func_args(operator.itemgetter(2), stripself=True), ["obj"] - ) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) From 46cddc6ecfbe9a0750676143ef789acf6c2e637d Mon Sep 17 00:00:00 2001 From: mmoriniere <maxime.moriniere@hotmail.fr> Date: Wed, 2 Oct 2024 10:04:03 +0200 Subject: [PATCH 1533/2083] Ignore SyntaxError as well when SPIDER_LOADER_WARN_ONLY is set to True (#6484) --- docs/news.rst | 6 ++++++ docs/topics/settings.rst | 2 +- scrapy/spiderloader.py | 2 +- tests/test_spiderloader/__init__.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 758b22d80..58b51c9ea 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,6 +8,12 @@ Release notes Scrapy VERSION (YYYY-MM-DD) --------------------------- +New features +~~~~~~~~~~~~ + +- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``, + ``SpiderLoader`` does not raise :exc:`SyntaxError` but emits a warning instead. + Deprecations ~~~~~~~~~~~~ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 904bd7ecc..02fca7ff4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1580,7 +1580,7 @@ SPIDER_LOADER_WARN_ONLY Default: ``False`` By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, -it will fail loudly if there is any ``ImportError`` exception. +it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception. But you can choose to silence this exception and turn it into a simple warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index b8fe65668..f5fd899b2 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -64,7 +64,7 @@ class SpiderLoader: try: for module in walk_modules(name): self._load_spiders(module) - except ImportError: + except (ImportError, SyntaxError): if self.warn_only: warnings.warn( f"\n{traceback.format_exc()}Could not load spiders " diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index f950739f2..32699d837 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -4,6 +4,7 @@ import tempfile import warnings from pathlib import Path from tempfile import mkdtemp +from unittest import mock from twisted.trial import unittest from zope.interface.verify import verifyObject @@ -136,6 +137,33 @@ class SpiderLoaderTest(unittest.TestCase): spiders = spider_loader.list() self.assertEqual(spiders, []) + def test_syntax_error_exception(self): + module = "tests.test_spiderloader.test_spiders.spider1" + with mock.patch.object(SpiderLoader, "_load_spiders") as m: + m.side_effect = SyntaxError + settings = Settings({"SPIDER_MODULES": [module]}) + self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings) + + def test_syntax_error_warning(self): + with warnings.catch_warnings(record=True) as w, mock.patch.object( + SpiderLoader, "_load_spiders" + ) as m: + m.side_effect = SyntaxError + module = "tests.test_spiderloader.test_spiders.spider1" + settings = Settings( + {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} + ) + spider_loader = SpiderLoader.from_settings(settings) + if str(w[0].message).startswith("_SixMetaPathImporter"): + # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, + # at least until all six versions we can import (including botocore.vendored.six) + # are updated to 1.16.0+ + w.pop(0) + self.assertIn("Could not load spiders from module", str(w[0].message)) + + spiders = spider_loader.list() + self.assertEqual(spiders, []) + class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): From 8c133fcf7e4f19d55d60dc6a090d75dab6db1a72 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 7 Oct 2024 23:04:48 +0500 Subject: [PATCH 1534/2083] Remove the installation dependency on setuptools. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 2d6d26b0c..f458a9de3 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,6 @@ install_requires = [ "zope.interface>=5.1.0", "protego>=0.1.15", "itemadapter>=0.1.0", - "setuptools", "packaging", "tldextract", "lxml>=4.4.1", From df6c51af0f518724151b69bd2d958a0c3fb18ff3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 8 Oct 2024 15:37:49 +0500 Subject: [PATCH 1535/2083] Use the 3.13 release. --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 2 +- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 10 +++++----- .github/workflows/tests-windows.yml | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 224d5cbbe..e912bf0cd 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.13.0-beta.1" + - python-version: "3.13" env: TOXENV: pylint - python-version: 3.8 @@ -24,7 +24,7 @@ jobs: - python-version: "3.12" # Keep in sync with .readthedocs.yml env: TOXENV: docs - - python-version: "3.13.0-beta.1" + - python-version: "3.13" env: TOXENV: twinecheck diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 75a7479eb..4c7bde147 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.13.0-beta.1" + python-version: "3.13" - run: | pip install --upgrade build twine python -m build diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 2e6e4265d..1f123824b 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13.0-beta.1"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 39af0c79f..9db2ad897 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -24,10 +24,10 @@ jobs: - python-version: "3.12" env: TOXENV: py - - python-version: "3.13.0-beta.1" + - python-version: "3.13" env: TOXENV: py - - python-version: "3.13.0-beta.1" + - python-version: "3.13" env: TOXENV: asyncio - python-version: pypy3.9 @@ -54,10 +54,10 @@ jobs: env: TOXENV: botocore-pinned - - python-version: "3.13.0-beta.1" + - python-version: "3.13" env: TOXENV: extra-deps - - python-version: "3.13.0-beta.1" + - python-version: "3.13" env: TOXENV: botocore @@ -70,7 +70,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: contains(matrix.python-version, 'pypy') || contains(matrix.python-version, 'beta') || contains(matrix.env.TOXENV, 'pinned') + if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index d32d19958..4e1034d77 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -27,10 +27,10 @@ jobs: - python-version: "3.12" env: TOXENV: py - - python-version: "3.13.0-beta.1" + - python-version: "3.13" env: TOXENV: py - - python-version: "3.13.0-beta.1" + - python-version: "3.13" env: TOXENV: asyncio From 29bb8692841491db388d4fa71f28b453a79bdab9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 8 Oct 2024 15:43:17 +0500 Subject: [PATCH 1536/2083] Remove the beta block. --- .github/workflows/checks.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e912bf0cd..03298e3cc 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -36,12 +36,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install system libraries - if: contains(matrix.python-version, 'beta') - run: | - sudo apt-get update - sudo apt-get install libxml2-dev libxslt-dev - - name: Run check env: ${{ matrix.env }} run: | From 87651fdf47403767b5b79f075237e7351ba4853b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 8 Oct 2024 16:04:26 +0500 Subject: [PATCH 1537/2083] Don't use types-setuptools. --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2d62f1cb7..80ef4a99e 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,6 @@ deps = typing-extensions==4.12.2 types-lxml==2024.8.7 types-Pygments==2.18.0.20240506 - types-setuptools==71.1.0.20240806 botocore-stubs==1.34.158 boto3-stubs[s3]==1.34.158 attrs >= 18.2.0 From 5ef54741729739e9a161d80e74d0076dfdb973cc Mon Sep 17 00:00:00 2001 From: Klaus Rettinghaus <klaus.rettinghaus@gmail.com> Date: Wed, 9 Oct 2024 20:38:50 +0200 Subject: [PATCH 1538/2083] update gh-action-pypi-publish --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 03e94f761..5ce48be61 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,6 +20,6 @@ jobs: pip install --upgrade build twine python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.9.0 + uses: pypa/gh-action-pypi-publish@v1.10.3 with: password: ${{ secrets.PYPI_TOKEN }} From 53916630723a86277836053e1c54fe50655f0bd5 Mon Sep 17 00:00:00 2001 From: Vsevolod Breus <vsevolodbreus1@gmail.com> Date: Wed, 16 Oct 2024 08:03:16 +0000 Subject: [PATCH 1539/2083] Drop Python 3.8 Support (#6472) --- .github/workflows/checks.yml | 4 +- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 12 +-- .github/workflows/tests-windows.yml | 5 +- .pre-commit-config.yaml | 2 +- README.rst | 2 +- docs/intro/install.rst | 2 +- scrapy/addons.py | 4 +- scrapy/cmdline.py | 26 +++--- scrapy/commands/__init__.py | 14 ++-- scrapy/commands/bench.py | 6 +- scrapy/commands/check.py | 3 +- scrapy/commands/crawl.py | 4 +- scrapy/commands/edit.py | 3 +- scrapy/commands/fetch.py | 8 +- scrapy/commands/genspider.py | 4 +- scrapy/commands/list.py | 4 +- scrapy/commands/parse.py | 41 ++++----- scrapy/commands/runspider.py | 4 +- scrapy/commands/settings.py | 3 +- scrapy/commands/shell.py | 8 +- scrapy/commands/startproject.py | 10 +-- scrapy/commands/version.py | 3 +- scrapy/contracts/__init__.py | 41 ++++----- scrapy/contracts/default.py | 14 ++-- scrapy/core/downloader/__init__.py | 29 ++----- scrapy/core/downloader/contextfactory.py | 6 +- scrapy/core/downloader/handlers/__init__.py | 28 +++---- scrapy/core/downloader/handlers/datauri.py | 4 +- scrapy/core/downloader/handlers/ftp.py | 4 +- scrapy/core/downloader/handlers/http10.py | 9 +- scrapy/core/downloader/handlers/http11.py | 14 ++-- scrapy/core/downloader/handlers/s3.py | 4 +- scrapy/core/downloader/middleware.py | 7 +- scrapy/core/downloader/tls.py | 4 +- scrapy/core/downloader/webclient.py | 6 +- scrapy/core/engine.py | 27 ++---- scrapy/core/http2/agent.py | 14 ++-- scrapy/core/http2/protocol.py | 22 ++--- scrapy/core/http2/stream.py | 12 +-- scrapy/core/scheduler.py | 20 ++--- scrapy/core/scraper.py | 31 ++----- scrapy/core/spidermw.py | 22 ++--- scrapy/crawler.py | 43 ++++------ scrapy/downloadermiddlewares/cookies.py | 5 +- .../downloadermiddlewares/defaultheaders.py | 8 +- .../downloadermiddlewares/httpcompression.py | 20 ++--- scrapy/downloadermiddlewares/httpproxy.py | 6 +- scrapy/downloadermiddlewares/offsite.py | 4 +- scrapy/downloadermiddlewares/redirect.py | 4 +- scrapy/downloadermiddlewares/retry.py | 10 +-- scrapy/downloadermiddlewares/robotstxt.py | 4 +- scrapy/downloadermiddlewares/stats.py | 4 +- scrapy/dupefilters.py | 4 +- scrapy/exporters.py | 17 ++-- scrapy/extension.py | 4 +- scrapy/extensions/closespider.py | 6 +- scrapy/extensions/feedexport.py | 84 ++++++++----------- scrapy/extensions/httpcache.py | 28 ++++--- scrapy/extensions/logstats.py | 4 +- scrapy/extensions/memusage.py | 8 +- scrapy/extensions/periodic_log.py | 34 ++++---- scrapy/extensions/postprocessing.py | 12 +-- scrapy/extensions/statsmailer.py | 8 +- scrapy/extensions/telnet.py | 8 +- scrapy/extensions/throttle.py | 4 +- scrapy/http/cookies.py | 22 ++--- scrapy/http/headers.py | 35 +++----- scrapy/http/request/__init__.py | 36 ++++---- scrapy/http/request/form.py | 34 +++----- scrapy/http/request/json_request.py | 14 ++-- scrapy/http/response/__init__.py | 50 ++++------- scrapy/http/response/text.py | 36 +++----- scrapy/item.py | 22 ++--- scrapy/linkextractors/__init__.py | 9 +- scrapy/linkextractors/lxmlhtml.py | 55 +++++------- scrapy/logformatter.py | 6 +- scrapy/mail.py | 33 +++----- scrapy/middleware.py | 29 ++----- scrapy/pipelines/__init__.py | 4 +- scrapy/pipelines/files.py | 61 +++++++------- scrapy/pipelines/images.py | 41 ++++----- scrapy/pipelines/media.py | 26 +++--- scrapy/pqueues.py | 42 ++++------ scrapy/resolver.py | 8 +- scrapy/responsetypes.py | 23 +++-- scrapy/selector/unified.py | 4 +- scrapy/settings/__init__.py | 40 ++++----- scrapy/shell.py | 13 +-- scrapy/signalmanager.py | 6 +- scrapy/spiderloader.py | 14 ++-- scrapy/spidermiddlewares/depth.py | 4 +- scrapy/spidermiddlewares/httperror.py | 6 +- scrapy/spidermiddlewares/offsite.py | 6 +- scrapy/spidermiddlewares/referer.py | 29 +++---- scrapy/spidermiddlewares/urllength.py | 4 +- scrapy/spiders/__init__.py | 8 +- scrapy/spiders/crawl.py | 36 +++----- scrapy/spiders/feed.py | 13 ++- scrapy/spiders/init.py | 3 +- scrapy/spiders/sitemap.py | 27 ++---- scrapy/squeues.py | 17 ++-- scrapy/statscollectors.py | 6 +- scrapy/utils/asyncgen.py | 5 +- scrapy/utils/conf.py | 42 ++++------ scrapy/utils/console.py | 28 ++++--- scrapy/utils/curl.py | 17 ++-- scrapy/utils/datatypes.py | 20 ++--- scrapy/utils/decorators.py | 4 +- scrapy/utils/defer.py | 45 ++++------ scrapy/utils/deprecate.py | 12 +-- scrapy/utils/engine.py | 6 +- scrapy/utils/iterators.py | 26 +++--- scrapy/utils/log.py | 31 +++---- scrapy/utils/misc.py | 28 ++----- scrapy/utils/ossignal.py | 5 +- scrapy/utils/project.py | 2 - scrapy/utils/python.py | 50 ++++------- scrapy/utils/reactor.py | 22 ++--- scrapy/utils/request.py | 25 ++---- scrapy/utils/response.py | 8 +- scrapy/utils/signal.py | 21 ++--- scrapy/utils/sitemap.py | 11 ++- scrapy/utils/spider.py | 28 ++----- scrapy/utils/test.py | 28 +++---- scrapy/utils/testproc.py | 8 +- scrapy/utils/trackref.py | 10 ++- scrapy/utils/url.py | 8 +- scrapy/utils/versions.py | 3 +- setup.py | 11 ++- tests/mocks/dummydbm.py | 6 +- tests/mockserver.py | 4 +- tests/test_addons.py | 4 +- tests/test_commands.py | 9 +- tests/test_crawler.py | 3 +- tests/test_downloader_handlers.py | 18 ++-- tests/test_http2_client_protocol.py | 4 +- tests/test_http_request.py | 6 +- tests/test_pipeline_crawl.py | 4 +- tests/test_pipeline_files.py | 9 +- tests/test_pipeline_images.py | 10 +-- tests/test_request_cb_kwargs.py | 4 +- tests/test_scheduler_base.py | 4 +- tests/test_settings/__init__.py | 7 +- tests/test_spidermiddleware.py | 6 +- tests/test_spidermiddleware_httperror.py | 3 +- tests/test_spidermiddleware_referer.py | 32 +++---- tests/test_utils_datatypes.py | 3 +- tests/test_utils_log.py | 9 +- tests/test_utils_request.py | 8 +- tests_typing/test_http_request.mypy-testing | 2 +- tests_typing/test_http_response.mypy-testing | 2 +- tox.ini | 14 ++-- 153 files changed, 1011 insertions(+), 1307 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2be6a9502..9240a16f4 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -15,10 +15,10 @@ jobs: - python-version: "3.12" env: TOXENV: pylint - - python-version: 3.8 + - python-version: "3.9" env: TOXENV: typing - - python-version: 3.8 + - python-version: "3.9" env: TOXENV: typing-tests - python-version: "3.12" # Keep in sync with .readthedocs.yml diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 8ebe7f1db..27ea0613d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 763de9eff..29c870e6a 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.9 + - python-version: "3.9" env: TOXENV: py - python-version: "3.10" @@ -35,19 +35,19 @@ jobs: TOXENV: pypy3 # pinned deps - - python-version: 3.8.17 + - python-version: 3.9.19 env: TOXENV: pinned - - python-version: 3.8.17 + - python-version: 3.9.19 env: TOXENV: asyncio-pinned - - python-version: pypy3.8 + - python-version: pypy3.9 env: TOXENV: pypy3-pinned - - python-version: 3.8.17 + - python-version: 3.9.19 env: TOXENV: extra-deps-pinned - - python-version: 3.8.17 + - python-version: 3.9.19 env: TOXENV: botocore-pinned diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 80d09e7a0..5728c6fd0 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -12,12 +12,9 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.8 + - python-version: "3.9" env: TOXENV: windows-pinned - - python-version: 3.9 - env: - TOXENV: py - python-version: "3.10" env: TOXENV: py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index addad838f..75529be05 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,4 +33,4 @@ repos: rev: v3.16.0 hooks: - id: pyupgrade - args: [--py38-plus, --keep-runtime-typing] + args: [--py39-plus, --keep-runtime-typing] diff --git a/README.rst b/README.rst index 14adff648..e640bce35 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ including a list of features. Requirements ============ -* Python 3.8+ +* Python 3.9+ * Works on Linux, Windows, macOS, BSD Install diff --git a/docs/intro/install.rst b/docs/intro/install.rst index e6c9a683b..ef541368a 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,7 +9,7 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.8+, either the CPython implementation (default) or +Scrapy requires Python 3.9+, either the CPython implementation (default) or the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: diff --git a/scrapy/addons.py b/scrapy/addons.py index f9ec58cea..7a1da3afc 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any from scrapy.exceptions import NotConfigured from scrapy.utils.conf import build_component_list @@ -20,7 +20,7 @@ class AddonManager: def __init__(self, crawler: Crawler) -> None: self.crawler: Crawler = crawler - self.addons: List[Any] = [] + self.addons: list[Any] = [] def load_settings(self, settings: Settings) -> None: """Load add-ons and configurations from a settings object and apply them. diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index e010b159a..b820eb7f9 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,7 +6,7 @@ import inspect import os import sys from importlib.metadata import entry_points -from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple, Type +from typing import TYPE_CHECKING, Optional import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -17,6 +17,8 @@ from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect if TYPE_CHECKING: + from collections.abc import Callable, Iterable + # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec @@ -28,7 +30,7 @@ if TYPE_CHECKING: class ScrapyArgumentParser(argparse.ArgumentParser): def _parse_optional( self, arg_string: str - ) -> Optional[Tuple[Optional[argparse.Action], str, Optional[str]]]: + ) -> Optional[tuple[Optional[argparse.Action], str, Optional[str]]]: # if starts with -: it means that is a parameter not a argument if arg_string[:2] == "-:": return None @@ -36,7 +38,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser): return super()._parse_optional(arg_string) -def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]: +def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]: # TODO: add `name` attribute to commands and merge this function with # scrapy.utils.spider.iter_spider_classes for module in walk_modules(module_name): @@ -50,8 +52,8 @@ def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]: yield obj -def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyCommand]: - d: Dict[str, ScrapyCommand] = {} +def _get_commands_from_module(module: str, inproject: bool) -> dict[str, ScrapyCommand]: + d: dict[str, ScrapyCommand] = {} for cmd in _iter_command_classes(module): if inproject or not cmd.requires_project: cmdname = cmd.__module__.split(".")[-1] @@ -61,8 +63,8 @@ def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyC def _get_commands_from_entry_points( inproject: bool, group: str = "scrapy.commands" -) -> Dict[str, ScrapyCommand]: - cmds: Dict[str, ScrapyCommand] = {} +) -> dict[str, ScrapyCommand]: + cmds: dict[str, ScrapyCommand] = {} if sys.version_info >= (3, 10): eps = entry_points(group=group) else: @@ -78,7 +80,7 @@ def _get_commands_from_entry_points( def _get_commands_dict( settings: BaseSettings, inproject: bool -) -> Dict[str, ScrapyCommand]: +) -> dict[str, ScrapyCommand]: cmds = _get_commands_from_module("scrapy.commands", inproject) cmds.update(_get_commands_from_entry_points(inproject)) cmds_module = settings["COMMANDS_MODULE"] @@ -87,7 +89,7 @@ def _get_commands_dict( return cmds -def _pop_command_name(argv: List[str]) -> Optional[str]: +def _pop_command_name(argv: list[str]) -> Optional[str]: i = 0 for arg in argv[1:]: if not arg.startswith("-"): @@ -146,7 +148,7 @@ def _run_print_help( def execute( - argv: Optional[List[str]] = None, settings: Optional[Settings] = None + argv: Optional[list[str]] = None, settings: Optional[Settings] = None ) -> None: if argv is None: argv = sys.argv @@ -189,7 +191,7 @@ def execute( sys.exit(cmd.exitcode) -def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) -> None: +def _run_command(cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace) -> None: if opts.profile: _run_command_profiled(cmd, args, opts) else: @@ -197,7 +199,7 @@ def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) def _run_command_profiled( - cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace + cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace ) -> None: if opts.profile: sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 0322390e5..a94db90b1 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -8,7 +8,7 @@ import argparse import builtins import os from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Optional from twisted.python import failure @@ -16,6 +16,8 @@ from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli if TYPE_CHECKING: + from collections.abc import Iterable + from scrapy.crawler import Crawler, CrawlerProcess @@ -24,7 +26,7 @@ class ScrapyCommand: crawler_process: Optional[CrawlerProcess] = None # default settings to be used for this command instead of global defaults - default_settings: Dict[str, Any] = {} + default_settings: dict[str, Any] = {} exitcode: int = 0 @@ -97,7 +99,7 @@ class ScrapyCommand: ) group.add_argument("--pdb", action="store_true", help="enable pdb on failure") - def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: @@ -122,7 +124,7 @@ class ScrapyCommand: if opts.pdb: failure.startDebugMode() - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: """ Entry point for running commands """ @@ -167,7 +169,7 @@ class BaseRunSpiderCommand(ScrapyCommand): help="format to use for dumping items", ) - def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: super().process_options(args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) @@ -207,7 +209,7 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): parts = self.format_part_strings(builtins.list(part_strings)) return super()._join_parts(parts) - def format_part_strings(self, part_strings: List[str]) -> List[str]: + def format_part_strings(self, part_strings: list[str]) -> list[str]: """ Underline and title case command line help message headers. """ diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index f91fec57e..4f6933006 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -4,7 +4,7 @@ import argparse import subprocess # nosec import sys import time -from typing import TYPE_CHECKING, Any, Iterable, List +from typing import TYPE_CHECKING, Any from urllib.parse import urlencode import scrapy @@ -13,6 +13,8 @@ from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor if TYPE_CHECKING: + from collections.abc import Iterable + from scrapy import Request @@ -26,7 +28,7 @@ class Command(ScrapyCommand): def short_desc(self) -> str: return "Run quick benchmark test" - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: with _BenchServer(): assert self.crawler_process self.crawler_process.crawl(_BenchSpider, total=100000) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 22c8abf7a..c7946605b 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,7 +1,6 @@ import argparse import time from collections import defaultdict -from typing import List from unittest import TextTestResult as _TextTestResult from unittest import TextTestRunner @@ -69,7 +68,7 @@ class Command(ScrapyCommand): help="print contract tests for all spiders", ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: # load contracts contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) conman = ContractsManager(load_object(c) for c in contracts) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index fe1864372..6b6a80bb5 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List, cast +from typing import TYPE_CHECKING, cast from twisted.python.failure import Failure @@ -20,7 +20,7 @@ class Command(BaseRunSpiderCommand): def short_desc(self) -> str: return "Run a spider" - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: raise UsageError() elif len(args) > 1: diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 04012bee8..34313d731 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,7 +1,6 @@ import argparse import os import sys -from typing import List from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError @@ -27,7 +26,7 @@ class Command(ScrapyCommand): sys.stderr.write(msg + os.linesep) self.exitcode = 1 - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 0bdc429da..a1806f626 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,7 +1,7 @@ from __future__ import annotations import sys -from typing import TYPE_CHECKING, Dict, List, Type +from typing import TYPE_CHECKING from w3lib.url import is_url @@ -48,7 +48,7 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def _print_headers(self, headers: Dict[bytes, List[bytes]], prefix: bytes) -> None: + def _print_headers(self, headers: dict[bytes, list[bytes]], prefix: bytes) -> None: for key, values in headers.items(): for value in values: self._print_bytes(prefix + b" " + key + b": " + value) @@ -65,7 +65,7 @@ class Command(ScrapyCommand): def _print_bytes(self, bytes_: bytes) -> None: sys.stdout.buffer.write(bytes_ + b"\n") - def run(self, args: List[str], opts: Namespace) -> None: + def run(self, args: list[str], opts: Namespace) -> None: if len(args) != 1 or not is_url(args[0]): raise UsageError() request = Request( @@ -81,7 +81,7 @@ class Command(ScrapyCommand): else: request.meta["handle_httpstatus_all"] = True - spidercls: Type[Spider] = DefaultSpider + spidercls: type[Spider] = DefaultSpider assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 6c3713f8f..a9b7a6eee 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,7 +4,7 @@ import shutil import string from importlib import import_module from pathlib import Path -from typing import List, Optional, Union, cast +from typing import Optional, Union, cast from urllib.parse import urlparse import scrapy @@ -87,7 +87,7 @@ class Command(ScrapyCommand): help="If the spider already exists, overwrite it with the template", ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if opts.list: self._list_templates() return diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 10330c92a..3b2f127c2 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from scrapy.commands import ScrapyCommand @@ -15,7 +15,7 @@ class Command(ScrapyCommand): def short_desc(self) -> str: return "List available spiders" - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: assert self.crawler_process for s in sorted(self.crawler_process.spider_loader.list()): print(s) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index fbd200d88..bd1fad14b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -5,20 +5,7 @@ import functools import inspect import json import logging -from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, - Coroutine, - Dict, - Iterable, - List, - Optional, - Tuple, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload from itemadapter import ItemAdapter, is_item from twisted.internet.defer import Deferred, maybeDeferred @@ -35,6 +22,8 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Coroutine, Iterable + from twisted.python.failure import Failure from scrapy.http.request import CallbackT @@ -50,8 +39,8 @@ class Command(BaseRunSpiderCommand): requires_project = True spider = None - items: Dict[int, List[Any]] = {} - requests: Dict[int, List[Request]] = {} + items: dict[int, list[Any]] = {} + requests: dict[int, list[Request]] = {} first_response = None @@ -166,11 +155,11 @@ class Command(BaseRunSpiderCommand): return d return arg_to_iter(deferred_from_coro(result)) - def add_items(self, lvl: int, new_items: List[Any]) -> None: + def add_items(self, lvl: int, new_items: list[Any]) -> None: old_items = self.items.get(lvl, []) self.items[lvl] = old_items + new_items - def add_requests(self, lvl: int, new_reqs: List[Request]) -> None: + def add_requests(self, lvl: int, new_reqs: list[Request]) -> None: old_reqs = self.requests.get(lvl, []) self.requests[lvl] = old_reqs + new_reqs @@ -219,7 +208,7 @@ class Command(BaseRunSpiderCommand): depth: int, spider: Spider, callback: CallbackT, - ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT]: + ) -> tuple[list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT]: items, requests = [], [] for x in spider_output: if is_item(x): @@ -232,7 +221,7 @@ class Command(BaseRunSpiderCommand): self, response: Response, callback: CallbackT, - cb_kwargs: Optional[Dict[str, Any]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) @@ -285,10 +274,10 @@ class Command(BaseRunSpiderCommand): def scraped_data( self, - args: Tuple[ - List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT + args: tuple[ + list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT ], - ) -> List[Any]: + ) -> list[Any]: items, requests, opts, depth, spider, callback = args if opts.pipelines: itemproc = self.pcrawler.engine.scraper.itemproc @@ -345,7 +334,7 @@ class Command(BaseRunSpiderCommand): def prepare_request( self, spider: Spider, request: Request, opts: argparse.Namespace ) -> Request: - def callback(response: Response, **cb_kwargs: Any) -> Deferred[List[Any]]: + def callback(response: Response, **cb_kwargs: Any) -> Deferred[list[Any]]: # memorize first request if not self.first_response: self.first_response = response @@ -376,7 +365,7 @@ class Command(BaseRunSpiderCommand): request.callback = callback return request - def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + def process_options(self, args: list[str], opts: argparse.Namespace) -> None: super().process_options(args, opts) self.process_request_meta(opts) @@ -404,7 +393,7 @@ class Command(BaseRunSpiderCommand): print_help=False, ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: # parse arguments if not len(args) == 1 or not is_url(args[0]): raise UsageError() diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 87acf9a01..14d58f311 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -4,7 +4,7 @@ import argparse import sys from importlib import import_module from pathlib import Path -from typing import TYPE_CHECKING, List, Union +from typing import TYPE_CHECKING, Union from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -41,7 +41,7 @@ class Command(BaseRunSpiderCommand): def long_desc(self) -> str: return "Run the spider defined in the given file" - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() filename = Path(args[0]) diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index dbda73b44..59f86b9a7 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,6 +1,5 @@ import argparse import json -from typing import List from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings @@ -46,7 +45,7 @@ class Command(ScrapyCommand): help="print setting value, interpreted as a list", ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: assert self.crawler_process settings = self.crawler_process.settings if opts.get: diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index f03cf997a..27e6d68ee 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -7,7 +7,7 @@ See documentation in docs/topics/shell.rst from __future__ import annotations from threading import Thread -from typing import TYPE_CHECKING, Any, Dict, List, Type +from typing import TYPE_CHECKING, Any from scrapy import Spider from scrapy.commands import ScrapyCommand @@ -56,13 +56,13 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def update_vars(self, vars: Dict[str, Any]) -> None: + def update_vars(self, vars: dict[str, Any]) -> None: """You can use this function to update the Scrapy objects that will be available in the shell """ pass - def run(self, args: List[str], opts: Namespace) -> None: + def run(self, args: list[str], opts: Namespace) -> None: url = args[0] if args else None if url: # first argument may be a local file @@ -71,7 +71,7 @@ class Command(ScrapyCommand): assert self.crawler_process spider_loader = self.crawler_process.spider_loader - spidercls: Type[Spider] = DefaultSpider + spidercls: type[Spider] = DefaultSpider if opts.spider: spidercls = spider_loader.load(opts.spider) elif url: diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 58c1aa28f..f7052cd18 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -6,14 +6,14 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION -from typing import List, Tuple, Union +from typing import Union import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase -TEMPLATES_TO_RENDER: Tuple[Tuple[str, ...], ...] = ( +TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( ("scrapy.cfg",), ("${project_name}", "settings.py.tmpl"), ("${project_name}", "items.py.tmpl"), @@ -86,7 +86,7 @@ class Command(ScrapyCommand): copystat(src, dst) _make_writable(dst) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) not in (1, 2): raise UsageError() @@ -107,9 +107,7 @@ class Command(ScrapyCommand): return self._copytree(Path(self.templates_dir), project_dir.resolve()) - # On 3.8 shutil.move doesn't fully support Path args, but it supports our use case - # See https://bugs.python.org/issue32689 - move(project_dir / "module", project_dir / project_name) # type: ignore[arg-type] + move(project_dir / "module", project_dir / project_name) for paths in TEMPLATES_TO_RENDER: tplfile = Path( project_dir, diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index f057e8544..571f4fda8 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,5 +1,4 @@ import argparse -from typing import List import scrapy from scrapy.commands import ScrapyCommand @@ -25,7 +24,7 @@ class Command(ScrapyCommand): help="also display twisted/python/platform info (useful for bug reports)", ) - def run(self, args: List[str], opts: argparse.Namespace) -> None: + def run(self, args: list[str], opts: argparse.Namespace) -> None: if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index a7e129948..ffe5053de 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -2,22 +2,11 @@ from __future__ import annotations import re import sys +from collections.abc import AsyncGenerator, Iterable from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, - Callable, - Dict, - Iterable, - List, - Optional, - Tuple, - Type, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, cast from unittest import TestCase, TestResult from scrapy.http import Request, Response @@ -25,6 +14,8 @@ from scrapy.utils.python import get_spec from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: + from collections.abc import Callable + from twisted.python.failure import Failure from scrapy import Spider @@ -33,13 +24,13 @@ if TYPE_CHECKING: class Contract: """Abstract class for contracts""" - request_cls: Optional[Type[Request]] = None + request_cls: Optional[type[Request]] = None name: str def __init__(self, method: Callable, *args: Any): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") self.testcase_post = _create_testcase(method, f"@{self.name} post-hook") - self.args: Tuple[Any, ...] = args + self.args: tuple[Any, ...] = args def add_pre_hook(self, request: Request, results: TestResult) -> Request: if hasattr(self, "pre_process"): @@ -47,7 +38,7 @@ class Contract: assert cb is not None @wraps(cb) - def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]: + def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: try: results.startTest(self.testcase_pre) self.pre_process(response) @@ -76,7 +67,7 @@ class Contract: assert cb is not None @wraps(cb) - def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]: + def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") @@ -98,18 +89,18 @@ class Contract: return request - def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: return args class ContractsManager: - contracts: Dict[str, Type[Contract]] = {} + contracts: dict[str, type[Contract]] = {} - def __init__(self, contracts: Iterable[Type[Contract]]): + def __init__(self, contracts: Iterable[type[Contract]]): for contract in contracts: self.contracts[contract.name] = contract - def tested_methods_from_spidercls(self, spidercls: Type[Spider]) -> List[str]: + def tested_methods_from_spidercls(self, spidercls: type[Spider]) -> list[str]: is_method = re.compile(r"^\s*@", re.MULTILINE).search methods = [] for key, value in getmembers(spidercls): @@ -118,8 +109,8 @@ class ContractsManager: return methods - def extract_contracts(self, method: Callable) -> List[Contract]: - contracts: List[Contract] = [] + def extract_contracts(self, method: Callable) -> list[Contract]: + contracts: list[Contract] = [] assert method.__doc__ is not None for line in method.__doc__.split("\n"): line = line.strip() @@ -137,8 +128,8 @@ class ContractsManager: def from_spider( self, spider: Spider, results: TestResult - ) -> List[Optional[Request]]: - requests: List[Optional[Request]] = [] + ) -> list[Optional[Request]]: + requests: list[Optional[Request]] = [] for method in self.tested_methods_from_spidercls(type(spider)): bound_method = spider.__getattribute__(method) try: diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index e7b11d426..87170d3c1 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,5 +1,5 @@ import json -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Optional from itemadapter import ItemAdapter, is_item @@ -16,7 +16,7 @@ class UrlContract(Contract): name = "url" - def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: args["url"] = self.args[0] return args @@ -30,7 +30,7 @@ class CallbackKeywordArgumentsContract(Contract): name = "cb_kwargs" - def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: args["cb_kwargs"] = json.loads(" ".join(self.args)) return args @@ -44,7 +44,7 @@ class MetadataContract(Contract): name = "meta" - def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]: + def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: args["meta"] = json.loads(" ".join(self.args)) return args @@ -63,7 +63,7 @@ class ReturnsContract(Contract): """ name = "returns" - object_type_verifiers: Dict[Optional[str], Callable[[Any], bool]] = { + object_type_verifiers: dict[Optional[str], Callable[[Any], bool]] = { "request": lambda x: isinstance(x, Request), "requests": lambda x: isinstance(x, Request), "item": is_item, @@ -90,7 +90,7 @@ class ReturnsContract(Contract): except IndexError: self.max_bound = float("inf") - def post_process(self, output: List[Any]) -> None: + def post_process(self, output: list[Any]) -> None: occurrences = 0 for x in output: if self.obj_type_verifier(x): @@ -116,7 +116,7 @@ class ScrapesContract(Contract): name = "scrapes" - def post_process(self, output: List[Any]) -> None: + def post_process(self, output: list[Any]) -> None: for x in output: if is_item(x): missing = [arg for arg in self.args if arg not in ItemAdapter(x)] diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 6786d7acf..77d57a8d8 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -5,18 +5,7 @@ import warnings from collections import deque from datetime import datetime from time import time -from typing import ( - TYPE_CHECKING, - Any, - Deque, - Dict, - Optional, - Set, - Tuple, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from twisted.internet import task from twisted.internet.defer import Deferred @@ -55,9 +44,9 @@ class Slot: self.randomize_delay: bool = randomize_delay self.throttle = throttle - self.active: Set[Request] = set() - self.queue: Deque[Tuple[Request, Deferred[Response]]] = deque() - self.transferring: Set[Request] = set() + self.active: set[Request] = set() + self.queue: deque[tuple[Request, Deferred[Response]]] = deque() + self.transferring: set[Request] = set() self.lastseen: float = 0 self.latercall = None @@ -95,7 +84,7 @@ class Slot: def _get_concurrency_delay( concurrency: int, spider: Spider, settings: BaseSettings -) -> Tuple[int, float]: +) -> tuple[int, float]: delay: float = settings.getfloat("DOWNLOAD_DELAY") if hasattr(spider, "download_delay"): delay = spider.download_delay @@ -112,8 +101,8 @@ class Downloader: def __init__(self, crawler: Crawler): self.settings: BaseSettings = crawler.settings self.signals: SignalManager = crawler.signals - self.slots: Dict[str, Slot] = {} - self.active: Set[Request] = set() + self.slots: dict[str, Slot] = {} + self.active: set[Request] = set() self.handlers: DownloadHandlers = DownloadHandlers(crawler) self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS") self.domain_concurrency: int = self.settings.getint( @@ -126,7 +115,7 @@ class Downloader: ) self._slot_gc_loop: task.LoopingCall = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) - self.per_slot_settings: Dict[str, Dict[str, Any]] = self.settings.getdict( + self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( "DOWNLOAD_SLOTS", {} ) @@ -146,7 +135,7 @@ class Downloader: def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency - def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]: + def _get_slot(self, request: Request, spider: Spider) -> tuple[str, Slot]: key = self.get_slot_key(request) if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 2b388a9f5..ba20c3c2c 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, List, Optional +from typing import TYPE_CHECKING, Any, Optional from OpenSSL import SSL from twisted.internet._sslverify import _setAcceptableProtocols @@ -154,10 +154,10 @@ class AcceptableProtocolsContextFactory: negotiation. """ - def __init__(self, context_factory: Any, acceptable_protocols: List[bytes]): + def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]): verifyObject(IPolicyForHTTPS, context_factory) self._wrapped_context_factory: Any = context_factory - self._acceptable_protocols: List[bytes] = acceptable_protocols + self._acceptable_protocols: list[bytes] = acceptable_protocols def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc( diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 70d356b83..c39e480f1 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -3,18 +3,8 @@ from __future__ import annotations import logging -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Generator, - Optional, - Protocol, - Type, - Union, - cast, -) +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Optional, Protocol, Union, cast from twisted.internet import defer @@ -25,6 +15,8 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from collections.abc import Generator + from twisted.internet.defer import Deferred from scrapy.crawler import Crawler @@ -43,16 +35,16 @@ class DownloadHandlerProtocol(Protocol): class DownloadHandlers: def __init__(self, crawler: Crawler): self._crawler: Crawler = crawler - self._schemes: Dict[str, Union[str, Callable[..., Any]]] = ( + self._schemes: dict[str, Union[str, Callable[..., Any]]] = ( {} ) # stores acceptable schemes on instancing - self._handlers: Dict[str, DownloadHandlerProtocol] = ( + self._handlers: dict[str, DownloadHandlerProtocol] = ( {} ) # stores instanced handlers for schemes - self._notconfigured: Dict[str, str] = {} # remembers failed handlers - handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values( + self._notconfigured: dict[str, str] = {} # remembers failed handlers + handlers: dict[str, Union[str, Callable[..., Any]]] = without_none_values( cast( - Dict[str, Union[str, Callable[..., Any]]], + dict[str, Union[str, Callable[..., Any]]], crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), ) ) @@ -81,7 +73,7 @@ class DownloadHandlers: ) -> Optional[DownloadHandlerProtocol]: path = self._schemes[scheme] try: - dhcls: Type[DownloadHandlerProtocol] = load_object(path) + dhcls: type[DownloadHandlerProtocol] = load_object(path) if skip_lazy and getattr(dhcls, "lazy", True): return None dh = build_from_crawler( diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index bf6879521..b3f286d87 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict +from typing import TYPE_CHECKING, Any from w3lib.url import parse_data_uri @@ -20,7 +20,7 @@ class DataURIDownloadHandler: uri = parse_data_uri(request.url) respcls = responsetypes.from_mimetype(uri.media_type) - resp_kwargs: Dict[str, Any] = {} + resp_kwargs: dict[str, Any] = {} if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text": charset = uri.media_type_parameters.get("charset") resp_kwargs["encoding"] = charset diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 69c2d88e1..bc06c7ef4 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -32,7 +32,7 @@ from __future__ import annotations import re from io import BytesIO -from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional +from typing import TYPE_CHECKING, Any, BinaryIO, Optional from urllib.parse import unquote from twisted.internet.protocol import ClientCreator, Protocol @@ -79,7 +79,7 @@ _CODE_RE = re.compile(r"\d+") class FTPDownloadHandler: lazy = False - CODE_MAPPING: Dict[str, int] = { + CODE_MAPPING: dict[str, int] = { "550": 404, "default": 503, } diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 8d7b0635c..58f7ad577 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,9 +1,8 @@ -"""Download handlers for http and https schemes -""" +"""Download handlers for http and https schemes""" from __future__ import annotations -from typing import TYPE_CHECKING, Type +from typing import TYPE_CHECKING from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode @@ -27,10 +26,10 @@ class HTTP10DownloadHandler: lazy = False def __init__(self, settings: BaseSettings, crawler: Crawler): - self.HTTPClientFactory: Type[ScrapyHTTPClientFactory] = load_object( + self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object( settings["DOWNLOADER_HTTPCLIENTFACTORY"] ) - self.ClientContextFactory: Type[ScrapyClientContextFactory] = load_object( + self.ClientContextFactory: type[ScrapyClientContextFactory] = load_object( settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] ) self._settings: BaseSettings = settings diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c06d90f01..f96dc7c98 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar, Union +from typing import TYPE_CHECKING, Any, Optional, TypedDict, TypeVar, Union from urllib.parse import urldefrag, urlunparse from twisted.internet import ssl @@ -52,7 +52,7 @@ _T = TypeVar("_T") class _ResultT(TypedDict): txresponse: TxResponse body: bytes - flags: Optional[List[str]] + flags: Optional[list[str]] certificate: Optional[ssl.Certificate] ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] failure: NotRequired[Optional[Failure]] @@ -143,10 +143,10 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): reactor: ReactorBase, host: str, port: int, - proxyConf: Tuple[str, int, Optional[bytes]], + proxyConf: tuple[str, int, Optional[bytes]], contextFactory: IPolicyForHTTPS, timeout: float = 30, - bindAddress: Optional[Tuple[str, int]] = None, + bindAddress: Optional[tuple[str, int]] = None, ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) @@ -254,14 +254,14 @@ class TunnelingAgent(Agent): self, *, reactor: ReactorBase, - proxyConf: Tuple[str, int, Optional[bytes]], + proxyConf: tuple[str, int, Optional[bytes]], contextFactory: IPolicyForHTTPS, connectTimeout: Optional[float] = None, bindAddress: Optional[bytes] = None, pool: Optional[HTTPConnectionPool] = None, ): super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) - self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf + self._proxyConf: tuple[str, int, Optional[bytes]] = proxyConf self._contextFactory: IPolicyForHTTPS = contextFactory def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint: @@ -621,7 +621,7 @@ class _ResponseReader(Protocol): self._crawler: Crawler = crawler def _finish_response( - self, flags: Optional[List[str]] = None, failure: Optional[Failure] = None + self, flags: Optional[list[str]] = None, failure: Optional[Failure] = None ) -> None: self._finished.callback( { diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index edf370193..fa660c63c 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Type +from typing import TYPE_CHECKING, Any, Optional from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured @@ -29,7 +29,7 @@ class S3DownloadHandler: aws_access_key_id: Optional[str] = None, aws_secret_access_key: Optional[str] = None, aws_session_token: Optional[str] = None, - httpdownloadhandler: Type[HTTPDownloadHandler] = HTTPDownloadHandler, + httpdownloadhandler: type[HTTPDownloadHandler] = HTTPDownloadHandler, **kw: Any, ): if not is_botocore_available(): diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 0bdb756c8..00d3bd1b0 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -6,7 +6,8 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, Callable, Generator, List, Union, cast +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks @@ -17,6 +18,8 @@ from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_from_coro, mustbe_deferred if TYPE_CHECKING: + from collections.abc import Generator + from twisted.python.failure import Failure from scrapy import Spider @@ -27,7 +30,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): component_name = "downloader middleware" @classmethod - def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("DOWNLOADER_MIDDLEWARES")) def _add_middleware(self, mw: Any) -> None: diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 33cea7263..1ae66f614 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict +from typing import Any from OpenSSL import SSL from service_identity.exceptions import CertificateError @@ -21,7 +21,7 @@ METHOD_TLSv11 = "TLSv1.1" METHOD_TLSv12 = "TLSv1.2" -openssl_methods: Dict[str, int] = { +openssl_methods: dict[str, int] = { METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 99502f0d2..509bda4e4 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -2,7 +2,7 @@ from __future__ import annotations import re from time import time -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from twisted.internet import defer @@ -18,7 +18,7 @@ if TYPE_CHECKING: from scrapy import Request -def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]: +def _parsed_url_args(parsed: ParseResult) -> tuple[bytes, bytes, bytes, int, bytes]: # Assume parsed is urlparse-d from Request.url, # which was passed via safe_url_string and is ascii-only. path_str = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, "")) @@ -33,7 +33,7 @@ def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, byt return scheme, netloc, host, port, path -def _parse(url: str) -> Tuple[bytes, bytes, bytes, int, bytes]: +def _parse(url: str) -> tuple[bytes, bytes, bytes, int, bytes]: """Return tuple of (scheme, netloc, host, port, path), all in bytes except for port which is int. Assume url is from Request.url, which was passed via safe_url_string diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index fd9a5f781..bb09d066f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -9,20 +9,7 @@ from __future__ import annotations import logging from time import time -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Generator, - Iterable, - Iterator, - Optional, - Set, - Type, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed @@ -42,6 +29,8 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: + from collections.abc import Callable, Generator, Iterable, Iterator + from scrapy.core.scheduler import BaseScheduler from scrapy.core.scraper import _HandleOutputDeferred from scrapy.crawler import Crawler @@ -63,7 +52,7 @@ class Slot: scheduler: BaseScheduler, ) -> None: self.closing: Optional[Deferred[None]] = None - self.inprogress: Set[Request] = set() + self.inprogress: set[Request] = set() self.start_requests: Optional[Iterator[Request]] = iter(start_requests) self.close_if_idle: bool = close_if_idle self.nextcall: CallLaterOnce[None] = nextcall @@ -106,10 +95,10 @@ class ExecutionEngine: self.spider: Optional[Spider] = None self.running: bool = False self.paused: bool = False - self.scheduler_cls: Type[BaseScheduler] = self._get_scheduler_class( + self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( crawler.settings ) - downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"]) + downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = ( @@ -117,10 +106,10 @@ class ExecutionEngine: ) self.start_time: Optional[float] = None - def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]: + def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler - scheduler_cls: Type[BaseScheduler] = load_object(settings["SCHEDULER"]) + scheduler_cls: type[BaseScheduler] = load_object(settings["SCHEDULER"]) if not issubclass(scheduler_cls, BaseScheduler): raise TypeError( f"The provided scheduler class ({settings['SCHEDULER']})" diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 640fb7129..b5ff55eb0 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections import deque -from typing import TYPE_CHECKING, Deque, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Optional from twisted.internet import defer from twisted.internet.defer import Deferred @@ -26,7 +26,7 @@ if TYPE_CHECKING: from scrapy.spiders import Spider -ConnectionKeyT = Tuple[bytes, bytes, int] +ConnectionKeyT = tuple[bytes, bytes, int] class H2ConnectionPool: @@ -36,11 +36,11 @@ class H2ConnectionPool: # Store a dictionary which is used to get the respective # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port) - self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {} + self._connections: dict[ConnectionKeyT, H2ClientProtocol] = {} # Save all requests that arrive before the connection is established - self._pending_requests: Dict[ - ConnectionKeyT, Deque[Deferred[H2ClientProtocol]] + self._pending_requests: dict[ + ConnectionKeyT, deque[Deferred[H2ClientProtocol]] ] = {} def get_connection( @@ -68,7 +68,7 @@ class H2ConnectionPool: ) -> Deferred[H2ClientProtocol]: self._pending_requests[key] = deque() - conn_lost_deferred: Deferred[List[BaseException]] = Deferred() + conn_lost_deferred: Deferred[list[BaseException]] = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) @@ -94,7 +94,7 @@ class H2ConnectionPool: return conn def _remove_connection( - self, errors: List[BaseException], key: ConnectionKeyT + self, errors: list[BaseException], key: ConnectionKeyT ) -> None: self._connections.pop(key) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 8aebbaab4..618423218 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -4,7 +4,7 @@ import ipaddress import itertools import logging from collections import deque -from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union from h2.config import H2Configuration from h2.connection import H2Connection @@ -91,7 +91,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self, uri: URI, settings: Settings, - conn_lost_deferred: Deferred[List[BaseException]], + conn_lost_deferred: Deferred[list[BaseException]], ) -> None: """ Arguments: @@ -102,7 +102,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): conn_lost_deferred -- Deferred fires with the reason: Failure to notify that connection was lost """ - self._conn_lost_deferred: Deferred[List[BaseException]] = conn_lost_deferred + self._conn_lost_deferred: Deferred[list[BaseException]] = conn_lost_deferred config = H2Configuration(client_side=True, header_encoding="utf-8") self.conn = H2Connection(config=config) @@ -113,19 +113,19 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._stream_id_generator = itertools.count(start=1, step=2) # Streams are stored in a dictionary keyed off their stream IDs - self.streams: Dict[int, Stream] = {} + self.streams: dict[int, Stream] = {} # If requests are received before connection is made we keep # all requests in a pool and send them as the connection is made - self._pending_request_stream_pool: Deque[Stream] = deque() + self._pending_request_stream_pool: deque[Stream] = deque() # Save an instance of errors raised which lead to losing the connection # We pass these instances to the streams ResponseFailed() failure - self._conn_lost_errors: List[BaseException] = [] + self._conn_lost_errors: list[BaseException] = [] # Some meta data of this connection # initialized when connection is successfully made - self.metadata: Dict[str, Any] = { + self.metadata: dict[str, Any] = { # Peer certificate instance "certificate": None, # Address of the server we are connected to which @@ -250,7 +250,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self.conn.initiate_connection() self._write_to_transport() - def _lose_connection_with_error(self, errors: List[BaseException]) -> None: + def _lose_connection_with_error(self, errors: list[BaseException]) -> None: """Helper function to lose the connection with the error sent as a reason""" self._conn_lost_errors += errors @@ -353,7 +353,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): self._pending_request_stream_pool.clear() self.conn.close_connection() - def _handle_events(self, events: List[Event]) -> None: + def _handle_events(self, events: list[Event]) -> None: """Private method which acts as a bridge between the events received from the HTTP/2 data and IH2EventsHandler @@ -442,7 +442,7 @@ class H2ClientFactory(Factory): self, uri: URI, settings: Settings, - conn_lost_deferred: Deferred[List[BaseException]], + conn_lost_deferred: Deferred[list[BaseException]], ) -> None: self.uri = uri self.settings = settings @@ -451,5 +451,5 @@ class H2ClientFactory(Factory): def buildProtocol(self, addr: IAddress) -> H2ClientProtocol: return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) - def acceptableProtocols(self) -> List[bytes]: + def acceptableProtocols(self) -> list[bytes]: return [PROTOCOL_NAME] diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index d8b5cc8eb..51ebdf489 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from enum import Enum from io import BytesIO -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Optional from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -113,7 +113,7 @@ class Stream: # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated - self.metadata: Dict[str, Any] = { + self.metadata: dict[str, Any] = { "request_content_length": ( 0 if self._request.body is None else len(self._request.body) ), @@ -134,7 +134,7 @@ class Stream: # Private variable used to build the response # this response is then converted to appropriate Response class # passed to the response deferred callback - self._response: Dict[str, Any] = { + self._response: dict[str, Any] = { # Data received frame by frame from the server is appended # and passed to the response Deferred when completely received. "body": BytesIO(), @@ -196,7 +196,7 @@ class Stream: == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' ) - def _get_request_headers(self) -> List[Tuple[str, str]]: + def _get_request_headers(self) -> list[tuple[str, str]]: url = urlparse_cached(self._request) path = url.path @@ -349,7 +349,7 @@ class Stream: self._response["flow_controlled_size"], self.stream_id ) - def receive_headers(self, headers: List[HeaderTuple]) -> None: + def receive_headers(self, headers: list[HeaderTuple]) -> None: for name, value in headers: self._response["headers"].appendlist(name, value) @@ -382,7 +382,7 @@ class Stream: def close( self, reason: StreamCloseReason, - errors: Optional[List[BaseException]] = None, + errors: Optional[list[BaseException]] = None, from_protocol: bool = False, ) -> None: """Based on the reason sent we will handle each case.""" diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index d4286c874..ced18fc05 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,7 +4,7 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, List, Optional, Type, cast +from typing import TYPE_CHECKING, Any, Optional, cast # working around https://github.com/sphinx-doc/sphinx/issues/10400 from twisted.internet.defer import Deferred # noqa: TC002 @@ -182,18 +182,18 @@ class Scheduler(BaseScheduler): self, dupefilter: BaseDupeFilter, jobdir: Optional[str] = None, - dqclass: Optional[Type[BaseQueue]] = None, - mqclass: Optional[Type[BaseQueue]] = None, + dqclass: Optional[type[BaseQueue]] = None, + mqclass: Optional[type[BaseQueue]] = None, logunser: bool = False, stats: Optional[StatsCollector] = None, - pqclass: Optional[Type[ScrapyPriorityQueue]] = None, + pqclass: Optional[type[ScrapyPriorityQueue]] = None, crawler: Optional[Crawler] = None, ): self.df: BaseDupeFilter = dupefilter self.dqdir: Optional[str] = self._dqdir(jobdir) - self.pqclass: Optional[Type[ScrapyPriorityQueue]] = pqclass - self.dqclass: Optional[Type[BaseQueue]] = dqclass - self.mqclass: Optional[Type[BaseQueue]] = mqclass + self.pqclass: Optional[type[ScrapyPriorityQueue]] = pqclass + self.dqclass: Optional[type[BaseQueue]] = dqclass + self.mqclass: Optional[type[BaseQueue]] = mqclass self.logunser: bool = logunser self.stats: Optional[StatsCollector] = stats self.crawler: Optional[Crawler] = crawler @@ -364,13 +364,13 @@ class Scheduler(BaseScheduler): return str(dqdir) return None - def _read_dqs_state(self, dqdir: str) -> List[int]: + def _read_dqs_state(self, dqdir: str) -> list[int]: path = Path(dqdir, "active.json") if not path.exists(): return [] with path.open(encoding="utf-8") as f: - return cast(List[int], json.load(f)) + return cast(list[int], json.load(f)) - def _write_dqs_state(self, dqdir: str, state: List[int]) -> None: + def _write_dqs_state(self, dqdir: str, state: list[int]) -> None: with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: json.dump(state, f) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7a51dbeb4..29d7cb0c8 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -5,23 +5,8 @@ from __future__ import annotations import logging from collections import deque -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Deque, - Generator, - Iterable, - Iterator, - List, - Optional, - Set, - Tuple, - Type, - TypeVar, - Union, - cast, -) +from collections.abc import AsyncIterable, Iterator +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks @@ -47,6 +32,8 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: + from collections.abc import Generator, Iterable + from scrapy.crawler import Crawler @@ -54,12 +41,12 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -_ParallelResult = List[Tuple[bool, Iterator[Any]]] +_ParallelResult = list[tuple[bool, Iterator[Any]]] if TYPE_CHECKING: # parameterized Deferreds require Twisted 21.7.0 _HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] - QueueTuple = Tuple[Union[Response, Failure], Request, _HandleOutputDeferred] + QueueTuple = tuple[Union[Response, Failure], Request, _HandleOutputDeferred] class Slot: @@ -69,8 +56,8 @@ class Slot: def __init__(self, max_active_size: int = 5000000): self.max_active_size = max_active_size - self.queue: Deque[QueueTuple] = deque() - self.active: Set[Request] = set() + self.queue: deque[QueueTuple] = deque() + self.active: set[Request] = set() self.active_size: int = 0 self.itemproc_size: int = 0 self.closing: Optional[Deferred[Spider]] = None @@ -113,7 +100,7 @@ class Scraper: self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler( crawler ) - itemproc_cls: Type[ItemPipelineManager] = load_object( + itemproc_cls: type[ItemPipelineManager] = load_object( crawler.settings["ITEM_PROCESSOR"] ) self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c9feac29c..223e4192e 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -7,22 +7,10 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging +from collections.abc import AsyncIterable, Callable, Iterable from inspect import isasyncgenfunction, iscoroutine from itertools import islice -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Callable, - Generator, - Iterable, - List, - Optional, - Tuple, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -42,6 +30,8 @@ from scrapy.utils.defer import ( from scrapy.utils.python import MutableAsyncChain, MutableChain if TYPE_CHECKING: + from collections.abc import Generator + from scrapy.settings import BaseSettings @@ -66,7 +56,7 @@ class SpiderMiddlewareManager(MiddlewareManager): self.downgrade_warning_done = False @classmethod - def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) def _add_middleware(self, mw: Any) -> None: @@ -349,7 +339,7 @@ class SpiderMiddlewareManager(MiddlewareManager): @staticmethod def _get_async_method_pair( mw: Any, methodname: str - ) -> Union[None, Callable, Tuple[Callable, Callable]]: + ) -> Union[None, Callable, tuple[Callable, Callable]]: normal_method: Optional[Callable] = getattr(mw, methodname, None) methodname_async = methodname + "_async" async_method: Optional[Callable] = getattr(mw, methodname_async, None) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ecb0a8150..b0a4932e1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,18 +4,7 @@ import logging import pprint import signal import warnings -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Generator, - Optional, - Set, - Type, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from twisted.internet.defer import ( Deferred, @@ -53,6 +42,8 @@ from scrapy.utils.reactor import ( ) if TYPE_CHECKING: + from collections.abc import Generator + from scrapy.utils.request import RequestFingerprinter @@ -64,8 +55,8 @@ _T = TypeVar("_T") class Crawler: def __init__( self, - spidercls: Type[Spider], - settings: Union[None, Dict[str, Any], Settings] = None, + spidercls: type[Spider], + settings: Union[None, dict[str, Any], Settings] = None, init_reactor: bool = False, ): if isinstance(spidercls, Spider): @@ -74,7 +65,7 @@ class Crawler: if isinstance(settings, dict) or settings is None: settings = Settings(settings) - self.spidercls: Type[Spider] = spidercls + self.spidercls: type[Spider] = spidercls self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) self._update_root_log_handler() @@ -112,7 +103,7 @@ class Crawler: self.__remove_handler = lambda: logging.root.removeHandler(handler) self.signals.connect(self.__remove_handler, signals.engine_stopped) - lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) + lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) self.logformatter = lf_cls.from_crawler(self) self.request_fingerprinter = build_from_crawler( @@ -256,18 +247,18 @@ class CrawlerRunner: verifyClass(ISpiderLoader, loader_cls) return loader_cls.from_settings(settings.frozencopy()) - def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): + def __init__(self, settings: Union[dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings self.spider_loader = self._get_spider_loader(settings) - self._crawlers: Set[Crawler] = set() - self._active: Set[Deferred[None]] = set() + self._crawlers: set[Crawler] = set() + self._active: set[Deferred[None]] = set() self.bootstrap_failed = False def crawl( self, - crawler_or_spidercls: Union[Type[Spider], str, Crawler], + crawler_or_spidercls: Union[type[Spider], str, Crawler], *args: Any, **kwargs: Any, ) -> Deferred[None]: @@ -314,7 +305,7 @@ class CrawlerRunner: return d.addBoth(_done) def create_crawler( - self, crawler_or_spidercls: Union[Type[Spider], str, Crawler] + self, crawler_or_spidercls: Union[type[Spider], str, Crawler] ) -> Crawler: """ Return a :class:`~scrapy.crawler.Crawler` object. @@ -335,11 +326,11 @@ class CrawlerRunner: return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) - def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: + def _create_crawler(self, spidercls: Union[str, type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) # temporary cast until self.spider_loader is typed - return Crawler(cast(Type[Spider], spidercls), self.settings) + return Crawler(cast(type[Spider], spidercls), self.settings) def stop(self) -> Deferred[Any]: """ @@ -387,7 +378,7 @@ class CrawlerProcess(CrawlerRunner): def __init__( self, - settings: Union[Dict[str, Any], Settings, None] = None, + settings: Union[dict[str, Any], Settings, None] = None, install_root_handler: bool = True, ): super().__init__(settings) @@ -416,14 +407,14 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._stop_reactor) - def _create_crawler(self, spidercls: Union[Type[Spider], str]) -> Crawler: + def _create_crawler(self, spidercls: Union[type[Spider], str]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True # temporary cast until self.spider_loader is typed return Crawler( - cast(Type[Spider], spidercls), self.settings, init_reactor=init_reactor + cast(type[Spider], spidercls), self.settings, init_reactor=init_reactor ) def start( diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 23140d263..e384793ee 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from collections import defaultdict -from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union +from typing import TYPE_CHECKING, Any, Optional, Union from tldextract import TLDExtract @@ -13,6 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterable, Sequence from http.cookiejar import Cookie # typing.Self requires Python 3.11 @@ -39,7 +40,7 @@ class CookiesMiddleware: """This middleware enables working with sites that need cookies""" def __init__(self, debug: bool = False): - self.jars: DefaultDict[Any, CookieJar] = defaultdict(CookieJar) + self.jars: defaultdict[Any, CookieJar] = defaultdict(CookieJar) self.debug: bool = debug @classmethod diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 49b9fdc05..312c1e026 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -6,11 +6,13 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Iterable, Tuple, Union +from typing import TYPE_CHECKING, Union from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -20,8 +22,8 @@ if TYPE_CHECKING: class DefaultHeadersMiddleware: - def __init__(self, headers: Iterable[Tuple[str, str]]): - self._headers: Iterable[Tuple[str, str]] = headers + def __init__(self, headers: Iterable[tuple[str, str]]): + self._headers: Iterable[tuple[str, str]] = headers @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 6b0a56f7f..b0cede97d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -3,7 +3,7 @@ from __future__ import annotations import warnings from itertools import chain from logging import getLogger -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union from scrapy import Request, Spider, signals from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -28,7 +28,7 @@ if TYPE_CHECKING: logger = getLogger(__name__) -ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] +ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate"] try: try: @@ -50,7 +50,7 @@ else: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be - sent/received from web sites""" + sent/received from websites""" def __init__( self, @@ -140,7 +140,7 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs: Dict[str, Any] = {"body": decoded_body} + kwargs: dict[str, Any] = {"body": decoded_body} if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable @@ -152,23 +152,23 @@ class HttpCompressionMiddleware: return response def _handle_encoding( - self, body: bytes, content_encoding: List[bytes], max_size: int - ) -> Tuple[bytes, List[bytes]]: + self, body: bytes, content_encoding: list[bytes], max_size: int + ) -> tuple[bytes, list[bytes]]: to_decode, to_keep = self._split_encodings(content_encoding) for encoding in to_decode: body = self._decode(body, encoding, max_size) return body, to_keep def _split_encodings( - self, content_encoding: List[bytes] - ) -> Tuple[List[bytes], List[bytes]]: - to_keep: List[bytes] = [ + self, content_encoding: list[bytes] + ) -> tuple[list[bytes], list[bytes]]: + to_keep: list[bytes] = [ encoding.strip().lower() for encoding in chain.from_iterable( encodings.split(b",") for encodings in content_encoding ) ] - to_decode: List[bytes] = [] + to_decode: list[bytes] = [] while to_keep: encoding = to_keep.pop() if encoding not in ACCEPTED_ENCODINGS: diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index a7af83f7d..b35ecbd54 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,7 +1,7 @@ from __future__ import annotations import base64 -from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional, Union from urllib.parse import unquote, urlunparse from urllib.request import ( # type: ignore[attr-defined] _parse_proxy, @@ -25,7 +25,7 @@ if TYPE_CHECKING: class HttpProxyMiddleware: def __init__(self, auth_encoding: Optional[str] = "latin-1"): self.auth_encoding: Optional[str] = auth_encoding - self.proxies: Dict[str, Tuple[Optional[bytes], str]] = {} + self.proxies: dict[str, tuple[Optional[bytes], str]] = {} for type_, url in getproxies().items(): try: self.proxies[type_] = self._get_proxy(url, type_) @@ -47,7 +47,7 @@ class HttpProxyMiddleware: ) return base64.b64encode(user_pass) - def _get_proxy(self, url: str, orig_type: str) -> Tuple[Optional[bytes], str]: + def _get_proxy(self, url: str, orig_type: str) -> tuple[Optional[bytes], str]: proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 6f67e3975..05ec4cad4 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging import re import warnings -from typing import TYPE_CHECKING, Set +from typing import TYPE_CHECKING from scrapy import Request, Spider, signals from scrapy.exceptions import IgnoreRequest @@ -31,7 +31,7 @@ class OffsiteMiddleware: def __init__(self, stats: StatsCollector): self.stats = stats - self.domains_seen: Set[str] = set() + self.domains_seen: set[str] = set() def spider_opened(self, spider: Spider) -> None: self.host_regex: re.Pattern[str] = self.get_host_regex(spider) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 53081237c..6437485cf 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, List, Union, cast +from typing import TYPE_CHECKING, Any, Union, cast from urllib.parse import urljoin from w3lib.url import safe_url_string @@ -180,7 +180,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): def __init__(self, settings: BaseSettings): super().__init__(settings) - self._ignore_tags: List[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") + self._ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS") self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY") def process_response( diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 8d7b7293c..c32624371 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -7,14 +7,14 @@ RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, -once the spider has finished crawling all regular (non failed) pages. +once the spider has finished crawling all regular (non-failed) pages. """ from __future__ import annotations import warnings from logging import Logger, getLogger -from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, Optional, Union from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.settings import BaseSettings, Settings @@ -35,7 +35,7 @@ if TYPE_CHECKING: retry_logger = getLogger(__name__) -def backwards_compatibility_getattr(self: Any, name: str) -> Tuple[Any, ...]: +def backwards_compatibility_getattr(self: Any, name: str) -> tuple[Any, ...]: if name == "EXCEPTIONS_TO_RETRY": warnings.warn( "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " @@ -60,7 +60,7 @@ def get_retry_request( request: Request, *, spider: Spider, - reason: Union[str, Exception, Type[Exception]] = "unspecified", + reason: Union[str, Exception, type[Exception]] = "unspecified", max_retry_times: Optional[int] = None, priority_adjust: Optional[int] = None, logger: Logger = retry_logger, @@ -187,7 +187,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): def _retry( self, request: Request, - reason: Union[str, Exception, Type[Exception]], + reason: Union[str, Exception, type[Exception]], spider: Spider, ) -> Optional[Request]: max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 73757162f..421c58e68 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, Dict, Optional, TypeVar, Union +from typing import TYPE_CHECKING, Optional, TypeVar, Union from twisted.internet.defer import Deferred, maybeDeferred @@ -45,7 +45,7 @@ class RobotsTxtMiddleware: "ROBOTSTXT_USER_AGENT", None ) self.crawler: Crawler = crawler - self._parsers: Dict[ + self._parsers: dict[ str, Union[RobotParser, Deferred[Optional[RobotParser]], None] ] = {} self._parserimpl: RobotParser = load_object( diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 0faae7b5a..ab5655393 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, List, Tuple, Union +from typing import TYPE_CHECKING, Union from twisted.web import http @@ -19,7 +19,7 @@ if TYPE_CHECKING: def get_header_size( - headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]] + headers: dict[str, Union[list[Union[str, bytes]], tuple[Union[str, bytes], ...]]] ) -> int: size = 0 for key, value in headers.items(): diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 40ea48510..28118977d 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from pathlib import Path -from typing import TYPE_CHECKING, Optional, Set +from typing import TYPE_CHECKING, Optional from scrapy.utils.job import job_dir from scrapy.utils.request import ( @@ -56,7 +56,7 @@ class RFPDupeFilter(BaseDupeFilter): self.fingerprinter: RequestFingerprinterProtocol = ( fingerprinter or RequestFingerprinter() ) - self.fingerprints: Set[str] = set() + self.fingerprints: set[str] = set() self.logdupes = True self.debug = debug self.logger = logging.getLogger(__name__) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index fb4998099..ee0033dfb 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -6,9 +6,10 @@ import csv import marshal import pickle # nosec import pprint +from collections.abc import Callable, Iterable, Mapping from io import BytesIO, TextIOWrapper from json import JSONEncoder -from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union +from typing import Any, Optional, Union from xml.sax.saxutils import XMLGenerator # nosec from xml.sax.xmlreader import AttributesImpl # nosec @@ -32,10 +33,10 @@ __all__ = [ class BaseItemExporter: def __init__(self, *, dont_fail: bool = False, **kwargs: Any): - self._kwargs: Dict[str, Any] = kwargs + self._kwargs: dict[str, Any] = kwargs self._configure(kwargs, dont_fail=dont_fail) - def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: + def _configure(self, options: dict[str, Any], dont_fail: bool = False) -> None: """Configure the exporter by popping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) @@ -66,7 +67,7 @@ class BaseItemExporter: def _get_serialized_fields( self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None - ) -> Iterable[Tuple[str, Any]]: + ) -> Iterable[tuple[str, Any]]: """Return the fields to export as an iterable of tuples (name, serialized_value) """ @@ -339,7 +340,7 @@ class PythonItemExporter(BaseItemExporter): .. _msgpack: https://pypi.org/project/msgpack/ """ - def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: + def _configure(self, options: dict[str, Any], dont_fail: bool = False) -> None: super()._configure(options, dont_fail) if not self.encoding: self.encoding = "utf-8" @@ -363,10 +364,10 @@ class PythonItemExporter(BaseItemExporter): return to_unicode(value, encoding=self.encoding) return value - def _serialize_item(self, item: Any) -> Iterable[Tuple[Union[str, bytes], Any]]: + def _serialize_item(self, item: Any) -> Iterable[tuple[Union[str, bytes], Any]]: for key, value in ItemAdapter(item).items(): yield key, self._serialize_value(value) - def export_item(self, item: Any) -> Dict[Union[str, bytes], Any]: # type: ignore[override] - result: Dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item)) + def export_item(self, item: Any) -> dict[Union[str, bytes], Any]: # type: ignore[override] + result: dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item)) return result diff --git a/scrapy/extension.py b/scrapy/extension.py index 8c81ab356..9f978fa32 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -6,7 +6,7 @@ See documentation in docs/topics/extensions.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list @@ -19,5 +19,5 @@ class ExtensionManager(MiddlewareManager): component_name = "extension" @classmethod - def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: return build_component_list(settings.getwithbase("EXTENSIONS")) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index cef5527b7..dff8bc97e 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -8,7 +8,7 @@ from __future__ import annotations import logging from collections import defaultdict -from typing import TYPE_CHECKING, Any, DefaultDict, Dict +from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured @@ -30,7 +30,7 @@ class CloseSpider: def __init__(self, crawler: Crawler): self.crawler: Crawler = crawler - self.close_on: Dict[str, Any] = { + self.close_on: dict[str, Any] = { "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"), "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), @@ -44,7 +44,7 @@ class CloseSpider: if not any(self.close_on.values()): raise NotConfigured - self.counter: DefaultDict[str, int] = defaultdict(int) + self.counter: defaultdict[str, int] = defaultdict(int) if self.close_on.get("errorcount"): crawler.signals.connect(self.error_count, signal=signals.spider_error) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0d7f5bfd4..b1001dabb 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -10,25 +10,11 @@ import logging import re import sys import warnings +from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Optional, - Protocol, - Tuple, - Type, - TypeVar, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, Optional, Protocol, TypeVar, Union, cast from urllib.parse import unquote, urlparse from twisted.internet.defer import Deferred, DeferredList, maybeDeferred @@ -50,6 +36,8 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: + from collections.abc import Iterable + from _typeshed import OpenBinaryMode from twisted.python.failure import Failure @@ -70,7 +58,7 @@ except ImportError: logger = logging.getLogger(__name__) -UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]] +UriParamsCallableT = Callable[[dict[str, Any], Spider], Optional[dict[str, Any]]] _StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") @@ -79,7 +67,7 @@ def build_storage( builder: Callable[..., _StorageT], uri: str, *args: Any, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, preargs: Iterable[Any] = (), **kwargs: Any, ) -> _StorageT: @@ -96,10 +84,10 @@ class ItemFilter: :type feed_options: dict """ - feed_options: Optional[Dict[str, Any]] - item_classes: Tuple[type, ...] + feed_options: Optional[dict[str, Any]] + item_classes: tuple[type, ...] - def __init__(self, feed_options: Optional[Dict[str, Any]]) -> None: + def __init__(self, feed_options: Optional[dict[str, Any]]) -> None: self.feed_options = feed_options if feed_options is not None: self.item_classes = tuple( @@ -141,7 +129,7 @@ class IFeedStorage(Interface): class FeedStorageProtocol(Protocol): """Reimplementation of ``IFeedStorage`` that can be used in type hints.""" - def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + def __init__(self, uri: str, *, feed_options: Optional[dict[str, Any]] = None): """Initialize the storage with the parameters given in the URI and the feed-specific options (see :setting:`FEEDS`)""" @@ -176,7 +164,7 @@ class StdoutFeedStorage: uri: str, _stdout: Optional[IO[bytes]] = None, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, ): if not _stdout: _stdout = sys.stdout.buffer @@ -198,7 +186,7 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None): + def __init__(self, uri: str, *, feed_options: Optional[dict[str, Any]] = None): self.path: str = file_uri_to_path(uri) feed_options = feed_options or {} self.write_mode: OpenBinaryMode = ( @@ -225,7 +213,7 @@ class S3FeedStorage(BlockingFeedStorage): acl: Optional[str] = None, endpoint_url: Optional[str] = None, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, session_token: Optional[str] = None, region_name: Optional[str] = None, ): @@ -291,7 +279,7 @@ class S3FeedStorage(BlockingFeedStorage): crawler: Crawler, uri: str, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, ) -> Self: return build_storage( cls, @@ -307,7 +295,7 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - kwargs: Dict[str, Any] + kwargs: dict[str, Any] if IS_BOTO3_AVAILABLE: kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} self.s3_client.upload_fileobj( @@ -354,7 +342,7 @@ class FTPFeedStorage(BlockingFeedStorage): uri: str, use_active_mode: bool = False, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, ): u = urlparse(uri) if not u.hostname: @@ -373,7 +361,7 @@ class FTPFeedStorage(BlockingFeedStorage): crawler: Crawler, uri: str, *, - feed_options: Optional[Dict[str, Any]] = None, + feed_options: Optional[dict[str, Any]] = None, ) -> Self: return build_storage( cls, @@ -405,9 +393,9 @@ class FeedSlot: batch_id: int, uri_template: str, filter: ItemFilter, - feed_options: Dict[str, Any], + feed_options: dict[str, Any], spider: Spider, - exporters: Dict[str, Type[BaseItemExporter]], + exporters: dict[str, type[BaseItemExporter]], settings: BaseSettings, crawler: Crawler, ): @@ -422,9 +410,9 @@ class FeedSlot: self.uri: str = uri self.filter: ItemFilter = filter # exporter params - self.feed_options: Dict[str, Any] = feed_options + self.feed_options: dict[str, Any] = feed_options self.spider: Spider = spider - self.exporters: Dict[str, Type[BaseItemExporter]] = exporters + self.exporters: dict[str, type[BaseItemExporter]] = exporters self.settings: BaseSettings = settings self.crawler: Crawler = crawler # flags @@ -460,7 +448,7 @@ class FeedSlot: self._exporting = True def _get_instance( - self, objcls: Type[BaseItemExporter], *args: Any, **kwargs: Any + self, objcls: type[BaseItemExporter], *args: Any, **kwargs: Any ) -> BaseItemExporter: return build_from_crawler(objcls, self.crawler, *args, **kwargs) @@ -483,7 +471,7 @@ _FeedSlot = create_deprecated_class( class FeedExporter: - _pending_deferreds: List[Deferred[None]] = [] + _pending_deferreds: list[Deferred[None]] = [] @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -497,8 +485,8 @@ class FeedExporter: self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.feeds = {} - self.slots: List[FeedSlot] = [] - self.filters: Dict[str, ItemFilter] = {} + self.slots: list[FeedSlot] = [] + self.filters: dict[str, ItemFilter] = {} if not self.settings["FEEDS"] and not self.settings["FEED_URI"]: raise NotConfigured @@ -530,10 +518,10 @@ class FeedExporter: ) self.filters[uri] = self._load_filter(feed_options) - self.storages: Dict[str, Type[FeedStorageProtocol]] = self._load_components( + self.storages: dict[str, type[FeedStorageProtocol]] = self._load_components( "FEED_STORAGES" ) - self.exporters: Dict[str, Type[BaseItemExporter]] = self._load_components( + self.exporters: dict[str, type[BaseItemExporter]] = self._load_components( "FEED_EXPORTERS" ) for uri, feed_options in self.feeds.items(): @@ -631,7 +619,7 @@ class FeedExporter: self, batch_id: int, uri: str, - feed_options: Dict[str, Any], + feed_options: dict[str, Any], spider: Spider, uri_template: str, ) -> FeedSlot: @@ -696,9 +684,9 @@ class FeedExporter: slots.append(slot) self.slots = slots - def _load_components(self, setting_prefix: str) -> Dict[str, Any]: + def _load_components(self, setting_prefix: str) -> dict[str, Any]: conf = without_none_values( - cast(Dict[str, str], self.settings.getwithbase(setting_prefix)) + cast(dict[str, str], self.settings.getwithbase(setting_prefix)) ) d = {} for k, v in conf.items(): @@ -732,7 +720,7 @@ class FeedExporter: return False return True - def _storage_supported(self, uri: str, feed_options: Dict[str, Any]) -> bool: + def _storage_supported(self, uri: str, feed_options: dict[str, Any]) -> bool: scheme = urlparse(uri).scheme if scheme in self.storages or PureWindowsPath(uri).drive: try: @@ -748,7 +736,7 @@ class FeedExporter: return False def _get_storage( - self, uri: str, feed_options: Dict[str, Any] + self, uri: str, feed_options: dict[str, Any] ) -> FeedStorageProtocol: """Fork of create_instance specific to feed storage classes @@ -759,7 +747,7 @@ class FeedExporter: crawler = getattr(self, "crawler", None) def build_instance( - builder: Type[FeedStorageProtocol], *preargs: Any + builder: type[FeedStorageProtocol], *preargs: Any ) -> FeedStorageProtocol: return build_storage( builder, uri, feed_options=feed_options, preargs=preargs @@ -784,7 +772,7 @@ class FeedExporter: spider: Spider, uri_params_function: Union[str, UriParamsCallableT, None], slot: Optional[FeedSlot] = None, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: params = {} for k in dir(spider): params[k] = getattr(spider, k) @@ -800,9 +788,9 @@ class FeedExporter: new_params = uripar_function(params, spider) return new_params if new_params is not None else params - def _load_filter(self, feed_options: Dict[str, Any]) -> ItemFilter: + def _load_filter(self, feed_options: dict[str, Any]) -> ItemFilter: # load the item filter if declared else load the default filter class - item_filter_class: Type[ItemFilter] = load_object( + item_filter_class: type[ItemFilter] = load_object( feed_options.get("item_filter", ItemFilter) ) return item_filter_class(feed_options) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 448d5f1ab..a72f9db51 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -9,7 +9,7 @@ from importlib import import_module from pathlib import Path from time import time from types import ModuleType -from typing import IO, TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast +from typing import IO, TYPE_CHECKING, Any, Optional, Union, cast from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict @@ -22,6 +22,8 @@ from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.request import RequestFingerprinter if TYPE_CHECKING: + from collections.abc import Callable + # typing.Concatenate requires Python 3.10 from typing_extensions import Concatenate @@ -35,8 +37,8 @@ logger = logging.getLogger(__name__) class DummyPolicy: def __init__(self, settings: BaseSettings): - self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") - self.ignore_http_codes: List[int] = [ + self.ignore_schemes: list[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self.ignore_http_codes: list[int] = [ int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES") ] @@ -62,18 +64,18 @@ class RFC2616Policy: def __init__(self, settings: BaseSettings): self.always_store: bool = settings.getbool("HTTPCACHE_ALWAYS_STORE") - self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") + self.ignore_schemes: list[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") self._cc_parsed: WeakKeyDictionary[ - Union[Request, Response], Dict[bytes, Optional[bytes]] + Union[Request, Response], dict[bytes, Optional[bytes]] ] = WeakKeyDictionary() - self.ignore_response_cache_controls: List[bytes] = [ + self.ignore_response_cache_controls: list[bytes] = [ to_bytes(cc) for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS") ] def _parse_cachecontrol( self, r: Union[Request, Response] - ) -> Dict[bytes, Optional[bytes]]: + ) -> dict[bytes, Optional[bytes]]: if r not in self._cc_parsed: cch = r.headers.get(b"Cache-Control", b"") assert cch is not None @@ -189,7 +191,7 @@ class RFC2616Policy: if b"ETag" in cachedresponse.headers: request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"] - def _get_max_age(self, cc: Dict[bytes, Optional[bytes]]) -> Optional[int]: + def _get_max_age(self, cc: dict[bytes, Optional[bytes]]) -> Optional[int]: try: return max(0, int(cc[b"max-age"])) # type: ignore[arg-type] except (KeyError, ValueError): @@ -298,7 +300,7 @@ class DbmCacheStorage: self.db[f"{key}_data"] = pickle.dumps(data, protocol=4) self.db[f"{key}_time"] = str(time()) - def _read_data(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: + def _read_data(self, spider: Spider, request: Request) -> Optional[dict[str, Any]]: key = self._fingerprinter.fingerprint(request).hex() db = self.db tkey = f"{key}_time" @@ -309,7 +311,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return None # expired - return cast(Dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec + return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec class FilesystemCacheStorage: @@ -385,7 +387,7 @@ class FilesystemCacheStorage: key = self._fingerprinter.fingerprint(request).hex() return str(Path(self.cachedir, spider.name, key[0:2], key)) - def _read_meta(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]: + def _read_meta(self, spider: Spider, request: Request) -> Optional[dict[str, Any]]: rpath = Path(self._get_request_path(spider, request)) metapath = rpath / "pickled_meta" if not metapath.exists(): @@ -394,10 +396,10 @@ class FilesystemCacheStorage: if 0 < self.expiration_secs < time() - mtime: return None # expired with self._open(metapath, "rb") as f: - return cast(Dict[str, Any], pickle.load(f)) # nosec + return cast(dict[str, Any], pickle.load(f)) # nosec -def parse_cachecontrol(header: bytes) -> Dict[bytes, Optional[bytes]]: +def parse_cachecontrol(header: bytes) -> dict[bytes, Optional[bytes]]: """Parse Cache-Control header https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index c4f43482d..01484481b 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional, Union from twisted.internet import task @@ -81,7 +81,7 @@ class LogStats: def calculate_final_stats( self, spider: Spider - ) -> Union[Tuple[None, None], Tuple[float, float]]: + ) -> Union[tuple[None, None], tuple[float, float]]: start_time = self.stats.get_value("start_time") finished_time = self.stats.get_value("finished_time") diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 25f63ecc6..73d864d5d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -11,7 +11,7 @@ import socket import sys from importlib import import_module from pprint import pformat -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from twisted.internet import task @@ -42,7 +42,7 @@ class MemoryUsage: self.crawler: Crawler = crawler self.warned: bool = False - self.notify_mails: List[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") + self.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 self.check_interval: float = crawler.settings.getfloat( @@ -66,7 +66,7 @@ class MemoryUsage: def engine_started(self) -> None: assert self.crawler.stats self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) - self.tasks: List[task.LoopingCall] = [] + self.tasks: list[task.LoopingCall] = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) @@ -141,7 +141,7 @@ class MemoryUsage: self.crawler.stats.set_value("memusage/warning_notified", 1) self.warned = True - def _send_report(self, rcpts: List[str], subject: str) -> None: + def _send_report(self, rcpts: list[str], subject: str) -> None: """send notification mail with some additional useful info""" assert self.crawler.engine assert self.crawler.stats diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 80c0a3b26..fba12bec7 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from datetime import datetime, timezone from json import JSONEncoder -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union from twisted.internet import task @@ -29,8 +29,8 @@ class PeriodicLog: self, stats: StatsCollector, interval: float = 60.0, - ext_stats: Dict[str, Any] = {}, - ext_delta: Dict[str, Any] = {}, + ext_stats: dict[str, Any] = {}, + ext_delta: dict[str, Any] = {}, ext_timing_enabled: bool = False, ): self.stats: StatsCollector = stats @@ -39,11 +39,11 @@ class PeriodicLog: self.task: Optional[task.LoopingCall] = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) - self.ext_stats_include: List[str] = ext_stats.get("include", []) - self.ext_stats_exclude: List[str] = ext_stats.get("exclude", []) + self.ext_stats_include: list[str] = ext_stats.get("include", []) + self.ext_stats_exclude: list[str] = ext_stats.get("exclude", []) self.ext_delta_enabled: bool = bool(ext_delta) - self.ext_delta_include: List[str] = ext_delta.get("include", []) - self.ext_delta_exclude: List[str] = ext_delta.get("exclude", []) + self.ext_delta_include: list[str] = ext_delta.get("include", []) + self.ext_delta_exclude: list[str] = ext_delta.get("exclude", []) self.ext_timing_enabled: bool = ext_timing_enabled @classmethod @@ -52,7 +52,7 @@ class PeriodicLog: if not interval: raise NotConfigured try: - ext_stats: Optional[Dict[str, Any]] = crawler.settings.getdict( + ext_stats: Optional[dict[str, Any]] = crawler.settings.getdict( "PERIODIC_LOG_STATS" ) except (TypeError, ValueError): @@ -62,7 +62,7 @@ class PeriodicLog: else None ) try: - ext_delta: Optional[Dict[str, Any]] = crawler.settings.getdict( + ext_delta: Optional[dict[str, Any]] = crawler.settings.getdict( "PERIODIC_LOG_DELTA" ) except (TypeError, ValueError): @@ -93,14 +93,14 @@ class PeriodicLog: def spider_opened(self, spider: Spider) -> None: self.time_prev: datetime = datetime.now(tz=timezone.utc) - self.delta_prev: Dict[str, Union[int, float]] = {} - self.stats_prev: Dict[str, Union[int, float]] = {} + self.delta_prev: dict[str, Union[int, float]] = {} + self.stats_prev: dict[str, Union[int, float]] = {} self.task = task.LoopingCall(self.log) self.task.start(self.interval) def log(self) -> None: - data: Dict[str, Any] = {} + data: dict[str, Any] = {} if self.ext_timing_enabled: data.update(self.log_timing()) if self.ext_delta_enabled: @@ -109,8 +109,8 @@ class PeriodicLog: data.update(self.log_crawler_stats()) logger.info(self.encoder.encode(data)) - def log_delta(self) -> Dict[str, Any]: - num_stats: Dict[str, Union[int, float]] = { + def log_delta(self) -> dict[str, Any]: + num_stats: dict[str, Union[int, float]] = { k: v for k, v in self.stats._stats.items() if isinstance(v, (int, float)) @@ -120,7 +120,7 @@ class PeriodicLog: self.delta_prev = num_stats return {"delta": delta} - def log_timing(self) -> Dict[str, Any]: + def log_timing(self) -> dict[str, Any]: now = datetime.now(tz=timezone.utc) time = { "log_interval": self.interval, @@ -132,7 +132,7 @@ class PeriodicLog: self.time_prev = now return {"time": time} - def log_crawler_stats(self) -> Dict[str, Any]: + def log_crawler_stats(self) -> dict[str, Any]: stats = { k: v for k, v in self.stats._stats.items() @@ -141,7 +141,7 @@ class PeriodicLog: return {"stats": stats} def param_allowed( - self, stat_name: str, include: List[str], exclude: List[str] + self, stat_name: str, include: list[str], exclude: list[str] ) -> bool: if not include and not exclude: return True diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index ac12ad829..16067f82b 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -6,7 +6,7 @@ from bz2 import BZ2File from gzip import GzipFile from io import IOBase from lzma import LZMAFile -from typing import IO, Any, BinaryIO, Dict, List, cast +from typing import IO, Any, BinaryIO, cast from scrapy.utils.misc import load_object @@ -24,7 +24,7 @@ class GzipPlugin: See :py:class:`gzip.GzipFile` for more info about parameters. """ - def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__(self, file: BinaryIO, feed_options: dict[str, Any]) -> None: self.file = file self.feed_options = feed_options compress_level = self.feed_options.get("gzip_compresslevel", 9) @@ -56,7 +56,7 @@ class Bz2Plugin: See :py:class:`bz2.BZ2File` for more info about parameters. """ - def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__(self, file: BinaryIO, feed_options: dict[str, Any]) -> None: self.file = file self.feed_options = feed_options compress_level = self.feed_options.get("bz2_compresslevel", 9) @@ -88,7 +88,7 @@ class LZMAPlugin: See :py:class:`lzma.LZMAFile` for more info about parameters. """ - def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + def __init__(self, file: BinaryIO, feed_options: dict[str, Any]) -> None: self.file = file self.feed_options = feed_options @@ -126,7 +126,7 @@ class PostProcessingManager(IOBase): """ def __init__( - self, plugins: List[Any], file: IO[bytes], feed_options: Dict[str, Any] + self, plugins: list[Any], file: IO[bytes], feed_options: dict[str, Any] ) -> None: self.plugins = self._load_plugins(plugins) self.file = file @@ -156,7 +156,7 @@ class PostProcessingManager(IOBase): def writable(self) -> bool: return True - def _load_plugins(self, plugins: List[Any]) -> List[Any]: + def _load_plugins(self, plugins: list[Any]) -> list[Any]: plugins = [load_object(plugin) for plugin in plugins] return plugins diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index cad607514..c8fefe792 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -6,7 +6,7 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address from __future__ import annotations -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Optional from scrapy import Spider, signals from scrapy.exceptions import NotConfigured @@ -23,14 +23,14 @@ if TYPE_CHECKING: class StatsMailer: - def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender): + def __init__(self, stats: StatsCollector, recipients: list[str], mail: MailSender): self.stats: StatsCollector = stats - self.recipients: List[str] = recipients + self.recipients: list[str] = recipients self.mail: MailSender = mail @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - recipients: List[str] = crawler.settings.getlist("STATSMAILER_RCPTS") + recipients: list[str] = crawler.settings.getlist("STATSMAILER_RCPTS") if not recipients: raise NotConfigured mail: MailSender = MailSender.from_settings(crawler.settings) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index c64a0b417..07dc5880b 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -10,7 +10,7 @@ import binascii import logging import os import pprint -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any from twisted.internet import protocol from twisted.internet.tcp import Port @@ -45,7 +45,7 @@ class TelnetConsole(protocol.ServerFactory): self.crawler: Crawler = crawler self.noisy: bool = False - self.portrange: List[int] = [ + self.portrange: list[int] = [ int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT") ] self.host: str = crawler.settings["TELNETCONSOLE_HOST"] @@ -98,10 +98,10 @@ class TelnetConsole(protocol.ServerFactory): return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal()) - def _get_telnet_vars(self) -> Dict[str, Any]: + def _get_telnet_vars(self) -> dict[str, Any]: # Note: if you add entries here also update topics/telnetconsole.rst assert self.crawler.engine - telnet_vars: Dict[str, Any] = { + telnet_vars: dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, "slot": self.crawler.engine.slot, diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 6ce9ce63a..6b5fd181d 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured @@ -90,7 +90,7 @@ class AutoThrottle: def _get_slot( self, request: Request, spider: Spider - ) -> Tuple[Optional[str], Optional[Slot]]: + ) -> tuple[Optional[str], Optional[Slot]]: key: Optional[str] = request.meta.get("download_slot") if key is None: return None, None diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index cc88a9420..b5388a918 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -5,22 +5,14 @@ import time from http.cookiejar import Cookie from http.cookiejar import CookieJar as _CookieJar from http.cookiejar import CookiePolicy, DefaultCookiePolicy -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterator, - List, - Optional, - Sequence, - Tuple, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, cast from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -83,7 +75,7 @@ class CookieJar: self.jar.clear_expired_cookies() @property - def _cookies(self) -> Dict[str, Dict[str, Dict[str, Cookie]]]: + def _cookies(self) -> dict[str, dict[str, dict[str, Cookie]]]: return self.jar._cookies # type: ignore[attr-defined,no-any-return] def clear_session_cookies(self) -> None: @@ -118,7 +110,7 @@ class CookieJar: self.jar.set_cookie_if_ok(cookie, WrappedRequest(request)) # type: ignore[arg-type] -def potential_domain_matches(domain: str) -> List[str]: +def potential_domain_matches(domain: str) -> list[str]: """Potential domain matches for a cookie >>> potential_domain_matches('www.example.com') @@ -200,7 +192,7 @@ class WrappedRequest: value = self.request.headers.get(name, default) return to_unicode(value, errors="replace") if value is not None else None - def header_items(self) -> List[Tuple[str, List[str]]]: + def header_items(self) -> list[tuple[str, list[str]]]: return [ ( to_unicode(k, errors="replace"), @@ -220,7 +212,7 @@ class WrappedResponse: def info(self) -> Self: return self - def get_all(self, name: str, default: Any = None) -> List[str]: + def get_all(self, name: str, default: Any = None) -> list[str]: return [ to_unicode(v, errors="replace") for v in self.response.headers.getlist(name) ] diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 85b9229d3..1dcbcb966 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,18 +1,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Dict, - Iterable, - List, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, AnyStr, Optional, Union, cast from w3lib.http import headers_dict_to_raw @@ -20,6 +9,8 @@ from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -34,17 +25,17 @@ class Headers(CaselessDict): def __init__( self, - seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, encoding: str = "utf-8", ): self.encoding: str = encoding super().__init__(seq) def update( # type: ignore[override] - self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]] + self, seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]]] ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq - iseq: Dict[bytes, List[bytes]] = {} + iseq: dict[bytes, list[bytes]] = {} for k, v in seq: iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) super().update(iseq) @@ -53,7 +44,7 @@ class Headers(CaselessDict): """Normalize key to bytes""" return self._tobytes(key.title()) - def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> List[bytes]: + def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> list[bytes]: """Normalize values to bytes""" _value: Iterable[_RawValueT] if value is None: @@ -78,19 +69,19 @@ class Headers(CaselessDict): def __getitem__(self, key: AnyStr) -> Optional[bytes]: try: - return cast(List[bytes], super().__getitem__(key))[-1] + return cast(list[bytes], super().__getitem__(key))[-1] except IndexError: return None def get(self, key: AnyStr, def_val: Any = None) -> Optional[bytes]: try: - return cast(List[bytes], super().get(key, def_val))[-1] + return cast(list[bytes], super().get(key, def_val))[-1] except IndexError: return None - def getlist(self, key: AnyStr, def_val: Any = None) -> List[bytes]: + def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]: try: - return cast(List[bytes], super().__getitem__(key)) + return cast(list[bytes], super().__getitem__(key)) except KeyError: if def_val is not None: return self.normvalue(def_val) @@ -109,10 +100,10 @@ class Headers(CaselessDict): lst.extend(self.normvalue(value)) self[key] = lst - def items(self) -> Iterable[Tuple[bytes, List[bytes]]]: # type: ignore[override] + def items(self) -> Iterable[tuple[bytes, list[bytes]]]: # type: ignore[override] return ((k, self.getlist(k)) for k in self.keys()) - def values(self) -> List[Optional[bytes]]: # type: ignore[override] + def values(self) -> list[Optional[bytes]]: # type: ignore[override] return [ self[k] for k in self.keys() # pylint: disable=consider-using-dict-items ] diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 9381a6cb3..aac8d3e50 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -12,14 +12,8 @@ from typing import ( TYPE_CHECKING, Any, AnyStr, - Dict, - Iterable, - List, - Mapping, NoReturn, Optional, - Tuple, - Type, TypedDict, TypeVar, Union, @@ -36,7 +30,7 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Iterable, Mapping from twisted.python.failure import Failure @@ -57,7 +51,7 @@ class VerboseCookie(TypedDict): secure: NotRequired[bool] -CookiesT = Union[Dict[str, str], List[VerboseCookie]] +CookiesT = Union[dict[str, str], list[VerboseCookie]] RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") @@ -92,7 +86,7 @@ class Request(object_ref): executed by the Downloader, thus generating a :class:`Response`. """ - attributes: Tuple[str, ...] = ( + attributes: tuple[str, ...] = ( "url", "callback", "method", @@ -120,16 +114,16 @@ class Request(object_ref): url: str, callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - flags: Optional[List[str]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, + flags: Optional[list[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, ) -> None: self._encoding: str = encoding # this one has to be set first self.method: str = str(method).upper() @@ -152,20 +146,20 @@ class Request(object_ref): self.headers: Headers = Headers(headers or {}, encoding=encoding) self.dont_filter: bool = dont_filter - self._meta: Optional[Dict[str, Any]] = dict(meta) if meta else None - self._cb_kwargs: Optional[Dict[str, Any]] = ( + self._meta: Optional[dict[str, Any]] = dict(meta) if meta else None + self._cb_kwargs: Optional[dict[str, Any]] = ( dict(cb_kwargs) if cb_kwargs else None ) - self.flags: List[str] = [] if flags is None else list(flags) + self.flags: list[str] = [] if flags is None else list(flags) @property - def cb_kwargs(self) -> Dict[str, Any]: + def cb_kwargs(self) -> dict[str, Any]: if self._cb_kwargs is None: self._cb_kwargs = {} return self._cb_kwargs @property - def meta(self) -> Dict[str, Any]: + def meta(self) -> dict[str, Any]: if self._meta is None: self._meta = {} return self._meta @@ -207,14 +201,14 @@ class Request(object_ref): @overload def replace( - self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + self, *args: Any, cls: type[RequestTypeVar], **kwargs: Any ) -> RequestTypeVar: ... @overload def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + self, *args: Any, cls: Optional[type[Request]] = None, **kwargs: Any ) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: @@ -261,7 +255,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]: + def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a8c242e8b..d9c913672 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,17 +7,8 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - List, - Optional, - Tuple, - Union, - cast, -) +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from lxml.html import FormElement # nosec @@ -31,6 +22,7 @@ from scrapy.http.request import Request from scrapy.utils.python import is_listlike, to_bytes if TYPE_CHECKING: + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -38,8 +30,8 @@ if TYPE_CHECKING: FormdataVType = Union[str, Iterable[str]] -FormdataKVType = Tuple[str, FormdataVType] -FormdataType = Optional[Union[Dict[str, FormdataVType], List[FormdataKVType]]] +FormdataKVType = tuple[str, FormdataVType] +FormdataType = Optional[Union[dict[str, FormdataVType], list[FormdataKVType]]] class FormRequest(Request): @@ -74,7 +66,7 @@ class FormRequest(Request): formid: Optional[str] = None, formnumber: int = 0, formdata: FormdataType = None, - clickdata: Optional[Dict[str, Union[str, int]]] = None, + clickdata: Optional[dict[str, Union[str, int]]] = None, dont_click: bool = False, formxpath: Optional[str] = None, formcss: Optional[str] = None, @@ -168,8 +160,8 @@ def _get_inputs( form: FormElement, formdata: FormdataType, dont_click: bool, - clickdata: Optional[Dict[str, Union[str, int]]], -) -> List[FormdataKVType]: + clickdata: Optional[dict[str, Union[str, int]]], +) -> list[FormdataKVType]: """Return a list of key-value pairs for the inputs found in the given form.""" try: formdata_keys = dict(formdata or ()).keys() @@ -187,7 +179,7 @@ def _get_inputs( ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={"re": "http://exslt.org/regular-expressions"}, ) - values: List[FormdataKVType] = [ + values: list[FormdataKVType] = [ (k, "" if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys @@ -205,7 +197,7 @@ def _get_inputs( def _value( ele: Union[InputElement, SelectElement, TextareaElement] -) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: +) -> tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: n = ele.name v = ele.value if ele.tag == "select": @@ -215,7 +207,7 @@ def _value( def _select_value( ele: SelectElement, n: Optional[str], v: Union[None, str, MultipleSelectOptions] -) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: +) -> tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected @@ -226,8 +218,8 @@ def _select_value( def _get_clickable( - clickdata: Optional[Dict[str, Union[str, int]]], form: FormElement -) -> Optional[Tuple[str, str]]: + clickdata: Optional[dict[str, Union[str, int]]], form: FormElement +) -> Optional[tuple[str, str]]: """ Returns the clickable element specified in clickdata, if the latter is given. If not, it returns the first diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 057a4f897..48862534e 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -10,7 +10,7 @@ from __future__ import annotations import copy import json import warnings -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, overload +from typing import TYPE_CHECKING, Any, Optional, overload from scrapy.http.request import Request, RequestTypeVar @@ -20,14 +20,14 @@ if TYPE_CHECKING: class JsonRequest(Request): - attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) + attributes: tuple[str, ...] = Request.attributes + ("dumps_kwargs",) def __init__( - self, *args: Any, dumps_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any + self, *args: Any, dumps_kwargs: Optional[dict[str, Any]] = None, **kwargs: Any ) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault("sort_keys", True) - self._dumps_kwargs: Dict[str, Any] = dumps_kwargs + self._dumps_kwargs: dict[str, Any] = dumps_kwargs body_passed = kwargs.get("body", None) is not None data: Any = kwargs.pop("data", None) @@ -47,19 +47,19 @@ class JsonRequest(Request): ) @property - def dumps_kwargs(self) -> Dict[str, Any]: + def dumps_kwargs(self) -> dict[str, Any]: return self._dumps_kwargs @overload def replace( - self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + self, *args: Any, cls: type[RequestTypeVar], **kwargs: Any ) -> RequestTypeVar: ... @overload def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + self, *args: Any, cls: Optional[type[Request]] = None, **kwargs: Any ) -> Request: body_passed = kwargs.get("body", None) is not None data: Any = kwargs.pop("data", None) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 92e4852b6..c69945e2d 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,22 +7,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Callable, - Dict, - Iterable, - List, - Mapping, - Optional, - Tuple, - Type, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, AnyStr, Optional, TypeVar, Union, overload from urllib.parse import urljoin from scrapy.exceptions import NotSupported @@ -32,6 +17,7 @@ from scrapy.link import Link from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Mapping from ipaddress import IPv4Address, IPv6Address from twisted.internet.ssl import Certificate @@ -52,7 +38,7 @@ class Response(object_ref): downloaded (by the Downloader) and fed to the Spiders for processing. """ - attributes: Tuple[str, ...] = ( + attributes: tuple[str, ...] = ( "url", "status", "headers", @@ -74,9 +60,9 @@ class Response(object_ref): self, url: str, status: int = 200, - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: bytes = b"", - flags: Optional[List[str]] = None, + flags: Optional[list[str]] = None, request: Optional[Request] = None, certificate: Optional[Certificate] = None, ip_address: Union[IPv4Address, IPv6Address, None] = None, @@ -87,13 +73,13 @@ class Response(object_ref): self._set_body(body) self._set_url(url) self.request: Optional[Request] = request - self.flags: List[str] = [] if flags is None else list(flags) + self.flags: list[str] = [] if flags is None else list(flags) self.certificate: Optional[Certificate] = certificate self.ip_address: Union[IPv4Address, IPv6Address, None] = ip_address self.protocol: Optional[str] = protocol @property - def cb_kwargs(self) -> Dict[str, Any]: + def cb_kwargs(self) -> dict[str, Any]: try: return self.request.cb_kwargs # type: ignore[union-attr] except AttributeError: @@ -103,7 +89,7 @@ class Response(object_ref): ) @property - def meta(self) -> Dict[str, Any]: + def meta(self) -> dict[str, Any]: try: return self.request.meta # type: ignore[union-attr] except AttributeError: @@ -149,14 +135,14 @@ class Response(object_ref): @overload def replace( - self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any + self, *args: Any, cls: type[ResponseTypeVar], **kwargs: Any ) -> ResponseTypeVar: ... @overload def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any + self, *args: Any, cls: Optional[type[Response]] = None, **kwargs: Any ) -> Response: """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: @@ -200,16 +186,16 @@ class Response(object_ref): url: Union[str, Link], callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, + flags: Optional[list[str]] = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -253,16 +239,16 @@ class Response(object_ref): urls: Iterable[Union[str, Link]], callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, + flags: Optional[list[str]] = None, ) -> Iterable[Request]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 588695002..680c1f602 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,21 +8,9 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations import json +from collections.abc import Iterable from contextlib import suppress -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Callable, - Dict, - Iterable, - List, - Mapping, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, AnyStr, Optional, Union, cast from urllib.parse import urljoin import parsel @@ -41,6 +29,8 @@ from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: + from collections.abc import Callable, Mapping + from twisted.python.failure import Failure from scrapy.http.request import CallbackT, CookiesT, Request @@ -54,7 +44,7 @@ class TextResponse(Response): _DEFAULT_ENCODING = "ascii" _cached_decoded_json = _NONE - attributes: Tuple[str, ...] = Response.attributes + ("encoding",) + attributes: tuple[str, ...] = Response.attributes + ("encoding",) def __init__(self, *args: Any, **kwargs: Any): self._encoding: Optional[str] = kwargs.pop("encoding", None) @@ -183,16 +173,16 @@ class TextResponse(Response): url: Union[str, Link, parsel.Selector], callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, + flags: Optional[list[str]] = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -236,16 +226,16 @@ class TextResponse(Response): urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None, callback: Optional[CallbackT] = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[CookiesT] = None, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, - flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, + flags: Optional[list[str]] = None, css: Optional[str] = None, xpath: Optional[str] = None, ) -> Iterable[Request]: diff --git a/scrapy/item.py b/scrapy/item.py index 3f93809e7..f77002d18 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -7,27 +7,21 @@ See documentation in docs/topics/item.rst from __future__ import annotations from abc import ABCMeta +from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterator, - KeysView, - MutableMapping, - NoReturn, - Tuple, -) +from typing import TYPE_CHECKING, Any, NoReturn from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + from collections.abc import Iterator, KeysView + # typing.Self requires Python 3.11 from typing_extensions import Self -class Field(Dict[str, Any]): +class Field(dict[str, Any]): """Container of field metadata""" @@ -38,7 +32,7 @@ class ItemMeta(ABCMeta): """ def __new__( - mcs, class_name: str, bases: Tuple[type, ...], attrs: Dict[str, Any] + mcs, class_name: str, bases: tuple[type, ...], attrs: dict[str, Any] ) -> ItemMeta: classcell = attrs.pop("__classcell__", None) new_bases = tuple(base._class for base in bases if hasattr(base, "_class")) @@ -83,10 +77,10 @@ class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks. """ - fields: Dict[str, Field] + fields: dict[str, Field] def __init__(self, *args: Any, **kwargs: Any): - self._values: Dict[str, Any] = {} + self._values: dict[str, Any] = {} if args or kwargs: # avoid creating dict for most common case for k, v in dict(*args, **kwargs).items(): self[k] = v diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index d59005edd..1c7e96ae0 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -6,8 +6,13 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ -import re -from typing import Iterable, Pattern +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + from re import Pattern # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index d27a132b3..73673b1c6 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -6,20 +6,10 @@ from __future__ import annotations import logging import operator +import re +from collections.abc import Callable, Iterable from functools import partial -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Iterable, - List, - Optional, - Pattern, - Set, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urljoin, urlparse from lxml import etree # nosec @@ -28,13 +18,14 @@ from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link -from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re +from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain if TYPE_CHECKING: + from lxml.html import HtmlElement # nosec from scrapy import Selector @@ -98,7 +89,7 @@ class LxmlParserLinkExtractor: def _iter_links( self, document: HtmlElement - ) -> Iterable[Tuple[HtmlElement, str, str]]: + ) -> Iterable[tuple[HtmlElement, str, str]]: for el in document.iter(etree.Element): if not self.scan_tag(_nons(el.tag)): continue @@ -114,8 +105,8 @@ class LxmlParserLinkExtractor: response_url: str, response_encoding: str, base_url: str, - ) -> List[Link]: - links: List[Link] = [] + ) -> list[Link]: + links: list[Link] = [] # hacky way to get the underlying lxml parsed document for el, attr, attr_val in self._iter_links(selector.root): # pseudo lxml.html.HtmlElement.make_links_absolute(base_url) @@ -145,26 +136,26 @@ class LxmlParserLinkExtractor: links.append(link) return self._deduplicate_if_needed(links) - def extract_links(self, response: TextResponse) -> List[Link]: + def extract_links(self, response: TextResponse) -> list[Link]: base_url = get_base_url(response) return self._extract_links( response.selector, response.url, response.encoding, base_url ) - def _process_links(self, links: List[Link]) -> List[Link]: + def _process_links(self, links: list[Link]) -> list[Link]: """Normalize and filter extracted links The subclass should override it if necessary """ return self._deduplicate_if_needed(links) - def _deduplicate_if_needed(self, links: List[Link]) -> List[Link]: + def _deduplicate_if_needed(self, links: list[Link]) -> list[Link]: if self.unique: return unique_list(links, key=self.link_key) return links -_RegexT = Union[str, Pattern[str]] +_RegexT = Union[str, re.Pattern[str]] _RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]] @@ -197,13 +188,13 @@ class LxmlLinkExtractor: strip=strip, canonicalized=not canonicalize, ) - self.allow_res: List[Pattern[str]] = self._compile_regexes(allow) - self.deny_res: List[Pattern[str]] = self._compile_regexes(deny) + self.allow_res: list[re.Pattern[str]] = self._compile_regexes(allow) + self.deny_res: list[re.Pattern[str]] = self._compile_regexes(deny) - self.allow_domains: Set[str] = set(arg_to_iter(allow_domains)) - self.deny_domains: Set[str] = set(arg_to_iter(deny_domains)) + self.allow_domains: set[str] = set(arg_to_iter(allow_domains)) + self.deny_domains: set[str] = set(arg_to_iter(deny_domains)) - self.restrict_xpaths: Tuple[str, ...] = tuple(arg_to_iter(restrict_xpaths)) + self.restrict_xpaths: tuple[str, ...] = tuple(arg_to_iter(restrict_xpaths)) self.restrict_xpaths += tuple( map(self._csstranslator.css_to_xpath, arg_to_iter(restrict_css)) ) @@ -211,11 +202,11 @@ class LxmlLinkExtractor: if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS self.canonicalize: bool = canonicalize - self.deny_extensions: Set[str] = {"." + e for e in arg_to_iter(deny_extensions)} - self.restrict_text: List[Pattern[str]] = self._compile_regexes(restrict_text) + self.deny_extensions: set[str] = {"." + e for e in arg_to_iter(deny_extensions)} + self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text) @staticmethod - def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[Pattern[str]]: + def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> list[re.Pattern[str]]: return [ x if isinstance(x, re.Pattern) else re.compile(x) for x in arg_to_iter(value) @@ -257,7 +248,7 @@ class LxmlLinkExtractor: denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] return any(allowed) and not any(denied) - def _process_links(self, links: List[Link]) -> List[Link]: + def _process_links(self, links: list[Link]) -> list[Link]: links = [x for x in links if self._link_allowed(x)] if self.canonicalize: for link in links: @@ -265,10 +256,10 @@ class LxmlLinkExtractor: links = self.link_extractor._process_links(links) return links - def _extract_links(self, *args: Any, **kwargs: Any) -> List[Link]: + def _extract_links(self, *args: Any, **kwargs: Any) -> list[Link]: return self.link_extractor._extract_links(*args, **kwargs) - def extract_links(self, response: TextResponse) -> List[Link]: + def extract_links(self, response: TextResponse) -> list[Link]: """Returns a list of :class:`~scrapy.link.Link` objects from the specified :class:`response <scrapy.http.Response>`. diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index fea7003e5..2b838d8e2 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging import os -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union +from typing import TYPE_CHECKING, Any, Optional, TypedDict, Union from twisted.python.failure import Failure @@ -31,7 +31,7 @@ DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" class LogFormatterResult(TypedDict): level: int msg: str - args: Union[Dict[str, Any], Tuple[Any, ...]] + args: Union[dict[str, Any], tuple[Any, ...]] class LogFormatter: @@ -181,7 +181,7 @@ class LogFormatter: .. versionadded:: 2.0 """ - args: Dict[str, Any] = {"request": request} + args: dict[str, Any] = {"request": request} if errmsg: msg = DOWNLOADERRORMSG_LONG args["errmsg"] = errmsg diff --git a/scrapy/mail.py b/scrapy/mail.py index c020732f9..f33cf2939 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -14,18 +14,7 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - List, - Optional, - Sequence, - Tuple, - Union, -) +from typing import IO, TYPE_CHECKING, Any, Optional, Union from twisted import version as twisted_version from twisted.internet import ssl @@ -36,6 +25,8 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes if TYPE_CHECKING: + from collections.abc import Callable, Sequence + # imports twisted.internet.reactor from twisted.mail.smtp import ESMTPSenderFactory from twisted.python.failure import Failure @@ -95,11 +86,11 @@ class MailSender: def send( self, - to: Union[str, List[str]], + to: Union[str, list[str]], subject: str, body: str, - cc: Union[str, List[str], None] = None, - attachs: Sequence[Tuple[str, str, IO[Any]]] = (), + cc: Union[str, list[str], None] = None, + attachs: Sequence[tuple[str, str, IO[Any]]] = (), mimetype: str = "text/plain", charset: Optional[str] = None, _callback: Optional[Callable[..., None]] = None, @@ -164,7 +155,7 @@ class MailSender: return dfd def _sent_ok( - self, result: Any, to: List[str], cc: List[str], subject: str, nattachs: int + self, result: Any, to: list[str], cc: list[str], subject: str, nattachs: int ) -> None: logger.info( "Mail sent OK: To=%(mailto)s Cc=%(mailcc)s " @@ -180,8 +171,8 @@ class MailSender: def _sent_failed( self, failure: Failure, - to: List[str], - cc: List[str], + to: list[str], + cc: list[str], subject: str, nattachs: int, ) -> Failure: @@ -200,7 +191,7 @@ class MailSender: ) return failure - def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred[Any]: + def _sendmail(self, to_addrs: list[str], msg: bytes) -> Deferred[Any]: from twisted.internet import reactor msg_io = BytesIO(msg) @@ -218,11 +209,11 @@ class MailSender: return d def _create_sender_factory( - self, to_addrs: List[str], msg: IO[bytes], d: Deferred[Any] + self, to_addrs: list[str], msg: IO[bytes], d: Deferred[Any] ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory - factory_keywords: Dict[str, Any] = { + factory_keywords: dict[str, Any] = { "heloFallback": True, "requireAuthentication": False, "requireTransportSecurity": self.smtptls, diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 2296db90e..825d6b4c8 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -3,26 +3,15 @@ from __future__ import annotations import logging import pprint from collections import defaultdict, deque -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Deque, - Dict, - Iterable, - List, - Optional, - Tuple, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from scrapy.exceptions import NotConfigured from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object if TYPE_CHECKING: + from collections.abc import Callable, Iterable + from twisted.internet.defer import Deferred # typing.Concatenate and typing.ParamSpec require Python 3.10 @@ -51,14 +40,14 @@ class MiddlewareManager: self.middlewares = middlewares # Only process_spider_output and process_spider_exception can be None. # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. - self.methods: Dict[ - str, Deque[Union[None, Callable, Tuple[Callable, Callable]]] + self.methods: dict[ + str, deque[Union[None, Callable, tuple[Callable, Callable]]] ] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @classmethod - def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: raise NotImplementedError @classmethod @@ -107,7 +96,7 @@ class MiddlewareManager: def _process_parallel( self, methodname: str, obj: _T, *args: Any - ) -> Deferred[List[_T2]]: + ) -> Deferred[list[_T2]]: methods = cast( "Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname] ) @@ -119,8 +108,8 @@ class MiddlewareManager: ) return process_chain(methods, obj, *args) - def open_spider(self, spider: Spider) -> Deferred[List[None]]: + def open_spider(self, spider: Spider) -> Deferred[list[None]]: return self._process_parallel("open_spider", spider) - def close_spider(self, spider: Spider) -> Deferred[List[None]]: + def close_spider(self, spider: Spider) -> Deferred[list[None]]: return self._process_parallel("close_spider", spider) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 480a5a58c..01f8bd2c8 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -6,7 +6,7 @@ See documentation in docs/item-pipeline.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list @@ -23,7 +23,7 @@ class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" @classmethod - def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]: + def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: return build_component_list(settings.getwithbase("ITEM_PIPELINES")) def _add_middleware(self, pipe: Any) -> None: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 1a13aeaf2..9314856c1 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -21,15 +21,9 @@ from typing import ( IO, TYPE_CHECKING, Any, - Callable, - DefaultDict, - Dict, - List, NoReturn, Optional, Protocol, - Set, - Type, TypedDict, Union, cast, @@ -53,6 +47,7 @@ from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str if TYPE_CHECKING: + from collections.abc import Callable from os import PathLike from twisted.python.failure import Failure @@ -104,8 +99,8 @@ class FilesStoreProtocol(Protocol): path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> Optional[Deferred[Any]]: ... def stat_file( @@ -120,7 +115,7 @@ class FSFilesStore: basedir = basedir.split("://", 1)[1] self.basedir: str = basedir self._mkdir(Path(self.basedir)) - self.created_directories: DefaultDict[MediaPipeline.SpiderInfo, Set[str]] = ( + self.created_directories: defaultdict[MediaPipeline.SpiderInfo, set[str]] = ( defaultdict(set) ) @@ -129,8 +124,8 @@ class FSFilesStore: path: Union[str, PathLike[str]], buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> None: absolute_path = self._get_filesystem_path(path) self._mkdir(absolute_path.parent, info) @@ -157,7 +152,7 @@ class FSFilesStore: def _mkdir( self, dirname: Path, domain: Optional[MediaPipeline.SpiderInfo] = None ) -> None: - seen: Set[str] = self.created_directories[domain] if domain else set() + seen: set[str] = self.created_directories[domain] if domain else set() if str(dirname) not in seen: if not dirname.exists(): dirname.mkdir(parents=True) @@ -201,7 +196,7 @@ class S3FilesStore: def stat_file( self, path: str, info: MediaPipeline.SpiderInfo ) -> Deferred[StatInfo]: - def _onsuccess(boto_key: Dict[str, Any]) -> StatInfo: + def _onsuccess(boto_key: dict[str, Any]) -> StatInfo: checksum = boto_key["ETag"].strip('"') last_modified = boto_key["LastModified"] modified_stamp = time.mktime(last_modified.timetuple()) @@ -209,10 +204,10 @@ class S3FilesStore: return self._get_boto_key(path).addCallback(_onsuccess) - def _get_boto_key(self, path: str) -> Deferred[Dict[str, Any]]: + def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]: key_name = f"{self.prefix}{path}" return cast( - "Deferred[Dict[str, Any]]", + "Deferred[dict[str, Any]]", deferToThread( self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined] ), @@ -223,8 +218,8 @@ class S3FilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> Deferred[Any]: """Upload file to S3 storage""" key_name = f"{self.prefix}{path}" @@ -242,7 +237,7 @@ class S3FilesStore: **extra, ) - def _headers_to_botocore_kwargs(self, headers: Dict[str, Any]) -> Dict[str, Any]: + def _headers_to_botocore_kwargs(self, headers: dict[str, Any]) -> dict[str, Any]: """Convert headers to botocore keyword arguments.""" # This is required while we need to support both boto and botocore. mapping = CaseInsensitiveDict( @@ -274,7 +269,7 @@ class S3FilesStore: "X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation", } ) - extra: Dict[str, Any] = {} + extra: dict[str, Any] = {} for key, value in headers.items(): try: kwarg = mapping[key] @@ -332,7 +327,7 @@ class GCSFilesStore: deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess), ) - def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str: + def _get_content_type(self, headers: Optional[dict[str, str]]) -> str: if headers and "Content-Type" in headers: return headers["Content-Type"] return "application/octet-stream" @@ -345,8 +340,8 @@ class GCSFilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> Deferred[Any]: blob_path = self._get_blob_path(path) blob = self.bucket.blob(blob_path) @@ -385,8 +380,8 @@ class FTPFilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + meta: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, ) -> Deferred[Any]: path = f"{self.basedir}/{path}" return deferToThread( @@ -443,7 +438,7 @@ class FilesPipeline(MediaPipeline): MEDIA_NAME: str = "file" EXPIRES: int = 90 - STORE_SCHEMES: Dict[str, Type[FilesStoreProtocol]] = { + STORE_SCHEMES: dict[str, type[FilesStoreProtocol]] = { "": FSFilesStore, "file": FSFilesStore, "s3": S3FilesStore, @@ -457,7 +452,7 @@ class FilesPipeline(MediaPipeline): self, store_uri: Union[str, PathLike[str]], download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, Dict[str, Any], None] = None, + settings: Union[Settings, dict[str, Any], None] = None, ): store_uri = _to_string(store_uri) if not store_uri: @@ -486,7 +481,7 @@ class FilesPipeline(MediaPipeline): @classmethod def from_settings(cls, settings: Settings) -> Self: - s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) + s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -496,14 +491,14 @@ class FilesPipeline(MediaPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["FILES_STORE_S3_ACL"] - gcs_store: Type[GCSFilesStore] = cast( - Type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + gcs_store: type[GCSFilesStore] = cast( + type[GCSFilesStore], cls.STORE_SCHEMES["gs"] ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None - ftp_store: Type[FTPFilesStore] = cast( - Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + ftp_store: type[FTPFilesStore] = cast( + type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] @@ -660,7 +655,7 @@ class FilesPipeline(MediaPipeline): # Overridable Interface def get_media_requests( self, item: Any, info: MediaPipeline.SpiderInfo - ) -> List[Request]: + ) -> list[Request]: urls = ItemAdapter(item).get(self.files_urls_field, []) return [Request(u, callback=NO_CALLBACK) for u in urls] @@ -680,7 +675,7 @@ class FilesPipeline(MediaPipeline): return checksum def item_completed( - self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 166f81314..f2fe4396b 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,19 +11,7 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Optional, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from itemadapter import ItemAdapter @@ -42,6 +30,7 @@ from scrapy.settings import Settings from scrapy.utils.python import get_func_args, to_bytes if TYPE_CHECKING: + from collections.abc import Callable, Iterable from os import PathLike from PIL import Image @@ -79,7 +68,7 @@ class ImagesPipeline(FilesPipeline): MIN_WIDTH: int = 0 MIN_HEIGHT: int = 0 EXPIRES: int = 90 - THUMBS: Dict[str, Tuple[int, int]] = {} + THUMBS: dict[str, tuple[int, int]] = {} DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" @@ -87,7 +76,7 @@ class ImagesPipeline(FilesPipeline): self, store_uri: Union[str, PathLike[str]], download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, Dict[str, Any], None] = None, + settings: Union[Settings, dict[str, Any], None] = None, ): try: from PIL import Image @@ -127,7 +116,7 @@ class ImagesPipeline(FilesPipeline): self.min_height: int = settings.getint( resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT ) - self.thumbs: Dict[str, Tuple[int, int]] = settings.get( + self.thumbs: dict[str, tuple[int, int]] = settings.get( resolve("IMAGES_THUMBS"), self.THUMBS ) @@ -135,7 +124,7 @@ class ImagesPipeline(FilesPipeline): @classmethod def from_settings(cls, settings: Settings) -> Self: - s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"]) + s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] @@ -145,14 +134,14 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_VERIFY = settings["AWS_VERIFY"] s3store.POLICY = settings["IMAGES_STORE_S3_ACL"] - gcs_store: Type[GCSFilesStore] = cast( - Type[GCSFilesStore], cls.STORE_SCHEMES["gs"] + gcs_store: type[GCSFilesStore] = cast( + type[GCSFilesStore], cls.STORE_SCHEMES["gs"] ) gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None - ftp_store: Type[FTPFilesStore] = cast( - Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] + ftp_store: type[FTPFilesStore] = cast( + type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] ) ftp_store.FTP_USERNAME = settings["FTP_USER"] ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] @@ -202,7 +191,7 @@ class ImagesPipeline(FilesPipeline): info: MediaPipeline.SpiderInfo, *, item: Any = None, - ) -> Iterable[Tuple[str, Image.Image, BytesIO]]: + ) -> Iterable[tuple[str, Image.Image, BytesIO]]: path = self.file_path(request, response=response, info=info, item=item) orig_image = self._Image.open(BytesIO(response.body)) @@ -246,9 +235,9 @@ class ImagesPipeline(FilesPipeline): def convert_image( self, image: Image.Image, - size: Optional[Tuple[int, int]] = None, + size: Optional[tuple[int, int]] = None, response_body: Optional[BytesIO] = None, - ) -> Tuple[Image.Image, BytesIO]: + ) -> tuple[Image.Image, BytesIO]: if response_body is None: warnings.warn( f"{self.__class__.__name__}.convert_image() method called in a deprecated way, " @@ -288,12 +277,12 @@ class ImagesPipeline(FilesPipeline): def get_media_requests( self, item: Any, info: MediaPipeline.SpiderInfo - ) -> List[Request]: + ) -> list[Request]: urls = ItemAdapter(item).get(self.images_urls_field, []) return [Request(u, callback=NO_CALLBACK) for u in urls] def item_completed( - self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo + self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo ) -> Any: with suppress(KeyError): ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 6bd3ed9b4..b30cf9264 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -7,15 +7,9 @@ from collections import defaultdict from typing import ( TYPE_CHECKING, Any, - Callable, - DefaultDict, - Dict, - List, Literal, NoReturn, Optional, - Set, - Tuple, TypedDict, TypeVar, Union, @@ -33,6 +27,8 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter if TYPE_CHECKING: + from collections.abc import Callable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -52,7 +48,7 @@ class FileInfo(TypedDict): status: str -FileInfoOrError = Union[Tuple[Literal[True], FileInfo], Tuple[Literal[False], Failure]] +FileInfoOrError = Union[tuple[Literal[True], FileInfo], tuple[Literal[False], Failure]] logger = logging.getLogger(__name__) @@ -67,16 +63,16 @@ class MediaPipeline(ABC): class SpiderInfo: def __init__(self, spider: Spider): self.spider: Spider = spider - self.downloading: Set[bytes] = set() - self.downloaded: Dict[bytes, Union[FileInfo, Failure]] = {} - self.waiting: DefaultDict[bytes, List[Deferred[FileInfo]]] = defaultdict( + self.downloading: set[bytes] = set() + self.downloaded: dict[bytes, Union[FileInfo, Failure]] = {} + self.waiting: defaultdict[bytes, list[Deferred[FileInfo]]] = defaultdict( list ) def __init__( self, download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, Dict[str, Any], None] = None, + settings: Union[Settings, dict[str, Any], None] = None, ): self.download_func = download_func @@ -129,12 +125,12 @@ class MediaPipeline(ABC): def process_item( self, item: Any, spider: Spider - ) -> Deferred[List[FileInfoOrError]]: + ) -> Deferred[list[FileInfoOrError]]: info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) dlist = [self._process_request(r, info, item) for r in requests] dfd = cast( - "Deferred[List[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True) + "Deferred[list[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True) ) return dfd.addCallback(self.item_completed, item, info) @@ -252,7 +248,7 @@ class MediaPipeline(ABC): raise NotImplementedError() @abstractmethod - def get_media_requests(self, item: Any, info: SpiderInfo) -> List[Request]: + def get_media_requests(self, item: Any, info: SpiderInfo) -> list[Request]: """Returns the media requests to download""" raise NotImplementedError() @@ -276,7 +272,7 @@ class MediaPipeline(ABC): raise NotImplementedError() def item_completed( - self, results: List[FileInfoOrError], item: Any, info: SpiderInfo + self, results: list[FileInfoOrError], item: Any, info: SpiderInfo ) -> Any: """Called per item when all media requests has been processed""" if self.LOG_FAILED_RESULTS: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 58a47ef0f..e1bb21fb1 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -2,23 +2,15 @@ from __future__ import annotations import hashlib import logging -from typing import ( - TYPE_CHECKING, - Dict, - Iterable, - List, - Optional, - Protocol, - Tuple, - Type, - cast, -) +from typing import TYPE_CHECKING, Optional, Protocol, cast from scrapy import Request from scrapy.core.downloader import Downloader from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -87,7 +79,7 @@ class ScrapyPriorityQueue: def from_crawler( cls, crawler: Crawler, - downstream_queue_cls: Type[QueueProtocol], + downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), ) -> Self: @@ -96,14 +88,14 @@ class ScrapyPriorityQueue: def __init__( self, crawler: Crawler, - downstream_queue_cls: Type[QueueProtocol], + downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), ): self.crawler: Crawler = crawler - self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self.key: str = key - self.queues: Dict[int, QueueProtocol] = {} + self.queues: dict[int, QueueProtocol] = {} self.curprio: Optional[int] = None self.init_prios(startprios) @@ -160,8 +152,8 @@ class ScrapyPriorityQueue: # Protocols can't declare optional members return cast(Request, queue.peek()) # type: ignore[attr-defined] - def close(self) -> List[int]: - active: List[int] = [] + def close(self) -> list[int]: + active: list[int] = [] for p, q in self.queues.items(): active.append(p) q.close() @@ -176,7 +168,7 @@ class DownloaderInterface: assert crawler.engine self.downloader: Downloader = crawler.engine.downloader - def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]: + def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]: return [(self._active_downloads(slot), slot) for slot in possible_slots] def get_slot_key(self, request: Request) -> str: @@ -199,18 +191,18 @@ class DownloaderAwarePriorityQueue: def from_crawler( cls, crawler: Crawler, - downstream_queue_cls: Type[QueueProtocol], + downstream_queue_cls: type[QueueProtocol], key: str, - startprios: Optional[Dict[str, Iterable[int]]] = None, + startprios: Optional[dict[str, Iterable[int]]] = None, ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) def __init__( self, crawler: Crawler, - downstream_queue_cls: Type[QueueProtocol], + downstream_queue_cls: type[QueueProtocol], key: str, - slot_startprios: Optional[Dict[str, Iterable[int]]] = None, + slot_startprios: Optional[dict[str, Iterable[int]]] = None, ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( @@ -229,11 +221,11 @@ class DownloaderAwarePriorityQueue: ) self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) - self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls + self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self.key: str = key self.crawler: Crawler = crawler - self.pqueues: Dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue + self.pqueues: dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue for slot, startprios in (slot_startprios or {}).items(): self.pqueues[slot] = self.pqfactory(slot, startprios) @@ -281,7 +273,7 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] return queue.peek() - def close(self) -> Dict[str, List[int]]: + def close(self) -> dict[str, list[int]]: active = {slot: queue.close() for slot, queue in self.pqueues.items()} self.pqueues.clear() return active diff --git a/scrapy/resolver.py b/scrapy/resolver.py index d5eedf9b1..97fa74bc2 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type +from typing import TYPE_CHECKING, Any, Optional from twisted.internet import defer from twisted.internet.base import ReactorBase, ThreadedResolver @@ -16,6 +16,8 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache if TYPE_CHECKING: + from collections.abc import Sequence + from twisted.internet.defer import Deferred # typing.Self requires Python 3.11 @@ -82,7 +84,7 @@ class _CachingResolutionReceiver: def __init__(self, resolutionReceiver: IResolutionReceiver, hostName: str): self.resolutionReceiver: IResolutionReceiver = resolutionReceiver self.hostName: str = hostName - self.addresses: List[IAddress] = [] + self.addresses: list[IAddress] = [] def resolutionBegan(self, resolution: IHostResolution) -> None: self.resolutionReceiver.resolutionBegan(resolution) @@ -126,7 +128,7 @@ class CachingHostnameResolver: resolutionReceiver: IResolutionReceiver, hostName: str, portNumber: int = 0, - addressTypes: Optional[Sequence[Type[IAddress]]] = None, + addressTypes: Optional[Sequence[type[IAddress]]] = None, transportSemantics: str = "TCP", ) -> IHostResolution: try: diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 702e50536..7154f2b95 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -3,15 +3,20 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ +from __future__ import annotations + from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data -from typing import Dict, Mapping, Optional, Type, Union +from typing import TYPE_CHECKING, Optional, Union from scrapy.http import Response from scrapy.utils.misc import load_object from scrapy.utils.python import binary_is_text, to_bytes, to_unicode +if TYPE_CHECKING: + from collections.abc import Mapping + class ResponseTypes: CLASSES = { @@ -32,7 +37,7 @@ class ResponseTypes: } def __init__(self) -> None: - self.classes: Dict[str, Type[Response]] = {} + self.classes: dict[str, type[Response]] = {} self.mimetypes: MimeTypes = MimeTypes() mimedata = get_data("scrapy", "mime.types") if not mimedata: @@ -43,7 +48,7 @@ class ResponseTypes: for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) - def from_mimetype(self, mimetype: str) -> Type[Response]: + def from_mimetype(self, mimetype: str) -> type[Response]: """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response @@ -54,7 +59,7 @@ class ResponseTypes: def from_content_type( self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = None - ) -> Type[Response]: + ) -> type[Response]: """Return the most appropriate Response class from an HTTP Content-Type header""" if content_encoding: @@ -66,7 +71,7 @@ class ResponseTypes: def from_content_disposition( self, content_disposition: Union[str, bytes] - ) -> Type[Response]: + ) -> type[Response]: try: filename = ( to_unicode(content_disposition, encoding="latin-1", errors="replace") @@ -78,7 +83,7 @@ class ResponseTypes: except IndexError: return Response - def from_headers(self, headers: Mapping[bytes, bytes]) -> Type[Response]: + def from_headers(self, headers: Mapping[bytes, bytes]) -> type[Response]: """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response @@ -91,14 +96,14 @@ class ResponseTypes: cls = self.from_content_disposition(headers[b"Content-Disposition"]) return cls - def from_filename(self, filename: str) -> Type[Response]: + def from_filename(self, filename: str) -> type[Response]: """Return the most appropriate Response class from a file name""" mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) return Response - def from_body(self, body: bytes) -> Type[Response]: + def from_body(self, body: bytes) -> type[Response]: """Try to guess the appropriate response based on the body content. This method is a bit magic and could be improved in the future, but it's not meant to be used except for special cases where response types @@ -122,7 +127,7 @@ class ResponseTypes: url: Optional[str] = None, filename: Optional[str] = None, body: Optional[bytes] = None, - ) -> Type[Response]: + ) -> type[Response]: """Guess the most appropriate Response class based on the given arguments.""" cls = Response diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index bfddb87cb..0a3eae409 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -2,7 +2,7 @@ XPath selectors based on lxml """ -from typing import Any, Optional, Type, Union +from typing import Any, Optional, Union from parsel import Selector as _ParselSelector @@ -23,7 +23,7 @@ def _st(response: Optional[TextResponse], st: Optional[str]) -> str: def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse: - rt: Type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse + rt: type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 6703c569f..b7e3763fb 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -2,22 +2,10 @@ from __future__ import annotations import copy import json +from collections.abc import Iterable, Iterator, Mapping, MutableMapping from importlib import import_module from pprint import pformat -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - Iterator, - List, - Mapping, - MutableMapping, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from scrapy.settings import default_settings @@ -37,7 +25,7 @@ if TYPE_CHECKING: _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None] -SETTINGS_PRIORITIES: Dict[str, int] = { +SETTINGS_PRIORITIES: dict[str, int] = { "default": 0, "command": 10, "addon": 15, @@ -192,8 +180,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return float(self.get(name, default)) def getlist( - self, name: _SettingsKeyT, default: Optional[List[Any]] = None - ) -> List[Any]: + self, name: _SettingsKeyT, default: Optional[list[Any]] = None + ) -> list[Any]: """ Get a setting value as a list. If the setting original type is a list, a copy of it will be returned. If it's a string it will be split by ",". @@ -213,8 +201,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return list(value) def getdict( - self, name: _SettingsKeyT, default: Optional[Dict[Any, Any]] = None - ) -> Dict[Any, Any]: + self, name: _SettingsKeyT, default: Optional[dict[Any, Any]] = None + ) -> dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a dictionary, a copy of it will be returned. If it is a string it will be @@ -238,8 +226,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def getdictorlist( self, name: _SettingsKeyT, - default: Union[Dict[Any, Any], List[Any], Tuple[Any], None] = None, - ) -> Union[Dict[Any, Any], List[Any]]: + default: Union[dict[Any, Any], list[Any], tuple[Any], None] = None, + ) -> Union[dict[Any, Any], list[Any]]: """Get a setting value as either a :class:`dict` or a :class:`list`. If the setting is already a dict or a list, a copy of it will be @@ -412,7 +400,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self._assert_mutability() if isinstance(values, str): - values = cast(Dict[_SettingsKeyT, Any], json.loads(values)) + values = cast(dict[_SettingsKeyT, Any], json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): @@ -477,7 +465,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def __len__(self) -> int: return len(self.attributes) - def _to_dict(self) -> Dict[_SettingsKeyT, Any]: + def _to_dict(self) -> dict[_SettingsKeyT, Any]: return { self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items() @@ -490,7 +478,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): else str(key_value) ) - def copy_to_dict(self) -> Dict[_SettingsKeyT, Any]: + def copy_to_dict(self) -> dict[_SettingsKeyT, Any]: """ Make a copy of current settings and convert to a dict. @@ -553,7 +541,7 @@ class Settings(BaseSettings): self.update(values, priority) -def iter_default_settings() -> Iterable[Tuple[str, Any]]: +def iter_default_settings() -> Iterable[tuple[str, Any]]: """Return the default settings as an iterator of (name, value) tuples""" for name in dir(default_settings): if name.isupper(): @@ -562,7 +550,7 @@ def iter_default_settings() -> Iterable[Tuple[str, Any]]: def overridden_settings( settings: Mapping[_SettingsKeyT, Any] -) -> Iterable[Tuple[str, Any]]: +) -> Iterable[tuple[str, Any]]: """Return an iterable of the settings that have been overridden""" for name, defvalue in iter_default_settings(): value = settings[name] diff --git a/scrapy/shell.py b/scrapy/shell.py index b7e46274f..dc402e678 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -8,7 +8,7 @@ from __future__ import annotations import os import signal -from typing import Any, Callable, Dict, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union from itemadapter import is_item from twisted.internet import defer, threads @@ -27,25 +27,28 @@ from scrapy.utils.misc import load_object from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser +if TYPE_CHECKING: + from collections.abc import Callable + class Shell: - relevant_classes: Tuple[type, ...] = (Crawler, Spider, Request, Response, Settings) + relevant_classes: tuple[type, ...] = (Crawler, Spider, Request, Response, Settings) def __init__( self, crawler: Crawler, - update_vars: Optional[Callable[[Dict[str, Any]], None]] = None, + update_vars: Optional[Callable[[dict[str, Any]], None]] = None, code: Optional[str] = None, ): self.crawler: Crawler = crawler - self.update_vars: Callable[[Dict[str, Any]], None] = update_vars or ( + self.update_vars: Callable[[dict[str, Any]], None] = update_vars or ( lambda x: None ) self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) self.spider: Optional[Spider] = None self.inthread: bool = not threadable.isInIOThread() self.code: Optional[str] = code - self.vars: Dict[str, Any] = {} + self.vars: dict[str, Any] = {} def start( self, diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index b2c6dea5d..e106418d6 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, List, Tuple +from typing import TYPE_CHECKING, Any from pydispatch import dispatcher @@ -40,7 +40,7 @@ class SignalManager: kwargs.setdefault("sender", self.sender) dispatcher.disconnect(receiver, signal, **kwargs) - def send_catch_log(self, signal: Any, **kwargs: Any) -> List[Tuple[Any, Any]]: + def send_catch_log(self, signal: Any, **kwargs: Any) -> list[tuple[Any, Any]]: """ Send a signal, catch exceptions and log them. @@ -52,7 +52,7 @@ class SignalManager: def send_catch_log_deferred( self, signal: Any, **kwargs: Any - ) -> Deferred[List[Tuple[Any, Any]]]: + ) -> Deferred[list[tuple[Any, Any]]]: """ Like :meth:`send_catch_log` but supports returning :class:`~twisted.internet.defer.Deferred` objects from signal handlers. diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index f5fd899b2..210e729a1 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,7 +3,7 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type +from typing import TYPE_CHECKING from zope.interface import implementer @@ -29,10 +29,10 @@ class SpiderLoader: """ def __init__(self, settings: BaseSettings): - self.spider_modules: List[str] = settings.getlist("SPIDER_MODULES") + self.spider_modules: list[str] = settings.getlist("SPIDER_MODULES") self.warn_only: bool = settings.getbool("SPIDER_LOADER_WARN_ONLY") - self._spiders: Dict[str, Type[Spider]] = {} - self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list) + self._spiders: dict[str, type[Spider]] = {} + self._found: defaultdict[str, list[tuple[str, str]]] = defaultdict(list) self._load_all_spiders() def _check_name_duplicates(self) -> None: @@ -80,7 +80,7 @@ class SpiderLoader: def from_settings(cls, settings: BaseSettings) -> Self: return cls(settings) - def load(self, spider_name: str) -> Type[Spider]: + def load(self, spider_name: str) -> type[Spider]: """ Return the Spider class for the given spider name. If the spider name is not found, raise a KeyError. @@ -90,7 +90,7 @@ class SpiderLoader: except KeyError: raise KeyError(f"Spider not found: {spider_name}") - def find_by_request(self, request: Request) -> List[str]: + def find_by_request(self, request: Request) -> list[str]: """ Return the list of spider names that can handle the given request. """ @@ -98,7 +98,7 @@ class SpiderLoader: name for name, cls in self._spiders.items() if cls.handles_request(request) ] - def list(self) -> List[str]: + def list(self) -> list[str]: """ Return a list with the names of all spiders available in the project. """ diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index c5b7f0749..3164c1c03 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -7,11 +7,13 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable +from typing import TYPE_CHECKING, Any from scrapy.http import Request, Response if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index ea1686c25..afab2eac2 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -7,11 +7,13 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Optional from scrapy.exceptions import IgnoreRequest if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -39,7 +41,7 @@ class HttpErrorMiddleware: def __init__(self, settings: BaseSettings): self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL") - self.handle_httpstatus_list: List[int] = settings.getlist( + self.handle_httpstatus_list: list[int] = settings.getlist( "HTTPERROR_ALLOWED_CODES" ) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 379c5d0a3..d3ed64ef5 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -9,7 +9,7 @@ from __future__ import annotations import logging import re import warnings -from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set +from typing import TYPE_CHECKING, Any from scrapy import Spider, signals from scrapy.exceptions import ScrapyDeprecationWarning @@ -23,6 +23,8 @@ warnings.warn( ) if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -109,7 +111,7 @@ class OffsiteMiddleware: def spider_opened(self, spider: Spider) -> None: self.host_regex: re.Pattern[str] = self.get_host_regex(spider) - self.domains_seen: Set[str] = set() + self.domains_seen: set[str] = set() class URLWarning(Warning): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index d35cf8f71..8784e4b05 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -6,18 +6,7 @@ originated it. from __future__ import annotations import warnings -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Dict, - Iterable, - Optional, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urlparse from w3lib.url import safe_url_string @@ -30,6 +19,8 @@ from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -37,7 +28,7 @@ if TYPE_CHECKING: from scrapy.settings import BaseSettings -LOCAL_SCHEMES: Tuple[str, ...] = ( +LOCAL_SCHEMES: tuple[str, ...] = ( "about", "blob", "data", @@ -56,7 +47,7 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default" class ReferrerPolicy: - NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES name: str def referrer(self, response_url: str, request_url: str) -> Optional[str]: @@ -291,11 +282,11 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3") + NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3") name: str = POLICY_SCRAPY_DEFAULT -_policy_classes: Dict[str, Type[ReferrerPolicy]] = { +_policy_classes: dict[str, type[ReferrerPolicy]] = { p.name: p for p in ( NoReferrerPolicy, @@ -316,14 +307,14 @@ _policy_classes[""] = NoReferrerWhenDowngradePolicy def _load_policy_class( policy: str, warning_only: bool = False -) -> Optional[Type[ReferrerPolicy]]: +) -> Optional[type[ReferrerPolicy]]: """ Expect a string for the path to the policy class, otherwise try to interpret the string as a standard value from https://www.w3.org/TR/referrer-policy/#referrer-policies """ try: - return cast(Type[ReferrerPolicy], load_object(policy)) + return cast(type[ReferrerPolicy], load_object(policy)) except ValueError: tokens = [token.strip() for token in policy.lower().split(",")] # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header @@ -341,7 +332,7 @@ def _load_policy_class( class RefererMiddleware: def __init__(self, settings: Optional[BaseSettings] = None): - self.default_policy: Type[ReferrerPolicy] = DefaultReferrerPolicy + self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy if settings is not None: settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) assert settings_policy diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 34df54ca7..191adb6cd 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -7,12 +7,14 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable +from typing import TYPE_CHECKING, Any from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index d977acd26..8220aca28 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, cast +from typing import TYPE_CHECKING, Any, Optional, cast from scrapy import signals from scrapy.http import Request, Response @@ -15,6 +15,8 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: + from collections.abc import Iterable + from twisted.internet.defer import Deferred # typing.Self requires Python 3.11 @@ -32,7 +34,7 @@ class Spider(object_ref): """ name: str - custom_settings: Optional[Dict[_SettingsKeyT, Any]] = None + custom_settings: Optional[dict[_SettingsKeyT, Any]] = None def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: @@ -41,7 +43,7 @@ class Spider(object_ref): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) if not hasattr(self, "start_urls"): - self.start_urls: List[str] = [] + self.start_urls: list[str] = [] @property def logger(self) -> SpiderLoggerAdapter: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 2639f14b2..d628f49f6 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -1,6 +1,6 @@ """ This modules implements the CrawlSpider which is the recommended spider to use -for scraping typical web sites that requires crawling pages. +for scraping typical websites that requires crawling pages. See documentation in docs/topics/spiders.rst """ @@ -8,22 +8,8 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import copy -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - Awaitable, - Callable, - Dict, - Iterable, - List, - Optional, - Sequence, - Set, - TypeVar, - Union, - cast, -) +from collections.abc import AsyncIterable, Awaitable, Callable +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from twisted.python.failure import Failure @@ -35,6 +21,8 @@ from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -43,7 +31,7 @@ if TYPE_CHECKING: _T = TypeVar("_T") -ProcessLinksT = Callable[[List[Link]], List[Link]] +ProcessLinksT = Callable[[list[Link]], list[Link]] ProcessRequestT = Callable[[Request, Response], Optional[Request]] @@ -75,7 +63,7 @@ class Rule: self, link_extractor: Optional[LinkExtractor] = None, callback: Union[CallbackT, str, None] = None, - cb_kwargs: Optional[Dict[str, Any]] = None, + cb_kwargs: Optional[dict[str, Any]] = None, follow: Optional[bool] = None, process_links: Union[ProcessLinksT, str, None] = None, process_request: Union[ProcessRequestT, str, None] = None, @@ -84,7 +72,7 @@ class Rule: self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor self.callback: Union[CallbackT, str, None] = callback self.errback: Union[Callable[[Failure], Any], str, None] = errback - self.cb_kwargs: Dict[str, Any] = cb_kwargs or {} + self.cb_kwargs: dict[str, Any] = cb_kwargs or {} self.process_links: Union[ProcessLinksT, str] = process_links or _identity self.process_request: Union[ProcessRequestT, str] = ( process_request or _identity_process_request @@ -105,7 +93,7 @@ class Rule: class CrawlSpider(Spider): rules: Sequence[Rule] = () - _rules: List[Rule] + _rules: list[Rule] _follow_links: bool def __init__(self, *a: Any, **kw: Any): @@ -139,9 +127,9 @@ class CrawlSpider(Spider): def _requests_to_follow(self, response: Response) -> Iterable[Optional[Request]]: if not isinstance(response, HtmlResponse): return - seen: Set[Link] = set() + seen: set[Link] = set() for rule_index, rule in enumerate(self._rules): - links: List[Link] = [ + links: list[Link] = [ lnk for lnk in rule.link_extractor.extract_links(response) if lnk not in seen @@ -170,7 +158,7 @@ class CrawlSpider(Spider): self, response: Response, callback: Optional[CallbackT], - cb_kwargs: Dict[str, Any], + cb_kwargs: dict[str, Any], follow: bool = True, ) -> AsyncIterable[Any]: if callback: diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 9dd8a5d68..0ddef1f32 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -5,7 +5,9 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional from scrapy.exceptions import NotConfigured, NotSupported from scrapy.http import Response, TextResponse @@ -14,6 +16,9 @@ from scrapy.spiders import Spider from scrapy.utils.iterators import csviter, xmliter_lxml from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + class XMLFeedSpider(Spider): """ @@ -27,7 +32,7 @@ class XMLFeedSpider(Spider): iterator: str = "iternodes" itertag: str = "item" - namespaces: Sequence[Tuple[str, str]] = () + namespaces: Sequence[tuple[str, str]] = () def process_results( self, response: Response, results: Iterable[Any] @@ -118,7 +123,7 @@ class CSVFeedSpider(Spider): quotechar: Optional[str] = ( None # When this is None, python's csv module's default quotechar is used ) - headers: Optional[List[str]] = None + headers: Optional[list[str]] = None def process_results( self, response: Response, results: Iterable[Any] @@ -130,7 +135,7 @@ class CSVFeedSpider(Spider): """This method has the same purpose as the one in XMLFeedSpider""" return response - def parse_row(self, response: Response, row: Dict[str, str]) -> Any: + def parse_row(self, response: Response, row: dict[str, str]) -> Any: """This method must be overridden with your custom spider functionality""" raise NotImplementedError diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index ce0f1bbaa..ebe288b83 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Iterable, Optional, cast +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, Optional, cast from scrapy import Request from scrapy.spiders import Spider diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 1542ef79c..945539d7b 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -2,18 +2,7 @@ from __future__ import annotations import logging import re -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - List, - Optional, - Sequence, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from scrapy.http import Request, Response, XmlResponse from scrapy.spiders import Spider @@ -22,6 +11,8 @@ from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -34,7 +25,7 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): sitemap_urls: Sequence[str] = () sitemap_rules: Sequence[ - Tuple[Union[re.Pattern[str], str], Union[str, CallbackT]] + tuple[Union[re.Pattern[str], str], Union[str, CallbackT]] ] = [("", "parse")] sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""] sitemap_alternate_links: bool = False @@ -54,20 +45,20 @@ class SitemapSpider(Spider): def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self._cbs: List[Tuple[re.Pattern[str], CallbackT]] = [] + self._cbs: list[tuple[re.Pattern[str], CallbackT]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): c = cast("CallbackT", getattr(self, c)) self._cbs.append((regex(r), c)) - self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] + self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) def sitemap_filter( - self, entries: Iterable[Dict[str, Any]] - ) -> Iterable[Dict[str, Any]]: + self, entries: Iterable[dict[str, Any]] + ) -> Iterable[dict[str, Any]]: """This method can be used to filter sitemap entries by their attributes, for example, you can filter locs with lastmod greater than a given date (see docs). @@ -142,7 +133,7 @@ def regex(x: Union[re.Pattern[str], str]) -> re.Pattern[str]: return x -def iterloc(it: Iterable[Dict[str, Any]], alt: bool = False) -> Iterable[str]: +def iterloc(it: Iterable[dict[str, Any]], alt: bool = False) -> Iterable[str]: for d in it: yield d["loc"] diff --git a/scrapy/squeues.py b/scrapy/squeues.py index d3e7896c5..767a53db8 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -7,13 +7,14 @@ from __future__ import annotations import marshal import pickle # nosec from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Optional, Union from queuelib import queue from scrapy.utils.request import request_from_dict if TYPE_CHECKING: + from collections.abc import Callable from os import PathLike # typing.Self requires Python 3.11 @@ -23,7 +24,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler -def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: +def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]: class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any): dirname = Path(path).parent @@ -35,10 +36,10 @@ def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]: def _serializable_queue( - queue_class: Type[queue.BaseQueue], + queue_class: type[queue.BaseQueue], serialize: Callable[[Any], bytes], deserialize: Callable[[bytes], Any], -) -> Type[queue.BaseQueue]: +) -> type[queue.BaseQueue]: class SerializableQueue(queue_class): # type: ignore[valid-type,misc] def push(self, obj: Any) -> None: s = serialize(obj) @@ -71,8 +72,8 @@ def _serializable_queue( def _scrapy_serialization_queue( - queue_class: Type[queue.BaseQueue], -) -> Type[queue.BaseQueue]: + queue_class: type[queue.BaseQueue], +) -> type[queue.BaseQueue]: class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] def __init__(self, crawler: Crawler, key: str): self.spider = crawler.spider @@ -110,8 +111,8 @@ def _scrapy_serialization_queue( def _scrapy_non_serialization_queue( - queue_class: Type[queue.BaseQueue], -) -> Type[queue.BaseQueue]: + queue_class: type[queue.BaseQueue], +) -> type[queue.BaseQueue]: class ScrapyRequestQueue(queue_class): # type: ignore[valid-type,misc] @classmethod def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 88e72f366..63c82ec6d 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -6,7 +6,7 @@ from __future__ import annotations import logging import pprint -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Optional if TYPE_CHECKING: from scrapy import Spider @@ -16,7 +16,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -StatsT = Dict[str, Any] +StatsT = dict[str, Any] class StatsCollector: @@ -71,7 +71,7 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): def __init__(self, crawler: Crawler): super().__init__(crawler) - self.spider_stats: Dict[str, StatsT] = {} + self.spider_stats: dict[str, StatsT] = {} def _persist_stats(self, stats: StatsT, spider: Spider) -> None: self.spider_stats[spider.name] = stats diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 67c8e1a01..f1505e4bd 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,9 +1,10 @@ -from typing import AsyncGenerator, AsyncIterable, Iterable, List, TypeVar, Union +from collections.abc import AsyncGenerator, AsyncIterable, Iterable +from typing import TypeVar, Union _T = TypeVar("_T") -async def collect_asyncgen(result: AsyncIterable[_T]) -> List[_T]: +async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]: results = [] async for x in result: results.append(x) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index c63b69995..463bbb5df 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,35 +1,29 @@ +from __future__ import annotations + import numbers import os import sys import warnings +from collections.abc import Iterable from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import ( - Any, - Callable, - Collection, - Dict, - Iterable, - List, - Mapping, - MutableMapping, - Optional, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + from collections.abc import Collection, Mapping, MutableMapping + def build_component_list( compdict: MutableMapping[Any, Any], custom: Any = None, convert: Callable[[Any], Any] = update_classpath, -) -> List[Any]: +) -> list[Any]: """Compose a component list from a { class: order } dictionary.""" def _check_components(complist: Collection[Any]) -> None: @@ -39,7 +33,7 @@ def build_component_list( "please update your settings" ) - def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, Dict[Any, Any]]: + def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, dict[Any, Any]]: if isinstance(compdict, BaseSettings): compbs = BaseSettings() for k, v in compdict.items(): @@ -84,7 +78,7 @@ def build_component_list( return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] -def arglist_to_dict(arglist: List[str]) -> Dict[str, str]: +def arglist_to_dict(arglist: list[str]) -> dict[str, str]: """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a dict """ @@ -130,7 +124,7 @@ def get_config(use_closest: bool = True) -> ConfigParser: return cfg -def get_sources(use_closest: bool = True) -> List[str]: +def get_sources(use_closest: bool = True) -> list[str]: xdg_config_home = ( os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser() ) @@ -146,8 +140,8 @@ def get_sources(use_closest: bool = True) -> List[str]: def feed_complete_default_values_from_settings( - feed: Dict[str, Any], settings: BaseSettings -) -> Dict[str, Any]: + feed: dict[str, Any], settings: BaseSettings +) -> dict[str, Any]: out = feed.copy() out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT")) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) @@ -164,17 +158,17 @@ def feed_complete_default_values_from_settings( def feed_process_params_from_cli( settings: BaseSettings, - output: List[str], + output: list[str], output_format: Optional[str] = None, - overwrite_output: Optional[List[str]] = None, -) -> Dict[str, Dict[str, Any]]: + overwrite_output: Optional[list[str]] = None, +) -> dict[str, dict[str, Any]]: """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary suitable to be used as the FEEDS setting. """ valid_output_formats: Iterable[str] = without_none_values( - cast(Dict[str, str], settings.getwithbase("FEED_EXPORTERS")) + cast(dict[str, str], settings.getwithbase("FEED_EXPORTERS")) ).keys() def check_valid_format(output_format: str) -> None: @@ -223,7 +217,7 @@ def feed_process_params_from_cli( "URIs are specified" ) - result: Dict[str, Dict[str, Any]] = {} + result: dict[str, dict[str, Any]] = {} for element in output: try: feed_uri, feed_format = element.rsplit(":", 1) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 328219831..3b5596ab7 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,12 +1,18 @@ +from __future__ import annotations + +from collections.abc import Callable from functools import wraps -from typing import Any, Callable, Dict, Iterable, Optional +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + from collections.abc import Iterable EmbedFuncT = Callable[..., None] -KnownShellsT = Dict[str, Callable[..., EmbedFuncT]] +KnownShellsT = dict[str, Callable[..., EmbedFuncT]] def _embed_ipython_shell( - namespace: Dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start an IPython Shell""" try: @@ -21,7 +27,7 @@ def _embed_ipython_shell( ) @wraps(_embed_ipython_shell) - def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: config = load_default_config() # Always use .instance() to ensure _instance propagation to all parents # this is needed for <TAB> completion works well for new imports @@ -37,26 +43,26 @@ def _embed_ipython_shell( def _embed_bpython_shell( - namespace: Dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a bpython shell""" import bpython @wraps(_embed_bpython_shell) - def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: bpython.embed(locals_=namespace, banner=banner) return wrapper def _embed_ptpython_shell( - namespace: Dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a ptpython shell""" import ptpython.repl @wraps(_embed_ptpython_shell) - def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: print(banner) ptpython.repl.embed(locals=namespace) @@ -64,7 +70,7 @@ def _embed_ptpython_shell( def _embed_standard_shell( - namespace: Dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a standard python shell""" import code @@ -79,7 +85,7 @@ def _embed_standard_shell( readline.parse_and_bind("tab:complete") @wraps(_embed_standard_shell) - def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: code.interact(banner=banner, local=namespace) return wrapper @@ -114,7 +120,7 @@ def get_shell_embed_func( def start_python_console( - namespace: Optional[Dict[str, Any]] = None, + namespace: Optional[dict[str, Any]] = None, banner: str = "", shells: Optional[Iterable[str]] = None, ) -> None: diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index c10e48511..9c7f63848 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,12 +1,17 @@ +from __future__ import annotations + import argparse import warnings from http.cookies import SimpleCookie from shlex import split -from typing import Any, Dict, List, NoReturn, Optional, Sequence, Tuple, Union +from typing import TYPE_CHECKING, Any, NoReturn, Optional, Union from urllib.parse import urlparse from w3lib.http import basic_auth_header +if TYPE_CHECKING: + from collections.abc import Sequence + class DataAction(argparse.Action): def __call__( @@ -51,9 +56,9 @@ for argument in safe_to_ignore_arguments: def _parse_headers_and_cookies( parsed_args: argparse.Namespace, -) -> Tuple[List[Tuple[str, bytes]], Dict[str, str]]: - headers: List[Tuple[str, bytes]] = [] - cookies: Dict[str, str] = {} +) -> tuple[list[tuple[str, bytes]], dict[str, str]]: + headers: list[tuple[str, bytes]] = [] + cookies: dict[str, str] = {} for header in parsed_args.headers or (): name, val = header.split(":", 1) name = name.strip() @@ -73,7 +78,7 @@ def _parse_headers_and_cookies( def curl_to_request_kwargs( curl_command: str, ignore_unknown_options: bool = True -) -> Dict[str, Any]: +) -> dict[str, Any]: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command @@ -107,7 +112,7 @@ def curl_to_request_kwargs( method = parsed_args.method or "GET" - result: Dict[str, Any] = {"method": method.upper(), "url": url} + result: dict[str, Any] = {"method": method.upper(), "url": url} headers, cookies = _parse_headers_and_cookies(parsed_args) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index d06887610..c78325676 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -10,23 +10,15 @@ from __future__ import annotations import collections import warnings import weakref +from collections import OrderedDict from collections.abc import Mapping -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Iterable, - Optional, - OrderedDict, - Sequence, - Tuple, - TypeVar, - Union, -) +from typing import TYPE_CHECKING, Any, AnyStr, Optional, TypeVar, Union from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -52,7 +44,7 @@ class CaselessDict(dict): def __init__( self, - seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, + seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, ): super().__init__() if seq: @@ -92,7 +84,7 @@ class CaselessDict(dict): return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # type: ignore[arg-type] # doesn't fully implement MutableMapping.update() - def update(self, seq: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]]]) -> None: # type: ignore[override] + def update(self, seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]]]) -> None: # type: ignore[override] seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super().update(iseq) diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 2240f0b58..0f4d0beda 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -2,7 +2,7 @@ from __future__ import annotations import warnings from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from twisted.internet.defer import Deferred, maybeDeferred from twisted.internet.threads import deferToThread @@ -10,6 +10,8 @@ from twisted.internet.threads import deferToThread from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: + from collections.abc import Callable + # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 33ec23cec..3a0dee8f1 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -8,28 +8,10 @@ import asyncio import inspect import warnings from asyncio import Future +from collections.abc import Awaitable, Coroutine, Iterable, Iterator from functools import wraps from types import CoroutineType -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - AsyncIterator, - Awaitable, - Callable, - Coroutine, - Dict, - Generic, - Iterable, - Iterator, - List, - Optional, - Tuple, - TypeVar, - Union, - cast, - overload, -) +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, Union, cast, overload from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred @@ -40,6 +22,8 @@ from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: + from collections.abc import AsyncIterable, AsyncIterator, Callable + from twisted.python.failure import Failure # typing.Concatenate and typing.ParamSpec require Python 3.10 @@ -47,6 +31,7 @@ if TYPE_CHECKING: _P = ParamSpec("_P") + _T = TypeVar("_T") _T2 = TypeVar("_T2") @@ -134,7 +119,7 @@ def parallel( callable: Callable[Concatenate[_T, _P], _T2], *args: _P.args, **named: _P.kwargs, -) -> Deferred[List[Tuple[bool, Iterator[_T2]]]]: +) -> Deferred[list[tuple[bool, Iterator[_T2]]]]: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -145,7 +130,7 @@ def parallel( return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter(Iterator[Deferred], Generic[_T]): +class _AsyncCooperatorAdapter(Iterator, Generic[_T]): """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more @@ -200,10 +185,10 @@ class _AsyncCooperatorAdapter(Iterator[Deferred], Generic[_T]): ): self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() self.callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]] = callable - self.callable_args: Tuple[Any, ...] = callable_args - self.callable_kwargs: Dict[str, Any] = callable_kwargs + self.callable_args: tuple[Any, ...] = callable_args + self.callable_kwargs: dict[str, Any] = callable_kwargs self.finished: bool = False - self.waiting_deferreds: List[Deferred[Any]] = [] + self.waiting_deferreds: list[Deferred[Any]] = [] self.anext_deferred: Optional[Deferred[_T]] = None def _callback(self, result: _T) -> None: @@ -255,13 +240,13 @@ def parallel_async( callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], *args: _P.args, **named: _P.kwargs, -) -> Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]]: +) -> Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]]: """Like ``parallel`` but for async iterators""" coop = Cooperator() work: Iterator[Deferred[Any]] = _AsyncCooperatorAdapter( async_iterable, callable, *args, **named ) - dl: Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList( + dl: Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList( [coop.coiterate(work) for _ in range(count)] ) return dl @@ -311,15 +296,15 @@ def process_parallel( input: _T, *a: _P.args, **kw: _P.kwargs, -) -> Deferred[List[_T2]]: +) -> Deferred[list[_T2]]: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d: Deferred[List[Tuple[bool, _T2]]] = DeferredList( + d: Deferred[list[tuple[bool, _T2]]] = DeferredList( dfds, fireOnOneErrback=True, consumeErrors=True ) - d2: Deferred[List[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) + d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) d2.addErrback(lambda f: f.value.subFailure) return d2 diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index e0f2ac763..9b0d476a1 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -2,7 +2,7 @@ import inspect import warnings -from typing import Any, Dict, List, Optional, Tuple, Type, overload +from typing import Any, Optional, overload from scrapy.exceptions import ScrapyDeprecationWarning @@ -20,8 +20,8 @@ def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> No def create_deprecated_class( name: str, new_class: type, - clsdict: Optional[Dict[str, Any]] = None, - warn_category: Type[Warning] = ScrapyDeprecationWarning, + clsdict: Optional[dict[str, Any]] = None, + warn_category: type[Warning] = ScrapyDeprecationWarning, warn_once: bool = True, old_class_path: Optional[str] = None, new_class_path: Optional[str] = None, @@ -59,14 +59,14 @@ def create_deprecated_class( warned_on_subclass: bool = False def __new__( - metacls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any] + metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any] ) -> type: cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls - def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any]): + def __init__(cls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]): meta = cls.__class__ old = meta.deprecated_class if old in bases and not (warn_once and meta.warned_on_subclass): @@ -134,7 +134,7 @@ def _clspath(cls: type, forced: Optional[str] = None) -> str: return f"{cls.__module__}.{cls.__name__}" -DEPRECATION_RULES: List[Tuple[str, str]] = [] +DEPRECATION_RULES: list[tuple[str, str]] = [] @overload diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 770ee0b1b..1430ed8d6 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -4,13 +4,13 @@ from __future__ import annotations # used in global tests code from time import time # noqa: F401 -from typing import TYPE_CHECKING, Any, List, Tuple +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from scrapy.core.engine import ExecutionEngine -def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: +def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: """Return a report of the current engine status""" tests = [ "time()-engine.start_time", @@ -29,7 +29,7 @@ def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: "engine.scraper.slot.needs_backout()", ] - checks: List[Tuple[str, Any]] = [] + checks: list[tuple[str, Any]] = [] for test in tests: try: checks += [(test, eval(test))] # nosec diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 41a842386..a4d339adc 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,19 +1,10 @@ +from __future__ import annotations + import csv import logging import re from io import StringIO -from typing import ( - Any, - Callable, - Dict, - Iterator, - List, - Literal, - Optional, - Union, - cast, - overload, -) +from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast, overload from warnings import warn from lxml import etree # nosec @@ -23,6 +14,9 @@ from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + logger = logging.getLogger(__name__) @@ -59,7 +53,7 @@ def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selecto ) header_end_idx = re_rsearch(HEADER_END_RE, text) header_end = text[header_end_idx[1] :].strip() if header_end_idx else "" - namespaces: Dict[str, str] = {} + namespaces: dict[str, str] = {} if header_end: for tagname in reversed(re.findall(END_TAG_RE, header_end)): assert header_end_idx @@ -162,10 +156,10 @@ class _StreamReader: def csviter( obj: Union[Response, str, bytes], delimiter: Optional[str] = None, - headers: Optional[List[str]] = None, + headers: Optional[list[str]] = None, encoding: Optional[str] = None, quotechar: Optional[str] = None, -) -> Iterator[Dict[str, str]]: +) -> Iterator[dict[str, str]]: """Returns an iterator of dictionaries from the given csv object obj can be: @@ -191,7 +185,7 @@ def csviter( lines = StringIO(_body_or_str(obj, unicode=True)) - kwargs: Dict[str, Any] = {} + kwargs: dict[str, Any] = {} if delimiter: kwargs["delimiter"] = delimiter if quotechar: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 4a70de6b4..2b90c6b36 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -2,20 +2,10 @@ from __future__ import annotations import logging import sys +from collections.abc import MutableMapping from logging.config import dictConfig from types import TracebackType -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - Optional, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -25,6 +15,7 @@ from scrapy.settings import Settings, _SettingsKeyT from scrapy.utils.versions import scrapy_components_versions if TYPE_CHECKING: + from scrapy.crawler import Crawler from scrapy.logformatter import LogFormatterResult @@ -34,7 +25,7 @@ logger = logging.getLogger(__name__) def failure_to_exc_info( failure: Failure, -) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]: +) -> Optional[tuple[type[BaseException], BaseException, Optional[TracebackType]]]: """Extract exc_info from Failure instances""" if isinstance(failure, Failure): assert failure.type @@ -48,7 +39,7 @@ def failure_to_exc_info( class TopLevelFormatter(logging.Filter): - """Keep only top level loggers's name (direct children from root) from + """Keep only top level loggers' name (direct children from root) from records. This filter will replace Scrapy loggers' names with 'scrapy'. This mimics @@ -59,8 +50,8 @@ class TopLevelFormatter(logging.Filter): ``loggers`` list where it should act. """ - def __init__(self, loggers: Optional[List[str]] = None): - self.loggers: List[str] = loggers or [] + def __init__(self, loggers: Optional[list[str]] = None): + self.loggers: list[str] = loggers or [] def filter(self, record: logging.LogRecord) -> bool: if any(record.name.startswith(logger + ".") for logger in self.loggers): @@ -89,7 +80,7 @@ DEFAULT_LOGGING = { def configure_logging( - settings: Union[Settings, Dict[_SettingsKeyT, Any], None] = None, + settings: Union[Settings, dict[_SettingsKeyT, Any], None] = None, install_root_handler: bool = True, ) -> None: """ @@ -240,7 +231,7 @@ class LogCounterHandler(logging.Handler): def logformatter_adapter( logkws: LogFormatterResult, -) -> Tuple[int, str, Union[Dict[str, Any], Tuple[Any, ...]]]: +) -> tuple[int, str, Union[dict[str, Any], tuple[Any, ...]]]: """ Helper that takes the dictionary output from the methods in LogFormatter and adapts it into a tuple of positional arguments for logger.log calls, @@ -251,7 +242,7 @@ def logformatter_adapter( message = logkws.get("msg") or "" # NOTE: This also handles 'args' being an empty dict, that case doesn't # play well in logger.log calls - args = cast(Dict[str, Any], logkws) if not logkws.get("args") else logkws["args"] + args = cast(dict[str, Any], logkws) if not logkws.get("args") else logkws["args"] return (level, message, args) @@ -259,7 +250,7 @@ def logformatter_adapter( class SpiderLoggerAdapter(logging.LoggerAdapter): def process( self, msg: str, kwargs: MutableMapping[str, Any] - ) -> Tuple[str, MutableMapping[str, Any]]: + ) -> tuple[str, MutableMapping[str, Any]]: """Method that augments logging with additional 'extra' data""" if isinstance(kwargs.get("extra"), MutableMapping): kwargs["extra"].update(self.extra) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 3c787e50f..e5e00512a 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -9,31 +9,19 @@ import os import re import warnings from collections import deque +from collections.abc import Iterable from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Deque, - Iterable, - Iterator, - List, - Optional, - Type, - TypeVar, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, Optional, TypeVar, Union, cast from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.datatypes import LocalWeakReferencedCache if TYPE_CHECKING: + from collections.abc import Callable, Iterator from types import ModuleType from scrapy import Spider @@ -91,7 +79,7 @@ def load_object(path: Union[str, Callable[..., Any]]) -> Any: return obj -def walk_modules(path: str) -> List[ModuleType]: +def walk_modules(path: str) -> list[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that exception is thrown back. @@ -99,7 +87,7 @@ def walk_modules(path: str) -> List[ModuleType]: For example: walk_modules('scrapy.utils') """ - mods: List[ModuleType] = [] + mods: list[ModuleType] = [] mod = import_module(path) mods.append(mod) if hasattr(mod, "__path__"): @@ -186,7 +174,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): def build_from_crawler( - objcls: Type[T], crawler: Crawler, /, *args: Any, **kwargs: Any + objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any ) -> T: """Construct a class instance using its ``from_crawler`` constructor. @@ -209,7 +197,7 @@ def build_from_crawler( def build_from_settings( - objcls: Type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any + objcls: type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any ) -> T: """Construct a class instance using its ``from_settings`` constructor. @@ -250,7 +238,7 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]: """Similar to ``ast.walk``, but walks only function body and skips nested functions defined within the node. """ - todo: Deque[ast.AST] = deque([node]) + todo: deque[ast.AST] = deque([node]) walked_func_def = False while todo: node = todo.popleft() diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 5985a847e..cff5eb629 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,13 +1,14 @@ import signal +from collections.abc import Callable from types import FrameType -from typing import Any, Callable, Dict, Optional, Union +from typing import Any, Optional, Union # copy of _HANDLER from typeshed/stdlib/signal.pyi SignalHandlerT = Union[ Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None ] -signal_names: Dict[int, str] = {} +signal_names: dict[int, str] = {} for signame in dir(signal): if signame.startswith("SIG") and not signame.startswith("SIG_"): signum = getattr(signal, signame) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index efb6af299..c9e5eb857 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import os import warnings from importlib import import_module diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index f56950fdd..91c5d67f5 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -4,36 +4,22 @@ This module contains essential stuff that should've come with Python itself ;) from __future__ import annotations -import collections.abc import gc import inspect import re import sys import weakref +from collections.abc import AsyncIterable, Iterable, Mapping from functools import partial, wraps from itertools import chain -from typing import ( - TYPE_CHECKING, - Any, - AsyncIterable, - AsyncIterator, - Callable, - Dict, - Iterable, - Iterator, - List, - Mapping, - Optional, - Pattern, - Tuple, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable, Iterator + from re import Pattern + # typing.Concatenate and typing.ParamSpec require Python 3.10 from typing_extensions import Concatenate, ParamSpec @@ -44,7 +30,7 @@ _KT = TypeVar("_KT") _VT = TypeVar("_VT") -def flatten(x: Iterable[Any]) -> List[Any]: +def flatten(x: Iterable[Any]) -> list[Any]: """flatten(sequence) -> list Returns a single, flat list which contains all elements retrieved @@ -99,10 +85,10 @@ def is_listlike(x: Any) -> bool: return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) -def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> List[_T]: +def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> list[_T]: """efficient function to uniquify a list preserving item order""" seen = set() - result: List[_T] = [] + result: list[_T] = [] for item in list_: seenkey = key(item) if seenkey in seen: @@ -147,7 +133,7 @@ def to_bytes( def re_rsearch( pattern: Union[str, Pattern[str]], text: str, chunk_size: int = 1024 -) -> Optional[Tuple[int, int]]: +) -> Optional[tuple[int, int]]: """ This function does a reverse search in a text using a regular expression given in the attribute 'pattern'. @@ -161,7 +147,7 @@ def re_rsearch( the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter() -> Iterable[Tuple[str, int]]: + def _chunk_iter() -> Iterable[tuple[str, int]]: offset = len(text) while True: offset -= chunk_size * 1024 @@ -215,12 +201,12 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) -def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str]: +def get_func_args(func: Callable[..., Any], stripself: bool = False) -> list[str]: """Return the argument name list of a callable object""" if not callable(func): raise TypeError(f"func must be callable, got '{type(func).__name__}'") - args: List[str] = [] + args: list[str] = [] try: sig = inspect.signature(func) except ValueError: @@ -245,7 +231,7 @@ def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str return args -def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: +def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]: """Returns (args, kwargs) tuple for a function >>> import re >>> get_spec(re.match) @@ -274,7 +260,7 @@ def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: else: raise TypeError(f"{type(func)} is not callable") - defaults: Tuple[Any, ...] = spec.defaults or () + defaults: tuple[Any, ...] = spec.defaults or () firstdefault = len(spec.args) - len(defaults) args = spec.args[:firstdefault] @@ -283,7 +269,7 @@ def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: def equal_attributes( - obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable[[Any], Any]]]] + obj1: Any, obj2: Any, attributes: Optional[list[Union[str, Callable[[Any], Any]]]] ) -> bool: """Compare two objects attributes""" # not attributes given return False by default @@ -303,7 +289,7 @@ def equal_attributes( @overload -def without_none_values(iterable: Mapping[_KT, _VT]) -> Dict[_KT, _VT]: ... +def without_none_values(iterable: Mapping[_KT, _VT]) -> dict[_KT, _VT]: ... @overload @@ -312,13 +298,13 @@ def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ... def without_none_values( iterable: Union[Mapping[_KT, _VT], Iterable[_KT]] -) -> Union[Dict[_KT, _VT], Iterable[_KT]]: +) -> Union[dict[_KT, _VT], Iterable[_KT]]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have value ``None`` have been removed. """ - if isinstance(iterable, collections.abc.Mapping): + if isinstance(iterable, Mapping): return {k: v for k, v in iterable.items() if v is not None} else: # the iterable __init__ must take another iterable diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index a627db601..ed2fb5959 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -3,18 +3,7 @@ from __future__ import annotations import asyncio import sys from contextlib import suppress -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Generic, - List, - Optional, - Tuple, - Type, - TypeVar, -) +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error @@ -25,6 +14,7 @@ from scrapy.utils.misc import load_object if TYPE_CHECKING: from asyncio import AbstractEventLoop, AbstractEventLoopPolicy + from collections.abc import Callable from twisted.internet.protocol import ServerFactory from twisted.internet.tcp import Port @@ -37,7 +27,7 @@ if TYPE_CHECKING: _T = TypeVar("_T") -def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] +def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor @@ -62,8 +52,8 @@ class CallLaterOnce(Generic[_T]): def __init__(self, func: Callable[_P, _T], *a: _P.args, **kw: _P.kwargs): self._func: Callable[_P, _T] = func - self._a: Tuple[Any, ...] = a - self._kw: Dict[str, Any] = kw + self._a: tuple[Any, ...] = a + self._kw: dict[str, Any] = kw self._call: Optional[DelayedCall] = None def schedule(self, delay: float = 0) -> None: @@ -142,7 +132,7 @@ def _get_asyncio_event_loop() -> AbstractEventLoop: def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: - event_loop_class: Type[AbstractEventLoop] = load_object(event_loop_path) + event_loop_class: type[AbstractEventLoop] = load_object(event_loop_path) event_loop = event_loop_class() asyncio.set_event_loop(event_loop) else: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 99ca3b7a0..052a3721a 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -8,18 +8,7 @@ from __future__ import annotations import hashlib import json import warnings -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - List, - Optional, - Protocol, - Tuple, - Type, - Union, -) +from typing import TYPE_CHECKING, Any, Optional, Protocol, Union from urllib.parse import urlunparse from weakref import WeakKeyDictionary @@ -33,6 +22,8 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -47,7 +38,7 @@ def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[b _fingerprint_cache: WeakKeyDictionary[ - Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes] + Request, dict[tuple[Optional[tuple[bytes, ...]], bool], bytes] ] _fingerprint_cache = WeakKeyDictionary() @@ -88,7 +79,7 @@ def fingerprint( If you want to include them, set the keep_fragments argument to True (for instance when handling requests with a headless browser). """ - processed_include_headers: Optional[Tuple[bytes, ...]] = None + processed_include_headers: Optional[tuple[bytes, ...]] = None if include_headers: processed_include_headers = tuple( to_bytes(h.lower()) for h in sorted(include_headers) @@ -98,7 +89,7 @@ def fingerprint( if cache_key not in cache: # To decode bytes reliably (JSON does not support bytes), regardless of # character encoding, we use bytes.hex() - headers: Dict[str, List[str]] = {} + headers: dict[str, list[str]] = {} if processed_include_headers: for header in processed_include_headers: if header in request.headers: @@ -194,13 +185,13 @@ def referer_str(request: Request) -> Optional[str]: return to_unicode(referrer, errors="replace") -def request_from_dict(d: Dict[str, Any], *, spider: Optional[Spider] = None) -> Request: +def request_from_dict(d: dict[str, Any], *, spider: Optional[Spider] = None) -> Request: """Create a :class:`~scrapy.Request` object from a dict. If a spider is given, it will try to resolve the callbacks looking at the spider for methods with the same name. """ - request_cls: Type[Request] = load_object(d["_class"]) if "_class" in d else Request + request_cls: type[Request] = load_object(d["_class"]) if "_class" in d else Request kwargs = {key: value for key, value in d.items() if key in request_cls.attributes} if d.get("callback") and spider: kwargs["callback"] = _get_method(spider, d["callback"]) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 320059b3a..0ca9d07a4 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -9,7 +9,7 @@ import os import re import tempfile import webbrowser -from typing import TYPE_CHECKING, Any, Callable, Iterable, Tuple, Union +from typing import TYPE_CHECKING, Any, Union from weakref import WeakKeyDictionary from twisted.web import http @@ -18,6 +18,8 @@ from w3lib import html from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: + from collections.abc import Callable, Iterable + from scrapy.http import Response, TextResponse _baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary() @@ -34,14 +36,14 @@ def get_base_url(response: TextResponse) -> str: _metaref_cache: WeakKeyDictionary[ - Response, Union[Tuple[None, None], Tuple[float, str]] + Response, Union[tuple[None, None], tuple[float, str]] ] = WeakKeyDictionary() def get_meta_refresh( response: TextResponse, ignore_tags: Iterable[str] = ("script", "noscript"), -) -> Union[Tuple[None, None], Tuple[float, str]]: +) -> Union[tuple[None, None], tuple[float, str]]: """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 4310c1d56..c1d3bfffb 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -2,10 +2,9 @@ from __future__ import annotations -import collections.abc import logging +from collections.abc import Sequence from typing import Any as TypingAny -from typing import List, Tuple from pydispatch.dispatcher import ( Anonymous, @@ -30,19 +29,15 @@ def send_catch_log( sender: TypingAny = Anonymous, *arguments: TypingAny, **named: TypingAny, -) -> List[Tuple[TypingAny, TypingAny]]: +) -> list[tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. """ dont_log = named.pop("dont_log", ()) - dont_log = ( - tuple(dont_log) - if isinstance(dont_log, collections.abc.Sequence) - else (dont_log,) - ) + dont_log = tuple(dont_log) if isinstance(dont_log, Sequence) else (dont_log,) dont_log += (StopDownload,) spider = named.get("spider", None) - responses: List[Tuple[TypingAny, TypingAny]] = [] + responses: list[tuple[TypingAny, TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): result: TypingAny try: @@ -76,7 +71,7 @@ def send_catch_log_deferred( sender: TypingAny = Anonymous, *arguments: TypingAny, **named: TypingAny, -) -> Deferred[List[Tuple[TypingAny, TypingAny]]]: +) -> Deferred[list[tuple[TypingAny, TypingAny]]]: """Like send_catch_log but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were fired. @@ -94,14 +89,14 @@ def send_catch_log_deferred( dont_log = named.pop("dont_log", None) spider = named.get("spider", None) - dfds: List[Deferred[Tuple[TypingAny, TypingAny]]] = [] + dfds: list[Deferred[tuple[TypingAny, TypingAny]]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): d: Deferred[TypingAny] = maybeDeferred_coro( robustApply, receiver, signal=signal, sender=sender, *arguments, **named ) d.addErrback(logerror, receiver) # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html - d2: Deferred[Tuple[TypingAny, TypingAny]] = d.addBoth( + d2: Deferred[tuple[TypingAny, TypingAny]] = d.addBoth( lambda result: ( receiver, # pylint: disable=cell-var-from-loop # noqa: B023 result, @@ -109,7 +104,7 @@ def send_catch_log_deferred( ) dfds.append(d2) dl = DeferredList(dfds) - d3: Deferred[List[Tuple[TypingAny, TypingAny]]] = dl.addCallback( + d3: Deferred[list[tuple[TypingAny, TypingAny]]] = dl.addCallback( lambda out: [x[1] for x in out] ) return d3 diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 7a91afe59..1f70fcf69 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -5,11 +5,16 @@ Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ -from typing import Any, Dict, Iterable, Iterator, Optional, Union +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional, Union from urllib.parse import urljoin import lxml.etree # nosec +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index @@ -23,9 +28,9 @@ class Sitemap: rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt - def __iter__(self) -> Iterator[Dict[str, Any]]: + def __iter__(self) -> Iterator[dict[str, Any]]: for elem in self._root.getchildren(): - d: Dict[str, Any] = {} + d: dict[str, Any] = {} for el in elem.getchildren(): tag = el.tag name = tag.split("}", 1)[1] if "}" in tag else tag diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index ce754fad3..02dbb2e90 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,24 +2,14 @@ from __future__ import annotations import inspect import logging -from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, - Iterable, - Literal, - Optional, - Type, - TypeVar, - Union, - overload, -) +from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union, overload from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Iterable from types import CoroutineType, ModuleType from twisted.internet.defer import Deferred @@ -58,7 +48,7 @@ def iterate_spider_output( return arg_to_iter(deferred_from_coro(result)) -def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]: +def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ @@ -80,10 +70,10 @@ def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]: def spidercls_for_request( spider_loader: SpiderLoader, request: Request, - default_spidercls: Type[Spider], + default_spidercls: type[Spider], log_none: bool = ..., log_multiple: bool = ..., -) -> Type[Spider]: ... +) -> type[Spider]: ... @overload @@ -93,7 +83,7 @@ def spidercls_for_request( default_spidercls: Literal[None], log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: ... +) -> Optional[type[Spider]]: ... @overload @@ -103,16 +93,16 @@ def spidercls_for_request( *, log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: ... +) -> Optional[type[Spider]]: ... def spidercls_for_request( spider_loader: SpiderLoader, request: Request, - default_spidercls: Optional[Type[Spider]] = None, + default_spidercls: Optional[type[Spider]] = None, log_none: bool = False, log_multiple: bool = False, -) -> Optional[Type[Spider]]: +) -> Optional[type[Spider]]: """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 30f235592..860a2e3dd 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -9,17 +9,7 @@ import os from importlib import import_module from pathlib import Path from posixpath import split -from typing import ( - TYPE_CHECKING, - Any, - Awaitable, - Dict, - List, - Optional, - Tuple, - Type, - TypeVar, -) +from typing import TYPE_CHECKING, Any, Optional, TypeVar from unittest import TestCase, mock from twisted.trial.unittest import SkipTest @@ -29,6 +19,8 @@ from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available if TYPE_CHECKING: + from collections.abc import Awaitable + from twisted.internet.defer import Deferred from twisted.web.client import Response as TxResponse @@ -48,7 +40,7 @@ def skip_if_no_boto() -> None: def get_gcs_content_and_delete( bucket: Any, path: str -) -> Tuple[bytes, List[Dict[str, str]], Any]: +) -> tuple[bytes, list[dict[str, str]], Any]: from google.cloud import storage client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) @@ -75,7 +67,7 @@ def get_ftp_content_and_delete( ftp.login(username, password) if use_active_mode: ftp.set_pasv(False) - ftp_data: List[bytes] = [] + ftp_data: list[bytes] = [] def buffer_data(data: bytes) -> None: ftp_data.append(data) @@ -92,8 +84,8 @@ class TestSpider(Spider): def get_crawler( - spidercls: Optional[Type[Spider]] = None, - settings_dict: Optional[Dict[str, Any]] = None, + spidercls: Optional[type[Spider]] = None, + settings_dict: Optional[dict[str, Any]] = None, prevent_warnings: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it @@ -103,7 +95,7 @@ def get_crawler( from scrapy.crawler import CrawlerRunner # Set by default settings that prevent deprecation warnings. - settings: Dict[str, Any] = {} + settings: dict[str, Any] = {} settings.update(settings_dict or {}) runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or TestSpider) @@ -118,7 +110,7 @@ def get_pythonpath() -> str: return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get("PYTHONPATH", "") -def get_testenv() -> Dict[str, str]: +def get_testenv() -> dict[str, str]: """Return a OS environment dict suitable to fork processes that need to import this installation of Scrapy, instead of a system installed one. """ @@ -143,7 +135,7 @@ def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: return getter -def mock_google_cloud_storage() -> Tuple[Any, Any, Any]: +def mock_google_cloud_storage() -> tuple[Any, Any, Any]: """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. """ diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index bb269a9f5..dfc823725 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,13 +2,15 @@ from __future__ import annotations import os import sys -from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, cast +from typing import TYPE_CHECKING, Optional, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated from twisted.internet.protocol import ProcessProtocol if TYPE_CHECKING: + from collections.abc import Iterable + from twisted.python.failure import Failure @@ -36,8 +38,8 @@ class ProcessTest: return pp.deferred def _process_finished( - self, pp: TestProcessProtocol, cmd: List[str], check_code: bool - ) -> Tuple[int, bytes, bytes]: + self, pp: TestProcessProtocol, cmd: list[str], check_code: bool + ) -> tuple[int, bytes, bytes]: if pp.exitcode and check_code: msg = f"process {cmd} exit with code {pp.exitcode}" msg += f"\n>>> stdout <<<\n{pp.out.decode()}" diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 9ff9a273f..5eec1c10f 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -9,19 +9,23 @@ and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). """ +from __future__ import annotations + from collections import defaultdict from operator import itemgetter from time import time -from typing import TYPE_CHECKING, Any, DefaultDict, Iterable +from typing import TYPE_CHECKING, Any from weakref import WeakKeyDictionary if TYPE_CHECKING: + from collections.abc import Iterable + # typing.Self requires Python 3.11 from typing_extensions import Self NoneType = type(None) -live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) +live_refs: defaultdict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) class object_ref: @@ -29,7 +33,7 @@ class object_ref: __slots__ = () - def __new__(cls, *args: Any, **kwargs: Any) -> "Self": + def __new__(cls, *args: Any, **kwargs: Any) -> Self: obj = object.__new__(cls) live_refs[cls][obj] = time() return obj diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 9d97cb12f..41d268baa 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,8 +6,10 @@ Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ +from __future__ import annotations + import re -from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast +from typing import TYPE_CHECKING, Optional, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this @@ -18,6 +20,8 @@ from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Iterable + from scrapy import Spider @@ -33,7 +37,7 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: return any((host == d) or (host.endswith(f".{d}")) for d in domains) -def url_is_from_spider(url: UrlT, spider: Type["Spider"]) -> bool: +def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool: """Return True if the url belongs to the given spider""" return url_is_from_any_domain( url, [spider.name] + list(getattr(spider, "allowed_domains", [])) diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 42e5e9be4..4e9e29286 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -1,6 +1,5 @@ import platform import sys -from typing import List, Tuple import cryptography import cssselect @@ -13,7 +12,7 @@ import scrapy from scrapy.utils.ssl import get_openssl_version -def scrapy_components_versions() -> List[Tuple[str, str]]: +def scrapy_components_versions() -> list[tuple[str, str]]: lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) diff --git a/setup.py b/setup.py index f458a9de3..ec9ac6597 100644 --- a/setup.py +++ b/setup.py @@ -6,12 +6,12 @@ version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() install_requires = [ - "Twisted>=18.9.0", - "cryptography>=36.0.0", + "Twisted>=21.7.0", + "cryptography>=37.0.0", "cssselect>=0.9.1", "itemloaders>=1.0.1", "parsel>=1.5.0", - "pyOpenSSL>=21.0.0", + "pyOpenSSL>=22.0.0", "queuelib>=1.4.2", "service_identity>=18.1.0", "w3lib>=1.17.0", @@ -20,7 +20,7 @@ install_requires = [ "itemadapter>=0.1.0", "packaging", "tldextract", - "lxml>=4.4.1", + "lxml>=4.6.0", "defusedxml>=0.7.1", ] extras_require = { @@ -58,7 +58,6 @@ setup( "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -69,7 +68,7 @@ setup( "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ], - python_requires=">=3.8", + python_requires=">=3.9", install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index bde3de228..a7f7f1356 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -1,7 +1,7 @@ """DBM-like dummy module""" -import collections -from typing import Any, DefaultDict +from collections import defaultdict +from typing import Any class DummyDB(dict): @@ -14,7 +14,7 @@ class DummyDB(dict): error = KeyError -_DATABASES: DefaultDict[Any, DummyDB] = collections.defaultdict(DummyDB) +_DATABASES: defaultdict[Any, DummyDB] = defaultdict(DummyDB) def open(file, flag="r", mode=0o666): diff --git a/tests/mockserver.py b/tests/mockserver.py index 6ec46aa3d..f5c12787a 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -9,7 +9,7 @@ from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING from urllib.parse import urlencode from OpenSSL import SSL @@ -37,7 +37,7 @@ def getarg(request, name, default=None, type=None): return default -def get_mockserver_env() -> Dict[str, str]: +def get_mockserver_env() -> dict[str, str]: """Return a OS environment dict suitable to run mockserver processes.""" tests_path = Path(__file__).parent.parent diff --git a/tests/test_addons.py b/tests/test_addons.py index f1b01bc5c..775f629b3 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,5 +1,5 @@ import itertools -from typing import Any, Dict +from typing import Any from unittest.mock import patch from twisted.internet.defer import inlineCallbacks @@ -17,7 +17,7 @@ class SimpleAddon: pass -def get_addon_cls(config: Dict[str, Any]) -> type: +def get_addon_cls(config: dict[str, Any]) -> type: class AddonWithConfig: def update_settings(self, settings: BaseSettings): settings.update(config, priority="addon") diff --git a/tests/test_commands.py b/tests/test_commands.py index a23b7f4a9..6ec7c21b0 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import inspect import json @@ -13,7 +15,7 @@ from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import TemporaryFile, mkdtemp from threading import Timer -from typing import Dict, Iterator, Optional, Union +from typing import TYPE_CHECKING, Optional, Union from unittest import skipIf from pytest import mark @@ -27,6 +29,9 @@ from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv from tests.test_crawler import ExceptionSpider, NoRequestsSpider +if TYPE_CHECKING: + from collections.abc import Iterator + class CommandSettings(unittest.TestCase): def setUp(self): @@ -194,7 +199,7 @@ class StartprojectTest(ProjectTest): def get_permissions_dict( path: Union[str, os.PathLike], renamings=None, ignore=None -) -> Dict[str, str]: +) -> dict[str, str]: def get_permissions(path: Path) -> str: return oct(path.stat().st_mode) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c87e65758..69bfb7eb3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -6,7 +6,6 @@ import subprocess import sys import warnings from pathlib import Path -from typing import List import pytest from packaging.version import parse as parse_version @@ -651,7 +650,7 @@ class ScriptRunnerMixin: script_dir: Path cwd = os.getcwd() - def get_script_args(self, script_name: str, *script_args: str) -> List[str]: + def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name return [sys.executable, str(script_path)] + list(script_args) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 884491d01..f14a10a32 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -4,7 +4,7 @@ import shutil import sys from pathlib import Path from tempfile import mkdtemp, mkstemp -from typing import Optional, Type +from typing import Optional from unittest import SkipTest, mock from testfixtures import LogCapture @@ -218,7 +218,7 @@ class DuplicateHeaderResource(resource.Resource): class HttpTestCase(unittest.TestCase): scheme = "http" - download_handler_cls: Type = HTTPDownloadHandler + download_handler_cls: type = HTTPDownloadHandler # only used for HTTPS tests keyfile = "keys/localhost.key" @@ -428,7 +428,7 @@ class HttpTestCase(unittest.TestCase): class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" - download_handler_cls: Type = HTTP10DownloadHandler + download_handler_cls: type = HTTP10DownloadHandler def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -445,7 +445,7 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls: Type = HTTP11DownloadHandler + download_handler_cls: type = HTTP11DownloadHandler def test_download_without_maxsize_limit(self): request = Request(self.getURL("file")) @@ -645,7 +645,7 @@ class Https11InvalidDNSPattern(Https11TestCase): class Https11CustomCiphers(unittest.TestCase): scheme = "https" - download_handler_cls: Type = HTTP11DownloadHandler + download_handler_cls: type = HTTP11DownloadHandler keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" @@ -740,7 +740,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): - download_handler_cls: Type = HTTPDownloadHandler + download_handler_cls: type = HTTPDownloadHandler expected_http_proxy_request_body = b"http://example.com" def setUp(self): @@ -783,14 +783,14 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): - download_handler_cls: Type = HTTP10DownloadHandler + download_handler_cls: type = HTTP10DownloadHandler def test_download_with_proxy_https_noconnect(self): raise unittest.SkipTest("noconnect is not supported in HTTP10DownloadHandler") class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls: Type = HTTP11DownloadHandler + download_handler_cls: type = HTTP11DownloadHandler @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -845,7 +845,7 @@ class S3AnonTestCase(unittest.TestCase): class S3TestCase(unittest.TestCase): - download_handler_cls: Type = S3DownloadHandler + download_handler_cls: type = S3DownloadHandler # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7ea3fe8c9..1f998de1a 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -8,7 +8,7 @@ import string from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING from unittest import mock, skipIf from urllib.parse import urlencode @@ -152,7 +152,7 @@ class QueryParams(LeafResource): request.setHeader("Content-Type", "application/json; charset=UTF-8") request.setHeader("Content-Encoding", "UTF-8") - query_params: Dict[str, str] = {} + query_params: dict[str, str] = {} assert request.args is not None for k, v in request.args.items(): query_params[str(k, "utf-8")] = str(v[0], "utf-8") diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 7ce73e6ff..d0fb17f1f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,7 +3,7 @@ import re import unittest import warnings import xmlrpc.client -from typing import Any, Dict, List +from typing import Any from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes @@ -23,8 +23,8 @@ from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): request_class = Request default_method = "GET" - default_headers: Dict[bytes, List[bytes]] = {} - default_meta: Dict[str, Any] = {} + default_headers: dict[bytes, list[bytes]] = {} + default_meta: dict[str, Any] = {} def test_init(self): # Request requires url in the __init__ method diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index cd3442dd4..83e22b070 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,7 +1,7 @@ import shutil from pathlib import Path from tempfile import mkdtemp -from typing import Optional, Set +from typing import Optional from testfixtures import LogCapture from twisted.internet import defer @@ -57,7 +57,7 @@ class FileDownloadCrawlTestCase(TestCase): store_setting_key = "FILES_STORE" media_key = "files" media_urls_key = "file_urls" - expected_checksums: Optional[Set[str]] = { + expected_checksums: Optional[set[str]] = { "5547178b89448faf0015a13f904c936e", "c2281c83670e31d8aaab7cb642b824db", "ed3f6538dc15d4d9179dae57319edc5f", diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 0babde4d9..6ce7fc059 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -7,7 +7,6 @@ from io import BytesIO from pathlib import Path from shutil import rmtree from tempfile import mkdtemp -from typing import Dict, List from unittest import mock from urllib.parse import urlparse @@ -309,11 +308,11 @@ class FilesPipelineTestCaseFieldsDataClass( class FilesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - file_urls: List[str] = attr.ib(default=lambda: []) - files: List[Dict[str, str]] = attr.ib(default=lambda: []) + file_urls: list[str] = attr.ib(default=lambda: []) + files: list[dict[str, str]] = attr.ib(default=lambda: []) # overridden fields - custom_file_urls: List[str] = attr.ib(default=lambda: []) - custom_files: List[Dict[str, str]] = attr.ib(default=lambda: []) + custom_file_urls: list[str] = attr.ib(default=lambda: []) + custom_files: list[dict[str, str]] = attr.ib(default=lambda: []) class FilesPipelineTestCaseFieldsAttrsItem( diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 7d7c78920..296a6fae0 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -5,7 +5,7 @@ import random import warnings from shutil import rmtree from tempfile import mkdtemp -from typing import Dict, List, Optional +from typing import Optional from unittest.mock import patch import attr @@ -406,11 +406,11 @@ class ImagesPipelineTestCaseFieldsDataClass( class ImagesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - image_urls: List[str] = attr.ib(default=lambda: []) - images: List[Dict[str, str]] = attr.ib(default=lambda: []) + image_urls: list[str] = attr.ib(default=lambda: []) + images: list[dict[str, str]] = attr.ib(default=lambda: []) # overridden fields - custom_image_urls: List[str] = attr.ib(default=lambda: []) - custom_images: List[Dict[str, str]] = attr.ib(default=lambda: []) + custom_image_urls: list[str] = attr.ib(default=lambda: []) + custom_images: list[dict[str, str]] = attr.ib(default=lambda: []) class ImagesPipelineTestCaseFieldsAttrsItem( diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 7299972f6..8c0e5764a 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,5 +1,3 @@ -from typing import List - from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -64,7 +62,7 @@ class KeywordArgumentsSpider(MockServerSpider): }, } - checks: List[bool] = [] + checks: list[bool] = [] def start_requests(self): data = {"key": "value", "number": 123, "callback": "some_callback"} diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 5db2e4e50..4fd293ec7 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Optional from unittest import TestCase from urllib.parse import urljoin @@ -20,7 +20,7 @@ URLS = [urljoin("https://example.org", p) for p in PATHS] class MinimalScheduler: def __init__(self) -> None: - self.requests: Dict[bytes, Request] = {} + self.requests: dict[bytes, Request] = {} def has_pending_requests(self) -> bool: return bool(self.requests) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 9ee248538..503c29e32 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -105,9 +105,10 @@ class BaseSettingsTest(unittest.TestCase): def test_set_calls_settings_attributes_methods_on_update(self): attr = SettingsAttribute("value", 10) - with mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object( - attr, "set" - ) as mock_set: + with ( + mock.patch.object(attr, "__setattr__") as mock_setattr, + mock.patch.object(attr, "set") as mock_set, + ): self.settings.attributes = {"TEST_OPTION": attr} for priority in (0, 10, 20): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 9dbffe353..41228b5f2 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,4 +1,4 @@ -import collections.abc +from collections.abc import AsyncIterator, Iterable from typing import Optional, Union from unittest import mock @@ -147,7 +147,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, collections.abc.Iterable) + self.assertIsInstance(result, Iterable) result_list = list(result) self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) @@ -161,7 +161,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, collections.abc.AsyncIterator) + self.assertIsInstance(result, AsyncIterator) result_list = yield deferred_from_coro(collect_asyncgen(result)) self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 044455415..01a2b4bb4 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,5 +1,4 @@ import logging -from typing import Set from unittest import TestCase from testfixtures import LogCapture @@ -17,7 +16,7 @@ from tests.spiders import MockServerSpider class _HttpErrorSpider(MockServerSpider): name = "httperror" - bypass_status_codes: Set[int] = set() + bypass_status_codes: set[int] = set() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 5797edfbd..e73e7ff4c 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,5 +1,5 @@ import warnings -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional from unittest import TestCase from urllib.parse import urlparse @@ -32,10 +32,10 @@ from scrapy.spiders import Spider class TestRefererMiddleware(TestCase): - req_meta: Dict[str, Any] = {} - resp_headers: Dict[str, str] = {} - settings: Dict[str, Any] = {} - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + req_meta: dict[str, Any] = {} + resp_headers: dict[str, str] = {} + settings: dict[str, Any] = {} + scenarii: list[tuple[str, str, Optional[bytes]]] = [ ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] @@ -65,7 +65,7 @@ class MixinDefault: with some additional filtering of s3:// """ - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ ("https://example.com/", "https://scrapy.org/", b"https://example.com/"), ("http://example.com/", "http://scrapy.org/", b"http://example.com/"), ("http://example.com/", "https://scrapy.org/", b"http://example.com/"), @@ -86,7 +86,7 @@ class MixinDefault: class MixinNoReferrer: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ ("https://example.com/page.html", "https://example.com/", None), ("http://www.example.com/", "https://scrapy.org/", None), ("http://www.example.com/", "http://scrapy.org/", None), @@ -96,7 +96,7 @@ class MixinNoReferrer: class MixinNoReferrerWhenDowngrade: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # TLS to TLS: send non-empty referrer ( "https://example.com/page.html", @@ -178,7 +178,7 @@ class MixinNoReferrerWhenDowngrade: class MixinSameOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -247,7 +247,7 @@ class MixinSameOrigin: class MixinOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) ( "https://example.com/page.html", @@ -271,7 +271,7 @@ class MixinOrigin: class MixinStrictOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades ( "https://example.com/page.html", @@ -299,7 +299,7 @@ class MixinStrictOrigin: class MixinOriginWhenCrossOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -406,7 +406,7 @@ class MixinOriginWhenCrossOrigin: class MixinStrictOriginWhenCrossOrigin: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -518,7 +518,7 @@ class MixinStrictOriginWhenCrossOrigin: class MixinUnsafeUrl: - scenarii: List[Tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, Optional[bytes]]] = [ # TLS to TLS: send referrer ( "https://example.com/sekrit.html", @@ -968,8 +968,8 @@ class TestPolicyHeaderPrecedence004( class TestReferrerOnRedirect(TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} - scenarii: List[ - Tuple[str, str, Tuple[Tuple[int, str], ...], Optional[bytes], Optional[bytes]] + scenarii: list[ + tuple[str, str, tuple[tuple[int, str], ...], Optional[bytes], Optional[bytes]] ] = [ # type: ignore[assignment] ( "http://scrapytest.org/1", # parent diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index be5c6de81..fb7c90f80 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,8 +1,7 @@ import copy import unittest import warnings -from collections.abc import Mapping, MutableMapping -from typing import Iterator +from collections.abc import Iterator, Mapping, MutableMapping from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 0f75bdb5c..76820eabf 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import json import logging import re import sys import unittest from io import StringIO -from typing import Any, Dict, Mapping, MutableMapping +from typing import TYPE_CHECKING, Any from unittest import TestCase import pytest @@ -21,6 +23,9 @@ from scrapy.utils.log import ( from scrapy.utils.test import get_crawler from tests.spiders import LogSpider +if TYPE_CHECKING: + from collections.abc import Mapping, MutableMapping + class FailureToExcInfoTest(unittest.TestCase): def test_failure(self): @@ -133,7 +138,7 @@ class StreamLoggerTest(unittest.TestCase): ), ) def test_spider_logger_adapter_process( - base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: Dict + base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict ): logger = logging.getLogger("test") spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 633077eec..ca3bca0b2 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -2,7 +2,7 @@ import json import unittest import warnings from hashlib import sha1 -from typing import Dict, Optional, Tuple, Union +from typing import Optional, Union from weakref import WeakKeyDictionary from scrapy.http import Request @@ -57,11 +57,11 @@ class FingerprintTest(unittest.TestCase): function: staticmethod = staticmethod(fingerprint) cache: Union[ - "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]", - "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]", + "WeakKeyDictionary[Request, dict[tuple[Optional[tuple[bytes, ...]], bool], bytes]]", + "WeakKeyDictionary[Request, dict[tuple[Optional[tuple[bytes, ...]], bool], str]]", ] = _fingerprint_cache default_cache_key = (None, False) - known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( + known_hashes: tuple[tuple[Request, Union[bytes, str], dict], ...] = ( ( Request("http://example.org"), b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD", diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index 665db9088..3926c830f 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -16,7 +16,7 @@ class MyRequest2(Request): @pytest.mark.mypy_testing def mypy_test_headers(): - Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[tuple[str, Any]], None]" Request("data:,", headers=None) Request("data:,", headers={}) Request("data:,", headers=[]) diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index d58ac1027..88aedbd3e 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse @pytest.mark.mypy_testing def mypy_test_headers(): - Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[tuple[str, Any]], None]" Response("data:,", headers=None) Response("data:,", headers={}) Response("data:,", headers=[]) diff --git a/tox.ini b/tox.ini index 80ef4a99e..dad15c6ab 100644 --- a/tox.ini +++ b/tox.ini @@ -61,7 +61,7 @@ commands = mypy {posargs: scrapy tests} [testenv:typing-tests] -basepython = python3.8 +basepython = python3.9 deps = {[test-requirements]deps} {[testenv:typing]deps} @@ -94,21 +94,21 @@ commands = twine check dist/* [pinned] -basepython = python3.8 +basepython = python3.9 deps = - cryptography==36.0.0 + cryptography==37.0.0 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - pyOpenSSL==21.0.0 + pyOpenSSL==22.0.0 queuelib==1.4.2 service_identity==18.1.0 - Twisted[http2]==18.9.0 + Twisted[http2]==21.7.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.4.1 + lxml==4.6.0 {[test-requirements]deps} # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies @@ -194,7 +194,7 @@ commands = pytest {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] -basepython = pypy3.8 +basepython = pypy3.9 deps = {[pinned]deps} PyPyDispatcher==2.1.0 From 9bd5e5bcdbad47aee3e5d141c74e5d029502904c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 16 Oct 2024 14:50:57 +0500 Subject: [PATCH 1540/2083] Revert uvloop restrictions. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 451860653..dad15c6ab 100644 --- a/tox.ini +++ b/tox.ini @@ -149,8 +149,8 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - uvloop; platform_system != "Windows" and python_version < '3.13' - bpython # optional for shell wrapper tests + uvloop; platform_system != "Windows" + bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests From f65e64a7243d725d35bbf86ca6f5ae4c350dbcc5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 16 Oct 2024 21:38:43 +0500 Subject: [PATCH 1541/2083] Misc typing improvements. (#6494) --- scrapy/commands/check.py | 5 ++- scrapy/commands/genspider.py | 4 +-- scrapy/commands/parse.py | 6 ++-- scrapy/core/engine.py | 4 +-- scrapy/core/scraper.py | 6 ++-- scrapy/core/spidermw.py | 4 +-- scrapy/crawler.py | 34 +++++++++---------- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/pipelines/files.py | 2 +- scrapy/utils/defer.py | 6 +++- tox.ini | 8 ++--- 13 files changed, 46 insertions(+), 39 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index c7946605b..1ce155da7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -13,8 +13,7 @@ from scrapy.utils.misc import load_object, set_environ class TextTestResult(_TextTestResult): def printSummary(self, start: float, stop: float) -> None: write = self.stream.write - # _WritelnDecorator isn't implemented in typeshed yet - writeln = self.stream.writeln # type: ignore[attr-defined] + writeln = self.stream.writeln run = self.testsRun plural = "s" if run != 1 else "" @@ -84,7 +83,7 @@ class Command(ScrapyCommand): with set_environ(SCRAPY_CHECK="true"): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) - spidercls.start_requests = lambda s: conman.from_spider(s, result) + spidercls.start_requests = lambda s: conman.from_spider(s, result) # type: ignore[assignment,method-assign,return-value] tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index a9b7a6eee..2ac281212 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,7 +4,7 @@ import shutil import string from importlib import import_module from pathlib import Path -from typing import Optional, Union, cast +from typing import Any, Optional, Union, cast from urllib.parse import urlparse import scrapy @@ -122,7 +122,7 @@ class Command(ScrapyCommand): name: str, url: str, template_name: str, - ): + ) -> dict[str, Any]: capitalized_module = "".join(s.capitalize() for s in module.split("_")) return { "project_name": self.settings.get("BOT_NAME"), diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index bd1fad14b..ff2bb8ab9 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -38,9 +38,10 @@ _T = TypeVar("_T") class Command(BaseRunSpiderCommand): requires_project = True - spider = None + spider: Optional[Spider] = None items: dict[int, list[Any]] = {} requests: dict[int, list[Request]] = {} + spidercls: Optional[type[Spider]] first_response = None @@ -261,10 +262,11 @@ class Command(BaseRunSpiderCommand): yield self.prepare_request(spider, Request(url), opts) if self.spidercls: - self.spidercls.start_requests = _start_requests + self.spidercls.start_requests = _start_requests # type: ignore[assignment,method-assign] def start_parsing(self, url: str, opts: argparse.Namespace) -> None: assert self.crawler_process + assert self.spidercls self.crawler_process.crawl(self.spidercls, **opts.spargs) self.pcrawler = list(self.crawler_process.crawlers)[0] self.crawler_process.start() diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index bb09d066f..f3d74eccf 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -100,7 +100,7 @@ class ExecutionEngine: ) downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) - self.scraper = Scraper(crawler) + self.scraper: Scraper = Scraper(crawler) self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = ( spider_closed_callback ) @@ -325,7 +325,7 @@ class ExecutionEngine: raise RuntimeError(f"No open spider to crawl: {request}") d: Deferred[Union[Response, Request]] = self._download(request) # Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type - d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[arg-type] + d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[call-overload] return d2 def _downloaded( diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 29d7cb0c8..71a0d6aeb 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -55,7 +55,7 @@ class Slot: MIN_RESPONSE_SIZE = 1024 def __init__(self, max_active_size: int = 5000000): - self.max_active_size = max_active_size + self.max_active_size: int = max_active_size self.queue: deque[QueueTuple] = deque() self.active: set[Request] = set() self.active_size: int = 0 @@ -316,7 +316,9 @@ class Scraper: ) return None - def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]: + def start_itemproc( + self, item: Any, *, response: Optional[Response] + ) -> Deferred[Any]: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 223e4192e..3c8513042 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -72,7 +72,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_input( self, - scrape_func: ScrapeFunc, + scrape_func: ScrapeFunc[_T], response: Response, request: Request, spider: Spider, @@ -306,7 +306,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def scrape_response( self, - scrape_func: ScrapeFunc, + scrape_func: ScrapeFunc[_T], response: Response, request: Request, spider: Spider, diff --git a/scrapy/crawler.py b/scrapy/crawler.py index b0a4932e1..e75ef52ac 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -42,8 +42,9 @@ from scrapy.utils.reactor import ( ) if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import Generator, Iterable + from scrapy.spiderloader import SpiderLoader from scrapy.utils.request import RequestFingerprinter @@ -178,16 +179,18 @@ class Crawler: yield maybeDeferred(self.engine.stop) @staticmethod - def _get_component(component_class, components): + def _get_component( + component_class: type[_T], components: Iterable[Any] + ) -> Optional[_T]: for component in components: if isinstance(component, component_class): return component return None - def get_addon(self, cls): + def get_addon(self, cls: type[_T]) -> Optional[_T]: return self._get_component(cls, self.addons.addons) - def get_downloader_middleware(self, cls): + def get_downloader_middleware(self, cls: type[_T]) -> Optional[_T]: if not self.engine: raise RuntimeError( "Crawler.get_downloader_middleware() can only be called after " @@ -195,7 +198,7 @@ class Crawler: ) return self._get_component(cls, self.engine.downloader.middleware.middlewares) - def get_extension(self, cls): + def get_extension(self, cls: type[_T]) -> Optional[_T]: if not self.extensions: raise RuntimeError( "Crawler.get_extension() can only be called after the " @@ -203,7 +206,7 @@ class Crawler: ) return self._get_component(cls, self.extensions.middlewares) - def get_item_pipeline(self, cls): + def get_item_pipeline(self, cls: type[_T]) -> Optional[_T]: if not self.engine: raise RuntimeError( "Crawler.get_item_pipeline() can only be called after the " @@ -211,7 +214,7 @@ class Crawler: ) return self._get_component(cls, self.engine.scraper.itemproc.middlewares) - def get_spider_middleware(self, cls): + def get_spider_middleware(self, cls: type[_T]) -> Optional[_T]: if not self.engine: raise RuntimeError( "Crawler.get_spider_middleware() can only be called after the " @@ -240,18 +243,18 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings: BaseSettings): + def _get_spider_loader(settings: BaseSettings) -> SpiderLoader: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) verifyClass(ISpiderLoader, loader_cls) - return loader_cls.from_settings(settings.frozencopy()) + return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy())) def __init__(self, settings: Union[dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) - self.settings = settings - self.spider_loader = self._get_spider_loader(settings) + self.settings: Settings = settings + self.spider_loader: SpiderLoader = self._get_spider_loader(settings) self._crawlers: set[Crawler] = set() self._active: set[Deferred[None]] = set() self.bootstrap_failed = False @@ -329,8 +332,7 @@ class CrawlerRunner: def _create_crawler(self, spidercls: Union[str, type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - # temporary cast until self.spider_loader is typed - return Crawler(cast(type[Spider], spidercls), self.settings) + return Crawler(spidercls, self.settings) def stop(self) -> Deferred[Any]: """ @@ -384,7 +386,7 @@ class CrawlerProcess(CrawlerRunner): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) - self._initialized_reactor = False + self._initialized_reactor: bool = False def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor @@ -413,9 +415,7 @@ class CrawlerProcess(CrawlerRunner): init_reactor = not self._initialized_reactor self._initialized_reactor = True # temporary cast until self.spider_loader is typed - return Crawler( - cast(type[Spider], spidercls), self.settings, init_reactor=init_reactor - ) + return Crawler(spidercls, self.settings, init_reactor=init_reactor) def start( self, stop_after_crawl: bool = True, install_signal_handlers: bool = True diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index b0cede97d..d913ca25d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -88,7 +88,7 @@ class HttpCompressionMiddleware: crawler.signals.connect(mw.open_spider, signals.spider_opened) return mw - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> None: if hasattr(spider, "download_maxsize"): self._max_size = spider.download_maxsize if hasattr(spider, "download_warnsize"): diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 421c58e68..81ba009d6 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -67,7 +67,7 @@ class RobotsTxtMiddleware: if request.url.startswith("data:") or request.url.startswith("file:"): return None d: Deferred[Optional[RobotParser]] = maybeDeferred( - self.robot_parser, request, spider # type: ignore[arg-type] + self.robot_parser, request, spider # type: ignore[call-overload] ) d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider) return d2 diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index b1001dabb..7bfcbe6f3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -578,7 +578,7 @@ class FeedExporter: return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[arg-type] + d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[call-overload] d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 9314856c1..32e9ffe7c 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -550,7 +550,7 @@ class FilesPipeline(MediaPipeline): path = self.file_path(request, info=info, item=item) # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type - dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[arg-type] + dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload] dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess) dfd2.addErrback(lambda _: None) dfd2.addErrback( diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 3a0dee8f1..aeacadb1c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -305,7 +305,11 @@ def process_parallel( dfds, fireOnOneErrback=True, consumeErrors=True ) d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) - d2.addErrback(lambda f: f.value.subFailure) + + def eb(failure: Failure) -> Failure: + return failure.value.subFailure + + d2.addErrback(eb) return d2 diff --git a/tox.ini b/tox.ini index dad15c6ab..79f72a0f2 100644 --- a/tox.ini +++ b/tox.ini @@ -46,12 +46,12 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.11.1 + mypy==1.12.0 typing-extensions==4.12.2 - types-lxml==2024.8.7 + types-lxml==2024.9.16 types-Pygments==2.18.0.20240506 - botocore-stubs==1.34.158 - boto3-stubs[s3]==1.34.158 + botocore-stubs==1.35.39 + boto3-stubs[s3]==1.35.39 attrs >= 18.2.0 Pillow >= 10.3.0 pyOpenSSL >= 24.2.1 From c8e87ab21a216c546baa797b9a4e6fe27751a4d3 Mon Sep 17 00:00:00 2001 From: Julian Ste <31321934+julian-st@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:03:16 +0200 Subject: [PATCH 1542/2083] Fixed typos (#6497) --- docs/faq.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index d394406e8..0b650f522 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -269,7 +269,7 @@ To dump into a CSV file:: scrapy crawl myspider -O items.csv -To dump into a XML file:: +To dump into an XML file:: scrapy crawl myspider -O items.xml @@ -417,8 +417,8 @@ How can I make a blank request? blank_request = Request("data:,") -In this case, the URL is set to a data URI scheme. Data URLs allow you to include data -in-line in web pages as if they were external resources. The "data:" scheme with an empty +In this case, the URL is set to a data URI scheme. Data URLs allow you to include data +inline within web pages, similar to external resources. The "data:" scheme with an empty content (",") essentially creates a request to a data URL without any specific content. From c9095ef927bc42e8f23c5d02c05a7b918f7aa5bf Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 17 Oct 2024 21:22:34 +0500 Subject: [PATCH 1543/2083] Remove --keep-runtime-typing from pyupgrade. --- .pre-commit-config.yaml | 4 +- scrapy/cmdline.py | 10 +-- scrapy/commands/__init__.py | 6 +- scrapy/commands/genspider.py | 8 +- scrapy/commands/parse.py | 22 +++--- scrapy/commands/runspider.py | 4 +- scrapy/commands/startproject.py | 5 +- scrapy/contracts/__init__.py | 12 ++- scrapy/contracts/default.py | 6 +- scrapy/core/downloader/__init__.py | 12 ++- scrapy/core/downloader/contextfactory.py | 6 +- scrapy/core/downloader/handlers/__init__.py | 12 +-- scrapy/core/downloader/handlers/ftp.py | 8 +- scrapy/core/downloader/handlers/http11.py | 62 ++++++++-------- scrapy/core/downloader/handlers/http2.py | 8 +- scrapy/core/downloader/handlers/s3.py | 8 +- scrapy/core/downloader/middleware.py | 14 ++-- scrapy/core/downloader/webclient.py | 6 +- scrapy/core/engine.py | 41 +++++------ scrapy/core/http2/agent.py | 10 +-- scrapy/core/http2/protocol.py | 8 +- scrapy/core/http2/stream.py | 4 +- scrapy/core/scheduler.py | 46 ++++++------ scrapy/core/scraper.py | 51 ++++++------- scrapy/core/spidermw.py | 56 +++++++------- scrapy/crawler.py | 40 +++++----- scrapy/downloadermiddlewares/ajaxcrawl.py | 4 +- scrapy/downloadermiddlewares/cookies.py | 8 +- .../downloadermiddlewares/defaultheaders.py | 4 +- .../downloadermiddlewares/downloadtimeout.py | 4 +- scrapy/downloadermiddlewares/httpauth.py | 4 +- scrapy/downloadermiddlewares/httpcache.py | 16 ++-- .../downloadermiddlewares/httpcompression.py | 10 +-- scrapy/downloadermiddlewares/httpproxy.py | 20 ++--- scrapy/downloadermiddlewares/redirect.py | 6 +- scrapy/downloadermiddlewares/retry.py | 18 ++--- scrapy/downloadermiddlewares/robotstxt.py | 22 +++--- scrapy/downloadermiddlewares/stats.py | 10 +-- scrapy/downloadermiddlewares/useragent.py | 4 +- scrapy/dupefilters.py | 12 +-- scrapy/exporters.py | 28 +++---- scrapy/extensions/corestats.py | 4 +- scrapy/extensions/debug.py | 6 +- scrapy/extensions/feedexport.py | 72 +++++++++--------- scrapy/extensions/httpcache.py | 26 +++---- scrapy/extensions/logstats.py | 6 +- scrapy/extensions/periodic_log.py | 14 ++-- scrapy/extensions/spiderstate.py | 6 +- scrapy/extensions/statsmailer.py | 4 +- scrapy/extensions/throttle.py | 6 +- scrapy/http/cookies.py | 12 +-- scrapy/http/headers.py | 14 ++-- scrapy/http/request/__init__.py | 33 ++++----- scrapy/http/request/form.py | 32 ++++---- scrapy/http/request/json_request.py | 6 +- scrapy/http/request/rpc.py | 6 +- scrapy/http/response/__init__.py | 66 ++++++++--------- scrapy/http/response/text.py | 73 +++++++++---------- scrapy/linkextractors/lxmlhtml.py | 28 +++---- scrapy/loader/__init__.py | 8 +- scrapy/logformatter.py | 14 ++-- scrapy/mail.py | 22 +++--- scrapy/middleware.py | 12 ++- scrapy/pipelines/files.py | 72 ++++++++---------- scrapy/pipelines/images.py | 24 +++--- scrapy/pipelines/media.py | 27 ++++--- scrapy/pqueues.py | 18 ++--- scrapy/resolver.py | 4 +- scrapy/responsetypes.py | 14 ++-- scrapy/robotstxt.py | 30 ++++---- scrapy/selector/unified.py | 16 ++-- scrapy/settings/__init__.py | 38 ++++------ scrapy/shell.py | 34 ++++----- scrapy/spidermiddlewares/httperror.py | 4 +- scrapy/spidermiddlewares/referer.py | 36 +++++---- scrapy/spiders/__init__.py | 10 +-- scrapy/spiders/crawl.py | 38 +++++----- scrapy/spiders/feed.py | 8 +- scrapy/spiders/init.py | 4 +- scrapy/spiders/sitemap.py | 14 ++-- scrapy/squeues.py | 14 ++-- scrapy/statscollectors.py | 30 ++++---- scrapy/utils/asyncgen.py | 8 +- scrapy/utils/conf.py | 12 +-- scrapy/utils/console.py | 8 +- scrapy/utils/curl.py | 6 +- scrapy/utils/datatypes.py | 14 ++-- scrapy/utils/defer.py | 16 ++-- scrapy/utils/deprecate.py | 14 ++-- scrapy/utils/httpobj.py | 6 +- scrapy/utils/iterators.py | 34 ++++----- scrapy/utils/job.py | 6 +- scrapy/utils/log.py | 14 ++-- scrapy/utils/misc.py | 6 +- scrapy/utils/ossignal.py | 2 + scrapy/utils/project.py | 5 +- scrapy/utils/python.py | 22 +++--- scrapy/utils/reactor.py | 8 +- scrapy/utils/request.py | 14 ++-- scrapy/utils/response.py | 12 +-- scrapy/utils/sitemap.py | 6 +- scrapy/utils/spider.py | 14 ++-- scrapy/utils/ssl.py | 4 +- scrapy/utils/template.py | 4 +- scrapy/utils/test.py | 8 +- scrapy/utils/testproc.py | 8 +- scrapy/utils/url.py | 4 +- .../CrawlerProcess/asyncio_deferred_signal.py | 5 +- tests/spiders.py | 13 ++-- tests/test_commands.py | 8 +- tests/test_downloader_handlers.py | 5 +- tests/test_feedexport.py | 4 +- tests/test_linkextractors.py | 5 +- tests/test_loader.py | 5 +- tests/test_pipeline_crawl.py | 7 +- tests/test_pipeline_images.py | 5 +- tests/test_pipeline_media.py | 4 +- tests/test_scheduler.py | 7 +- tests/test_scheduler_base.py | 5 +- tests/test_spidermiddleware.py | 17 +++-- tests/test_spidermiddleware_referer.py | 26 ++++--- tests/test_utils_request.py | 13 ++-- 122 files changed, 947 insertions(+), 981 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75529be05..fbd710f6f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,7 +30,7 @@ repos: additional_dependencies: - black==24.4.2 - repo: https://github.com/asottile/pyupgrade - rev: v3.16.0 + rev: v3.18.0 hooks: - id: pyupgrade - args: [--py39-plus, --keep-runtime-typing] + args: [--py39-plus] diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index b820eb7f9..b6f19a37f 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,7 +6,7 @@ import inspect import os import sys from importlib.metadata import entry_points -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -30,7 +30,7 @@ if TYPE_CHECKING: class ScrapyArgumentParser(argparse.ArgumentParser): def _parse_optional( self, arg_string: str - ) -> Optional[tuple[Optional[argparse.Action], str, Optional[str]]]: + ) -> tuple[argparse.Action | None, str, str | None] | None: # if starts with -: it means that is a parameter not a argument if arg_string[:2] == "-:": return None @@ -89,7 +89,7 @@ def _get_commands_dict( return cmds -def _pop_command_name(argv: list[str]) -> Optional[str]: +def _pop_command_name(argv: list[str]) -> str | None: i = 0 for arg in argv[1:]: if not arg.startswith("-"): @@ -147,9 +147,7 @@ def _run_print_help( sys.exit(2) -def execute( - argv: Optional[list[str]] = None, settings: Optional[Settings] = None -) -> None: +def execute(argv: list[str] | None = None, settings: Settings | None = None) -> None: if argv is None: argv = sys.argv diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index a94db90b1..eccbef040 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -8,7 +8,7 @@ import argparse import builtins import os from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from twisted.python import failure @@ -23,7 +23,7 @@ if TYPE_CHECKING: class ScrapyCommand: requires_project: bool = False - crawler_process: Optional[CrawlerProcess] = None + crawler_process: CrawlerProcess | None = None # default settings to be used for this command instead of global defaults default_settings: dict[str, Any] = {} @@ -195,7 +195,7 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): prog: str, indent_increment: int = 2, max_help_position: int = 24, - width: Optional[int] = None, + width: int | None = None, ): super().__init__( prog, diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2ac281212..b286e703e 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import argparse import os import shutil import string from importlib import import_module from pathlib import Path -from typing import Any, Optional, Union, cast +from typing import Any, cast from urllib.parse import urlparse import scrapy @@ -140,7 +142,7 @@ class Command(ScrapyCommand): name: str, url: str, template_name: str, - template_file: Union[str, os.PathLike], + template_file: str | os.PathLike, ) -> None: """Generate the spider module, based on the given template""" tvars = self._generate_template_variables(module, name, url, template_name) @@ -161,7 +163,7 @@ class Command(ScrapyCommand): if spiders_module: print(f"in module:\n {spiders_module.__name__}.{module}") - def _find_template(self, template: str) -> Optional[Path]: + def _find_template(self, template: str) -> Path | None: template_file = Path(self.templates_dir, f"{template}.tmpl") if template_file.exists(): return template_file diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ff2bb8ab9..2059dcf75 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -5,7 +5,7 @@ import functools import inspect import json import logging -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, TypeVar, overload from itemadapter import ItemAdapter, is_item from twisted.internet.defer import Deferred, maybeDeferred @@ -38,10 +38,10 @@ _T = TypeVar("_T") class Command(BaseRunSpiderCommand): requires_project = True - spider: Optional[Spider] = None + spider: Spider | None = None items: dict[int, list[Any]] = {} requests: dict[int, list[Request]] = {} - spidercls: Optional[type[Spider]] + spidercls: type[Spider] | None first_response = None @@ -137,13 +137,13 @@ class Command(BaseRunSpiderCommand): @overload def iterate_spider_output( - self, result: Union[AsyncGenerator[_T, None], Coroutine[Any, Any, _T]] + self, result: AsyncGenerator[_T] | Coroutine[Any, Any, _T] ) -> Deferred[_T]: ... @overload def iterate_spider_output(self, result: _T) -> Iterable[Any]: ... - def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred[Any]]: + def iterate_spider_output(self, result: Any) -> Iterable[Any] | Deferred[Any]: if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) @@ -164,7 +164,7 @@ class Command(BaseRunSpiderCommand): old_reqs = self.requests.get(lvl, []) self.requests[lvl] = old_reqs + new_reqs - def print_items(self, lvl: Optional[int] = None, colour: bool = True) -> None: + def print_items(self, lvl: int | None = None, colour: bool = True) -> None: if lvl is None: items = [item for lst in self.items.values() for item in lst] else: @@ -173,7 +173,7 @@ class Command(BaseRunSpiderCommand): print("# Scraped Items ", "-" * 60) display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour) - def print_requests(self, lvl: Optional[int] = None, colour: bool = True) -> None: + def print_requests(self, lvl: int | None = None, colour: bool = True) -> None: if lvl is None: if self.requests: requests = self.requests[max(self.requests)] @@ -222,7 +222,7 @@ class Command(BaseRunSpiderCommand): self, response: Response, callback: CallbackT, - cb_kwargs: Optional[dict[str, Any]] = None, + cb_kwargs: dict[str, Any] | None = None, ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) @@ -230,7 +230,7 @@ class Command(BaseRunSpiderCommand): def get_callback_from_rules( self, spider: Spider, response: Response - ) -> Union[CallbackT, str, None]: + ) -> CallbackT | str | None: if getattr(spider, "rules", None): for rule in spider.rules: # type: ignore[attr-defined] if rule.link_extractor.matches(response.url): @@ -303,9 +303,9 @@ class Command(BaseRunSpiderCommand): *, spider: Spider, opts: argparse.Namespace, - response: Optional[Response] = None, + response: Response | None = None, ) -> CallbackT: - cb: Union[str, CallbackT, None] = None + cb: str | CallbackT | None = None if response: cb = response.meta["_callback"] if not cb: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 14d58f311..7ec56899c 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -4,7 +4,7 @@ import argparse import sys from importlib import import_module from pathlib import Path -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -15,7 +15,7 @@ if TYPE_CHECKING: from types import ModuleType -def _import_file(filepath: Union[str, PathLike[str]]) -> ModuleType: +def _import_file(filepath: str | PathLike[str]) -> ModuleType: abspath = Path(filepath).resolve() if abspath.suffix not in (".py", ".pyw"): raise ValueError(f"Not a Python source file: {abspath}") diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index f7052cd18..f54c02369 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import os import re @@ -6,7 +8,6 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION -from typing import Union import scrapy from scrapy.commands import ScrapyCommand @@ -24,7 +25,7 @@ TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") -def _make_writable(path: Union[str, os.PathLike]) -> None: +def _make_writable(path: str | os.PathLike) -> None: current_permissions = os.stat(path).st_mode os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index ffe5053de..c20c02ca6 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -6,7 +6,7 @@ from collections.abc import AsyncGenerator, Iterable from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast from unittest import TestCase, TestResult from scrapy.http import Request, Response @@ -24,7 +24,7 @@ if TYPE_CHECKING: class Contract: """Abstract class for contracts""" - request_cls: Optional[type[Request]] = None + request_cls: type[Request] | None = None name: str def __init__(self, method: Callable, *args: Any): @@ -126,10 +126,8 @@ class ContractsManager: return contracts - def from_spider( - self, spider: Spider, results: TestResult - ) -> list[Optional[Request]]: - requests: list[Optional[Request]] = [] + def from_spider(self, spider: Spider, results: TestResult) -> list[Request | None]: + requests: list[Request | None] = [] for method in self.tested_methods_from_spidercls(type(spider)): bound_method = spider.__getattribute__(method) try: @@ -140,7 +138,7 @@ class ContractsManager: return requests - def from_method(self, method: Callable, results: TestResult) -> Optional[Request]: + def from_method(self, method: Callable, results: TestResult) -> Request | None: contracts = self.extract_contracts(method) if contracts: request_cls = Request diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 87170d3c1..6f357ba20 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,5 +1,7 @@ +from __future__ import annotations + import json -from typing import Any, Callable, Optional +from typing import Any, Callable from itemadapter import ItemAdapter, is_item @@ -63,7 +65,7 @@ class ReturnsContract(Contract): """ name = "returns" - object_type_verifiers: dict[Optional[str], Callable[[Any], bool]] = { + object_type_verifiers: dict[str | None, Callable[[Any], bool]] = { "request": lambda x: isinstance(x, Request), "requests": lambda x: isinstance(x, Request), "item": is_item, diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 77d57a8d8..1cc0422b7 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -5,7 +5,7 @@ import warnings from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from twisted.internet import task from twisted.internet.defer import Deferred @@ -37,7 +37,7 @@ class Slot: delay: float, randomize_delay: bool, *, - throttle: Optional[bool] = None, + throttle: bool | None = None, ): self.concurrency: int = concurrency self.delay: float = delay @@ -119,15 +119,13 @@ class Downloader: "DOWNLOAD_SLOTS", {} ) - def fetch( - self, request: Request, spider: Spider - ) -> Deferred[Union[Response, Request]]: + def fetch(self, request: Request, spider: Spider) -> Deferred[Response | Request]: def _deactivate(response: _T) -> _T: self.active.remove(request) return response self.active.add(request) - dfd: Deferred[Union[Response, Request]] = self.middleware.download( + dfd: Deferred[Response | Request] = self.middleware.download( self._enqueue_request, request, spider ) return dfd.addBoth(_deactivate) @@ -164,7 +162,7 @@ class Downloader: return key - def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str: + def _get_slot_key(self, request: Request, spider: Spider | None) -> str: warnings.warn( "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.", ScrapyDeprecationWarning, diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index ba20c3c2c..f80f832a7 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from OpenSSL import SSL from twisted.internet._sslverify import _setAcceptableProtocols @@ -49,7 +49,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): self, method: int = SSL.SSLv23_METHOD, tls_verbose_logging: bool = False, - tls_ciphers: Optional[str] = None, + tls_ciphers: str | None = None, *args: Any, **kwargs: Any, ): @@ -73,7 +73,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): tls_verbose_logging: bool = settings.getbool( "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" ) - tls_ciphers: Optional[str] = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + tls_ciphers: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] return cls( # type: ignore[misc] method=method, tls_verbose_logging=tls_verbose_logging, diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index c39e480f1..218f44bbb 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -4,7 +4,7 @@ from __future__ import annotations import logging from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Optional, Protocol, Union, cast +from typing import TYPE_CHECKING, Any, Protocol, cast from twisted.internet import defer @@ -35,16 +35,16 @@ class DownloadHandlerProtocol(Protocol): class DownloadHandlers: def __init__(self, crawler: Crawler): self._crawler: Crawler = crawler - self._schemes: dict[str, Union[str, Callable[..., Any]]] = ( + self._schemes: dict[str, str | Callable[..., Any]] = ( {} ) # stores acceptable schemes on instancing self._handlers: dict[str, DownloadHandlerProtocol] = ( {} ) # stores instanced handlers for schemes self._notconfigured: dict[str, str] = {} # remembers failed handlers - handlers: dict[str, Union[str, Callable[..., Any]]] = without_none_values( + handlers: dict[str, str | Callable[..., Any]] = without_none_values( cast( - dict[str, Union[str, Callable[..., Any]]], + "dict[str, str | Callable[..., Any]]", crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), ) ) @@ -54,7 +54,7 @@ class DownloadHandlers: crawler.signals.connect(self._close, signals.engine_stopped) - def _get_handler(self, scheme: str) -> Optional[DownloadHandlerProtocol]: + def _get_handler(self, scheme: str) -> DownloadHandlerProtocol | None: """Lazy-load the downloadhandler for a scheme only on the first request for that scheme. """ @@ -70,7 +70,7 @@ class DownloadHandlers: def _load_handler( self, scheme: str, skip_lazy: bool = False - ) -> Optional[DownloadHandlerProtocol]: + ) -> DownloadHandlerProtocol | None: path = self._schemes[scheme] try: dhcls: type[DownloadHandlerProtocol] = load_object(path) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index bc06c7ef4..70a769771 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -32,7 +32,7 @@ from __future__ import annotations import re from io import BytesIO -from typing import TYPE_CHECKING, Any, BinaryIO, Optional +from typing import TYPE_CHECKING, Any, BinaryIO from urllib.parse import unquote from twisted.internet.protocol import ClientCreator, Protocol @@ -56,8 +56,8 @@ if TYPE_CHECKING: class ReceivedDataProtocol(Protocol): - def __init__(self, filename: Optional[str] = None): - self.__filename: Optional[str] = filename + def __init__(self, filename: str | None = None): + self.__filename: str | None = filename self.body: BinaryIO = open(filename, "wb") if filename else BytesIO() self.size: int = 0 @@ -66,7 +66,7 @@ class ReceivedDataProtocol(Protocol): self.size += len(data) @property - def filename(self) -> Optional[str]: + def filename(self) -> str | None: return self.__filename def close(self) -> None: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index f96dc7c98..bd3200e9f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, Optional, TypedDict, TypeVar, Union +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar from urllib.parse import urldefrag, urlunparse from twisted.internet import ssl @@ -52,10 +52,10 @@ _T = TypeVar("_T") class _ResultT(TypedDict): txresponse: TxResponse body: bytes - flags: Optional[list[str]] - certificate: Optional[ssl.Certificate] - ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] - failure: NotRequired[Optional[Failure]] + flags: list[str] | None + certificate: ssl.Certificate | None + ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None + failure: NotRequired[Failure | None] class HTTP11DownloadHandler: @@ -143,10 +143,10 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): reactor: ReactorBase, host: str, port: int, - proxyConf: tuple[str, int, Optional[bytes]], + proxyConf: tuple[str, int, bytes | None], contextFactory: IPolicyForHTTPS, timeout: float = 30, - bindAddress: Optional[tuple[str, int]] = None, + bindAddress: tuple[str, int] | None = None, ): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) @@ -220,7 +220,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def tunnel_request_data( - host: str, port: int, proxy_auth_header: Optional[bytes] = None + host: str, port: int, proxy_auth_header: bytes | None = None ) -> bytes: r""" Return binary content of a CONNECT request. @@ -254,14 +254,14 @@ class TunnelingAgent(Agent): self, *, reactor: ReactorBase, - proxyConf: tuple[str, int, Optional[bytes]], + proxyConf: tuple[str, int, bytes | None], contextFactory: IPolicyForHTTPS, - connectTimeout: Optional[float] = None, - bindAddress: Optional[bytes] = None, - pool: Optional[HTTPConnectionPool] = None, + connectTimeout: float | None = None, + bindAddress: bytes | None = None, + pool: HTTPConnectionPool | None = None, ): super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) - self._proxyConf: tuple[str, int, Optional[bytes]] = proxyConf + self._proxyConf: tuple[str, int, bytes | None] = proxyConf self._contextFactory: IPolicyForHTTPS = contextFactory def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint: @@ -281,8 +281,8 @@ class TunnelingAgent(Agent): endpoint: TCP4ClientEndpoint, method: bytes, parsedURI: bytes, - headers: Optional[TxHeaders], - bodyProducer: Optional[IBodyProducer], + headers: TxHeaders | None, + bodyProducer: IBodyProducer | None, requestPath: bytes, ) -> Deferred[TxResponse]: # proxy host and port are required for HTTP pool `key` @@ -305,9 +305,9 @@ class ScrapyProxyAgent(Agent): self, reactor: ReactorBase, proxyURI: bytes, - connectTimeout: Optional[float] = None, - bindAddress: Optional[bytes] = None, - pool: Optional[HTTPConnectionPool] = None, + connectTimeout: float | None = None, + bindAddress: bytes | None = None, + pool: HTTPConnectionPool | None = None, ): super().__init__( reactor=reactor, @@ -321,8 +321,8 @@ class ScrapyProxyAgent(Agent): self, method: bytes, uri: bytes, - headers: Optional[TxHeaders] = None, - bodyProducer: Optional[IBodyProducer] = None, + headers: TxHeaders | None = None, + bodyProducer: IBodyProducer | None = None, ) -> Deferred[TxResponse]: """ Issue a new request via the configured proxy. @@ -350,8 +350,8 @@ class ScrapyAgent: *, contextFactory: IPolicyForHTTPS, connectTimeout: float = 10, - bindAddress: Optional[bytes] = None, - pool: Optional[HTTPConnectionPool] = None, + bindAddress: bytes | None = None, + pool: HTTPConnectionPool | None = None, maxsize: int = 0, warnsize: int = 0, fail_on_dataloss: bool = True, @@ -359,12 +359,12 @@ class ScrapyAgent: ): self._contextFactory: IPolicyForHTTPS = contextFactory self._connectTimeout: float = connectTimeout - self._bindAddress: Optional[bytes] = bindAddress - self._pool: Optional[HTTPConnectionPool] = pool + self._bindAddress: bytes | None = bindAddress + self._pool: HTTPConnectionPool | None = pool self._maxsize: int = maxsize self._warnsize: int = warnsize self._fail_on_dataloss: bool = fail_on_dataloss - self._txresponse: Optional[TxResponse] = None + self._txresponse: TxResponse | None = None self._crawler: Crawler = crawler def _get_agent(self, request: Request, timeout: float) -> Agent: @@ -462,7 +462,7 @@ class ScrapyAgent: def _cb_bodyready( self, txresponse: TxResponse, request: Request - ) -> Union[_ResultT, Deferred[_ResultT]]: + ) -> _ResultT | Deferred[_ResultT]: headers_received_result = self._crawler.signals.send_catch_log( signal=signals.headers_received, headers=self._headers_from_twisted_response(txresponse), @@ -551,7 +551,7 @@ class ScrapyAgent: def _cb_bodydone( self, result: _ResultT, request: Request, url: str - ) -> Union[Response, Failure]: + ) -> Response | Failure: headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) try: @@ -614,14 +614,12 @@ class _ResponseReader(Protocol): self._fail_on_dataloss_warned: bool = False self._reached_warnsize: bool = False self._bytes_received: int = 0 - self._certificate: Optional[ssl.Certificate] = None - self._ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] = ( - None - ) + self._certificate: ssl.Certificate | None = None + self._ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None self._crawler: Crawler = crawler def _finish_response( - self, flags: Optional[list[str]] = None, failure: Optional[Failure] = None + self, flags: list[str] | None = None, failure: Failure | None = None ) -> None: self._finished.callback( { diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 4722c612d..f0f9ceeb7 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,7 +1,7 @@ from __future__ import annotations from time import time -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from urllib.parse import urldefrag from twisted.internet.error import TimeoutError @@ -60,8 +60,8 @@ class ScrapyH2Agent: context_factory: IPolicyForHTTPS, pool: H2ConnectionPool, connect_timeout: int = 10, - bind_address: Optional[bytes] = None, - crawler: Optional[Crawler] = None, + bind_address: bytes | None = None, + crawler: Crawler | None = None, ) -> None: self._context_factory = context_factory self._connect_timeout = connect_timeout @@ -69,7 +69,7 @@ class ScrapyH2Agent: self._pool = pool self._crawler = crawler - def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent: + def _get_agent(self, request: Request, timeout: float | None) -> H2Agent: from twisted.internet import reactor bind_address = request.meta.get("bindaddress") or self._bind_address diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index fa660c63c..870a26f04 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured @@ -26,9 +26,9 @@ class S3DownloadHandler: settings: BaseSettings, *, crawler: Crawler, - aws_access_key_id: Optional[str] = None, - aws_secret_access_key: Optional[str] = None, - aws_session_token: Optional[str] = None, + aws_access_key_id: str | None = None, + aws_secret_access_key: str | None = None, + aws_session_token: str | None = None, httpdownloadhandler: type[HTTPDownloadHandler] = HTTPDownloadHandler, **kw: Any, ): diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 00d3bd1b0..60e7adb2f 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Union, cast +from typing import TYPE_CHECKING, Any, cast from twisted.internet.defer import Deferred, inlineCallbacks @@ -46,11 +46,11 @@ class DownloaderMiddlewareManager(MiddlewareManager): download_func: Callable[[Request, Spider], Deferred[Response]], request: Request, spider: Spider, - ) -> Deferred[Union[Response, Request]]: + ) -> Deferred[Response | Request]: @inlineCallbacks def process_request( request: Request, - ) -> Generator[Deferred[Any], Any, Union[Response, Request]]: + ) -> Generator[Deferred[Any], Any, Response | Request]: for method in self.methods["process_request"]: method = cast(Callable, method) response = yield deferred_from_coro( @@ -69,8 +69,8 @@ class DownloaderMiddlewareManager(MiddlewareManager): @inlineCallbacks def process_response( - response: Union[Response, Request] - ) -> Generator[Deferred[Any], Any, Union[Response, Request]]: + response: Response | Request, + ) -> Generator[Deferred[Any], Any, Response | Request]: if response is None: raise TypeError("Received None in process_response") elif isinstance(response, Request): @@ -93,7 +93,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): @inlineCallbacks def process_exception( failure: Failure, - ) -> Generator[Deferred[Any], Any, Union[Failure, Response, Request]]: + ) -> Generator[Deferred[Any], Any, Failure | Response | Request]: exception = failure.value for method in self.methods["process_exception"]: method = cast(Callable, method) @@ -111,7 +111,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response return failure - deferred: Deferred[Union[Response, Request]] = mustbe_deferred( + deferred: Deferred[Response | Request] = mustbe_deferred( process_request, request ) deferred.addErrback(process_exception) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 509bda4e4..ee10ae73b 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -2,7 +2,7 @@ from __future__ import annotations import re from time import time -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from twisted.internet import defer @@ -144,9 +144,9 @@ class ScrapyHTTPClientFactory(ClientFactory): # converting to bytes to comply to Twisted interface self.url: bytes = to_bytes(self._url, encoding="ascii") self.method: bytes = to_bytes(request.method, encoding="ascii") - self.body: Optional[bytes] = request.body or None + self.body: bytes | None = request.body or None self.headers: Headers = Headers(request.headers) - self.response_headers: Optional[Headers] = None + self.response_headers: Headers | None = None self.timeout: float = request.meta.get("download_timeout") or timeout self.start_time: float = time() self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback( diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index f3d74eccf..d056a00ba 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -9,7 +9,7 @@ from __future__ import annotations import logging from time import time -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed @@ -18,7 +18,7 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.downloader import Downloader -from scrapy.core.scraper import Scraper +from scrapy.core.scraper import Scraper, _HandleOutputDeferred from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter @@ -32,7 +32,6 @@ if TYPE_CHECKING: from collections.abc import Callable, Generator, Iterable, Iterator from scrapy.core.scheduler import BaseScheduler - from scrapy.core.scraper import _HandleOutputDeferred from scrapy.crawler import Crawler from scrapy.settings import BaseSettings from scrapy.spiders import Spider @@ -51,9 +50,9 @@ class Slot: nextcall: CallLaterOnce[None], scheduler: BaseScheduler, ) -> None: - self.closing: Optional[Deferred[None]] = None + self.closing: Deferred[None] | None = None self.inprogress: set[Request] = set() - self.start_requests: Optional[Iterator[Request]] = iter(start_requests) + self.start_requests: Iterator[Request] | None = iter(start_requests) self.close_if_idle: bool = close_if_idle self.nextcall: CallLaterOnce[None] = nextcall self.scheduler: BaseScheduler = scheduler @@ -84,15 +83,15 @@ class ExecutionEngine: def __init__( self, crawler: Crawler, - spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]], + spider_closed_callback: Callable[[Spider], Deferred[None] | None], ) -> None: self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter - self.slot: Optional[Slot] = None - self.spider: Optional[Spider] = None + self.slot: Slot | None = None + self.spider: Spider | None = None self.running: bool = False self.paused: bool = False self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -101,10 +100,10 @@ class ExecutionEngine: downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) self.scraper: Scraper = Scraper(crawler) - self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = ( + self._spider_closed_callback: Callable[[Spider], Deferred[None] | None] = ( spider_closed_callback ) - self.start_time: Optional[float] = None + self.start_time: float | None = None def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler @@ -218,7 +217,7 @@ class ExecutionEngine: or self.scraper.slot.needs_backout() ) - def _next_request_from_scheduler(self) -> Optional[Deferred[None]]: + def _next_request_from_scheduler(self) -> Deferred[None] | None: assert self.slot is not None # typing assert self.spider is not None # typing @@ -226,7 +225,7 @@ class ExecutionEngine: if request is None: return None - d: Deferred[Union[Response, Request]] = self._download(request) + d: Deferred[Response | Request] = self._download(request) d.addBoth(self._handle_downloader_output, request) d.addErrback( lambda f: logger.info( @@ -260,8 +259,8 @@ class ExecutionEngine: return d2 def _handle_downloader_output( - self, result: Union[Request, Response, Failure], request: Request - ) -> Optional[_HandleOutputDeferred]: + self, result: Request | Response | Failure, request: Request + ) -> _HandleOutputDeferred | None: assert self.spider is not None # typing if not isinstance(result, (Request, Response, Failure)): @@ -323,24 +322,24 @@ class ExecutionEngine: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - d: Deferred[Union[Response, Request]] = self._download(request) + d: Deferred[Response | Request] = self._download(request) # Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[call-overload] return d2 def _downloaded( - self, result: Union[Response, Request, Failure], request: Request - ) -> Union[Deferred[Response], Response, Failure]: + self, result: Response | Request | Failure, request: Request + ) -> Deferred[Response] | Response | Failure: assert self.slot is not None # typing self.slot.remove_request(request) return self.download(result) if isinstance(result, Request) else result - def _download(self, request: Request) -> Deferred[Union[Response, Request]]: + def _download(self, request: Request) -> Deferred[Response | Request]: assert self.slot is not None # typing self.slot.add_request(request) - def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: + def _on_success(result: Response | Request) -> Response | Request: if not isinstance(result, (Response, Request)): raise TypeError( f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}" @@ -368,9 +367,7 @@ class ExecutionEngine: return _ assert self.spider is not None - dwld: Deferred[Union[Response, Request]] = self.downloader.fetch( - request, self.spider - ) + dwld: Deferred[Response | Request] = self.downloader.fetch(request, self.spider) dwld.addCallback(_on_success) dwld.addBoth(_on_complete) return dwld diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index b5ff55eb0..45f32daaa 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections import deque -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from twisted.internet import defer from twisted.internet.defer import Deferred @@ -121,8 +121,8 @@ class H2Agent: reactor: ReactorBase, pool: H2ConnectionPool, context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, - bind_address: Optional[bytes] = None, + connect_timeout: float | None = None, + bind_address: bytes | None = None, ) -> None: self._reactor = reactor self._pool = pool @@ -165,8 +165,8 @@ class ScrapyProxyH2Agent(H2Agent): proxy_uri: URI, pool: H2ConnectionPool, context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, - bind_address: Optional[bytes] = None, + connect_timeout: float | None = None, + bind_address: bytes | None = None, ) -> None: super().__init__( reactor=reactor, diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 618423218..23335b7b2 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -4,7 +4,7 @@ import ipaddress import itertools import logging from collections import deque -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from h2.config import H2Configuration from h2.connection import H2Connection @@ -63,7 +63,7 @@ class InvalidNegotiatedProtocol(H2Error): class RemoteTerminatedConnection(H2Error): def __init__( self, - remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], + remote_ip_address: IPv4Address | IPv6Address | None, event: ConnectionTerminated, ) -> None: self.remote_ip_address = remote_ip_address @@ -74,9 +74,7 @@ class RemoteTerminatedConnection(H2Error): class MethodNotAllowed405(H2Error): - def __init__( - self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]] - ) -> None: + def __init__(self, remote_ip_address: IPv4Address | IPv6Address | None) -> None: self.remote_ip_address = remote_ip_address def __str__(self) -> str: diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 51ebdf489..a4dc89c18 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from enum import Enum from io import BytesIO -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from h2.errors import ErrorCodes from h2.exceptions import H2Error, ProtocolError, StreamClosedError @@ -382,7 +382,7 @@ class Stream: def close( self, reason: StreamCloseReason, - errors: Optional[list[BaseException]] = None, + errors: list[BaseException] | None = None, from_protocol: bool = False, ) -> None: """Based on the reason sent we will handle each case.""" diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index ced18fc05..bebee1236 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,7 +4,7 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast # working around https://github.com/sphinx-doc/sphinx/issues/10400 from twisted.internet.defer import Deferred # noqa: TC002 @@ -73,7 +73,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ return cls() - def open(self, spider: Spider) -> Optional[Deferred[None]]: + def open(self, spider: Spider) -> Deferred[None] | None: """ Called when the spider is opened by the engine. It receives the spider instance as argument and it's useful to execute initialization code. @@ -83,7 +83,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ pass - def close(self, reason: str) -> Optional[Deferred[None]]: + def close(self, reason: str) -> Deferred[None] | None: """ Called when the spider is closed by the engine. It receives the reason why the crawl finished as argument and it's useful to execute cleaning code. @@ -115,7 +115,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): raise NotImplementedError() @abstractmethod - def next_request(self) -> Optional[Request]: + def next_request(self) -> Request | None: """ Return the next :class:`~scrapy.http.Request` to be processed, or ``None`` to indicate that there are no requests to be considered ready at the moment. @@ -181,22 +181,22 @@ class Scheduler(BaseScheduler): def __init__( self, dupefilter: BaseDupeFilter, - jobdir: Optional[str] = None, - dqclass: Optional[type[BaseQueue]] = None, - mqclass: Optional[type[BaseQueue]] = None, + jobdir: str | None = None, + dqclass: type[BaseQueue] | None = None, + mqclass: type[BaseQueue] | None = None, logunser: bool = False, - stats: Optional[StatsCollector] = None, - pqclass: Optional[type[ScrapyPriorityQueue]] = None, - crawler: Optional[Crawler] = None, + stats: StatsCollector | None = None, + pqclass: type[ScrapyPriorityQueue] | None = None, + crawler: Crawler | None = None, ): self.df: BaseDupeFilter = dupefilter - self.dqdir: Optional[str] = self._dqdir(jobdir) - self.pqclass: Optional[type[ScrapyPriorityQueue]] = pqclass - self.dqclass: Optional[type[BaseQueue]] = dqclass - self.mqclass: Optional[type[BaseQueue]] = mqclass + self.dqdir: str | None = self._dqdir(jobdir) + self.pqclass: type[ScrapyPriorityQueue] | None = pqclass + self.dqclass: type[BaseQueue] | None = dqclass + self.mqclass: type[BaseQueue] | None = mqclass self.logunser: bool = logunser - self.stats: Optional[StatsCollector] = stats - self.crawler: Optional[Crawler] = crawler + self.stats: StatsCollector | None = stats + self.crawler: Crawler | None = crawler @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -218,7 +218,7 @@ class Scheduler(BaseScheduler): def has_pending_requests(self) -> bool: return len(self) > 0 - def open(self, spider: Spider) -> Optional[Deferred[None]]: + def open(self, spider: Spider) -> Deferred[None] | None: """ (1) initialize the memory queue (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory @@ -226,10 +226,10 @@ class Scheduler(BaseScheduler): """ self.spider: Spider = spider self.mqs: ScrapyPriorityQueue = self._mq() - self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None + self.dqs: ScrapyPriorityQueue | None = self._dq() if self.dqdir else None return self.df.open() - def close(self, reason: str) -> Optional[Deferred[None]]: + def close(self, reason: str) -> Deferred[None] | None: """ (1) dump pending requests to disk if there is a disk queue (2) return the result of the dupefilter's ``close`` method @@ -263,7 +263,7 @@ class Scheduler(BaseScheduler): self.stats.inc_value("scheduler/enqueued", spider=self.spider) return True - def next_request(self) -> Optional[Request]: + def next_request(self) -> Request | None: """ Return a :class:`~scrapy.http.Request` object from the memory queue, falling back to the disk queue if the memory queue is empty. @@ -272,7 +272,7 @@ class Scheduler(BaseScheduler): Increment the appropriate stats, such as: ``scheduler/dequeued``, ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. """ - request: Optional[Request] = self.mqs.pop() + request: Request | None = self.mqs.pop() assert self.stats is not None if request is not None: self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider) @@ -318,7 +318,7 @@ class Scheduler(BaseScheduler): def _mqpush(self, request: Request) -> None: self.mqs.push(request) - def _dqpop(self) -> Optional[Request]: + def _dqpop(self) -> Request | None: if self.dqs is not None: return self.dqs.pop() return None @@ -355,7 +355,7 @@ class Scheduler(BaseScheduler): ) return q - def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: + def _dqdir(self, jobdir: str | None) -> str | None: """Return a folder name to keep disk queue state at""" if jobdir: dqdir = Path(jobdir, "requests.queue") diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 71a0d6aeb..83dad0c0b 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -6,7 +6,7 @@ from __future__ import annotations import logging from collections import deque from collections.abc import AsyncIterable, Iterator -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeVar, Union, cast from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks @@ -42,11 +42,8 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") _ParallelResult = list[tuple[bool, Iterator[Any]]] - -if TYPE_CHECKING: - # parameterized Deferreds require Twisted 21.7.0 - _HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] - QueueTuple = tuple[Union[Response, Failure], Request, _HandleOutputDeferred] +_HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] +QueueTuple = tuple[Union[Response, Failure], Request, _HandleOutputDeferred] class Slot: @@ -60,10 +57,10 @@ class Slot: self.active: set[Request] = set() self.active_size: int = 0 self.itemproc_size: int = 0 - self.closing: Optional[Deferred[Spider]] = None + self.closing: Deferred[Spider] | None = None def add_response_request( - self, result: Union[Response, Failure], request: Request + self, result: Response | Failure, request: Request ) -> _HandleOutputDeferred: deferred: _HandleOutputDeferred = Deferred() self.queue.append((result, request, deferred)) @@ -78,9 +75,7 @@ class Slot: self.active.add(request) return response, request, deferred - def finish_response( - self, result: Union[Response, Failure], request: Request - ) -> None: + def finish_response(self, result: Response | Failure, request: Request) -> None: self.active.remove(request) if isinstance(result, Response): self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -96,7 +91,7 @@ class Slot: class Scraper: def __init__(self, crawler: Crawler) -> None: - self.slot: Optional[Slot] = None + self.slot: Slot | None = None self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler( crawler ) @@ -135,7 +130,7 @@ class Scraper: self.slot.closing.callback(spider) def enqueue_scrape( - self, result: Union[Response, Failure], request: Request, spider: Spider + self, result: Response | Failure, request: Request, spider: Spider ) -> _HandleOutputDeferred: if self.slot is None: raise RuntimeError("Scraper slot not assigned") @@ -167,7 +162,7 @@ class Scraper: self._scrape(response, request, spider).chainDeferred(deferred) def _scrape( - self, result: Union[Response, Failure], request: Request, spider: Spider + self, result: Response | Failure, request: Request, spider: Spider ) -> _HandleOutputDeferred: """ Handle the downloaded response or failure through the spider callback/errback @@ -176,7 +171,7 @@ class Scraper: raise TypeError( f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) - dfd: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = self._scrape2( + dfd: Deferred[Iterable[Any] | AsyncIterable[Any]] = self._scrape2( result, request, spider ) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) @@ -186,8 +181,8 @@ class Scraper: return dfd2 def _scrape2( - self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]: + self, result: Response | Failure, request: Request, spider: Spider + ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: """ Handle the different cases of request's result been a Response or a Failure """ @@ -202,8 +197,8 @@ class Scraper: return dfd def call_spider( - self, result: Union[Response, Failure], request: Request, spider: Spider - ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]: + self, result: Response | Failure, request: Request, spider: Spider + ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: dfd: Deferred[Any] if isinstance(result, Response): if getattr(result, "request", None) is None: @@ -222,7 +217,7 @@ class Scraper: if request.errback: warn_on_generator_with_return_value(spider, request.errback) dfd.addErrback(request.errback) - dfd2: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = dfd.addCallback( + dfd2: Deferred[Iterable[Any] | AsyncIterable[Any]] = dfd.addCallback( iterate_spider_output ) return dfd2 @@ -231,7 +226,7 @@ class Scraper: self, _failure: Failure, request: Request, - response: Union[Response, Failure], + response: Response | Failure, spider: Spider, ) -> None: exc = _failure.value @@ -258,14 +253,14 @@ class Scraper: def handle_spider_output( self, - result: Union[Iterable[_T], AsyncIterable[_T]], + result: Iterable[_T] | AsyncIterable[_T], request: Request, response: Response, spider: Spider, ) -> _HandleOutputDeferred: if not result: return defer_succeed(None) - it: Union[Iterable[_T], AsyncIterable[_T]] + it: Iterable[_T] | AsyncIterable[_T] dfd: Deferred[_ParallelResult] if isinstance(result, AsyncIterable): it = aiter_errback( @@ -296,7 +291,7 @@ class Scraper: def _process_spidermw_output( self, output: Any, request: Request, response: Response, spider: Spider - ) -> Optional[Deferred[Any]]: + ) -> Deferred[Any] | None: """Process each Request/Item (given in the output parameter) returned from the given spider """ @@ -316,9 +311,7 @@ class Scraper: ) return None - def start_itemproc( - self, item: Any, *, response: Optional[Response] - ) -> Deferred[Any]: + def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[Any]: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come @@ -337,7 +330,7 @@ class Scraper: download_failure: Failure, request: Request, spider: Spider, - ) -> Union[Failure, None]: + ) -> Failure | None: """Log and silence errors that come from the engine (typically download errors that got propagated thru here). @@ -371,7 +364,7 @@ class Scraper: return None def _itemproc_finished( - self, output: Any, item: Any, response: Optional[Response], spider: Spider + self, output: Any, item: Any, response: Response | None, spider: Spider ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 3c8513042..1edfe1c51 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -10,7 +10,7 @@ import logging from collections.abc import AsyncIterable, Callable, Iterable from inspect import isasyncgenfunction, iscoroutine from itertools import islice -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeVar, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -76,7 +76,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Union[Iterable[_T], AsyncIterable[_T]]: + ) -> Iterable[_T] | AsyncIterable[_T]: for method in self.methods["process_spider_input"]: method = cast(Callable, method) try: @@ -97,10 +97,10 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - iterable: Union[Iterable[_T], AsyncIterable[_T]], + iterable: Iterable[_T] | AsyncIterable[_T], exception_processor_index: int, - recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]], - ) -> Union[Iterable[_T], AsyncIterable[_T]]: + recover_to: MutableChain[_T] | MutableAsyncChain[_T], + ) -> Iterable[_T] | AsyncIterable[_T]: def process_sync(iterable: Iterable[_T]) -> Iterable[_T]: try: yield from iterable @@ -142,7 +142,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, _failure: Failure, start_index: int = 0, - ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]: + ) -> Failure | MutableChain[_T] | MutableAsyncChain[_T]: exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -158,7 +158,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = ( + dfd: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = ( self._process_spider_output( response, spider, result, method_index + 1 ) @@ -192,12 +192,12 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Union[Iterable[_T], AsyncIterable[_T]], + result: Iterable[_T] | AsyncIterable[_T], start_index: int = 0, - ) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]: + ) -> Generator[Deferred[Any], Any, MutableChain[_T] | MutableAsyncChain[_T]]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - recovered: Union[MutableChain[_T], MutableAsyncChain[_T]] + recovered: MutableChain[_T] | MutableAsyncChain[_T] last_result_is_async = isinstance(result, AsyncIterable) if last_result_is_async: recovered = MutableAsyncChain() @@ -248,10 +248,10 @@ class SpiderMiddlewareManager(MiddlewareManager): # might fail directly if the output value is not a generator result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result: Union[ - Failure, MutableChain[_T], MutableAsyncChain[_T] - ] = self._process_spider_exception( - response, spider, Failure(ex), method_index + 1 + exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = ( + self._process_spider_exception( + response, spider, Failure(ex), method_index + 1 + ) ) if isinstance(exception_result, Failure): raise @@ -283,9 +283,9 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Union[Iterable[_T], AsyncIterable[_T]], - ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]: - recovered: Union[MutableChain[_T], MutableAsyncChain[_T]] + result: Iterable[_T] | AsyncIterable[_T], + ) -> MutableChain[_T] | MutableAsyncChain[_T]: + recovered: MutableChain[_T] | MutableAsyncChain[_T] if isinstance(result, AsyncIterable): recovered = MutableAsyncChain() else: @@ -293,7 +293,7 @@ class SpiderMiddlewareManager(MiddlewareManager): result = self._evaluate_iterable(response, spider, result, 0, recovered) result = await maybe_deferred_to_future( cast( - "Deferred[Union[Iterable[_T], AsyncIterable[_T]]]", + "Deferred[Iterable[_T] | AsyncIterable[_T]]", self._process_spider_output(response, spider, result), ) ) @@ -310,22 +310,22 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]]: + ) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]: async def process_callback_output( - result: Union[Iterable[_T], AsyncIterable[_T]] - ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]: + result: Iterable[_T] | AsyncIterable[_T], + ) -> MutableChain[_T] | MutableAsyncChain[_T]: return await self._process_callback_output(response, spider, result) def process_spider_exception( _failure: Failure, - ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]: + ) -> Failure | MutableChain[_T] | MutableAsyncChain[_T]: return self._process_spider_exception(response, spider, _failure) - dfd: Deferred[Union[Iterable[_T], AsyncIterable[_T]]] = mustbe_deferred( + dfd: Deferred[Iterable[_T] | AsyncIterable[_T]] = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) - dfd2: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = ( - dfd.addCallback(deferred_f_from_coro_f(process_callback_output)) + dfd2: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = dfd.addCallback( + deferred_f_from_coro_f(process_callback_output) ) dfd2.addErrback(process_spider_exception) return dfd2 @@ -339,10 +339,10 @@ class SpiderMiddlewareManager(MiddlewareManager): @staticmethod def _get_async_method_pair( mw: Any, methodname: str - ) -> Union[None, Callable, tuple[Callable, Callable]]: - normal_method: Optional[Callable] = getattr(mw, methodname, None) + ) -> None | Callable | tuple[Callable, Callable]: + normal_method: Callable | None = getattr(mw, methodname, None) methodname_async = methodname + "_async" - async_method: Optional[Callable] = getattr(mw, methodname_async, None) + async_method: Callable | None = getattr(mw, methodname_async, None) if not async_method: return normal_method if not normal_method: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e75ef52ac..701dccf57 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,7 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from twisted.internet.defer import ( Deferred, @@ -57,7 +57,7 @@ class Crawler: def __init__( self, spidercls: type[Spider], - settings: Union[None, dict[str, Any], Settings] = None, + settings: None | dict[str, Any] | Settings = None, init_reactor: bool = False, ): if isinstance(spidercls, Spider): @@ -78,12 +78,12 @@ class Crawler: self.crawling: bool = False self._started: bool = False - self.extensions: Optional[ExtensionManager] = None - self.stats: Optional[StatsCollector] = None - self.logformatter: Optional[LogFormatter] = None - self.request_fingerprinter: Optional[RequestFingerprinter] = None - self.spider: Optional[Spider] = None - self.engine: Optional[ExecutionEngine] = None + self.extensions: ExtensionManager | None = None + self.stats: StatsCollector | None = None + self.logformatter: LogFormatter | None = None + self.request_fingerprinter: RequestFingerprinter | None = None + self.spider: Spider | None = None + self.engine: ExecutionEngine | None = None def _update_root_log_handler(self) -> None: if get_scrapy_root_handler() is not None: @@ -181,16 +181,16 @@ class Crawler: @staticmethod def _get_component( component_class: type[_T], components: Iterable[Any] - ) -> Optional[_T]: + ) -> _T | None: for component in components: if isinstance(component, component_class): return component return None - def get_addon(self, cls: type[_T]) -> Optional[_T]: + def get_addon(self, cls: type[_T]) -> _T | None: return self._get_component(cls, self.addons.addons) - def get_downloader_middleware(self, cls: type[_T]) -> Optional[_T]: + def get_downloader_middleware(self, cls: type[_T]) -> _T | None: if not self.engine: raise RuntimeError( "Crawler.get_downloader_middleware() can only be called after " @@ -198,7 +198,7 @@ class Crawler: ) return self._get_component(cls, self.engine.downloader.middleware.middlewares) - def get_extension(self, cls: type[_T]) -> Optional[_T]: + def get_extension(self, cls: type[_T]) -> _T | None: if not self.extensions: raise RuntimeError( "Crawler.get_extension() can only be called after the " @@ -206,7 +206,7 @@ class Crawler: ) return self._get_component(cls, self.extensions.middlewares) - def get_item_pipeline(self, cls: type[_T]) -> Optional[_T]: + def get_item_pipeline(self, cls: type[_T]) -> _T | None: if not self.engine: raise RuntimeError( "Crawler.get_item_pipeline() can only be called after the " @@ -214,7 +214,7 @@ class Crawler: ) return self._get_component(cls, self.engine.scraper.itemproc.middlewares) - def get_spider_middleware(self, cls: type[_T]) -> Optional[_T]: + def get_spider_middleware(self, cls: type[_T]) -> _T | None: if not self.engine: raise RuntimeError( "Crawler.get_spider_middleware() can only be called after the " @@ -250,7 +250,7 @@ class CrawlerRunner: verifyClass(ISpiderLoader, loader_cls) return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy())) - def __init__(self, settings: Union[dict[str, Any], Settings, None] = None): + def __init__(self, settings: dict[str, Any] | Settings | None = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings: Settings = settings @@ -261,7 +261,7 @@ class CrawlerRunner: def crawl( self, - crawler_or_spidercls: Union[type[Spider], str, Crawler], + crawler_or_spidercls: type[Spider] | str | Crawler, *args: Any, **kwargs: Any, ) -> Deferred[None]: @@ -308,7 +308,7 @@ class CrawlerRunner: return d.addBoth(_done) def create_crawler( - self, crawler_or_spidercls: Union[type[Spider], str, Crawler] + self, crawler_or_spidercls: type[Spider] | str | Crawler ) -> Crawler: """ Return a :class:`~scrapy.crawler.Crawler` object. @@ -329,7 +329,7 @@ class CrawlerRunner: return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) - def _create_crawler(self, spidercls: Union[str, type[Spider]]) -> Crawler: + def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) @@ -380,7 +380,7 @@ class CrawlerProcess(CrawlerRunner): def __init__( self, - settings: Union[dict[str, Any], Settings, None] = None, + settings: dict[str, Any] | Settings | None = None, install_root_handler: bool = True, ): super().__init__(settings) @@ -409,7 +409,7 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._stop_reactor) - def _create_crawler(self, spidercls: Union[type[Spider], str]) -> Crawler: + def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 5fc7f31a3..b813baf86 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging import re -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from w3lib import html @@ -43,7 +43,7 @@ class AjaxCrawlMiddleware: def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if not isinstance(response, HtmlResponse) or response.status != 200: return response diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index e384793ee..545dcaac9 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from collections import defaultdict -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from tldextract import TLDExtract @@ -70,7 +70,7 @@ class CookiesMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: if request.meta.get("dont_merge_cookies", False): return None @@ -87,7 +87,7 @@ class CookiesMiddleware: def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if request.meta.get("dont_merge_cookies", False): return response @@ -123,7 +123,7 @@ class CookiesMiddleware: msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={"spider": spider}) - def _format_cookie(self, cookie: VerboseCookie, request: Request) -> Optional[str]: + def _format_cookie(self, cookie: VerboseCookie, request: Request) -> str | None: """ Given a dict consisting of cookie components, return its string representation. Decode from bytes if necessary. diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 312c1e026..d58b4490b 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -6,7 +6,7 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from scrapy.utils.python import without_none_values @@ -32,7 +32,7 @@ class DefaultHeadersMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: for k, v in self._headers: request.headers.setdefault(k, v) return None diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index ee7a24825..28456c697 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -6,7 +6,7 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from scrapy import Request, Spider, signals @@ -33,7 +33,7 @@ class DownloadTimeoutMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: if self._timeout: request.meta.setdefault("download_timeout", self._timeout) return None diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 39165e155..b74140ee1 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -6,7 +6,7 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from w3lib.http import basic_auth_header @@ -40,7 +40,7 @@ class HttpAuthMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: auth = getattr(self, "auth", None) if auth and b"Authorization" not in request.headers: if not self.domain or url_is_from_any_domain(request.url, [self.domain]): diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 8377a3c1d..3892dba23 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,7 +1,7 @@ from __future__ import annotations from email.utils import formatdate -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from twisted.internet import defer from twisted.internet.error import ( @@ -69,7 +69,7 @@ class HttpCacheMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: if request.meta.get("dont_cache", False): return None @@ -79,7 +79,7 @@ class HttpCacheMiddleware: return None # Look for cached response and check if expired - cachedresponse: Optional[Response] = self.storage.retrieve_response( + cachedresponse: Response | None = self.storage.retrieve_response( spider, request ) if cachedresponse is None: @@ -103,7 +103,7 @@ class HttpCacheMiddleware: def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if request.meta.get("dont_cache", False): return response @@ -118,7 +118,7 @@ class HttpCacheMiddleware: response.headers["Date"] = formatdate(usegmt=True) # Do not validate first-hand responses - cachedresponse: Optional[Response] = request.meta.pop("cached_response", None) + cachedresponse: Response | None = request.meta.pop("cached_response", None) if cachedresponse is None: self.stats.inc_value("httpcache/firsthand", spider=spider) self._cache_response(spider, response, request, cachedresponse) @@ -134,8 +134,8 @@ class HttpCacheMiddleware: def process_exception( self, request: Request, exception: Exception, spider: Spider - ) -> Union[Request, Response, None]: - cachedresponse: Optional[Response] = request.meta.pop("cached_response", None) + ) -> Request | Response | None: + cachedresponse: Response | None = request.meta.pop("cached_response", None) if cachedresponse is not None and isinstance( exception, self.DOWNLOAD_EXCEPTIONS ): @@ -148,7 +148,7 @@ class HttpCacheMiddleware: spider: Spider, response: Response, request: Request, - cachedresponse: Optional[Response], + cachedresponse: Response | None, ) -> None: if self.policy.should_cache_response(response, request): self.stats.inc_value("httpcache/store", spider=spider) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index d913ca25d..84678b8e9 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -3,7 +3,7 @@ from __future__ import annotations import warnings from itertools import chain from logging import getLogger -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -54,9 +54,9 @@ class HttpCompressionMiddleware: def __init__( self, - stats: Optional[StatsCollector] = None, + stats: StatsCollector | None = None, *, - crawler: Optional[Crawler] = None, + crawler: Crawler | None = None, ): if not crawler: self.stats = stats @@ -96,13 +96,13 @@ class HttpCompressionMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) return None def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if request.method == "HEAD": return response if isinstance(response, Response): diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index b35ecbd54..2f3f2db47 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,7 +1,7 @@ from __future__ import annotations import base64 -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from urllib.parse import unquote, urlunparse from urllib.request import ( # type: ignore[attr-defined] _parse_proxy, @@ -23,9 +23,9 @@ if TYPE_CHECKING: class HttpProxyMiddleware: - def __init__(self, auth_encoding: Optional[str] = "latin-1"): - self.auth_encoding: Optional[str] = auth_encoding - self.proxies: dict[str, tuple[Optional[bytes], str]] = {} + def __init__(self, auth_encoding: str | None = "latin-1"): + self.auth_encoding: str | None = auth_encoding + self.proxies: dict[str, tuple[bytes | None, str]] = {} for type_, url in getproxies().items(): try: self.proxies[type_] = self._get_proxy(url, type_) @@ -38,7 +38,7 @@ class HttpProxyMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("HTTPPROXY_ENABLED"): raise NotConfigured - auth_encoding: Optional[str] = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") + auth_encoding: str | None = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") return cls(auth_encoding) def _basic_auth_header(self, username: str, password: str) -> bytes: @@ -47,7 +47,7 @@ class HttpProxyMiddleware: ) return base64.b64encode(user_pass) - def _get_proxy(self, url: str, orig_type: str) -> tuple[Optional[bytes], str]: + def _get_proxy(self, url: str, orig_type: str) -> tuple[bytes | None, str]: proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) @@ -60,7 +60,7 @@ class HttpProxyMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: creds, proxy_url, scheme = None, None, None if "proxy" in request.meta: if request.meta["proxy"] is not None: @@ -82,9 +82,9 @@ class HttpProxyMiddleware: def _set_proxy_and_creds( self, request: Request, - proxy_url: Optional[str], - creds: Optional[bytes], - scheme: Optional[str], + proxy_url: str | None, + creds: bytes | None, + scheme: str | None, ) -> None: if scheme: request.meta["_scheme_proxy"] = True diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 6437485cf..0b883b43a 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Union, cast +from typing import TYPE_CHECKING, Any, cast from urllib.parse import urljoin from w3lib.url import safe_url_string @@ -144,7 +144,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if ( request.meta.get("dont_redirect", False) or response.status in getattr(spider, "handle_httpstatus_list", []) @@ -185,7 +185,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if ( request.meta.get("dont_redirect", False) or request.method == "HEAD" diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index c32624371..7c0e2280c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -14,7 +14,7 @@ from __future__ import annotations import warnings from logging import Logger, getLogger -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.settings import BaseSettings, Settings @@ -60,12 +60,12 @@ def get_retry_request( request: Request, *, spider: Spider, - reason: Union[str, Exception, type[Exception]] = "unspecified", - max_retry_times: Optional[int] = None, - priority_adjust: Optional[int] = None, + reason: str | Exception | type[Exception] = "unspecified", + max_retry_times: int | None = None, + priority_adjust: int | None = None, logger: Logger = retry_logger, stats_base_key: str = "retry", -) -> Optional[Request]: +) -> Request | None: """ Returns a new :class:`~scrapy.Request` object to retry the specified request, or ``None`` if retries of the specified request have been @@ -167,7 +167,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: if request.meta.get("dont_retry", False): return response if response.status in self.retry_http_codes: @@ -177,7 +177,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): def process_exception( self, request: Request, exception: Exception, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( "dont_retry", False ): @@ -187,9 +187,9 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): def _retry( self, request: Request, - reason: Union[str, Exception, type[Exception]], + reason: str | Exception | type[Exception], spider: Spider, - ) -> Optional[Request]: + ) -> Request | None: max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) priority_adjust = request.meta.get("priority_adjust", self.priority_adjust) return get_retry_request( diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 81ba009d6..ea9f47d69 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional, TypeVar, Union +from typing import TYPE_CHECKING, TypeVar from twisted.internet.defer import Deferred, maybeDeferred @@ -41,13 +41,11 @@ class RobotsTxtMiddleware: if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy") - self._robotstxt_useragent: Optional[str] = crawler.settings.get( + self._robotstxt_useragent: str | None = crawler.settings.get( "ROBOTSTXT_USER_AGENT", None ) self.crawler: Crawler = crawler - self._parsers: dict[ - str, Union[RobotParser, Deferred[Optional[RobotParser]], None] - ] = {} + self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {} self._parserimpl: RobotParser = load_object( crawler.settings.get("ROBOTSTXT_PARSER") ) @@ -61,24 +59,24 @@ class RobotsTxtMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Optional[Deferred[None]]: + ) -> Deferred[None] | None: if request.meta.get("dont_obey_robotstxt"): return None if request.url.startswith("data:") or request.url.startswith("file:"): return None - d: Deferred[Optional[RobotParser]] = maybeDeferred( + d: Deferred[RobotParser | None] = maybeDeferred( self.robot_parser, request, spider # type: ignore[call-overload] ) d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider) return d2 def process_request_2( - self, rp: Optional[RobotParser], request: Request, spider: Spider + self, rp: RobotParser | None, request: Request, spider: Spider ) -> None: if rp is None: return - useragent: Union[str, bytes, None] = self._robotstxt_useragent + useragent: str | bytes | None = self._robotstxt_useragent if not useragent: useragent = request.headers.get(b"User-Agent", self._default_useragent) assert useragent is not None @@ -94,7 +92,7 @@ class RobotsTxtMiddleware: def robot_parser( self, request: Request, spider: Spider - ) -> Union[RobotParser, Deferred[Optional[RobotParser]], None]: + ) -> RobotParser | Deferred[RobotParser | None] | None: url = urlparse_cached(request) netloc = url.netloc @@ -117,9 +115,9 @@ class RobotsTxtMiddleware: parser = self._parsers[netloc] if isinstance(parser, Deferred): - d: Deferred[Optional[RobotParser]] = Deferred() + d: Deferred[RobotParser | None] = Deferred() - def cb(result: Optional[RobotParser]) -> Optional[RobotParser]: + def cb(result: RobotParser | None) -> RobotParser | None: d.callback(result) return result diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index ab5655393..fb0f30620 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from twisted.web import http @@ -19,7 +19,7 @@ if TYPE_CHECKING: def get_header_size( - headers: dict[str, Union[list[Union[str, bytes]], tuple[Union[str, bytes], ...]]] + headers: dict[str, list[str | bytes] | tuple[str | bytes, ...]] ) -> int: size = 0 for key, value in headers.items(): @@ -47,7 +47,7 @@ class DownloaderStats: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: self.stats.inc_value("downloader/request_count", spider=spider) self.stats.inc_value( f"downloader/request_method_count/{request.method}", spider=spider @@ -58,7 +58,7 @@ class DownloaderStats: def process_response( self, request: Request, response: Response, spider: Spider - ) -> Union[Request, Response]: + ) -> Request | Response: self.stats.inc_value("downloader/response_count", spider=spider) self.stats.inc_value( f"downloader/response_status_count/{response.status}", spider=spider @@ -75,7 +75,7 @@ class DownloaderStats: def process_exception( self, request: Request, exception: Exception, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: ex_class = global_object_name(exception.__class__) self.stats.inc_value("downloader/exception_count", spider=spider) self.stats.inc_value( diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 109f1a4d9..ba379f862 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from scrapy import Request, Spider, signals @@ -31,7 +31,7 @@ class UserAgentMiddleware: def process_request( self, request: Request, spider: Spider - ) -> Union[Request, Response, None]: + ) -> Request | Response | None: if self.user_agent: request.headers.setdefault(b"User-Agent", self.user_agent) return None diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 28118977d..d37d2741a 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging from pathlib import Path -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from scrapy.utils.job import job_dir from scrapy.utils.request import ( @@ -31,10 +31,10 @@ class BaseDupeFilter: def request_seen(self, request: Request) -> bool: return False - def open(self) -> Optional[Deferred[None]]: + def open(self) -> Deferred[None] | None: pass - def close(self, reason: str) -> Optional[Deferred[None]]: + def close(self, reason: str) -> Deferred[None] | None: pass def log(self, request: Request, spider: Spider) -> None: @@ -47,10 +47,10 @@ class RFPDupeFilter(BaseDupeFilter): def __init__( self, - path: Optional[str] = None, + path: str | None = None, debug: bool = False, *, - fingerprinter: Optional[RequestFingerprinterProtocol] = None, + fingerprinter: RequestFingerprinterProtocol | None = None, ) -> None: self.file = None self.fingerprinter: RequestFingerprinterProtocol = ( @@ -70,7 +70,7 @@ class RFPDupeFilter(BaseDupeFilter): cls, settings: BaseSettings, *, - fingerprinter: Optional[RequestFingerprinterProtocol] = None, + fingerprinter: RequestFingerprinterProtocol | None = None, ) -> Self: debug = settings.getbool("DUPEFILTER_DEBUG") return cls(job_dir(settings), debug, fingerprinter=fingerprinter) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index ee0033dfb..c9350a956 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -2,6 +2,8 @@ Item Exporters are used to export/serialize items into different formats. """ +from __future__ import annotations + import csv import marshal import pickle # nosec @@ -9,7 +11,7 @@ import pprint from collections.abc import Callable, Iterable, Mapping from io import BytesIO, TextIOWrapper from json import JSONEncoder -from typing import Any, Optional, Union +from typing import Any from xml.sax.saxutils import XMLGenerator # nosec from xml.sax.xmlreader import AttributesImpl # nosec @@ -41,12 +43,12 @@ class BaseItemExporter: If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) """ - self.encoding: Optional[str] = options.pop("encoding", None) - self.fields_to_export: Union[Mapping[str, str], Iterable[str], None] = ( - options.pop("fields_to_export", None) + self.encoding: str | None = options.pop("encoding", None) + self.fields_to_export: Mapping[str, str] | Iterable[str] | None = options.pop( + "fields_to_export", None ) self.export_empty_fields: bool = options.pop("export_empty_fields", False) - self.indent: Optional[int] = options.pop("indent", None) + self.indent: int | None = options.pop("indent", None) if not dont_fail and options: raise TypeError(f"Unexpected options: {', '.join(options.keys())}") @@ -54,7 +56,7 @@ class BaseItemExporter: raise NotImplementedError def serialize_field( - self, field: Union[Mapping[str, Any], Field], name: str, value: Any + self, field: Mapping[str, Any] | Field, name: str, value: Any ) -> Any: serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x) return serializer(value) @@ -66,7 +68,7 @@ class BaseItemExporter: pass def _get_serialized_fields( - self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None + self, item: Any, default_value: Any = None, include_empty: bool | None = None ) -> Iterable[tuple[str, Any]]: """Return the fields to export as an iterable of tuples (name, serialized_value) @@ -225,7 +227,7 @@ class CsvItemExporter(BaseItemExporter): file: BytesIO, include_headers_line: bool = True, join_multivalued: str = ",", - errors: Optional[str] = None, + errors: str | None = None, **kwargs: Any, ): super().__init__(dont_fail=True, **kwargs) @@ -245,7 +247,7 @@ class CsvItemExporter(BaseItemExporter): self._join_multivalued = join_multivalued def serialize_field( - self, field: Union[Mapping[str, Any], Field], name: str, value: Any + self, field: Mapping[str, Any] | Field, name: str, value: Any ) -> Any: serializer: Callable[[Any], Any] = field.get("serializer", self._join_if_needed) return serializer(value) @@ -346,7 +348,7 @@ class PythonItemExporter(BaseItemExporter): self.encoding = "utf-8" def serialize_field( - self, field: Union[Mapping[str, Any], Field], name: str, value: Any + self, field: Mapping[str, Any] | Field, name: str, value: Any ) -> Any: serializer: Callable[[Any], Any] = field.get( "serializer", self._serialize_value @@ -364,10 +366,10 @@ class PythonItemExporter(BaseItemExporter): return to_unicode(value, encoding=self.encoding) return value - def _serialize_item(self, item: Any) -> Iterable[tuple[Union[str, bytes], Any]]: + def _serialize_item(self, item: Any) -> Iterable[tuple[str | bytes, Any]]: for key, value in ItemAdapter(item).items(): yield key, self._serialize_value(value) - def export_item(self, item: Any) -> dict[Union[str, bytes], Any]: # type: ignore[override] - result: dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item)) + def export_item(self, item: Any) -> dict[str | bytes, Any]: # type: ignore[override] + result: dict[str | bytes, Any] = dict(self._get_serialized_fields(item)) return result diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 6ef2d0382..779cd5d1c 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -5,7 +5,7 @@ Extension for collecting core stats like items scraped and start/finish times from __future__ import annotations from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from scrapy import Spider, signals @@ -20,7 +20,7 @@ if TYPE_CHECKING: class CoreStats: def __init__(self, stats: StatsCollector): self.stats: StatsCollector = stats - self.start_time: Optional[datetime] = None + self.start_time: datetime | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index c54871e02..d3c225bcd 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -12,7 +12,7 @@ import sys import threading import traceback from pdb import Pdb -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from scrapy.utils.engine import format_engine_status from scrapy.utils.trackref import format_live_refs @@ -43,7 +43,7 @@ class StackTraceDump: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def dump_stacktrace(self, signum: int, frame: Optional[FrameType]) -> None: + def dump_stacktrace(self, signum: int, frame: FrameType | None) -> None: assert self.crawler.engine log_args = { "stackdumps": self._thread_stacks(), @@ -75,6 +75,6 @@ class Debugger: # win32 platforms don't support SIGUSR signals pass - def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None: + def _enter_debugger(self, signum: int, frame: FrameType | None) -> None: assert frame Pdb().set_trace(frame.f_back) # noqa: T100 diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 7bfcbe6f3..eb1698ce5 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -14,7 +14,7 @@ from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, TYPE_CHECKING, Any, Optional, Protocol, TypeVar, Union, cast +from typing import IO, TYPE_CHECKING, Any, Optional, Protocol, TypeVar, cast from urllib.parse import unquote, urlparse from twisted.internet.defer import Deferred, DeferredList, maybeDeferred @@ -67,7 +67,7 @@ def build_storage( builder: Callable[..., _StorageT], uri: str, *args: Any, - feed_options: Optional[dict[str, Any]] = None, + feed_options: dict[str, Any] | None = None, preargs: Iterable[Any] = (), **kwargs: Any, ) -> _StorageT: @@ -84,10 +84,10 @@ class ItemFilter: :type feed_options: dict """ - feed_options: Optional[dict[str, Any]] + feed_options: dict[str, Any] | None item_classes: tuple[type, ...] - def __init__(self, feed_options: Optional[dict[str, Any]]) -> None: + def __init__(self, feed_options: dict[str, Any] | None) -> None: self.feed_options = feed_options if feed_options is not None: self.item_classes = tuple( @@ -129,7 +129,7 @@ class IFeedStorage(Interface): class FeedStorageProtocol(Protocol): """Reimplementation of ``IFeedStorage`` that can be used in type hints.""" - def __init__(self, uri: str, *, feed_options: Optional[dict[str, Any]] = None): + def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): """Initialize the storage with the parameters given in the URI and the feed-specific options (see :setting:`FEEDS`)""" @@ -137,7 +137,7 @@ class FeedStorageProtocol(Protocol): """Open the storage for the given spider. It must return a file-like object that will be used for the exporters""" - def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: + def store(self, file: IO[bytes]) -> Deferred[None] | None: """Store the given file stream""" @@ -150,7 +150,7 @@ class BlockingFeedStorage: return NamedTemporaryFile(prefix="feed-", dir=path) - def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: + def store(self, file: IO[bytes]) -> Deferred[None] | None: return deferToThread(self._store_in_thread, file) def _store_in_thread(self, file: IO[bytes]) -> None: @@ -162,9 +162,9 @@ class StdoutFeedStorage: def __init__( self, uri: str, - _stdout: Optional[IO[bytes]] = None, + _stdout: IO[bytes] | None = None, *, - feed_options: Optional[dict[str, Any]] = None, + feed_options: dict[str, Any] | None = None, ): if not _stdout: _stdout = sys.stdout.buffer @@ -180,13 +180,13 @@ class StdoutFeedStorage: def open(self, spider: Spider) -> IO[bytes]: return self._stdout - def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: + def store(self, file: IO[bytes]) -> Deferred[None] | None: pass @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri: str, *, feed_options: Optional[dict[str, Any]] = None): + def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): self.path: str = file_uri_to_path(uri) feed_options = feed_options or {} self.write_mode: OpenBinaryMode = ( @@ -199,7 +199,7 @@ class FileFeedStorage: dirname.mkdir(parents=True) return Path(self.path).open(self.write_mode) - def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: + def store(self, file: IO[bytes]) -> Deferred[None] | None: file.close() return None @@ -208,27 +208,27 @@ class S3FeedStorage(BlockingFeedStorage): def __init__( self, uri: str, - access_key: Optional[str] = None, - secret_key: Optional[str] = None, - acl: Optional[str] = None, - endpoint_url: Optional[str] = None, + access_key: str | None = None, + secret_key: str | None = None, + acl: str | None = None, + endpoint_url: str | None = None, *, - feed_options: Optional[dict[str, Any]] = None, - session_token: Optional[str] = None, - region_name: Optional[str] = None, + feed_options: dict[str, Any] | None = None, + session_token: str | None = None, + region_name: str | None = None, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") u = urlparse(uri) assert u.hostname self.bucketname: str = u.hostname - self.access_key: Optional[str] = u.username or access_key - self.secret_key: Optional[str] = u.password or secret_key - self.session_token: Optional[str] = session_token + self.access_key: str | None = u.username or access_key + self.secret_key: str | None = u.password or secret_key + self.session_token: str | None = session_token self.keyname: str = u.path[1:] # remove first "/" - self.acl: Optional[str] = acl - self.endpoint_url: Optional[str] = endpoint_url - self.region_name: Optional[str] = region_name + self.acl: str | None = acl + self.endpoint_url: str | None = endpoint_url + self.region_name: str | None = region_name # It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client, # there seems to be no good way to infer it statically. self.s3_client: Any @@ -279,7 +279,7 @@ class S3FeedStorage(BlockingFeedStorage): crawler: Crawler, uri: str, *, - feed_options: Optional[dict[str, Any]] = None, + feed_options: dict[str, Any] | None = None, ) -> Self: return build_storage( cls, @@ -310,9 +310,9 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri: str, project_id: Optional[str], acl: Optional[str]): - self.project_id: Optional[str] = project_id - self.acl: Optional[str] = acl + def __init__(self, uri: str, project_id: str | None, acl: str | None): + self.project_id: str | None = project_id + self.acl: str | None = acl u = urlparse(uri) assert u.hostname self.bucket_name: str = u.hostname @@ -342,7 +342,7 @@ class FTPFeedStorage(BlockingFeedStorage): uri: str, use_active_mode: bool = False, *, - feed_options: Optional[dict[str, Any]] = None, + feed_options: dict[str, Any] | None = None, ): u = urlparse(uri) if not u.hostname: @@ -361,7 +361,7 @@ class FTPFeedStorage(BlockingFeedStorage): crawler: Crawler, uri: str, *, - feed_options: Optional[dict[str, Any]] = None, + feed_options: dict[str, Any] | None = None, ) -> Self: return build_storage( cls, @@ -399,8 +399,8 @@ class FeedSlot: settings: BaseSettings, crawler: Crawler, ): - self.file: Optional[IO[bytes]] = None - self.exporter: Optional[BaseItemExporter] = None + self.file: IO[bytes] | None = None + self.exporter: BaseItemExporter | None = None self.storage: FeedStorageProtocol = storage # feed params self.batch_id: int = batch_id @@ -558,7 +558,7 @@ class FeedExporter: self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) ) - def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred[None]]: + def _close_slot(self, slot: FeedSlot, spider: Spider) -> Deferred[None] | None: def get_file(slot_: FeedSlot) -> IO[bytes]: assert slot_.file if isinstance(slot_.file, PostProcessingManager): @@ -770,8 +770,8 @@ class FeedExporter: def _get_uri_params( self, spider: Spider, - uri_params_function: Union[str, UriParamsCallableT, None], - slot: Optional[FeedSlot] = None, + uri_params_function: str | UriParamsCallableT | None, + slot: FeedSlot | None = None, ) -> dict[str, Any]: params = {} for k in dir(spider): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index a72f9db51..0e6120c21 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -9,7 +9,7 @@ from importlib import import_module from pathlib import Path from time import time from types import ModuleType -from typing import IO, TYPE_CHECKING, Any, Optional, Union, cast +from typing import IO, TYPE_CHECKING, Any, cast from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict @@ -66,16 +66,14 @@ class RFC2616Policy: self.always_store: bool = settings.getbool("HTTPCACHE_ALWAYS_STORE") self.ignore_schemes: list[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") self._cc_parsed: WeakKeyDictionary[ - Union[Request, Response], dict[bytes, Optional[bytes]] + Request | Response, dict[bytes, bytes | None] ] = WeakKeyDictionary() self.ignore_response_cache_controls: list[bytes] = [ to_bytes(cc) for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS") ] - def _parse_cachecontrol( - self, r: Union[Request, Response] - ) -> dict[bytes, Optional[bytes]]: + def _parse_cachecontrol(self, r: Request | Response) -> dict[bytes, bytes | None]: if r not in self._cc_parsed: cch = r.headers.get(b"Cache-Control", b"") assert cch is not None @@ -191,7 +189,7 @@ class RFC2616Policy: if b"ETag" in cachedresponse.headers: request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"] - def _get_max_age(self, cc: dict[bytes, Optional[bytes]]) -> Optional[int]: + def _get_max_age(self, cc: dict[bytes, bytes | None]) -> int | None: try: return max(0, int(cc[b"max-age"])) # type: ignore[arg-type] except (KeyError, ValueError): @@ -275,7 +273,7 @@ class DbmCacheStorage: def close_spider(self, spider: Spider) -> None: self.db.close() - def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]: + def retrieve_response(self, spider: Spider, request: Request) -> Response | None: data = self._read_data(spider, request) if data is None: return None # not cached @@ -300,7 +298,7 @@ class DbmCacheStorage: self.db[f"{key}_data"] = pickle.dumps(data, protocol=4) self.db[f"{key}_time"] = str(time()) - def _read_data(self, spider: Spider, request: Request) -> Optional[dict[str, Any]]: + def _read_data(self, spider: Spider, request: Request) -> dict[str, Any] | None: key = self._fingerprinter.fingerprint(request).hex() db = self.db tkey = f"{key}_time" @@ -320,9 +318,7 @@ class FilesystemCacheStorage: self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") # https://github.com/python/mypy/issues/10740 - self._open: Callable[ - Concatenate[Union[str, os.PathLike], str, ...], IO[bytes] - ] = ( + self._open: Callable[Concatenate[str | os.PathLike, str, ...], IO[bytes]] = ( gzip.open if self.use_gzip else open # type: ignore[assignment] ) @@ -339,7 +335,7 @@ class FilesystemCacheStorage: def close_spider(self, spider: Spider) -> None: pass - def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]: + def retrieve_response(self, spider: Spider, request: Request) -> Response | None: """Return response if present in cache, or None otherwise.""" metadata = self._read_meta(spider, request) if metadata is None: @@ -387,7 +383,7 @@ class FilesystemCacheStorage: key = self._fingerprinter.fingerprint(request).hex() return str(Path(self.cachedir, spider.name, key[0:2], key)) - def _read_meta(self, spider: Spider, request: Request) -> Optional[dict[str, Any]]: + def _read_meta(self, spider: Spider, request: Request) -> dict[str, Any] | None: rpath = Path(self._get_request_path(spider, request)) metapath = rpath / "pickled_meta" if not metapath.exists(): @@ -399,7 +395,7 @@ class FilesystemCacheStorage: return cast(dict[str, Any], pickle.load(f)) # nosec -def parse_cachecontrol(header: bytes) -> dict[bytes, Optional[bytes]]: +def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]: """Parse Cache-Control header https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 @@ -419,7 +415,7 @@ def parse_cachecontrol(header: bytes) -> dict[bytes, Optional[bytes]]: return directives -def rfc1123_to_epoch(date_str: Union[str, bytes, None]) -> Optional[int]: +def rfc1123_to_epoch(date_str: str | bytes | None) -> int | None: try: date_str = to_unicode(date_str, encoding="ascii") # type: ignore[arg-type] return mktime_tz(parsedate_tz(date_str)) # type: ignore[arg-type] diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 01484481b..e829d8b92 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from twisted.internet import task @@ -29,7 +29,7 @@ class LogStats: self.stats: StatsCollector = stats self.interval: float = interval self.multiplier: float = 60.0 / self.interval - self.task: Optional[task.LoopingCall] = None + self.task: task.LoopingCall | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -81,7 +81,7 @@ class LogStats: def calculate_final_stats( self, spider: Spider - ) -> Union[tuple[None, None], tuple[float, float]]: + ) -> tuple[None, None] | tuple[float, float]: start_time = self.stats.get_value("start_time") finished_time = self.stats.get_value("finished_time") diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index fba12bec7..f2e3782a4 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging from datetime import datetime, timezone from json import JSONEncoder -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from twisted.internet import task @@ -36,7 +36,7 @@ class PeriodicLog: self.stats: StatsCollector = stats self.interval: float = interval self.multiplier: float = 60.0 / self.interval - self.task: Optional[task.LoopingCall] = None + self.task: task.LoopingCall | None = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) self.ext_stats_include: list[str] = ext_stats.get("include", []) @@ -52,7 +52,7 @@ class PeriodicLog: if not interval: raise NotConfigured try: - ext_stats: Optional[dict[str, Any]] = crawler.settings.getdict( + ext_stats: dict[str, Any] | None = crawler.settings.getdict( "PERIODIC_LOG_STATS" ) except (TypeError, ValueError): @@ -62,7 +62,7 @@ class PeriodicLog: else None ) try: - ext_delta: Optional[dict[str, Any]] = crawler.settings.getdict( + ext_delta: dict[str, Any] | None = crawler.settings.getdict( "PERIODIC_LOG_DELTA" ) except (TypeError, ValueError): @@ -93,8 +93,8 @@ class PeriodicLog: def spider_opened(self, spider: Spider) -> None: self.time_prev: datetime = datetime.now(tz=timezone.utc) - self.delta_prev: dict[str, Union[int, float]] = {} - self.stats_prev: dict[str, Union[int, float]] = {} + self.delta_prev: dict[str, int | float] = {} + self.stats_prev: dict[str, int | float] = {} self.task = task.LoopingCall(self.log) self.task.start(self.interval) @@ -110,7 +110,7 @@ class PeriodicLog: logger.info(self.encoder.encode(data)) def log_delta(self) -> dict[str, Any]: - num_stats: dict[str, Union[int, float]] = { + num_stats: dict[str, int | float] = { k: v for k, v in self.stats._stats.items() if isinstance(v, (int, float)) diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 567efd7a1..642919be9 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -2,7 +2,7 @@ from __future__ import annotations import pickle # nosec from pathlib import Path -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from scrapy import Spider, signals from scrapy.exceptions import NotConfigured @@ -18,8 +18,8 @@ if TYPE_CHECKING: class SpiderState: """Store and load spider state during a scraping job""" - def __init__(self, jobdir: Optional[str] = None): - self.jobdir: Optional[str] = jobdir + def __init__(self, jobdir: str | None = None): + self.jobdir: str | None = jobdir @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index c8fefe792..600eebcf2 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -6,7 +6,7 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from scrapy import Spider, signals from scrapy.exceptions import NotConfigured @@ -39,7 +39,7 @@ class StatsMailer: crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider: Spider) -> Optional[Deferred[None]]: + def spider_closed(self, spider: Spider) -> Deferred[None] | None: spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 6b5fd181d..d4b4f0e9d 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured @@ -90,8 +90,8 @@ class AutoThrottle: def _get_slot( self, request: Request, spider: Spider - ) -> tuple[Optional[str], Optional[Slot]]: - key: Optional[str] = request.meta.get("download_slot") + ) -> tuple[str | None, Slot | None]: + key: str | None = request.meta.get("download_slot") if key is None: return None, None assert self.crawler.engine diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index b5388a918..56941ad51 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -5,7 +5,7 @@ import time from http.cookiejar import Cookie from http.cookiejar import CookieJar as _CookieJar from http.cookiejar import CookiePolicy, DefaultCookiePolicy -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -28,7 +28,7 @@ IPV4_RE = re.compile(r"\.\d+$", re.ASCII) class CookieJar: def __init__( self, - policy: Optional[CookiePolicy] = None, + policy: CookiePolicy | None = None, check_expired_frequency: int = 10000, ): self.policy: CookiePolicy = policy or DefaultCookiePolicy() @@ -83,9 +83,9 @@ class CookieJar: def clear( self, - domain: Optional[str] = None, - path: Optional[str] = None, - name: Optional[str] = None, + domain: str | None = None, + path: str | None = None, + name: str | None = None, ) -> None: self.jar.clear(domain, path, name) @@ -188,7 +188,7 @@ class WrappedRequest: def has_header(self, name: str) -> bool: return name in self.request.headers - def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]: + def get_header(self, name: str, default: str | None = None) -> str | None: value = self.request.headers.get(name, default) return to_unicode(value, errors="replace") if value is not None else None diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 1dcbcb966..29ba9533b 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, Optional, Union, cast +from typing import TYPE_CHECKING, Any, AnyStr, Union, cast from w3lib.http import headers_dict_to_raw @@ -25,14 +25,14 @@ class Headers(CaselessDict): def __init__( self, - seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, + seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, encoding: str = "utf-8", ): self.encoding: str = encoding super().__init__(seq) def update( # type: ignore[override] - self, seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]]] + self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] ) -> None: seq = seq.items() if isinstance(seq, Mapping) else seq iseq: dict[bytes, list[bytes]] = {} @@ -44,7 +44,7 @@ class Headers(CaselessDict): """Normalize key to bytes""" return self._tobytes(key.title()) - def normvalue(self, value: Union[_RawValueT, Iterable[_RawValueT]]) -> list[bytes]: + def normvalue(self, value: _RawValueT | Iterable[_RawValueT]) -> list[bytes]: """Normalize values to bytes""" _value: Iterable[_RawValueT] if value is None: @@ -67,13 +67,13 @@ class Headers(CaselessDict): return str(x).encode(self.encoding) raise TypeError(f"Unsupported value type: {type(x)}") - def __getitem__(self, key: AnyStr) -> Optional[bytes]: + def __getitem__(self, key: AnyStr) -> bytes | None: try: return cast(list[bytes], super().__getitem__(key))[-1] except IndexError: return None - def get(self, key: AnyStr, def_val: Any = None) -> Optional[bytes]: + def get(self, key: AnyStr, def_val: Any = None) -> bytes | None: try: return cast(list[bytes], super().get(key, def_val))[-1] except IndexError: @@ -103,7 +103,7 @@ class Headers(CaselessDict): def items(self) -> Iterable[tuple[bytes, list[bytes]]]: # type: ignore[override] return ((k, self.getlist(k)) for k in self.keys()) - def values(self) -> list[Optional[bytes]]: # type: ignore[override] + def values(self) -> list[bytes | None]: # type: ignore[override] return [ self[k] for k in self.keys() # pylint: disable=consider-using-dict-items ] diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index aac8d3e50..ed225555c 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -13,7 +13,6 @@ from typing import ( Any, AnyStr, NoReturn, - Optional, TypedDict, TypeVar, Union, @@ -112,18 +111,18 @@ class Request(object_ref): def __init__( self, url: str, - callback: Optional[CallbackT] = None, + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[CookiesT] = None, - meta: Optional[dict[str, Any]] = None, + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable[[Failure], Any]] = None, - flags: Optional[list[str]] = None, - cb_kwargs: Optional[dict[str, Any]] = None, + errback: Callable[[Failure], Any] | None = None, + flags: list[str] | None = None, + cb_kwargs: dict[str, Any] | None = None, ) -> None: self._encoding: str = encoding # this one has to be set first self.method: str = str(method).upper() @@ -139,17 +138,15 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") - self.callback: Optional[CallbackT] = callback - self.errback: Optional[Callable[[Failure], Any]] = errback + self.callback: CallbackT | None = callback + self.errback: Callable[[Failure], Any] | None = errback self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) self.dont_filter: bool = dont_filter - self._meta: Optional[dict[str, Any]] = dict(meta) if meta else None - self._cb_kwargs: Optional[dict[str, Any]] = ( - dict(cb_kwargs) if cb_kwargs else None - ) + self._meta: dict[str, Any] | None = dict(meta) if meta else None + self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None self.flags: list[str] = [] if flags is None else list(flags) @property @@ -186,7 +183,7 @@ class Request(object_ref): def body(self) -> bytes: return self._body - def _set_body(self, body: Optional[Union[str, bytes]]) -> None: + def _set_body(self, body: str | bytes | None) -> None: self._body = b"" if body is None else to_bytes(body, self.encoding) @property @@ -208,7 +205,7 @@ class Request(object_ref): def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[type[Request]] = None, **kwargs: Any + self, *args: Any, cls: type[Request] | None = None, **kwargs: Any ) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: @@ -255,7 +252,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> dict[str, Any]: + def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d9c913672..2fabf08d1 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -62,14 +62,14 @@ class FormRequest(Request): def from_response( cls, response: TextResponse, - formname: Optional[str] = None, - formid: Optional[str] = None, + formname: str | None = None, + formid: str | None = None, formnumber: int = 0, formdata: FormdataType = None, - clickdata: Optional[dict[str, Union[str, int]]] = None, + clickdata: dict[str, str | int] | None = None, dont_click: bool = False, - formxpath: Optional[str] = None, - formcss: Optional[str] = None, + formxpath: str | None = None, + formcss: str | None = None, **kwargs: Any, ) -> Self: kwargs.setdefault("encoding", response.encoding) @@ -92,7 +92,7 @@ class FormRequest(Request): return cls(url=url, method=method, formdata=formdata, **kwargs) -def _get_form_url(form: FormElement, url: Optional[str]) -> str: +def _get_form_url(form: FormElement, url: str | None) -> str: assert form.base_url is not None # typing if url is None: action = form.get("action") @@ -113,10 +113,10 @@ def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str: def _get_form( response: TextResponse, - formname: Optional[str], - formid: Optional[str], + formname: str | None, + formid: str | None, formnumber: int, - formxpath: Optional[str], + formxpath: str | None, ) -> FormElement: """Find the wanted form element within the given response.""" root = response.selector.root @@ -160,7 +160,7 @@ def _get_inputs( form: FormElement, formdata: FormdataType, dont_click: bool, - clickdata: Optional[dict[str, Union[str, int]]], + clickdata: dict[str, str | int] | None, ) -> list[FormdataKVType]: """Return a list of key-value pairs for the inputs found in the given form.""" try: @@ -196,8 +196,8 @@ def _get_inputs( def _value( - ele: Union[InputElement, SelectElement, TextareaElement] -) -> tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: + ele: InputElement | SelectElement | TextareaElement, +) -> tuple[str | None, None | str | MultipleSelectOptions]: n = ele.name v = ele.value if ele.tag == "select": @@ -206,8 +206,8 @@ def _value( def _select_value( - ele: SelectElement, n: Optional[str], v: Union[None, str, MultipleSelectOptions] -) -> tuple[Optional[str], Union[None, str, MultipleSelectOptions]]: + ele: SelectElement, n: str | None, v: None | str | MultipleSelectOptions +) -> tuple[str | None, None | str | MultipleSelectOptions]: multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected @@ -218,8 +218,8 @@ def _select_value( def _get_clickable( - clickdata: Optional[dict[str, Union[str, int]]], form: FormElement -) -> Optional[tuple[str, str]]: + clickdata: dict[str, str | int] | None, form: FormElement +) -> tuple[str, str] | None: """ Returns the clickable element specified in clickdata, if the latter is given. If not, it returns the first diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 48862534e..289c60591 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -10,7 +10,7 @@ from __future__ import annotations import copy import json import warnings -from typing import TYPE_CHECKING, Any, Optional, overload +from typing import TYPE_CHECKING, Any, overload from scrapy.http.request import Request, RequestTypeVar @@ -23,7 +23,7 @@ class JsonRequest(Request): attributes: tuple[str, ...] = Request.attributes + ("dumps_kwargs",) def __init__( - self, *args: Any, dumps_kwargs: Optional[dict[str, Any]] = None, **kwargs: Any + self, *args: Any, dumps_kwargs: dict[str, Any] | None = None, **kwargs: Any ) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault("sort_keys", True) @@ -59,7 +59,7 @@ class JsonRequest(Request): def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[type[Request]] = None, **kwargs: Any + self, *args: Any, cls: type[Request] | None = None, **kwargs: Any ) -> Request: body_passed = kwargs.get("body", None) is not None data: Any = kwargs.pop("data", None) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 096ecd370..01fe740a8 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -5,8 +5,10 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +from __future__ import annotations + import xmlrpc.client as xmlrpclib -from typing import Any, Optional +from typing import Any import defusedxml.xmlrpc @@ -19,7 +21,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any): + def __init__(self, *args: Any, encoding: str | None = None, **kwargs: Any): if "body" not in kwargs and "params" in kwargs: kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs} kwargs["body"] = xmlrpclib.dumps(**kw) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c69945e2d..d50388548 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, AnyStr, Optional, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload from urllib.parse import urljoin from scrapy.exceptions import NotSupported @@ -60,23 +60,23 @@ class Response(object_ref): self, url: str, status: int = 200, - headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, body: bytes = b"", - flags: Optional[list[str]] = None, - request: Optional[Request] = None, - certificate: Optional[Certificate] = None, - ip_address: Union[IPv4Address, IPv6Address, None] = None, - protocol: Optional[str] = None, + flags: list[str] | None = None, + request: Request | None = None, + certificate: Certificate | None = None, + ip_address: IPv4Address | IPv6Address | None = None, + protocol: str | None = None, ): self.headers: Headers = Headers(headers or {}) self.status: int = int(status) self._set_body(body) self._set_url(url) - self.request: Optional[Request] = request + self.request: Request | None = request self.flags: list[str] = [] if flags is None else list(flags) - self.certificate: Optional[Certificate] = certificate - self.ip_address: Union[IPv4Address, IPv6Address, None] = ip_address - self.protocol: Optional[str] = protocol + self.certificate: Certificate | None = certificate + self.ip_address: IPv4Address | IPv6Address | None = ip_address + self.protocol: str | None = protocol @property def cb_kwargs(self) -> dict[str, Any]: @@ -114,7 +114,7 @@ class Response(object_ref): def body(self) -> bytes: return self._body - def _set_body(self, body: Optional[bytes]) -> None: + def _set_body(self, body: bytes | None) -> None: if body is None: self._body = b"" elif not isinstance(body, bytes): @@ -142,7 +142,7 @@ class Response(object_ref): def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( - self, *args: Any, cls: Optional[type[Response]] = None, **kwargs: Any + self, *args: Any, cls: type[Response] | None = None, **kwargs: Any ) -> Response: """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: @@ -183,19 +183,19 @@ class Response(object_ref): def follow( self, - url: Union[str, Link], - callback: Optional[CallbackT] = None, + url: str | Link, + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[CookiesT] = None, - meta: Optional[dict[str, Any]] = None, - encoding: Optional[str] = "utf-8", + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, + encoding: str | None = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[dict[str, Any]] = None, - flags: Optional[list[str]] = None, + errback: Callable[[Failure], Any] | None = None, + cb_kwargs: dict[str, Any] | None = None, + flags: list[str] | None = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -236,19 +236,19 @@ class Response(object_ref): def follow_all( self, - urls: Iterable[Union[str, Link]], - callback: Optional[CallbackT] = None, + urls: Iterable[str | Link], + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[CookiesT] = None, - meta: Optional[dict[str, Any]] = None, - encoding: Optional[str] = "utf-8", + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, + encoding: str | None = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[dict[str, Any]] = None, - flags: Optional[list[str]] = None, + errback: Callable[[Failure], Any] | None = None, + cb_kwargs: dict[str, Any] | None = None, + flags: list[str] | None = None, ) -> Iterable[Request]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 680c1f602..c713f6188 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,9 +8,8 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations import json -from collections.abc import Iterable from contextlib import suppress -from typing import TYPE_CHECKING, Any, AnyStr, Optional, Union, cast +from typing import TYPE_CHECKING, Any, AnyStr, cast from urllib.parse import urljoin import parsel @@ -24,16 +23,16 @@ from w3lib.encoding import ( from w3lib.html import strip_html5_whitespace from scrapy.http.response import Response -from scrapy.link import Link from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: - from collections.abc import Callable, Mapping + from collections.abc import Callable, Iterable, Mapping from twisted.python.failure import Failure from scrapy.http.request import CallbackT, CookiesT, Request + from scrapy.link import Link from scrapy.selector import Selector, SelectorList @@ -47,13 +46,13 @@ class TextResponse(Response): attributes: tuple[str, ...] = Response.attributes + ("encoding",) def __init__(self, *args: Any, **kwargs: Any): - self._encoding: Optional[str] = kwargs.pop("encoding", None) - self._cached_benc: Optional[str] = None - self._cached_ubody: Optional[str] = None - self._cached_selector: Optional[Selector] = None + self._encoding: str | None = kwargs.pop("encoding", None) + self._cached_benc: str | None = None + self._cached_ubody: str | None = None + self._cached_selector: Selector | None = None super().__init__(*args, **kwargs) - def _set_body(self, body: Union[str, bytes, None]) -> None: + def _set_body(self, body: str | bytes | None) -> None: self._body: bytes = b"" # used by encoding detection if isinstance(body, str): if self._encoding is None: @@ -69,7 +68,7 @@ class TextResponse(Response): def encoding(self) -> str: return self._declared_encoding() or self._body_inferred_encoding() - def _declared_encoding(self) -> Optional[str]: + def _declared_encoding(self) -> str | None: return ( self._encoding or self._bom_encoding() @@ -104,7 +103,7 @@ class TextResponse(Response): return urljoin(get_base_url(self), url) @memoizemethod_noargs - def _headers_encoding(self) -> Optional[str]: + def _headers_encoding(self) -> str | None: content_type = cast(bytes, self.headers.get(b"Content-Type", b"")) return http_content_type_encoding(to_unicode(content_type, encoding="latin-1")) @@ -123,7 +122,7 @@ class TextResponse(Response): self._cached_ubody = ubody return self._cached_benc - def _auto_detect_fun(self, text: bytes) -> Optional[str]: + def _auto_detect_fun(self, text: bytes) -> str | None: for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"): try: text.decode(enc) @@ -133,11 +132,11 @@ class TextResponse(Response): return None @memoizemethod_noargs - def _body_declared_encoding(self) -> Optional[str]: + def _body_declared_encoding(self) -> str | None: return html_body_declared_encoding(self.body) @memoizemethod_noargs - def _bom_encoding(self) -> Optional[str]: + def _bom_encoding(self) -> str | None: return read_bom(self.body)[0] @property @@ -170,19 +169,19 @@ class TextResponse(Response): def follow( self, - url: Union[str, Link, parsel.Selector], - callback: Optional[CallbackT] = None, + url: str | Link | parsel.Selector, + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[CookiesT] = None, - meta: Optional[dict[str, Any]] = None, - encoding: Optional[str] = None, + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, + encoding: str | None = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[dict[str, Any]] = None, - flags: Optional[list[str]] = None, + errback: Callable[[Failure], Any] | None = None, + cb_kwargs: dict[str, Any] | None = None, + flags: list[str] | None = None, ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -223,21 +222,21 @@ class TextResponse(Response): def follow_all( self, - urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None, - callback: Optional[CallbackT] = None, + urls: Iterable[str | Link] | parsel.SelectorList | None = None, + callback: CallbackT | None = None, method: str = "GET", - headers: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, - body: Optional[Union[bytes, str]] = None, - cookies: Optional[CookiesT] = None, - meta: Optional[dict[str, Any]] = None, - encoding: Optional[str] = None, + headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: bytes | str | None = None, + cookies: CookiesT | None = None, + meta: dict[str, Any] | None = None, + encoding: str | None = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable[[Failure], Any]] = None, - cb_kwargs: Optional[dict[str, Any]] = None, - flags: Optional[list[str]] = None, - css: Optional[str] = None, - xpath: Optional[str] = None, + errback: Callable[[Failure], Any] | None = None, + cb_kwargs: dict[str, Any] | None = None, + flags: list[str] | None = None, + css: str | None = None, + xpath: str | None = None, ) -> Iterable[Request]: """ A generator that produces :class:`~.Request` instances to follow all @@ -279,7 +278,7 @@ class TextResponse(Response): with suppress(_InvalidSelector): urls.append(_url_from_selector(sel)) return super().follow_all( - urls=cast(Iterable[Union[str, Link]], urls), + urls=cast("Iterable[str | Link]", urls), callback=callback, method=method, headers=headers, diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 73673b1c6..192f937ce 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -9,7 +9,7 @@ import operator import re from collections.abc import Callable, Iterable from functools import partial -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Union, cast from urllib.parse import urljoin, urlparse from lxml import etree # nosec @@ -58,9 +58,9 @@ def _canonicalize_link_url(link: Link) -> str: class LxmlParserLinkExtractor: def __init__( self, - tag: Union[str, Callable[[str], bool]] = "a", - attr: Union[str, Callable[[str], bool]] = "href", - process: Optional[Callable[[Any], Any]] = None, + tag: str | Callable[[str], bool] = "a", + attr: str | Callable[[str], bool] = "href", + process: Callable[[Any], Any] | None = None, unique: bool = False, strip: bool = True, canonicalized: bool = False, @@ -166,18 +166,18 @@ class LxmlLinkExtractor: self, allow: _RegexOrSeveralT = (), deny: _RegexOrSeveralT = (), - allow_domains: Union[str, Iterable[str]] = (), - deny_domains: Union[str, Iterable[str]] = (), - restrict_xpaths: Union[str, Iterable[str]] = (), - tags: Union[str, Iterable[str]] = ("a", "area"), - attrs: Union[str, Iterable[str]] = ("href",), + allow_domains: str | Iterable[str] = (), + deny_domains: str | Iterable[str] = (), + restrict_xpaths: str | Iterable[str] = (), + tags: str | Iterable[str] = ("a", "area"), + attrs: str | Iterable[str] = ("href",), canonicalize: bool = False, unique: bool = True, - process_value: Optional[Callable[[Any], Any]] = None, - deny_extensions: Union[str, Iterable[str], None] = None, - restrict_css: Union[str, Iterable[str]] = (), + process_value: Callable[[Any], Any] | None = None, + deny_extensions: str | Iterable[str] | None = None, + restrict_css: str | Iterable[str] = (), strip: bool = True, - restrict_text: Optional[_RegexOrSeveralT] = None, + restrict_text: _RegexOrSeveralT | None = None, ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) self.link_extractor = LxmlParserLinkExtractor( @@ -206,7 +206,7 @@ class LxmlLinkExtractor: self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text) @staticmethod - def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> list[re.Pattern[str]]: + def _compile_regexes(value: _RegexOrSeveralT | None) -> list[re.Pattern[str]]: return [ x if isinstance(x, re.Pattern) else re.compile(x) for x in arg_to_iter(value) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 9644cc093..d35720a45 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -6,7 +6,7 @@ See documentation in docs/topics/loaders.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any import itemloaders @@ -92,9 +92,9 @@ class ItemLoader(itemloaders.ItemLoader): def __init__( self, item: Any = None, - selector: Optional[Selector] = None, - response: Optional[TextResponse] = None, - parent: Optional[itemloaders.ItemLoader] = None, + selector: Selector | None = None, + response: TextResponse | None = None, + parent: itemloaders.ItemLoader | None = None, **context: Any, ): if selector is None and response is not None: diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 2b838d8e2..544f4adfe 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging import os -from typing import TYPE_CHECKING, Any, Optional, TypedDict, Union +from typing import TYPE_CHECKING, Any, TypedDict from twisted.python.failure import Failure @@ -31,7 +31,7 @@ DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" class LogFormatterResult(TypedDict): level: int msg: str - args: Union[dict[str, Any], tuple[Any, ...]] + args: dict[str, Any] | tuple[Any, ...] class LogFormatter: @@ -93,7 +93,7 @@ class LogFormatter: } def scraped( - self, item: Any, response: Union[Response, Failure, None], spider: Spider + self, item: Any, response: Response | Failure | None, spider: Spider ) -> LogFormatterResult: """Logs a message when an item is scraped by a spider.""" src: Any @@ -116,7 +116,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Optional[Response], + response: Response | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" @@ -133,7 +133,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Optional[Response], + response: Response | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing @@ -153,7 +153,7 @@ class LogFormatter: self, failure: Failure, request: Request, - response: Union[Response, Failure], + response: Response | Failure, spider: Spider, ) -> LogFormatterResult: """Logs an error message from a spider. @@ -174,7 +174,7 @@ class LogFormatter: failure: Failure, request: Request, spider: Spider, - errmsg: Optional[str] = None, + errmsg: str | None = None, ) -> LogFormatterResult: """Logs a download error message from a spider (typically coming from the engine). diff --git a/scrapy/mail.py b/scrapy/mail.py index f33cf2939..1e65b1623 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -14,7 +14,7 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from typing import IO, TYPE_CHECKING, Any, Optional, Union +from typing import IO, TYPE_CHECKING, Any from twisted import version as twisted_version from twisted.internet import ssl @@ -45,7 +45,7 @@ logger = logging.getLogger(__name__) COMMASPACE = ", " -def _to_bytes_or_none(text: Union[str, bytes, None]) -> Optional[bytes]: +def _to_bytes_or_none(text: str | bytes | None) -> bytes | None: if text is None: return None return to_bytes(text) @@ -56,8 +56,8 @@ class MailSender: self, smtphost: str = "localhost", mailfrom: str = "scrapy@localhost", - smtpuser: Optional[str] = None, - smtppass: Optional[str] = None, + smtpuser: str | None = None, + smtppass: str | None = None, smtpport: int = 25, smtptls: bool = False, smtpssl: bool = False, @@ -65,8 +65,8 @@ class MailSender: ): self.smtphost: str = smtphost self.smtpport: int = smtpport - self.smtpuser: Optional[bytes] = _to_bytes_or_none(smtpuser) - self.smtppass: Optional[bytes] = _to_bytes_or_none(smtppass) + self.smtpuser: bytes | None = _to_bytes_or_none(smtpuser) + self.smtppass: bytes | None = _to_bytes_or_none(smtppass) self.smtptls: bool = smtptls self.smtpssl: bool = smtpssl self.mailfrom: str = mailfrom @@ -86,15 +86,15 @@ class MailSender: def send( self, - to: Union[str, list[str]], + to: str | list[str], subject: str, body: str, - cc: Union[str, list[str], None] = None, + cc: str | list[str] | None = None, attachs: Sequence[tuple[str, str, IO[Any]]] = (), mimetype: str = "text/plain", - charset: Optional[str] = None, - _callback: Optional[Callable[..., None]] = None, - ) -> Optional[Deferred[None]]: + charset: str | None = None, + _callback: Callable[..., None] | None = None, + ) -> Deferred[None] | None: from twisted.internet import reactor msg: MIMEBase diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 825d6b4c8..39f26717a 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging import pprint from collections import defaultdict, deque -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeVar, cast from scrapy.exceptions import NotConfigured from scrapy.utils.defer import process_chain, process_parallel @@ -40,9 +40,9 @@ class MiddlewareManager: self.middlewares = middlewares # Only process_spider_output and process_spider_exception can be None. # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. - self.methods: dict[ - str, deque[Union[None, Callable, tuple[Callable, Callable]]] - ] = defaultdict(deque) + self.methods: dict[str, deque[None | Callable | tuple[Callable, Callable]]] = ( + defaultdict(deque) + ) for mw in middlewares: self._add_middleware(mw) @@ -51,9 +51,7 @@ class MiddlewareManager: raise NotImplementedError @classmethod - def from_settings( - cls, settings: Settings, crawler: Optional[Crawler] = None - ) -> Self: + def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self: mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 32e9ffe7c..4a8639c22 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -17,17 +17,7 @@ from contextlib import suppress from ftplib import FTP from io import BytesIO from pathlib import Path -from typing import ( - IO, - TYPE_CHECKING, - Any, - NoReturn, - Optional, - Protocol, - TypedDict, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, NoReturn, Protocol, TypedDict, cast from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -61,7 +51,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def _to_string(path: Union[str, PathLike[str]]) -> str: +def _to_string(path: str | PathLike[str]) -> str: return str(path) # convert a Path object to string @@ -99,17 +89,17 @@ class FilesStoreProtocol(Protocol): path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[dict[str, Any]] = None, - headers: Optional[dict[str, str]] = None, - ) -> Optional[Deferred[Any]]: ... + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + ) -> Deferred[Any] | None: ... def stat_file( self, path: str, info: MediaPipeline.SpiderInfo - ) -> Union[StatInfo, Deferred[StatInfo]]: ... + ) -> StatInfo | Deferred[StatInfo]: ... class FSFilesStore: - def __init__(self, basedir: Union[str, PathLike[str]]): + def __init__(self, basedir: str | PathLike[str]): basedir = _to_string(basedir) if "://" in basedir: basedir = basedir.split("://", 1)[1] @@ -121,18 +111,18 @@ class FSFilesStore: def persist_file( self, - path: Union[str, PathLike[str]], + path: str | PathLike[str], buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[dict[str, Any]] = None, - headers: Optional[dict[str, str]] = None, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ) -> None: absolute_path = self._get_filesystem_path(path) self._mkdir(absolute_path.parent, info) absolute_path.write_bytes(buf.getvalue()) def stat_file( - self, path: Union[str, PathLike[str]], info: MediaPipeline.SpiderInfo + self, path: str | PathLike[str], info: MediaPipeline.SpiderInfo ) -> StatInfo: absolute_path = self._get_filesystem_path(path) try: @@ -145,12 +135,12 @@ class FSFilesStore: return {"last_modified": last_modified, "checksum": checksum} - def _get_filesystem_path(self, path: Union[str, PathLike[str]]) -> Path: + def _get_filesystem_path(self, path: str | PathLike[str]) -> Path: path_comps = _to_string(path).split("/") return Path(self.basedir, *path_comps) def _mkdir( - self, dirname: Path, domain: Optional[MediaPipeline.SpiderInfo] = None + self, dirname: Path, domain: MediaPipeline.SpiderInfo | None = None ) -> None: seen: set[str] = self.created_directories[domain] if domain else set() if str(dirname) not in seen: @@ -218,8 +208,8 @@ class S3FilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[dict[str, Any]] = None, - headers: Optional[dict[str, str]] = None, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ) -> Deferred[Any]: """Upload file to S3 storage""" key_name = f"{self.prefix}{path}" @@ -327,7 +317,7 @@ class GCSFilesStore: deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess), ) - def _get_content_type(self, headers: Optional[dict[str, str]]) -> str: + def _get_content_type(self, headers: dict[str, str] | None) -> str: if headers and "Content-Type" in headers: return headers["Content-Type"] return "application/octet-stream" @@ -340,8 +330,8 @@ class GCSFilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[dict[str, Any]] = None, - headers: Optional[dict[str, str]] = None, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ) -> Deferred[Any]: blob_path = self._get_blob_path(path) blob = self.bucket.blob(blob_path) @@ -356,9 +346,9 @@ class GCSFilesStore: class FTPFilesStore: - FTP_USERNAME: Optional[str] = None - FTP_PASSWORD: Optional[str] = None - USE_ACTIVE_MODE: Optional[bool] = None + FTP_USERNAME: str | None = None + FTP_PASSWORD: str | None = None + USE_ACTIVE_MODE: bool | None = None def __init__(self, uri: str): if not uri.startswith("ftp://"): @@ -380,8 +370,8 @@ class FTPFilesStore: path: str, buf: BytesIO, info: MediaPipeline.SpiderInfo, - meta: Optional[dict[str, Any]] = None, - headers: Optional[dict[str, str]] = None, + meta: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ) -> Deferred[Any]: path = f"{self.basedir}/{path}" return deferToThread( @@ -450,9 +440,9 @@ class FilesPipeline(MediaPipeline): def __init__( self, - store_uri: Union[str, PathLike[str]], - download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, dict[str, Any], None] = None, + store_uri: str | PathLike[str], + download_func: Callable[[Request, Spider], Response] | None = None, + settings: Settings | dict[str, Any] | None = None, ): store_uri = _to_string(store_uri) if not store_uri: @@ -517,8 +507,8 @@ class FilesPipeline(MediaPipeline): def media_to_download( self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None - ) -> Deferred[Optional[FileInfo]]: - def _onsuccess(result: StatInfo) -> Optional[FileInfo]: + ) -> Deferred[FileInfo | None]: + def _onsuccess(result: StatInfo) -> FileInfo | None: if not result: return None # returning None force download @@ -551,7 +541,7 @@ class FilesPipeline(MediaPipeline): path = self.file_path(request, info=info, item=item) # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload] - dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess) + dfd2: Deferred[FileInfo | None] = dfd.addCallback(_onsuccess) dfd2.addErrback(lambda _: None) dfd2.addErrback( lambda f: logger.error( @@ -684,8 +674,8 @@ class FilesPipeline(MediaPipeline): def file_path( self, request: Request, - response: Optional[Response] = None, - info: Optional[MediaPipeline.SpiderInfo] = None, + response: Response | None = None, + info: MediaPipeline.SpiderInfo | None = None, *, item: Any = None, ) -> str: diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index f2fe4396b..bbba7d1e1 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,7 +11,7 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, cast from itemadapter import ItemAdapter @@ -74,9 +74,9 @@ class ImagesPipeline(FilesPipeline): def __init__( self, - store_uri: Union[str, PathLike[str]], - download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, dict[str, Any], None] = None, + store_uri: str | PathLike[str], + download_func: Callable[[Request, Spider], Response] | None = None, + settings: Settings | dict[str, Any] | None = None, ): try: from PIL import Image @@ -120,7 +120,7 @@ class ImagesPipeline(FilesPipeline): resolve("IMAGES_THUMBS"), self.THUMBS ) - self._deprecated_convert_image: Optional[bool] = None + self._deprecated_convert_image: bool | None = None @classmethod def from_settings(cls, settings: Settings) -> Self: @@ -168,7 +168,7 @@ class ImagesPipeline(FilesPipeline): *, item: Any = None, ) -> str: - checksum: Optional[str] = None + checksum: str | None = None for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) @@ -235,8 +235,8 @@ class ImagesPipeline(FilesPipeline): def convert_image( self, image: Image.Image, - size: Optional[tuple[int, int]] = None, - response_body: Optional[BytesIO] = None, + size: tuple[int, int] | None = None, + response_body: BytesIO | None = None, ) -> tuple[Image.Image, BytesIO]: if response_body is None: warnings.warn( @@ -291,8 +291,8 @@ class ImagesPipeline(FilesPipeline): def file_path( self, request: Request, - response: Optional[Response] = None, - info: Optional[MediaPipeline.SpiderInfo] = None, + response: Response | None = None, + info: MediaPipeline.SpiderInfo | None = None, *, item: Any = None, ) -> str: @@ -303,8 +303,8 @@ class ImagesPipeline(FilesPipeline): self, request: Request, thumb_id: str, - response: Optional[Response] = None, - info: Optional[MediaPipeline.SpiderInfo] = None, + response: Response | None = None, + info: MediaPipeline.SpiderInfo | None = None, *, item: Any = None, ) -> str: diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index b30cf9264..61eddffa7 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -9,7 +9,6 @@ from typing import ( Any, Literal, NoReturn, - Optional, TypedDict, TypeVar, Union, @@ -44,7 +43,7 @@ _T = TypeVar("_T") class FileInfo(TypedDict): url: str path: str - checksum: Optional[str] + checksum: str | None status: str @@ -64,15 +63,15 @@ class MediaPipeline(ABC): def __init__(self, spider: Spider): self.spider: Spider = spider self.downloading: set[bytes] = set() - self.downloaded: dict[bytes, Union[FileInfo, Failure]] = {} + self.downloaded: dict[bytes, FileInfo | Failure] = {} self.waiting: defaultdict[bytes, list[Deferred[FileInfo]]] = defaultdict( list ) def __init__( self, - download_func: Optional[Callable[[Request, Spider], Response]] = None, - settings: Union[Settings, dict[str, Any], None] = None, + download_func: Callable[[Request, Spider], Response] | None = None, + settings: Settings | dict[str, Any] | None = None, ): self.download_func = download_func @@ -94,8 +93,8 @@ class MediaPipeline(ABC): def _key_for_pipe( self, key: str, - base_class_name: Optional[str] = None, - settings: Optional[Settings] = None, + base_class_name: str | None = None, + settings: Settings | None = None, ) -> str: class_name = self.__class__.__name__ formatted_key = f"{class_name.upper()}_{key}" @@ -161,7 +160,7 @@ class MediaPipeline(ABC): # Download request checking media_to_download hook output first info.downloading.add(fp) - dfd: Deferred[Optional[FileInfo]] = mustbe_deferred( + dfd: Deferred[FileInfo | None] = mustbe_deferred( self.media_to_download, request, info, item=item ) dfd2: Deferred[FileInfo] = dfd.addCallback( @@ -182,8 +181,8 @@ class MediaPipeline(ABC): request.meta["handle_httpstatus_all"] = True def _check_media_to_download( - self, result: Optional[FileInfo], request: Request, info: SpiderInfo, item: Any - ) -> Union[FileInfo, Deferred[FileInfo]]: + self, result: FileInfo | None, request: Request, info: SpiderInfo, item: Any + ) -> FileInfo | Deferred[FileInfo]: if result is not None: return result dfd: Deferred[Response] @@ -201,7 +200,7 @@ class MediaPipeline(ABC): return dfd2 def _cache_result_and_execute_waiters( - self, result: Union[FileInfo, Failure], fp: bytes, info: SpiderInfo + self, result: FileInfo | Failure, fp: bytes, info: SpiderInfo ) -> None: if isinstance(result, Failure): # minimize cached information for failure @@ -243,7 +242,7 @@ class MediaPipeline(ABC): @abstractmethod def media_to_download( self, request: Request, info: SpiderInfo, *, item: Any = None - ) -> Deferred[Optional[FileInfo]]: + ) -> Deferred[FileInfo | None]: """Check request before starting download""" raise NotImplementedError() @@ -291,8 +290,8 @@ class MediaPipeline(ABC): def file_path( self, request: Request, - response: Optional[Response] = None, - info: Optional[SpiderInfo] = None, + response: Response | None = None, + info: SpiderInfo | None = None, *, item: Any = None, ) -> str: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index e1bb21fb1..28e2073a2 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -2,7 +2,7 @@ from __future__ import annotations import hashlib import logging -from typing import TYPE_CHECKING, Optional, Protocol, cast +from typing import TYPE_CHECKING, Protocol, cast from scrapy import Request from scrapy.core.downloader import Downloader @@ -42,7 +42,7 @@ class QueueProtocol(Protocol): def push(self, request: Request) -> None: ... - def pop(self) -> Optional[Request]: ... + def pop(self) -> Request | None: ... def close(self) -> None: ... @@ -96,7 +96,7 @@ class ScrapyPriorityQueue: self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self.key: str = key self.queues: dict[int, QueueProtocol] = {} - self.curprio: Optional[int] = None + self.curprio: int | None = None self.init_prios(startprios) def init_prios(self, startprios: Iterable[int]) -> None: @@ -127,7 +127,7 @@ class ScrapyPriorityQueue: if self.curprio is None or priority < self.curprio: self.curprio = priority - def pop(self) -> Optional[Request]: + def pop(self) -> Request | None: if self.curprio is None: return None q = self.queues[self.curprio] @@ -139,7 +139,7 @@ class ScrapyPriorityQueue: self.curprio = min(prios) if prios else None return m - def peek(self) -> Optional[Request]: + def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -193,7 +193,7 @@ class DownloaderAwarePriorityQueue: crawler: Crawler, downstream_queue_cls: type[QueueProtocol], key: str, - startprios: Optional[dict[str, Iterable[int]]] = None, + startprios: dict[str, Iterable[int]] | None = None, ) -> Self: return cls(crawler, downstream_queue_cls, key, startprios) @@ -202,7 +202,7 @@ class DownloaderAwarePriorityQueue: crawler: Crawler, downstream_queue_cls: type[QueueProtocol], key: str, - slot_startprios: Optional[dict[str, Iterable[int]]] = None, + slot_startprios: dict[str, Iterable[int]] | None = None, ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( @@ -239,7 +239,7 @@ class DownloaderAwarePriorityQueue: startprios, ) - def pop(self) -> Optional[Request]: + def pop(self) -> Request | None: stats = self._downloader_interface.stats(self.pqueues) if not stats: @@ -259,7 +259,7 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] queue.push(request) - def peek(self) -> Optional[Request]: + def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 97fa74bc2..99a6cc5f6 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from twisted.internet import defer from twisted.internet.base import ReactorBase, ThreadedResolver @@ -128,7 +128,7 @@ class CachingHostnameResolver: resolutionReceiver: IResolutionReceiver, hostName: str, portNumber: int = 0, - addressTypes: Optional[Sequence[type[IAddress]]] = None, + addressTypes: Sequence[type[IAddress]] | None = None, transportSemantics: str = "TCP", ) -> IHostResolution: try: diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 7154f2b95..3f6f030a5 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -8,7 +8,7 @@ from __future__ import annotations from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from scrapy.http import Response from scrapy.utils.misc import load_object @@ -58,7 +58,7 @@ class ResponseTypes: return self.classes.get(basetype, Response) def from_content_type( - self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = None + self, content_type: str | bytes, content_encoding: bytes | None = None ) -> type[Response]: """Return the most appropriate Response class from an HTTP Content-Type header""" @@ -70,7 +70,7 @@ class ResponseTypes: return self.from_mimetype(mimetype) def from_content_disposition( - self, content_disposition: Union[str, bytes] + self, content_disposition: str | bytes ) -> type[Response]: try: filename = ( @@ -123,10 +123,10 @@ class ResponseTypes: def from_args( self, - headers: Optional[Mapping[bytes, bytes]] = None, - url: Optional[str] = None, - filename: Optional[str] = None, - body: Optional[bytes] = None, + headers: Mapping[bytes, bytes] | None = None, + url: str | None = None, + filename: str | None = None, + body: bytes | None = None, ) -> type[Response]: """Guess the most appropriate Response class based on the given arguments.""" diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0d282dc37..a0e5fc671 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging import sys from abc import ABCMeta, abstractmethod -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) def decode_robotstxt( - robotstxt_body: bytes, spider: Optional[Spider], to_native_str_type: bool = False + robotstxt_body: bytes, spider: Spider | None, to_native_str_type: bool = False ) -> str: try: if to_native_str_type: @@ -57,7 +57,7 @@ class RobotParser(metaclass=ABCMeta): pass @abstractmethod - def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. :param url: Absolute URL @@ -70,10 +70,10 @@ class RobotParser(metaclass=ABCMeta): class PythonRobotParser(RobotParser): - def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): + def __init__(self, robotstxt_body: bytes, spider: Spider | None): from urllib.robotparser import RobotFileParser - self.spider: Optional[Spider] = spider + self.spider: Spider | None = spider body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) self.rp: RobotFileParser = RobotFileParser() self.rp.parse(body_decoded.splitlines()) @@ -84,18 +84,18 @@ class PythonRobotParser(RobotParser): o = cls(robotstxt_body, spider) return o - def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(user_agent, url) class ReppyRobotParser(RobotParser): - def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): + def __init__(self, robotstxt_body: bytes, spider: Spider | None): warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2) from reppy.robots import Robots - self.spider: Optional[Spider] = spider + self.spider: Spider | None = spider self.rp = Robots.parse("", robotstxt_body) @classmethod @@ -104,15 +104,15 @@ class ReppyRobotParser(RobotParser): o = cls(robotstxt_body, spider) return o - def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: return self.rp.allowed(url, user_agent) class RerpRobotParser(RobotParser): - def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): + def __init__(self, robotstxt_body: bytes, spider: Spider | None): from robotexclusionrulesparser import RobotExclusionRulesParser - self.spider: Optional[Spider] = spider + self.spider: Spider | None = spider self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser() body_decoded = decode_robotstxt(robotstxt_body, spider) self.rp.parse(body_decoded) @@ -123,17 +123,17 @@ class RerpRobotParser(RobotParser): o = cls(robotstxt_body, spider) return o - def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.is_allowed(user_agent, url) class ProtegoRobotParser(RobotParser): - def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]): + def __init__(self, robotstxt_body: bytes, spider: Spider | None): from protego import Protego - self.spider: Optional[Spider] = spider + self.spider: Spider | None = spider body_decoded = decode_robotstxt(robotstxt_body, spider) self.rp = Protego.parse(body_decoded) @@ -143,7 +143,7 @@ class ProtegoRobotParser(RobotParser): o = cls(robotstxt_body, spider) return o - def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool: + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 0a3eae409..db9014b41 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -2,7 +2,9 @@ XPath selectors based on lxml """ -from typing import Any, Optional, Union +from __future__ import annotations + +from typing import Any from parsel import Selector as _ParselSelector @@ -16,13 +18,13 @@ __all__ = ["Selector", "SelectorList"] _NOT_SET = object() -def _st(response: Optional[TextResponse], st: Optional[str]) -> str: +def _st(response: TextResponse | None, st: str | None) -> str: if st is None: return "xml" if isinstance(response, XmlResponse) else "html" return st -def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse: +def _response_from_text(text: str | bytes, st: str | None) -> TextResponse: rt: type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) @@ -71,10 +73,10 @@ class Selector(_ParselSelector, object_ref): def __init__( self, - response: Optional[TextResponse] = None, - text: Optional[str] = None, - type: Optional[str] = None, - root: Optional[Any] = _NOT_SET, + response: TextResponse | None = None, + text: str | None = None, + type: str | None = None, + root: Any | None = _NOT_SET, **kwargs: Any, ): if response is not None and text is not None: diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b7e3763fb..274ced3e3 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -5,7 +5,7 @@ import json from collections.abc import Iterable, Iterator, Mapping, MutableMapping from importlib import import_module from pprint import pformat -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Union, cast from scrapy.settings import default_settings @@ -35,7 +35,7 @@ SETTINGS_PRIORITIES: dict[str, int] = { } -def get_settings_priority(priority: Union[int, str]) -> int: +def get_settings_priority(priority: int | str) -> int: """ Small helper function that looks up a given string priority in the :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its @@ -97,9 +97,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): __default = object() - def __init__( - self, values: _SettingsInputT = None, priority: Union[int, str] = "project" - ): + def __init__(self, values: _SettingsInputT = None, priority: int | str = "project"): self.frozen: bool = False self.attributes: dict[_SettingsKeyT, SettingsAttribute] = {} if values: @@ -180,7 +178,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return float(self.get(name, default)) def getlist( - self, name: _SettingsKeyT, default: Optional[list[Any]] = None + self, name: _SettingsKeyT, default: list[Any] | None = None ) -> list[Any]: """ Get a setting value as a list. If the setting original type is a list, a @@ -201,7 +199,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return list(value) def getdict( - self, name: _SettingsKeyT, default: Optional[dict[Any, Any]] = None + self, name: _SettingsKeyT, default: dict[Any, Any] | None = None ) -> dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a @@ -226,8 +224,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def getdictorlist( self, name: _SettingsKeyT, - default: Union[dict[Any, Any], list[Any], tuple[Any], None] = None, - ) -> Union[dict[Any, Any], list[Any]]: + default: dict[Any, Any] | list[Any] | tuple[Any] | None = None, + ) -> dict[Any, Any] | list[Any]: """Get a setting value as either a :class:`dict` or a :class:`list`. If the setting is already a dict or a list, a copy of it will be @@ -278,7 +276,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): compbs.update(self[name]) return compbs - def getpriority(self, name: _SettingsKeyT) -> Optional[int]: + def getpriority(self, name: _SettingsKeyT) -> int | None: """ Return the current numerical priority value of a setting, or ``None`` if the given ``name`` does not exist. @@ -305,7 +303,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): self.set(name, value) def set( - self, name: _SettingsKeyT, value: Any, priority: Union[int, str] = "project" + self, name: _SettingsKeyT, value: Any, priority: int | str = "project" ) -> None: """ Store a key/value attribute with a given priority. @@ -338,7 +336,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): self, name: _SettingsKeyT, default: Any = None, - priority: Union[int, str] = "project", + priority: int | str = "project", ) -> Any: if name not in self: self.set(name, default, priority) @@ -346,13 +344,11 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return self.attributes[name].value - def setdict( - self, values: _SettingsInputT, priority: Union[int, str] = "project" - ) -> None: + def setdict(self, values: _SettingsInputT, priority: int | str = "project") -> None: self.update(values, priority) def setmodule( - self, module: Union[ModuleType, str], priority: Union[int, str] = "project" + self, module: ModuleType | str, priority: int | str = "project" ) -> None: """ Store settings from a module with a given priority. @@ -376,7 +372,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): self.set(key, getattr(module, key), priority) # BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports - def update(self, values: _SettingsInputT, priority: Union[int, str] = "project") -> None: # type: ignore[override] + def update(self, values: _SettingsInputT, priority: int | str = "project") -> None: # type: ignore[override] """ Store key/value pairs with a given priority. @@ -409,9 +405,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): for name, value in values.items(): self.set(name, value, priority) - def delete( - self, name: _SettingsKeyT, priority: Union[int, str] = "project" - ) -> None: + def delete(self, name: _SettingsKeyT, priority: int | str = "project") -> None: if name not in self: raise KeyError(name) self._assert_mutability() @@ -525,9 +519,7 @@ class Settings(BaseSettings): described on :ref:`topics-settings-ref` already populated. """ - def __init__( - self, values: _SettingsInputT = None, priority: Union[int, str] = "project" - ): + def __init__(self, values: _SettingsInputT = None, priority: int | str = "project"): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user diff --git a/scrapy/shell.py b/scrapy/shell.py index dc402e678..31349c4ff 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -8,7 +8,7 @@ from __future__ import annotations import os import signal -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from itemadapter import is_item from twisted.internet import defer, threads @@ -37,25 +37,25 @@ class Shell: def __init__( self, crawler: Crawler, - update_vars: Optional[Callable[[dict[str, Any]], None]] = None, - code: Optional[str] = None, + update_vars: Callable[[dict[str, Any]], None] | None = None, + code: str | None = None, ): self.crawler: Crawler = crawler self.update_vars: Callable[[dict[str, Any]], None] = update_vars or ( lambda x: None ) self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) - self.spider: Optional[Spider] = None + self.spider: Spider | None = None self.inthread: bool = not threadable.isInIOThread() - self.code: Optional[str] = code + self.code: str | None = code self.vars: dict[str, Any] = {} def start( self, - url: Optional[str] = None, - request: Optional[Request] = None, - response: Optional[Response] = None, - spider: Optional[Spider] = None, + url: str | None = None, + request: Request | None = None, + response: Response | None = None, + spider: Spider | None = None, redirect: bool = True, ) -> None: # disable accidental Ctrl-C key press from shutting down the engine @@ -97,9 +97,7 @@ class Shell: self.vars, shells=shells, banner=self.vars.pop("banner", "") ) - def _schedule( - self, request: Request, spider: Optional[Spider] - ) -> defer.Deferred[Any]: + def _schedule(self, request: Request, spider: Spider | None) -> defer.Deferred[Any]: if is_asyncio_reactor_installed(): # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] @@ -111,7 +109,7 @@ class Shell: self.crawler.engine.crawl(request) return d - def _open_spider(self, request: Request, spider: Optional[Spider]) -> Spider: + def _open_spider(self, request: Request, spider: Spider | None) -> Spider: if self.spider: return self.spider @@ -126,8 +124,8 @@ class Shell: def fetch( self, - request_or_url: Union[Request, str], - spider: Optional[Spider] = None, + request_or_url: Request | str, + spider: Spider | None = None, redirect: bool = True, **kwargs: Any, ) -> None: @@ -155,9 +153,9 @@ class Shell: def populate_vars( self, - response: Optional[Response] = None, - request: Optional[Request] = None, - spider: Optional[Spider] = None, + response: Response | None = None, + request: Request | None = None, + spider: Spider | None = None, ) -> None: import scrapy diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index afab2eac2..42619ec7f 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from scrapy.exceptions import IgnoreRequest @@ -65,7 +65,7 @@ class HttpErrorMiddleware: def process_spider_exception( self, response: Response, exception: Exception, spider: Spider - ) -> Optional[Iterable[Any]]: + ) -> Iterable[Any] | None: if isinstance(exception, HttpError): assert spider.crawler.stats spider.crawler.stats.inc_value("httperror/response_ignored_count") diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 8784e4b05..bdf1f168a 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -6,7 +6,7 @@ originated it. from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, cast from urllib.parse import urlparse from w3lib.url import safe_url_string @@ -50,20 +50,20 @@ class ReferrerPolicy: NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES name: str - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: raise NotImplementedError() - def stripped_referrer(self, url: str) -> Optional[str]: + def stripped_referrer(self, url: str) -> str | None: if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: return self.strip_url(url) return None - def origin_referrer(self, url: str) -> Optional[str]: + def origin_referrer(self, url: str) -> str | None: if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: return self.origin(url) return None - def strip_url(self, url: str, origin_only: bool = False) -> Optional[str]: + def strip_url(self, url: str, origin_only: bool = False) -> str | None: """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -87,7 +87,7 @@ class ReferrerPolicy: origin_only=origin_only, ) - def origin(self, url: str) -> Optional[str]: + def origin(self, url: str) -> str | None: """Return serialized origin (scheme, host, path) for a request or response URL.""" return self.strip_url(url, origin_only=True) @@ -113,7 +113,7 @@ class NoReferrerPolicy(ReferrerPolicy): name: str = POLICY_NO_REFERRER - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: return None @@ -134,7 +134,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: if not self.tls_protected(response_url) or self.tls_protected(request_url): return self.stripped_referrer(response_url) return None @@ -153,7 +153,7 @@ class SameOriginPolicy(ReferrerPolicy): name: str = POLICY_SAME_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: if self.origin(response_url) == self.origin(request_url): return self.stripped_referrer(response_url) return None @@ -171,7 +171,7 @@ class OriginPolicy(ReferrerPolicy): name: str = POLICY_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: return self.origin_referrer(response_url) @@ -191,7 +191,7 @@ class StrictOriginPolicy(ReferrerPolicy): name: str = POLICY_STRICT_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) @@ -215,7 +215,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) @@ -242,7 +242,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) @@ -271,7 +271,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): name: str = POLICY_UNSAFE_URL - def referrer(self, response_url: str, request_url: str) -> Optional[str]: + def referrer(self, response_url: str, request_url: str) -> str | None: return self.stripped_referrer(response_url) @@ -307,7 +307,7 @@ _policy_classes[""] = NoReferrerWhenDowngradePolicy def _load_policy_class( policy: str, warning_only: bool = False -) -> Optional[type[ReferrerPolicy]]: +) -> type[ReferrerPolicy] | None: """ Expect a string for the path to the policy class, otherwise try to interpret the string as a standard value @@ -331,7 +331,7 @@ def _load_policy_class( class RefererMiddleware: - def __init__(self, settings: Optional[BaseSettings] = None): + def __init__(self, settings: BaseSettings | None = None): self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy if settings is not None: settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) @@ -349,9 +349,7 @@ class RefererMiddleware: return mw - def policy( - self, resp_or_url: Union[Response, str], request: Request - ) -> ReferrerPolicy: + def policy(self, resp_or_url: Response | str, request: Request) -> ReferrerPolicy: """ Determine Referrer-Policy to use from a parent Response (or URL), and a Request to be sent. diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 8220aca28..6136dabc7 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast from scrapy import signals from scrapy.http import Request, Response @@ -34,9 +34,9 @@ class Spider(object_ref): """ name: str - custom_settings: Optional[dict[_SettingsKeyT, Any]] = None + custom_settings: dict[_SettingsKeyT, Any] | None = None - def __init__(self, name: Optional[str] = None, **kwargs: Any): + def __init__(self, name: str | None = None, **kwargs: Any): if name is not None: self.name: str = name elif not getattr(self, "name", None): @@ -103,10 +103,10 @@ class Spider(object_ref): return url_is_from_spider(request.url, cls) @staticmethod - def close(spider: Spider, reason: str) -> Optional[Deferred[None]]: + def close(spider: Spider, reason: str) -> Deferred[None] | None: closed = getattr(spider, "closed", None) if callable(closed): - return cast("Optional[Deferred[None]]", closed(reason)) + return cast("Deferred[None] | None", closed(reason)) return None def __repr__(self) -> str: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d628f49f6..087049425 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -9,7 +9,7 @@ from __future__ import annotations import copy from collections.abc import AsyncIterable, Awaitable, Callable -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast from twisted.python.failure import Failure @@ -39,15 +39,11 @@ def _identity(x: _T) -> _T: return x -def _identity_process_request( - request: Request, response: Response -) -> Optional[Request]: +def _identity_process_request(request: Request, response: Response) -> Request | None: return request -def _get_method( - method: Union[Callable, str, None], spider: Spider -) -> Optional[Callable]: +def _get_method(method: Callable | str | None, spider: Spider) -> Callable | None: if callable(method): return method if isinstance(method, str): @@ -61,20 +57,20 @@ _default_link_extractor = LinkExtractor() class Rule: def __init__( self, - link_extractor: Optional[LinkExtractor] = None, - callback: Union[CallbackT, str, None] = None, - cb_kwargs: Optional[dict[str, Any]] = None, - follow: Optional[bool] = None, - process_links: Union[ProcessLinksT, str, None] = None, - process_request: Union[ProcessRequestT, str, None] = None, - errback: Union[Callable[[Failure], Any], str, None] = None, + link_extractor: LinkExtractor | None = None, + callback: CallbackT | str | None = None, + cb_kwargs: dict[str, Any] | None = None, + follow: bool | None = None, + process_links: ProcessLinksT | str | None = None, + process_request: ProcessRequestT | str | None = None, + errback: Callable[[Failure], Any] | str | None = None, ): self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor - self.callback: Union[CallbackT, str, None] = callback - self.errback: Union[Callable[[Failure], Any], str, None] = errback + self.callback: CallbackT | str | None = callback + self.errback: Callable[[Failure], Any] | str | None = errback self.cb_kwargs: dict[str, Any] = cb_kwargs or {} - self.process_links: Union[ProcessLinksT, str] = process_links or _identity - self.process_request: Union[ProcessRequestT, str] = ( + self.process_links: ProcessLinksT | str = process_links or _identity + self.process_request: ProcessRequestT | str = ( process_request or _identity_process_request ) self.follow: bool = follow if follow is not None else not callback @@ -124,7 +120,7 @@ class CrawlSpider(Spider): meta={"rule": rule_index, "link_text": link.text}, ) - def _requests_to_follow(self, response: Response) -> Iterable[Optional[Request]]: + def _requests_to_follow(self, response: Response) -> Iterable[Request | None]: if not isinstance(response, HtmlResponse): return seen: set[Link] = set() @@ -157,7 +153,7 @@ class CrawlSpider(Spider): async def _parse_response( self, response: Response, - callback: Optional[CallbackT], + callback: CallbackT | None, cb_kwargs: dict[str, Any], follow: bool = True, ) -> AsyncIterable[Any]: @@ -176,7 +172,7 @@ class CrawlSpider(Spider): yield request_or_item def _handle_failure( - self, failure: Failure, errback: Optional[Callable[[Failure], Any]] + self, failure: Failure, errback: Callable[[Failure], Any] | None ) -> Iterable[Any]: if errback: results = errback(failure) or () diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 0ddef1f32..395183613 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from scrapy.exceptions import NotConfigured, NotSupported from scrapy.http import Response, TextResponse @@ -117,13 +117,13 @@ class CSVFeedSpider(Spider): and the file's headers. """ - delimiter: Optional[str] = ( + delimiter: str | None = ( None # When this is None, python's csv module's default delimiter is used ) - quotechar: Optional[str] = ( + quotechar: str | None = ( None # When this is None, python's csv module's default quotechar is used ) - headers: Optional[list[str]] = None + headers: list[str] | None = None def process_results( self, response: Response, results: Iterable[Any] diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index ebe288b83..4ec2919f7 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast from scrapy import Request from scrapy.spiders import Spider @@ -18,7 +18,7 @@ class InitSpider(Spider): self._postinit_reqs: Iterable[Request] = super().start_requests() return cast(Iterable[Request], iterate_spider_output(self.init_request())) - def initialized(self, response: Optional[Response] = None) -> Any: + def initialized(self, response: Response | None = None) -> Any: """This method must be set as the callback of your last initialization request. See self.init_request() docstring for more info. """ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 945539d7b..91c7e3be9 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging import re -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse from scrapy.spiders import Spider @@ -24,10 +24,10 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): sitemap_urls: Sequence[str] = () - sitemap_rules: Sequence[ - tuple[Union[re.Pattern[str], str], Union[str, CallbackT]] - ] = [("", "parse")] - sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""] + sitemap_rules: Sequence[tuple[re.Pattern[str] | str, str | CallbackT]] = [ + ("", "parse") + ] + sitemap_follow: Sequence[re.Pattern[str] | str] = [""] sitemap_alternate_links: bool = False _max_size: int _warn_size: int @@ -93,7 +93,7 @@ class SitemapSpider(Spider): yield Request(loc, callback=c) break - def _get_sitemap_body(self, response: Response) -> Optional[bytes]: + def _get_sitemap_body(self, response: Response) -> bytes | None: """Return the sitemap body contained in the given response, or None if the response is not a sitemap. """ @@ -127,7 +127,7 @@ class SitemapSpider(Spider): return None -def regex(x: Union[re.Pattern[str], str]) -> re.Pattern[str]: +def regex(x: re.Pattern[str] | str) -> re.Pattern[str]: if isinstance(x, str): return re.compile(x) return x diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 767a53db8..7732187fd 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -7,7 +7,7 @@ from __future__ import annotations import marshal import pickle # nosec from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from queuelib import queue @@ -26,7 +26,7 @@ if TYPE_CHECKING: def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]: class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] - def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any): + def __init__(self, path: str | PathLike, *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) @@ -45,13 +45,13 @@ def _serializable_queue( s = serialize(obj) super().push(s) - def pop(self) -> Optional[Any]: + def pop(self) -> Any | None: s = super().pop() if s: return deserialize(s) return None - def peek(self) -> Optional[Any]: + def peek(self) -> Any | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -89,13 +89,13 @@ def _scrapy_serialization_queue( request_dict = request.to_dict(spider=self.spider) super().push(request_dict) - def pop(self) -> Optional[Request]: + def pop(self) -> Request | None: request = super().pop() if not request: return None return request_from_dict(request, spider=self.spider) - def peek(self) -> Optional[Request]: + def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. @@ -118,7 +118,7 @@ def _scrapy_non_serialization_queue( def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: return cls() - def peek(self) -> Optional[Any]: + def peek(self) -> Any | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 63c82ec6d..f3dd0f8e7 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -6,7 +6,7 @@ from __future__ import annotations import logging import pprint -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from scrapy import Spider @@ -25,32 +25,32 @@ class StatsCollector: self._stats: StatsT = {} def get_value( - self, key: str, default: Any = None, spider: Optional[Spider] = None + self, key: str, default: Any = None, spider: Spider | None = None ) -> Any: return self._stats.get(key, default) - def get_stats(self, spider: Optional[Spider] = None) -> StatsT: + def get_stats(self, spider: Spider | None = None) -> StatsT: return self._stats - def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def set_value(self, key: str, value: Any, spider: Spider | None = None) -> None: self._stats[key] = value - def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None: + def set_stats(self, stats: StatsT, spider: Spider | None = None) -> None: self._stats = stats def inc_value( - self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None + self, key: str, count: int = 1, start: int = 0, spider: Spider | None = None ) -> None: d = self._stats d[key] = d.setdefault(key, start) + count - def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def max_value(self, key: str, value: Any, spider: Spider | None = None) -> None: self._stats[key] = max(self._stats.setdefault(key, value), value) - def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def min_value(self, key: str, value: Any, spider: Spider | None = None) -> None: self._stats[key] = min(self._stats.setdefault(key, value), value) - def clear_stats(self, spider: Optional[Spider] = None) -> None: + def clear_stats(self, spider: Spider | None = None) -> None: self._stats.clear() def open_spider(self, spider: Spider) -> None: @@ -79,23 +79,23 @@ class MemoryStatsCollector(StatsCollector): class DummyStatsCollector(StatsCollector): def get_value( - self, key: str, default: Any = None, spider: Optional[Spider] = None + self, key: str, default: Any = None, spider: Spider | None = None ) -> Any: return default - def set_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def set_value(self, key: str, value: Any, spider: Spider | None = None) -> None: pass - def set_stats(self, stats: StatsT, spider: Optional[Spider] = None) -> None: + def set_stats(self, stats: StatsT, spider: Spider | None = None) -> None: pass def inc_value( - self, key: str, count: int = 1, start: int = 0, spider: Optional[Spider] = None + self, key: str, count: int = 1, start: int = 0, spider: Spider | None = None ) -> None: pass - def max_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def max_value(self, key: str, value: Any, spider: Spider | None = None) -> None: pass - def min_value(self, key: str, value: Any, spider: Optional[Spider] = None) -> None: + def min_value(self, key: str, value: Any, spider: Spider | None = None) -> None: pass diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index f1505e4bd..905959c25 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from collections.abc import AsyncGenerator, AsyncIterable, Iterable -from typing import TypeVar, Union +from typing import TypeVar _T = TypeVar("_T") @@ -12,8 +14,8 @@ async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]: async def as_async_generator( - it: Union[Iterable[_T], AsyncIterable[_T]] -) -> AsyncGenerator[_T, None]: + it: Iterable[_T] | AsyncIterable[_T], +) -> AsyncGenerator[_T]: """Wraps an iterable (sync or async) into an async generator.""" if isinstance(it, AsyncIterable): async for r in it: diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 463bbb5df..64cd31c4b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -8,7 +8,7 @@ from collections.abc import Iterable from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Callable, cast from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings @@ -33,7 +33,7 @@ def build_component_list( "please update your settings" ) - def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, dict[Any, Any]]: + def _map_keys(compdict: Mapping[Any, Any]) -> BaseSettings | dict[Any, Any]: if isinstance(compdict, BaseSettings): compbs = BaseSettings() for k, v in compdict.items(): @@ -86,8 +86,8 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]: def closest_scrapy_cfg( - path: Union[str, os.PathLike] = ".", - prevpath: Optional[Union[str, os.PathLike]] = None, + path: str | os.PathLike = ".", + prevpath: str | os.PathLike | None = None, ) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents @@ -159,8 +159,8 @@ def feed_complete_default_values_from_settings( def feed_process_params_from_cli( settings: BaseSettings, output: list[str], - output_format: Optional[str] = None, - overwrite_output: Optional[list[str]] = None, + output_format: str | None = None, + overwrite_output: list[str] | None = None, ) -> dict[str, dict[str, Any]]: """ Receives feed export params (from the 'crawl' or 'runspider' commands), diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 3b5596ab7..aecd3fdb7 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -2,7 +2,7 @@ from __future__ import annotations from collections.abc import Callable from functools import wraps -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from collections.abc import Iterable @@ -100,7 +100,7 @@ DEFAULT_PYTHON_SHELLS: KnownShellsT = { def get_shell_embed_func( - shells: Optional[Iterable[str]] = None, known_shells: Optional[KnownShellsT] = None + shells: Iterable[str] | None = None, known_shells: KnownShellsT | None = None ) -> Any: """Return the first acceptable shell-embed function from a given list of shell names. @@ -120,9 +120,9 @@ def get_shell_embed_func( def start_python_console( - namespace: Optional[dict[str, Any]] = None, + namespace: dict[str, Any] | None = None, banner: str = "", - shells: Optional[Iterable[str]] = None, + shells: Iterable[str] | None = None, ) -> None: """Start Python console bound to the given namespace. Readline support and tab completion will be used on Unix, if available. diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 9c7f63848..bfdd4dc8a 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -4,7 +4,7 @@ import argparse import warnings from http.cookies import SimpleCookie from shlex import split -from typing import TYPE_CHECKING, Any, NoReturn, Optional, Union +from typing import TYPE_CHECKING, Any, NoReturn from urllib.parse import urlparse from w3lib.http import basic_auth_header @@ -18,8 +18,8 @@ class DataAction(argparse.Action): self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, - values: Union[str, Sequence[Any], None], - option_string: Optional[str] = None, + values: str | Sequence[Any] | None, + option_string: str | None = None, ) -> None: value = str(values) if value.startswith("$"): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index c78325676..98ecb2f02 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -12,7 +12,7 @@ import warnings import weakref from collections import OrderedDict from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, Optional, TypeVar, Union +from typing import TYPE_CHECKING, Any, AnyStr, TypeVar from scrapy.exceptions import ScrapyDeprecationWarning @@ -44,7 +44,7 @@ class CaselessDict(dict): def __init__( self, - seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]], None] = None, + seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, ): super().__init__() if seq: @@ -84,7 +84,7 @@ class CaselessDict(dict): return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # type: ignore[arg-type] # doesn't fully implement MutableMapping.update() - def update(self, seq: Union[Mapping[AnyStr, Any], Iterable[tuple[AnyStr, Any]]]) -> None: # type: ignore[override] + def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None: # type: ignore[override] seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super().update(iseq) @@ -145,9 +145,9 @@ class LocalCache(OrderedDict[_KT, _VT]): Older items expires first. """ - def __init__(self, limit: Optional[int] = None): + def __init__(self, limit: int | None = None): super().__init__() - self.limit: Optional[int] = limit + self.limit: int | None = limit def __setitem__(self, key: _KT, value: _VT) -> None: if self.limit: @@ -168,7 +168,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): it cannot be instantiated with an initial dictionary. """ - def __init__(self, limit: Optional[int] = None): + def __init__(self, limit: int | None = None): super().__init__() self.data: LocalCache = LocalCache(limit=limit) @@ -178,7 +178,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): except TypeError: pass # key is not weak-referenceable, skip caching - def __getitem__(self, key: _KT) -> Optional[_VT]: # type: ignore[override] + def __getitem__(self, key: _KT) -> _VT | None: # type: ignore[override] try: return super().__getitem__(key) except (TypeError, KeyError): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index aeacadb1c..9ca6c6a24 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -11,7 +11,7 @@ from asyncio import Future from collections.abc import Awaitable, Coroutine, Iterable, Iterator from functools import wraps from types import CoroutineType -from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar, Union, cast, overload +from typing import TYPE_CHECKING, Any, Generic, TypeVar, Union, cast, overload from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred @@ -93,7 +93,7 @@ def mustbe_deferred( def mustbe_deferred( - f: Callable[_P, Union[Deferred[_T], Coroutine[Deferred[Any], Any, _T], _T]], + f: Callable[_P, Deferred[_T] | Coroutine[Deferred[Any], Any, _T] | _T], *args: _P.args, **kw: _P.kwargs, ) -> Deferred[_T]: @@ -179,17 +179,17 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def __init__( self, aiterable: AsyncIterable[_T], - callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], + callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], *callable_args: _P.args, **callable_kwargs: _P.kwargs, ): self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() - self.callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]] = callable + self.callable: Callable[Concatenate[_T, _P], Deferred[Any] | None] = callable self.callable_args: tuple[Any, ...] = callable_args self.callable_kwargs: dict[str, Any] = callable_kwargs self.finished: bool = False self.waiting_deferreds: list[Deferred[Any]] = [] - self.anext_deferred: Optional[Deferred[_T]] = None + self.anext_deferred: Deferred[_T] | None = None def _callback(self, result: _T) -> None: # This gets called when the result from aiterator.__anext__() is available. @@ -237,7 +237,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def parallel_async( async_iterable: AsyncIterable[_T], count: int, - callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], + callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], *args: _P.args, **named: _P.kwargs, ) -> Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]]: @@ -362,7 +362,7 @@ def deferred_from_coro(o: _CT) -> Deferred: ... def deferred_from_coro(o: _T) -> _T: ... -def deferred_from_coro(o: _T) -> Union[Deferred, _T]: +def deferred_from_coro(o: _T) -> Deferred | _T: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, Deferred): return o @@ -433,7 +433,7 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]: return d.asFuture(_get_asyncio_event_loop()) -def maybe_deferred_to_future(d: Deferred[_T]) -> Union[Deferred[_T], Future[_T]]: +def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: """ .. versionadded:: 2.6.0 diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 9b0d476a1..32430cd6c 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -1,8 +1,10 @@ """Some helpers for deprecation messages""" +from __future__ import annotations + import inspect import warnings -from typing import Any, Optional, overload +from typing import Any, overload from scrapy.exceptions import ScrapyDeprecationWarning @@ -20,11 +22,11 @@ def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> No def create_deprecated_class( name: str, new_class: type, - clsdict: Optional[dict[str, Any]] = None, + clsdict: dict[str, Any] | None = None, warn_category: type[Warning] = ScrapyDeprecationWarning, warn_once: bool = True, - old_class_path: Optional[str] = None, - new_class_path: Optional[str] = None, + old_class_path: str | None = None, + new_class_path: str | None = None, subclass_warn_message: str = "{cls} inherits from deprecated class {old}, please inherit from {new}.", instance_warn_message: str = "{cls} is deprecated, instantiate {new} instead.", ) -> type: @@ -55,7 +57,7 @@ def create_deprecated_class( # https://github.com/python/mypy/issues/4177 class DeprecatedClass(new_class.__class__): # type: ignore[misc, name-defined] - deprecated_class: Optional[type] = None + deprecated_class: type | None = None warned_on_subclass: bool = False def __new__( @@ -128,7 +130,7 @@ def create_deprecated_class( return deprecated_cls -def _clspath(cls: type, forced: Optional[str] = None) -> str: +def _clspath(cls: type, forced: str | None = None) -> str: if forced is not None: return forced return f"{cls.__module__}.{cls.__name__}" diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index 3cf9585ec..58b4539bf 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from urllib.parse import ParseResult, urlparse from weakref import WeakKeyDictionary @@ -10,12 +10,12 @@ if TYPE_CHECKING: from scrapy.http import Request, Response -_urlparse_cache: WeakKeyDictionary[Union[Request, Response], ParseResult] = ( +_urlparse_cache: WeakKeyDictionary[Request | Response, ParseResult] = ( WeakKeyDictionary() ) -def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult: +def urlparse_cached(request_or_response: Request | Response) -> ParseResult: """Return urlparse.urlparse caching the result, where the argument can be a Request or Response object """ diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index a4d339adc..ba58d939c 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -4,7 +4,7 @@ import csv import logging import re from io import StringIO -from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast, overload +from typing import TYPE_CHECKING, Any, Literal, cast, overload from warnings import warn from lxml import etree # nosec @@ -20,7 +20,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selector]: +def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]: """Return a iterator of Selector's over all nodes of a XML document, given the name of the node to iterate. Useful for parsing XML feeds. @@ -77,9 +77,9 @@ def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selecto def xmliter_lxml( - obj: Union[Response, str, bytes], + obj: Response | str | bytes, nodename: str, - namespace: Optional[str] = None, + namespace: str | None = None, prefix: str = "x", ) -> Iterator[Selector]: reader = _StreamReader(obj) @@ -120,9 +120,9 @@ def xmliter_lxml( class _StreamReader: - def __init__(self, obj: Union[Response, str, bytes]): + def __init__(self, obj: Response | str | bytes): self._ptr: int = 0 - self._text: Union[str, bytes] + self._text: str | bytes if isinstance(obj, TextResponse): self._text, self.encoding = obj.body, obj.encoding elif isinstance(obj, Response): @@ -154,11 +154,11 @@ class _StreamReader: def csviter( - obj: Union[Response, str, bytes], - delimiter: Optional[str] = None, - headers: Optional[list[str]] = None, - encoding: Optional[str] = None, - quotechar: Optional[str] = None, + obj: Response | str | bytes, + delimiter: str | None = None, + headers: list[str] | None = None, + encoding: str | None = None, + quotechar: str | None = None, ) -> Iterator[dict[str, str]]: """Returns an iterator of dictionaries from the given csv object @@ -214,22 +214,18 @@ def csviter( @overload -def _body_or_str(obj: Union[Response, str, bytes]) -> str: ... +def _body_or_str(obj: Response | str | bytes) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: ... +def _body_or_str(obj: Response | str | bytes, unicode: Literal[True]) -> str: ... @overload -def _body_or_str( - obj: Union[Response, str, bytes], unicode: Literal[False] -) -> bytes: ... +def _body_or_str(obj: Response | str | bytes, unicode: Literal[False]) -> bytes: ... -def _body_or_str( - obj: Union[Response, str, bytes], unicode: bool = True -) -> Union[str, bytes]: +def _body_or_str(obj: Response | str | bytes, unicode: bool = True) -> str | bytes: expected_types = (Response, str, bytes) if not isinstance(obj, expected_types): expected_types_str = " or ".join(t.__name__ for t in expected_types) diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 488c7994b..37e6aeb51 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,14 +1,14 @@ from __future__ import annotations from pathlib import Path -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING if TYPE_CHECKING: from scrapy.settings import BaseSettings -def job_dir(settings: BaseSettings) -> Optional[str]: - path: Optional[str] = settings["JOBDIR"] +def job_dir(settings: BaseSettings) -> str | None: + path: str | None = settings["JOBDIR"] if not path: return None if not Path(path).exists(): diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2b90c6b36..c3808426a 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -5,7 +5,7 @@ import sys from collections.abc import MutableMapping from logging.config import dictConfig from types import TracebackType -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Optional, cast from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) def failure_to_exc_info( failure: Failure, -) -> Optional[tuple[type[BaseException], BaseException, Optional[TracebackType]]]: +) -> tuple[type[BaseException], BaseException, TracebackType | None] | None: """Extract exc_info from Failure instances""" if isinstance(failure, Failure): assert failure.type @@ -50,7 +50,7 @@ class TopLevelFormatter(logging.Filter): ``loggers`` list where it should act. """ - def __init__(self, loggers: Optional[list[str]] = None): + def __init__(self, loggers: list[str] | None = None): self.loggers: list[str] = loggers or [] def filter(self, record: logging.LogRecord) -> bool: @@ -80,7 +80,7 @@ DEFAULT_LOGGING = { def configure_logging( - settings: Union[Settings, dict[_SettingsKeyT, Any], None] = None, + settings: Settings | dict[_SettingsKeyT, Any] | None = None, install_root_handler: bool = True, ) -> None: """ @@ -125,7 +125,7 @@ def configure_logging( install_scrapy_root_handler(settings) -_scrapy_root_handler: Optional[logging.Handler] = None +_scrapy_root_handler: logging.Handler | None = None def install_scrapy_root_handler(settings: Settings) -> None: @@ -141,7 +141,7 @@ def install_scrapy_root_handler(settings: Settings) -> None: logging.root.addHandler(_scrapy_root_handler) -def get_scrapy_root_handler() -> Optional[logging.Handler]: +def get_scrapy_root_handler() -> logging.Handler | None: return _scrapy_root_handler @@ -231,7 +231,7 @@ class LogCounterHandler(logging.Handler): def logformatter_adapter( logkws: LogFormatterResult, -) -> tuple[int, str, Union[dict[str, Any], tuple[Any, ...]]]: +) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]: """ Helper that takes the dictionary output from the methods in LogFormatter and adapts it into a tuple of positional arguments for logger.log calls, diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index e5e00512a..1ab30f097 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -14,7 +14,7 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from typing import IO, TYPE_CHECKING, Any, Optional, TypeVar, Union, cast +from typing import IO, TYPE_CHECKING, Any, TypeVar, cast from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item @@ -46,7 +46,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]: return [arg] -def load_object(path: Union[str, Callable[..., Any]]) -> Any: +def load_object(path: str | Callable[..., Any]) -> Any: """Load an object given its absolute object path, and return it. The object can be the import path of a class, function, variable or an @@ -126,7 +126,7 @@ def md5sum(file: IO[bytes]) -> str: return m.hexdigest() -def rel_has_nofollow(rel: Optional[str]) -> bool: +def rel_has_nofollow(rel: str | None) -> bool: """Return True if link rel attribute has nofollow type""" return rel is not None and "nofollow" in rel.replace(",", " ").split() diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index cff5eb629..ad758b783 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import signal from collections.abc import Callable from types import FrameType diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index c9e5eb857..0139720b7 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,8 +1,9 @@ +from __future__ import annotations + import os import warnings from importlib import import_module from pathlib import Path -from typing import Union from scrapy.exceptions import NotConfigured from scrapy.settings import Settings @@ -45,7 +46,7 @@ def project_data_dir(project: str = "default") -> str: return str(d) -def data_path(path: Union[str, os.PathLike[str]], createdir: bool = False) -> str: +def data_path(path: str | os.PathLike[str], createdir: bool = False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 91c5d67f5..6268af728 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -12,7 +12,7 @@ import weakref from collections.abc import AsyncIterable, Iterable, Mapping from functools import partial, wraps from itertools import chain -from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, TypeVar, overload from scrapy.utils.asyncgen import as_async_generator @@ -99,7 +99,7 @@ def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> list[ def to_unicode( - text: Union[str, bytes], encoding: Optional[str] = None, errors: str = "strict" + text: str | bytes, encoding: str | None = None, errors: str = "strict" ) -> str: """Return the unicode representation of a bytes object ``text``. If ``text`` is already an unicode object, return it as-is.""" @@ -116,7 +116,7 @@ def to_unicode( def to_bytes( - text: Union[str, bytes], encoding: Optional[str] = None, errors: str = "strict" + text: str | bytes, encoding: str | None = None, errors: str = "strict" ) -> bytes: """Return the binary representation of ``text``. If ``text`` is already a bytes object, return it as-is.""" @@ -132,8 +132,8 @@ def to_bytes( def re_rsearch( - pattern: Union[str, Pattern[str]], text: str, chunk_size: int = 1024 -) -> Optional[tuple[int, int]]: + pattern: str | Pattern[str], text: str, chunk_size: int = 1024 +) -> tuple[int, int] | None: """ This function does a reverse search in a text using a regular expression given in the attribute 'pattern'. @@ -269,7 +269,7 @@ def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]: def equal_attributes( - obj1: Any, obj2: Any, attributes: Optional[list[Union[str, Callable[[Any], Any]]]] + obj1: Any, obj2: Any, attributes: list[str | Callable[[Any], Any]] | None ) -> bool: """Compare two objects attributes""" # not attributes given return False by default @@ -297,8 +297,8 @@ def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ... def without_none_values( - iterable: Union[Mapping[_KT, _VT], Iterable[_KT]] -) -> Union[dict[_KT, _VT], Iterable[_KT]]: + iterable: Mapping[_KT, _VT] | Iterable[_KT] +) -> dict[_KT, _VT] | Iterable[_KT]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have @@ -354,7 +354,7 @@ class MutableChain(Iterable[_T]): async def _async_chain( - *iterables: Union[Iterable[_T], AsyncIterable[_T]] + *iterables: Iterable[_T] | AsyncIterable[_T], ) -> AsyncIterator[_T]: for it in iterables: async for o in as_async_generator(it): @@ -366,10 +366,10 @@ class MutableAsyncChain(AsyncIterable[_T]): Similar to MutableChain but for async iterables """ - def __init__(self, *args: Union[Iterable[_T], AsyncIterable[_T]]): + def __init__(self, *args: Iterable[_T] | AsyncIterable[_T]): self.data: AsyncIterator[_T] = _async_chain(*args) - def extend(self, *iterables: Union[Iterable[_T], AsyncIterable[_T]]) -> None: + def extend(self, *iterables: Iterable[_T] | AsyncIterable[_T]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) def __aiter__(self) -> AsyncIterator[_T]: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ed2fb5959..18bb583b8 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio import sys from contextlib import suppress -from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar +from typing import TYPE_CHECKING, Any, Generic, TypeVar from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error @@ -54,7 +54,7 @@ class CallLaterOnce(Generic[_T]): self._func: Callable[_P, _T] = func self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw - self._call: Optional[DelayedCall] = None + self._call: DelayedCall | None = None def schedule(self, delay: float = 0) -> None: from twisted.internet import reactor @@ -107,7 +107,7 @@ def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: return policy -def install_reactor(reactor_path: str, event_loop_path: Optional[str] = None) -> None: +def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> None: """Installs the :mod:`~twisted.internet.reactor` with the specified import path. Also installs the asyncio event loop with the specified import path if the asyncio reactor is enabled""" @@ -129,7 +129,7 @@ def _get_asyncio_event_loop() -> AbstractEventLoop: return set_asyncio_event_loop(None) -def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop: +def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: event_loop_class: type[AbstractEventLoop] = load_object(event_loop_path) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 052a3721a..82bdcb0f9 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -8,7 +8,7 @@ from __future__ import annotations import hashlib import json import warnings -from typing import TYPE_CHECKING, Any, Optional, Protocol, Union +from typing import TYPE_CHECKING, Any, Protocol from urllib.parse import urlunparse from weakref import WeakKeyDictionary @@ -38,7 +38,7 @@ def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[b _fingerprint_cache: WeakKeyDictionary[ - Request, dict[tuple[Optional[tuple[bytes, ...]], bool], bytes] + Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes] ] _fingerprint_cache = WeakKeyDictionary() @@ -46,7 +46,7 @@ _fingerprint_cache = WeakKeyDictionary() def fingerprint( request: Request, *, - include_headers: Optional[Iterable[Union[bytes, str]]] = None, + include_headers: Iterable[bytes | str] | None = None, keep_fragments: bool = False, ) -> bytes: """ @@ -79,7 +79,7 @@ def fingerprint( If you want to include them, set the keep_fragments argument to True (for instance when handling requests with a headless browser). """ - processed_include_headers: Optional[tuple[bytes, ...]] = None + processed_include_headers: tuple[bytes, ...] | None = None if include_headers: processed_include_headers = tuple( to_bytes(h.lower()) for h in sorted(include_headers) @@ -129,7 +129,7 @@ class RequestFingerprinter: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def __init__(self, crawler: Optional[Crawler] = None): + def __init__(self, crawler: Crawler | None = None): if crawler: implementation = crawler.settings.get( "REQUEST_FINGERPRINTER_IMPLEMENTATION" @@ -177,7 +177,7 @@ def request_httprepr(request: Request) -> bytes: return s -def referer_str(request: Request) -> Optional[str]: +def referer_str(request: Request) -> str | None: """Return Referer HTTP header suitable for logging.""" referrer = request.headers.get("Referer") if referrer is None: @@ -185,7 +185,7 @@ def referer_str(request: Request) -> Optional[str]: return to_unicode(referrer, errors="replace") -def request_from_dict(d: dict[str, Any], *, spider: Optional[Spider] = None) -> Request: +def request_from_dict(d: dict[str, Any], *, spider: Spider | None = None) -> Request: """Create a :class:`~scrapy.Request` object from a dict. If a spider is given, it will try to resolve the callbacks looking at the diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 0ca9d07a4..ecc83d1c8 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -9,7 +9,7 @@ import os import re import tempfile import webbrowser -from typing import TYPE_CHECKING, Any, Union +from typing import TYPE_CHECKING, Any from weakref import WeakKeyDictionary from twisted.web import http @@ -35,15 +35,15 @@ def get_base_url(response: TextResponse) -> str: return _baseurl_cache[response] -_metaref_cache: WeakKeyDictionary[ - Response, Union[tuple[None, None], tuple[float, str]] -] = WeakKeyDictionary() +_metaref_cache: WeakKeyDictionary[Response, tuple[None, None] | tuple[float, str]] = ( + WeakKeyDictionary() +) def get_meta_refresh( response: TextResponse, ignore_tags: Iterable[str] = ("script", "noscript"), -) -> Union[tuple[None, None], tuple[float, str]]: +) -> tuple[None, None] | tuple[float, str]: """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] @@ -53,7 +53,7 @@ def get_meta_refresh( return _metaref_cache[response] -def response_status_message(status: Union[bytes, float, int, str]) -> str: +def response_status_message(status: bytes | float | int | str) -> str: """Return status code plus status text descriptive message""" status_int = int(status) message = http.RESPONSES.get(status_int, "Unknown Status") diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 1f70fcf69..c572580ae 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -7,7 +7,7 @@ SitemapSpider, its API is subject to change without notice. from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any from urllib.parse import urljoin import lxml.etree # nosec @@ -20,7 +20,7 @@ class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" - def __init__(self, xmltext: Union[str, bytes]): + def __init__(self, xmltext: str | bytes): xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) @@ -46,7 +46,7 @@ class Sitemap: def sitemap_urls_from_robots( - robots_text: str, base_url: Optional[str] = None + robots_text: str, base_url: str | None = None ) -> Iterable[str]: """Return an iterator over all sitemap urls contained in the given robots.txt file diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 02dbb2e90..e58eb8134 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,7 +2,7 @@ from __future__ import annotations import inspect import logging -from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, Literal, TypeVar, overload from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro @@ -25,7 +25,7 @@ _T = TypeVar("_T") # https://stackoverflow.com/questions/60222982 @overload -def iterate_spider_output(result: AsyncGenerator[_T, None]) -> AsyncGenerator[_T, None]: ... # type: ignore[overload-overlap] +def iterate_spider_output(result: AsyncGenerator[_T]) -> AsyncGenerator[_T]: ... # type: ignore[overload-overlap] @overload @@ -38,7 +38,7 @@ def iterate_spider_output(result: _T) -> Iterable[Any]: ... def iterate_spider_output( result: Any, -) -> Union[Iterable[Any], AsyncGenerator[_T, None], Deferred[_T]]: +) -> Iterable[Any] | AsyncGenerator[_T] | Deferred[_T]: if inspect.isasyncgen(result): return result if inspect.iscoroutine(result): @@ -83,7 +83,7 @@ def spidercls_for_request( default_spidercls: Literal[None], log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[type[Spider]]: ... +) -> type[Spider] | None: ... @overload @@ -93,16 +93,16 @@ def spidercls_for_request( *, log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[type[Spider]]: ... +) -> type[Spider] | None: ... def spidercls_for_request( spider_loader: SpiderLoader, request: Request, - default_spidercls: Optional[type[Spider]] = None, + default_spidercls: type[Spider] | None = None, log_none: bool = False, log_multiple: bool = False, -) -> Optional[type[Spider]]: +) -> type[Spider] | None: """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 2c3a259c1..7d46cbd4f 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL @@ -26,7 +26,7 @@ def x509name_to_string(x509name: X509Name) -> str: return ffi_buf_to_string(result_buffer) -def get_temp_key_info(ssl_object: Any) -> Optional[str]: +def get_temp_key_info(ssl_object: Any) -> str | None: # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"): # removed in cryptography 40.0.0 diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 08f3f2dc9..3e4dae5c8 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -5,13 +5,13 @@ from __future__ import annotations import re import string from pathlib import Path -from typing import TYPE_CHECKING, Any, Union +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from os import PathLike -def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None: +def render_templatefile(path: str | PathLike, **kwargs: Any) -> None: path_obj = Path(path) raw = path_obj.read_text("utf8") diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 860a2e3dd..d65f2a76d 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -9,7 +9,7 @@ import os from importlib import import_module from pathlib import Path from posixpath import split -from typing import TYPE_CHECKING, Any, Optional, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from unittest import TestCase, mock from twisted.trial.unittest import SkipTest @@ -84,8 +84,8 @@ class TestSpider(Spider): def get_crawler( - spidercls: Optional[type[Spider]] = None, - settings_dict: Optional[dict[str, Any]] = None, + spidercls: type[Spider] | None = None, + settings_dict: dict[str, Any] | None = None, prevent_warnings: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it @@ -120,7 +120,7 @@ def get_testenv() -> dict[str, str]: def assert_samelines( - testcase: TestCase, text1: str, text2: str, msg: Optional[str] = None + testcase: TestCase, text1: str, text2: str, msg: str | None = None ) -> None: """Asserts text1 and text2 have the same lines, ignoring differences in line endings between platforms diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index dfc823725..05e04e2d1 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,7 +2,7 @@ from __future__ import annotations import os import sys -from typing import TYPE_CHECKING, Optional, cast +from typing import TYPE_CHECKING, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated @@ -15,7 +15,7 @@ if TYPE_CHECKING: class ProcessTest: - command: Optional[str] = None + command: str | None = None prefix = [sys.executable, "-m", "scrapy.cmdline"] cwd = os.getcwd() # trial chdirs to temp dir @@ -23,7 +23,7 @@ class ProcessTest: self, args: Iterable[str], check_code: bool = True, - settings: Optional[str] = None, + settings: str | None = None, ) -> Deferred[TestProcessProtocol]: from twisted.internet import reactor @@ -54,7 +54,7 @@ class TestProcessProtocol(ProcessProtocol): self.deferred: Deferred[TestProcessProtocol] = Deferred() self.out: bytes = b"" self.err: bytes = b"" - self.exitcode: Optional[int] = None + self.exitcode: int | None = None def outReceived(self, data: bytes) -> None: self.out += data diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 41d268baa..e0a2973f7 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -9,7 +9,7 @@ to the w3lib.url module. Always import those from there instead. from __future__ import annotations import re -from typing import TYPE_CHECKING, Optional, Union, cast +from typing import TYPE_CHECKING, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this @@ -50,7 +50,7 @@ def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: return any(lowercase_path.endswith(ext) for ext in extensions) -def parse_url(url: UrlT, encoding: Optional[str] = None) -> ParseResult: +def parse_url(url: UrlT, encoding: str | None = None) -> ParseResult: """Return urlparsed url from the given argument (which could be an already parsed url) """ diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index 1afef4d24..028e3a08a 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -1,6 +1,7 @@ +from __future__ import annotations + import asyncio import sys -from typing import Optional from scrapy import Spider from scrapy.crawler import CrawlerProcess @@ -31,7 +32,7 @@ class UrlSpider(Spider): if __name__ == "__main__": - ASYNCIO_EVENT_LOOP: Optional[str] + ASYNCIO_EVENT_LOOP: str | None try: ASYNCIO_EVENT_LOOP = sys.argv[1] except IndexError: diff --git a/tests/spiders.py b/tests/spiders.py index 5d5792858..cc54240ef 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -2,9 +2,10 @@ Some spiders used for testing and benchmarking """ +from __future__ import annotations + import asyncio import time -from typing import Optional from urllib.parse import urlencode from twisted.internet import defer @@ -82,19 +83,19 @@ class DelaySpider(MetaSpider): class LogSpider(MetaSpider): name = "log_spider" - def log_debug(self, message: str, extra: Optional[dict] = None): + def log_debug(self, message: str, extra: dict | None = None): self.logger.debug(message, extra=extra) - def log_info(self, message: str, extra: Optional[dict] = None): + def log_info(self, message: str, extra: dict | None = None): self.logger.info(message, extra=extra) - def log_warning(self, message: str, extra: Optional[dict] = None): + def log_warning(self, message: str, extra: dict | None = None): self.logger.warning(message, extra=extra) - def log_error(self, message: str, extra: Optional[dict] = None): + def log_error(self, message: str, extra: dict | None = None): self.logger.error(message, extra=extra) - def log_critical(self, message: str, extra: Optional[dict] = None): + def log_critical(self, message: str, extra: dict | None = None): self.logger.critical(message, extra=extra) def parse(self, response): diff --git a/tests/test_commands.py b/tests/test_commands.py index 6ec7c21b0..e7df7b6e8 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -15,7 +15,7 @@ from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import TemporaryFile, mkdtemp from threading import Timer -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from unittest import skipIf from pytest import mark @@ -117,9 +117,7 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) - def find_in_file( - self, filename: Union[str, os.PathLike], regex - ) -> Optional[re.Match]: + def find_in_file(self, filename: str | os.PathLike, regex) -> re.Match | None: """Find first pattern occurrence in file""" pattern = re.compile(regex) with Path(filename).open("r", encoding="utf-8") as f: @@ -198,7 +196,7 @@ class StartprojectTest(ProjectTest): def get_permissions_dict( - path: Union[str, os.PathLike], renamings=None, ignore=None + path: str | os.PathLike, renamings=None, ignore=None ) -> dict[str, str]: def get_permissions(path: Path) -> str: return oct(path.stat().st_mode) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index f14a10a32..19cea97ec 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,10 +1,11 @@ +from __future__ import annotations + import contextlib import os import shutil import sys from pathlib import Path from tempfile import mkdtemp, mkstemp -from typing import Optional from unittest import SkipTest, mock from testfixtures import LogCapture @@ -692,7 +693,7 @@ class Https11CustomCiphers(unittest.TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" - settings_dict: Optional[dict] = None + settings_dict: dict | None = None def setUp(self): self.mockserver = MockServer() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ea3ed3b05..f59412ab4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -18,7 +18,7 @@ from io import BytesIO from logging import getLogger from pathlib import Path from string import ascii_letters, digits -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from unittest import mock from urllib.parse import quote, urljoin from urllib.request import pathname2url @@ -66,7 +66,7 @@ def printf_escape(string): return string.replace("%", "%%") -def build_url(path: Union[str, PathLike]) -> str: +def build_url(path: str | PathLike) -> str: path_str = str(path) if path_str[0] != "/": path_str = "/" + path_str diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index b1043c111..ed3394b01 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,7 +1,8 @@ +from __future__ import annotations + import pickle import re import unittest -from typing import Optional from packaging.version import Version from pytest import mark @@ -16,7 +17,7 @@ from tests import get_testdata # a hack to skip base class tests in pytest class Base: class LinkExtractorTestCase(unittest.TestCase): - extractor_cls: Optional[type] = None + extractor_cls: type | None = None def setUp(self): body = get_testdata("link_extractor", "linkextractor.html") diff --git a/tests/test_loader.py b/tests/test_loader.py index 8db929dcf..aca428bbe 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,6 +1,7 @@ +from __future__ import annotations + import dataclasses import unittest -from typing import Optional import attr from itemadapter import ItemAdapter @@ -88,7 +89,7 @@ class BasicItemLoaderTest(unittest.TestCase): class InitializationTestMixin: - item_class: Optional[type] = None + item_class: type | None = None def test_keep_single_value(self): """Loaded item should contain values from the initial item""" diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 83e22b070..5cf4a63aa 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,7 +1,8 @@ +from __future__ import annotations + import shutil from pathlib import Path from tempfile import mkdtemp -from typing import Optional from testfixtures import LogCapture from twisted.internet import defer @@ -57,7 +58,7 @@ class FileDownloadCrawlTestCase(TestCase): store_setting_key = "FILES_STORE" media_key = "files" media_urls_key = "file_urls" - expected_checksums: Optional[set[str]] = { + expected_checksums: set[str] | None = { "5547178b89448faf0015a13f904c936e", "c2281c83670e31d8aaab7cb642b824db", "ed3f6538dc15d4d9179dae57319edc5f", @@ -216,7 +217,7 @@ class FileDownloadCrawlTestCase(TestCase): self.assertIn("ZeroDivisionError", str(log)) -skip_pillow: Optional[str] +skip_pillow: str | None try: from PIL import Image # noqa: imported just to check for the import error except ImportError: diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 296a6fae0..2c3b191fe 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import dataclasses import hashlib import io @@ -5,7 +7,6 @@ import random import warnings from shutil import rmtree from tempfile import mkdtemp -from typing import Optional from unittest.mock import patch import attr @@ -19,7 +20,7 @@ from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop from scrapy.settings import Settings from scrapy.utils.python import to_bytes -skip_pillow: Optional[str] +skip_pillow: str | None try: from PIL import Image except ImportError: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 127775f43..0faf6d015 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,4 +1,4 @@ -from typing import Optional +from __future__ import annotations from testfixtures import LogCapture from twisted.internet import reactor @@ -20,7 +20,7 @@ from scrapy.utils.test import get_crawler try: from PIL import Image # noqa: imported just to check for the import error except ImportError: - skip_pillow: Optional[str] = ( + skip_pillow: str | None = ( "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" ) else: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 9b7bad4bf..6b7cd5dac 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,8 +1,9 @@ +from __future__ import annotations + import collections import shutil import tempfile import unittest -from typing import Optional from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -60,7 +61,7 @@ class MockCrawler(Crawler): class SchedulerHandler: - priority_queue_cls: Optional[str] = None + priority_queue_cls: str | None = None jobdir = None def create_scheduler(self): @@ -254,7 +255,7 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): class DownloaderAwareSchedulerTestMixin: - priority_queue_cls: Optional[str] = "scrapy.pqueues.DownloaderAwarePriorityQueue" + priority_queue_cls: str | None = "scrapy.pqueues.DownloaderAwarePriorityQueue" reopen = False def test_logic(self): diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 4fd293ec7..b48a65e67 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,4 +1,5 @@ -from typing import Optional +from __future__ import annotations + from unittest import TestCase from urllib.parse import urljoin @@ -32,7 +33,7 @@ class MinimalScheduler: return True return False - def next_request(self) -> Optional[Request]: + def next_request(self) -> Request | None: if self.has_pending_requests(): fp, request = self.requests.popitem() return request diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 41228b5f2..1a80eb7be 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,5 +1,6 @@ +from __future__ import annotations + from collections.abc import AsyncIterator, Iterable -from typing import Optional, Union from unittest import mock from testfixtures import LogCapture @@ -112,11 +113,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): Should work for process_spider_output and, when it's supported, process_start_requests. """ - ITEM_TYPE: Union[type, tuple] + ITEM_TYPE: type | tuple RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod - def _construct_mw_setting(*mw_classes, start_index: Optional[int] = None): + def _construct_mw_setting(*mw_classes, start_index: int | None = None): if start_index is None: start_index = 10 return {i: c for c, i in enumerate(mw_classes, start=start_index)} @@ -127,7 +128,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): yield {"foo": 3} @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + def _get_middleware_result(self, *mw_classes, start_index: int | None = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) self.crawler = get_crawler( Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} @@ -141,7 +142,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): @defer.inlineCallbacks def _test_simple_base( - self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None + self, *mw_classes, downgrade: bool = False, start_index: int | None = None ): with LogCapture() as log: result = yield self._get_middleware_result( @@ -155,7 +156,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): @defer.inlineCallbacks def _test_asyncgen_base( - self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None + self, *mw_classes, downgrade: bool = False, start_index: int | None = None ): with LogCapture() as log: result = yield self._get_middleware_result( @@ -337,7 +338,7 @@ class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): yield {"name": "test item"} @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + def _get_middleware_result(self, *mw_classes, start_index: int | None = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) self.crawler = get_crawler( Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} @@ -441,7 +442,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + def _get_middleware_result(self, *mw_classes, start_index: int | None = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting}) self.spider = self.crawler._create_spider("foo") diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index e73e7ff4c..facbaa60d 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,5 +1,7 @@ +from __future__ import annotations + import warnings -from typing import Any, Optional +from typing import Any from unittest import TestCase from urllib.parse import urlparse @@ -35,7 +37,7 @@ class TestRefererMiddleware(TestCase): req_meta: dict[str, Any] = {} resp_headers: dict[str, str] = {} settings: dict[str, Any] = {} - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] @@ -65,7 +67,7 @@ class MixinDefault: with some additional filtering of s3:// """ - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ ("https://example.com/", "https://scrapy.org/", b"https://example.com/"), ("http://example.com/", "http://scrapy.org/", b"http://example.com/"), ("http://example.com/", "https://scrapy.org/", b"http://example.com/"), @@ -86,7 +88,7 @@ class MixinDefault: class MixinNoReferrer: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ ("https://example.com/page.html", "https://example.com/", None), ("http://www.example.com/", "https://scrapy.org/", None), ("http://www.example.com/", "http://scrapy.org/", None), @@ -96,7 +98,7 @@ class MixinNoReferrer: class MixinNoReferrerWhenDowngrade: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # TLS to TLS: send non-empty referrer ( "https://example.com/page.html", @@ -178,7 +180,7 @@ class MixinNoReferrerWhenDowngrade: class MixinSameOrigin: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -247,7 +249,7 @@ class MixinSameOrigin: class MixinOrigin: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) ( "https://example.com/page.html", @@ -271,7 +273,7 @@ class MixinOrigin: class MixinStrictOrigin: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades ( "https://example.com/page.html", @@ -299,7 +301,7 @@ class MixinStrictOrigin: class MixinOriginWhenCrossOrigin: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -406,7 +408,7 @@ class MixinOriginWhenCrossOrigin: class MixinStrictOriginWhenCrossOrigin: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # Same origin (protocol, host, port): send referrer ( "https://example.com/page.html", @@ -518,7 +520,7 @@ class MixinStrictOriginWhenCrossOrigin: class MixinUnsafeUrl: - scenarii: list[tuple[str, str, Optional[bytes]]] = [ + scenarii: list[tuple[str, str, bytes | None]] = [ # TLS to TLS: send referrer ( "https://example.com/sekrit.html", @@ -969,7 +971,7 @@ class TestPolicyHeaderPrecedence004( class TestReferrerOnRedirect(TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} scenarii: list[ - tuple[str, str, tuple[tuple[int, str], ...], Optional[bytes], Optional[bytes]] + tuple[str, str, tuple[tuple[int, str], ...], bytes | None, bytes | None] ] = [ # type: ignore[assignment] ( "http://scrapytest.org/1", # parent diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index ca3bca0b2..7156b13d0 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,8 +1,9 @@ +from __future__ import annotations + import json import unittest import warnings from hashlib import sha1 -from typing import Optional, Union from weakref import WeakKeyDictionary from scrapy.http import Request @@ -56,12 +57,12 @@ class FingerprintTest(unittest.TestCase): maxDiff = None function: staticmethod = staticmethod(fingerprint) - cache: Union[ - "WeakKeyDictionary[Request, dict[tuple[Optional[tuple[bytes, ...]], bool], bytes]]", - "WeakKeyDictionary[Request, dict[tuple[Optional[tuple[bytes, ...]], bool], str]]", - ] = _fingerprint_cache + cache: ( + WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]] + | WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]] + ) = _fingerprint_cache default_cache_key = (None, False) - known_hashes: tuple[tuple[Request, Union[bytes, str], dict], ...] = ( + known_hashes: tuple[tuple[Request, bytes | str, dict], ...] = ( ( Request("http://example.org"), b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD", From 7196a11f5321d05b79c9dedc29398a200d00c911 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 17 Oct 2024 21:51:13 +0500 Subject: [PATCH 1544/2083] Reorder unions with None. --- scrapy/core/spidermw.py | 2 +- scrapy/crawler.py | 2 +- scrapy/http/request/form.py | 6 +++--- scrapy/middleware.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1edfe1c51..f7947d35d 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -339,7 +339,7 @@ class SpiderMiddlewareManager(MiddlewareManager): @staticmethod def _get_async_method_pair( mw: Any, methodname: str - ) -> None | Callable | tuple[Callable, Callable]: + ) -> Callable | tuple[Callable, Callable] | None: normal_method: Callable | None = getattr(mw, methodname, None) methodname_async = methodname + "_async" async_method: Callable | None = getattr(mw, methodname_async, None) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 701dccf57..3e5657d22 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -57,7 +57,7 @@ class Crawler: def __init__( self, spidercls: type[Spider], - settings: None | dict[str, Any] | Settings = None, + settings: dict[str, Any] | Settings | None = None, init_reactor: bool = False, ): if isinstance(spidercls, Spider): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2fabf08d1..29743565d 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -197,7 +197,7 @@ def _get_inputs( def _value( ele: InputElement | SelectElement | TextareaElement, -) -> tuple[str | None, None | str | MultipleSelectOptions]: +) -> tuple[str | None, str | MultipleSelectOptions | None]: n = ele.name v = ele.value if ele.tag == "select": @@ -206,8 +206,8 @@ def _value( def _select_value( - ele: SelectElement, n: str | None, v: None | str | MultipleSelectOptions -) -> tuple[str | None, None | str | MultipleSelectOptions]: + ele: SelectElement, n: str | None, v: str | MultipleSelectOptions | None +) -> tuple[str | None, str | MultipleSelectOptions | None]: multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 39f26717a..b6a427895 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -40,7 +40,7 @@ class MiddlewareManager: self.middlewares = middlewares # Only process_spider_output and process_spider_exception can be None. # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. - self.methods: dict[str, deque[None | Callable | tuple[Callable, Callable]]] = ( + self.methods: dict[str, deque[Callable | tuple[Callable, Callable] | None]] = ( defaultdict(deque) ) for mw in middlewares: From 7e07d48cc5bfb4e07e1319334884ab420a2616c0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 17 Oct 2024 23:22:37 +0500 Subject: [PATCH 1545/2083] Small 3.7 and 3.8 cleanup. --- scrapy/utils/reactor.py | 6 ++---- tests/CrawlerProcess/asyncio_enabled_reactor.py | 2 +- .../asyncio_enabled_reactor_different_loop.py | 2 +- tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py | 2 +- tox.ini | 3 --- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 18bb583b8..f8904a9aa 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -97,10 +97,8 @@ def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: policy = asyncio.get_event_loop_policy() - if ( - sys.version_info >= (3, 8) - and sys.platform == "win32" - and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy) + if sys.platform == "win32" and not isinstance( + policy, asyncio.WindowsSelectorEventLoopPolicy ): policy = asyncio.WindowsSelectorEventLoopPolicy() asyncio.set_event_loop_policy(policy) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 01d23c963..f013eed27 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -3,7 +3,7 @@ import sys from twisted.internet import asyncioreactor -if sys.version_info >= (3, 8) and sys.platform == "win32": +if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index 9dc8ce46b..e9d6d8875 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -4,7 +4,7 @@ import sys from twisted.internet import asyncioreactor from twisted.python import log -if sys.version_info >= (3, 8) and sys.platform == "win32": +if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index be9c83b95..c72a0a17c 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -4,7 +4,7 @@ import sys from twisted.internet import asyncioreactor from uvloop import Loop -if sys.version_info >= (3, 8) and sys.platform == "win32": +if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.set_event_loop(Loop()) asyncioreactor.install(asyncio.get_event_loop()) diff --git a/tox.ini b/tox.ini index 79f72a0f2..fbbce48d4 100644 --- a/tox.ini +++ b/tox.ini @@ -26,9 +26,6 @@ deps = # mitmproxy does not support PyPy mitmproxy; implementation_name != 'pypy' - # https://github.com/pallets/werkzeug/pull/2768 breaks flask, required by - # mitmproxy. - werkzeug < 3; python_version < '3.9' and implementation_name != 'pypy' passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From 5759b3f0f2b0a45588e7ae7cd455ee5e7d4f531c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 17 Oct 2024 23:41:23 +0500 Subject: [PATCH 1546/2083] Drop Reppy. --- docs/topics/downloader-middleware.rst | 32 -------------------- scrapy/robotstxt.py | 20 ------------ tests/test_downloadermiddleware_robotstxt.py | 13 +------- tests/test_robotstxt_interface.py | 24 --------------- 4 files changed, 1 insertion(+), 88 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index c31f7fe43..13064ccdd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1086,7 +1086,6 @@ RobotsTxtMiddleware * :ref:`Protego <protego-parser>` (default) * :ref:`RobotFileParser <python-robotfileparser>` * :ref:`Robotexclusionrulesparser <rerp-parser>` - * :ref:`Reppy <reppy-parser>` (deprecated) You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`. @@ -1154,37 +1153,6 @@ In order to use this parser, set: * :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser`` -.. _reppy-parser: - -Reppy parser -~~~~~~~~~~~~ - -Based on `Reppy <https://github.com/seomoz/reppy/>`_: - -* is a Python wrapper around `Robots Exclusion Protocol Parser for C++ - <https://github.com/seomoz/rep-cpp>`_ - -* is compliant with `Martijn Koster's 1996 draft specification - <https://www.robotstxt.org/norobots-rfc.txt>`_ - -* supports wildcard matching - -* uses the length based rule - -Native implementation, provides better speed than Protego. - -In order to use this parser: - -* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy`` - - .. warning:: `Upstream issue #122 - <https://github.com/seomoz/reppy/issues/122>`_ prevents reppy usage in Python 3.9+. - Because of this the Reppy parser is deprecated. - -* Set :setting:`ROBOTSTXT_PARSER` setting to - ``scrapy.robotstxt.ReppyRobotParser`` - - .. _rerp-parser: Robotexclusionrulesparser diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index a0e5fc671..f0a6e7467 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -4,9 +4,7 @@ import logging import sys from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING -from warnings import warn -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import to_unicode if TYPE_CHECKING: @@ -90,24 +88,6 @@ class PythonRobotParser(RobotParser): return self.rp.can_fetch(user_agent, url) -class ReppyRobotParser(RobotParser): - def __init__(self, robotstxt_body: bytes, spider: Spider | None): - warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2) - from reppy.robots import Robots - - self.spider: Spider | None = spider - self.rp = Robots.parse("", robotstxt_body) - - @classmethod - def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: - spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o - - def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: - return self.rp.allowed(url, user_agent) - - class RerpRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): from robotexclusionrulesparser import RobotExclusionRulesParser diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index e166cc000..12b541456 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings -from tests.test_robotstxt_interface import reppy_available, rerp_available +from tests.test_robotstxt_interface import rerp_available class RobotsTxtMiddlewareTest(unittest.TestCase): @@ -254,14 +254,3 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): self.crawler.settings.set( "ROBOTSTXT_PARSER", "scrapy.robotstxt.RerpRobotParser" ) - - -class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest): - if not reppy_available(): - skip = "Reppy parser is not installed" - - def setUp(self): - super().setUp() - self.crawler.settings.set( - "ROBOTSTXT_PARSER", "scrapy.robotstxt.ReppyRobotParser" - ) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 28ad910a8..541979dcc 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -3,15 +3,6 @@ from twisted.trial import unittest from scrapy.robotstxt import decode_robotstxt -def reppy_available(): - # check if reppy parser is installed - try: - from reppy.robots import Robots # noqa: F401 - except ImportError: - return False - return True - - def rerp_available(): # check if robotexclusionrulesparser is installed try: @@ -169,21 +160,6 @@ class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): raise unittest.SkipTest("RobotFileParser does not support wildcards.") -class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase): - if not reppy_available(): - skip = "Reppy parser is not installed" - - def setUp(self): - from scrapy.robotstxt import ReppyRobotParser - - super()._setUp(ReppyRobotParser) - - def test_order_based_precedence(self): - raise unittest.SkipTest( - "Reppy does not support order based directives precedence." - ) - - class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): if not rerp_available(): skip = "Rerp parser is not installed" From 677e9772070ec8a92033f66dff45d7c421763203 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 18 Oct 2024 00:03:32 +0500 Subject: [PATCH 1547/2083] Remove dead links to the Reppy doc from the release notes. --- docs/news.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 58b51c9ea..2bbca77cc 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1530,7 +1530,7 @@ Documentation - Provided better context and instructions to disable the :setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`) -- Documented that :ref:`reppy-parser` does not support Python 3.9+. +- Documented that Reppy parser does not support Python 3.9+. (:issue:`5226`, :issue:`5231`) - Documented :ref:`the scheduler component <topics-scheduler>`. @@ -3344,7 +3344,7 @@ New features * A new :setting:`ROBOTSTXT_PARSER` setting allows choosing which robots.txt_ parser to use. It includes built-in support for :ref:`RobotFileParser <python-robotfileparser>`, - :ref:`Protego <protego-parser>` (default), :ref:`Reppy <reppy-parser>`, and + :ref:`Protego <protego-parser>` (default), Reppy, and :ref:`Robotexclusionrulesparser <rerp-parser>`, and allows you to :ref:`implement support for additional parsers <support-for-new-robots-parser>` (:issue:`754`, :issue:`2669`, From 04d0411bf7538ebe8e81771ecf9c6792c71c863b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 21 Oct 2024 15:30:49 +0500 Subject: [PATCH 1548/2083] Filter test-time warnings. (#6501) --- tests/test_crawl.py | 2 +- tests/test_crawler.py | 12 +++--------- tests/test_downloadermiddleware_offsite.py | 10 ++++++++-- tests/test_dupefilters.py | 6 ------ tests/test_pipeline_crawl.py | 1 - tests/test_scheduler.py | 1 - tests/test_spiderloader/__init__.py | 1 - tests/test_utils_asyncio.py | 2 +- tests/test_utils_datatypes.py | 6 ++++++ tests/test_utils_misc/__init__.py | 3 +++ 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 125709571..1f81a6073 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -428,7 +428,7 @@ with multiples lines @defer.inlineCallbacks def test_crawl_multiple(self): - runner = CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"}) + runner = CrawlerRunner() runner.crawl( SimpleSpider, self.mockserver.url("/status?n=200"), diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 69bfb7eb3..92a201fd1 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -6,6 +6,7 @@ import subprocess import sys import warnings from pathlib import Path +from typing import Any import pytest from packaging.version import parse as parse_version @@ -28,10 +29,7 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer, get_mockserver_env -# To prevent warnings. -BASE_SETTINGS = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", -} +BASE_SETTINGS: dict[str, Any] = {} def get_raw_crawler(spidercls=None, settings_dict=None): @@ -478,8 +476,6 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { "LOG_LEVEL": "INFO", "LOG_FILE": str(log_file), - # settings to avoid extra warnings - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } configure_logging() @@ -582,7 +578,7 @@ class NoRequestsSpider(scrapy.Spider): @mark.usefixtures("reactor_pytest") class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): - return CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"}) + return CrawlerRunner() @inlineCallbacks def test_crawler_runner_bootstrap_successful(self): @@ -631,7 +627,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase): CrawlerRunner( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } ) else: @@ -640,7 +635,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase): runner = CrawlerRunner( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } ) yield runner.crawl(NoRequestsSpider) diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index d4669f450..fec56a39f 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -1,3 +1,5 @@ +import warnings + import pytest from scrapy import Request, Spider @@ -87,7 +89,9 @@ def test_process_request_invalid_domains(): allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) mw = OffsiteMiddleware.from_crawler(crawler) - mw.spider_opened(spider) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + mw.spider_opened(spider) request = Request("https://a.example") assert mw.process_request(request, spider) is None for letter in ("b", "c"): @@ -175,7 +179,9 @@ def test_request_scheduled_invalid_domains(): allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) mw = OffsiteMiddleware.from_crawler(crawler) - mw.spider_opened(spider) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + mw.spider_opened(spider) request = Request("https://a.example") assert mw.request_scheduled(request, spider) is None for letter in ("b", "c"): diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index f617fc027..9ba8bd64f 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -50,7 +50,6 @@ class RFPDupeFilterTest(unittest.TestCase): settings = { "DUPEFILTER_DEBUG": True, "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) @@ -61,7 +60,6 @@ class RFPDupeFilterTest(unittest.TestCase): settings = { "DUPEFILTER_DEBUG": True, "DUPEFILTER_CLASS": FromSettingsRFPDupeFilter, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) @@ -71,7 +69,6 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_direct_scheduler(self): settings = { "DUPEFILTER_CLASS": DirectDupeFilter, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) @@ -176,7 +173,6 @@ class RFPDupeFilterTest(unittest.TestCase): settings = { "DUPEFILTER_DEBUG": False, "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) @@ -205,7 +201,6 @@ class RFPDupeFilterTest(unittest.TestCase): settings = { "DUPEFILTER_DEBUG": True, "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) @@ -243,7 +238,6 @@ class RFPDupeFilterTest(unittest.TestCase): with LogCapture() as log: settings = { "DUPEFILTER_DEBUG": True, - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 5cf4a63aa..696ef8cab 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -71,7 +71,6 @@ class FileDownloadCrawlTestCase(TestCase): # prepare a directory for storing files self.tmpmediastore = Path(mkdtemp()) self.settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", "ITEM_PIPELINES": {self.pipeline_class: 1}, self.store_setting_key: str(self.tmpmediastore), } diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 6b7cd5dac..387bc7c20 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -53,7 +53,6 @@ class MockCrawler(Crawler): "SCHEDULER_PRIORITY_QUEUE": priority_queue_cls, "JOBDIR": jobdir, "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 32699d837..d2ff9ba48 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -103,7 +103,6 @@ class SpiderLoaderTest(unittest.TestCase): runner = CrawlerRunner( { "SPIDER_MODULES": [module], - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } ) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 65e352053..1c93829e9 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,8 +1,8 @@ import asyncio import warnings -from unittest import TestCase from pytest import mark +from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index fb7c90f80..10dc6f270 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -3,6 +3,8 @@ import unittest import warnings from collections.abc import Iterator, Mapping, MutableMapping +import pytest + from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.utils.datatypes import ( @@ -90,12 +92,14 @@ class CaseInsensitiveDictMixin: self.assertRaises(KeyError, d.__getitem__, "key_LOWER") self.assertRaises(KeyError, d.__getitem__, "key_lower") + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_getdefault(self): d = CaselessDict() self.assertEqual(d.get("c", 5), 5) d["c"] = 10 self.assertEqual(d.get("c", 5), 10) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_setdefault(self): d = CaselessDict({"a": 1, "b": 2}) @@ -212,11 +216,13 @@ class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): self.assertEqual(list(iterkeys), ["AsDf", "FoO"]) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): dict_class = CaselessDict def test_deprecation_message(self): with warnings.catch_warnings(record=True) as caught: + warnings.filterwarnings("always", category=ScrapyDeprecationWarning) self.dict_class({"foo": "bar"}) self.assertEqual(len(caught), 1) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index ee3314d8e..4d8e71521 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -4,6 +4,8 @@ import unittest from pathlib import Path from unittest import mock +import pytest + from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, @@ -97,6 +99,7 @@ class UtilsMiscTestCase(unittest.TestCase): list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")] ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_create_instance(self): settings = mock.MagicMock() crawler = mock.MagicMock(spec_set=["settings"]) From d10c58ff38b88bf1cb67503645e9cb00a59d970f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 22 Oct 2024 19:07:21 +0500 Subject: [PATCH 1549/2083] Bump pyftpdlib to the version supporting Python 3.13 on Windows. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index fbbce48d4..a526fc120 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ minversion = 1.7.0 deps = attrs pexpect >= 4.8.0 - pyftpdlib >= 1.5.8 + pyftpdlib >= 2.0.1 pygments pytest pytest-cov==4.0.0 From 0523e1616d32182499a2dcd3fb98b38bd3c74041 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 29 Oct 2024 14:16:03 +0500 Subject: [PATCH 1550/2083] Explictly set html_baseurl on RTD. (#6507) --- docs/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index dcd2c9a3a..3de50e54e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,7 +8,7 @@ # # All configuration values have a default; values that are commented out # serve to show the default. - +import os import sys from pathlib import Path @@ -186,6 +186,8 @@ html_css_files = [ "custom.css", ] +# Set canonical URL from the Read the Docs Domain +html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "") # Options for LaTeX output # ------------------------ From fcb5ab6cffa8cec7c731bbd81419635fa2f2ece0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 29 Oct 2024 14:21:07 +0500 Subject: [PATCH 1551/2083] Remove code for unsupported Twisted. (#6510) --- scrapy/mail.py | 7 +------ tests/test_mail.py | 8 +------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 1e65b1623..ce7beb773 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -16,10 +16,8 @@ from email.utils import formatdate from io import BytesIO from typing import IO, TYPE_CHECKING, Any -from twisted import version as twisted_version from twisted.internet import ssl from twisted.internet.defer import Deferred -from twisted.python.versions import Version from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes @@ -217,12 +215,9 @@ class MailSender: "heloFallback": True, "requireAuthentication": False, "requireTransportSecurity": self.smtptls, + "hostname": self.smtphost, } - # Newer versions of twisted require the hostname to use STARTTLS - if twisted_version >= Version("twisted", 21, 2, 0): - factory_keywords["hostname"] = self.smtphost - factory = ESMTPSenderFactory( self.smtpuser, self.smtppass, diff --git a/tests/test_mail.py b/tests/test_mail.py index ff1505397..c6af2b1b8 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -2,11 +2,8 @@ import unittest from email.charset import Charset from io import BytesIO -from twisted import version as twisted_version from twisted.internet import defer from twisted.internet._sslverify import ClientTLSOptions -from twisted.internet.ssl import ClientContextFactory -from twisted.python.versions import Version from scrapy.mail import MailSender @@ -159,10 +156,7 @@ class MailSenderTest(unittest.TestCase): ) context = factory.buildProtocol("test@scrapy.org").context - if twisted_version >= Version("twisted", 21, 2, 0): - self.assertIsInstance(context, ClientTLSOptions) - else: - self.assertIsInstance(context, ClientContextFactory) + self.assertIsInstance(context, ClientTLSOptions) if __name__ == "__main__": From 5bbf8124ac6785b824b005ad1380039c963c2af1 Mon Sep 17 00:00:00 2001 From: ThunderMind <46158218+ThunderMind2019@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:28:00 +0500 Subject: [PATCH 1552/2083] Updated deprecated ast.NameConstant with ast.Constant #6305 (#6463) --- scrapy/utils/misc.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 1ab30f097..516218347 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -263,9 +263,7 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: def returns_none(return_node: ast.Return) -> bool: value = return_node.value - return ( - value is None or isinstance(value, ast.NameConstant) and value.value is None - ) + return value is None or isinstance(value, ast.Constant) and value.value is None if inspect.isgeneratorfunction(callable): func = callable From 65ecd5d5287491cb0c44541252a127144438da01 Mon Sep 17 00:00:00 2001 From: Rohitkr117 <145501871+Rohitkr117@users.noreply.github.com> Date: Tue, 29 Oct 2024 23:38:38 +0530 Subject: [PATCH 1553/2083] Fixes for Twisted Version Check and Typing Issues (#6511) --- scrapy/pipelines/media.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 61eddffa7..b10ec147b 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -15,8 +15,10 @@ from typing import ( cast, ) +from twisted import version as twisted_version from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure +from twisted.python.versions import Version from scrapy.http.request import NO_CALLBACK, Request from scrapy.settings import Settings @@ -206,8 +208,8 @@ class MediaPipeline(ABC): # minimize cached information for failure result.cleanFailure() result.frames = [] - result.stack = [] - + if twisted_version <= Version("twisted", 24, 10, 0): + result.stack = [] # type: ignore[method-assign] # This code fixes a memory leak by avoiding to keep references to # the Request and Response objects on the Media Pipeline cache. # From 12b087b0f23d91a16c7382baeba96d5bf32ab946 Mon Sep 17 00:00:00 2001 From: Rohitkr117 <145501871+Rohitkr117@users.noreply.github.com> Date: Wed, 30 Oct 2024 00:00:32 +0530 Subject: [PATCH 1554/2083] Added ignore statements for Windows specific typing issues (#6516) --- scrapy/extensions/debug.py | 6 +++--- scrapy/utils/console.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index d3c225bcd..6948c394c 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -33,8 +33,8 @@ class StackTraceDump: def __init__(self, crawler: Crawler): self.crawler: Crawler = crawler try: - signal.signal(signal.SIGUSR2, self.dump_stacktrace) - signal.signal(signal.SIGQUIT, self.dump_stacktrace) + signal.signal(signal.SIGUSR2, self.dump_stacktrace) # type: ignore[attr-defined] + signal.signal(signal.SIGQUIT, self.dump_stacktrace) # type: ignore[attr-defined] except AttributeError: # win32 platforms don't support SIGUSR signals pass @@ -70,7 +70,7 @@ class StackTraceDump: class Debugger: def __init__(self) -> None: try: - signal.signal(signal.SIGUSR2, self._enter_debugger) + signal.signal(signal.SIGUSR2, self._enter_debugger) # type: ignore[attr-defined] except AttributeError: # win32 platforms don't support SIGUSR signals pass diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index aecd3fdb7..6b9b4114f 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -82,7 +82,7 @@ def _embed_standard_shell( else: import rlcompleter # noqa: F401 - readline.parse_and_bind("tab:complete") + readline.parse_and_bind("tab:complete") # type: ignore[attr-defined] @wraps(_embed_standard_shell) def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: From d2bdbad8c8cc5e5b4b9d3a79c94e2411a44e94be Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:28:35 -0300 Subject: [PATCH 1555/2083] Deprecate `scrapy.twisted_version` (#6512) * Deprecate scrapy.twisted_version * fix: typing * remove typing * raise default exception if attribute is not found * remove redudant () * add tests * rollback exception raised * add filterwarnings again * change order * lint --- scrapy/__init__.py | 21 +++++++++++++++++---- tests/test_scrapy__getattr__.py | 13 +++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 tests/test_scrapy__getattr__.py diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 1c1a5c2cc..921296502 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -6,8 +6,6 @@ import pkgutil import sys import warnings -from twisted import version as _txv - # Declare top-level shortcuts from scrapy.http import FormRequest, Request from scrapy.item import Field, Item @@ -17,7 +15,6 @@ from scrapy.spiders import Spider __all__ = [ "__version__", "version_info", - "twisted_version", "Spider", "Request", "FormRequest", @@ -30,7 +27,23 @@ __all__ = [ # Scrapy and Twisted versions __version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip() version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split(".")) -twisted_version = (_txv.major, _txv.minor, _txv.micro) + + +def __getattr__(name: str): + if name == "twisted_version": + import warnings + + from twisted import version as _txv + + from scrapy.exceptions import ScrapyDeprecationWarning + + warnings.warn( + "The scrapy.twisted_version attribute is deprecated, use twisted.version instead", + ScrapyDeprecationWarning, + ) + return _txv.major, _txv.minor, _txv.micro + + raise AttributeError # Ignore noisy twisted deprecation warnings diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py new file mode 100644 index 000000000..979c42267 --- /dev/null +++ b/tests/test_scrapy__getattr__.py @@ -0,0 +1,13 @@ +import warnings + + +def test_deprecated_twisted_version(): + with warnings.catch_warnings(record=True) as warns: + from scrapy import twisted_version + + assert twisted_version is not None + assert isinstance(twisted_version, tuple) + assert ( + "The scrapy.twisted_version attribute is deprecated, use twisted.version instead" + in warns[0].message.args + ) From d85c39f5bcd728915fccece86f2b2e4ef37c0e53 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 31 Oct 2024 18:06:22 +0500 Subject: [PATCH 1556/2083] Deprecation removals. (#6500) * Deprecation removals. * Clean up the default pytest filterwarnings. * Remove test_get_images_old(). * Redo boto-requiring test filtering. * Remove an unused function. * Improve the Crawler.crawl() error message. * Fix the test. --- conftest.py | 24 ++++ docs/topics/commands.rst | 5 - extras/scrapy_zsh_completion | 2 - pytest.ini | 6 +- scrapy/commands/__init__.py | 9 +- scrapy/crawler.py | 8 +- .../downloadermiddlewares/httpcompression.py | 18 +-- scrapy/downloadermiddlewares/retry.py | 45 ++---- scrapy/extensions/feedexport.py | 80 +++-------- scrapy/pipelines/images.py | 57 ++------ scrapy/utils/conf.py | 48 +------ scrapy/utils/reactor.py | 19 +-- scrapy/utils/request.py | 10 +- tests/test_crawler.py | 8 +- tests/test_downloader_handlers.py | 7 +- ...st_downloadermiddleware_httpcompression.py | 29 +--- tests/test_downloadermiddleware_retry.py | 32 ----- tests/test_feedexport.py | 65 ++------- tests/test_pipeline_files.py | 7 +- tests/test_pipeline_images.py | 135 +----------------- tests/test_utils_conf.py | 72 +--------- tox.ini | 4 +- 22 files changed, 103 insertions(+), 587 deletions(-) diff --git a/conftest.py b/conftest.py index 2ab3dffd4..77b0e033b 100644 --- a/conftest.py +++ b/conftest.py @@ -89,6 +89,30 @@ def requires_uvloop(request): pytest.skip("uvloop is not installed") +@pytest.fixture(autouse=True) +def requires_botocore(request): + if not request.node.get_closest_marker("requires_botocore"): + return + try: + import botocore + + del botocore + except ImportError: + pytest.skip("botocore is not installed") + + +@pytest.fixture(autouse=True) +def requires_boto3(request): + if not request.node.get_closest_marker("requires_boto3"): + return + try: + import boto3 + + del boto3 + except ImportError: + pytest.skip("boto3 is not installed") + + def pytest_configure(config): if config.getoption("--reactor") == "asyncio": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 6eb4af9bd..6ffb8ae93 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -278,8 +278,6 @@ Supported options: * ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) -* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O`` - Usage examples:: $ scrapy crawl myspider @@ -291,9 +289,6 @@ Usage examples:: $ scrapy crawl -O myfile:json myspider [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ] - $ scrapy crawl -o myfile -t csv myspider - [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] - .. command:: check check diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion index e2f2dc82b..82eb77cc0 100644 --- a/extras/scrapy_zsh_completion +++ b/extras/scrapy_zsh_completion @@ -41,7 +41,6 @@ _scrapy() { (runspider) local options=( {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' - {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)' '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' '1:spider file:_files -g \*.py' ) @@ -99,7 +98,6 @@ _scrapy() { (crawl) local options=( {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' - {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)' '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' '1:spider:_scrapy_spiders' ) diff --git a/pytest.ini b/pytest.ini index 16983be5e..824c0e9e9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -21,8 +21,6 @@ markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed requires_uvloop: marks tests as only enabled when uvloop is known to be working + requires_botocore: marks tests that need botocore (but not boto3) + requires_boto3: marks tests that need botocore and boto3 filterwarnings = - ignore:scrapy.downloadermiddlewares.decompression is deprecated - ignore:Module scrapy.utils.reqser is deprecated - ignore:typing.re is deprecated - ignore:typing.io is deprecated diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index eccbef040..56199cc01 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -162,12 +162,6 @@ class BaseRunSpiderCommand(ScrapyCommand): help="dump scraped items into FILE, overwriting any existing file," " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)", ) - parser.add_argument( - "-t", - "--output-format", - metavar="FORMAT", - help="format to use for dumping items", - ) def process_options(self, args: list[str], opts: argparse.Namespace) -> None: super().process_options(args, opts) @@ -179,8 +173,7 @@ class BaseRunSpiderCommand(ScrapyCommand): feeds = feed_process_params_from_cli( self.settings, opts.output, - opts.output_format, - opts.overwrite_output, + overwrite_output=opts.overwrite_output, ) self.settings.set("FEEDS", feeds, priority="cmdline") diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 3e5657d22..de0cf543e 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging import pprint import signal -import warnings from typing import TYPE_CHECKING, Any, TypeVar, cast from twisted.internet.defer import ( @@ -17,7 +16,6 @@ from zope.interface.verify import verifyClass from scrapy import Spider, signals from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.logformatter import LogFormatter @@ -142,10 +140,8 @@ class Crawler: if self.crawling: raise RuntimeError("Crawling already taking place") if self._started: - warnings.warn( - "Running Crawler.crawl() more than once is deprecated.", - ScrapyDeprecationWarning, - stacklevel=2, + raise RuntimeError( + "Cannot run Crawler.crawl() more than once on the same instance." ) self.crawling = self._started = True diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 84678b8e9..a65757972 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,5 @@ from __future__ import annotations -import warnings from itertools import chain from logging import getLogger from typing import TYPE_CHECKING, Any @@ -15,7 +14,6 @@ from scrapy.utils._compression import ( _unbrotli, _unzstd, ) -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip if TYPE_CHECKING: @@ -72,21 +70,7 @@ class HttpCompressionMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COMPRESSION_ENABLED"): raise NotConfigured - try: - return cls(crawler=crawler) - except TypeError: - warnings.warn( - "HttpCompressionMiddleware subclasses must either modify " - "their '__init__' method to support a 'crawler' parameter or " - "reimplement their 'from_crawler' method.", - ScrapyDeprecationWarning, - ) - mw = cls() - mw.stats = crawler.stats - mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE") - mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE") - crawler.signals.connect(mw.open_spider, signals.spider_opened) - return mw + return cls(crawler=crawler) def open_spider(self, spider: Spider) -> None: if hasattr(spider, "download_maxsize"): diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 7c0e2280c..9fab172a8 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -12,12 +12,10 @@ once the spider has finished crawling all regular (non-failed) pages. from __future__ import annotations -import warnings from logging import Logger, getLogger -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.settings import BaseSettings, Settings +from scrapy.exceptions import NotConfigured from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message @@ -29,33 +27,13 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.http.request import Request + from scrapy.settings import BaseSettings from scrapy.spiders import Spider retry_logger = getLogger(__name__) -def backwards_compatibility_getattr(self: Any, name: str) -> tuple[Any, ...]: - if name == "EXCEPTIONS_TO_RETRY": - warnings.warn( - "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " - "Use the RETRY_EXCEPTIONS setting instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return tuple( - load_object(x) if isinstance(x, str) else x - for x in Settings().getlist("RETRY_EXCEPTIONS") - ) - raise AttributeError( - f"{self.__class__.__name__!r} object has no attribute {name!r}" - ) - - -class BackwardsCompatibilityMetaclass(type): - __getattr__ = backwards_compatibility_getattr - - def get_retry_request( request: Request, *, @@ -144,22 +122,17 @@ def get_retry_request( return None -class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): +class RetryMiddleware: def __init__(self, settings: BaseSettings): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured self.max_retry_times = settings.getint("RETRY_TIMES") self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") - - try: - self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY") - except AttributeError: - # If EXCEPTIONS_TO_RETRY is not "overridden" - self.exceptions_to_retry = tuple( - load_object(x) if isinstance(x, str) else x - for x in settings.getlist("RETRY_EXCEPTIONS") - ) + self.exceptions_to_retry = tuple( + load_object(x) if isinstance(x, str) else x + for x in settings.getlist("RETRY_EXCEPTIONS") + ) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -199,5 +172,3 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): max_retry_times=max_retry_times, priority_adjust=priority_adjust, ) - - __getattr__ = backwards_compatibility_getattr diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index eb1698ce5..6ab88dbb4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -26,10 +26,8 @@ from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.settings import Settings -from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import maybe_deferred_to_future -from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import build_from_crawler, load_object @@ -48,13 +46,6 @@ if TYPE_CHECKING: from scrapy.exporters import BaseItemExporter from scrapy.settings import BaseSettings -try: - import boto3 # noqa: F401 - - IS_BOTO3_AVAILABLE = True -except ImportError: - IS_BOTO3_AVAILABLE = False - logger = logging.getLogger(__name__) @@ -217,8 +208,10 @@ class S3FeedStorage(BlockingFeedStorage): session_token: str | None = None, region_name: str | None = None, ): - if not is_botocore_available(): - raise NotConfigured("missing botocore library") + try: + import boto3.session + except ImportError: + raise NotConfigured("missing boto3 library") u = urlparse(uri) assert u.hostname self.bucketname: str = u.hostname @@ -229,42 +222,16 @@ class S3FeedStorage(BlockingFeedStorage): self.acl: str | None = acl self.endpoint_url: str | None = endpoint_url self.region_name: str | None = region_name - # It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client, - # there seems to be no good way to infer it statically. - self.s3_client: Any - if IS_BOTO3_AVAILABLE: - import boto3.session - - boto3_session = boto3.session.Session() - - self.s3_client = boto3_session.client( - "s3", - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - aws_session_token=self.session_token, - endpoint_url=self.endpoint_url, - region_name=self.region_name, - ) - else: - warnings.warn( - "`botocore` usage has been deprecated for S3 feed " - "export, please use `boto3` to avoid problems", - category=ScrapyDeprecationWarning, - ) - - import botocore.session - - botocore_session = botocore.session.get_session() - - self.s3_client = botocore_session.create_client( - "s3", - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - aws_session_token=self.session_token, - endpoint_url=self.endpoint_url, - region_name=self.region_name, - ) + boto3_session = boto3.session.Session() + self.s3_client = boto3_session.client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + region_name=self.region_name, + ) if feed_options and feed_options.get("overwrite", True) is False: logger.warning( @@ -295,17 +262,10 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - kwargs: dict[str, Any] - if IS_BOTO3_AVAILABLE: - kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} - self.s3_client.upload_fileobj( - Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs - ) - else: - kwargs = {"ACL": self.acl} if self.acl else {} - self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs - ) + kwargs: dict[str, Any] = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} + self.s3_client.upload_fileobj( + Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs + ) file.close() @@ -464,12 +424,6 @@ class FeedSlot: self._exporting = False -_FeedSlot = create_deprecated_class( - name="_FeedSlot", - new_class=FeedSlot, -) - - class FeedExporter: _pending_deferreds: list[Deferred[None]] = [] diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index bbba7d1e1..2c4c9376e 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -8,14 +8,13 @@ from __future__ import annotations import functools import hashlib -import warnings from contextlib import suppress from io import BytesIO from typing import TYPE_CHECKING, Any, cast from itemadapter import ItemAdapter -from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import ( @@ -27,7 +26,7 @@ from scrapy.pipelines.files import ( _md5sum, ) from scrapy.settings import Settings -from scrapy.utils.python import get_func_args, to_bytes +from scrapy.utils.python import to_bytes if TYPE_CHECKING: from collections.abc import Callable, Iterable @@ -42,18 +41,6 @@ if TYPE_CHECKING: from scrapy.pipelines.media import FileInfoOrError, MediaPipeline -class NoimagesDrop(DropItem): - """Product with no images exception""" - - def __init__(self, *args: Any, **kwargs: Any): - warnings.warn( - "The NoimagesDrop class is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - super().__init__(*args, **kwargs) - - class ImageException(FileException): """General image error exception""" @@ -120,8 +107,6 @@ class ImagesPipeline(FilesPipeline): resolve("IMAGES_THUMBS"), self.THUMBS ) - self._deprecated_convert_image: bool | None = None - @classmethod def from_settings(cls, settings: Settings) -> Self: s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) @@ -203,49 +188,25 @@ class ImagesPipeline(FilesPipeline): f"{self.min_width}x{self.min_height})" ) - if self._deprecated_convert_image is None: - self._deprecated_convert_image = "response_body" not in get_func_args( - self.convert_image - ) - if self._deprecated_convert_image: - warnings.warn( - f"{self.__class__.__name__}.convert_image() method overridden in a deprecated way, " - "overridden method does not accept response_body argument.", - category=ScrapyDeprecationWarning, - ) - - if self._deprecated_convert_image: - image, buf = self.convert_image(orig_image) - else: - image, buf = self.convert_image( - orig_image, response_body=BytesIO(response.body) - ) + image, buf = self.convert_image( + orig_image, response_body=BytesIO(response.body) + ) yield path, image, buf for thumb_id, size in self.thumbs.items(): thumb_path = self.thumb_path( request, thumb_id, response=response, info=info, item=item ) - if self._deprecated_convert_image: - thumb_image, thumb_buf = self.convert_image(image, size) - else: - thumb_image, thumb_buf = self.convert_image(image, size, buf) + thumb_image, thumb_buf = self.convert_image(image, size, response_body=buf) yield thumb_path, thumb_image, thumb_buf def convert_image( self, image: Image.Image, size: tuple[int, int] | None = None, - response_body: BytesIO | None = None, + *, + response_body: BytesIO, ) -> tuple[Image.Image, BytesIO]: - if response_body is None: - warnings.warn( - f"{self.__class__.__name__}.convert_image() method called in a deprecated way, " - "method called without response_body argument.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if image.format in ("PNG", "WEBP") and image.mode == "RGBA": background = self._Image.new("RGBA", image.size, (255, 255, 255)) background.paste(image, image) @@ -268,7 +229,7 @@ class ImagesPipeline(FilesPipeline): except AttributeError: resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined] image.thumbnail(size, resampling_filter) - elif response_body is not None and image.format == "JPEG": + elif image.format == "JPEG": return image, response_body buf = BytesIO() diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 64cd31c4b..91a49c652 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -3,14 +3,13 @@ from __future__ import annotations import numbers import os import sys -import warnings from collections.abc import Iterable from configparser import ConfigParser from operator import itemgetter from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, cast -from scrapy.exceptions import ScrapyDeprecationWarning, UsageError +from scrapy.exceptions import UsageError from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values @@ -21,7 +20,7 @@ if TYPE_CHECKING: def build_component_list( compdict: MutableMapping[Any, Any], - custom: Any = None, + *, convert: Callable[[Any], Any] = update_classpath, ) -> list[Any]: """Compose a component list from a { class: order } dictionary.""" @@ -60,19 +59,6 @@ def build_component_list( "please provide a real number or None instead" ) - if custom is not None: - warnings.warn( - "The 'custom' attribute of build_component_list() is deprecated. " - "Please merge its value into 'compdict' manually or change your " - "code to use Settings.getwithbase().", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if isinstance(custom, (list, tuple)): - _check_components(custom) - return type(custom)(convert(c) for c in custom) # type: ignore[return-value] - compdict.update(custom) - _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] @@ -159,7 +145,7 @@ def feed_complete_default_values_from_settings( def feed_process_params_from_cli( settings: BaseSettings, output: list[str], - output_format: str | None = None, + *, overwrite_output: list[str] | None = None, ) -> dict[str, dict[str, Any]]: """ @@ -186,37 +172,9 @@ def feed_process_params_from_cli( raise UsageError( "Please use only one of -o/--output and -O/--overwrite-output" ) - if output_format: - raise UsageError( - "-t/--output-format is a deprecated command line option" - " and does not work in combination with -O/--overwrite-output." - " To specify a format please specify it after a colon at the end of the" - " output URI (i.e. -O <URI>:<FORMAT>)." - " Example working in the tutorial: " - "scrapy crawl quotes -O quotes.json:json" - ) output = overwrite_output overwrite = True - if output_format: - if len(output) == 1: - check_valid_format(output_format) - message = ( - "The -t/--output-format command line option is deprecated in favor of " - "specifying the output format within the output URI using the -o/--output or the" - " -O/--overwrite-output option (i.e. -o/-O <URI>:<FORMAT>). See the documentation" - " of the -o or -O option or the following examples for more information. " - "Examples working in the tutorial: " - "scrapy crawl quotes -o quotes.csv:csv or " - "scrapy crawl quotes -O quotes.json:json" - ) - warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) - return {output[0]: {"format": output_format}} - raise UsageError( - "The -t command-line option cannot be used if multiple output " - "URIs are specified" - ) - result: dict[str, dict[str, Any]] = {} for element in output: try: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index f8904a9aa..e7bd0b232 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -4,12 +4,11 @@ import asyncio import sys from contextlib import suppress from typing import TYPE_CHECKING, Any, Generic, TypeVar -from warnings import catch_warnings, filterwarnings, warn +from warnings import catch_warnings, filterwarnings from twisted.internet import asyncioreactor, error from twisted.internet.base import DelayedCall -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object if TYPE_CHECKING: @@ -79,22 +78,6 @@ def set_asyncio_event_loop_policy() -> None: _get_asyncio_event_loop_policy() -def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: - warn( - "Call to deprecated function " - "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n" - "\n" - "Please use get_event_loop, new_event_loop and set_event_loop" - " from asyncio instead, as the corresponding policy methods may lead" - " to unexpected behaviour.\n" - "This function is replaced by set_asyncio_event_loop_policy and" - " is meant to be used only when the reactor is being installed.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return _get_asyncio_event_loop_policy() - - def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: policy = asyncio.get_event_loop_policy() if sys.platform == "win32" and not isinstance( diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 82bdcb0f9..e80cbbb89 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -30,17 +30,9 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler -def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]: - for header in headers: - if header in request.headers: - yield header - yield from request.headers.getlist(header) - - _fingerprint_cache: WeakKeyDictionary[ Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes] -] -_fingerprint_cache = WeakKeyDictionary() +] = WeakKeyDictionary() def fingerprint( diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 92a201fd1..37348778c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -8,7 +8,6 @@ import warnings from pathlib import Path from typing import Any -import pytest from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn from pytest import mark, raises @@ -82,13 +81,10 @@ class CrawlerTestCase(BaseCrawlerTest): Crawler(DefaultSpider()) @inlineCallbacks - def test_crawler_crawl_twice_deprecated(self): + def test_crawler_crawl_twice_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) yield crawler.crawl() - with pytest.warns( - ScrapyDeprecationWarning, - match=r"Running Crawler.crawl\(\) more than once is deprecated", - ): + with raises(RuntimeError, match="more than once on the same instance"): yield crawler.crawl() def test_get_addon(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 19cea97ec..6a7597e9f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -8,6 +8,7 @@ from pathlib import Path from tempfile import mkdtemp, mkstemp from unittest import SkipTest, mock +import pytest from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal from twisted.internet import defer, error, reactor @@ -32,7 +33,7 @@ from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes -from scrapy.utils.test import get_crawler, skip_if_no_boto +from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import ( Echo, @@ -824,9 +825,9 @@ class HttpDownloadHandlerMock: return request +@pytest.mark.requires_botocore class S3AnonTestCase(unittest.TestCase): def setUp(self): - skip_if_no_boto() crawler = get_crawler() self.s3reqh = build_from_crawler( S3DownloadHandler, @@ -845,6 +846,7 @@ class S3AnonTestCase(unittest.TestCase): self.assertEqual(httpreq.url, "http://aws-publicdatasets.s3.amazonaws.com/") +@pytest.mark.requires_botocore class S3TestCase(unittest.TestCase): download_handler_cls: type = S3DownloadHandler @@ -856,7 +858,6 @@ class S3TestCase(unittest.TestCase): AWS_SECRET_ACCESS_KEY = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o" def setUp(self): - skip_if_no_boto() crawler = get_crawler() s3reqh = build_from_crawler( S3DownloadHandler, diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 7c36f748e..934af6590 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -3,7 +3,6 @@ from io import BytesIO from logging import WARNING from pathlib import Path from unittest import SkipTest, TestCase -from warnings import catch_warnings from testfixtures import LogCapture from w3lib.encoding import resolve_encoding @@ -12,7 +11,7 @@ from scrapy.downloadermiddlewares.httpcompression import ( ACCEPTED_ENCODINGS, HttpCompressionMiddleware, ) -from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider @@ -700,29 +699,3 @@ class HttpCompressionTest(TestCase): except ImportError: raise SkipTest("no zstd support (zstandard)") self._test_download_warnsize_request_meta("zstd") - - -class HttpCompressionSubclassTest(TestCase): - def test_init_missing_stats(self): - class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware): - def __init__(self): - super().__init__() - - crawler = get_crawler(Spider) - with catch_warnings(record=True) as caught_warnings: - HttpCompressionMiddlewareSubclass.from_crawler(crawler) - messages = tuple( - str(warning.message) - for warning in caught_warnings - if warning.category is ScrapyDeprecationWarning - ) - self.assertEqual( - messages, - ( - ( - "HttpCompressionMiddleware subclasses must either modify " - "their '__init__' method to support a 'crawler' parameter " - "or reimplement their 'from_crawler' method." - ), - ), - ) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 661175840..a010865ef 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,6 +1,5 @@ import logging import unittest -import warnings from testfixtures import LogCapture from twisted.internet import defer @@ -122,37 +121,6 @@ class RetryTest(unittest.TestCase): req = Request(f"http://www.scrapytest.org/{exc.__name__}") self._test_retry_exception(req, exc("foo"), mw) - def test_exception_to_retry_custom_middleware(self): - exc = ValueError - - with warnings.catch_warnings(record=True) as warns: - - class MyRetryMiddleware(RetryMiddleware): - EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,) - - self.assertEqual(len(warns), 1) - - mw2 = MyRetryMiddleware.from_crawler(self.crawler) - req = Request(f"http://www.scrapytest.org/{exc.__name__}") - req = mw2.process_exception(req, exc("foo"), self.spider) - assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 1) - - def test_exception_to_retry_custom_middleware_self(self): - class MyRetryMiddleware(RetryMiddleware): - def process_exception(self, request, exception, spider): - if isinstance(exception, self.EXCEPTIONS_TO_RETRY): - return self._retry(request, exception, spider) - - exc = OSError - mw2 = MyRetryMiddleware.from_crawler(self.crawler) - req = Request(f"http://www.scrapytest.org/{exc.__name__}") - with warnings.catch_warnings(record=True) as warns: - req = mw2.process_exception(req, exc("foo"), self.spider) - assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 1) - self.assertEqual(len(warns), 1) - def _test_retry_exception(self, req, exception, mw=None): if mw is None: mw = self.mw diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f59412ab4..790c347fb 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -37,7 +37,6 @@ from scrapy import signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( - IS_BOTO3_AVAILABLE, BlockingFeedStorage, FeedExporter, FeedSlot, @@ -50,7 +49,7 @@ from scrapy.extensions.feedexport import ( ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto +from scrapy.utils.test import get_crawler, mock_google_cloud_storage from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider @@ -240,10 +239,8 @@ class BlockingFeedStorageTest(unittest.TestCase): self.assertRaises(OSError, b.open, spider=spider) +@pytest.mark.requires_boto3 class S3FeedStorageTest(unittest.TestCase): - def setUp(self): - skip_if_no_boto() - def test_parse_credentials(self): aws_credentials = { "AWS_ACCESS_KEY_ID": "settings_key", @@ -292,38 +289,12 @@ class S3FeedStorageTest(unittest.TestCase): file = mock.MagicMock() - if IS_BOTO3_AVAILABLE: - storage.s3_client = mock.MagicMock() - yield storage.store(file) - self.assertEqual( - storage.s3_client.upload_fileobj.call_args, - mock.call(Bucket=bucket, Key=key, Fileobj=file), - ) - else: - from botocore.stub import Stubber - - with Stubber(storage.s3_client) as stub: - stub.add_response( - "put_object", - expected_params={ - "Body": file, - "Bucket": bucket, - "Key": key, - }, - service_response={}, - ) - - yield storage.store(file) - - stub.assert_no_pending_responses() - self.assertEqual( - file.method_calls, - [ - mock.call.seek(0), - # The call to read does not happen with Stubber - mock.call.close(), - ], - ) + storage.s3_client = mock.MagicMock() + yield storage.store(file) + self.assertEqual( + storage.s3_client.upload_fileobj.call_args, + mock.call(Bucket=bucket, Key=key, Fileobj=file), + ) def test_init_without_acl(self): storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") @@ -459,14 +430,11 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - if IS_BOTO3_AVAILABLE: - acl = ( - storage.s3_client.upload_fileobj.call_args[1] - .get("ExtraArgs", {}) - .get("ACL") - ) - else: - acl = storage.s3_client.put_object.call_args[1].get("ACL") + acl = ( + storage.s3_client.upload_fileobj.call_args[1] + .get("ExtraArgs", {}) + .get("ACL") + ) self.assertIsNone(acl) @defer.inlineCallbacks @@ -480,10 +448,7 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - if IS_BOTO3_AVAILABLE: - acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] - else: - acl = storage.s3_client.put_object.call_args[1]["ACL"] + acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] self.assertEqual(acl, "custom-acl") def test_overwrite_default(self): @@ -2647,9 +2612,9 @@ class BatchDeliveriesTest(FeedExportTestBase): crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12 ) + @pytest.mark.requires_boto3 @defer.inlineCallbacks def test_s3_export(self): - skip_if_no_boto() bucket = "mybucket" items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 6ce7fc059..47840caaa 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -11,6 +11,7 @@ from unittest import mock from urllib.parse import urlparse import attr +import pytest from itemadapter import ItemAdapter from twisted.internet import defer from twisted.trial import unittest @@ -30,7 +31,6 @@ from scrapy.utils.test import ( get_crawler, get_ftp_content_and_delete, get_gcs_content_and_delete, - skip_if_no_boto, ) from tests.mockserver import MockFTPServer @@ -507,11 +507,10 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(fs_store.basedir, str(path)) +@pytest.mark.requires_botocore class TestS3FilesStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): - skip_if_no_boto() - bucket = "mybucket" key = "export.csv" uri = f"s3://{bucket}/{key}" @@ -557,8 +556,6 @@ class TestS3FilesStore(unittest.TestCase): @defer.inlineCallbacks def test_stat(self): - skip_if_no_boto() - bucket = "mybucket" key = "export.csv" uri = f"s3://{bucket}/{key}" diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 2c3b191fe..7561e1fd4 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -1,24 +1,19 @@ from __future__ import annotations import dataclasses -import hashlib import io import random -import warnings from shutil import rmtree from tempfile import mkdtemp -from unittest.mock import patch import attr from itemadapter import ItemAdapter from twisted.trial import unittest -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.item import Field, Item -from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop +from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.settings import Settings -from scrapy.utils.python import to_bytes skip_pillow: str | None try: @@ -159,7 +154,7 @@ class ImagesPipelineTestCase(unittest.TestCase): with self.assertRaises(ImageException): next(self.pipeline.get_images(response=resp3, request=req, info=object())) - def test_get_images_new(self): + def test_get_images(self): self.pipeline.min_width = 0 self.pipeline.min_height = 0 self.pipeline.thumbs = {"small": (20, 20)} @@ -185,101 +180,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(thumb_img, thumb_img) self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue()) - def test_get_images_old(self): - self.pipeline.thumbs = {"small": (20, 20)} - orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) - resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) - req = Request(url="https://dev.mydeco.com/mydeco.gif") - - def overridden_convert_image(image, size=None): - im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0)) - return im, buf - - with patch.object(self.pipeline, "convert_image", overridden_convert_image): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - get_images_gen = self.pipeline.get_images( - response=resp, request=req, info=object() - ) - path, new_im, new_buf = next(get_images_gen) - self.assertEqual( - path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" - ) - self.assertEqual(orig_im.mode, new_im.mode) - self.assertEqual(orig_im.getcolors(), new_im.getcolors()) - self.assertEqual(buf.getvalue(), new_buf.getvalue()) - - thumb_path, thumb_img, thumb_buf = next(get_images_gen) - self.assertEqual( - thumb_path, - "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg", - ) - self.assertEqual(orig_im.mode, thumb_img.mode) - self.assertEqual(orig_im.getcolors(), thumb_img.getcolors()) - self.assertEqual(buf.getvalue(), thumb_buf.getvalue()) - - expected_warning_msg = ( - ".convert_image() method overridden in a deprecated way, " - "overridden method does not accept response_body argument." - ) - self.assertEqual( - len( - [ - warning - for warning in w - if expected_warning_msg in str(warning.message) - ] - ), - 1, - ) - - def test_convert_image_old(self): - # tests for old API - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - SIZE = (100, 100) - # straight forward case: RGB and JPEG - COLOUR = (0, 127, 255) - im, _ = _create_image("JPEG", "RGB", SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) - - # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) - self.assertEqual(thumbnail.mode, "RGB") - self.assertEqual(thumbnail.size, (10, 10)) - - # transparency case: RGBA and PNG - COLOUR = (0, 127, 255, 50) - im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # transparency case with palette: P and PNG - COLOUR = (0, 127, 255, 50) - im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR) - im = im.convert("P") - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # ensure that we received deprecation warnings - expected_warning_msg = ".convert_image() method called in a deprecated way" - self.assertTrue( - len( - [ - warning - for warning in w - if expected_warning_msg in str(warning.message) - ] - ) - == 4 - ) - - def test_convert_image_new(self): - # tests for new API + def test_convert_image(self): SIZE = (100, 100) # straight forward case: RGB and JPEG COLOUR = (0, 127, 255) @@ -313,19 +214,6 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) -class DeprecatedImagesPipeline(ImagesPipeline): - def file_key(self, url): - return self.image_key(url) - - def image_key(self, url): - image_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return f"empty/{image_guid}.jpg" - - def thumb_key(self, url, thumb_id): - thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return f"thumbsup/{thumb_id}/{thumb_guid}.jpg" - - class ImagesPipelineTestCaseFieldsMixin: skip = skip_pillow @@ -627,23 +515,6 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value) -class NoimagesDropTestCase(unittest.TestCase): - def test_deprecation_warning(self): - arg = "" - with warnings.catch_warnings(record=True) as w: - NoimagesDrop(arg) - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, ScrapyDeprecationWarning) - with warnings.catch_warnings(record=True) as w: - - class SubclassedNoimagesDrop(NoimagesDrop): - pass - - SubclassedNoimagesDrop(arg) - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, ScrapyDeprecationWarning) - - def _create_image(format, *a, **kw): buf = io.BytesIO() Image.new(*a, **kw).save(buf, format) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index dc3f01d57..2ce7948eb 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,9 +1,6 @@ import unittest -import warnings -import pytest - -from scrapy.exceptions import ScrapyDeprecationWarning, UsageError +from scrapy.exceptions import UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( arglist_to_dict, @@ -20,50 +17,6 @@ class BuildComponentListTest(unittest.TestCase): build_component_list(d, convert=lambda x: x), ["one", "four", "three"] ) - def test_backward_compatible_build_dict(self): - base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None} - custom = {"two": None, "three": 8, "four": 4} - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - self.assertEqual( - build_component_list(base, custom, convert=lambda x: x), - ["one", "four", "five", "three"], - ) - - def test_return_list(self): - custom = ["a", "b", "c"] - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - self.assertEqual( - build_component_list(None, custom, convert=lambda x: x), custom - ) - - def test_map_dict(self): - custom = {"one": 1, "two": 2, "three": 3} - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - self.assertEqual( - build_component_list({}, custom, convert=lambda x: x.upper()), - ["ONE", "TWO", "THREE"], - ) - - def test_map_list(self): - custom = ["a", "b", "c"] - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - self.assertEqual( - build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"] - ) - - def test_duplicate_components_in_dict(self): - duplicate_dict = {"one": 1, "two": 2, "ONE": 4} - with self.assertRaises(ValueError): - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - build_component_list({}, duplicate_dict, convert=lambda x: x.lower()) - - def test_duplicate_components_in_list(self): - duplicate_list = ["a", "b", "a"] - with self.assertRaises(ValueError) as cm: - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - build_component_list(None, duplicate_list, convert=lambda x: x) - self.assertIn(str(duplicate_list), str(cm.exception)) - def test_duplicate_components_in_basesettings(self): # Higher priority takes precedence duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0) @@ -92,11 +45,6 @@ class BuildComponentListTest(unittest.TestCase): "c": 22222222222222222222, } self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"]) - # raise exception for invalid values - d = {"one": "5"} - with self.assertRaises(ValueError): - with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): - build_component_list({}, d, convert=lambda x: x) class UtilsConfTestCase(unittest.TestCase): @@ -115,7 +63,6 @@ class FeedExportConfigTestCase(unittest.TestCase): feed_process_params_from_cli, settings, ["items.dat"], - "noformat", ) def test_feed_export_config_mismatch(self): @@ -125,18 +72,8 @@ class FeedExportConfigTestCase(unittest.TestCase): feed_process_params_from_cli, settings, ["items1.dat", "items2.dat"], - "noformat", ) - def test_feed_export_config_backward_compatible(self): - with warnings.catch_warnings(record=True) as cw: - settings = Settings() - self.assertEqual( - {"items.dat": {"format": "csv"}}, - feed_process_params_from_cli(settings, ["items.dat"], "csv"), - ) - self.assertEqual(cw[0].category, ScrapyDeprecationWarning) - def test_feed_export_config_explicit_formats(self): settings = Settings() self.assertEqual( @@ -174,7 +111,9 @@ class FeedExportConfigTestCase(unittest.TestCase): settings = Settings() self.assertEqual( {"output.json": {"format": "json", "overwrite": True}}, - feed_process_params_from_cli(settings, [], None, ["output.json"]), + feed_process_params_from_cli( + settings, [], overwrite_output=["output.json"] + ), ) def test_output_and_overwrite_output(self): @@ -182,8 +121,7 @@ class FeedExportConfigTestCase(unittest.TestCase): feed_process_params_from_cli( Settings(), ["output1.json"], - None, - ["output2.json"], + overwrite_output=["output2.json"], ) def test_feed_complete_default_values_from_settings_empty(self): diff --git a/tox.ini b/tox.ini index a526fc120..5783a0e61 100644 --- a/tox.ini +++ b/tox.ini @@ -241,7 +241,7 @@ deps = {[testenv]deps} botocore>=1.4.87 commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} [testenv:botocore-pinned] basepython = {[pinned]basepython} @@ -252,4 +252,4 @@ install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} From 7701e590fbc5ac4d5da8512b07dc4e81e0d9c6c1 Mon Sep 17 00:00:00 2001 From: Rohit Kumar Singh <145501871+Rohitkr117@users.noreply.github.com> Date: Sat, 2 Nov 2024 11:15:27 +0530 Subject: [PATCH 1557/2083] Documentation added for Spider State in extensions.rst (#6522) * Documentation added for Spider State in extensions.rst * Made correction in documentation for Spiderstate * Added appropriate intro for Spider state extension * Added reference for spiderstate extension * Added Spiderstate extension hyperlink refrence in jobs.rst --- docs/topics/extensions.rst | 26 ++++++++++++++++++++++++++ docs/topics/jobs.rst | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 7b34a19d5..9cbc9663d 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -243,6 +243,32 @@ An extension for debugging memory usage. It collects information about: To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The info will be stored in the stats. +.. _topics-extensions-ref-spiderstate: + +Spider state extension +~~~~~~~~~~~~~~~~~~~~~~ + +.. module:: scrapy.extensions.spiderstate + :synopsis: Spider state extension + +.. class:: SpiderState + +Manages spider state data by loading it before a crawl and saving it after. + +Give a value to the :setting:`JOBDIR` setting to enable this extension. +When enabled, this extension manages the :attr:`~scrapy.Spider.state` +attribute of your :class:`~scrapy.Spider` instance: + +- When your spider closes (:signal:`spider_closed`), the contents of its + :attr:`~scrapy.Spider.state` attribute are serialized into a file named + ``spider.state`` in the :setting:`JOBDIR` folder. +- When your spider opens (:signal:`spider_opened`), if a previously-generated + ``spider.state`` file exists in the :setting:`JOBDIR` folder, it is loaded + into the :attr:`~scrapy.Spider.state` attribute. + + +For an example, see :ref:`topics-keeping-persistent-state-between-batches`. + Close spider extension ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index c7fc1ea48..0e705dc64 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -46,7 +46,7 @@ Keeping persistent state between batches Sometimes you'll want to keep some persistent spider state between pause/resume batches. You can use the ``spider.state`` attribute for that, which should be a -dict. There's a built-in extension that takes care of serializing, storing and +dict. There's :ref:`a built-in extension <topics-extensions-ref-spiderstate>` that takes care of serializing, storing and loading that attribute from the job directory, when the spider starts and stops. From ce5a132f12341a4118edb7c8ae3b7c2a27306057 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 4 Nov 2024 15:40:07 +0500 Subject: [PATCH 1558/2083] Run and fix linkcheck. (#6524) --- README.rst | 8 +++--- docs/conf.py | 1 + docs/contributing.rst | 10 ++++---- docs/faq.rst | 11 ++++---- docs/index.rst | 2 +- docs/intro/install.rst | 4 +-- docs/intro/overview.rst | 2 +- docs/intro/tutorial.rst | 4 +-- docs/news.rst | 31 +++++++++++------------ docs/topics/architecture.rst | 6 ++--- docs/topics/broad-crawls.rst | 4 +-- docs/topics/deploy.rst | 2 +- docs/topics/developer-tools.rst | 2 +- docs/topics/downloader-middleware.rst | 12 ++++----- docs/topics/dynamic-content.rst | 7 +++-- docs/topics/extensions.rst | 4 --- docs/topics/feed-exports.rst | 9 +++---- docs/topics/item-pipeline.rst | 2 +- docs/topics/items.rst | 2 +- docs/topics/media-pipeline.rst | 8 +++--- docs/topics/selectors.rst | 8 +++--- docs/topics/settings.rst | 10 ++++---- extras/coverage-report.sh | 2 +- scrapy/downloadermiddlewares/ajaxcrawl.py | 4 +-- scrapy/http/request/__init__.py | 2 +- scrapy/utils/request.py | 8 +++--- scrapy/utils/url.py | 3 +-- tests/test_http_request.py | 2 +- tests/test_pipeline_crawl.py | 4 +-- tests/test_pipeline_images.py | 4 +-- tests/test_pipeline_media.py | 2 +- 31 files changed, 79 insertions(+), 101 deletions(-) diff --git a/README.rst b/README.rst index e640bce35..3f468953e 100644 --- a/README.rst +++ b/README.rst @@ -6,11 +6,11 @@ Scrapy ====== .. image:: https://img.shields.io/pypi/v/Scrapy.svg - :target: https://pypi.python.org/pypi/Scrapy + :target: https://pypi.org/pypi/Scrapy :alt: PyPI Version .. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg - :target: https://pypi.python.org/pypi/Scrapy + :target: https://pypi.org/pypi/Scrapy :alt: Supported Python Versions .. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg @@ -27,7 +27,7 @@ Scrapy :alt: Windows .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg - :target: https://pypi.python.org/pypi/Scrapy + :target: https://pypi.org/pypi/Scrapy :alt: Wheel Status .. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg @@ -111,4 +111,4 @@ See https://scrapy.org/companies/ for a list. Commercial Support ================== -See https://scrapy.org/support/ for details. \ No newline at end of file +See https://scrapy.org/support/ for details. diff --git a/docs/conf.py b/docs/conf.py index 3de50e54e..7a5166053 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -231,6 +231,7 @@ linkcheck_ignore = [ r"http://localhost:\d+", "http://hg.scrapy.org", "http://directory.google.com/", + r"https://github.com/scrapy/scrapy/issues/\d+", ] diff --git a/docs/contributing.rst b/docs/contributing.rst index d728338da..e8ffe83b4 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -154,7 +154,7 @@ by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE`` (replace 'upstream' with a remote name for scrapy repository, ``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE`` with a name of the branch you want to create locally). -See also: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally. +See also: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally. When writing GitHub pull requests, try to keep titles short but descriptive. E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" @@ -182,8 +182,8 @@ Scrapy: * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. - See https://help.github.com/en/github/using-git/setting-your-username-in-git for - setup instructions. + See https://docs.github.com/en/get-started/getting-started-with-git/setting-your-username-in-git + for setup instructions. .. _scrapy-pre-commit: @@ -317,8 +317,8 @@ And their unit-tests are in:: .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues -.. _PEP 257: https://www.python.org/dev/peps/pep-0257/ -.. _pull request: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request +.. _PEP 257: https://peps.python.org/pep-0257/ +.. _pull request: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist .. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 diff --git a/docs/faq.rst b/docs/faq.rst index 0b650f522..f81ec3601 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -23,7 +23,7 @@ comparing `jinja2`_ to `Django`_. .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ .. _lxml: https://lxml.de/ -.. _jinja2: https://palletsprojects.com/p/jinja/ +.. _jinja2: https://palletsprojects.com/projects/jinja/ .. _Django: https://www.djangoproject.com/ Can I use Scrapy with BeautifulSoup? @@ -148,7 +148,7 @@ middleware with a :ref:`custom downloader middleware instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into a complex regular expression. -- If you can `meet the installation requirements`_, use pyre2_ instead of +- If you can meet the installation requirements, use pyre2_ instead of Python’s re_ to compile your URL-filtering regular expression. See :issue:`1908`. @@ -166,9 +166,8 @@ See also `other suggestions at StackOverflow "myproject.middlewares.CustomOffsiteMiddleware": 50, } -.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation .. _pyre2: https://github.com/andreasvc/pyre2 -.. _re: https://docs.python.org/library/re.html +.. _re: https://docs.python.org/3/library/re.html Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- @@ -282,7 +281,7 @@ The ``__VIEWSTATE`` parameter is used in sites built with ASP.NET/VB.NET. For more info on how it works see `this page`_. Also, here's an `example spider`_ which scrapes one of these sites. -.. _this page: https://metacpan.org/pod/release/ECARROLL/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm +.. _this page: https://metacpan.org/release/ECARROLL/HTML-TreeBuilderX-ASP_NET-0.09/view/lib/HTML/TreeBuilderX/ASP_NET.pm .. _example spider: https://github.com/AmbientLighter/rpn-fas/blob/master/fas/spiders/rnp.py What's the best way to parse big XML/CSV data feeds? @@ -432,7 +431,7 @@ See :issue:`2680`. .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 -.. _Python standard library modules: https://docs.python.org/py-modindex.html +.. _Python standard library modules: https://docs.python.org/3/py-modindex.html .. _Python package: https://pypi.org/ .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) diff --git a/docs/index.rst b/docs/index.rst index 8798aebd1..1a9cf636c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -33,7 +33,7 @@ Having trouble? We'd like to help! .. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues -.. _Scrapy Discord: https://discord.gg/mv3yErfpvq +.. _Scrapy Discord: https://discord.com/invite/mv3yErfpvq First steps diff --git a/docs/intro/install.rst b/docs/intro/install.rst index ef541368a..82a0e18c5 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -267,10 +267,10 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_. .. _lxml: https://lxml.de/index.html .. _parsel: https://pypi.org/project/parsel/ .. _w3lib: https://pypi.org/project/w3lib/ -.. _twisted: https://twistedmatrix.com/trac/ +.. _twisted: https://twisted.org/ .. _cryptography: https://cryptography.io/en/latest/ .. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/ -.. _setuptools: https://pypi.python.org/pypi/setuptools +.. _setuptools: https://pypi.org/pypi/setuptools .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ .. _Anaconda: https://docs.anaconda.com/anaconda/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index cd17b1968..d05e46551 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -152,6 +152,6 @@ interest! .. _join the community: https://scrapy.org/community/ .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping -.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html +.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/welcome/ecs .. _Amazon S3: https://aws.amazon.com/s3/ .. _Sitemaps: https://www.sitemaps.org/index.html diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index dd1efd3b3..6e6caebf1 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -369,7 +369,7 @@ recommend `this tutorial to learn XPath through examples <http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how to think in XPath" <http://plasmasturm.org/log/xpath101/>`_. -.. _XPath: https://www.w3.org/TR/xpath/all/ +.. _XPath: https://www.w3.org/TR/xpath-10/ .. _CSS: https://www.w3.org/TR/selectors Extracting quotes and authors @@ -541,7 +541,7 @@ for Item Pipelines has been set up for you when the project is created, in ``tutorial/pipelines.py``. Though you don't need to implement any item pipelines if you just want to store the scraped items. -.. _JSON Lines: http://jsonlines.org +.. _JSON Lines: https://jsonlines.org .. _JQ: https://stedolan.github.io/jq diff --git a/docs/news.rst b/docs/news.rst index 2bbca77cc..3c9e58cca 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1069,7 +1069,7 @@ Documentation (:issue:`3582`, :issue:`5432`). .. _Common Crawl: https://commoncrawl.org/ - .. _Google cache: http://www.googleguide.com/cached_pages.html + .. _Google cache: https://www.googleguide.com/cached_pages.html - The new :ref:`topics-components` topic covers enforcing requirements on Scrapy components, like :ref:`downloader middlewares @@ -1426,7 +1426,7 @@ New features (:setting:`AWS_SESSION_TOKEN`) and endpoint customization (:setting:`AWS_ENDPOINT_URL`). (:issue:`4998`, :issue:`5210`) - .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys + .. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html - New :setting:`LOG_FILE_APPEND` setting to allow truncating the log file. (:issue:`5279`) @@ -1572,7 +1572,7 @@ Documentation - ``quotes.toscrape.com`` references now use HTTPS instead of HTTP. (:issue:`5395`, :issue:`5396`) -- Added a link to `our Discord server <https://discord.gg/mv3yErfpvq>`_ +- Added a link to `our Discord server <https://discord.com/invite/mv3yErfpvq>`_ to :ref:`getting-help`. (:issue:`5421`, :issue:`5422`) - The pronunciation of the project name is now :ref:`officially @@ -1763,7 +1763,7 @@ Bug fixes with lower indentation than the following code. (:issue:`4477`, :issue:`4935`) -- The `Content-Length <https://tools.ietf.org/html/rfc2616#section-14.13>`_ +- The `Content-Length <https://datatracker.ietf.org/doc/html/rfc2616#section-14.13>`_ header is no longer omitted from responses when using the default, HTTP/1.1 download handler (see :setting:`DOWNLOAD_HANDLERS`). (:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`) @@ -2263,7 +2263,7 @@ Documentation * Simplified the code example in :ref:`topics-loaders-dataclass` (:issue:`4652`) -.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT +.. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format Quality assurance @@ -2490,7 +2490,7 @@ Quality assurance * Added a `Pylint <https://www.pylint.org/>`_ job to Travis CI (:issue:`3727`) -* Added a `Mypy <http://mypy-lang.org/>`_ job to Travis CI (:issue:`4637`) +* Added a `Mypy <https://mypy-lang.org/>`_ job to Travis CI (:issue:`4637`) * Made use of set literals in tests (:issue:`4573`) @@ -2997,7 +2997,7 @@ Quality assurance * Cleaned up code (:issue:`3937`, :issue:`4208`, :issue:`4209`, :issue:`4210`, :issue:`4212`, :issue:`4369`, :issue:`4376`, :issue:`4378`) -.. _Bandit: https://bandit.readthedocs.io/ +.. _Bandit: https://bandit.readthedocs.io/en/latest/ .. _Flake8: https://flake8.pycqa.org/en/latest/ @@ -4172,7 +4172,7 @@ Docs - Update Contributing docs, document new support channels (:issue:`2762`, issue:`3038`) - Include references to Scrapy subreddit in the docs -- Fix broken links; use https:// for external links +- Fix broken links; use ``https://`` for external links (:issue:`2978`, :issue:`2982`, :issue:`2958`) - Document CloseSpider extension better (:issue:`2759`) - Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example @@ -4773,7 +4773,7 @@ This 1.1 release brings a lot of interesting features and bug fixes: - Don't retry bad requests (HTTP 400) by default (:issue:`1289`). If you need the old behavior, add ``400`` to :setting:`RETRY_HTTP_CODES`. - Fix shell files argument handling (:issue:`1710`, :issue:`1550`). - If you try ``scrapy shell index.html`` it will try to load the URL http://index.html, + If you try ``scrapy shell index.html`` it will try to load the URL ``http://index.html``, use ``scrapy shell ./index.html`` to load a local file. - Robots.txt compliance is now enabled by default for newly-created projects (:issue:`1724`). Scrapy will also wait for robots.txt to be downloaded @@ -5449,7 +5449,7 @@ Scrapy 0.24.5 (2015-02-25) Scrapy 0.24.4 (2014-08-09) -------------------------- -- pem file is used by mockserver and required by scrapy bench (:commit:`5eddc68`) +- pem file is used by mockserver and required by scrapy bench (:commit:`5eddc68b63`) - scrapy bench needs scrapy.tests* (:commit:`d6cb999`) Scrapy 0.24.3 (2014-08-09) @@ -5970,7 +5970,7 @@ Scrapy changes: - nested items now fully supported in JSON and JSONLines exporters - added :reqmeta:`cookiejar` Request meta key to support multiple cookie sessions per spider - decoupled encoding detection code to `w3lib.encoding`_, and ported Scrapy code to use that module -- dropped support for Python 2.5. See https://blog.scrapinghub.com/2012/02/27/scrapy-0-15-dropping-support-for-python-2-5/ +- dropped support for Python 2.5. See https://www.zyte.com/blog/scrapy-0-15-dropping-support-for-python-2-5/ - dropped support for Twisted 2.5 - added :setting:`REFERER_ENABLED` setting, to control referer middleware - changed default user agent to: ``Scrapy/VERSION (+http://scrapy.org)`` @@ -6048,7 +6048,7 @@ Scrapy 0.14 New features and settings ~~~~~~~~~~~~~~~~~~~~~~~~~ -- Support for `AJAX crawlable urls`_ +- Support for AJAX crawlable urls - New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`) - added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``) - Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`) @@ -6319,11 +6319,10 @@ Scrapy 0.7 First release of Scrapy. -.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 .. _boto3: https://github.com/boto/boto3 .. _botocore: https://github.com/boto/botocore .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding -.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ +.. _ClientForm: https://pypi.org/project/ClientForm/ .. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request .. _cryptography: https://cryptography.io/en/latest/ .. _docstrings: https://docs.python.org/3/glossary.html#term-docstring @@ -6335,7 +6334,7 @@ First release of Scrapy. .. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr -.. _PEP 257: https://www.python.org/dev/peps/pep-0257/ +.. _PEP 257: https://peps.python.org/pep-0257/ .. _Pillow: https://python-pillow.org/ .. _pyOpenSSL: https://www.pyopenssl.org/en/stable/ .. _queuelib: https://github.com/scrapy/queuelib @@ -6347,7 +6346,7 @@ First release of Scrapy. .. _service_identity: https://service-identity.readthedocs.io/en/stable/ .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ -.. _Twisted: https://twistedmatrix.com/trac/ +.. _Twisted: https://twisted.org/ .. _w3lib: https://github.com/scrapy/w3lib .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py .. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 0c3a7ed88..0370dc538 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -168,9 +168,7 @@ For more information about asynchronous programming and Twisted see these links: * :doc:`twisted:core/howto/defer-intro` -* `Twisted - hello, asynchronous programming`_ * `Twisted Introduction - Krondo`_ -.. _Twisted: https://twistedmatrix.com/trac/ -.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ -.. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/ +.. _Twisted: https://twisted.org/ +.. _Twisted Introduction - Krondo: https://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/ diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 750aae554..0286c3354 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -186,7 +186,7 @@ Enable crawling of "Ajax Crawlable Pages" ========================================= Some pages (up to 1%, based on empirical data from year 2013) declare -themselves as `ajax crawlable`_. This means they provide plain HTML +themselves as ajax crawlable. This means they provide plain HTML version of content that is usually available only via AJAX. Pages can indicate it in two ways: @@ -206,8 +206,6 @@ AjaxCrawlMiddleware helps to crawl them correctly. It is turned OFF by default because it has some performance overhead, and enabling it for focused crawls doesn't make much sense. -.. _ajax crawlable: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started - .. _broad-crawls-bfo: Crawl in BFO order diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index 961d6dc01..f3515b4be 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -54,6 +54,6 @@ just like ``scrapyd-deploy``. .. _scrapyd-client: https://github.com/scrapy/scrapyd-client .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html .. _shub: https://shub.readthedocs.io/en/latest/ -.. _Zyte: https://zyte.com/ +.. _Zyte: https://www.zyte.com/ .. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/ .. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index a15ee1059..89a4d32d8 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -278,7 +278,7 @@ into our ``url``. In more complex websites, it could be difficult to easily reproduce the requests, as we could need to add ``headers`` or ``cookies`` to make it work. -In those cases you can export the requests in `cURL <https://curl.haxx.se/>`_ +In those cases you can export the requests in `cURL <https://curl.se/>`_ format, by right-clicking on each of them in the network tool and using the :meth:`~scrapy.Request.from_curl()` method to generate an equivalent request: diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 13064ccdd..b184a629e 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1105,7 +1105,7 @@ Parsers vary in several aspects: * Support for wildcard matching -* Usage of `length based rule <https://developers.google.com/search/reference/robots_txt#order-of-precedence-for-group-member-lines>`_: +* Usage of `length based rule <https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt#order-of-precedence-for-rules>`_: in particular for ``Allow`` and ``Disallow`` directives, where the most specific rule based on the length of the path trumps the less specific (shorter) rule @@ -1123,7 +1123,7 @@ Based on `Protego <https://github.com/scrapy/protego>`_: * implemented in Python * is compliant with `Google's Robots.txt Specification - <https://developers.google.com/search/reference/robots_txt>`_ + <https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt>`_ * supports wildcard matching @@ -1158,7 +1158,7 @@ In order to use this parser, set: Robotexclusionrulesparser ~~~~~~~~~~~~~~~~~~~~~~~~~ -Based on `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_: +Based on `Robotexclusionrulesparser <https://pypi.org/project/robotexclusionrulesparser/>`_: * implemented in Python @@ -1171,7 +1171,7 @@ Based on `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_: In order to use this parser: -* Install `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ by running +* Install ``Robotexclusionrulesparser`` by running ``pip install robotexclusionrulesparser`` * Set :setting:`ROBOTSTXT_PARSER` setting to @@ -1231,9 +1231,7 @@ AjaxCrawlMiddleware .. class:: AjaxCrawlMiddleware Middleware that finds 'AJAX crawlable' page variants based - on meta-fragment html tag. See - https://developers.google.com/search/docs/ajax-crawling/docs/getting-started - for more info. + on meta-fragment html tag. .. note:: diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index a99f1e222..75d980835 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -85,9 +85,8 @@ It might be enough to yield a :class:`~scrapy.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and form parameters (see :class:`~scrapy.FormRequest`) of that request. -As all major browsers allow to export the requests in `cURL -<https://curl.haxx.se/>`_ format, Scrapy incorporates the method -:meth:`~scrapy.Request.from_curl()` to generate an equivalent +As all major browsers allow to export the requests in curl_ format, Scrapy +incorporates the method :meth:`~scrapy.Request.from_curl()` to generate an equivalent :class:`~scrapy.Request` from a cURL command. To get more information visit :ref:`request from curl <requests-from-curl>` inside the network tool section. @@ -289,7 +288,7 @@ We recommend using `scrapy-playwright`_ for a better integration. .. _JavaScript: https://en.wikipedia.org/wiki/JavaScript .. _Splash: https://github.com/scrapinghub/splash .. _chompjs: https://github.com/Nykakin/chompjs -.. _curl: https://curl.haxx.se/ +.. _curl: https://curl.se/ .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser .. _js2xml: https://github.com/scrapinghub/js2xml .. _playwright-python: https://github.com/microsoft/playwright-python diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 9cbc9663d..c47a3226a 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -546,8 +546,4 @@ Invokes a :doc:`Python debugger <library/pdb>` inside a running Scrapy process w signal is received. After the debugger is exited, the Scrapy process continues running normally. -For more info see `Debugging in Python`_. - This extension only works on POSIX-compliant platforms (i.e. not Windows). - -.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/ diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 922b765db..07a3f3678 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -213,7 +213,7 @@ passed through the following settings: - :setting:`AWS_SECRET_ACCESS_KEY` - :setting:`AWS_SESSION_TOKEN` (only needed for `temporary security credentials`_) -.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys +.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html You can also define a custom ACL, custom endpoint, and region name for exported feeds using these settings: @@ -248,7 +248,7 @@ The feeds are stored on `Google Cloud Storage`_. - Required external libraries: `google-cloud-storage`_. -For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_. +For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication>`_. You can set a *Project ID* and *Access Control List (ACL)* through the following settings: @@ -516,8 +516,7 @@ as a fallback value if that key is not provided for a specific feed definition: .. note:: Some FTP servers may not support appending to files (the ``APPE`` FTP command). - - :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported - <https://forums.aws.amazon.com/message.jspa?messageID=540395>`_) + - :ref:`topics-feed-storage-s3`: ``True`` (appending is not supported) - :ref:`topics-feed-storage-gcs`: ``True`` (appending is not supported) @@ -816,5 +815,5 @@ source spider in the feed URI: .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ .. _boto3: https://github.com/boto/boto3 -.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl .. _Google Cloud Storage: https://cloud.google.com/storage/ diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 58c922e0d..310f153e8 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -175,7 +175,7 @@ method and how to clean up the resources properly. return item .. _MongoDB: https://www.mongodb.com/ -.. _pymongo: https://api.mongodb.com/python/current/ +.. _pymongo: https://pymongo.readthedocs.io/en/stable/ .. _ScreenshotPipeline: diff --git a/docs/topics/items.rst b/docs/topics/items.rst index f13a7b5b1..39a95815c 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -221,7 +221,7 @@ the :attr:`Item.fields` attribute. `attr.ib`_ for additional information. .. _dataclasses.field: https://docs.python.org/3/library/dataclasses.html#dataclasses.field - .. _attr.ib: https://www.attrs.org/en/stable/api.html#attr.ib + .. _attr.ib: https://www.attrs.org/en/stable/api-attr.html#attr.ib Working with Item objects diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index c96dd0f99..f086a943e 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -261,7 +261,7 @@ policy: For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. You can also use other S3-like storages. Storages like self-hosted `Minio`_ or -`s3.scality`_. All you need to do is set endpoint option in you Scrapy +`Zenko CloudServer`_. All you need to do is set endpoint option in you Scrapy settings: .. code-block:: python @@ -276,9 +276,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S AWS_VERIFY = False # or True (None by default) .. _botocore: https://github.com/boto/botocore -.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl .. _Minio: https://github.com/minio/minio -.. _s3.scality: https://s3.scality.com/ +.. _Zenko CloudServer: https://www.zenko.io/cloudserver/ .. _media-pipeline-gcs: @@ -303,7 +303,7 @@ For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_I For information about authentication, see this `documentation`_. -.. _documentation: https://cloud.google.com/docs/authentication/production +.. _documentation: https://cloud.google.com/docs/authentication You can modify the Access Control List (ACL) policy used for the stored files, which is defined by the :setting:`FILES_STORE_GCS_ACL` and diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 0aae41cc8..202b0823a 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -591,7 +591,7 @@ Another common case would be to extract all direct ``<p>`` children: For more details about relative XPaths see the `Location Paths`_ section in the XPath specification. -.. _Location Paths: https://www.w3.org/TR/xpath/all/#location-paths +.. _Location Paths: https://www.w3.org/TR/xpath-10/#location-paths When querying by class, consider using CSS ------------------------------------------ @@ -727,7 +727,7 @@ But using the ``.`` to mean the node, works: >>> sel.xpath("//a[contains(., 'Next Page')]").getall() ['<a href="#">Click here to go to the <strong>Next Page</strong></a>'] -.. _`XPath string function`: https://www.w3.org/TR/xpath/all/#section-String-Functions +.. _`XPath string function`: https://www.w3.org/TR/xpath-10/#section-String-Functions .. _topics-selectors-xpath-variables: @@ -801,8 +801,8 @@ This is how the file starts:: ... You can see several namespace declarations including a default -"http://www.w3.org/2005/Atom" and another one using the "gd:" prefix for -"http://schemas.google.com/g/2005". +``"http://www.w3.org/2005/Atom"`` and another one using the ``gd:`` prefix for +``"http://schemas.google.com/g/2005"``. .. highlight:: python diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 02fca7ff4..116e8226e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -288,7 +288,7 @@ The AWS security token used by code that requires access to `Amazon Web services such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using `temporary security credentials`_. -.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys +.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html .. setting:: AWS_ENDPOINT_URL @@ -617,7 +617,7 @@ necessary to access certain HTTPS websites: for example, you may need to use ``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a specific cipher that is not included in ``DEFAULT`` if a website requires it. -.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT +.. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format .. setting:: DOWNLOADER_CLIENT_TLS_METHOD @@ -829,9 +829,9 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2: - No support for the :signal:`bytes_received` and :signal:`headers_received` signals. -.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 +.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption -.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 +.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2 .. setting:: DOWNLOAD_SLOTS @@ -1074,7 +1074,7 @@ in ``Request`` meta. some FTP servers explicitly ask for the user's e-mail address and will not allow login with the "guest" password. -.. _RFC 1635: https://tools.ietf.org/html/rfc1635 +.. _RFC 1635: https://datatracker.ietf.org/doc/html/rfc1635 .. reqmeta:: ftp_user .. setting:: FTP_USER diff --git a/extras/coverage-report.sh b/extras/coverage-report.sh index 842d0e46e..7eaa214cf 100755 --- a/extras/coverage-report.sh +++ b/extras/coverage-report.sh @@ -1,6 +1,6 @@ # Run tests, generate coverage report and open it on a browser # -# Requires: coverage 3.3 or above from https://pypi.python.org/pypi/coverage +# Requires: coverage 3.3 or above from https://pypi.org/pypi/coverage coverage run --branch $(which trial) --reporter=text tests coverage html -i diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index b813baf86..166192b4f 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -24,7 +24,6 @@ logger = logging.getLogger(__name__) class AjaxCrawlMiddleware: """ Handle 'AJAX crawlable' pages marked as crawlable via meta tag. - For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. """ def __init__(self, settings: BaseSettings): @@ -70,8 +69,7 @@ class AjaxCrawlMiddleware: def _has_ajax_crawlable_variant(self, response: Response) -> bool: """ - Return True if a page without hash fragment could be "AJAX crawlable" - according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. + Return True if a page without hash fragment could be "AJAX crawlable". """ body = response.text[: self.lookup_bytes] return _has_ajaxcrawlable_meta(body) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index ed225555c..9c29ea4d1 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -222,7 +222,7 @@ class Request(object_ref): **kwargs: Any, ) -> Self: """Create a Request object from a string containing a `cURL - <https://curl.haxx.se/>`_ command. It populates the HTTP method, the + <https://curl.se/>`_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same arguments as the :class:`Request` class, taking preference and overriding the values of the same arguments contained in the cURL diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index e80cbbb89..7848b9318 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -46,17 +46,15 @@ def fingerprint( The request fingerprint is a hash that uniquely identifies the resource the request points to. For example, take the following two urls: - - http://www.example.com/query?id=111&cat=222 - http://www.example.com/query?cat=222&id=111 + ``http://www.example.com/query?id=111&cat=222``, + ``http://www.example.com/query?cat=222&id=111``. Even though those are two different URLs both point to the same resource and are equivalent (i.e. they should return the same response). Another example are cookies used to store session ids. Suppose the following page is only accessible to authenticated users: - - http://www.example.com/members/offers.html + ``http://www.example.com/members/offers.html``. Lots of sites use a cookie to store the session id, which adds a random component to the HTTP Request and thus should be ignored when calculating diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index e0a2973f7..9dc177cf1 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -61,8 +61,7 @@ def parse_url(url: UrlT, encoding: str | None = None) -> ParseResult: def escape_ajax(url: str) -> str: """ - Return the crawlable url according to: - https://developers.google.com/webmasters/ajax-crawling/docs/getting-started + Return the crawlable url >>> escape_ajax("www.example.com/ajax.html#!key=value") 'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue' diff --git a/tests/test_http_request.py b/tests/test_http_request.py index d0fb17f1f..9997b7ab3 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -143,7 +143,7 @@ class RequestTest(unittest.TestCase): # percent-escaping sequences that do not match valid UTF-8 sequences # should be kept untouched (just upper-cased perhaps) # - # See https://tools.ietf.org/html/rfc3987#section-3.2 + # See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2 # # "Conversions from URIs to IRIs MUST NOT use any character encoding # other than UTF-8 in steps 3 and 4, even if it might be possible to diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 696ef8cab..7add27aa7 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -220,9 +220,7 @@ skip_pillow: str | None try: from PIL import Image # noqa: imported just to check for the import error except ImportError: - skip_pillow = ( - "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" - ) + skip_pillow = "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow" else: skip_pillow = None diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 7561e1fd4..dfeead999 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -19,9 +19,7 @@ skip_pillow: str | None try: from PIL import Image except ImportError: - skip_pillow = ( - "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" - ) + skip_pillow = "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow" else: encoders = {"jpeg_encoder", "jpeg_decoder"} if not encoders.issubset(set(Image.core.__dict__)): # type: ignore[attr-defined] diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 0faf6d015..c979e45d7 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -21,7 +21,7 @@ try: from PIL import Image # noqa: imported just to check for the import error except ImportError: skip_pillow: str | None = ( - "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow" ) else: skip_pillow = None From e7f5ae0b34ef87503884967f8b6c031d3f213c3e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 4 Nov 2024 16:17:56 +0500 Subject: [PATCH 1559/2083] Update the outdated Item docstring. (#6427) * Update the outdated Item doscstring. * Fix the reference links in items.html. --- docs/topics/items.rst | 47 ++++++++++++++++--------------------------- scrapy/item.py | 21 ++++++++++--------- 2 files changed, 29 insertions(+), 39 deletions(-) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 39a95815c..7cc476863 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -42,39 +42,27 @@ Item objects :class:`Item` provides a :class:`dict`-like API plus additional features that make it the most feature-complete item type: -.. class:: scrapy.item.Item([arg]) -.. class:: scrapy.Item([arg]) +.. autoclass:: scrapy.Item + :members: copy, deepcopy, fields + :undoc-members: - :class:`Item` objects replicate the standard :class:`dict` API, including - its ``__init__`` method. +:class:`Item` objects replicate the standard :class:`dict` API, including +its ``__init__`` method. - :class:`Item` allows the defining of field names, so that: +:class:`Item` allows the defining of field names, so that: - - :class:`KeyError` is raised when using undefined field names (i.e. - prevents typos going unnoticed) +- :class:`KeyError` is raised when using undefined field names (i.e. + prevents typos going unnoticed) - - :ref:`Item exporters <topics-exporters>` can export all fields by - default even if the first scraped object does not have values for all - of them +- :ref:`Item exporters <topics-exporters>` can export all fields by + default even if the first scraped object does not have values for all + of them - :class:`Item` also allows the defining of field metadata, which can be used to - :ref:`customize serialization <topics-exporters-field-serialization>`. +:class:`Item` also allows the defining of field metadata, which can be used to +:ref:`customize serialization <topics-exporters-field-serialization>`. - :mod:`trackref` tracks :class:`Item` objects to help find memory leaks - (see :ref:`topics-leaks-trackrefs`). - - :class:`Item` objects also provide the following additional API members: - - .. automethod:: copy - - .. automethod:: deepcopy - - .. attribute:: fields - - A dictionary containing *all declared fields* for this Item, not only - those populated. The keys are the field names and the values are the - :class:`Field` objects used in the :ref:`Item declaration - <topics-items-declaring>`. +:mod:`trackref` tracks :class:`Item` objects to help find memory leaks +(see :ref:`topics-leaks-trackrefs`). Example: @@ -205,10 +193,9 @@ documentation to see which metadata keys are used by each component. It's important to note that the :class:`Field` objects used to declare the item do not stay assigned as class attributes. Instead, they can be accessed through -the :attr:`Item.fields` attribute. +the :attr:`~scrapy.Item.fields` attribute. -.. class:: scrapy.item.Field([arg]) -.. class:: scrapy.Field([arg]) +.. autoclass:: scrapy.Field The :class:`Field` class is just an alias to the built-in :class:`dict` class and doesn't provide any extra functionality or attributes. In other words, diff --git a/scrapy/item.py b/scrapy/item.py index f77002d18..1cc0ae584 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -55,16 +55,13 @@ class ItemMeta(ABCMeta): class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): - """ - Base class for scraped items. + """Base class for scraped items. - In Scrapy, an object is considered an ``item`` if it is an instance of either - :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a - spider callback is evaluated, only instances of :class:`Item` or - :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`. - - If you need instances of a custom class to be considered items by Scrapy, - you must inherit from either :class:`Item` or :class:`dict`. + In Scrapy, an object is considered an ``item`` if it's supported by the + `itemadapter`_ library. For example, when the output of a spider callback + is evaluated, only such objects are passed to :ref:`item pipelines + <topics-item-pipeline>`. :class:`Item` is one of the classes supported by + `itemadapter`_ by default. Items must declare :class:`Field` attributes, which are processed and stored in the ``fields`` attribute. This restricts the set of allowed field names @@ -75,8 +72,14 @@ class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): Unlike instances of :class:`dict`, instances of :class:`Item` may be :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks. + + .. _itemadapter: https://github.com/scrapy/itemadapter """ + #: A dictionary containing *all declared fields* for this Item, not only + #: those populated. The keys are the field names and the values are the + #: :class:`Field` objects used in the :ref:`Item declaration + #: <topics-items-declaring>`. fields: dict[str, Field] def __init__(self, *args: Any, **kwargs: Any): From d2156696c45e023479ae1bdee8623bb6212e975c Mon Sep 17 00:00:00 2001 From: Rohit Kumar Singh <145501871+Rohitkr117@users.noreply.github.com> Date: Mon, 4 Nov 2024 21:39:45 +0530 Subject: [PATCH 1560/2083] Deprecate unused scrapy utils (#6519) * Added deprecation warnings for unused Scrapy.utils * Grammatical corrections * Exceptions class connected * Deprecation of ScrapyJSONDecoder * request_authenticate function deprecation * Making all warning similar * Added ignore statements for deprecation warning in tests * Missing stacklevel attr. added * Added Deprecation message --- scrapy/utils/misc.py | 2 +- scrapy/utils/python.py | 17 +++++++++++++++++ scrapy/utils/request.py | 7 ++++++- scrapy/utils/serialize.py | 10 +++++++++- scrapy/utils/test.py | 7 +++++++ tests/test_utils_python.py | 2 ++ tests/test_utils_request.py | 3 +++ 7 files changed, 45 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 516218347..12c09839f 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -111,7 +111,7 @@ def md5sum(file: IO[bytes]) -> str: """ warnings.warn( ( - "The scrapy.utils.misc.md5sum function is deprecated, and will be " + "The scrapy.utils.misc.md5sum function is deprecated and will be " "removed in a future version of Scrapy." ), ScrapyDeprecationWarning, diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 6268af728..d970f5da5 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,12 +8,14 @@ import gc import inspect import re import sys +import warnings import weakref from collections.abc import AsyncIterable, Iterable, Mapping from functools import partial, wraps from itertools import chain from typing import TYPE_CHECKING, Any, TypeVar, overload +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: @@ -47,6 +49,11 @@ def flatten(x: Iterable[Any]) -> list[Any]: >>> flatten(["foo", ["baz", 42], "bar"]) ['foo', 'baz', 42, 'bar'] """ + warnings.warn( + "The flatten function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) return list(iflatten(x)) @@ -54,6 +61,11 @@ def iflatten(x: Iterable[Any]) -> Iterable[Any]: """iflatten(sequence) -> iterator Similar to ``.flatten()``, but returns iterator instead""" + warnings.warn( + "The iflatten function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) for el in x: if is_listlike(el): yield from iflatten(el) @@ -272,6 +284,11 @@ def equal_attributes( obj1: Any, obj2: Any, attributes: list[str | Callable[[Any], Any]] | None ) -> bool: """Compare two objects attributes""" + warnings.warn( + "The equal_attributes function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) # not attributes given return False by default if not attributes: return False diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 7848b9318..20e3151da 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -130,7 +130,7 @@ class RequestFingerprinter: if implementation != "SENTINEL": message = ( "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n" - "And it will be removed in future version of Scrapy." + "It will be removed in a future version of Scrapy." ) warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) self._fingerprint = fingerprint @@ -147,6 +147,11 @@ def request_authenticate( """Authenticate the given request (in place) using the HTTP basic access authentication mechanism (RFC 2617) and the given username and password """ + warnings.warn( + "The request_authenticate function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) request.headers["Authorization"] = basic_auth_header(username, password) diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 3b4f67f00..308e351c6 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -1,11 +1,13 @@ import datetime import decimal import json +import warnings from typing import Any from itemadapter import ItemAdapter, is_item from twisted.internet import defer +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response @@ -36,4 +38,10 @@ class ScrapyJSONEncoder(json.JSONEncoder): class ScrapyJSONDecoder(json.JSONDecoder): - pass + def __init__(self, *args, **kwargs): + warnings.warn( + "The ScrapyJSONDecoder class is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + super().__init__(*args, **kwargs) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index d65f2a76d..92b73a91a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio import os +import warnings from importlib import import_module from pathlib import Path from posixpath import split @@ -16,6 +17,7 @@ from twisted.trial.unittest import SkipTest from scrapy import Spider from scrapy.crawler import Crawler +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available if TYPE_CHECKING: @@ -125,6 +127,11 @@ def assert_samelines( """Asserts text1 and text2 have the same lines, ignoring differences in line endings between platforms """ + warnings.warn( + "The assert_samelines function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 5681ff9a4..f80f2517a 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -3,6 +3,7 @@ import operator import platform import sys +import pytest from twisted.trial import unittest from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen @@ -151,6 +152,7 @@ class BinaryIsTextTest(unittest.TestCase): class UtilsPythonTestCase(unittest.TestCase): + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_equal_attributes(self): class Obj: pass diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 7156b13d0..965d050a4 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -6,6 +6,8 @@ import warnings from hashlib import sha1 from weakref import WeakKeyDictionary +import pytest + from scrapy.http import Request from scrapy.utils.python import to_bytes from scrapy.utils.request import ( @@ -19,6 +21,7 @@ from scrapy.utils.test import get_crawler class UtilsRequestTest(unittest.TestCase): + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_request_authenticate(self): r = Request("http://www.example.com") request_authenticate(r, "someuser", "somepass") From f57fc454beb4d7746002bb69457cf8add6cc3bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 5 Nov 2024 19:44:30 +0100 Subject: [PATCH 1561/2083] Replace Slot.throttle with Request.meta['dont_throttle'] --- docs/topics/autothrottle.rst | 15 ++++++++++----- docs/topics/request-response.rst | 1 + docs/topics/settings.rst | 10 +--------- scrapy/core/downloader/__init__.py | 10 ++-------- scrapy/extensions/throttle.py | 6 +++++- tests/test_core_downloader.py | 2 +- tests/test_downloaderslotssettings.py | 1 - tests/test_extension_throttle.py | 23 ++++++++++++----------- 8 files changed, 32 insertions(+), 36 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 8a13b8976..fbfdd0647 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -47,12 +47,17 @@ effect, but there are some important differences: AutoThrottle doesn't have these issues. -Disabling throttling on a downloader slot -========================================= +.. reqmeta:: dont_throttle -It is possible to disable AutoThrottle for a specific download slot at run time -by setting its ``throttle`` attribute to ``False``, e.g. using -:setting:`DOWNLOAD_SLOTS`. +Disabling the throttling of a request +===================================== + +To disable AutoThrottle for a specific request, set the ``dont_throttle`` +request metadata key to ``True``: + +.. code-block:: python + + yield Request("https://example.com", meta={"dont_throttle": True}) Note, however, that AutoThrottle still determines the starting delay of every slot by setting the ``download_delay`` attribute on the running spider. You diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 3c2843bc1..18b5cbdd0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -675,6 +675,7 @@ Those are: * :reqmeta:`dont_obey_robotstxt` * :reqmeta:`dont_redirect` * :reqmeta:`dont_retry` +* :reqmeta:`dont_throttle` * :reqmeta:`download_fail_on_dataloss` * :reqmeta:`download_latency` * :reqmeta:`download_maxsize` diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 116e8226e..cce4a7b3e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -845,12 +845,7 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: .. code-block:: python DOWNLOAD_SLOTS = { - "quotes.toscrape.com": { - "concurrency": 1, - "delay": 2, - "randomize_delay": False, - "throttle": False, - }, + "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, } @@ -862,9 +857,6 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` - There is no global setting for ``throttle``, whose default value is - ``None``. - .. setting:: DOWNLOAD_TIMEOUT diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 1cc0422b7..5040741e2 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -36,13 +36,10 @@ class Slot: concurrency: int, delay: float, randomize_delay: bool, - *, - throttle: bool | None = None, ): self.concurrency: int = concurrency self.delay: float = delay self.randomize_delay: bool = randomize_delay - self.throttle = throttle self.active: set[Request] = set() self.queue: deque[tuple[Request, Deferred[Response]]] = deque() @@ -67,15 +64,13 @@ class Slot: return ( f"{cls_name}(concurrency={self.concurrency!r}, " f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r}, " - f"throttle={self.throttle!r})" + f"randomize_delay={self.randomize_delay!r})" ) def __str__(self) -> str: return ( f"<downloader.Slot concurrency={self.concurrency!r} " f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} " - f"throttle={self.throttle!r} " f"len(active)={len(self.active)} len(queue)={len(self.queue)} " f"len(transferring)={len(self.transferring)} " f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>" @@ -146,8 +141,7 @@ class Downloader: slot_settings.get("delay", delay), ) randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) - throttle = slot_settings.get("throttle", None) - new_slot = Slot(conc, delay, randomize_delay, throttle=throttle) + new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot return key, self.slots[key] diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index d4b4f0e9d..fbac48b1e 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -64,7 +64,11 @@ class AutoThrottle: ) -> None: key, slot = self._get_slot(request, spider) latency = request.meta.get("download_latency") - if latency is None or slot is None or slot.throttle is False: + if ( + latency is None + or slot is None + or request.meta.get("dont_throttle", False) is True + ): return olddelay = slot.delay diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 81cff4947..d929a9369 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -8,5 +8,5 @@ class SlotTest(unittest.TestCase): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) self.assertEqual( repr(slot), - "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)", + "Slot(concurrency=8, delay=0.10, randomize_delay=True)", ) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index ea8c5b4f0..55f9ecac9 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -80,7 +80,6 @@ def test_params(): "concurrency": 1, "delay": 2, "randomize_delay": False, - "throttle": False, } settings = { "DOWNLOAD_SLOTS": { diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index 722a05c26..602b48e78 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -157,17 +157,20 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) @pytest.mark.parametrize( - ("meta", "slot", "throttle"), + ("meta", "slot"), ( - ({}, None, None), - ({"download_latency": 1.0}, None, None), - ({"download_slot": "foo"}, None, None), - ({"download_slot": "foo"}, "foo", None), - ({"download_latency": 1.0, "download_slot": "foo"}, None, None), - ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False), + ({}, None), + ({"download_latency": 1.0}, None), + ({"download_slot": "foo"}, None), + ({"download_slot": "foo"}, "foo"), + ({"download_latency": 1.0, "download_slot": "foo"}, None), + ( + {"download_latency": 1.0, "download_slot": "foo", "dont_throttle": True}, + "foo", + ), ), ) -def test_skipped(meta, slot, throttle): +def test_skipped(meta, slot): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) spider = TestSpider() @@ -178,9 +181,7 @@ def test_skipped(meta, slot, throttle): crawler.engine.downloader = Mock() crawler.engine.downloader.slots = {} if slot is not None: - _slot = Mock() - _slot.throttle = throttle - crawler.engine.downloader.slots[slot] = _slot + crawler.engine.downloader.slots[slot] = object() at._adjust_delay = None # Raise exception if called. at._response_downloaded(None, request, spider) From 5862216bb1c4717b5f4eebe8c410ad8cef60c6d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 5 Nov 2024 19:55:28 +0100 Subject: [PATCH 1562/2083] Fix docs example --- docs/topics/autothrottle.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index fbfdd0647..9f9114e83 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -57,7 +57,7 @@ request metadata key to ``True``: .. code-block:: python - yield Request("https://example.com", meta={"dont_throttle": True}) + Request("https://example.com", meta={"dont_throttle": True}) Note, however, that AutoThrottle still determines the starting delay of every slot by setting the ``download_delay`` attribute on the running spider. You From b244ea7ac028e2aae69d7014a808d49fa26d7c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 5 Nov 2024 20:05:58 +0100 Subject: [PATCH 1563/2083] Add the missing import to the docs example --- docs/topics/autothrottle.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 9f9114e83..48d742f63 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -57,6 +57,8 @@ request metadata key to ``True``: .. code-block:: python + from scrapy import Request + Request("https://example.com", meta={"dont_throttle": True}) Note, however, that AutoThrottle still determines the starting delay of every From 2a4b7fe0f8b2e1ce8c43998aad503f2b0b68495b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 7 Nov 2024 16:17:16 +0100 Subject: [PATCH 1564/2083] =?UTF-8?q?dont=5Fthrottle=20=E2=86=92=20autothr?= =?UTF-8?q?ottle=5Fdont=5Fadjust=5Fdelay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/autothrottle.rst | 8 ++++---- docs/topics/request-response.rst | 2 +- scrapy/extensions/throttle.py | 2 +- tests/test_extension_throttle.py | 6 +++++- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 48d742f63..cfd6440f2 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -47,19 +47,19 @@ effect, but there are some important differences: AutoThrottle doesn't have these issues. -.. reqmeta:: dont_throttle +.. reqmeta:: autothrottle_dont_adjust_delay Disabling the throttling of a request ===================================== -To disable AutoThrottle for a specific request, set the ``dont_throttle`` -request metadata key to ``True``: +To disable AutoThrottle for a specific request, set the +``autothrottle_dont_adjust_delay`` request metadata key to ``True``: .. code-block:: python from scrapy import Request - Request("https://example.com", meta={"dont_throttle": True}) + Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) Note, however, that AutoThrottle still determines the starting delay of every slot by setting the ``download_delay`` attribute on the running spider. You diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 18b5cbdd0..7c15b67e8 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -668,6 +668,7 @@ are some special keys recognized by Scrapy and its built-in extensions. Those are: +* :reqmeta:`autothrottle_dont_adjust_delay` * :reqmeta:`bindaddress` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` @@ -675,7 +676,6 @@ Those are: * :reqmeta:`dont_obey_robotstxt` * :reqmeta:`dont_redirect` * :reqmeta:`dont_retry` -* :reqmeta:`dont_throttle` * :reqmeta:`download_fail_on_dataloss` * :reqmeta:`download_latency` * :reqmeta:`download_maxsize` diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index fbac48b1e..cdb0671ae 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -67,7 +67,7 @@ class AutoThrottle: if ( latency is None or slot is None - or request.meta.get("dont_throttle", False) is True + or request.meta.get("autothrottle_dont_adjust_delay", False) is True ): return diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index 602b48e78..f2c9dc063 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -165,7 +165,11 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) ({"download_slot": "foo"}, "foo"), ({"download_latency": 1.0, "download_slot": "foo"}, None), ( - {"download_latency": 1.0, "download_slot": "foo", "dont_throttle": True}, + { + "download_latency": 1.0, + "download_slot": "foo", + "autothrottle_dont_adjust_delay": True, + }, "foo", ), ), From dc3ebb6cf76daa1953418af5aae3b83ffc12d02a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 7 Nov 2024 16:38:48 +0100 Subject: [PATCH 1565/2083] Refactor the docs --- docs/topics/autothrottle.rst | 57 ++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index cfd6440f2..5bd72fa15 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -21,9 +21,14 @@ Design goals How it works ============ -AutoThrottle extension adjusts download delays dynamically to make spider send -:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent requests on average -to each remote website. +Scrapy allows defining the concurrency and delay of different download slots, +e.g. through the :setting:`DOWNLOAD_SLOTS` setting. By default requests are +assigned to slots based on their URL domain, although it is possible to +customize the download slot of any request. + +The AutoThrottle extension adjusts the delay of each download slot dynamically, +to make your spider send :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent +requests on average to each remote website. It uses download latency to compute the delays. The main idea is the following: if a server needs ``latency`` seconds to respond, a client @@ -47,25 +52,6 @@ effect, but there are some important differences: AutoThrottle doesn't have these issues. -.. reqmeta:: autothrottle_dont_adjust_delay - -Disabling the throttling of a request -===================================== - -To disable AutoThrottle for a specific request, set the -``autothrottle_dont_adjust_delay`` request metadata key to ``True``: - -.. code-block:: python - - from scrapy import Request - - Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) - -Note, however, that AutoThrottle still determines the starting delay of every -slot by setting the ``download_delay`` attribute on the running spider. You -might want to set a custom value for the ``delay`` attribute of the slot, e.g. -using :setting:`DOWNLOAD_SLOTS`. - Throttling algorithm ==================== @@ -99,6 +85,33 @@ callback, for example, and unable to attend downloads. However, these latencies should still give a reasonable estimate of how busy Scrapy (and ultimately, the server) is, and this extension builds on that premise. +.. reqmeta:: autothrottle_dont_adjust_delay + +Prevent specific requests from triggering slot delay adjustments +================================================================ + +AutoThrottle adjusts the delay of download slots based on the latencies of +responses that belong to that download slot. The only exceptions are non-200 +responses, which are only taken into account to increase that delay, but +ignored if they would decrease that delay. + +You can also set the ``autothrottle_dont_adjust_delay`` request metadata key to +``True`` in any request to prevent its response latency from impacting the +delay of its download slot: + +.. code-block:: python + + from scrapy import Request + + Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) + +Note, however, that AutoThrottle still determines the starting delay of every +download slot by setting the ``download_delay`` attribute on the running +spider. If you want AutoThrottle not to impact a download slot at all, in +addition to setting this meta key in all requests that use that download slot, +you might want to set a custom value for the ``delay`` attribute of that +download slot, e.g. using :setting:`DOWNLOAD_SLOTS`. + Settings ======== From b042ad255db139adc740cd97047b6607889f9f1c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 11 Nov 2024 15:49:52 +0500 Subject: [PATCH 1566/2083] Address some previously ignored pylint messages. (#6531) --- conftest.py | 2 +- pylintrc | 9 ------- scrapy/commands/crawl.py | 2 +- scrapy/commands/edit.py | 3 ++- scrapy/commands/parse.py | 3 +-- scrapy/contracts/__init__.py | 8 ++++-- scrapy/core/downloader/handlers/__init__.py | 5 ++-- scrapy/core/downloader/handlers/ftp.py | 5 +++- scrapy/core/downloader/middleware.py | 2 +- scrapy/core/engine.py | 4 +-- scrapy/core/scheduler.py | 3 +-- scrapy/core/spidermw.py | 13 +++++----- scrapy/extensions/feedexport.py | 2 +- scrapy/http/request/form.py | 6 ++--- scrapy/mail.py | 4 +-- scrapy/pipelines/files.py | 3 +-- scrapy/resolver.py | 11 ++++----- scrapy/settings/__init__.py | 6 ++--- scrapy/spidermiddlewares/referer.py | 5 ++-- scrapy/utils/conf.py | 3 +-- scrapy/utils/console.py | 3 ++- scrapy/utils/defer.py | 3 +-- scrapy/utils/display.py | 7 +++--- scrapy/utils/log.py | 1 + scrapy/utils/python.py | 5 ++-- scrapy/utils/reactor.py | 2 +- scrapy/utils/response.py | 5 ++-- scrapy/utils/trackref.py | 1 + scrapy/utils/url.py | 2 +- tests/spiders.py | 2 ++ tests/test_crawler.py | 12 ++++++--- tests/test_downloadermiddleware_cookies.py | 2 +- tests/test_linkextractors.py | 3 +-- tests/test_loader_deprecated.py | 3 +-- tests/test_logformatter.py | 3 +-- tests/test_pipeline_files.py | 27 ++++++++++----------- tests/test_request_dict.py | 7 +++--- tests/test_spidermiddleware_referer.py | 1 + tests/test_utils_datatypes.py | 1 + tests/test_utils_defer.py | 1 + 40 files changed, 91 insertions(+), 99 deletions(-) diff --git a/conftest.py b/conftest.py index 77b0e033b..3af072318 100644 --- a/conftest.py +++ b/conftest.py @@ -57,7 +57,7 @@ def pytest_addoption(parser): def reactor_pytest(request): if not request.cls: # doctests - return + return None request.cls.reactor_pytest = request.config.getoption("--reactor") return request.cls.reactor_pytest diff --git a/pylintrc b/pylintrc index c60e4e16a..e927b903c 100644 --- a/pylintrc +++ b/pylintrc @@ -18,14 +18,12 @@ disable=abstract-method, disallowed-name, duplicate-code, # https://github.com/PyCQA/pylint/issues/214 eval-used, - expression-not-assigned, fixme, function-redefined, global-statement, implicit-str-concat, import-error, import-outside-toplevel, - inconsistent-return-statements, inherit-non-class, invalid-name, invalid-overridden-method, @@ -37,25 +35,20 @@ disable=abstract-method, logging-not-lazy, lost-exception, missing-docstring, - no-else-raise, - no-else-return, no-member, no-method-argument, no-name-in-module, no-self-argument, no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268 not-callable, - pointless-exception-statement, pointless-statement, pointless-string-statement, protected-access, raise-missing-from, - redefined-argument-from-local, redefined-builtin, redefined-outer-name, reimported, signature-differs, - super-init-not-called, too-few-public-methods, too-many-ancestors, too-many-arguments, @@ -73,9 +66,7 @@ disable=abstract-method, unreachable, unused-argument, unused-import, - unused-private-member, unused-variable, - unused-wildcard-import, used-before-assignment, useless-return, wildcard-import, diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 6b6a80bb5..0d71ab6c6 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -23,7 +23,7 @@ class Command(BaseRunSpiderCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: raise UsageError() - elif len(args) > 1: + if len(args) > 1: raise UsageError( "running 'scrapy crawl' with more than one spider is not supported" ) diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 34313d731..438375e02 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -35,7 +35,8 @@ class Command(ScrapyCommand): try: spidercls = self.crawler_process.spider_loader.load(args[0]) except KeyError: - return self._err(f"Spider not found: {args[0]}") + self._err(f"Spider not found: {args[0]}") + return sfile = sys.modules[spidercls.__module__].__file__ assert sfile diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 2059dcf75..fba294851 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -399,8 +399,7 @@ class Command(BaseRunSpiderCommand): # parse arguments if not len(args) == 1 or not is_url(args[0]): raise UsageError() - else: - url = args[0] + url = args[0] # prepare spidercls self.set_spidercls(url, opts) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index c20c02ca6..9071395e3 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -38,7 +38,9 @@ class Contract: assert cb is not None @wraps(cb) - def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: + def wrapper( # pylint: disable=inconsistent-return-statements + response: Response, **cb_kwargs: Any + ) -> list[Any]: try: results.startTest(self.testcase_pre) self.pre_process(response) @@ -67,7 +69,9 @@ class Contract: assert cb is not None @wraps(cb) - def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: + def wrapper( # pylint: disable=inconsistent-return-statements + response: Response, **cb_kwargs: Any + ) -> list[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 218f44bbb..20377ac06 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -92,9 +92,8 @@ class DownloadHandlers: ) self._notconfigured[scheme] = str(ex) return None - else: - self._handlers[scheme] = dh - return dh + self._handlers[scheme] = dh + return dh def download_request(self, request: Request, spider: Spider) -> Deferred[Response]: scheme = urlparse_cached(request).scheme diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 70a769771..598659b4d 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -70,7 +70,10 @@ class ReceivedDataProtocol(Protocol): return self.__filename def close(self) -> None: - self.body.close() if self.filename else self.body.seek(0) + if self.filename: + self.body.close() + else: + self.body.seek(0) _CODE_RE = re.compile(r"\d+") diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 60e7adb2f..db4191385 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -73,7 +73,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): ) -> Generator[Deferred[Any], Any, Response | Request]: if response is None: raise TypeError("Received None in process_response") - elif isinstance(response, Request): + if isinstance(response, Request): return response for method in self.methods["process_response"]: diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index d056a00ba..60cffae35 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -172,7 +172,7 @@ class ExecutionEngine: assert self.spider is not None # typing if self.paused: - return None + return while ( not self._needs_backout() @@ -418,7 +418,7 @@ class ExecutionEngine: if isinstance(x, Failure) and isinstance(x.value, ex) } if DontCloseSpider in detected_ex: - return None + return if self.spider_is_idle(): ex = detected_ex.get(CloseSpider, CloseSpider(reason="finished")) assert isinstance(ex, CloseSpider) # typing diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index bebee1236..f09d1903c 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -312,8 +312,7 @@ class Scheduler(BaseScheduler): assert self.stats is not None self.stats.inc_value("scheduler/unserializable", spider=self.spider) return False - else: - return True + return True def _mqpush(self, request: Request) -> None: self.mqs.push(request) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index f7947d35d..a63ee40bf 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -174,14 +174,13 @@ class SpiderMiddlewareManager(MiddlewareManager): # _process_spider_exception too, which complicates the architecture msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" raise _InvalidOutput(msg) - elif result is None: + if result is None: continue - else: - msg = ( - f"{method.__qualname__} must return None " - f"or an iterable, got {type(result)}" - ) - raise _InvalidOutput(msg) + msg = ( + f"{method.__qualname__} must return None " + f"or an iterable, got {type(result)}" + ) + raise _InvalidOutput(msg) return _failure # This method cannot be made async def, as _process_spider_exception relies on the Deferred result diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 6ab88dbb4..af06b7790 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -105,7 +105,7 @@ class ItemFilter: class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" - def __init__(uri, *, feed_options=None): + def __init__(uri, *, feed_options=None): # pylint: disable=super-init-not-called """Initialize the storage with the parameters given in the URI and the feed-specific options (see :setting:`FEEDS`)""" diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 29743565d..10ad1305e 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -152,8 +152,7 @@ def _get_form( form = forms[formnumber] except IndexError: raise IndexError(f"Form number {formnumber} not found in {response}") - else: - return cast(FormElement, form) + return cast(FormElement, form) def _get_inputs( @@ -264,5 +263,4 @@ def _get_clickable( f"Multiple elements found ({el!r}) matching the " f"criteria in clickdata: {clickdata!r}" ) - else: - raise ValueError(f"No clickable element matching clickdata: {clickdata!r}") + raise ValueError(f"No clickable element matching clickdata: {clickdata!r}") diff --git a/scrapy/mail.py b/scrapy/mail.py index ce7beb773..10dc7fed2 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -117,8 +117,8 @@ class MailSender: if charset: msg.set_charset(charset) msg.attach(MIMEText(body, "plain", charset or "us-ascii")) - for attach_name, mimetype, f in attachs: - part = MIMEBase(*mimetype.split("/")) + for attach_name, attach_mimetype, f in attachs: + part = MIMEBase(*attach_mimetype.split("/")) part.set_payload(f.read()) Encoders.encode_base64(part) part.add_header( diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4a8639c22..73cf37d28 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -265,8 +265,7 @@ class S3FilesStore: kwarg = mapping[key] except KeyError: raise TypeError(f'Header "{key}" is not supported by botocore') - else: - extra[kwarg] = value + extra[kwarg] = value return extra diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 99a6cc5f6..0e8260736 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -141,9 +141,8 @@ class CachingHostnameResolver: addressTypes, transportSemantics, ) - else: - resolutionReceiver.resolutionBegan(HostResolution(hostName)) - for addr in addresses: - resolutionReceiver.addressResolved(addr) - resolutionReceiver.resolutionComplete() - return resolutionReceiver + resolutionReceiver.resolutionBegan(HostResolution(hostName)) + for addr in addresses: + resolutionReceiver.addressResolved(addr) + resolutionReceiver.resolutionComplete() + return resolutionReceiver diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 274ced3e3..3ebdb351a 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -501,11 +501,9 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): except KeyError: if default is self.__default: raise - return default - else: - self.__delitem__(name) - return value + self.__delitem__(name) + return value class Settings(BaseSettings): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index bdf1f168a..720217c97 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -325,9 +325,8 @@ def _load_policy_class( msg = f"Could not load referrer policy {policy!r}" if not warning_only: raise RuntimeError(msg) - else: - warnings.warn(msg, RuntimeWarning) - return None + warnings.warn(msg, RuntimeWarning) + return None class RefererMiddleware: diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 91a49c652..e621525f2 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -44,8 +44,7 @@ def build_component_list( "convert to the same " "object, please update your settings" ) - else: - compbs.set(convert(k), v, priority=prio) + compbs.set(convert(k), v, priority=prio) return compbs _check_components(compdict) return {convert(k): v for k, v in compdict.items()} diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 6b9b4114f..95844a48c 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -101,7 +101,7 @@ DEFAULT_PYTHON_SHELLS: KnownShellsT = { def get_shell_embed_func( shells: Iterable[str] | None = None, known_shells: KnownShellsT | None = None -) -> Any: +) -> EmbedFuncT | None: """Return the first acceptable shell-embed function from a given list of shell names. """ @@ -117,6 +117,7 @@ def get_shell_embed_func( return known_shells[shell]() except ImportError: continue + return None def start_python_console( diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 9ca6c6a24..9f1b816c8 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -109,8 +109,7 @@ def mustbe_deferred( return defer_fail(failure.Failure(e)) except Exception: return defer_fail(failure.Failure()) - else: - return defer_result(result) + return defer_result(result) def parallel( diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 596cf89e4..39f46270b 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -36,11 +36,10 @@ def _colorize(text: str, colorize: bool = True) -> str: from pygments import highlight except ImportError: return text - else: - from pygments.formatters import TerminalFormatter - from pygments.lexers import PythonLexer + from pygments.formatters import TerminalFormatter + from pygments.lexers import PythonLexer - return highlight(text, PythonLexer(), TerminalFormatter()) + return highlight(text, PythonLexer(), TerminalFormatter()) def pformat(obj: Any, *args: Any, **kwargs: Any) -> str: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index c3808426a..a40b835cd 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -51,6 +51,7 @@ class TopLevelFormatter(logging.Filter): """ def __init__(self, loggers: list[str] | None = None): + super().__init__() self.loggers: list[str] = loggers or [] def filter(self, record: logging.LogRecord) -> bool: diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index d970f5da5..3864d054f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -323,9 +323,8 @@ def without_none_values( """ if isinstance(iterable, Mapping): return {k: v for k, v in iterable.items() if v is not None} - else: - # the iterable __init__ must take another iterable - return type(iterable)(v for v in iterable if v is not None) # type: ignore[call-arg] + # the iterable __init__ must take another iterable + return type(iterable)(v for v in iterable if v is not None) # type: ignore[call-arg] def global_object_name(obj: Any) -> str: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index e7bd0b232..ac4358410 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -26,7 +26,7 @@ if TYPE_CHECKING: _T = TypeVar("_T") -def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] +def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index ecc83d1c8..7c8ca51f2 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -66,9 +66,8 @@ def _remove_html_comments(body: bytes) -> bytes: end = body.find(b"-->", start + 1) if end == -1: return body[:start] - else: - body = body[:start] + body[end + 3 :] - start = body.find(b"<!--") + body = body[:start] + body[end + 3 :] + start = body.find(b"<!--") return body diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 5eec1c10f..b04214c51 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -66,6 +66,7 @@ def get_oldest(class_name: str) -> Any: if not wdict: break return min(wdict.items(), key=itemgetter(1))[0] + return None def iter_all(class_name: str) -> Iterable[Any]: diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 9dc177cf1..a5cc22c1c 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -14,7 +14,7 @@ from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code -from w3lib.url import * +from w3lib.url import * # pylint: disable=unused-wildcard-import from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode diff --git a/tests/spiders.py b/tests/spiders.py index cc54240ef..63c7a6f9b 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -377,11 +377,13 @@ class SingleRequestSpider(MetaSpider): return self.callback_func(response) if "next" in response.meta: return response.meta["next"] + return None def on_error(self, failure): self.meta["failure"] = failure if callable(self.errback_func): return self.errback_func(failure) + return None class DuplicateStartRequestsSpider(MockServerSpider): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 37348778c..853acf2de 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -143,7 +143,8 @@ class CrawlerTestCase(BaseCrawlerTest): def from_crawler(cls, crawler): return cls(crawler=crawler) - def __init__(self, crawler): + def __init__(self, crawler, **kwargs: Any): + super().__init__(**kwargs) self.crawler = crawler def start_requests(self): @@ -223,7 +224,8 @@ class CrawlerTestCase(BaseCrawlerTest): def from_crawler(cls, crawler): return cls(crawler=crawler) - def __init__(self, crawler): + def __init__(self, crawler, **kwargs: Any): + super().__init__(**kwargs) self.crawler = crawler def start_requests(self): @@ -301,7 +303,8 @@ class CrawlerTestCase(BaseCrawlerTest): def from_crawler(cls, crawler): return cls(crawler=crawler) - def __init__(self, crawler): + def __init__(self, crawler, **kwargs: Any): + super().__init__(**kwargs) self.crawler = crawler def start_requests(self): @@ -379,7 +382,8 @@ class CrawlerTestCase(BaseCrawlerTest): def from_crawler(cls, crawler): return cls(crawler=crawler) - def __init__(self, crawler): + def __init__(self, crawler, **kwargs: Any): + super().__init__(**kwargs) self.crawler = crawler def start_requests(self): diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 6e343d035..772769690 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -25,7 +25,7 @@ def _cookie_to_set_cookie_value(cookie): for key in ("name", "value", "path", "domain"): if cookie.get(key) is None: if key in ("name", "value"): - return + return None continue if isinstance(cookie[key], (bool, float, int, str)): decoded[key] = str(cookie[key]) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index ed3394b01..a83cfb56c 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -436,8 +436,7 @@ class Base: def process_value(value): m = re.search(r"javascript:goToPage\('(.*?)'", value) - if m: - return m.group(1) + return m.group(1) if m else None lx = self.extractor_cls(process_value=process_value) self.assertEqual( diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 0d245bec9..f9b841a61 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -69,8 +69,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_load_item_ignore_none_field_values(self): def validate_sku(value): # Let's assume a SKU is only digits. - if value.isdigit(): - return value + return value if value.isdigit() else None class MyLoader(ItemLoader): name_out = Compose(lambda vs: vs[0]) # take first which allows empty values diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 56810f2ff..5a92521cc 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -198,8 +198,7 @@ class DropSomeItemsPipeline: if self.drop: self.drop = False raise DropItem("Ignoring item") - else: - self.drop = True + self.drop = True class ShowOrSkipMessagesTestCase(TwistedTestCase): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 47840caaa..80bb9e939 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -627,20 +627,19 @@ class TestGCSFilesStore(unittest.TestCase): import google.cloud.storage # noqa except ModuleNotFoundError: raise unittest.SkipTest("google-cloud-storage is not installed") - else: - with mock.patch("google.cloud.storage") as _: - with mock.patch("scrapy.pipelines.files.time") as _: - uri = "gs://my_bucket/my_prefix/" - store = GCSFilesStore(uri) - store.bucket = mock.Mock() - path = "full/my_data.txt" - yield store.persist_file( - path, mock.Mock(), info=None, meta=None, headers=None - ) - yield store.stat_file(path, info=None) - expected_blob_path = store.prefix + path - store.bucket.blob.assert_called_with(expected_blob_path) - store.bucket.get_blob.assert_called_with(expected_blob_path) + with mock.patch("google.cloud.storage") as _: + with mock.patch("scrapy.pipelines.files.time") as _: + uri = "gs://my_bucket/my_prefix/" + store = GCSFilesStore(uri) + store.bucket = mock.Mock() + path = "full/my_data.txt" + yield store.persist_file( + path, mock.Mock(), info=None, meta=None, headers=None + ) + yield store.stat_file(path, info=None) + expected_blob_path = store.prefix + path + store.bucket.blob.assert_called_with(expected_blob_path) + store.bucket.get_blob.assert_called_with(expected_blob_path) class TestFTPFileStore(unittest.TestCase): diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index d3f416347..854805cf7 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -159,7 +159,7 @@ class RequestSerializationTest(unittest.TestCase): class TestSpiderMixin: - def __mixin_callback(self, response): + def __mixin_callback(self, response): # pylint: disable=unused-private-member pass @@ -191,7 +191,8 @@ class TestSpider(Spider, TestSpiderMixin): __parse_item_reference = private_parse_item __handle_error_reference = private_handle_error - def __init__(self): + def __init__(self, **kwargs): + super().__init__(**kwargs) self.delegated_callback = TestSpiderDelegation().delegated_callback def parse_item(self, response): @@ -200,5 +201,5 @@ class TestSpider(Spider, TestSpiderMixin): def handle_error(self, failure): pass - def __parse_item_private(self, response): + def __parse_item_private(self, response): # pylint: disable=unused-private-member pass diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index facbaa60d..23b0c17c6 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -686,6 +686,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): return b"https://python.org/" if scheme == "http": return b"http://python.org/" + return None class TestSettingsCustomPolicy(TestRefererMiddleware): diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 10dc6f270..5a76593c3 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -158,6 +158,7 @@ class CaseInsensitiveDictMixin: def _normvalue(self, value): if value is not None: return value + 1 + return None normvalue = _normvalue # deprecated CaselessDict class diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index ec0399865..3f153bdc0 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -182,6 +182,7 @@ class AsyncCooperatorTest(unittest.TestCase): return dfd # simulate trivial sync processing results.append(o) + return None @staticmethod def get_async_iterable(length): From eda3a89b3fe3e88ed8b90d032a2f25b92a1b79ca Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 11 Nov 2024 16:44:47 +0500 Subject: [PATCH 1567/2083] Remove build_from_settings(). --- scrapy/middleware.py | 28 +++++++++++++++++++++---- scrapy/utils/misc.py | 25 ++--------------------- tests/test_middleware.py | 8 +++----- tests/test_utils_misc/__init__.py | 34 ------------------------------- tests/test_webclient.py | 33 ++++++++++++------------------ 5 files changed, 42 insertions(+), 86 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index b6a427895..9e994703d 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -2,12 +2,13 @@ from __future__ import annotations import logging import pprint +import warnings from collections import defaultdict, deque from typing import TYPE_CHECKING, Any, TypeVar, cast -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.utils.defer import process_chain, process_parallel -from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from collections.abc import Callable, Iterable @@ -20,7 +21,7 @@ if TYPE_CHECKING: from scrapy import Spider from scrapy.crawler import Crawler - from scrapy.settings import Settings + from scrapy.settings import BaseSettings, Settings _P = ParamSpec("_P") @@ -50,8 +51,27 @@ class MiddlewareManager: def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: raise NotImplementedError + @staticmethod + def _build_from_settings(objcls: type[_T], settings: BaseSettings) -> _T: + if hasattr(objcls, "from_settings"): + instance = objcls.from_settings(settings) # type: ignore[attr-defined] + method_name = "from_settings" + else: + instance = objcls() + method_name = "__new__" + if instance is None: + raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") + return cast(_T, instance) + @classmethod def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self: + if crawler is None: + warnings.warn( + "Calling MiddlewareManager.from_settings() without a Crawler instance is deprecated." + " As this method will be deprecated in the future, please switch to from_crawler().", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] @@ -61,7 +81,7 @@ class MiddlewareManager: if crawler is not None: mw = build_from_crawler(mwcls, crawler) else: - mw = build_from_settings(mwcls, settings) + mw = MiddlewareManager._build_from_settings(mwcls, settings) middlewares.append(mw) enabled.append(clspath) except NotConfigured as e: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 12c09839f..efb475131 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -26,7 +26,6 @@ if TYPE_CHECKING: from scrapy import Spider from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes @@ -150,7 +149,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): """ warnings.warn( "The create_instance() function is deprecated. " - "Please use build_from_crawler() or build_from_settings() instead.", + "Please use build_from_crawler() instead.", category=ScrapyDeprecationWarning, stacklevel=2, ) @@ -176,7 +175,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): def build_from_crawler( objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any ) -> T: - """Construct a class instance using its ``from_crawler`` constructor. + """Construct a class instance using its ``from_crawler`` or ``from_settings`` constructor. ``*args`` and ``**kwargs`` are forwarded to the constructor. @@ -196,26 +195,6 @@ def build_from_crawler( return cast(T, instance) -def build_from_settings( - objcls: type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any -) -> T: - """Construct a class instance using its ``from_settings`` constructor. - - ``*args`` and ``**kwargs`` are forwarded to the constructor. - - Raises ``TypeError`` if the resulting instance is ``None``. - """ - if hasattr(objcls, "from_settings"): - instance = objcls.from_settings(settings, *args, **kwargs) # type: ignore[attr-defined] - method_name = "from_settings" - else: - instance = objcls(*args, **kwargs) - method_name = "__new__" - if instance is None: - raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return cast(T, instance) - - @contextmanager def set_environ(**kwargs: str) -> Iterator[None]: """Temporarily set environment variables inside the context manager and diff --git a/tests/test_middleware.py b/tests/test_middleware.py index a42c7b3d1..3a1cf19ad 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -2,7 +2,7 @@ from twisted.trial import unittest from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager -from scrapy.settings import Settings +from scrapy.utils.test import get_crawler class M1: @@ -23,8 +23,6 @@ class M2: def close_spider(self, spider): pass - pass - class M3: def process(self, response, request, spider): @@ -83,7 +81,7 @@ class MiddlewareManagerTest(unittest.TestCase): self.assertEqual(mwman.middlewares, (m1, m2, m3)) def test_enabled_from_settings(self): - settings = Settings() - mwman = TestMiddlewareManager.from_settings(settings) + crawler = get_crawler() + mwman = TestMiddlewareManager.from_crawler(crawler) classes = [x.__class__ for x in mwman.middlewares] self.assertEqual(classes, [M1, M3]) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 4d8e71521..f71b2b034 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -10,7 +10,6 @@ from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, build_from_crawler, - build_from_settings, create_instance, load_object, rel_has_nofollow, @@ -197,39 +196,6 @@ class UtilsMiscTestCase(unittest.TestCase): with self.assertRaises(TypeError): build_from_crawler(m, crawler, *args, **kwargs) - def test_build_from_settings(self): - settings = mock.MagicMock() - args = (True, 100.0) - kwargs = {"key": "val"} - - def _test_with_settings(mock, settings): - build_from_settings(mock, settings, *args, **kwargs) - if hasattr(mock, "from_settings"): - mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - self.assertEqual(mock.call_count, 0) - else: - mock.assert_called_once_with(*args, **kwargs) - - # Check usage of correct constructor using three mocks: - # 1. with no alternative constructors - # 2. with from_settings() constructor - # 3. with from_settings() and from_crawler() constructor - spec_sets = ( - ["__qualname__"], - ["__qualname__", "from_settings"], - ["__qualname__", "from_settings", "from_crawler"], - ) - for specs in spec_sets: - m = mock.MagicMock(spec_set=specs) - _test_with_settings(m, settings) - m.reset_mock() - - # Check adoption of crawler settings - m = mock.MagicMock(spec_set=["__qualname__", "from_settings"]) - m.from_settings.return_value = None - with self.assertRaises(TypeError): - build_from_settings(m, settings, *args, **kwargs) - def test_set_environ(self): assert os.environ.get("some_test_environ") is None with set_environ(some_test_environ="test_value"): diff --git a/tests/test_webclient.py b/tests/test_webclient.py index cce119001..1797d5e1f 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -9,25 +9,18 @@ from tempfile import mkdtemp import OpenSSL.SSL from twisted.internet import defer, reactor +from twisted.internet.defer import inlineCallbacks +from twisted.internet.testing import StringTransport +from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util -try: - from twisted.internet.testing import StringTransport -except ImportError: - # deprecated in Twisted 19.7.0 - # (remove once we bump our requirement past that version) - from twisted.test.proto_helpers import StringTransport - -from twisted.internet.defer import inlineCallbacks -from twisted.protocols.policies import WrappingFactory - from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from scrapy.http import Headers, Request -from scrapy.settings import Settings -from scrapy.utils.misc import build_from_settings +from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.test import get_crawler from tests.mockserver import ( BrokenDownloadResource, ErrorResource, @@ -469,22 +462,22 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): def testPayload(self): s = "0123456789" * 10 - settings = Settings({"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers}) - client_context_factory = build_from_settings( - ScrapyClientContextFactory, settings + crawler = get_crawler( + settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers} ) + client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) return getPage( self.getURL("payload"), body=s, contextFactory=client_context_factory ).addCallback(self.assertEqual, to_bytes(s)) def testPayloadDisabledCipher(self): s = "0123456789" * 10 - settings = Settings( - {"DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384"} - ) - client_context_factory = build_from_settings( - ScrapyClientContextFactory, settings + crawler = get_crawler( + settings_dict={ + "DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384" + } ) + client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) d = getPage( self.getURL("payload"), body=s, contextFactory=client_context_factory ) From 83d4939d41ab8790587f721755a74f883cc04e31 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 11 Nov 2024 22:14:42 +0500 Subject: [PATCH 1568/2083] Deprecate scrapy.extensions.feedexport.build_storage() and simplify _get_storage(). --- scrapy/extensions/feedexport.py | 50 ++++++++------------------------- 1 file changed, 12 insertions(+), 38 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 6ab88dbb4..27f0b79ae 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -62,6 +62,11 @@ def build_storage( preargs: Iterable[Any] = (), **kwargs: Any, ) -> _StorageT: + warnings.warn( + "scrapy.extensions.feedexport.build_storage() is deprecated, call the builder directly.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) kwargs["feed_options"] = feed_options return builder(*preargs, uri, *args, **kwargs) @@ -248,8 +253,7 @@ class S3FeedStorage(BlockingFeedStorage): *, feed_options: dict[str, Any] | None = None, ) -> Self: - return build_storage( - cls, + return cls( uri, access_key=crawler.settings["AWS_ACCESS_KEY_ID"], secret_key=crawler.settings["AWS_SECRET_ACCESS_KEY"], @@ -323,10 +327,9 @@ class FTPFeedStorage(BlockingFeedStorage): *, feed_options: dict[str, Any] | None = None, ) -> Self: - return build_storage( - cls, + return cls( uri, - crawler.settings.getbool("FEED_STORAGE_FTP_ACTIVE"), + use_active_mode=crawler.settings.getbool("FEED_STORAGE_FTP_ACTIVE"), feed_options=feed_options, ) @@ -407,15 +410,12 @@ class FeedSlot: self.exporter.start_exporting() self._exporting = True - def _get_instance( - self, objcls: type[BaseItemExporter], *args: Any, **kwargs: Any - ) -> BaseItemExporter: - return build_from_crawler(objcls, self.crawler, *args, **kwargs) - def _get_exporter( self, file: IO[bytes], format: str, *args: Any, **kwargs: Any ) -> BaseItemExporter: - return self._get_instance(self.exporters[format], file, *args, **kwargs) + return build_from_crawler( + self.exporters[format], self.crawler, file, *args, **kwargs + ) def finish_exporting(self) -> None: if self._exporting: @@ -692,34 +692,8 @@ class FeedExporter: def _get_storage( self, uri: str, feed_options: dict[str, Any] ) -> FeedStorageProtocol: - """Fork of create_instance specific to feed storage classes - - It supports not passing the *feed_options* parameters to classes that - do not support it, and issuing a deprecation warning instead. - """ feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) - crawler = getattr(self, "crawler", None) - - def build_instance( - builder: type[FeedStorageProtocol], *preargs: Any - ) -> FeedStorageProtocol: - return build_storage( - builder, uri, feed_options=feed_options, preargs=preargs - ) - - instance: FeedStorageProtocol - if crawler and hasattr(feedcls, "from_crawler"): - instance = build_instance(feedcls.from_crawler, crawler) - method_name = "from_crawler" - elif hasattr(feedcls, "from_settings"): - instance = build_instance(feedcls.from_settings, self.settings) - method_name = "from_settings" - else: - instance = build_instance(feedcls) - method_name = "__new__" - if instance is None: - raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None") - return instance + return build_from_crawler(feedcls, self.crawler, uri, feed_options=feed_options) def _get_uri_params( self, From f796d8780c75543eadb6cf3689c7d0ca02896f0c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 12 Nov 2024 21:08:04 +0500 Subject: [PATCH 1569/2083] Add tests for MediaPipeline.from_crawler() and related code. --- tests/test_pipeline_files.py | 62 ++++++++++++++++++++++ tests/test_pipeline_media.py | 99 ++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 47840caaa..83eaa1fdd 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -2,6 +2,7 @@ import dataclasses import os import random import time +import warnings from datetime import datetime from io import BytesIO from pathlib import Path @@ -16,6 +17,7 @@ from itemadapter import ItemAdapter from twisted.internet import defer from twisted.trial import unittest +from scrapy import Spider from scrapy.http import Request, Response from scrapy.item import Field, Item from scrapy.pipelines.files import ( @@ -687,3 +689,63 @@ def _prepare_request_object(item_url, flags=None): item_url, meta={"response": Response(item_url, status=200, body=b"data", flags=flags)}, ) + + +# this is separate from the one in test_pipeline_media.py to specifically test FilesPipeline subclasses +class BuildFromCrawlerTestCase(unittest.TestCase): + def setUp(self): + self.tempdir = mkdtemp() + self.crawler = get_crawler(Spider, {"FILES_STORE": self.tempdir}) + + def tearDown(self): + rmtree(self.tempdir) + + def test_simple(self): + class Pipeline(FilesPipeline): + pass + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + self.assertEqual(len(w), 0) + assert pipe.store + + def test_has_from_settings(self): + class Pipeline(FilesPipeline): + @classmethod + def from_settings(cls, settings): + o = super().from_settings(settings) + o._from_settings_called = True + return o + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + self.assertEqual(len(w), 0) + assert pipe.store + assert pipe._from_settings_called + + @pytest.mark.xfail( + reason="No way to override MediaPipeline.from_crawler having non-trivial __init__" + ) + def test_has_from_crawler_and_init(self): + class Pipeline(FilesPipeline): + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + store_uri = settings["FILES_STORE"] + # you can either call super().from_crawler() or cls.__init__() but you need both + o = cls(store_uri, settings=settings) + o._from_crawler_called = True + return o + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + # this and the next assert will fail as MediaPipeline.from_crawler() wasn't called + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + self.assertEqual(len(w), 0) + assert pipe.store + assert pipe._from_crawler_called diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index c979e45d7..920b4246e 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,5 +1,8 @@ from __future__ import annotations +import warnings + +import pytest from testfixtures import LogCapture from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks @@ -410,3 +413,99 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): self._assert_request_no3xx( UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True} ) + + +class BuildFromCrawlerTestCase(unittest.TestCase): + def setUp(self): + self.crawler = get_crawler(Spider, {"FILES_STORE": "/foo"}) + + def test_simple(self): + class Pipeline(UserDefinedPipeline): + pass + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + self.assertEqual(len(w), 0) + + def test_has_from_settings(self): + class Pipeline(UserDefinedPipeline): + @classmethod + def from_settings(cls, settings): + o = cls() + o._from_settings_called = True + return o + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + self.assertEqual(len(w), 0) + assert pipe._from_settings_called + + def test_has_from_settings_and_init(self): + class Pipeline(UserDefinedPipeline): + def __init__(self, store_uri, settings): + super().__init__() + self._init_called = True + + @classmethod + def from_settings(cls, settings): + store_uri = settings["FILES_STORE"] + o = cls(store_uri, settings=settings) + o._from_settings_called = True + return o + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + self.assertEqual(len(w), 0) + assert pipe._from_settings_called + assert pipe._init_called + + @pytest.mark.xfail( + reason="No way to override MediaPipeline.from_crawler having non-trivial __init__" + ) + def test_has_from_crawler_and_init(self): + class Pipeline(UserDefinedPipeline): + def __init__(self, store_uri, settings): + super().__init__() + self._init_called = True + + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + store_uri = settings["FILES_STORE"] + # you can either call super().from_crawler() or cls.__init__() but you need both + o = cls(store_uri, settings=settings) + o._from_crawler_called = True + return o + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + # this and the next assert will fail as super().from_crawler() wasn't called + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + self.assertEqual(len(w), 0) + assert pipe._from_crawler_called + assert pipe._init_called + + def test_has_from_crawler(self): + class Pipeline(UserDefinedPipeline): + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + o = super().from_crawler(crawler) + o._from_crawler_called = True + o.store_uri = settings["FILES_STORE"] + return o + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + # this and the next assert will fail as MediaPipeline.from_crawler() wasn't called + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + self.assertEqual(len(w), 0) + assert pipe._from_crawler_called From 499e7e8aa685b2c8ba60576707e6fbf8ed8180ba Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 12 Nov 2024 21:12:32 +0500 Subject: [PATCH 1570/2083] Add from_crawler() to components that only had from_settings(). --- docs/topics/email.rst | 16 ++++++------ docs/topics/request-response.rst | 9 +------ scrapy/core/downloader/contextfactory.py | 26 ++++++++++++++++++++ scrapy/dupefilters.py | 31 +++++++++++++++++++++--- scrapy/extensions/memusage.py | 2 +- scrapy/extensions/statsmailer.py | 2 +- scrapy/mail.py | 16 ++++++++++++ scrapy/spidermiddlewares/urllength.py | 17 ++++++++++++- tests/test_dupefilters.py | 18 -------------- tests/test_spidermiddleware_urllength.py | 7 ++---- 10 files changed, 99 insertions(+), 45 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index d6a7ad354..8f7a2357a 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -27,13 +27,13 @@ the standard ``__init__`` method: mailer = MailSender() -Or you can instantiate it passing a Scrapy settings object, which will respect -the :ref:`settings <topics-email-settings>`: +Or you can instantiate it passing a :class:`scrapy.Crawler` instance, which +will respect the :ref:`settings <topics-email-settings>`: .. skip: start .. code-block:: python - mailer = MailSender.from_settings(settings) + mailer = MailSender.from_crawler(crawler) And here is how to use it to send an e-mail (without attachments): @@ -81,13 +81,13 @@ rest of the framework. :param smtpssl: enforce using a secure SSL connection :type smtpssl: bool - .. classmethod:: from_settings(settings) + .. classmethod:: from_crawler(crawler) - Instantiate using a Scrapy settings object, which will respect - :ref:`these Scrapy settings <topics-email-settings>`. + Instantiate using a :class:`scrapy.Crawler` instance, which will + respect :ref:`these Scrapy settings <topics-email-settings>`. - :param settings: the e-mail recipients - :type settings: :class:`scrapy.settings.Settings` object + :param crawler: the crawler + :type settings: :class:`scrapy.Crawler` object .. method:: send(to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 7c15b67e8..710e2e131 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -488,7 +488,7 @@ A request fingerprinter is a class that must implement the following method: :param request: request to fingerprint :type request: scrapy.http.Request -Additionally, it may also implement the following methods: +Additionally, it may also implement the following method: .. classmethod:: from_crawler(cls, crawler) :noindex: @@ -504,13 +504,6 @@ Additionally, it may also implement the following methods: :param crawler: crawler that uses this request fingerprinter :type crawler: :class:`~scrapy.crawler.Crawler` object -.. classmethod:: from_settings(cls, settings) - - If present, and ``from_crawler`` is not defined, this class method is called - to create a request fingerprinter instance from a - :class:`~scrapy.settings.Settings` object. It must return a new instance of - the request fingerprinter. - .. currentmodule:: scrapy.http The :meth:`fingerprint` method of the default request fingerprinter, diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index f80f832a7..8e17eab9a 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,6 +21,7 @@ from scrapy.core.downloader.tls import ( ScrapyClientTLSOptions, openssl_methods, ) +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: @@ -69,6 +70,31 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): method: int = SSL.SSLv23_METHOD, *args: Any, **kwargs: Any, + ) -> Self: + warnings.warn( + f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return cls._from_settings(settings, method, *args, **kwargs) + + @classmethod + def from_crawler( + cls, + crawler: Crawler, + method: int = SSL.SSLv23_METHOD, + *args: Any, + **kwargs: Any, + ) -> Self: + return cls._from_settings(crawler.settings, method, *args, **kwargs) + + @classmethod + def _from_settings( + cls, + settings: BaseSettings, + method: int = SSL.SSLv23_METHOD, + *args: Any, + **kwargs: Any, ) -> Self: tls_verbose_logging: bool = settings.getbool( "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index d37d2741a..7b8eea135 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,9 +1,11 @@ from __future__ import annotations import logging +import warnings from pathlib import Path from typing import TYPE_CHECKING +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.job import job_dir from scrapy.utils.request import ( RequestFingerprinter, @@ -26,6 +28,15 @@ if TYPE_CHECKING: class BaseDupeFilter: @classmethod def from_settings(cls, settings: BaseSettings) -> Self: + warnings.warn( + f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return cls() + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: return cls() def request_seen(self, request: Request) -> bool: @@ -72,17 +83,31 @@ class RFPDupeFilter(BaseDupeFilter): *, fingerprinter: RequestFingerprinterProtocol | None = None, ) -> Self: - debug = settings.getbool("DUPEFILTER_DEBUG") - return cls(job_dir(settings), debug, fingerprinter=fingerprinter) + warnings.warn( + f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return cls._from_settings(settings, fingerprinter=fingerprinter) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: assert crawler.request_fingerprinter - return cls.from_settings( + return cls._from_settings( crawler.settings, fingerprinter=crawler.request_fingerprinter, ) + @classmethod + def _from_settings( + cls, + settings: BaseSettings, + *, + fingerprinter: RequestFingerprinterProtocol | None = None, + ) -> Self: + debug = settings.getbool("DUPEFILTER_DEBUG") + return cls(job_dir(settings), debug, fingerprinter=fingerprinter) + def request_seen(self, request: Request) -> bool: fp = self.request_fingerprint(request) if fp in self.fingerprints: diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 73d864d5d..d7f810107 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -48,7 +48,7 @@ class MemoryUsage: self.check_interval: float = crawler.settings.getfloat( "MEMUSAGE_CHECK_INTERVAL_SECONDS" ) - self.mail: MailSender = MailSender.from_settings(crawler.settings) + self.mail: MailSender = MailSender.from_crawler(crawler) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 600eebcf2..221628642 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -33,7 +33,7 @@ class StatsMailer: recipients: list[str] = crawler.settings.getlist("STATSMAILER_RCPTS") if not recipients: raise NotConfigured - mail: MailSender = MailSender.from_settings(crawler.settings) + mail: MailSender = MailSender.from_crawler(crawler) assert crawler.stats o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) diff --git a/scrapy/mail.py b/scrapy/mail.py index ce7beb773..3c40fea34 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -7,6 +7,7 @@ See documentation in docs/topics/email.rst from __future__ import annotations import logging +import warnings from email import encoders as Encoders from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart @@ -19,6 +20,7 @@ from typing import IO, TYPE_CHECKING, Any from twisted.internet import ssl from twisted.internet.defer import Deferred +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes @@ -32,6 +34,7 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.crawler import Crawler from scrapy.settings import BaseSettings @@ -72,6 +75,19 @@ class MailSender: @classmethod def from_settings(cls, settings: BaseSettings) -> Self: + warnings.warn( + f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return cls._from_settings(settings) + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls._from_settings(crawler.settings) + + @classmethod + def _from_settings(cls, settings: BaseSettings) -> Self: return cls( smtphost=settings["MAIL_HOST"], mailfrom=settings["MAIL_FROM"], diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 191adb6cd..a1cd1bb7c 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -7,9 +7,10 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging +import warnings from typing import TYPE_CHECKING, Any -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response if TYPE_CHECKING: @@ -19,6 +20,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy import Spider + from scrapy.crawler import Crawler from scrapy.settings import BaseSettings @@ -31,6 +33,19 @@ class UrlLengthMiddleware: @classmethod def from_settings(cls, settings: BaseSettings) -> Self: + warnings.warn( + f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return cls._from_settings(settings) + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls._from_settings(crawler.settings) + + @classmethod + def _from_settings(cls, settings: BaseSettings) -> Self: maxlength = settings.getint("URLLENGTH_LIMIT") if not maxlength: raise NotConfigured diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 9ba8bd64f..4fd648f48 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -33,14 +33,6 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter): return df -class FromSettingsRFPDupeFilter(RFPDupeFilter): - @classmethod - def from_settings(cls, settings, *, fingerprinter=None): - df = super().from_settings(settings, fingerprinter=fingerprinter) - df.method = "from_settings" - return df - - class DirectDupeFilter: method = "n/a" @@ -56,16 +48,6 @@ class RFPDupeFilterTest(unittest.TestCase): self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, "from_crawler") - def test_df_from_settings_scheduler(self): - settings = { - "DUPEFILTER_DEBUG": True, - "DUPEFILTER_CLASS": FromSettingsRFPDupeFilter, - } - crawler = get_crawler(settings_dict=settings) - scheduler = Scheduler.from_crawler(crawler) - self.assertTrue(scheduler.df.debug) - self.assertEqual(scheduler.df.method, "from_settings") - def test_df_direct_scheduler(self): settings = { "DUPEFILTER_CLASS": DirectDupeFilter, diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 9111e4c82..1a0f2e223 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -3,7 +3,6 @@ from unittest import TestCase from testfixtures import LogCapture from scrapy.http import Request, Response -from scrapy.settings import Settings from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.spiders import Spider from scrapy.utils.test import get_crawler @@ -12,12 +11,10 @@ from scrapy.utils.test import get_crawler class TestUrlLengthMiddleware(TestCase): def setUp(self): self.maxlength = 25 - settings = Settings({"URLLENGTH_LIMIT": self.maxlength}) - - crawler = get_crawler(Spider) + crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": self.maxlength}) self.spider = crawler._create_spider("foo") self.stats = crawler.stats - self.mw = UrlLengthMiddleware.from_settings(settings) + self.mw = UrlLengthMiddleware.from_crawler(crawler) self.response = Response("http://scrapytest.org") self.short_url_req = Request("http://scrapytest.org/") From eda1a8a7c5b3b61aedd5fcc2e3950b7af28a2926 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 12 Nov 2024 22:57:39 +0500 Subject: [PATCH 1571/2083] Deprecate MiddlewareManager.from_settings(). --- scrapy/middleware.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 9e994703d..2b67dcd21 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -65,13 +65,19 @@ class MiddlewareManager: @classmethod def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self: - if crawler is None: - warnings.warn( - "Calling MiddlewareManager.from_settings() without a Crawler instance is deprecated." - " As this method will be deprecated in the future, please switch to from_crawler().", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) + warnings.warn( + f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return cls._from_settings(settings, crawler) + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls._from_settings(crawler.settings, crawler) + + @classmethod + def _from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self: mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] @@ -102,10 +108,6 @@ class MiddlewareManager: ) return cls(*middlewares) - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls.from_settings(crawler.settings, crawler) - def _add_middleware(self, mw: Any) -> None: if hasattr(mw, "open_spider"): self.methods["open_spider"].append(mw.open_spider) From 8700a5b7a92582fb1dc2a8fad7e41aa79948258b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 12 Nov 2024 23:25:53 +0500 Subject: [PATCH 1572/2083] Deprecate build_from_crawler() calling from_settings(). --- scrapy/middleware.py | 8 ++++++++ scrapy/utils/misc.py | 8 ++++++++ tests/test_utils_request.py | 5 ++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 2b67dcd21..91411506f 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -54,6 +54,14 @@ class MiddlewareManager: @staticmethod def _build_from_settings(objcls: type[_T], settings: BaseSettings) -> _T: if hasattr(objcls, "from_settings"): + warnings.warn( + f"{objcls.__qualname__} has from_settings() but not from_crawler()." + " This is deprecated and calling from_settings() will be removed in a future" + " Scrapy version. You can implement a simple from_crawler() that calls" + " from_settings() with crawler.settings.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) instance = objcls.from_settings(settings) # type: ignore[attr-defined] method_name = "from_settings" else: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index efb475131..a408a205d 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -185,6 +185,14 @@ def build_from_crawler( instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined] method_name = "from_crawler" elif hasattr(objcls, "from_settings"): + warnings.warn( + f"{objcls.__qualname__} has from_settings() but not from_crawler()." + " This is deprecated and calling from_settings() will be removed in a future" + " Scrapy version. You can implement a simple from_crawler() that calls" + " from_settings() with crawler.settings.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) instance = objcls.from_settings(crawler.settings, *args, **kwargs) # type: ignore[attr-defined] method_name = "from_settings" else: diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 965d050a4..0a3e3b00b 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -8,6 +8,7 @@ from weakref import WeakKeyDictionary import pytest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.utils.python import to_bytes from scrapy.utils.request import ( @@ -384,7 +385,9 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } - crawler = get_crawler(settings_dict=settings) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) From 261c4b61dc48353346c1e0387d0783ac15ab459d Mon Sep 17 00:00:00 2001 From: Robert Palmer <Awriter247@gmail.com> Date: Wed, 13 Nov 2024 12:47:39 -0500 Subject: [PATCH 1573/2083] Enhancement: Update docs to include IgnoreRequest details (#6506) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves <adrian@chaves.io> --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index b184a629e..9eace3be0 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -876,7 +876,7 @@ REDIRECT_MAX_TIMES Default: ``20`` The maximum number of redirections that will be followed for a single request. -After this maximum, the request's response is returned as is. +If maximum redirections are exceeded, the request is aborted and ignored. MetaRefreshMiddleware --------------------- From 28fafbb8c56257eb6f09b8cbcb919483d5b30a11 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 14 Nov 2024 01:29:51 +0500 Subject: [PATCH 1574/2083] Modernize the media pipeline initialization API. --- scrapy/pipelines/files.py | 49 +++++++++++++++++++++-- scrapy/pipelines/images.py | 27 ++++++++++--- scrapy/pipelines/media.py | 46 +++++++++++++++++---- tests/test_pipeline_files.py | 75 +++++++++++++++++++++-------------- tests/test_pipeline_images.py | 47 +++++++++++++--------- tests/test_pipeline_media.py | 47 ++++++++++++++-------- 6 files changed, 211 insertions(+), 80 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4a8639c22..f83037e6c 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -12,6 +12,7 @@ import hashlib import logging import mimetypes import time +import warnings from collections import defaultdict from contextlib import suppress from ftplib import FTP @@ -24,16 +25,17 @@ from itemadapter import ItemAdapter from twisted.internet.defer import Deferred, maybeDeferred from twisted.internet.threads import deferToThread -from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict +from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.python import to_bytes +from scrapy.utils.python import get_func_args, to_bytes from scrapy.utils.request import referer_str if TYPE_CHECKING: @@ -46,6 +48,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy import Spider + from scrapy.crawler import Crawler logger = logging.getLogger(__name__) @@ -443,6 +446,8 @@ class FilesPipeline(MediaPipeline): store_uri: str | PathLike[str], download_func: Callable[[Request, Spider], Response] | None = None, settings: Settings | dict[str, Any] | None = None, + *, + crawler: Crawler | None = None, ): store_uri = _to_string(store_uri) if not store_uri: @@ -467,10 +472,35 @@ class FilesPipeline(MediaPipeline): resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD ) - super().__init__(download_func=download_func, settings=settings) + super().__init__( + download_func=download_func, settings=settings, crawler=crawler + ) @classmethod def from_settings(cls, settings: Settings) -> Self: + warnings.warn( + f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return cls._from_settings(settings, None) + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + if method_is_overridden(cls, FilesPipeline, "from_settings"): + warnings.warn( + f"{cls.__name__} overrides FilesPipeline.from_settings()." + f" This method is deprecated and won't be called in future Scrapy versions," + f" please update your code so that it overrides from_crawler() instead.", + category=ScrapyDeprecationWarning, + ) + o = cls.from_settings(crawler.settings) + o._finish_init(crawler) + return o + return cls._from_settings(crawler.settings, crawler) + + @classmethod + def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self: s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] @@ -495,7 +525,18 @@ class FilesPipeline(MediaPipeline): ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") store_uri = settings["FILES_STORE"] - return cls(store_uri, settings=settings) + if "crawler" in get_func_args(cls.__init__): + o = cls(store_uri, settings=settings, crawler=crawler) + else: + o = cls(store_uri, settings=settings) + if crawler: + o._finish_init(crawler) + warnings.warn( + f"{cls.__qualname__}.__init__() doesn't take a crawler argument." + " This is deprecated and the argument will be required in future Scrapy versions.", + category=ScrapyDeprecationWarning, + ) + return o def _get_store(self, uri: str) -> FilesStoreProtocol: if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 2c4c9376e..71da6a196 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -8,13 +8,14 @@ from __future__ import annotations import functools import hashlib +import warnings from contextlib import suppress from io import BytesIO from typing import TYPE_CHECKING, Any, cast from itemadapter import ItemAdapter -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import ( @@ -26,7 +27,7 @@ from scrapy.pipelines.files import ( _md5sum, ) from scrapy.settings import Settings -from scrapy.utils.python import to_bytes +from scrapy.utils.python import get_func_args, to_bytes if TYPE_CHECKING: from collections.abc import Callable, Iterable @@ -38,6 +39,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy import Spider + from scrapy.crawler import Crawler from scrapy.pipelines.media import FileInfoOrError, MediaPipeline @@ -64,6 +66,8 @@ class ImagesPipeline(FilesPipeline): store_uri: str | PathLike[str], download_func: Callable[[Request, Spider], Response] | None = None, settings: Settings | dict[str, Any] | None = None, + *, + crawler: Crawler | None = None, ): try: from PIL import Image @@ -74,7 +78,9 @@ class ImagesPipeline(FilesPipeline): "ImagesPipeline requires installing Pillow 4.0.0 or later" ) - super().__init__(store_uri, settings=settings, download_func=download_func) + super().__init__( + store_uri, settings=settings, download_func=download_func, crawler=crawler + ) if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -108,7 +114,7 @@ class ImagesPipeline(FilesPipeline): ) @classmethod - def from_settings(cls, settings: Settings) -> Self: + def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self: s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] @@ -133,7 +139,18 @@ class ImagesPipeline(FilesPipeline): ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") store_uri = settings["IMAGES_STORE"] - return cls(store_uri, settings=settings) + if "crawler" in get_func_args(cls.__init__): + o = cls(store_uri, settings=settings, crawler=crawler) + else: + o = cls(store_uri, settings=settings) + if crawler: + o._finish_init(crawler) + warnings.warn( + f"{cls.__qualname__}.__init__() doesn't take a crawler argument." + " This is deprecated and the argument will be required in future Scrapy versions.", + category=ScrapyDeprecationWarning, + ) + return o def file_downloaded( self, diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index b10ec147b..99abed09e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -2,6 +2,7 @@ from __future__ import annotations import functools import logging +import warnings from abc import ABC, abstractmethod from collections import defaultdict from typing import ( @@ -20,12 +21,14 @@ from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure from twisted.python.versions import Version +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http.request import NO_CALLBACK, Request from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import defer_result, mustbe_deferred from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter +from scrapy.utils.python import get_func_args if TYPE_CHECKING: from collections.abc import Callable @@ -38,7 +41,6 @@ if TYPE_CHECKING: from scrapy.http import Response from scrapy.utils.request import RequestFingerprinter - _T = TypeVar("_T") @@ -51,13 +53,13 @@ class FileInfo(TypedDict): FileInfoOrError = Union[tuple[Literal[True], FileInfo], tuple[Literal[False], Failure]] - logger = logging.getLogger(__name__) class MediaPipeline(ABC): crawler: Crawler _fingerprinter: RequestFingerprinter + _modern_init = False LOG_FAILED_RESULTS: bool = True @@ -74,6 +76,8 @@ class MediaPipeline(ABC): self, download_func: Callable[[Request, Spider], Response] | None = None, settings: Settings | dict[str, Any] | None = None, + *, + crawler: Crawler | None = None, ): self.download_func = download_func @@ -87,6 +91,28 @@ class MediaPipeline(ABC): ) self._handle_statuses(self.allow_redirects) + if crawler: + # TODO use crawler.settings + self._finish_init(crawler) + self._modern_init = True + else: + warnings.warn( + f"MediaPipeline.__init__() was called without the crawler argument" + f" when creating {self.__class__.__qualname__}." + f" This is deprecated and the argument will be required in future Scrapy versions.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + def _finish_init(self, crawler: Crawler) -> None: + # This was done in from_crawler() before 2.12, now it's done in __init__() + # if the crawler was passed to it and may be needed to be called in other + # deprecated code paths explicitly too. After the crawler argument of __init__() + # becomes mandatory this should be inlined there. + self.crawler = crawler + assert crawler.request_fingerprinter + self._fingerprinter = crawler.request_fingerprinter + def _handle_statuses(self, allow_redirects: bool) -> None: self.handle_httpstatus_list = None if allow_redirects: @@ -112,13 +138,19 @@ class MediaPipeline(ABC): @classmethod def from_crawler(cls, crawler: Crawler) -> Self: pipe: Self - try: + if hasattr(cls, "from_settings"): pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined] - except AttributeError: + elif "crawler" in get_func_args(cls.__init__): + pipe = cls(crawler=crawler) + else: pipe = cls() - pipe.crawler = crawler - assert crawler.request_fingerprinter - pipe._fingerprinter = crawler.request_fingerprinter + warnings.warn( + f"{cls.__qualname__}.__init__() doesn't take a crawler argument." + " This is deprecated and the argument will be required in future Scrapy versions.", + category=ScrapyDeprecationWarning, + ) + if not pipe._modern_init: + pipe._finish_init(crawler) return pipe def open_spider(self, spider: Spider) -> None: diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 83eaa1fdd..5e94f9271 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -17,7 +17,6 @@ from itemadapter import ItemAdapter from twisted.internet import defer from twisted.trial import unittest -from scrapy import Spider from scrapy.http import Request, Response from scrapy.item import Field, Item from scrapy.pipelines.files import ( @@ -27,7 +26,6 @@ from scrapy.pipelines.files import ( GCSFilesStore, S3FilesStore, ) -from scrapy.settings import Settings from scrapy.utils.test import ( assert_gcs_environ, get_crawler, @@ -219,8 +217,8 @@ class FilesPipelineTestCase(unittest.TestCase): def file_path(self, request, response=None, info=None, item=None): return f'full/{item.get("path")}' - file_path = CustomFilesPipeline.from_settings( - Settings({"FILES_STORE": self.tempdir}) + file_path = CustomFilesPipeline.from_crawler( + get_crawler(None, {"FILES_STORE": self.tempdir}) ).file_path item = {"path": "path-to-store-file"} request = Request("http://example.com") @@ -237,7 +235,9 @@ class FilesPipelineTestCaseFieldsMixin: def test_item_fields_default(self): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) - pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir})) + pipeline = FilesPipeline.from_crawler( + get_crawler(None, {"FILES_STORE": self.tempdir}) + ) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) results = [(True, {"url": url})] @@ -249,13 +249,14 @@ class FilesPipelineTestCaseFieldsMixin: def test_item_fields_override_settings(self): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", custom_file_urls=[url]) - pipeline = FilesPipeline.from_settings( - Settings( + pipeline = FilesPipeline.from_crawler( + get_crawler( + None, { "FILES_STORE": self.tempdir, "FILES_URLS_FIELD": "custom_file_urls", "FILES_RESULT_FIELD": "custom_files", - } + }, ) ) requests = list(pipeline.get_media_requests(item, None)) @@ -373,8 +374,10 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): different settings. """ custom_settings = self._generate_fake_settings() - another_pipeline = FilesPipeline.from_settings(Settings(custom_settings)) - one_pipeline = FilesPipeline(self.tempdir) + another_pipeline = FilesPipeline.from_crawler( + get_crawler(None, custom_settings) + ) + one_pipeline = FilesPipeline(self.tempdir, crawler=get_crawler(None)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: default_value = self.default_cls_settings[pipe_attr] self.assertEqual(getattr(one_pipeline, pipe_attr), default_value) @@ -387,7 +390,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): If subclasses override class attributes and there are no special settings those values should be kept. """ pipe_cls = self._generate_fake_pipeline() - pipe = pipe_cls.from_settings(Settings({"FILES_STORE": self.tempdir})) + pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": self.tempdir})) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: custom_value = getattr(pipe, pipe_ins_attr) self.assertNotEqual(custom_value, self.default_cls_settings[pipe_attr]) @@ -400,7 +403,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): """ pipeline_cls = self._generate_fake_pipeline() settings = self._generate_fake_settings() - pipeline = pipeline_cls.from_settings(Settings(settings)) + pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: value = getattr(pipeline, pipe_ins_attr) setting_value = settings.get(settings_attr) @@ -416,8 +419,8 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): class UserDefinedFilesPipeline(FilesPipeline): pass - user_pipeline = UserDefinedFilesPipeline.from_settings( - Settings({"FILES_STORE": self.tempdir}) + user_pipeline = UserDefinedFilesPipeline.from_crawler( + get_crawler(None, {"FILES_STORE": self.tempdir}) ) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. @@ -435,7 +438,9 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): prefix = UserDefinedFilesPipeline.__name__.upper() settings = self._generate_fake_settings(prefix=prefix) - user_pipeline = UserDefinedFilesPipeline.from_settings(Settings(settings)) + user_pipeline = UserDefinedFilesPipeline.from_crawler( + get_crawler(None, settings) + ) for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = settings.get(prefix + "_" + settings_attr) @@ -450,7 +455,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline_cls = self._generate_fake_pipeline() prefix = pipeline_cls.__name__.upper() settings = self._generate_fake_settings(prefix=prefix) - user_pipeline = pipeline_cls.from_settings(Settings(settings)) + user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for ( pipe_cls_attr, settings_attr, @@ -465,8 +470,8 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): DEFAULT_FILES_RESULT_FIELD = "this" DEFAULT_FILES_URLS_FIELD = "that" - pipeline = UserDefinedFilesPipeline.from_settings( - Settings({"FILES_STORE": self.tempdir}) + pipeline = UserDefinedFilesPipeline.from_crawler( + get_crawler(None, {"FILES_STORE": self.tempdir}) ) self.assertEqual( pipeline.files_result_field, @@ -486,7 +491,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): class UserPipe(FilesPipeline): pass - pipeline_cls = UserPipe.from_settings(Settings(settings)) + pipeline_cls = UserPipe.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: expected_value = settings.get(settings_attr) @@ -497,8 +502,8 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): def file_path(self, request, response=None, info=None, *, item=None): return Path("subdir") / Path(request.url).name - pipeline = CustomFilesPipelineWithPathLikeDir.from_settings( - Settings({"FILES_STORE": Path("./Temp")}) + pipeline = CustomFilesPipelineWithPathLikeDir.from_crawler( + get_crawler(None, {"FILES_STORE": Path("./Temp")}) ) request = Request("http://example.com/image01.jpg") self.assertEqual(pipeline.file_path(request), Path("subdir/image01.jpg")) @@ -695,7 +700,7 @@ def _prepare_request_object(item_url, flags=None): class BuildFromCrawlerTestCase(unittest.TestCase): def setUp(self): self.tempdir = mkdtemp() - self.crawler = get_crawler(Spider, {"FILES_STORE": self.tempdir}) + self.crawler = get_crawler(None, {"FILES_STORE": self.tempdir}) def tearDown(self): rmtree(self.tempdir) @@ -711,8 +716,23 @@ class BuildFromCrawlerTestCase(unittest.TestCase): self.assertEqual(len(w), 0) assert pipe.store + def test_has_old_init(self): + class Pipeline(FilesPipeline): + def __init__(self, store_uri, download_func=None, settings=None): + super().__init__(store_uri, download_func, settings) + self._init_called = True + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + self.assertEqual(len(w), 2) + assert pipe._init_called + def test_has_from_settings(self): class Pipeline(FilesPipeline): + _from_settings_called = False + @classmethod def from_settings(cls, settings): o = super().from_settings(settings) @@ -723,27 +743,24 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + self.assertEqual(len(w), 3) assert pipe.store assert pipe._from_settings_called - @pytest.mark.xfail( - reason="No way to override MediaPipeline.from_crawler having non-trivial __init__" - ) def test_has_from_crawler_and_init(self): class Pipeline(FilesPipeline): + _from_crawler_called = False + @classmethod def from_crawler(cls, crawler): settings = crawler.settings store_uri = settings["FILES_STORE"] - # you can either call super().from_crawler() or cls.__init__() but you need both - o = cls(store_uri, settings=settings) + o = cls(store_uri, settings=settings, crawler=crawler) o._from_crawler_called = True return o with warnings.catch_warnings(record=True) as w: pipe = Pipeline.from_crawler(self.crawler) - # this and the next assert will fail as MediaPipeline.from_crawler() wasn't called assert pipe.crawler == self.crawler assert pipe._fingerprinter self.assertEqual(len(w), 0) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index dfeead999..3f18c83f7 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -14,6 +14,7 @@ from scrapy.http import Request, Response from scrapy.item import Field, Item from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.settings import Settings +from scrapy.utils.test import get_crawler skip_pillow: str | None try: @@ -33,7 +34,8 @@ class ImagesPipelineTestCase(unittest.TestCase): def setUp(self): self.tempdir = mkdtemp() - self.pipeline = ImagesPipeline(self.tempdir) + crawler = get_crawler() + self.pipeline = ImagesPipeline(self.tempdir, crawler=crawler) def tearDown(self): rmtree(self.tempdir) @@ -123,8 +125,8 @@ class ImagesPipelineTestCase(unittest.TestCase): ): return f"thumb/{thumb_id}/{item.get('path')}" - thumb_path = CustomImagesPipeline.from_settings( - Settings({"IMAGES_STORE": self.tempdir}) + thumb_path = CustomImagesPipeline.from_crawler( + get_crawler(None, {"IMAGES_STORE": self.tempdir}) ).thumb_path item = {"path": "path-to-store-file"} request = Request("http://example.com") @@ -218,8 +220,8 @@ class ImagesPipelineTestCaseFieldsMixin: def test_item_fields_default(self): url = "http://www.example.com/images/1.jpg" item = self.item_class(name="item1", image_urls=[url]) - pipeline = ImagesPipeline.from_settings( - Settings({"IMAGES_STORE": "s3://example/images/"}) + pipeline = ImagesPipeline.from_crawler( + get_crawler(None, {"IMAGES_STORE": "s3://example/images/"}) ) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) @@ -232,13 +234,14 @@ class ImagesPipelineTestCaseFieldsMixin: def test_item_fields_override_settings(self): url = "http://www.example.com/images/1.jpg" item = self.item_class(name="item1", custom_image_urls=[url]) - pipeline = ImagesPipeline.from_settings( - Settings( + pipeline = ImagesPipeline.from_crawler( + get_crawler( + None, { "IMAGES_STORE": "s3://example/images/", "IMAGES_URLS_FIELD": "custom_image_urls", "IMAGES_RESULT_FIELD": "custom_images", - } + }, ) ) requests = list(pipeline.get_media_requests(item, None)) @@ -390,8 +393,10 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): """ custom_settings = self._generate_fake_settings() default_settings = Settings() - default_sts_pipe = ImagesPipeline(self.tempdir, settings=default_settings) - user_sts_pipe = ImagesPipeline.from_settings(Settings(custom_settings)) + default_sts_pipe = ImagesPipeline( + self.tempdir, settings=default_settings, crawler=get_crawler(None) # TODO + ) + user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_default_value = self.default_pipeline_settings.get(pipe_attr) custom_value = custom_settings.get(settings_attr) @@ -407,7 +412,9 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): from class attributes. """ pipeline_cls = self._generate_fake_pipeline_subclass() - pipeline = pipeline_cls.from_settings(Settings({"IMAGES_STORE": self.tempdir})) + pipeline = pipeline_cls.from_crawler( + get_crawler(None, {"IMAGES_STORE": self.tempdir}) + ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to class attribute (uppercase). attr_value = getattr(pipeline, pipe_attr.lower()) @@ -421,7 +428,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): """ pipeline_cls = self._generate_fake_pipeline_subclass() settings = self._generate_fake_settings() - pipeline = pipeline_cls.from_settings(Settings(settings)) + pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to # value defined in settings. @@ -439,8 +446,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): class UserDefinedImagePipeline(ImagesPipeline): pass - user_pipeline = UserDefinedImagePipeline.from_settings( - Settings({"IMAGES_STORE": self.tempdir}) + user_pipeline = UserDefinedImagePipeline.from_crawler( + get_crawler(None, {"IMAGES_STORE": self.tempdir}) ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. @@ -458,7 +465,9 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): prefix = UserDefinedImagePipeline.__name__.upper() settings = self._generate_fake_settings(prefix=prefix) - user_pipeline = UserDefinedImagePipeline.from_settings(Settings(settings)) + user_pipeline = UserDefinedImagePipeline.from_crawler( + get_crawler(None, settings) + ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = settings.get(prefix + "_" + settings_attr) @@ -473,7 +482,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline_cls = self._generate_fake_pipeline_subclass() prefix = pipeline_cls.__name__.upper() settings = self._generate_fake_settings(prefix=prefix) - user_pipeline = pipeline_cls.from_settings(Settings(settings)) + user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: custom_value = settings.get(prefix + "_" + settings_attr) self.assertNotEqual(custom_value, self.default_pipeline_settings[pipe_attr]) @@ -484,8 +493,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): DEFAULT_IMAGES_URLS_FIELD = "something" DEFAULT_IMAGES_RESULT_FIELD = "something_else" - pipeline = UserDefinedImagePipeline.from_settings( - Settings({"IMAGES_STORE": self.tempdir}) + pipeline = UserDefinedImagePipeline.from_crawler( + get_crawler(None, {"IMAGES_STORE": self.tempdir}) ) self.assertEqual( pipeline.images_result_field, @@ -506,7 +515,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): class UserPipe(ImagesPipeline): pass - pipeline_cls = UserPipe.from_settings(Settings(settings)) + pipeline_cls = UserPipe.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_value = settings.get(settings_attr) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 920b4246e..a825de92a 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -2,7 +2,6 @@ from __future__ import annotations import warnings -import pytest from testfixtures import LogCapture from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks @@ -14,7 +13,6 @@ from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException from scrapy.pipelines.media import MediaPipeline -from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all @@ -178,8 +176,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase): class MockedMediaPipeline(UserDefinedPipeline): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self, *args, crawler=None, **kwargs): + super().__init__(*args, crawler=crawler, **kwargs) self._mockcalled = [] def download(self, request, info): @@ -380,7 +378,8 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): def _assert_request_no3xx(self, pipeline_class, settings): - pipe = pipeline_class(settings=Settings(settings)) + crawler = get_crawler(None, settings) + pipe = pipeline_class(settings=settings, crawler=crawler) # TODO request = Request("http://url") pipe._modify_media_request(request) @@ -417,7 +416,7 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): class BuildFromCrawlerTestCase(unittest.TestCase): def setUp(self): - self.crawler = get_crawler(Spider, {"FILES_STORE": "/foo"}) + self.crawler = get_crawler(None, {"FILES_STORE": "/foo"}) def test_simple(self): class Pipeline(UserDefinedPipeline): @@ -429,8 +428,23 @@ class BuildFromCrawlerTestCase(unittest.TestCase): assert pipe._fingerprinter self.assertEqual(len(w), 0) + def test_has_old_init(self): + class Pipeline(UserDefinedPipeline): + def __init__(self): + super().__init__() + self._init_called = True + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + self.assertEqual(len(w), 2) + assert pipe._init_called + def test_has_from_settings(self): class Pipeline(UserDefinedPipeline): + _from_settings_called = False + @classmethod def from_settings(cls, settings): o = cls() @@ -441,11 +455,13 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + self.assertEqual(len(w), 1) assert pipe._from_settings_called def test_has_from_settings_and_init(self): class Pipeline(UserDefinedPipeline): + _from_settings_called = False + def __init__(self, store_uri, settings): super().__init__() self._init_called = True @@ -461,31 +477,28 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + self.assertEqual(len(w), 1) assert pipe._from_settings_called assert pipe._init_called - @pytest.mark.xfail( - reason="No way to override MediaPipeline.from_crawler having non-trivial __init__" - ) def test_has_from_crawler_and_init(self): class Pipeline(UserDefinedPipeline): - def __init__(self, store_uri, settings): - super().__init__() + _from_crawler_called = False + + def __init__(self, store_uri, settings, *, crawler): + super().__init__(crawler=crawler) self._init_called = True @classmethod def from_crawler(cls, crawler): settings = crawler.settings store_uri = settings["FILES_STORE"] - # you can either call super().from_crawler() or cls.__init__() but you need both - o = cls(store_uri, settings=settings) + o = cls(store_uri, settings=settings, crawler=crawler) o._from_crawler_called = True return o with warnings.catch_warnings(record=True) as w: pipe = Pipeline.from_crawler(self.crawler) - # this and the next assert will fail as super().from_crawler() wasn't called assert pipe.crawler == self.crawler assert pipe._fingerprinter self.assertEqual(len(w), 0) @@ -494,6 +507,8 @@ class BuildFromCrawlerTestCase(unittest.TestCase): def test_has_from_crawler(self): class Pipeline(UserDefinedPipeline): + _from_crawler_called = False + @classmethod def from_crawler(cls, crawler): settings = crawler.settings From 6aa4d2b4ab28b8f657645de613295ea2498e8cee Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 14 Nov 2024 02:01:04 +0500 Subject: [PATCH 1575/2083] Prefer crawler.settings over settings in media pipelines. --- scrapy/pipelines/files.py | 18 +++++++++++++++--- scrapy/pipelines/images.py | 19 ++++++++++++++++--- scrapy/pipelines/media.py | 13 +++++++++++-- tests/test_pipeline_files.py | 2 +- tests/test_pipeline_images.py | 6 +----- tests/test_pipeline_media.py | 3 +-- 6 files changed, 45 insertions(+), 16 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index f83037e6c..3b730c432 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -453,7 +453,17 @@ class FilesPipeline(MediaPipeline): if not store_uri: raise NotConfigured - if isinstance(settings, dict) or settings is None: + if crawler is not None: + if settings is not None: + warnings.warn( + f"FilesPipeline.__init__() was called with a crawler instance and a settings instance" + f" when creating {self.__class__.__qualname__}. The settings instance will be ignored" + f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + settings = crawler.settings + elif isinstance(settings, dict) or settings is None: settings = Settings(settings) cls_name = "FilesPipeline" self.store: FilesStoreProtocol = self._get_store(store_uri) @@ -473,7 +483,9 @@ class FilesPipeline(MediaPipeline): ) super().__init__( - download_func=download_func, settings=settings, crawler=crawler + download_func=download_func, + settings=settings if not crawler else None, + crawler=crawler, ) @classmethod @@ -526,7 +538,7 @@ class FilesPipeline(MediaPipeline): store_uri = settings["FILES_STORE"] if "crawler" in get_func_args(cls.__init__): - o = cls(store_uri, settings=settings, crawler=crawler) + o = cls(store_uri, crawler=crawler) else: o = cls(store_uri, settings=settings) if crawler: diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 71da6a196..fa26133bb 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -79,10 +79,23 @@ class ImagesPipeline(FilesPipeline): ) super().__init__( - store_uri, settings=settings, download_func=download_func, crawler=crawler + store_uri, + settings=settings if not crawler else None, + download_func=download_func, + crawler=crawler, ) - if isinstance(settings, dict) or settings is None: + if crawler is not None: + if settings is not None: + warnings.warn( + f"ImagesPipeline.__init__() was called with a crawler instance and a settings instance" + f" when creating {self.__class__.__qualname__}. The settings instance will be ignored" + f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + settings = crawler.settings + elif isinstance(settings, dict) or settings is None: settings = Settings(settings) resolve = functools.partial( @@ -140,7 +153,7 @@ class ImagesPipeline(FilesPipeline): store_uri = settings["IMAGES_STORE"] if "crawler" in get_func_args(cls.__init__): - o = cls(store_uri, settings=settings, crawler=crawler) + o = cls(store_uri, crawler=crawler) else: o = cls(store_uri, settings=settings) if crawler: diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 99abed09e..70c52d090 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -81,7 +81,17 @@ class MediaPipeline(ABC): ): self.download_func = download_func - if isinstance(settings, dict) or settings is None: + if crawler is not None: + if settings is not None: + warnings.warn( + f"MediaPipeline.__init__() was called with a crawler instance and a settings instance" + f" when creating {self.__class__.__qualname__}. The settings instance will be ignored" + f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + settings = crawler.settings + elif isinstance(settings, dict) or settings is None: settings = Settings(settings) resolve = functools.partial( self._key_for_pipe, base_class_name="MediaPipeline", settings=settings @@ -92,7 +102,6 @@ class MediaPipeline(ABC): self._handle_statuses(self.allow_redirects) if crawler: - # TODO use crawler.settings self._finish_init(crawler) self._modern_init = True else: diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 5e94f9271..9dcb3e4d1 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -755,7 +755,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): def from_crawler(cls, crawler): settings = crawler.settings store_uri = settings["FILES_STORE"] - o = cls(store_uri, settings=settings, crawler=crawler) + o = cls(store_uri, crawler=crawler) o._from_crawler_called = True return o diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 3f18c83f7..3ffef4102 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -13,7 +13,6 @@ from twisted.trial import unittest from scrapy.http import Request, Response from scrapy.item import Field, Item from scrapy.pipelines.images import ImageException, ImagesPipeline -from scrapy.settings import Settings from scrapy.utils.test import get_crawler skip_pillow: str | None @@ -392,10 +391,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): have different settings. """ custom_settings = self._generate_fake_settings() - default_settings = Settings() - default_sts_pipe = ImagesPipeline( - self.tempdir, settings=default_settings, crawler=get_crawler(None) # TODO - ) + default_sts_pipe = ImagesPipeline(self.tempdir, crawler=get_crawler(None)) user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_default_value = self.default_pipeline_settings.get(pipe_attr) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index a825de92a..58a2d3678 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -378,8 +378,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): def _assert_request_no3xx(self, pipeline_class, settings): - crawler = get_crawler(None, settings) - pipe = pipeline_class(settings=settings, crawler=crawler) # TODO + pipe = pipeline_class(crawler=get_crawler(None, settings)) request = Request("http://url") pipe._modify_media_request(request) From 2ad5f0c12bfafc66fda6eb71790f447d2f7b8c13 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 14 Nov 2024 13:03:04 +0500 Subject: [PATCH 1576/2083] Extract duplicated code. --- scrapy/pipelines/files.py | 33 ++++++++++++++++++--------------- scrapy/pipelines/images.py | 35 +++-------------------------------- 2 files changed, 21 insertions(+), 47 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 3b730c432..065d822f3 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -29,7 +29,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWar from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline -from scrapy.settings import Settings +from scrapy.settings import BaseSettings, Settings from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.deprecate import method_is_overridden @@ -513,6 +513,23 @@ class FilesPipeline(MediaPipeline): @classmethod def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self: + cls._update_stores(settings) + store_uri = settings["FILES_STORE"] + if "crawler" in get_func_args(cls.__init__): + o = cls(store_uri, crawler=crawler) + else: + o = cls(store_uri, settings=settings) + if crawler: + o._finish_init(crawler) + warnings.warn( + f"{cls.__qualname__}.__init__() doesn't take a crawler argument." + " This is deprecated and the argument will be required in future Scrapy versions.", + category=ScrapyDeprecationWarning, + ) + return o + + @classmethod + def _update_stores(cls, settings: BaseSettings) -> None: s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] @@ -536,20 +553,6 @@ class FilesPipeline(MediaPipeline): ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") - store_uri = settings["FILES_STORE"] - if "crawler" in get_func_args(cls.__init__): - o = cls(store_uri, crawler=crawler) - else: - o = cls(store_uri, settings=settings) - if crawler: - o._finish_init(crawler) - warnings.warn( - f"{cls.__qualname__}.__init__() doesn't take a crawler argument." - " This is deprecated and the argument will be required in future Scrapy versions.", - category=ScrapyDeprecationWarning, - ) - return o - def _get_store(self, uri: str) -> FilesStoreProtocol: if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir scheme = "file" diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index fa26133bb..7defafb26 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,21 +11,14 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any from itemadapter import ItemAdapter from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.files import ( - FileException, - FilesPipeline, - FTPFilesStore, - GCSFilesStore, - S3FilesStore, - _md5sum, -) +from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum from scrapy.settings import Settings from scrapy.utils.python import get_func_args, to_bytes @@ -128,29 +121,7 @@ class ImagesPipeline(FilesPipeline): @classmethod def _from_settings(cls, settings: Settings, crawler: Crawler | None) -> Self: - s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"]) - s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"] - s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"] - s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"] - s3store.AWS_ENDPOINT_URL = settings["AWS_ENDPOINT_URL"] - s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"] - s3store.AWS_USE_SSL = settings["AWS_USE_SSL"] - s3store.AWS_VERIFY = settings["AWS_VERIFY"] - s3store.POLICY = settings["IMAGES_STORE_S3_ACL"] - - gcs_store: type[GCSFilesStore] = cast( - type[GCSFilesStore], cls.STORE_SCHEMES["gs"] - ) - gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"] - gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None - - ftp_store: type[FTPFilesStore] = cast( - type[FTPFilesStore], cls.STORE_SCHEMES["ftp"] - ) - ftp_store.FTP_USERNAME = settings["FTP_USER"] - ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"] - ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") - + cls._update_stores(settings) store_uri = settings["IMAGES_STORE"] if "crawler" in get_func_args(cls.__init__): o = cls(store_uri, crawler=crawler) From 929d665a74333434c9cede7133ea4f0707dbf9a6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 14 Nov 2024 19:35:56 +0500 Subject: [PATCH 1577/2083] Address PR feedback. --- scrapy/middleware.py | 8 -------- scrapy/pipelines/files.py | 8 ++++---- scrapy/pipelines/images.py | 6 +++--- scrapy/pipelines/media.py | 8 ++++---- 4 files changed, 11 insertions(+), 19 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 91411506f..2b67dcd21 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -54,14 +54,6 @@ class MiddlewareManager: @staticmethod def _build_from_settings(objcls: type[_T], settings: BaseSettings) -> _T: if hasattr(objcls, "from_settings"): - warnings.warn( - f"{objcls.__qualname__} has from_settings() but not from_crawler()." - " This is deprecated and calling from_settings() will be removed in a future" - " Scrapy version. You can implement a simple from_crawler() that calls" - " from_settings() with crawler.settings.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) instance = objcls.from_settings(settings) # type: ignore[attr-defined] method_name = "from_settings" else: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 065d822f3..196b54acb 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -35,7 +35,7 @@ from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.python import get_func_args, to_bytes +from scrapy.utils.python import get_func_args, global_object_name, to_bytes from scrapy.utils.request import referer_str if TYPE_CHECKING: @@ -457,7 +457,7 @@ class FilesPipeline(MediaPipeline): if settings is not None: warnings.warn( f"FilesPipeline.__init__() was called with a crawler instance and a settings instance" - f" when creating {self.__class__.__qualname__}. The settings instance will be ignored" + f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored" f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.", category=ScrapyDeprecationWarning, stacklevel=2, @@ -501,7 +501,7 @@ class FilesPipeline(MediaPipeline): def from_crawler(cls, crawler: Crawler) -> Self: if method_is_overridden(cls, FilesPipeline, "from_settings"): warnings.warn( - f"{cls.__name__} overrides FilesPipeline.from_settings()." + f"{global_object_name(cls)} overrides FilesPipeline.from_settings()." f" This method is deprecated and won't be called in future Scrapy versions," f" please update your code so that it overrides from_crawler() instead.", category=ScrapyDeprecationWarning, @@ -522,7 +522,7 @@ class FilesPipeline(MediaPipeline): if crawler: o._finish_init(crawler) warnings.warn( - f"{cls.__qualname__}.__init__() doesn't take a crawler argument." + f"{global_object_name(cls)}.__init__() doesn't take a crawler argument." " This is deprecated and the argument will be required in future Scrapy versions.", category=ScrapyDeprecationWarning, ) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 7defafb26..e86e7c493 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -20,7 +20,7 @@ from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum from scrapy.settings import Settings -from scrapy.utils.python import get_func_args, to_bytes +from scrapy.utils.python import get_func_args, global_object_name, to_bytes if TYPE_CHECKING: from collections.abc import Callable, Iterable @@ -82,7 +82,7 @@ class ImagesPipeline(FilesPipeline): if settings is not None: warnings.warn( f"ImagesPipeline.__init__() was called with a crawler instance and a settings instance" - f" when creating {self.__class__.__qualname__}. The settings instance will be ignored" + f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored" f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.", category=ScrapyDeprecationWarning, stacklevel=2, @@ -130,7 +130,7 @@ class ImagesPipeline(FilesPipeline): if crawler: o._finish_init(crawler) warnings.warn( - f"{cls.__qualname__}.__init__() doesn't take a crawler argument." + f"{global_object_name(cls)}.__init__() doesn't take a crawler argument." " This is deprecated and the argument will be required in future Scrapy versions.", category=ScrapyDeprecationWarning, ) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 70c52d090..6d7808c31 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -28,7 +28,7 @@ from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import defer_result, mustbe_deferred from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import get_func_args +from scrapy.utils.python import get_func_args, global_object_name if TYPE_CHECKING: from collections.abc import Callable @@ -85,7 +85,7 @@ class MediaPipeline(ABC): if settings is not None: warnings.warn( f"MediaPipeline.__init__() was called with a crawler instance and a settings instance" - f" when creating {self.__class__.__qualname__}. The settings instance will be ignored" + f" when creating {global_object_name(self.__class__)}. The settings instance will be ignored" f" and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.", category=ScrapyDeprecationWarning, stacklevel=2, @@ -107,7 +107,7 @@ class MediaPipeline(ABC): else: warnings.warn( f"MediaPipeline.__init__() was called without the crawler argument" - f" when creating {self.__class__.__qualname__}." + f" when creating {global_object_name(self.__class__)}." f" This is deprecated and the argument will be required in future Scrapy versions.", category=ScrapyDeprecationWarning, stacklevel=2, @@ -154,7 +154,7 @@ class MediaPipeline(ABC): else: pipe = cls() warnings.warn( - f"{cls.__qualname__}.__init__() doesn't take a crawler argument." + f"{global_object_name(cls)}.__init__() doesn't take a crawler argument." " This is deprecated and the argument will be required in future Scrapy versions.", category=ScrapyDeprecationWarning, ) From bfcee452b0f90dc3c642604bb77cd37f22ac0af1 Mon Sep 17 00:00:00 2001 From: Nicholas Laustrup <124007393+nicklaustrup@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:40:12 -0800 Subject: [PATCH 1578/2083] Added failing test cases to tests/test_contracts.py and fixed corresponding methods + removed pylint comments --- scrapy/contracts/__init__.py | 22 +++++--------- tests/test_contracts.py | 58 ++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 15 deletions(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 9071395e3..3b4f932a0 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -38,9 +38,7 @@ class Contract: assert cb is not None @wraps(cb) - def wrapper( # pylint: disable=inconsistent-return-statements - response: Response, **cb_kwargs: Any - ) -> list[Any]: + def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: try: results.startTest(self.testcase_pre) self.pre_process(response) @@ -51,13 +49,10 @@ class Contract: results.addError(self.testcase_pre, sys.exc_info()) else: results.addSuccess(self.testcase_pre) - finally: - cb_result = cb(response, **cb_kwargs) - if isinstance(cb_result, (AsyncGenerator, CoroutineType)): - raise TypeError("Contracts don't support async callbacks") - return list( # pylint: disable=return-in-finally - cast(Iterable[Any], iterate_spider_output(cb_result)) - ) + cb_result = cb(response, **cb_kwargs) + if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + raise TypeError("Contracts don't support async callbacks") + return list(cast(Iterable[Any], iterate_spider_output(cb_result))) request.callback = wrapper @@ -69,9 +64,7 @@ class Contract: assert cb is not None @wraps(cb) - def wrapper( # pylint: disable=inconsistent-return-statements - response: Response, **cb_kwargs: Any - ) -> list[Any]: + def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") @@ -86,8 +79,7 @@ class Contract: results.addError(self.testcase_post, sys.exc_info()) else: results.addSuccess(self.testcase_post) - finally: - return output # pylint: disable=return-in-finally + return output request.callback = wrapper diff --git a/tests/test_contracts.py b/tests/test_contracts.py index d578b3af4..b0cb92d12 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -556,3 +556,61 @@ class ContractsManagerTest(unittest.TestCase): requests = self.conman.from_spider(spider, self.results) self.assertTrue(requests) + + +class CustomFailContractPreProcess(Contract): + name = "test_contract" + + def pre_process(self, response): + raise KeyboardInterrupt("Pre-process exception") + + +class CustomFailContractPostProcess(Contract): + name = "test_contract" + + def post_process(self, response): + raise KeyboardInterrupt("Post-process exception") + + +class CustomContractPrePostProcess(unittest.TestCase): + + def setUp(self): + self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) + + def test_pre_hook_keyboard_interrupt(self): + spider = TestSpider() + response = ResponseMock() + contract = CustomFailContractPreProcess(spider.returns_request) + conman = ContractsManager([contract]) + + try: + request = conman.from_method(spider.returns_request, self.results) + contract.add_pre_hook(request, self.results) + # Expect this to raise a KeyboardInterrupt + request.callback(response, **request.cb_kwargs) + except KeyboardInterrupt as e: + self.assertEqual(str(e), "Pre-process exception") + else: + self.fail("KeyboardInterrupt not raised") + + self.assertFalse(self.results.failures) + self.assertFalse(self.results.errors) + + def test_post_hook_keyboard_interrupt(self): + spider = TestSpider() + response = ResponseMock() + contract = CustomFailContractPostProcess(spider.returns_request) + conman = ContractsManager([contract]) + + try: + request = conman.from_method(spider.returns_request, self.results) + contract.add_post_hook(request, self.results) + # Expect this to raise a KeyboardInterrupt + request.callback(response, **request.cb_kwargs) + except KeyboardInterrupt as e: + self.assertEqual(str(e), "Post-process exception") + else: + self.fail("KeyboardInterrupt not raised") + + self.assertFalse(self.results.failures) + self.assertFalse(self.results.errors) From dc4d6d16ead45932a564ea37eef03da92714f5cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com> Date: Fri, 15 Nov 2024 00:09:00 -0300 Subject: [PATCH 1579/2083] Verified PyPI releases (a.k.a. PEP740) --- .github/workflows/publish.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2cd556516..8e01ffd88 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,16 +10,20 @@ concurrency: jobs: publish: + name: Upload release to PyPI runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/Scrapy + permissions: + id-token: write steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.13" - run: | - pip install --upgrade build twine + python -m pip install --upgrade build python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.10.3 - with: - password: ${{ secrets.PYPI_TOKEN }} + uses: pypa/gh-action-pypi-publish@release/v1 From feea3a0f67f8e6f32ae6452f485709db16146c5e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 15 Nov 2024 21:08:18 +0500 Subject: [PATCH 1580/2083] Commit mitmproxy-dhparam.pem. --- tests/keys/mitmproxy-dhparam.pem | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 tests/keys/mitmproxy-dhparam.pem diff --git a/tests/keys/mitmproxy-dhparam.pem b/tests/keys/mitmproxy-dhparam.pem new file mode 100644 index 000000000..c10121fbf --- /dev/null +++ b/tests/keys/mitmproxy-dhparam.pem @@ -0,0 +1,14 @@ + +-----BEGIN DH PARAMETERS----- +MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3 +O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv +j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ +Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB +chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC +ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq +o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX +IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv +A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8 +6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I +rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI= +-----END DH PARAMETERS----- From 10089c6fe2028b879f9f60e9598fa580ef6a3e33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 18 Nov 2024 09:07:32 +0100 Subject: [PATCH 1581/2083] 2.12 release notes (#6226) * Cover 2.12 in the release notes up to 9bb973dc54766a0f8d10eca0947d11f195c1a1be * Add one more highlight * Better merge of the news entries. * Cover 2.12 in the release notes up to 642af40. * Cover 2.12 in the release notes up to 7a0a34b. * Cover 2.12 in the release notes up to b4bad97. * Add not yet merged PRs #6463, #6507, #6511 to the 2.12 release notes. * Cover 2.12 in the release notes up to d85c39f, small fixes. * Cover 2.12 in the release notes up to d215669. * Cover #6527 in the release notes. * Address PR feedback. * Cover recent PRs. * Finalize the 2.12.0 release notes, small additional fixes. --------- Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name> --- docs/news.rst | 550 +++++++++++++++++++++++++- docs/topics/addons.rst | 7 +- docs/topics/api.rst | 4 +- docs/topics/components.rst | 16 +- docs/topics/downloader-middleware.rst | 4 - docs/topics/spider-middleware.rst | 2 +- scrapy/crawler.py | 42 ++ scrapy/extensions/feedexport.py | 6 +- scrapy/pipelines/media.py | 14 +- scrapy/utils/misc.py | 2 + scrapy/utils/python.py | 3 +- tests/test_pipeline_media.py | 29 +- 12 files changed, 647 insertions(+), 32 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 3c9e58cca..025eb09ba 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,23 +3,555 @@ Release notes ============= -.. _release-VERSION: +.. _release-2.12.0: -Scrapy VERSION (YYYY-MM-DD) ---------------------------- +Scrapy 2.12.0 (2024-11-18) +-------------------------- -New features -~~~~~~~~~~~~ +Highlights: -- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``, - ``SpiderLoader`` does not raise :exc:`SyntaxError` but emits a warning instead. +- Dropped support for Python 3.8, added support for Python 3.13 + +- :meth:`~scrapy.Spider.start_requests` can now yield items + +- Added :class:`~scrapy.http.JsonResponse` + +- Added :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM` + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for Python 3.8. + (:issue:`6466`, :issue:`6472`) + +- Added support for Python 3.13. + (:issue:`6166`) + +- Minimum versions increased for these dependencies: + + - Twisted_: 18.9.0 → 21.7.0 + + - cryptography_: 36.0.0 → 37.0.0 + + - pyOpenSSL_: 21.0.0 → 22.0.0 + + - lxml_: 4.4.1 → 4.6.0 + +- Removed ``setuptools`` from the dependency list. + (:issue:`6487`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- User-defined cookies for HTTPS requests will have the ``secure`` flag set + to ``True`` unless it's set to ``False`` explictly. This is important when + these cookies are reused in HTTP requests, e.g. after a redirect to an HTTP + URL. + (:issue:`6357`) + +- The Reppy-based ``robots.txt`` parser, + ``scrapy.robotstxt.ReppyRobotParser``, was removed, as it doesn't support + Python 3.9+. + (:issue:`5230`, :issue:`6099`, :issue:`6499`) + +- The initialization API of :class:`scrapy.pipelines.media.MediaPipeline` and + its subclasses was improved and it's possible that some previously working + usage scenarios will no longer work. It can only affect you if you define + custom subclasses of ``MediaPipeline`` or create instances of these + pipelines via ``from_settings()`` or ``__init__()`` calls instead of + ``from_crawler()`` calls. + + Previously, ``MediaPipeline.from_crawler()`` called the ``from_settings()`` + method if it existed or the ``__init__()`` method otherwise, and then did + some additional initialization using the ``crawler`` instance. If the + ``from_settings()`` method existed (like in ``FilesPipeline``) it called + ``__init__()`` to create the instance. It wasn't possible to override + ``from_crawler()`` without calling ``MediaPipeline.from_crawler()`` from it + which, in turn, couldn't be called in some cases (including subclasses of + ``FilesPipeline``). + + Now, in line with the general usage of ``from_crawler()`` and + ``from_settings()`` and the deprecation of the latter the recommended + initialization order is the following one: + + - All ``__init__()`` methods should take a ``crawler`` argument. If they + also take a ``settings`` argument they should ignore it, using + ``crawler.settings`` instead. When they call ``__init__()`` of the base + class they should pass the ``crawler`` argument to it too. + - A ``from_settings()`` method shouldn't be defined. Class-specific + initialization code should go into either an overriden ``from_crawler()`` + method or into ``__init__()``. + - It's now possible to override ``from_crawler()`` and it's not necessary + to call ``MediaPipeline.from_crawler()`` in it if other recommendations + were followed. + - If pipeline instances were created with ``from_settings()`` or + ``__init__()`` calls (which wasn't supported even before, as it missed + important initialization code), they should now be created with + ``from_crawler()`` calls. + + (:issue:`6540`) + +- The ``response_body`` argument of :meth:`ImagesPipeline.convert_image + <scrapy.pipelines.images.ImagesPipeline.convert_image>` is now + positional-only, as it was changed from optional to required. + (:issue:`6500`) + +- The ``convert`` argument of :func:`scrapy.utils.conf.build_component_list` + is now positional-only, as the preceding argument (``custom``) was removed. + (:issue:`6500`) + +- The ``overwrite_output`` argument of + :func:`scrapy.utils.conf.feed_process_params_from_cli` is now + positional-only, as the preceding argument (``output_format``) was removed. + (:issue:`6500`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed the ``scrapy.utils.request.request_fingerprint()`` function, + deprecated in Scrapy 2.7.0. + (:issue:`6212`, :issue:`6213`) + +- Removed support for value ``"2.6"`` of setting + ``REQUEST_FINGERPRINTER_IMPLEMENTATION``, deprecated in Scrapy 2.7.0. + (:issue:`6212`, :issue:`6213`) + +- :class:`~scrapy.dupefilters.RFPDupeFilter` subclasses now require + supporting the ``fingerprinter`` parameter in their ``__init__`` method, + introduced in Scrapy 2.7.0. + (:issue:`6102`, :issue:`6113`) + +- Removed the ``scrapy.downloadermiddlewares.decompression`` module, + deprecated in Scrapy 2.7.0. + (:issue:`6100`, :issue:`6113`) + +- Removed the ``scrapy.utils.response.response_httprepr()`` function, + deprecated in Scrapy 2.6.0. + (:issue:`6111`, :issue:`6116`) + +- Spiders with spider-level HTTP authentication, i.e. with the ``http_user`` + or ``http_pass`` attributes, must now define ``http_auth_domain`` as well, + which was introduced in Scrapy 2.5.1. + (:issue:`6103`, :issue:`6113`) + +- :ref:`Media pipelines <topics-media-pipeline>` methods ``file_path()``, + ``file_downloaded()``, ``get_images()``, ``image_downloaded()``, + ``media_downloaded()``, ``media_to_download()``, and ``thumb_path()`` must + now support an ``item`` parameter, added in Scrapy 2.4.0. + (:issue:`6107`, :issue:`6113`) + +- The ``__init__()`` and ``from_crawler()`` methods of :ref:`feed storage + backend classes <topics-feed-storage>` must now support the keyword-only + ``feed_options`` parameter, introduced in Scrapy 2.4.0. + (:issue:`6105`, :issue:`6113`) + +- Removed the ``scrapy.loader.common`` and ``scrapy.loader.processors`` + modules, deprecated in Scrapy 2.3.0. + (:issue:`6106`, :issue:`6113`) + +- Removed the ``scrapy.utils.misc.extract_regex()`` function, deprecated in + Scrapy 2.3.0. + (:issue:`6106`, :issue:`6113`) + +- Removed the ``scrapy.http.JSONRequest`` class, replaced with + ``JsonRequest`` in Scrapy 1.8.0. + (:issue:`6110`, :issue:`6113`) + +- ``scrapy.utils.log.logformatter_adapter`` no longer supports missing + ``args``, ``level``, or ``msg`` parameters, and no longer supports a + ``format`` parameter, all scenarios that were deprecated in Scrapy 1.0.0. + (:issue:`6109`, :issue:`6116`) + +- A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that + does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface + will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at + run time. Non-compliant classes have been triggering a deprecation warning + since Scrapy 1.0.0. + (:issue:`6101`, :issue:`6113`) + +- Removed the ``--output-format``/``-t`` command line option, deprecated in + Scrapy 2.1.0. ``-O <URI>:<FORMAT>`` should be used instead. + (:issue:`6500`) + +- Running :meth:`~scrapy.crawler.Crawler.crawl` more than once on the same + :class:`~scrapy.crawler.Crawler` instance, deprecated in Scrapy 2.11.0, now + raises an exception. + (:issue:`6500`) + +- Subclassing + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + without support for the ``crawler`` argument in ``__init__()`` and without + a custom ``from_crawler()`` method, deprecated in Scrapy 2.5.0, is no + longer allowed. + (:issue:`6500`) + +- Removed the ``EXCEPTIONS_TO_RETRY`` attribute of + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`, deprecated in + Scrapy 2.10.0. + (:issue:`6500`) + +- Removed support for :ref:`S3 feed exports <topics-feed-storage-s3>` without + the boto3_ package installed, deprecated in Scrapy 2.10.0. + (:issue:`6500`) + +- Removed the ``scrapy.extensions.feedexport._FeedSlot`` class, deprecated in + Scrapy 2.10.0. + (:issue:`6500`) + +- Removed the ``scrapy.pipelines.images.NoimagesDrop`` exception, deprecated + in Scrapy 2.8.0. + (:issue:`6500`) + +- The ``response_body`` argument of :meth:`ImagesPipeline.convert_image + <scrapy.pipelines.images.ImagesPipeline.convert_image>` is now required, + not passing it was deprecated in Scrapy 2.8.0. + (:issue:`6500`) + +- Removed the ``custom`` argument of + :func:`scrapy.utils.conf.build_component_list`, deprecated in Scrapy + 2.10.0. + (:issue:`6500`) + +- Removed the ``scrapy.utils.reactor.get_asyncio_event_loop_policy()`` + function, deprecated in Scrapy 2.9.0. Use :func:`asyncio.get_event_loop` + and related standard library functions instead. + (:issue:`6500`) Deprecations ~~~~~~~~~~~~ -- :meth:`scrapy.core.downloader.Downloader._get_slot_key` is deprecated, use +- The ``from_settings()`` methods of the :ref:`Scrapy components + <topics-components>` that have them are now deprecated. ``from_crawler()`` + should now be used instead. Affected components: + + - :class:`scrapy.dupefilters.RFPDupeFilter` + - :class:`scrapy.mail.MailSender` + - :class:`scrapy.middleware.MiddlewareManager` + - :class:`scrapy.core.downloader.contextfactory.ScrapyClientContextFactory` + - :class:`scrapy.pipelines.files.FilesPipeline` + - :class:`scrapy.pipelines.images.ImagesPipeline` + - :class:`scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` + + (:issue:`6540`) + +- It's now deprecated to have a ``from_settings()`` method but no + ``from_crawler()`` method in 3rd-party :ref:`Scrapy components + <topics-components>`. You can define a simple ``from_crawler()`` method + that calls ``cls.from_settings(crawler.settings)`` to fix this if you don't + want to refactor the code. Note that if you have a ``from_crawler()`` + method Scrapy will not call the ``from_settings()`` method so the latter + can be removed. + (:issue:`6540`) + +- The initialization API of :class:`scrapy.pipelines.media.MediaPipeline` and + its subclasses was improved and some old usage scenarios are now deprecated + (see also the "Backward-incompatible changes" section). Specifically: + + - It's deprecated to define an ``__init__()`` method that doesn't take a + ``crawler`` argument. + - It's deprecated to call an ``__init__()`` method without passing a + ``crawler`` argument. If it's passed, it's also deprecated to pass a + ``settings`` argument, which will be ignored anyway. + - Calling ``from_settings()`` is deprecated, use ``from_crawler()`` + instead. + - Overriding ``from_settings()`` is deprecated, override ``from_crawler()`` + instead. + + (:issue:`6540`) + +- The ``REQUEST_FINGERPRINTER_IMPLEMENTATION`` setting is now deprecated. + (:issue:`6212`, :issue:`6213`) + +- The ``scrapy.utils.misc.create_instance()`` function is now deprecated, use + :func:`scrapy.utils.misc.build_from_crawler` instead. + (:issue:`5523`, :issue:`5884`, :issue:`6162`, :issue:`6169`, :issue:`6540`) + +- ``scrapy.core.downloader.Downloader._get_slot_key()`` is deprecated, use :meth:`scrapy.core.downloader.Downloader.get_slot_key` instead. - (:issue:`6340`) + (:issue:`6340`, :issue:`6352`) + +- ``scrapy.utils.defer.process_chain_both()`` is now deprecated. + (:issue:`6397`) + +- ``scrapy.twisted_version`` is now deprecated, you should instead use + :attr:`twisted.version` directly (but note that it's an + ``incremental.Version`` object, not a tuple). + (:issue:`6509`, :issue:`6512`) + +- ``scrapy.utils.python.flatten()`` and ``scrapy.utils.python.iflatten()`` + are now deprecated. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.python.equal_attributes()`` is now deprecated. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.request.request_authenticate()`` is now deprecated, you + should instead just set the ``Authorization`` header directly. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.serialize.ScrapyJSONDecoder`` is now deprecated, it didn't + contain any code since Scrapy 1.0.0. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.test.assert_samelines()`` is now deprecated. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.extensions.feedexport.build_storage()`` is now deprecated. You can + instead call the builder callable directly. + (:issue:`6540`) + +New features +~~~~~~~~~~~~ + +- :meth:`~scrapy.Spider.start_requests` can now yield items. + (:issue:`5289`, :issue:`6417`) + +- Added a new :class:`~scrapy.http.Response` subclass, + :class:`~scrapy.http.JsonResponse`, for responses with a `JSON MIME type + <https://mimesniff.spec.whatwg.org/#json-mime-type>`_. + (:issue:`6069`, :issue:`6171`, :issue:`6174`) + +- The :class:`~scrapy.extensions.logstats.LogStats` extension now adds + ``items_per_minute`` and ``responses_per_minute`` to the :ref:`stats + <topics-stats>` when the spider closes. + (:issue:`4110`, :issue:`4111`) + +- Added :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM` which allows closing the + spider if no items were scraped in a set amount of time. + (:issue:`6434`) + +- User-defined cookies can now include the ``secure`` field. + (:issue:`6357`) + +- Added component getters to :class:`~scrapy.crawler.Crawler`: + :meth:`~scrapy.crawler.Crawler.get_addon`, + :meth:`~scrapy.crawler.Crawler.get_downloader_middleware`, + :meth:`~scrapy.crawler.Crawler.get_extension`, + :meth:`~scrapy.crawler.Crawler.get_item_pipeline`, + :meth:`~scrapy.crawler.Crawler.get_spider_middleware`. + (:issue:`6181`) + +- Slot delay updates by the :ref:`AutoThrottle extension + <topics-autothrottle>` based on response latencies can now be disabled for + specific requests via the :reqmeta:`autothrottle_dont_adjust_delay` meta + key. + (:issue:`6246`, :issue:`6527`) + +- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``, + :class:`~scrapy.spiderloader.SpiderLoader` does not raise + :exc:`SyntaxError` but emits a warning instead. + (:issue:`6483`, :issue:`6484`) + +- Added support for multiple-compressed responses (ones with several + encodings in the ``Content-Encoding`` header). + (:issue:`5143`, :issue:`5964`, :issue:`6063`) + +- Added support for multiple standard values in :setting:`REFERRER_POLICY`. + (:issue:`6381`) + +- Added support for brotlicffi_ (previously named brotlipy_). brotli_ is + still recommended but only brotlicffi_ works on PyPy. + (:issue:`6263`, :issue:`6269`) + + .. _brotlicffi: https://github.com/python-hyper/brotlicffi + +- Added :class:`~scrapy.contracts.default.MetadataContract` that sets the + request meta. + (:issue:`6468`, :issue:`6469`) + +Improvements +~~~~~~~~~~~~ + +- Extended the list of file extensions that + :class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` + ignores by default. + (:issue:`6074`, :issue:`6125`) + +- :func:`scrapy.utils.httpobj.urlparse_cached` is now used in more places + instead of :func:`urllib.parse.urlparse`. + (:issue:`6228`, :issue:`6229`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.pipelines.media.MediaPipeline` is now an abstract class and + its methods that were expected to be overridden in subclasses are now + abstract methods. + (:issue:`6365`, :issue:`6368`) + +- Fixed handling of invalid ``@``-prefixed lines in contract extraction. + (:issue:`6383`, :issue:`6388`) + +- Importing ``scrapy.extensions.telnet`` no longer installs the default + reactor. + (:issue:`6432``) + +- Reduced log verbosity for dropped requests that was increased in 2.11.2. + (:issue:`6433`, :issue:`6475`) + +Documentation +~~~~~~~~~~~~~ + +- Added ``SECURITY.md`` that documents the security policy. + (:issue:`5364`, :issue:`6051`) + +- Example code for :ref:`running Scrapy from a script <run-from-script>` no + longer imports ``twisted.internet.reactor`` at the top level, which caused + problems with non-default reactors when this code was used unmodified. + (:issue:`6361`, :issue:`6374`) + +- Documented the :class:`~scrapy.extensions.spiderstate.SpiderState` + extension. + (:issue:`6278`, :issue:`6522`) + +- Other documentation improvements and fixes. + (:issue:`5920`, + :issue:`6094`, + :issue:`6177`, + :issue:`6200`, + :issue:`6207`, + :issue:`6216`, + :issue:`6223`, + :issue:`6317`, + :issue:`6328`, + :issue:`6389`, + :issue:`6394`, + :issue:`6402`, + :issue:`6411`, + :issue:`6427`, + :issue:`6429`, + :issue:`6440`, + :issue:`6448`, + :issue:`6449`, + :issue:`6462`, + :issue:`6497`, + :issue:`6506`, + :issue:`6507`, + :issue:`6524`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added ``py.typed``, in line with `PEP 561 + <https://peps.python.org/pep-0561/>`_. + (:issue:`6058`, :issue:`6059`) + +- Fully covered the code with type hints (except for the most complicated + parts, mostly related to ``twisted.web.http`` and other Twisted parts + without type hints). + (:issue:`5989`, + :issue:`6097`, + :issue:`6127`, + :issue:`6129`, + :issue:`6130`, + :issue:`6133`, + :issue:`6143`, + :issue:`6191`, + :issue:`6268`, + :issue:`6274`, + :issue:`6275`, + :issue:`6276`, + :issue:`6279`, + :issue:`6325`, + :issue:`6326`, + :issue:`6333`, + :issue:`6335`, + :issue:`6336`, + :issue:`6337`, + :issue:`6341`, + :issue:`6353`, + :issue:`6356`, + :issue:`6370`, + :issue:`6371`, + :issue:`6384`, + :issue:`6385`, + :issue:`6387`, + :issue:`6391`, + :issue:`6395`, + :issue:`6414`, + :issue:`6422`, + :issue:`6460`, + :issue:`6466`, + :issue:`6472`, + :issue:`6494`, + :issue:`6498`, + :issue:`6516`) + +- Improved Bandit_ checks. + (:issue:`6260`, :issue:`6264`, :issue:`6265`) + +- Added pyupgrade_ to the ``pre-commit`` configuration. + (:issue:`6392`) + + .. _pyupgrade: https://github.com/asottile/pyupgrade + +- Added ``flake8-bugbear``, ``flake8-comprehensions``, ``flake8-debugger``, + ``flake8-docstrings``, ``flake8-string-format`` and + ``flake8-type-checking`` to the ``pre-commit`` configuration. + (:issue:`6406`, :issue:`6413`) + +- CI and test improvements and fixes. + (:issue:`5285`, + :issue:`5454`, + :issue:`5997`, + :issue:`6078`, + :issue:`6084`, + :issue:`6087`, + :issue:`6132`, + :issue:`6153`, + :issue:`6154`, + :issue:`6201`, + :issue:`6231`, + :issue:`6232`, + :issue:`6235`, + :issue:`6236`, + :issue:`6242`, + :issue:`6245`, + :issue:`6253`, + :issue:`6258`, + :issue:`6259`, + :issue:`6270`, + :issue:`6272`, + :issue:`6286`, + :issue:`6290`, + :issue:`6296` + :issue:`6367`, + :issue:`6372`, + :issue:`6403`, + :issue:`6416`, + :issue:`6435`, + :issue:`6489`, + :issue:`6501`, + :issue:`6504`, + :issue:`6511`, + :issue:`6543`, + :issue:`6545`) + +- Code cleanups. + (:issue:`6196`, + :issue:`6197`, + :issue:`6198`, + :issue:`6199`, + :issue:`6254`, + :issue:`6257`, + :issue:`6285`, + :issue:`6305`, + :issue:`6343`, + :issue:`6349`, + :issue:`6386`, + :issue:`6415`, + :issue:`6463`, + :issue:`6470`, + :issue:`6499`, + :issue:`6505`, + :issue:`6510`, + :issue:`6531`, + :issue:`6542`) + +Other +~~~~~ + +- Issue tracker improvements. (:issue:`6066`) .. _release-2.11.2: diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index d2fc41003..14b4aa8ba 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -157,6 +157,7 @@ Use a fallback component: .. code-block:: python from scrapy.core.downloader.handlers.http import HTTPDownloadHandler + from scrapy.utils.misc import build_from_crawler FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" @@ -167,11 +168,7 @@ Use a fallback component: def __init__(self, settings, crawler): dhcls = load_object(settings.get(FALLBACK_SETTING)) - self._fallback_handler = create_instance( - dhcls, - settings=None, - crawler=crawler, - ) + self._fallback_handler = build_from_crawler(dhcls, crawler) def download_request(self, request, spider): if request.meta.get("my_params"): diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 175c877de..f7cffb61b 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -26,7 +26,9 @@ contains a dictionary of all available extensions and their order similar to how you :ref:`configure the downloader middlewares <topics-downloader-middleware-setting>`. -.. class:: Crawler(spidercls, settings) +.. autoclass:: Crawler + :members: get_addon, get_downloader_middleware, get_extension, + get_item_pipeline, get_spider_middleware The Crawler object must be instantiated with a :class:`scrapy.Spider` subclass and a diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 478dd9647..d34b3884b 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -4,8 +4,8 @@ Components ========== -A Scrapy component is any class whose objects are created using -:func:`scrapy.utils.misc.create_instance`. +A Scrapy component is any class whose objects are built using +:func:`~scrapy.utils.misc.build_from_crawler`. That includes the classes that you may assign to the following settings: @@ -84,3 +84,15 @@ If your requirement is a minimum Scrapy version, you may use f"method of spider middlewares as an asynchronous " f"generator." ) + +API reference +============= + +The following function can be used to create an instance of a component class: + +.. autofunction:: scrapy.utils.misc.build_from_crawler + +The following function can also be useful when implementing a component, to +report the import path of the component class, e.g. when reporting problems: + +.. autofunction:: scrapy.utils.python.global_object_name diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 9eace3be0..11a3fcb94 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -926,10 +926,6 @@ Meta tags within these tags are ignored. The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from ``[]`` to ``["noscript"]``. -.. versionchanged:: VERSION - The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from - ``[]`` to ``['noscript']``. - .. setting:: METAREFRESH_MAXDELAY METAREFRESH_MAXDELAY diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8f39bcd53..2b59cabe1 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -358,7 +358,7 @@ Acceptable values for REFERRER_POLICY - either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass — a custom policy or one of the built-in ones (see classes below), -- or one of the standard W3C-defined string values, +- or one or more comma-separated standard W3C-defined string values, - or the special ``"scrapy-default"``. ======================================= ======================================================================== diff --git a/scrapy/crawler.py b/scrapy/crawler.py index de0cf543e..1ad837a47 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -184,9 +184,23 @@ class Crawler: return None def get_addon(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of an :ref:`add-on <topics-addons>` of + the specified class or a subclass, or ``None`` if none is found. + + .. versionadded:: 2.12 + """ return self._get_component(cls, self.addons.addons) def get_downloader_middleware(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of a :ref:`downloader middleware + <topics-downloader-middleware>` of the specified class or a subclass, + or ``None`` if none is found. + + .. versionadded:: 2.12 + + This method can only be called after the crawl engine has been created, + e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. + """ if not self.engine: raise RuntimeError( "Crawler.get_downloader_middleware() can only be called after " @@ -195,6 +209,16 @@ class Crawler: return self._get_component(cls, self.engine.downloader.middleware.middlewares) def get_extension(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of an :ref:`extension + <topics-extensions>` of the specified class or a subclass, + or ``None`` if none is found. + + .. versionadded:: 2.12 + + This method can only be called after the extension manager has been + created, e.g. at signals :signal:`engine_started` or + :signal:`spider_opened`. + """ if not self.extensions: raise RuntimeError( "Crawler.get_extension() can only be called after the " @@ -203,6 +227,15 @@ class Crawler: return self._get_component(cls, self.extensions.middlewares) def get_item_pipeline(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of a :ref:`item pipeline + <topics-item-pipeline>` of the specified class or a subclass, or + ``None`` if none is found. + + .. versionadded:: 2.12 + + This method can only be called after the crawl engine has been created, + e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. + """ if not self.engine: raise RuntimeError( "Crawler.get_item_pipeline() can only be called after the " @@ -211,6 +244,15 @@ class Crawler: return self._get_component(cls, self.engine.scraper.itemproc.middlewares) def get_spider_middleware(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of a :ref:`spider middleware + <topics-spider-middleware>` of the specified class or a subclass, or + ``None`` if none is found. + + .. versionadded:: 2.12 + + This method can only be called after the crawl engine has been created, + e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. + """ if not self.engine: raise RuntimeError( "Crawler.get_spider_middleware() can only be called after the " diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 6a7704687..0cf44aed8 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -692,8 +692,10 @@ class FeedExporter: def _get_storage( self, uri: str, feed_options: dict[str, Any] ) -> FeedStorageProtocol: - feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) - return build_from_crawler(feedcls, self.crawler, uri, feed_options=feed_options) + """Build a storage object for the specified *uri* with the specified + *feed_options*.""" + cls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) + return build_from_crawler(cls, self.crawler, uri, feed_options=feed_options) def _get_uri_params( self, diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 6d7808c31..691a1cbf2 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -149,6 +149,15 @@ class MediaPipeline(ABC): pipe: Self if hasattr(cls, "from_settings"): pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined] + warnings.warn( + f"{global_object_name(cls)} has from_settings() and either doesn't have" + " from_crawler() or calls MediaPipeline.from_crawler() from it," + " so from_settings() was used to create the instance of it." + " This is deprecated and calling from_settings() will be removed" + " in a future Scrapy version. Please move the initialization code into" + " from_crawler() or __init__().", + category=ScrapyDeprecationWarning, + ) elif "crawler" in get_func_args(cls.__init__): pipe = cls(crawler=crawler) else: @@ -249,7 +258,7 @@ class MediaPipeline(ABC): # minimize cached information for failure result.cleanFailure() result.frames = [] - if twisted_version <= Version("twisted", 24, 10, 0): + if twisted_version < Version("twisted", 24, 10, 0): result.stack = [] # type: ignore[method-assign] # This code fixes a memory leak by avoiding to keep references to # the Request and Response objects on the Media Pipeline cache. @@ -269,9 +278,6 @@ class MediaPipeline(ABC): # To avoid keeping references to the Response and therefore Request # objects on the Media Pipeline cache, we should wipe the context of # the encapsulated exception when it is a StopIteration instance - # - # This problem does not occur in Python 2.7 since we don't have - # Exception Chaining (https://www.python.org/dev/peps/pep-3134/). context = getattr(result.value, "__context__", None) if isinstance(context, StopIteration): result.value.__context__ = None diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index a408a205d..eefadd07d 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -177,6 +177,8 @@ def build_from_crawler( ) -> T: """Construct a class instance using its ``from_crawler`` or ``from_settings`` constructor. + .. versionadded:: 2.12 + ``*args`` and ``**kwargs`` are forwarded to the constructor. Raises ``TypeError`` if the resulting instance is ``None``. diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 3864d054f..b9babb08f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -328,8 +328,7 @@ def without_none_values( def global_object_name(obj: Any) -> str: - """ - Return full name of a global object. + """Return the full import path of the given class. >>> from scrapy import Request >>> global_object_name(Request) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 58a2d3678..cb1e2f9a1 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -454,9 +454,34 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 1) + self.assertEqual(len(w), 2) assert pipe._from_settings_called + def test_has_from_settings_and_from_crawler(self): + class Pipeline(UserDefinedPipeline): + _from_settings_called = False + _from_crawler_called = False + + @classmethod + def from_settings(cls, settings): + o = cls() + o._from_settings_called = True + return o + + @classmethod + def from_crawler(cls, crawler): + o = super().from_crawler(crawler) + o._from_crawler_called = True + return o + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + self.assertEqual(len(w), 2) + assert pipe._from_settings_called + assert pipe._from_crawler_called + def test_has_from_settings_and_init(self): class Pipeline(UserDefinedPipeline): _from_settings_called = False @@ -476,7 +501,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 1) + self.assertEqual(len(w), 2) assert pipe._from_settings_called assert pipe._init_called From b1f9e56693cd2000ddcea922306f726f3e9339af Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 18 Nov 2024 13:08:05 +0500 Subject: [PATCH 1582/2083] =?UTF-8?q?Bump=20version:=202.11.2=20=E2=86=92?= =?UTF-8?q?=202.12.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- SECURITY.md | 4 ++-- scrapy/VERSION | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 599cd0cff..f83e3e890 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.11.2 +current_version = 2.12.0 commit = True tag = True tag_name = {new_version} diff --git a/SECURITY.md b/SECURITY.md index 51305d95e..bc64dec7b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,8 +4,8 @@ | Version | Supported | | ------- | ------------------ | -| 2.11.x | :white_check_mark: | -| < 2.11.x | :x: | +| 2.12.x | :white_check_mark: | +| < 2.12.x | :x: | ## Reporting a Vulnerability diff --git a/scrapy/VERSION b/scrapy/VERSION index 9e5bb77a3..d8b698973 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.11.2 +2.12.0 From efb53aafdcaae058962c6189ddecb3dc62b02c31 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 18 Nov 2024 15:39:49 +0500 Subject: [PATCH 1583/2083] Fix a typo that broke PDF builds. --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 025eb09ba..2bf65272f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -384,7 +384,7 @@ Bug fixes - Importing ``scrapy.extensions.telnet`` no longer installs the default reactor. - (:issue:`6432``) + (:issue:`6432`) - Reduced log verbosity for dropped requests that was increased in 2.11.2. (:issue:`6433`, :issue:`6475`) From 8c23da943c5e892515f4fa2eb57229839802010a Mon Sep 17 00:00:00 2001 From: Swayam Gupta <78016781+swayam0322@users.noreply.github.com> Date: Tue, 19 Nov 2024 19:51:15 +0530 Subject: [PATCH 1584/2083] Integrating configs into pyproject.toml (#6547) --- .bandit.yml | 7 - .bumpversion.cfg | 11 -- .coveragerc | 12 -- .isort.cfg | 2 - .pre-commit-config.yaml | 3 +- MANIFEST.in | 1 - pylintrc | 73 --------- pyproject.toml | 235 ++++++++++++++++++++++++++++ pytest.ini | 26 --- setup.cfg | 24 --- setup.py | 75 --------- tests/test_crawler.py | 2 +- tests/test_spiderloader/__init__.py | 7 +- tox.ini | 2 +- 14 files changed, 243 insertions(+), 237 deletions(-) delete mode 100644 .bandit.yml delete mode 100644 .bumpversion.cfg delete mode 100644 .coveragerc delete mode 100644 .isort.cfg delete mode 100644 pylintrc create mode 100644 pyproject.toml delete mode 100644 pytest.ini delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.bandit.yml b/.bandit.yml deleted file mode 100644 index b7f1817e0..000000000 --- a/.bandit.yml +++ /dev/null @@ -1,7 +0,0 @@ -skips: -- B101 # assert_used, needed for mypy -- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 -- B503 # ssl_with_bad_defaults -exclude_dirs: ['tests'] diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index f83e3e890..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,11 +0,0 @@ -[bumpversion] -current_version = 2.12.0 -commit = True -tag = True -tag_name = {new_version} - -[bumpversion:file:scrapy/VERSION] - -[bumpversion:file:SECURITY.md] -parse = (?P<major>\d+)\.(?P<minor>\d+)\.x -serialize = {major}.{minor}.x diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index f9ad353d5..000000000 --- a/.coveragerc +++ /dev/null @@ -1,12 +0,0 @@ -[run] -branch = true -include = scrapy/* -omit = - tests/* -disable_warnings = include-ignored - -[report] -# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 -exclude_lines = - pragma: no cover - if TYPE_CHECKING: diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index f238bf7ea..000000000 --- a/.isort.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[settings] -profile = black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fbd710f6f..b411f4927 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,8 @@ repos: rev: 1.7.9 hooks: - id: bandit - args: [-r, -c, .bandit.yml] + args: ["-c", "pyproject.toml"] + additional_dependencies: ["bandit[toml]"] - repo: https://github.com/PyCQA/flake8 rev: 7.1.0 hooks: diff --git a/MANIFEST.in b/MANIFEST.in index 06971e39c..7700ae7bd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,7 +10,6 @@ include scrapy/py.typed include codecov.yml include conftest.py -include pytest.ini include tox.ini recursive-include scrapy/templates * diff --git a/pylintrc b/pylintrc deleted file mode 100644 index e927b903c..000000000 --- a/pylintrc +++ /dev/null @@ -1,73 +0,0 @@ -[MASTER] -persistent=no -jobs=1 # >1 hides results - -[MESSAGES CONTROL] -disable=abstract-method, - arguments-differ, - arguments-renamed, - attribute-defined-outside-init, - bad-classmethod-argument, - bare-except, - broad-except, - broad-exception-raised, - c-extension-no-member, - consider-using-with, - cyclic-import, - dangerous-default-value, - disallowed-name, - duplicate-code, # https://github.com/PyCQA/pylint/issues/214 - eval-used, - fixme, - function-redefined, - global-statement, - implicit-str-concat, - import-error, - import-outside-toplevel, - inherit-non-class, - invalid-name, - invalid-overridden-method, - isinstance-second-argument-not-valid-type, - keyword-arg-before-vararg, - line-too-long, - logging-format-interpolation, - logging-fstring-interpolation, - logging-not-lazy, - lost-exception, - missing-docstring, - no-member, - no-method-argument, - no-name-in-module, - no-self-argument, - no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268 - not-callable, - pointless-statement, - pointless-string-statement, - protected-access, - raise-missing-from, - redefined-builtin, - redefined-outer-name, - reimported, - signature-differs, - too-few-public-methods, - too-many-ancestors, - too-many-arguments, - too-many-branches, - too-many-format-args, - too-many-function-args, - too-many-instance-attributes, - too-many-lines, - too-many-locals, - too-many-public-methods, - too-many-return-statements, - unbalanced-tuple-unpacking, - unnecessary-dunder-call, - unnecessary-pass, - unreachable, - unused-argument, - unused-import, - unused-variable, - used-before-assignment, - useless-return, - wildcard-import, - wrong-import-position diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..f25715e76 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,235 @@ +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "Scrapy" +dynamic = ["version"] +description = "A high-level Web Crawling and Web Scraping framework" +dependencies = [ + "Twisted>=21.7.0", + "cryptography>=37.0.0", + "cssselect>=0.9.1", + "itemloaders>=1.0.1", + "parsel>=1.5.0", + "pyOpenSSL>=22.0.0", + "queuelib>=1.4.2", + "service_identity>=18.1.0", + "w3lib>=1.17.0", + "zope.interface>=5.1.0", + "protego>=0.1.15", + "itemadapter>=0.1.0", + "packaging", + "tldextract", + "lxml>=4.6.0", + "defusedxml>=0.7.1", + # Platform-specific dependencies + 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', + 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', +] +classifiers = [ + "Framework :: Scrapy", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries :: Application Frameworks", + "Topic :: Software Development :: Libraries :: Python Modules", +] +readme = "README.rst" +requires-python = ">=3.9" +authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }] +maintainers = [{ name = "Pablo Hoffman", email = "pablo@pablohoffman.com" }] + +[project.urls] +Homepage = "https://scrapy.org/" +Documentation = "https://docs.scrapy.org/" +Source = "https://github.com/scrapy/scrapy" +Tracker = "https://github.com/scrapy/scrapy/issues" +Changelog = "https://github.com/scrapy/scrapy/commits/master/" +releasenotes = "https://docs.scrapy.org/en/latest/news.html" + +[project.scripts] +scrapy = "scrapy.cmdline:execute" + +[tool.setuptools.packages.find] +where = ["."] +include = ["scrapy", "scrapy.*",] + +[tool.setuptools.dynamic] +version = {file = "./scrapy/VERSION"} + +[tool.mypy] +ignore_missing_imports = true + +# Interface classes are hard to support + +[[tool.mypy.overrides]] +module = "twisted.internet.interfaces" +follow_imports = "skip" + +[[tool.mypy.overrides]] +module = "scrapy.interfaces" +ignore_errors = true + +[[tool.mypy.overrides]] +module = "twisted.internet.reactor" +follow_imports = "skip" + +# FIXME: remove the following section once the issues are solved +[[tool.mypy.overrides]] +module = "scrapy.settings.default_settings" +ignore_errors = true + +[tool.bandit] +skips = [ + "B101", # assert_used, needed for mypy + "B321", # ftplib, https://github.com/scrapy/scrapy/issues/4180 + "B402", # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 + "B411", # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 + "B503", # ssl_with_bad_defaults +] +exclude_dirs = ["tests"] + +[tool.bumpversion] +current_version = "2.12.0" +commit = true +tag = true +tag_name = "{new_version}" + +[[tool.bumpversion.files]] +filename = "scrapy/VERSION" + +[[tool.bumpversion.files]] +filename = "SECURITY.md" +parse = """(?P<major>0|[1-9]\\d*)\\.(?P<minor>0|[1-9]\\d*)""" +serialize = ["{major}.{minor}"] + +[tool.coverage.run] +branch = true +include = ["scrapy/*"] +omit = ["tests/*"] +disable_warnings = ["include-ignored"] + +[tool.coverage.report] +# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 +exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"] + +[tool.isort] +profile = "black" + +[tool.pylint.MASTER] +persistent = "no" +jobs = 1 # >1 hides results + +[tool.pylint."MESSAGES CONTROL"] +disable = [ + "abstract-method", + "arguments-differ", + "arguments-renamed", + "attribute-defined-outside-init", + "bad-classmethod-argument", + "bare-except", + "broad-except", + "broad-exception-raised", + "c-extension-no-member", + "consider-using-with", + "cyclic-import", + "dangerous-default-value", + "disallowed-name", + "duplicate-code", # https://github.com/PyCQA/pylint/issues/214 + "eval-used", + "fixme", + "function-redefined", + "global-statement", + "implicit-str-concat", + "import-error", + "import-outside-toplevel", + "inherit-non-class", + "invalid-name", + "invalid-overridden-method", + "isinstance-second-argument-not-valid-type", + "keyword-arg-before-vararg", + "line-too-long", + "logging-format-interpolation", + "logging-fstring-interpolation", + "logging-not-lazy", + "lost-exception", + "missing-docstring", + "no-member", + "no-method-argument", + "no-name-in-module", + "no-self-argument", + "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268 + "not-callable", + "pointless-statement", + "pointless-string-statement", + "protected-access", + "raise-missing-from", + "redefined-builtin", + "redefined-outer-name", + "reimported", + "signature-differs", + "too-few-public-methods", + "too-many-ancestors", + "too-many-arguments", + "too-many-branches", + "too-many-format-args", + "too-many-function-args", + "too-many-instance-attributes", + "too-many-lines", + "too-many-locals", + "too-many-public-methods", + "too-many-return-statements", + "unbalanced-tuple-unpacking", + "unnecessary-dunder-call", + "unnecessary-pass", + "unreachable", + "unused-argument", + "unused-import", + "unused-variable", + "used-before-assignment", + "useless-return", + "wildcard-import", + "wrong-import-position", +] + +[tool.pytest.ini_options] +xfail_strict = true +usefixtures = "chdir" +python_files = ["test_*.py", "__init__.py"] +python_classes = [] +addopts = [ + "--assert=plain", + "--ignore=docs/_ext", + "--ignore=docs/conf.py", + "--ignore=docs/news.rst", + "--ignore=docs/topics/dynamic-content.rst", + "--ignore=docs/topics/items.rst", + "--ignore=docs/topics/leaks.rst", + "--ignore=docs/topics/loaders.rst", + "--ignore=docs/topics/selectors.rst", + "--ignore=docs/topics/shell.rst", + "--ignore=docs/topics/stats.rst", + "--ignore=docs/topics/telnetconsole.rst", + "--ignore=docs/utils", +] +markers = [ + "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed", + "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed", + "requires_uvloop: marks tests as only enabled when uvloop is known to be working", + "requires_botocore: marks tests that need botocore (but not boto3)", + "requires_boto3: marks tests that need botocore and boto3", +] +filterwarnings = [] \ No newline at end of file diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 824c0e9e9..000000000 --- a/pytest.ini +++ /dev/null @@ -1,26 +0,0 @@ -[pytest] -xfail_strict = true -usefixtures = chdir -python_files=test_*.py __init__.py -python_classes= -addopts = - --assert=plain - --ignore=docs/_ext - --ignore=docs/conf.py - --ignore=docs/news.rst - --ignore=docs/topics/dynamic-content.rst - --ignore=docs/topics/items.rst - --ignore=docs/topics/leaks.rst - --ignore=docs/topics/loaders.rst - --ignore=docs/topics/selectors.rst - --ignore=docs/topics/shell.rst - --ignore=docs/topics/stats.rst - --ignore=docs/topics/telnetconsole.rst - --ignore=docs/utils -markers = - only_asyncio: marks tests as only enabled when --reactor=asyncio is passed - only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed - requires_uvloop: marks tests as only enabled when uvloop is known to be working - requires_botocore: marks tests that need botocore (but not boto3) - requires_boto3: marks tests that need botocore and boto3 -filterwarnings = diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 151e784c6..000000000 --- a/setup.cfg +++ /dev/null @@ -1,24 +0,0 @@ -[bdist_rpm] -doc_files = docs AUTHORS INSTALL LICENSE README.rst - -[bdist_wheel] -universal=1 - -[mypy] -ignore_missing_imports = true - -# Interface classes are hard to support - -[mypy-twisted.internet.interfaces] -follow_imports = skip - -[mypy-scrapy.interfaces] -ignore_errors = True - -[mypy-twisted.internet.reactor] -follow_imports = skip - -# FIXME: remove the following sections once the issues are solved - -[mypy-scrapy.settings.default_settings] -ignore_errors = True diff --git a/setup.py b/setup.py deleted file mode 100644 index 6cc1150a5..000000000 --- a/setup.py +++ /dev/null @@ -1,75 +0,0 @@ -from pathlib import Path - -from setuptools import find_packages, setup - -version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() - - -install_requires = [ - "Twisted>=21.7.0", - "cryptography>=37.0.0", - "cssselect>=0.9.1", - "itemloaders>=1.0.1", - "parsel>=1.5.0", - "pyOpenSSL>=22.0.0", - "queuelib>=1.4.2", - "service_identity>=18.1.0", - "w3lib>=1.17.0", - "zope.interface>=5.1.0", - "protego>=0.1.15", - "itemadapter>=0.1.0", - "packaging", - "tldextract", - "lxml>=4.6.0", - "defusedxml>=0.7.1", -] -extras_require = { - ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], - ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"], -} - - -setup( - name="Scrapy", - version=version, - url="https://scrapy.org", - project_urls={ - "Documentation": "https://docs.scrapy.org/", - "Source": "https://github.com/scrapy/scrapy", - "Tracker": "https://github.com/scrapy/scrapy/issues", - }, - description="A high-level Web Crawling and Web Scraping framework", - long_description=open("README.rst", encoding="utf-8").read(), - author="Scrapy developers", - author_email="pablo@pablohoffman.com", - maintainer="Pablo Hoffman", - maintainer_email="pablo@pablohoffman.com", - license="BSD", - packages=find_packages(exclude=("tests", "tests.*")), - include_package_data=True, - zip_safe=False, - entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]}, - classifiers=[ - "Framework :: Scrapy", - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Software Development :: Libraries :: Application Frameworks", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - python_requires=">=3.9", - install_requires=install_requires, - extras_require=extras_require, -) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 853acf2de..a77531f62 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -899,7 +899,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.expect_exact("shutting down gracefully") # sending the second signal too fast often causes problems d = Deferred() - reactor.callLater(0.1, d.callback, None) + reactor.callLater(0.01, d.callback, None) yield d p.kill(sig) p.expect_exact("forcing unclean shutdown") diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d2ff9ba48..9b53b9b96 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -144,9 +144,10 @@ class SpiderLoaderTest(unittest.TestCase): self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings) def test_syntax_error_warning(self): - with warnings.catch_warnings(record=True) as w, mock.patch.object( - SpiderLoader, "_load_spiders" - ) as m: + with ( + warnings.catch_warnings(record=True) as w, + mock.patch.object(SpiderLoader, "_load_spiders") as m, + ): m.side_effect = SyntaxError module = "tests.test_spiderloader.test_spiders.spider1" settings = Settings( diff --git a/tox.ini b/tox.ini index 5783a0e61..4e1a99473 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,7 @@ deps = {[testenv:extra-deps]deps} pylint==3.2.5 commands = - pylint conftest.py docs extras scrapy setup.py tests + pylint conftest.py docs extras scrapy tests [testenv:twinecheck] basepython = python3 From 4dcc04be48b5c9ed096e91bf2928f5e421ce2153 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 24 Nov 2024 12:44:48 +0400 Subject: [PATCH 1585/2083] Add tests for DOWNLOADER_CLIENT_TLS_METHOD, remove dead code. --- scrapy/core/downloader/contextfactory.py | 11 +---- tests/test_webclient.py | 60 +++++++++++++++++++++++- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 8e17eab9a..d44c663bb 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -111,18 +111,9 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): def getCertificateOptions(self) -> CertificateOptions: # setting verify=True will require you to provide CAs # to verify against; in other words: it's not that simple - - # backward-compatible SSL/TLS method: - # - # * this will respect `method` attribute in often recommended - # `ScrapyClientContextFactory` subclass - # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133) - # - # * getattr() for `_ssl_method` attribute for context factories - # not calling super().__init__ return CertificateOptions( verify=False, - method=getattr(self, "method", getattr(self, "_ssl_method", None)), + method=self._ssl_method, fixBrokenPeers=True, acceptableCiphers=self.tls_ciphers, ) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 1797d5e1f..1cad68b9c 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -3,21 +3,29 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ +from __future__ import annotations + import shutil from pathlib import Path from tempfile import mkdtemp +from typing import Any import OpenSSL.SSL +from pytest import raises from twisted.internet import defer, reactor -from twisted.internet.defer import inlineCallbacks +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.internet.testing import StringTransport from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util from scrapy.core.downloader import webclient as client -from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory +from scrapy.core.downloader.contextfactory import ( + ScrapyClientContextFactory, + load_context_factory_from_settings, +) from scrapy.http import Headers, Request +from scrapy.settings import Settings from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.test import get_crawler @@ -482,3 +490,51 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): self.getURL("payload"), body=s, contextFactory=client_context_factory ) return self.assertFailure(d, OpenSSL.SSL.Error) + + +class WebClientTLSMethodTestCase(WebClientSSLTestCase): + def _assert_factory_works( + self, client_context_factory: ScrapyClientContextFactory + ) -> Deferred[Any]: + s = "0123456789" * 10 + return getPage( + self.getURL("payload"), body=s, contextFactory=client_context_factory + ).addCallback(self.assertEqual, to_bytes(s)) + + def test_setting_default(self): + crawler = get_crawler() + settings = Settings() + client_context_factory = load_context_factory_from_settings(settings, crawler) + assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD + return self._assert_factory_works(client_context_factory) + + def test_setting_none(self): + crawler = get_crawler() + settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": None}) + with raises(KeyError): + load_context_factory_from_settings(settings, crawler) + + def test_setting_bad(self): + crawler = get_crawler() + settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) + with raises(KeyError): + load_context_factory_from_settings(settings, crawler) + + def test_setting_explicit(self): + crawler = get_crawler() + settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"}) + client_context_factory = load_context_factory_from_settings(settings, crawler) + assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD + return self._assert_factory_works(client_context_factory) + + def test_direct_from_crawler(self): + # the setting is ignored + crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) + client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) + assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD + return self._assert_factory_works(client_context_factory) + + def test_direct_init(self): + client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) + assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD + return self._assert_factory_works(client_context_factory) From cc146b9df7c6039ab0e0654b5844f5978e5bc565 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 9 Dec 2024 13:47:47 +0400 Subject: [PATCH 1586/2083] Add ruff with basic rules. (#6565) --- .pre-commit-config.yaml | 4 ++++ pyproject.toml | 23 ++++++++++++++++++++++- scrapy/http/request/form.py | 2 +- tests/test_pipeline_crawl.py | 2 +- tests/test_pipeline_media.py | 2 +- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b411f4927..ec8693c00 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,8 @@ repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.1 + hooks: + - id: ruff - repo: https://github.com/PyCQA/bandit rev: 1.7.9 hooks: diff --git a/pyproject.toml b/pyproject.toml index f25715e76..b6c02472d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -232,4 +232,25 @@ markers = [ "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", ] -filterwarnings = [] \ No newline at end of file +filterwarnings = [] + +[tool.ruff.lint] +extend-select = [ +] +ignore = [ +] + +[tool.ruff.lint.per-file-ignores] +# Exclude files that are meant to provide top-level imports +"scrapy/__init__.py" = ["E402"] +"scrapy/core/downloader/handlers/http.py" = ["F401"] +"scrapy/http/__init__.py" = ["F401"] +"scrapy/linkextractors/__init__.py" = ["E402", "F401"] +"scrapy/selector/__init__.py" = ["F401"] +"scrapy/spiders/__init__.py" = ["E402", "F401"] + +# Issues pending a review: +"docs/conf.py" = ["E402"] +"scrapy/utils/url.py" = ["F403", "F405"] +"tests/CrawlerRunner/change_reactor.py" = ["E402"] +"tests/test_loader.py" = ["E741"] diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 10ad1305e..b3c3d7c7a 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -186,7 +186,7 @@ def _get_inputs( if not dont_click: clickable = _get_clickable(clickdata, form) - if clickable and clickable[0] not in formdata and not clickable[0] is None: + if clickable and clickable[0] not in formdata and clickable[0] is not None: values.append(clickable) formdata_items = formdata.items() if isinstance(formdata, dict) else formdata diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 7add27aa7..9e1b1ab5b 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -218,7 +218,7 @@ class FileDownloadCrawlTestCase(TestCase): skip_pillow: str | None try: - from PIL import Image # noqa: imported just to check for the import error + from PIL import Image # noqa: F401 except ImportError: skip_pillow = "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow" else: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index cb1e2f9a1..dd8f1084a 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -19,7 +19,7 @@ from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler try: - from PIL import Image # noqa: imported just to check for the import error + from PIL import Image # noqa: F401 except ImportError: skip_pillow: str | None = ( "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow" From 5680bee968d91239da27fff00aa33a9d891f4109 Mon Sep 17 00:00:00 2001 From: Emery Berger <emery.berger@gmail.com> Date: Mon, 9 Dec 2024 05:01:00 -0500 Subject: [PATCH 1587/2083] Made path absolute to enable running pytest from a different directory. (#6567) --- conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 3af072318..e9765962a 100644 --- a/conftest.py +++ b/conftest.py @@ -24,7 +24,9 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with Path("tests/ignores.txt").open(encoding="utf-8") as reader: +base_dir = Path(__file__).parent +ignore_file_path = base_dir / "tests" / "ignores.txt" +with ignore_file_path.open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != "#": From c184f12ab5cb6239fc7b2f27ad75be45930dc871 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 1 Dec 2024 11:59:36 +0500 Subject: [PATCH 1588/2083] Add flake8-bugbear rules to ruff. --- pyproject.toml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b6c02472d..1cbf4ac13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -236,8 +236,26 @@ filterwarnings = [] [tool.ruff.lint] extend-select = [ + # flake8-bugbear + "B", ] ignore = [ + # Assigning to `os.environ` doesn't clear the environment. + "B003", + # Do not use mutable data structures for argument defaults. + "B006", + # Loop control variable not used within the loop body. + "B007", + # Do not perform function calls in argument defaults. + "B008", + # Star-arg unpacking after a keyword argument is strongly discouraged. + "B026", + # Found useless expression. + "B018", + # No explicit stacklevel argument found. + "B028", + # Within an `except` clause, raise exceptions with `raise ... from` + "B904", ] [tool.ruff.lint.per-file-ignores] From e53d6f09bc584f752200a11b686ec628b8c4c09d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 1 Dec 2024 12:02:11 +0500 Subject: [PATCH 1589/2083] Add flake8-comprehensions and flake8-debugger rules to ruff. --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1cbf4ac13..9d88b4e80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -238,6 +238,10 @@ filterwarnings = [] extend-select = [ # flake8-bugbear "B", + # flake8-comprehensions + "C4", + # flake8-debugger + "T10", ] ignore = [ # Assigning to `os.environ` doesn't clear the environment. From d6bf1464b875d40be489568bede8e8319ff770c9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 1 Dec 2024 12:12:13 +0500 Subject: [PATCH 1590/2083] Add pydocstyle/flake8-docstrings rules to ruff. --- pyproject.toml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9d88b4e80..41ba77094 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -240,6 +240,8 @@ extend-select = [ "B", # flake8-comprehensions "C4", + # pydocstyle + "D", # flake8-debugger "T10", ] @@ -260,6 +262,38 @@ ignore = [ "B028", # Within an `except` clause, raise exceptions with `raise ... from` "B904", + # Missing docstring in public module + "D100", + # Missing docstring in public class + "D101", + # Missing docstring in public method + "D102", + # Missing docstring in public function + "D103", + # Missing docstring in public package + "D104", + # Missing docstring in magic method + "D105", + # Missing docstring in public nested class + "D106", + # Missing docstring in __init__ + "D107", + # One-line docstring should fit on one line with quotes + "D200", + # No blank lines allowed after function docstring + "D202", + # 1 blank line required between summary line and description + "D205", + # Multi-line docstring closing quotes should be on a separate line + "D209", + # First line should end with a period + "D400", + # First line should be in imperative mood; try rephrasing + "D401", + # First line should not be the function's "signature" + "D402", + # First word of the first line should be properly capitalized + "D403", ] [tool.ruff.lint.per-file-ignores] @@ -276,3 +310,6 @@ ignore = [ "scrapy/utils/url.py" = ["F403", "F405"] "tests/CrawlerRunner/change_reactor.py" = ["E402"] "tests/test_loader.py" = ["E741"] + +[tool.ruff.lint.pydocstyle] +convention = "pep257" From d47f142d0ffff0e8f87eee67c4c0e7edbc71613e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 1 Dec 2024 12:28:57 +0500 Subject: [PATCH 1591/2083] Add flake8-type-checking rules to ruff. --- pyproject.toml | 2 ++ scrapy/commands/bench.py | 2 +- scrapy/commands/fetch.py | 3 ++- scrapy/commands/genspider.py | 6 ++++-- scrapy/commands/parse.py | 2 +- scrapy/commands/runspider.py | 2 +- scrapy/commands/shell.py | 3 ++- scrapy/commands/startproject.py | 5 ++++- scrapy/core/downloader/__init__.py | 2 +- scrapy/core/downloader/handlers/__init__.py | 3 +-- scrapy/core/engine.py | 9 ++++----- scrapy/core/scraper.py | 6 +++--- scrapy/crawler.py | 4 ++-- scrapy/exporters.py | 6 ++++-- scrapy/extensions/feedexport.py | 3 +-- scrapy/extensions/httpcache.py | 6 +++--- scrapy/extensions/periodic_log.py | 3 ++- scrapy/extensions/telnet.py | 2 +- scrapy/http/request/__init__.py | 4 ++-- scrapy/pqueues.py | 2 +- scrapy/utils/conf.py | 3 +-- scrapy/utils/reactor.py | 2 +- scrapy/utils/test.py | 3 ++- 23 files changed, 46 insertions(+), 37 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 41ba77094..4d20e5c1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -244,6 +244,8 @@ extend-select = [ "D", # flake8-debugger "T10", + # flake8-type-checking + "TC", ] ignore = [ # Assigning to `os.environ` doesn't clear the environment. diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 4f6933006..b96c63eb7 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,6 +1,5 @@ from __future__ import annotations -import argparse import subprocess # nosec import sys import time @@ -13,6 +12,7 @@ from scrapy.http import Response, TextResponse from scrapy.linkextractors import LinkExtractor if TYPE_CHECKING: + import argparse from collections.abc import Iterable from scrapy import Request diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index a1806f626..05e5e53e9 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -5,7 +5,6 @@ from typing import TYPE_CHECKING from w3lib.url import is_url -from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.http import Request, Response @@ -15,6 +14,8 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request if TYPE_CHECKING: from argparse import ArgumentParser, Namespace + from scrapy import Spider + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index b286e703e..2e70b2865 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,12 +1,11 @@ from __future__ import annotations -import argparse import os import shutil import string from importlib import import_module from pathlib import Path -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast from urllib.parse import urlparse import scrapy @@ -14,6 +13,9 @@ from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase +if TYPE_CHECKING: + import argparse + def sanitize_module_name(module_name: str) -> str: """Sanitize the given module name, by replacing dashes and points diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index fba294851..fc16e46d1 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,6 +1,5 @@ from __future__ import annotations -import argparse import functools import inspect import json @@ -22,6 +21,7 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: + import argparse from collections.abc import AsyncGenerator, Coroutine, Iterable from twisted.python.failure import Failure diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 7ec56899c..55211f8d7 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,6 +1,5 @@ from __future__ import annotations -import argparse import sys from importlib import import_module from pathlib import Path @@ -11,6 +10,7 @@ from scrapy.exceptions import UsageError from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: + import argparse from os import PathLike from types import ModuleType diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 27e6d68ee..4ca015f5e 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -9,7 +9,6 @@ from __future__ import annotations from threading import Thread from typing import TYPE_CHECKING, Any -from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.shell import Shell @@ -19,6 +18,8 @@ from scrapy.utils.url import guess_scheme if TYPE_CHECKING: from argparse import ArgumentParser, Namespace + from scrapy import Spider + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index f54c02369..6da877610 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,6 +1,5 @@ from __future__ import annotations -import argparse import os import re import string @@ -8,12 +7,16 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION +from typing import TYPE_CHECKING import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase +if TYPE_CHECKING: + import argparse + TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( ("scrapy.cfg",), ("${project_name}", "settings.py.tmpl"), diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 5040741e2..434b316e9 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -15,7 +15,6 @@ from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.resolver import dnscache -from scrapy.signalmanager import SignalManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached @@ -23,6 +22,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.settings import BaseSettings + from scrapy.signalmanager import SignalManager _T = TypeVar("_T") diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 20377ac06..7f3da67eb 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -from collections.abc import Callable from typing import TYPE_CHECKING, Any, Protocol, cast from twisted.internet import defer @@ -15,7 +14,7 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import Callable, Generator from twisted.internet.defer import Deferred diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 60cffae35..5480df72c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -17,13 +17,9 @@ from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals -from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper, _HandleOutputDeferred from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response -from scrapy.logformatter import LogFormatter -from scrapy.settings import Settings -from scrapy.signalmanager import SignalManager from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce @@ -31,9 +27,12 @@ from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: from collections.abc import Callable, Generator, Iterable, Iterator + from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings + from scrapy.logformatter import LogFormatter + from scrapy.settings import BaseSettings, Settings + from scrapy.signalmanager import SignalManager from scrapy.spiders import Spider diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 83dad0c0b..03301717d 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -16,9 +16,6 @@ from scrapy import Spider, signals from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response -from scrapy.logformatter import LogFormatter -from scrapy.pipelines import ItemPipelineManager -from scrapy.signalmanager import SignalManager from scrapy.utils.defer import ( aiter_errback, defer_fail, @@ -35,6 +32,9 @@ if TYPE_CHECKING: from collections.abc import Generator, Iterable from scrapy.crawler import Crawler + from scrapy.logformatter import LogFormatter + from scrapy.pipelines import ItemPipelineManager + from scrapy.signalmanager import SignalManager logger = logging.getLogger(__name__) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1ad837a47..05af1bf8a 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -18,10 +18,8 @@ from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader -from scrapy.logformatter import LogFormatter from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager -from scrapy.statscollectors import StatsCollector from scrapy.utils.log import ( LogCounterHandler, configure_logging, @@ -42,7 +40,9 @@ from scrapy.utils.reactor import ( if TYPE_CHECKING: from collections.abc import Generator, Iterable + from scrapy.logformatter import LogFormatter from scrapy.spiderloader import SpiderLoader + from scrapy.statscollectors import StatsCollector from scrapy.utils.request import RequestFingerprinter diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c9350a956..9380b7e78 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -10,8 +10,7 @@ import pickle # nosec import pprint from collections.abc import Callable, Iterable, Mapping from io import BytesIO, TextIOWrapper -from json import JSONEncoder -from typing import Any +from typing import TYPE_CHECKING, Any from xml.sax.saxutils import XMLGenerator # nosec from xml.sax.xmlreader import AttributesImpl # nosec @@ -21,6 +20,9 @@ from scrapy.item import Field, Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder +if TYPE_CHECKING: + from json import JSONEncoder + __all__ = [ "BaseItemExporter", "PprintItemExporter", diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0cf44aed8..f6415ad8e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -25,7 +25,6 @@ from zope.interface import Interface, implementer from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.settings import Settings from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.ftp import ftp_store_file @@ -44,7 +43,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.exporters import BaseItemExporter - from scrapy.settings import BaseSettings + from scrapy.settings import BaseSettings, Settings logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 0e6120c21..0edcce888 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -2,13 +2,11 @@ from __future__ import annotations import gzip import logging -import os import pickle # nosec from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path from time import time -from types import ModuleType from typing import IO, TYPE_CHECKING, Any, cast from weakref import WeakKeyDictionary @@ -19,10 +17,11 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.request import RequestFingerprinter if TYPE_CHECKING: + import os from collections.abc import Callable + from types import ModuleType # typing.Concatenate requires Python 3.10 from typing_extensions import Concatenate @@ -30,6 +29,7 @@ if TYPE_CHECKING: from scrapy.http.request import Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider + from scrapy.utils.request import RequestFingerprinter logger = logging.getLogger(__name__) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index f2e3782a4..7cf08a1bb 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -2,7 +2,6 @@ from __future__ import annotations import logging from datetime import datetime, timezone -from json import JSONEncoder from typing import TYPE_CHECKING, Any from twisted.internet import task @@ -13,6 +12,8 @@ from scrapy.utils.serialize import ScrapyJSONEncoder if TYPE_CHECKING: # typing.Self requires Python 3.11 + from json import JSONEncoder + from typing_extensions import Self from scrapy.crawler import Crawler diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 07dc5880b..89c83d20d 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -13,7 +13,6 @@ import pprint from typing import TYPE_CHECKING, Any from twisted.internet import protocol -from twisted.internet.tcp import Port from scrapy import signals from scrapy.exceptions import NotConfigured @@ -24,6 +23,7 @@ from scrapy.utils.trackref import print_live_refs if TYPE_CHECKING: from twisted.conch import telnet + from twisted.internet.tcp import Port # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 9c29ea4d1..107469504 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -21,7 +21,6 @@ from typing import ( from w3lib.url import safe_url_string -import scrapy from scrapy.http.headers import Headers from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes @@ -37,6 +36,7 @@ if TYPE_CHECKING: # typing.NotRequired and typing.Self require Python 3.11 from typing_extensions import Concatenate, NotRequired, Self + from scrapy import Spider from scrapy.http import Response CallbackT = Callable[Concatenate[Response, ...], Any] @@ -252,7 +252,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]: + def to_dict(self, *, spider: Spider | None = None) -> dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 28e2073a2..4dea5afea 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -5,7 +5,6 @@ import logging from typing import TYPE_CHECKING, Protocol, cast from scrapy import Request -from scrapy.core.downloader import Downloader from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: @@ -14,6 +13,7 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.core.downloader import Downloader from scrapy.crawler import Crawler logger = logging.getLogger(__name__) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index e621525f2..a86aad51c 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -3,7 +3,6 @@ from __future__ import annotations import numbers import os import sys -from collections.abc import Iterable from configparser import ConfigParser from operator import itemgetter from pathlib import Path @@ -15,7 +14,7 @@ from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values if TYPE_CHECKING: - from collections.abc import Collection, Mapping, MutableMapping + from collections.abc import Collection, Iterable, Mapping, MutableMapping def build_component_list( diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ac4358410..2102ce798 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar from warnings import catch_warnings, filterwarnings from twisted.internet import asyncioreactor, error -from twisted.internet.base import DelayedCall from scrapy.utils.misc import load_object @@ -15,6 +14,7 @@ if TYPE_CHECKING: from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from collections.abc import Callable + from twisted.internet.base import DelayedCall from twisted.internet.protocol import ServerFactory from twisted.internet.tcp import Port diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 92b73a91a..a7b84baef 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -16,7 +16,6 @@ from unittest import TestCase, mock from twisted.trial.unittest import SkipTest from scrapy import Spider -from scrapy.crawler import Crawler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available @@ -26,6 +25,8 @@ if TYPE_CHECKING: from twisted.internet.defer import Deferred from twisted.web.client import Response as TxResponse + from scrapy.crawler import Crawler + _T = TypeVar("_T") From 0d7a5e760d5f7761fc819e788f24448349b4c129 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 10 Dec 2024 01:42:03 +0500 Subject: [PATCH 1592/2083] Fix building docs. --- scrapy/http/request/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 107469504..a96a215f4 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -21,6 +21,8 @@ from typing import ( from w3lib.url import safe_url_string +# a workaround for the docs "more than one target found" problem +import scrapy # noqa: TC001 from scrapy.http.headers import Headers from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes @@ -36,7 +38,6 @@ if TYPE_CHECKING: # typing.NotRequired and typing.Self require Python 3.11 from typing_extensions import Concatenate, NotRequired, Self - from scrapy import Spider from scrapy.http import Response CallbackT = Callable[Concatenate[Response, ...], Any] @@ -252,7 +253,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Spider | None = None) -> dict[str, Any]: + def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. From ba30f64268c011387959e939032315b9462da638 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 10 Dec 2024 14:52:16 +0500 Subject: [PATCH 1593/2083] Remove flake8. --- .flake8 | 82 ----------------------------------------- .pre-commit-config.yaml | 11 ------ 2 files changed, 93 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index c4814f13a..000000000 --- a/.flake8 +++ /dev/null @@ -1,82 +0,0 @@ -[flake8] - -max-line-length = 119 -extend-select = TC, TC1 -ignore = - # black disagrees with flake8 about these - E203, E501, E701, E704, W503 - - # Assigning to `os.environ` doesn't clear the environment. - B003 - # Do not use mutable data structures for argument defaults. - B006 - # Loop control variable not used within the loop body. - B007 - # Do not perform function calls in argument defaults. - B008 - # return/continue/break inside finally blocks cause exceptions to be - # silenced. - B012 - # Star-arg unpacking after a keyword argument is strongly discouraged - B026 - # No explicit stacklevel argument found. - B028 - - # docstring does contain unindexed parameters - P102 - # other string does contain unindexed parameters - P103 - - # Missing docstring in public module - D100 - # Missing docstring in public class - D101 - # Missing docstring in public method - D102 - # Missing docstring in public function - D103 - # Missing docstring in public package - D104 - # Missing docstring in magic method - D105 - # Missing docstring in public nested class - D106 - # Missing docstring in __init__ - D107 - # One-line docstring should fit on one line with quotes - D200 - # No blank lines allowed after function docstring - D202 - # 1 blank line required between summary line and description - D205 - # Multi-line docstring closing quotes should be on a separate line - D209 - # First line should end with a period - D400 - # First line should be in imperative mood; try rephrasing - D401 - # First line should not be the function's "signature" - D402 - # First word of the first line should be properly capitalized - D403 - - # Annotation in typing.cast() should be a string literal - TC006 -exclude = - docs/conf.py - -per-file-ignores = -# Exclude files that are meant to provide top-level imports -# E402: Module level import not at top of file -# F401: Module imported but unused - scrapy/__init__.py:E402 - scrapy/core/downloader/handlers/http.py:F401 - scrapy/http/__init__.py:F401 - scrapy/linkextractors/__init__.py:E402,F401 - scrapy/selector/__init__.py:F401 - scrapy/spiders/__init__.py:E402,F401 - tests/CrawlerRunner/change_reactor.py:E402 - - # Issues pending a review: - scrapy/utils/url.py:F403,F405 - tests/test_loader.py:E741 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec8693c00..49db3f610 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,17 +9,6 @@ repos: - id: bandit args: ["-c", "pyproject.toml"] additional_dependencies: ["bandit[toml]"] -- repo: https://github.com/PyCQA/flake8 - rev: 7.1.0 - hooks: - - id: flake8 - additional_dependencies: - - flake8-bugbear - - flake8-comprehensions - - flake8-debugger - - flake8-docstrings - - flake8-string-format - - flake8-type-checking - repo: https://github.com/psf/black.git rev: 24.4.2 hooks: From f4d8d6d8acf8ed26230c3c2b2b51425659de7105 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 10 Dec 2024 14:58:45 +0500 Subject: [PATCH 1594/2083] Tidy up noqa comments. --- pyproject.toml | 1 - tests/CrawlerRunner/change_reactor.py | 4 ++-- tests/test_feedexport.py | 6 +++--- tests/test_item.py | 4 +--- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4d20e5c1c..1378bab50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -310,7 +310,6 @@ ignore = [ # Issues pending a review: "docs/conf.py" = ["E402"] "scrapy/utils/url.py" = ["F403", "F405"] -"tests/CrawlerRunner/change_reactor.py" = ["E402"] "tests/test_loader.py" = ["E741"] [tool.ruff.lint.pydocstyle] diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py index b20aa0c7c..de76e13e8 100644 --- a/tests/CrawlerRunner/change_reactor.py +++ b/tests/CrawlerRunner/change_reactor.py @@ -17,7 +17,7 @@ class NoRequestsSpider(Spider): configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"}) -from scrapy.utils.reactor import install_reactor +from scrapy.utils.reactor import install_reactor # noqa: E402 install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") @@ -25,7 +25,7 @@ runner = CrawlerRunner() d = runner.crawl(NoRequestsSpider) -from twisted.internet import reactor +from twisted.internet import reactor # noqa: E402 d.addBoth(callback=lambda _: reactor.stop()) reactor.run() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 790c347fb..c3d429c2b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -473,7 +473,7 @@ class S3FeedStorageTest(unittest.TestCase): class GCSFeedStorageTest(unittest.TestCase): def test_parse_settings(self): try: - from google.cloud.storage import Client # noqa + from google.cloud.storage import Client # noqa: F401 except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") @@ -487,7 +487,7 @@ class GCSFeedStorageTest(unittest.TestCase): def test_parse_empty_acl(self): try: - from google.cloud.storage import Client # noqa + from google.cloud.storage import Client # noqa: F401 except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") @@ -504,7 +504,7 @@ class GCSFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): try: - from google.cloud.storage import Client # noqa + from google.cloud.storage import Client # noqa: F401 except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") diff --git a/tests/test_item.py b/tests/test_item.py index daf5d4f59..13243b67f 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -273,9 +273,7 @@ class ItemMetaTest(unittest.TestCase): def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 - return ( - __class__ # noqa https://github.com/scrapy/scrapy/issues/2836 - ) + return __class__ MyItem() From cde0845ab2ac390ad7671f50f52e7b6033d4bbc1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 10 Dec 2024 22:53:27 +0400 Subject: [PATCH 1595/2083] Ruff: migrate pyupgrade and bandit, enable some other rules (#6577) --- .pre-commit-config.yaml | 11 -------- pyproject.toml | 39 ++++++++++++++++++++------- scrapy/commands/bench.py | 6 ++--- scrapy/commands/edit.py | 2 +- scrapy/commands/genspider.py | 2 +- scrapy/core/downloader/__init__.py | 2 +- scrapy/downloadermiddlewares/retry.py | 2 +- scrapy/exporters.py | 6 ++--- scrapy/extensions/feedexport.py | 2 +- scrapy/extensions/httpcache.py | 6 ++--- scrapy/extensions/spiderstate.py | 4 +-- scrapy/http/request/form.py | 12 +++++---- scrapy/http/response/__init__.py | 2 +- scrapy/http/response/text.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 4 +-- scrapy/pipelines/files.py | 6 ++--- scrapy/pipelines/images.py | 4 +-- scrapy/pqueues.py | 2 +- scrapy/settings/default_settings.py | 2 +- scrapy/shell.py | 2 +- scrapy/squeues.py | 2 +- scrapy/utils/benchserver.py | 2 +- scrapy/utils/engine.py | 2 +- scrapy/utils/iterators.py | 14 +++++----- scrapy/utils/misc.py | 2 +- scrapy/utils/python.py | 2 +- scrapy/utils/request.py | 4 ++- scrapy/utils/response.py | 2 +- scrapy/utils/sitemap.py | 4 +-- scrapy/utils/url.py | 2 +- scrapy/utils/versions.py | 2 +- tests/test_commands.py | 2 +- tests/test_engine.py | 4 +-- tests/test_pipeline_files.py | 2 +- tests/test_robotstxt_interface.py | 4 +-- tests/test_utils_trackref.py | 20 +++++++------- tests/test_webclient.py | 4 +-- 37 files changed, 103 insertions(+), 89 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 49db3f610..b273e269b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,12 +3,6 @@ repos: rev: v0.8.1 hooks: - id: ruff -- repo: https://github.com/PyCQA/bandit - rev: 1.7.9 - hooks: - - id: bandit - args: ["-c", "pyproject.toml"] - additional_dependencies: ["bandit[toml]"] - repo: https://github.com/psf/black.git rev: 24.4.2 hooks: @@ -23,8 +17,3 @@ repos: - id: blacken-docs additional_dependencies: - black==24.4.2 -- repo: https://github.com/asottile/pyupgrade - rev: v3.18.0 - hooks: - - id: pyupgrade - args: [--py39-plus] diff --git a/pyproject.toml b/pyproject.toml index 1378bab50..977792178 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,16 +92,6 @@ follow_imports = "skip" module = "scrapy.settings.default_settings" ignore_errors = true -[tool.bandit] -skips = [ - "B101", # assert_used, needed for mypy - "B321", # ftplib, https://github.com/scrapy/scrapy/issues/4180 - "B402", # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 - "B411", # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 - "B503", # ssl_with_bad_defaults -] -exclude_dirs = ["tests"] - [tool.bumpversion] current_version = "2.12.0" commit = true @@ -242,10 +232,30 @@ extend-select = [ "C4", # pydocstyle "D", + # flake8-future-annotations + "FA", + # refurb + "FURB", + # flake8-implicit-str-concat + "ISC", + # flake8-logging + "LOG", + # pygrep-hooks + "PGH", + # flake8-quotes + "Q", + # flake8-bandit + "S", + # flake8-slots + "SLOT", # flake8-debugger "T10", # flake8-type-checking "TC", + # pyupgrade + "UP", + # flake8-2020 + "YTT", ] ignore = [ # Assigning to `os.environ` doesn't clear the environment. @@ -296,6 +306,12 @@ ignore = [ "D402", # First word of the first line should be properly capitalized "D403", + # Use of `assert` detected; needed for mypy + "S101", + # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180 + "S321", + # Argument default set to insecure SSL protocol + "S503", ] [tool.ruff.lint.per-file-ignores] @@ -307,6 +323,9 @@ ignore = [ "scrapy/selector/__init__.py" = ["F401"] "scrapy/spiders/__init__.py" = ["E402", "F401"] +# Skip bandit in tests +"tests/**" = ["S"] + # Issues pending a review: "docs/conf.py" = ["E402"] "scrapy/utils/url.py" = ["F403", "F405"] diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index b96c63eb7..714bc38da 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,6 +1,6 @@ from __future__ import annotations -import subprocess # nosec +import subprocess import sys import time from typing import TYPE_CHECKING, Any @@ -40,9 +40,9 @@ class _BenchServer: from scrapy.utils.test import get_testenv pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] - self.proc = subprocess.Popen( + self.proc = subprocess.Popen( # noqa: S603 pargs, stdout=subprocess.PIPE, env=get_testenv() - ) # nosec + ) assert self.proc.stdout self.proc.stdout.readline() diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 438375e02..0e046cece 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -41,4 +41,4 @@ class Command(ScrapyCommand): sfile = sys.modules[spidercls.__module__].__file__ assert sfile sfile = sfile.replace(".pyc", ".py") - self.exitcode = os.system(f'{editor} "{sfile}"') # nosec + self.exitcode = os.system(f'{editor} "{sfile}"') # noqa: S605 diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2e70b2865..38f917c7e 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -118,7 +118,7 @@ class Command(ScrapyCommand): if template_file: self._genspider(module, name, url, opts.template, template_file) if opts.edit: - self.exitcode = os.system(f'scrapy edit "{name}"') # nosec + self.exitcode = os.system(f'scrapy edit "{name}"') # noqa: S605 def _generate_template_variables( self, diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 434b316e9..78dc16df6 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -52,7 +52,7 @@ class Slot: def download_delay(self) -> float: if self.randomize_delay: - return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosec + return random.uniform(0.5 * self.delay, 1.5 * self.delay) # noqa: S311 return self.delay def close(self) -> None: diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 9fab172a8..723fe5e93 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -115,7 +115,7 @@ def get_retry_request( return new_request stats.inc_value(f"{stats_base_key}/max_reached") logger.error( - "Gave up retrying %(request)s (failed %(retry_times)d times): " "%(reason)s", + "Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s", {"request": request, "retry_times": retry_times, "reason": reason}, extra={"spider": spider}, ) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 9380b7e78..b6997ef67 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -6,13 +6,13 @@ from __future__ import annotations import csv import marshal -import pickle # nosec +import pickle import pprint from collections.abc import Callable, Iterable, Mapping from io import BytesIO, TextIOWrapper from typing import TYPE_CHECKING, Any -from xml.sax.saxutils import XMLGenerator # nosec -from xml.sax.xmlreader import AttributesImpl # nosec +from xml.sax.saxutils import XMLGenerator +from xml.sax.xmlreader import AttributesImpl from itemadapter import ItemAdapter, is_item diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index f6415ad8e..edea7cc39 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -681,7 +681,7 @@ class FeedExporter: return True except NotConfigured as e: logger.error( - "Disabled feed storage scheme: %(scheme)s. " "Reason: %(reason)s", + "Disabled feed storage scheme: %(scheme)s. Reason: %(reason)s", {"scheme": scheme, "reason": str(e)}, ) else: diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 0edcce888..965d6434b 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -2,7 +2,7 @@ from __future__ import annotations import gzip import logging -import pickle # nosec +import pickle from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path @@ -309,7 +309,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return None # expired - return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # nosec + return cast(dict[str, Any], pickle.loads(db[f"{key}_data"])) # noqa: S301 class FilesystemCacheStorage: @@ -392,7 +392,7 @@ class FilesystemCacheStorage: if 0 < self.expiration_secs < time() - mtime: return None # expired with self._open(metapath, "rb") as f: - return cast(dict[str, Any], pickle.load(f)) # nosec + return cast(dict[str, Any], pickle.load(f)) # noqa: S301 def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]: diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 642919be9..7b8756572 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,6 +1,6 @@ from __future__ import annotations -import pickle # nosec +import pickle from pathlib import Path from typing import TYPE_CHECKING @@ -41,7 +41,7 @@ class SpiderState: def spider_opened(self, spider: Spider) -> None: if self.jobdir and Path(self.statefn).exists(): with Path(self.statefn).open("rb") as f: - spider.state = pickle.load(f) # type: ignore[attr-defined] # nosec + spider.state = pickle.load(f) # type: ignore[attr-defined] # noqa: S301 else: spider.state = {} # type: ignore[attr-defined] diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b3c3d7c7a..de3b24de0 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -11,11 +11,13 @@ from collections.abc import Iterable from typing import TYPE_CHECKING, Any, Optional, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit -from lxml.html import FormElement # nosec -from lxml.html import InputElement # nosec -from lxml.html import MultipleSelectOptions # nosec -from lxml.html import SelectElement # nosec -from lxml.html import TextareaElement # nosec +from lxml.html import ( + FormElement, + InputElement, + MultipleSelectOptions, + SelectElement, + TextareaElement, +) from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index d50388548..387805f57 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -107,7 +107,7 @@ class Response(object_ref): self._url: str = url else: raise TypeError( - f"{type(self).__name__} url must be str, " f"got {type(url).__name__}" + f"{type(self).__name__} url must be str, got {type(url).__name__}" ) @property diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index c713f6188..f954b5e9e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -308,7 +308,7 @@ def _url_from_selector(sel: parsel.Selector) -> str: raise _InvalidSelector(f"Unsupported selector: {sel}") if sel.root.tag not in ("a", "link"): raise _InvalidSelector( - "Only <a> and <link> elements are supported; " f"got <{sel.root.tag}>" + f"Only <a> and <link> elements are supported; got <{sel.root.tag}>" ) href = sel.root.get("href") if href is None: diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 192f937ce..bd96ccf19 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -12,7 +12,7 @@ from functools import partial from typing import TYPE_CHECKING, Any, Union, cast from urllib.parse import urljoin, urlparse -from lxml import etree # nosec +from lxml import etree from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string @@ -26,7 +26,7 @@ from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain if TYPE_CHECKING: - from lxml.html import HtmlElement # nosec + from lxml.html import HtmlElement from scrapy import Selector from scrapy.http import TextResponse diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index bebf6039b..16bd45c00 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -66,7 +66,7 @@ def _md5sum(file: IO[bytes]) -> str: >>> _md5sum(BytesIO(b'file content to hash')) '784406af91dd5a54fbb9c84c2236595a' """ - m = hashlib.md5() # nosec + m = hashlib.md5() # noqa: S324 while True: d = file.read(8096) if not d: @@ -399,7 +399,7 @@ class FTPFilesStore: ftp.set_pasv(False) file_path = f"{self.basedir}/{path}" last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) - m = hashlib.md5() # nosec + m = hashlib.md5() # noqa: S324 ftp.retrbinary(f"RETR {file_path}", m.update) return {"last_modified": last_modified, "checksum": m.hexdigest()} # The file doesn't exist @@ -734,7 +734,7 @@ class FilesPipeline(MediaPipeline): *, item: Any = None, ) -> str: - media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec + media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324 media_ext = Path(request.url).suffix # Handles empty and wild extensions by trying to guess the # mime type then extension or default to empty string otherwise diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e86e7c493..29dc13f0a 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -258,7 +258,7 @@ class ImagesPipeline(FilesPipeline): *, item: Any = None, ) -> str: - image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec + image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324 return f"full/{image_guid}.jpg" def thumb_path( @@ -270,5 +270,5 @@ class ImagesPipeline(FilesPipeline): *, item: Any = None, ) -> str: - thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec + thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324 return f"thumbs/{thumb_id}/{thumb_guid}.jpg" diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 4dea5afea..5b2f81335 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -33,7 +33,7 @@ def _path_safe(text: str) -> str: pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part - unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # nosec + unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # noqa: S324 return "-".join([pathable_slot, unique_slot]) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7ba0128a5..89ab21fbe 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -178,7 +178,7 @@ FILES_STORE_S3_ACL = "private" FILES_STORE_GCS_ACL = "" FTP_USER = "anonymous" -FTP_PASSWORD = "guest" # nosec +FTP_PASSWORD = "guest" # noqa: S105 FTP_PASSIVE_MODE = True GCS_PROJECT_ID = None diff --git a/scrapy/shell.py b/scrapy/shell.py index 31349c4ff..5d0ab1e4d 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -70,7 +70,7 @@ class Shell: else: self.populate_vars() if self.code: - print(eval(self.code, globals(), self.vars)) # nosec + print(eval(self.code, globals(), self.vars)) # noqa: S307 else: """ Detect interactive shell setting in scrapy.cfg diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 7732187fd..80bb37e93 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -5,7 +5,7 @@ Scheduler queues from __future__ import annotations import marshal -import pickle # nosec +import pickle from pathlib import Path from typing import TYPE_CHECKING, Any diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 550516141..923ec005e 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -15,7 +15,7 @@ class Root(Resource): def render(self, request: Request) -> bytes: total = _getarg(request, b"total", 100, int) show = _getarg(request, b"show", 10, int) - nlist = [random.randint(1, total) for _ in range(show)] # nosec + nlist = [random.randint(1, total) for _ in range(show)] # noqa: S311 request.write(b"<html><head></head><body>") assert request.args is not None args = request.args.copy() diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 1430ed8d6..1948009e8 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -32,7 +32,7 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: checks: list[tuple[str, Any]] = [] for test in tests: try: - checks += [(test, eval(test))] # nosec + checks += [(test, eval(test))] # noqa: S307 except Exception as e: checks += [(test, f"{type(e).__name__} (exception)")] diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index ba58d939c..e8ed7b60a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -7,7 +7,7 @@ from io import StringIO from typing import TYPE_CHECKING, Any, Literal, cast, overload from warnings import warn -from lxml import etree # nosec +from lxml import etree from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse @@ -41,10 +41,10 @@ def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]: nodename_patt = re.escape(nodename) - DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) - HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S) - END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S) - NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) + DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.DOTALL) + HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.DOTALL) + END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.DOTALL) + NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.DOTALL) text = _body_or_str(obj) document_header_match = re.search(DOCUMENT_HEADER_RE, text) @@ -58,7 +58,9 @@ def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]: for tagname in reversed(re.findall(END_TAG_RE, header_end)): assert header_end_idx tag = re.search( - rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S + rf"<\s*{tagname}.*?xmlns[:=][^>]*>", + text[: header_end_idx[1]], + re.DOTALL, ) if tag: for x in re.findall(NAMESPACE_RE, tag.group()): diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index eefadd07d..5ce4863f6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -116,7 +116,7 @@ def md5sum(file: IO[bytes]) -> str: ScrapyDeprecationWarning, stacklevel=2, ) - m = hashlib.md5() # nosec + m = hashlib.md5() # noqa: S324 while True: d = file.read(8096) if not d: diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index b9babb08f..511511301 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -136,7 +136,7 @@ def to_bytes( return text if not isinstance(text, str): raise TypeError( - "to_bytes must receive a str or bytes " f"object, got {type(text).__name__}" + f"to_bytes must receive a str or bytes object, got {type(text).__name__}" ) if encoding is None: encoding = "utf-8" diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 20e3151da..ad811e804 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -94,7 +94,9 @@ def fingerprint( "headers": headers, } fingerprint_json = json.dumps(fingerprint_data, sort_keys=True) - cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() # nosec + cache[cache_key] = hashlib.sha1( # noqa: S324 + fingerprint_json.encode() + ).digest() return cache[cache_key] diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 7c8ca51f2..a7ad4544d 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -104,7 +104,7 @@ def open_in_browser( elif isinstance(response, TextResponse): ext = ".txt" else: - raise TypeError("Unsupported response type: " f"{response.__class__.__name__}") + raise TypeError(f"Unsupported response type: {response.__class__.__name__}") fd, fname = tempfile.mkstemp(ext) os.write(fd, body) os.close(fd) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index c572580ae..b60fe929e 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -10,7 +10,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any from urllib.parse import urljoin -import lxml.etree # nosec +import lxml.etree if TYPE_CHECKING: from collections.abc import Iterable, Iterator @@ -24,7 +24,7 @@ class Sitemap: xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) - self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # nosec + self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320 rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index a5cc22c1c..2539f30c7 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -89,7 +89,7 @@ def escape_ajax(url: str) -> str: def add_http_if_no_scheme(url: str) -> str: """Add http as the default scheme if it is missing from the url.""" - match = re.match(r"^\w+://", url, flags=re.I) + match = re.match(r"^\w+://", url, flags=re.IGNORECASE) if not match: parts = urlparse(url) scheme = "http:" if parts.netloc else "http://" diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 4e9e29286..996a5cdb3 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -3,7 +3,7 @@ import sys import cryptography import cssselect -import lxml.etree # nosec +import lxml.etree import parsel import twisted import w3lib diff --git a/tests/test_commands.py b/tests/test_commands.py index e7df7b6e8..32b69de8a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -238,7 +238,7 @@ class StartprojectTemplatesTest(ProjectTest): args = ["--set", f"TEMPLATES_DIR={self.tmpl}"] p, out, err = self.proc("startproject", self.project_name, *args) self.assertIn( - f"New Scrapy project '{self.project_name}', " "using template directory", + f"New Scrapy project '{self.project_name}', using template directory", out, ) self.assertIn(self.tmpl_proj, out) diff --git a/tests/test_engine.py b/tests/test_engine.py index 2ebc0b5e4..8d645eada 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -67,8 +67,8 @@ class TestSpider(Spider): allowed_domains = ["scrapytest.org", "localhost"] itemurl_re = re.compile(r"item\d+.html") - name_re = re.compile(r"<h1>(.*?)</h1>", re.M) - price_re = re.compile(r">Price: \$(.*?)<", re.M) + name_re = re.compile(r"<h1>(.*?)</h1>", re.MULTILINE) + price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE) item_cls: type = TestItem diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 96a2c42b7..2be5e09bc 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -631,7 +631,7 @@ class TestGCSFilesStore(unittest.TestCase): """ assert_gcs_environ() try: - import google.cloud.storage # noqa + import google.cloud.storage # noqa: F401 except ModuleNotFoundError: raise unittest.SkipTest("google-cloud-storage is not installed") with mock.patch("google.cloud.storage") as _: diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 541979dcc..e127cc2e3 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -66,14 +66,14 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second")) def test_length_based_precedence(self): - robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page" + robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) self.assertTrue(rp.allowed("https://www.site.local/page", "*")) def test_order_based_precedence(self): - robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page" + robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 35d1508c6..ef07d625f 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -21,9 +21,9 @@ class TrackrefTestCase(unittest.TestCase): trackref.live_refs.clear() def test_format_live_refs(self): - o1 = Foo() # NOQA - o2 = Bar() # NOQA - o3 = Foo() # NOQA + o1 = Foo() # noqa: F841 + o2 = Bar() # noqa: F841 + o3 = Foo() # noqa: F841 self.assertEqual( trackref.format_live_refs(), """\ @@ -50,7 +50,7 @@ Bar 1 oldest: 0s ago @mock.patch("sys.stdout", new_callable=StringIO) def test_print_live_refs_with_objects(self, stdout): - o1 = Foo() # NOQA + o1 = Foo() # noqa: F841 trackref.print_live_refs() self.assertEqual( stdout.getvalue(), @@ -61,11 +61,11 @@ Foo 1 oldest: 0s ago\n\n""", ) def test_get_oldest(self): - o1 = Foo() # NOQA + o1 = Foo() # noqa: F841 o1_time = time() - o2 = Bar() # NOQA + o2 = Bar() # noqa: F841 o3_time = time() if o3_time <= o1_time: @@ -74,15 +74,15 @@ Foo 1 oldest: 0s ago\n\n""", if o3_time <= o1_time: raise SkipTest("time.time is not precise enough") - o3 = Foo() # NOQA + o3 = Foo() # noqa: F841 self.assertIs(trackref.get_oldest("Foo"), o1) self.assertIs(trackref.get_oldest("Bar"), o2) self.assertIsNone(trackref.get_oldest("XXX")) def test_iter_all(self): - o1 = Foo() # NOQA - o2 = Bar() # NOQA - o3 = Foo() # NOQA + o1 = Foo() # noqa: F841 + o2 = Bar() # noqa: F841 + o3 = Foo() # noqa: F841 self.assertEqual( set(trackref.iter_all("Foo")), {o1, o3}, diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 1cad68b9c..0a594aa7c 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -161,7 +161,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): # test minimal sent headers factory = client.ScrapyHTTPClientFactory(Request("http://foo/bar")) - self._test(factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"\r\n") + self._test(factory, b"GET /bar HTTP/1.0\r\nHost: foo\r\n\r\n") # test a simple POST with body and content-type factory = client.ScrapyHTTPClientFactory( @@ -191,7 +191,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): self._test( factory, - b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Content-Length: 0\r\n" b"\r\n", + b"POST /bar HTTP/1.0\r\nHost: foo\r\nContent-Length: 0\r\n\r\n", ) # test with single and multivalued headers From 5c2df5cf2aec8ea1ef25c066c08920a6adbd42ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 12 Dec 2024 11:38:30 +0100 Subject: [PATCH 1596/2083] Contributing: add a section on finding work (#6575) --- docs/contributing.rst | 87 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 8 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index e8ffe83b4..f5c1c74b8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -74,18 +74,81 @@ guidelines when you're going to report a new bug. .. _Minimal, Complete, and Verifiable example: https://stackoverflow.com/help/mcve +.. _find-work: + +Finding work +============ + +If you have decided to make a contribution to Scrapy, but you do not know what +to contribute, you have a few options to find pending work: + +- Check out the `contribution GitHub page`_, which lists open issues tagged + as **good first issue**. + + .. _contribution GitHub page: https://github.com/scrapy/scrapy/contribute + + There are also `help wanted issues`_ but mind that some may require + familiarity with the Scrapy code base. You can also target any other issue + provided it is not tagged as **discuss**. + +- If you enjoy writing documentation, there are `documentation issues`_ as + well, but mind that some may require familiarity with the Scrapy code base + as well. + + .. _documentation issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3Adocs+ + +- If you enjoy :ref:`writing automated tests <write-tests>`, you can work on + increasing our `test coverage`_. + +- If you enjoy code cleanup, we welcome fixes for issues detected by our + static analysis tools. See ``pyproject.toml`` for silenced issues that may + need addressing. + + Mind that some issues we do not aim to address at all, and usually include + a comment on them explaining the reason; not to confuse with comments that + state what the issue is about, for non-descriptive issue codes. + +If you have found an issue, make sure you read the entire issue thread before +you ask questions. That includes related issues and pull requests that show up +in the issue thread when the issue is mentioned elsewhere. + +We do not assign issues, and you do not need to announce that you are going to +start working on an issue either. If you want to work on an issue, just go +ahead and :ref:`write a patch for it <writing-patches>`. + +Do not discard an issue simply because there is an open pull request for it. +Check if open pull requests are active first. And even if some are active, if +you think you can build a better implementation, feel free to create a pull +request with your approach. + +If you decide to work on something without an open issue, please: + +- Do not create an issue to work on code coverage or code cleanup, create a + pull request directly. + +- Do not create both an issue and a pull request right away. Either open an + issue first to get feedback on whether or not the issue is worth + addressing, and create a pull request later only if the feedback from the + team is positive, or create only a pull request, if you think a discussion + will be easier over your code. + +- Do not add docstrings for the sake of adding docstrings, or only to address + silenced Ruff issues. We expect docstrings to exist only when they add + something significant to readers, such as explaining something that is not + easier to understand from reading the corresponding code, summarizing a + long, hard-to-read implementation, providing context about calling code, or + indicating purposely uncaught exceptions from called code. + +- Do not add tests that use as much mocking as possible just to touch a given + line of code and hence improve line coverage. While we do aim to maximize + test coverage, tests should be written for real scenarios, with minimum + mocking. We usually prefer end-to-end tests. + .. _writing-patches: Writing patches =============== -Scrapy has a list of `good first issues`_ and `help wanted issues`_ that you -can work on. These issues are a great way to get started with contributing to -Scrapy. If you're new to the codebase, you may want to focus on documentation -or testing-related issues, as they are always useful and can help you get -more familiar with the project. You can also check Scrapy's `test coverage`_ -to see which areas may benefit from more tests. - The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged. Well-written patches should: @@ -131,6 +194,14 @@ Remember to explain what was fixed or the new functionality (what it is, why it's needed, etc). The more info you include, the easier will be for core developers to understand and accept your patch. +If your pull request aims to resolve an open issue, `link it accordingly +<https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword>`__, +e.g.: + +.. code-block:: none + + Resolves #123 + You can also discuss the new functionality (or bug fix) before creating the patch, but it's always good to have a patch ready to illustrate your arguments and show that you have put some additional thought into the subject. A good @@ -242,6 +313,7 @@ Documentation about deprecated features must be removed as those features are deprecated, so that new readers do not run into it. New deprecations and deprecation removals are documented in the :ref:`release notes <news>`. +.. _write-tests: Tests ===== @@ -320,6 +392,5 @@ And their unit-tests are in:: .. _PEP 257: https://peps.python.org/pep-0257/ .. _pull request: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist -.. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 .. _test coverage: https://app.codecov.io/gh/scrapy/scrapy From 802c67072cb1fb47c8f1033dd51cab557c72b6a4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 12 Dec 2024 20:12:55 +0500 Subject: [PATCH 1597/2083] Enable ruff --fix. --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b273e269b..39b9a33aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,6 +3,7 @@ repos: rev: v0.8.1 hooks: - id: ruff + args: [ --fix ] - repo: https://github.com/psf/black.git rev: 24.4.2 hooks: From 897e124a27772b9a710f501954f50e3c66e27c79 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 12 Dec 2024 20:22:03 +0500 Subject: [PATCH 1598/2083] Add flake8-return rules to ruff. --- pyproject.toml | 2 ++ scrapy/commands/parse.py | 5 +++-- scrapy/downloadermiddlewares/offsite.py | 2 +- scrapy/extensions/feedexport.py | 3 +-- scrapy/extensions/httpcache.py | 6 ++---- scrapy/extensions/postprocessing.py | 3 +-- scrapy/linkextractors/lxmlhtml.py | 3 +-- scrapy/robotstxt.py | 9 +++------ scrapy/utils/reactor.py | 2 +- tests/spiders.py | 2 +- tests/test_downloadermiddleware.py | 3 +-- tests/test_downloadermiddleware_robotstxt.py | 3 +-- tests/test_feedexport.py | 6 ++---- tests/test_http_response.py | 6 ++---- tests/test_proxy_connect.py | 3 +-- tests/test_request_cb_kwargs.py | 2 -- tests/test_request_left.py | 3 +-- tests/test_spidermiddleware.py | 3 +-- tests/test_spidermiddleware_output_chain.py | 3 --- 19 files changed, 25 insertions(+), 44 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 977792178..9a4a91a7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -244,6 +244,8 @@ extend-select = [ "PGH", # flake8-quotes "Q", + # flake8-return + "RET", # flake8-bandit "S", # flake8-slots diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index fc16e46d1..62d094411 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -225,8 +225,9 @@ class Command(BaseRunSpiderCommand): cb_kwargs: dict[str, Any] | None = None, ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} - d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) - return d + return maybeDeferred( + self.iterate_spider_output, callback(response, **cb_kwargs) + ) def get_callback_from_rules( self, spider: Spider, response: Response diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 05ec4cad4..a69f531a7 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -41,7 +41,7 @@ class OffsiteMiddleware: def process_request(self, request: Request, spider: Spider) -> None: if request.dont_filter or self.should_follow(request, spider): - return None + return domain = urlparse_cached(request).hostname if domain and domain not in self.domains_seen: self.domains_seen.add(domain) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index edea7cc39..b6e6f55a6 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -586,7 +586,7 @@ class FeedExporter: :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ storage = self._get_storage(uri, feed_options) - slot = FeedSlot( + return FeedSlot( storage=storage, uri=uri, format=feed_options["format"], @@ -600,7 +600,6 @@ class FeedExporter: settings=self.settings, crawler=self.crawler, ) - return slot def item_scraped(self, item: Any, spider: Spider) -> None: slots = [] diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 965d6434b..929807de8 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -282,8 +282,7 @@ class DbmCacheStorage: headers = Headers(data["headers"]) body = data["body"] respcls = responsetypes.from_args(headers=headers, url=url, body=body) - response = respcls(url=url, headers=headers, status=status, body=body) - return response + return respcls(url=url, headers=headers, status=status, body=body) def store_response( self, spider: Spider, request: Request, response: Response @@ -349,8 +348,7 @@ class FilesystemCacheStorage: status = metadata["status"] headers = Headers(headers_raw_to_dict(rawheaders)) respcls = responsetypes.from_args(headers=headers, url=url, body=body) - response = respcls(url=url, headers=headers, status=status, body=body) - return response + return respcls(url=url, headers=headers, status=status, body=body) def store_response( self, spider: Spider, request: Request, response: Response diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 16067f82b..b1fa160c8 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -157,8 +157,7 @@ class PostProcessingManager(IOBase): return True def _load_plugins(self, plugins: list[Any]) -> list[Any]: - plugins = [load_object(plugin) for plugin in plugins] - return plugins + return [load_object(plugin) for plugin in plugins] def _get_head_plugin(self) -> Any: prev = self.file diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index bd96ccf19..f195dbdd7 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -253,8 +253,7 @@ class LxmlLinkExtractor: if self.canonicalize: for link in links: link.url = canonicalize_url(link.url) - links = self.link_extractor._process_links(links) - return links + return self.link_extractor._process_links(links) def _extract_links(self, *args: Any, **kwargs: Any) -> list[Link]: return self.link_extractor._extract_links(*args, **kwargs) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index f0a6e7467..844969c6d 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -79,8 +79,7 @@ class PythonRobotParser(RobotParser): @classmethod def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o + return cls(robotstxt_body, spider) def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) @@ -100,8 +99,7 @@ class RerpRobotParser(RobotParser): @classmethod def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o + return cls(robotstxt_body, spider) def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) @@ -120,8 +118,7 @@ class ProtegoRobotParser(RobotParser): @classmethod def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: spider = None if not crawler else crawler.spider - o = cls(robotstxt_body, spider) - return o + return cls(robotstxt_body, spider) def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 2102ce798..2d781cc27 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -36,7 +36,7 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: return reactor.listenTCP(0, factory, interface=host) if len(portrange) == 1: return reactor.listenTCP(portrange[0], factory, interface=host) - for x in range(portrange[0], portrange[1] + 1): + for x in range(portrange[0], portrange[1] + 1): # noqa: RET503 try: return reactor.listenTCP(x, factory, interface=host) except error.CannotListenError: diff --git a/tests/spiders.py b/tests/spiders.py index 63c7a6f9b..0180cf757 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -175,7 +175,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): status = await get_from_asyncio_queue(response.status) self.logger.info(f"Got response {status}, req_id {req_id}") if req_id > 0: - return + return None reqs = [] for i in range(1, 3): req = Request(self.start_urls[0], dont_filter=True, meta={"req_id": i}) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index dd3f8ceb9..8987a76fb 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -250,8 +250,7 @@ class MiddlewareUsingCoro(ManagerTestCase): class CoroMiddleware: async def process_request(self, request, spider): await asyncio.sleep(0.1) - result = await get_from_asyncio_queue(resp) - return result + return await get_from_asyncio_queue(resp) self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 12b541456..535e07c1f 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -116,7 +116,7 @@ Disallow: /some/randome/page.html def test_robotstxt_garbage(self): # garbage response should be discarded, equal 'allow all' middleware = RobotsTxtMiddleware(self._get_garbage_crawler()) - deferred = DeferredList( + return DeferredList( [ self.assertNotIgnored(Request("http://site.local"), middleware), self.assertNotIgnored(Request("http://site.local/allowed"), middleware), @@ -127,7 +127,6 @@ Disallow: /some/randome/page.html ], fireOnOneErrback=True, ) - return deferred def _get_emptybody_crawler(self): crawler = self.crawler diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c3d429c2b..031d6180d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -134,8 +134,7 @@ class FTPFeedStorageTest(unittest.TestCase): name = "test_spider" crawler = get_crawler(settings_dict=settings) - spider = TestSpider.from_crawler(crawler) - return spider + return TestSpider.from_crawler(crawler) def _store(self, uri, content, feed_options=None, settings=None): crawler = get_crawler(settings_dict=settings or {}) @@ -210,8 +209,7 @@ class BlockingFeedStorageTest(unittest.TestCase): name = "test_spider" crawler = get_crawler(settings_dict=settings) - spider = TestSpider.from_crawler(crawler) - return spider + return TestSpider.from_crawler(crawler) def test_default_temp_dir(self): b = BlockingFeedStorage() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index b8a277295..679cc8238 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -342,13 +342,11 @@ class BaseResponseTest(unittest.TestCase): def _links_response(self): body = get_testdata("link_extractor", "linkextractor.html") - resp = self.response_class("http://example.com/index", body=body) - return resp + return self.response_class("http://example.com/index", body=body) def _links_response_no_href(self): body = get_testdata("link_extractor", "linkextractor_no_href.html") - resp = self.response_class("http://example.com/index", body=body) - return resp + return self.response_class("http://example.com/index", body=body) class TextResponseTest(BaseResponseTest): diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 93f006c76..26bd6332c 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -48,8 +48,7 @@ sys.exit(mitmdump()) ) line = self.proc.stdout.readline().decode("utf-8") host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1) - address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}" - return address + return f"http://{self.auth_user}:{self.auth_pass}@{host_port}" def stop(self): self.proc.kill() diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 8c0e5764a..b178c928b 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -16,7 +16,6 @@ class InjectArgumentsDownloaderMiddleware: def process_request(self, request, spider): if request.callback.__name__ == "parse_downloader_mw": request.cb_kwargs["from_process_request"] = True - return None def process_response(self, request, response, spider): if request.callback.__name__ == "parse_downloader_mw": @@ -39,7 +38,6 @@ class InjectArgumentsSpiderMiddleware: request = response.request if request.callback.__name__ == "parse_spider_mw": request.cb_kwargs["from_process_spider_input"] = True - return None def process_spider_output(self, response, result, spider): for element in result: diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 54155f7ef..ba1b70695 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -18,8 +18,7 @@ class SignalCatcherSpider(Spider): @classmethod def from_crawler(cls, crawler, *args, **kwargs): - spider = cls(crawler, *args, **kwargs) - return spider + return cls(crawler, *args, **kwargs) def on_request_left(self, request, spider): self.caught_times += 1 diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 1a80eb7be..1aca0fe54 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -37,8 +37,7 @@ class SpiderMiddlewareTestCase(TestCase): results = [] dfd.addBoth(results.append) self._wait(dfd) - ret = results[0] - return ret + return results[0] class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index fad5dcaac..670c41f2b 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -12,7 +12,6 @@ class LogExceptionMiddleware: spider.logger.info( "Middleware: %s exception caught", exception.__class__.__name__ ) - return None # ================================================================================ @@ -170,7 +169,6 @@ class _GeneratorDoNothingMiddleware: def process_spider_exception(self, response, exception, spider): method = f"{self.__class__.__name__}.process_spider_exception" spider.logger.info("%s: %s caught", method, exception.__class__.__name__) - return None class GeneratorFailMiddleware: @@ -240,7 +238,6 @@ class _NotGeneratorDoNothingMiddleware: def process_spider_exception(self, response, exception, spider): method = f"{self.__class__.__name__}.process_spider_exception" spider.logger.info("%s: %s caught", method, exception.__class__.__name__) - return None class NotGeneratorFailMiddleware: From e7595837a60b896eb0efed4b452b0bdd62e8039f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 12 Dec 2024 20:25:16 +0500 Subject: [PATCH 1599/2083] Add flake8-raise rules to ruff. --- pyproject.toml | 2 ++ scrapy/commands/crawl.py | 2 +- scrapy/commands/edit.py | 2 +- scrapy/commands/fetch.py | 2 +- scrapy/commands/genspider.py | 2 +- scrapy/commands/parse.py | 2 +- scrapy/commands/runspider.py | 2 +- scrapy/commands/startproject.py | 2 +- scrapy/core/scheduler.py | 6 +++--- scrapy/pipelines/media.py | 10 +++++----- scrapy/resolver.py | 2 +- scrapy/spidermiddlewares/referer.py | 2 +- tests/test_addons.py | 2 +- tests/test_downloader_handlers.py | 2 +- tests/test_downloadermiddleware.py | 2 +- tests/test_downloadermiddleware_httpcompression.py | 2 +- tests/test_spidermiddleware.py | 2 +- tests/test_spidermiddleware_output_chain.py | 12 ++++++------ tests/test_squeues_request.py | 2 +- .../test_return_with_argument_inside_generator.py | 2 +- 20 files changed, 32 insertions(+), 30 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9a4a91a7e..131684724 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -246,6 +246,8 @@ extend-select = [ "Q", # flake8-return "RET", + # flake8-raise + "RSE", # flake8-bandit "S", # flake8-slots diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 0d71ab6c6..86d4cc41c 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -22,7 +22,7 @@ class Command(BaseRunSpiderCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) < 1: - raise UsageError() + raise UsageError if len(args) > 1: raise UsageError( "running 'scrapy crawl' with more than one spider is not supported" diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 0e046cece..d153a5271 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -28,7 +28,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: - raise UsageError() + raise UsageError editor = self.settings["EDITOR"] assert self.crawler_process diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 05e5e53e9..8a8d04ff6 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -68,7 +68,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: Namespace) -> None: if len(args) != 1 or not is_url(args[0]): - raise UsageError() + raise UsageError request = Request( args[0], callback=self._print_response, diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 38f917c7e..d7dc104c2 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -101,7 +101,7 @@ class Command(ScrapyCommand): print(template_file.read_text(encoding="utf-8")) return if len(args) != 2: - raise UsageError() + raise UsageError name, url = args[0:2] url = verify_url_scheme(url) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 62d094411..cc5c1350b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -399,7 +399,7 @@ class Command(BaseRunSpiderCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: # parse arguments if not len(args) == 1 or not is_url(args[0]): - raise UsageError() + raise UsageError url = args[0] # prepare spidercls diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 55211f8d7..bf8e41020 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -43,7 +43,7 @@ class Command(BaseRunSpiderCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: - raise UsageError() + raise UsageError filename = Path(args[0]) if not filename.exists(): raise UsageError(f"File not found: {filename}\n") diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 6da877610..5cb73f0d2 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -92,7 +92,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) not in (1, 2): - raise UsageError() + raise UsageError project_name = args[0] diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index f09d1903c..823677175 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -98,7 +98,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ ``True`` if the scheduler has enqueued requests, ``False`` otherwise """ - raise NotImplementedError() + raise NotImplementedError @abstractmethod def enqueue_request(self, request: Request) -> bool: @@ -112,7 +112,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): For reference, the default Scrapy scheduler returns ``False`` when the request is rejected by the dupefilter. """ - raise NotImplementedError() + raise NotImplementedError @abstractmethod def next_request(self) -> Request | None: @@ -124,7 +124,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): to the downloader in the current reactor cycle. The engine will continue calling ``next_request`` until ``has_pending_requests`` is ``False``. """ - raise NotImplementedError() + raise NotImplementedError class Scheduler(BaseScheduler): diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 691a1cbf2..b16f1cb84 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -293,12 +293,12 @@ class MediaPipeline(ABC): self, request: Request, info: SpiderInfo, *, item: Any = None ) -> Deferred[FileInfo | None]: """Check request before starting download""" - raise NotImplementedError() + raise NotImplementedError @abstractmethod def get_media_requests(self, item: Any, info: SpiderInfo) -> list[Request]: """Returns the media requests to download""" - raise NotImplementedError() + raise NotImplementedError @abstractmethod def media_downloaded( @@ -310,14 +310,14 @@ class MediaPipeline(ABC): item: Any = None, ) -> FileInfo: """Handler for success downloads""" - raise NotImplementedError() + raise NotImplementedError @abstractmethod def media_failed( self, failure: Failure, request: Request, info: SpiderInfo ) -> NoReturn: """Handler for failed downloads""" - raise NotImplementedError() + raise NotImplementedError def item_completed( self, results: list[FileInfoOrError], item: Any, info: SpiderInfo @@ -345,4 +345,4 @@ class MediaPipeline(ABC): item: Any = None, ) -> str: """Returns the path where downloaded media should be stored""" - raise NotImplementedError() + raise NotImplementedError diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 0e8260736..f5f00ab0f 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -76,7 +76,7 @@ class HostResolution: self.name: str = name def cancel(self) -> None: - raise NotImplementedError() + raise NotImplementedError @provider(IResolutionReceiver) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 720217c97..93b7fcf17 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -51,7 +51,7 @@ class ReferrerPolicy: name: str def referrer(self, response_url: str, request_url: str) -> str | None: - raise NotImplementedError() + raise NotImplementedError def stripped_referrer(self, url: str) -> str | None: if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: diff --git a/tests/test_addons.py b/tests/test_addons.py index 775f629b3..17949997c 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -64,7 +64,7 @@ class AddonManagerTest(unittest.TestCase): def test_notconfigured(self): class NotConfiguredAddon: def update_settings(self, settings): - raise NotConfigured() + raise NotConfigured settings_dict = { "ADDONS": {NotConfiguredAddon: 0}, diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 6a7597e9f..3fcba4ef2 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -894,7 +894,7 @@ class S3TestCase(unittest.TestCase): except Exception as e: self.assertIsInstance(e, (TypeError, NotConfigured)) else: - raise AssertionError() + raise AssertionError def test_request_signing1(self): # gets an object from the johnsmith bucket. diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 8987a76fb..e650b4936 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -178,7 +178,7 @@ class ProcessExceptionInvalidOutput(ManagerTestCase): class InvalidProcessExceptionMiddleware: def process_request(self, request, spider): - raise Exception() + raise Exception def process_exception(self, request, exception, spider): return 1 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 934af6590..78d0dd99d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -59,7 +59,7 @@ class HttpCompressionTest(TestCase): def _getresponse(self, coding): if coding not in FORMAT: - raise ValueError() + raise ValueError samplefile, contentencoding = FORMAT[coding] diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 1aca0fe54..af3b7543d 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -78,7 +78,7 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): class RaiseExceptionProcessSpiderOutputMiddleware: def process_spider_output(self, response, result, spider): - raise Exception() + raise Exception self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 670c41f2b..4c19d167f 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -43,7 +43,7 @@ class RecoverySpider(Spider): yield {"test": 1} self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail")) if not response.meta.get("dont_fail"): - raise TabError() + raise TabError class RecoveryAsyncGenSpider(RecoverySpider): @@ -59,7 +59,7 @@ class RecoveryAsyncGenSpider(RecoverySpider): class FailProcessSpiderInputMiddleware: def process_spider_input(self, response, spider): spider.logger.info("Middleware: will raise IndexError") - raise IndexError() + raise IndexError class ProcessSpiderInputSpiderWithoutErrback(Spider): @@ -109,14 +109,14 @@ class GeneratorCallbackSpider(Spider): def parse(self, response): yield {"test": 1} yield {"test": 2} - raise ImportError() + raise ImportError class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider): async def parse(self, response): yield {"test": 1} yield {"test": 2} - raise ImportError() + raise ImportError # ================================================================================ @@ -176,7 +176,7 @@ class GeneratorFailMiddleware: for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r - raise LookupError() + raise LookupError def process_spider_exception(self, response, exception, spider): method = f"{self.__class__.__name__}.process_spider_exception" @@ -246,7 +246,7 @@ class NotGeneratorFailMiddleware: for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") out.append(r) - raise ReferenceError() + raise ReferenceError return out def process_spider_exception(self, response, exception, spider): diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 499ca46b8..02ea8027f 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -41,7 +41,7 @@ class BaseQueueTestCase(unittest.TestCase): class RequestQueueTestMixin: def queue(self): - raise NotImplementedError() + raise NotImplementedError def test_one_element_with_peek(self): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 484757035..c7774751e 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -10,7 +10,7 @@ from scrapy.utils.misc import ( def _indentation_error(*args, **kwargs): - raise IndentationError() + raise IndentationError def top_level_return_something(): From 93644f2c30ee74a68584c9e4cdfd0827c2187d34 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 12 Dec 2024 20:27:04 +0500 Subject: [PATCH 1600/2083] Add flake8-pie rules to ruff. --- pyproject.toml | 2 ++ scrapy/commands/shell.py | 1 - scrapy/core/scheduler.py | 2 -- scrapy/dupefilters.py | 1 - scrapy/exceptions.py | 14 -------------- scrapy/robotstxt.py | 2 -- tests/spiders.py | 4 ++-- tests/test_contracts.py | 6 ------ tests/test_downloadermiddleware_retry.py | 2 +- tests/test_extension_telnet.py | 2 +- tests/test_pipeline_files.py | 8 ++++---- tests/test_pipeline_images.py | 8 ++++---- 12 files changed, 14 insertions(+), 38 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 131684724..b3dd9f057 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -242,6 +242,8 @@ extend-select = [ "LOG", # pygrep-hooks "PGH", + # flake8-pie + "PIE", # flake8-quotes "Q", # flake8-return diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 4ca015f5e..3047ae396 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -61,7 +61,6 @@ class Command(ScrapyCommand): """You can use this function to update the Scrapy objects that will be available in the shell """ - pass def run(self, args: list[str], opts: Namespace) -> None: url = args[0] if args else None diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 823677175..fcc94879a 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -81,7 +81,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): :param spider: the spider object for the current crawl :type spider: :class:`~scrapy.spiders.Spider` """ - pass def close(self, reason: str) -> Deferred[None] | None: """ @@ -91,7 +90,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): :param reason: a string which describes the reason why the spider was closed :type reason: :class:`str` """ - pass @abstractmethod def has_pending_requests(self) -> bool: diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 7b8eea135..caf69daf4 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -50,7 +50,6 @@ class BaseDupeFilter: def log(self, request: Request, spider: Spider) -> None: """Log that a request has been filtered""" - pass class RFPDupeFilter(BaseDupeFilter): diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index e7ecdbe0c..96566ba86 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -13,8 +13,6 @@ from typing import Any class NotConfigured(Exception): """Indicates a missing configuration situation""" - pass - class _InvalidOutput(TypeError): """ @@ -22,8 +20,6 @@ class _InvalidOutput(TypeError): Internal and undocumented, it should not be raised or caught by user code. """ - pass - # HTTP and crawling @@ -35,8 +31,6 @@ class IgnoreRequest(Exception): class DontCloseSpider(Exception): """Request the spider not to be closed yet""" - pass - class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" @@ -64,14 +58,10 @@ class StopDownload(Exception): class DropItem(Exception): """Drop item from the item pipeline""" - pass - class NotSupported(Exception): """Indicates a feature or method is not supported""" - pass - # Commands @@ -89,10 +79,6 @@ class ScrapyDeprecationWarning(Warning): DeprecationWarning is silenced on Python 2.7+ """ - pass - class ContractFail(AssertionError): """Error raised in case of a failing contract""" - - pass diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 844969c6d..417c9c142 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -52,7 +52,6 @@ class RobotParser(metaclass=ABCMeta): :param robotstxt_body: content of a robots.txt_ file. :type robotstxt_body: bytes """ - pass @abstractmethod def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: @@ -64,7 +63,6 @@ class RobotParser(metaclass=ABCMeta): :param user_agent: User agent :type user_agent: str or bytes """ - pass class PythonRobotParser(RobotParser): diff --git a/tests/spiders.py b/tests/spiders.py index 0180cf757..3c44d7da5 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -393,8 +393,8 @@ class DuplicateStartRequestsSpider(MockServerSpider): dupe_factor = 3 def start_requests(self): - for i in range(0, self.distinct_urls): - for j in range(0, self.dupe_factor): + for i in range(self.distinct_urls): + for j in range(self.dupe_factor): url = self.mockserver.url(f"/echo?headers=1&body=test{i}") yield Request(url, dont_filter=self.dont_filter) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b0cb92d12..743889234 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -178,27 +178,23 @@ class TestSpider(Spider): """method with no url @returns items 1 1 """ - pass def custom_form(self, response): """ @url http://scrapy.org @custom_form """ - pass def invalid_regex(self, response): """method with invalid regex @ Scrapy is awsome """ - pass def invalid_regex_with_valid_contract(self, response): """method with invalid regex @ scrapy is awsome @url http://scrapy.org """ - pass def returns_request_meta(self, response): """method which returns request @@ -235,7 +231,6 @@ class CustomContractSuccessSpider(Spider): """ @custom_success_contract """ - pass class CustomContractFailSpider(Spider): @@ -245,7 +240,6 @@ class CustomContractFailSpider(Spider): """ @custom_fail_contract """ - pass class InheritsTestSpider(TestSpider): diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index a010865ef..c99f19b03 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -265,7 +265,7 @@ class MaxRetryTimesTest(unittest.TestCase): spider = spider or self.spider middleware = middleware or self.mw - for i in range(0, max_retry_times): + for i in range(max_retry_times): req = middleware.process_exception(req, exception, spider) assert isinstance(req, Request) diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 9fd680e9f..8c897c223 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -13,7 +13,7 @@ class TelnetExtensionTest(unittest.TestCase): console = TelnetConsole(crawler) # This function has some side effects we don't need for this test - console._get_telnet_vars = lambda: {} + console._get_telnet_vars = dict console.start_listening() protocol = console.protocol() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 2be5e09bc..a6c5f0a94 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -311,11 +311,11 @@ class FilesPipelineTestCaseFieldsDataClass( class FilesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - file_urls: list[str] = attr.ib(default=lambda: []) - files: list[dict[str, str]] = attr.ib(default=lambda: []) + file_urls: list[str] = attr.ib(default=list) + files: list[dict[str, str]] = attr.ib(default=list) # overridden fields - custom_file_urls: list[str] = attr.ib(default=lambda: []) - custom_files: list[dict[str, str]] = attr.ib(default=lambda: []) + custom_file_urls: list[str] = attr.ib(default=list) + custom_files: list[dict[str, str]] = attr.ib(default=list) class FilesPipelineTestCaseFieldsAttrsItem( diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 3ffef4102..3d049843a 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -295,11 +295,11 @@ class ImagesPipelineTestCaseFieldsDataClass( class ImagesPipelineTestAttrsItem: name = attr.ib(default="") # default fields - image_urls: list[str] = attr.ib(default=lambda: []) - images: list[dict[str, str]] = attr.ib(default=lambda: []) + image_urls: list[str] = attr.ib(default=list) + images: list[dict[str, str]] = attr.ib(default=list) # overridden fields - custom_image_urls: list[str] = attr.ib(default=lambda: []) - custom_images: list[dict[str, str]] = attr.ib(default=lambda: []) + custom_image_urls: list[str] = attr.ib(default=list) + custom_images: list[dict[str, str]] = attr.ib(default=list) class ImagesPipelineTestCaseFieldsAttrsItem( From c2832ed1316b25813870a3ef8ebc15473a35d82f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 12 Dec 2024 20:44:28 +0500 Subject: [PATCH 1601/2083] Add flake8-pyi rules to ruff. --- pyproject.toml | 2 ++ scrapy/downloadermiddlewares/robotstxt.py | 4 +--- scrapy/link.py | 4 +--- scrapy/pipelines/media.py | 13 +------------ scrapy/utils/response.py | 2 +- tests/test_scheduler.py | 11 ++++++++--- 6 files changed, 14 insertions(+), 22 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b3dd9f057..973d43162 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -244,6 +244,8 @@ extend-select = [ "PGH", # flake8-pie "PIE", + # flake8-pyi + "PYI", # flake8-quotes "Q", # flake8-return diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index ea9f47d69..9411cff14 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING from twisted.internet.defer import Deferred, maybeDeferred @@ -31,8 +31,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -_T = TypeVar("_T") - class RobotsTxtMiddleware: DOWNLOAD_PRIORITY: int = 1000 diff --git a/scrapy/link.py b/scrapy/link.py index 4bdbc1823..1a569f892 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -5,8 +5,6 @@ For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ -from typing import Any - class Link: """Link objects represent an extracted link by the LinkExtractor. @@ -39,7 +37,7 @@ class Link: self.fragment: str = fragment self.nofollow: bool = nofollow - def __eq__(self, other: Any) -> bool: + def __eq__(self, other: object) -> bool: if not isinstance(other, Link): raise NotImplementedError return ( diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index b16f1cb84..5438b8522 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -5,16 +5,7 @@ import logging import warnings from abc import ABC, abstractmethod from collections import defaultdict -from typing import ( - TYPE_CHECKING, - Any, - Literal, - NoReturn, - TypedDict, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Literal, NoReturn, TypedDict, Union, cast from twisted import version as twisted_version from twisted.internet.defer import Deferred, DeferredList @@ -41,8 +32,6 @@ if TYPE_CHECKING: from scrapy.http import Response from scrapy.utils.request import RequestFingerprinter -_T = TypeVar("_T") - class FileInfo(TypedDict): url: str diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index a7ad4544d..76a6b7de6 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -53,7 +53,7 @@ def get_meta_refresh( return _metaref_cache[response] -def response_status_message(status: bytes | float | int | str) -> str: +def response_status_message(status: bytes | float | str) -> str: """Return status code plus status text descriptive message""" status_int = int(status) message = http.RESPONSES.get(status_int, "Unknown Status") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 387bc7c20..8bd1480ad 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,9 +1,9 @@ from __future__ import annotations -import collections import shutil import tempfile import unittest +from typing import Any, NamedTuple from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -18,8 +18,13 @@ from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -MockEngine = collections.namedtuple("MockEngine", ["downloader"]) -MockSlot = collections.namedtuple("MockSlot", ["active"]) + +class MockEngine(NamedTuple): + downloader: MockDownloader + + +class MockSlot(NamedTuple): + active: list[Any] class MockDownloader: From 1e4c81e9dce7dfc06c8ab1b89d85da32f0cb54de Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 12 Dec 2024 21:03:34 +0500 Subject: [PATCH 1602/2083] Add Perflint rules to ruff. --- pyproject.toml | 4 ++++ scrapy/utils/asyncgen.py | 5 +---- scrapy/utils/python.py | 3 +-- tests/test_feedexport.py | 4 ++-- tests/test_spidermiddleware.py | 5 +---- tests/test_utils_asyncgen.py | 4 +--- tests/test_utils_iterators.py | 30 ++++++++++++++---------------- tests/test_utils_python.py | 7 ------- 8 files changed, 24 insertions(+), 38 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 973d43162..b63a3631b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -240,6 +240,8 @@ extend-select = [ "ISC", # flake8-logging "LOG", + # Perflint + "PERF", # pygrep-hooks "PGH", # flake8-pie @@ -314,6 +316,8 @@ ignore = [ "D402", # First word of the first line should be properly capitalized "D403", + # `try`-`except` within a loop incurs performance overhead + "PERF203", # Use of `assert` detected; needed for mypy "S101", # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180 diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 905959c25..237bd8331 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -7,10 +7,7 @@ _T = TypeVar("_T") async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]: - results = [] - async for x in result: - results.append(x) - return results + return [x async for x in result] async def as_async_generator( diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 511511301..e954b625c 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -235,8 +235,7 @@ def get_func_args(func: Callable[..., Any], stripself: bool = False) -> list[str continue args.append(name) else: - for name in sig.parameters.keys(): - args.append(name) + args = list(sig.parameters) if stripself and args and args[0] == "self": args = args[1:] diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 031d6180d..2debbe0d7 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1757,13 +1757,13 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): crawler = get_crawler(spider_cls, settings) yield crawler.crawl() - for file_path, feed_options in FEEDS.items(): + for file_path in FEEDS: content[str(file_path)] = ( Path(file_path).read_bytes() if Path(file_path).exists() else None ) finally: - for file_path in FEEDS.keys(): + for file_path in FEEDS: if not Path(file_path).exists(): continue diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index af3b7543d..f2a57bd88 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -289,10 +289,7 @@ class ProcessSpiderOutputNonIterableMiddleware: class ProcessSpiderOutputCoroutineMiddleware: async def process_spider_output(self, response, result, spider): - results = [] - for r in result: - results.append(r) - return results + return result class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 9ae66c57c..8adeea5c0 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -8,9 +8,7 @@ class AsyncgenUtilsTest(unittest.TestCase): @deferred_f_from_coro_f async def test_as_async_generator(self): ag = as_async_generator(range(42)) - results = [] - async for i in ag: - results.append(i) + results = [i async for i in ag] self.assertEqual(results, list(range(42))) @deferred_f_from_coro_f diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ec377bb19..4c81e3a2f 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -26,15 +26,14 @@ class XmliterBaseTestCase: """ response = XmlResponse(url="http://example.com", body=body) - attrs = [] - for x in self.xmliter(response, "product"): - attrs.append( - ( - x.attrib["id"], - x.xpath("name/text()").getall(), - x.xpath("./type/text()").getall(), - ) + attrs = [ + ( + x.attrib["id"], + x.xpath("name/text()").getall(), + x.xpath("./type/text()").getall(), ) + for x in self.xmliter(response, "product") + ] self.assertEqual( attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])] @@ -99,15 +98,14 @@ class XmliterBaseTestCase: # Unicode body needs encoding information XmlResponse(url="http://example.com", body=body, encoding="utf-8"), ): - attrs = [] - for x in self.xmliter(r, "þingflokkur"): - attrs.append( - ( - x.attrib["id"], - x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(), - x.xpath("./tímabil/fyrstaþing/text()").getall(), - ) + attrs = [ + ( + x.attrib["id"], + x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(), + x.xpath("./tímabil/fyrstaþing/text()").getall(), ) + for x in self.xmliter(r, "þingflokkur") + ] self.assertEqual( attrs, diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index f80f2517a..83004cec4 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -58,13 +58,6 @@ class MutableAsyncChainTest(unittest.TestCase): for i in range(5, 7): yield i - @staticmethod - async def collect_asyncgen_exc(asyncgen): - results = [] - async for x in asyncgen: - results.append(x) - return results - @deferred_f_from_coro_f async def test_mutableasyncchain(self): m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7))) From c003fc0841367227aff3c2c1192fe60abbd1a458 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 12 Dec 2024 21:07:01 +0500 Subject: [PATCH 1603/2083] Add flake8 warning rules to ruff. --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b63a3631b..c0297e192 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -264,6 +264,8 @@ extend-select = [ "TC", # pyupgrade "UP", + # pycodestyle warnings + "W", # flake8-2020 "YTT", ] From 7dd92e6e4341dc6d5ba70a20d7e89edb3a87fba9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 16 Dec 2024 12:44:36 +0400 Subject: [PATCH 1604/2083] Add pylint rules to ruff, refresh the ignore list of pylint itself (#6584) --- pyproject.toml | 32 +++++++++++-------- scrapy/__init__.py | 2 +- scrapy/cmdline.py | 2 +- scrapy/exporters.py | 7 ++-- scrapy/utils/defer.py | 2 +- scrapy/utils/deprecate.py | 2 +- scrapy/utils/log.py | 2 +- scrapy/utils/reactor.py | 5 ++- scrapy/utils/spider.py | 4 --- scrapy/utils/url.py | 6 ++-- .../test_spider/pipelines.py | 2 +- tests/test_downloader_handlers.py | 5 --- tests/test_downloadermiddleware.py | 2 +- tests/test_extension_periodic_log.py | 1 - tests/test_feedexport.py | 2 +- tests/test_loader_deprecated.py | 4 +-- tests/test_spidermiddleware.py | 2 +- tests/test_spidermiddleware_output_chain.py | 1 - tests/test_utils_asyncio.py | 2 +- tests/test_utils_defer.py | 2 +- ...t_return_with_argument_inside_generator.py | 7 ---- tests/test_utils_spider.py | 2 +- tests/test_utils_url.py | 2 -- 23 files changed, 41 insertions(+), 57 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c0297e192..7dc1f6ec3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,9 @@ profile = "black" [tool.pylint.MASTER] persistent = "no" jobs = 1 # >1 hides results +extension-pkg-allow-list=[ + "lxml", +] [tool.pylint."MESSAGES CONTROL"] disable = [ @@ -129,11 +132,7 @@ disable = [ "arguments-differ", "arguments-renamed", "attribute-defined-outside-init", - "bad-classmethod-argument", - "bare-except", - "broad-except", - "broad-exception-raised", - "c-extension-no-member", + "broad-exception-caught", "consider-using-with", "cyclic-import", "dangerous-default-value", @@ -141,9 +140,6 @@ disable = [ "duplicate-code", # https://github.com/PyCQA/pylint/issues/214 "eval-used", "fixme", - "function-redefined", - "global-statement", - "implicit-str-concat", "import-error", "import-outside-toplevel", "inherit-non-class", @@ -155,7 +151,6 @@ disable = [ "logging-format-interpolation", "logging-fstring-interpolation", "logging-not-lazy", - "lost-exception", "missing-docstring", "no-member", "no-method-argument", @@ -169,13 +164,11 @@ disable = [ "raise-missing-from", "redefined-builtin", "redefined-outer-name", - "reimported", "signature-differs", "too-few-public-methods", "too-many-ancestors", "too-many-arguments", "too-many-branches", - "too-many-format-args", "too-many-function-args", "too-many-instance-attributes", "too-many-lines", @@ -184,14 +177,11 @@ disable = [ "too-many-return-statements", "unbalanced-tuple-unpacking", "unnecessary-dunder-call", - "unnecessary-pass", - "unreachable", "unused-argument", "unused-import", "unused-variable", "used-before-assignment", "useless-return", - "wildcard-import", "wrong-import-position", ] @@ -246,6 +236,8 @@ extend-select = [ "PGH", # flake8-pie "PIE", + # pylint + "PL", # flake8-pyi "PYI", # flake8-quotes @@ -320,6 +312,18 @@ ignore = [ "D403", # `try`-`except` within a loop incurs performance overhead "PERF203", + # Too many return statements + "PLR0911", + # Too many branches + "PLR0912", + # Too many arguments in function definition + "PLR0913", + # Too many statements + "PLR0915", + # Magic value used in comparison + "PLR2004", + # `for` loop variable overwritten by assignment target + "PLW2901", # Use of `assert` detected; needed for mypy "S101", # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 921296502..c19710a6a 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -31,7 +31,7 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("." def __getattr__(name: str): if name == "twisted_version": - import warnings + import warnings # pylint: disable=reimported from twisted import version as _txv diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index b6f19a37f..9a24871de 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -74,7 +74,7 @@ def _get_commands_from_entry_points( if inspect.isclass(obj): cmds[entry_point.name] = obj() else: - raise Exception(f"Invalid entry point {entry_point.name}") + raise ValueError(f"Invalid entry point {entry_point.name}") return cmds diff --git a/scrapy/exporters.py b/scrapy/exporters.py index b6997ef67..cdb7ac159 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -92,11 +92,10 @@ class BaseItemExporter: field_iter = ( (x, y) for x, y in self.fields_to_export.items() if x in item ) + elif include_empty: + field_iter = self.fields_to_export else: - if include_empty: - field_iter = self.fields_to_export - else: - field_iter = (x for x in self.fields_to_export if x in item) + field_iter = (x for x in self.fields_to_export if x in item) for field_name in field_iter: if isinstance(field_name, str): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 9f1b816c8..000ab5c65 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -398,7 +398,7 @@ def maybeDeferred_coro( """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) - except: # noqa: E722,B001 + except: # noqa: E722 # pylint: disable=bare-except return defer.fail(failure.Failure(captureVars=Deferred.debug)) if isinstance(result, Deferred): diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 32430cd6c..0a0acc742 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -60,7 +60,7 @@ def create_deprecated_class( deprecated_class: type | None = None warned_on_subclass: bool = False - def __new__( + def __new__( # pylint: disable=bad-classmethod-argument metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any] ) -> type: cls = super().__new__(metacls, name, bases, clsdict_) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index a40b835cd..6165d1f72 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -130,7 +130,7 @@ _scrapy_root_handler: logging.Handler | None = None def install_scrapy_root_handler(settings: Settings) -> None: - global _scrapy_root_handler + global _scrapy_root_handler # noqa: PLW0603 # pylint: disable=global-statement if ( _scrapy_root_handler is not None diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 2d781cc27..66a06a9f0 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -149,12 +149,11 @@ def verify_installed_reactor(reactor_path: str) -> None: reactor_class = load_object(reactor_path) if not reactor.__class__ == reactor_class: - msg = ( + raise RuntimeError( "The installed reactor " f"({reactor.__module__}.{reactor.__class__.__name__}) does not " f"match the requested one ({reactor_path})" ) - raise Exception(msg) def verify_installed_asyncio_event_loop(loop_path: str) -> None: @@ -168,7 +167,7 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None: f".{reactor._asyncioEventloop.__class__.__qualname__}" ) specified = f"{loop_class.__module__}.{loop_class.__qualname__}" - raise Exception( + raise RuntimeError( "Scrapy found an asyncio Twisted reactor already " f"installed, and its event loop class ({installed}) does " "not match the one specified in the ASYNCIO_EVENT_LOOP " diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index e58eb8134..5277a292c 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -52,10 +52,6 @@ def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ - # this needs to be imported here until get rid of the spider manager - # singleton in scrapy.spider.spiders - from scrapy.spiders import Spider - for obj in vars(module).values(): if ( inspect.isclass(obj) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 2539f30c7..1cbfbfd99 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -14,7 +14,7 @@ from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code -from w3lib.url import * # pylint: disable=unused-wildcard-import +from w3lib.url import * # pylint: disable=unused-wildcard-import,wildcard-import from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode @@ -50,7 +50,9 @@ def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: return any(lowercase_path.endswith(ext) for ext in extensions) -def parse_url(url: UrlT, encoding: str | None = None) -> ParseResult: +def parse_url( # pylint: disable=function-redefined + url: UrlT, encoding: str | None = None +) -> ParseResult: """Return urlparsed url from the given argument (which could be an already parsed url) """ diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py index af15cac68..3e29c70ed 100644 --- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py @@ -8,7 +8,7 @@ class TestSpiderPipeline: class TestSpiderExceptionPipeline: def open_spider(self, spider): - raise Exception("exception") + raise RuntimeError("exception") def process_item(self, item, spider): return item diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 3fcba4ef2..8ecba41bf 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -349,11 +349,6 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL("host"), headers={"Host": host}) return self.download_request(request, Spider("foo")).addCallback(_test) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"localhost") - return d - def test_content_length_zero_bodyless_post_request_headers(self): """Tests if "Content-Length: 0" is sent for bodyless POST requests. diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index e650b4936..c581e7596 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -178,7 +178,7 @@ class ProcessExceptionInvalidOutput(ManagerTestCase): class InvalidProcessExceptionMiddleware: def process_request(self, request, spider): - raise Exception + raise RuntimeError def process_exception(self, request, exception, spider): return 1 diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index b7312bbcd..15129e31f 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -192,4 +192,3 @@ class TestPeriodicLog(unittest.TestCase): {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: "downloader/" in k and "bytes" not in k, ) - # diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2debbe0d7..b087aaab1 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -727,7 +727,7 @@ class ExceptionJsonItemExporter(JsonItemExporter): """JsonItemExporter that throws an exception every time export_item is called.""" def export_item(self, _): - raise Exception("foo") + raise RuntimeError("foo") class FeedExportTest(FeedExportTestBase): diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index f9b841a61..4bf22f6a0 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -330,10 +330,10 @@ class BasicItemLoaderTest(unittest.TestCase): il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), "Mar Ta") - class TakeFirstItemLoader(TestItemLoader): + class TakeFirstItemLoader2(TestItemLoader): name_out = Join("<br>") - il = TakeFirstItemLoader() + il = TakeFirstItemLoader2() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), "Mar<br>Ta") diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index f2a57bd88..ba64ba721 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -78,7 +78,7 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): class RaiseExceptionProcessSpiderOutputMiddleware: def process_spider_output(self, response, result, spider): - raise Exception + raise RuntimeError self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 4c19d167f..e51957497 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -247,7 +247,6 @@ class NotGeneratorFailMiddleware: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") out.append(r) raise ReferenceError - return out def process_spider_exception(self, response, exception, spider): method = f"{self.__class__.__name__}.process_spider_exception" diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 1c93829e9..e00f69573 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -26,7 +26,7 @@ class AsyncioTest(TestCase): with warnings.catch_warnings(record=True) as w: install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") self.assertEqual(len(w), 0) - from twisted.internet import reactor + from twisted.internet import reactor # pylint: disable=reimported assert original_reactor == reactor diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 3f153bdc0..e4ab97e5d 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -153,7 +153,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase): @mark.xfail(reason="Checks that the test is actually executed", strict=True) @deferred_f_from_coro_f async def test_deferred_f_from_coro_f_xfail(self): - raise Exception("This is expected to be raised") + raise RuntimeError("This is expected to be raised") class AsyncCooperatorTest(unittest.TestCase): diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index c7774751e..480729d11 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -32,7 +32,6 @@ def top_level_return_none(): https://example.org """ yield url - return def generator_that_returns_stuff(): @@ -103,11 +102,9 @@ https://example.org def test_generators_return_none(self): def f2(): yield 1 - return None def g2(): yield 1 - return def h2(): yield 1 @@ -132,7 +129,6 @@ https://example.org https://example.org """ yield url - return def l2(): return @@ -181,12 +177,10 @@ https://example.org @decorator def f3(): yield 1 - return None @decorator def g3(): yield 1 - return @decorator def h3(): @@ -215,7 +209,6 @@ https://example.org https://example.org """ yield url - return @decorator def l3(): diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index dd1d26448..ae59d0137 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -26,7 +26,7 @@ class UtilsSpidersTestCase(unittest.TestCase): self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o]) def test_iter_spider_classes(self): - import tests.test_utils_spider # pylint: disable=import-self + import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self it = iter_spider_classes(tests.test_utils_spider) self.assertEqual(set(it), {MySpider1, MySpider2}) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 59a95b0e2..a15ad749d 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -327,8 +327,6 @@ def create_guess_scheme_t(args): def create_skipped_scheme_t(args): def do_expected(self): raise unittest.SkipTest(args[2]) - url = guess_scheme(args[0]) - assert url.startswith(args[1]) return do_expected From 21b9ba717c1687a889879232f226c75fb4dbe0bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 16 Dec 2024 14:46:23 +0100 Subject: [PATCH 1605/2083] Allow customizing logged software versions (#6582) Co-authored-by: Grammy Jiang <grammy.jiang@gmail.com> Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name> --- docs/topics/settings.rst | 19 ++++++++++ scrapy/commands/version.py | 4 +- scrapy/settings/default_settings.py | 12 ++++++ scrapy/utils/log.py | 14 +++---- scrapy/utils/versions.py | 59 ++++++++++++++++++----------- tests/test_crawler.py | 26 +++++++++++++ 6 files changed, 103 insertions(+), 31 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index cce4a7b3e..76904a26e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1228,6 +1228,25 @@ Default: ``False`` If ``True``, the logs will just contain the root path. If it is set to ``False`` then it displays the component responsible for the log output +.. setting:: LOG_VERSIONS + +LOG_VERSIONS +------------ + +Default: ``["lxml", "libxml2", "cssselect", "parsel", "w3lib", "Twisted", "Python", "pyOpenSSL", "cryptography", "Platform"]`` + +Logs the installed versions of the specified items. + +An item can be any installed Python package. + +The following special items are also supported: + +- ``libxml2`` + +- ``Platform`` (:func:`platform.platform`) + +- ``Python`` + .. setting:: LOGSTATS_INTERVAL LOGSTATS_INTERVAL diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 571f4fda8..713a78ad9 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -2,7 +2,7 @@ import argparse import scrapy from scrapy.commands import ScrapyCommand -from scrapy.utils.versions import scrapy_components_versions +from scrapy.utils.versions import get_versions class Command(ScrapyCommand): @@ -26,7 +26,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: if opts.verbose: - versions = scrapy_components_versions() + versions = get_versions() width = max(len(n) for (n, _) in versions) for name, version in versions: print(f"{name:<{width}} : {version}") diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 89ab21fbe..0bbde118e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -219,6 +219,18 @@ LOG_LEVEL = "DEBUG" LOG_FILE = None LOG_FILE_APPEND = True LOG_SHORT_NAMES = False +LOG_VERSIONS = [ + "lxml", + "libxml2", + "cssselect", + "parsel", + "w3lib", + "Twisted", + "Python", + "pyOpenSSL", + "cryptography", + "Platform", +] SCHEDULER_DEBUG = False diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 6165d1f72..d51231b82 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import pprint import sys from collections.abc import MutableMapping from logging.config import dictConfig @@ -12,7 +13,7 @@ from twisted.python.failure import Failure import scrapy from scrapy.settings import Settings, _SettingsKeyT -from scrapy.utils.versions import scrapy_components_versions +from scrapy.utils.versions import get_versions if TYPE_CHECKING: @@ -174,12 +175,11 @@ def log_scrapy_info(settings: Settings) -> None: "Scrapy %(version)s started (bot: %(bot)s)", {"version": scrapy.__version__, "bot": settings["BOT_NAME"]}, ) - versions = [ - f"{name} {version}" - for name, version in scrapy_components_versions() - if name != "Scrapy" - ] - logger.info("Versions: %(versions)s", {"versions": ", ".join(versions)}) + software = settings.getlist("LOG_VERSIONS") + if not software: + return + versions = pprint.pformat(dict(get_versions(software)), sort_dicts=False) + logger.info(f"Versions:\n{versions}") def log_reactor_info() -> None: diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 996a5cdb3..ff1f9b346 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -1,31 +1,46 @@ +from __future__ import annotations + import platform import sys +from importlib.metadata import version +from warnings import warn -import cryptography -import cssselect import lxml.etree -import parsel -import twisted -import w3lib -import scrapy +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.settings.default_settings import LOG_VERSIONS from scrapy.utils.ssl import get_openssl_version +_DEFAULT_SOFTWARE = ["Scrapy"] + LOG_VERSIONS + + +def _version(item): + lowercase_item = item.lower() + if lowercase_item == "libxml2": + return ".".join(map(str, lxml.etree.LIBXML_VERSION)) + if lowercase_item == "platform": + return platform.platform() + if lowercase_item == "pyopenssl": + return get_openssl_version() + if lowercase_item == "python": + return sys.version.replace("\n", "- ") + return version(item) + + +def get_versions( + software: list | None = None, +) -> list[tuple[str, str]]: + software = software or _DEFAULT_SOFTWARE + return [(item, _version(item)) for item in software] + def scrapy_components_versions() -> list[tuple[str, str]]: - lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) - libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) - - return [ - ("Scrapy", scrapy.__version__), - ("lxml", lxml_version), - ("libxml2", libxml2_version), - ("cssselect", cssselect.__version__), - ("parsel", parsel.__version__), - ("w3lib", w3lib.__version__), - ("Twisted", twisted.version.short()), - ("Python", sys.version.replace("\n", "- ")), - ("pyOpenSSL", get_openssl_version()), - ("cryptography", cryptography.__version__), - ("Platform", platform.platform()), - ] + warn( + ( + "scrapy.utils.versions.scrapy_components_versions() is deprecated, " + "use scrapy.utils.versions.get_versions() instead." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + return get_versions() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index a77531f62..f3e5ebf5d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,6 +1,7 @@ import logging import os import platform +import re import signal import subprocess import sys @@ -923,3 +924,28 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): log, ) self.assertIn("DEBUG: Using asyncio event loop", log) + + +@mark.parametrize( + ["settings", "items"], + ( + ({}, default_settings.LOG_VERSIONS), + ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]), + ({"LOG_VERSIONS": []}, None), + ), +) +def test_log_scrapy_info(settings, items, caplog): + with caplog.at_level("INFO"): + CrawlerProcess(settings) + assert ( + caplog.records[0].getMessage() + == f"Scrapy {scrapy.__version__} started (bot: scrapybot)" + ), repr(caplog.records[0].msg) + if not items: + assert len(caplog.records) == 1 + return + version_string = caplog.records[1].getMessage() + expected_items_pattern = "',\n '".join( + f"{item}': '[^']+('\n +'[^']+)*" for item in items + ) + assert re.search(r"^Versions:\n{'" + expected_items_pattern + "'}$", version_string) From a195af304d2823cc686fd7354790e52239208d82 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 18 Dec 2024 03:50:44 -0300 Subject: [PATCH 1606/2083] Deprecate w3lib objects importable from scrapy.utils.url (#6586) --- scrapy/utils/url.py | 50 ++++++++++++++++++++--------------------- tests/test_utils_url.py | 25 +++++++++++++++++++++ 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 1cbfbfd99..3bf831c26 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -1,36 +1,47 @@ """ This module contains general purpose URL functions not found in the standard library. - -Some of the functions that used to be imported from this module have been moved -to the w3lib.url module. Always import those from there instead. """ from __future__ import annotations import re -from typing import TYPE_CHECKING, Union, cast +import warnings +from importlib import import_module +from typing import TYPE_CHECKING, Union from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse -# scrapy.utils.url was moved to w3lib.url and import * ensures this -# move doesn't break old code -from w3lib.url import * # pylint: disable=unused-wildcard-import,wildcard-import -from w3lib.url import _safe_chars, _unquotepath # noqa: F401 +from w3lib.url import __all__ as _public_w3lib_objects +from w3lib.url import add_or_replace_parameter as _add_or_replace_parameter +from w3lib.url import any_to_uri as _any_to_uri +from w3lib.url import parse_url as _parse_url + +from scrapy.exceptions import ScrapyDeprecationWarning + + +def __getattr__(name: str): + if name in ("_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects): + obj_type = "attribute" if name == "_safe_chars" else "function" + warnings.warn( + f"The scrapy.utils.url.{name} {obj_type} is deprecated, use w3lib.url.{name} instead.", + ScrapyDeprecationWarning, + ) + return getattr(import_module("w3lib.url"), name) + + raise AttributeError -from scrapy.utils.python import to_unicode if TYPE_CHECKING: from collections.abc import Iterable from scrapy import Spider - UrlT = Union[str, bytes, ParseResult] def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: """Return True if the url belongs to any of the given domains""" - host = parse_url(url).netloc.lower() + host = _parse_url(url).netloc.lower() if not host: return False domains = [d.lower() for d in domains] @@ -46,21 +57,10 @@ def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool: def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: """Return True if the url ends with one of the extensions provided""" - lowercase_path = parse_url(url).path.lower() + lowercase_path = _parse_url(url).path.lower() return any(lowercase_path.endswith(ext) for ext in extensions) -def parse_url( # pylint: disable=function-redefined - url: UrlT, encoding: str | None = None -) -> ParseResult: - """Return urlparsed url from the given argument (which could be an already - parsed url) - """ - if isinstance(url, ParseResult): - return url - return cast(ParseResult, urlparse(to_unicode(url, encoding))) - - def escape_ajax(url: str) -> str: """ Return the crawlable url @@ -86,7 +86,7 @@ def escape_ajax(url: str) -> str: defrag, frag = urldefrag(url) if not frag.startswith("!"): return url - return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:]) + return _add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:]) def add_http_if_no_scheme(url: str) -> str: @@ -146,7 +146,7 @@ def guess_scheme(url: str) -> str: """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" if _is_filesystem_path(url): - return any_to_uri(url) + return _any_to_uri(url) return add_http_if_no_scheme(url) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index a15ad749d..62e2b5c1e 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,10 +1,14 @@ import unittest +import warnings + +import pytest from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.spiders import Spider from scrapy.utils.misc import arg_to_iter from scrapy.utils.url import ( _is_filesystem_path, + _public_w3lib_objects, add_http_if_no_scheme, guess_scheme, strip_url, @@ -607,5 +611,26 @@ class IsPathTestCase(unittest.TestCase): ) +@pytest.mark.parametrize( + "obj_name", + [ + "_unquotepath", + "_safe_chars", + "parse_url", + *_public_w3lib_objects, + ], +) +def test_deprecated_imports_from_w3lib(obj_name): + with warnings.catch_warnings(record=True) as warns: + obj_type = "attribute" if obj_name == "_safe_chars" else "function" + message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead." + + from importlib import import_module + + getattr(import_module("scrapy.utils.url"), obj_name) + + assert message in warns[0].message.args + + if __name__ == "__main__": unittest.main() From c5ed0fd45cfcee15cf59eb92f40c12ca29ecc890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 18 Dec 2024 17:05:51 +0100 Subject: [PATCH 1607/2083] Add ADDONS to the settings template for new projects --- scrapy/templates/project/module/settings.py.tmpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index b4779e555..0bb31ffaa 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -12,6 +12,8 @@ BOT_NAME = "$project_name" SPIDER_MODULES = ["$project_name.spiders"] NEWSPIDER_MODULE = "$project_name.spiders" +ADDONS = {} + # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = "$project_name (+http://www.yourdomain.com)" From cc484efd43b0f8ba0dc89904a7d38086775c44ae Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 29 Dec 2024 14:15:16 +0500 Subject: [PATCH 1608/2083] Replace isort with the ruff isort rules. --- .pre-commit-config.yaml | 4 ---- pyproject.toml | 5 ++--- scrapy/core/downloader/handlers/http11.py | 9 +++++++-- scrapy/http/cookies.py | 3 +-- tests/CrawlerRunner/ip_address.py | 3 +-- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 39b9a33aa..d253f61c6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,10 +8,6 @@ repos: rev: 24.4.2 hooks: - id: black -- repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort - repo: https://github.com/adamchainz/blacken-docs rev: 1.18.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 7dc1f6ec3..a2dabcf4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,9 +116,6 @@ disable_warnings = ["include-ignored"] # https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"] -[tool.isort] -profile = "black" - [tool.pylint.MASTER] persistent = "no" jobs = 1 # >1 hides results @@ -226,6 +223,8 @@ extend-select = [ "FA", # refurb "FURB", + # isort + "I", # flake8-implicit-str-concat "ISC", # flake8-logging diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index bd3200e9f..9f65794fe 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -17,9 +17,14 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure -from twisted.web.client import URI, Agent, HTTPConnectionPool +from twisted.web.client import ( + URI, + Agent, + HTTPConnectionPool, + ResponseDone, + ResponseFailed, +) from twisted.web.client import Response as TxResponse -from twisted.web.client import ResponseDone, ResponseFailed from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 56941ad51..60322fe6e 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -2,9 +2,8 @@ from __future__ import annotations import re import time -from http.cookiejar import Cookie +from http.cookiejar import Cookie, CookiePolicy, DefaultCookiePolicy from http.cookiejar import CookieJar as _CookieJar -from http.cookiejar import CookiePolicy, DefaultCookiePolicy from typing import TYPE_CHECKING, Any, cast from scrapy.utils.httpobj import urlparse_cached diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 5bf7512bc..2f1bb7713 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,9 +1,8 @@ from urllib.parse import urlparse from twisted.internet import reactor -from twisted.names import cache +from twisted.names import cache, resolve from twisted.names import hosts as hostsModule -from twisted.names import resolve from twisted.names.client import Resolver from twisted.python.runtime import platform From 4a0c05749c72662bb82cdb01d9a0ef1ff6416b6a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 29 Dec 2024 14:29:27 +0500 Subject: [PATCH 1609/2083] Bump tool versions. --- .github/workflows/checks.yml | 2 +- .pre-commit-config.yaml | 8 ++++---- .readthedocs.yml | 2 +- pyproject.toml | 1 + tox.ini | 6 +++--- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ff279e9fd..a064bf5b2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -21,7 +21,7 @@ jobs: - python-version: "3.9" env: TOXENV: typing-tests - - python-version: "3.12" # Keep in sync with .readthedocs.yml + - python-version: "3.13" # Keep in sync with .readthedocs.yml env: TOXENV: docs - python-version: "3.13" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d253f61c6..c76c613d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,16 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.1 + rev: v0.8.4 hooks: - id: ruff args: [ --fix ] - repo: https://github.com/psf/black.git - rev: 24.4.2 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/adamchainz/blacken-docs - rev: 1.18.0 + rev: 1.19.1 hooks: - id: blacken-docs additional_dependencies: - - black==24.4.2 + - black==24.10.0 diff --git a/.readthedocs.yml b/.readthedocs.yml index 0c544df7e..5ec6eafbb 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.12" # Keep in sync with .github/workflows/checks.yml + python: "3.13" # Keep in sync with .github/workflows/checks.yml python: install: diff --git a/pyproject.toml b/pyproject.toml index a2dabcf4b..ad85e5c75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,6 +170,7 @@ disable = [ "too-many-instance-attributes", "too-many-lines", "too-many-locals", + "too-many-positional-arguments", "too-many-public-methods", "too-many-return-statements", "unbalanced-tuple-unpacking", diff --git a/tox.ini b/tox.ini index 4e1a99473..24b674085 100644 --- a/tox.ini +++ b/tox.ini @@ -77,15 +77,15 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==3.2.5 + pylint==3.3.3 commands = pylint conftest.py docs extras scrapy tests [testenv:twinecheck] basepython = python3 deps = - twine==5.1.1 - build==1.2.1 + twine==6.0.1 + build==1.2.2.post1 commands = python -m build --sdist twine check dist/* From f7af7b282d6d3b36689cc192e6f78c065e32fb89 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 29 Dec 2024 16:45:26 +0500 Subject: [PATCH 1610/2083] Bump mypy and stubs. --- scrapy/downloadermiddlewares/cookies.py | 13 +++++++------ scrapy/extensions/telnet.py | 2 +- scrapy/http/request/__init__.py | 8 ++++---- scrapy/utils/sitemap.py | 4 +++- tox.ini | 8 ++++---- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 545dcaac9..43348f632 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -131,25 +131,26 @@ class CookiesMiddleware: decoded = {} flags = set() for key in ("name", "value", "path", "domain"): - if cookie.get(key) is None: + value = cookie.get(key) + if value is None: if key in ("name", "value"): msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" logger.warning(msg) return None continue - # https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168 - if isinstance(cookie[key], (bool, float, int, str)): # type: ignore[literal-required] - decoded[key] = str(cookie[key]) # type: ignore[literal-required] + if isinstance(value, (bool, float, int, str)): + decoded[key] = str(value) else: + assert isinstance(value, bytes) try: - decoded[key] = cookie[key].decode("utf8") # type: ignore[literal-required] + decoded[key] = value.decode("utf8") except UnicodeDecodeError: logger.warning( "Non UTF-8 encoded cookie found in request %s: %s", request, cookie, ) - decoded[key] = cookie[key].decode("latin1", errors="replace") # type: ignore[literal-required] + decoded[key] = value.decode("latin1", errors="replace") for flag in ("secure",): value = cookie.get(flag, _UNSET) if value is _UNSET or not value: diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 89c83d20d..ee28d86ba 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -75,7 +75,7 @@ class TelnetConsole(protocol.ServerFactory): def stop_listening(self) -> None: self.port.stopListening() - def protocol(self) -> telnet.TelnetTransport: # type: ignore[override] + def protocol(self) -> telnet.TelnetTransport: # these import twisted.internet.reactor from twisted.conch import manhole, telnet from twisted.conch.insults import insults diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a96a215f4..3d6cf4816 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -44,10 +44,10 @@ if TYPE_CHECKING: class VerboseCookie(TypedDict): - name: str - value: str - domain: NotRequired[str] - path: NotRequired[str] + name: str | bytes + value: str | bytes | bool | float | int + domain: NotRequired[str | bytes] + path: NotRequired[str | bytes] secure: NotRequired[bool] diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index b60fe929e..e0d9f4595 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -26,13 +26,15 @@ class Sitemap: ) self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320 rt = self._root.tag - self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt + assert isinstance(rt, str) + self.type = rt.split("}", 1)[1] if "}" in rt else rt def __iter__(self) -> Iterator[dict[str, Any]]: for elem in self._root.getchildren(): d: dict[str, Any] = {} for el in elem.getchildren(): tag = el.tag + assert isinstance(tag, str) name = tag.split("}", 1)[1] if "}" in tag else tag if name == "link": diff --git a/tox.ini b/tox.ini index 24b674085..39ab1ccd4 100644 --- a/tox.ini +++ b/tox.ini @@ -43,12 +43,12 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.12.0 + mypy==1.14.0 typing-extensions==4.12.2 - types-lxml==2024.9.16 + types-lxml==2024.12.13 types-Pygments==2.18.0.20240506 - botocore-stubs==1.35.39 - boto3-stubs[s3]==1.35.39 + botocore-stubs==1.35.90 + boto3-stubs[s3]==1.35.90 attrs >= 18.2.0 Pillow >= 10.3.0 pyOpenSSL >= 24.2.1 From 838ff99f37d88214829018c6a7dd2a84fdb418b4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 1 Jan 2025 21:31:04 +0500 Subject: [PATCH 1611/2083] Enable RUF Ruff rules. --- pyproject.toml | 10 ++++++++++ scrapy/__init__.py | 12 ++++++------ scrapy/commands/crawl.py | 5 ++--- scrapy/commands/parse.py | 2 +- scrapy/commands/runspider.py | 2 +- scrapy/downloadermiddlewares/redirect.py | 12 +++++++----- scrapy/exporters.py | 8 ++++---- scrapy/extensions/debug.py | 2 +- scrapy/http/request/json_request.py | 2 +- scrapy/http/response/text.py | 2 +- scrapy/link.py | 2 +- scrapy/logformatter.py | 4 ++-- scrapy/pipelines/media.py | 3 +-- scrapy/spidermiddlewares/referer.py | 8 +++----- scrapy/utils/misc.py | 4 +++- scrapy/utils/reactor.py | 2 +- scrapy/utils/request.py | 3 ++- scrapy/utils/testproc.py | 2 +- scrapy/utils/url.py | 2 +- scrapy/utils/versions.py | 2 +- tests/test_cmdline/__init__.py | 2 +- tests/test_commands.py | 4 ++-- tests/test_crawler.py | 2 +- tests/test_downloadermiddleware_retry.py | 2 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_exporters.py | 2 +- tests/test_http_response.py | 2 +- tests/test_utils_trackref.py | 8 ++++---- 28 files changed, 62 insertions(+), 51 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ad85e5c75..065382205 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -246,6 +246,8 @@ extend-select = [ "RET", # flake8-raise "RSE", + # Ruff-specific rules + "RUF", # flake8-bandit "S", # flake8-slots @@ -324,6 +326,14 @@ ignore = [ "PLR2004", # `for` loop variable overwritten by assignment target "PLW2901", + # String contains ambiguous {}. + "RUF001", + # Docstring contains ambiguous {}. + "RUF002", + # Comment contains ambiguous {}. + "RUF003", + # Mutable class attributes should be annotated with `typing.ClassVar` + "RUF012", # Use of `assert` detected; needed for mypy "S101", # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index c19710a6a..256504c9c 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -13,14 +13,14 @@ from scrapy.selector import Selector from scrapy.spiders import Spider __all__ = [ + "Field", + "FormRequest", + "Item", + "Request", + "Selector", + "Spider", "__version__", "version_info", - "Spider", - "Request", - "FormRequest", - "Selector", - "Item", - "Field", ] diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 86d4cc41c..184bd5ca4 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -39,9 +39,8 @@ class Command(BaseRunSpiderCommand): else: self.crawler_process.start() - if ( - self.crawler_process.bootstrap_failed - or hasattr(self.crawler_process, "has_exception") + if self.crawler_process.bootstrap_failed or ( + hasattr(self.crawler_process, "has_exception") and self.crawler_process.has_exception ): self.exitcode = 1 diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index cc5c1350b..f996d1806 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -269,7 +269,7 @@ class Command(BaseRunSpiderCommand): assert self.crawler_process assert self.spidercls self.crawler_process.crawl(self.spidercls, **opts.spargs) - self.pcrawler = list(self.crawler_process.crawlers)[0] + self.pcrawler = next(iter(self.crawler_process.crawlers)) self.crawler_process.start() if not self.first_response: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index bf8e41020..357ca8b37 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -20,7 +20,7 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType: if abspath.suffix not in (".py", ".pyw"): raise ValueError(f"Not a Python source file: {abspath}") dirname = str(abspath.parent) - sys.path = [dirname] + sys.path + sys.path = [dirname, *sys.path] try: module = import_module(abspath.stem) finally: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 0b883b43a..612426371 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -101,12 +101,14 @@ class BaseRedirectMiddleware: if ttl and redirects <= self.max_redirect_times: redirected.meta["redirect_times"] = redirects redirected.meta["redirect_ttl"] = ttl - 1 - redirected.meta["redirect_urls"] = request.meta.get("redirect_urls", []) + [ - request.url + redirected.meta["redirect_urls"] = [ + *request.meta.get("redirect_urls", []), + request.url, + ] + redirected.meta["redirect_reasons"] = [ + *request.meta.get("redirect_reasons", []), + reason, ] - redirected.meta["redirect_reasons"] = request.meta.get( - "redirect_reasons", [] - ) + [reason] redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust logger.debug( diff --git a/scrapy/exporters.py b/scrapy/exporters.py index cdb7ac159..834a05ae9 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -25,13 +25,13 @@ if TYPE_CHECKING: __all__ = [ "BaseItemExporter", - "PprintItemExporter", - "PickleItemExporter", "CsvItemExporter", - "XmlItemExporter", - "JsonLinesItemExporter", "JsonItemExporter", + "JsonLinesItemExporter", "MarshalItemExporter", + "PickleItemExporter", + "PprintItemExporter", + "XmlItemExporter", ] diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 6948c394c..5ca07394f 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -77,4 +77,4 @@ class Debugger: def _enter_debugger(self, signum: int, frame: FrameType | None) -> None: assert frame - Pdb().set_trace(frame.f_back) # noqa: T100 + Pdb().set_trace(frame.f_back) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 289c60591..e5b63ef14 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: class JsonRequest(Request): - attributes: tuple[str, ...] = Request.attributes + ("dumps_kwargs",) + attributes: tuple[str, ...] = (*Request.attributes, "dumps_kwargs") def __init__( self, *args: Any, dumps_kwargs: dict[str, Any] | None = None, **kwargs: Any diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index f954b5e9e..476f1754e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -43,7 +43,7 @@ class TextResponse(Response): _DEFAULT_ENCODING = "ascii" _cached_decoded_json = _NONE - attributes: tuple[str, ...] = Response.attributes + ("encoding",) + attributes: tuple[str, ...] = (*Response.attributes, "encoding") def __init__(self, *args: Any, **kwargs: Any): self._encoding: str | None = kwargs.pop("encoding", None) diff --git a/scrapy/link.py b/scrapy/link.py index 1a569f892..9c272ab2f 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -24,7 +24,7 @@ class Link: of the anchor tag. """ - __slots__ = ["url", "text", "fragment", "nofollow"] + __slots__ = ["fragment", "nofollow", "text", "url"] def __init__( self, url: str, text: str = "", fragment: str = "", nofollow: bool = False diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 544f4adfe..76f9c7856 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -76,8 +76,8 @@ class LogFormatter: self, request: Request, response: Response, spider: Spider ) -> LogFormatterResult: """Logs a message when the crawler finds a webpage.""" - request_flags = f" {str(request.flags)}" if request.flags else "" - response_flags = f" {str(response.flags)}" if response.flags else "" + request_flags = f" {request.flags!s}" if request.flags else "" + response_flags = f" {response.flags!s}" if response.flags else "" return { "level": logging.DEBUG, "msg": CRAWLEDMSG, diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 5438b8522..0f3329db1 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -127,8 +127,7 @@ class MediaPipeline(ABC): if ( not base_class_name or class_name == base_class_name - or settings - and not settings.get(formatted_key) + or (settings and not settings.get(formatted_key)) ): return key return formatted_key diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 93b7fcf17..18cc991bf 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -195,8 +195,7 @@ class StrictOriginPolicy(ReferrerPolicy): if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) - or not self.tls_protected(response_url) - ): + ) or not self.tls_protected(response_url): return self.origin_referrer(response_url) return None @@ -249,8 +248,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) - or not self.tls_protected(response_url) - ): + ) or not self.tls_protected(response_url): return self.origin_referrer(response_url) return None @@ -282,7 +280,7 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3") + NOREFERRER_SCHEMES: tuple[str, ...] = (*LOCAL_SCHEMES, "file", "s3") name: str = POLICY_SCRAPY_DEFAULT diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 5ce4863f6..d319e7950 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -252,7 +252,9 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: def returns_none(return_node: ast.Return) -> bool: value = return_node.value - return value is None or isinstance(value, ast.Constant) and value.value is None + return value is None or ( + isinstance(value, ast.Constant) and value.value is None + ) if inspect.isgeneratorfunction(callable): func = callable diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 66a06a9f0..679e38206 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -100,7 +100,7 @@ def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> No asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") - installer_path = module + ["install"] + installer_path = [*module, "install"] installer = load_object(".".join(installer_path)) with suppress(error.ReactorAlreadyInstalledError): installer() diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index ad811e804..7f2b178f5 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -229,7 +229,8 @@ def request_to_curl(request: Request) -> str: cookies = f"--cookie '{cookie}'" elif isinstance(request.cookies, list): cookie = "; ".join( - f"{list(c.keys())[0]}={list(c.values())[0]}" for c in request.cookies + f"{next(iter(c.keys()))}={next(iter(c.values()))}" + for c in request.cookies ) cookies = f"--cookie '{cookie}'" diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 05e04e2d1..3b1035eab 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -31,7 +31,7 @@ class ProcessTest: if settings is not None: env["SCRAPY_SETTINGS_MODULE"] = settings assert self.command - cmd = self.prefix + [self.command] + list(args) + cmd = [*self.prefix, self.command, *args] pp = TestProcessProtocol() pp.deferred.addCallback(self._process_finished, cmd, check_code) reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 3bf831c26..d487849bb 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -51,7 +51,7 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool: """Return True if the url belongs to the given spider""" return url_is_from_any_domain( - url, [spider.name] + list(getattr(spider, "allowed_domains", [])) + url, [spider.name, *getattr(spider, "allowed_domains", [])] ) diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index ff1f9b346..052321ae3 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -11,7 +11,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings.default_settings import LOG_VERSIONS from scrapy.utils.ssl import get_openssl_version -_DEFAULT_SOFTWARE = ["Scrapy"] + LOG_VERSIONS +_DEFAULT_SOFTWARE = ["Scrapy", *LOG_VERSIONS] def _version(item): diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 4835e936b..acd524ea4 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -21,7 +21,7 @@ class CmdlineTest(unittest.TestCase): def _execute(self, *new_args, **kwargs): encoding = sys.stdout.encoding or "utf-8" - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs) comm = proc.communicate()[0].strip() return comm.decode(encoding) diff --git a/tests/test_commands.py b/tests/test_commands.py index 32b69de8a..9d5720b98 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -87,13 +87,13 @@ class ProjectTest(unittest.TestCase): def call(self, *new_args, **kwargs): with TemporaryFile() as out: - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) return subprocess.call( args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs ) def proc(self, *new_args, **popen_kwargs): - args = (sys.executable, "-m", "scrapy.cmdline") + new_args + args = (sys.executable, "-m", "scrapy.cmdline", *new_args) p = subprocess.Popen( args, cwd=popen_kwargs.pop("cwd", self.cwd), diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f3e5ebf5d..8b3a6eeca 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -647,7 +647,7 @@ class ScriptRunnerMixin: def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name - return [sys.executable, str(script_path)] + list(script_args) + return [sys.executable, str(script_path), *script_args] def run_script(self, script_name: str, *script_args: str) -> str: args = self.get_script_args(script_name, *script_args) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index c99f19b03..1eb7dcf9d 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -114,7 +114,7 @@ class RetryTest(unittest.TestCase): def test_exception_to_retry_added(self): exc = ValueError settings_dict = { - "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc], + "RETRY_EXCEPTIONS": [*RETRY_EXCEPTIONS, exc], } crawler = get_crawler(Spider, settings_dict=settings_dict) mw = RetryMiddleware.from_crawler(crawler) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 55f9ecac9..879bc8697 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -31,7 +31,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): def start_requests(self): self.times = {None: []} - slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None] + slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None] for slot in slots: url = self.mockserver.url(f"/?downloader_slot={slot}") diff --git a/tests/test_exporters.py b/tests/test_exporters.py index fa9389044..522c6638d 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -116,7 +116,7 @@ class BaseItemExporterTest(unittest.TestCase): ) ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1") - _, name = list(ie._get_serialized_fields(self.i))[0] + _, name = next(iter(ie._get_serialized_fields(self.i))) assert isinstance(name, str) self.assertEqual(name, "John\xa3") diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 679cc8238..0730cff3a 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -960,7 +960,7 @@ class XmlResponseTest(TextResponseTest): class CustomResponse(TextResponse): - attributes = TextResponse.attributes + ("foo", "bar") + attributes = (*TextResponse.attributes, "foo", "bar") def __init__(self, *args, **kwargs) -> None: self.foo = kwargs.pop("foo", None) diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index ef07d625f..58efad585 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -61,11 +61,11 @@ Foo 1 oldest: 0s ago\n\n""", ) def test_get_oldest(self): - o1 = Foo() # noqa: F841 + o1 = Foo() o1_time = time() - o2 = Bar() # noqa: F841 + o2 = Bar() o3_time = time() if o3_time <= o1_time: @@ -80,9 +80,9 @@ Foo 1 oldest: 0s ago\n\n""", self.assertIsNone(trackref.get_oldest("XXX")) def test_iter_all(self): - o1 = Foo() # noqa: F841 + o1 = Foo() o2 = Bar() # noqa: F841 - o3 = Foo() # noqa: F841 + o3 = Foo() self.assertEqual( set(trackref.iter_all("Foo")), {o1, o3}, From f44ca39fa23f07c857a34309d79db6394c5faefb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 1 Jan 2025 21:50:02 +0500 Subject: [PATCH 1612/2083] Enable FLY Ruff rules. --- pyproject.toml | 2 ++ scrapy/pqueues.py | 2 +- tests/test_spidermiddleware_referer.py | 25 +++---------------------- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 065382205..a0b37b966 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -222,6 +222,8 @@ extend-select = [ "D", # flake8-future-annotations "FA", + # flynt + "FLY", # refurb "FURB", # isort diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 5b2f81335..a04e0107b 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -34,7 +34,7 @@ def _path_safe(text: str) -> str: # as we replace some letters we can get collision for different slots # add we add unique part unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # noqa: S324 - return "-".join([pathable_slot, unique_slot]) + return f"{pathable_slot}-{unique_slot}" class QueueProtocol(Protocol): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 23b0c17c6..cefd33e4e 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -891,14 +891,7 @@ class TestSettingsPolicyByName(TestCase): # test parsing without space(s) after the comma settings1 = Settings( { - "REFERRER_POLICY": ",".join( - [ - "some-custom-unknown-policy", - POLICY_SAME_ORIGIN, - POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, - "another-custom-unknown-policy", - ] - ) + "REFERRER_POLICY": f"some-custom-unknown-policy,{POLICY_SAME_ORIGIN},{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN},another-custom-unknown-policy" } ) mw1 = RefererMiddleware(settings1) @@ -907,13 +900,7 @@ class TestSettingsPolicyByName(TestCase): # test parsing with space(s) after the comma settings2 = Settings( { - "REFERRER_POLICY": ", ".join( - [ - POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, - "another-custom-unknown-policy", - POLICY_UNSAFE_URL, - ] - ) + "REFERRER_POLICY": f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}, another-custom-unknown-policy, {POLICY_UNSAFE_URL}" } ) mw2 = RefererMiddleware(settings2) @@ -922,13 +909,7 @@ class TestSettingsPolicyByName(TestCase): def test_multiple_policy_tokens_all_invalid(self): settings = Settings( { - "REFERRER_POLICY": ",".join( - [ - "some-custom-unknown-policy", - "another-custom-unknown-policy", - "yet-another-custom-unknown-policy", - ] - ) + "REFERRER_POLICY": "some-custom-unknown-policy,another-custom-unknown-policy,yet-another-custom-unknown-policy" } ) with self.assertRaises(RuntimeError): From 273620488ced7dd1a3c8a1c5022d17f0cc7f9496 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 1 Jan 2025 22:03:42 +0500 Subject: [PATCH 1613/2083] Enable PTH Ruff rules. --- pyproject.toml | 2 ++ scrapy/commands/genspider.py | 2 +- scrapy/commands/startproject.py | 7 +++---- scrapy/core/downloader/handlers/ftp.py | 15 +++++++++------ scrapy/utils/testproc.py | 2 +- tests/test_crawler.py | 2 -- tests/test_downloader_handlers.py | 2 +- tests/test_utils_project.py | 2 +- 8 files changed, 18 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a0b37b966..08b4b09b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -240,6 +240,8 @@ extend-select = [ "PIE", # pylint "PL", + # flake8-use-pathlib + "PTH", # flake8-pyi "PYI", # flake8-quotes diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index d7dc104c2..2a1dea997 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -154,7 +154,7 @@ class Command(ScrapyCommand): spiders_dir = Path(spiders_module.__file__).parent.resolve() else: spiders_module = None - spiders_dir = Path(".") + spiders_dir = Path() spider_file = f"{spiders_dir / module}.py" shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 5cb73f0d2..e0c004580 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import re import string from importlib.util import find_spec @@ -28,9 +27,9 @@ TEMPLATES_TO_RENDER: tuple[tuple[str, ...], ...] = ( IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") -def _make_writable(path: str | os.PathLike) -> None: - current_permissions = os.stat(path).st_mode - os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) +def _make_writable(path: Path) -> None: + current_permissions = path.stat().st_mode + path.chmod(current_permissions | OWNER_WRITE_PERMISSION) class Command(ScrapyCommand): diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 598659b4d..0ad10baff 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -32,6 +32,7 @@ from __future__ import annotations import re from io import BytesIO +from pathlib import Path from typing import TYPE_CHECKING, Any, BinaryIO from urllib.parse import unquote @@ -56,9 +57,11 @@ if TYPE_CHECKING: class ReceivedDataProtocol(Protocol): - def __init__(self, filename: str | None = None): - self.__filename: str | None = filename - self.body: BinaryIO = open(filename, "wb") if filename else BytesIO() + def __init__(self, filename: bytes | None = None): + self.__filename: bytes | None = filename + self.body: BinaryIO = ( + Path(filename.decode()).open("wb") if filename else BytesIO() + ) self.size: int = 0 def dataReceived(self, data: bytes) -> None: @@ -66,7 +69,7 @@ class ReceivedDataProtocol(Protocol): self.size += len(data) @property - def filename(self) -> str | None: + def filename(self) -> bytes | None: return self.__filename def close(self) -> None: @@ -128,8 +131,8 @@ class FTPDownloadHandler: ) -> Response: self.result = result protocol.close() - headers = {"local filename": protocol.filename or "", "size": protocol.size} - body = to_bytes(protocol.filename or protocol.body.read()) + headers = {"local filename": protocol.filename or b"", "size": protocol.size} + body = protocol.filename or protocol.body.read() respcls = responsetypes.from_args(url=request.url, body=body) # hints for Headers-related types may need to be fixed to not use AnyStr return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type] diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 3b1035eab..85d7c940f 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: class ProcessTest: command: str | None = None prefix = [sys.executable, "-m", "scrapy.cmdline"] - cwd = os.getcwd() # trial chdirs to temp dir + cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109 def execute( self, diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 8b3a6eeca..6c3fe96b0 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,4 @@ import logging -import os import platform import re import signal @@ -643,7 +642,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: script_dir: Path - cwd = os.getcwd() def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8ecba41bf..05b64e704 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -116,7 +116,7 @@ class FileTestCase(unittest.TestCase): def tearDown(self): os.close(self.fd) - os.remove(self.tmpname) + Path(self.tmpname).unlink() def test_download(self): def _test(response): diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 3831f4c21..1d149d48d 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -12,7 +12,7 @@ from scrapy.utils.project import data_path, get_project_settings @contextlib.contextmanager def inside_a_project(): - prev_dir = os.getcwd() + prev_dir = Path.cwd() project_dir = tempfile.mkdtemp() try: From c87354cd46afdba35ca104a62b0c2bbfa6bd6f64 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 1 Jan 2025 23:05:07 +0500 Subject: [PATCH 1614/2083] Enable SIM Ruff rules. --- pyproject.toml | 8 +++++++ scrapy/cmdline.py | 4 +--- scrapy/commands/parse.py | 11 ++++----- scrapy/commands/startproject.py | 5 +--- scrapy/core/downloader/handlers/http11.py | 5 +--- scrapy/core/engine.py | 9 +++----- scrapy/core/spidermw.py | 5 +--- scrapy/crawler.py | 6 ++--- scrapy/downloadermiddlewares/httpauth.py | 9 +++++--- scrapy/downloadermiddlewares/httpproxy.py | 5 +--- scrapy/exporters.py | 5 +--- scrapy/extensions/debug.py | 7 +++--- scrapy/extensions/feedexport.py | 5 ++-- scrapy/extensions/httpcache.py | 5 +--- scrapy/extensions/periodic_log.py | 5 +--- scrapy/http/cookies.py | 5 ++-- scrapy/http/request/json_request.py | 4 ++-- scrapy/linkextractors/lxmlhtml.py | 13 ++++++----- scrapy/mail.py | 8 +++---- scrapy/pipelines/files.py | 6 ++--- scrapy/shell.py | 5 ++-- scrapy/spidermiddlewares/referer.py | 9 ++++---- scrapy/utils/_compression.py | 5 ++-- scrapy/utils/datatypes.py | 6 ++--- scrapy/utils/signal.py | 4 ++-- scrapy/utils/url.py | 14 ++++++++---- tests/test_addons.py | 28 ++++++++++++----------- tests/test_downloader_handlers.py | 7 +++--- tests/test_feedexport.py | 12 ++++------ tests/test_http_request.py | 5 +--- tests/test_pipeline_files.py | 28 ++++++++++++----------- tests/test_settings/__init__.py | 2 +- tests/test_spiderloader/__init__.py | 5 ++-- tests/test_utils_deprecate.py | 12 ++++++---- tests/test_utils_iterators.py | 2 +- 35 files changed, 128 insertions(+), 146 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 08b4b09b2..a75f3b6db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -254,6 +254,8 @@ extend-select = [ "RUF", # flake8-bandit "S", + # flake8-simplify + "SIM", # flake8-slots "SLOT", # flake8-debugger @@ -344,6 +346,12 @@ ignore = [ "S321", # Argument default set to insecure SSL protocol "S503", + # Use capitalized environment variable + "SIM112", + # Use a context manager for opening files + "SIM115", + # Yoda condition detected + "SIM300", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 9a24871de..48f462c65 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -90,12 +90,10 @@ def _get_commands_dict( def _pop_command_name(argv: list[str]) -> str | None: - i = 0 - for arg in argv[1:]: + for i, arg in enumerate(argv[1:]): if not arg.startswith("-"): del argv[i] return arg - i += 1 return None diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index f996d1806..61aea3ee4 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -174,13 +174,12 @@ class Command(BaseRunSpiderCommand): display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour) def print_requests(self, lvl: int | None = None, colour: bool = True) -> None: - if lvl is None: - if self.requests: - requests = self.requests[max(self.requests)] - else: - requests = [] - else: + if lvl is not None: requests = self.requests.get(lvl, []) + elif self.requests: + requests = self.requests[max(self.requests)] + else: + requests = [] print("# Requests ", "-" * 65) display.pprint(requests, colorize=colour) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e0c004580..1adc1530f 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -95,10 +95,7 @@ class Command(ScrapyCommand): project_name = args[0] - if len(args) == 2: - project_dir = Path(args[1]) - else: - project_dir = Path(args[0]) + project_dir = Path(args[-1]) if (project_dir / "scrapy.cfg").exists(): self.exitcode = 1 diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9f65794fe..aa8a1a2a4 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -424,10 +424,7 @@ class ScrapyAgent: headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): headers.removeHeader(b"Proxy-Authorization") - if request.body: - bodyproducer = _RequestBodyProducer(request.body) - else: - bodyproducer = None + bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() d: Deferred[TxResponse] = agent.request( method, to_bytes(url, encoding="ascii"), headers, bodyproducer diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 5480df72c..61f444e31 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -291,9 +291,7 @@ class ExecutionEngine: return False if self.slot.start_requests is not None: # not all start requests are handled return False - if self.slot.scheduler.has_pending_requests(): - return False - return True + return not self.slot.scheduler.has_pending_requests() def crawl(self, request: Request) -> None: """Inject the request into the spider <-> downloader pipeline""" @@ -388,9 +386,8 @@ class ExecutionEngine: ) self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider - if hasattr(scheduler, "open"): - if d := scheduler.open(spider): - yield d + if hasattr(scheduler, "open") and (d := scheduler.open(spider)): + yield d yield self.scraper.open_spider(spider) assert self.crawler.stats self.crawler.stats.open_spider(spider) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index a63ee40bf..4b2520aa1 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -198,10 +198,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # chain, they went through it already from the process_spider_exception method recovered: MutableChain[_T] | MutableAsyncChain[_T] last_result_is_async = isinstance(result, AsyncIterable) - if last_result_is_async: - recovered = MutableAsyncChain() - else: - recovered = MutableChain() + recovered = MutableAsyncChain() if last_result_is_async else MutableChain() # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async. # 1. def foo. Sync iterables are passed as is, async ones are downgraded. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 05af1bf8a..0a28c4549 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib import logging import pprint import signal @@ -503,7 +504,6 @@ class CrawlerProcess(CrawlerRunner): def _stop_reactor(self, _: Any = None) -> None: from twisted.internet import reactor - try: + # raised if already stopped or in shutdown stage + with contextlib.suppress(RuntimeError): reactor.stop() - except RuntimeError: # raised if already stopped or in shutdown stage - pass diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index b74140ee1..80107261b 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -42,7 +42,10 @@ class HttpAuthMiddleware: self, request: Request, spider: Spider ) -> Request | Response | None: auth = getattr(self, "auth", None) - if auth and b"Authorization" not in request.headers: - if not self.domain or url_is_from_any_domain(request.url, [self.domain]): - request.headers[b"Authorization"] = auth + if ( + auth + and b"Authorization" not in request.headers + and (not self.domain or url_is_from_any_domain(request.url, [self.domain])) + ): + request.headers[b"Authorization"] = auth return None diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 2f3f2db47..cb7fa8c90 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -51,10 +51,7 @@ class HttpProxyMiddleware: proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", "")) - if user: - creds = self._basic_auth_header(user, password) - else: - creds = None + creds = self._basic_auth_header(user, password) if user else None return creds, proxy_url diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 834a05ae9..46c6aa3fa 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -81,10 +81,7 @@ class BaseItemExporter: include_empty = self.export_empty_fields if self.fields_to_export is None: - if include_empty: - field_iter = item.field_names() - else: - field_iter = item.keys() + field_iter = item.field_names() if include_empty else item.keys() elif isinstance(self.fields_to_export, Mapping): if include_empty: field_iter = self.fields_to_export.items() diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 5ca07394f..afaf81928 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst from __future__ import annotations +import contextlib import logging import signal import sys @@ -69,11 +70,9 @@ class StackTraceDump: class Debugger: def __init__(self) -> None: - try: + # win32 platforms don't support SIGUSR signals + with contextlib.suppress(AttributeError): signal.signal(signal.SIGUSR2, self._enter_debugger) # type: ignore[attr-defined] - except AttributeError: - # win32 platforms don't support SIGUSR signals - pass def _enter_debugger(self, signum: int, frame: FrameType | None) -> None: assert frame diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index b6e6f55a6..8a3d607b0 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst from __future__ import annotations +import contextlib import logging import re import sys @@ -642,10 +643,8 @@ class FeedExporter: ) d = {} for k, v in conf.items(): - try: + with contextlib.suppress(NotConfigured): d[k] = load_object(v) - except NotConfigured: - pass return d def _exporter_supported(self, format: str) -> bool: diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 929807de8..fe2cbcb86 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -89,10 +89,7 @@ class RFC2616Policy: return False cc = self._parse_cachecontrol(request) # obey user-agent directive "Cache-Control: no-store" - if b"no-store" in cc: - return False - # Any other is eligible for caching - return True + return b"no-store" not in cc def should_cache_response(self, response: Response, request: Request) -> bool: # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 7cf08a1bb..f97577442 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -151,10 +151,7 @@ class PeriodicLog: return False if exclude and not include: return True - for p in include: - if p in stat_name: - return True - return False + return any(p in stat_name for p in include) def spider_closed(self, spider: Spider, reason: str) -> None: self.log() diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 60322fe6e..b7c3b9d37 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -64,9 +64,8 @@ class CookieJar: cookies += self.jar._cookies_for_domain(host, wreq) # type: ignore[attr-defined] attrs = self.jar._cookie_attrs(cookies) # type: ignore[attr-defined] - if attrs: - if not wreq.has_header("Cookie"): - wreq.add_unredirected_header("Cookie", "; ".join(attrs)) + if attrs and not wreq.has_header("Cookie"): + wreq.add_unredirected_header("Cookie", "; ".join(attrs)) self.processed += 1 if self.processed % self.check_expired_frequency == 0: diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index e5b63ef14..e26cbe05b 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -29,7 +29,7 @@ class JsonRequest(Request): dumps_kwargs.setdefault("sort_keys", True) self._dumps_kwargs: dict[str, Any] = dumps_kwargs - body_passed = kwargs.get("body", None) is not None + body_passed = kwargs.get("body") is not None data: Any = kwargs.pop("data", None) data_passed: bool = data is not None @@ -61,7 +61,7 @@ class JsonRequest(Request): def replace( self, *args: Any, cls: type[Request] | None = None, **kwargs: Any ) -> Request: - body_passed = kwargs.get("body", None) is not None + body_passed = kwargs.get("body") is not None data: Any = kwargs.pop("data", None) data_passed: bool = data is not None diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index f195dbdd7..4fd932b88 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -41,9 +41,12 @@ _collect_string_content = etree.XPath("string()") def _nons(tag: Any) -> Any: - if isinstance(tag, str): - if tag[0] == "{" and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE: - return tag.split("}")[-1] + if ( + isinstance(tag, str) + and tag[0] == "{" + and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE + ): + return tag.split("}")[-1] return tag @@ -230,9 +233,7 @@ class LxmlLinkExtractor: parsed_url, self.deny_extensions ): return False - if self.restrict_text and not _matches(link.text, self.restrict_text): - return False - return True + return not self.restrict_text or _matches(link.text, self.restrict_text) def matches(self, url: str) -> bool: if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): diff --git a/scrapy/mail.py b/scrapy/mail.py index a3c642401..be2423965 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -111,11 +111,9 @@ class MailSender: ) -> Deferred[None] | None: from twisted.internet import reactor - msg: MIMEBase - if attachs: - msg = MIMEMultipart() - else: - msg = MIMENonMultipart(*mimetype.split("/", 1)) + msg: MIMEBase = ( + MIMEMultipart() if attachs else MIMENonMultipart(*mimetype.split("/", 1)) + ) to = list(arg_to_iter(to)) cc = list(arg_to_iter(cc)) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 16bd45c00..a10117590 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -553,10 +553,8 @@ class FilesPipeline(MediaPipeline): ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE") def _get_store(self, uri: str) -> FilesStoreProtocol: - if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir - scheme = "file" - else: - scheme = urlparse(uri).scheme + # to support win32 paths like: C:\\some\dir + scheme = "file" if Path(uri).is_absolute() else urlparse(uri).scheme store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) diff --git a/scrapy/shell.py b/scrapy/shell.py index 5d0ab1e4d..4a5b9e9cf 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst from __future__ import annotations +import contextlib import os import signal from typing import TYPE_CHECKING, Any @@ -143,12 +144,10 @@ class Shell: else: request.meta["handle_httpstatus_all"] = True response = None - try: + with contextlib.suppress(IgnoreRequest): response, spider = threads.blockingCallFromThread( reactor, self._schedule, request, spider ) - except IgnoreRequest: - pass self.populate_vars(response, request, spider) def populate_vars( diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 18cc991bf..a3a1e5b92 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -360,11 +360,10 @@ class RefererMiddleware: - otherwise, the policy from settings is used. """ policy_name = request.meta.get("referrer_policy") - if policy_name is None: - if isinstance(resp_or_url, Response): - policy_header = resp_or_url.headers.get("Referrer-Policy") - if policy_header is not None: - policy_name = to_unicode(policy_header.decode("latin1")) + if policy_name is None and isinstance(resp_or_url, Response): + policy_header = resp_or_url.headers.get("Referrer-Policy") + if policy_header is not None: + policy_name = to_unicode(policy_header.decode("latin1")) if policy_name is None: return self.default_policy() diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 591737b8e..6b09f36ff 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -1,3 +1,4 @@ +import contextlib import zlib from io import BytesIO from warnings import warn @@ -37,10 +38,8 @@ else: return decompressor.process(data) -try: +with contextlib.suppress(ImportError): import zstandard -except ImportError: - pass _CHUNK_SIZE = 65536 # 64 KiB diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 98ecb2f02..3d0e0d3c7 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -8,6 +8,7 @@ This module must not depend on any module outside the Standard Library. from __future__ import annotations import collections +import contextlib import warnings import weakref from collections import OrderedDict @@ -173,10 +174,9 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): self.data: LocalCache = LocalCache(limit=limit) def __setitem__(self, key: _KT, value: _VT) -> None: - try: + # if raised, key is not weak-referenceable, skip caching + with contextlib.suppress(TypeError): super().__setitem__(key, value) - except TypeError: - pass # key is not weak-referenceable, skip caching def __getitem__(self, key: _KT) -> _VT | None: # type: ignore[override] try: diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index c1d3bfffb..5fd176a3f 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -36,7 +36,7 @@ def send_catch_log( dont_log = named.pop("dont_log", ()) dont_log = tuple(dont_log) if isinstance(dont_log, Sequence) else (dont_log,) dont_log += (StopDownload,) - spider = named.get("spider", None) + spider = named.get("spider") responses: list[tuple[TypingAny, TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): result: TypingAny @@ -88,7 +88,7 @@ def send_catch_log_deferred( return failure dont_log = named.pop("dont_log", None) - spider = named.get("spider", None) + spider = named.get("spider") dfds: list[Deferred[tuple[TypingAny, TypingAny]]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): d: Deferred[TypingAny] = maybeDeferred_coro( diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index d487849bb..db2749d79 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -173,13 +173,19 @@ def strip_url( parsed_url.username or parsed_url.password ): netloc = netloc.split("@")[-1] - if strip_default_port and parsed_url.port: - if (parsed_url.scheme, parsed_url.port) in ( + + if ( + strip_default_port + and parsed_url.port + and (parsed_url.scheme, parsed_url.port) + in ( ("http", 80), ("https", 443), ("ftp", 21), - ): - netloc = netloc.replace(f":{parsed_url.port}", "") + ) + ): + netloc = netloc.replace(f":{parsed_url.port}", "") + return urlunparse( ( parsed_url.scheme, diff --git a/tests/test_addons.py b/tests/test_addons.py index 17949997c..a0caa3511 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -166,19 +166,21 @@ class AddonManagerTest(unittest.TestCase): def update_settings(self, settings): pass - with patch("scrapy.addons.logger") as logger_mock: - with patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock: - settings_dict = { - "ADDONS": {LoggedAddon: 1}, - } - addon = LoggedAddon() - build_from_crawler_mock.return_value = addon - crawler = get_crawler(settings_dict=settings_dict) - logger_mock.info.assert_called_once_with( - "Enabled addons:\n%(addons)s", - {"addons": [addon]}, - extra={"crawler": crawler}, - ) + with ( + patch("scrapy.addons.logger") as logger_mock, + patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock, + ): + settings_dict = { + "ADDONS": {LoggedAddon: 1}, + } + addon = LoggedAddon() + build_from_crawler_mock.return_value = addon + crawler = get_crawler(settings_dict=settings_dict) + logger_mock.info.assert_called_once_with( + "Enabled addons:\n%(addons)s", + {"addons": [addon]}, + extra={"crawler": crawler}, + ) @inlineCallbacks def test_enable_addon_in_spider(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 05b64e704..0dcbeaec1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -530,9 +530,10 @@ class Http11TestCase(HttpTestCase): d = self.download_request(request, Spider("foo")) def checkDataLoss(failure): - if failure.check(ResponseFailed): - if any(r.check(_DataLoss) for r in failure.value.reasons): - return None + if failure.check(ResponseFailed) and any( + r.check(_DataLoss) for r in failure.value.reasons + ): + return None return failure d.addCallback(lambda _: self.fail("No DataLoss exception")) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b087aaab1..0f149f172 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -756,7 +756,7 @@ class FeedExportTest(FeedExportTestBase): ) finally: - for file_path in FEEDS.keys(): + for file_path in FEEDS: if not Path(file_path).exists(): continue @@ -1229,15 +1229,13 @@ class FeedExportTest(FeedExportTestBase): class CustomFilter2(scrapy.extensions.feedexport.ItemFilter): def accepts(self, item): - if "foo" not in item.fields: - return False - return True + return "foo" in item.fields class CustomFilter3(scrapy.extensions.feedexport.ItemFilter): def accepts(self, item): - if isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1": - return True - return False + return ( + isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1" + ) formats = { "json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]', diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9997b7ab3..c5929c339 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1488,10 +1488,7 @@ def _buildresponse(body, **kwargs): def _qs(req, encoding="utf-8", to_unicode=False): - if req.method == "POST": - qs = req.body - else: - qs = req.url.partition("?")[2] + qs = req.body if req.method == "POST" else req.url.partition("?")[2] uqs = unquote_to_bytes(qs) if to_unicode: uqs = uqs.decode(encoding) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index a6c5f0a94..4c3fc36b6 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -634,19 +634,21 @@ class TestGCSFilesStore(unittest.TestCase): import google.cloud.storage # noqa: F401 except ModuleNotFoundError: raise unittest.SkipTest("google-cloud-storage is not installed") - with mock.patch("google.cloud.storage") as _: - with mock.patch("scrapy.pipelines.files.time") as _: - uri = "gs://my_bucket/my_prefix/" - store = GCSFilesStore(uri) - store.bucket = mock.Mock() - path = "full/my_data.txt" - yield store.persist_file( - path, mock.Mock(), info=None, meta=None, headers=None - ) - yield store.stat_file(path, info=None) - expected_blob_path = store.prefix + path - store.bucket.blob.assert_called_with(expected_blob_path) - store.bucket.get_blob.assert_called_with(expected_blob_path) + with ( + mock.patch("google.cloud.storage"), + mock.patch("scrapy.pipelines.files.time"), + ): + uri = "gs://my_bucket/my_prefix/" + store = GCSFilesStore(uri) + store.bucket = mock.Mock() + path = "full/my_data.txt" + yield store.persist_file( + path, mock.Mock(), info=None, meta=None, headers=None + ) + yield store.stat_file(path, info=None) + expected_blob_path = store.prefix + path + store.bucket.blob.assert_called_with(expected_blob_path) + store.bucket.get_blob.assert_called_with(expected_blob_path) class TestFTPFileStore(unittest.TestCase): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 503c29e32..96d59c911 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -170,7 +170,7 @@ class BaseSettingsTest(unittest.TestCase): self.assertCountEqual(self.settings.attributes.keys(), ctrl_attributes.keys()) - for key in ctrl_attributes.keys(): + for key in ctrl_attributes: attr = self.settings.attributes[key] ctrl_attr = ctrl_attributes[key] self.assertEqual(attr.value, ctrl_attr.value) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 9b53b9b96..d5aac34eb 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,3 +1,4 @@ +import contextlib import shutil import sys import tempfile @@ -22,10 +23,8 @@ module_dir = Path(__file__).resolve().parent def _copytree(source: Path, target: Path): - try: + with contextlib.suppress(shutil.Error): shutil.copytree(source, target) - except shutil.Error: - pass class SpiderLoaderTest(unittest.TestCase): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index eedb6f6af..dc5fbd3c3 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -259,12 +259,14 @@ class WarnWhenSubclassedTest(unittest.TestCase): self.assertIn("foo.Bar", str(w[1].message)) def test_inspect_stack(self): - with mock.patch("inspect.stack", side_effect=IndexError): - with warnings.catch_warnings(record=True) as w: - DeprecatedName = create_deprecated_class("DeprecatedName", NewName) + with ( + mock.patch("inspect.stack", side_effect=IndexError), + warnings.catch_warnings(record=True) as w, + ): + DeprecatedName = create_deprecated_class("DeprecatedName", NewName) - class SubClass(DeprecatedName): - pass + class SubClass(DeprecatedName): + pass self.assertIn("Error detecting parent module", str(w[0].message)) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 4c81e3a2f..12507c6a3 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -366,7 +366,7 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all(isinstance(k, str) for k in result_row.keys())) + self.assertTrue(all(isinstance(k, str) for k in result_row)) self.assertTrue(all(isinstance(v, str) for v in result_row.values())) def test_csviter_delimiter(self): From b70443f2d06b1b0ad8c474fc5e8a424e363bdd81 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 2 Jan 2025 00:31:26 +0500 Subject: [PATCH 1615/2083] Split ruff and pylint ignores into two categories, some pylint cleanup. --- docs/_ext/scrapydocs.py | 1 + docs/conf.py | 2 + pyproject.toml | 81 +++++++++++++++++---------------- scrapy/contracts/__init__.py | 2 +- scrapy/extensions/feedexport.py | 2 + scrapy/extensions/telnet.py | 4 +- scrapy/interfaces.py | 2 + scrapy/shell.py | 17 ++++--- scrapy/utils/console.py | 2 +- scrapy/utils/deprecate.py | 1 + scrapy/utils/display.py | 1 + scrapy/utils/engine.py | 2 +- tests/test_contracts.py | 4 +- tests/test_exporters.py | 4 +- tests/test_item.py | 2 +- tests/test_link.py | 2 +- tests/test_scrapy__getattr__.py | 2 +- tests/test_selector.py | 2 +- tests/test_squeues_request.py | 8 ++-- 19 files changed, 77 insertions(+), 64 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index c23a89089..9b63f39f6 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,3 +1,4 @@ +# pylint: disable=import-error from operator import itemgetter from docutils import nodes diff --git a/docs/conf.py b/docs/conf.py index 7a5166053..d06828bcc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,6 +8,8 @@ # # All configuration values have a default; values that are commented out # serve to show the default. + +# pylint: disable=import-error import os import sys from pathlib import Path diff --git a/pyproject.toml b/pyproject.toml index a75f3b6db..88005ec4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,43 +125,30 @@ extension-pkg-allow-list=[ [tool.pylint."MESSAGES CONTROL"] disable = [ - "abstract-method", - "arguments-differ", - "arguments-renamed", + # Ones we want to ignore "attribute-defined-outside-init", "broad-exception-caught", "consider-using-with", "cyclic-import", - "dangerous-default-value", "disallowed-name", - "duplicate-code", # https://github.com/PyCQA/pylint/issues/214 - "eval-used", + "duplicate-code", # https://github.com/pylint-dev/pylint/issues/214 "fixme", - "import-error", "import-outside-toplevel", - "inherit-non-class", + "inherit-non-class", # false positives with create_deprecated_class() "invalid-name", "invalid-overridden-method", - "isinstance-second-argument-not-valid-type", - "keyword-arg-before-vararg", + "isinstance-second-argument-not-valid-type", # false positives with create_deprecated_class() "line-too-long", "logging-format-interpolation", "logging-fstring-interpolation", "logging-not-lazy", "missing-docstring", "no-member", - "no-method-argument", - "no-name-in-module", - "no-self-argument", - "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268 + "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268 "not-callable", - "pointless-statement", - "pointless-string-statement", "protected-access", - "raise-missing-from", "redefined-builtin", "redefined-outer-name", - "signature-differs", "too-few-public-methods", "too-many-ancestors", "too-many-arguments", @@ -173,14 +160,23 @@ disable = [ "too-many-positional-arguments", "too-many-public-methods", "too-many-return-statements", - "unbalanced-tuple-unpacking", - "unnecessary-dunder-call", "unused-argument", "unused-import", "unused-variable", - "used-before-assignment", - "useless-return", + "useless-return", # https://github.com/pylint-dev/pylint/issues/6530 "wrong-import-position", + + # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") + "abstract-method", + "arguments-differ", + "arguments-renamed", + "dangerous-default-value", + "keyword-arg-before-vararg", + "pointless-statement", + "raise-missing-from", + "unbalanced-tuple-unpacking", + "unnecessary-dunder-call", + "used-before-assignment", ] [tool.pytest.ini_options] @@ -270,22 +266,8 @@ extend-select = [ "YTT", ] ignore = [ - # Assigning to `os.environ` doesn't clear the environment. - "B003", - # Do not use mutable data structures for argument defaults. - "B006", - # Loop control variable not used within the loop body. - "B007", - # Do not perform function calls in argument defaults. - "B008", - # Star-arg unpacking after a keyword argument is strongly discouraged. - "B026", - # Found useless expression. - "B018", - # No explicit stacklevel argument found. - "B028", - # Within an `except` clause, raise exceptions with `raise ... from` - "B904", + # Ones we want to ignore + # Missing docstring in public module "D100", # Missing docstring in public class @@ -346,12 +328,31 @@ ignore = [ "S321", # Argument default set to insecure SSL protocol "S503", - # Use capitalized environment variable - "SIM112", # Use a context manager for opening files "SIM115", # Yoda condition detected "SIM300", + + # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") + + # Assigning to `os.environ` doesn't clear the environment. + "B003", + # Do not use mutable data structures for argument defaults. + "B006", + # Loop control variable not used within the loop body. + "B007", + # Do not perform function calls in argument defaults. + "B008", + # Found useless expression. + "B018", + # Star-arg unpacking after a keyword argument is strongly discouraged. + "B026", + # No explicit stacklevel argument found. + "B028", + # Within an `except` clause, raise exceptions with `raise ... from` + "B904", + # Use capitalized environment variable + "SIM112", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 3b4f932a0..bdb68c4ad 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -199,7 +199,7 @@ def _create_testcase(method: Callable, desc: str) -> TestCase: spider = method.__self__.name # type: ignore[attr-defined] class ContractTestCase(TestCase): - def __str__(_self) -> str: + def __str__(_self) -> str: # pylint: disable=no-self-argument return f"[{spider}] {method.__name__} ({desc})" name = f"{spider}_{method.__name__}" diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 8a3d607b0..c6e2aa0dd 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -110,6 +110,8 @@ class ItemFilter: class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" + # pylint: disable=no-self-argument + def __init__(uri, *, feed_options=None): # pylint: disable=super-init-not-called """Initialize the storage with the parameters given in the URI and the feed-specific options (see :setting:`FEEDS`)""" diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index ee28d86ba..189b1953b 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -84,7 +84,9 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers - def login(self_, credentials, mind, *interfaces): + def login( + self_, credentials, mind, *interfaces + ): # pylint: disable=no-self-argument if not ( credentials.username == self.username.encode("utf8") and credentials.checkPassword(self.password.encode("utf8")) diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index 9a2c5f170..13a4d822d 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,3 +1,5 @@ +# pylint:disable=no-method-argument,no-self-argument + from zope.interface import Interface diff --git a/scrapy/shell.py b/scrapy/shell.py index 4a5b9e9cf..5e5e57a9a 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -71,17 +71,16 @@ class Shell: else: self.populate_vars() if self.code: + # pylint: disable-next=eval-used print(eval(self.code, globals(), self.vars)) # noqa: S307 else: - """ - Detect interactive shell setting in scrapy.cfg - e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg - [settings] - # shell can be one of ipython, bpython or python; - # to be used as the interactive python console, if available. - # (default is ipython, fallbacks in the order listed above) - shell = python - """ + # Detect interactive shell setting in scrapy.cfg + # e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg + # [settings] + # # shell can be one of ipython, bpython or python; + # # to be used as the interactive python console, if available. + # # (default is ipython, fallbacks in the order listed above) + # shell = python cfg = get_config() section, option = "settings", "shell" env = os.environ.get("SCRAPY_PYTHON_SHELL") diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 95844a48c..7425543ff 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -59,7 +59,7 @@ def _embed_ptpython_shell( namespace: dict[str, Any] = {}, banner: str = "" ) -> EmbedFuncT: """Start a ptpython shell""" - import ptpython.repl + import ptpython.repl # pylint: disable=import-error @wraps(_embed_ptpython_shell) def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 0a0acc742..20d03cae6 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -57,6 +57,7 @@ def create_deprecated_class( # https://github.com/python/mypy/issues/4177 class DeprecatedClass(new_class.__class__): # type: ignore[misc, name-defined] + # pylint: disable=no-self-argument deprecated_class: type | None = None warned_on_subclass: bool = False diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 39f46270b..20744a604 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -30,6 +30,7 @@ def _tty_supports_color() -> bool: def _colorize(text: str, colorize: bool = True) -> str: + # pylint: disable=no-name-in-module if not colorize or not sys.stdout.isatty() or not _tty_supports_color(): return text try: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 1948009e8..52f29e22c 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -32,7 +32,7 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: checks: list[tuple[str, Any]] = [] for test in tests: try: - checks += [(test, eval(test))] # noqa: S307 + checks += [(test, eval(test))] # noqa: S307 # pylint: disable=eval-used except Exception as e: checks += [(test, f"{type(e).__name__} (exception)")] diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 743889234..f7581707b 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -517,8 +517,8 @@ class ContractsManagerTest(unittest.TestCase): super().__init__(*args, **kwargs) self.visited = 0 - def start_requests(s): - return self.conman.from_spider(s, self.results) + def start_requests(self_): # pylint: disable=no-self-argument + return self.conman.from_spider(self_, self.results) def parse_first(self, response): self.visited += 1 diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 522c6638d..970f8d2f5 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -216,7 +216,9 @@ class PprintItemExporterTest(BaseItemExporterTest): return PprintItemExporter(self.output, **kwargs) def _check_output(self): - self._assert_expected_item(eval(self.output.getvalue())) + self._assert_expected_item( + eval(self.output.getvalue()) # pylint: disable=eval-used + ) class PprintItemExporterDataclassTest(PprintItemExporterTest): diff --git a/tests/test_item.py b/tests/test_item.py index 13243b67f..35212c153 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -54,7 +54,7 @@ class ItemTest(unittest.TestCase): self.assertEqual(itemrepr, "{'name': 'John Doe', 'number': 123}") - i2 = eval(itemrepr) + i2 = eval(itemrepr) # pylint: disable=eval-used self.assertEqual(i2["name"], "John Doe") self.assertEqual(i2["number"], 123) diff --git a/tests/test_link.py b/tests/test_link.py index 7ba0851ae..35723bbd6 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -49,7 +49,7 @@ class LinkTest(unittest.TestCase): l1 = Link( "http://www.example.com", text="test", fragment="something", nofollow=True ) - l2 = eval(repr(l1)) + l2 = eval(repr(l1)) # pylint: disable=eval-used self._assert_same_links(l1, l2) def test_bytes_url(self): diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py index 979c42267..443e26a3c 100644 --- a/tests/test_scrapy__getattr__.py +++ b/tests/test_scrapy__getattr__.py @@ -3,7 +3,7 @@ import warnings def test_deprecated_twisted_version(): with warnings.catch_warnings(record=True) as warns: - from scrapy import twisted_version + from scrapy import twisted_version # pylint: disable=no-name-in-module assert twisted_version is not None assert isinstance(twisted_version, tuple) diff --git a/tests/test_selector.py b/tests/test_selector.py index 1b5f3f018..857c7d626 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -264,7 +264,7 @@ class JMESPathTestCase(unittest.TestCase): ) @pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath") - def test_jmespath_not_available(my_json_page) -> None: + def test_jmespath_not_available(self) -> None: body = """ { "website": {"name": "Example"} diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 02ea8027f..04eeae4dc 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -1,3 +1,7 @@ +""" +Queues that handle requests +""" + import shutil import tempfile import unittest @@ -16,10 +20,6 @@ from scrapy.squeues import ( ) from scrapy.utils.test import get_crawler -""" -Queues that handle requests -""" - class BaseQueueTestCase(unittest.TestCase): def setUp(self): From dc706d4fc307f0b51d8122f10dee6ad8e2653629 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 2 Jan 2025 12:32:25 +0500 Subject: [PATCH 1616/2083] Remove useless pylint: disable lines. --- pyproject.toml | 3 +++ scrapy/interfaces.py | 2 +- tests/test_scheduler.py | 2 -- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 88005ec4a..8c985753f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,6 +124,9 @@ extension-pkg-allow-list=[ ] [tool.pylint."MESSAGES CONTROL"] +enable = [ + "useless-suppression", +] disable = [ # Ones we want to ignore "attribute-defined-outside-init", diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index 13a4d822d..b4f1d9394 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,4 +1,4 @@ -# pylint:disable=no-method-argument,no-self-argument +# pylint: disable=no-method-argument,no-self-argument from zope.interface import Interface diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 8bd1480ad..3ac330ae2 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -277,14 +277,12 @@ class DownloaderAwareSchedulerTestMixin: downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() - # pylint: disable=protected-access slot = downloader.get_slot_key(request) dequeued_slots.append(slot) downloader.increment(slot) requests.append(request) for request in requests: - # pylint: disable=protected-access slot = downloader.get_slot_key(request) downloader.decrement(slot) From b10d46d280fbc84f7a1c50e116a1ed828aa286c9 Mon Sep 17 00:00:00 2001 From: Arthur <48801049+devfox-se@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:36:28 +0400 Subject: [PATCH 1617/2083] Fix the calculate_final_stats method (#6599) --- scrapy/extensions/logstats.py | 9 ++++++--- tests/test_logstats.py | 11 ++++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index e829d8b92..f2e1f57b8 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -83,12 +83,15 @@ class LogStats: self, spider: Spider ) -> tuple[None, None] | tuple[float, float]: start_time = self.stats.get_value("start_time") - finished_time = self.stats.get_value("finished_time") + finish_time = self.stats.get_value("finish_time") - if not start_time or not finished_time: + if not start_time or not finish_time: return None, None - mins_elapsed = (finished_time - start_time).seconds / 60 + mins_elapsed = (finish_time - start_time).seconds / 60 + + if mins_elapsed == 0: + return None, None items = self.stats.get_value("item_scraped_count", 0) pages = self.stats.get_value("response_received_count", 0) diff --git a/tests/test_logstats.py b/tests/test_logstats.py index d87285df7..a4b002e34 100644 --- a/tests/test_logstats.py +++ b/tests/test_logstats.py @@ -47,7 +47,7 @@ class TestLogStats(unittest.TestCase): # Simulate when spider closes after running for 30 mins self.stats.set_value("start_time", datetime.fromtimestamp(1655100172)) - self.stats.set_value("finished_time", datetime.fromtimestamp(1655101972)) + self.stats.set_value("finish_time", datetime.fromtimestamp(1655101972)) logstats.spider_closed(self.spider, "test reason") self.assertEqual(self.stats.get_value("responses_per_minute"), 172.9) self.assertEqual(self.stats.get_value("items_per_minute"), 116.4) @@ -60,3 +60,12 @@ class TestLogStats(unittest.TestCase): logstats.spider_closed(self.spider, "test reason") self.assertIsNone(self.stats.get_value("responses_per_minute")) self.assertIsNone(self.stats.get_value("items_per_minute")) + + def test_stats_calculation_no_elapsed_time(self): + """The stat values should be None since the elapsed time is 0.""" + logstats = LogStats.from_crawler(self.crawler) + self.stats.set_value("start_time", datetime.fromtimestamp(1655100172)) + self.stats.set_value("finish_time", datetime.fromtimestamp(1655100172)) + logstats.spider_closed(self.spider, "test reason") + self.assertIsNone(self.stats.get_value("responses_per_minute")) + self.assertIsNone(self.stats.get_value("items_per_minute")) From 6ae5b9267145e5b01d282f766fa90d129eb40390 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 2 Jan 2025 15:45:04 +0400 Subject: [PATCH 1618/2083] Drop the remaining unittest.main() blocks. (#6602) --- tests/test_downloadermiddleware_httpcache.py | 4 ---- tests/test_downloadermiddleware_redirect.py | 4 ---- tests/test_downloadermiddleware_retry.py | 4 ---- tests/test_exporters.py | 4 ---- tests/test_http_request.py | 4 ---- tests/test_item.py | 4 ---- tests/test_loader.py | 4 ---- tests/test_loader_deprecated.py | 4 ---- tests/test_logformatter.py | 4 ---- tests/test_mail.py | 4 ---- tests/test_responsetypes.py | 4 ---- tests/test_settings/__init__.py | 4 ---- tests/test_utils_conf.py | 4 ---- tests/test_utils_console.py | 4 ---- tests/test_utils_datatypes.py | 4 ---- tests/test_utils_httpobj.py | 4 ---- tests/test_utils_misc/__init__.py | 4 ---- tests/test_utils_request.py | 4 ---- tests/test_utils_sitemap.py | 4 ---- tests/test_utils_spider.py | 4 ---- tests/test_utils_template.py | 4 ---- tests/test_utils_url.py | 4 ---- 22 files changed, 88 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index f80eff3e6..ec4e87ffb 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -566,7 +566,3 @@ class RFC2616PolicyTest(DefaultStorageTest): res2 = self._process_requestresponse(mw, req0, None) self.assertEqualResponse(res1, res2) assert "cached" in res2.flags - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index e37da9715..7b19ab781 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1314,7 +1314,3 @@ def test_meta_refresh_schemes(url, location, target): else: assert isinstance(redirect, Request) assert redirect.url == target - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index c99f19b03..9b39b84d9 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -643,7 +643,3 @@ class GetRetryRequestTest(unittest.TestCase): f"{stats_key}/reason_count/{expected_reason}", ): self.assertEqual(spider.crawler.stats.get_value(stat), 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_exporters.py b/tests/test_exporters.py index fa9389044..0f70887af 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -699,7 +699,3 @@ class CustomExporterItemTest(unittest.TestCase): class CustomExporterDataclassTest(CustomExporterItemTest): item_class = TestDataClass - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9997b7ab3..d020a8911 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1717,7 +1717,3 @@ class JsonRequestTest(RequestTest): def tearDown(self): warnings.resetwarnings() super().tearDown() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_item.py b/tests/test_item.py index 13243b67f..3f10a724d 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -296,7 +296,3 @@ class ItemMetaClassCellRegression(unittest.TestCase): # TypeError: __class__ set to <class '__main__.MyItem'> # defining 'MyItem' as <class '__main__.MyItem'> super().__init__(*args, **kwargs) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_loader.py b/tests/test_loader.py index aca428bbe..824d7aecf 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -586,7 +586,3 @@ class FunctionProcessorTestCase(unittest.TestCase): lo.add_value("foo", " bar ") lo.add_value("foo", [" asdf ", " qwerty "]) self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 4bf22f6a0..8d4bd6bc1 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -715,7 +715,3 @@ class FunctionProcessorTestCase(unittest.TestCase): lo.add_value("foo", " bar ") lo.add_value("foo", [" asdf ", " qwerty "]) self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 5a92521cc..61a9f3f8d 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -234,7 +234,3 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc)) self.assertNotIn("Crawled (200) <GET http://127.0.0.1:", str(lc)) self.assertNotIn("Dropped: Ignoring item", str(lc)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_mail.py b/tests/test_mail.py index c6af2b1b8..cf40c342e 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -157,7 +157,3 @@ class MailSenderTest(unittest.TestCase): context = factory.buildProtocol("test@scrapy.org").context self.assertIsInstance(context, ClientTLSOptions) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 7be8150fc..f9f56ff97 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -126,7 +126,3 @@ class ResponseTypesTest(unittest.TestCase): self.assertEqual( responsetypes.mimetypes.guess_type("x.scrapytest")[0], "x-scrapy/test" ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 503c29e32..8bc48aa7b 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -497,7 +497,3 @@ class SettingsTest(unittest.TestCase): self.assertEqual( str(error.exception), "Trying to modify an immutable Settings object" ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 2ce7948eb..cbea41129 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -177,7 +177,3 @@ class FeedExportConfigTestCase(unittest.TestCase): "item_export_kwargs": {}, }, ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index dabd6054d..0bc86e1b9 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -38,7 +38,3 @@ class UtilsConsoleTestCase(unittest.TestCase): # default shell should be 'ipython' shell = get_shell_embed_func() self.assertEqual(shell.__name__, "_embed_ipython_shell") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 5a76593c3..e80381671 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -372,7 +372,3 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): for i, r in enumerate(refs): self.assertIn(r, cache) self.assertEqual(cache[r], i) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index b824972d5..741e69559 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -20,7 +20,3 @@ class HttpobjUtilsTest(unittest.TestCase): assert req1a is req1b assert req1a is not req2 assert req1a is not req2 - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index f71b2b034..478c1e73a 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -216,7 +216,3 @@ class UtilsMiscTestCase(unittest.TestCase): assert rel_has_nofollow("nofollowfoo") is False assert rel_has_nofollow("foonofollow") is False assert rel_has_nofollow("ugc, , nofollow") is True - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 0a3e3b00b..51bca9a31 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -481,7 +481,3 @@ class RequestToCurlTest(unittest.TestCase): " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" ) self._test_request(request_object, expected_curl_command) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index ce0de0722..69a459d8b 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -295,7 +295,3 @@ Disallow: /forum/active/ ) self.assertEqual(list(s), [{"loc": "http://127.0.0.1:8000/"}]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index ae59d0137..df8f37103 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -30,7 +30,3 @@ class UtilsSpidersTestCase(unittest.TestCase): it = iter_spider_classes(tests.test_utils_spider) self.assertEqual(set(it), {MySpider1, MySpider2}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index fc42c0d2f..5fbbd74da 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -33,7 +33,3 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): render_path.unlink() assert not render_path.exists() # Failure of test itself - - -if "__main__" == __name__: - unittest.main() diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 62e2b5c1e..94a59f883 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -630,7 +630,3 @@ def test_deprecated_imports_from_w3lib(obj_name): getattr(import_module("scrapy.utils.url"), obj_name) assert message in warns[0].message.args - - -if __name__ == "__main__": - unittest.main() From 176ae348c57a536d661ca1b469035e7e1ccbca6a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 2 Jan 2025 18:14:18 +0500 Subject: [PATCH 1619/2083] Reformat long REFERRER_POLICY. --- tests/test_spidermiddleware_referer.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index cefd33e4e..4945ac25d 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -891,7 +891,12 @@ class TestSettingsPolicyByName(TestCase): # test parsing without space(s) after the comma settings1 = Settings( { - "REFERRER_POLICY": f"some-custom-unknown-policy,{POLICY_SAME_ORIGIN},{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN},another-custom-unknown-policy" + "REFERRER_POLICY": ( + f"some-custom-unknown-policy," + f"{POLICY_SAME_ORIGIN}," + f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}," + f"another-custom-unknown-policy" + ) } ) mw1 = RefererMiddleware(settings1) @@ -900,7 +905,11 @@ class TestSettingsPolicyByName(TestCase): # test parsing with space(s) after the comma settings2 = Settings( { - "REFERRER_POLICY": f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}, another-custom-unknown-policy, {POLICY_UNSAFE_URL}" + "REFERRER_POLICY": ( + f"{POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}," + f" another-custom-unknown-policy," + f" {POLICY_UNSAFE_URL}" + ) } ) mw2 = RefererMiddleware(settings2) @@ -909,7 +918,11 @@ class TestSettingsPolicyByName(TestCase): def test_multiple_policy_tokens_all_invalid(self): settings = Settings( { - "REFERRER_POLICY": "some-custom-unknown-policy,another-custom-unknown-policy,yet-another-custom-unknown-policy" + "REFERRER_POLICY": ( + "some-custom-unknown-policy," + "another-custom-unknown-policy," + "yet-another-custom-unknown-policy" + ) } ) with self.assertRaises(RuntimeError): From 4d31277bc67169460dc2d8bca80946df8b355b8f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 3 Jan 2025 02:48:14 +0400 Subject: [PATCH 1620/2083] Explicitly mark re-exports. (#6579) --- pyproject.toml | 20 ++++++++++++-------- scrapy/core/downloader/handlers/http.py | 5 +++++ scrapy/http/__init__.py | 13 +++++++++++++ scrapy/linkextractors/__init__.py | 5 +++++ scrapy/selector/__init__.py | 5 +++++ scrapy/spiders/__init__.py | 9 +++++++++ tests/test_item.py | 3 ++- tests/test_utils_url.py | 2 +- 8 files changed, 52 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8c985753f..571a61f1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,9 +72,9 @@ version = {file = "./scrapy/VERSION"} [tool.mypy] ignore_missing_imports = true +implicit_reexport = false # Interface classes are hard to support - [[tool.mypy.overrides]] module = "twisted.internet.interfaces" follow_imports = "skip" @@ -92,6 +92,14 @@ follow_imports = "skip" module = "scrapy.settings.default_settings" ignore_errors = true +[[tool.mypy.overrides]] +module = "itemadapter" +implicit_reexport = true + +[[tool.mypy.overrides]] +module = "twisted" +implicit_reexport = true + [tool.bumpversion] current_version = "2.12.0" commit = true @@ -359,13 +367,9 @@ ignore = [ ] [tool.ruff.lint.per-file-ignores] -# Exclude files that are meant to provide top-level imports -"scrapy/__init__.py" = ["E402"] -"scrapy/core/downloader/handlers/http.py" = ["F401"] -"scrapy/http/__init__.py" = ["F401"] -"scrapy/linkextractors/__init__.py" = ["E402", "F401"] -"scrapy/selector/__init__.py" = ["F401"] -"scrapy/spiders/__init__.py" = ["E402", "F401"] +# Circular import workarounds +"scrapy/linkextractors/__init__.py" = ["E402"] +"scrapy/spiders/__init__.py" = ["E402"] # Skip bandit in tests "tests/**" = ["S"] diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 52535bd8b..93b96c779 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -2,3 +2,8 @@ from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import ( HTTP11DownloadHandler as HTTPDownloadHandler, ) + +__all__ = [ + "HTTP10DownloadHandler", + "HTTPDownloadHandler", +] diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index d0b726bad..0e5c2b53b 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -15,3 +15,16 @@ from scrapy.http.response.html import HtmlResponse from scrapy.http.response.json import JsonResponse from scrapy.http.response.text import TextResponse from scrapy.http.response.xml import XmlResponse + +__all__ = [ + "FormRequest", + "Headers", + "HtmlResponse", + "JsonRequest", + "JsonResponse", + "Request", + "Response", + "TextResponse", + "XmlResponse", + "XmlRpcRequest", +] diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 1c7e96ae0..b39859f7b 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -126,3 +126,8 @@ def _is_valid_url(url: str) -> bool: # Top-level imports from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor + +__all__ = [ + "IGNORED_EXTENSIONS", + "LinkExtractor", +] diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 85c500d66..7cfa3c364 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -4,3 +4,8 @@ Selectors # top-level imports from scrapy.selector.unified import Selector, SelectorList + +__all__ = [ + "Selector", + "SelectorList", +] diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 6136dabc7..e255e91cc 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -117,3 +117,12 @@ class Spider(object_ref): from scrapy.spiders.crawl import CrawlSpider, Rule from scrapy.spiders.feed import CSVFeedSpider, XMLFeedSpider from scrapy.spiders.sitemap import SitemapSpider + +__all__ = [ + "CSVFeedSpider", + "CrawlSpider", + "Rule", + "SitemapSpider", + "Spider", + "XMLFeedSpider", +] diff --git a/tests/test_item.py b/tests/test_item.py index 5a8ee095e..480412841 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,7 +1,8 @@ import unittest +from abc import ABCMeta from unittest import mock -from scrapy.item import ABCMeta, Field, Item, ItemMeta +from scrapy.item import Field, Item, ItemMeta class ItemTest(unittest.TestCase): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 94a59f883..314082742 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -6,7 +6,7 @@ import pytest from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.spiders import Spider from scrapy.utils.misc import arg_to_iter -from scrapy.utils.url import ( +from scrapy.utils.url import ( # type: ignore[attr-defined] _is_filesystem_path, _public_w3lib_objects, add_http_if_no_scheme, From f2234c5b96d4069d6881aacb7007ba3d305dfb0e Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Tue, 7 Jan 2025 06:40:49 -0300 Subject: [PATCH 1621/2083] Fix Crawler.request_fingerprinter typing (#6605) --- scrapy/crawler.py | 4 ++-- scrapy/extensions/httpcache.py | 6 ++++-- scrapy/pipelines/media.py | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 0a28c4549..f6dbe053a 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -44,7 +44,7 @@ if TYPE_CHECKING: from scrapy.logformatter import LogFormatter from scrapy.spiderloader import SpiderLoader from scrapy.statscollectors import StatsCollector - from scrapy.utils.request import RequestFingerprinter + from scrapy.utils.request import RequestFingerprinterProtocol logger = logging.getLogger(__name__) @@ -80,7 +80,7 @@ class Crawler: self.extensions: ExtensionManager | None = None self.stats: StatsCollector | None = None self.logformatter: LogFormatter | None = None - self.request_fingerprinter: RequestFingerprinter | None = None + self.request_fingerprinter: RequestFingerprinterProtocol | None = None self.spider: Spider | None = None self.engine: ExecutionEngine | None = None diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index fe2cbcb86..0cd16d737 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: from scrapy.http.request import Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider - from scrapy.utils.request import RequestFingerprinter + from scrapy.utils.request import RequestFingerprinterProtocol logger = logging.getLogger(__name__) @@ -265,7 +265,9 @@ class DbmCacheStorage: ) assert spider.crawler.request_fingerprinter - self._fingerprinter: RequestFingerprinter = spider.crawler.request_fingerprinter + self._fingerprinter: RequestFingerprinterProtocol = ( + spider.crawler.request_fingerprinter + ) def close_spider(self, spider: Spider) -> None: self.db.close() diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0f3329db1..e66b86ce6 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -30,7 +30,7 @@ if TYPE_CHECKING: from scrapy import Spider from scrapy.crawler import Crawler from scrapy.http import Response - from scrapy.utils.request import RequestFingerprinter + from scrapy.utils.request import RequestFingerprinterProtocol class FileInfo(TypedDict): @@ -47,7 +47,7 @@ logger = logging.getLogger(__name__) class MediaPipeline(ABC): crawler: Crawler - _fingerprinter: RequestFingerprinter + _fingerprinter: RequestFingerprinterProtocol _modern_init = False LOG_FAILED_RESULTS: bool = True From 4869315d102e2e92f50582ebc3aebdb82266f5b3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 7 Jan 2025 13:46:12 +0400 Subject: [PATCH 1622/2083] Install libjpeg-dev on pinned envs to be able to install Pillow. (#6607) --- .github/workflows/tests-ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index b2a5681df..ab6794d3c 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -73,7 +73,7 @@ jobs: if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update - sudo apt-get install libxml2-dev libxslt-dev + sudo apt-get install libxml2-dev libxslt-dev libjpeg-dev - name: Run tests env: ${{ matrix.env }} From 5d3aa80ad1f1dcd67a185158c6a7ceb53855eefd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 7 Jan 2025 13:52:26 +0400 Subject: [PATCH 1623/2083] Switch CI to codecov/codecov-action and enable it on Windows. (#6609) --- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- .github/workflows/tests-windows.yml | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 9e78e26e3..c28a99982 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -27,4 +27,4 @@ jobs: tox -e py - name: Upload coverage report - run: bash <(curl -s https://codecov.io/bash) + uses: codecov/codecov-action@v5 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index ab6794d3c..89d1e70ac 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -82,4 +82,4 @@ jobs: tox - name: Upload coverage report - run: bash <(curl -s https://codecov.io/bash) + uses: codecov/codecov-action@v5 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 67a32aac6..45e4ca157 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -44,3 +44,6 @@ jobs: run: | pip install -U tox tox + + - name: Upload coverage report + uses: codecov/codecov-action@v5 From 59fcb9b93c4971602b4a4afd4afdf08db7a7f2b7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 7 Jan 2025 15:18:18 +0400 Subject: [PATCH 1624/2083] Improve internal refs to scrapy.Request and scrapy.Selector (#6526) * Improve internal refs to scrapy.Selector. * Improve internal refs to scrapy.Request. * More scrapy.http fixes. * Fix FormRequest refs. * More fixes. * Simplifications. * Last fixes. * Add the parsel intersphinx. --- docs/conf.py | 1 + docs/news.rst | 139 +++++++++++++------------- docs/topics/downloader-middleware.rst | 2 +- docs/topics/dynamic-content.rst | 5 +- docs/topics/exceptions.rst | 2 +- docs/topics/request-response.rst | 137 ++++++++++++------------- docs/topics/selectors.rst | 16 +-- scrapy/core/scheduler.py | 4 +- scrapy/http/request/__init__.py | 12 +-- scrapy/http/response/__init__.py | 8 +- scrapy/http/response/text.py | 14 +-- scrapy/loader/__init__.py | 4 +- scrapy/utils/request.py | 10 +- 13 files changed, 176 insertions(+), 178 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d06828bcc..fd8165db3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -284,6 +284,7 @@ intersphinx_mapping = { "cryptography": ("https://cryptography.io/en/latest/", None), "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), + "parsel": ("https://parsel.readthedocs.io/en/latest/", None), "pytest": ("https://docs.pytest.org/en/latest", None), "python": ("https://docs.python.org/3", None), "sphinx": ("https://www.sphinx-doc.org/en/master", None), diff --git a/docs/news.rst b/docs/news.rst index 2bf65272f..924abb7a1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -635,7 +635,7 @@ Bug fixes exception if ``default`` is ``None``. (:issue:`6308`, :issue:`6310`) -- :class:`~scrapy.selector.Selector` now uses +- :class:`~scrapy.Selector` now uses :func:`scrapy.utils.response.get_base_url` to determine the base URL of a given :class:`~scrapy.http.Response`. (:issue:`6265`) @@ -653,7 +653,7 @@ Documentation - Add a FAQ entry about :ref:`creating blank requests <faq-blank-request>`. (:issue:`6203`, :issue:`6208`) -- Document that :attr:`scrapy.selector.Selector.type` can be ``"json"``. +- Document that :attr:`scrapy.Selector.type` can be ``"json"``. (:issue:`6328`, :issue:`6334`) Quality assurance @@ -734,7 +734,7 @@ Documentation - Improved documentation for :class:`~scrapy.crawler.Crawler` initialization changes made in the 2.11.0 release. (:issue:`6057`, :issue:`6147`) -- Extended documentation for :attr:`Request.meta <scrapy.http.Request.meta>`. +- Extended documentation for :attr:`.Request.meta`. (:issue:`5565`) - Fixed the :reqmeta:`dont_merge_cookies` documentation. (:issue:`5936`, @@ -1095,7 +1095,7 @@ New features :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) -- Added :meth:`TextResponse.jmespath`, a shortcut for JMESPath selectors +- Added :meth:`.TextResponse.jmespath`, a shortcut for JMESPath selectors available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`) - Added :signal:`feed_slot_closed` and :signal:`feed_exporter_closed` @@ -1275,7 +1275,7 @@ New features avoid confusion. (:issue:`5717`, :issue:`5722`, :issue:`5727`) -- The ``callback`` parameter of :class:`~scrapy.http.Request` can now be set +- The ``callback`` parameter of :class:`~scrapy.Request` can now be set to :func:`scrapy.http.request.NO_CALLBACK`, to distinguish it from ``None``, as the latter indicates that the default spider callback (:meth:`~scrapy.Spider.parse`) is to be used. @@ -1772,17 +1772,17 @@ Highlights: Security bug fixes ~~~~~~~~~~~~~~~~~~ -- When a :class:`~scrapy.http.Request` object with cookies defined gets a - redirect response causing a new :class:`~scrapy.http.Request` object to be +- When a :class:`~scrapy.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.Request` object to be scheduled, the cookies defined in the original - :class:`~scrapy.http.Request` object are no longer copied into the new - :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object are no longer copied into the new + :class:`~scrapy.Request` object. If you manually set the ``Cookie`` header on a - :class:`~scrapy.http.Request` object and the domain name of the redirect + :class:`~scrapy.Request` object and the domain name of the redirect URL is not an exact match for the domain of the URL of the original - :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped - from the new :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.Request` object. The old behavior could be exploited by an attacker to gain access to your cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more @@ -1795,10 +1795,10 @@ Security bug fixes ``example.com`` and any subdomain) by defining the shared domain suffix (e.g. ``example.com``) as the cookie domain when defining your cookies. See the documentation of the - :class:`~scrapy.http.Request` class for more information. + :class:`~scrapy.Request` class for more information. - When the domain of a cookie, either received in the ``Set-Cookie`` header - of a response or defined in a :class:`~scrapy.http.Request` object, is set + of a response or defined in a :class:`~scrapy.Request` object, is set to a `public suffix <https://publicsuffix.org/>`_, the cookie is now ignored unless the cookie domain is the same as the request domain. @@ -1849,7 +1849,7 @@ Backward-incompatible changes meet expectations, :exc:`TypeError` is now raised at startup time. Before, other exceptions would be raised at run time. (:issue:`3559`) -- The ``_encoding`` field of serialized :class:`~scrapy.http.Request` objects +- The ``_encoding`` field of serialized :class:`~scrapy.Request` objects is now named ``encoding``, in line with all other fields (:issue:`5130`) @@ -1879,7 +1879,7 @@ Deprecations - :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`) - Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new - :meth:`Request.to_dict <scrapy.http.Request.to_dict>` method. + :meth:`.Request.to_dict` method. - Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new :func:`scrapy.utils.request.request_from_dict` function. @@ -1984,9 +1984,9 @@ New features using ``queuelib`` 1.6.1 or later), the ``peek`` method raises :exc:`NotImplementedError`. -- :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have +- :class:`~scrapy.Request` and :class:`~scrapy.http.Response` now have an ``attributes`` attribute that makes subclassing easier. For - :class:`~scrapy.http.Request`, it also allows subclasses to work with + :class:`~scrapy.Request`, it also allows subclasses to work with :func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`, :issue:`5130`, :issue:`5218`) @@ -2452,14 +2452,13 @@ Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again - discards cookies defined in :attr:`Request.headers - <scrapy.http.Request.headers>`. + discards cookies defined in :attr:`.Request.headers`. We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it was reported that the current implementation could break existing code. If you need to set cookies for a request, use the :class:`Request.cookies - <scrapy.http.Request>` parameter. + <scrapy.Request>` parameter. A future version of Scrapy will include a new, better implementation of the reverted bug fix. @@ -2580,16 +2579,16 @@ New features :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` or :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception` - with a custom :class:`~scrapy.http.Request` object assigned to + with a custom :class:`~scrapy.Request` object assigned to :class:`response.request <scrapy.http.Response.request>`: - The response is handled by the callback of that custom - :class:`~scrapy.http.Request` object, instead of being handled by the - callback of the original :class:`~scrapy.http.Request` object + :class:`~scrapy.Request` object, instead of being handled by the + callback of the original :class:`~scrapy.Request` object - - That custom :class:`~scrapy.http.Request` object is now sent as the + - That custom :class:`~scrapy.Request` object is now sent as the ``request`` argument to the :signal:`response_received` signal, instead - of the original :class:`~scrapy.http.Request` object + of the original :class:`~scrapy.Request` object (:issue:`4529`, :issue:`4632`) @@ -2760,7 +2759,7 @@ New features * The :command:`parse` command now allows specifying an output file (:issue:`4317`, :issue:`4377`) -* :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and +* :meth:`.Request.from_curl` and :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support ``--data-raw`` (:issue:`4612`) @@ -2776,7 +2775,7 @@ Bug fixes :ref:`dataclass items <dataclass-items>` and :ref:`attr.s items <attrs-items>` (:issue:`4667`, :issue:`4668`) -* :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and +* :meth:`.Request.from_curl` and :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request method to ``POST`` when a request body is specified and no request method is specified (:issue:`4612`) @@ -2861,8 +2860,7 @@ Backward-incompatible changes Deprecations ~~~~~~~~~~~~ -* :meth:`TextResponse.body_as_unicode - <scrapy.http.TextResponse.body_as_unicode>` is now deprecated, use +* ``TextResponse.body_as_unicode()`` is now deprecated, use :attr:`TextResponse.text <scrapy.http.TextResponse.text>` instead (:issue:`4546`, :issue:`4555`, :issue:`4579`) @@ -2901,9 +2899,8 @@ New features * :ref:`Link extractors <topics-link-extractors>` are now serializable, as long as you do not use :ref:`lambdas <lambda>` for parameters; for - example, you can now pass link extractors in :attr:`Request.cb_kwargs - <scrapy.http.Request.cb_kwargs>` or - :attr:`Request.meta <scrapy.http.Request.meta>` when :ref:`persisting + example, you can now pass link extractors in :attr:`.Request.cb_kwargs` + or :attr:`.Request.meta` when :ref:`persisting scheduled requests <topics-jobs>` (:issue:`4554`) * Upgraded the :ref:`pickle protocol <pickle-protocols>` that Scrapy uses @@ -2922,11 +2919,11 @@ Bug fixes * :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer discards cookies defined in :attr:`Request.headers - <scrapy.http.Request.headers>` (:issue:`1992`, :issue:`2400`) + <scrapy.Request.headers>` (:issue:`1992`, :issue:`2400`) * :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter - of the ``__init__`` method of :class:`~scrapy.http.Request` + of the ``__init__`` method of :class:`~scrapy.Request` (:issue:`2400`, :issue:`3575`) * When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is @@ -2935,7 +2932,7 @@ Bug fixes * :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine syntax <topics/coroutines>` no longer need to return an iterable, and may - instead return a :class:`~scrapy.http.Request` object, an + instead return a :class:`~scrapy.Request` object, an :ref:`item <topics-items>`, or ``None`` (:issue:`4609`) * The :command:`startproject` command now ensures that the generated project @@ -2976,8 +2973,8 @@ Documentation :issue:`4587`) * The display-on-hover behavior of internal documentation references now also - covers links to :ref:`commands <topics-commands>`, :attr:`Request.meta - <scrapy.http.Request.meta>` keys, :ref:`settings <topics-settings>` and + covers links to :ref:`commands <topics-commands>`, :attr:`.Request.meta` + keys, :ref:`settings <topics-settings>` and :ref:`signals <topics-signals>` (:issue:`4495`, :issue:`4563`) * It is again possible to download the documentation for offline reading @@ -3262,7 +3259,7 @@ Deprecation removals ~~~~~~~~~~~~~~~~~~~~ * The :ref:`Scrapy shell <topics-shell>` no longer provides a `sel` proxy - object, use :meth:`response.selector <scrapy.http.Response.selector>` + object, use :meth:`response.selector <scrapy.http.TextResponse.selector>` instead (:issue:`4347`) * LevelDB support has been removed (:issue:`4112`) @@ -3332,10 +3329,10 @@ New features * The new :attr:`Response.cb_kwargs <scrapy.http.Response.cb_kwargs>` attribute serves as a shortcut for :attr:`Response.request.cb_kwargs - <scrapy.http.Request.cb_kwargs>` (:issue:`4331`) + <scrapy.Request.cb_kwargs>` (:issue:`4331`) * :meth:`Response.follow <scrapy.http.Response.follow>` now supports a - ``flags`` parameter, for consistency with :class:`~scrapy.http.Request` + ``flags`` parameter, for consistency with :class:`~scrapy.Request` (:issue:`4277`, :issue:`4279`) * :ref:`Item loader processors <topics-loaders-processors>` can now be @@ -3344,7 +3341,7 @@ New features * :class:`~scrapy.spiders.Rule` now accepts an ``errback`` parameter (:issue:`4000`) -* :class:`~scrapy.http.Request` no longer requires a ``callback`` parameter +* :class:`~scrapy.Request` no longer requires a ``callback`` parameter when an ``errback`` parameter is specified (:issue:`3586`, :issue:`4008`) * :class:`~scrapy.logformatter.LogFormatter` now supports some additional @@ -3416,7 +3413,7 @@ Bug fixes * Redirects to URLs starting with 3 slashes (``///``) are now supported (:issue:`4032`, :issue:`4042`) -* :class:`~scrapy.http.Request` no longer accepts strings as ``url`` simply +* :class:`~scrapy.Request` no longer accepts strings as ``url`` simply because they have a colon (:issue:`2552`, :issue:`4094`) * The correct encoding is now used for attach names in @@ -3462,7 +3459,7 @@ Documentation using :class:`~scrapy.crawler.CrawlerProcess` (:issue:`2149`, :issue:`2352`, :issue:`3146`, :issue:`3960`) -* Clarified the requirements for :class:`~scrapy.http.Request` objects +* Clarified the requirements for :class:`~scrapy.Request` objects :ref:`when using persistence <request-serialization>` (:issue:`4124`, :issue:`4139`) @@ -3731,17 +3728,17 @@ Scrapy 1.8.2 (2022-03-01) **Security bug fixes:** -- When a :class:`~scrapy.http.Request` object with cookies defined gets a - redirect response causing a new :class:`~scrapy.http.Request` object to be +- When a :class:`~scrapy.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.Request` object to be scheduled, the cookies defined in the original - :class:`~scrapy.http.Request` object are no longer copied into the new - :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object are no longer copied into the new + :class:`~scrapy.Request` object. If you manually set the ``Cookie`` header on a - :class:`~scrapy.http.Request` object and the domain name of the redirect + :class:`~scrapy.Request` object and the domain name of the redirect URL is not an exact match for the domain of the URL of the original - :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped - from the new :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.Request` object. The old behavior could be exploited by an attacker to gain access to your cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more @@ -3754,10 +3751,10 @@ Scrapy 1.8.2 (2022-03-01) ``example.com`` and any subdomain) by defining the shared domain suffix (e.g. ``example.com``) as the cookie domain when defining your cookies. See the documentation of the - :class:`~scrapy.http.Request` class for more information. + :class:`~scrapy.Request` class for more information. - When the domain of a cookie, either received in the ``Set-Cookie`` header - of a response or defined in a :class:`~scrapy.http.Request` object, is set + of a response or defined in a :class:`~scrapy.Request` object, is set to a `public suffix <https://publicsuffix.org/>`_, the cookie is now ignored unless the cookie domain is the same as the request domain. @@ -3815,7 +3812,7 @@ Highlights: * Dropped Python 3.4 support and updated minimum requirements; made Python 3.8 support official -* New :meth:`Request.from_curl <scrapy.http.Request.from_curl>` class method +* New :meth:`.Request.from_curl` class method * New :setting:`ROBOTSTXT_PARSER` and :setting:`ROBOTSTXT_USER_AGENT` settings * New :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` settings @@ -3869,7 +3866,7 @@ See also :ref:`1.8-deprecation-removals` below. New features ~~~~~~~~~~~~ -* A new :meth:`Request.from_curl <scrapy.http.Request.from_curl>` class +* A new :meth:`Request.from_curl <scrapy.Request.from_curl>` class method allows :ref:`creating a request from a cURL command <requests-from-curl>` (:issue:`2985`, :issue:`3862`) @@ -3898,9 +3895,8 @@ New features ``True`` to enable debug-level messages about TLS connection parameters after establishing HTTPS connections (:issue:`2111`, :issue:`3450`) -* Callbacks that receive keyword arguments - (see :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>`) can now be - tested using the new :class:`@cb_kwargs +* Callbacks that receive keyword arguments (see :attr:`.Request.cb_kwargs`) + can now be tested using the new :class:`@cb_kwargs <scrapy.contracts.default.CallbackKeywordArgumentsContract>` :ref:`spider contract <topics-contracts>` (:issue:`3985`, :issue:`3988`) @@ -4089,7 +4085,7 @@ Backward-incompatible changes * Non-default values for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting may stop working. Scheduler priority queue classes now need to handle - :class:`~scrapy.http.Request` objects instead of arbitrary Python data + :class:`~scrapy.Request` objects instead of arbitrary Python data structures. * An additional ``crawler`` parameter has been added to the ``__init__`` @@ -4111,7 +4107,7 @@ New features scheduling improvement on crawls targeting multiple web domains, at the cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) -* A new :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>` attribute +* A new :attr:`.Request.cb_kwargs` attribute provides a cleaner way to pass keyword arguments to callback methods (:issue:`1138`, :issue:`3563`) @@ -4192,7 +4188,7 @@ Bug fixes * Requests with private callbacks are now correctly unserialized from disk (:issue:`3790`) -* :meth:`FormRequest.from_response() <scrapy.http.FormRequest.from_response>` +* :meth:`.FormRequest.from_response` now handles invalid methods like major web browsers (:issue:`3777`, :issue:`3794`) @@ -4272,13 +4268,13 @@ The following deprecated APIs have been removed (:issue:`3578`): * From both ``scrapy.selector`` and ``scrapy.selector.lxmlsel``: - * ``HtmlXPathSelector`` (use :class:`~scrapy.selector.Selector`) + * ``HtmlXPathSelector`` (use :class:`~scrapy.Selector`) - * ``XmlXPathSelector`` (use :class:`~scrapy.selector.Selector`) + * ``XmlXPathSelector`` (use :class:`~scrapy.Selector`) - * ``XPathSelector`` (use :class:`~scrapy.selector.Selector`) + * ``XPathSelector`` (use :class:`~scrapy.Selector`) - * ``XPathSelectorList`` (use :class:`~scrapy.selector.Selector`) + * ``XPathSelectorList`` (use :class:`~scrapy.Selector`) * From ``scrapy.selector.csstranslator``: @@ -4288,7 +4284,7 @@ The following deprecated APIs have been removed (:issue:`3578`): * ``ScrapyXPathExpr`` (use parsel.csstranslator.XPathExpr_) -* From :class:`~scrapy.selector.Selector`: +* From :class:`~scrapy.Selector`: * ``_root`` (both the ``__init__`` method argument and the object property, use ``root``) @@ -4818,7 +4814,7 @@ New Features (:issue:`2535`) - New :ref:`response.follow <response-follow-example>` shortcut for creating requests (:issue:`1940`) -- Added ``flags`` argument and attribute to :class:`Request <scrapy.http.Request>` +- Added ``flags`` argument and attribute to :class:`~scrapy.Request` objects (:issue:`2047`) - Support Anonymous FTP (:issue:`2342`) - Added ``retry/count``, ``retry/max_reached`` and ``retry/reason_count/<reason>`` @@ -4860,7 +4856,7 @@ Bug fixes - LinkExtractor now strips leading and trailing whitespaces from attributes (:issue:`2547`, fixes :issue:`1614`) - Properly handle whitespaces in action attribute in - :class:`~scrapy.http.FormRequest` (:issue:`2548`) + :class:`~scrapy.FormRequest` (:issue:`2548`) - Buffer CONNECT response bytes from proxy until all HTTP headers are received (:issue:`2495`, fixes :issue:`2491`) - FTP downloader now works on Python 3, provided you use Twisted>=17.1 @@ -4902,8 +4898,7 @@ Documentation ~~~~~~~~~~~~~ - Binary mode is required for exporters (:issue:`2564`, fixes :issue:`2553`) -- Mention issue with :meth:`FormRequest.from_response - <scrapy.http.FormRequest.from_response>` due to bug in lxml (:issue:`2572`) +- Mention issue with :meth:`.FormRequest.from_response` due to bug in lxml (:issue:`2572`) - Use single quotes uniformly in templates (:issue:`2596`) - Document :reqmeta:`ftp_user` and :reqmeta:`ftp_password` meta keys (:issue:`2587`) - Removed section on deprecated ``contrib/`` (:issue:`2636`) @@ -5442,7 +5437,7 @@ Bugfixes - Support empty password for http_proxy config (:issue:`1274`). - Interpret ``application/x-json`` as ``TextResponse`` (:issue:`1333`). - Support link rel attribute with multiple values (:issue:`1201`). -- Fixed ``scrapy.http.FormRequest.from_response`` when there is a ``<base>`` +- Fixed ``scrapy.FormRequest.from_response`` when there is a ``<base>`` tag (:issue:`1564`). - Fixed :setting:`TEMPLATES_DIR` handling (:issue:`1575`). - Various ``FormRequest`` fixes (:issue:`1595`, :issue:`1596`, :issue:`1597`). @@ -6369,7 +6364,7 @@ Scrapy 0.18.0 (released 2013-08-09) - Moved persistent (on disk) queues to a separate project (queuelib_) which Scrapy now depends on - Add Scrapy commands using external libraries (:issue:`260`) - Added ``--pdb`` option to ``scrapy`` command line tool -- Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. +- Added :meth:`XPathSelector.remove_namespaces <scrapy.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. - Several improvements to spider contracts - New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections, - MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62 diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 11a3fcb94..af7885a45 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -80,7 +80,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`. middleware. :meth:`process_request` should either: return ``None``, return a - :class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request` + :class:`~scrapy.http.Response` object, return a :class:`~scrapy.Request` object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`. If it returns ``None``, Scrapy will continue processing this request, executing all diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 75d980835..801f6d06d 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -117,7 +117,8 @@ data from it depends on the type of response: - If the response is HTML, XML or JSON, use :ref:`selectors <topics-selectors>` as usual. -- If the response is JSON, use :func:`response.json()` to load the desired data: +- If the response is JSON, use :func:`response.json() + <scrapy.http.TextResponse.json>` to load the desired data: .. code-block:: python @@ -143,7 +144,7 @@ data from it depends on the type of response: - If the response is an image or another format based on images (e.g. PDF), read the response as bytes from - :attr:`response.body <scrapy.http.TextResponse.body>` and use an OCR + :attr:`response.body <scrapy.http.Response.body>` and use an OCR solution to extract the desired data as text. For example, you can use pytesseract_. To read a table from a PDF, diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index ea64edbe6..0b572ff95 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -105,7 +105,7 @@ response: In both cases, the response could have its body truncated: the body contains all bytes received up until the exception is raised, including the bytes received in the signal handler that raises the exception. Also, the response -object is marked with ``"download_stopped"`` in its :attr:`Response.flags` +object is marked with ``"download_stopped"`` in its :attr:`~scrapy.http.Response.flags` attribute. .. note:: ``fail`` is a keyword-only parameter, i.e. raising diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 710e2e131..1bb1a10a4 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -7,15 +7,15 @@ Requests and Responses .. module:: scrapy.http :synopsis: Request and Response classes -Scrapy uses :class:`Request` and :class:`Response` objects for crawling web +Scrapy uses :class:`~scrapy.Request` and :class:`Response` objects for crawling web sites. -Typically, :class:`Request` objects are generated in the spiders and pass +Typically, :class:`~scrapy.Request` objects are generated in the spiders and pass across the system until they reach the Downloader, which executes the request and returns a :class:`Response` object which travels back to the spider that issued the request. -Both :class:`Request` and :class:`Response` classes have subclasses which add +Both :class:`~scrapy.Request` and :class:`Response` classes have subclasses which add functionality not required in the base classes. These are described below in :ref:`topics-request-response-ref-request-subclasses` and :ref:`topics-request-response-ref-response-subclasses`. @@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and Request objects =============== -.. autoclass:: Request +.. autoclass:: scrapy.Request :param url: the URL of this request @@ -52,7 +52,7 @@ Request objects :param method: the HTTP method of this request. Defaults to ``'GET'``. :type method: str - :param meta: the initial values for the :attr:`Request.meta` attribute. If + :param meta: the initial values for the :attr:`.Request.meta` attribute. If given, the dict passed in this parameter will be shallow copied. :type meta: dict @@ -67,10 +67,10 @@ Request objects (for single valued headers) or lists (for multi-valued headers). If ``None`` is passed as value, the HTTP header will not be sent at all. - .. caution:: Cookies set via the ``Cookie`` header are not considered by the - :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies <scrapy.Request>` parameter. This is a known - current limitation that is being worked on. + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + ``cookies`` argument. This is a known current limitation that is being + worked on. :type headers: dict @@ -124,7 +124,7 @@ Request objects .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies <scrapy.Request>` parameter. This is a known + :class:`scrapy.Request.cookies <scrapy.Request>` parameter. This is a known current limitation that is being worked on. .. versionadded:: 2.6.0 @@ -172,7 +172,7 @@ Request objects A string containing the URL of this request. Keep in mind that this attribute contains the escaped URL, so it can differ from the URL passed in - the ``__init__`` method. + the ``__init__()`` method. This attribute is read-only. To change the URL of a Request use :meth:`replace`. @@ -184,7 +184,8 @@ Request objects .. attribute:: Request.headers - A dictionary-like object which contains the request headers. + A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains + the request headers. .. attribute:: Request.body @@ -240,8 +241,8 @@ Request objects A dictionary that contains arbitrary metadata for this request. Its contents will be passed to the Request's callback as keyword arguments. It is empty - for new Requests, which means by default callbacks only get a :class:`Response` - object as argument. + for new Requests, which means by default callbacks only get a + :class:`~scrapy.http.Response` object as argument. This dict is :doc:`shallow copied <library/copy>` when the request is cloned using the ``copy()`` or ``replace()`` methods, and can also be @@ -262,7 +263,7 @@ Request objects Return a Request object with the same members, except for those members given new values by whichever keyword arguments are specified. The - :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are shallow + :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow copied by default (unless new values are given as arguments). See also :ref:`topics-request-response-ref-request-callback-arguments`. @@ -305,7 +306,7 @@ Example: In some cases you may be interested in passing arguments to those callback functions so you can receive the arguments later, in the second callback. The following example shows how to achieve this by using the -:attr:`Request.cb_kwargs` attribute: +:attr:`.Request.cb_kwargs` attribute: .. code-block:: python @@ -326,10 +327,10 @@ The following example shows how to achieve this by using the foo=foo, ) -.. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``. - Prior to that, using :attr:`Request.meta` was recommended for passing - information around callbacks. After ``1.7``, :attr:`Request.cb_kwargs` - became the preferred way for handling user information, leaving :attr:`Request.meta` +.. caution:: :attr:`.Request.cb_kwargs` was introduced in version ``1.7``. + Prior to that, using :attr:`.Request.meta` was recommended for passing + information around callbacks. After ``1.7``, :attr:`.Request.cb_kwargs` + became the preferred way for handling user information, leaving :attr:`.Request.meta` for communication with components like middlewares and extensions. .. _topics-request-response-ref-errbacks: @@ -441,7 +442,7 @@ Request fingerprints There are some aspects of scraping, such as filtering out duplicate requests (see :setting:`DUPEFILTER_CLASS`) or caching responses (see :setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short, -unique identifier from a :class:`~scrapy.http.Request` object: a request +unique identifier from a :class:`~scrapy.Request` object: a request fingerprint. You often do not need to worry about request fingerprints, the default request @@ -486,7 +487,7 @@ A request fingerprinter is a class that must implement the following method: See also :ref:`request-fingerprint-restrictions`. :param request: request to fingerprint - :type request: scrapy.http.Request + :type request: scrapy.Request Additionally, it may also implement the following method: @@ -566,7 +567,7 @@ URL canonicalization or taking the request method or body into account: If you need to be able to override the request fingerprinting for arbitrary requests from your spider callbacks, you may implement a request fingerprinter -that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>` +that reads fingerprints from :attr:`request.meta <scrapy.Request.meta>` when available, and then falls back to :func:`scrapy.utils.request.fingerprint`. For example: @@ -581,10 +582,8 @@ when available, and then falls back to return request.meta["fingerprint"] return fingerprint(request) -If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6 -without using the deprecated ``'2.6'`` value of the -:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following -request fingerprinter: +If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6, use +the following request fingerprinter: .. code-block:: python @@ -628,7 +627,7 @@ The following built-in Scrapy components have such restrictions: :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`, the following directory structure is created: - - :attr:`Spider.name <scrapy.spiders.Spider.name>` + - :attr:`.Spider.name` - first byte of a request fingerprint as hexadecimal @@ -656,7 +655,7 @@ The following built-in Scrapy components have such restrictions: Request.meta special keys ========================= -The :attr:`Request.meta` attribute can contain any arbitrary data, but there +The :attr:`.Request.meta` attribute can contain any arbitrary data, but there are some special keys recognized by Scrapy and its built-in extensions. Those are: @@ -780,24 +779,25 @@ call their callback instead, like in this example, pass ``fail=False`` to the Request subclasses ================== -Here is the list of built-in :class:`Request` subclasses. You can also subclass +Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass it to implement your own custom functionality. FormRequest objects ------------------- -The FormRequest class extends the base :class:`Request` with functionality for +The FormRequest class extends the base :class:`~scrapy.Request` with functionality for dealing with HTML forms. It uses `lxml.html forms`_ to pre-populate form fields with form data from :class:`Response` objects. .. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms -.. class:: scrapy.http.request.form.FormRequest -.. class:: scrapy.http.FormRequest -.. class:: scrapy.FormRequest(url, [formdata, ...]) +.. currentmodule:: None - The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The - remaining arguments are the same as for the :class:`Request` class and are +.. class:: scrapy.FormRequest(url, [formdata, ...]) + :canonical: scrapy.http.request.form.FormRequest + + The :class:`~scrapy.FormRequest` class adds a new keyword parameter to the ``__init__()`` method. The + remaining arguments are the same as for the :class:`~scrapy.Request` class and are not documented here. :param formdata: is a dictionary (or iterable of (key, value) tuples) @@ -805,12 +805,12 @@ fields with form data from :class:`Response` objects. body of the request. :type formdata: dict or collections.abc.Iterable - The :class:`FormRequest` objects support the following class method in - addition to the standard :class:`Request` methods: + The :class:`~scrapy.FormRequest` objects support the following class method in + addition to the standard :class:`~scrapy.Request` methods: - .. classmethod:: FormRequest.from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) + .. classmethod:: from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) - Returns a new :class:`FormRequest` object with its form field values + Returns a new :class:`~scrapy.FormRequest` object with its form field values pre-populated with those found in the HTML ``<form>`` element contained in the given response. For an example see :ref:`topics-request-response-ref-request-userlogin`. @@ -832,7 +832,7 @@ fields with form data from :class:`Response` objects. :param response: the response containing a HTML form which will be used to pre-populate the form fields - :type response: :class:`Response` object + :type response: :class:`~scrapy.http.Response` object :param formname: if given, the form with name attribute set to this value will be used. :type formname: str @@ -869,7 +869,9 @@ fields with form data from :class:`Response` objects. :type dont_click: bool The other parameters of this class method are passed directly to the - :class:`FormRequest` ``__init__`` method. + :class:`~scrapy.FormRequest` ``__init__()`` method. + +.. currentmodule:: scrapy.http Request usage examples ---------------------- @@ -878,7 +880,7 @@ Using FormRequest to send data via HTTP POST ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you want to simulate a HTML Form POST in your spider and send a couple of -key-value fields, you can return a :class:`FormRequest` object (from your +key-value fields, you can return a :class:`~scrapy.FormRequest` object (from your spider) like this: .. skip: next @@ -901,7 +903,7 @@ It is usual for web sites to provide pre-populated form fields through ``<input type="hidden">`` elements, such as session related data or authentication tokens (for login pages). When scraping, you'll want these fields to be automatically pre-populated and only override a couple of them, such as the -user name and password. You can use the :meth:`FormRequest.from_response` +user name and password. You can use the :meth:`.FormRequest.from_response()` method for this job. Here's an example spider which uses it: .. code-block:: python @@ -936,21 +938,22 @@ method for this job. Here's an example spider which uses it: JsonRequest ----------- -The JsonRequest class extends the base :class:`Request` class with functionality for +The JsonRequest class extends the base :class:`~scrapy.Request` class with functionality for dealing with JSON requests. .. class:: JsonRequest(url, [... data, dumps_kwargs]) - The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The - remaining arguments are the same as for the :class:`Request` class and are + The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__()`` method. The + remaining arguments are the same as for the :class:`~scrapy.Request` class and are not documented here. Using the :class:`JsonRequest` will set the ``Content-Type`` header to ``application/json`` and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01`` :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body. - if :attr:`Request.body` argument is provided this parameter will be ignored. - if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be + If the :attr:`~scrapy.Request.body` argument is provided this parameter will be ignored. + If the :attr:`~scrapy.Request.body` argument is not provided and the + ``data`` argument is provided the :attr:`~scrapy.Request.method` will be set to ``'POST'`` automatically. :type data: object @@ -1002,7 +1005,7 @@ Response objects :type flags: list :param request: the initial value of the :attr:`Response.request` attribute. - This represents the :class:`Request` that generated this response. + This represents the :class:`~scrapy.Request` that generated this response. :type request: scrapy.Request :param certificate: an object representing the server's SSL certificate. @@ -1038,11 +1041,12 @@ Response objects .. attribute:: Response.headers - A dictionary-like object which contains the response headers. Values can - be accessed using :meth:`get` to return the first header value with the - specified name or :meth:`getlist` to return all header values with the - specified name. For example, this call will give you all cookies in the - headers:: + A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains + the response headers. Values can be accessed using + :meth:`~scrapy.http.headers.Headers.get` to return the first header value with + the specified name or :meth:`~scrapy.http.headers.Headers.getlist` to return + all header values with the specified name. For example, this call will give you + all cookies in the headers:: response.headers.getlist('Set-Cookie') @@ -1058,7 +1062,7 @@ Response objects .. attribute:: Response.request - The :class:`Request` object that generated this response. This attribute is + The :class:`~scrapy.Request` object that generated this response. This attribute is assigned in the Scrapy engine, after the response and the request have passed through all :ref:`Downloader Middlewares <topics-downloader-middleware>`. In particular, this means that: @@ -1077,34 +1081,33 @@ Response objects .. attribute:: Response.meta - A shortcut to the :attr:`Request.meta` attribute of the + A shortcut to the :attr:`~scrapy.Request.meta` attribute of the :attr:`Response.request` object (i.e. ``self.request.meta``). Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta` attribute is propagated along redirects and retries, so you will get - the original :attr:`Request.meta` sent from your spider. + the original :attr:`.Request.meta` sent from your spider. - .. seealso:: :attr:`Request.meta` attribute + .. seealso:: :attr:`.Request.meta` attribute .. attribute:: Response.cb_kwargs .. versionadded:: 2.0 - A shortcut to the :attr:`Request.cb_kwargs` attribute of the + A shortcut to the :attr:`~scrapy.Request.cb_kwargs` attribute of the :attr:`Response.request` object (i.e. ``self.request.cb_kwargs``). Unlike the :attr:`Response.request` attribute, the :attr:`Response.cb_kwargs` attribute is propagated along redirects and - retries, so you will get the original :attr:`Request.cb_kwargs` sent - from your spider. + retries, so you will get the original :attr:`.Request.cb_kwargs` sent from your spider. - .. seealso:: :attr:`Request.cb_kwargs` attribute + .. seealso:: :attr:`.Request.cb_kwargs` attribute .. attribute:: Response.flags A list that contains flags for this response. Flags are labels used for tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And - they're shown on the string representation of the Response (`__str__` + they're shown on the string representation of the Response (``__str__()`` method) which is used by the engine for logging. .. attribute:: Response.certificate @@ -1181,7 +1184,7 @@ TextResponse objects :class:`Response` class, which is meant to be used only for binary data, such as images, sounds or any media file. - :class:`TextResponse` objects support a new ``__init__`` method argument, in + :class:`TextResponse` objects support a new ``__init__()`` method argument, in addition to the base :class:`Response` objects. The remaining functionality is the same as for the :class:`Response` class and is not documented here. @@ -1219,7 +1222,7 @@ TextResponse objects A string with the encoding of this response. The encoding is resolved by trying the following mechanisms, in order: - 1. the encoding passed in the ``__init__`` method ``encoding`` argument + 1. the encoding passed in the ``__init__()`` method ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this encoding is not valid (i.e. unknown), it is ignored and the next @@ -1273,7 +1276,7 @@ TextResponse objects Constructs an absolute url by combining the Response's base url with a possible relative url. The base url shall be extracted from the - ``<base>`` tag, or just the Response's :attr:`url` if there is no such + ``<base>`` tag, or just :attr:`Response.url` if there is no such tag. diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 202b0823a..b95e6eab3 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -777,7 +777,7 @@ Removing namespaces When dealing with scraping projects, it is often quite convenient to get rid of namespaces altogether and just work with element names, to write more simple/convenient XPaths. You can use the -:meth:`Selector.remove_namespaces` method for that. +:meth:`.Selector.remove_namespaces` method for that. Let's show an example that illustrates this with the Python Insider blog atom feed. @@ -814,7 +814,7 @@ doesn't work (because the Atom XML namespace is obfuscating those nodes): >>> response.xpath("//link") [] -But once we call the :meth:`Selector.remove_namespaces` method, all +But once we call the :meth:`.Selector.remove_namespaces` method, all nodes can be accessed directly by their names: .. code-block:: pycon @@ -1046,7 +1046,7 @@ Built-in Selectors reference Selector objects ---------------- -.. autoclass:: Selector +.. autoclass:: scrapy.Selector .. automethod:: xpath @@ -1126,8 +1126,8 @@ Examples Selector examples on HTML response ---------------------------------- -Here are some :class:`Selector` examples to illustrate several concepts. -In all cases, we assume there is already a :class:`Selector` instantiated with +Here are some :class:`~scrapy.Selector` examples to illustrate several concepts. +In all cases, we assume there is already a :class:`~scrapy.Selector` instantiated with a :class:`~scrapy.http.HtmlResponse` object like this: .. code-block:: python @@ -1135,7 +1135,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this: sel = Selector(html_response) 1. Select all ``<h1>`` elements from an HTML response body, returning a list of - :class:`Selector` objects (i.e. a :class:`SelectorList` object): + :class:`~scrapy.Selector` objects (i.e. a :class:`SelectorList` object): .. code-block:: python @@ -1165,7 +1165,7 @@ Selector examples on XML response .. skip: start -Here are some examples to illustrate concepts for :class:`Selector` objects +Here are some examples to illustrate concepts for :class:`~scrapy.Selector` objects instantiated with an :class:`~scrapy.http.XmlResponse` object: .. code-block:: python @@ -1173,7 +1173,7 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object: sel = Selector(xml_response) 1. Select all ``<product>`` elements from an XML response body, returning a list - of :class:`Selector` objects (i.e. a :class:`SelectorList` object): + of :class:`~scrapy.Selector` objects (i.e. a :class:`SelectorList` object): .. code-block:: python diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index fcc94879a..4bb143dfd 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -115,7 +115,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): @abstractmethod def next_request(self) -> Request | None: """ - Return the next :class:`~scrapy.http.Request` to be processed, or ``None`` + Return the next :class:`~scrapy.Request` to be processed, or ``None`` to indicate that there are no requests to be considered ready at the moment. Returning ``None`` implies that no request from the scheduler will be sent @@ -263,7 +263,7 @@ class Scheduler(BaseScheduler): def next_request(self) -> Request | None: """ - Return a :class:`~scrapy.http.Request` object from the memory queue, + Return a :class:`~scrapy.Request` object from the memory queue, falling back to the disk queue if the memory queue is empty. Return ``None`` if there are no more enqueued requests. diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 3d6cf4816..4eee5ffbb 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -59,7 +59,7 @@ RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: """When assigned to the ``callback`` parameter of - :class:`~scrapy.http.Request`, it indicates that the request is not meant + :class:`~scrapy.Request`, it indicates that the request is not meant to have a spider callback at all. For example: @@ -83,7 +83,7 @@ def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: class Request(object_ref): """Represents an HTTP request, which is usually generated in a Spider and - executed by the Downloader, thus generating a :class:`Response`. + executed by the Downloader, thus generating a :class:`~scrapy.http.Response`. """ attributes: tuple[str, ...] = ( @@ -103,9 +103,9 @@ class Request(object_ref): ) """A tuple of :class:`str` objects containing the name of all public attributes of the class that are also keyword parameters of the - ``__init__`` method. + ``__init__()`` method. - Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and + Currently used by :meth:`.Request.replace`, :meth:`.Request.to_dict` and :func:`~scrapy.utils.request.request_from_dict`. """ @@ -233,7 +233,7 @@ class Request(object_ref): finding unknown options call this method by passing ``ignore_unknown_options=False``. - .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request` + .. caution:: Using :meth:`from_curl` from :class:`~scrapy.Request` subclasses, such as :class:`~scrapy.http.JsonRequest`, or :class:`~scrapy.http.XmlRpcRequest`, as well as having :ref:`downloader middlewares <topics-downloader-middleware>` @@ -244,7 +244,7 @@ class Request(object_ref): :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`, or :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`, - may modify the :class:`~scrapy.http.Request` object. + may modify the :class:`~scrapy.Request` object. To translate a cURL command into a Scrapy request, you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_. diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 387805f57..b84110b29 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -51,7 +51,7 @@ class Response(object_ref): ) """A tuple of :class:`str` objects containing the name of all public attributes of the class that are also keyword parameters of the - ``__init__`` method. + ``__init__()`` method. Currently used by :meth:`Response.replace`. """ @@ -199,8 +199,8 @@ class Response(object_ref): ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. - It accepts the same arguments as ``Request.__init__`` method, - but ``url`` can be a relative URL or a ``scrapy.link.Link`` object, + It accepts the same arguments as ``Request.__init__()`` method, + but ``url`` can be a relative URL or a :class:`~scrapy.link.Link` object, not only an absolute URL. :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` @@ -254,7 +254,7 @@ class Response(object_ref): .. versionadded:: 2.0 Return an iterable of :class:`~.Request` instances to follow all links - in ``urls``. It accepts the same arguments as ``Request.__init__`` method, + in ``urls``. It accepts the same arguments as ``Request.__init__()`` method, but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects, not only absolute URLs. diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 476f1754e..081223882 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -185,15 +185,15 @@ class TextResponse(Response): ) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. - It accepts the same arguments as ``Request.__init__`` method, + It accepts the same arguments as ``Request.__init__()`` method, but ``url`` can be not only an absolute URL, but also * a relative URL * a :class:`~scrapy.link.Link` object, e.g. the result of :ref:`topics-link-extractors` - * a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g. + * a :class:`~scrapy.Selector` object for a ``<link>`` or ``<a>`` element, e.g. ``response.css('a.my_link')[0]`` - * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g. + * an attribute :class:`~scrapy.Selector` (not SelectorList), e.g. ``response.css('a::attr(href)')[0]`` or ``response.xpath('//img/@src')[0]`` @@ -241,20 +241,20 @@ class TextResponse(Response): """ A generator that produces :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s - ``__init__`` method, except that each ``urls`` element does not need to be + ``__init__()`` method, except that each ``urls`` element does not need to be an absolute URL, it can be any of the following: * a relative URL * a :class:`~scrapy.link.Link` object, e.g. the result of :ref:`topics-link-extractors` - * a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g. + * a :class:`~scrapy.Selector` object for a ``<link>`` or ``<a>`` element, e.g. ``response.css('a.my_link')[0]`` - * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g. + * an attribute :class:`~scrapy.Selector` (not SelectorList), e.g. ``response.css('a::attr(href)')[0]`` or ``response.xpath('//img/@src')[0]`` In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction - within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted). + within the ``follow_all()`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted). Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or using the ``css`` or ``xpath`` parameters, this method will not produce requests for diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index d35720a45..2f5c0343b 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -32,7 +32,7 @@ class ItemLoader(itemloaders.ItemLoader): :param selector: The selector to extract data from, when using the :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or :meth:`replace_css` method. - :type selector: :class:`~scrapy.selector.Selector` object + :type selector: :class:`~scrapy.Selector` object :param response: The response used to construct the selector using the :attr:`default_selector_class`, unless the selector argument is given, @@ -79,7 +79,7 @@ class ItemLoader(itemloaders.ItemLoader): .. attribute:: selector - The :class:`~scrapy.selector.Selector` object to extract data from. + The :class:`~scrapy.Selector` object to extract data from. It's either the selector given in the ``__init__`` method or one created from the response given in the ``__init__`` method using the :attr:`default_selector_class`. This attribute is meant to be diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 7f2b178f5..9c1161968 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -1,6 +1,6 @@ """ This module provides some useful functions for working with -scrapy.http.Request objects +scrapy.Request objects """ from __future__ import annotations @@ -109,12 +109,10 @@ class RequestFingerprinter: It takes into account a canonical version (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url - <scrapy.http.Request.url>` and the values of :attr:`request.method - <scrapy.http.Request.method>` and :attr:`request.body - <scrapy.http.Request.body>`. It then generates an `SHA1 + <scrapy.Request.url>` and the values of :attr:`request.method + <scrapy.Request.method>` and :attr:`request.body + <scrapy.Request.body>`. It then generates an `SHA1 <https://en.wikipedia.org/wiki/SHA-1>`_ hash. - - .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`. """ @classmethod From 7dfbecd3924a0d7a9e555e9cc3618cdb06b5415d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 7 Jan 2025 19:11:10 +0500 Subject: [PATCH 1625/2083] Fix tracking of coverage in subprocesses. --- pyproject.toml | 6 ++++++ tests/__init__.py | 7 ------- tox.ini | 10 +++++----- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 571a61f1c..29e26399f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,6 +120,12 @@ include = ["scrapy/*"] omit = ["tests/*"] disable_warnings = ["include-ignored"] +[tool.coverage.paths] +source = [ + "scrapy", + ".tox/**/site-packages/scrapy" +] + [tool.coverage.report] # https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"] diff --git a/tests/__init__.py b/tests/__init__.py index 5f0c0f7ad..cd52ade58 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -15,13 +15,6 @@ os.environ["http_proxy"] = "" os.environ["https_proxy"] = "" os.environ["ftp_proxy"] = "" -# Absolutize paths to coverage config and output file because tests that -# spawn subprocesses also changes current working directory. -_sourceroot = Path(__file__).resolve().parent.parent -if "COV_CORE_CONFIG" in os.environ: - os.environ["COVERAGE_FILE"] = str(_sourceroot / ".coverage") - os.environ["COV_CORE_CONFIG"] = str(_sourceroot / os.environ["COV_CORE_CONFIG"]) - tests_datadir = str(Path(__file__).parent.resolve() / "sample_data") diff --git a/tox.ini b/tox.ini index 39ab1ccd4..de91c8b04 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ deps = pyftpdlib >= 2.0.1 pygments pytest - pytest-cov==4.0.0 + pytest-cov >= 4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures @@ -36,7 +36,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules + pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} @@ -115,7 +115,7 @@ setenv = install_command = python -I -m pip install {opts} {packages} commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} + pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] basepython = {[pinned]basepython} @@ -241,7 +241,7 @@ deps = {[testenv]deps} botocore>=1.4.87 commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} + pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} [testenv:botocore-pinned] basepython = {[pinned]basepython} @@ -252,4 +252,4 @@ install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} + pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} From 3154b08e90d9777dfe2879b8686b6fc63a793c84 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 7 Jan 2025 19:40:25 +0500 Subject: [PATCH 1626/2083] Improve coverage speed on Python 3.12+. --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index de91c8b04..cf5e19a61 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ minversion = 1.7.0 [test-requirements] deps = attrs + coverage >= 7.4.0 pexpect >= 4.8.0 pyftpdlib >= 2.0.1 pygments @@ -26,6 +27,8 @@ deps = # mitmproxy does not support PyPy mitmproxy; implementation_name != 'pypy' +setenv = + COVERAGE_CORE=sysmon passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From 1fc91bb46262118c9ff7aa2b4719d880f727699f Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal <kevinoxy@gmail.com> Date: Thu, 9 Jan 2025 03:28:51 +1100 Subject: [PATCH 1627/2083] new `allow_offsite` parameter in OffsiteMiddleware (#6151) * new 'allow_offsite' parameter in OffsiteMiddleware * document deprecated dont_filter flag in OffsiteMiddleware * avoid deprecating dont_filter in OffsiteMiddleware * Copy the code to the downloader mw. * Add tests for allow_offsite in the downloader mw. * Mark allow_offsite with reqmeta. --------- Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name> --- docs/topics/downloader-middleware.rst | 9 ++++--- docs/topics/request-response.rst | 7 ++--- scrapy/downloadermiddlewares/offsite.py | 6 ++++- scrapy/spidermiddlewares/offsite.py | 6 ++++- tests/test_downloadermiddleware_offsite.py | 31 ++++++++++++++++++++++ tests/test_spidermiddleware_offsite.py | 1 + 6 files changed, 52 insertions(+), 8 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index af7885a45..1ab8f588f 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -797,9 +797,12 @@ OffsiteMiddleware :attr:`~scrapy.Spider.allowed_domains` attribute, or the attribute is empty, the offsite middleware will allow all requests. - If the request has the :attr:`~scrapy.Request.dont_filter` attribute - set, the offsite middleware will allow the request even if its domain is not - listed in allowed domains. + .. reqmeta:: allow_offsite + + If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to + ``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then + the OffsiteMiddleware will allow the request even if its domain is not listed + in allowed domains. RedirectMiddleware ------------------ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1bb1a10a4..b187f3aaf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -145,9 +145,9 @@ Request objects :type priority: int :param dont_filter: indicates that this request should not be filtered by - the scheduler. This is used when you want to perform an identical - request multiple times, to ignore the duplicates filter. Use it with - care, or you will get into crawling loops. Default to ``False``. + the scheduler or some middlewares. This is used when you want to perform + an identical request multiple times, to ignore the duplicates filter. + Use it with care, or you will get into crawling loops. Default to ``False``. :type dont_filter: bool :param errback: a function that will be called if any exception was @@ -660,6 +660,7 @@ are some special keys recognized by Scrapy and its built-in extensions. Those are: +* :reqmeta:`allow_offsite` * :reqmeta:`autothrottle_dont_adjust_delay` * :reqmeta:`bindaddress` * :reqmeta:`cookiejar` diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index a69f531a7..a2cff65e7 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -40,7 +40,11 @@ class OffsiteMiddleware: self.process_request(request, spider) def process_request(self, request: Request, spider: Spider) -> None: - if request.dont_filter or self.should_follow(request, spider): + if ( + request.dont_filter + or request.meta.get("allow_offsite") + or self.should_follow(request, spider) + ): return domain = urlparse_cached(request).hostname if domain and domain not in self.domains_seen: diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index d3ed64ef5..95e753830 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -61,7 +61,11 @@ class OffsiteMiddleware: def _filter(self, request: Any, spider: Spider) -> bool: if not isinstance(request, Request): return True - if request.dont_filter or self.should_follow(request, spider): + if ( + request.dont_filter + or request.meta.get("allow_offsite") + or self.should_follow(request, spider) + ): return True domain = urlparse_cached(request).hostname if domain and domain not in self.domains_seen: diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index fec56a39f..23a1d06da 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -64,6 +64,37 @@ def test_process_request_dont_filter(value, filtered): assert mw.process_request(request, spider) is None +@pytest.mark.parametrize( + ("allow_offsite", "dont_filter", "filtered"), + ( + (True, UNSET, False), + (True, None, False), + (True, False, False), + (True, True, False), + (False, UNSET, True), + (False, None, True), + (False, False, True), + (False, True, False), + ), +) +def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): + crawler = get_crawler(Spider) + spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + kwargs = {"meta": {}} + if allow_offsite is not UNSET: + kwargs["meta"]["allow_offsite"] = allow_offsite + if dont_filter is not UNSET: + kwargs["dont_filter"] = dont_filter + request = Request("https://b.example", **kwargs) + if filtered: + with pytest.raises(IgnoreRequest): + mw.process_request(request, spider) + else: + assert mw.process_request(request, spider) is None + + @pytest.mark.parametrize( "value", ( diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 837f1c2c8..906928e01 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -29,6 +29,7 @@ class TestOffsiteMiddleware(TestCase): Request("http://scrapy.org/1"), Request("http://sub.scrapy.org/1"), Request("http://offsite.tld/letmepass", dont_filter=True), + Request("http://offsite-2.tld/allow", meta={"allow_offsite": True}), Request("http://scrapy.test.org/"), Request("http://scrapy.test.org:8000/"), ] From 402500b164efc01257679247d3dd1628a5f90f5e Mon Sep 17 00:00:00 2001 From: Ionut-Cezar Ciubotariu <ionut.cezar.ciubotariu@gmail.com> Date: Fri, 10 Jan 2025 20:08:27 +0200 Subject: [PATCH 1628/2083] Change unknown cmd message when outside project (#3426) * Change unknown cmd message when outside project * Simplification. * Move the import to the top level. * Reword the message. --------- Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name> --- scrapy/cmdline.py | 22 +++++++++++++++++++++- tests/test_commands.py | 22 +++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 48f462c65..065adccfb 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -89,6 +89,12 @@ def _get_commands_dict( return cmds +def _get_project_only_cmds(settings: BaseSettings) -> set[str]: + return set(_get_commands_dict(settings, inproject=True)) - set( + _get_commands_dict(settings, inproject=False) + ) + + def _pop_command_name(argv: list[str]) -> str | None: for i, arg in enumerate(argv[1:]): if not arg.startswith("-"): @@ -121,11 +127,25 @@ def _print_commands(settings: BaseSettings, inproject: bool) -> None: print('Use "scrapy <command> -h" to see more info about a command') +def _print_unknown_command_msg( + settings: BaseSettings, cmdname: str, inproject: bool +) -> None: + proj_only_cmds = _get_project_only_cmds(settings) + if cmdname in proj_only_cmds and not inproject: + cmd_list = ", ".join(sorted(proj_only_cmds)) + print( + f"The {cmdname} command is not available from this location.\n" + f"These commands are only available from within a project: {cmd_list}.\n" + ) + else: + print(f"Unknown command: {cmdname}\n") + + def _print_unknown_command( settings: BaseSettings, cmdname: str, inproject: bool ) -> None: _print_header(settings, inproject) - print(f"Unknown command: {cmdname}\n") + _print_unknown_command_msg(settings, cmdname, inproject) print('Use "scrapy" to see available commands') diff --git a/tests/test_commands.py b/tests/test_commands.py index 9d5720b98..1aae3222e 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -9,6 +9,7 @@ import re import subprocess import sys from contextlib import contextmanager +from io import StringIO from itertools import chain from pathlib import Path from shutil import copytree, rmtree @@ -16,12 +17,13 @@ from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import TemporaryFile, mkdtemp from threading import Timer from typing import TYPE_CHECKING -from unittest import skipIf +from unittest import mock, skipIf from pytest import mark from twisted.trial import unittest import scrapy +from scrapy.cmdline import _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings @@ -652,6 +654,24 @@ class MiscCommandsTest(CommandTest): def test_list(self): self.assertEqual(0, self.call("list")) + def test_command_not_found(self): + na_msg = """ +The list command is not available from this location. +These commands are only available from within a project: check, crawl, edit, list, parse. +""" + not_found_msg = """ +Unknown command: abc +""" + params = [ + ("list", 0, na_msg), + ("abc", 0, not_found_msg), + ("abc", 1, not_found_msg), + ] + for cmdname, inproject, message in params: + with mock.patch("sys.stdout", new=StringIO()) as out: + _print_unknown_command_msg(Settings(), cmdname, inproject) + self.assertEqual(out.getvalue().strip(), message.strip()) + class RunSpiderCommandTest(CommandTest): spider_filename = "myspider.py" From 98ba61256deceba7b04b938a97005258f4ef5c66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 14 Jan 2025 15:36:56 +0100 Subject: [PATCH 1629/2083] Deprecate BaseDupeFilter.log() and improve dupefilter docs (#4151) * Remove BaseDupeFilter.log() It is never called because request_seen() always returns False * Document the interface of DUPEFILTER_CLASS classes * Remove unnecessary BaseDupeFilter comments and add a short class description * Improve the documentation related to the DUPEFILTER_CLASS setting * Deprecate BaseDupeFilter.log * Update the docs * Fix the new code example * Remove typing to keep the example short Otherwise, it would have required yet another import line (from __future__ or typing). --------- Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name> --- docs/conf.py | 4 ++ docs/topics/settings.rst | 78 +++++++++++++++++++++++++++++++++++---- scrapy/dupefilters.py | 16 +++++++- tests/test_dupefilters.py | 19 +++++++++- 4 files changed, 108 insertions(+), 9 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index fd8165db3..8196b6934 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -258,6 +258,10 @@ coverage_ignore_pyobjects = [ # Base classes of downloader middlewares are implementation details that # are not meant for users. r"^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware", + # The interface methods of duplicate request filtering classes are already + # covered in the interface documentation part of the DUPEFILTER_CLASS + # setting documentation. + r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_settings|request_seen|open|close|log)$", # Private exception used by the command-line interface implementation. r"^scrapy\.exceptions\.UsageError", # Methods of BaseItemExporter subclasses are only documented in diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 76904a26e..06974f336 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -955,15 +955,79 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'`` The class used to detect and filter duplicate requests. -The default (``RFPDupeFilter``) filters based on the +The default, :class:`~scrapy.dupefilters.RFPDupeFilter`, filters based on the :setting:`REQUEST_FINGERPRINTER_CLASS` setting. -You can disable filtering of duplicate requests by setting -:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``. -Be very careful about this however, because you can get into crawling loops. -It's usually a better idea to set the ``dont_filter`` parameter to -``True`` on the specific :class:`~scrapy.Request` that should not be -filtered. +To change how duplicates are checked, you can point :setting:`DUPEFILTER_CLASS` +to a custom subclass of :class:`~scrapy.dupefilters.RFPDupeFilter` that +overrides its ``__init__`` method to use a :ref:`different request +fingerprinting class <custom-request-fingerprinter>`. For example: + +.. code-block:: python + + from scrapy.dupefilters import RFPDupeFilter + from scrapy.utils.request import fingerprint + + + class CustomRequestFingerprinter: + def fingerprint(self, request): + return fingerprint(request, include_headers=["X-ID"]) + + + class CustomDupeFilter(RFPDupeFilter): + + def __init__(self, path=None, debug=False, *, fingerprinter=None): + super().__init__( + path=path, debug=debug, fingerprinter=CustomRequestFingerprinter() + ) + +To disable duplicate request filtering set :setting:`DUPEFILTER_CLASS` to +``'scrapy.dupefilters.BaseDupeFilter'``. Note that not filtering out duplicate +requests may cause crawling loops. It is usually better to set +the ``dont_filter`` parameter to ``True`` on the ``__init__`` method of a +specific :class:`~scrapy.Request` object that should not be filtered out. + +A class assigned to :setting:`DUPEFILTER_CLASS` must implement the following +interface:: + + class MyDupeFilter: + + @classmethod + def from_settings(cls, settings): + """Returns an instance of this duplicate request filtering class + based on the current crawl settings.""" + return cls() + + def request_seen(self, request): + """Returns ``True`` if *request* is a duplicate of another request + seen in a previous call to :meth:`request_seen`, or ``False`` + otherwise.""" + return False + + def open(self): + """Called before the spider opens. It may return a deferred.""" + pass + + def close(self, reason): + """Called before the spider closes. It may return a deferred.""" + pass + + def log(self, request, spider): + """Logs that a request has been filtered out. + + It is called right after a call to :meth:`request_seen` that + returns ``True``. + + If :meth:`request_seen` always returns ``False``, such as in the + case of :class:`~scrapy.dupefilters.BaseDupeFilter`, this method + may be omitted. + """ + pass + +.. autoclass:: scrapy.dupefilters.BaseDupeFilter + +.. autoclass:: scrapy.dupefilters.RFPDupeFilter + .. setting:: DUPEFILTER_DEBUG diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index caf69daf4..a3e2c5eb4 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -4,6 +4,7 @@ import logging import warnings from pathlib import Path from typing import TYPE_CHECKING +from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.job import job_dir @@ -26,6 +27,9 @@ if TYPE_CHECKING: class BaseDupeFilter: + """Dummy duplicate request filtering class (:setting:`DUPEFILTER_CLASS`) + that does not filter out any request.""" + @classmethod def from_settings(cls, settings: BaseSettings) -> Self: warnings.warn( @@ -50,10 +54,19 @@ class BaseDupeFilter: def log(self, request: Request, spider: Spider) -> None: """Log that a request has been filtered""" + warn( + "Calling BaseDupeFilter.log() is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) class RFPDupeFilter(BaseDupeFilter): - """Request Fingerprint duplicates filter""" + """Duplicate request filtering class (:setting:`DUPEFILTER_CLASS`) that + filters out requests with the canonical + (:func:`w3lib.url.canonicalize_url`) :attr:`~scrapy.http.Request.url`, + :attr:`~scrapy.http.Request.method` and :attr:`~scrapy.http.Request.body`. + """ def __init__( self, @@ -117,6 +130,7 @@ class RFPDupeFilter(BaseDupeFilter): return False def request_fingerprint(self, request: Request) -> str: + """Returns a string that uniquely identifies the specified request.""" return self.fingerprinter.fingerprint(request).hex() def close(self, reason: str) -> None: diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 4fd648f48..703c23529 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -4,11 +4,13 @@ import sys import tempfile import unittest from pathlib import Path +from warnings import catch_warnings from testfixtures import LogCapture from scrapy.core.scheduler import Scheduler -from scrapy.dupefilters import RFPDupeFilter +from scrapy.dupefilters import BaseDupeFilter, RFPDupeFilter +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler @@ -252,3 +254,18 @@ class RFPDupeFilterTest(unittest.TestCase): ) dupefilter.close("finished") + + +class BaseDupeFilterTestCase(unittest.TestCase): + def test_log_deprecation(self): + dupefilter = _get_dupefilter( + settings={"DUPEFILTER_CLASS": BaseDupeFilter}, + ) + with catch_warnings(record=True) as warning_list: + dupefilter.log(None, None) + self.assertEqual(len(warning_list), 1) + self.assertEqual( + str(warning_list[0].message), + "Calling BaseDupeFilter.log() is deprecated.", + ) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) From 1c1e83895c15dc491c6c133982cde22d778dcae6 Mon Sep 17 00:00:00 2001 From: anubhav <protokoul@users.noreply.github.com> Date: Tue, 14 Jan 2025 21:10:24 +0530 Subject: [PATCH 1630/2083] Fix _pop_command_name (#6606) --- scrapy/cmdline.py | 7 +++---- tests/test_commands.py | 28 +++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 065adccfb..b08fd3409 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -96,10 +96,9 @@ def _get_project_only_cmds(settings: BaseSettings) -> set[str]: def _pop_command_name(argv: list[str]) -> str | None: - for i, arg in enumerate(argv[1:]): - if not arg.startswith("-"): - del argv[i] - return arg + for i in range(1, len(argv)): + if not argv[i].startswith("-"): + return argv.pop(i) return None diff --git a/tests/test_commands.py b/tests/test_commands.py index 1aae3222e..50f093043 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -23,7 +23,7 @@ from pytest import mark from twisted.trial import unittest import scrapy -from scrapy.cmdline import _print_unknown_command_msg +from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings @@ -1163,3 +1163,29 @@ class HelpMessageTest(CommandTest): for command in self.commands: _, out, _ = self.proc(command, "-h") self.assertIn("Usage", out) + + +class PopCommandNameTest(unittest.TestCase): + def test_valid_command(self): + argv = ["scrapy", "crawl", "my_spider"] + command = _pop_command_name(argv) + self.assertEqual(command, "crawl") + self.assertEqual(argv, ["scrapy", "my_spider"]) + + def test_no_command(self): + argv = ["scrapy"] + command = _pop_command_name(argv) + self.assertIsNone(command) + self.assertEqual(argv, ["scrapy"]) + + def test_option_before_command(self): + argv = ["scrapy", "-h", "crawl"] + command = _pop_command_name(argv) + self.assertEqual(command, "crawl") + self.assertEqual(argv, ["scrapy", "-h"]) + + def test_option_after_command(self): + argv = ["scrapy", "crawl", "-h"] + command = _pop_command_name(argv) + self.assertEqual(command, "crawl") + self.assertEqual(argv, ["scrapy", "-h"]) From ca345a3b73904ffd6d2e8ffb17c45ebb69639d26 Mon Sep 17 00:00:00 2001 From: anubhav <protokoul@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:38:18 +0530 Subject: [PATCH 1631/2083] Flexible severity of logging level when items are dropped (#6608) --- docs/topics/settings.rst | 32 +++++++++++++++ scrapy/exceptions.py | 6 +++ scrapy/logformatter.py | 6 ++- scrapy/settings/default_settings.py | 2 + tests/test_logformatter.py | 60 +++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 06974f336..8801434d8 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -418,6 +418,38 @@ This setting also affects :setting:`DOWNLOAD_DELAY` and :ref:`topics-autothrottle`: if :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, download delay is enforced per IP, not per domain. +.. setting:: DEFAULT_DROPITEM_LOG_LEVEL + +DEFAULT_DROPITEM_LOG_LEVEL +-------------------------- + +Default: ``"WARNING"`` + +Default :ref:`log level <levels>` of messages about dropped items. + +When an item is dropped by raising :exc:`scrapy.exceptions.DropItem` from the +:func:`process_item` method of an :ref:`item pipeline <topics-item-pipeline>`, +a message is logged, and by default its log level is the one configured in this +setting. + +You may specify this log level as an integer (e.g. ``20``), as a log level +constant (e.g. ``logging.INFO``) or as a string with the name of a log level +constant (e.g. ``"INFO"``). + +When writing an item pipeline, you can force a different log level by setting +:attr:`scrapy.exceptions.DropItem.log_level` in your +:exc:`scrapy.exceptions.DropItem` exception. For example: + +.. code-block:: python + + from scrapy.exceptions import DropItem + + + class MyPipeline: + def process_item(self, item, spider): + if not item.get("price"): + raise DropItem("Missing price data", log_level="INFO") + return item .. setting:: DEFAULT_ITEM_CLASS diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 96566ba86..f37f881a7 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -5,6 +5,8 @@ These exceptions are documented in docs/topics/exceptions.rst. Please don't add new exceptions here without documenting them there. """ +from __future__ import annotations + from typing import Any # Internal @@ -58,6 +60,10 @@ class StopDownload(Exception): class DropItem(Exception): """Drop item from the item pipeline""" + def __init__(self, message: str, log_level: str | None = None): + super().__init__(message) + self.log_level = log_level + class NotSupported(Exception): """Indicates a feature or method is not supported""" diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 76f9c7856..f10e91beb 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -120,8 +120,12 @@ class LogFormatter: spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" + if (level := getattr(exception, "log_level", None)) is None: + level = spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] + if isinstance(level, str): + level = getattr(logging, level) return { - "level": logging.WARNING, + "level": level, "msg": DROPPEDMSG, "args": { "exception": exception, diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 0bbde118e..7ef365f68 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -49,6 +49,8 @@ CONCURRENT_REQUESTS_PER_IP = 0 COOKIES_ENABLED = True COOKIES_DEBUG = False +DEFAULT_DROPITEM_LOG_LEVEL = "WARNING" + DEFAULT_ITEM_CLASS = "scrapy.item.Item" DEFAULT_REQUEST_HEADERS = { diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 61a9f3f8d..e5d077858 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -1,5 +1,7 @@ +import logging import unittest +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.python.failure import Failure @@ -26,6 +28,7 @@ class LogFormatterTestCase(unittest.TestCase): def setUp(self): self.formatter = LogFormatter() self.spider = Spider("default") + self.spider.crawler = get_crawler() def test_crawled_with_referer(self): req = Request("http://www.example.com") @@ -68,6 +71,62 @@ class LogFormatterTestCase(unittest.TestCase): assert all(isinstance(x, str) for x in lines) self.assertEqual(lines, ["Dropped: \u2018", "{}"]) + def test_dropitem_default_log_level(self): + item = {} + exception = DropItem("Test drop") + response = Response("http://www.example.com") + spider = Spider("foo") + spider.crawler = get_crawler(Spider) + + logkws = self.formatter.dropped(item, exception, response, spider) + self.assertEqual(logkws["level"], logging.WARNING) + + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = logging.INFO + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + self.assertEqual(logkws["level"], logging.INFO) + + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = "INFO" + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + self.assertEqual(logkws["level"], logging.INFO) + + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 10 + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + self.assertEqual(logkws["level"], logging.DEBUG) + + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 0 + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + self.assertEqual(logkws["level"], logging.NOTSET) + + unsupported_value = object() + spider.crawler.settings.frozen = False + spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = unsupported_value + spider.crawler.settings.frozen = True + logkws = self.formatter.dropped(item, exception, response, spider) + self.assertEqual(logkws["level"], unsupported_value) + + with pytest.raises(TypeError): + logging.log(logkws["level"], "message") + + def test_dropitem_custom_log_level(self): + item = {} + response = Response("http://www.example.com") + + exception = DropItem("Test drop", log_level="INFO") + logkws = self.formatter.dropped(item, exception, response, self.spider) + self.assertEqual(logkws["level"], logging.INFO) + + exception = DropItem("Test drop", log_level="ERROR") + logkws = self.formatter.dropped(item, exception, response, self.spider) + self.assertEqual(logkws["level"], logging.ERROR) + def test_item_error(self): # In practice, the complete traceback is shown by passing the # 'exc_info' argument to the logging function @@ -145,6 +204,7 @@ class LogformatterSubclassTest(LogFormatterTestCase): def setUp(self): self.formatter = LogFormatterSubclass() self.spider = Spider("default") + self.spider.crawler = get_crawler(Spider) def test_crawled_with_referer(self): req = Request("http://www.example.com") From d7168577b859d15a15c2bcbc6c4f607a62be7478 Mon Sep 17 00:00:00 2001 From: Rotzbua <Rotzbua@users.noreply.github.com> Date: Sun, 19 Jan 2025 13:50:53 +0100 Subject: [PATCH 1632/2083] chore(docs): migrate to RTD template v3 notable change: Drop support for all versions of Internet Explorer. --- docs/conf.py | 7 ------- docs/requirements.txt | 6 +++--- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 8196b6934..a3475a323 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -117,13 +117,6 @@ html_theme = "sphinx_rtd_theme" # documentation. # html_theme_options = {} -# Add any paths that contain custom themes here, relative to this directory. -# Add path to the RTD explicitly to robustify builds (otherwise might -# fail in a clean Debian build env) -import sphinx_rtd_theme - -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # The style sheet to use for HTML and HTML Help pages. A file of that name # must exist either in Sphinx' static/ path, or in one of the custom paths # given in html_static_path. diff --git a/docs/requirements.txt b/docs/requirements.txt index 5f683d34c..7ee897170 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ sphinx==6.2.1 -sphinx-hoverxref==1.3.0 -sphinx-notfound-page==1.0.0 -sphinx-rtd-theme==2.0.0 +sphinx-hoverxref==1.4.2 +sphinx-notfound-page==1.0.4 +sphinx-rtd-theme==3.0.2 From ee4f527f47111c18ddef0c2369c1dede7447a4e9 Mon Sep 17 00:00:00 2001 From: Rotzbua <Rotzbua@users.noreply.github.com> Date: Sun, 19 Jan 2025 14:58:02 +0100 Subject: [PATCH 1633/2083] fix(docs): pillow domain is shut down permanently See https://github.com/python-pillow/Pillow/issues/8585 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 924abb7a1..8230c3aef 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6862,7 +6862,7 @@ First release of Scrapy. .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr .. _PEP 257: https://peps.python.org/pep-0257/ -.. _Pillow: https://python-pillow.org/ +.. _Pillow: https://github.com/python-pillow/Pillow .. _pyOpenSSL: https://www.pyopenssl.org/en/stable/ .. _queuelib: https://github.com/scrapy/queuelib .. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml From e0c828b7f665d8c82e17787996634bc072e416ae Mon Sep 17 00:00:00 2001 From: Rotzbua <Rotzbua@users.noreply.github.com> Date: Mon, 20 Jan 2025 12:18:30 +0100 Subject: [PATCH 1634/2083] chore(docs): refactor config (#6623) --- .readthedocs.yml | 2 +- docs/Makefile | 104 +++-------------- docs/_ext/scrapyfixautodoc.py | 18 +++ docs/conf.py | 211 ++++++---------------------------- 4 files changed, 65 insertions(+), 270 deletions(-) create mode 100644 docs/_ext/scrapyfixautodoc.py diff --git a/.readthedocs.yml b/.readthedocs.yml index 5ec6eafbb..23e4cabea 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,7 +5,7 @@ sphinx: fail_on_warning: true build: - os: ubuntu-20.04 + os: ubuntu-24.04 tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python diff --git a/docs/Makefile b/docs/Makefile index 48401bac8..ed8809902 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,96 +1,20 @@ -# -# Makefile for Scrapy documentation [based on Python documentation Makefile] -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Minimal makefile for Sphinx documentation # -# You can set these variables from the command line. -PYTHON = python -SPHINXOPTS = -PAPER = -SOURCES = -SHELL = /usr/bin/env bash - -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ - -D latex_elements.papersize=$(PAPER) \ - $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) - -.PHONY: help update build html htmlhelp clean +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = build +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make <target>' where <target> is one of" - @echo " html to make standalone HTML files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " text to make plain text files" - @echo " changes to make an overview over all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " watch build HTML docs, open in browser and watch for changes" + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -build-dirs: - mkdir -p build/$(BUILDER) build/doctrees +.PHONY: help Makefile -build: build-dirs - sphinx-build $(ALLSPHINXOPTS) - @echo - -build-ignore-errors: build-dirs - -sphinx-build $(ALLSPHINXOPTS) - @echo - - -html: BUILDER = html -html: build - @echo "Build finished. The HTML pages are in build/html." - -htmlhelp: BUILDER = htmlhelp -htmlhelp: build - @echo "Build finished; now you can run HTML Help Workshop with the" \ - "build/htmlhelp/pydoc.hhp project file." - -latex: BUILDER = latex -latex: build - @echo "Build finished; the LaTeX files are in build/latex." - @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ - "run these through (pdf)latex." - -text: BUILDER = text -text: build - @echo "Build finished; the text files are in build/text." - -changes: BUILDER = changes -changes: build - @echo "The overview file is in build/changes." - -linkcheck: BUILDER = linkcheck -linkcheck: build - @echo "Link check complete; look for any errors in the above output " \ - "or in build/$(BUILDER)/output.txt" - -linkfix: BUILDER = linkcheck -linkfix: build-ignore-errors - $(PYTHON) utils/linkfix.py - @echo "Fixing redirecting links in docs has finished; check all " \ - "replacements before committing them" - -doctest: BUILDER = doctest -doctest: build - @echo "Testing of doctests in the sources finished, look at the " \ - "results in build/doctest/output.txt" - -pydoc-topics: BUILDER = pydoc-topics -pydoc-topics: build - @echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \ - "into the Lib/ directory" - -coverage: BUILDER = coverage -coverage: build - -htmlview: html - $(PYTHON) -c "import webbrowser; from pathlib import Path; \ - webbrowser.open(Path('build/html/index.html').resolve().as_uri())" - -clean: - -rm -rf build/* - -watch: htmlview - watchmedo shell-command -p '*.rst' -c 'make html' -R -D +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_ext/scrapyfixautodoc.py b/docs/_ext/scrapyfixautodoc.py new file mode 100644 index 000000000..d7a3fb514 --- /dev/null +++ b/docs/_ext/scrapyfixautodoc.py @@ -0,0 +1,18 @@ +""" +Must be included after 'sphinx.ext.autodoc'. Fixes unwanted 'alias of' behavior. +https://github.com/sphinx-doc/sphinx/issues/4422 +""" + +# pylint: disable=import-error +from sphinx.application import Sphinx + + +def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool: + if not skip: + # autodocs was generating a text "alias of" for the following members + return name in {"default_item_class", "default_selector_class"} + return skip + + +def setup(app: Sphinx) -> None: + app.connect("autodoc-skip-member", maybe_skip_member) diff --git a/docs/conf.py b/docs/conf.py index a3475a323..be5e07195 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,17 +1,12 @@ -# Scrapy documentation build configuration file, created by -# sphinx-quickstart on Mon Nov 24 12:02:52 2008. +# Configuration file for the Sphinx documentation builder. # -# This file is execfile()d with the current directory set to its containing dir. -# -# The contents of this file are pickled, so don't put values in the namespace -# that aren't pickleable (module imports are okay, they're removed automatically). -# -# All configuration values have a default; values that are commented out -# serve to show the default. +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html # pylint: disable=import-error import os import sys +from collections.abc import Sequence from pathlib import Path # If your extensions are in another directory, add it here. If the directory @@ -20,36 +15,30 @@ sys.path.append(str(Path(__file__).parent / "_ext")) sys.path.insert(0, str(Path(__file__).parent.parent)) -# General configuration -# --------------------- +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "Scrapy" +project_copyright = "Scrapy developers" +author = "Scrapy developers" + + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ "hoverxref.extension", "notfound.extension", "scrapydocs", "sphinx.ext.autodoc", + "scrapyfixautodoc", # Must be after "sphinx.ext.autodoc" "sphinx.ext.coverage", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", ] -# Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = "Scrapy" -copyright = "Scrapy developers" +exclude_patterns = ["build", "Thumbs.db", ".DS_Store"] # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -65,118 +54,17 @@ except ImportError: version = "" release = "" -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -language = "en" - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of documents that shouldn't be included in the build. -# unused_docs = [] - -exclude_patterns = ["build"] - -# List of directories, relative to source directory, that shouldn't be searched -# for source files. -exclude_trees = [".build"] - -# The reST default role (used for this markup: `text`) to use for all documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# List of Sphinx warnings that will not be raised suppress_warnings = ["epub.unknown_project_files"] -# Options for HTML output -# ----------------------- +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. html_theme = "sphinx_rtd_theme" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# The style sheet to use for HTML and HTML Help pages. A file of that name -# must exist either in Sphinx' static/ path, or in one of the custom paths -# given in html_static_path. -# html_style = 'scrapydoc.css' - -# The name for this set of Sphinx documents. If None, it defaults to -# "<project> v<release> documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. html_last_updated_fmt = "%b %d, %Y" -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_use_modindex = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, the reST sources are included in the HTML build as _sources/<name>. -html_copy_source = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a <link> tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = '' - -# Output file base name for HTML help builder. -htmlhelp_basename = "Scrapydoc" - html_css_files = [ "custom.css", ] @@ -184,14 +72,8 @@ html_css_files = [ # Set canonical URL from the Read the Docs Domain html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "") -# Options for LaTeX output -# ------------------------ - -# The paper size ('letter' or 'a4'). -# latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -# latex_font_size = '10pt' +# -- Options for LaTeX output ------------------------------------------------ +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-latex-output # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -199,39 +81,22 @@ latex_documents = [ ("index", "Scrapy.tex", "Scrapy Documentation", "Scrapy developers", "manual"), ] -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False +# -- Options for the linkcheck builder --------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-the-linkcheck-builder -# Additional stuff for the LaTeX preamble. -# latex_preamble = '' - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_use_modindex = True - - -# Options for the linkcheck builder -# --------------------------------- - -# A list of regular expressions that match URIs that should not be checked when -# doing a linkcheck build. linkcheck_ignore = [ r"http://localhost:\d+", "http://hg.scrapy.org", - "http://directory.google.com/", + r"https://github.com/scrapy/scrapy/commit/\w+", r"https://github.com/scrapy/scrapy/issues/\d+", ] +linkcheck_anchors_ignore_for_url = ["https://github.com/pyca/cryptography/issues/2692"] + +# -- Options for the Coverage extension -------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/coverage.html#configuration -# Options for the Coverage extension -# ---------------------------------- coverage_ignore_pyobjects = [ # Contract’s add_pre_hook and add_post_hook are not documented because # they should be transparent to contract developers, for whom pre_hook and @@ -272,8 +137,8 @@ coverage_ignore_pyobjects = [ ] -# Options for the InterSphinx extension -# ------------------------------------- +# -- Options for the InterSphinx extension ----------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration intersphinx_mapping = { "attrs": ("https://www.attrs.org/en/stable/", None), @@ -290,11 +155,11 @@ intersphinx_mapping = { "twistedapi": ("https://docs.twisted.org/en/stable/api/", None), "w3lib": ("https://w3lib.readthedocs.io/en/latest", None), } -intersphinx_disabled_reftypes = [] +intersphinx_disabled_reftypes: Sequence[str] = [] -# Options for sphinx-hoverxref options -# ------------------------------------ +# -- Options for sphinx-hoverxref extension ---------------------------------- +# https://sphinx-hoverxref.readthedocs.io/en/latest/configuration.html hoverxref_auto_ref = True hoverxref_role_types = { @@ -309,15 +174,3 @@ hoverxref_role_types = { "signal": "tooltip", } hoverxref_roles = ["command", "reqmeta", "setting", "signal"] - - -def setup(app): - app.connect("autodoc-skip-member", maybe_skip_member) - - -def maybe_skip_member(app, what, name, obj, skip, options): - if not skip: - # autodocs was generating a text "alias of" for the following members - # https://github.com/sphinx-doc/sphinx/issues/4422 - return name in {"default_item_class", "default_selector_class"} - return skip From 14219b1fca86f2229d2b69455bdbb4eb952cf504 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <laertefbk@gmail.com> Date: Wed, 22 Jan 2025 07:16:22 -0300 Subject: [PATCH 1635/2083] fix: test_s3_export fails with boto3 >= 1.36.0 --- tests/test_feedexport.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 0f149f172..81d05e2a3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2622,18 +2622,24 @@ class BatchDeliveriesTest(FeedExportTestBase): stubs = [] def open(self, *args, **kwargs): + from botocore import __version__ as botocore_version from botocore.stub import ANY, Stubber + from packaging.version import Version + + expected_params = { + "Body": ANY, + "Bucket": bucket, + "Key": ANY, + } + if Version(botocore_version) >= Version("1.36.0"): + expected_params["ChecksumAlgorithm"] = ANY stub = Stubber(self.s3_client) stub.activate() CustomS3FeedStorage.stubs.append(stub) stub.add_response( "put_object", - expected_params={ - "Body": ANY, - "Bucket": bucket, - "Key": ANY, - }, + expected_params=expected_params, service_response={}, ) return super().open(*args, **kwargs) From 9bc0029d27d8ed719e2cc2e9077a81450b995b96 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <95530227+guillermo-bondonno@users.noreply.github.com> Date: Wed, 22 Jan 2025 08:07:44 -0300 Subject: [PATCH 1636/2083] Allow updating pre-crawler settings from add-ons (#6568) --- docs/topics/addons.rst | 11 +- docs/topics/practices.rst | 22 +-- docs/topics/settings.rst | 151 +++++++++++++----- scrapy/addons.py | 22 ++- scrapy/crawler.py | 1 + tests/test_spiderloader/__init__.py | 19 +++ .../spiders_from_addons/__init__.py | 0 .../spiders_from_addons/spider0.py | 6 + 8 files changed, 171 insertions(+), 61 deletions(-) create mode 100644 tests/test_spiderloader/spiders_from_addons/__init__.py create mode 100644 tests/test_spiderloader/spiders_from_addons/spider0.py diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 14b4aa8ba..46cf1edbd 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -32,7 +32,7 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are Python classes that include the following method: +Add-ons are Python classes that include one or both of the following methods: .. method:: update_settings(settings) @@ -45,6 +45,15 @@ Add-ons are Python classes that include the following method: :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.Settings` +.. classmethod:: update_pre_crawler_settings(cls, settings) + + Use this class method instead of the :meth:`update_settings` method to + update :ref:`pre-crawler settings <pre-crawler-settings>` whose value is + used before the :class:`~scrapy.crawler.Crawler` object is created. + + :param settings: The settings object storing Scrapy/component configuration + :type settings: :class:`~scrapy.settings.BaseSettings` + They can also have the following method: .. classmethod:: from_crawler(cls, crawler) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 1500011e7..5f6798601 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -246,24 +246,10 @@ Same example but running the spiders sequentially by chaining the deferreds: crawl() reactor.run() # the script will block here until the last crawl call is finished -Different spiders can set different values for the same setting, but when they -run in the same process it may be impossible, by design or because of some -limitations, to use these different values. What happens in practice is -different for different settings: - -* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value - (:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the - default one) cannot be read from the per-spider settings. These are applied - when the :class:`~scrapy.crawler.CrawlerRunner` or - :class:`~scrapy.crawler.CrawlerProcess` object is created. -* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first - available value is used, and if a spider requests a different reactor an - exception will be raised. These are applied when the reactor is installed. -* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the - ones used by the resolver (:setting:`DNSCACHE_ENABLED`, - :setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy) - the first available value is used. These are applied when the reactor is - started. +.. note:: When running multiple spiders in the same process, :ref:`reactor + settings <reactor-settings>` should not have a different value per spider. + Also, :ref:`pre-crawler settings <pre-crawler-settings>` cannot be defined + per spider. .. seealso:: :ref:`run-from-script`. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8801434d8..a53e0806d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -33,42 +33,48 @@ Python :ref:`import search path <tut-searchpath>`. Populating the settings ======================= -Settings can be populated using different mechanisms, each of which having a -different precedence. Here is the list of them in decreasing order of -precedence: +Settings can be populated using different mechanisms, each of which has a +different precedence: - 1. Command line options (most precedence) - 2. Settings per-spider - 3. Project settings module - 4. Settings set by add-ons - 5. Default settings per-command - 6. Default global settings (less precedence) + 1. :ref:`Command-line settings <cli-settings>` (highest precedence) + 2. :ref:`Spider settings <spider-settings>` + 3. :ref:`Project settings <project-settings>` + 4. :ref:`Add-on settings <addon-settings>` + 5. :ref:`Command-specific default settings <cmd-default-settings>` + 6. :ref:`Global default settings <default-settings>` (lowest precedence) -The population of these settings sources is taken care of internally, but a -manual handling is possible using API calls. See the -:ref:`topics-api-settings` topic for reference. +.. _cli-settings: -These mechanisms are described in more detail below. +1. Command-line settings +------------------------ -1. Command line options ------------------------ +Settings set in the command line have the highest precedence, overriding any +other settings. -Arguments provided by the command line are the ones that take most precedence, -overriding any other options. You can explicitly override one (or more) -settings using the ``-s`` (or ``--set``) command line option. +You can explicitly override one or more settings using the ``-s`` (or +``--set``) command-line option. .. highlight:: sh Example:: - scrapy crawl myspider -s LOG_FILE=scrapy.log + scrapy crawl myspider -s LOG_LEVEL=INFO -s LOG_FILE=scrapy.log -2. Settings per-spider ----------------------- +.. _spider-settings: -Spiders (See the :ref:`topics-spiders` chapter for reference) can define their -own settings that will take precedence and override the project ones. One way -to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: +2. Spider settings +------------------ + +:ref:`Spiders <topics-spiders>` can define their own settings that will take +precedence and override the project ones. + +.. note:: :ref:`Pre-crawler settings <pre-crawler-settings>` cannot be defined + per spider, and :ref:`reactor settings <reactor-settings>` should not have + a different value per spider when :ref:`running multiple spiders in the + same process <run-multiple-spiders>`. + +One way to do so is by setting their :attr:`~scrapy.Spider.custom_settings` +attribute: .. code-block:: python @@ -83,7 +89,7 @@ to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: } It's often better to implement :meth:`~scrapy.Spider.update_settings` instead, -and settings set there should use the "spider" priority explicitly: +and settings set there should use the ``"spider"`` priority explicitly: .. code-block:: python @@ -121,27 +127,37 @@ arguments <spiderargs>` or other logic: ) return spider -3. Project settings module --------------------------- +.. _project-settings: -The project settings module is the standard configuration file for your Scrapy -project, it's where most of your custom settings will be populated. For a -standard Scrapy project, this means you'll be adding or changing the settings -in the ``settings.py`` file created for your project. +3. Project settings +------------------- -4. Settings set by add-ons --------------------------- +Scrapy projects include a settings module, usually a file called +``settings.py``, where you should populate most settings that apply to all your +spiders. + +.. seealso:: :ref:`topics-settings-module-envvar` + +.. _addon-settings: + +4. Add-on settings +------------------ :ref:`Add-ons <topics-addons>` can modify settings. They should do this with -this priority, though this is not enforced. +``"addon"`` priority where possible. -5. Default settings per-command -------------------------------- +.. _cmd-default-settings: -Each :doc:`Scrapy tool </topics/commands>` command can have its own default -settings, which override the global default settings. Those custom command -settings are specified in the ``default_settings`` attribute of the command -class. +5. Command-specific default settings +------------------------------------ + +Each :ref:`Scrapy command <topics-commands>` can have its own default settings, +which override the :ref:`global default settings <default-settings>`. + +Those command-specific default settings are specified in the +``default_settings`` attribute of each command class. + +.. _default-settings: 6. Default global settings -------------------------- @@ -234,6 +250,61 @@ example, proper setting names for a fictional robots.txt extension would be ``ROBOTSTXT_ENABLED``, ``ROBOTSTXT_OBEY``, ``ROBOTSTXT_CACHEDIR``, etc. +Special settings +================ + +The following settings work slightly differently than all other settings. + +.. _pre-crawler-settings: + +Pre-crawler settings +-------------------- + +**Pre-crawler settings** are settings used before the +:class:`~scrapy.crawler.Crawler` object is created. + +These settings cannot be :ref:`set from a spider <spider-settings>`. + +These settings are :setting:`SPIDER_LOADER_CLASS` and settings used by the +corresponding :ref:`component <topics-components>`, e.g. +:setting:`SPIDER_MODULES` and :setting:`SPIDER_LOADER_WARN_ONLY` for the +default component. + + +.. _reactor-settings: + +Reactor settings +---------------- + +**Reactor settings** are settings tied to the :doc:`Twisted reactor +<twisted:core/howto/reactor-basics>`. + +These settings can be defined from a spider. However, because only 1 reactor +can be used per process, these settings cannot use a different value per spider +when :ref:`running multiple spiders in the same process +<run-multiple-spiders>`. + +In general, if different spiders define different values, the first defined +value is used. However, if two spiders request a different reactor, an +exception is raised. + +These settings are: + +- :setting:`ASYNCIO_EVENT_LOOP` + +- :setting:`DNS_RESOLVER` and settings used by the corresponding + component, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE` + and :setting:`DNS_TIMEOUT` for the default one. + +- :setting:`REACTOR_THREADPOOL_MAXSIZE` + +- :setting:`TWISTED_REACTOR` + +:setting:`ASYNCIO_EVENT_LOOP` and :setting:`TWISTED_REACTOR` are used upon +installing the reactor. The rest of the settings are applied when starting +the reactor. + + .. _topics-settings-ref: Built-in settings reference diff --git a/scrapy/addons.py b/scrapy/addons.py index 7a1da3afc..1024d2dcd 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -9,7 +9,7 @@ from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler - from scrapy.settings import Settings + from scrapy.settings import BaseSettings, Settings logger = logging.getLogger(__name__) @@ -36,7 +36,8 @@ class AddonManager: try: addoncls = load_object(clspath) addon = build_from_crawler(addoncls, self.crawler) - addon.update_settings(settings) + if hasattr(addon, "update_settings"): + addon.update_settings(settings) self.addons.append(addon) except NotConfigured as e: if e.args: @@ -52,3 +53,20 @@ class AddonManager: }, extra={"crawler": self.crawler}, ) + + @classmethod + def load_pre_crawler_settings(cls, settings: BaseSettings): + """Update early settings that do not require a crawler instance, such as SPIDER_MODULES. + + Similar to the load_settings method, this loads each add-on configured in the + ``ADDONS`` setting and calls their 'update_pre_crawler_settings' class method if present. + This method doesn't have access to the crawler instance or the addons list. + + :param settings: The :class:`~scrapy.settings.BaseSettings` object from \ + which to read the early add-on configuration + :type settings: :class:`~scrapy.settings.Settings` + """ + for clspath in build_component_list(settings["ADDONS"]): + addoncls = load_object(clspath) + if hasattr(addoncls, "update_pre_crawler_settings"): + addoncls.update_pre_crawler_settings(settings) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index f6dbe053a..1aa68cb00 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -292,6 +292,7 @@ class CrawlerRunner: def __init__(self, settings: dict[str, Any] | Settings | None = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) + AddonManager.load_pre_crawler_settings(settings) self.settings: Settings = settings self.spider_loader: SpiderLoader = self._get_spider_loader(settings) self._crawlers: set[Crawler] = set() diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d5aac34eb..705f722b3 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -97,6 +97,25 @@ class SpiderLoaderTest(unittest.TestCase): self.spider_loader = SpiderLoader.from_settings(settings) assert len(self.spider_loader._spiders) == 0 + def test_load_spider_module_from_addons(self): + module = "tests.test_spiderloader.spiders_from_addons.spider0" + + class SpiderModuleAddon: + @classmethod + def update_pre_crawler_settings(cls, settings): + settings.set( + "SPIDER_MODULES", + [module], + "project", + ) + + runner = CrawlerRunner({"ADDONS": {SpiderModuleAddon: 1}}) + + crawler = runner.create_crawler("spider_from_addon") + self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) + self.assertEqual(crawler.spidercls.name, "spider_from_addon") + self.assertTrue(len(crawler.settings["SPIDER_MODULES"]) == 1) + def test_crawler_runner_loading(self): module = "tests.test_spiderloader.test_spiders.spider1" runner = CrawlerRunner( diff --git a/tests/test_spiderloader/spiders_from_addons/__init__.py b/tests/test_spiderloader/spiders_from_addons/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_spiderloader/spiders_from_addons/spider0.py b/tests/test_spiderloader/spiders_from_addons/spider0.py new file mode 100644 index 000000000..45c3f64a7 --- /dev/null +++ b/tests/test_spiderloader/spiders_from_addons/spider0.py @@ -0,0 +1,6 @@ +from scrapy.spiders import Spider + + +class SpiderFromAddon(Spider): + name = "spider_from_addon" + allowed_domains = ["scrapy1.org", "scrapy3.org"] From 7e61ff352439d3e5c85785fc26b5503e4fed67b8 Mon Sep 17 00:00:00 2001 From: Rotzbua <Rotzbua@users.noreply.github.com> Date: Wed, 22 Jan 2025 18:09:42 +0100 Subject: [PATCH 1637/2083] Upgrade Sphinx (#6624) --- docs/_ext/scrapydocs.py | 130 +++++++++++++++++++++------------------- docs/requirements.txt | 2 +- 2 files changed, 71 insertions(+), 61 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 9b63f39f6..4ceb003c7 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,63 +1,67 @@ # pylint: disable=import-error +from collections.abc import Sequence from operator import itemgetter +from typing import Any, TypedDict from docutils import nodes +from docutils.nodes import Element, General, Node, document from docutils.parsers.rst import Directive -from docutils.parsers.rst.roles import set_classes +from sphinx.application import Sphinx from sphinx.util.nodes import make_refnode -class settingslist_node(nodes.General, nodes.Element): +class SettingData(TypedDict): + docname: str + setting_name: str + refid: str + + +class SettingslistNode(General, Element): pass class SettingsListDirective(Directive): - def run(self): - return [settingslist_node("")] + def run(self) -> Sequence[Node]: + return [SettingslistNode()] -def is_setting_index(node): - if node.tagname == "index" and node["entries"]: +def is_setting_index(node: Node) -> bool: + if node.tagname == "index" and node["entries"]: # type: ignore[index,attr-defined] # index entries for setting directives look like: # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] - entry_type, info, refid = node["entries"][0][:3] + entry_type, info, refid = node["entries"][0][:3] # type: ignore[index] return entry_type == "pair" and info.endswith("; setting") return False -def get_setting_target(node): - # target nodes are placed next to the node in the doc tree - return node.parent[node.parent.index(node) + 1] - - -def get_setting_name_and_refid(node): +def get_setting_name_and_refid(node: Node) -> tuple[str, str]: """Extract setting name from directive index node""" - entry_type, info, refid = node["entries"][0][:3] + entry_type, info, refid = node["entries"][0][:3] # type: ignore[index] return info.replace("; setting", ""), refid -def collect_scrapy_settings_refs(app, doctree): +def collect_scrapy_settings_refs(app: Sphinx, doctree: document) -> None: env = app.builder.env if not hasattr(env, "scrapy_all_settings"): - env.scrapy_all_settings = [] - - for node in doctree.traverse(is_setting_index): - targetnode = get_setting_target(node) - assert isinstance(targetnode, nodes.target), "Next node is not a target" + emptyList: list[SettingData] = [] + env.scrapy_all_settings = emptyList # type: ignore[attr-defined] + for node in doctree.findall(is_setting_index): setting_name, refid = get_setting_name_and_refid(node) - env.scrapy_all_settings.append( - { - "docname": env.docname, - "setting_name": setting_name, - "refid": refid, - } + env.scrapy_all_settings.append( # type: ignore[attr-defined] + SettingData( + docname=env.docname, + setting_name=setting_name, + refid=refid, + ) ) -def make_setting_element(setting_data, app, fromdocname): +def make_setting_element( + setting_data: SettingData, app: Sphinx, fromdocname: str +) -> Any: refnode = make_refnode( app.builder, fromdocname, @@ -73,22 +77,56 @@ def make_setting_element(setting_data, app, fromdocname): return item -def replace_settingslist_nodes(app, doctree, fromdocname): +def replace_settingslist_nodes( + app: Sphinx, doctree: document, fromdocname: str +) -> None: env = app.builder.env - for node in doctree.traverse(settingslist_node): + for node in doctree.findall(SettingslistNode): settings_list = nodes.bullet_list() settings_list.extend( [ make_setting_element(d, app, fromdocname) - for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) + for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined] if fromdocname != d["docname"] ] ) node.replace_self(settings_list) -def setup(app): +def source_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/blob/master/" + text + node = nodes.reference(rawtext, text, refuri=ref, **options) + return [node], [] + + +def issue_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/issues/" + text + node = nodes.reference(rawtext, "issue " + text, refuri=ref) + return [node], [] + + +def commit_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/commit/" + text + node = nodes.reference(rawtext, "commit " + text, refuri=ref) + return [node], [] + + +def rev_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "http://hg.scrapy.org/scrapy/changeset/" + text + node = nodes.reference(rawtext, "r" + text, refuri=ref) + return [node], [] + + +def setup(app: Sphinx) -> None: app.add_crossref_type( directivename="setting", rolename="setting", @@ -114,36 +152,8 @@ def setup(app): app.add_role("issue", issue_role) app.add_role("rev", rev_role) - app.add_node(settingslist_node) + app.add_node(SettingslistNode) app.add_directive("settingslist", SettingsListDirective) app.connect("doctree-read", collect_scrapy_settings_refs) app.connect("doctree-resolved", replace_settingslist_nodes) - - -def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/blob/master/" + text - set_classes(options) - node = nodes.reference(rawtext, text, refuri=ref, **options) - return [node], [] - - -def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/issues/" + text - set_classes(options) - node = nodes.reference(rawtext, "issue " + text, refuri=ref, **options) - return [node], [] - - -def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/commit/" + text - set_classes(options) - node = nodes.reference(rawtext, "commit " + text, refuri=ref, **options) - return [node], [] - - -def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "http://hg.scrapy.org/scrapy/changeset/" + text - set_classes(options) - node = nodes.reference(rawtext, "r" + text, refuri=ref, **options) - return [node], [] diff --git a/docs/requirements.txt b/docs/requirements.txt index 7ee897170..e2abe76d9 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -sphinx==6.2.1 +sphinx==8.1.3 sphinx-hoverxref==1.4.2 sphinx-notfound-page==1.0.4 sphinx-rtd-theme==3.0.2 From d4b152bbf64591317d2d7ec9dfed0746e7bdb8e1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 23 Jan 2025 12:22:18 +0400 Subject: [PATCH 1638/2083] Drop PyPy 3.9, add a pypy3-extra-deps CI job. (#6613) --- .github/workflows/tests-ubuntu.yml | 8 ++++---- tox.ini | 28 +++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 89d1e70ac..6c7842217 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -30,9 +30,6 @@ jobs: - python-version: "3.13" env: TOXENV: asyncio - - python-version: pypy3.9 - env: - TOXENV: pypy3 - python-version: pypy3.10 env: TOXENV: pypy3 @@ -44,7 +41,7 @@ jobs: - python-version: 3.9.19 env: TOXENV: asyncio-pinned - - python-version: pypy3.9 + - python-version: pypy3.10 env: TOXENV: pypy3-pinned - python-version: 3.9.19 @@ -57,6 +54,9 @@ jobs: - python-version: "3.13" env: TOXENV: extra-deps + - python-version: pypy3.10 + env: + TOXENV: pypy3-extra-deps - python-version: "3.13" env: TOXENV: botocore diff --git a/tox.ini b/tox.ini index cf5e19a61..0f91db19d 100644 --- a/tox.ini +++ b/tox.ini @@ -118,6 +118,7 @@ setenv = install_command = python -I -m pip install {opts} {packages} commands = + ; tests for docs fail with parsel < 1.8.0 pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] @@ -191,14 +192,35 @@ setenv = [testenv:pypy3] basepython = pypy3 commands = + ; not enabling coverage as it significantly increases the run time pytest {posargs:--durations=10 docs scrapy tests} -[testenv:pypy3-pinned] -basepython = pypy3.9 +[testenv:pypy3-extra-deps] +basepython = pypy3 deps = - {[pinned]deps} + {[testenv:extra-deps]deps} +commands = {[testenv:pypy3]commands} + +[testenv:pypy3-pinned] +basepython = pypy3.10 +deps = + cryptography==41.0.5 + cssselect==0.9.1 + h2==3.1 + itemadapter==0.1.0 + parsel==1.5.0 + Protego==0.1.15 + pyOpenSSL==23.3.0 + queuelib==1.4.2 + service_identity==18.1.0 + Twisted[http2]==21.7.0 + w3lib==1.17.0 + zope.interface==5.1.0 + lxml==4.6.0 + {[test-requirements]deps} PyPyDispatcher==2.1.0 commands = + ; disabling both coverage and docs tests pytest {posargs:--durations=10 scrapy tests} install_command = {[pinned]install_command} setenv = From c03fb2abb8c354c56c4e8363fc602d49f956c280 Mon Sep 17 00:00:00 2001 From: anubhav <protokoul@users.noreply.github.com> Date: Thu, 23 Jan 2025 21:36:45 +0530 Subject: [PATCH 1639/2083] fix: added feed_options as a keyword argument to GCSFeedStorage. (#6628) --- scrapy/extensions/feedexport.py | 25 +++++++++++++++++++++++-- tests/test_feedexport.py | 15 +++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c6e2aa0dd..8bcd4e40d 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -276,7 +276,14 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri: str, project_id: str | None, acl: str | None): + def __init__( + self, + uri: str, + project_id: str | None, + acl: str | None, + *, + feed_options: dict[str, Any] | None = None, + ): self.project_id: str | None = project_id self.acl: str | None = acl u = urlparse(uri) @@ -284,12 +291,26 @@ class GCSFeedStorage(BlockingFeedStorage): self.bucket_name: str = u.hostname self.blob_name: str = u.path[1:] # remove first "/" + if feed_options and feed_options.get("overwrite", True) is False: + logger.warning( + "GCS does not support appending to files. To " + "suppress this warning, remove the overwrite " + "option from your FEEDS setting or set it to True." + ) + @classmethod - def from_crawler(cls, crawler: Crawler, uri: str) -> Self: + def from_crawler( + cls, + crawler: Crawler, + uri: str, + *, + feed_options: dict[str, Any] | None = None, + ) -> Self: return cls( uri, crawler.settings["GCS_PROJECT_ID"], crawler.settings["FEED_STORAGE_GCS_ACL"] or None, + feed_options=feed_options, ) def _store_in_thread(self, file: IO[bytes]) -> None: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 81d05e2a3..7edffa1f6 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -523,6 +523,21 @@ class GCSFeedStorageTest(unittest.TestCase): bucket_mock.blob.assert_called_once_with("export.csv") blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + def test_overwrite_default(self): + with LogCapture() as log: + GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl") + self.assertNotIn("GCS does not support appending to files", str(log)) + + def test_overwrite_false(self): + with LogCapture() as log: + GCSFeedStorage( + "gs://mybucket/export.csv", + "myproject-123", + "custom-acl", + feed_options={"overwrite": False}, + ) + self.assertIn("GCS does not support appending to files", str(log)) + class StdoutFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks From cec0aeca58730b592bec50299414d4bf30fc9ec0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 27 Jan 2025 14:07:09 +0400 Subject: [PATCH 1640/2083] Bump ruff, switch from black to ruff-format (#6631) --- .pre-commit-config.yaml | 7 +- scrapy/commands/genspider.py | 6 +- scrapy/core/downloader/handlers/__init__.py | 13 ++-- scrapy/core/http2/stream.py | 8 +-- scrapy/downloadermiddlewares/cookies.py | 3 +- scrapy/downloadermiddlewares/offsite.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 4 +- scrapy/downloadermiddlewares/stats.py | 2 +- scrapy/extensions/telnet.py | 4 +- scrapy/http/headers.py | 3 +- scrapy/http/request/form.py | 1 - scrapy/http/response/__init__.py | 3 +- scrapy/linkextractors/lxmlhtml.py | 1 - scrapy/pipelines/files.py | 4 +- scrapy/selector/unified.py | 3 +- scrapy/settings/__init__.py | 2 +- scrapy/settings/default_settings.py | 2 +- scrapy/spidermiddlewares/offsite.py | 2 +- scrapy/squeues.py | 16 +++-- scrapy/utils/curl.py | 5 +- scrapy/utils/defer.py | 2 +- scrapy/utils/iterators.py | 2 +- scrapy/utils/log.py | 1 - scrapy/utils/python.py | 7 +- tests/spiders.py | 6 +- tests/test_contracts.py | 3 +- tests/test_downloadermiddleware_httpproxy.py | 6 +- tests/test_downloadermiddleware_redirect.py | 24 +++---- tests/test_downloaderslotssettings.py | 6 +- tests/test_engine.py | 12 ++-- tests/test_engine_stop_download_headers.py | 6 +- tests/test_exporters.py | 4 +- tests/test_feedexport.py | 76 +++++++++----------- tests/test_http_request.py | 12 ++-- tests/test_http_response.py | 6 +- tests/test_item.py | 4 +- tests/test_pipeline_files.py | 2 +- tests/test_pipeline_media.py | 4 -- tests/test_robotstxt_interface.py | 7 +- tests/test_selector.py | 6 +- tests/test_utils_response.py | 30 ++++---- tests/test_utils_url.py | 6 +- 42 files changed, 151 insertions(+), 172 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c76c613d9..18402b908 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,10 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.4 + rev: v0.9.3 hooks: - id: ruff args: [ --fix ] -- repo: https://github.com/psf/black.git - rev: 24.10.0 - hooks: - - id: black + - id: ruff-format - repo: https://github.com/adamchainz/blacken-docs rev: 1.19.1 hooks: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2a1dea997..6d4aec3d8 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -188,9 +188,9 @@ class Command(ScrapyCommand): return True return False - assert ( - self.crawler_process is not None - ), "crawler_process must be set before calling run" + assert self.crawler_process is not None, ( + "crawler_process must be set before calling run" + ) try: spidercls = self.crawler_process.spider_loader.load(name) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 7f3da67eb..902f200b8 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -34,13 +34,12 @@ class DownloadHandlerProtocol(Protocol): class DownloadHandlers: def __init__(self, crawler: Crawler): self._crawler: Crawler = crawler - self._schemes: dict[str, str | Callable[..., Any]] = ( - {} - ) # stores acceptable schemes on instancing - self._handlers: dict[str, DownloadHandlerProtocol] = ( - {} - ) # stores instanced handlers for schemes - self._notconfigured: dict[str, str] = {} # remembers failed handlers + # stores acceptable schemes on instancing + self._schemes: dict[str, str | Callable[..., Any]] = {} + # stores instanced handlers for schemes + self._handlers: dict[str, DownloadHandlerProtocol] = {} + # remembers failed handlers + self._notconfigured: dict[str, str] = {} handlers: dict[str, str | Callable[..., Any]] = without_none_values( cast( "dict[str, str | Callable[..., Any]]", diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a4dc89c18..afca99dcf 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -193,7 +193,7 @@ class Stream: url.netloc == str(self._protocol.metadata["uri"].host, "utf-8") or url.netloc == str(self._protocol.metadata["uri"].netloc, "utf-8") or url.netloc - == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' + == f"{self._protocol.metadata['ip_address']}:{self._protocol.metadata['uri'].port}" ) def _get_request_headers(self) -> list[tuple[str, str]]: @@ -339,7 +339,7 @@ class Stream: if self._log_warnsize: self.metadata["reached_warnsize"] = True warning_msg = ( - f'Received more ({self._response["flow_controlled_size"]}) bytes than download ' + f"Received more ({self._response['flow_controlled_size']}) bytes than download " f"warn size ({self._download_warnsize}) in request {self._request}" ) logger.warning(warning_msg) @@ -445,7 +445,7 @@ class Stream: ResponseFailed( [ Failure( - f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM', + f"Remote peer {self._protocol.metadata['ip_address']} sent RST_STREAM", ProtocolError, ) ] @@ -465,7 +465,7 @@ class Stream: InvalidHostname( self._request, str(self._protocol.metadata["uri"].host, "utf-8"), - f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}', + f"{self._protocol.metadata['ip_address']}:{self._protocol.metadata['uri'].port}", ) ) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 43348f632..9156b8c3a 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -54,8 +54,7 @@ class CookiesMiddleware: ) -> None: for cookie in cookies: cookie_domain = cookie.domain - if cookie_domain.startswith("."): - cookie_domain = cookie_domain[1:] + cookie_domain = cookie_domain.removeprefix(".") hostname = urlparse_cached(request).hostname assert hostname is not None diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index a2cff65e7..787c46a60 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -89,5 +89,5 @@ class OffsiteMiddleware: warnings.warn(message) else: domains.append(re.escape(domain)) - regex = rf'^(.*\.)?({"|".join(domains)})$' + regex = rf"^(.*\.)?({'|'.join(domains)})$" return re.compile(regex) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 9411cff14..aba455bdd 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -63,7 +63,9 @@ class RobotsTxtMiddleware: if request.url.startswith("data:") or request.url.startswith("file:"): return None d: Deferred[RobotParser | None] = maybeDeferred( - self.robot_parser, request, spider # type: ignore[call-overload] + self.robot_parser, + request, + spider, # type: ignore[call-overload] ) d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider) return d2 diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index fb0f30620..cb5887a6f 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -19,7 +19,7 @@ if TYPE_CHECKING: def get_header_size( - headers: dict[str, list[str | bytes] | tuple[str | bytes, ...]] + headers: dict[str, list[str | bytes] | tuple[str | bytes, ...]], ) -> int: size = 0 for key, value in headers.items(): diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 189b1953b..ac832e025 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -84,9 +84,7 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers - def login( - self_, credentials, mind, *interfaces - ): # pylint: disable=no-self-argument + def login(self_, credentials, mind, *interfaces): # pylint: disable=no-self-argument if not ( credentials.username == self.username.encode("utf8") and credentials.checkPassword(self.password.encode("utf8")) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 29ba9533b..60b04753b 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -105,7 +105,8 @@ class Headers(CaselessDict): def values(self) -> list[bytes | None]: # type: ignore[override] return [ - self[k] for k in self.keys() # pylint: disable=consider-using-dict-items + self[k] + for k in self.keys() # pylint: disable=consider-using-dict-items ] def to_string(self) -> bytes: diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index de3b24de0..7681419c4 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -24,7 +24,6 @@ from scrapy.http.request import Request from scrapy.utils.python import is_listlike, to_bytes if TYPE_CHECKING: - # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index b84110b29..de2188ceb 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -94,8 +94,7 @@ class Response(object_ref): return self.request.meta # type: ignore[union-attr] except AttributeError: raise AttributeError( - "Response.meta not available, this response " - "is not tied to any request" + "Response.meta not available, this response is not tied to any request" ) @property diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 4fd932b88..814e31fec 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -25,7 +25,6 @@ from scrapy.utils.response import get_base_url from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain if TYPE_CHECKING: - from lxml.html import HtmlElement from scrapy import Selector diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index a10117590..888be81c3 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -202,7 +202,9 @@ class S3FilesStore: return cast( "Deferred[dict[str, Any]]", deferToThread( - self.s3_client.head_object, Bucket=self.bucket, Key=key_name # type: ignore[attr-defined] + self.s3_client.head_object, # type: ignore[attr-defined] + Bucket=self.bucket, + Key=key_name, ), ) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index db9014b41..f8365a87b 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -81,8 +81,7 @@ class Selector(_ParselSelector, object_ref): ): if response is not None and text is not None: raise ValueError( - f"{self.__class__.__name__}.__init__() received " - "both response and text" + f"{self.__class__.__name__}.__init__() received both response and text" ) st = _st(response, type) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 3ebdb351a..f31f824a8 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -539,7 +539,7 @@ def iter_default_settings() -> Iterable[tuple[str, Any]]: def overridden_settings( - settings: Mapping[_SettingsKeyT, Any] + settings: Mapping[_SettingsKeyT, Any], ) -> Iterable[tuple[str, Any]]: """Return an iterable of the settings that have been overridden""" for name, defvalue in iter_default_settings(): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7ef365f68..c473b369c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -333,7 +333,7 @@ TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) URLLENGTH_LIMIT = 2083 -USER_AGENT = f'Scrapy/{import_module("scrapy").__version__} (+https://scrapy.org)' +USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 95e753830..646beb911 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -110,7 +110,7 @@ class OffsiteMiddleware: warnings.warn(message, PortWarning) else: domains.append(re.escape(domain)) - regex = rf'^(.*\.)?({"|".join(domains)})$' + regex = rf"^(.*\.)?({'|'.join(domains)})$" return re.compile(regex) def spider_opened(self, spider: Spider) -> None: diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 80bb37e93..7007cd4b8 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -147,16 +147,24 @@ def _pickle_serialize(obj: Any) -> bytes: # queue.*Queue aren't subclasses of queue.BaseQueue _PickleFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type] + _with_mkdir(queue.FifoDiskQueue), # type: ignore[arg-type] + _pickle_serialize, + pickle.loads, ) _PickleLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads # type: ignore[arg-type] + _with_mkdir(queue.LifoDiskQueue), # type: ignore[arg-type] + _pickle_serialize, + pickle.loads, ) _MarshalFifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type] + _with_mkdir(queue.FifoDiskQueue), # type: ignore[arg-type] + marshal.dumps, + marshal.loads, ) _MarshalLifoSerializationDiskQueue = _serializable_queue( - _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads # type: ignore[arg-type] + _with_mkdir(queue.LifoDiskQueue), # type: ignore[arg-type] + marshal.dumps, + marshal.loads, ) # public queue classes diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index bfdd4dc8a..a563dc79a 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -22,8 +22,7 @@ class DataAction(argparse.Action): option_string: str | None = None, ) -> None: value = str(values) - if value.startswith("$"): - value = value[1:] + value = value.removeprefix("$") setattr(namespace, self.dest, value) @@ -96,7 +95,7 @@ def curl_to_request_kwargs( parsed_args, argv = curl_parser.parse_known_args(curl_args[1:]) if argv: - msg = f'Unrecognized options: {", ".join(argv)}' + msg = f"Unrecognized options: {', '.join(argv)}" if ignore_unknown_options: warnings.warn(msg) else: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 000ab5c65..8f52836c4 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -377,7 +377,7 @@ def deferred_from_coro(o: _T) -> Deferred | _T: def deferred_f_from_coro_f( - coro_f: Callable[_P, Coroutine[Any, Any, _T]] + coro_f: Callable[_P, Coroutine[Any, Any, _T]], ) -> Callable[_P, Deferred[_T]]: """Converts a coroutine function into a function that returns a Deferred. diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index e8ed7b60a..c646fc218 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -71,7 +71,7 @@ def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]: nodetext = ( document_header + match.group().replace( - nodename, f'{nodename} {" ".join(namespaces.values())}', 1 + nodename, f"{nodename} {' '.join(namespaces.values())}", 1 ) + header_end ) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index d51231b82..b865cf48d 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -16,7 +16,6 @@ from scrapy.settings import Settings, _SettingsKeyT from scrapy.utils.versions import get_versions if TYPE_CHECKING: - from scrapy.crawler import Crawler from scrapy.logformatter import LogFormatterResult diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index e954b625c..fcf582082 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -119,8 +119,7 @@ def to_unicode( return text if not isinstance(text, (bytes, str)): raise TypeError( - "to_unicode must receive a bytes or str " - f"object, got {type(text).__name__}" + f"to_unicode must receive a bytes or str object, got {type(text).__name__}" ) if encoding is None: encoding = "utf-8" @@ -183,7 +182,7 @@ _SelfT = TypeVar("_SelfT") def memoizemethod_noargs( - method: Callable[Concatenate[_SelfT, _P], _T] + method: Callable[Concatenate[_SelfT, _P], _T], ) -> Callable[Concatenate[_SelfT, _P], _T]: """Decorator to cache the result of a method (without arguments) using a weak reference to its object @@ -313,7 +312,7 @@ def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ... def without_none_values( - iterable: Mapping[_KT, _VT] | Iterable[_KT] + iterable: Mapping[_KT, _VT] | Iterable[_KT], ) -> dict[_KT, _VT] | Iterable[_KT]: """Return a copy of ``iterable`` with all ``None`` entries removed. diff --git a/tests/spiders.py b/tests/spiders.py index 3c44d7da5..da923de6e 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -338,9 +338,9 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_yielding: 2 / 0 - assert ( - self.seedsseen - ), "All start requests consumed before any download happened" + assert self.seedsseen, ( + "All start requests consumed before any download happened" + ) def parse(self, response): self.seedsseen.append(response.meta.get("seed")) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index f7581707b..fb16140be 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -529,7 +529,7 @@ class ContractsManagerTest(unittest.TestCase): return TestItem() with MockServer() as mockserver: - contract_doc = f'@url {mockserver.url("/status?n=200")}' + contract_doc = f"@url {mockserver.url('/status?n=200')}" TestSameUrlSpider.parse_first.__doc__ = contract_doc TestSameUrlSpider.parse_second.__doc__ = contract_doc @@ -567,7 +567,6 @@ class CustomFailContractPostProcess(Contract): class CustomContractPrePostProcess(unittest.TestCase): - def setUp(self): self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 0ea1ef5eb..97c276b48 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -94,7 +94,7 @@ class TestHttpProxyMiddleware(TestCase): def test_proxy_auth_encoding(self): # utf-8 encoding - os.environ["http_proxy"] = "https://m\u00E1n:pass@proxy:3128" + os.environ["http_proxy"] = "https://m\u00e1n:pass@proxy:3128" mw = HttpProxyMiddleware(auth_encoding="utf-8") req = Request("http://scrapytest.org") assert mw.process_request(req, spider) is None @@ -103,7 +103,7 @@ class TestHttpProxyMiddleware(TestCase): # proxy from request.meta req = Request( - "http://scrapytest.org", meta={"proxy": "https://\u00FCser:pass@proxy:3128"} + "http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"} ) assert mw.process_request(req, spider) is None self.assertEqual(req.meta["proxy"], "https://proxy:3128") @@ -120,7 +120,7 @@ class TestHttpProxyMiddleware(TestCase): # proxy from request.meta, latin-1 encoding req = Request( - "http://scrapytest.org", meta={"proxy": "https://\u00FCser:pass@proxy:3128"} + "http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"} ) assert mw.process_request(req, spider) is None self.assertEqual(req.meta["proxy"], "https://proxy:3128") diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 7b19ab781..eb3cdfc11 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -55,12 +55,12 @@ class Base: assert isinstance(req2, Request) self.assertEqual(req2.url, url2) self.assertEqual(req2.method, "GET") - assert ( - "Content-Type" not in req2.headers - ), "Content-Type header must not be present in redirected request" - assert ( - "Content-Length" not in req2.headers - ), "Content-Length header must not be present in redirected request" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" def test_max_redirect_times(self): @@ -1243,12 +1243,12 @@ class MetaRefreshMiddlewareTest(Base.Test): assert isinstance(req2, Request) self.assertEqual(req2.url, "http://example.org/newpage") self.assertEqual(req2.method, "GET") - assert ( - "Content-Type" not in req2.headers - ), "Content-Type header must not be present in redirected request" - assert ( - "Content-Length" not in req2.headers - ), "Content-Length header must not be present in redirected request" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" def test_ignore_tags_default(self): diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 879bc8697..0bb143f69 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -93,6 +93,6 @@ def test_params(): _, actual = downloader._get_slot(request, spider=None) expected = Slot(**params) for param in params: - assert getattr(expected, param) == getattr( - actual, param - ), f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}" + assert getattr(expected, param) == getattr(actual, param), ( + f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}" + ) diff --git a/tests/test_engine.py b/tests/test_engine.py index 8d645eada..95955f7be 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -294,9 +294,9 @@ class EngineTest(unittest.TestCase): ] urls_visited = {rp[0].url for rp in run.respplug} urls_expected = {run.geturl(p) for p in must_be_visited} - assert ( - urls_expected <= urls_visited - ), f"URLs not visited: {list(urls_expected - urls_visited)}" + assert urls_expected <= urls_visited, ( + f"URLs not visited: {list(urls_expected - urls_visited)}" + ) def _assert_scheduled_requests(self, run: CrawlerRun, count=None): self.assertEqual(count, len(run.reqplug)) @@ -496,9 +496,9 @@ def test_request_scheduled_signal(caplog): drop_request = Request("https://drop.example") caplog.set_level(DEBUG) engine._schedule_request(drop_request, spider) - assert scheduler.enqueued == [ - keep_request - ], f"{scheduler.enqueued!r} != [{keep_request!r}]" + assert scheduler.enqueued == [keep_request], ( + f"{scheduler.enqueued!r} != [{keep_request!r}]" + ) crawler.signals.disconnect(signal_handler, request_scheduled) diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 0bad5ba55..db35bd81e 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -67,6 +67,6 @@ class HeadersReceivedEngineTest(EngineTest): must_be_visited = ["/", "/redirect", "/redirected"] urls_visited = {rp[0].url for rp in run.respplug} urls_expected = {run.geturl(p) for p in must_be_visited} - assert ( - urls_expected <= urls_visited - ), f"URLs not visited: {list(urls_expected - urls_visited)}" + assert urls_expected <= urls_visited, ( + f"URLs not visited: {list(urls_expected - urls_visited)}" + ) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 1fbacfdfc..c2cab9b2a 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -390,14 +390,14 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_errors_default(self): with self.assertRaises(UnicodeEncodeError): self.assertExportResult( - item={"text": "W\u0275\u200Brd"}, + item={"text": "W\u0275\u200brd"}, expected=None, encoding="windows-1251", ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item={"text": "W\u0275\u200Brd"}, + item={"text": "W\u0275\u200brd"}, include_headers_line=False, expected="Wɵ​rd\r\n", encoding="windows-1251", diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 7edffa1f6..4f91795e4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1190,8 +1190,7 @@ class FeedExportTest(FeedExportTestBase): "csv": b"baz,egg,foo\r\n,spam1,bar1\r\n", "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]', "jsonlines": ( - b'{"foo": "bar1", "egg": "spam1"}\n' - b'{"hello": "world2", "foo": "bar2"}\n' + b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n' ), "xml": ( b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>' @@ -2289,9 +2288,9 @@ class BatchDeliveriesTest(FeedExportTestBase): settings.update( { "FEEDS": { - self._random_temp_filename() - / "jl" - / self._file_mark: {"format": "jl"}, + self._random_temp_filename() / "jl" / self._file_mark: { + "format": "jl" + }, }, } ) @@ -2311,9 +2310,9 @@ class BatchDeliveriesTest(FeedExportTestBase): settings.update( { "FEEDS": { - self._random_temp_filename() - / "csv" - / self._file_mark: {"format": "csv"}, + self._random_temp_filename() / "csv" / self._file_mark: { + "format": "csv" + }, }, } ) @@ -2331,9 +2330,9 @@ class BatchDeliveriesTest(FeedExportTestBase): settings.update( { "FEEDS": { - self._random_temp_filename() - / "xml" - / self._file_mark: {"format": "xml"}, + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml" + }, }, } ) @@ -2352,12 +2351,12 @@ class BatchDeliveriesTest(FeedExportTestBase): settings.update( { "FEEDS": { - self._random_temp_filename() - / "xml" - / self._file_mark: {"format": "xml"}, - self._random_temp_filename() - / "json" - / self._file_mark: {"format": "json"}, + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml" + }, + self._random_temp_filename() / "json" / self._file_mark: { + "format": "json" + }, }, } ) @@ -2384,9 +2383,9 @@ class BatchDeliveriesTest(FeedExportTestBase): settings.update( { "FEEDS": { - self._random_temp_filename() - / "pickle" - / self._file_mark: {"format": "pickle"}, + self._random_temp_filename() / "pickle" / self._file_mark: { + "format": "pickle" + }, }, } ) @@ -2406,9 +2405,9 @@ class BatchDeliveriesTest(FeedExportTestBase): settings.update( { "FEEDS": { - self._random_temp_filename() - / "marshal" - / self._file_mark: {"format": "marshal"}, + self._random_temp_filename() / "marshal" / self._file_mark: { + "format": "marshal" + }, }, } ) @@ -2455,9 +2454,9 @@ class BatchDeliveriesTest(FeedExportTestBase): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { "FEEDS": { - self._random_temp_filename() - / fmt - / self._file_mark: {"format": fmt}, + self._random_temp_filename() / fmt / self._file_mark: { + "format": fmt + }, }, "FEED_EXPORT_BATCH_ITEM_COUNT": 1, "FEED_STORE_EMPTY": False, @@ -2478,9 +2477,9 @@ class BatchDeliveriesTest(FeedExportTestBase): for fmt, expctd in formats: settings = { "FEEDS": { - self._random_temp_filename() - / fmt - / self._file_mark: {"format": fmt}, + self._random_temp_filename() / fmt / self._file_mark: { + "format": fmt + }, }, "FEED_STORE_EMPTY": True, "FEED_EXPORT_INDENT": None, @@ -2520,25 +2519,19 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = { "FEEDS": { - self._random_temp_filename() - / "json" - / self._file_mark: { + self._random_temp_filename() / "json" / self._file_mark: { "format": "json", "indent": 0, "fields": ["bar"], "encoding": "utf-8", }, - self._random_temp_filename() - / "xml" - / self._file_mark: { + self._random_temp_filename() / "xml" / self._file_mark: { "format": "xml", "indent": 2, "fields": ["foo"], "encoding": "latin-1", }, - self._random_temp_filename() - / "csv" - / self._file_mark: { + self._random_temp_filename() / "csv" / self._file_mark: { "format": "csv", "indent": None, "fields": ["foo", "bar"], @@ -2563,9 +2556,7 @@ class BatchDeliveriesTest(FeedExportTestBase): } settings = { "FEEDS": { - self._random_temp_filename() - / "json" - / self._file_mark: { + self._random_temp_filename() / "json" / self._file_mark: { "format": "json", "indent": None, "encoding": "utf-8", @@ -2591,8 +2582,7 @@ class BatchDeliveriesTest(FeedExportTestBase): ] settings = { "FEEDS": { - self._random_temp_filename() - / "%(batch_id)d": { + self._random_temp_filename() / "%(batch_id)d": { "format": "json", }, }, diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 34d3b25d5..9915aaca4 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -226,9 +226,9 @@ class RequestTest(unittest.TestCase): self.assertEqual(r1.flags, r2.flags) # make sure cb_kwargs dict is shallow copied - assert ( - r1.cb_kwargs is not r2.cb_kwargs - ), "cb_kwargs must be a shallow copy, not identical" + assert r1.cb_kwargs is not r2.cb_kwargs, ( + "cb_kwargs must be a shallow copy, not identical" + ) self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) # make sure meta dict is shallow copied @@ -236,9 +236,9 @@ class RequestTest(unittest.TestCase): self.assertEqual(r1.meta, r2.meta) # make sure headers attribute is shallow copied - assert ( - r1.headers is not r2.headers - ), "headers must be a shallow copy, not identical" + assert r1.headers is not r2.headers, ( + "headers must be a shallow copy, not identical" + ) self.assertEqual(r1.headers, r2.headers) self.assertEqual(r1.encoding, r2.encoding) self.assertEqual(r1.dont_filter, r2.dont_filter) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 0730cff3a..b157e9802 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -99,9 +99,9 @@ class BaseResponseTest(unittest.TestCase): self.assertEqual(r1.flags, r2.flags) # make sure headers attribute is shallow copied - assert ( - r1.headers is not r2.headers - ), "headers must be a shallow copy, not identical" + assert r1.headers is not r2.headers, ( + "headers must be a shallow copy, not identical" + ) self.assertEqual(r1.headers, r2.headers) def test_copy_meta(self): diff --git a/tests/test_item.py b/tests/test_item.py index 480412841..0399c8f8d 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -289,9 +289,7 @@ class ItemMetaTest(unittest.TestCase): class ItemMetaClassCellRegression(unittest.TestCase): def test_item_meta_classcell_regression(self): class MyItem(Item, metaclass=ItemMeta): - def __init__( - self, *args, **kwargs - ): # pylint: disable=useless-parent-delegation + def __init__(self, *args, **kwargs): # pylint: disable=useless-parent-delegation # This call to super() trigger the __classcell__ propagation # requirement. When not done properly raises an error: # TypeError: __class__ set to <class '__main__.MyItem'> diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4c3fc36b6..4c59fcfb7 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -215,7 +215,7 @@ class FilesPipelineTestCase(unittest.TestCase): class CustomFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, item=None): - return f'full/{item.get("path")}' + return f"full/{item.get('path')}" file_path = CustomFilesPipeline.from_crawler( get_crawler(None, {"FILES_STORE": self.tempdir}) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index dd8f1084a..c6fdd3767 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -35,7 +35,6 @@ def _mocked_download_func(request, info): class UserDefinedPipeline(MediaPipeline): - def media_to_download(self, request, info, *, item=None): pass @@ -376,7 +375,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): - def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(crawler=get_crawler(None, settings)) request = Request("http://url") @@ -403,11 +401,9 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): self.assertNotIn(status, request.meta["handle_httpstatus_list"]) def test_subclass_standard_setting(self): - self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) def test_subclass_specific_setting(self): - self._assert_request_no3xx( UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True} ) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index e127cc2e3..0d00ff660 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -27,10 +27,7 @@ class BaseRobotParserTest: def test_allowed(self): robotstxt_robotstxt_body = ( - b"User-agent: * \n" - b"Disallow: /disallowed \n" - b"Allow: /allowed \n" - b"Crawl-delay: 10" + b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10" ) rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body @@ -140,7 +137,7 @@ class DecodeRobotsTxtTest(unittest.TestCase): self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") def test_decode_non_utf8(self): - robotstxt_body = b"User-agent: *\n\xFFDisallow: /\n" + robotstxt_body = b"User-agent: *\n\xffDisallow: /\n" decoded_content = decode_robotstxt(robotstxt_body, spider=None) self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") diff --git a/tests/test_selector.py b/tests/test_selector.py index 857c7d626..4eda0460f 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -107,9 +107,9 @@ class SelectorTestCase(unittest.TestCase): """Check that classes are using slots and are weak-referenceable""" x = Selector(text="") weakref.ref(x) - assert not hasattr( - x, "__dict__" - ), f"{x.__class__.__name__} does not use __slots__" + assert not hasattr(x, "__dict__"), ( + f"{x.__class__.__name__} does not use __slots__" + ) def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, "received both response and text"): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 7ad86127b..db6866571 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -158,18 +158,18 @@ class ResponseUtilsTest(unittest.TestCase): ) assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url" - assert open_in_browser( - r2, _openfunc=check_base_url - ), "Inject base url with argumented head" - assert open_in_browser( - r3, _openfunc=check_base_url - ), "Inject unique base url with misleading tag" - assert open_in_browser( - r4, _openfunc=check_base_url - ), "Inject unique base url with misleading comment" - assert open_in_browser( - r5, _openfunc=check_base_url - ), "Inject unique base url with conditional comment" + assert open_in_browser(r2, _openfunc=check_base_url), ( + "Inject base url with argumented head" + ) + assert open_in_browser(r3, _openfunc=check_base_url), ( + "Inject unique base url with misleading tag" + ) + assert open_in_browser(r4, _openfunc=check_base_url), ( + "Inject unique base url with misleading comment" + ) + assert open_in_browser(r5, _openfunc=check_base_url), ( + "Inject unique base url with conditional comment" + ) def test_open_in_browser_redos_comment(self): MAX_CPU_TIME = 0.02 @@ -240,6 +240,6 @@ class ResponseUtilsTest(unittest.TestCase): ), ) def test_remove_html_comments(input_body, output_body): - assert ( - _remove_html_comments(input_body) == output_body - ), f"{_remove_html_comments(input_body)=} == {output_body=}" + assert _remove_html_comments(input_body) == output_body, ( + f"{_remove_html_comments(input_body)=} == {output_body=}" + ) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 314082742..4b9a98d79 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -321,9 +321,9 @@ class GuessSchemeTest(unittest.TestCase): def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) - assert url.startswith( - args[1] - ), f"Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`" + assert url.startswith(args[1]), ( + f"Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`" + ) return do_expected From 98a57e241879e1b56ef8bffeb8f85f868e91c1e9 Mon Sep 17 00:00:00 2001 From: Lidiane T <lidi.mayra@gmail.com> Date: Mon, 27 Jan 2025 10:21:30 +0000 Subject: [PATCH 1641/2083] Fix error when running `scrapy bench` (#6633) --- scrapy/commands/bench.py | 2 +- tests/test_commands.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 714bc38da..16dae6ac4 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -67,6 +67,6 @@ class _BenchSpider(scrapy.Spider): return [scrapy.Request(url, dont_filter=True)] def parse(self, response: Response) -> Any: - assert isinstance(Response, TextResponse) + assert isinstance(response, TextResponse) for link in self.link_extractor.extract_links(response): yield scrapy.Request(link.url, callback=self.parse) diff --git a/tests/test_commands.py b/tests/test_commands.py index 50f093043..872b54d04 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1034,6 +1034,7 @@ class BenchCommandTest(CommandTest): ) self.assertIn("INFO: Crawled", log) self.assertNotIn("Unhandled Error", log) + self.assertNotIn("log_count/ERROR", log) class ViewCommandTest(CommandTest): From d27c6b46b11c2ceaa61b35372ad8a3c98185aa18 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 27 Jan 2025 21:25:47 +0500 Subject: [PATCH 1642/2083] Deprecate HTTP/1.0 support. --- scrapy/core/downloader/handlers/http10.py | 7 +++++++ scrapy/core/downloader/webclient.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 58f7ad577..0fbe5fc23 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -2,8 +2,10 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode @@ -26,6 +28,11 @@ class HTTP10DownloadHandler: lazy = False def __init__(self, settings: BaseSettings, crawler: Crawler): + warnings.warn( + "HTTP10DownloadHandler is deprecated and will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object( settings["DOWNLOADER_HTTPCLIENTFACTORY"] ) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index ee10ae73b..aaaf68152 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +import warnings from time import time from typing import TYPE_CHECKING from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse @@ -9,6 +10,7 @@ from twisted.internet import defer from twisted.internet.protocol import ClientFactory from twisted.web.http import HTTPClient +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached @@ -49,6 +51,14 @@ def _parse(url: str) -> tuple[bytes, bytes, bytes, int, bytes]: class ScrapyHTTPPageGetter(HTTPClient): delimiter = b"\n" + def __init__(self): + warnings.warn( + "ScrapyHTTPPageGetter is deprecated and will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + super().__init__() + def connectionMade(self): self.headers = Headers() # bucket for response headers @@ -140,6 +150,12 @@ class ScrapyHTTPClientFactory(ClientFactory): self.path = self.url def __init__(self, request: Request, timeout: float = 180): + warnings.warn( + "ScrapyHTTPClientFactory is deprecated and will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self._url: str = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface self.url: bytes = to_bytes(self._url, encoding="ascii") From 16b998f9ca8b928b03f9f8be659ac01d4f2f623f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 28 Jan 2025 01:33:00 +0500 Subject: [PATCH 1643/2083] Sort out webclient tests. --- scrapy/core/downloader/contextfactory.py | 1 + scrapy/core/downloader/webclient.py | 2 + tests/test_core_downloader.py | 134 +++++++++++++++++++++++ tests/test_downloader_handlers.py | 2 + tests/test_webclient.py | 89 ++------------- 5 files changed, 146 insertions(+), 82 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index d44c663bb..b01ee97f3 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -121,6 +121,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: + # FIXME ctx: SSL.Context = self.getCertificateOptions().getContext() ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index aaaf68152..09751ea1a 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,3 +1,5 @@ +"""Deprecated HTTP/1.0 helper classes used by HTTP10DownloadHandler.""" + from __future__ import annotations import re diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index d929a9369..0a0c0a4f0 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -1,6 +1,31 @@ +from __future__ import annotations + +import shutil +from pathlib import Path +from tempfile import mkdtemp + +import OpenSSL.SSL +import pytest +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks +from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest +from twisted.web import server, static +from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody +from twisted.web.client import Response as TxResponse from scrapy.core.downloader import Slot +from scrapy.core.downloader.contextfactory import ( + ScrapyClientContextFactory, + load_context_factory_from_settings, +) +from scrapy.core.downloader.handlers.http11 import _RequestBodyProducer +from scrapy.settings import Settings +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.python import to_bytes +from scrapy.utils.test import get_crawler +from tests.mockserver import PayloadResource, ssl_context_factory class SlotTest(unittest.TestCase): @@ -10,3 +35,112 @@ class SlotTest(unittest.TestCase): repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)", ) + + +class ContextFactoryBaseTestCase(unittest.TestCase): + context_factory = None + + def _listen(self, site): + return reactor.listenSSL( + 0, + site, + contextFactory=self.context_factory or ssl_context_factory(), + interface="127.0.0.1", + ) + + def getURL(self, path): + return f"https://127.0.0.1:{self.portno}/{path}" + + def setUp(self): + self.tmpname = Path(mkdtemp()) + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) + r.putChild(b"payload", PayloadResource()) + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.port = self._listen(self.wrapper) + self.portno = self.port.getHost().port + + @inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + shutil.rmtree(self.tmpname) + + @staticmethod + async def get_page( + url: str, + client_context_factory: BrowserLikePolicyForHTTPS, + body: str | None = None, + ) -> bytes: + agent = Agent(reactor, contextFactory=client_context_factory) + body_producer = _RequestBodyProducer(body.encode()) if body else None + response: TxResponse = await maybe_deferred_to_future( + agent.request(b"GET", url.encode(), bodyProducer=body_producer) + ) + return await maybe_deferred_to_future(readBody(response)) # type: ignore[arg-type] + + +class ContextFactoryTestCase(ContextFactoryBaseTestCase): + @deferred_f_from_coro_f + async def testPayload(self): + s = "0123456789" * 10 + crawler = get_crawler() + settings = Settings() + client_context_factory = load_context_factory_from_settings(settings, crawler) + body = await self.get_page( + self.getURL("payload"), client_context_factory, body=s + ) + self.assertEqual(body, to_bytes(s)) + + +class ContextFactoryTLSMethodTestCase(ContextFactoryBaseTestCase): + async def _assert_factory_works( + self, client_context_factory: ScrapyClientContextFactory + ) -> None: + s = "0123456789" * 10 + body = await self.get_page( + self.getURL("payload"), client_context_factory, body=s + ) + self.assertEqual(body, to_bytes(s)) + + @deferred_f_from_coro_f + async def test_setting_default(self): + crawler = get_crawler() + settings = Settings() + client_context_factory = load_context_factory_from_settings(settings, crawler) + assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD + await self._assert_factory_works(client_context_factory) + + def test_setting_none(self): + crawler = get_crawler() + settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": None}) + with pytest.raises(KeyError): + load_context_factory_from_settings(settings, crawler) + + def test_setting_bad(self): + crawler = get_crawler() + settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) + with pytest.raises(KeyError): + load_context_factory_from_settings(settings, crawler) + + @deferred_f_from_coro_f + async def test_setting_explicit(self): + crawler = get_crawler() + settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"}) + client_context_factory = load_context_factory_from_settings(settings, crawler) + assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD + await self._assert_factory_works(client_context_factory) + + @deferred_f_from_coro_f + async def test_direct_from_crawler(self): + # the setting is ignored + crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) + client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) + assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD + await self._assert_factory_works(client_context_factory) + + @deferred_f_from_coro_f + async def test_direct_init(self): + client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) + assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD + await self._assert_factory_works(client_context_factory) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0dcbeaec1..64f615bfe 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -422,6 +422,7 @@ class HttpTestCase(unittest.TestCase): return self.download_request(request, Spider("foo")).addCallback(_test) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" @@ -780,6 +781,7 @@ class HttpProxyTestCase(unittest.TestCase): return self.download_request(request, Spider("foo")).addCallback(_test) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class Http10ProxyTestCase(HttpProxyTestCase): download_handler_cls: type = HTTP10DownloadHandler diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 0a594aa7c..fa19b350b 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -8,12 +8,11 @@ from __future__ import annotations import shutil from pathlib import Path from tempfile import mkdtemp -from typing import Any import OpenSSL.SSL -from pytest import raises +import pytest from twisted.internet import defer, reactor -from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.internet.defer import inlineCallbacks from twisted.internet.testing import StringTransport from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest @@ -22,10 +21,8 @@ from twisted.web import resource, server, static, util from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ( ScrapyClientContextFactory, - load_context_factory_from_settings, ) from scrapy.http import Headers, Request -from scrapy.settings import Settings from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.test import get_crawler @@ -38,6 +35,7 @@ from tests.mockserver import ( PayloadResource, ssl_context_factory, ) +from tests.test_core_downloader import ContextFactoryBaseTestCase def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): @@ -129,6 +127,7 @@ class ParseUrlTestCase(unittest.TestCase): self.assertEqual(client._parse(url), test, url) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class ScrapyHTTPPageGetterTests(unittest.TestCase): def test_earlyHeaders(self): # basic test stolen from twisted HTTPageGetter @@ -272,6 +271,7 @@ class EncodingResource(resource.Resource): return body.encode(self.out_encoding) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class WebClientTestCase(unittest.TestCase): def _listen(self, site): return reactor.listenTCP(0, site, interface="127.0.0.1") @@ -427,35 +427,8 @@ class WebClientTestCase(unittest.TestCase): ) -class WebClientSSLTestCase(unittest.TestCase): - context_factory = None - - def _listen(self, site): - return reactor.listenSSL( - 0, - site, - contextFactory=self.context_factory or ssl_context_factory(), - interface="127.0.0.1", - ) - - def getURL(self, path): - return f"https://127.0.0.1:{self.portno}/{path}" - - def setUp(self): - self.tmpname = Path(mkdtemp()) - (self.tmpname / "file").write_bytes(b"0123456789") - r = static.File(str(self.tmpname)) - r.putChild(b"payload", PayloadResource()) - self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) - self.port = self._listen(self.wrapper) - self.portno = self.port.getHost().port - - @inlineCallbacks - def tearDown(self): - yield self.port.stopListening() - shutil.rmtree(self.tmpname) - +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class WebClientSSLTestCase(ContextFactoryBaseTestCase): def testPayload(self): s = "0123456789" * 10 return getPage(self.getURL("payload"), body=s).addCallback( @@ -490,51 +463,3 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): self.getURL("payload"), body=s, contextFactory=client_context_factory ) return self.assertFailure(d, OpenSSL.SSL.Error) - - -class WebClientTLSMethodTestCase(WebClientSSLTestCase): - def _assert_factory_works( - self, client_context_factory: ScrapyClientContextFactory - ) -> Deferred[Any]: - s = "0123456789" * 10 - return getPage( - self.getURL("payload"), body=s, contextFactory=client_context_factory - ).addCallback(self.assertEqual, to_bytes(s)) - - def test_setting_default(self): - crawler = get_crawler() - settings = Settings() - client_context_factory = load_context_factory_from_settings(settings, crawler) - assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD - return self._assert_factory_works(client_context_factory) - - def test_setting_none(self): - crawler = get_crawler() - settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": None}) - with raises(KeyError): - load_context_factory_from_settings(settings, crawler) - - def test_setting_bad(self): - crawler = get_crawler() - settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) - with raises(KeyError): - load_context_factory_from_settings(settings, crawler) - - def test_setting_explicit(self): - crawler = get_crawler() - settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"}) - client_context_factory = load_context_factory_from_settings(settings, crawler) - assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD - return self._assert_factory_works(client_context_factory) - - def test_direct_from_crawler(self): - # the setting is ignored - crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) - client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) - assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD - return self._assert_factory_works(client_context_factory) - - def test_direct_init(self): - client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) - assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD - return self._assert_factory_works(client_context_factory) From bc1aeeefc970fbd123699e0cb6d8486141bf8418 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 28 Jan 2025 01:54:43 +0500 Subject: [PATCH 1644/2083] Deprecate overriding ScrapyClientContextFactory.getContext(). --- scrapy/core/downloader/contextfactory.py | 9 ++++++++- tests/test_core_downloader.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index b01ee97f3..d1ba6208a 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -22,6 +22,7 @@ from scrapy.core.downloader.tls import ( openssl_methods, ) from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: @@ -62,6 +63,13 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers) else: self.tls_ciphers = DEFAULT_CIPHERS + if method_is_overridden(type(self), ScrapyClientContextFactory, "getContext"): + warnings.warn( + "Overriding ScrapyClientContextFactory.getContext() is deprecated and that method" + " will be removed in a future Scrapy version. Override creatorForNetloc() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) @classmethod def from_settings( @@ -121,7 +129,6 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: - # FIXME ctx: SSL.Context = self.getCertificateOptions().getContext() ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 0a0c0a4f0..e67337fc7 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -1,8 +1,10 @@ from __future__ import annotations import shutil +import warnings from pathlib import Path from tempfile import mkdtemp +from typing import Any import OpenSSL.SSL import pytest @@ -92,6 +94,22 @@ class ContextFactoryTestCase(ContextFactoryBaseTestCase): ) self.assertEqual(body, to_bytes(s)) + def test_override_getContext(self): + class MyFactory(ScrapyClientContextFactory): + def getContext( + self, hostname: Any = None, port: Any = None + ) -> OpenSSL.SSL.Context: + ctx: OpenSSL.SSL.Context = super().getContext(hostname, port) + return ctx + + with warnings.catch_warnings(record=True) as w: + MyFactory() + self.assertEqual(len(w), 1) + self.assertIn( + "Overriding ScrapyClientContextFactory.getContext() is deprecated", + str(w[0].message), + ) + class ContextFactoryTLSMethodTestCase(ContextFactoryBaseTestCase): async def _assert_factory_works( From 0d2d2892badb2b6f47c9b597564510871c7f1518 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 28 Jan 2025 02:08:49 +0500 Subject: [PATCH 1645/2083] Silence the readBody warning. --- tests/test_core_downloader.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index e67337fc7..dffba303f 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -9,7 +9,7 @@ from typing import Any import OpenSSL.SSL import pytest from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import server, static @@ -79,7 +79,15 @@ class ContextFactoryBaseTestCase(unittest.TestCase): response: TxResponse = await maybe_deferred_to_future( agent.request(b"GET", url.encode(), bodyProducer=body_producer) ) - return await maybe_deferred_to_future(readBody(response)) # type: ignore[arg-type] + with warnings.catch_warnings(): + # https://github.com/twisted/twisted/issues/8227 + warnings.filterwarnings( + "ignore", + category=DeprecationWarning, + message=r".*does not have an abortConnection method", + ) + d: Deferred[bytes] = readBody(response) # type: ignore[arg-type] + return await maybe_deferred_to_future(d) class ContextFactoryTestCase(ContextFactoryBaseTestCase): From 0a80871c3a2b353d870e337715bbbeedf6bc216e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 28 Jan 2025 22:22:09 +0500 Subject: [PATCH 1646/2083] Remove scrapy.core.downloader.webclient._parse(). --- scrapy/core/downloader/handlers/http11.py | 24 ++++---- scrapy/core/downloader/handlers/http2.py | 7 +-- scrapy/core/downloader/webclient.py | 56 +++++++----------- tests/test_downloader_handlers.py | 3 - tests/test_webclient.py | 71 +---------------------- 5 files changed, 38 insertions(+), 123 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index aa8a1a2a4..74a6e54ee 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -9,7 +9,7 @@ from contextlib import suppress from io import BytesIO from time import time from typing import TYPE_CHECKING, Any, TypedDict, TypeVar -from urllib.parse import urldefrag, urlunparse +from urllib.parse import urldefrag, urlparse from twisted.internet import ssl from twisted.internet.defer import CancelledError, Deferred, succeed @@ -32,11 +32,12 @@ from zope.interface import implementer from scrapy import Request, Spider, signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings -from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import StopDownload from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.url import add_http_if_no_scheme if TYPE_CHECKING: from twisted.internet.base import ReactorBase @@ -378,12 +379,15 @@ class ScrapyAgent: bindaddress = request.meta.get("bindaddress") or self._bindAddress proxy = request.meta.get("proxy") if proxy: - proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) - scheme = _parse(request.url)[0] - proxyHost_str = to_unicode(proxyHost) - if scheme == b"https": + proxy = add_http_if_no_scheme(proxy) + proxy_parsed = urlparse(proxy) + proxy_host = proxy_parsed.hostname + proxy_port = proxy_parsed.port + if not proxy_port: + proxy_port = 443 if proxy_parsed.scheme == "https" else 80 + if urlparse_cached(request).scheme == "https": proxyAuth = request.headers.get(b"Proxy-Authorization", None) - proxyConf = (proxyHost_str, proxyPort, proxyAuth) + proxyConf = (proxy_host, proxy_port, proxyAuth) return self._TunnelingAgent( reactor=reactor, proxyConf=proxyConf, @@ -392,13 +396,9 @@ class ScrapyAgent: bindAddress=bindaddress, pool=self._pool, ) - proxyScheme = proxyScheme or b"http" - proxyURI = urlunparse( - (proxyScheme, proxyNetloc, proxyParams, b"", b"", b"") - ) return self._ProxyAgent( reactor=reactor, - proxyURI=to_bytes(proxyURI, encoding="ascii"), + proxyURI=to_bytes(proxy, encoding="ascii"), connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool, diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index f0f9ceeb7..d0a95ee9d 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -8,8 +8,8 @@ from twisted.internet.error import TimeoutError from twisted.web.client import URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings -from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes if TYPE_CHECKING: @@ -75,10 +75,7 @@ class ScrapyH2Agent: bind_address = request.meta.get("bindaddress") or self._bind_address proxy = request.meta.get("proxy") if proxy: - _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) - scheme = _parse(request.url)[0] - - if scheme == b"https": + if urlparse_cached(request).scheme == "https": # ToDo raise NotImplementedError( "Tunneling via CONNECT method using HTTP/2.0 is not yet supported" diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 09751ea1a..e5c2255af 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -2,11 +2,10 @@ from __future__ import annotations -import re import warnings from time import time from typing import TYPE_CHECKING -from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse +from urllib.parse import urldefrag, urlparse, urlunparse from twisted.internet import defer from twisted.internet.protocol import ClientFactory @@ -22,34 +21,6 @@ if TYPE_CHECKING: from scrapy import Request -def _parsed_url_args(parsed: ParseResult) -> tuple[bytes, bytes, bytes, int, bytes]: - # Assume parsed is urlparse-d from Request.url, - # which was passed via safe_url_string and is ascii-only. - path_str = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, "")) - path = to_bytes(path_str, encoding="ascii") - assert parsed.hostname is not None - host = to_bytes(parsed.hostname, encoding="ascii") - port = parsed.port - scheme = to_bytes(parsed.scheme, encoding="ascii") - netloc = to_bytes(parsed.netloc, encoding="ascii") - if port is None: - port = 443 if scheme == b"https" else 80 - return scheme, netloc, host, port, path - - -def _parse(url: str) -> tuple[bytes, bytes, bytes, int, bytes]: - """Return tuple of (scheme, netloc, host, port, path), - all in bytes except for port which is int. - Assume url is from Request.url, which was passed via safe_url_string - and is ascii-only. - """ - url = url.strip() - if not re.match(r"^\w+://", url): - url = "//" + url - parsed = urlparse(url) - return _parsed_url_args(parsed) - - class ScrapyHTTPPageGetter(HTTPClient): delimiter = b"\n" @@ -142,14 +113,29 @@ class ScrapyHTTPClientFactory(ClientFactory): ) def _set_connection_attributes(self, request): - parsed = urlparse_cached(request) - self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args( - parsed - ) proxy = request.meta.get("proxy") if proxy: - self.scheme, _, self.host, self.port, _ = _parse(proxy) + proxy_parsed = urlparse(to_bytes(proxy, encoding="ascii")) + self.scheme = proxy_parsed.scheme + self.host = proxy_parsed.hostname + self.port = proxy_parsed.port + self.netloc = proxy_parsed.netloc + if self.port is None: + self.port = 443 if proxy_parsed.scheme == b"https" else 80 self.path = self.url + else: + parsed = urlparse_cached(request) + path_str = urlunparse( + ("", "", parsed.path or "/", parsed.params, parsed.query, "") + ) + self.path = to_bytes(path_str, encoding="ascii") + assert parsed.hostname is not None + self.host = to_bytes(parsed.hostname, encoding="ascii") + self.port = parsed.port + self.scheme = to_bytes(parsed.scheme, encoding="ascii") + self.netloc = to_bytes(parsed.netloc, encoding="ascii") + if self.port is None: + self.port = 443 if self.scheme == b"https" else 80 def __init__(self, request: Request, timeout: float = 180): warnings.warn( diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 64f615bfe..ae2030fe6 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -785,9 +785,6 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): download_handler_cls: type = HTTP10DownloadHandler - def test_download_with_proxy_https_noconnect(self): - raise unittest.SkipTest("noconnect is not supported in HTTP10DownloadHandler") - class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls: type = HTTP11DownloadHandler diff --git a/tests/test_webclient.py b/tests/test_webclient.py index fa19b350b..1b4ad2f2f 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -8,6 +8,7 @@ from __future__ import annotations import shutil from pathlib import Path from tempfile import mkdtemp +from urllib.parse import urlparse import OpenSSL.SSL import pytest @@ -61,72 +62,6 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): ).deferred -class ParseUrlTestCase(unittest.TestCase): - """Test URL parsing facility and defaults values.""" - - def _parse(self, url): - f = client.ScrapyHTTPClientFactory(Request(url)) - return (f.scheme, f.netloc, f.host, f.port, f.path) - - def testParse(self): - lip = "127.0.0.1" - tests = ( - ( - "http://127.0.0.1?c=v&c2=v2#fragment", - ("http", lip, lip, 80, "/?c=v&c2=v2"), - ), - ( - "http://127.0.0.1/?c=v&c2=v2#fragment", - ("http", lip, lip, 80, "/?c=v&c2=v2"), - ), - ( - "http://127.0.0.1/foo?c=v&c2=v2#frag", - ("http", lip, lip, 80, "/foo?c=v&c2=v2"), - ), - ( - "http://127.0.0.1:100?c=v&c2=v2#fragment", - ("http", lip + ":100", lip, 100, "/?c=v&c2=v2"), - ), - ( - "http://127.0.0.1:100/?c=v&c2=v2#frag", - ("http", lip + ":100", lip, 100, "/?c=v&c2=v2"), - ), - ( - "http://127.0.0.1:100/foo?c=v&c2=v2#frag", - ("http", lip + ":100", lip, 100, "/foo?c=v&c2=v2"), - ), - ("http://127.0.0.1", ("http", lip, lip, 80, "/")), - ("http://127.0.0.1/", ("http", lip, lip, 80, "/")), - ("http://127.0.0.1/foo", ("http", lip, lip, 80, "/foo")), - ("http://127.0.0.1?param=value", ("http", lip, lip, 80, "/?param=value")), - ("http://127.0.0.1/?param=value", ("http", lip, lip, 80, "/?param=value")), - ( - "http://127.0.0.1:12345/foo", - ("http", lip + ":12345", lip, 12345, "/foo"), - ), - ("http://spam:12345/foo", ("http", "spam:12345", "spam", 12345, "/foo")), - ( - "http://spam.test.org/foo", - ("http", "spam.test.org", "spam.test.org", 80, "/foo"), - ), - ("https://127.0.0.1/foo", ("https", lip, lip, 443, "/foo")), - ( - "https://127.0.0.1/?param=value", - ("https", lip, lip, 443, "/?param=value"), - ), - ("https://127.0.0.1:12345/", ("https", lip + ":12345", lip, 12345, "/")), - ( - "http://scrapytest.org/foo ", - ("http", "scrapytest.org", "scrapytest.org", 80, "/foo"), - ), - ("http://egg:7890 ", ("http", "egg:7890", "egg", 7890, "/")), - ) - - for url, test in tests: - test = tuple(to_bytes(x) if not isinstance(x, int) else x for x in test) - self.assertEqual(client._parse(url), test, url) - - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class ScrapyHTTPPageGetterTests(unittest.TestCase): def test_earlyHeaders(self): @@ -388,9 +323,9 @@ class WebClientTestCase(unittest.TestCase): def testFactoryInfo(self): url = self.getURL("file") - _, _, host, port, _ = client._parse(url) + parsed = urlparse(url) factory = client.ScrapyHTTPClientFactory(Request(url)) - reactor.connectTCP(to_unicode(host), port, factory) + reactor.connectTCP(parsed.hostname, parsed.port, factory) return factory.deferred.addCallback(self._cbFactoryInfo, factory) def _cbFactoryInfo(self, ignoredResult, factory): From 200d76afa96a78899faf9e2c30ef45273b71c600 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sat, 1 Feb 2025 16:07:55 +0500 Subject: [PATCH 1647/2083] Refactor EngineTest tests. --- tests/test_engine.py | 82 +++++++++++----------- tests/test_engine_stop_download_bytes.py | 4 +- tests/test_engine_stop_download_headers.py | 4 +- 3 files changed, 46 insertions(+), 44 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 95955f7be..91ce2c0de 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -243,46 +243,7 @@ class CrawlerRun: self.signals_caught[sig] = signalargs -class EngineTest(unittest.TestCase): - @defer.inlineCallbacks - def test_crawler(self): - for spider in ( - TestSpider, - DictItemsSpider, - AttrsItemsSpider, - DataClassItemsSpider, - ): - run = CrawlerRun(spider) - yield run.run() - self._assert_visited_urls(run) - self._assert_scheduled_requests(run, count=9) - self._assert_downloaded_responses(run, count=9) - self._assert_scraped_items(run) - self._assert_signals_caught(run) - self._assert_bytes_received(run) - - @defer.inlineCallbacks - def test_crawler_dupefilter(self): - run = CrawlerRun(TestDupeFilterSpider) - yield run.run() - self._assert_scheduled_requests(run, count=8) - self._assert_dropped_requests(run) - - @defer.inlineCallbacks - def test_crawler_itemerror(self): - run = CrawlerRun(ItemZeroDivisionErrorSpider) - yield run.run() - self._assert_items_error(run) - - @defer.inlineCallbacks - def test_crawler_change_close_reason_on_idle(self): - run = CrawlerRun(ChangeCloseReasonSpider) - yield run.run() - self.assertEqual( - {"spider": run.spider, "reason": "custom_reason"}, - run.signals_caught[signals.spider_closed], - ) - +class EngineTestBase(unittest.TestCase): def _assert_visited_urls(self, run: CrawlerRun): must_be_visited = [ "/", @@ -422,6 +383,47 @@ class EngineTest(unittest.TestCase): run.signals_caught[signals.spider_closed], ) + +class EngineTest(EngineTestBase): + @defer.inlineCallbacks + def test_crawler(self): + for spider in ( + TestSpider, + DictItemsSpider, + AttrsItemsSpider, + DataClassItemsSpider, + ): + run = CrawlerRun(spider) + yield run.run() + self._assert_visited_urls(run) + self._assert_scheduled_requests(run, count=9) + self._assert_downloaded_responses(run, count=9) + self._assert_scraped_items(run) + self._assert_signals_caught(run) + self._assert_bytes_received(run) + + @defer.inlineCallbacks + def test_crawler_dupefilter(self): + run = CrawlerRun(TestDupeFilterSpider) + yield run.run() + self._assert_scheduled_requests(run, count=8) + self._assert_dropped_requests(run) + + @defer.inlineCallbacks + def test_crawler_itemerror(self): + run = CrawlerRun(ItemZeroDivisionErrorSpider) + yield run.run() + self._assert_items_error(run) + + @defer.inlineCallbacks + def test_crawler_change_close_reason_on_idle(self): + run = CrawlerRun(ChangeCloseReasonSpider) + yield run.run() + self.assertEqual( + {"spider": run.spider, "reason": "custom_reason"}, + run.signals_caught[signals.spider_closed], + ) + @defer.inlineCallbacks def test_close_downloader(self): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 8dbb5b7ea..8bf225ab1 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -7,7 +7,7 @@ from tests.test_engine import ( CrawlerRun, DataClassItemsSpider, DictItemsSpider, - EngineTest, + EngineTestBase, TestSpider, ) @@ -18,7 +18,7 @@ class BytesReceivedCrawlerRun(CrawlerRun): raise StopDownload(fail=False) -class BytesReceivedEngineTest(EngineTest): +class BytesReceivedEngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index db35bd81e..4efb6b7a8 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -7,7 +7,7 @@ from tests.test_engine import ( CrawlerRun, DataClassItemsSpider, DictItemsSpider, - EngineTest, + EngineTestBase, TestSpider, ) @@ -18,7 +18,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun): raise StopDownload(fail=False) -class HeadersReceivedEngineTest(EngineTest): +class HeadersReceivedEngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( From 1a0dfbd32e8d96299c4f0f3d16cb2d52a73de339 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 2 Feb 2025 13:28:34 +0500 Subject: [PATCH 1648/2083] Reuse mockserver instances in test_feedexport.py. --- tests/test_feedexport.py | 80 ++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 49 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 4f91795e4..ae52a3e18 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -13,7 +13,6 @@ import tempfile import warnings from abc import ABC, abstractmethod from collections import defaultdict -from contextlib import ExitStack from io import BytesIO from logging import getLogger from pathlib import Path @@ -623,8 +622,6 @@ class LogOnStoreFileStorage: class FeedExportTestBase(ABC, unittest.TestCase): - __test__ = False - class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() @@ -641,8 +638,11 @@ class FeedExportTestBase(ABC, unittest.TestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp() + self.mockserver = MockServer() + self.mockserver.__enter__() def tearDown(self): + self.mockserver.__exit__(None, None, None) shutil.rmtree(self.temp_dir, ignore_errors=True) @defer.inlineCallbacks @@ -746,8 +746,6 @@ class ExceptionJsonItemExporter(JsonItemExporter): class FeedExportTest(FeedExportTestBase): - __test__ = True - @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data.""" @@ -760,10 +758,9 @@ class FeedExportTest(FeedExportTestBase): content = {} try: - with MockServer() as s: - spider_cls.start_urls = [s.url("/")] - crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + yield crawler.crawl() for file_path, feed_options in FEEDS.items(): content[feed_options["format"]] = ( @@ -890,8 +887,7 @@ class FeedExportTest(FeedExportTestBase): }, } crawler = get_crawler(ItemSpider, settings) - with MockServer() as mockserver: - yield crawler.crawl(mockserver=mockserver) + yield crawler.crawl(mockserver=self.mockserver) self.assertIn( "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() ) @@ -909,15 +905,11 @@ class FeedExportTest(FeedExportTestBase): }, } crawler = get_crawler(ItemSpider, settings) - with ExitStack() as stack: - mockserver = stack.enter_context(MockServer()) - stack.enter_context( - mock.patch( - "scrapy.extensions.feedexport.FileFeedStorage.store", - side_effect=KeyError("foo"), - ) - ) - yield crawler.crawl(mockserver=mockserver) + with mock.patch( + "scrapy.extensions.feedexport.FileFeedStorage.store", + side_effect=KeyError("foo"), + ): + yield crawler.crawl(mockserver=self.mockserver) self.assertIn( "feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats() ) @@ -938,8 +930,8 @@ class FeedExportTest(FeedExportTestBase): }, } crawler = get_crawler(ItemSpider, settings) - with MockServer() as mockserver, mock.patch.object(S3FeedStorage, "store"): - yield crawler.crawl(mockserver=mockserver) + with mock.patch.object(S3FeedStorage, "store"): + yield crawler.crawl(mockserver=self.mockserver) self.assertIn( "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() ) @@ -1730,8 +1722,6 @@ class FeedExportTest(FeedExportTestBase): class FeedPostProcessedExportsTest(FeedExportTestBase): - __test__ = True - items = [{"foo": "bar"}] expected = b"foo\r\nbar\r\n" @@ -1764,10 +1754,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): content = {} try: - with MockServer() as s: - spider_cls.start_urls = [s.url("/")] - crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + yield crawler.crawl() for file_path in FEEDS: content[str(file_path)] = ( @@ -2253,7 +2242,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): class BatchDeliveriesTest(FeedExportTestBase): - __test__ = True _file_mark = "_%(batch_time)s_#%(batch_id)02d_" @defer.inlineCallbacks @@ -2265,21 +2253,17 @@ class BatchDeliveriesTest(FeedExportTestBase): build_url(file_path): feed for file_path, feed in FEEDS.items() } content = defaultdict(list) - try: - with MockServer() as s: - spider_cls.start_urls = [s.url("/")] - crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + yield crawler.crawl() - for path, feed in FEEDS.items(): - dir_name = Path(path).parent - if not dir_name.exists(): - content[feed["format"]] = [] - continue - for file in sorted(dir_name.iterdir()): - content[feed["format"]].append(file.read_bytes()) - finally: - self.tearDown() + for path, feed in FEEDS.items(): + dir_name = Path(path).parent + if not dir_name.exists(): + content[feed["format"]] = [] + continue + for file in sorted(dir_name.iterdir()): + content[feed["format"]].append(file.read_bytes()) return content @defer.inlineCallbacks @@ -2604,8 +2588,7 @@ class BatchDeliveriesTest(FeedExportTestBase): "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } crawler = get_crawler(ItemSpider, settings) - with MockServer() as mockserver: - yield crawler.crawl(total=2, mockserver=mockserver) + yield crawler.crawl(total=2, mockserver=self.mockserver) self.assertIn( "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() ) @@ -2675,10 +2658,9 @@ class BatchDeliveriesTest(FeedExportTestBase): def parse(self, response): yield from items - with MockServer() as server: - TestSpider.start_urls = [server.url("/")] - crawler = get_crawler(TestSpider, settings) - yield crawler.crawl() + TestSpider.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(TestSpider, settings) + yield crawler.crawl() self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)) for stub in CustomS3FeedStorage.stubs[:-1]: From 783b98dedaea65e0c6658d9d588328dd887c2c8a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 2 Feb 2025 14:10:09 +0500 Subject: [PATCH 1649/2083] Make mockserver instances per-class. --- tests/test_closespider.py | 12 +++++++----- tests/test_crawl.py | 24 ++++++++++++++---------- tests/test_downloader_handlers.py | 12 +++++++----- tests/test_downloaderslotssettings.py | 16 ++++++++++------ tests/test_feedexport.py | 12 +++++++++--- tests/test_logformatter.py | 14 +++++++++----- tests/test_pipeline_crawl.py | 13 +++++++++---- tests/test_pipelines.py | 12 +++++++----- tests/test_proxy_connect.py | 12 +++++++++--- tests/test_request_attribute_binding.py | 12 +++++++----- tests/test_request_cb_kwargs.py | 12 +++++++----- tests/test_request_left.py | 12 +++++++----- tests/test_signals.py | 16 ++++++++++------ tests/test_spidermiddleware_httperror.py | 12 +++++++----- 14 files changed, 119 insertions(+), 72 deletions(-) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 9a837350f..ecde301d1 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -13,12 +13,14 @@ from tests.spiders import ( class TestCloseSpider(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_closespider_itemcount(self): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1f81a6073..cd2a559a8 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -56,12 +56,14 @@ from tests.spiders import ( class CrawlTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_follow_all(self): @@ -448,12 +450,14 @@ with multiples lines class CrawlSpiderTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def _run_spider(self, spider_cls): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index ae2030fe6..1549059f0 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -693,12 +693,14 @@ class Http11MockServerTestCase(unittest.TestCase): settings_dict: dict | None = None - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_download_with_content_length(self): diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 0bb143f69..4f8b005d7 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -50,13 +50,17 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): class CrawlTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - self.runner = CrawlerRunner() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + def setUp(self): + self.runner = CrawlerRunner() @defer.inlineCallbacks def test_delay(self): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ae52a3e18..1620d2d41 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -636,13 +636,19 @@ class FeedExportTestBase(ABC, unittest.TestCase): filename = "".join(chars) return Path(self.temp_dir, inter_dir, filename) + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + def setUp(self): self.temp_dir = tempfile.mkdtemp() - self.mockserver = MockServer() - self.mockserver.__enter__() def tearDown(self): - self.mockserver.__exit__(None, None, None) shutil.rmtree(self.temp_dir, ignore_errors=True) @defer.inlineCallbacks diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index e5d077858..962692a31 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -262,9 +262,16 @@ class DropSomeItemsPipeline: class ShowOrSkipMessagesTestCase(TwistedTestCase): + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() self.base_settings = { "LOG_LEVEL": "DEBUG", "ITEM_PIPELINES": { @@ -272,9 +279,6 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): }, } - def tearDown(self): - self.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks def test_show_messages(self): crawler = get_crawler(ItemSpider, self.base_settings) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 9e1b1ab5b..84d714e5c 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -64,10 +64,16 @@ class FileDownloadCrawlTestCase(TestCase): "ed3f6538dc15d4d9179dae57319edc5f", } - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + def setUp(self): # prepare a directory for storing files self.tmpmediastore = Path(mkdtemp()) self.settings = { @@ -80,7 +86,6 @@ class FileDownloadCrawlTestCase(TestCase): def tearDown(self): shutil.rmtree(self.tmpmediastore) self.items = [] - self.mockserver.__exit__(None, None, None) def _on_item_scraped(self, item): self.items.append(item) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 5ab288c1a..222b19e7f 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -77,12 +77,14 @@ class ItemSpider(Spider): class PipelineTestCase(unittest.TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) def _on_item_scraped(self, item): self.assertIsInstance(item, dict) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 26bd6332c..6ed7e93a6 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -62,14 +62,21 @@ def _wrong_credentials(proxy_url): class ProxyConnectTestCase(TestCase): + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + def setUp(self): try: import mitmproxy # noqa: F401 except ImportError: self.skipTest("mitmproxy is not installed") - self.mockserver = MockServer() - self.mockserver.__enter__() self._oldenv = os.environ.copy() self._proxy = MitmProxy() @@ -78,7 +85,6 @@ class ProxyConnectTestCase(TestCase): os.environ["http_proxy"] = proxy_url def tearDown(self): - self.mockserver.__exit__(None, None, None) self._proxy.stop() os.environ = self._oldenv diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index d65d74206..0072660a7 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -57,12 +57,14 @@ class AlternativeCallbacksMiddleware: class CrawlTestCase(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_response_200(self): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index b178c928b..a21cb43ff 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -154,12 +154,14 @@ class KeywordArgumentsSpider(MockServerSpider): class CallbackKeywordArgumentsTestCase(TestCase): maxDiff = None - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_callback_kwargs(self): diff --git a/tests/test_request_left.py b/tests/test_request_left.py index ba1b70695..cf4c8a2d5 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -25,12 +25,14 @@ class SignalCatcherSpider(Spider): class TestCatching(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_success(self): diff --git a/tests/test_signals.py b/tests/test_signals.py index 0df104600..1e693c094 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -21,13 +21,17 @@ class ItemSpider(Spider): class AsyncSignalTestCase(unittest.TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - self.items = [] + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + def setUp(self): + self.items = [] async def _on_item_scraped(self, item): item = await get_from_asyncio_queue(item) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 01a2b4bb4..307054de7 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -172,12 +172,14 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): class TestHttpErrorMiddlewareIntegrational(TrialTestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_middleware_works(self): From df688910e0499b0a874d220fe00ed7355a70fce0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 2 Feb 2025 18:48:26 +0500 Subject: [PATCH 1650/2083] Remove a duplicate test. --- tests/test_downloadermiddleware_httpcache.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index ec4e87ffb..a0886d9e9 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -14,7 +14,7 @@ from scrapy.utils.test import get_crawler class _BaseTest(unittest.TestCase): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" def setUp(self): @@ -161,11 +161,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): self.assertEqual(storage.dbmodule.__name__, self.dbm_module) -class FilesystemStorageTest(DefaultStorageTest): - storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" - - -class FilesystemStorageGzipTest(FilesystemStorageTest): +class FilesystemStorageGzipTest(DefaultStorageTest): def _get_settings(self, **new_settings): new_settings.setdefault("HTTPCACHE_GZIP", True) return super()._get_settings(**new_settings) From 393ff96e45ffcb738bc9f8d3240cd4d999bd07df Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 7 Nov 2024 21:21:17 +0500 Subject: [PATCH 1651/2083] Deprecate AjaxCrawlMiddleware. --- docs/topics/broad-crawls.rst | 24 ------------------- docs/topics/downloader-middleware.rst | 3 +-- scrapy/downloadermiddlewares/ajaxcrawl.py | 11 +++++++-- ...test_downloadermiddleware_ajaxcrawlable.py | 3 +++ 4 files changed, 13 insertions(+), 28 deletions(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 0286c3354..248e38b61 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -182,30 +182,6 @@ To disable redirects use: REDIRECT_ENABLED = False -Enable crawling of "Ajax Crawlable Pages" -========================================= - -Some pages (up to 1%, based on empirical data from year 2013) declare -themselves as ajax crawlable. This means they provide plain HTML -version of content that is usually available only via AJAX. -Pages can indicate it in two ways: - -1) by using ``#!`` in URL - this is the default way; -2) by using a special meta tag - this way is used on - "main", "index" website pages. - -Scrapy handles (1) automatically; to handle (2) enable -:ref:`AjaxCrawlMiddleware <ajaxcrawl-middleware>`: - -.. code-block:: python - - AJAXCRAWL_ENABLED = True - -When doing broad crawls it's common to crawl a lot of "index" web pages; -AjaxCrawlMiddleware helps to crawl them correctly. -It is turned OFF by default because it has some performance overhead, -and enabling it for focused crawls doesn't make much sense. - .. _broad-crawls-bfo: Crawl in BFO order diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1ab8f588f..ca597291f 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1249,8 +1249,7 @@ AJAXCRAWL_ENABLED Default: ``False`` -Whether the AjaxCrawlMiddleware will be enabled. You may want to -enable it for :ref:`broad crawls <topics-broad-crawls>`. +Whether the AjaxCrawlMiddleware will be enabled. HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 166192b4f..c6a55732d 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -3,10 +3,11 @@ from __future__ import annotations import logging import re from typing import TYPE_CHECKING +from warnings import warn from w3lib import html -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response if TYPE_CHECKING: @@ -30,6 +31,13 @@ class AjaxCrawlMiddleware: if not settings.getbool("AJAXCRAWL_ENABLED"): raise NotConfigured + warn( + "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware is deprecated" + " and will be removed in a future Scrapy version.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + # XXX: Google parses at least first 100k bytes; scrapy's redirect # middleware parses first 4k. 4k turns out to be insufficient # for this middleware, and parsing 100k could be slow. @@ -75,7 +83,6 @@ class AjaxCrawlMiddleware: return _has_ajaxcrawlable_meta(body) -# XXX: move it to w3lib? _ajax_crawlable_re: re.Pattern[str] = re.compile( r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>' ) diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 043dc0a12..63bd158f6 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -1,5 +1,7 @@ import unittest +import pytest + from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider @@ -8,6 +10,7 @@ from scrapy.utils.test import get_crawler __doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"] +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class AjaxCrawlMiddlewareTest(unittest.TestCase): def setUp(self): crawler = get_crawler(Spider, {"AJAXCRAWL_ENABLED": True}) From 4842bcbf1da41029a604d1bf743d4dc893960d39 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 2 Feb 2025 23:23:51 +0500 Subject: [PATCH 1652/2083] Deprecate and disable escape_ajax(). --- scrapy/downloadermiddlewares/ajaxcrawl.py | 4 ++-- scrapy/http/request/__init__.py | 4 +--- scrapy/utils/url.py | 6 ++++++ tests/test_http_request.py | 12 ------------ 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index c6a55732d..e7a8962a1 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -9,6 +9,7 @@ from w3lib import html from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response +from scrapy.utils.url import escape_ajax if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -64,8 +65,7 @@ class AjaxCrawlMiddleware: if not self._has_ajax_crawlable_variant(response): return response - # scrapy already handles #! links properly - ajax_crawl_request = request.replace(url=request.url + "#!") + ajax_crawl_request = request.replace(url=escape_ajax(request.url + "#!")) logger.debug( "Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s", {"ajax_crawl_request": ajax_crawl_request, "request": request}, diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 4eee5ffbb..e24f6874d 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -27,7 +27,6 @@ from scrapy.http.headers import Headers from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref -from scrapy.utils.url import escape_ajax if TYPE_CHECKING: from collections.abc import Callable, Iterable, Mapping @@ -170,8 +169,7 @@ class Request(object_ref): if not isinstance(url, str): raise TypeError(f"Request url must be str, got {type(url).__name__}") - s = safe_url_string(url, self.encoding) - self._url = escape_ajax(s) + self._url = safe_url_string(url, self.encoding) if ( "://" not in self._url diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index db2749d79..1348cc992 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -10,6 +10,7 @@ import warnings from importlib import import_module from typing import TYPE_CHECKING, Union from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse +from warnings import warn from w3lib.url import __all__ as _public_w3lib_objects from w3lib.url import add_or_replace_parameter as _add_or_replace_parameter @@ -83,6 +84,11 @@ def escape_ajax(url: str) -> str: >>> escape_ajax("www.example.com/ajax.html") 'www.example.com/ajax.html' """ + warn( + "escape_ajax() is deprecated and will be removed in a future Scrapy version.", + ScrapyDeprecationWarning, + stacklevel=2, + ) defrag, frag = urldefrag(url) if not frag.startswith("!"): return url diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9915aaca4..a8ab8240f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -187,18 +187,6 @@ class RequestTest(unittest.TestCase): assert isinstance(r4.body, bytes) self.assertEqual(r4.body, b"Price: \xa3100") - def test_ajax_url(self): - # ascii url - r = self.request_class(url="http://www.example.com/ajax.html#!key=value") - self.assertEqual( - r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue" - ) - # unicode url - r = self.request_class(url="http://www.example.com/ajax.html#!key=value") - self.assertEqual( - r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue" - ) - def test_copy(self): """Test Request copy""" From 76a8badd24cea6509df24e070f7fc06f47ee9ac3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 3 Feb 2025 14:55:10 +0500 Subject: [PATCH 1653/2083] Add a deprecation notice to the AjaxCrawlMiddleware docs. --- docs/topics/downloader-middleware.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index ca597291f..33308940c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1251,6 +1251,10 @@ Default: ``False`` Whether the AjaxCrawlMiddleware will be enabled. + .. note:: + + This middleware is deprecated and will be removed in a future Scrapy release. + HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From ba5df629a2004ca0d919d8b7f0a7f5725448e50a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 3 Feb 2025 19:11:47 +0400 Subject: [PATCH 1654/2083] Refactor downloader tests (#6647) * Make download handler test base classes abstract. * Small cleanup. * Don't run the full test suite for special HTTP cases. * Don't run tests in imported base classes. * Remove an obsolete service_identity check. * Move FTP imports back to the top level. * Simplify the H2DownloadHandler import. * Forbig pytest 8.2.x. * Revert "Simplify the H2DownloadHandler import." This reverts commit ed187046ac53c395c7423c0f5e6fb2bc7c27838f. --- pyproject.toml | 3 + tests/test_downloader_handlers.py | 150 +++++++++++++----------- tests/test_downloader_handlers_http2.py | 134 +++++++++++---------- tox.ini | 2 +- 4 files changed, 149 insertions(+), 140 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 29e26399f..1072730c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -180,6 +180,7 @@ disable = [ "unused-argument", "unused-import", "unused-variable", + "useless-import-alias", # used as a hint to mypy "useless-return", # https://github.com/pylint-dev/pylint/issues/6530 "wrong-import-position", @@ -319,6 +320,8 @@ ignore = [ "D403", # `try`-`except` within a loop incurs performance overhead "PERF203", + # Import alias does not rename original package + "PLC0414", # Too many return statements "PLR0911", # Too many branches diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 1549059f0..323a51002 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -4,6 +4,7 @@ import contextlib import os import shutil import sys +from abc import ABC, abstractmethod from pathlib import Path from tempfile import mkdtemp, mkstemp from unittest import SkipTest, mock @@ -12,17 +13,18 @@ import pytest from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal from twisted.internet import defer, error, reactor +from twisted.protocols.ftp import FTPFactory, FTPRealm from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util -from twisted.web._newclient import ResponseFailed +from twisted.web.client import ResponseFailed from twisted.web.http import _DataLoss from w3lib.url import path_to_file_uri -from scrapy.core.downloader.handlers import DownloadHandlers +from scrapy.core.downloader.handlers import DownloadHandlerProtocol, DownloadHandlers from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler -from scrapy.core.downloader.handlers.http import HTTPDownloadHandler +from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler @@ -183,10 +185,7 @@ class BrokenDownloadResource(resource.Resource): def closeConnection(request): # We have to force a disconnection for HTTP/1.1 clients. Otherwise # client keeps the connection open waiting for more data. - if hasattr(request.channel, "loseConnection"): # twisted >=16.3.0 - request.channel.loseConnection() - else: - request.channel.transport.loseConnection() + request.channel.loseConnection() request.finish() @@ -218,14 +217,18 @@ class DuplicateHeaderResource(resource.Resource): return b"" -class HttpTestCase(unittest.TestCase): +class HttpTestCase(unittest.TestCase, ABC): scheme = "http" - download_handler_cls: type = HTTPDownloadHandler # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + def setUp(self): self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") @@ -426,7 +429,9 @@ class HttpTestCase(unittest.TestCase): class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" - download_handler_cls: type = HTTP10DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP10DownloadHandler def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -443,7 +448,9 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls: type = HTTP11DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler def test_download_without_maxsize_limit(self): request = Request(self.getURL("file")) @@ -604,50 +611,16 @@ class Https11TestCase(Http11TestCase): yield download_handler.close() -class Https11WrongHostnameTestCase(Http11TestCase): - scheme = "https" - - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = "keys/example-com.key.pem" - certfile = "keys/example-com.cert.pem" - - -class Https11InvalidDNSId(Https11TestCase): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super().setUp() - self.host = "127.0.0.1" - - -class Https11InvalidDNSPattern(Https11TestCase): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = "keys/localhost.ip.key" - certfile = "keys/localhost.ip.crt" - - def setUp(self): - try: - from service_identity.exceptions import CertificateError # noqa: F401 - except ImportError: - raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' - 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' - ) - super().setUp() - - -class Https11CustomCiphers(unittest.TestCase): - scheme = "https" - download_handler_cls: type = HTTP11DownloadHandler +class SimpleHttpsTest(unittest.TestCase): + """Base class for special cases tested with just one simple request""" keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" + cipher_string: str | None = None + + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler def setUp(self): self.tmpname = Path(mkdtemp()) @@ -659,14 +632,16 @@ class Https11CustomCiphers(unittest.TestCase): 0, self.site, ssl_context_factory( - self.keyfile, self.certfile, cipher_string="CAMELLIA256-SHA" + self.keyfile, self.certfile, cipher_string=self.cipher_string ), interface=self.host, ) self.portno = self.port.getHost().port - crawler = get_crawler( - settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"} - ) + if self.cipher_string is not None: + settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string} + else: + settings_dict = None + crawler = get_crawler(settings_dict=settings_dict) self.download_handler = build_from_crawler(self.download_handler_cls, crawler) self.download_request = self.download_handler.download_request @@ -678,7 +653,7 @@ class Https11CustomCiphers(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return f"{self.scheme}://{self.host}:{self.portno}/{path}" + return f"https://{self.host}:{self.portno}/{path}" def test_download(self): request = Request(self.getURL("file")) @@ -688,10 +663,40 @@ class Https11CustomCiphers(unittest.TestCase): return d +class Https11WrongHostnameTestCase(SimpleHttpsTest): + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = "keys/example-com.key.pem" + certfile = "keys/example-com.cert.pem" + + +class Https11InvalidDNSId(SimpleHttpsTest): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super().setUp() + self.host = "127.0.0.1" + + +class Https11InvalidDNSPattern(SimpleHttpsTest): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = "keys/localhost.ip.key" + certfile = "keys/localhost.ip.crt" + + +class Https11CustomCiphers(SimpleHttpsTest): + cipher_string = "CAMELLIA256-SHA" + + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" settings_dict: dict | None = None + is_secure = False @classmethod def setUpClass(cls): @@ -709,7 +714,8 @@ class Http11MockServerTestCase(unittest.TestCase): # download it yield crawler.crawl( seed=Request( - url=self.mockserver.url("/partial"), meta={"download_maxsize": 1000} + url=self.mockserver.url("/partial", is_secure=self.is_secure), + meta={"download_maxsize": 1000}, ) ) failure = crawler.spider.meta["failure"] @@ -718,7 +724,9 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download(self): crawler = get_crawler(SingleRequestSpider, self.settings_dict) - yield crawler.crawl(seed=Request(url=self.mockserver.url(""))) + yield crawler.crawl( + seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) + ) failure = crawler.spider.meta.get("failure") self.assertTrue(failure is None) reason = crawler.spider.meta["close_reason"] @@ -740,10 +748,14 @@ class UriResource(resource.Resource): return b"" -class HttpProxyTestCase(unittest.TestCase): - download_handler_cls: type = HTTPDownloadHandler +class HttpProxyTestCase(unittest.TestCase, ABC): expected_http_proxy_request_body = b"http://example.com" + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + def setUp(self): site = server.Site(UriResource(), timeout=None) wrapper = WrappingFactory(site) @@ -785,11 +797,15 @@ class HttpProxyTestCase(unittest.TestCase): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class Http10ProxyTestCase(HttpProxyTestCase): - download_handler_cls: type = HTTP10DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP10DownloadHandler class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls: type = HTTP11DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -1008,10 +1024,6 @@ class BaseFTPTestCase(unittest.TestCase): ) def setUp(self): - from twisted.protocols.ftp import FTPFactory, FTPRealm - - from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler - # setup dirs and test file self.directory = Path(mkdtemp()) userdir = self.directory / self.username @@ -1155,10 +1167,6 @@ class AnonymousFTPTestCase(BaseFTPTestCase): req_meta = {} def setUp(self): - from twisted.protocols.ftp import FTPFactory, FTPRealm - - from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler - # setup dir and test file self.directory = Path(mkdtemp()) for filename, content in self.test_files: diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 322075043..174bf841e 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -1,7 +1,7 @@ import json -from unittest import mock, skipIf +from unittest import mock -from pytest import mark +import pytest from testfixtures import LogCapture from twisted.internet import defer, error, reactor from twisted.trial import unittest @@ -9,30 +9,60 @@ from twisted.web import server from twisted.web.error import SchemeNotSupported from twisted.web.http import H2_ENABLED +from scrapy.core.downloader.handlers import DownloadHandlerProtocol from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory from tests.test_downloader_handlers import ( - Http11MockServerTestCase, - Http11ProxyTestCase, - Https11CustomCiphers, - Https11TestCase, UriResource, ) +pytestmark = pytest.mark.skipif( + not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" +) -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") -class Https2TestCase(Https11TestCase): + +class BaseTestClasses: + # A hack to prevent tests from the imported classes to run here too. + # See https://stackoverflow.com/q/1323455/113586 for other ways. + from tests.test_downloader_handlers import ( + Http11MockServerTestCase as Http11MockServerTestCase, + ) + from tests.test_downloader_handlers import ( + Http11ProxyTestCase as Http11ProxyTestCase, + ) + from tests.test_downloader_handlers import ( + Https11CustomCiphers as Https11CustomCiphers, + ) + from tests.test_downloader_handlers import ( + Https11InvalidDNSId as Https11InvalidDNSId, + ) + from tests.test_downloader_handlers import ( + Https11InvalidDNSPattern as Https11InvalidDNSPattern, + ) + from tests.test_downloader_handlers import ( + Https11TestCase as Https11TestCase, + ) + from tests.test_downloader_handlers import ( + Https11WrongHostnameTestCase as Https11WrongHostnameTestCase, + ) + + +def _get_dh() -> type[DownloadHandlerProtocol]: + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + + return H2DownloadHandler + + +class Https2TestCase(BaseTestClasses.Https11TestCase): scheme = "https" HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - @classmethod - def setUpClass(cls): - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - cls.download_handler_cls = H2DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -99,7 +129,7 @@ class Https2TestCase(Https11TestCase): return defer.DeferredList([d1, d2]) - @mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") + @pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") def test_connect_request(self): request = Request(self.getURL("file"), method="CONNECT") d = self.download_request(request, Spider("foo")) @@ -150,61 +180,31 @@ class Https2TestCase(Https11TestCase): return d -class Https2WrongHostnameTestCase(Https2TestCase): - tls_log_message = ( - 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' - 'subject "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' - ) - - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = "keys/example-com.key.pem" - certfile = "keys/example-com.cert.pem" +class Https2WrongHostnameTestCase(BaseTestClasses.Https11WrongHostnameTestCase): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() -class Https2InvalidDNSId(Https2TestCase): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super().setUp() - self.host = "127.0.0.1" +class Https2InvalidDNSId(BaseTestClasses.Https11InvalidDNSId): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() -class Https2InvalidDNSPattern(Https2TestCase): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = "keys/localhost.ip.key" - certfile = "keys/localhost.ip.crt" - - def setUp(self): - try: - from service_identity.exceptions import CertificateError # noqa: F401 - except ImportError: - raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' - 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' - ) - super().setUp() +class Https2InvalidDNSPattern(BaseTestClasses.Https11InvalidDNSPattern): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") -class Https2CustomCiphers(Https11CustomCiphers): - scheme = "https" - - @classmethod - def setUpClass(cls): - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - cls.download_handler_cls = H2DownloadHandler +class Https2CustomCiphers(BaseTestClasses.Https11CustomCiphers): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() -class Http2MockServerTestCase(Http11MockServerTestCase): +class Http2MockServerTestCase(BaseTestClasses.Http11MockServerTestCase): """HTTP 2.0 test case with MockServer""" settings_dict = { @@ -212,10 +212,10 @@ class Http2MockServerTestCase(Http11MockServerTestCase): "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler" } } + is_secure = True -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") -class Https2ProxyTestCase(Http11ProxyTestCase): +class Https2ProxyTestCase(BaseTestClasses.Http11ProxyTestCase): # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" @@ -225,11 +225,9 @@ class Https2ProxyTestCase(Http11ProxyTestCase): expected_http_proxy_request_body = b"/" - @classmethod - def setUpClass(cls): - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - cls.download_handler_cls = H2DownloadHandler + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() def setUp(self): site = server.Site(UriResource(), timeout=None) diff --git a/tox.ini b/tox.ini index 0f91db19d..82ad84c90 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ deps = pexpect >= 4.8.0 pyftpdlib >= 2.0.1 pygments - pytest + pytest != 8.2.* # https://github.com/pytest-dev/pytest/issues/12275 pytest-cov >= 4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 From 9d35428770326a3e833a2720c4f641fa70b58d29 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <laertefbk@gmail.com> Date: Wed, 5 Feb 2025 06:48:56 -0300 Subject: [PATCH 1655/2083] Remove deprecated signals --- scrapy/signals.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scrapy/signals.py b/scrapy/signals.py index 0090f1c8b..8ef0f34f0 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -24,12 +24,3 @@ item_dropped = object() item_error = object() feed_slot_closed = object() feed_exporter_closed = object() - -# for backward compatibility -stats_spider_opened = spider_opened -stats_spider_closing = spider_closed -stats_spider_closed = spider_closed - -item_passed = item_scraped - -request_received = request_scheduled From 2eb3c75c697685af595b08023b4dc27d49403274 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <laertefbk@gmail.com> Date: Wed, 5 Feb 2025 13:16:51 -0300 Subject: [PATCH 1656/2083] Remove AjaxCrawlMiddleware mention from built-in downloader middleware --- docs/topics/downloader-middleware.rst | 76 +++++++-------------------- 1 file changed, 20 insertions(+), 56 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 33308940c..ab7e6a0ec 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -763,6 +763,26 @@ HttpProxyMiddleware Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. +HttpProxyMiddleware settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: HTTPPROXY_ENABLED +.. setting:: HTTPPROXY_AUTH_ENCODING + +HTTPPROXY_ENABLED +^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether or not to enable the :class:`HttpProxyMiddleware`. + +HTTPPROXY_AUTH_ENCODING +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``"latin-1"`` + +The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. + OffsiteMiddleware ----------------- @@ -1220,60 +1240,4 @@ UserAgentMiddleware In order for a spider to override the default user agent, its ``user_agent`` attribute must be set. -.. _ajaxcrawl-middleware: - -AjaxCrawlMiddleware -------------------- - -.. module:: scrapy.downloadermiddlewares.ajaxcrawl - -.. class:: AjaxCrawlMiddleware - - Middleware that finds 'AJAX crawlable' page variants based - on meta-fragment html tag. - - .. note:: - - Scrapy finds 'AJAX crawlable' pages for URLs like - ``'http://example.com/!#foo=bar'`` even without this middleware. - AjaxCrawlMiddleware is necessary when URL doesn't contain ``'!#'``. - This is often a case for 'index' or 'main' website pages. - -AjaxCrawlMiddleware Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. setting:: AJAXCRAWL_ENABLED - -AJAXCRAWL_ENABLED -^^^^^^^^^^^^^^^^^ - -Default: ``False`` - -Whether the AjaxCrawlMiddleware will be enabled. - - .. note:: - - This middleware is deprecated and will be removed in a future Scrapy release. - -HttpProxyMiddleware settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. setting:: HTTPPROXY_ENABLED -.. setting:: HTTPPROXY_AUTH_ENCODING - -HTTPPROXY_ENABLED -^^^^^^^^^^^^^^^^^ - -Default: ``True`` - -Whether or not to enable the :class:`HttpProxyMiddleware`. - -HTTPPROXY_AUTH_ENCODING -^^^^^^^^^^^^^^^^^^^^^^^ - -Default: ``"latin-1"`` - -The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. - - .. _DBM: https://en.wikipedia.org/wiki/Dbm From 4e0a3087e4f4f2bc118d0f09b71e7440e78c42d7 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 6 Feb 2025 07:47:39 -0300 Subject: [PATCH 1657/2083] fix: Reactor info logged twice (#6657) * fix: Reactor info logged twice * Change condition syntax * Simplify logic * Format --- scrapy/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1aa68cb00..1873c90d3 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -120,12 +120,12 @@ class Crawler: install_reactor(reactor_class, event_loop) else: from twisted.internet import reactor # noqa: F401 - log_reactor_info() if reactor_class: verify_installed_reactor(reactor_class) if is_asyncio_reactor_installed() and event_loop: verify_installed_asyncio_event_loop(event_loop) + if self._init_reactor or reactor_class: log_reactor_info() self.extensions = ExtensionManager.from_crawler(self) From f041f26a6ff636b764d2bf584ddbc9b9e4334d1b Mon Sep 17 00:00:00 2001 From: anubhav <protokoul@users.noreply.github.com> Date: Thu, 6 Feb 2025 22:37:07 +0530 Subject: [PATCH 1658/2083] Support dark mode in the documentation (#6653) --- docs/_static/custom.css | 48 ++++++++++++++++++++++++++++++++++++++++- docs/conf.py | 3 +++ docs/requirements.txt | 1 + 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 64f16939c..1c2859deb 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -7,4 +7,50 @@ } .rst-content dl p + ol, .rst-content dl p + ul { margin-top: -6px; /* Compensates margin-top: 12px of p */ -} \ No newline at end of file +} + +/*override some styles in +sphinx-rtd-dark-mode/static/dark_mode_css/general.css*/ +.theme-switcher { + right: 0.4em !important; + top: 0.6em !important; + -webkit-box-shadow: 0px 3px 14px 4px rgba(0, 0, 0, 0.30) !important; + box-shadow: 0px 3px 14px 4px rgba(0, 0, 0, 0.30) !important; + height: 2em !important; + width: 2em !important; +} + +/*place the toggle button for dark mode +at the bottom right corner on small screens*/ +@media (max-width: 768px) { + .theme-switcher { + right: 0.4em !important; + bottom: 2.6em !important; + top: auto !important; + } +} + +/*persist blue color at the top left used in +default rtd theme*/ +html[data-theme="dark"] .wy-side-nav-search, +html[data-theme="dark"] .wy-nav-top { + background-color: #1d577d !important; +} + +/*all the styles below used to present +API objects nicely in dark mode*/ +html[data-theme="dark"] .sig.sig-object { + border-left-color: #3e4446 !important; + background-color: #202325 !important +} + +html[data-theme="dark"] .sig-name, +html[data-theme="dark"] .sig-prename, +html[data-theme="dark"] .property, +html[data-theme="dark"] .sig-param, +html[data-theme="dark"] .sig-paren, +html[data-theme="dark"] .sig-return-icon, +html[data-theme="dark"] .sig-return-typehint, +html[data-theme="dark"] .optional { + color: #e8e6e3 !important +} diff --git a/docs/conf.py b/docs/conf.py index be5e07195..1167ce050 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,6 +35,7 @@ extensions = [ "sphinx.ext.coverage", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", + "sphinx_rtd_dark_mode", ] templates_path = ["_templates"] @@ -174,3 +175,5 @@ hoverxref_role_types = { "signal": "tooltip", } hoverxref_roles = ["command", "reqmeta", "setting", "signal"] + +default_dark_mode = False diff --git a/docs/requirements.txt b/docs/requirements.txt index e2abe76d9..103fb08d6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,3 +2,4 @@ sphinx==8.1.3 sphinx-hoverxref==1.4.2 sphinx-notfound-page==1.0.4 sphinx-rtd-theme==3.0.2 +sphinx-rtd-dark-mode==1.3.0 From d8978d405c32ee63375e09bf0b66b1e803da3d08 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sat, 8 Feb 2025 18:41:27 +0500 Subject: [PATCH 1659/2083] Improve diagnostics for sync-only spider middlewares. --- scrapy/core/spidermw.py | 30 ++++++++++++++++++------------ tests/test_spidermiddleware.py | 16 ++++++++++------ 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 4b2520aa1..c7706bb7b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -27,7 +27,7 @@ from scrapy.utils.defer import ( maybe_deferred_to_future, mustbe_deferred, ) -from scrapy.utils.python import MutableAsyncChain, MutableChain +from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_name if TYPE_CHECKING: from collections.abc import Generator @@ -51,10 +51,6 @@ def _isiterable(o: Any) -> bool: class SpiderMiddlewareManager(MiddlewareManager): component_name = "spider middleware" - def __init__(self, *middlewares: Any): - super().__init__(*middlewares) - self.downgrade_warning_done = False - @classmethod def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) @@ -227,12 +223,13 @@ class SpiderMiddlewareManager(MiddlewareManager): # Iterable -> AsyncIterable result = as_async_generator(result) elif need_downgrade: - if not self.downgrade_warning_done: - logger.warning( - f"Async iterable passed to {method.__qualname__} " - f"was downgraded to a non-async one" - ) - self.downgrade_warning_done = True + logger.warning( + f"Async iterable passed to {method.__qualname__} was" + f" downgraded to a non-async one. This is deprecated and will" + f" stop working in a future version of Scrapy. Please see" + f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares" + f" for more information." + ) assert isinstance(result, AsyncIterable) # AsyncIterable -> Iterable result = yield deferred_from_coro(collect_asyncgen(result)) @@ -340,10 +337,19 @@ class SpiderMiddlewareManager(MiddlewareManager): methodname_async = methodname + "_async" async_method: Callable | None = getattr(mw, methodname_async, None) if not async_method: + if normal_method and not isasyncgenfunction(normal_method): + logger.warning( + f"Middleware {global_object_name(mw.__class__)} doesn't support" + f" asynchronous spider output, this is deprecated and will stop" + f" working in a future version of Scrapy. The middleware should" + f" be updated to support it. Please see" + f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares" + f" for more information." + ) return normal_method if not normal_method: logger.error( - f"Middleware {mw.__qualname__} has {methodname_async} " + f"Middleware {global_object_name(mw.__class__)} has {methodname_async} " f"without {methodname}, skipping this method." ) return None diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index ba64ba721..a8507c789 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -152,6 +152,10 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) self.assertEqual("downgraded to a non-async" in str(log), downgrade) + self.assertEqual( + "doesn't support asynchronous spider output" in str(log), + ProcessSpiderOutputSimpleMiddleware in mw_classes, + ) @defer.inlineCallbacks def _test_asyncgen_base( @@ -376,21 +380,21 @@ class UniversalMiddlewareManagerTest(TestCase): self.mwman = SpiderMiddlewareManager() def test_simple_mw(self): - mw = ProcessSpiderOutputSimpleMiddleware + mw = ProcessSpiderOutputSimpleMiddleware() self.mwman._add_middleware(mw) self.assertEqual( self.mwman.methods["process_spider_output"][0], mw.process_spider_output ) def test_async_mw(self): - mw = ProcessSpiderOutputAsyncGenMiddleware + mw = ProcessSpiderOutputAsyncGenMiddleware() self.mwman._add_middleware(mw) self.assertEqual( self.mwman.methods["process_spider_output"][0], mw.process_spider_output ) def test_universal_mw(self): - mw = ProcessSpiderOutputUniversalMiddleware + mw = ProcessSpiderOutputUniversalMiddleware() self.mwman._add_middleware(mw) self.assertEqual( self.mwman.methods["process_spider_output"][0], @@ -399,7 +403,7 @@ class UniversalMiddlewareManagerTest(TestCase): def test_universal_mw_no_sync(self): with LogCapture() as log: - self.mwman._add_middleware(UniversalMiddlewareNoSync) + self.mwman._add_middleware(UniversalMiddlewareNoSync()) self.assertIn( "UniversalMiddlewareNoSync has process_spider_output_async" " without process_spider_output", @@ -408,7 +412,7 @@ class UniversalMiddlewareManagerTest(TestCase): self.assertEqual(self.mwman.methods["process_spider_output"][0], None) def test_universal_mw_both_sync(self): - mw = UniversalMiddlewareBothSync + mw = UniversalMiddlewareBothSync() with LogCapture() as log: self.mwman._add_middleware(mw) self.assertIn( @@ -422,7 +426,7 @@ class UniversalMiddlewareManagerTest(TestCase): def test_universal_mw_both_async(self): with LogCapture() as log: - self.mwman._add_middleware(UniversalMiddlewareBothAsync) + self.mwman._add_middleware(UniversalMiddlewareBothAsync()) self.assertIn( "UniversalMiddlewareBothAsync.process_spider_output " "is an async generator function while process_spider_output_async exists", From ede9e9c3c3f9a9049fea2a6be0339b2c7434b8a1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 11 Feb 2025 23:07:25 +0500 Subject: [PATCH 1660/2083] Use full method names in all spidermw log messages. --- scrapy/core/spidermw.py | 16 ++++++++-------- scrapy/utils/python.py | 4 +++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c7706bb7b..86d11c0e0 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -79,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager): result = method(response=response, spider=spider) if result is not None: msg = ( - f"{method.__qualname__} must return None " + f"{global_object_name(method)} must return None " f"or raise an exception, got {type(result)}" ) raise _InvalidOutput(msg) @@ -168,12 +168,12 @@ class SpiderMiddlewareManager(MiddlewareManager): ) # we forbid waiting here because otherwise we would need to return a deferred from # _process_spider_exception too, which complicates the architecture - msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" + msg = f"Async iterable returned from {global_object_name(method)} cannot be downgraded" raise _InvalidOutput(msg) if result is None: continue msg = ( - f"{method.__qualname__} must return None " + f"{global_object_name(method)} must return None " f"or an iterable, got {type(result)}" ) raise _InvalidOutput(msg) @@ -224,7 +224,7 @@ class SpiderMiddlewareManager(MiddlewareManager): result = as_async_generator(result) elif need_downgrade: logger.warning( - f"Async iterable passed to {method.__qualname__} was" + f"Async iterable passed to {global_object_name(method)} was" f" downgraded to a non-async one. This is deprecated and will" f" stop working in a future version of Scrapy. Please see" f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares" @@ -257,12 +257,12 @@ class SpiderMiddlewareManager(MiddlewareManager): if iscoroutine(result): result.close() # Silence warning about not awaiting msg = ( - f"{method.__qualname__} must be an asynchronous " + f"{global_object_name(method)} must be an asynchronous " f"generator (i.e. use yield)" ) else: msg = ( - f"{method.__qualname__} must return an iterable, got " + f"{global_object_name(method)} must return an iterable, got " f"{type(result)}" ) raise _InvalidOutput(msg) @@ -355,13 +355,13 @@ class SpiderMiddlewareManager(MiddlewareManager): return None if not isasyncgenfunction(async_method): logger.error( - f"{async_method.__qualname__} is not " + f"{global_object_name(async_method)} is not " f"an async generator function, skipping this method." ) return normal_method if isasyncgenfunction(normal_method): logger.error( - f"{normal_method.__qualname__} is an async " + f"{global_object_name(normal_method)} is an async " f"generator function while {methodname_async} exists, " f"skipping both methods." ) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index fcf582082..2e6869779 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -326,11 +326,13 @@ def without_none_values( def global_object_name(obj: Any) -> str: - """Return the full import path of the given class. + """Return the full import path of the given object. >>> from scrapy import Request >>> global_object_name(Request) 'scrapy.http.request.Request' + >>> global_object_name(Request.replace) + 'scrapy.http.request.Request.replace' """ return f"{obj.__module__}.{obj.__qualname__}" From 7d5b189c1147e8aad632d4ef6759cc391d2017ac Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 14 Feb 2025 19:40:06 +0400 Subject: [PATCH 1661/2083] Fix getting annotations for _parse_sitemap() at the runtime. (#6671) * Fix getting annotations for _parse_sitemap() at the runtime. * Split off the callback annotations test. --- scrapy/spiders/sitemap.py | 5 +++-- tests/test_poet.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tests/test_poet.py diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 91c7e3be9..39033ac3c 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -2,6 +2,9 @@ from __future__ import annotations import logging import re + +# Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation +from collections.abc import Iterable, Sequence # noqa: TC003 from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse @@ -11,8 +14,6 @@ from scrapy.utils.gz import gunzip, gzip_magic_number from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots if TYPE_CHECKING: - from collections.abc import Iterable, Sequence - # typing.Self requires Python 3.11 from typing_extensions import Self diff --git a/tests/test_poet.py b/tests/test_poet.py new file mode 100644 index 000000000..9601c75a1 --- /dev/null +++ b/tests/test_poet.py @@ -0,0 +1,20 @@ +"""Tests that make sure parts needed for the scrapy-poet stack work.""" + +from typing import get_type_hints + +from scrapy import Spider +from scrapy.spiders import CrawlSpider, CSVFeedSpider, SitemapSpider, XMLFeedSpider + + +def test_callbacks(): + """Making sure annotations on all non-abstract callbacks can be resolved.""" + + for cb in [ + Spider._parse, + CrawlSpider._parse, + CrawlSpider._callback, + XMLFeedSpider._parse, + CSVFeedSpider._parse, + SitemapSpider._parse_sitemap, + ]: + get_type_hints(cb) From a898331d14f889c1d4860cf1a364ba28285090a4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 16 Feb 2025 23:28:58 +0400 Subject: [PATCH 1662/2083] Preparations for switching to direct pytest. (#6678) --- conftest.py | 2 +- pyproject.toml | 11 +++- scrapy/utils/test.py | 9 +-- tests/test_commands.py | 4 +- tests/test_contracts.py | 50 ++++++++--------- tests/test_crawl.py | 26 +++++---- tests/test_crawler.py | 36 ++++++------ tests/test_downloadermiddleware.py | 6 +- ...test_downloadermiddleware_ajaxcrawlable.py | 2 - tests/test_downloadermiddleware_httpauth.py | 12 ++-- tests/test_downloadermiddleware_httpcache.py | 6 +- tests/test_downloadermiddleware_httpproxy.py | 3 +- tests/test_downloadermiddleware_offsite.py | 28 +++++----- tests/test_downloadermiddleware_redirect.py | 4 +- tests/test_engine.py | 30 +++++----- tests/test_engine_stop_download_bytes.py | 4 +- tests/test_engine_stop_download_headers.py | 4 +- tests/test_exporters.py | 40 +++++++------- tests/test_extension_periodic_log.py | 4 +- tests/test_extension_throttle.py | 55 +++++++++---------- tests/test_http2_client_protocol.py | 3 +- tests/test_http_response.py | 12 ++-- tests/test_linkextractors.py | 4 +- tests/test_loader.py | 54 +++++++++--------- tests/test_loader_deprecated.py | 46 ++++++++-------- tests/test_middleware.py | 8 +-- tests/test_pipelines.py | 6 +- tests/test_request_dict.py | 28 ++++++---- tests/test_scheduler_base.py | 6 +- tests/test_signals.py | 4 +- tests/test_squeues.py | 26 ++++----- tests/test_utils_asyncio.py | 6 +- tests/test_utils_datatypes.py | 2 - tests/test_utils_defer.py | 4 +- tests/test_utils_log.py | 4 +- tests/test_utils_misc/__init__.py | 2 - tests/test_utils_python.py | 2 - tests/test_utils_response.py | 8 +-- tests/test_utils_signal.py | 6 +- tests/test_utils_template.py | 2 - tests/test_utils_url.py | 2 - 41 files changed, 285 insertions(+), 286 deletions(-) diff --git a/conftest.py b/conftest.py index e9765962a..a08ad9d05 100644 --- a/conftest.py +++ b/conftest.py @@ -41,7 +41,7 @@ if not H2_ENABLED: ) -@pytest.fixture() +@pytest.fixture def chdir(tmpdir): """Change to pytest-provided temporary directory""" tmpdir.chdir() diff --git a/pyproject.toml b/pyproject.toml index 1072730c0..ad62ea212 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,8 +200,7 @@ disable = [ [tool.pytest.ini_options] xfail_strict = true usefixtures = "chdir" -python_files = ["test_*.py", "__init__.py"] -python_classes = [] +python_files = ["test_*.py", "test_*/__init__.py"] addopts = [ "--assert=plain", "--ignore=docs/_ext", @@ -254,6 +253,8 @@ extend-select = [ "PIE", # pylint "PL", + # flake8-pytest-style + "PT", # flake8-use-pathlib "PTH", # flake8-pyi @@ -373,6 +374,12 @@ ignore = [ "B904", # Use capitalized environment variable "SIM112", + + # Temporarily silenced PT rules + # Use a regular `assert` instead of unittest-style `assertEqual` + "PT009", + # Use `pytest.raises` instead of unittest-style `assertRaises` + "PT027", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index a7b84baef..e89786103 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -15,9 +15,10 @@ from unittest import TestCase, mock from twisted.trial.unittest import SkipTest -from scrapy import Spider from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available +from scrapy.utils.deprecate import create_deprecated_class +from scrapy.utils.spider import DefaultSpider if TYPE_CHECKING: from collections.abc import Awaitable @@ -25,6 +26,7 @@ if TYPE_CHECKING: from twisted.internet.defer import Deferred from twisted.web.client import Response as TxResponse + from scrapy import Spider from scrapy.crawler import Crawler @@ -82,8 +84,7 @@ def get_ftp_content_and_delete( return b"".join(ftp_data) -class TestSpider(Spider): - name = "test" +TestSpider = create_deprecated_class("TestSpider", DefaultSpider) def get_crawler( @@ -101,7 +102,7 @@ def get_crawler( settings: dict[str, Any] = {} settings.update(settings_dict or {}) runner = CrawlerRunner(settings) - crawler = runner.create_crawler(spidercls or TestSpider) + crawler = runner.create_crawler(spidercls or DefaultSpider) crawler._apply_settings() return crawler diff --git a/tests/test_commands.py b/tests/test_commands.py index 872b54d04..1a0db1e03 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -19,7 +19,7 @@ from threading import Timer from typing import TYPE_CHECKING from unittest import mock, skipIf -from pytest import mark +import pytest from twisted.trial import unittest import scrapy @@ -822,7 +822,7 @@ class MySpider(scrapy.Spider): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log ) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_custom_asyncio_loop_enabled_true(self): log = self.get_log( self.debug_log_spider, diff --git a/tests/test_contracts.py b/tests/test_contracts.py index fb16140be..0f7d7b54c 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -21,7 +21,7 @@ from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -class TestItem(Item): +class DemoItem(Item): name = Field() url = Field() @@ -58,7 +58,7 @@ class CustomFormContract(Contract): return args -class TestSpider(Spider): +class DemoSpider(Spider): name = "demo_spider" def returns_request(self, response): @@ -80,7 +80,7 @@ class TestSpider(Spider): @url http://scrapy.org @returns items 1 1 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_request_cb_kwargs(self, response, url): """method which returns request @@ -96,7 +96,7 @@ class TestSpider(Spider): @cb_kwargs {"name": "Scrapy"} @returns items 1 1 """ - return TestItem(name=name, url=response.url) + return DemoItem(name=name, url=response.url) def returns_item_cb_kwargs_error_unexpected_keyword(self, response): """method which returns item @@ -104,14 +104,14 @@ class TestSpider(Spider): @cb_kwargs {"arg": "value"} @returns items 1 1 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_item_cb_kwargs_error_missing_argument(self, response, arg): """method which returns item @url http://scrapy.org @returns items 1 1 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_dict_item(self, response): """method which returns item @@ -125,7 +125,7 @@ class TestSpider(Spider): @url http://scrapy.org @returns items 0 0 """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def returns_dict_fail(self, response): """method which returns item @@ -140,7 +140,7 @@ class TestSpider(Spider): @returns items 1 1 @scrapes name url """ - return TestItem(name="test", url=response.url) + return DemoItem(name="test", url=response.url) def scrapes_dict_item_ok(self, response): """returns item with name and url @@ -156,7 +156,7 @@ class TestSpider(Spider): @returns items 1 1 @scrapes name url """ - return TestItem(url=response.url) + return DemoItem(url=response.url) def scrapes_dict_item_fail(self, response): """returns item with no name @@ -212,7 +212,7 @@ class TestSpider(Spider): @meta {"key": "example"} @returns items 1 1 """ - return TestItem(name="example", url=response.url) + return DemoItem(name="example", url=response.url) def returns_error_missing_meta(self, response): """method which depends of metadata be defined @@ -242,7 +242,7 @@ class CustomContractFailSpider(Spider): """ -class InheritsTestSpider(TestSpider): +class InheritsDemoSpider(DemoSpider): name = "inherits_demo_spider" @@ -274,7 +274,7 @@ class ContractsManagerTest(unittest.TestCase): self.assertTrue(self.results.errors) def test_contracts(self): - spider = TestSpider() + spider = DemoSpider() # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request) @@ -293,7 +293,7 @@ class ContractsManagerTest(unittest.TestCase): self.assertEqual(request, None) def test_cb_kwargs(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # extract contracts correctly @@ -356,7 +356,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_meta(self): - spider = TestSpider() + spider = DemoSpider() # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request_meta) @@ -402,7 +402,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_returns(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # returns_item @@ -431,7 +431,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_fail() def test_returns_async(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() request = self.conman.from_method(spider.returns_request_async, self.results) @@ -439,7 +439,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_scrapes(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # scrapes_item_ok @@ -472,7 +472,7 @@ class ContractsManagerTest(unittest.TestCase): assert message in self.results.failures[-1][-1] def test_regex(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() # invalid regex @@ -494,7 +494,7 @@ class ContractsManagerTest(unittest.TestCase): self.should_error() def test_errback(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() try: @@ -522,11 +522,11 @@ class ContractsManagerTest(unittest.TestCase): def parse_first(self, response): self.visited += 1 - return TestItem() + return DemoItem() def parse_second(self, response): self.visited += 1 - return TestItem() + return DemoItem() with MockServer() as mockserver: contract_doc = f"@url {mockserver.url('/status?n=200')}" @@ -540,13 +540,13 @@ class ContractsManagerTest(unittest.TestCase): self.assertEqual(crawler.spider.visited, 2) def test_form_contract(self): - spider = TestSpider() + spider = DemoSpider() request = self.conman.from_method(spider.custom_form, self.results) self.assertEqual(request.method, "POST") self.assertIsInstance(request, FormRequest) def test_inherited_contracts(self): - spider = InheritsTestSpider() + spider = InheritsDemoSpider() requests = self.conman.from_spider(spider, self.results) self.assertTrue(requests) @@ -571,7 +571,7 @@ class CustomContractPrePostProcess(unittest.TestCase): self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) def test_pre_hook_keyboard_interrupt(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPreProcess(spider.returns_request) conman = ContractsManager([contract]) @@ -590,7 +590,7 @@ class CustomContractPrePostProcess(unittest.TestCase): self.assertFalse(self.results.errors) def test_post_hook_keyboard_interrupt(self): - spider = TestSpider() + spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPostProcess(spider.returns_request) conman = ContractsManager([contract]) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index cd2a559a8..3aca2bbce 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -6,7 +6,7 @@ from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse -from pytest import mark +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.ssl import Certificate @@ -536,7 +536,7 @@ class CrawlSpiderTestCase(TestCase): ) self.assertIn("Got response 200", str(log)) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse(self): crawler = get_crawler( @@ -551,7 +551,7 @@ class CrawlSpiderTestCase(TestCase): ) self.assertIn("Got response 200", str(log)) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_items_list(self): log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) @@ -559,7 +559,7 @@ class CrawlSpiderTestCase(TestCase): self.assertIn({"id": 1}, items) self.assertIn({"id": 2}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): items = [] @@ -576,7 +576,7 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("Got response 200", str(log)) self.assertIn({"foo": 42}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) @@ -584,7 +584,7 @@ class CrawlSpiderTestCase(TestCase): itemcount = stats.get_value("item_scraped_count") self.assertEqual(itemcount, 1) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) @@ -594,7 +594,7 @@ class CrawlSpiderTestCase(TestCase): for i in range(10): self.assertIn({"foo": i}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_exc(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) @@ -606,7 +606,7 @@ class CrawlSpiderTestCase(TestCase): for i in range(7): self.assertIn({"foo": i}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) @@ -618,20 +618,20 @@ class CrawlSpiderTestCase(TestCase): for i in [10, 30, 122]: self.assertIn({"index2": i}, items) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): self.assertIn(f"Got response 200, req_id {req_id}", str(log)) - @mark.only_not_asyncio() + @pytest.mark.only_not_asyncio @defer.inlineCallbacks def test_async_def_deferred_direct(self): _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) self.assertEqual(items, [{"code": 200}]) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_deferred_wrapped(self): log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) @@ -659,7 +659,9 @@ class CrawlSpiderTestCase(TestCase): self.assertEqual(cert.getSubject().commonName, b"localhost") self.assertEqual(cert.getIssuer().commonName, b"localhost") - @mark.xfail(reason="Responses with no body return early and contain no certificate") + @pytest.mark.xfail( + reason="Responses with no body return early and contain no certificate" + ) @defer.inlineCallbacks def test_response_ssl_certificate_empty_response(self): crawler = get_crawler(SingleRequestSpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6c3fe96b0..425188d32 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -8,9 +8,9 @@ import warnings from pathlib import Path from typing import Any +import pytest from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn -from pytest import mark, raises from twisted.internet.defer import Deferred, inlineCallbacks from twisted.trial import unittest from w3lib import __version__ as w3lib_version @@ -77,14 +77,14 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") def test_crawler_rejects_spider_objects(self): - with raises(ValueError): + with pytest.raises(ValueError, match="spidercls argument must be a class"): Crawler(DefaultSpider()) @inlineCallbacks def test_crawler_crawl_twice_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) yield crawler.crawl() - with raises(RuntimeError, match="more than once on the same instance"): + with pytest.raises(RuntimeError, match="more than once on the same instance"): yield crawler.crawl() def test_get_addon(self): @@ -203,7 +203,7 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): + with pytest.raises(RuntimeError): yield crawler.crawl() @inlineCallbacks @@ -282,7 +282,7 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): + with pytest.raises(RuntimeError): yield crawler.crawl() @inlineCallbacks @@ -361,7 +361,7 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): + with pytest.raises(RuntimeError): yield crawler.crawl() @inlineCallbacks @@ -440,7 +440,7 @@ class CrawlerTestCase(BaseCrawlerTest): raise crawler = get_raw_crawler(MySpider, BASE_SETTINGS) - with raises(RuntimeError): + with pytest.raises(RuntimeError): yield crawler.crawl() @@ -575,7 +575,7 @@ class NoRequestsSpider(scrapy.Spider): return [] -@mark.usefixtures("reactor_pytest") +@pytest.mark.usefixtures("reactor_pytest") class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): return CrawlerRunner() @@ -744,7 +744,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log ) - @mark.skipif( + @pytest.mark.skipif( parse_version(w3lib_version) >= parse_version("2.0.0"), reason="w3lib 2.0.0 and later do not allow invalid domains.", ) @@ -781,7 +781,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.selectreactor.SelectReactor", log ) - @mark.skipif( + @pytest.mark.skipif( platform.system() == "Windows", reason="PollReactor is not supported on Windows" ) def test_twisted_reactor_poll(self): @@ -820,7 +820,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): log, ) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") self.assertIn("Spider closed (finished)", log) @@ -829,7 +829,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") self.assertIn("Spider closed (finished)", log) @@ -839,7 +839,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_asyncio_enabled_reactor_same_loop(self): log = self.run_script("asyncio_enabled_reactor_same_loop.py") self.assertIn("Spider closed (finished)", log) @@ -848,7 +848,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.requires_uvloop + @pytest.mark.requires_uvloop def test_asyncio_enabled_reactor_different_loop(self): log = self.run_script("asyncio_enabled_reactor_different_loop.py") self.assertNotIn("Spider closed (finished)", log) @@ -924,13 +924,13 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("DEBUG: Using asyncio event loop", log) -@mark.parametrize( - ["settings", "items"], - ( +@pytest.mark.parametrize( + ("settings", "items"), + [ ({}, default_settings.LOG_VERSIONS), ({"LOG_VERSIONS": ["itemadapter"]}, ["itemadapter"]), ({"LOG_VERSIONS": []}, None), - ), + ], ) def test_log_scrapy_info(settings, items, caplog): with caplog.at_level("INFO"): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index c581e7596..42051042c 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,7 +1,7 @@ import asyncio from unittest import mock -from pytest import mark +import pytest from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.python.failure import Failure @@ -220,7 +220,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.assertFalse(download_func.called) -@mark.usefixtures("reactor_pytest") +@pytest.mark.usefixtures("reactor_pytest") class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" @@ -243,7 +243,7 @@ class MiddlewareUsingCoro(ManagerTestCase): self.assertIs(results[0], resp) self.assertFalse(download_func.called) - @mark.only_asyncio() + @pytest.mark.only_asyncio def test_asyncdef_asyncio(self): resp = Response("http://example.com/index.html") diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 63bd158f6..76fcece4f 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -7,8 +7,6 @@ from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -__doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class AjaxCrawlMiddlewareTest(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 500af6536..581fc1974 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -7,18 +7,18 @@ from scrapy.http import Request from scrapy.spiders import Spider -class TestSpiderLegacy(Spider): +class LegacySpider(Spider): http_user = "foo" http_pass = "bar" -class TestSpider(Spider): +class DomainSpider(Spider): http_user = "foo" http_pass = "bar" http_auth_domain = "example.com" -class TestSpiderAny(Spider): +class AnyDomainSpider(Spider): http_user = "foo" http_pass = "bar" http_auth_domain = None @@ -26,7 +26,7 @@ class TestSpiderAny(Spider): class HttpAuthMiddlewareLegacyTest(unittest.TestCase): def setUp(self): - self.spider = TestSpiderLegacy("foo") + self.spider = LegacySpider("foo") def test_auth(self): with self.assertRaises(AttributeError): @@ -37,7 +37,7 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): class HttpAuthMiddlewareTest(unittest.TestCase): def setUp(self): self.mw = HttpAuthMiddleware() - self.spider = TestSpider("foo") + self.spider = DomainSpider("foo") self.mw.spider_opened(self.spider) def tearDown(self): @@ -67,7 +67,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase): class HttpAuthAnyMiddlewareTest(unittest.TestCase): def setUp(self): self.mw = HttpAuthMiddleware() - self.spider = TestSpiderAny("foo") + self.spider = AnyDomainSpider("foo") self.mw.spider_opened(self.spider) def tearDown(self): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index a0886d9e9..74db93f8a 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -353,7 +353,8 @@ class RFC2616PolicyTest(DefaultStorageTest): resc = mw.storage.retrieve_response(self.spider, req0) if shouldcache: self.assertEqualResponse(resc, res1) - assert "cached" in res2.flags and res2.status != 304 + assert "cached" in res2.flags + assert res2.status != 304 else: self.assertFalse(resc) assert "cached" not in res2.flags @@ -376,7 +377,8 @@ class RFC2616PolicyTest(DefaultStorageTest): resc = mw.storage.retrieve_response(self.spider, req0) if shouldcache: self.assertEqualResponse(resc, res1) - assert "cached" in res2.flags and res2.status != 304 + assert "cached" in res2.flags + assert res2.status != 304 else: self.assertFalse(resc) assert "cached" not in res2.flags diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 97c276b48..f0826ef5b 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -131,7 +131,8 @@ class TestHttpProxyMiddleware(TestCase): mw = HttpProxyMiddleware() req = Request("http://noproxy.com", meta={"proxy": None}) assert mw.process_request(req, spider) is None - assert "proxy" in req.meta and req.meta["proxy"] is None + assert "proxy" in req.meta + assert req.meta["proxy"] is None def test_no_proxy(self): os.environ["http_proxy"] = "https://proxy.for.http:3128" diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index 23a1d06da..cace52a27 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -12,7 +12,7 @@ UNSET = object() @pytest.mark.parametrize( ("allowed_domain", "url", "allowed"), - ( + [ ("example.com", "http://example.com/1", True), ("example.com", "http://example.org/1", False), ("example.com", "http://sub.example.com/1", True), @@ -24,7 +24,7 @@ UNSET = object() ("example.com", "http://example.com.example", False), ("a.example", "http://nota.example", False), ("b.a.example", "http://notb.a.example", False), - ), + ], ) def test_process_request_domain_filtering(allowed_domain, url, allowed): crawler = get_crawler(Spider) @@ -41,12 +41,12 @@ def test_process_request_domain_filtering(allowed_domain, url, allowed): @pytest.mark.parametrize( ("value", "filtered"), - ( + [ (UNSET, True), (None, True), (False, True), (True, False), - ), + ], ) def test_process_request_dont_filter(value, filtered): crawler = get_crawler(Spider) @@ -66,7 +66,7 @@ def test_process_request_dont_filter(value, filtered): @pytest.mark.parametrize( ("allow_offsite", "dont_filter", "filtered"), - ( + [ (True, UNSET, False), (True, None, False), (True, False, False), @@ -75,7 +75,7 @@ def test_process_request_dont_filter(value, filtered): (False, None, True), (False, False, True), (False, True, False), - ), + ], ) def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): crawler = get_crawler(Spider) @@ -97,11 +97,11 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): @pytest.mark.parametrize( "value", - ( + [ UNSET, None, [], - ), + ], ) def test_process_request_no_allowed_domains(value): crawler = get_crawler(Spider) @@ -133,7 +133,7 @@ def test_process_request_invalid_domains(): @pytest.mark.parametrize( ("allowed_domain", "url", "allowed"), - ( + [ ("example.com", "http://example.com/1", True), ("example.com", "http://example.org/1", False), ("example.com", "http://sub.example.com/1", True), @@ -145,7 +145,7 @@ def test_process_request_invalid_domains(): ("example.com", "http://example.com.example", False), ("a.example", "http://nota.example", False), ("b.a.example", "http://notb.a.example", False), - ), + ], ) def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): crawler = get_crawler(Spider) @@ -162,12 +162,12 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): @pytest.mark.parametrize( ("value", "filtered"), - ( + [ (UNSET, True), (None, True), (False, True), (True, False), - ), + ], ) def test_request_scheduled_dont_filter(value, filtered): crawler = get_crawler(Spider) @@ -187,11 +187,11 @@ def test_request_scheduled_dont_filter(value, filtered): @pytest.mark.parametrize( "value", - ( + [ UNSET, None, [], - ), + ], ) def test_request_scheduled_no_allowed_domains(value): crawler = get_crawler(Spider) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index eb3cdfc11..f950906e9 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1278,7 +1278,7 @@ class MetaRefreshMiddlewareTest(Base.Test): @pytest.mark.parametrize( SCHEME_PARAMS, - ( + [ *REDIRECT_SCHEME_CASES, # data/file/ftp/s3/foo → * does not redirect *( @@ -1300,7 +1300,7 @@ class MetaRefreshMiddlewareTest(Base.Test): for scheme in NON_HTTP_SCHEMES for location in ("//example.com/b", "/b") ), - ), + ], ) def test_meta_refresh_schemes(url, location, target): crawler = get_crawler(Spider) diff --git a/tests/test_engine.py b/tests/test_engine.py index 91ce2c0de..e9470493f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -42,7 +42,7 @@ from scrapy.utils.test import get_crawler from tests import get_testdata, tests_datadir -class TestItem(Item): +class MyItem(Item): name = Field() url = Field() price = Field() @@ -62,7 +62,7 @@ class DataClassItem: price: int = 0 -class TestSpider(Spider): +class MySpider(Spider): name = "scrapytest.org" allowed_domains = ["scrapytest.org", "localhost"] @@ -70,7 +70,7 @@ class TestSpider(Spider): name_re = re.compile(r"<h1>(.*?)</h1>", re.MULTILINE) price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE) - item_cls: type = TestItem + item_cls: type = MyItem def parse(self, response): xlink = LinkExtractor() @@ -91,24 +91,24 @@ class TestSpider(Spider): return adapter.item -class TestDupeFilterSpider(TestSpider): +class DupeFilterSpider(MySpider): def start_requests(self): return (Request(url) for url in self.start_urls) # no dont_filter=True -class DictItemsSpider(TestSpider): +class DictItemsSpider(MySpider): item_cls = dict -class AttrsItemsSpider(TestSpider): +class AttrsItemsSpider(MySpider): item_cls = AttrsItem -class DataClassItemsSpider(TestSpider): +class DataClassItemsSpider(MySpider): item_cls = DataClassItem -class ItemZeroDivisionErrorSpider(TestSpider): +class ItemZeroDivisionErrorSpider(MySpider): custom_settings = { "ITEM_PIPELINES": { "tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300, @@ -116,7 +116,7 @@ class ItemZeroDivisionErrorSpider(TestSpider): } -class ChangeCloseReasonSpider(TestSpider): +class ChangeCloseReasonSpider(MySpider): @classmethod def from_crawler(cls, crawler, *args, **kwargs): spider = cls(*args, **kwargs) @@ -388,7 +388,7 @@ class EngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( - TestSpider, + MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider, @@ -404,7 +404,7 @@ class EngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler_dupefilter(self): - run = CrawlerRun(TestDupeFilterSpider) + run = CrawlerRun(DupeFilterSpider) yield run.run() self._assert_scheduled_requests(run, count=8) self._assert_dropped_requests(run) @@ -426,13 +426,13 @@ class EngineTest(EngineTestBase): @defer.inlineCallbacks def test_close_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.close() @defer.inlineCallbacks def test_start_already_running_exception(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) + e = ExecutionEngine(get_crawler(MySpider), lambda _: None) + yield e.open_spider(MySpider(), []) e.start() try: yield self.assertFailure(e.start(), RuntimeError).addBoth( @@ -486,7 +486,7 @@ def test_request_scheduled_signal(caplog): if "drop" in request.url: raise IgnoreRequest - spider = TestSpider() + spider = MySpider() crawler = get_crawler(spider.__class__) engine = ExecutionEngine(crawler, lambda _: None) engine.downloader._slot_gc_loop.stop() diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 8bf225ab1..5dd04c310 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -8,7 +8,7 @@ from tests.test_engine import ( DataClassItemsSpider, DictItemsSpider, EngineTestBase, - TestSpider, + MySpider, ) @@ -22,7 +22,7 @@ class BytesReceivedEngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( - TestSpider, + MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider, diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 4efb6b7a8..06929d1e4 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -8,7 +8,7 @@ from tests.test_engine import ( DataClassItemsSpider, DictItemsSpider, EngineTestBase, - TestSpider, + MySpider, ) @@ -22,7 +22,7 @@ class HeadersReceivedEngineTest(EngineTestBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( - TestSpider, + MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider, diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c2cab9b2a..eb8d309b6 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -31,7 +31,7 @@ def custom_serializer(value): return str(int(value) + 2) -class TestItem(Item): +class MyItem(Item): name = Field() age = Field() @@ -42,7 +42,7 @@ class CustomFieldItem(Item): @dataclasses.dataclass -class TestDataClass: +class MyDataClass: name: str age: int @@ -54,7 +54,7 @@ class CustomFieldDataclass: class BaseItemExporterTest(unittest.TestCase): - item_class: type = TestItem + item_class: type = MyItem custom_field_item_class: type = CustomFieldItem def setUp(self): @@ -138,7 +138,7 @@ class BaseItemExporterTest(unittest.TestCase): class BaseItemExporterDataclassTest(BaseItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -207,7 +207,7 @@ class PythonItemExporterTest(BaseItemExporterTest): class PythonItemExporterDataclassTest(PythonItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -222,7 +222,7 @@ class PprintItemExporterTest(BaseItemExporterTest): class PprintItemExporterDataclassTest(PprintItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -259,7 +259,7 @@ class PickleItemExporterTest(BaseItemExporterTest): class PickleItemExporterDataclassTest(PickleItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -286,7 +286,7 @@ class MarshalItemExporterTest(BaseItemExporterTest): class MarshalItemExporterDataclassTest(MarshalItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -406,7 +406,7 @@ class CsvItemExporterTest(BaseItemExporterTest): class CsvItemExporterDataclassTest(CsvItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -517,7 +517,7 @@ class XmlItemExporterTest(BaseItemExporterTest): class XmlItemExporterDataclassTest(XmlItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -563,7 +563,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass @@ -595,11 +595,11 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) def test_two_items_with_failure_between(self): - i1 = TestItem(name="Joseph\xa3", age="22") - i2 = TestItem( + i1 = MyItem(name="Joseph\xa3", age="22") + i2 = MyItem( name="Maria", age=1j ) # Invalid datetimes didn't consistently fail between Python versions - i3 = TestItem(name="Jesus", age="44") + i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) self.assertRaises(TypeError, self.ie.export_item, i2) @@ -652,9 +652,9 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest): return JsonItemExporter(self.output, **kwargs) def test_two_items_with_failure_between(self): - i1 = TestItem(name="Joseph", age="22") - i2 = TestItem(name="\u263a", age="11") - i3 = TestItem(name="Jesus", age="44") + i1 = MyItem(name="Joseph", age="22") + i2 = MyItem(name="\u263a", age="11") + i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2) @@ -665,12 +665,12 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest): class JsonItemExporterDataclassTest(JsonItemExporterTest): - item_class = TestDataClass + item_class = MyDataClass custom_field_item_class = CustomFieldDataclass class CustomExporterItemTest(unittest.TestCase): - item_class: type = TestItem + item_class: type = MyItem def setUp(self): if self.item_class is None: @@ -700,4 +700,4 @@ class CustomExporterItemTest(unittest.TestCase): class CustomExporterDataclassTest(CustomExporterItemTest): - item_class = TestDataClass + item_class = MyDataClass diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 15129e31f..ca5ffdc26 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -51,7 +51,7 @@ stats_dump_2 = { } -class TestExtPeriodicLog(PeriodicLog): +class CustomPeriodicLog(PeriodicLog): def set_a(self): self.stats._stats = stats_dump_1 @@ -62,7 +62,7 @@ class TestExtPeriodicLog(PeriodicLog): def extension(settings=None): crawler = Crawler(MetaSpider, settings=settings) crawler._apply_settings() - return TestExtPeriodicLog.from_crawler(crawler) + return CustomPeriodicLog.from_crawler(crawler) class TestPeriodicLog(unittest.TestCase): diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index f2c9dc063..4874f284a 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -13,15 +13,12 @@ from scrapy.settings.default_settings import ( DOWNLOAD_DELAY, ) from scrapy.utils.misc import build_from_crawler +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler as _get_crawler UNSET = object() -class TestSpider(Spider): - name = "test" - - def get_crawler(settings=None, spidercls=None): settings = settings or {} settings["AUTOTHROTTLE_ENABLED"] = True @@ -30,11 +27,11 @@ def get_crawler(settings=None, spidercls=None): @pytest.mark.parametrize( ("value", "expected"), - ( + [ (UNSET, False), (False, False), (True, True), - ), + ], ) def test_enabled(value, expected): settings = {} @@ -50,10 +47,10 @@ def test_enabled(value, expected): @pytest.mark.parametrize( "value", - ( + [ 0.0, -1.0, - ), + ], ) def test_target_concurrency_invalid(value): settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": value} @@ -64,13 +61,13 @@ def test_target_concurrency_invalid(value): @pytest.mark.parametrize( ("spider", "setting", "expected"), - ( + [ (UNSET, UNSET, DOWNLOAD_DELAY), (1.0, UNSET, 1.0), (UNSET, 1.0, 1.0), (1.0, 2.0, 1.0), (3.0, 2.0, 3.0), - ), + ], ) def test_mindelay_definition(spider, setting, expected): settings = {} @@ -91,10 +88,10 @@ def test_mindelay_definition(spider, setting, expected): @pytest.mark.parametrize( ("value", "expected"), - ( + [ (UNSET, AUTOTHROTTLE_MAX_DELAY), (1.0, 1.0), - ), + ], ) def test_maxdelay_definition(value, expected): settings = {} @@ -102,13 +99,13 @@ def test_maxdelay_definition(value, expected): settings["AUTOTHROTTLE_MAX_DELAY"] = value crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - at._spider_opened(TestSpider()) + at._spider_opened(DefaultSpider()) assert at.maxdelay == expected @pytest.mark.parametrize( ("min_spider", "min_setting", "start_setting", "expected"), - ( + [ (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), @@ -134,7 +131,7 @@ def test_maxdelay_definition(value, expected): AUTOTHROTTLE_START_DELAY + 2.0, AUTOTHROTTLE_START_DELAY + 2.0, ), - ), + ], ) def test_startdelay_definition(min_spider, min_setting, start_setting, expected): settings = {} @@ -158,7 +155,7 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) @pytest.mark.parametrize( ("meta", "slot"), - ( + [ ({}, None), ({"download_latency": 1.0}, None), ({"download_slot": "foo"}, None), @@ -172,12 +169,12 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) }, "foo", ), - ), + ], ) def test_skipped(meta, slot): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) request = Request("https://example.com", meta=meta) @@ -193,7 +190,7 @@ def test_skipped(meta, slot): @pytest.mark.parametrize( ("download_latency", "target_concurrency", "slot_delay", "expected"), - ( + [ (2.0, 2.0, 1.0, 1.0), (1.0, 2.0, 1.0, 0.75), (4.0, 2.0, 1.0, 2.0), @@ -201,13 +198,13 @@ def test_skipped(meta, slot): (2.0, 4.0, 1.0, 0.75), (2.0, 2.0, 0.5, 1.0), (2.0, 2.0, 2.0, 1.5), - ), + ], ) def test_adjustment(download_latency, target_concurrency, slot_delay, expected): settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) @@ -227,11 +224,11 @@ def test_adjustment(download_latency, target_concurrency, slot_delay, expected): @pytest.mark.parametrize( ("mindelay", "maxdelay", "expected"), - ( + [ (0.5, 2.0, 1.0), (0.25, 0.5, 0.5), (2.0, 4.0, 2.0), - ), + ], ) def test_adjustment_limits(mindelay, maxdelay, expected): download_latency, target_concurrency, slot_delay = (2.0, 2.0, 1.0) @@ -243,7 +240,7 @@ def test_adjustment_limits(mindelay, maxdelay, expected): } crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) @@ -263,11 +260,11 @@ def test_adjustment_limits(mindelay, maxdelay, expected): @pytest.mark.parametrize( ("download_latency", "target_concurrency", "slot_delay", "expected"), - ( + [ (2.0, 2.0, 1.0, 1.0), (1.0, 2.0, 1.0, 1.0), # Instead of 0.75 (4.0, 2.0, 1.0, 2.0), - ), + ], ) def test_adjustment_bad_response( download_latency, target_concurrency, slot_delay, expected @@ -275,7 +272,7 @@ def test_adjustment_bad_response( settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": download_latency, "download_slot": "foo"} request = Request("https://example.com", meta=meta) @@ -297,7 +294,7 @@ def test_debug(caplog): settings = {"AUTOTHROTTLE_DEBUG": True} crawler = get_crawler(settings) at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": 1.0, "download_slot": "foo"} request = Request("https://example.com", meta=meta) @@ -327,7 +324,7 @@ def test_debug(caplog): def test_debug_disabled(caplog): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) - spider = TestSpider() + spider = DefaultSpider() at._spider_opened(spider) meta = {"download_latency": 1.0, "download_slot": "foo"} request = Request("https://example.com", meta=meta) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 1f998de1a..ddc772236 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -258,7 +258,8 @@ class Https2ClientProtocolTestCase(TestCase): :param path: Should have / at the starting compulsorily if not empty :return: Complete url """ - assert len(path) > 0 and (path[0] == "/" or path[0] == "&") + assert len(path) > 0 + assert path[0] == "/" or path[0] == "&" return f"{self.scheme}://{self.hostname}:{self.port_number}{path}" def make_request(self, request: Request) -> Deferred: diff --git a/tests/test_http_response.py b/tests/test_http_response.py index b157e9802..dde883451 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -2,8 +2,8 @@ import codecs import unittest from unittest import mock +import pytest from packaging.version import Version as parse_version -from pytest import mark from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding @@ -218,7 +218,7 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com") self.assertRaises(ValueError, r.follow, None) - @mark.xfail( + @pytest.mark.xfail( parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, @@ -226,7 +226,7 @@ class BaseResponseTest(unittest.TestCase): def test_follow_whitespace_url(self): self._assert_followed_url("foo ", "http://example.com/foo") - @mark.xfail( + @pytest.mark.xfail( parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, @@ -473,10 +473,8 @@ class TextResponseTest(BaseResponseTest): self._assert_response_encoding(r5, "utf-8") self._assert_response_encoding(r8, "utf-8") self._assert_response_encoding(r9, "cp1252") - assert ( - r4._body_inferred_encoding() is not None - and r4._body_inferred_encoding() != "ascii" - ) + assert r4._body_inferred_encoding() is not None + assert r4._body_inferred_encoding() != "ascii" self._assert_response_values(r1, "utf-8", "\xa3") self._assert_response_values(r2, "utf-8", "\xa3") self._assert_response_values(r3, "iso-8859-1", "\xa3") diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index a83cfb56c..e751e0a63 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -4,8 +4,8 @@ import pickle import re import unittest +import pytest from packaging.version import Version -from pytest import mark from w3lib import __version__ as w3lib_version from scrapy.http import HtmlResponse, XmlResponse @@ -930,7 +930,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ], ) - @mark.skipif( + @pytest.mark.skipif( Version(w3lib_version) < Version("2.0.0"), reason=( "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain " diff --git a/tests/test_loader.py b/tests/test_loader.py index 824d7aecf..b52d5ea2e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -18,12 +18,12 @@ class NameItem(Item): name = Field() -class TestItem(NameItem): +class SummaryItem(NameItem): url = Field() summary = Field() -class TestNestedItem(Item): +class NestedItem(Item): name = Field() name_div = Field() name_value = Field() @@ -38,20 +38,20 @@ class AttrsNameItem: @dataclasses.dataclass -class TestDataClass: +class NameDataClass: name: list = dataclasses.field(default_factory=list) # test item loaders class NameItemLoader(ItemLoader): - default_item_class = TestItem + default_item_class = SummaryItem class NestedItemLoader(ItemLoader): - default_item_class = TestNestedItem + default_item_class = NestedItem -class TestItemLoader(NameItemLoader): +class ProcessorItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title()) @@ -68,11 +68,11 @@ def processor_with_args(value, other=None, loader_context=None): class BasicItemLoaderTest(unittest.TestCase): def test_add_value_on_unknown_field(self): - il = TestItemLoader() + il = ProcessorItemLoader() self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"]) def test_load_item_using_default_loader(self): - i = TestItem() + i = SummaryItem() i["summary"] = "lala" il = ItemLoader(item=i) il.add_value("name", "marta") @@ -82,7 +82,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(item["name"], ["marta"]) def test_load_item_using_custom_loader(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", "marta") item = il.load_item() self.assertEqual(item["name"], ["Marta"]) @@ -194,7 +194,7 @@ class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase): - item_class = TestDataClass + item_class = NameDataClass class BaseNoInputReprocessingLoader(ItemLoader): @@ -289,11 +289,11 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_init_method(self): - l = TestItemLoader() + l = ProcessorItemLoader() self.assertEqual(l.selector, None) def test_init_method_errors(self): - l = TestItemLoader() + l = ProcessorItemLoader() self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href") self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href") self.assertRaises(RuntimeError, l.get_xpath, "//a/@href") @@ -303,7 +303,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_with_selector(self): sel = Selector(text="<html><body><div>marta</div></body></html>") - l = TestItemLoader(selector=sel) + l = ProcessorItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_xpath("name", "//div/text()") @@ -311,7 +311,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_with_selector_css(self): sel = Selector(text="<html><body><div>marta</div></body></html>") - l = TestItemLoader(selector=sel) + l = ProcessorItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_css("name", "div::text") @@ -320,18 +320,18 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_with_base_response(self): """Selector should be None after initialization""" response = Response("https://scrapy.org") - l = TestItemLoader(response=response) + l = ProcessorItemLoader(response=response) self.assertIs(l.selector, None) def test_init_method_with_response(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath("name", "//div/text()") self.assertEqual(l.get_output_value("name"), ["Marta"]) def test_init_method_with_response_css(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css("name", "div::text") @@ -350,12 +350,12 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_add_xpath_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_xpath("name", "//div/text()", re="ma") self.assertEqual(l.get_output_value("name"), ["Ma"]) def test_replace_xpath(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath("name", "//div/text()") self.assertEqual(l.get_output_value("name"), ["Marta"]) @@ -366,7 +366,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) def test_get_xpath(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"]) self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph") self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa") @@ -376,14 +376,14 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_replace_xpath_multi_fields(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x}) self.assertEqual(l.get_output_value("name"), ["Marta"]) l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x}) self.assertEqual(l.get_output_value("name"), ["Paragraph"]) def test_replace_xpath_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath("name", "//div/text()") self.assertEqual(l.get_output_value("name"), ["Marta"]) @@ -391,7 +391,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("name"), ["Ma"]) def test_add_css_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_css("name", "div::text", re="ma") self.assertEqual(l.get_output_value("name"), ["Ma"]) @@ -399,7 +399,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) def test_replace_css(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css("name", "div::text") self.assertEqual(l.get_output_value("name"), ["Marta"]) @@ -415,7 +415,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) def test_get_css(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertEqual(l.get_css("p::text"), ["paragraph"]) self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph") self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa") @@ -427,7 +427,7 @@ class SelectortemLoaderTest(unittest.TestCase): ) def test_replace_css_multi_fields(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x}) self.assertEqual(l.get_output_value("name"), ["Marta"]) l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x}) @@ -439,7 +439,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) def test_replace_css_re(self): - l = TestItemLoader(response=self.response) + l = ProcessorItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css("url", "a::attr(href)") self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 8d4bd6bc1..1e504f539 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -24,17 +24,17 @@ class NameItem(Item): name = Field() -class TestItem(NameItem): +class SummaryItem(NameItem): url = Field() summary = Field() # test item loaders class NameItemLoader(ItemLoader): - default_item_class = TestItem + default_item_class = SummaryItem -class TestItemLoader(NameItemLoader): +class ProcessorItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title()) @@ -51,7 +51,7 @@ def processor_with_args(value, other=None, loader_context=None): class BasicItemLoaderTest(unittest.TestCase): def test_load_item_using_default_loader(self): - i = TestItem() + i = SummaryItem() i["summary"] = "lala" il = ItemLoader(item=i) il.add_value("name", "marta") @@ -61,7 +61,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(item["name"], ["marta"]) def test_load_item_using_custom_loader(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", "marta") item = il.load_item() self.assertEqual(item["name"], ["Marta"]) @@ -125,7 +125,7 @@ class BasicItemLoaderTest(unittest.TestCase): ) def test_add_value(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", "marta") self.assertEqual(il.get_collected_values("name"), ["Marta"]) self.assertEqual(il.get_output_value("name"), ["Marta"]) @@ -146,7 +146,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_collected_values("name"), [0]) def test_replace_value(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.replace_value("name", "marta") self.assertEqual(il.get_collected_values("name"), ["Marta"]) self.assertEqual(il.get_output_value("name"), ["Marta"]) @@ -229,7 +229,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["mart"]) def test_input_processor_inheritance(self): - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(lambda v: v.lower()) il = ChildItemLoader() @@ -265,8 +265,8 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["marta"]) def test_extend_custom_input_processors(self): - class ChildItemLoader(TestItemLoader): - name_in = MapCompose(TestItemLoader.name_in, str.swapcase) + class ChildItemLoader(ProcessorItemLoader): + name_in = MapCompose(ProcessorItemLoader.name_in, str.swapcase) il = ChildItemLoader() il.add_value("name", "marta") @@ -283,11 +283,11 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["MART"]) def test_output_processor_using_function(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - class TakeFirstItemLoader(TestItemLoader): + class TakeFirstItemLoader(ProcessorItemLoader): name_out = " ".join il = TakeFirstItemLoader() @@ -296,7 +296,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_output_processor_error(self): class TestItemLoader(ItemLoader): - default_item_class = TestItem + default_item_class = SummaryItem name_out = MapCompose(float) il = TestItemLoader() @@ -319,18 +319,18 @@ class BasicItemLoaderTest(unittest.TestCase): assert expected_exc_str in s, s def test_output_processor_using_classes(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - class TakeFirstItemLoader(TestItemLoader): + class TakeFirstItemLoader(ProcessorItemLoader): name_out = Join() il = TakeFirstItemLoader() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), "Mar Ta") - class TakeFirstItemLoader2(TestItemLoader): + class TakeFirstItemLoader2(ProcessorItemLoader): name_out = Join("<br>") il = TakeFirstItemLoader2() @@ -338,11 +338,11 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), "Mar<br>Ta") def test_default_output_processor(self): - il = TestItemLoader() + il = ProcessorItemLoader() il.add_value("name", ["mar", "ta"]) self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - class LalaItemLoader(TestItemLoader): + class LalaItemLoader(ProcessorItemLoader): default_output_processor = Identity() il = LalaItemLoader() @@ -350,7 +350,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) def test_loader_context_on_declaration(self): - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(processor_with_args, key="val") il = ChildItemLoader() @@ -360,7 +360,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("url"), ["val"]) def test_loader_context_on_instantiation(self): - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(processor_with_args) il = ChildItemLoader(key="val") @@ -370,7 +370,7 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(il.get_output_value("url"), ["val"]) def test_loader_context_on_assign(self): - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(processor_with_args) il = ChildItemLoader() @@ -384,10 +384,10 @@ class BasicItemLoaderTest(unittest.TestCase): def processor(value, loader_context): return loader_context["item"]["name"] - class ChildItemLoader(TestItemLoader): + class ChildItemLoader(ProcessorItemLoader): url_in = MapCompose(processor) - it = TestItem(name="marta") + it = SummaryItem(name="marta") il = ChildItemLoader(item=it) il.add_value("url", "text") self.assertEqual(il.get_output_value("url"), ["marta"]) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3a1cf19ad..0cc532570 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -40,7 +40,7 @@ class MOff: raise NotConfigured("foo") -class TestMiddlewareManager(MiddlewareManager): +class MyMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): return [M1, MOff, M3] @@ -54,7 +54,7 @@ class TestMiddlewareManager(MiddlewareManager): class MiddlewareManagerTest(unittest.TestCase): def test_init(self): m1, m2, m3 = M1(), M2(), M3() - mwman = TestMiddlewareManager(m1, m2, m3) + mwman = MyMiddlewareManager(m1, m2, m3) self.assertEqual( list(mwman.methods["open_spider"]), [m1.open_spider, m2.open_spider] ) @@ -64,7 +64,7 @@ class MiddlewareManagerTest(unittest.TestCase): self.assertEqual(list(mwman.methods["process"]), [m1.process, m3.process]) def test_methods(self): - mwman = TestMiddlewareManager(M1(), M2(), M3()) + mwman = MyMiddlewareManager(M1(), M2(), M3()) self.assertEqual( [x.__self__.__class__ for x in mwman.methods["open_spider"]], [M1, M2] ) @@ -82,6 +82,6 @@ class MiddlewareManagerTest(unittest.TestCase): def test_enabled_from_settings(self): crawler = get_crawler() - mwman = TestMiddlewareManager.from_crawler(crawler) + mwman = MyMiddlewareManager.from_crawler(crawler) classes = [x.__class__ for x in mwman.middlewares] self.assertEqual(classes, [M1, M3]) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 222b19e7f..0ae86235c 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -1,6 +1,6 @@ import asyncio -from pytest import mark +import pytest from twisted.internet import defer from twisted.internet.defer import Deferred from twisted.trial import unittest @@ -118,14 +118,14 @@ class PipelineTestCase(unittest.TestCase): yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(self.items), 1) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_asyncdef_asyncio_pipeline(self): crawler = self._create_crawler(AsyncDefAsyncioPipeline) yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(self.items), 1) - @mark.only_not_asyncio() + @pytest.mark.only_not_asyncio @defer.inlineCallbacks def test_asyncdef_not_asyncio_pipeline(self): crawler = self._create_crawler(AsyncDefNotAsyncioPipeline) diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 854805cf7..85133038a 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -11,7 +11,7 @@ class CustomRequest(Request): class RequestSerializationTest(unittest.TestCase): def setUp(self): - self.spider = TestSpider() + self.spider = MethodsSpider() def test_basic(self): r = Request("http://www.example.com") @@ -96,18 +96,22 @@ class RequestSerializationTest(unittest.TestCase): def test_private_reference_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._TestSpider__parse_item_reference, - errback=self.spider._TestSpider__handle_error_reference, + callback=self.spider._MethodsSpider__parse_item_reference, + errback=self.spider._MethodsSpider__handle_error_reference, ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) - self.assertEqual(request_dict["callback"], "_TestSpider__parse_item_reference") - self.assertEqual(request_dict["errback"], "_TestSpider__handle_error_reference") + self.assertEqual( + request_dict["callback"], "_MethodsSpider__parse_item_reference" + ) + self.assertEqual( + request_dict["errback"], "_MethodsSpider__handle_error_reference" + ) def test_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._TestSpider__parse_item_private, + callback=self.spider._MethodsSpider__parse_item_private, errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -115,7 +119,7 @@ class RequestSerializationTest(unittest.TestCase): def test_mixin_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._TestSpiderMixin__mixin_callback, + callback=self.spider._SpiderMixin__mixin_callback, errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -152,18 +156,18 @@ class RequestSerializationTest(unittest.TestCase): def test_callback_not_available(self): """Callback method is not available in the spider passed to from_dict""" - spider = TestSpiderDelegation() + spider = SpiderDelegation() r = Request("http://www.example.com", callback=spider.delegated_callback) d = r.to_dict(spider=spider) self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo")) -class TestSpiderMixin: +class SpiderMixin: def __mixin_callback(self, response): # pylint: disable=unused-private-member pass -class TestSpiderDelegation: +class SpiderDelegation: def delegated_callback(self, response): pass @@ -184,7 +188,7 @@ def private_handle_error(failure): pass -class TestSpider(Spider, TestSpiderMixin): +class MethodsSpider(Spider, SpiderMixin): name = "test" parse_item_reference = parse_item handle_error_reference = handle_error @@ -193,7 +197,7 @@ class TestSpider(Spider, TestSpiderMixin): def __init__(self, **kwargs): super().__init__(**kwargs) - self.delegated_callback = TestSpiderDelegation().delegated_callback + self.delegated_callback = SpiderDelegation().delegated_callback def parse_item(self, response): pass diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index b48a65e67..7c72805e2 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -51,8 +51,8 @@ class SimpleScheduler(MinimalScheduler): return len(self.requests) -class TestSpider(Spider): - name = "test" +class PathsSpider(Spider): + name = "paths" def __init__(self, mockserver, *args, **kwargs): super().__init__(*args, **kwargs) @@ -155,7 +155,7 @@ class MinimalSchedulerCrawlTest(TwistedTestCase): "SCHEDULER": self.scheduler_cls, } with LogCapture() as log: - crawler = get_crawler(TestSpider, settings) + crawler = get_crawler(PathsSpider, settings) yield crawler.crawl(mockserver) for path in PATHS: self.assertIn(f"{{'path': '{path}'}}", str(log)) diff --git a/tests/test_signals.py b/tests/test_signals.py index 1e693c094..a508eb41a 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,4 +1,4 @@ -from pytest import mark +import pytest from twisted.internet import defer from twisted.trial import unittest @@ -37,7 +37,7 @@ class AsyncSignalTestCase(unittest.TestCase): item = await get_from_asyncio_queue(item) self.items.append(item) - @mark.only_asyncio() + @pytest.mark.only_asyncio @defer.inlineCallbacks def test_simple_pipeline(self): crawler = get_crawler(ItemSpider) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 4ce7cc9a4..a2e7ae65d 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -15,7 +15,7 @@ from scrapy.squeues import ( ) -class TestItem(Item): +class MyItem(Item): name = Field() @@ -23,8 +23,8 @@ def _test_procesor(x): return x + x -class TestLoader(ItemLoader): - default_item_class = TestItem +class MyLoader(ItemLoader): + default_item_class = MyItem name_out = staticmethod(_test_procesor) @@ -80,19 +80,19 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): def test_serialize_item(self): q = self.queue() - i = TestItem(name="foo") + i = MyItem(name="foo") q.push(i) i2 = q.pop() - assert isinstance(i2, TestItem) + assert isinstance(i2, MyItem) self.assertEqual(i, i2) def test_serialize_loader(self): q = self.queue() - loader = TestLoader() + loader = MyLoader() q.push(loader) loader2 = q.pop() - assert isinstance(loader2, TestLoader) - assert loader2.default_item_class is TestItem + assert isinstance(loader2, MyLoader) + assert loader2.default_item_class is MyItem self.assertEqual(loader2.name_out("x"), "xx") def test_serialize_request_recursive(self): @@ -161,19 +161,19 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def test_serialize_item(self): q = self.queue() - i = TestItem(name="foo") + i = MyItem(name="foo") q.push(i) i2 = q.pop() - assert isinstance(i2, TestItem) + assert isinstance(i2, MyItem) self.assertEqual(i, i2) def test_serialize_loader(self): q = self.queue() - loader = TestLoader() + loader = MyLoader() q.push(loader) loader2 = q.pop() - assert isinstance(loader2, TestLoader) - assert loader2.default_item_class is TestItem + assert isinstance(loader2, MyLoader) + assert loader2.default_item_class is MyItem self.assertEqual(loader2.name_out("x"), "xx") def test_serialize_request_recursive(self): diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index e00f69573..ecac0df9c 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,7 +1,7 @@ import asyncio import warnings -from pytest import mark +import pytest from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f @@ -12,7 +12,7 @@ from scrapy.utils.reactor import ( ) -@mark.usefixtures("reactor_pytest") +@pytest.mark.usefixtures("reactor_pytest") class AsyncioTest(TestCase): def test_is_asyncio_reactor_installed(self): # the result should depend only on the pytest --reactor argument @@ -30,7 +30,7 @@ class AsyncioTest(TestCase): assert original_reactor == reactor - @mark.only_asyncio() + @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_set_asyncio_event_loop(self): install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index e80381671..fadbc6daa 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -16,8 +16,6 @@ from scrapy.utils.datatypes import ( ) from scrapy.utils.python import garbage_collect -__doctests__ = ["scrapy.utils.datatypes"] - class CaseInsensitiveDictMixin: def test_init_dict(self): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index e4ab97e5d..3a1030fcf 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,6 +1,6 @@ import random -from pytest import mark +import pytest from twisted.internet import defer, reactor from twisted.python.failure import Failure from twisted.trial import unittest @@ -150,7 +150,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase): async def test_deferred_f_from_coro_f_generator(self): yield - @mark.xfail(reason="Checks that the test is actually executed", strict=True) + @pytest.mark.xfail(reason="Checks that the test is actually executed", strict=True) @deferred_f_from_coro_f async def test_deferred_f_from_coro_f_xfail(self): raise RuntimeError("This is expected to be raised") diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 76820eabf..06e88bd10 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -119,7 +119,7 @@ class StreamLoggerTest(unittest.TestCase): @pytest.mark.parametrize( ("base_extra", "log_extra", "expected_extra"), - ( + [ ( {"spider": "test"}, {"extra": {"log_extra": "info"}}, @@ -135,7 +135,7 @@ class StreamLoggerTest(unittest.TestCase): {"extra": {"spider": "test2"}}, {"extra": {"spider": "test"}}, ), - ), + ], ) def test_spider_logger_adapter_process( base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 478c1e73a..e25bdfe3f 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -17,8 +17,6 @@ from scrapy.utils.misc import ( walk_modules, ) -__doctests__ = ["scrapy.utils.misc"] - class UtilsMiscTestCase(unittest.TestCase): def test_load_object_class(self): diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 83004cec4..a693d6b53 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -20,8 +20,6 @@ from scrapy.utils.python import ( without_none_values, ) -__doctests__ = ["scrapy.utils.python"] - class MutableChainTest(unittest.TestCase): def test_mutablechain(self): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index db6866571..c6ba8cbbb 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -15,8 +15,6 @@ from scrapy.utils.response import ( response_status_message, ) -__doctests__ = ["scrapy.utils.response"] - class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") @@ -207,8 +205,8 @@ class ResponseUtilsTest(unittest.TestCase): @pytest.mark.parametrize( - "input_body,output_body", - ( + ("input_body", "output_body"), + [ ( b"a<!--", b"a", @@ -237,7 +235,7 @@ class ResponseUtilsTest(unittest.TestCase): b"a<!--b--><!--c-->d", b"ad", ), - ), + ], ) def test_remove_html_comments(input_body, output_body): assert _remove_html_comments(input_body) == output_body, ( diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 60232f10b..858813e83 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,7 +1,7 @@ import asyncio +import pytest from pydispatch import dispatcher -from pytest import mark from testfixtures import LogCapture from twisted.internet import defer, reactor from twisted.python.failure import Failure @@ -67,7 +67,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): return d -@mark.usefixtures("reactor_pytest") +@pytest.mark.usefixtures("reactor_pytest") class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) @@ -76,7 +76,7 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): return "OK" -@mark.only_asyncio() +@pytest.mark.only_asyncio class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 5fbbd74da..fc6c33200 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -5,8 +5,6 @@ from tempfile import mkdtemp from scrapy.utils.template import render_templatefile -__doctests__ = ["scrapy.utils.template"] - class UtilsRenderTemplateFileTestCase(unittest.TestCase): def setUp(self): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 4b9a98d79..e99ef40c4 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -17,8 +17,6 @@ from scrapy.utils.url import ( # type: ignore[attr-defined] url_is_from_spider, ) -__doctests__ = ["scrapy.utils.url"] - class UrlUtilsTest(unittest.TestCase): def test_url_is_from_any_domain(self): From 8c34e6d9a4994abda0059f76dbbdb64c2e8a9751 Mon Sep 17 00:00:00 2001 From: Matt Winter <MattWinter@gmail.com> Date: Wed, 19 Feb 2025 04:17:37 -0500 Subject: [PATCH 1663/2083] curl: add support for parsing -b,--cookie (#6684) --- scrapy/utils/curl.py | 9 +++++++++ tests/test_utils_curl.py | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index a563dc79a..a40ee8997 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -36,6 +36,7 @@ curl_parser = CurlParser() curl_parser.add_argument("url") curl_parser.add_argument("-H", "--header", dest="headers", action="append") curl_parser.add_argument("-X", "--request", dest="method") +curl_parser.add_argument("-b", "--cookie", dest="cookies", action="append") curl_parser.add_argument("-d", "--data", "--data-raw", dest="data", action=DataAction) curl_parser.add_argument("-u", "--user", dest="auth") @@ -68,6 +69,14 @@ def _parse_headers_and_cookies( else: headers.append((name, val)) + for cookie_param in parsed_args.cookies or (): + # curl can treat this parameter as either "key=value; key2=value2" pairs, or a filename. + # Scrapy will only support key-value pairs. + if "=" not in cookie_param: + continue + for name, morsel in SimpleCookie(cookie_param).items(): + cookies[name] = morsel.value + if parsed_args.auth: user, password = parsed_args.auth.split(":", 1) headers.append(("Authorization", basic_auth_header(user, password))) diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 1816db29b..5d99161bf 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -49,8 +49,8 @@ class CurlToRequestKwargsTest(unittest.TestCase): "ml,application/xhtml+xml,application/xml;q=0.9,image/webp,image/a" "png,*/*;q=0.8' -H 'Referer: http://httpbin.org/' -H 'Cookie: _gau" "ges_unique_year=1; _gauges_unique=1; _gauges_unique_month=1; _gau" - "ges_unique_hour=1; _gauges_unique_day=1' -H 'Connection: keep-ali" - "ve' --compressed" + "ges_unique_hour=1' -H 'Connection: keep-alive' --compressed -b '_" + "gauges_unique_day=1'" ) expected_result = { "method": "GET", From c200458f24a2e55dece5df5c22f39a0d1b4ff341 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 25 Feb 2025 04:16:23 +0500 Subject: [PATCH 1664/2083] Add more docs for updating sync spider middlewares. (#6688) --- docs/topics/coroutines.rst | 46 +++++++++++++++++++++++++++++++++----- scrapy/core/spidermw.py | 4 ++-- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a65bab3ca..57aa3a62d 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -238,16 +238,52 @@ active spider middlewares must either have their ``process_spider_output`` method defined as an asynchronous generator or :ref:`define a process_spider_output_async method <universal-spider-middleware>`. -.. note:: When using third-party spider middlewares that only define a - synchronous ``process_spider_output`` method, consider - :ref:`making them universal <universal-spider-middleware>` through - :ref:`subclassing <tut-inheritance>`. +.. _sync-async-spider-middleware-users: +For middleware users +-------------------- + +If you have asynchronous callbacks or use asynchronous-only spider middlewares +you should make sure the asynchronous-to-synchronous conversions +:ref:`described above <sync-async-spider-middleware>` don't happen. To do this, +make sure all spider middlewares you use support asynchronous spider output. +Even if you don't have asynchronous callbacks and don't use asynchronous-only +spider middlewares in your project, it's still a good idea to make sure all +middlewares you use support asynchronous spider output, so that it will be easy +to start using asynchronous callbacks in the future. Because of this, Scrapy +logs a warning when it detects a synchronous-only spider middleware. + +If you want to update middlewares you wrote, see the :ref:`following section +<sync-async-spider-middleware-authors>`. If you have 3rd-party middlewares that +aren't yet updated by their authors, you can :ref:`subclass <tut-inheritance>` +them to make them :ref:`universal <universal-spider-middleware>` and use the +subclasses in your projects. + +.. _sync-async-spider-middleware-authors: + +For middleware authors +---------------------- + +If you have a spider middleware that defines a synchronous +``process_spider_output`` method, you should update it to support asynchronous +spider output for :ref:`better compatibility <sync-async-spider-middleware>`, +even if you don't yet use it with asynchronous callbacks, especially if you +publish this middleware for other people to use. You have two options for this: + +1. Make the middleware asynchronous, by making the ``process_spider_output`` + method an :term:`asynchronous generator`. +2. Make the middleware universal, as described in the :ref:`next section + <universal-spider-middleware>`. + +If your middleware won't be used in projects with synchronous-only middlewares, +e.g. because it's an internal middleware and you know that all other +middlewares in your projects are already updated, it's safe to choose the first +option. Otherwise, it's better to choose the second option. .. _universal-spider-middleware: Universal spider middlewares -============================ +---------------------------- .. versionadded:: 2.7 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 86d11c0e0..85a3b5895 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -227,7 +227,7 @@ class SpiderMiddlewareManager(MiddlewareManager): f"Async iterable passed to {global_object_name(method)} was" f" downgraded to a non-async one. This is deprecated and will" f" stop working in a future version of Scrapy. Please see" - f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares" + f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users" f" for more information." ) assert isinstance(result, AsyncIterable) @@ -343,7 +343,7 @@ class SpiderMiddlewareManager(MiddlewareManager): f" asynchronous spider output, this is deprecated and will stop" f" working in a future version of Scrapy. The middleware should" f" be updated to support it. Please see" - f" https://docs.scrapy.org/en/latest/topics/coroutines.html#mixing-synchronous-and-asynchronous-spider-middlewares" + f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users" f" for more information." ) return normal_method From 391af6afcca232aed82eda516b035dbbd39cdcb1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 27 Feb 2025 22:37:01 +0500 Subject: [PATCH 1665/2083] Unknown encoding handling in HttpCompressionMiddleware, restore x-gzip support (#6618) * Unknown encoding handling in HttpCompressionMiddleware. * Implement the changes for unknown encoding handling. * Restore support for Content-Encoding: x-gzip. * Simplify the decoding logic. * Add tests for the unsupported encoding warning. * Add a test for the "no zstandard" warning. --- .../downloadermiddlewares/httpcompression.py | 32 +++++-- ...st_downloadermiddleware_httpcompression.py | 84 ++++++++++++++++++- 2 files changed, 108 insertions(+), 8 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index a65757972..58891b952 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -111,6 +111,8 @@ class HttpCompressionMiddleware: f"({len(decoded_body)} B) is larger than the " f"download warning size ({warn_size} B)." ) + if content_encoding: + self._warn_unknown_encoding(response, content_encoding) response.headers["Content-Encoding"] = content_encoding if self.stats: self.stats.inc_value( @@ -143,9 +145,11 @@ class HttpCompressionMiddleware: body = self._decode(body, encoding, max_size) return body, to_keep + @staticmethod def _split_encodings( - self, content_encoding: list[bytes] + content_encoding: list[bytes], ) -> tuple[list[bytes], list[bytes]]: + supported_encodings = {*ACCEPTED_ENCODINGS, b"x-gzip"} to_keep: list[bytes] = [ encoding.strip().lower() for encoding in chain.from_iterable( @@ -155,19 +159,35 @@ class HttpCompressionMiddleware: to_decode: list[bytes] = [] while to_keep: encoding = to_keep.pop() - if encoding not in ACCEPTED_ENCODINGS: + if encoding not in supported_encodings: to_keep.append(encoding) return to_decode, to_keep to_decode.append(encoding) return to_decode, to_keep - def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes: + @staticmethod + def _decode(body: bytes, encoding: bytes, max_size: int) -> bytes: if encoding in {b"gzip", b"x-gzip"}: return gunzip(body, max_size=max_size) if encoding == b"deflate": return _inflate(body, max_size=max_size) - if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS: + if encoding == b"br": return _unbrotli(body, max_size=max_size) - if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS: + if encoding == b"zstd": return _unzstd(body, max_size=max_size) - return body + # shouldn't be reached + return body # pragma: no cover + + def _warn_unknown_encoding( + self, response: Response, encodings: list[bytes] + ) -> None: + encodings_str = b",".join(encodings).decode() + msg = ( + f"{self.__class__.__name__} cannot decode the response for {response.url} " + f"from unsupported encoding(s) '{encodings_str}'." + ) + if b"br" in encodings: + msg += " You need to install brotli or brotlicffi to decode 'br'." + if b"zstd" in encodings: + msg += " You need to install zstandard to decode 'zstd'." + logger.warning(msg) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 78d0dd99d..a1c5883ec 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -23,7 +23,7 @@ SAMPLEDIR = Path(tests_datadir, "compressed") FORMAT = { "gzip": ("html-gzip.bin", "gzip"), - "x-gzip": ("html-gzip.bin", "gzip"), + "x-gzip": ("html-gzip.bin", "x-gzip"), "rawdeflate": ("html-rawdeflate.bin", "deflate"), "zlibdeflate": ("html-zlibdeflate.bin", "deflate"), "gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"), @@ -145,6 +145,41 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) + def test_process_response_br_unsupported(self): + try: + try: + import brotli # noqa: F401 + + raise SkipTest("Requires not having brotli support") + except ImportError: + import brotlicffi # noqa: F401 + + raise SkipTest("Requires not having brotli support") + except ImportError: + pass + response = self._getresponse("br") + request = response.request + self.assertEqual(response.headers["Content-Encoding"], b"br") + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + newresponse = self.mw.process_response(request, response, self.spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "HttpCompressionMiddleware cannot decode the response for" + " http://scrapytest.org/ from unsupported encoding(s) 'br'." + " You need to install brotli or brotlicffi to decode 'br'." + ), + ), + ) + assert newresponse is not response + self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"br"]) + def test_process_response_zstd(self): try: import zstandard # noqa: F401 @@ -166,6 +201,36 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b"<!DOCTYPE") assert "Content-Encoding" not in newresponse.headers + def test_process_response_zstd_unsupported(self): + try: + import zstandard # noqa: F401 + + raise SkipTest("Requires not having zstandard support") + except ImportError: + pass + response = self._getresponse("zstd-static-content-size") + request = response.request + self.assertEqual(response.headers["Content-Encoding"], b"zstd") + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + newresponse = self.mw.process_response(request, response, self.spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "HttpCompressionMiddleware cannot decode the response for" + " http://scrapytest.org/ from unsupported encoding(s) 'zstd'." + " You need to install zstandard to decode 'zstd'." + ), + ), + ) + assert newresponse is not response + self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"zstd"]) + def test_process_response_rawdeflate(self): response = self._getresponse("rawdeflate") request = response.request @@ -221,7 +286,22 @@ class HttpCompressionTest(TestCase): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = [b"gzip, foo, deflate"] request = response.request - newresponse = self.mw.process_response(request, response, self.spider) + with LogCapture( + "scrapy.downloadermiddlewares.httpcompression", + propagate=False, + level=WARNING, + ) as log: + newresponse = self.mw.process_response(request, response, self.spider) + log.check( + ( + "scrapy.downloadermiddlewares.httpcompression", + "WARNING", + ( + "HttpCompressionMiddleware cannot decode the response for" + " http://scrapytest.org/ from unsupported encoding(s) 'gzip,foo'." + ), + ), + ) assert newresponse is not response self.assertEqual( newresponse.headers.getlist("Content-Encoding"), [b"gzip", b"foo"] From 8d92c28a16c78a1ca7531679488fbd979f308661 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 27 Feb 2025 23:13:04 +0500 Subject: [PATCH 1666/2083] Switch to pytest.raises(). (#6680) * Switch to pytest.raises(). * Add matches= to broad pytest.raises(). * Adjust the test_nonserializable_object() regex for Python <= 3.11. * Adjust the test_nonserializable_object() regex for PyPy. * Adjust other test exception regexes for PyPy. * Cleanup. --- pyproject.toml | 2 - scrapy/pqueues.py | 2 +- tests/test_crawler.py | 33 ++-- tests/test_downloader_handlers_http2.py | 2 +- tests/test_downloadermiddleware.py | 4 +- tests/test_downloadermiddleware_cookies.py | 9 +- tests/test_downloadermiddleware_httpauth.py | 5 +- tests/test_downloadermiddleware_httpcache.py | 7 +- ...st_downloadermiddleware_httpcompression.py | 37 ++--- tests/test_downloadermiddleware_redirect.py | 10 +- tests/test_downloadermiddleware_retry.py | 3 +- tests/test_downloadermiddleware_robotstxt.py | 4 +- tests/test_exporters.py | 14 +- tests/test_feedexport.py | 12 +- tests/test_http2_client_protocol.py | 5 +- tests/test_http_headers.py | 25 ++- tests/test_http_request.py | 149 ++++++++---------- tests/test_http_response.py | 113 +++++++------ tests/test_item.py | 29 ++-- tests/test_link.py | 4 +- tests/test_loader.py | 22 ++- tests/test_loader_deprecated.py | 53 +++++-- tests/test_logstats.py | 5 +- tests/test_pipeline_images.py | 7 +- tests/test_pqueues.py | 9 +- tests/test_request_dict.py | 18 ++- tests/test_scheduler.py | 10 +- tests/test_scheduler_base.py | 22 +-- tests/test_selector.py | 2 +- tests/test_settings/__init__.py | 37 +++-- tests/test_spider.py | 12 +- tests/test_spiderloader/__init__.py | 12 +- tests/test_spidermiddleware.py | 16 +- tests/test_spidermiddleware_httperror.py | 25 +-- tests/test_spidermiddleware_referer.py | 6 +- tests/test_spiderstate.py | 4 +- tests/test_squeues.py | 12 +- tests/test_squeues_request.py | 13 +- tests/test_utils_conf.py | 28 ++-- tests/test_utils_curl.py | 24 ++- tests/test_utils_datatypes.py | 13 +- tests/test_utils_deprecate.py | 5 +- tests/test_utils_gz.py | 7 +- tests/test_utils_iterators.py | 16 +- tests/test_utils_misc/__init__.py | 20 ++- tests/test_utils_python.py | 6 +- tests/test_utils_response.py | 3 +- 47 files changed, 482 insertions(+), 394 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ad62ea212..82d8056f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -378,8 +378,6 @@ ignore = [ # Temporarily silenced PT rules # Use a regular `assert` instead of unittest-style `assertEqual` "PT009", - # Use `pytest.raises` instead of unittest-style `assertRaises` - "PT027", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index a04e0107b..324a9b955 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -214,7 +214,7 @@ class DownloaderAwarePriorityQueue: "DownloaderAwarePriorityQueue accepts " "``slot_startprios`` as a dict; " f"{slot_startprios.__class__!r} instance " - "is passed. Most likely, it means the state is" + "is passed. Most likely, it means the state is " "created by an incompatible priority queue. " "Only a crawl started with the same priority " "queue class can be resumed." diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 425188d32..df5ebfa7b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -185,9 +185,8 @@ class CrawlerTestCase(BaseCrawlerTest): def test_get_downloader_middleware_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises( - RuntimeError, crawler.get_downloader_middleware, DefaultSpider - ) + with pytest.raises(RuntimeError): + crawler.get_downloader_middleware(DefaultSpider) @inlineCallbacks def test_get_downloader_middleware_no_engine(self): @@ -266,7 +265,8 @@ class CrawlerTestCase(BaseCrawlerTest): def test_get_extension_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider) + with pytest.raises(RuntimeError): + crawler.get_extension(DefaultSpider) @inlineCallbacks def test_get_extension_no_engine(self): @@ -345,7 +345,8 @@ class CrawlerTestCase(BaseCrawlerTest): def test_get_item_pipeline_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider) + with pytest.raises(RuntimeError): + crawler.get_item_pipeline(DefaultSpider) @inlineCallbacks def test_get_item_pipeline_no_engine(self): @@ -424,7 +425,8 @@ class CrawlerTestCase(BaseCrawlerTest): def test_get_spider_middleware_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) - self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider) + with pytest.raises(RuntimeError): + crawler.get_spider_middleware(DefaultSpider) @inlineCallbacks def test_get_spider_middleware_no_engine(self): @@ -537,7 +539,8 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, } ) - self.assertRaises(MultipleInvalid, CrawlerRunner, settings) + with pytest.raises(MultipleInvalid): + CrawlerRunner(settings) def test_crawler_runner_accepts_dict(self): runner = CrawlerRunner({"foo": "bar"}) @@ -630,13 +633,15 @@ class CrawlerRunnerHasSpider(unittest.TestCase): } ) else: - msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" - with self.assertRaisesRegex(Exception, msg): - runner = CrawlerRunner( - settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - } - ) + runner = CrawlerRunner( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + ) + with pytest.raises( + Exception, + match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", + ): yield runner.crawl(NoRequestsSpider) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 174bf841e..17d5c2d0a 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -248,5 +248,5 @@ class Https2ProxyTestCase(BaseTestClasses.Http11ProxyTestCase): @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): - with self.assertRaises(NotImplementedError): + with pytest.raises(NotImplementedError): yield super().test_download_with_proxy_https_timeout() diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 42051042c..49498375c 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,4 +1,5 @@ import asyncio +from gzip import BadGzipFile from unittest import mock import pytest @@ -106,7 +107,8 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - self.assertRaises(OSError, self._download, request=req, response=resp) + with pytest.raises(BadGzipFile): + self._download(request=req, response=resp) class ResponseFromProcessRequestTest(ManagerTestCase): diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 772769690..694a669d4 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -83,11 +83,10 @@ class CookiesMiddlewareTest(TestCase): self.assertEqual(req2.headers.get("Cookie"), b"C1=value1") def test_setting_false_cookies_enabled(self): - self.assertRaises( - NotConfigured, - CookiesMiddleware.from_crawler, - get_crawler(settings_dict={"COOKIES_ENABLED": False}), - ) + with pytest.raises(NotConfigured): + CookiesMiddleware.from_crawler( + get_crawler(settings_dict={"COOKIES_ENABLED": False}) + ) def test_setting_default_cookies_enabled(self): self.assertIsInstance( diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 581fc1974..0f1489344 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,5 +1,6 @@ import unittest +import pytest from w3lib.http import basic_auth_header from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware @@ -29,8 +30,8 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): self.spider = LegacySpider("foo") def test_auth(self): - with self.assertRaises(AttributeError): - mw = HttpAuthMiddleware() + mw = HttpAuthMiddleware() + with pytest.raises(AttributeError): mw.spider_opened(self.spider) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 74db93f8a..de3a9689b 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -5,6 +5,8 @@ import time import unittest from contextlib import contextmanager +import pytest + from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.http import HtmlResponse, Request, Response @@ -192,9 +194,8 @@ class DummyPolicyTest(_BaseTest): def test_middleware_ignore_missing(self): with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: - self.assertRaises( - IgnoreRequest, mw.process_request, self.request, self.spider - ) + with pytest.raises(IgnoreRequest): + mw.process_request(self.request, self.spider) mw.process_response(self.request, self.response, self.spider) response = mw.process_request(self.request, self.spider) assert isinstance(response, HtmlResponse) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a1c5883ec..b3e3b98d7 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -4,6 +4,7 @@ from logging import WARNING from pathlib import Path from unittest import SkipTest, TestCase +import pytest from testfixtures import LogCapture from w3lib.encoding import resolve_encoding @@ -87,11 +88,10 @@ class HttpCompressionTest(TestCase): ) def test_setting_false_compression_enabled(self): - self.assertRaises( - NotConfigured, - HttpCompressionMiddleware.from_crawler, - get_crawler(settings_dict={"COMPRESSION_ENABLED": False}), - ) + with pytest.raises(NotConfigured): + HttpCompressionMiddleware.from_crawler( + get_crawler(settings_dict={"COMPRESSION_ENABLED": False}) + ) def test_setting_default_compression_enabled(self): self.assertIsInstance( @@ -520,13 +520,8 @@ class HttpCompressionTest(TestCase): mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") - self.assertRaises( - IgnoreRequest, - mw.process_response, - response.request, - response, - spider, - ) + with pytest.raises(IgnoreRequest): + mw.process_response(response.request, response, spider) def test_compression_bomb_setting_br(self): try: @@ -561,13 +556,8 @@ class HttpCompressionTest(TestCase): mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") - self.assertRaises( - IgnoreRequest, - mw.process_response, - response.request, - response, - spider, - ) + with pytest.raises(IgnoreRequest): + mw.process_response(response.request, response, spider) def test_compression_bomb_spider_attr_br(self): try: @@ -600,13 +590,8 @@ class HttpCompressionTest(TestCase): response = self._getresponse(f"bomb-{compression_id}") response.meta["download_maxsize"] = 10_000_000 - self.assertRaises( - IgnoreRequest, - mw.process_response, - response.request, - response, - spider, - ) + with pytest.raises(IgnoreRequest): + mw.process_response(response.request, response, spider) def test_compression_bomb_request_meta_br(self): try: diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index f950906e9..47abeee7a 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -72,9 +72,8 @@ class Base: assert isinstance(req, Request) assert "redirect_times" in req.meta self.assertEqual(req.meta["redirect_times"], 1) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp, self.spider) def test_ttl(self): self.mw.max_redirect_times = 100 @@ -83,9 +82,8 @@ class Base: req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - self.assertRaises( - IgnoreRequest, self.mw.process_response, req, rsp, self.spider - ) + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp, self.spider) def test_redirect_urls(self): req1 = Request("http://scrapytest.org/first") diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 6b9b39413..36f48db69 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,6 +1,7 @@ import logging import unittest +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.error import ( @@ -407,7 +408,7 @@ class GetRetryRequestTest(unittest.TestCase): def test_no_spider(self): request = Request("https://example.com") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): get_retry_request(request) # pylint: disable=missing-kwoa def test_max_retry_times_setting(self): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 535e07c1f..9b95400fd 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,5 +1,6 @@ from unittest import mock +import pytest from twisted.internet import error, reactor from twisted.internet.defer import Deferred, DeferredList, maybeDeferred from twisted.python import failure @@ -26,7 +27,8 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def test_robotstxt_settings(self): self.crawler.settings = Settings() self.crawler.settings.set("USER_AGENT", "CustomAgent") - self.assertRaises(NotConfigured, RobotsTxtMiddleware, self.crawler) + with pytest.raises(NotConfigured): + RobotsTxtMiddleware(self.crawler) def _get_successful_crawler(self): crawler = self.crawler diff --git a/tests/test_exporters.py b/tests/test_exporters.py index eb8d309b6..48728e078 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -10,6 +10,7 @@ from io import BytesIO from typing import Any import lxml.etree +import pytest from itemadapter import ItemAdapter from scrapy.exporters import ( @@ -147,7 +148,7 @@ class PythonItemExporterTest(BaseItemExporterTest): return PythonItemExporter(**kwargs) def test_invalid_option(self): - with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"): + with pytest.raises(TypeError, match="Unexpected options: invalid_option"): PythonItemExporter(invalid_option="something") def test_nested_item(self): @@ -388,7 +389,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ) def test_errors_default(self): - with self.assertRaises(UnicodeEncodeError): + with pytest.raises(UnicodeEncodeError): self.assertExportResult( item={"text": "W\u0275\u200brd"}, expected=None, @@ -549,7 +550,8 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie = self._get_exporter(sort_keys=True) self.test_export_item() self._check_output() - self.assertRaises(TypeError, self._get_exporter, foo_unknown_keyword_bar=True) + with pytest.raises(TypeError): + self._get_exporter(foo_unknown_keyword_bar=True) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -602,7 +604,8 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) - self.assertRaises(TypeError, self.ie.export_item, i2) + with pytest.raises(TypeError): + self.ie.export_item(i2) self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) @@ -657,7 +660,8 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest): i3 = MyItem(name="Jesus", age="44") self.ie.start_exporting() self.ie.export_item(i1) - self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2) + with pytest.raises(UnicodeEncodeError): + self.ie.export_item(i2) self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin")) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1620d2d41..b4c1b9631 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -233,7 +233,8 @@ class BlockingFeedStorageTest(unittest.TestCase): invalid_path = tests_path / "invalid_path" spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) - self.assertRaises(OSError, b.open, spider=spider) + with pytest.raises(OSError, match="Not a Directory:"): + b.open(spider=spider) @pytest.mark.requires_boto3 @@ -2437,7 +2438,8 @@ class BatchDeliveriesTest(FeedExportTestBase): "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } crawler = get_crawler(settings_dict=settings) - self.assertRaises(NotConfigured, FeedExporter, crawler) + with pytest.raises(NotConfigured): + FeedExporter(crawler) @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): @@ -2758,7 +2760,7 @@ class FeedExportInitTest(unittest.TestCase): }, } crawler = get_crawler(settings_dict=settings) - with self.assertRaises(NotConfigured): + with pytest.raises(NotConfigured): FeedExporter.from_crawler(crawler) def test_unsupported_format(self): @@ -2770,7 +2772,7 @@ class FeedExportInitTest(unittest.TestCase): }, } crawler = get_crawler(settings_dict=settings) - with self.assertRaises(NotConfigured): + with pytest.raises(NotConfigured): FeedExporter.from_crawler(crawler) def test_absolute_pathlib_as_uri(self): @@ -2863,7 +2865,7 @@ class URIParamsTest: with warnings.catch_warnings(): warnings.simplefilter("error", ScrapyDeprecationWarning) - with self.assertRaises(KeyError): + with pytest.raises(KeyError): feed_exporter.open_spider(spider) def test_params_as_is(self): diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index ddc772236..0881bbeca 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING from unittest import mock, skipIf from urllib.parse import urlencode +import pytest from twisted.internet import reactor from twisted.internet.defer import ( CancelledError, @@ -406,7 +407,7 @@ class Https2ClientProtocolTestCase(TestCase): "scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2" ): request = Request(url=self.get_url("/status?n=200")) - with self.assertRaises(ResponseFailed): + with pytest.raises(ResponseFailed): yield self.make_request(request) def test_cancel_request(self): @@ -560,7 +561,7 @@ class Https2ClientProtocolTestCase(TestCase): return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) def test_invalid_request_type(self): - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.make_request("https://InvalidDataTypePassed.com") def test_query_parameters(self): diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 7db1eb8c5..0bbbcda46 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -1,6 +1,8 @@ import copy import unittest +import pytest + from scrapy.http import Headers @@ -13,7 +15,8 @@ class HeadersTest(unittest.TestCase): assert h["Content-Type"] assert h["Content-Length"] - self.assertRaises(KeyError, h.__getitem__, "Accept") + with pytest.raises(KeyError): + h["Accept"] self.assertEqual(h.get("Accept"), None) self.assertEqual(h.getlist("Accept"), []) @@ -152,15 +155,11 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h1.getlist("hey"), [b"5"]) def test_invalid_value(self): - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers, {"foo": object()} - ) - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers().__setitem__, "foo", object() - ) - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers().setdefault, "foo", object() - ) - self.assertRaisesRegex( - TypeError, "Unsupported value type", Headers().setlist, "foo", [object()] - ) + with pytest.raises(TypeError, match="Unsupported value type"): + Headers({"foo": object()}) + with pytest.raises(TypeError, match="Unsupported value type"): + Headers()["foo"] = object() + with pytest.raises(TypeError, match="Unsupported value type"): + Headers().setdefault("foo", object()) + with pytest.raises(TypeError, match="Unsupported value type"): + Headers().setlist("foo", [object()]) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a8ab8240f..e5291157d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -7,6 +7,8 @@ from typing import Any from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes +import pytest + from scrapy.http import ( FormRequest, Headers, @@ -28,10 +30,12 @@ class RequestTest(unittest.TestCase): def test_init(self): # Request requires url in the __init__ method - self.assertRaises(Exception, self.request_class) + with pytest.raises(TypeError): + self.request_class() # url argument must be basestring - self.assertRaises(TypeError, self.request_class, 123) + with pytest.raises(TypeError): + self.request_class(123) r = self.request_class("http://www.example.com") r = self.request_class("http://www.example.com") @@ -64,9 +68,13 @@ class RequestTest(unittest.TestCase): self.request_class("data:,Hello%2C%20World!") def test_url_no_scheme(self): - self.assertRaises(ValueError, self.request_class, "foo") - self.assertRaises(ValueError, self.request_class, "/foo/") - self.assertRaises(ValueError, self.request_class, "/foo:bar") + msg = "Missing scheme in request url:" + with pytest.raises(ValueError, match=msg): + self.request_class("foo") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo/") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo:bar") def test_headers(self): # Different ways of setting headers attribute @@ -273,8 +281,10 @@ class RequestTest(unittest.TestCase): def test_immutable_attributes(self): r = self.request_class("http://example.com") - self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com") - self.assertRaises(AttributeError, setattr, r, "body", "xxx") + with pytest.raises(AttributeError): + r.url = "http://example2.com" + with pytest.raises(AttributeError): + r.body = "xxx" def test_callback_and_errback(self): def a_function(): @@ -309,11 +319,11 @@ class RequestTest(unittest.TestCase): self.assertIs(r5.errback, NO_CALLBACK) def test_callback_and_errback_type(self): - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.request_class("http://example.com", callback="a_function") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.request_class("http://example.com", errback="a_function") - with self.assertRaises(TypeError): + with pytest.raises(TypeError): self.request_class( url="http://example.com", callback="a_function", @@ -321,7 +331,7 @@ class RequestTest(unittest.TestCase): ) def test_no_callback(self): - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): NO_CALLBACK() def test_from_curl(self): @@ -403,13 +413,11 @@ class RequestTest(unittest.TestCase): # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z - self.assertRaises( - ValueError, - lambda: self.request_class.from_curl( + with pytest.raises(ValueError, match="Unrecognized options:"): + self.request_class.from_curl( 'curl -X PATCH "http://example.org" --foo -z', ignore_unknown_options=False, - ), - ) + ) class FormRequestTest(RequestTest): @@ -428,7 +436,7 @@ class FormRequestTest(RequestTest): data = (("a", "one"), ("a", "two"), ("b", "2")) url = self.request_class( "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data - ).url.split("#")[0] + ).url.split("#", maxsplit=1)[0] fs = _qs(self.request_class(url, method="GET", formdata=data)) self.assertEqual(set(fs[b"a"]), {b"one", b"two"}) self.assertEqual(fs[b"b"], [b"2"]) @@ -897,12 +905,11 @@ class FormRequestTest(RequestTest): <input type="submit" name="clickable2" value="clicked2"> </form>""" ) - self.assertRaises( + with pytest.raises( ValueError, - self.request_class.from_response, - response, - clickdata={"type": "submit"}, - ) + match="Multiple elements found .* matching the criteria in clickdata", + ): + self.request_class.from_response(response, clickdata={"type": "submit"}) def test_from_response_non_matching_clickdata(self): response = _buildresponse( @@ -910,12 +917,12 @@ class FormRequestTest(RequestTest): <input type="submit" name="clickable" value="clicked"> </form>""" ) - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - clickdata={"nonexistent": "notme"}, - ) + with pytest.raises( + ValueError, match="No clickable element matching clickdata:" + ): + self.request_class.from_response( + response, clickdata={"nonexistent": "notme"} + ) def test_from_response_nr_index_clickdata(self): response = _buildresponse( @@ -937,13 +944,15 @@ class FormRequestTest(RequestTest): </form> """ ) - self.assertRaises( - ValueError, self.request_class.from_response, response, clickdata={"nr": 1} - ) + with pytest.raises( + ValueError, match="No clickable element matching clickdata:" + ): + self.request_class.from_response(response, clickdata={"nr": 1}) def test_from_response_errors_noform(self): response = _buildresponse("""<html></html>""") - self.assertRaises(ValueError, self.request_class.from_response, response) + with pytest.raises(ValueError, match="No <form> element found in"): + self.request_class.from_response(response) def test_from_response_invalid_html5(self): response = _buildresponse( @@ -963,9 +972,8 @@ class FormRequestTest(RequestTest): <input type="hidden" name="test2" value="xxx"> </form>""" ) - self.assertRaises( - IndexError, self.request_class.from_response, response, formnumber=1 - ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formnumber=1) def test_from_response_noformname(self): response = _buildresponse( @@ -1021,13 +1029,8 @@ class FormRequestTest(RequestTest): <input type="hidden" name="two" value="2"> </form>""" ) - self.assertRaises( - IndexError, - self.request_class.from_response, - response, - formname="form3", - formnumber=2, - ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formname="form3", formnumber=2) def test_from_response_formid_exists(self): response = _buildresponse( @@ -1086,13 +1089,8 @@ class FormRequestTest(RequestTest): <input type="hidden" name="two" value="2"> </form>""" ) - self.assertRaises( - IndexError, - self.request_class.from_response, - response, - formid="form3", - formnumber=2, - ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formid="form3", formnumber=2) def test_from_response_select(self): res = _buildresponse( @@ -1245,12 +1243,10 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs[b"three"], [b"3"]) - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - formxpath="//form/input[@name='abc']", - ) + with pytest.raises(ValueError, match="No <form> element found with"): + self.request_class.from_response( + response, formxpath="//form/input[@name='abc']" + ) def test_from_response_unicode_xpath(self): response = _buildresponse(b'<form name="\xd1\x8a"></form>') @@ -1261,13 +1257,8 @@ class FormRequestTest(RequestTest): self.assertEqual(fs, {}) xpath = "//form[@name='\u03b1']" - self.assertRaisesRegex( - ValueError, - re.escape(xpath), - self.request_class.from_response, - response, - formxpath=xpath, - ) + with pytest.raises(ValueError, match=re.escape(xpath)): + self.request_class.from_response(response, formxpath=xpath) def test_from_response_button_submit(self): response = _buildresponse( @@ -1393,12 +1384,8 @@ class FormRequestTest(RequestTest): fs = _qs(r1) self.assertEqual(fs[b"three"], [b"3"]) - self.assertRaises( - ValueError, - self.request_class.from_response, - response, - formcss="input[name='abc']", - ) + with pytest.raises(ValueError, match="No <form> element found with"): + self.request_class.from_response(response, formcss="input[name='abc']") def test_from_response_valid_form_methods(self): form_methods = [ @@ -1424,13 +1411,11 @@ class FormRequestTest(RequestTest): </form> </body></html>""" ) - with self.assertRaises(ValueError) as context: + with pytest.raises( + ValueError, match="formdata should be a dict or iterable of tuples" + ): FormRequest.from_response(response, formdata=123) - self.assertIn( - "formdata should be a dict or iterable of tuples", str(context.exception) - ) - def test_form_response_with_custom_invalid_formdata_value_error(self): """Test that a ValueError is raised for fault-inducing iterable formdata input""" response = _buildresponse( @@ -1441,13 +1426,11 @@ class FormRequestTest(RequestTest): </body></html>""" ) - with self.assertRaises(ValueError) as context: + with pytest.raises( + ValueError, match="formdata should be a dict or iterable of tuples" + ): FormRequest.from_response(response, formdata=("a",)) - self.assertIn( - "formdata should be a dict or iterable of tuples", str(context.exception) - ) - def test_get_form_with_xpath_no_form_parent(self): """Test that _get_from raised a ValueError when an XPath selects an element not nested within a <form> and no <form> parent is found""" @@ -1462,11 +1445,9 @@ class FormRequestTest(RequestTest): </body></html>""" ) - with self.assertRaises(ValueError) as context: + with pytest.raises(ValueError, match="No <form> element found with"): FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') - self.assertIn("No <form> element found with", str(context.exception)) - def _buildresponse(body, **kwargs): kwargs.setdefault("body", body) @@ -1507,8 +1488,10 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=("response",), methodresponse="login") self._test_request(params=("pas£",), encoding="utf-8") self._test_request(params=(None,), allow_none=1) - self.assertRaises(TypeError, self._test_request) - self.assertRaises(TypeError, self._test_request, params=(None,)) + with pytest.raises(TypeError): + self._test_request() + with pytest.raises(TypeError): + self._test_request(params=(None,)) def test_latin1(self): self._test_request(params=("pas£",), encoding="latin1") diff --git a/tests/test_http_response.py b/tests/test_http_response.py index dde883451..5a943f084 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -27,14 +27,15 @@ class BaseResponseTest(unittest.TestCase): def test_init(self): # Response requires url in the constructor - self.assertRaises(Exception, self.response_class) + with pytest.raises(TypeError): + self.response_class() self.assertTrue( isinstance(self.response_class("http://example.com/"), self.response_class) ) - self.assertRaises(TypeError, self.response_class, b"http://example.com") - self.assertRaises( - TypeError, self.response_class, url="http://example.com", body={} - ) + with pytest.raises(TypeError): + self.response_class(b"http://example.com") + with pytest.raises(TypeError): + self.response_class(url="http://example.com", body={}) # body can be str or None self.assertTrue( isinstance( @@ -77,12 +78,8 @@ class BaseResponseTest(unittest.TestCase): self.assertEqual(r.status, 301) r = self.response_class("http://www.example.com", status="301") self.assertEqual(r.status, 301) - self.assertRaises( - ValueError, - self.response_class, - "http://example.com", - status="lala200", - ) + with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): + self.response_class("http://example.com", status="lala200") def test_copy(self): """Test Response copy""" @@ -122,14 +119,12 @@ class BaseResponseTest(unittest.TestCase): def test_unavailable_meta(self): r1 = self.response_class("http://www.example.com", body=b"Some body") - with self.assertRaisesRegex(AttributeError, r"Response\.meta not available"): + with pytest.raises(AttributeError, match=r"Response\.meta not available"): r1.meta def test_unavailable_cb_kwargs(self): r1 = self.response_class("http://www.example.com", body=b"Some body") - with self.assertRaisesRegex( - AttributeError, r"Response\.cb_kwargs not available" - ): + with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"): r1.cb_kwargs def test_copy_inherited_classes(self): @@ -179,8 +174,10 @@ class BaseResponseTest(unittest.TestCase): def test_immutable_attributes(self): r = self.response_class("http://example.com") - self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com") - self.assertRaises(AttributeError, setattr, r, "body", "xxx") + with pytest.raises(AttributeError): + r.url = "http://example2.com" + with pytest.raises(AttributeError): + r.body = "xxx" def test_urljoin(self): """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" @@ -192,10 +189,14 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com", body=b"hello") if self.response_class == Response: msg = "Response content isn't text" - self.assertRaisesRegex(AttributeError, msg, getattr, r, "text") - self.assertRaisesRegex(NotSupported, msg, r.css, "body") - self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body") - self.assertRaisesRegex(NotSupported, msg, r.jmespath, "body") + with pytest.raises(AttributeError, match=msg): + r.text + with pytest.raises(NotSupported, match=msg): + r.css("body") + with pytest.raises(NotSupported, match=msg): + r.xpath("//body") + with pytest.raises(NotSupported, match=msg): + r.jmespath("body") else: r.text r.css("body") @@ -216,7 +217,8 @@ class BaseResponseTest(unittest.TestCase): def test_follow_None_url(self): r = self.response_class("http://example.com") - self.assertRaises(ValueError, r.follow, None) + with pytest.raises(ValueError, match="url can't be None"): + r.follow(None) @pytest.mark.xfail( parse_version(w3lib_version) < parse_version("2.1.1"), @@ -279,18 +281,20 @@ class BaseResponseTest(unittest.TestCase): def test_follow_all_invalid(self): r = self.response_class("http://example.com") if self.response_class == Response: - with self.assertRaises(TypeError): + with pytest.raises(TypeError): list(r.follow_all(urls=None)) - with self.assertRaises(TypeError): + with pytest.raises(TypeError): list(r.follow_all(urls=12345)) - with self.assertRaises(ValueError): + with pytest.raises(ValueError, match="url can't be None"): list(r.follow_all(urls=[None])) else: - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): list(r.follow_all(urls=None)) - with self.assertRaises(TypeError): + with pytest.raises(TypeError): list(r.follow_all(urls=12345)) - with self.assertRaises(ValueError): + with pytest.raises(ValueError, match="url can't be None"): list(r.follow_all(urls=[None])) def test_follow_all_whitespace(self): @@ -399,12 +403,8 @@ class TextResponseTest(BaseResponseTest): "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 " "\u0442\u0435\u043a\u0441\u0442" ) - self.assertRaises( - TypeError, - self.response_class, - "http://www.example.com", - body="unicode body", - ) + with pytest.raises(TypeError): + self.response_class("http://www.example.com", body="unicode body") original_string = unicode_string.encode("cp1251") r1 = self.response_class( @@ -483,12 +483,8 @@ class TextResponseTest(BaseResponseTest): self._assert_response_values(r9, "cp1252", "€") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies - self.assertRaises( - TypeError, - self.response_class, - "http://www.example.com", - body="\xa3", - ) + with pytest.raises(TypeError): + self.response_class("http://www.example.com", body="\xa3") def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" @@ -679,20 +675,20 @@ class TextResponseTest(BaseResponseTest): self._assert_followed_url(sel, url, response=resp) # non-a elements are not supported - self.assertRaises(ValueError, resp.follow, resp.css("div")[0]) + with pytest.raises( + ValueError, match="Only <a> and <link> elements are supported" + ): + resp.follow(resp.css("div")[0]) def test_follow_selector_list(self): resp = self._links_response() - self.assertRaisesRegex(ValueError, "SelectorList", resp.follow, resp.css("a")) + with pytest.raises(ValueError, match="SelectorList"): + resp.follow(resp.css("a")) def test_follow_selector_invalid(self): resp = self._links_response() - self.assertRaisesRegex( - ValueError, - "Unsupported", - resp.follow, - resp.xpath("count(//div)")[0], - ) + with pytest.raises(ValueError, match="Unsupported"): + resp.follow(resp.xpath("count(//div)")[0]) def test_follow_selector_attribute(self): resp = self._links_response() @@ -704,7 +700,8 @@ class TextResponseTest(BaseResponseTest): url="http://example.com", body=b"<html><body><a name=123>click me</a></body></html>", ) - self.assertRaisesRegex(ValueError, "no href", resp.follow, resp.css("a")[0]) + with pytest.raises(ValueError, match="no href"): + resp.follow(resp.css("a")[0]) def test_follow_whitespace_selector(self): resp = self.response_class( @@ -812,7 +809,9 @@ class TextResponseTest(BaseResponseTest): def test_follow_all_too_many_arguments(self): response = self._links_response() - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): response.follow_all( css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]', @@ -825,7 +824,9 @@ class TextResponseTest(BaseResponseTest): text_body = b"""<html><body>text</body></html>""" text_response = self.response_class("http://www.example.com", body=text_body) - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="(Expecting value|Unexpected '<'): line 1" + ): text_response.json() def test_cache_json_response(self): @@ -1023,10 +1024,8 @@ class CustomResponseTest(TextResponseTest): self.assertEqual(r4.bar, "bar") self.assertIsNone(r4.lost) - with self.assertRaises(TypeError) as ctx: + with pytest.raises( + TypeError, + match=r"__init__\(\) got an unexpected keyword argument 'unknown'", + ): r1.replace(unknown="unknown") - self.assertTrue( - str(ctx.exception).endswith( - "__init__() got an unexpected keyword argument 'unknown'" - ) - ) diff --git a/tests/test_item.py b/tests/test_item.py index 0399c8f8d..47c5c3db6 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -2,6 +2,8 @@ import unittest from abc import ABCMeta from unittest import mock +import pytest + from scrapy.item import Field, Item, ItemMeta @@ -22,7 +24,8 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - self.assertRaises(KeyError, i.__getitem__, "name") + with pytest.raises(KeyError): + i["name"] i2 = TestItem(name="john doe") self.assertEqual(i2["name"], "john doe") @@ -33,15 +36,18 @@ class ItemTest(unittest.TestCase): i4 = TestItem(i3) self.assertEqual(i4["name"], "john doe") - self.assertRaises(KeyError, TestItem, {"name": "john doe", "other": "foo"}) + with pytest.raises(KeyError): + TestItem({"name": "john doe", "other": "foo"}) def test_invalid_field(self): class TestItem(Item): pass i = TestItem() - self.assertRaises(KeyError, i.__setitem__, "field", "text") - self.assertRaises(KeyError, i.__getitem__, "field") + with pytest.raises(KeyError): + i["field"] = "text" + with pytest.raises(KeyError): + i["field"] def test_repr(self): class TestItem(Item): @@ -72,14 +78,16 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - self.assertRaises(AttributeError, getattr, i, "name") + with pytest.raises(AttributeError): + i.name def test_raise_setattr(self): class TestItem(Item): name = Field() i = TestItem() - self.assertRaises(AttributeError, setattr, i, "name", "john") + with pytest.raises(AttributeError): + i.name = "john" def test_custom_methods(self): class TestItem(Item): @@ -92,7 +100,8 @@ class ItemTest(unittest.TestCase): self["name"] = name i = TestItem() - self.assertRaises(KeyError, i.get_name) + with pytest.raises(KeyError): + i.get_name() i["name"] = "lala" self.assertEqual(i.get_name(), "lala") i.change_name("other") @@ -223,7 +232,8 @@ class ItemTest(unittest.TestCase): class D(B, C): pass - self.assertRaises(KeyError, D, not_allowed="value") + with pytest.raises(KeyError): + D(not_allowed="value") self.assertEqual(D(save="X")["save"], "X") self.assertEqual(D.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) @@ -231,7 +241,8 @@ class ItemTest(unittest.TestCase): class E(C, B): pass - self.assertRaises(KeyError, E, not_allowed="value") + with pytest.raises(KeyError): + E(not_allowed="value") self.assertEqual(E(save="X")["save"], "X") self.assertEqual(E.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) diff --git a/tests/test_link.py b/tests/test_link.py index 35723bbd6..ed9d27a37 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -1,5 +1,7 @@ import unittest +import pytest + from scrapy.link import Link @@ -53,5 +55,5 @@ class LinkTest(unittest.TestCase): self._assert_same_links(l1, l2) def test_bytes_url(self): - with self.assertRaises(TypeError): + with pytest.raises(TypeError): Link(b"http://www.example.com/\xc2\xa3") diff --git a/tests/test_loader.py b/tests/test_loader.py index b52d5ea2e..1a933bb8d 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -4,6 +4,7 @@ import dataclasses import unittest import attr +import pytest from itemadapter import ItemAdapter from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst @@ -69,7 +70,8 @@ def processor_with_args(value, other=None, loader_context=None): class BasicItemLoaderTest(unittest.TestCase): def test_add_value_on_unknown_field(self): il = ProcessorItemLoader() - self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"]) + with pytest.raises(KeyError): + il.add_value("wrong_field", ["lala", "lolo"]) def test_load_item_using_default_loader(self): i = SummaryItem() @@ -294,12 +296,18 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_errors(self): l = ProcessorItemLoader() - self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href") - self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href") - self.assertRaises(RuntimeError, l.get_xpath, "//a/@href") - self.assertRaises(RuntimeError, l.add_css, "name", "#name::text") - self.assertRaises(RuntimeError, l.replace_css, "name", "#name::text") - self.assertRaises(RuntimeError, l.get_css, "#name::text") + with pytest.raises(RuntimeError): + l.add_xpath("url", "//a/@href") + with pytest.raises(RuntimeError): + l.replace_xpath("url", "//a/@href") + with pytest.raises(RuntimeError): + l.get_xpath("//a/@href") + with pytest.raises(RuntimeError): + l.add_css("name", "#name::text") + with pytest.raises(RuntimeError): + l.replace_css("name", "#name::text") + with pytest.raises(RuntimeError): + l.get_css("#name::text") def test_init_method_with_selector(self): sel = Selector(text="<html><body><div>marta</div></body></html>") diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 1e504f539..0d7921b1d 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -6,6 +6,7 @@ Once we remove the references from scrapy, we can remove these tests. import unittest from functools import partial +import pytest from itemloaders.processors import ( Compose, Identity, @@ -435,7 +436,13 @@ class BasicItemLoaderTest(unittest.TestCase): name_in = MapCompose(float) il = TestItemLoader() - self.assertRaises(ValueError, il.add_value, "name", ["marta", "other"]) + with pytest.raises( + ValueError, + match="Error with input processor MapCompose: .* " + "error='ValueError: Error in MapCompose .* " + "error='ValueError: could not convert", + ): + il.add_value("name", ["marta", "other"]) def test_error_output_processor(self): class TestItem(Item): @@ -447,7 +454,12 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() il.add_value("name", "marta") - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, + match="Error with output processor: .* " + "error='ValueError: Error in Compose .* " + "error='ValueError: could not convert", + ): il.load_item() def test_error_processor_as_argument(self): @@ -458,9 +470,13 @@ class BasicItemLoaderTest(unittest.TestCase): default_item_class = TestItem il = TestItemLoader() - self.assertRaises( - ValueError, il.add_value, "name", ["marta", "other"], Compose(float) - ) + with pytest.raises( + ValueError, + match=r"Error with processor Compose .* " + r"error='ValueError: Error in Compose .* " + r"error='TypeError: float\(\) argument", + ): + il.add_value("name", ["marta", "other"], Compose(float)) class InitializationFromDictTest(unittest.TestCase): @@ -630,7 +646,8 @@ class ProcessorsTest(unittest.TestCase): def test_join(self): proc = Join() - self.assertRaises(TypeError, proc, [None, "", "hello", "world"]) + with pytest.raises(TypeError): + proc([None, "", "hello", "world"]) self.assertEqual(proc(["", "hello", "world"]), " hello world") self.assertEqual(proc(["hello", "world"]), "hello world") self.assertIsInstance(proc(["hello", "world"]), str) @@ -641,9 +658,17 @@ class ProcessorsTest(unittest.TestCase): proc = Compose(str.upper) self.assertEqual(proc(None), None) proc = Compose(str.upper, stop_on_none=False) - self.assertRaises(ValueError, proc, None) + with pytest.raises( + ValueError, + match="Error in Compose with .* error='TypeError: (descriptor 'upper'|'str' object expected)", + ): + proc(None) proc = Compose(str.upper, lambda x: x + 1) - self.assertRaises(ValueError, proc, "hello") + with pytest.raises( + ValueError, + match="Error in Compose with .* error='TypeError: (can only|unsupported operand)", + ): + proc("hello") def test_mapcompose(self): def filter_world(x): @@ -657,9 +682,17 @@ class ProcessorsTest(unittest.TestCase): proc = MapCompose(filter_world, str.upper) self.assertEqual(proc(None), []) proc = MapCompose(filter_world, str.upper) - self.assertRaises(ValueError, proc, [1]) + with pytest.raises( + ValueError, + match="Error in MapCompose with .* error='TypeError: (descriptor 'upper'|'str' object expected)", + ): + proc([1]) proc = MapCompose(filter_world, lambda x: x + 1) - self.assertRaises(ValueError, proc, "hello") + with pytest.raises( + ValueError, + match="Error in MapCompose with .* error='TypeError: (can only|unsupported operand)", + ): + proc("hello") class SelectJmesTestCase(unittest.TestCase): diff --git a/tests/test_logstats.py b/tests/test_logstats.py index a4b002e34..6bc5b6f1f 100644 --- a/tests/test_logstats.py +++ b/tests/test_logstats.py @@ -1,6 +1,8 @@ import unittest from datetime import datetime +import pytest + from scrapy.extensions.logstats import LogStats from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider @@ -18,8 +20,9 @@ class TestLogStats(unittest.TestCase): def test_stats_calculations(self): logstats = LogStats.from_crawler(self.crawler) - with self.assertRaises(AttributeError): + with pytest.raises(AttributeError): logstats.pagesprev + with pytest.raises(AttributeError): logstats.itemsprev logstats.spider_opened(self.spider) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 3d049843a..1d89e44ce 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -7,6 +7,7 @@ from shutil import rmtree from tempfile import mkdtemp import attr +import pytest from itemadapter import ItemAdapter from twisted.trial import unittest @@ -146,11 +147,11 @@ class ImagesPipelineTestCase(unittest.TestCase): resp3 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf3.getvalue()) req = Request(url="https://dev.mydeco.com/mydeco.gif") - with self.assertRaises(ImageException): + with pytest.raises(ImageException): next(self.pipeline.get_images(response=resp1, request=req, info=object())) - with self.assertRaises(ImageException): + with pytest.raises(ImageException): next(self.pipeline.get_images(response=resp2, request=req, info=object())) - with self.assertRaises(ImageException): + with pytest.raises(ImageException): next(self.pipeline.get_images(response=resp3, request=req, info=object())) def test_get_images(self): diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 1584014b8..c223c4562 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -1,6 +1,7 @@ import tempfile import unittest +import pytest import queuelib from scrapy.http.request import Request @@ -40,9 +41,9 @@ class PriorityQueueTest(unittest.TestCase): self.crawler, FifoMemoryQueue, temp_dir ) queue.push(Request("https://example.org")) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): queue.peek() queue.close() @@ -129,9 +130,9 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") self.queue.push(Request("https://example.org")) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): self.queue.peek() diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 85133038a..2c605a015 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,5 +1,7 @@ import unittest +import pytest + from scrapy import Request, Spider from scrapy.http import FormRequest, JsonRequest from scrapy.utils.request import request_from_dict @@ -134,11 +136,15 @@ class RequestSerializationTest(unittest.TestCase): def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) - self.assertRaises(ValueError, r.to_dict, spider=self.spider) + with pytest.raises( + ValueError, match="is not an instance method in: <MethodsSpider" + ): + r.to_dict(spider=self.spider) def test_unserializable_callback2(self): r = Request("http://www.example.com", callback=self.spider.parse_item) - self.assertRaises(ValueError, r.to_dict, spider=None) + with pytest.raises(ValueError, match="is not an instance method in: None"): + r.to_dict(spider=None) def test_unserializable_callback3(self): """Parser method is removed or replaced dynamically.""" @@ -152,14 +158,18 @@ class RequestSerializationTest(unittest.TestCase): spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) spider.parse = None - self.assertRaises(ValueError, r.to_dict, spider=spider) + with pytest.raises(ValueError, match="is not an instance method in: <MySpider"): + r.to_dict(spider=spider) def test_callback_not_available(self): """Callback method is not available in the spider passed to from_dict""" spider = SpiderDelegation() r = Request("http://www.example.com", callback=spider.delegated_callback) d = r.to_dict(spider=spider) - self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo")) + with pytest.raises( + ValueError, match="Method 'delegated_callback' not found in: <Spider" + ): + request_from_dict(d, spider=Spider("foo")) class SpiderMixin: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 3ac330ae2..f2f8b96cd 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -5,6 +5,7 @@ import tempfile import unittest from typing import Any, NamedTuple +import pytest from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -229,7 +230,10 @@ class TestMigration(unittest.TestCase): next_scheduler_handler.create_scheduler() def test_migration(self): - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, + match="DownloaderAwarePriorityQueue accepts ``slot_startprios`` as a dict", + ): self._migration(self.tmpdir) @@ -351,5 +355,7 @@ class TestIncompatibility(unittest.TestCase): scheduler.open(spider) def test_incompatibility(self): - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP" + ): self._incompatible() diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 7c72805e2..c2bb8cec5 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -3,6 +3,7 @@ from __future__ import annotations from unittest import TestCase from urllib.parse import urljoin +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase as TwistedTestCase @@ -75,13 +76,12 @@ class BaseSchedulerTest(TestCase, InterfaceCheckMixin): def test_methods(self): self.assertIsNone(self.scheduler.open(Spider("foo"))) self.assertIsNone(self.scheduler.close("finished")) - self.assertRaises(NotImplementedError, self.scheduler.has_pending_requests) - self.assertRaises( - NotImplementedError, - self.scheduler.enqueue_request, - Request("https://example.org"), - ) - self.assertRaises(NotImplementedError, self.scheduler.next_request) + with pytest.raises(NotImplementedError): + self.scheduler.has_pending_requests() + with pytest.raises(NotImplementedError): + self.scheduler.enqueue_request(Request("https://example.org")) + with pytest.raises(NotImplementedError): + self.scheduler.next_request() class MinimalSchedulerTest(TestCase, InterfaceCheckMixin): @@ -89,15 +89,15 @@ class MinimalSchedulerTest(TestCase, InterfaceCheckMixin): self.scheduler = MinimalScheduler() def test_open_close(self): - with self.assertRaises(AttributeError): + with pytest.raises(AttributeError): self.scheduler.open(Spider("foo")) - with self.assertRaises(AttributeError): + with pytest.raises(AttributeError): self.scheduler.close("finished") def test_len(self): - with self.assertRaises(AttributeError): + with pytest.raises(AttributeError): self.scheduler.__len__() - with self.assertRaises(TypeError): + with pytest.raises(TypeError): len(self.scheduler) def test_enqueue_dequeue(self): diff --git a/tests/test_selector.py b/tests/test_selector.py index 4eda0460f..2d7a1442e 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -112,7 +112,7 @@ class SelectorTestCase(unittest.TestCase): ) def test_selector_bad_args(self): - with self.assertRaisesRegex(ValueError, "received both response and text"): + with pytest.raises(ValueError, match="received both response and text"): Selector(TextResponse(url="http://example.com", body=b""), text="") diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 5c8a19d9b..b7a316eee 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -235,9 +235,9 @@ class BaseSettingsTest(unittest.TestCase): self.assertIn("key_highprio", settings) del settings["key_highprio"] self.assertNotIn("key_highprio", settings) - with self.assertRaises(KeyError): + with pytest.raises(KeyError): settings.delete("notkey") - with self.assertRaises(KeyError): + with pytest.raises(KeyError): del settings["notkey"] def test_get(self): @@ -303,9 +303,19 @@ class BaseSettingsTest(unittest.TestCase): self.assertEqual(settings.getdict("TEST_DICT2"), {"key1": "val1", "ke2": 3}) self.assertEqual(settings.getdict("TEST_DICT3"), {}) self.assertEqual(settings.getdict("TEST_DICT3", {"key1": 5}), {"key1": 5}) - self.assertRaises(ValueError, settings.getdict, "TEST_LIST1") - self.assertRaises(ValueError, settings.getbool, "TEST_ENABLED_WRONG") - self.assertRaises(ValueError, settings.getbool, "TEST_DISABLED_WRONG") + with pytest.raises( + ValueError, + match="dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected", + ): + settings.getdict("TEST_LIST1") + with pytest.raises( + ValueError, match="Supported values for boolean settings are" + ): + settings.getbool("TEST_ENABLED_WRONG") + with pytest.raises( + ValueError, match="Supported values for boolean settings are" + ): + settings.getbool("TEST_DISABLED_WRONG") def test_getpriority(self): settings = BaseSettings({"key": "value"}, priority=99) @@ -381,11 +391,10 @@ class BaseSettingsTest(unittest.TestCase): def test_freeze(self): self.settings.freeze() - with self.assertRaises(TypeError) as cm: + with pytest.raises( + TypeError, match="Trying to modify an immutable Settings object" + ): self.settings.set("TEST_BOOL", False) - self.assertEqual( - str(cm.exception), "Trying to modify an immutable Settings object" - ) def test_frozencopy(self): frozencopy = self.settings.frozencopy() @@ -476,7 +485,7 @@ class SettingsTest(unittest.TestCase): def test_pop_item_with_default_value(self): settings = Settings() - with self.assertRaises(KeyError): + with pytest.raises(KeyError): settings.pop("DUMMY_CONFIG") dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value") @@ -491,9 +500,7 @@ class SettingsTest(unittest.TestCase): settings.freeze() - with self.assertRaises(TypeError) as error: + with pytest.raises( + TypeError, match="Trying to modify an immutable Settings object" + ): settings.pop("OTHER_DUMMY_CONFIG") - - self.assertEqual( - str(error.exception), "Trying to modify an immutable Settings object" - ) diff --git a/tests/test_spider.py b/tests/test_spider.py index 18a863350..af29872a8 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any from unittest import mock +import pytest from testfixtures import LogCapture from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest @@ -57,8 +58,11 @@ class SpiderTest(unittest.TestCase): def test_spider_without_name(self): """``__init__`` method arguments are assigned to spider attributes""" - self.assertRaises(ValueError, self.spider_class) - self.assertRaises(ValueError, self.spider_class, somearg="foo") + msg = "must have a name" + with pytest.raises(ValueError, match=msg): + self.spider_class() + with pytest.raises(ValueError, match=msg): + self.spider_class(somearg="foo") def test_from_crawler_crawler_and_settings_population(self): crawler = get_crawler() @@ -475,7 +479,7 @@ class CrawlSpiderTest(SpiderTest): spider = self.spider_class("example.com") spider.start_url = "https://www.example.com" - with self.assertRaisesRegex(AttributeError, r"^Crawling could not start.*$"): + with pytest.raises(AttributeError, match=r"^Crawling could not start.*$"): list(spider.start_requests()) @@ -825,5 +829,5 @@ class NoParseMethodSpiderTest(unittest.TestCase): resp = TextResponse(url="http://www.example.com/random_url", body=text) exc_msg = "Spider.parse callback is not defined" - with self.assertRaisesRegex(NotImplementedError, exc_msg): + with pytest.raises(NotImplementedError, match=exc_msg): spider.parse(resp) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 705f722b3..b103e9ed0 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -7,6 +7,7 @@ from pathlib import Path from tempfile import mkdtemp from unittest import mock +import pytest from twisted.trial import unittest from zope.interface.verify import verifyObject @@ -124,9 +125,8 @@ class SpiderLoaderTest(unittest.TestCase): } ) - self.assertRaisesRegex( - KeyError, "Spider not found", runner.create_crawler, "spider2" - ) + with pytest.raises(KeyError, match="Spider not found"): + runner.create_crawler("spider2") crawler = runner.create_crawler("spider1") self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) @@ -135,7 +135,8 @@ class SpiderLoaderTest(unittest.TestCase): def test_bad_spider_modules_exception(self): module = "tests.test_spiderloader.test_spiders.doesnotexist" settings = Settings({"SPIDER_MODULES": [module]}) - self.assertRaises(ImportError, SpiderLoader.from_settings, settings) + with pytest.raises(ImportError): + SpiderLoader.from_settings(settings) def test_bad_spider_modules_warning(self): with warnings.catch_warnings(record=True) as w: @@ -159,7 +160,8 @@ class SpiderLoaderTest(unittest.TestCase): with mock.patch.object(SpiderLoader, "_load_spiders") as m: m.side_effect = SyntaxError settings = Settings({"SPIDER_MODULES": [module]}) - self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings) + with pytest.raises(SyntaxError): + SpiderLoader.from_settings(settings) def test_syntax_error_warning(self): with ( diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index a8507c789..a9f3876bb 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterable from unittest import mock +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.python.failure import Failure @@ -299,12 +300,9 @@ class ProcessSpiderOutputCoroutineMiddleware: class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): @defer.inlineCallbacks def test_non_iterable(self): - with self.assertRaisesRegex( + with pytest.raises( _InvalidOutput, - ( - r"\.process_spider_output must return an iterable, got <class " - r"'NoneType'>" - ), + match=r"\.process_spider_output must return an iterable, got <class 'NoneType'>", ): yield self._get_middleware_result( ProcessSpiderOutputNonIterableMiddleware, @@ -312,9 +310,9 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): @defer.inlineCallbacks def test_coroutine(self): - with self.assertRaisesRegex( + with pytest.raises( _InvalidOutput, - r"\.process_spider_output must be an asynchronous generator", + match=r"\.process_spider_output must be an asynchronous generator", ): yield self._get_middleware_result( ProcessSpiderOutputCoroutineMiddleware, @@ -518,8 +516,8 @@ class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase): @defer.inlineCallbacks def _test_asyncgen_nodowngrade(self, *mw_classes): - with self.assertRaisesRegex( - _InvalidOutput, "Async iterable returned from .+ cannot be downgraded" + with pytest.raises( + _InvalidOutput, match="Async iterable returned from .+ cannot be downgraded" ): yield self._get_middleware_result(*mw_classes) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 307054de7..f9eb93d6b 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,6 +1,7 @@ import logging from unittest import TestCase +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase as TrialTestCase @@ -68,9 +69,8 @@ class TestHttpErrorMiddleware(TestCase): def test_process_spider_input(self): self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises( - HttpError, self.mw.process_spider_input, self.res404, self.spider - ) + with pytest.raises(HttpError): + self.mw.process_spider_input(self.res404, self.spider) def test_process_spider_exception(self): self.assertEqual( @@ -105,9 +105,8 @@ class TestHttpErrorMiddlewareSettings(TestCase): def test_process_spider_input(self): self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises( - HttpError, self.mw.process_spider_input, self.res404, self.spider - ) + with pytest.raises(HttpError): + self.mw.process_spider_input(self.res404, self.spider) self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): @@ -120,14 +119,14 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402.request = request self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + with pytest.raises(HttpError): + self.mw.process_spider_input(res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) - self.assertRaises( - HttpError, self.mw.process_spider_input, self.res402, self.spider - ) + with pytest.raises(HttpError): + self.mw.process_spider_input(self.res402, self.spider) class TestHttpErrorMiddlewareHandleAll(TestCase): @@ -151,7 +150,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402.request = request self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + with pytest.raises(HttpError): + self.mw.process_spider_input(res402, self.spider) def test_httperror_allow_all_false(self): crawler = get_crawler(_HttpErrorSpider) @@ -167,7 +167,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request_httpstatus_true - self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider) + with pytest.raises(HttpError): + mw.process_spider_input(res404, self.spider) self.assertIsNone(mw.process_spider_input(res402, self.spider)) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 4945ac25d..01a87c645 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -5,6 +5,8 @@ from typing import Any from unittest import TestCase from urllib.parse import urlparse +import pytest + from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.http import Request, Response from scrapy.settings import Settings @@ -884,7 +886,7 @@ class TestSettingsPolicyByName(TestCase): def test_invalid_name(self): settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"}) - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): RefererMiddleware(settings) def test_multiple_policy_tokens(self): @@ -925,7 +927,7 @@ class TestSettingsPolicyByName(TestCase): ) } ) - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): RefererMiddleware(settings) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 59d18d92e..72692afab 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -2,6 +2,7 @@ import shutil from datetime import datetime, timezone from tempfile import mkdtemp +import pytest from twisted.trial import unittest from scrapy.exceptions import NotConfigured @@ -42,4 +43,5 @@ class SpiderStateTest(unittest.TestCase): def test_not_configured(self): crawler = get_crawler(Spider) - self.assertRaises(NotConfigured, SpiderState.from_crawler, crawler) + with pytest.raises(NotConfigured): + SpiderState.from_crawler(crawler) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index a2e7ae65d..8556b75dd 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,6 +1,7 @@ import pickle import sys +import pytest from queuelib.tests import test_queue as t from scrapy.http import Request @@ -30,10 +31,17 @@ class MyLoader(ItemLoader): def nonserializable_object_test(self): q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + with pytest.raises( + ValueError, + match="unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as", + ): + q.push(lambda x: x) # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) sel = Selector(text="<html><body><p>some text</p></body></html>") - self.assertRaises(ValueError, q.push, sel) + with pytest.raises( + ValueError, match="unmarshallable object|can't pickle Selector objects" + ): + q.push(sel) class FifoDiskQueueTestMixin: diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 04eeae4dc..88f6657d8 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -6,6 +6,7 @@ import shutil import tempfile import unittest +import pytest import queuelib from scrapy.http import Request @@ -69,9 +70,9 @@ class RequestQueueTestMixin: req = Request("http://www.example.com") q.push(req) self.assertEqual(len(q), 1) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): q.peek() self.assertEqual(q.pop().url, req.url) @@ -120,9 +121,9 @@ class FifoQueueMixin(RequestQueueTestMixin): q.push(req1) q.push(req2) q.push(req3) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): q.peek() self.assertEqual(len(q), 3) @@ -176,9 +177,9 @@ class LifoQueueMixin(RequestQueueTestMixin): q.push(req1) q.push(req2) q.push(req3) - with self.assertRaises( + with pytest.raises( NotImplementedError, - msg="The underlying queue class does not implement 'peek'", + match="The underlying queue class does not implement 'peek'", ): q.peek() self.assertEqual(len(q), 3) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index cbea41129..e27bb7b74 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,5 +1,7 @@ import unittest +import pytest + from scrapy.exceptions import UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( @@ -32,7 +34,9 @@ class BuildComponentListTest(unittest.TestCase): ) # Same priority raises ValueError duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20) - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="Some paths in .* convert to the same object" + ): build_component_list(duplicate_bs, convert=lambda x: x.lower()) def test_valid_numbers(self): @@ -58,21 +62,13 @@ class UtilsConfTestCase(unittest.TestCase): class FeedExportConfigTestCase(unittest.TestCase): def test_feed_export_config_invalid_format(self): settings = Settings() - self.assertRaises( - UsageError, - feed_process_params_from_cli, - settings, - ["items.dat"], - ) + with pytest.raises(UsageError): + feed_process_params_from_cli(settings, ["items.dat"]) def test_feed_export_config_mismatch(self): settings = Settings() - self.assertRaises( - UsageError, - feed_process_params_from_cli, - settings, - ["items1.dat", "items2.dat"], - ) + with pytest.raises(UsageError): + feed_process_params_from_cli(settings, ["items1.dat", "items2.dat"]) def test_feed_export_config_explicit_formats(self): settings = Settings() @@ -117,11 +113,9 @@ class FeedExportConfigTestCase(unittest.TestCase): ) def test_output_and_overwrite_output(self): - with self.assertRaises(UsageError): + with pytest.raises(UsageError): feed_process_params_from_cli( - Settings(), - ["output1.json"], - overwrite_output=["output2.json"], + Settings(), ["output1.json"], overwrite_output=["output2.json"] ) def test_feed_complete_default_values_from_settings_empty(self): diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 5d99161bf..a5b438645 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,6 +1,7 @@ import unittest import warnings +import pytest from w3lib.http import basic_auth_header from scrapy import Request @@ -205,11 +206,11 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) def test_too_few_arguments_error(self): - self.assertRaisesRegex( + with pytest.raises( ValueError, - r"too few arguments|the following arguments are required:\s*url", - lambda: curl_to_request_kwargs("curl"), - ) + match=r"too few arguments|the following arguments are required:\s*url", + ): + curl_to_request_kwargs("curl") def test_ignore_unknown_options(self): # case 1: ignore_unknown_options=True: @@ -220,16 +221,11 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) # case 2: ignore_unknown_options=False (raise exception): - self.assertRaisesRegex( - ValueError, - "Unrecognized options:.*--bar.*--baz", - lambda: curl_to_request_kwargs( + with pytest.raises(ValueError, match="Unrecognized options:.*--bar.*--baz"): + curl_to_request_kwargs( "curl --bar --baz http://www.example.com", ignore_unknown_options=False - ), - ) + ) def test_must_start_with_curl_error(self): - self.assertRaises( - ValueError, - lambda: curl_to_request_kwargs("carl -X POST http://example.org"), - ) + with pytest.raises(ValueError, match="A curl command must start"): + curl_to_request_kwargs("carl -X POST http://example.org") diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index fadbc6daa..2e35d339a 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -87,8 +87,10 @@ class CaseInsensitiveDictMixin: def test_delete(self): d = self.dict_class({"key_lower": 1}) del d["key_LOWER"] - self.assertRaises(KeyError, d.__getitem__, "key_LOWER") - self.assertRaises(KeyError, d.__getitem__, "key_lower") + with pytest.raises(KeyError): + d["key_LOWER"] + with pytest.raises(KeyError): + d["key_lower"] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_getdefault(self): @@ -138,7 +140,8 @@ class CaseInsensitiveDictMixin: d = self.dict_class() d["a"] = 1 self.assertEqual(d.pop("A"), 1) - self.assertRaises(KeyError, d.pop, "A") + with pytest.raises(KeyError): + d.pop("A") def test_normkey(self): class MyDict(self.dict_class): @@ -279,8 +282,8 @@ class SequenceExcludeTest(unittest.TestCase): self.assertIn(set("bar"), d) # supplied sequence is a set, so checking for list (non)inclusion fails - self.assertRaises(TypeError, (0, 1, 2) in d) - self.assertRaises(TypeError, d.__contains__, ["a", "b", "c"]) + with pytest.raises(TypeError): + ["a", "b", "c"] in d # noqa: B015 for v in [-3, "test", 1.1]: self.assertNotIn(v, d) diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index dc5fbd3c3..e917b6947 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -3,6 +3,8 @@ import unittest import warnings from unittest import mock +import pytest + from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import create_deprecated_class, update_classpath @@ -181,7 +183,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a) assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1) - self.assertRaises(TypeError, issubclass, object(), DeprecatedName) + with pytest.raises(TypeError): + issubclass(object(), DeprecatedName) def test_isinstance(self): with warnings.catch_warnings(): diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 7b7a25db8..d40cae9c7 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,6 +1,8 @@ import unittest +from gzip import BadGzipFile from pathlib import Path +import pytest from w3lib.encoding import html_to_unicode from scrapy.http import Response @@ -27,9 +29,8 @@ class GunzipTest(unittest.TestCase): assert text.endswith(b"</html") def test_gunzip_no_gzip_file_raises(self): - self.assertRaises( - OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes() - ) + with pytest.raises(BadGzipFile): + gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes()) def test_gunzip_truncated_short(self): r1 = Response( diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 12507c6a3..9ad30617a 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -215,7 +215,7 @@ class XmliterBaseTestCase: """ response = XmlResponse(url="http://mydummycompany.com", body=body) my_iter = self.xmliter(response, "g:link_image") - with self.assertRaises(StopIteration): + with pytest.raises(StopIteration): next(my_iter) @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") @@ -228,13 +228,14 @@ class XmliterBaseTestCase: iter = self.xmliter(body, "product") next(iter) next(iter) - - self.assertRaises(StopIteration, next, iter) + with pytest.raises(StopIteration): + next(iter) @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") - self.assertRaises(TypeError, next, i) + with pytest.raises(TypeError): + next(i) @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_encoding(self): @@ -344,7 +345,8 @@ class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") - self.assertRaises(TypeError, next, i) + with pytest.raises(TypeError): + next(i) class UtilsCsvTestCase(unittest.TestCase): @@ -491,8 +493,8 @@ class UtilsCsvTestCase(unittest.TestCase): next(iter) next(iter) next(iter) - - self.assertRaises(StopIteration, next, iter) + with pytest.raises(StopIteration): + next(iter) def test_csviter_encoding(self): body1 = get_testdata("feeds", "feed-sample4.csv") diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index e25bdfe3f..a67e16962 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -32,9 +32,12 @@ class UtilsMiscTestCase(unittest.TestCase): self.assertIs(obj, load_object) def test_load_object_exceptions(self): - self.assertRaises(ImportError, load_object, "nomodule999.mod.function") - self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999") - self.assertRaises(TypeError, load_object, {}) + with pytest.raises(ImportError): + load_object("nomodule999.mod.function") + with pytest.raises(NameError): + load_object("scrapy.utils.misc.load_object999") + with pytest.raises(TypeError): + load_object({}) def test_walk_modules(self): mods = walk_modules("tests.test_utils_misc.test_walk_modules") @@ -59,7 +62,8 @@ class UtilsMiscTestCase(unittest.TestCase): ] self.assertEqual({m.__name__ for m in mods}, set(expected)) - self.assertRaises(ImportError, walk_modules, "nomodule999") + with pytest.raises(ImportError): + walk_modules("nomodule999") def test_walk_modules_egg(self): egg = str(Path(__file__).parent / "test.egg") @@ -148,11 +152,13 @@ class UtilsMiscTestCase(unittest.TestCase): create_instance(m, None, crawler, *args, **kwargs) m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs) - with self.assertRaises(ValueError): + with pytest.raises( + ValueError, match="Specify at least one of settings and crawler" + ): create_instance(m, None, None) m.from_settings.return_value = None - with self.assertRaises(TypeError): + with pytest.raises(TypeError): create_instance(m, settings, None) def test_build_from_crawler(self): @@ -191,7 +197,7 @@ class UtilsMiscTestCase(unittest.TestCase): # Check adoption of crawler m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"]) m.from_crawler.return_value = None - with self.assertRaises(TypeError): + with pytest.raises(TypeError): build_from_crawler(m, crawler, *args, **kwargs) def test_set_environ(self): diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index a693d6b53..3b0739276 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -87,7 +87,8 @@ class ToUnicodeTest(unittest.TestCase): self.assertEqual(to_unicode("\xf1e\xf1e\xf1e"), "\xf1e\xf1e\xf1e") def test_converting_a_strange_object_should_raise_TypeError(self): - self.assertRaises(TypeError, to_unicode, 423) + with pytest.raises(TypeError): + to_unicode(423) def test_errors_argument(self): self.assertEqual(to_unicode(b"a\xedb", "utf-8", errors="replace"), "a\ufffdb") @@ -104,7 +105,8 @@ class ToBytesTest(unittest.TestCase): self.assertEqual(to_bytes(b"lel\xf1e"), b"lel\xf1e") def test_converting_a_strange_object_should_raise_TypeError(self): - self.assertRaises(TypeError, to_bytes, unittest) + with pytest.raises(TypeError): + to_bytes(pytest) def test_errors_argument(self): self.assertEqual(to_bytes("a\ufffdb", "latin-1", errors="replace"), b"a?b") diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index c6ba8cbbb..af7906781 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -35,7 +35,8 @@ class ResponseUtilsTest(unittest.TestCase): assert open_in_browser(response, _openfunc=browser_open), "Browser not called" resp = Response(url, body=body) - self.assertRaises(TypeError, open_in_browser, resp, debug=True) + with pytest.raises(TypeError): + open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg def test_get_meta_refresh(self): r1 = HtmlResponse( From 87db3f2fd6f9d365208a69a0de31181f1ea70e43 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 28 Feb 2025 15:18:55 +0500 Subject: [PATCH 1667/2083] Add SpiderLoaderProtocol. (#6694) --- scrapy/crawler.py | 10 ++++++---- scrapy/spiderloader.py | 19 ++++++++++++++++++- scrapy/utils/spider.py | 10 +++++----- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1873c90d3..1ec1e31dc 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -42,7 +42,7 @@ if TYPE_CHECKING: from collections.abc import Generator, Iterable from scrapy.logformatter import LogFormatter - from scrapy.spiderloader import SpiderLoader + from scrapy.spiderloader import SpiderLoaderProtocol from scrapy.statscollectors import StatsCollector from scrapy.utils.request import RequestFingerprinterProtocol @@ -282,19 +282,21 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings: BaseSettings) -> SpiderLoader: + def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) verifyClass(ISpiderLoader, loader_cls) - return cast("SpiderLoader", loader_cls.from_settings(settings.frozencopy())) + return cast( + "SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()) + ) def __init__(self, settings: dict[str, Any] | Settings | None = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) AddonManager.load_pre_crawler_settings(settings) self.settings: Settings = settings - self.spider_loader: SpiderLoader = self._get_spider_loader(settings) + self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings) self._crawlers: set[Crawler] = set() self._active: set[Deferred[None]] = set() self.bootstrap_failed = False diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 210e729a1..f537e0593 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,7 +3,7 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol from zope.interface import implementer @@ -21,6 +21,23 @@ if TYPE_CHECKING: from scrapy.settings import BaseSettings +class SpiderLoaderProtocol(Protocol): + @classmethod + def from_settings(cls, settings: BaseSettings) -> Self: + """Return an instance of the class for the given settings""" + + def load(self, spider_name: str) -> type[Spider]: + """Return the Spider class for the given spider name. If the spider + name is not found, it must raise a KeyError.""" + + def list(self) -> list[str]: + """Return a list with the names of all spiders available in the + project""" + + def find_by_request(self, request: Request) -> __builtins__.list[str]: + """Return the list of spiders names that can handle the given request""" + + @implementer(ISpiderLoader) class SpiderLoader: """ diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 5277a292c..74fd0e354 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from twisted.internet.defer import Deferred from scrapy import Request - from scrapy.spiderloader import SpiderLoader + from scrapy.spiderloader import SpiderLoaderProtocol logger = logging.getLogger(__name__) @@ -64,7 +64,7 @@ def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]: @overload def spidercls_for_request( - spider_loader: SpiderLoader, + spider_loader: SpiderLoaderProtocol, request: Request, default_spidercls: type[Spider], log_none: bool = ..., @@ -74,7 +74,7 @@ def spidercls_for_request( @overload def spidercls_for_request( - spider_loader: SpiderLoader, + spider_loader: SpiderLoaderProtocol, request: Request, default_spidercls: Literal[None], log_none: bool = ..., @@ -84,7 +84,7 @@ def spidercls_for_request( @overload def spidercls_for_request( - spider_loader: SpiderLoader, + spider_loader: SpiderLoaderProtocol, request: Request, *, log_none: bool = ..., @@ -93,7 +93,7 @@ def spidercls_for_request( def spidercls_for_request( - spider_loader: SpiderLoader, + spider_loader: SpiderLoaderProtocol, request: Request, default_spidercls: type[Spider] | None = None, log_none: bool = False, From a5731c1944d8aa7b1921c543b7ad616ad131853e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 2 Mar 2025 21:04:12 +0500 Subject: [PATCH 1668/2083] Move most of the test utils inside tests. --- scrapy/utils/test.py | 28 +++++++++++++++ scrapy/utils/testproc.py | 9 +++++ scrapy/utils/testsite.py | 10 +++++- tests/test_command_fetch.py | 4 +-- tests/test_command_parse.py | 4 +-- tests/test_command_shell.py | 4 +-- tests/test_command_version.py | 2 +- tests/test_feedexport.py | 21 +++++++++-- tests/test_pipeline_files.py | 51 +++++++++++++++++++++++--- tests/utils/__init__.py | 0 tests/utils/testproc.py | 67 +++++++++++++++++++++++++++++++++++ tests/utils/testsite.py | 47 ++++++++++++++++++++++++ 12 files changed, 232 insertions(+), 15 deletions(-) create mode 100644 tests/utils/__init__.py create mode 100644 tests/utils/testproc.py create mode 100644 tests/utils/testsite.py diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index e89786103..db1f5c419 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -34,11 +34,23 @@ _T = TypeVar("_T") def assert_gcs_environ() -> None: + warnings.warn( + "The assert_gcs_environ() function is deprecated and will be removed in a future version of Scrapy." + " Check GCS_PROJECT_ID directly.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) if "GCS_PROJECT_ID" not in os.environ: raise SkipTest("GCS_PROJECT_ID not found") def skip_if_no_boto() -> None: + warnings.warn( + "The skip_if_no_boto() function is deprecated and will be removed in a future version of Scrapy." + " Check scrapy.utils.boto.is_botocore_available() directly.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) if not is_botocore_available(): raise SkipTest("missing botocore library") @@ -48,6 +60,11 @@ def get_gcs_content_and_delete( ) -> tuple[bytes, list[dict[str, str]], Any]: from google.cloud import storage + warnings.warn( + "The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) bucket = client.get_bucket(bucket) blob = bucket.get_blob(path) @@ -67,6 +84,11 @@ def get_ftp_content_and_delete( ) -> bytes: from ftplib import FTP + warnings.warn( + "The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) ftp = FTP() ftp.connect(host, port) ftp.login(username, password) @@ -150,6 +172,12 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: """ from google.cloud.storage import Blob, Bucket, Client + warnings.warn( + "The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + client_mock = mock.create_autospec(Client) bucket_mock = mock.create_autospec(Bucket) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 85d7c940f..10f764ab8 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -2,18 +2,27 @@ from __future__ import annotations import os import sys +import warnings from typing import TYPE_CHECKING, cast from twisted.internet.defer import Deferred from twisted.internet.error import ProcessTerminated from twisted.internet.protocol import ProcessProtocol +from scrapy.exceptions import ScrapyDeprecationWarning + if TYPE_CHECKING: from collections.abc import Iterable from twisted.python.failure import Failure +warnings.warn( + "The scrapy.utils.testproc module is deprecated.", + ScrapyDeprecationWarning, +) + + class ProcessTest: command: str | None = None prefix = [sys.executable, "-m", "scrapy.cmdline"] diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index ca1f68116..f12b301fd 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -1,7 +1,15 @@ +import warnings from urllib.parse import urljoin from twisted.web import resource, server, static, util +from scrapy.exceptions import ScrapyDeprecationWarning + +warnings.warn( + "The scrapy.utils.testsite module is deprecated.", + ScrapyDeprecationWarning, +) + class SiteTest: def setUp(self): @@ -48,7 +56,7 @@ def test_site(): if __name__ == "__main__": - from twisted.internet import reactor + from twisted.internet import reactor # pylint: disable=ungrouped-imports port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") print(f"http://localhost:{port.getHost().port}/") diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index d2027d1c2..a4d7fdd30 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -1,8 +1,8 @@ from twisted.internet import defer from twisted.trial import unittest -from scrapy.utils.testproc import ProcessTest -from scrapy.utils.testsite import SiteTest +from tests.utils.testproc import ProcessTest +from tests.utils.testsite import SiteTest class FetchTest(ProcessTest, SiteTest, unittest.TestCase): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 9356d6b79..9f2c7fa13 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -7,9 +7,9 @@ from twisted.internet import defer from scrapy.commands import parse from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.testproc import ProcessTest -from scrapy.utils.testsite import SiteTest from tests.test_commands import CommandTest +from tests.utils.testproc import ProcessTest +from tests.utils.testsite import SiteTest def _textmode(bstr): diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7918d94b2..9ca5e05dc 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -7,10 +7,10 @@ from pexpect.popen_spawn import PopenSpawn from twisted.internet import defer from twisted.trial import unittest -from scrapy.utils.testproc import ProcessTest -from scrapy.utils.testsite import SiteTest from tests import NON_EXISTING_RESOLVABLE, tests_datadir from tests.mockserver import MockServer +from tests.utils.testproc import ProcessTest +from tests.utils.testsite import SiteTest class ShellTest(ProcessTest, SiteTest, unittest.TestCase): diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 18c1c531c..917f457cb 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -4,7 +4,7 @@ from twisted.internet import defer from twisted.trial import unittest import scrapy -from scrapy.utils.testproc import ProcessTest +from tests.utils.testproc import ProcessTest class VersionTest(ProcessTest, unittest.TestCase): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b4c1b9631..8e008ab98 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -17,7 +17,7 @@ from io import BytesIO from logging import getLogger from pathlib import Path from string import ascii_letters, digits -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from unittest import mock from urllib.parse import quote, urljoin from urllib.request import pathname2url @@ -48,7 +48,7 @@ from scrapy.extensions.feedexport import ( ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_crawler, mock_google_cloud_storage +from scrapy.utils.test import get_crawler from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider @@ -71,6 +71,23 @@ def build_url(path: str | PathLike) -> str: return urljoin("file:", path_str) +def mock_google_cloud_storage() -> tuple[Any, Any, Any]: + """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob + classes and set their proper return values. + """ + from google.cloud.storage import Blob, Bucket, Client + + client_mock = mock.create_autospec(Client) + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + return (client_mock, bucket_mock, blob_mock) + + class FileFeedStorageTest(unittest.TestCase): def test_store_file_uri(self): path = Path(self.mktemp()).resolve() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4c59fcfb7..05fd17207 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -6,8 +6,10 @@ import warnings from datetime import datetime from io import BytesIO from pathlib import Path +from posixpath import split from shutil import rmtree from tempfile import mkdtemp +from typing import Any from unittest import mock from urllib.parse import urlparse @@ -27,16 +29,54 @@ from scrapy.pipelines.files import ( S3FilesStore, ) from scrapy.utils.test import ( - assert_gcs_environ, get_crawler, - get_ftp_content_and_delete, - get_gcs_content_and_delete, ) from tests.mockserver import MockFTPServer from .test_pipeline_media import _mocked_download_func +def get_gcs_content_and_delete( + bucket: Any, path: str +) -> tuple[bytes, list[dict[str, str]], Any]: + from google.cloud import storage + + client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) + bucket = client.get_bucket(bucket) + blob = bucket.get_blob(path) + content = blob.download_as_string() + acl = list(blob.acl) # loads acl before it will be deleted + bucket.delete_blob(path) + return content, acl, blob + + +def get_ftp_content_and_delete( + path: str, + host: str, + port: int, + username: str, + password: str, + use_active_mode: bool = False, +) -> bytes: + from ftplib import FTP + + ftp = FTP() + ftp.connect(host, port) + ftp.login(username, password) + if use_active_mode: + ftp.set_pasv(False) + ftp_data: list[bytes] = [] + + def buffer_data(data: bytes) -> None: + ftp_data.append(data) + + ftp.retrbinary(f"RETR {path}", buffer_data) + dirname, filename = split(path) + ftp.cwd(dirname) + ftp.delete(filename) + return b"".join(ftp_data) + + class FilesPipelineTestCase(unittest.TestCase): def setUp(self): self.tempdir = mkdtemp() @@ -597,10 +637,12 @@ class TestS3FilesStore(unittest.TestCase): stub.assert_no_pending_responses() +@pytest.mark.skipif( + "GCS_PROJECT_ID" not in os.environ, reason="GCS_PROJECT_ID not found" +) class TestGCSFilesStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): - assert_gcs_environ() uri = os.environ.get("GCS_TEST_FILE_URI") if not uri: raise unittest.SkipTest("No GCS URI available for testing") @@ -629,7 +671,6 @@ class TestGCSFilesStore(unittest.TestCase): """Test to make sure that paths used to store files is the same as the one used to get already uploaded files. """ - assert_gcs_environ() try: import google.cloud.storage # noqa: F401 except ModuleNotFoundError: diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/utils/testproc.py b/tests/utils/testproc.py new file mode 100644 index 000000000..85d7c940f --- /dev/null +++ b/tests/utils/testproc.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import os +import sys +from typing import TYPE_CHECKING, cast + +from twisted.internet.defer import Deferred +from twisted.internet.error import ProcessTerminated +from twisted.internet.protocol import ProcessProtocol + +if TYPE_CHECKING: + from collections.abc import Iterable + + from twisted.python.failure import Failure + + +class ProcessTest: + command: str | None = None + prefix = [sys.executable, "-m", "scrapy.cmdline"] + cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109 + + def execute( + self, + args: Iterable[str], + check_code: bool = True, + settings: str | None = None, + ) -> Deferred[TestProcessProtocol]: + from twisted.internet import reactor + + env = os.environ.copy() + if settings is not None: + env["SCRAPY_SETTINGS_MODULE"] = settings + assert self.command + cmd = [*self.prefix, self.command, *args] + pp = TestProcessProtocol() + pp.deferred.addCallback(self._process_finished, cmd, check_code) + reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) + return pp.deferred + + def _process_finished( + self, pp: TestProcessProtocol, cmd: list[str], check_code: bool + ) -> tuple[int, bytes, bytes]: + if pp.exitcode and check_code: + msg = f"process {cmd} exit with code {pp.exitcode}" + msg += f"\n>>> stdout <<<\n{pp.out.decode()}" + msg += "\n" + msg += f"\n>>> stderr <<<\n{pp.err.decode()}" + raise RuntimeError(msg) + return cast(int, pp.exitcode), pp.out, pp.err + + +class TestProcessProtocol(ProcessProtocol): + def __init__(self) -> None: + self.deferred: Deferred[TestProcessProtocol] = Deferred() + self.out: bytes = b"" + self.err: bytes = b"" + self.exitcode: int | None = None + + def outReceived(self, data: bytes) -> None: + self.out += data + + def errReceived(self, data: bytes) -> None: + self.err += data + + def processEnded(self, status: Failure) -> None: + self.exitcode = cast(ProcessTerminated, status.value).exitCode + self.deferred.callback(self) diff --git a/tests/utils/testsite.py b/tests/utils/testsite.py new file mode 100644 index 000000000..473738773 --- /dev/null +++ b/tests/utils/testsite.py @@ -0,0 +1,47 @@ +from urllib.parse import urljoin + +from twisted.web import resource, server, static, util + + +class SiteTest: + def setUp(self): + from twisted.internet import reactor + + super().setUp() + self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") + self.baseurl = f"http://localhost:{self.site.getHost().port}/" + + def tearDown(self): + super().tearDown() + self.site.stopListening() + + def url(self, path: str) -> str: + return urljoin(self.baseurl, path) + + +class NoMetaRefreshRedirect(util.Redirect): + def render(self, request: server.Request) -> bytes: + content = util.Redirect.render(self, request) + return content.replace( + b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' + ) + + +def test_site(): + r = resource.Resource() + r.putChild(b"text", static.Data(b"Works", "text/plain")) + r.putChild( + b"html", + static.Data( + b"<body><p class='one'>Works</p><p class='two'>World</p></body>", + "text/html", + ), + ) + r.putChild( + b"enc-gb18030", + static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"), + ) + r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) + r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) + return server.Site(r) From 93c076047bf5e3169feb2c29aca24e71bab0f8f0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 2 Mar 2025 21:19:24 +0500 Subject: [PATCH 1669/2083] Add scrapy/utils/testproc.py to collect_ignore to silence a warning. --- conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conftest.py b/conftest.py index a08ad9d05..f33ffb1a4 100644 --- a/conftest.py +++ b/conftest.py @@ -13,6 +13,7 @@ def _py_files(folder): collect_ignore = [ # not a test, but looks like a test + "scrapy/utils/testproc.py", "scrapy/utils/testsite.py", "tests/ftpserver.py", "tests/mockserver.py", From d161d1d47d445272cba35ed81eec068ed4be8b1a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 4 Mar 2025 13:31:26 +0500 Subject: [PATCH 1670/2083] Convert tests/test_utils* to plain asserts. (#6695) --- tests/test_utils_asyncgen.py | 6 +- tests/test_utils_asyncio.py | 9 +- tests/test_utils_conf.py | 126 +++---- tests/test_utils_console.py | 20 +- tests/test_utils_curl.py | 11 +- tests/test_utils_datatypes.py | 207 ++++++----- tests/test_utils_defer.py | 46 ++- tests/test_utils_deprecate.py | 83 +++-- tests/test_utils_display.py | 20 +- tests/test_utils_gz.py | 19 +- tests/test_utils_httpobj.py | 3 +- tests/test_utils_iterators.py | 274 +++++++-------- tests/test_utils_log.py | 43 ++- tests/test_utils_misc/__init__.py | 51 ++- ...t_return_with_argument_inside_generator.py | 63 ++-- tests/test_utils_project.py | 13 +- tests/test_utils_python.py | 113 +++--- tests/test_utils_request.py | 106 +++--- tests/test_utils_response.py | 27 +- tests/test_utils_serialize.py | 27 +- tests/test_utils_signal.py | 22 +- tests/test_utils_sitemap.py | 178 +++++----- tests/test_utils_spider.py | 14 +- tests/test_utils_template.py | 11 +- tests/test_utils_trackref.py | 42 ++- tests/test_utils_url.py | 323 +++++++----------- 26 files changed, 835 insertions(+), 1022 deletions(-) diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 8adeea5c0..9b5a25b3a 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -4,15 +4,15 @@ from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import deferred_f_from_coro_f -class AsyncgenUtilsTest(unittest.TestCase): +class TestAsyncgenUtils(unittest.TestCase): @deferred_f_from_coro_f async def test_as_async_generator(self): ag = as_async_generator(range(42)) results = [i async for i in ag] - self.assertEqual(results, list(range(42))) + assert results == list(range(42)) @deferred_f_from_coro_f async def test_collect_asyncgen(self): ag = as_async_generator(range(42)) results = await collect_asyncgen(ag) - self.assertEqual(results, list(range(42))) + assert results == list(range(42)) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index ecac0df9c..a65a36219 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -2,7 +2,6 @@ import asyncio import warnings import pytest -from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( @@ -13,19 +12,17 @@ from scrapy.utils.reactor import ( @pytest.mark.usefixtures("reactor_pytest") -class AsyncioTest(TestCase): +class TestAsyncio: def test_is_asyncio_reactor_installed(self): # the result should depend only on the pytest --reactor argument - self.assertEqual( - is_asyncio_reactor_installed(), self.reactor_pytest == "asyncio" - ) + assert is_asyncio_reactor_installed() == (self.reactor_pytest == "asyncio") def test_install_asyncio_reactor(self): from twisted.internet import reactor as original_reactor with warnings.catch_warnings(record=True) as w: install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - self.assertEqual(len(w), 0) + assert len(w) == 0 from twisted.internet import reactor # pylint: disable=reimported assert original_reactor == reactor diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index e27bb7b74..26f158380 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,5 +1,3 @@ -import unittest - import pytest from scrapy.exceptions import UsageError @@ -12,26 +10,24 @@ from scrapy.utils.conf import ( ) -class BuildComponentListTest(unittest.TestCase): +class TestBuildComponentList: def test_build_dict(self): d = {"one": 1, "two": None, "three": 8, "four": 4} - self.assertEqual( - build_component_list(d, convert=lambda x: x), ["one", "four", "three"] - ) + assert build_component_list(d, convert=lambda x: x) == ["one", "four", "three"] def test_duplicate_components_in_basesettings(self): # Higher priority takes precedence duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0) duplicate_bs.set("ONE", 4, priority=10) - self.assertEqual( - build_component_list(duplicate_bs, convert=lambda x: x.lower()), - ["two", "one"], - ) + assert build_component_list(duplicate_bs, convert=lambda x: x.lower()) == [ + "two", + "one", + ] duplicate_bs.set("one", duplicate_bs["one"], priority=20) - self.assertEqual( - build_component_list(duplicate_bs, convert=lambda x: x.lower()), - ["one", "two"], - ) + assert build_component_list(duplicate_bs, convert=lambda x: x.lower()) == [ + "one", + "two", + ] # Same priority raises ValueError duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20) with pytest.raises( @@ -42,24 +38,24 @@ class BuildComponentListTest(unittest.TestCase): def test_valid_numbers(self): # work well with None and numeric values d = {"a": 10, "b": None, "c": 15, "d": 5.0} - self.assertEqual(build_component_list(d, convert=lambda x: x), ["d", "a", "c"]) + assert build_component_list(d, convert=lambda x: x) == ["d", "a", "c"] d = { "a": 33333333333333333333, "b": 11111111111111111111, "c": 22222222222222222222, } - self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"]) + assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"] -class UtilsConfTestCase(unittest.TestCase): +class TestUtilsConf: def test_arglist_to_dict(self): - self.assertEqual( - arglist_to_dict(["arg1=val1", "arg2=val2"]), - {"arg1": "val1", "arg2": "val2"}, - ) + assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == { + "arg1": "val1", + "arg2": "val2", + } -class FeedExportConfigTestCase(unittest.TestCase): +class TestFeedExportConfig: def test_feed_export_config_invalid_format(self): settings = Settings() with pytest.raises(UsageError): @@ -72,44 +68,36 @@ class FeedExportConfigTestCase(unittest.TestCase): def test_feed_export_config_explicit_formats(self): settings = Settings() - self.assertEqual( - { - "items_1.dat": {"format": "json"}, - "items_2.dat": {"format": "xml"}, - "items_3.dat": {"format": "csv"}, - }, - feed_process_params_from_cli( - settings, ["items_1.dat:json", "items_2.dat:xml", "items_3.dat:csv"] - ), + assert { + "items_1.dat": {"format": "json"}, + "items_2.dat": {"format": "xml"}, + "items_3.dat": {"format": "csv"}, + } == feed_process_params_from_cli( + settings, ["items_1.dat:json", "items_2.dat:xml", "items_3.dat:csv"] ) def test_feed_export_config_implicit_formats(self): settings = Settings() - self.assertEqual( - { - "items_1.json": {"format": "json"}, - "items_2.xml": {"format": "xml"}, - "items_3.csv": {"format": "csv"}, - }, - feed_process_params_from_cli( - settings, ["items_1.json", "items_2.xml", "items_3.csv"] - ), + assert { + "items_1.json": {"format": "json"}, + "items_2.xml": {"format": "xml"}, + "items_3.csv": {"format": "csv"}, + } == feed_process_params_from_cli( + settings, ["items_1.json", "items_2.xml", "items_3.csv"] ) def test_feed_export_config_stdout(self): settings = Settings() - self.assertEqual( - {"stdout:": {"format": "pickle"}}, - feed_process_params_from_cli(settings, ["-:pickle"]), + assert {"stdout:": {"format": "pickle"}} == feed_process_params_from_cli( + settings, ["-:pickle"] ) def test_feed_export_config_overwrite(self): settings = Settings() - self.assertEqual( - {"output.json": {"format": "json", "overwrite": True}}, - feed_process_params_from_cli( - settings, [], overwrite_output=["output.json"] - ), + assert { + "output.json": {"format": "json", "overwrite": True} + } == feed_process_params_from_cli( + settings, [], overwrite_output=["output.json"] ) def test_output_and_overwrite_output(self): @@ -131,18 +119,15 @@ class FeedExportConfigTestCase(unittest.TestCase): } ) new_feed = feed_complete_default_values_from_settings(feed, settings) - self.assertEqual( - new_feed, - { - "encoding": "custom encoding", - "fields": ["f1", "f2", "f3"], - "indent": 42, - "store_empty": True, - "uri_params": (1, 2, 3, 4), - "batch_item_count": 2, - "item_export_kwargs": {}, - }, - ) + assert new_feed == { + "encoding": "custom encoding", + "fields": ["f1", "f2", "f3"], + "indent": 42, + "store_empty": True, + "uri_params": (1, 2, 3, 4), + "batch_item_count": 2, + "item_export_kwargs": {}, + } def test_feed_complete_default_values_from_settings_non_empty(self): feed = { @@ -159,15 +144,12 @@ class FeedExportConfigTestCase(unittest.TestCase): } ) new_feed = feed_complete_default_values_from_settings(feed, settings) - self.assertEqual( - new_feed, - { - "encoding": "other encoding", - "fields": None, - "indent": 42, - "store_empty": True, - "uri_params": None, - "batch_item_count": 2, - "item_export_kwargs": {}, - }, - ) + assert new_feed == { + "encoding": "other encoding", + "fields": None, + "indent": 42, + "store_empty": True, + "uri_params": None, + "batch_item_count": 2, + "item_export_kwargs": {}, + } diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index 0bc86e1b9..6598bdce7 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -1,4 +1,4 @@ -import unittest +import pytest from scrapy.utils.console import get_shell_embed_func @@ -18,23 +18,23 @@ except ImportError: ipy = False -class UtilsConsoleTestCase(unittest.TestCase): +class TestUtilsConsole: def test_get_shell_embed_func(self): shell = get_shell_embed_func(["invalid"]) - self.assertEqual(shell, None) + assert shell is None shell = get_shell_embed_func(["invalid", "python"]) - self.assertTrue(callable(shell)) - self.assertEqual(shell.__name__, "_embed_standard_shell") + assert callable(shell) + assert shell.__name__ == "_embed_standard_shell" - @unittest.skipIf(not bpy, "bpython not available in testenv") + @pytest.mark.skipif(not bpy, reason="bpython not available in testenv") def test_get_shell_embed_func2(self): shell = get_shell_embed_func(["bpython"]) - self.assertTrue(callable(shell)) - self.assertEqual(shell.__name__, "_embed_bpython_shell") + assert callable(shell) + assert shell.__name__ == "_embed_bpython_shell" - @unittest.skipIf(not ipy, "IPython not available in testenv") + @pytest.mark.skipif(not ipy, reason="IPython not available in testenv") def test_get_shell_embed_func3(self): # default shell should be 'ipython' shell = get_shell_embed_func() - self.assertEqual(shell.__name__, "_embed_ipython_shell") + assert shell.__name__ == "_embed_ipython_shell" diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index a5b438645..e8dd88049 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,4 +1,3 @@ -import unittest import warnings import pytest @@ -8,16 +7,16 @@ from scrapy import Request from scrapy.utils.curl import curl_to_request_kwargs -class CurlToRequestKwargsTest(unittest.TestCase): +class TestCurlToRequestKwargs: maxDiff = 5000 def _test_command(self, curl_command, expected_result): result = curl_to_request_kwargs(curl_command) - self.assertEqual(result, expected_result) + assert result == expected_result try: Request(**result) except TypeError as e: - self.fail(f"Request kwargs are not correct {e}") + pytest.fail(f"Request kwargs are not correct {e}") def test_get(self): curl_command = "curl http://example.org/" @@ -203,7 +202,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): def test_get_silent(self): curl_command = 'curl --silent "www.example.com"' expected_result = {"method": "GET", "url": "http://www.example.com"} - self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) + assert curl_to_request_kwargs(curl_command) == expected_result def test_too_few_arguments_error(self): with pytest.raises( @@ -218,7 +217,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): warnings.simplefilter("ignore") curl_command = "curl --bar --baz http://www.example.com" expected_result = {"method": "GET", "url": "http://www.example.com"} - self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) + assert curl_to_request_kwargs(curl_command) == expected_result # case 2: ignore_unknown_options=False (raise exception): with pytest.raises(ValueError, match="Unrecognized options:.*--bar.*--baz"): diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 2e35d339a..75b6b0e99 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,5 +1,4 @@ import copy -import unittest import warnings from collections.abc import Iterator, Mapping, MutableMapping @@ -17,18 +16,18 @@ from scrapy.utils.datatypes import ( from scrapy.utils.python import garbage_collect -class CaseInsensitiveDictMixin: +class CaseInsensitiveDictBase: def test_init_dict(self): seq = {"red": 1, "black": 3} d = self.dict_class(seq) - self.assertEqual(d["red"], 1) - self.assertEqual(d["black"], 3) + assert d["red"] == 1 + assert d["black"] == 3 def test_init_pair_sequence(self): seq = (("red", 1), ("black", 3)) d = self.dict_class(seq) - self.assertEqual(d["red"], 1) - self.assertEqual(d["black"], 3) + assert d["red"] == 1 + assert d["black"] == 3 def test_init_mapping(self): class MyMapping(Mapping): @@ -46,8 +45,8 @@ class CaseInsensitiveDictMixin: seq = MyMapping(red=1, black=3) d = self.dict_class(seq) - self.assertEqual(d["red"], 1) - self.assertEqual(d["black"], 3) + assert d["red"] == 1 + assert d["black"] == 3 def test_init_mutable_mapping(self): class MyMutableMapping(MutableMapping): @@ -71,18 +70,18 @@ class CaseInsensitiveDictMixin: seq = MyMutableMapping(red=1, black=3) d = self.dict_class(seq) - self.assertEqual(d["red"], 1) - self.assertEqual(d["black"], 3) + assert d["red"] == 1 + assert d["black"] == 3 def test_caseless(self): d = self.dict_class() d["key_Lower"] = 1 - self.assertEqual(d["KEy_loWer"], 1) - self.assertEqual(d.get("KEy_loWer"), 1) + assert d["KEy_loWer"] == 1 + assert d.get("KEy_loWer") == 1 d["KEY_LOWER"] = 3 - self.assertEqual(d["key_Lower"], 3) - self.assertEqual(d.get("key_Lower"), 3) + assert d["key_Lower"] == 3 + assert d.get("key_Lower") == 3 def test_delete(self): d = self.dict_class({"key_lower": 1}) @@ -95,41 +94,41 @@ class CaseInsensitiveDictMixin: @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_getdefault(self): d = CaselessDict() - self.assertEqual(d.get("c", 5), 5) + assert d.get("c", 5) == 5 d["c"] = 10 - self.assertEqual(d.get("c", 5), 10) + assert d.get("c", 5) == 10 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_setdefault(self): d = CaselessDict({"a": 1, "b": 2}) r = d.setdefault("A", 5) - self.assertEqual(r, 1) - self.assertEqual(d["A"], 1) + assert r == 1 + assert d["A"] == 1 r = d.setdefault("c", 5) - self.assertEqual(r, 5) - self.assertEqual(d["C"], 5) + assert r == 5 + assert d["C"] == 5 def test_fromkeys(self): keys = ("a", "b") d = self.dict_class.fromkeys(keys) - self.assertEqual(d["A"], None) - self.assertEqual(d["B"], None) + assert d["A"] is None + assert d["B"] is None d = self.dict_class.fromkeys(keys, 1) - self.assertEqual(d["A"], 1) - self.assertEqual(d["B"], 1) + assert d["A"] == 1 + assert d["B"] == 1 instance = self.dict_class() d = instance.fromkeys(keys) - self.assertEqual(d["A"], None) - self.assertEqual(d["B"], None) + assert d["A"] is None + assert d["B"] is None d = instance.fromkeys(keys, 1) - self.assertEqual(d["A"], 1) - self.assertEqual(d["B"], 1) + assert d["A"] == 1 + assert d["B"] == 1 def test_contains(self): d = self.dict_class() @@ -139,7 +138,7 @@ class CaseInsensitiveDictMixin: def test_pop(self): d = self.dict_class() d["a"] = 1 - self.assertEqual(d.pop("A"), 1) + assert d.pop("A") == 1 with pytest.raises(KeyError): d.pop("A") @@ -152,7 +151,7 @@ class CaseInsensitiveDictMixin: d = MyDict() d["key-one"] = 2 - self.assertEqual(list(d.keys()), ["Key-One"]) + assert list(d.keys()) == ["Key-One"] def test_normvalue(self): class MyDict(self.dict_class): @@ -164,62 +163,60 @@ class CaseInsensitiveDictMixin: normvalue = _normvalue # deprecated CaselessDict class d = MyDict({"key": 1}) - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 d = MyDict() d["key"] = 1 - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 d = MyDict() d.setdefault("key", 1) - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 d = MyDict() d.update({"key": 1}) - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 d = MyDict.fromkeys(("key",), 1) - self.assertEqual(d["key"], 2) - self.assertEqual(d.get("key"), 2) + assert d["key"] == 2 + assert d.get("key") == 2 def test_copy(self): h1 = self.dict_class({"header1": "value"}) h2 = copy.copy(h1) assert isinstance(h2, self.dict_class) - self.assertEqual(h1, h2) - self.assertEqual(h1.get("header1"), h2.get("header1")) - self.assertEqual(h1.get("header1"), h2.get("HEADER1")) + assert h1 == h2 + assert h1.get("header1") == h2.get("header1") + assert h1.get("header1") == h2.get("HEADER1") h3 = h1.copy() assert isinstance(h3, self.dict_class) - self.assertEqual(h1, h3) - self.assertEqual(h1.get("header1"), h3.get("header1")) - self.assertEqual(h1.get("header1"), h3.get("HEADER1")) + assert h1 == h3 + assert h1.get("header1") == h3.get("header1") + assert h1.get("header1") == h3.get("HEADER1") -class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): +class TestCaseInsensitiveDict(CaseInsensitiveDictBase): dict_class = CaseInsensitiveDict def test_repr(self): d1 = self.dict_class({"foo": "bar"}) - self.assertEqual(repr(d1), "<CaseInsensitiveDict: {'foo': 'bar'}>") + assert repr(d1) == "<CaseInsensitiveDict: {'foo': 'bar'}>" d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) - self.assertEqual( - repr(d2), "<CaseInsensitiveDict: {'AsDf': 'QwErTy', 'FoO': 'bAr'}>" - ) + assert repr(d2) == "<CaseInsensitiveDict: {'AsDf': 'QwErTy', 'FoO': 'bAr'}>" def test_iter(self): d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) iterkeys = iter(d) - self.assertIsInstance(iterkeys, Iterator) - self.assertEqual(list(iterkeys), ["AsDf", "FoO"]) + assert isinstance(iterkeys, Iterator) + assert list(iterkeys) == ["AsDf", "FoO"] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): +class TestCaselessDict(CaseInsensitiveDictBase): dict_class = CaselessDict def test_deprecation_message(self): @@ -227,93 +224,93 @@ class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): warnings.filterwarnings("always", category=ScrapyDeprecationWarning) self.dict_class({"foo": "bar"}) - self.assertEqual(len(caught), 1) - self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) - self.assertEqual( - "scrapy.utils.datatypes.CaselessDict is deprecated," - " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", - str(caught[0].message), + assert len(caught) == 1 + assert issubclass(caught[0].category, ScrapyDeprecationWarning) + assert ( + str(caught[0].message) + == "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead" ) -class SequenceExcludeTest(unittest.TestCase): +class TestSequenceExclude: def test_list(self): seq = [1, 2, 3] d = SequenceExclude(seq) - self.assertIn(0, d) - self.assertIn(4, d) - self.assertNotIn(2, d) + assert 0 in d + assert 4 in d + assert 2 not in d def test_range(self): seq = range(10, 20) d = SequenceExclude(seq) - self.assertIn(5, d) - self.assertIn(20, d) - self.assertNotIn(15, d) + assert 5 in d + assert 20 in d + assert 15 not in d def test_range_step(self): seq = range(10, 20, 3) d = SequenceExclude(seq) are_not_in = [v for v in range(10, 20, 3) if v in d] - self.assertEqual([], are_not_in) + assert are_not_in == [] are_not_in = [v for v in range(10, 20) if v in d] - self.assertEqual([11, 12, 14, 15, 17, 18], are_not_in) + assert are_not_in == [11, 12, 14, 15, 17, 18] def test_string_seq(self): seq = "cde" d = SequenceExclude(seq) chars = "".join(v for v in "abcdefg" if v in d) - self.assertEqual("abfg", chars) + assert chars == "abfg" def test_stringset_seq(self): seq = set("cde") d = SequenceExclude(seq) chars = "".join(v for v in "abcdefg" if v in d) - self.assertEqual("abfg", chars) + assert chars == "abfg" def test_set(self): """Anything that is not in the supplied sequence will evaluate as 'in' the container.""" seq = {-3, "test", 1.1} d = SequenceExclude(seq) - self.assertIn(0, d) - self.assertIn("foo", d) - self.assertIn(3.14, d) - self.assertIn(set("bar"), d) + assert 0 in d + assert "foo" in d + assert 3.14 in d + assert set("bar") in d # supplied sequence is a set, so checking for list (non)inclusion fails with pytest.raises(TypeError): ["a", "b", "c"] in d # noqa: B015 for v in [-3, "test", 1.1]: - self.assertNotIn(v, d) + assert v not in d -class LocalCacheTest(unittest.TestCase): +class TestLocalCache: def test_cache_with_limit(self): cache = LocalCache(limit=2) cache["a"] = 1 cache["b"] = 2 cache["c"] = 3 - self.assertEqual(len(cache), 2) - self.assertNotIn("a", cache) - self.assertIn("b", cache) - self.assertIn("c", cache) - self.assertEqual(cache["b"], 2) - self.assertEqual(cache["c"], 3) + assert len(cache) == 2 + assert "a" not in cache + assert "b" in cache + assert "c" in cache + assert cache["b"] == 2 + assert cache["c"] == 3 def test_cache_without_limit(self): maximum = 10**4 cache = LocalCache() for x in range(maximum): cache[str(x)] = x - self.assertEqual(len(cache), maximum) + assert len(cache) == maximum for x in range(maximum): - self.assertIn(str(x), cache) - self.assertEqual(cache[str(x)], x) + assert str(x) in cache + assert cache[str(x)] == x -class LocalWeakReferencedCacheTest(unittest.TestCase): +class TestLocalWeakReferencedCache: def test_cache_with_limit(self): cache = LocalWeakReferencedCache(limit=2) r1 = Request("https://example.org") @@ -322,19 +319,19 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): cache[r1] = 1 cache[r2] = 2 cache[r3] = 3 - self.assertEqual(len(cache), 2) - self.assertNotIn(r1, cache) - self.assertIn(r2, cache) - self.assertIn(r3, cache) - self.assertEqual(cache[r1], None) - self.assertEqual(cache[r2], 2) - self.assertEqual(cache[r3], 3) + assert len(cache) == 2 + assert r1 not in cache + assert r2 in cache + assert r3 in cache + assert cache[r1] is None + assert cache[r2] == 2 + assert cache[r3] == 3 del r2 # PyPy takes longer to collect dead references garbage_collect() - self.assertEqual(len(cache), 1) + assert len(cache) == 1 def test_cache_non_weak_referenceable_objects(self): cache = LocalWeakReferencedCache() @@ -344,10 +341,10 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): cache[k1] = 1 cache[k2] = 2 cache[k3] = 3 - self.assertNotIn(k1, cache) - self.assertNotIn(k2, cache) - self.assertNotIn(k3, cache) - self.assertEqual(len(cache), 0) + assert k1 not in cache + assert k2 not in cache + assert k3 not in cache + assert len(cache) == 0 def test_cache_without_limit(self): max = 10**4 @@ -356,10 +353,10 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): for x in range(max): refs.append(Request(f"https://example.org/{x}")) cache[refs[-1]] = x - self.assertEqual(len(cache), max) + assert len(cache) == max for i, r in enumerate(refs): - self.assertIn(r, cache) - self.assertEqual(cache[r], i) + assert r in cache + assert cache[r] == i del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable # delete half of the objects, make sure that is reflected in the cache @@ -369,7 +366,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): # PyPy takes longer to collect dead references garbage_collect() - self.assertEqual(len(cache), max // 2) + assert len(cache) == max // 2 for i, r in enumerate(refs): - self.assertIn(r, cache) - self.assertEqual(cache[r], i) + assert r in cache + assert cache[r] == i diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 3a1030fcf..36bd8ced9 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -18,7 +18,7 @@ from scrapy.utils.defer import ( ) -class MustbeDeferredTest(unittest.TestCase): +class TestMustbeDeferred(unittest.TestCase): def test_success_function(self): steps = [] @@ -66,23 +66,19 @@ def eb1(failure, arg1, arg2): return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})" -class DeferUtilsTest(unittest.TestCase): +class TestDeferUtils(unittest.TestCase): @defer.inlineCallbacks def test_process_chain(self): x = yield process_chain([cb1, cb2, cb3], "res", "v1", "v2") - self.assertEqual(x, "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)") + assert x == "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)" - gotexc = False - try: + with pytest.raises(TypeError): yield process_chain([cb1, cb_fail, cb3], "res", "v1", "v2") - except TypeError: - gotexc = True - self.assertTrue(gotexc) @defer.inlineCallbacks def test_process_parallel(self): x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") - self.assertEqual(x, ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"]) + assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"] def test_process_parallel_failure(self): d = process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2") @@ -90,15 +86,15 @@ class DeferUtilsTest(unittest.TestCase): return d -class IterErrbackTest(unittest.TestCase): +class TestIterErrback: def test_iter_errback_good(self): def itergood(): yield from range(10) errors = [] out = list(iter_errback(itergood(), errors.append)) - self.assertEqual(out, list(range(10))) - self.assertFalse(errors) + assert out == list(range(10)) + assert not errors def test_iter_errback_bad(self): def iterbad(): @@ -109,12 +105,12 @@ class IterErrbackTest(unittest.TestCase): errors = [] out = list(iter_errback(iterbad(), errors.append)) - self.assertEqual(out, [0, 1, 2, 3, 4]) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0].value, ZeroDivisionError) + assert out == [0, 1, 2, 3, 4] + assert len(errors) == 1 + assert isinstance(errors[0].value, ZeroDivisionError) -class AiterErrbackTest(unittest.TestCase): +class TestAiterErrback(unittest.TestCase): @deferred_f_from_coro_f async def test_aiter_errback_good(self): async def itergood(): @@ -123,8 +119,8 @@ class AiterErrbackTest(unittest.TestCase): errors = [] out = await collect_asyncgen(aiter_errback(itergood(), errors.append)) - self.assertEqual(out, list(range(10))) - self.assertFalse(errors) + assert out == list(range(10)) + assert not errors @deferred_f_from_coro_f async def test_iter_errback_bad(self): @@ -136,12 +132,12 @@ class AiterErrbackTest(unittest.TestCase): errors = [] out = await collect_asyncgen(aiter_errback(iterbad(), errors.append)) - self.assertEqual(out, [0, 1, 2, 3, 4]) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0].value, ZeroDivisionError) + assert out == [0, 1, 2, 3, 4] + assert len(errors) == 1 + assert isinstance(errors[0].value, ZeroDivisionError) -class AsyncDefTestsuiteTest(unittest.TestCase): +class TestAsyncDefTestsuite(unittest.TestCase): @deferred_f_from_coro_f async def test_deferred_f_from_coro_f(self): pass @@ -156,7 +152,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase): raise RuntimeError("This is expected to be raised") -class AsyncCooperatorTest(unittest.TestCase): +class TestAsyncCooperator(unittest.TestCase): """This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage. parallel_async is called with the results of a callback (so an iterable of items, requests and None, @@ -207,7 +203,7 @@ class AsyncCooperatorTest(unittest.TestCase): ait = self.get_async_iterable(length) dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) yield dl - self.assertEqual(list(range(length)), sorted(results)) + assert list(range(length)) == sorted(results) @defer.inlineCallbacks def test_delays(self): @@ -216,4 +212,4 @@ class AsyncCooperatorTest(unittest.TestCase): ait = self.get_async_iterable_with_delays(length) dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) yield dl - self.assertEqual(list(range(length)), sorted(results)) + assert list(range(length)) == sorted(results) diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index e917b6947..52c165bb4 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,5 +1,4 @@ import inspect -import unittest import warnings from unittest import mock @@ -21,7 +20,7 @@ class NewName(SomeBaseClass): pass -class WarnWhenSubclassedTest(unittest.TestCase): +class TestWarnWhenSubclassed: def _mywarnings(self, w, category=MyWarning): return [x for x in w if x.category is MyWarning] @@ -30,7 +29,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): create_deprecated_class("Deprecated", NewName) w = self._mywarnings(w) - self.assertEqual(w, []) + assert w == [] def test_subclassing_warning_message(self): Deprecated = create_deprecated_class( @@ -43,15 +42,14 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass w = self._mywarnings(w) - self.assertEqual(len(w), 1) - self.assertEqual( - str(w[0].message), - "tests.test_utils_deprecate.UserClass inherits from " + assert len(w) == 1 + assert ( + str(w[0].message) == "tests.test_utils_deprecate.UserClass inherits from " "deprecated class tests.test_utils_deprecate.Deprecated, " "please inherit from tests.test_utils_deprecate.NewName." - " (warning only on first subclass, there may be others)", + " (warning only on first subclass, there may be others)" ) - self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1]) + assert w[0].lineno == inspect.getsourcelines(UserClass)[1] def test_custom_class_paths(self): Deprecated = create_deprecated_class( @@ -70,11 +68,11 @@ class WarnWhenSubclassedTest(unittest.TestCase): _ = Deprecated() w = self._mywarnings(w) - self.assertEqual(len(w), 2) - self.assertIn("foo.NewClass", str(w[0].message)) - self.assertIn("bar.OldClass", str(w[0].message)) - self.assertIn("foo.NewClass", str(w[1].message)) - self.assertIn("bar.OldClass", str(w[1].message)) + assert len(w) == 2 + assert "foo.NewClass" in str(w[0].message) + assert "bar.OldClass" in str(w[0].message) + assert "foo.NewClass" in str(w[1].message) + assert "bar.OldClass" in str(w[1].message) def test_subclassing_warns_only_on_direct_children(self): Deprecated = create_deprecated_class( @@ -90,8 +88,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass w = self._mywarnings(w) - self.assertEqual(len(w), 1) - self.assertIn("UserClass", str(w[0].message)) + assert len(w) == 1 + assert "UserClass" in str(w[0].message) def test_subclassing_warns_once_by_default(self): Deprecated = create_deprecated_class( @@ -110,8 +108,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass w = self._mywarnings(w) - self.assertEqual(len(w), 1) - self.assertIn("UserClass", str(w[0].message)) + assert len(w) == 1 + assert "UserClass" in str(w[0].message) def test_warning_on_instance(self): Deprecated = create_deprecated_class( @@ -130,13 +128,12 @@ class WarnWhenSubclassedTest(unittest.TestCase): _ = UserClass() # subclass instances don't warn w = self._mywarnings(w) - self.assertEqual(len(w), 1) - self.assertEqual( - str(w[0].message), - "tests.test_utils_deprecate.Deprecated is deprecated, " - "instantiate tests.test_utils_deprecate.NewName instead.", + assert len(w) == 1 + assert ( + str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, " + "instantiate tests.test_utils_deprecate.NewName instead." ) - self.assertEqual(w[0].lineno, lineno) + assert w[0].lineno == lineno def test_warning_auto_message(self): with warnings.catch_warnings(record=True) as w: @@ -146,8 +143,8 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass msg = str(w[0].message) - self.assertIn("tests.test_utils_deprecate.NewName", msg) - self.assertIn("tests.test_utils_deprecate.Deprecated", msg) + assert "tests.test_utils_deprecate.NewName" in msg + assert "tests.test_utils_deprecate.Deprecated" in msg def test_issubclass(self): with warnings.catch_warnings(): @@ -225,7 +222,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): warnings.simplefilter("ignore", ScrapyDeprecationWarning) Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"}) - self.assertEqual(Deprecated.foo, "bar") + assert Deprecated.foo == "bar" def test_deprecate_a_class_with_custom_metaclass(self): Meta1 = type("Meta1", (type,), {}) @@ -246,7 +243,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): ) w = self._mywarnings(w) - self.assertEqual(len(w), 0, str(map(str, w))) + assert len(w) == 0, str(map(str, w)) with warnings.catch_warnings(record=True) as w: AlsoDeprecated() @@ -255,11 +252,11 @@ class WarnWhenSubclassedTest(unittest.TestCase): pass w = self._mywarnings(w) - self.assertEqual(len(w), 2) - self.assertIn("AlsoDeprecated", str(w[0].message)) - self.assertIn("foo.Bar", str(w[0].message)) - self.assertIn("AlsoDeprecated", str(w[1].message)) - self.assertIn("foo.Bar", str(w[1].message)) + assert len(w) == 2 + assert "AlsoDeprecated" in str(w[0].message) + assert "foo.Bar" in str(w[0].message) + assert "AlsoDeprecated" in str(w[1].message) + assert "foo.Bar" in str(w[1].message) def test_inspect_stack(self): with ( @@ -271,7 +268,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): class SubClass(DeprecatedName): pass - self.assertIn("Error detecting parent module", str(w[0].message)) + assert "Error detecting parent module" in str(w[0].message) @mock.patch( @@ -281,27 +278,27 @@ class WarnWhenSubclassedTest(unittest.TestCase): ("scrapy.contrib.", "scrapy.extensions."), ], ) -class UpdateClassPathTest(unittest.TestCase): +class TestUpdateClassPath: def test_old_path_gets_fixed(self): with warnings.catch_warnings(record=True) as w: output = update_classpath("scrapy.contrib.debug.Debug") - self.assertEqual(output, "scrapy.extensions.debug.Debug") - self.assertEqual(len(w), 1) - self.assertIn("scrapy.contrib.debug.Debug", str(w[0].message)) - self.assertIn("scrapy.extensions.debug.Debug", str(w[0].message)) + assert output == "scrapy.extensions.debug.Debug" + assert len(w) == 1 + assert "scrapy.contrib.debug.Debug" in str(w[0].message) + assert "scrapy.extensions.debug.Debug" in str(w[0].message) def test_sorted_replacement(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) output = update_classpath("scrapy.contrib.pipeline.Pipeline") - self.assertEqual(output, "scrapy.pipelines.Pipeline") + assert output == "scrapy.pipelines.Pipeline" def test_unmatched_path_stays_the_same(self): with warnings.catch_warnings(record=True) as w: output = update_classpath("scrapy.unmatched.Path") - self.assertEqual(output, "scrapy.unmatched.Path") - self.assertEqual(len(w), 0) + assert output == "scrapy.unmatched.Path" + assert len(w) == 0 def test_returns_nonstring(self): for notastring in [None, True, [1, 2, 3], object()]: - self.assertEqual(update_classpath(notastring), notastring) + assert update_classpath(notastring) == notastring diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index d1bf64828..cea564653 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -1,10 +1,10 @@ from io import StringIO -from unittest import TestCase, mock +from unittest import mock from scrapy.utils.display import pformat, pprint -class TestDisplay(TestCase): +class TestDisplay: object = {"a": 1} colorized_strings = { ( @@ -26,15 +26,15 @@ class TestDisplay(TestCase): @mock.patch("sys.stdout.isatty") def test_pformat(self, isatty): isatty.return_value = True - self.assertIn(pformat(self.object), self.colorized_strings) + assert pformat(self.object) in self.colorized_strings @mock.patch("sys.stdout.isatty") def test_pformat_dont_colorize(self, isatty): isatty.return_value = True - self.assertEqual(pformat(self.object, colorize=False), self.plain_string) + assert pformat(self.object, colorize=False) == self.plain_string def test_pformat_not_tty(self): - self.assertEqual(pformat(self.object), self.plain_string) + assert pformat(self.object) == self.plain_string @mock.patch("sys.platform", "win32") @mock.patch("platform.version") @@ -42,7 +42,7 @@ class TestDisplay(TestCase): def test_pformat_old_windows(self, isatty, version): isatty.return_value = True version.return_value = "10.0.14392" - self.assertIn(pformat(self.object), self.colorized_strings) + assert pformat(self.object) in self.colorized_strings @mock.patch("sys.platform", "win32") @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") @@ -54,7 +54,7 @@ class TestDisplay(TestCase): isatty.return_value = True version.return_value = "10.0.14393" terminal_processing.return_value = False - self.assertEqual(pformat(self.object), self.plain_string) + assert pformat(self.object) == self.plain_string @mock.patch("sys.platform", "win32") @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") @@ -64,7 +64,7 @@ class TestDisplay(TestCase): isatty.return_value = True version.return_value = "10.0.14393" terminal_processing.return_value = True - self.assertIn(pformat(self.object), self.colorized_strings) + assert pformat(self.object) in self.colorized_strings @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") @@ -81,10 +81,10 @@ class TestDisplay(TestCase): return real_import(name, globals, locals, fromlist, level) builtins.__import__ = mock_import - self.assertEqual(pformat(self.object), self.plain_string) + assert pformat(self.object) == self.plain_string builtins.__import__ = real_import def test_pprint(self): with mock.patch("sys.stdout", new=StringIO()) as mock_out: pprint(self.object) - self.assertEqual(mock_out.getvalue(), "{'a': 1}\n") + assert mock_out.getvalue() == "{'a': 1}\n" diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index d40cae9c7..c43ed152b 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,4 +1,3 @@ -import unittest from gzip import BadGzipFile from pathlib import Path @@ -12,17 +11,17 @@ from tests import tests_datadir SAMPLEDIR = Path(tests_datadir, "compressed") -class GunzipTest(unittest.TestCase): +class TestGunzip: def test_gunzip_basic(self): r1 = Response( "http://www.example.com", body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(), ) - self.assertTrue(gzip_magic_number(r1)) + assert gzip_magic_number(r1) r2 = Response("http://www.example.com", body=gunzip(r1.body)) - self.assertFalse(gzip_magic_number(r2)) - self.assertEqual(len(r2.body), 9950) + assert not gzip_magic_number(r2) + assert len(r2.body) == 9950 def test_gunzip_truncated(self): text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes()) @@ -37,15 +36,15 @@ class GunzipTest(unittest.TestCase): "http://www.example.com", body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(), ) - self.assertTrue(gzip_magic_number(r1)) + assert gzip_magic_number(r1) r2 = Response("http://www.example.com", body=gunzip(r1.body)) assert r2.body.endswith(b"</html>") - self.assertFalse(gzip_magic_number(r2)) + assert not gzip_magic_number(r2) def test_is_gzipped_empty(self): r1 = Response("http://www.example.com") - self.assertFalse(gzip_magic_number(r1)) + assert not gzip_magic_number(r1) def test_gunzip_illegal_eof(self): text = html_to_unicode( @@ -54,5 +53,5 @@ class GunzipTest(unittest.TestCase): expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text( encoding="utf-8" ) - self.assertEqual(len(text), len(expected_text)) - self.assertEqual(text, expected_text) + assert len(text) == len(expected_text) + assert text == expected_text diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 741e69559..0c05ef7d6 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -1,11 +1,10 @@ -import unittest from urllib.parse import urlparse from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached -class HttpobjUtilsTest(unittest.TestCase): +class TestHttpobjUtils: def test_urlparse_cached(self): url = "http://www.example.com/index.html" request1 = Request(url) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 9ad30617a..fa0d37866 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,5 +1,4 @@ import pytest -from twisted.trial import unittest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse, XmlResponse @@ -7,7 +6,7 @@ from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata -class XmliterBaseTestCase: +class XmliterBase: @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter(self): body = b""" @@ -35,9 +34,10 @@ class XmliterBaseTestCase: for x in self.xmliter(response, "product") ] - self.assertEqual( - attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])] - ) + assert attrs == [ + ("001", ["Name 1"], ["Type 1"]), + ("002", ["Name 2"], ["Type 2"]), + ] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unusual_node(self): @@ -51,7 +51,7 @@ class XmliterBaseTestCase: nodenames = [ e.xpath("name()").getall() for e in self.xmliter(response, "matchme...") ] - self.assertEqual(nodenames, [["matchme..."]]) + assert nodenames == [["matchme..."]] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unicode(self): @@ -107,10 +107,11 @@ class XmliterBaseTestCase: for x in self.xmliter(r, "þingflokkur") ] - self.assertEqual( - attrs, - [("26", ["-"], ["80"]), ("21", ["Ab"], ["76"]), ("27", ["A"], ["27"])], - ) + assert attrs == [ + ("26", ["-"], ["80"]), + ("21", ["Ab"], ["76"]), + ("27", ["A"], ["27"]), + ] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_text(self): @@ -119,10 +120,10 @@ class XmliterBaseTestCase: "<products><product>one</product><product>two</product></products>" ) - self.assertEqual( - [x.xpath("text()").getall() for x in self.xmliter(body, "product")], - [["one"], ["two"]], - ) + assert [x.xpath("text()").getall() for x in self.xmliter(body, "product")] == [ + ["one"], + ["two"], + ] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaces(self): @@ -148,21 +149,19 @@ class XmliterBaseTestCase: my_iter = self.xmliter(response, "item") node = next(my_iter) node.register_namespace("g", "http://base.google.com/ns/1.0") - self.assertEqual(node.xpath("title/text()").getall(), ["Item 1"]) - self.assertEqual(node.xpath("description/text()").getall(), ["This is item 1"]) - self.assertEqual( - node.xpath("link/text()").getall(), - ["http://www.mydummycompany.com/items/1"], - ) - self.assertEqual( - node.xpath("g:image_link/text()").getall(), - ["http://www.mydummycompany.com/images/item1.jpg"], - ) - self.assertEqual(node.xpath("g:id/text()").getall(), ["ITEM_1"]) - self.assertEqual(node.xpath("g:price/text()").getall(), ["400"]) - self.assertEqual(node.xpath("image_link/text()").getall(), []) - self.assertEqual(node.xpath("id/text()").getall(), []) - self.assertEqual(node.xpath("price/text()").getall(), []) + assert node.xpath("title/text()").getall() == ["Item 1"] + assert node.xpath("description/text()").getall() == ["This is item 1"] + assert node.xpath("link/text()").getall() == [ + "http://www.mydummycompany.com/items/1" + ] + assert node.xpath("g:image_link/text()").getall() == [ + "http://www.mydummycompany.com/images/item1.jpg" + ] + assert node.xpath("g:id/text()").getall() == ["ITEM_1"] + assert node.xpath("g:price/text()").getall() == ["400"] + assert node.xpath("image_link/text()").getall() == [] + assert node.xpath("id/text()").getall() == [] + assert node.xpath("price/text()").getall() == [] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename(self): @@ -188,10 +187,9 @@ class XmliterBaseTestCase: my_iter = self.xmliter(response, "g:image_link") node = next(my_iter) node.register_namespace("g", "http://base.google.com/ns/1.0") - self.assertEqual( - node.xpath("text()").extract(), - ["http://www.mydummycompany.com/images/item1.jpg"], - ) + assert node.xpath("text()").extract() == [ + "http://www.mydummycompany.com/images/item1.jpg" + ] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename_missing(self): @@ -246,13 +244,13 @@ class XmliterBaseTestCase: b"</xml>\n\n" ) response = XmlResponse("http://www.example.com", body=body) - self.assertEqual( - next(self.xmliter(response, "item")).get(), - "<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>", + assert ( + next(self.xmliter(response, "item")).get() + == "<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>" ) -class XmliterTestCase(XmliterBaseTestCase, unittest.TestCase): +class TestXmliter(XmliterBase): xmliter = staticmethod(xmliter) def test_deprecation(self): @@ -269,7 +267,7 @@ class XmliterTestCase(XmliterBaseTestCase, unittest.TestCase): next(self.xmliter(body, "product")) -class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): +class TestLxmlXmliter(XmliterBase): xmliter = staticmethod(xmliter_lxml) def test_xmliter_iterate_namespace(self): @@ -293,21 +291,19 @@ class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): response = XmlResponse(url="http://mydummycompany.com", body=body) no_namespace_iter = self.xmliter(response, "image_link") - self.assertEqual(len(list(no_namespace_iter)), 0) + assert len(list(no_namespace_iter)) == 0 namespace_iter = self.xmliter( response, "image_link", "http://base.google.com/ns/1.0" ) node = next(namespace_iter) - self.assertEqual( - node.xpath("text()").getall(), - ["http://www.mydummycompany.com/images/item1.jpg"], - ) + assert node.xpath("text()").getall() == [ + "http://www.mydummycompany.com/images/item1.jpg" + ] node = next(namespace_iter) - self.assertEqual( - node.xpath("text()").getall(), - ["http://www.mydummycompany.com/images/item2.jpg"], - ) + assert node.xpath("text()").getall() == [ + "http://www.mydummycompany.com/images/item2.jpg" + ] def test_xmliter_namespaces_prefix(self): body = b""" @@ -332,16 +328,16 @@ class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): my_iter = self.xmliter(response, "table", "http://www.w3.org/TR/html4/", "h") node = next(my_iter) - self.assertEqual(len(node.xpath("h:tr/h:td").getall()), 2) - self.assertEqual(node.xpath("h:tr/h:td[1]/text()").getall(), ["Apples"]) - self.assertEqual(node.xpath("h:tr/h:td[2]/text()").getall(), ["Bananas"]) + assert len(node.xpath("h:tr/h:td").getall()) == 2 + assert node.xpath("h:tr/h:td[1]/text()").getall() == ["Apples"] + assert node.xpath("h:tr/h:td[2]/text()").getall() == ["Bananas"] my_iter = self.xmliter( response, "table", "http://www.w3schools.com/furniture", "f" ) node = next(my_iter) - self.assertEqual(node.xpath("f:name/text()").getall(), ["African Coffee Table"]) + assert node.xpath("f:name/text()").getall() == ["African Coffee Table"] def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") @@ -349,42 +345,36 @@ class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): next(i) -class UtilsCsvTestCase(unittest.TestCase): +class TestUtilsCsv: def test_csviter_defaults(self): body = get_testdata("feeds", "feed-sample3.csv") response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) result = list(csv) - self.assertEqual( - result, - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert result == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all(isinstance(k, str) for k in result_row)) - self.assertTrue(all(isinstance(v, str) for v in result_row.values())) + assert all(isinstance(k, str) for k in result_row) + assert all(isinstance(v, str) for v in result_row.values()) def test_csviter_delimiter(self): body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t") response = TextResponse(url="http://example.com/", body=body) csv = csviter(response, delimiter="\t") - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_quotechar(self): body1 = get_testdata("feeds", "feed-sample6.csv") @@ -393,62 +383,50 @@ class UtilsCsvTestCase(unittest.TestCase): response1 = TextResponse(url="http://example.com/", body=body1) csv1 = csviter(response1, quotechar="'") - self.assertEqual( - list(csv1), - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv1) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] response2 = TextResponse(url="http://example.com/", body=body2) csv2 = csviter(response2, delimiter="|", quotechar="'") - self.assertEqual( - list(csv2), - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv2) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_wrong_quotechar(self): body = get_testdata("feeds", "feed-sample6.csv") response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - self.assertEqual( - list(csv), - [ - {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, - { - "'id'": "2", - "'name'": "'unicode'", - "'value'": "'\xfan\xedc\xf3d\xe9\u203d'", - }, - {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"}, - {"'id'": "4", "'name'": "'empty'", "'value'": ""}, - ], - ) + assert list(csv) == [ + {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, + { + "'id'": "2", + "'name'": "'unicode'", + "'value'": "'\xfan\xedc\xf3d\xe9\u203d'", + }, + {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"}, + {"'id'": "4", "'name'": "'empty'", "'value'": ""}, + ] def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t") response = Response(url="http://example.com/", body=body) csv = csviter(response, delimiter="\t") - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_headers(self): sample = get_testdata("feeds", "feed-sample3.csv").splitlines() @@ -457,15 +435,12 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body) csv = csviter(response, headers=[h.decode("utf-8") for h in headers]) - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_falserow(self): body = get_testdata("feeds", "feed-sample3.csv") @@ -474,15 +449,12 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "alpha", "value": "foobar"}, - {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, - {"id": "3", "name": "multi", "value": "foo\nbar"}, - {"id": "4", "name": "empty", "value": ""}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "alpha", "value": "foobar"}, + {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, + {"id": "3", "name": "multi", "value": "foo\nbar"}, + {"id": "4", "name": "empty", "value": ""}, + ] def test_csviter_exception(self): body = get_testdata("feeds", "feed-sample3.csv") @@ -504,30 +476,24 @@ class UtilsCsvTestCase(unittest.TestCase): url="http://example.com/", body=body1, encoding="latin1" ) csv = csviter(response) - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "latin1", "value": "test"}, - {"id": "2", "name": "something", "value": "\xf1\xe1\xe9\xf3"}, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "latin1", "value": "test"}, + {"id": "2", "name": "something", "value": "\xf1\xe1\xe9\xf3"}, + ] response = TextResponse(url="http://example.com/", body=body2, encoding="cp852") csv = csviter(response) - self.assertEqual( - list(csv), - [ - {"id": "1", "name": "cp852", "value": "test"}, - { - "id": "2", - "name": "something", - "value": "\u255a\u2569\u2569\u2569\u2550\u2550\u2557", - }, - ], - ) + assert list(csv) == [ + {"id": "1", "name": "cp852", "value": "test"}, + { + "id": "2", + "name": "something", + "value": "\u255a\u2569\u2569\u2569\u2550\u2550\u2557", + }, + ] -class TestHelper(unittest.TestCase): +class TestHelper: bbody = b"utf8-body" ubody = bbody.decode("utf8") txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8") @@ -541,11 +507,9 @@ class TestHelper(unittest.TestCase): self._assert_type_and_value(r2, self.ubody, obj) r3 = _body_or_str(obj, unicode=False) self._assert_type_and_value(r3, self.bbody, obj) - self.assertTrue(type(r1) is type(r2)) - self.assertTrue(type(r1) is not type(r3)) + assert type(r1) is type(r2) + assert type(r1) is not type(r3) def _assert_type_and_value(self, a, b, obj): - self.assertTrue( - type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}" - ) - self.assertEqual(a, b) + assert type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}" + assert a == b diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 06e88bd10..af50fed7a 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -7,7 +7,6 @@ import sys import unittest from io import StringIO from typing import TYPE_CHECKING, Any -from unittest import TestCase import pytest from testfixtures import LogCapture @@ -27,7 +26,7 @@ if TYPE_CHECKING: from collections.abc import Mapping, MutableMapping -class FailureToExcInfoTest(unittest.TestCase): +class TestFailureToExcInfo: def test_failure(self): try: 0 / 0 @@ -35,14 +34,14 @@ class FailureToExcInfoTest(unittest.TestCase): exc_info = sys.exc_info() failure = Failure() - self.assertTupleEqual(exc_info, failure_to_exc_info(failure)) + assert exc_info == failure_to_exc_info(failure) def test_non_failure(self): - self.assertIsNone(failure_to_exc_info("test")) + assert failure_to_exc_info("test") is None -class TopLevelFormatterTest(unittest.TestCase): - def setUp(self): +class TestTopLevelFormatter: + def setup_method(self): self.handler = LogCapture() self.handler.addFilter(TopLevelFormatter(["test"])) @@ -71,8 +70,8 @@ class TopLevelFormatterTest(unittest.TestCase): log.check(("different", "WARNING", "test log msg")) -class LogCounterHandlerTest(unittest.TestCase): - def setUp(self): +class TestLogCounterHandler: + def setup_method(self): settings = {"LOG_LEVEL": "WARNING"} self.logger = logging.getLogger("test") self.logger.setLevel(logging.NOTSET) @@ -81,24 +80,24 @@ class LogCounterHandlerTest(unittest.TestCase): self.handler = LogCounterHandler(self.crawler) self.logger.addHandler(self.handler) - def tearDown(self): + def teardown_method(self): self.logger.propagate = True self.logger.removeHandler(self.handler) def test_init(self): - self.assertIsNone(self.crawler.stats.get_value("log_count/DEBUG")) - self.assertIsNone(self.crawler.stats.get_value("log_count/INFO")) - self.assertIsNone(self.crawler.stats.get_value("log_count/WARNING")) - self.assertIsNone(self.crawler.stats.get_value("log_count/ERROR")) - self.assertIsNone(self.crawler.stats.get_value("log_count/CRITICAL")) + assert self.crawler.stats.get_value("log_count/DEBUG") is None + assert self.crawler.stats.get_value("log_count/INFO") is None + assert self.crawler.stats.get_value("log_count/WARNING") is None + assert self.crawler.stats.get_value("log_count/ERROR") is None + assert self.crawler.stats.get_value("log_count/CRITICAL") is None def test_accepted_level(self): self.logger.error("test log msg") - self.assertEqual(self.crawler.stats.get_value("log_count/ERROR"), 1) + assert self.crawler.stats.get_value("log_count/ERROR") == 1 def test_filtered_out_level(self): self.logger.debug("test log msg") - self.assertIsNone(self.crawler.stats.get_value("log_count/INFO")) + assert self.crawler.stats.get_value("log_count/INFO") is None class StreamLoggerTest(unittest.TestCase): @@ -152,8 +151,8 @@ def test_spider_logger_adapter_process( assert result_kwargs == expected_extra -class LoggingTestCase(TestCase): - def setUp(self): +class TestLogging: + def setup_method(self): self.log_stream = StringIO() handler = logging.StreamHandler(self.log_stream) logger = logging.getLogger("log_spider") @@ -163,7 +162,7 @@ class LoggingTestCase(TestCase): self.logger = logger self.spider = LogSpider() - def tearDown(self): + def teardown_method(self): self.logger.removeHandler(self.handler) def test_debug_logging(self): @@ -202,8 +201,8 @@ class LoggingTestCase(TestCase): assert log_contents == f"{log_message}\n" -class LoggingWithExtraTestCase(TestCase): - def setUp(self): +class TestLoggingWithExtra: + def setup_method(self): self.log_stream = StringIO() handler = logging.StreamHandler(self.log_stream) formatter = logging.Formatter( @@ -218,7 +217,7 @@ class LoggingWithExtraTestCase(TestCase): self.spider = LogSpider() self.regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$") - def tearDown(self): + def teardown_method(self): self.logger.removeHandler(self.handler) def test_debug_logging(self): diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index a67e16962..b330819d9 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -1,6 +1,5 @@ import os import sys -import unittest from pathlib import Path from unittest import mock @@ -18,18 +17,18 @@ from scrapy.utils.misc import ( ) -class UtilsMiscTestCase(unittest.TestCase): +class TestUtilsMisc: def test_load_object_class(self): obj = load_object(Field) - self.assertIs(obj, Field) + assert obj is Field obj = load_object("scrapy.item.Field") - self.assertIs(obj, Field) + assert obj is Field def test_load_object_function(self): obj = load_object(load_object) - self.assertIs(obj, load_object) + assert obj is load_object obj = load_object("scrapy.utils.misc.load_object") - self.assertIs(obj, load_object) + assert obj is load_object def test_load_object_exceptions(self): with pytest.raises(ImportError): @@ -47,20 +46,20 @@ class UtilsMiscTestCase(unittest.TestCase): "tests.test_utils_misc.test_walk_modules.mod.mod0", "tests.test_utils_misc.test_walk_modules.mod1", ] - self.assertEqual({m.__name__ for m in mods}, set(expected)) + assert {m.__name__ for m in mods} == set(expected) mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod") expected = [ "tests.test_utils_misc.test_walk_modules.mod", "tests.test_utils_misc.test_walk_modules.mod.mod0", ] - self.assertEqual({m.__name__ for m in mods}, set(expected)) + assert {m.__name__ for m in mods} == set(expected) mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod1") expected = [ "tests.test_utils_misc.test_walk_modules.mod1", ] - self.assertEqual({m.__name__ for m in mods}, set(expected)) + assert {m.__name__ for m in mods} == set(expected) with pytest.raises(ImportError): walk_modules("nomodule999") @@ -76,7 +75,7 @@ class UtilsMiscTestCase(unittest.TestCase): "testegg.spiders.b", "testegg", ] - self.assertEqual({m.__name__ for m in mods}, set(expected)) + assert {m.__name__ for m in mods} == set(expected) finally: sys.path.remove(egg) @@ -90,15 +89,13 @@ class UtilsMiscTestCase(unittest.TestCase): assert hasattr(arg_to_iter([1, 2, 3]), "__iter__") assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__") - self.assertEqual(list(arg_to_iter(None)), []) - self.assertEqual(list(arg_to_iter("lala")), ["lala"]) - self.assertEqual(list(arg_to_iter(100)), [100]) - self.assertEqual(list(arg_to_iter(c for c in "abc")), ["a", "b", "c"]) - self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3]) - self.assertEqual(list(arg_to_iter({"a": 1})), [{"a": 1}]) - self.assertEqual( - list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")] - ) + assert not list(arg_to_iter(None)) + assert list(arg_to_iter("lala")) == ["lala"] + assert list(arg_to_iter(100)) == [100] + assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"] + assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3] + assert list(arg_to_iter({"a": 1})) == [{"a": 1}] + assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")] @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_create_instance(self): @@ -110,10 +107,10 @@ class UtilsMiscTestCase(unittest.TestCase): def _test_with_settings(mock, settings): create_instance(mock, settings, None, *args, **kwargs) if hasattr(mock, "from_crawler"): - self.assertEqual(mock.from_crawler.call_count, 0) + assert mock.from_crawler.call_count == 0 if hasattr(mock, "from_settings"): mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - self.assertEqual(mock.call_count, 0) + assert mock.call_count == 0 else: mock.assert_called_once_with(*args, **kwargs) @@ -122,11 +119,11 @@ class UtilsMiscTestCase(unittest.TestCase): if hasattr(mock, "from_crawler"): mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) if hasattr(mock, "from_settings"): - self.assertEqual(mock.from_settings.call_count, 0) - self.assertEqual(mock.call_count, 0) + assert mock.from_settings.call_count == 0 + assert mock.call_count == 0 elif hasattr(mock, "from_settings"): mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - self.assertEqual(mock.call_count, 0) + assert mock.call_count == 0 else: mock.assert_called_once_with(*args, **kwargs) @@ -172,11 +169,11 @@ class UtilsMiscTestCase(unittest.TestCase): if hasattr(mock, "from_crawler"): mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs) if hasattr(mock, "from_settings"): - self.assertEqual(mock.from_settings.call_count, 0) - self.assertEqual(mock.call_count, 0) + assert mock.from_settings.call_count == 0 + assert mock.call_count == 0 elif hasattr(mock, "from_settings"): mock.from_settings.assert_called_once_with(settings, *args, **kwargs) - self.assertEqual(mock.call_count, 0) + assert mock.call_count == 0 else: mock.assert_called_once_with(*args, **kwargs) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 480729d11..81a83c3d7 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,4 +1,3 @@ -import unittest import warnings from functools import partial from unittest import mock @@ -40,7 +39,7 @@ def generator_that_returns_stuff(): return 3 -class UtilsMiscPy3TestCase(unittest.TestCase): +class TestUtilsMisc: def test_generators_return_something(self): def f1(): yield 1 @@ -77,27 +76,27 @@ https://example.org with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_something) - self.assertEqual(len(w), 1) - self.assertIn( - 'The "NoneType.top_level_return_something" method is a generator', - str(w[0].message), + assert len(w) == 1 + assert ( + 'The "NoneType.top_level_return_something" method is a generator' + in str(w[0].message) ) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, f1) - self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.f1" method is a generator', str(w[0].message)) + assert len(w) == 1 + assert 'The "NoneType.f1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, g1) - self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.g1" method is a generator', str(w[0].message)) + assert len(w) == 1 + assert 'The "NoneType.g1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, h1) - self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.h1" method is a generator', str(w[0].message)) + assert len(w) == 1 + assert 'The "NoneType.h1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, i1) - self.assertEqual(len(w), 1) - self.assertIn('The "NoneType.i1" method is a generator', str(w[0].message)) + assert len(w) == 1 + assert 'The "NoneType.i1" method is a generator' in str(w[0].message) def test_generators_return_none(self): def f2(): @@ -144,28 +143,28 @@ https://example.org with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_none) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, f2) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, g2) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, h2) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, i2) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, j2) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, k2) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, l2) - self.assertEqual(len(w), 0) + assert len(w) == 0 def test_generators_return_none_with_decorator(self): def decorator(func): @@ -225,28 +224,28 @@ https://example.org with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_none) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, f3) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, g3) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, h3) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, i3) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, j3) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, k3) - self.assertEqual(len(w), 0) + assert len(w) == 0 with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, l3) - self.assertEqual(len(w), 0) + assert len(w) == 0 @mock.patch( "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error @@ -254,8 +253,8 @@ https://example.org def test_indentation_error(self): with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_none) - self.assertEqual(len(w), 1) - self.assertIn("Unable to determine", str(w[0].message)) + assert len(w) == 1 + assert "Unable to determine" in str(w[0].message) def test_partial(self): def cb(arg1, arg2): diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 1d149d48d..aa250be69 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -2,7 +2,6 @@ import contextlib import os import shutil import tempfile -import unittest import warnings from pathlib import Path @@ -25,21 +24,21 @@ def inside_a_project(): shutil.rmtree(project_dir) -class ProjectUtilsTest(unittest.TestCase): +class TestProjectUtils: def test_data_path_outside_project(self): - self.assertEqual(str(Path(".scrapy", "somepath")), data_path("somepath")) + assert str(Path(".scrapy", "somepath")) == data_path("somepath") abspath = str(Path(os.path.sep, "absolute", "path")) - self.assertEqual(abspath, data_path(abspath)) + assert abspath == data_path(abspath) def test_data_path_inside_project(self): with inside_a_project() as proj_path: expected = Path(proj_path, ".scrapy", "somepath") - self.assertEqual(expected.resolve(), Path(data_path("somepath")).resolve()) + assert expected.resolve() == Path(data_path("somepath")).resolve() abspath = str(Path(os.path.sep, "absolute", "path").resolve()) - self.assertEqual(abspath, data_path(abspath)) + assert abspath == data_path(abspath) -class GetProjectSettingsTestCase(unittest.TestCase): +class TestGetProjectSettings: def test_valid_envvar(self): value = "tests.test_cmdline.settings" envvars = { diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3b0739276..291646ad7 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -21,18 +21,18 @@ from scrapy.utils.python import ( ) -class MutableChainTest(unittest.TestCase): +class TestMutableChain: def test_mutablechain(self): m = MutableChain(range(2), [2, 3], (4, 5)) m.extend(range(6, 7)) m.extend([7, 8]) m.extend([9, 10], (11, 12)) - self.assertEqual(next(m), 0) - self.assertEqual(m.__next__(), 1) - self.assertEqual(list(m), list(range(2, 13))) + assert next(m) == 0 + assert m.__next__() == 1 + assert list(m) == list(range(2, 13)) -class MutableAsyncChainTest(unittest.TestCase): +class TestMutableAsyncChain(unittest.TestCase): @staticmethod async def g1(): for i in range(3): @@ -62,9 +62,9 @@ class MutableAsyncChainTest(unittest.TestCase): m.extend(self.g2()) m.extend(self.g3()) - self.assertEqual(await m.__anext__(), 0) + assert await m.__anext__() == 0 results = await collect_asyncgen(m) - self.assertEqual(results, list(range(1, 10))) + assert results == list(range(1, 10)) @deferred_f_from_coro_f async def test_mutableasyncchain_exc(self): @@ -73,46 +73,46 @@ class MutableAsyncChainTest(unittest.TestCase): m.extend(self.g3()) results = await collect_asyncgen(aiter_errback(m, lambda _: None)) - self.assertEqual(results, list(range(5))) + assert results == list(range(5)) -class ToUnicodeTest(unittest.TestCase): +class TestToUnicode: def test_converting_an_utf8_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b"lel\xc3\xb1e"), "lel\xf1e") + assert to_unicode(b"lel\xc3\xb1e") == "lel\xf1e" def test_converting_a_latin_1_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b"lel\xf1e", "latin-1"), "lel\xf1e") + assert to_unicode(b"lel\xf1e", "latin-1") == "lel\xf1e" def test_converting_a_unicode_to_unicode_should_return_the_same_object(self): - self.assertEqual(to_unicode("\xf1e\xf1e\xf1e"), "\xf1e\xf1e\xf1e") + assert to_unicode("\xf1e\xf1e\xf1e") == "\xf1e\xf1e\xf1e" - def test_converting_a_strange_object_should_raise_TypeError(self): + def test_converting_a_strange_object_should_raise_type_error(self): with pytest.raises(TypeError): to_unicode(423) def test_errors_argument(self): - self.assertEqual(to_unicode(b"a\xedb", "utf-8", errors="replace"), "a\ufffdb") + assert to_unicode(b"a\xedb", "utf-8", errors="replace") == "a\ufffdb" -class ToBytesTest(unittest.TestCase): +class TestToBytes: def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self): - self.assertEqual(to_bytes("\xa3 49"), b"\xc2\xa3 49") + assert to_bytes("\xa3 49") == b"\xc2\xa3 49" def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self): - self.assertEqual(to_bytes("\xa3 49", "latin-1"), b"\xa3 49") + assert to_bytes("\xa3 49", "latin-1") == b"\xa3 49" def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self): - self.assertEqual(to_bytes(b"lel\xf1e"), b"lel\xf1e") + assert to_bytes(b"lel\xf1e") == b"lel\xf1e" - def test_converting_a_strange_object_should_raise_TypeError(self): + def test_converting_a_strange_object_should_raise_type_error(self): with pytest.raises(TypeError): to_bytes(pytest) def test_errors_argument(self): - self.assertEqual(to_bytes("a\ufffdb", "latin-1", errors="replace"), b"a?b") + assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b" -class MemoizedMethodTest(unittest.TestCase): +class TestMemoizedMethod: def test_memoizemethod_noargs(self): class A: @memoizemethod_noargs @@ -130,7 +130,7 @@ class MemoizedMethodTest(unittest.TestCase): assert one is not three -class BinaryIsTextTest(unittest.TestCase): +class TestBinaryIsText: def test_binaryistext(self): assert binary_is_text(b"hello") @@ -144,7 +144,7 @@ class BinaryIsTextTest(unittest.TestCase): assert not binary_is_text(b"\x02\xa3") -class UtilsPythonTestCase(unittest.TestCase): +class TestUtilsPython: @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_equal_attributes(self): class Obj: @@ -153,31 +153,31 @@ class UtilsPythonTestCase(unittest.TestCase): a = Obj() b = Obj() # no attributes given return False - self.assertFalse(equal_attributes(a, b, [])) + assert not equal_attributes(a, b, []) # nonexistent attributes - self.assertFalse(equal_attributes(a, b, ["x", "y"])) + assert not equal_attributes(a, b, ["x", "y"]) a.x = 1 b.x = 1 # equal attribute - self.assertTrue(equal_attributes(a, b, ["x"])) + assert equal_attributes(a, b, ["x"]) b.y = 2 # obj1 has no attribute y - self.assertFalse(equal_attributes(a, b, ["x", "y"])) + assert not equal_attributes(a, b, ["x", "y"]) a.y = 2 # equal attributes - self.assertTrue(equal_attributes(a, b, ["x", "y"])) + assert equal_attributes(a, b, ["x", "y"]) a.y = 1 # different attributes - self.assertFalse(equal_attributes(a, b, ["x", "y"])) + assert not equal_attributes(a, b, ["x", "y"]) # test callable a.meta = {} b.meta = {} - self.assertTrue(equal_attributes(a, b, ["meta"])) + assert equal_attributes(a, b, ["meta"]) # compare ['meta']['a'] a.meta["z"] = 1 @@ -189,10 +189,10 @@ class UtilsPythonTestCase(unittest.TestCase): def compare_z(obj): return get_z(get_meta(obj)) - self.assertTrue(equal_attributes(a, b, [compare_z, "x"])) + assert equal_attributes(a, b, [compare_z, "x"]) # fail z equality a.meta["z"] = 2 - self.assertFalse(equal_attributes(a, b, [compare_z, "x"])) + assert not equal_attributes(a, b, [compare_z, "x"]) def test_get_func_args(self): def f1(a, b, c): @@ -221,36 +221,35 @@ class UtilsPythonTestCase(unittest.TestCase): partial_f2 = functools.partial(f1, b=None) partial_f3 = functools.partial(partial_f2, None) - self.assertEqual(get_func_args(f1), ["a", "b", "c"]) - self.assertEqual(get_func_args(f2), ["a", "b", "c"]) - self.assertEqual(get_func_args(f3), ["a", "b", "c"]) - self.assertEqual(get_func_args(A), ["a", "b", "c"]) - self.assertEqual(get_func_args(a.method), ["a", "b", "c"]) - self.assertEqual(get_func_args(partial_f1), ["b", "c"]) - self.assertEqual(get_func_args(partial_f2), ["a", "c"]) - self.assertEqual(get_func_args(partial_f3), ["c"]) - self.assertEqual(get_func_args(cal), ["a", "b", "c"]) - self.assertEqual(get_func_args(object), []) - self.assertEqual(get_func_args(str.split, stripself=True), ["sep", "maxsplit"]) - self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"]) + assert get_func_args(f1) == ["a", "b", "c"] + assert get_func_args(f2) == ["a", "b", "c"] + assert get_func_args(f3) == ["a", "b", "c"] + assert get_func_args(A) == ["a", "b", "c"] + assert get_func_args(a.method) == ["a", "b", "c"] + assert get_func_args(partial_f1) == ["b", "c"] + assert get_func_args(partial_f2) == ["a", "c"] + assert get_func_args(partial_f3) == ["c"] + assert get_func_args(cal) == ["a", "b", "c"] + assert get_func_args(object) == [] + assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"] + assert get_func_args(" ".join, stripself=True) == ["iterable"] if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy": # the correct and correctly extracted signature - self.assertEqual( - get_func_args(operator.itemgetter(2), stripself=True), ["obj"] - ) + assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"] elif platform.python_implementation() == "CPython": # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951) - self.assertIn( - get_func_args(operator.itemgetter(2), stripself=True), - [[], ["args", "kwargs"]], - ) + assert get_func_args(operator.itemgetter(2), stripself=True) in [ + [], + ["args", "kwargs"], + ] def test_without_none_values(self): - self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) - self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4)) - self.assertEqual( - without_none_values({"one": 1, "none": None, "three": 3, "four": 4}), - {"one": 1, "three": 3, "four": 4}, - ) + assert without_none_values([1, None, 3, 4]) == [1, 3, 4] + assert without_none_values((1, None, 3, 4)) == (1, 3, 4) + assert without_none_values({"one": 1, "none": None, "three": 3, "four": 4}) == { + "one": 1, + "three": 3, + "four": 4, + } diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 51bca9a31..5b8509753 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import unittest import warnings from hashlib import sha1 from weakref import WeakKeyDictionary @@ -21,23 +20,23 @@ from scrapy.utils.request import ( from scrapy.utils.test import get_crawler -class UtilsRequestTest(unittest.TestCase): +class TestUtilsRequest: @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_request_authenticate(self): r = Request("http://www.example.com") request_authenticate(r, "someuser", "somepass") - self.assertEqual(r.headers["Authorization"], b"Basic c29tZXVzZXI6c29tZXBhc3M=") + assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M=" def test_request_httprepr(self): r1 = Request("http://www.example.com") - self.assertEqual( - request_httprepr(r1), b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n" + assert ( + request_httprepr(r1) == b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n" ) r1 = Request("http://www.example.com/some/page.html?arg=1") - self.assertEqual( - request_httprepr(r1), - b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + assert ( + request_httprepr(r1) + == b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n" ) r1 = Request( @@ -46,9 +45,9 @@ class UtilsRequestTest(unittest.TestCase): headers={"Content-type": b"text/html"}, body=b"Some body", ) - self.assertEqual( - request_httprepr(r1), - b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body", + assert ( + request_httprepr(r1) + == b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body" ) def test_request_httprepr_for_non_http_request(self): @@ -57,7 +56,7 @@ class UtilsRequestTest(unittest.TestCase): request_httprepr(Request("ftp://localhost/tmp/foo.txt")) -class FingerprintTest(unittest.TestCase): +class TestFingerprint: maxDiff = None function: staticmethod = staticmethod(fingerprint) @@ -147,23 +146,23 @@ class FingerprintTest(unittest.TestCase): def test_query_string_key_order(self): r1 = Request("http://www.example.com/query?id=111&cat=222") r2 = Request("http://www.example.com/query?cat=222&id=111") - self.assertEqual(self.function(r1), self.function(r1)) - self.assertEqual(self.function(r1), self.function(r2)) + assert self.function(r1) == self.function(r1) + assert self.function(r1) == self.function(r2) def test_query_string_key_without_value(self): r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78132,199") r2 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") - self.assertNotEqual(self.function(r1), self.function(r2)) + assert self.function(r1) != self.function(r2) def test_caching(self): r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") - self.assertEqual(self.function(r1), self.cache[r1][self.default_cache_key]) + assert self.function(r1) == self.cache[r1][self.default_cache_key] def test_header(self): r1 = Request("http://www.example.com/members/offers.html") r2 = Request("http://www.example.com/members/offers.html") r2.headers["SESSIONID"] = b"somehash" - self.assertEqual(self.function(r1), self.function(r2)) + assert self.function(r1) == self.function(r2) def test_headers(self): r1 = Request("http://www.example.com/") @@ -173,36 +172,35 @@ class FingerprintTest(unittest.TestCase): r3.headers["Accept-Language"] = b"en" r3.headers["SESSIONID"] = b"somehash" - self.assertEqual(self.function(r1), self.function(r2), self.function(r3)) + assert self.function(r1) == self.function(r2) == self.function(r3) - self.assertEqual( - self.function(r1), self.function(r1, include_headers=["Accept-Language"]) + assert self.function(r1) == self.function( + r1, include_headers=["Accept-Language"] ) - self.assertNotEqual( - self.function(r1), self.function(r2, include_headers=["Accept-Language"]) + assert self.function(r1) != self.function( + r2, include_headers=["Accept-Language"] ) - self.assertEqual( - self.function(r3, include_headers=["accept-language", "sessionid"]), - self.function(r3, include_headers=["SESSIONID", "Accept-Language"]), - ) + assert self.function( + r3, include_headers=["accept-language", "sessionid"] + ) == self.function(r3, include_headers=["SESSIONID", "Accept-Language"]) def test_fragment(self): r1 = Request("http://www.example.com/test.html") r2 = Request("http://www.example.com/test.html#fragment") - self.assertEqual(self.function(r1), self.function(r2)) - self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True)) - self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True)) - self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True)) + assert self.function(r1) == self.function(r2) + assert self.function(r1) == self.function(r1, keep_fragments=True) + assert self.function(r2) != self.function(r2, keep_fragments=True) + assert self.function(r1) != self.function(r2, keep_fragments=True) def test_method_and_body(self): r1 = Request("http://www.example.com") r2 = Request("http://www.example.com", method="POST") r3 = Request("http://www.example.com", method="POST", body=b"request body") - self.assertNotEqual(self.function(r1), self.function(r2)) - self.assertNotEqual(self.function(r2), self.function(r3)) + assert self.function(r1) != self.function(r2) + assert self.function(r2) != self.function(r3) def test_request_replace(self): # cached fingerprint must be cleared on request copy @@ -210,7 +208,7 @@ class FingerprintTest(unittest.TestCase): fp1 = self.function(r1) r2 = r1.replace(url="http://www.example.com/other") fp2 = self.function(r2) - self.assertNotEqual(fp1, fp2) + assert fp1 != fp2 def test_part_separation(self): # An old implementation used to serialize request data in a way that @@ -219,7 +217,7 @@ class FingerprintTest(unittest.TestCase): fp1 = self.function(r1) r2 = Request("http://www.example.com/f", body=b"oo") fp2 = self.function(r2) - self.assertNotEqual(fp1, fp2) + assert fp1 != fp2 def test_hashes(self): """Test hardcoded hashes, to make sure future changes to not introduce @@ -228,7 +226,7 @@ class FingerprintTest(unittest.TestCase): self.function(request, **kwargs) for request, _, kwargs in self.known_hashes ] expected = [_fingerprint for _, _fingerprint, _ in self.known_hashes] - self.assertEqual(actual, expected) + assert actual == expected REQUEST_OBJECTS_TO_TEST = ( @@ -260,13 +258,12 @@ REQUEST_OBJECTS_TO_TEST = ( ) -class RequestFingerprinterTestCase(unittest.TestCase): +class TestRequestFingerprinter: def test_default_implementation(self): crawler = get_crawler() request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - fingerprint(request), + assert crawler.request_fingerprinter.fingerprint(request) == fingerprint( + request ) def test_deprecated_implementation(self): @@ -276,14 +273,13 @@ class RequestFingerprinterTestCase(unittest.TestCase): with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - fingerprint(request), + assert crawler.request_fingerprinter.fingerprint(request) == fingerprint( + request ) - self.assertTrue(logged_warnings) + assert logged_warnings -class CustomRequestFingerprinterTestCase(unittest.TestCase): +class TestCustomRequestFingerprinter: def test_include_headers(self): class RequestFingerprinter: def fingerprint(self, request): @@ -298,7 +294,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com", headers={"X-ID": "2"}) fp2 = crawler.request_fingerprinter.fingerprint(r2) - self.assertNotEqual(fp1, fp2) + assert fp1 != fp2 def test_dont_canonicalize(self): class RequestFingerprinter: @@ -320,7 +316,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp1 = crawler.request_fingerprinter.fingerprint(r1) r2 = Request("http://www.example.com?a=2&a=1") fp2 = crawler.request_fingerprinter.fingerprint(r2) - self.assertNotEqual(fp1, fp2) + assert fp1 != fp2 def test_meta(self): class RequestFingerprinter: @@ -342,10 +338,10 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp3 = crawler.request_fingerprinter.fingerprint(r3) r4 = Request("http://www.example.com", meta={"fingerprint": "b"}) fp4 = crawler.request_fingerprinter.fingerprint(r4) - self.assertNotEqual(fp1, fp2) - self.assertNotEqual(fp1, fp4) - self.assertNotEqual(fp2, fp4) - self.assertEqual(fp2, fp3) + assert fp1 != fp2 + assert fp1 != fp4 + assert fp2 != fp4 + assert fp2 == fp3 def test_from_crawler(self): class RequestFingerprinter: @@ -367,7 +363,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) - self.assertEqual(fingerprint, settings["FINGERPRINT"]) + assert fingerprint == settings["FINGERPRINT"] def test_from_settings(self): class RequestFingerprinter: @@ -391,7 +387,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) - self.assertEqual(fingerprint, settings["FINGERPRINT"]) + assert fingerprint == settings["FINGERPRINT"] def test_from_crawler_and_settings(self): class RequestFingerprinter: @@ -418,13 +414,13 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) - self.assertEqual(fingerprint, settings["FINGERPRINT"]) + assert fingerprint == settings["FINGERPRINT"] -class RequestToCurlTest(unittest.TestCase): +class TestRequestToCurl: def _test_request(self, request_object, expected_curl_command): curl_command = request_to_curl(request_object) - self.assertEqual(curl_command, expected_curl_command) + assert curl_command == expected_curl_command def test_get(self): request_object = Request("https://www.example.com") diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index af7906781..80f2f25d5 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,4 +1,3 @@ -import unittest from pathlib import Path from time import process_time from urllib.parse import urlparse @@ -16,7 +15,7 @@ from scrapy.utils.response import ( ) -class ResponseUtilsTest(unittest.TestCase): +class TestResponseUtils: dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") def test_open_in_browser(self): @@ -28,7 +27,7 @@ class ResponseUtilsTest(unittest.TestCase): if not path or not Path(path).exists(): path = burl.replace("file://", "") bbody = Path(path).read_bytes() - self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody) + assert b'<base href="' + to_bytes(url) + b'">' in bbody return True response = HtmlResponse(url, body=body) @@ -68,9 +67,9 @@ class ResponseUtilsTest(unittest.TestCase): </script> """, ) - self.assertEqual(get_meta_refresh(r1), (5.0, "http://example.org/newpage")) - self.assertEqual(get_meta_refresh(r2), (None, None)) - self.assertEqual(get_meta_refresh(r3), (None, None)) + assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage") + assert get_meta_refresh(r2) == (None, None) + assert get_meta_refresh(r3) == (None, None) def test_get_base_url(self): resp = HtmlResponse( @@ -81,19 +80,19 @@ class ResponseUtilsTest(unittest.TestCase): <body>blahablsdfsal&</body> </html>""", ) - self.assertEqual(get_base_url(resp), "http://www.example.com/img/") + assert get_base_url(resp) == "http://www.example.com/img/" resp2 = HtmlResponse( "http://www.example.com", body=b""" <html><body>blahablsdfsal&</body></html>""", ) - self.assertEqual(get_base_url(resp2), "http://www.example.com") + assert get_base_url(resp2) == "http://www.example.com" def test_response_status_message(self): - self.assertEqual(response_status_message(200), "200 OK") - self.assertEqual(response_status_message(404), "404 Not Found") - self.assertEqual(response_status_message(573), "573 Unknown Status") + assert response_status_message(200) == "200 OK" + assert response_status_message(404) == "404 Not Found" + assert response_status_message(573) == "573 Unknown Status" def test_inject_base_url(self): url = "http://www.example.com" @@ -103,7 +102,7 @@ class ResponseUtilsTest(unittest.TestCase): if not path or not Path(path).exists(): path = burl.replace("file://", "") bbody = Path(path).read_bytes() - self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1) + assert bbody.count(b'<base href="' + to_bytes(url) + b'">') == 1 return True r1 = HtmlResponse( @@ -185,7 +184,7 @@ class ResponseUtilsTest(unittest.TestCase): open_in_browser(response, lambda url: True) end_time = process_time() - self.assertLess(end_time - start_time, MAX_CPU_TIME) + assert end_time - start_time < MAX_CPU_TIME def test_open_in_browser_redos_head(self): MAX_CPU_TIME = 0.02 @@ -202,7 +201,7 @@ class ResponseUtilsTest(unittest.TestCase): open_in_browser(response, lambda url: True) end_time = process_time() - self.assertLess(end_time - start_time, MAX_CPU_TIME) + assert end_time - start_time < MAX_CPU_TIME @pytest.mark.parametrize( diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index 055db4e5b..2ee3850b0 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -1,7 +1,6 @@ import dataclasses import datetime import json -import unittest from decimal import Decimal import attr @@ -11,8 +10,8 @@ from scrapy.http import Request, Response from scrapy.utils.serialize import ScrapyJSONEncoder -class JsonEncoderTestCase(unittest.TestCase): - def setUp(self): +class TestJsonEncoder: + def setup_method(self): self.encoder = ScrapyJSONEncoder(sort_keys=True) def test_encode_decode(self): @@ -39,24 +38,22 @@ class JsonEncoderTestCase(unittest.TestCase): (s, ss), (dt_set, dt_sets), ]: - self.assertEqual( - self.encoder.encode(input), json.dumps(output, sort_keys=True) - ) + assert self.encoder.encode(input) == json.dumps(output, sort_keys=True) def test_encode_deferred(self): - self.assertIn("Deferred", self.encoder.encode(defer.Deferred())) + assert "Deferred" in self.encoder.encode(defer.Deferred()) def test_encode_request(self): r = Request("http://www.example.com/lala") rs = self.encoder.encode(r) - self.assertIn(r.method, rs) - self.assertIn(r.url, rs) + assert r.method in rs + assert r.url in rs def test_encode_response(self): r = Response("http://www.example.com/lala") rs = self.encoder.encode(r) - self.assertIn(r.url, rs) - self.assertIn(str(r.status), rs) + assert r.url in rs + assert str(r.status) in rs def test_encode_dataclass_item(self) -> None: @dataclasses.dataclass @@ -67,9 +64,7 @@ class JsonEncoderTestCase(unittest.TestCase): item = TestDataClass(name="Product", url="http://product.org", price=1) encoded = self.encoder.encode(item) - self.assertEqual( - encoded, '{"name": "Product", "price": 1, "url": "http://product.org"}' - ) + assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}' def test_encode_attrs_item(self): @attr.s @@ -80,6 +75,4 @@ class JsonEncoderTestCase(unittest.TestCase): item = AttrsItem(name="Product", url="http://product.org", price=1) encoded = self.encoder.encode(item) - self.assertEqual( - encoded, '{"name": "Product", "price": 1, "url": "http://product.org"}' - ) + assert encoded == '{"name": "Product", "price": 1, "url": "http://product.org"}' diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 858813e83..751a77031 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -11,7 +11,7 @@ from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.utils.test import get_from_asyncio_queue -class SendCatchLogTest(unittest.TestCase): +class TestSendCatchLog(unittest.TestCase): @defer.inlineCallbacks def test_send_catch_log(self): test_signal = object() @@ -29,13 +29,13 @@ class SendCatchLogTest(unittest.TestCase): assert self.error_handler in handlers_called assert self.ok_handler in handlers_called - self.assertEqual(len(log.records), 1) + assert len(log.records) == 1 record = log.records[0] - self.assertIn("error_handler", record.getMessage()) - self.assertEqual(record.levelname, "ERROR") - self.assertEqual(result[0][0], self.error_handler) - self.assertIsInstance(result[0][1], Failure) - self.assertEqual(result[1], (self.ok_handler, "OK")) + assert "error_handler" in record.getMessage() + assert record.levelname == "ERROR" + assert result[0][0] == self.error_handler # pylint: disable=comparison-with-callable + assert isinstance(result[0][1], Failure) + assert result[1] == (self.ok_handler, "OK") dispatcher.disconnect(self.error_handler, signal=test_signal) dispatcher.disconnect(self.ok_handler, signal=test_signal) @@ -53,7 +53,7 @@ class SendCatchLogTest(unittest.TestCase): return "OK" -class SendCatchLogDeferredTest(SendCatchLogTest): +class SendCatchLogDeferredTest(TestSendCatchLog): def _get_result(self, signal, *a, **kw): return send_catch_log_deferred(signal, *a, **kw) @@ -85,7 +85,7 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): return await get_from_asyncio_queue("OK") -class SendCatchLogTest2(unittest.TestCase): +class TestSendCatchLog2: def test_error_logged_if_deferred_not_supported(self): def test_handler(): return defer.Deferred() @@ -94,6 +94,6 @@ class SendCatchLogTest2(unittest.TestCase): dispatcher.connect(test_handler, test_signal) with LogCapture() as log: send_catch_log(test_signal) - self.assertEqual(len(log.records), 1) - self.assertIn("Cannot return deferreds from signal handler", str(log)) + assert len(log.records) == 1 + assert "Cannot return deferreds from signal handler" in str(log) dispatcher.disconnect(test_handler, test_signal) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 69a459d8b..36d612009 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -1,9 +1,7 @@ -import unittest - from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots -class SitemapTest(unittest.TestCase): +class TestSitemap: def test_sitemap(self): s = Sitemap( b"""<?xml version="1.0" encoding="UTF-8"?> @@ -23,23 +21,20 @@ class SitemapTest(unittest.TestCase): </urlset>""" ) assert s.type == "urlset" - self.assertEqual( - list(s), - [ - { - "priority": "1", - "loc": "http://www.example.com/", - "lastmod": "2009-08-16", - "changefreq": "daily", - }, - { - "priority": "0.8", - "loc": "http://www.example.com/Special-Offers.html", - "lastmod": "2009-08-16", - "changefreq": "weekly", - }, - ], - ) + assert list(s) == [ + { + "priority": "1", + "loc": "http://www.example.com/", + "lastmod": "2009-08-16", + "changefreq": "daily", + }, + { + "priority": "0.8", + "loc": "http://www.example.com/Special-Offers.html", + "lastmod": "2009-08-16", + "changefreq": "weekly", + }, + ] def test_sitemap_index(self): s = Sitemap( @@ -56,19 +51,16 @@ class SitemapTest(unittest.TestCase): </sitemapindex>""" ) assert s.type == "sitemapindex" - self.assertEqual( - list(s), - [ - { - "loc": "http://www.example.com/sitemap1.xml.gz", - "lastmod": "2004-10-01T18:23:17+00:00", - }, - { - "loc": "http://www.example.com/sitemap2.xml.gz", - "lastmod": "2005-01-01", - }, - ], - ) + assert list(s) == [ + { + "loc": "http://www.example.com/sitemap1.xml.gz", + "lastmod": "2004-10-01T18:23:17+00:00", + }, + { + "loc": "http://www.example.com/sitemap2.xml.gz", + "lastmod": "2005-01-01", + }, + ] def test_sitemap_strip(self): """Assert we can deal with trailing spaces inside <loc> tags - we've @@ -90,18 +82,15 @@ class SitemapTest(unittest.TestCase): </urlset> """ ) - self.assertEqual( - list(s), - [ - { - "priority": "1", - "loc": "http://www.example.com/", - "lastmod": "2009-08-16", - "changefreq": "daily", - }, - {"loc": "http://www.example.com/2", "lastmod": ""}, - ], - ) + assert list(s) == [ + { + "priority": "1", + "loc": "http://www.example.com/", + "lastmod": "2009-08-16", + "changefreq": "daily", + }, + {"loc": "http://www.example.com/2", "lastmod": ""}, + ] def test_sitemap_wrong_ns(self): """We have seen sitemaps with wrongs ns. Presumably, Google still works @@ -122,18 +111,15 @@ class SitemapTest(unittest.TestCase): </urlset> """ ) - self.assertEqual( - list(s), - [ - { - "priority": "1", - "loc": "http://www.example.com/", - "lastmod": "2009-08-16", - "changefreq": "daily", - }, - {"loc": "http://www.example.com/2", "lastmod": ""}, - ], - ) + assert list(s) == [ + { + "priority": "1", + "loc": "http://www.example.com/", + "lastmod": "2009-08-16", + "changefreq": "daily", + }, + {"loc": "http://www.example.com/2", "lastmod": ""}, + ] def test_sitemap_wrong_ns2(self): """We have seen sitemaps with wrongs ns. Presumably, Google still works @@ -155,18 +141,15 @@ class SitemapTest(unittest.TestCase): """ ) assert s.type == "urlset" - self.assertEqual( - list(s), - [ - { - "priority": "1", - "loc": "http://www.example.com/", - "lastmod": "2009-08-16", - "changefreq": "daily", - }, - {"loc": "http://www.example.com/2", "lastmod": ""}, - ], - ) + assert list(s) == [ + { + "priority": "1", + "loc": "http://www.example.com/", + "lastmod": "2009-08-16", + "changefreq": "daily", + }, + {"loc": "http://www.example.com/2", "lastmod": ""}, + ] def test_sitemap_urls_from_robots(self): robots = """User-agent: * @@ -187,15 +170,14 @@ Sitemap: /sitemap-relative-url.xml Disallow: /forum/search/ Disallow: /forum/active/ """ - self.assertEqual( - list(sitemap_urls_from_robots(robots, base_url="http://example.com")), - [ - "http://example.com/sitemap.xml", - "http://example.com/sitemap-product-index.xml", - "http://example.com/sitemap-uppercase.xml", - "http://example.com/sitemap-relative-url.xml", - ], - ) + assert list( + sitemap_urls_from_robots(robots, base_url="http://example.com") + ) == [ + "http://example.com/sitemap.xml", + "http://example.com/sitemap-product-index.xml", + "http://example.com/sitemap-uppercase.xml", + "http://example.com/sitemap-relative-url.xml", + ] def test_sitemap_blanklines(self): """Assert we can deal with starting blank lines before <xml> tag""" @@ -224,14 +206,11 @@ Disallow: /forum/active/ </sitemapindex> """ ) - self.assertEqual( - list(s), - [ - {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"}, - {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"}, - {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"}, - ], - ) + assert list(s) == [ + {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"}, + {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"}, + {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"}, + ] def test_comment(self): s = Sitemap( @@ -245,7 +224,7 @@ Disallow: /forum/active/ </urlset>""" ) - self.assertEqual(list(s), [{"loc": "http://www.example.com/"}]) + assert list(s) == [{"loc": "http://www.example.com/"}] def test_alternate(self): s = Sitemap( @@ -265,19 +244,16 @@ Disallow: /forum/active/ </urlset>""" ) - self.assertEqual( - list(s), - [ - { - "loc": "http://www.example.com/english/", - "alternate": [ - "http://www.example.com/deutsch/", - "http://www.example.com/schweiz-deutsch/", - "http://www.example.com/english/", - ], - } - ], - ) + assert list(s) == [ + { + "loc": "http://www.example.com/english/", + "alternate": [ + "http://www.example.com/deutsch/", + "http://www.example.com/schweiz-deutsch/", + "http://www.example.com/english/", + ], + } + ] def test_xml_entity_expansion(self): s = Sitemap( @@ -294,4 +270,4 @@ Disallow: /forum/active/ """ ) - self.assertEqual(list(s), [{"loc": "http://127.0.0.1:8000/"}]) + assert list(s) == [{"loc": "http://127.0.0.1:8000/"}] diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index df8f37103..43e603f6c 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -1,5 +1,3 @@ -import unittest - from scrapy import Spider from scrapy.http import Request from scrapy.item import Item @@ -14,19 +12,19 @@ class MySpider2(Spider): name = "myspider2" -class UtilsSpidersTestCase(unittest.TestCase): +class TestUtilsSpiders: def test_iterate_spider_output(self): i = Item() r = Request("http://scrapytest.org") o = object() - self.assertEqual(list(iterate_spider_output(i)), [i]) - self.assertEqual(list(iterate_spider_output(r)), [r]) - self.assertEqual(list(iterate_spider_output(o)), [o]) - self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o]) + assert list(iterate_spider_output(i)) == [i] + assert list(iterate_spider_output(r)) == [r] + assert list(iterate_spider_output(o)) == [o] + assert list(iterate_spider_output([r, i, o])) == [r, i, o] def test_iter_spider_classes(self): import tests.test_utils_spider # noqa: PLW0406 # pylint: disable=import-self it = iter_spider_classes(tests.test_utils_spider) - self.assertEqual(set(it), {MySpider1, MySpider2}) + assert set(it) == {MySpider1, MySpider2} diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index fc6c33200..0b845fdb0 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,4 +1,3 @@ -import unittest from pathlib import Path from shutil import rmtree from tempfile import mkdtemp @@ -6,11 +5,11 @@ from tempfile import mkdtemp from scrapy.utils.template import render_templatefile -class UtilsRenderTemplateFileTestCase(unittest.TestCase): - def setUp(self): +class TestUtilsRenderTemplateFile: + def setup_method(self): self.tmp_path = mkdtemp() - def tearDown(self): + def teardown_method(self): rmtree(self.tmp_path) def test_simple_render(self): @@ -26,8 +25,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): render_templatefile(template_path, **context) - self.assertFalse(template_path.exists()) - self.assertEqual(render_path.read_text(encoding="utf8"), rendered) + assert not template_path.exists() + assert render_path.read_text(encoding="utf8") == rendered render_path.unlink() assert not render_path.exists() # Failure of test itself diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 58efad585..a945163ef 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,4 +1,3 @@ -import unittest from io import StringIO from time import sleep, time from unittest import mock @@ -16,48 +15,48 @@ class Bar(trackref.object_ref): pass -class TrackrefTestCase(unittest.TestCase): - def setUp(self): +class TestTrackref: + def setup_method(self): trackref.live_refs.clear() def test_format_live_refs(self): o1 = Foo() # noqa: F841 o2 = Bar() # noqa: F841 o3 = Foo() # noqa: F841 - self.assertEqual( - trackref.format_live_refs(), - """\ + assert ( + trackref.format_live_refs() + == """\ Live References Bar 1 oldest: 0s ago Foo 2 oldest: 0s ago -""", +""" ) - self.assertEqual( - trackref.format_live_refs(ignore=Foo), - """\ + assert ( + trackref.format_live_refs(ignore=Foo) + == """\ Live References Bar 1 oldest: 0s ago -""", +""" ) @mock.patch("sys.stdout", new_callable=StringIO) def test_print_live_refs_empty(self, stdout): trackref.print_live_refs() - self.assertEqual(stdout.getvalue(), "Live References\n\n\n") + assert stdout.getvalue() == "Live References\n\n\n" @mock.patch("sys.stdout", new_callable=StringIO) def test_print_live_refs_with_objects(self, stdout): o1 = Foo() # noqa: F841 trackref.print_live_refs() - self.assertEqual( - stdout.getvalue(), - """\ + assert ( + stdout.getvalue() + == """\ Live References -Foo 1 oldest: 0s ago\n\n""", +Foo 1 oldest: 0s ago\n\n""" ) def test_get_oldest(self): @@ -75,15 +74,12 @@ Foo 1 oldest: 0s ago\n\n""", raise SkipTest("time.time is not precise enough") o3 = Foo() # noqa: F841 - self.assertIs(trackref.get_oldest("Foo"), o1) - self.assertIs(trackref.get_oldest("Bar"), o2) - self.assertIsNone(trackref.get_oldest("XXX")) + assert trackref.get_oldest("Foo") is o1 + assert trackref.get_oldest("Bar") is o2 + assert trackref.get_oldest("XXX") is None def test_iter_all(self): o1 = Foo() o2 = Bar() # noqa: F841 o3 = Foo() - self.assertEqual( - set(trackref.iter_all("Foo")), - {o1, o3}, - ) + assert set(trackref.iter_all("Foo")) == {o1, o3} diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index e99ef40c4..5841d6866 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -18,301 +18,240 @@ from scrapy.utils.url import ( # type: ignore[attr-defined] ) -class UrlUtilsTest(unittest.TestCase): +class TestUrlUtils: def test_url_is_from_any_domain(self): url = "http://www.wheele-bin-art.co.uk/get/product/123" - self.assertTrue(url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])) - self.assertFalse(url_is_from_any_domain(url, ["art.co.uk"])) + assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"]) + assert not url_is_from_any_domain(url, ["art.co.uk"]) url = "http://wheele-bin-art.co.uk/get/product/123" - self.assertTrue(url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])) - self.assertFalse(url_is_from_any_domain(url, ["art.co.uk"])) + assert url_is_from_any_domain(url, ["wheele-bin-art.co.uk"]) + assert not url_is_from_any_domain(url, ["art.co.uk"]) url = "http://www.Wheele-Bin-Art.co.uk/get/product/123" - self.assertTrue(url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"])) - self.assertTrue(url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"])) + assert url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"]) + assert url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"]) url = "http://192.169.0.15:8080/mypage.html" - self.assertTrue(url_is_from_any_domain(url, ["192.169.0.15:8080"])) - self.assertFalse(url_is_from_any_domain(url, ["192.169.0.15"])) + assert url_is_from_any_domain(url, ["192.169.0.15:8080"]) + assert not url_is_from_any_domain(url, ["192.169.0.15"]) url = ( "javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20" "javascript:%20document.orderform_2581_1190810811.submit%28%29" ) - self.assertFalse(url_is_from_any_domain(url, ["testdomain.com"])) - self.assertFalse( - url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"]) - ) + assert not url_is_from_any_domain(url, ["testdomain.com"]) + assert not url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"]) def test_url_is_from_spider(self): spider = Spider(name="example.com") - self.assertTrue( - url_is_from_spider("http://www.example.com/some/page.html", spider) - ) - self.assertTrue( - url_is_from_spider("http://sub.example.com/some/page.html", spider) - ) - self.assertFalse( - url_is_from_spider("http://www.example.org/some/page.html", spider) - ) - self.assertFalse( - url_is_from_spider("http://www.example.net/some/page.html", spider) - ) + assert url_is_from_spider("http://www.example.com/some/page.html", spider) + assert url_is_from_spider("http://sub.example.com/some/page.html", spider) + assert not url_is_from_spider("http://www.example.org/some/page.html", spider) + assert not url_is_from_spider("http://www.example.net/some/page.html", spider) def test_url_is_from_spider_class_attributes(self): class MySpider(Spider): name = "example.com" - self.assertTrue( - url_is_from_spider("http://www.example.com/some/page.html", MySpider) - ) - self.assertTrue( - url_is_from_spider("http://sub.example.com/some/page.html", MySpider) - ) - self.assertFalse( - url_is_from_spider("http://www.example.org/some/page.html", MySpider) - ) - self.assertFalse( - url_is_from_spider("http://www.example.net/some/page.html", MySpider) - ) + assert url_is_from_spider("http://www.example.com/some/page.html", MySpider) + assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider) + assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider) + assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider) def test_url_is_from_spider_with_allowed_domains(self): spider = Spider( name="example.com", allowed_domains=["example.org", "example.net"] ) - self.assertTrue( - url_is_from_spider("http://www.example.com/some/page.html", spider) - ) - self.assertTrue( - url_is_from_spider("http://sub.example.com/some/page.html", spider) - ) - self.assertTrue(url_is_from_spider("http://example.com/some/page.html", spider)) - self.assertTrue( - url_is_from_spider("http://www.example.org/some/page.html", spider) - ) - self.assertTrue( - url_is_from_spider("http://www.example.net/some/page.html", spider) - ) - self.assertFalse( - url_is_from_spider("http://www.example.us/some/page.html", spider) - ) + assert url_is_from_spider("http://www.example.com/some/page.html", spider) + assert url_is_from_spider("http://sub.example.com/some/page.html", spider) + assert url_is_from_spider("http://example.com/some/page.html", spider) + assert url_is_from_spider("http://www.example.org/some/page.html", spider) + assert url_is_from_spider("http://www.example.net/some/page.html", spider) + assert not url_is_from_spider("http://www.example.us/some/page.html", spider) spider = Spider( name="example.com", allowed_domains={"example.com", "example.net"} ) - self.assertTrue( - url_is_from_spider("http://www.example.com/some/page.html", spider) - ) + assert url_is_from_spider("http://www.example.com/some/page.html", spider) spider = Spider( name="example.com", allowed_domains=("example.com", "example.net") ) - self.assertTrue( - url_is_from_spider("http://www.example.com/some/page.html", spider) - ) + assert url_is_from_spider("http://www.example.com/some/page.html", spider) def test_url_is_from_spider_with_allowed_domains_class_attributes(self): class MySpider(Spider): name = "example.com" allowed_domains = ("example.org", "example.net") - self.assertTrue( - url_is_from_spider("http://www.example.com/some/page.html", MySpider) - ) - self.assertTrue( - url_is_from_spider("http://sub.example.com/some/page.html", MySpider) - ) - self.assertTrue( - url_is_from_spider("http://example.com/some/page.html", MySpider) - ) - self.assertTrue( - url_is_from_spider("http://www.example.org/some/page.html", MySpider) - ) - self.assertTrue( - url_is_from_spider("http://www.example.net/some/page.html", MySpider) - ) - self.assertFalse( - url_is_from_spider("http://www.example.us/some/page.html", MySpider) - ) + assert url_is_from_spider("http://www.example.com/some/page.html", MySpider) + assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider) + assert url_is_from_spider("http://example.com/some/page.html", MySpider) + assert url_is_from_spider("http://www.example.org/some/page.html", MySpider) + assert url_is_from_spider("http://www.example.net/some/page.html", MySpider) + assert not url_is_from_spider("http://www.example.us/some/page.html", MySpider) def test_url_has_any_extension(self): deny_extensions = {"." + e for e in arg_to_iter(IGNORED_EXTENSIONS)} - self.assertTrue( - url_has_any_extension( - "http://www.example.com/archive.tar.gz", deny_extensions - ) + assert url_has_any_extension( + "http://www.example.com/archive.tar.gz", deny_extensions ) - self.assertTrue( - url_has_any_extension("http://www.example.com/page.doc", deny_extensions) + assert url_has_any_extension("http://www.example.com/page.doc", deny_extensions) + assert url_has_any_extension("http://www.example.com/page.pdf", deny_extensions) + assert not url_has_any_extension( + "http://www.example.com/page.htm", deny_extensions ) - self.assertTrue( - url_has_any_extension("http://www.example.com/page.pdf", deny_extensions) - ) - self.assertFalse( - url_has_any_extension("http://www.example.com/page.htm", deny_extensions) - ) - self.assertFalse( - url_has_any_extension("http://www.example.com/", deny_extensions) - ) - self.assertFalse( - url_has_any_extension( - "http://www.example.com/page.doc.html", deny_extensions - ) + assert not url_has_any_extension("http://www.example.com/", deny_extensions) + assert not url_has_any_extension( + "http://www.example.com/page.doc.html", deny_extensions ) -class AddHttpIfNoScheme(unittest.TestCase): +class TestAddHttpIfNoScheme: def test_add_scheme(self): - self.assertEqual( - add_http_if_no_scheme("www.example.com"), "http://www.example.com" - ) + assert add_http_if_no_scheme("www.example.com") == "http://www.example.com" def test_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme("example.com"), "http://example.com") + assert add_http_if_no_scheme("example.com") == "http://example.com" def test_path(self): - self.assertEqual( - add_http_if_no_scheme("www.example.com/some/page.html"), - "http://www.example.com/some/page.html", + assert ( + add_http_if_no_scheme("www.example.com/some/page.html") + == "http://www.example.com/some/page.html" ) def test_port(self): - self.assertEqual( - add_http_if_no_scheme("www.example.com:80"), "http://www.example.com:80" + assert ( + add_http_if_no_scheme("www.example.com:80") == "http://www.example.com:80" ) def test_fragment(self): - self.assertEqual( - add_http_if_no_scheme("www.example.com/some/page#frag"), - "http://www.example.com/some/page#frag", + assert ( + add_http_if_no_scheme("www.example.com/some/page#frag") + == "http://www.example.com/some/page#frag" ) def test_query(self): - self.assertEqual( - add_http_if_no_scheme("www.example.com/do?a=1&b=2&c=3"), - "http://www.example.com/do?a=1&b=2&c=3", + assert ( + add_http_if_no_scheme("www.example.com/do?a=1&b=2&c=3") + == "http://www.example.com/do?a=1&b=2&c=3" ) def test_username_password(self): - self.assertEqual( - add_http_if_no_scheme("username:password@www.example.com"), - "http://username:password@www.example.com", + assert ( + add_http_if_no_scheme("username:password@www.example.com") + == "http://username:password@www.example.com" ) def test_complete_url(self): - self.assertEqual( + assert ( add_http_if_no_scheme( "username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" - ), - "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag", + ) + == "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" ) def test_preserve_http(self): - self.assertEqual( - add_http_if_no_scheme("http://www.example.com"), "http://www.example.com" + assert ( + add_http_if_no_scheme("http://www.example.com") == "http://www.example.com" ) def test_preserve_http_without_subdomain(self): - self.assertEqual( - add_http_if_no_scheme("http://example.com"), "http://example.com" - ) + assert add_http_if_no_scheme("http://example.com") == "http://example.com" def test_preserve_http_path(self): - self.assertEqual( - add_http_if_no_scheme("http://www.example.com/some/page.html"), - "http://www.example.com/some/page.html", + assert ( + add_http_if_no_scheme("http://www.example.com/some/page.html") + == "http://www.example.com/some/page.html" ) def test_preserve_http_port(self): - self.assertEqual( - add_http_if_no_scheme("http://www.example.com:80"), - "http://www.example.com:80", + assert ( + add_http_if_no_scheme("http://www.example.com:80") + == "http://www.example.com:80" ) def test_preserve_http_fragment(self): - self.assertEqual( - add_http_if_no_scheme("http://www.example.com/some/page#frag"), - "http://www.example.com/some/page#frag", + assert ( + add_http_if_no_scheme("http://www.example.com/some/page#frag") + == "http://www.example.com/some/page#frag" ) def test_preserve_http_query(self): - self.assertEqual( - add_http_if_no_scheme("http://www.example.com/do?a=1&b=2&c=3"), - "http://www.example.com/do?a=1&b=2&c=3", + assert ( + add_http_if_no_scheme("http://www.example.com/do?a=1&b=2&c=3") + == "http://www.example.com/do?a=1&b=2&c=3" ) def test_preserve_http_username_password(self): - self.assertEqual( - add_http_if_no_scheme("http://username:password@www.example.com"), - "http://username:password@www.example.com", + assert ( + add_http_if_no_scheme("http://username:password@www.example.com") + == "http://username:password@www.example.com" ) def test_preserve_http_complete_url(self): - self.assertEqual( + assert ( add_http_if_no_scheme( "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" - ), - "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag", + ) + == "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" ) def test_protocol_relative(self): - self.assertEqual( - add_http_if_no_scheme("//www.example.com"), "http://www.example.com" - ) + assert add_http_if_no_scheme("//www.example.com") == "http://www.example.com" def test_protocol_relative_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme("//example.com"), "http://example.com") + assert add_http_if_no_scheme("//example.com") == "http://example.com" def test_protocol_relative_path(self): - self.assertEqual( - add_http_if_no_scheme("//www.example.com/some/page.html"), - "http://www.example.com/some/page.html", + assert ( + add_http_if_no_scheme("//www.example.com/some/page.html") + == "http://www.example.com/some/page.html" ) def test_protocol_relative_port(self): - self.assertEqual( - add_http_if_no_scheme("//www.example.com:80"), "http://www.example.com:80" + assert ( + add_http_if_no_scheme("//www.example.com:80") == "http://www.example.com:80" ) def test_protocol_relative_fragment(self): - self.assertEqual( - add_http_if_no_scheme("//www.example.com/some/page#frag"), - "http://www.example.com/some/page#frag", + assert ( + add_http_if_no_scheme("//www.example.com/some/page#frag") + == "http://www.example.com/some/page#frag" ) def test_protocol_relative_query(self): - self.assertEqual( - add_http_if_no_scheme("//www.example.com/do?a=1&b=2&c=3"), - "http://www.example.com/do?a=1&b=2&c=3", + assert ( + add_http_if_no_scheme("//www.example.com/do?a=1&b=2&c=3") + == "http://www.example.com/do?a=1&b=2&c=3" ) def test_protocol_relative_username_password(self): - self.assertEqual( - add_http_if_no_scheme("//username:password@www.example.com"), - "http://username:password@www.example.com", + assert ( + add_http_if_no_scheme("//username:password@www.example.com") + == "http://username:password@www.example.com" ) def test_protocol_relative_complete_url(self): - self.assertEqual( + assert ( add_http_if_no_scheme( "//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" - ), - "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag", + ) + == "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" ) def test_preserve_https(self): - self.assertEqual( - add_http_if_no_scheme("https://www.example.com"), "https://www.example.com" + assert ( + add_http_if_no_scheme("https://www.example.com") + == "https://www.example.com" ) def test_preserve_ftp(self): - self.assertEqual( - add_http_if_no_scheme("ftp://www.example.com"), "ftp://www.example.com" - ) + assert add_http_if_no_scheme("ftp://www.example.com") == "ftp://www.example.com" -class GuessSchemeTest(unittest.TestCase): +class TestGuessScheme: pass @@ -361,7 +300,7 @@ for k, args in enumerate( ): t_method = create_guess_scheme_t(args) t_method.__name__ = f"test_uri_{k:03}" - setattr(GuessSchemeTest, t_method.__name__, t_method) + setattr(TestGuessScheme, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation for k, skip_args in enumerate( @@ -376,29 +315,29 @@ for k, skip_args in enumerate( ): t_method = create_skipped_scheme_t(skip_args) t_method.__name__ = f"test_uri_skipped_{k:03}" - setattr(GuessSchemeTest, t_method.__name__, t_method) + setattr(TestGuessScheme, t_method.__name__, t_method) -class StripUrl(unittest.TestCase): +class TestStripUrl: def test_noop(self): - self.assertEqual( - strip_url("http://www.example.com/index.html"), - "http://www.example.com/index.html", + assert ( + strip_url("http://www.example.com/index.html") + == "http://www.example.com/index.html" ) def test_noop_query_string(self): - self.assertEqual( - strip_url("http://www.example.com/index.html?somekey=somevalue"), - "http://www.example.com/index.html?somekey=somevalue", + assert ( + strip_url("http://www.example.com/index.html?somekey=somevalue") + == "http://www.example.com/index.html?somekey=somevalue" ) def test_fragments(self): - self.assertEqual( + assert ( strip_url( "http://www.example.com/index.html?somekey=somevalue#section", strip_fragment=False, - ), - "http://www.example.com/index.html?somekey=somevalue#section", + ) + == "http://www.example.com/index.html?somekey=somevalue#section" ) def test_path(self): @@ -407,7 +346,7 @@ class StripUrl(unittest.TestCase): ("http://www.example.com", False, "http://www.example.com"), ("http://www.example.com", True, "http://www.example.com/"), ]: - self.assertEqual(strip_url(input_url, origin_only=origin), output_url) + assert strip_url(input_url, origin_only=origin) == output_url def test_credentials(self): for i, o in [ @@ -424,7 +363,7 @@ class StripUrl(unittest.TestCase): "ftp://www.example.com/index.html?somekey=somevalue", ), ]: - self.assertEqual(strip_url(i, strip_credentials=True), o) + assert strip_url(i, strip_credentials=True) == o def test_credentials_encoded_delims(self): for i, o in [ @@ -447,7 +386,7 @@ class StripUrl(unittest.TestCase): "ftp://www.example.com/index.html?somekey=somevalue", ), ]: - self.assertEqual(strip_url(i, strip_credentials=True), o) + assert strip_url(i, strip_credentials=True) == o def test_default_ports_creds_off(self): for i, o in [ @@ -484,7 +423,7 @@ class StripUrl(unittest.TestCase): "ftp://www.example.com:221/file.txt", ), ]: - self.assertEqual(strip_url(i), o) + assert strip_url(i) == o def test_default_ports(self): for i, o in [ @@ -521,9 +460,7 @@ class StripUrl(unittest.TestCase): "ftp://username:password@www.example.com:221/file.txt", ), ]: - self.assertEqual( - strip_url(i, strip_default_port=True, strip_credentials=False), o - ) + assert strip_url(i, strip_default_port=True, strip_credentials=False) == o def test_default_ports_keep(self): for i, o in [ @@ -560,9 +497,7 @@ class StripUrl(unittest.TestCase): "ftp://username:password@www.example.com:221/file.txt", ), ]: - self.assertEqual( - strip_url(i, strip_default_port=False, strip_credentials=False), o - ) + assert strip_url(i, strip_default_port=False, strip_credentials=False) == o def test_origin_only(self): for i, o in [ @@ -583,10 +518,10 @@ class StripUrl(unittest.TestCase): "https://www.example.com/", ), ]: - self.assertEqual(strip_url(i, origin_only=True), o) + assert strip_url(i, origin_only=True) == o -class IsPathTestCase(unittest.TestCase): +class TestIsPath: def test_path(self): for input_value, output_value in ( # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell @@ -604,9 +539,7 @@ class IsPathTestCase(unittest.TestCase): (r"C:\user\docs\somefile.ext:alternate_stream_name", True), (r"https://example.com", False), ): - self.assertEqual( - _is_filesystem_path(input_value), output_value, input_value - ) + assert _is_filesystem_path(input_value) == output_value, input_value @pytest.mark.parametrize( From 0c9200094e0764023cd34e72d4012f1c0450d8ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 5 Mar 2025 10:31:59 +0100 Subject: [PATCH 1671/2083] Extend BaseSettings with utils for add-ons (#6614) --- docs/topics/addons.rst | 30 +- docs/topics/settings.rst | 51 +++ scrapy/settings/__init__.py | 116 ++++++- scrapy/utils/conf.py | 3 +- tests/test_settings/__init__.py | 590 ++++++++++++++++++++++++++++++++ tox.ini | 2 +- 6 files changed, 779 insertions(+), 13 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 46cf1edbd..8ec7b0295 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -76,15 +76,11 @@ The settings set by the add-on should use the ``addon`` priority (see settings.set("DNSCACHE_ENABLED", True, "addon") This allows users to override these settings in the project or spider -configuration. This is not possible with settings that are mutable objects, -such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases -you can provide an add-on-specific setting that governs whether the add-on will -modify :setting:`ITEM_PIPELINES`:: +configuration. - class MyAddon: - def update_settings(self, settings): - if settings.getbool("MYADDON_ENABLE_PIPELINE"): - settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 +When editing the value of a setting instead of overriding it entirely, it is +usually best to leave its priority unchanged. For example, when editing a +:ref:`component priority dictionary <component-priority-dictionaries>`. If the ``update_settings`` method raises :exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes @@ -127,12 +123,28 @@ Add-on examples Set some basic configuration: +.. skip: next .. code-block:: python + from myproject.pipelines import MyPipeline + + class MyAddon: def update_settings(self, settings): - settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 settings.set("DNSCACHE_ENABLED", True, "addon") + settings.remove_from_list("METAREFRESH_IGNORE_TAGS", "noscript") + settings.setdefault_in_component_priority_dict( + "ITEM_PIPELINES", MyPipeline, 200 + ) + +.. tip:: When editing a :ref:`component priority dictionary + <component-priority-dictionaries>` setting, like :setting:`ITEM_PIPELINES`, + consider using setting methods like + :meth:`~scrapy.settings.BaseSettings.replace_in_component_priority_dict`, + :meth:`~scrapy.settings.BaseSettings.set_in_component_priority_dict` + and + :meth:`~scrapy.settings.BaseSettings.setdefault_in_component_priority_dict` + to avoid mistakes. Check dependencies: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a53e0806d..7646aca4f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -250,6 +250,57 @@ example, proper setting names for a fictional robots.txt extension would be ``ROBOTSTXT_ENABLED``, ``ROBOTSTXT_OBEY``, ``ROBOTSTXT_CACHEDIR``, etc. +.. _component-priority-dictionaries: + +Component priority dictionaries +=============================== + +A **component priority dictionary** is a :class:`dict` where keys are +:ref:`components <topics-components>` and values are component priorities. For +example: + +.. skip: next +.. code-block:: python + + { + "path.to.ComponentA": None, + ComponentB: 100, + } + +A component can be specified either as a class object or through an import +path. + +.. warning:: Component priority dictionaries are regular :class:`dict` objects. + Be careful not to define the same component more than once, e.g. with + different import path strings or defining both an import path and a + :class:`type` object. + +A priority can be an :class:`int` or :data:`None`. + +A component with priority 1 goes *before* a component with priority 2. What +going before entails, however, depends on the corresponding setting. For +example, in the :setting:`DOWNLOADER_MIDDLEWARES` setting, components have +their +:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request` +method executed before that of later components, but have their +:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` +method executed after that of later components. + +A component with priority :data:`None` is disabled. + +Some component priority dictionaries get merged with some built-in value. For +example, :setting:`DOWNLOADER_MIDDLEWARES` is merged with +:setting:`DOWNLOADER_MIDDLEWARES_BASE`. This is where :data:`None` comes in +handy, allowing you to disable a component from the base setting in the regular +setting: + +.. code-block:: python + + DOWNLOADER_MIDDLEWARES = { + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": None, + } + + Special settings ================ diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index f31f824a8..cc4853c8f 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -8,6 +8,7 @@ from pprint import pformat from typing import TYPE_CHECKING, Any, Union, cast from scrapy.settings import default_settings +from scrapy.utils.misc import load_object # The key types are restricted in BaseSettings._get_key() to ones supported by JSON, # see https://github.com/scrapy/scrapy/issues/5383. @@ -111,6 +112,31 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def __contains__(self, name: Any) -> bool: return name in self.attributes + def add_to_list(self, name: _SettingsKeyT, item: Any) -> None: + """Append *item* to the :class:`list` setting with the specified *name* + if *item* is not already in that list. + + This change is applied regardless of the priority of the *name* + setting. The setting priority is not affected by this change either. + """ + value: list[str] = self.getlist(name) + if item not in value: + self.set(name, [*value, item], self.getpriority(name) or 0) + + def remove_from_list(self, name: _SettingsKeyT, item: Any) -> None: + """Remove *item* from the :class:`list` setting with the specified + *name*. + + If *item* is missing, raise :exc:`ValueError`. + + This change is applied regardless of the priority of the *name* + setting. The setting priority is not affected by this change either. + """ + value: list[str] = self.getlist(name) + if item not in value: + raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).") + self.set(name, [v for v in value if v != item], self.getpriority(name) or 0) + def get(self, name: _SettingsKeyT, default: Any = None) -> Any: """ Get a setting value without affecting its original type. @@ -181,8 +207,9 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): self, name: _SettingsKeyT, default: list[Any] | None = None ) -> list[Any]: """ - Get a setting value as a list. If the setting original type is a list, a - copy of it will be returned. If it's a string it will be split by ",". + Get a setting value as a list. If the setting original type is a list, + a copy of it will be returned. If it's a string it will be split by + ",". If it is an empty string, an empty list will be returned. For example, settings populated through environment variables set to ``'one,two'`` will return a list ['one', 'two'] when using this method. @@ -194,6 +221,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): :type default: object """ value = self.get(name, default or []) + if not value: + return [] if isinstance(value, str): value = value.split(",") return list(value) @@ -299,6 +328,47 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return max(cast(int, self.getpriority(name)) for name in self) return get_settings_priority("default") + def replace_in_component_priority_dict( + self, + name: _SettingsKeyT, + old_cls: type, + new_cls: type, + priority: int | None = None, + ) -> None: + """Replace *old_cls* with *new_cls* in the *name* :ref:`component + priority dictionary <component-priority-dictionaries>`. + + If *old_cls* is missing, or has :data:`None` as value, :exc:`KeyError` + is raised. + + If *old_cls* was present as an import string, even more than once, + those keys are dropped and replaced by *new_cls*. + + If *priority* is specified, that is the value assigned to *new_cls* in + the component priority dictionary. Otherwise, the value of *old_cls* is + used. If *old_cls* was present multiple times (possible with import + strings) with different values, the value assigned to *new_cls* is one + of them, with no guarantee about which one it is. + + This change is applied regardless of the priority of the *name* + setting. The setting priority is not affected by this change either. + """ + component_priority_dict = self.getdict(name) + old_priority = None + for cls_or_path in tuple(component_priority_dict): + if load_object(cls_or_path) != old_cls: + continue + if (old_priority := component_priority_dict.pop(cls_or_path)) is None: + break + if old_priority is None: + raise KeyError( + f"{old_cls} not found in the {name} setting ({component_priority_dict!r})." + ) + component_priority_dict[new_cls] = ( + old_priority if priority is None else priority + ) + self.set(name, component_priority_dict, priority=self.getpriority(name) or 0) + def __setitem__(self, name: _SettingsKeyT, value: Any) -> None: self.set(name, value) @@ -332,6 +402,30 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): else: self.attributes[name].set(value, priority) + def set_in_component_priority_dict( + self, name: _SettingsKeyT, cls: type, priority: int | None + ) -> None: + """Set the *cls* component in the *name* :ref:`component priority + dictionary <component-priority-dictionaries>` setting with *priority*. + + If *cls* already exists, its value is updated. + + If *cls* was present as an import string, even more than once, those + keys are dropped and replaced by *cls*. + + This change is applied regardless of the priority of the *name* + setting. The setting priority is not affected by this change either. + """ + component_priority_dict = self.getdict(name) + for cls_or_path in tuple(component_priority_dict): + if not isinstance(cls_or_path, str): + continue + _cls = load_object(cls_or_path) + if _cls == cls: + del component_priority_dict[cls_or_path] + component_priority_dict[cls] = priority + self.set(name, component_priority_dict, self.getpriority(name) or 0) + def setdefault( self, name: _SettingsKeyT, @@ -344,6 +438,24 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return self.attributes[name].value + def setdefault_in_component_priority_dict( + self, name: _SettingsKeyT, cls: type, priority: int | None + ) -> None: + """Set the *cls* component in the *name* :ref:`component priority + dictionary <component-priority-dictionaries>` setting with *priority* + if not already defined (even as an import string). + + If *cls* is not already defined, it is set regardless of the priority + of the *name* setting. The setting priority is not affected by this + change either. + """ + component_priority_dict = self.getdict(name) + for cls_or_path in tuple(component_priority_dict): + if load_object(cls_or_path) == cls: + return + component_priority_dict[cls] = priority + self.set(name, component_priority_dict, self.getpriority(name) or 0) + def setdict(self, values: _SettingsInputT, priority: int | str = "project") -> None: self.update(values, priority) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index a86aad51c..891cbb485 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -22,7 +22,8 @@ def build_component_list( *, convert: Callable[[Any], Any] = update_classpath, ) -> list[Any]: - """Compose a component list from a { class: order } dictionary.""" + """Compose a component list from a :ref:`component priority dictionary + <component-priority-dictionaries>`.""" def _check_components(complist: Collection[Any]) -> None: if len({convert(c) for c in complist}) != len(complist): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index b7a316eee..909b365a9 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -260,6 +260,7 @@ class BaseSettingsTest(unittest.TestCase): "TEST_FLOAT2": "123.45", "TEST_LIST1": ["one", "two"], "TEST_LIST2": "one,two", + "TEST_LIST3": "", "TEST_STR": "value", "TEST_DICT1": {"key1": "val1", "ke2": 3}, "TEST_DICT2": '{"key1": "val1", "ke2": 3}', @@ -292,6 +293,7 @@ class BaseSettingsTest(unittest.TestCase): self.assertEqual(settings.getfloat("TEST_FLOATx", 55.0), 55.0) self.assertEqual(settings.getlist("TEST_LIST1"), ["one", "two"]) self.assertEqual(settings.getlist("TEST_LIST2"), ["one", "two"]) + self.assertEqual(settings.getlist("TEST_LIST3"), []) self.assertEqual(settings.getlist("TEST_LISTx"), []) self.assertEqual(settings.getlist("TEST_LISTx", ["default"]), ["default"]) self.assertEqual(settings["TEST_STR"], "value") @@ -504,3 +506,591 @@ class SettingsTest(unittest.TestCase): TypeError, match="Trying to modify an immutable Settings object" ): settings.pop("OTHER_DUMMY_CONFIG") + + +@pytest.mark.parametrize( + ("before", "name", "item", "after"), + [ + ({}, "FOO", "BAR", {"FOO": ["BAR"]}), + ({"FOO": []}, "FOO", "BAR", {"FOO": ["BAR"]}), + ({"FOO": ["BAR"]}, "FOO", "BAZ", {"FOO": ["BAR", "BAZ"]}), + ({"FOO": ["BAR"]}, "FOO", "BAR", {"FOO": ["BAR"]}), + ({"FOO": ""}, "FOO", "BAR", {"FOO": ["BAR"]}), + ({"FOO": "BAR"}, "FOO", "BAR", {"FOO": "BAR"}), + ({"FOO": "BAR"}, "FOO", "BAZ", {"FOO": ["BAR", "BAZ"]}), + ({"FOO": "BAR,BAZ"}, "FOO", "BAZ", {"FOO": "BAR,BAZ"}), + ({"FOO": "BAR,BAZ"}, "FOO", "QUX", {"FOO": ["BAR", "BAZ", "QUX"]}), + ], +) +def test_add_to_list(before, name, item, after): + settings = BaseSettings(before, priority=0) + settings.add_to_list(name, item) + expected_priority = settings.getpriority(name) or 0 + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings, ( + f"{settings[name]=} != {expected_settings[name]=}" + ) + assert settings.getpriority(name) == expected_settings.getpriority(name) + + +@pytest.mark.parametrize( + ("before", "name", "item", "after"), + [ + ({}, "FOO", "BAR", ValueError), + ({"FOO": ["BAR"]}, "FOO", "BAR", {"FOO": []}), + ({"FOO": ["BAR"]}, "FOO", "BAZ", ValueError), + ({"FOO": ["BAR", "BAZ"]}, "FOO", "BAR", {"FOO": ["BAZ"]}), + ({"FOO": ""}, "FOO", "BAR", ValueError), + ({"FOO": "[]"}, "FOO", "BAR", ValueError), + ({"FOO": "BAR"}, "FOO", "BAR", {"FOO": []}), + ({"FOO": "BAR"}, "FOO", "BAZ", ValueError), + ({"FOO": "BAR,BAZ"}, "FOO", "BAR", {"FOO": ["BAZ"]}), + ], +) +def test_remove_from_list(before, name, item, after): + settings = BaseSettings(before, priority=0) + + if isinstance(after, type) and issubclass(after, Exception): + with pytest.raises(after): + settings.remove_from_list(name, item) + return + + settings.remove_from_list(name, item) + expected_priority = settings.getpriority(name) or 0 + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings, ( + f"{settings[name]=} != {expected_settings[name]=}" + ) + assert settings.getpriority(name) == expected_settings.getpriority(name) + + +class Component1: + pass + + +Component1Alias = Component1 + + +class Component1Subclass(Component1): + pass + + +Component1SubclassAlias = Component1Subclass + + +class Component2: + pass + + +class Component3: + pass + + +class Component4: + pass + + +@pytest.mark.parametrize( + ("before", "name", "old_cls", "new_cls", "priority", "after"), + [ + ({}, "FOO", Component1, Component2, None, KeyError), + ( + {"FOO": {Component1: 1}}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": {Component1: 1}}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + {"FOO": {"tests.test_settings.Component1": 1}}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": {Component1Alias: 1}}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": {Component1Alias: 1}}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + {"FOO": {"tests.test_settings.Component1Alias": 1}}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": {"tests.test_settings.Component1Alias": 1}}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + { + "FOO": { + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": 2, + } + }, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 2}}, + ), + ( + { + "FOO": { + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": 2, + } + }, + "FOO", + Component1, + Component2, + 3, + {"FOO": {Component2: 3}}, + ), + ( + {"FOO": '{"tests.test_settings.Component1": 1}'}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": '{"tests.test_settings.Component1": 1}'}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + {"FOO": '{"tests.test_settings.Component1Alias": 1}'}, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 1}}, + ), + ( + {"FOO": '{"tests.test_settings.Component1Alias": 1}'}, + "FOO", + Component1, + Component2, + 2, + {"FOO": {Component2: 2}}, + ), + ( + { + "FOO": '{"tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2}' + }, + "FOO", + Component1, + Component2, + None, + {"FOO": {Component2: 2}}, + ), + ( + { + "FOO": '{"tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2}' + }, + "FOO", + Component1, + Component2, + 3, + {"FOO": {Component2: 3}}, + ), + # If old_cls has None as value, raise KeyError. + ( + {"FOO": {Component1: None}}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + ( + {"FOO": '{"tests.test_settings.Component1": null}'}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + ( + {"FOO": {Component1: None, "tests.test_settings.Component1": None}}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + ( + {"FOO": {Component1: 1, "tests.test_settings.Component1": None}}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + ( + {"FOO": {Component1: None, "tests.test_settings.Component1": 1}}, + "FOO", + Component1, + Component2, + None, + KeyError, + ), + # Unrelated components are kept as is, as expected. + ( + { + "FOO": { + Component1: 1, + "tests.test_settings.Component2": 2, + Component3: 3, + } + }, + "FOO", + Component3, + Component4, + None, + { + "FOO": { + Component1: 1, + "tests.test_settings.Component2": 2, + Component4: 3, + } + }, + ), + ], +) +def test_replace_in_component_priority_dict( + before, name, old_cls, new_cls, priority, after +): + settings = BaseSettings(before, priority=0) + + if isinstance(after, type) and issubclass(after, Exception): + with pytest.raises(after): + settings.replace_in_component_priority_dict( + name, old_cls, new_cls, priority + ) + return + + expected_priority = settings.getpriority(name) or 0 + settings.replace_in_component_priority_dict(name, old_cls, new_cls, priority) + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings + assert settings.getpriority(name) == expected_settings.getpriority(name) + + +@pytest.mark.parametrize( + ("before", "name", "cls", "priority", "after"), + [ + # Set + ({}, "FOO", Component1, None, {"FOO": {Component1: None}}), + ({}, "FOO", Component1, 0, {"FOO": {Component1: 0}}), + ({}, "FOO", Component1, 1, {"FOO": {Component1: 1}}), + # Add + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + None, + {"FOO": {Component1: 0, Component2: None}}, + ), + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + 0, + {"FOO": {Component1: 0, Component2: 0}}, + ), + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + 1, + {"FOO": {Component1: 0, Component2: 1}}, + ), + # Replace + ( + { + "FOO": { + Component1: None, + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + Component1Subclass: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + "FOO", + Component1, + None, + { + "FOO": { + Component1: None, + Component1Subclass: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + ), + ( + { + "FOO": { + Component1: 0, + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": None, + Component1Subclass: 0, + "tests.test_settings.Component1Subclass": 1, + "tests.test_settings.Component1SubclassAlias": None, + } + }, + "FOO", + Component1, + 0, + { + "FOO": { + Component1: 0, + Component1Subclass: 0, + "tests.test_settings.Component1Subclass": 1, + "tests.test_settings.Component1SubclassAlias": None, + } + }, + ), + ( + { + "FOO": { + Component1: 1, + "tests.test_settings.Component1": None, + "tests.test_settings.Component1Alias": 0, + Component1Subclass: 1, + "tests.test_settings.Component1Subclass": None, + "tests.test_settings.Component1SubclassAlias": 0, + } + }, + "FOO", + Component1, + 1, + { + "FOO": { + Component1: 1, + Component1Subclass: 1, + "tests.test_settings.Component1Subclass": None, + "tests.test_settings.Component1SubclassAlias": 0, + } + }, + ), + # String-based setting values + ( + {"FOO": '{"tests.test_settings.Component1": 0}'}, + "FOO", + Component2, + None, + {"FOO": {"tests.test_settings.Component1": 0, Component2: None}}, + ), + ( + { + "FOO": """{ + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1 + }""" + }, + "FOO", + Component1, + None, + { + "FOO": { + Component1: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + ), + ], +) +def test_set_in_component_priority_dict(before, name, cls, priority, after): + settings = BaseSettings(before, priority=0) + expected_priority = settings.getpriority(name) or 0 + settings.set_in_component_priority_dict(name, cls, priority) + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings + assert settings.getpriority(name) == expected_settings.getpriority(name), ( + f"{settings.getpriority(name)=} != {expected_settings.getpriority(name)=}" + ) + + +@pytest.mark.parametrize( + ("before", "name", "cls", "priority", "after"), + [ + # Set + ({}, "FOO", Component1, None, {"FOO": {Component1: None}}), + ({}, "FOO", Component1, 0, {"FOO": {Component1: 0}}), + ({}, "FOO", Component1, 1, {"FOO": {Component1: 1}}), + # Add + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + None, + {"FOO": {Component1: 0, Component2: None}}, + ), + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + 0, + {"FOO": {Component1: 0, Component2: 0}}, + ), + ( + {"FOO": {Component1: 0}}, + "FOO", + Component2, + 1, + {"FOO": {Component1: 0, Component2: 1}}, + ), + # Keep + ( + { + "FOO": { + Component1: None, + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + Component1Subclass: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + "FOO", + Component1, + None, + { + "FOO": { + Component1: None, + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + Component1Subclass: None, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1, + } + }, + ), + ( + { + "FOO": { + Component1: 0, + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": None, + Component1Subclass: 0, + "tests.test_settings.Component1Subclass": 1, + "tests.test_settings.Component1SubclassAlias": None, + } + }, + "FOO", + Component1, + 0, + { + "FOO": { + Component1: 0, + "tests.test_settings.Component1": 1, + "tests.test_settings.Component1Alias": None, + Component1Subclass: 0, + "tests.test_settings.Component1Subclass": 1, + "tests.test_settings.Component1SubclassAlias": None, + } + }, + ), + ( + { + "FOO": { + Component1: 1, + "tests.test_settings.Component1": None, + "tests.test_settings.Component1Alias": 0, + Component1Subclass: 1, + "tests.test_settings.Component1Subclass": None, + "tests.test_settings.Component1SubclassAlias": 0, + } + }, + "FOO", + Component1, + 1, + { + "FOO": { + Component1: 1, + "tests.test_settings.Component1": None, + "tests.test_settings.Component1Alias": 0, + Component1Subclass: 1, + "tests.test_settings.Component1Subclass": None, + "tests.test_settings.Component1SubclassAlias": 0, + } + }, + ), + # String-based setting values + ( + {"FOO": '{"tests.test_settings.Component1": 0}'}, + "FOO", + Component2, + None, + {"FOO": {"tests.test_settings.Component1": 0, Component2: None}}, + ), + ( + { + "FOO": """{ + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1 + }""" + }, + "FOO", + Component1, + None, + { + "FOO": """{ + "tests.test_settings.Component1": 0, + "tests.test_settings.Component1Alias": 1, + "tests.test_settings.Component1Subclass": 0, + "tests.test_settings.Component1SubclassAlias": 1 + }""" + }, + ), + ], +) +def test_setdefault_in_component_priority_dict(before, name, cls, priority, after): + settings = BaseSettings(before, priority=0) + expected_priority = settings.getpriority(name) or 0 + settings.setdefault_in_component_priority_dict(name, cls, priority) + expected_settings = BaseSettings(after, priority=expected_priority) + assert settings == expected_settings + assert settings.getpriority(name) == expected_settings.getpriority(name) diff --git a/tox.ini b/tox.ini index 82ad84c90..041fcffca 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules + pytest --cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml {posargs:--durations=10 docs scrapy tests} --doctest-modules install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} From 1843a4f75358a76fe8e4624f8f4dc26084d19b85 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 6 Mar 2025 23:50:14 +0400 Subject: [PATCH 1672/2083] Converting tests to plain asserts, part 3. (#6700) --- tests/test_downloadermiddleware.py | 54 +- ...test_downloadermiddleware_ajaxcrawlable.py | 18 +- tests/test_downloadermiddleware_cookies.py | 78 +- ...est_downloadermiddleware_defaultheaders.py | 10 +- ...st_downloadermiddleware_downloadtimeout.py | 12 +- tests/test_downloadermiddleware_httpauth.py | 30 +- tests/test_downloadermiddleware_httpcache.py | 51 +- ...st_downloadermiddleware_httpcompression.py | 88 +-- tests/test_downloadermiddleware_httpproxy.py | 166 ++-- tests/test_downloadermiddleware_redirect.py | 732 +++++++++--------- tests/test_downloadermiddleware_retry.py | 87 +-- tests/test_downloadermiddleware_robotstxt.py | 8 +- tests/test_downloadermiddleware_stats.py | 14 +- tests/test_downloadermiddleware_useragent.py | 10 +- 14 files changed, 638 insertions(+), 720 deletions(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 49498375c..8e718ad5b 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -16,7 +16,7 @@ from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue -class ManagerTestCase(TestCase): +class TestManagerBase(TestCase): settings_dict = None def setUp(self): @@ -51,14 +51,14 @@ class ManagerTestCase(TestCase): return ret -class DefaultsTest(ManagerTestCase): +class TestDefaults(TestManagerBase): """Tests default behavior with default settings""" def test_request_response(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) ret = self._download(req, resp) - self.assertTrue(isinstance(ret, Response), "Non-response returned") + assert isinstance(ret, Response), "Non-response returned" def test_3xx_and_invalid_gzipped_body_must_redirect(self): """Regression test for a failure when redirecting a compressed @@ -86,11 +86,9 @@ class DefaultsTest(ManagerTestCase): }, ) ret = self._download(request=req, response=resp) - self.assertTrue(isinstance(ret, Request), f"Not redirected: {ret!r}") - self.assertEqual( - to_bytes(ret.url), - resp.headers["Location"], - "Not redirected to location header", + assert isinstance(ret, Request), f"Not redirected: {ret!r}" + assert to_bytes(ret.url) == resp.headers["Location"], ( + "Not redirected to location header" ) def test_200_and_invalid_gzipped_body_must_fail(self): @@ -111,7 +109,7 @@ class DefaultsTest(ManagerTestCase): self._download(request=req, response=resp) -class ResponseFromProcessRequestTest(ManagerTestCase): +class TestResponseFromProcessRequest(TestManagerBase): """Tests middleware returning a response from process_request.""" def test_download_func_not_called(self): @@ -130,11 +128,11 @@ class ResponseFromProcessRequestTest(ManagerTestCase): dfd.addBoth(results.append) self._wait(dfd) - self.assertIs(results[0], resp) - self.assertFalse(download_func.called) + assert results[0] is resp + assert not download_func.called -class ProcessRequestInvalidOutput(ManagerTestCase): +class TestProcessRequestInvalidOutput(TestManagerBase): """Invalid return value for process_request method should raise an exception""" def test_invalid_process_request(self): @@ -149,11 +147,11 @@ class ProcessRequestInvalidOutput(ManagerTestCase): dfd = self.mwman.download(download_func, req, self.spider) results = [] dfd.addBoth(results.append) - self.assertIsInstance(results[0], Failure) - self.assertIsInstance(results[0].value, _InvalidOutput) + assert isinstance(results[0], Failure) + assert isinstance(results[0].value, _InvalidOutput) -class ProcessResponseInvalidOutput(ManagerTestCase): +class TestProcessResponseInvalidOutput(TestManagerBase): """Invalid return value for process_response method should raise an exception""" def test_invalid_process_response(self): @@ -168,11 +166,11 @@ class ProcessResponseInvalidOutput(ManagerTestCase): dfd = self.mwman.download(download_func, req, self.spider) results = [] dfd.addBoth(results.append) - self.assertIsInstance(results[0], Failure) - self.assertIsInstance(results[0].value, _InvalidOutput) + assert isinstance(results[0], Failure) + assert isinstance(results[0].value, _InvalidOutput) -class ProcessExceptionInvalidOutput(ManagerTestCase): +class TestProcessExceptionInvalidOutput(TestManagerBase): """Invalid return value for process_exception method should raise an exception""" def test_invalid_process_exception(self): @@ -190,11 +188,11 @@ class ProcessExceptionInvalidOutput(ManagerTestCase): dfd = self.mwman.download(download_func, req, self.spider) results = [] dfd.addBoth(results.append) - self.assertIsInstance(results[0], Failure) - self.assertIsInstance(results[0].value, _InvalidOutput) + assert isinstance(results[0], Failure) + assert isinstance(results[0].value, _InvalidOutput) -class MiddlewareUsingDeferreds(ManagerTestCase): +class TestMiddlewareUsingDeferreds(TestManagerBase): """Middlewares using Deferreds should work""" def test_deferred(self): @@ -218,12 +216,12 @@ class MiddlewareUsingDeferreds(ManagerTestCase): dfd.addBoth(results.append) self._wait(dfd) - self.assertIs(results[0], resp) - self.assertFalse(download_func.called) + assert results[0] is resp + assert not download_func.called @pytest.mark.usefixtures("reactor_pytest") -class MiddlewareUsingCoro(ManagerTestCase): +class TestMiddlewareUsingCoro(TestManagerBase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): @@ -242,8 +240,8 @@ class MiddlewareUsingCoro(ManagerTestCase): dfd.addBoth(results.append) self._wait(dfd) - self.assertIs(results[0], resp) - self.assertFalse(download_func.called) + assert results[0] is resp + assert not download_func.called @pytest.mark.only_asyncio def test_asyncdef_asyncio(self): @@ -262,5 +260,5 @@ class MiddlewareUsingCoro(ManagerTestCase): dfd.addBoth(results.append) self._wait(dfd) - self.assertIs(results[0], resp) - self.assertFalse(download_func.called) + assert results[0] is resp + assert not download_func.called diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 76fcece4f..44084f1e8 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -1,5 +1,3 @@ -import unittest - import pytest from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware @@ -9,8 +7,8 @@ from scrapy.utils.test import get_crawler @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class AjaxCrawlMiddlewareTest(unittest.TestCase): - def setUp(self): +class TestAjaxCrawlMiddleware: + def setup_method(self): crawler = get_crawler(Spider, {"AJAXCRAWL_ENABLED": True}) self.spider = crawler._create_spider("foo") self.mw = AjaxCrawlMiddleware.from_crawler(crawler) @@ -26,13 +24,13 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase): def test_non_get(self): req, resp = self._req_resp("http://example.com/", {"method": "HEAD"}) resp2 = self.mw.process_response(req, resp, self.spider) - self.assertEqual(resp, resp2) + assert resp == resp2 def test_binary_response(self): req = Request("http://example.com/") resp = Response("http://example.com/", body=b"foobar\x00\x01\x02", request=req) resp2 = self.mw.process_response(req, resp, self.spider) - self.assertIs(resp, resp2) + assert resp is resp2 def test_ajaxcrawl(self): req, resp = self._req_resp( @@ -41,8 +39,8 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase): {"body": self._ajaxcrawlable_body()}, ) req2 = self.mw.process_response(req, resp, self.spider) - self.assertEqual(req2.url, "http://example.com/?_escaped_fragment_=") - self.assertEqual(req2.meta["foo"], "bar") + assert req2.url == "http://example.com/?_escaped_fragment_=" + assert req2.meta["foo"] == "bar" def test_ajaxcrawl_loop(self): req, resp = self._req_resp( @@ -53,7 +51,7 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase): resp3 = self.mw.process_response(req2, resp2, self.spider) assert isinstance(resp3, HtmlResponse), (resp3.__class__, resp3) - self.assertEqual(resp3.request.url, "http://example.com/?_escaped_fragment_=") + assert resp3.request.url == "http://example.com/?_escaped_fragment_=" assert resp3 is resp2 def test_noncrawlable_body(self): @@ -61,4 +59,4 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase): "http://example.com/", {}, {"body": b"<html></html>"} ) resp2 = self.mw.process_response(req, resp, self.spider) - self.assertIs(resp, resp2) + assert resp is resp2 diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 694a669d4..8bf3a1f09 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,5 +1,4 @@ import logging -from unittest import TestCase import pytest from testfixtures import LogCapture @@ -53,19 +52,19 @@ def _cookies_to_set_cookie_list(cookies): return filter(None, (_cookie_to_set_cookie_value(cookie) for cookie in cookies)) -class CookiesMiddlewareTest(TestCase): +class TestCookiesMiddleware: def assertCookieValEqual(self, first, second, msg=None): def split_cookies(cookies): return sorted([s.strip() for s in to_bytes(cookies).split(b";")]) - return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg) + assert split_cookies(first) == split_cookies(second), msg - def setUp(self): + def setup_method(self): self.spider = Spider("foo") self.mw = CookiesMiddleware() self.redirect_middleware = RedirectMiddleware(settings=Settings()) - def tearDown(self): + def teardown_method(self): del self.mw del self.redirect_middleware @@ -80,7 +79,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request("http://scrapytest.org/sub1/") assert self.mw.process_request(req2, self.spider) is None - self.assertEqual(req2.headers.get("Cookie"), b"C1=value1") + assert req2.headers.get("Cookie") == b"C1=value1" def test_setting_false_cookies_enabled(self): with pytest.raises(NotConfigured): @@ -89,12 +88,12 @@ class CookiesMiddlewareTest(TestCase): ) def test_setting_default_cookies_enabled(self): - self.assertIsInstance( + assert isinstance( CookiesMiddleware.from_crawler(get_crawler()), CookiesMiddleware ) def test_setting_true_cookies_enabled(self): - self.assertIsInstance( + assert isinstance( CookiesMiddleware.from_crawler( get_crawler(settings_dict={"COOKIES_ENABLED": True}) ), @@ -161,7 +160,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request("http://scrapytest.org/sub1/") assert self.mw.process_request(req2, self.spider) is None - self.assertIn("Cookie", req2.headers) + assert "Cookie" in req2.headers def test_dont_merge_cookies(self): # merge some cookies into jar @@ -185,12 +184,12 @@ class CookiesMiddlewareTest(TestCase): # check that cookies are merged back req = Request("http://scrapytest.org/mergeme") assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get("Cookie"), b"C1=value1") + assert req.headers.get("Cookie") == b"C1=value1" # check that cookies are merged when dont_merge_cookies is passed as 0 req = Request("http://scrapytest.org/mergeme", meta={"dont_merge_cookies": 0}) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get("Cookie"), b"C1=value1") + assert req.headers.get("Cookie") == b"C1=value1" def test_complex_cookies(self): # merge some cookies into jar @@ -230,7 +229,7 @@ class CookiesMiddlewareTest(TestCase): # embed C2 for scrapytest.org/bar req = Request("http://scrapytest.org/bar") self.mw.process_request(req, self.spider) - self.assertEqual(req.headers.get("Cookie"), b"C2=value2") + assert req.headers.get("Cookie") == b"C2=value2" # embed nothing for scrapytest.org/baz req = Request("http://scrapytest.org/baz") @@ -240,7 +239,7 @@ class CookiesMiddlewareTest(TestCase): def test_merge_request_cookies(self): req = Request("http://scrapytest.org/", cookies={"galleta": "salada"}) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get("Cookie"), b"galleta=salada") + assert req.headers.get("Cookie") == b"galleta=salada" headers = {"Set-Cookie": "C1=value1; path=/"} res = Response("http://scrapytest.org/", headers=headers) @@ -260,7 +259,7 @@ class CookiesMiddlewareTest(TestCase): meta={"cookiejar": "store1"}, ) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers.get("Cookie"), b"galleta=salada") + assert req.headers.get("Cookie") == b"galleta=salada" headers = {"Set-Cookie": "C1=value1; path=/"} res = Response("http://scrapytest.org/", headers=headers, request=req) @@ -278,7 +277,7 @@ class CookiesMiddlewareTest(TestCase): meta={"cookiejar": "store2"}, ) assert self.mw.process_request(req3, self.spider) is None - self.assertEqual(req3.headers.get("Cookie"), b"galleta=dulce") + assert req3.headers.get("Cookie") == b"galleta=dulce" headers = {"Set-Cookie": "C2=value2; path=/"} res2 = Response("http://scrapytest.org/", headers=headers, request=req3) @@ -302,22 +301,22 @@ class CookiesMiddlewareTest(TestCase): req5_2 = Request("http://scrapytest.org:1104/some-redirected-path") assert self.mw.process_request(req5_2, self.spider) is None - self.assertEqual(req5_2.headers.get("Cookie"), b"C1=value1") + assert req5_2.headers.get("Cookie") == b"C1=value1" req5_3 = Request("http://scrapytest.org/some-redirected-path") assert self.mw.process_request(req5_3, self.spider) is None - self.assertEqual(req5_3.headers.get("Cookie"), b"C1=value1") + assert req5_3.headers.get("Cookie") == b"C1=value1" # skip cookie retrieval for not http request req6 = Request("file:///scrapy/sometempfile") assert self.mw.process_request(req6, self.spider) is None - self.assertEqual(req6.headers.get("Cookie"), None) + assert req6.headers.get("Cookie") is None def test_local_domain(self): request = Request("http://example-host/", cookies={"currencyCookie": "USD"}) assert self.mw.process_request(request, self.spider) is None - self.assertIn("Cookie", request.headers) - self.assertEqual(b"currencyCookie=USD", request.headers["Cookie"]) + assert "Cookie" in request.headers + assert request.headers["Cookie"] == b"currencyCookie=USD" @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): @@ -474,7 +473,7 @@ class CookiesMiddlewareTest(TestCase): request1 = Request(cookies=input_cookies, **source) self.mw.process_request(request1, self.spider) cookies = request1.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies1 else None) + assert cookies == (b"a=b" if cookies1 else None) response = Response( headers={ @@ -482,21 +481,18 @@ class CookiesMiddlewareTest(TestCase): }, **target, ) - self.assertEqual( - self.mw.process_response(request1, response, self.spider), - response, - ) + assert self.mw.process_response(request1, response, self.spider) == response request2 = self.redirect_middleware.process_response( request1, response, self.spider, ) - self.assertIsInstance(request2, Request) + assert isinstance(request2, Request) self.mw.process_request(request2, self.spider) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies2 else None) + assert cookies == (b"a=b" if cookies2 else None) def test_cookie_redirect_same_domain(self): self._test_cookie_redirect( @@ -573,10 +569,10 @@ class CookiesMiddlewareTest(TestCase): response, self.spider, ) - self.assertIsInstance(request2, Request) + assert isinstance(request2, Request) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies2 else None) + assert cookies == (b"a=b" if cookies2 else None) def test_cookie_header_redirect_same_domain(self): self._test_cookie_header_redirect( @@ -626,12 +622,12 @@ class CookiesMiddlewareTest(TestCase): request1 = Request(url1, cookies=input_cookies) self.mw.process_request(request1, self.spider) cookies = request1.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies1 else None) + assert cookies == (b"a=b" if cookies1 else None) request2 = Request(url2) self.mw.process_request(request2, self.spider) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies2 else None) + assert cookies == (b"a=b" if cookies2 else None) def test_user_set_cookie_domain_suffix_private(self): self._test_user_set_cookie_domain_followup( @@ -692,15 +688,12 @@ class CookiesMiddlewareTest(TestCase): "Set-Cookie": _cookies_to_set_cookie_list(input_cookies), } response = Response(url1, status=200, headers=headers) - self.assertEqual( - self.mw.process_response(request1, response, self.spider), - response, - ) + assert self.mw.process_response(request1, response, self.spider) == response request2 = Request(url2) self.mw.process_request(request2, self.spider) actual_cookies = request2.headers.get("Cookie") - self.assertEqual(actual_cookies, b"a=b" if cookies else None) + assert actual_cookies == (b"a=b" if cookies else None) def test_server_set_cookie_domain_suffix_private(self): self._test_server_set_cookie_domain_followup( @@ -752,30 +745,27 @@ class CookiesMiddlewareTest(TestCase): request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies) self.mw.process_request(request1, self.spider) cookies = request1.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies1 else None) + assert cookies == (b"a=b" if cookies1 else None) response = Response( f"{from_scheme}://a.example", headers={"Location": f"{to_scheme}://a.example"}, status=301, ) - self.assertEqual( - self.mw.process_response(request1, response, self.spider), - response, - ) + assert self.mw.process_response(request1, response, self.spider) == response request2 = self.redirect_middleware.process_response( request1, response, self.spider, ) - self.assertIsInstance(request2, Request) + assert isinstance(request2, Request) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies2 else None) + assert cookies == (b"a=b" if cookies2 else None) self.mw.process_request(request2, self.spider) cookies = request2.headers.get("Cookie") - self.assertEqual(cookies, b"a=b" if cookies3 else None) + assert cookies == (b"a=b" if cookies3 else None) def test_cookie_redirect_secure_undefined_downgrade(self): self._test_cookie_redirect_scheme_change( diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 27d6224b4..5716e3631 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request from scrapy.spiders import Spider @@ -7,7 +5,7 @@ from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler -class TestDefaultHeadersMiddleware(TestCase): +class TestDefaultHeadersMiddleware: def get_defaults_spider_mw(self): crawler = get_crawler(Spider) spider = crawler._create_spider("foo") @@ -21,15 +19,15 @@ class TestDefaultHeadersMiddleware(TestCase): defaults, spider, mw = self.get_defaults_spider_mw() req = Request("http://www.scrapytest.org") mw.process_request(req, spider) - self.assertEqual(req.headers, defaults) + assert req.headers == defaults def test_update_headers(self): defaults, spider, mw = self.get_defaults_spider_mw() headers = {"Accept-Language": ["es"], "Test-Header": ["test"]} bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]} req = Request("http://www.scrapytest.org", headers=headers) - self.assertEqual(req.headers, bytes_headers) + assert req.headers == bytes_headers mw.process_request(req, spider) defaults.update(bytes_headers) - self.assertEqual(req.headers, defaults) + assert req.headers == defaults diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 44458ade8..31323c8fa 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -1,12 +1,10 @@ -import unittest - from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class DownloadTimeoutMiddlewareTest(unittest.TestCase): +class TestDownloadTimeoutMiddleware: def get_request_spider_mw(self, settings=None): crawler = get_crawler(Spider, settings) spider = crawler._create_spider("foo") @@ -17,20 +15,20 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): req, spider, mw = self.get_request_spider_mw() mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get("download_timeout"), 180) + assert req.meta.get("download_timeout") == 180 def test_string_download_timeout(self): req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"}) mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get("download_timeout"), 20.1) + assert req.meta.get("download_timeout") == 20.1 def test_spider_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() spider.download_timeout = 2 mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get("download_timeout"), 2) + assert req.meta.get("download_timeout") == 2 def test_request_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() @@ -38,4 +36,4 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): mw.spider_opened(spider) req.meta["download_timeout"] = 1 assert mw.process_request(req, spider) is None - self.assertEqual(req.meta.get("download_timeout"), 1) + assert req.meta.get("download_timeout") == 1 diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 0f1489344..9154e1850 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,5 +1,3 @@ -import unittest - import pytest from w3lib.http import basic_auth_header @@ -25,8 +23,8 @@ class AnyDomainSpider(Spider): http_auth_domain = None -class HttpAuthMiddlewareLegacyTest(unittest.TestCase): - def setUp(self): +class TestHttpAuthMiddlewareLegacy: + def setup_method(self): self.spider = LegacySpider("foo") def test_auth(self): @@ -35,51 +33,51 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): mw.spider_opened(self.spider) -class HttpAuthMiddlewareTest(unittest.TestCase): - def setUp(self): +class TestHttpAuthMiddleware: + def setup_method(self): self.mw = HttpAuthMiddleware() self.spider = DomainSpider("foo") self.mw.spider_opened(self.spider) - def tearDown(self): + def teardown_method(self): del self.mw def test_no_auth(self): req = Request("http://example-noauth.com/") assert self.mw.process_request(req, self.spider) is None - self.assertNotIn("Authorization", req.headers) + assert "Authorization" not in req.headers def test_auth_domain(self): req = Request("http://example.com/") assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_subdomain(self): req = Request("http://foo.example.com/") assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_already_set(self): req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Digest 123") + assert req.headers["Authorization"] == b"Digest 123" -class HttpAuthAnyMiddlewareTest(unittest.TestCase): - def setUp(self): +class TestHttpAuthAnyMiddleware: + def setup_method(self): self.mw = HttpAuthMiddleware() self.spider = AnyDomainSpider("foo") self.mw.spider_opened(self.spider) - def tearDown(self): + def teardown_method(self): del self.mw def test_auth(self): req = Request("http://example.com/") assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_already_set(self): req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers["Authorization"], b"Digest 123") + assert req.headers["Authorization"] == b"Digest 123" diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index de3a9689b..5fac88ed7 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -2,7 +2,6 @@ import email.utils import shutil import tempfile import time -import unittest from contextlib import contextmanager import pytest @@ -15,11 +14,11 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class _BaseTest(unittest.TestCase): +class TestBase: storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" - def setUp(self): + def setup_method(self): self.yesterday = email.utils.formatdate(time.time() - 86400) self.today = email.utils.formatdate() self.tomorrow = email.utils.formatdate(time.time() + 86400) @@ -35,7 +34,7 @@ class _BaseTest(unittest.TestCase): ) self.crawler.stats.open_spider(self.spider) - def tearDown(self): + def teardown_method(self): self.crawler.stats.close_spider(self.spider, "") shutil.rmtree(self.tmpdir) @@ -72,44 +71,42 @@ class _BaseTest(unittest.TestCase): mw.spider_closed(self.spider) def assertEqualResponse(self, response1, response2): - self.assertEqual(response1.url, response2.url) - self.assertEqual(response1.status, response2.status) - self.assertEqual(response1.headers, response2.headers) - self.assertEqual(response1.body, response2.body) + assert response1.url == response2.url + assert response1.status == response2.status + assert response1.headers == response2.headers + assert response1.body == response2.body def assertEqualRequest(self, request1, request2): - self.assertEqual(request1.url, request2.url) - self.assertEqual(request1.headers, request2.headers) - self.assertEqual(request1.body, request2.body) + assert request1.url == request2.url + assert request1.headers == request2.headers + assert request1.body == request2.body def assertEqualRequestButWithCacheValidators(self, request1, request2): - self.assertEqual(request1.url, request2.url) + assert request1.url == request2.url assert b"If-None-Match" not in request1.headers assert b"If-Modified-Since" not in request1.headers assert any( h in request2.headers for h in (b"If-None-Match", b"If-Modified-Since") ) - self.assertEqual(request1.body, request2.body) + assert request1.body == request2.body def test_dont_cache(self): with self._middleware() as mw: self.request.meta["dont_cache"] = True mw.process_response(self.request, self.response, self.spider) - self.assertEqual( - mw.storage.retrieve_response(self.spider, self.request), None - ) + assert mw.storage.retrieve_response(self.spider, self.request) is None with self._middleware() as mw: self.request.meta["dont_cache"] = False mw.process_response(self.request, self.response, self.spider) if mw.policy.should_cache_response(self.response, self.request): - self.assertIsInstance( + assert isinstance( mw.storage.retrieve_response(self.spider, self.request), self.response.__class__, ) -class DefaultStorageTest(_BaseTest): +class TestDefaultStorage(TestBase): def test_storage(self): with self._storage() as storage: request2 = self.request.copy() @@ -142,15 +139,15 @@ class DefaultStorageTest(_BaseTest): ) storage.store_response(self.spider, self.request, response) cached_response = storage.retrieve_response(self.spider, self.request) - self.assertIsInstance(cached_response, HtmlResponse) + assert isinstance(cached_response, HtmlResponse) self.assertEqualResponse(response, cached_response) -class DbmStorageTest(DefaultStorageTest): +class TestDbmStorage(TestDefaultStorage): storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" -class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): +class TestDbmStorageWithCustomDbmModule(TestDbmStorage): dbm_module = "tests.mocks.dummydbm" def _get_settings(self, **new_settings): @@ -160,16 +157,16 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): def test_custom_dbm_module_loaded(self): # make sure our dbm module has been loaded with self._storage() as storage: - self.assertEqual(storage.dbmodule.__name__, self.dbm_module) + assert storage.dbmodule.__name__ == self.dbm_module -class FilesystemStorageGzipTest(DefaultStorageTest): +class TestFilesystemStorageGzip(TestDefaultStorage): def _get_settings(self, **new_settings): new_settings.setdefault("HTTPCACHE_GZIP", True) return super()._get_settings(**new_settings) -class DummyPolicyTest(_BaseTest): +class TestDummyPolicy(TestBase): policy_class = "scrapy.extensions.httpcache.DummyPolicy" def test_middleware(self): @@ -261,7 +258,7 @@ class DummyPolicyTest(_BaseTest): assert "cached" in response.flags -class RFC2616PolicyTest(DefaultStorageTest): +class TestRFC2616Policy(TestDefaultStorage): policy_class = "scrapy.extensions.httpcache.RFC2616Policy" def _process_requestresponse(self, mw, request, response): @@ -357,7 +354,7 @@ class RFC2616PolicyTest(DefaultStorageTest): assert "cached" in res2.flags assert res2.status != 304 else: - self.assertFalse(resc) + assert not resc assert "cached" not in res2.flags # cache unconditionally unless response contains no-store or is a 304 @@ -381,7 +378,7 @@ class RFC2616PolicyTest(DefaultStorageTest): assert "cached" in res2.flags assert res2.status != 304 else: - self.assertFalse(resc) + assert not resc assert "cached" not in res2.flags def test_cached_and_fresh(self): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index b3e3b98d7..e7427c5ac 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -2,7 +2,7 @@ from gzip import GzipFile from io import BytesIO from logging import WARNING from pathlib import Path -from unittest import SkipTest, TestCase +from unittest import SkipTest import pytest from testfixtures import LogCapture @@ -51,8 +51,8 @@ FORMAT = { } -class HttpCompressionTest(TestCase): - def setUp(self): +class TestHttpCompression: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("scrapytest.org") self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) @@ -81,10 +81,8 @@ class HttpCompressionTest(TestCase): return response def assertStatsEqual(self, key, value): - self.assertEqual( - self.crawler.stats.get_value(key, spider=self.spider), - value, - str(self.crawler.stats.get_stats(self.spider)), + assert self.crawler.stats.get_value(key, spider=self.spider) == value, str( + self.crawler.stats.get_stats(self.spider) ) def test_setting_false_compression_enabled(self): @@ -94,13 +92,13 @@ class HttpCompressionTest(TestCase): ) def test_setting_default_compression_enabled(self): - self.assertIsInstance( + assert isinstance( HttpCompressionMiddleware.from_crawler(get_crawler()), HttpCompressionMiddleware, ) def test_setting_true_compression_enabled(self): - self.assertIsInstance( + assert isinstance( HttpCompressionMiddleware.from_crawler( get_crawler(settings_dict={"COMPRESSION_ENABLED": True}) ), @@ -111,15 +109,13 @@ class HttpCompressionTest(TestCase): request = Request("http://scrapytest.org") assert "Accept-Encoding" not in request.headers self.mw.process_request(request, self.spider) - self.assertEqual( - request.headers.get("Accept-Encoding"), b", ".join(ACCEPTED_ENCODINGS) - ) + assert request.headers.get("Accept-Encoding") == b", ".join(ACCEPTED_ENCODINGS) def test_process_response_gzip(self): response = self._getresponse("gzip") request = response.request - self.assertEqual(response.headers["Content-Encoding"], b"gzip") + assert response.headers["Content-Encoding"] == b"gzip" newresponse = self.mw.process_response(request, response, self.spider) assert newresponse is not response assert newresponse.body.startswith(b"<!DOCTYPE") @@ -137,7 +133,7 @@ class HttpCompressionTest(TestCase): raise SkipTest("no brotli") response = self._getresponse("br") request = response.request - self.assertEqual(response.headers["Content-Encoding"], b"br") + assert response.headers["Content-Encoding"] == b"br" newresponse = self.mw.process_response(request, response, self.spider) assert newresponse is not response assert newresponse.body.startswith(b"<!DOCTYPE") @@ -159,7 +155,7 @@ class HttpCompressionTest(TestCase): pass response = self._getresponse("br") request = response.request - self.assertEqual(response.headers["Content-Encoding"], b"br") + assert response.headers["Content-Encoding"] == b"br" with LogCapture( "scrapy.downloadermiddlewares.httpcompression", propagate=False, @@ -178,7 +174,7 @@ class HttpCompressionTest(TestCase): ), ) assert newresponse is not response - self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"br"]) + assert newresponse.headers.getlist("Content-Encoding") == [b"br"] def test_process_response_zstd(self): try: @@ -191,7 +187,7 @@ class HttpCompressionTest(TestCase): continue response = self._getresponse(check_key) request = response.request - self.assertEqual(response.headers["Content-Encoding"], b"zstd") + assert response.headers["Content-Encoding"] == b"zstd" newresponse = self.mw.process_response(request, response, self.spider) if raw_content is None: raw_content = newresponse.body @@ -210,7 +206,7 @@ class HttpCompressionTest(TestCase): pass response = self._getresponse("zstd-static-content-size") request = response.request - self.assertEqual(response.headers["Content-Encoding"], b"zstd") + assert response.headers["Content-Encoding"] == b"zstd" with LogCapture( "scrapy.downloadermiddlewares.httpcompression", propagate=False, @@ -229,13 +225,13 @@ class HttpCompressionTest(TestCase): ), ) assert newresponse is not response - self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"zstd"]) + assert newresponse.headers.getlist("Content-Encoding") == [b"zstd"] def test_process_response_rawdeflate(self): response = self._getresponse("rawdeflate") request = response.request - self.assertEqual(response.headers["Content-Encoding"], b"deflate") + assert response.headers["Content-Encoding"] == b"deflate" newresponse = self.mw.process_response(request, response, self.spider) assert newresponse is not response assert newresponse.body.startswith(b"<!DOCTYPE") @@ -247,7 +243,7 @@ class HttpCompressionTest(TestCase): response = self._getresponse("zlibdeflate") request = response.request - self.assertEqual(response.headers["Content-Encoding"], b"deflate") + assert response.headers["Content-Encoding"] == b"deflate" newresponse = self.mw.process_response(request, response, self.spider) assert newresponse is not response assert newresponse.body.startswith(b"<!DOCTYPE") @@ -272,7 +268,7 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) assert newresponse is not response - self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"uuencode"]) + assert newresponse.headers.getlist("Content-Encoding") == [b"uuencode"] def test_multi_compression_single_header(self): response = self._getresponse("gzip-deflate") @@ -303,9 +299,7 @@ class HttpCompressionTest(TestCase): ), ) assert newresponse is not response - self.assertEqual( - newresponse.headers.getlist("Content-Encoding"), [b"gzip", b"foo"] - ) + assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"] def test_multi_compression_multiple_header(self): response = self._getresponse("gzip-deflate") @@ -322,9 +316,7 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) assert newresponse is not response - self.assertEqual( - newresponse.headers.getlist("Content-Encoding"), [b"gzip", b"foo"] - ) + assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"] def test_multi_compression_single_and_multiple_header(self): response = self._getresponse("gzip-deflate-gzip") @@ -341,9 +333,7 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) assert newresponse is not response - self.assertEqual( - newresponse.headers.getlist("Content-Encoding"), [b"gzip", b"foo"] - ) + assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"] def test_process_response_encoding_inside_body(self): headers = { @@ -365,8 +355,8 @@ class HttpCompressionTest(TestCase): newresponse = self.mw.process_response(request, response, self.spider) assert isinstance(newresponse, HtmlResponse) - self.assertEqual(newresponse.body, plainbody) - self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) + assert newresponse.body == plainbody + assert newresponse.encoding == resolve_encoding("gb2312") self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) @@ -390,8 +380,8 @@ class HttpCompressionTest(TestCase): newresponse = self.mw.process_response(request, response, self.spider) assert isinstance(newresponse, HtmlResponse) - self.assertEqual(newresponse.body, plainbody) - self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) + assert newresponse.body == plainbody + assert newresponse.encoding == resolve_encoding("gb2312") self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) @@ -413,8 +403,8 @@ class HttpCompressionTest(TestCase): newresponse = self.mw.process_response(request, response, self.spider) assert isinstance(newresponse, respcls) - self.assertEqual(newresponse.body, plainbody) - self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) + assert newresponse.body == plainbody + assert newresponse.encoding == resolve_encoding("gb2312") self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) @@ -424,9 +414,9 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) - self.assertIsNot(newresponse, response) - self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE")) - self.assertNotIn("Content-Encoding", newresponse.headers) + assert newresponse is not response + assert newresponse.body.startswith(b"<!DOCTYPE") + assert "Content-Encoding" not in newresponse.headers self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) @@ -436,9 +426,9 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) - self.assertIsNot(newresponse, response) - self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE")) - self.assertNotIn("Content-Encoding", newresponse.headers) + assert newresponse is not response + assert newresponse.body.startswith(b"<!DOCTYPE") + assert "Content-Encoding" not in newresponse.headers self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) @@ -448,9 +438,9 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) - self.assertIsNot(newresponse, response) - self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE")) - self.assertNotIn("Content-Encoding", newresponse.headers) + assert newresponse is not response + assert newresponse.body.startswith(b"<!DOCTYPE") + assert "Content-Encoding" not in newresponse.headers self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 74837) @@ -496,7 +486,7 @@ class HttpCompressionTest(TestCase): request = Request("http://www.example.com/") newresponse = self.mw.process_response(request, response, self.spider) - self.assertEqual(gunzip(newresponse.body), plainbody) + assert gunzip(newresponse.body) == plainbody self.assertStatsEqual("httpcompression/response_count", 1) self.assertStatsEqual("httpcompression/response_bytes", 230) @@ -507,8 +497,8 @@ class HttpCompressionTest(TestCase): request.method = "HEAD" response = response.replace(body=None) newresponse = self.mw.process_response(request, response, self.spider) - self.assertIs(newresponse, response) - self.assertEqual(response.body, b"") + assert newresponse is response + assert response.body == b"" self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index f0826ef5b..31d81e73d 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -1,7 +1,6 @@ import os import pytest -from twisted.trial.unittest import TestCase from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured @@ -12,13 +11,13 @@ from scrapy.utils.test import get_crawler spider = Spider("foo") -class TestHttpProxyMiddleware(TestCase): +class TestHttpProxyMiddleware: failureException = AssertionError # type: ignore[assignment] - def setUp(self): + def setup_method(self): self._oldenv = os.environ.copy() - def tearDown(self): + def teardown_method(self): os.environ = self._oldenv def test_not_enabled(self): @@ -33,8 +32,8 @@ class TestHttpProxyMiddleware(TestCase): for url in ("http://e.com", "https://e.com", "file:///tmp/a"): req = Request(url) assert mw.process_request(req, spider) is None - self.assertEqual(req.url, url) - self.assertEqual(req.meta, {}) + assert req.url == url + assert req.meta == {} def test_environment_proxies(self): os.environ["http_proxy"] = http_proxy = "https://proxy.for.http:3128" @@ -49,32 +48,32 @@ class TestHttpProxyMiddleware(TestCase): ]: req = Request(url) assert mw.process_request(req, spider) is None - self.assertEqual(req.url, url) - self.assertEqual(req.meta.get("proxy"), proxy) + assert req.url == url + assert req.meta.get("proxy") == proxy def test_proxy_precedence_meta(self): os.environ["http_proxy"] = "https://proxy.com" mw = HttpProxyMiddleware() req = Request("http://scrapytest.org", meta={"proxy": "https://new.proxy:3128"}) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {"proxy": "https://new.proxy:3128"}) + assert req.meta == {"proxy": "https://new.proxy:3128"} def test_proxy_auth(self): os.environ["http_proxy"] = "https://user:pass@proxy:3128" mw = HttpProxyMiddleware() req = Request("http://scrapytest.org") assert mw.process_request(req, spider) is None - self.assertEqual(req.meta["proxy"], "https://proxy:3128") - self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic dXNlcjpwYXNz") + assert req.meta["proxy"] == "https://proxy:3128" + assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcjpwYXNz" # proxy from request.meta req = Request( "http://scrapytest.org", meta={"proxy": "https://username:password@proxy:3128"}, ) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta["proxy"], "https://proxy:3128") - self.assertEqual( - req.headers.get("Proxy-Authorization"), b"Basic dXNlcm5hbWU6cGFzc3dvcmQ=" + assert req.meta["proxy"] == "https://proxy:3128" + assert ( + req.headers.get("Proxy-Authorization") == b"Basic dXNlcm5hbWU6cGFzc3dvcmQ=" ) def test_proxy_auth_empty_passwd(self): @@ -82,15 +81,15 @@ class TestHttpProxyMiddleware(TestCase): mw = HttpProxyMiddleware() req = Request("http://scrapytest.org") assert mw.process_request(req, spider) is None - self.assertEqual(req.meta["proxy"], "https://proxy:3128") - self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic dXNlcjo=") + assert req.meta["proxy"] == "https://proxy:3128" + assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcjo=" # proxy from request.meta req = Request( "http://scrapytest.org", meta={"proxy": "https://username:@proxy:3128"} ) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta["proxy"], "https://proxy:3128") - self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic dXNlcm5hbWU6") + assert req.meta["proxy"] == "https://proxy:3128" + assert req.headers.get("Proxy-Authorization") == b"Basic dXNlcm5hbWU6" def test_proxy_auth_encoding(self): # utf-8 encoding @@ -98,33 +97,31 @@ class TestHttpProxyMiddleware(TestCase): mw = HttpProxyMiddleware(auth_encoding="utf-8") req = Request("http://scrapytest.org") assert mw.process_request(req, spider) is None - self.assertEqual(req.meta["proxy"], "https://proxy:3128") - self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic bcOhbjpwYXNz") + assert req.meta["proxy"] == "https://proxy:3128" + assert req.headers.get("Proxy-Authorization") == b"Basic bcOhbjpwYXNz" # proxy from request.meta req = Request( "http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"} ) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta["proxy"], "https://proxy:3128") - self.assertEqual( - req.headers.get("Proxy-Authorization"), b"Basic w7xzZXI6cGFzcw==" - ) + assert req.meta["proxy"] == "https://proxy:3128" + assert req.headers.get("Proxy-Authorization") == b"Basic w7xzZXI6cGFzcw==" # default latin-1 encoding mw = HttpProxyMiddleware(auth_encoding="latin-1") req = Request("http://scrapytest.org") assert mw.process_request(req, spider) is None - self.assertEqual(req.meta["proxy"], "https://proxy:3128") - self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic beFuOnBhc3M=") + assert req.meta["proxy"] == "https://proxy:3128" + assert req.headers.get("Proxy-Authorization") == b"Basic beFuOnBhc3M=" # proxy from request.meta, latin-1 encoding req = Request( "http://scrapytest.org", meta={"proxy": "https://\u00fcser:pass@proxy:3128"} ) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta["proxy"], "https://proxy:3128") - self.assertEqual(req.headers.get("Proxy-Authorization"), b"Basic /HNlcjpwYXNz") + assert req.meta["proxy"] == "https://proxy:3128" + assert req.headers.get("Proxy-Authorization") == b"Basic /HNlcjpwYXNz" def test_proxy_already_seted(self): os.environ["http_proxy"] = "https://proxy.for.http:3128" @@ -157,7 +154,7 @@ class TestHttpProxyMiddleware(TestCase): os.environ["no_proxy"] = "*" req = Request("http://noproxy.com", meta={"proxy": "http://proxy.com"}) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {"proxy": "http://proxy.com"}) + assert req.meta == {"proxy": "http://proxy.com"} def test_no_proxy_invalid_values(self): os.environ["no_proxy"] = "/var/run/docker.sock" @@ -172,8 +169,8 @@ class TestHttpProxyMiddleware(TestCase): assert middleware.process_request(request, spider) is None request.meta["proxy"] = "https://example.com" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.com") - self.assertNotIn(b"Proxy-Authorization", request.headers) + assert request.meta["proxy"] == "https://example.com" + assert b"Proxy-Authorization" not in request.headers def test_add_proxy_with_credentials(self): middleware = HttpProxyMiddleware() @@ -181,15 +178,12 @@ class TestHttpProxyMiddleware(TestCase): assert middleware.process_request(request, spider) is None request.meta["proxy"] = "https://user1:password1@example.com" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.com") + assert request.meta["proxy"] == "https://example.com" encoded_credentials = middleware._basic_auth_header( "user1", "password1", ) - self.assertEqual( - request.headers["Proxy-Authorization"], - b"Basic " + encoded_credentials, - ) + assert request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials def test_remove_proxy_without_credentials(self): middleware = HttpProxyMiddleware() @@ -200,8 +194,8 @@ class TestHttpProxyMiddleware(TestCase): assert middleware.process_request(request, spider) is None request.meta["proxy"] = None assert middleware.process_request(request, spider) is None - self.assertIsNone(request.meta["proxy"]) - self.assertNotIn(b"Proxy-Authorization", request.headers) + assert request.meta["proxy"] is None + assert b"Proxy-Authorization" not in request.headers def test_remove_proxy_with_credentials(self): middleware = HttpProxyMiddleware() @@ -212,8 +206,8 @@ class TestHttpProxyMiddleware(TestCase): assert middleware.process_request(request, spider) is None request.meta["proxy"] = None assert middleware.process_request(request, spider) is None - self.assertIsNone(request.meta["proxy"]) - self.assertNotIn(b"Proxy-Authorization", request.headers) + assert request.meta["proxy"] is None + assert b"Proxy-Authorization" not in request.headers def test_add_credentials(self): """If the proxy request meta switches to a proxy URL with the same @@ -228,15 +222,12 @@ class TestHttpProxyMiddleware(TestCase): request.meta["proxy"] = "https://user1:password1@example.com" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.com") + assert request.meta["proxy"] == "https://example.com" encoded_credentials = middleware._basic_auth_header( "user1", "password1", ) - self.assertEqual( - request.headers["Proxy-Authorization"], - b"Basic " + encoded_credentials, - ) + assert request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials def test_change_credentials(self): """If the proxy request meta switches to a proxy URL with different @@ -249,15 +240,12 @@ class TestHttpProxyMiddleware(TestCase): assert middleware.process_request(request, spider) is None request.meta["proxy"] = "https://user2:password2@example.com" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.com") + assert request.meta["proxy"] == "https://example.com" encoded_credentials = middleware._basic_auth_header( "user2", "password2", ) - self.assertEqual( - request.headers["Proxy-Authorization"], - b"Basic " + encoded_credentials, - ) + assert request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials def test_remove_credentials(self): """If the proxy request meta switches to a proxy URL with the same @@ -276,21 +264,18 @@ class TestHttpProxyMiddleware(TestCase): request.meta["proxy"] = "https://example.com" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.com") + assert request.meta["proxy"] == "https://example.com" encoded_credentials = middleware._basic_auth_header( "user1", "password1", ) - self.assertEqual( - request.headers["Proxy-Authorization"], - b"Basic " + encoded_credentials, - ) + assert request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials request.meta["proxy"] = "https://example.com" del request.headers[b"Proxy-Authorization"] assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.com") - self.assertNotIn(b"Proxy-Authorization", request.headers) + assert request.meta["proxy"] == "https://example.com" + assert b"Proxy-Authorization" not in request.headers def test_change_proxy_add_credentials(self): middleware = HttpProxyMiddleware() @@ -302,15 +287,12 @@ class TestHttpProxyMiddleware(TestCase): request.meta["proxy"] = "https://user1:password1@example.org" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.org") + assert request.meta["proxy"] == "https://example.org" encoded_credentials = middleware._basic_auth_header( "user1", "password1", ) - self.assertEqual( - request.headers["Proxy-Authorization"], - b"Basic " + encoded_credentials, - ) + assert request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials def test_change_proxy_keep_credentials(self): middleware = HttpProxyMiddleware() @@ -322,21 +304,18 @@ class TestHttpProxyMiddleware(TestCase): request.meta["proxy"] = "https://user1:password1@example.org" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.org") + assert request.meta["proxy"] == "https://example.org" encoded_credentials = middleware._basic_auth_header( "user1", "password1", ) - self.assertEqual( - request.headers["Proxy-Authorization"], - b"Basic " + encoded_credentials, - ) + assert request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials # Make sure, indirectly, that _auth_proxy is updated. request.meta["proxy"] = "https://example.com" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.com") - self.assertNotIn(b"Proxy-Authorization", request.headers) + assert request.meta["proxy"] == "https://example.com" + assert b"Proxy-Authorization" not in request.headers def test_change_proxy_change_credentials(self): middleware = HttpProxyMiddleware() @@ -348,15 +327,12 @@ class TestHttpProxyMiddleware(TestCase): request.meta["proxy"] = "https://user2:password2@example.org" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.org") + assert request.meta["proxy"] == "https://example.org" encoded_credentials = middleware._basic_auth_header( "user2", "password2", ) - self.assertEqual( - request.headers["Proxy-Authorization"], - b"Basic " + encoded_credentials, - ) + assert request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials def test_change_proxy_remove_credentials(self): """If the proxy request meta switches to a proxy URL with a different @@ -369,8 +345,8 @@ class TestHttpProxyMiddleware(TestCase): assert middleware.process_request(request, spider) is None request.meta["proxy"] = "https://example.org" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta, {"proxy": "https://example.org"}) - self.assertNotIn(b"Proxy-Authorization", request.headers) + assert request.meta == {"proxy": "https://example.org"} + assert b"Proxy-Authorization" not in request.headers def test_change_proxy_remove_credentials_preremoved_header(self): """Corner case of proxy switch with credentials removal where the @@ -388,8 +364,8 @@ class TestHttpProxyMiddleware(TestCase): request.meta["proxy"] = "https://example.org" del request.headers[b"Proxy-Authorization"] assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta, {"proxy": "https://example.org"}) - self.assertNotIn(b"Proxy-Authorization", request.headers) + assert request.meta == {"proxy": "https://example.org"} + assert b"Proxy-Authorization" not in request.headers def test_proxy_authentication_header_undefined_proxy(self): middleware = HttpProxyMiddleware() @@ -398,8 +374,8 @@ class TestHttpProxyMiddleware(TestCase): headers={"Proxy-Authorization": "Basic foo"}, ) assert middleware.process_request(request, spider) is None - self.assertNotIn("proxy", request.meta) - self.assertNotIn(b"Proxy-Authorization", request.headers) + assert "proxy" not in request.meta + assert b"Proxy-Authorization" not in request.headers def test_proxy_authentication_header_disabled_proxy(self): middleware = HttpProxyMiddleware() @@ -409,8 +385,8 @@ class TestHttpProxyMiddleware(TestCase): meta={"proxy": None}, ) assert middleware.process_request(request, spider) is None - self.assertIsNone(request.meta["proxy"]) - self.assertNotIn(b"Proxy-Authorization", request.headers) + assert request.meta["proxy"] is None + assert b"Proxy-Authorization" not in request.headers def test_proxy_authentication_header_proxy_without_credentials(self): """As long as the proxy URL in request metadata remains the same, the @@ -423,17 +399,17 @@ class TestHttpProxyMiddleware(TestCase): meta={"proxy": "https://example.com"}, ) assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.com") - self.assertEqual(request.headers["Proxy-Authorization"], b"Basic foo") + assert request.meta["proxy"] == "https://example.com" + assert request.headers["Proxy-Authorization"] == b"Basic foo" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.com") - self.assertEqual(request.headers["Proxy-Authorization"], b"Basic foo") + assert request.meta["proxy"] == "https://example.com" + assert request.headers["Proxy-Authorization"] == b"Basic foo" request.headers["Proxy-Authorization"] = b"Basic bar" assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.com") - self.assertEqual(request.headers["Proxy-Authorization"], b"Basic bar") + assert request.meta["proxy"] == "https://example.com" + assert request.headers["Proxy-Authorization"] == b"Basic bar" def test_proxy_authentication_header_proxy_with_same_credentials(self): middleware = HttpProxyMiddleware() @@ -447,11 +423,8 @@ class TestHttpProxyMiddleware(TestCase): meta={"proxy": "https://user1:password1@example.com"}, ) assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.com") - self.assertEqual( - request.headers["Proxy-Authorization"], - b"Basic " + encoded_credentials, - ) + assert request.meta["proxy"] == "https://example.com" + assert request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials def test_proxy_authentication_header_proxy_with_different_credentials(self): middleware = HttpProxyMiddleware() @@ -465,12 +438,11 @@ class TestHttpProxyMiddleware(TestCase): meta={"proxy": "https://user2:password2@example.com"}, ) assert middleware.process_request(request, spider) is None - self.assertEqual(request.meta["proxy"], "https://example.com") + assert request.meta["proxy"] == "https://example.com" encoded_credentials2 = middleware._basic_auth_header( "user2", "password2", ) - self.assertEqual( - request.headers["Proxy-Authorization"], - b"Basic " + encoded_credentials2, + assert ( + request.headers["Proxy-Authorization"] == b"Basic " + encoded_credentials2 ) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 47abeee7a..a47459eda 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,4 +1,3 @@ -import unittest from itertools import chain, product import pytest @@ -16,12 +15,12 @@ from scrapy.utils.test import get_crawler class Base: - class Test(unittest.TestCase): + class Test: def test_priority_adjust(self): req = Request("http://a.com") rsp = self.get_response(req, "http://a.com/redirected") req2 = self.mw.process_response(req, rsp, self.spider) - self.assertGreater(req2.priority, req.priority) + assert req2.priority > req.priority def test_dont_redirect(self): url = "http://www.example.com/301" @@ -53,8 +52,8 @@ class Base: req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, "GET") + assert req2.url == url2 + assert req2.method == "GET" assert "Content-Type" not in req2.headers, ( "Content-Type header must not be present in redirected request" ) @@ -71,7 +70,7 @@ class Base: req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) assert "redirect_times" in req.meta - self.assertEqual(req.meta["redirect_times"], 1) + assert req.meta["redirect_times"] == 1 with pytest.raises(IgnoreRequest): self.mw.process_response(req, rsp, self.spider) @@ -92,15 +91,13 @@ class Base: rsp2 = self.get_response(req1, "/redirected2") req3 = self.mw.process_response(req2, rsp2, self.spider) - self.assertEqual(req2.url, "http://scrapytest.org/redirected") - self.assertEqual( - req2.meta["redirect_urls"], ["http://scrapytest.org/first"] - ) - self.assertEqual(req3.url, "http://scrapytest.org/redirected2") - self.assertEqual( - req3.meta["redirect_urls"], - ["http://scrapytest.org/first", "http://scrapytest.org/redirected"], - ) + assert req2.url == "http://scrapytest.org/redirected" + assert req2.meta["redirect_urls"] == ["http://scrapytest.org/first"] + assert req3.url == "http://scrapytest.org/redirected2" + assert req3.meta["redirect_urls"] == [ + "http://scrapytest.org/first", + "http://scrapytest.org/redirected", + ] def test_redirect_reasons(self): req1 = Request("http://scrapytest.org/first") @@ -108,8 +105,8 @@ class Base: req2 = self.mw.process_response(req1, rsp1, self.spider) rsp2 = self.get_response(req2, "/redirected2") req3 = self.mw.process_response(req2, rsp2, self.spider) - self.assertEqual(req2.meta["redirect_reasons"], [self.reason]) - self.assertEqual(req3.meta["redirect_reasons"], [self.reason, self.reason]) + assert req2.meta["redirect_reasons"] == [self.reason] + assert req3.meta["redirect_reasons"] == [self.reason, self.reason] def test_cross_origin_header_dropping(self): safe_headers = {"A": "B"} @@ -129,10 +126,8 @@ class Base: internal_redirect_request = self.mw.process_response( original_request, internal_response, self.spider ) - self.assertIsInstance(internal_redirect_request, Request) - self.assertEqual( - original_request.headers, internal_redirect_request.headers - ) + assert isinstance(internal_redirect_request, Request) + assert original_request.headers == internal_redirect_request.headers # Redirects to the same origin (same scheme, same domain, same port) # keep all headers also when the scheme is http. @@ -144,8 +139,8 @@ class Base: http_redirect_request = self.mw.process_response( http_request, http_response, self.spider ) - self.assertIsInstance(http_redirect_request, Request) - self.assertEqual(http_request.headers, http_redirect_request.headers) + assert isinstance(http_redirect_request, Request) + assert http_request.headers == http_redirect_request.headers # For default ports, whether the port is explicit or implicit does not # affect the outcome, it is still the same origin. @@ -155,10 +150,8 @@ class Base: to_explicit_port_redirect_request = self.mw.process_response( original_request, to_explicit_port_response, self.spider ) - self.assertIsInstance(to_explicit_port_redirect_request, Request) - self.assertEqual( - original_request.headers, to_explicit_port_redirect_request.headers - ) + assert isinstance(to_explicit_port_redirect_request, Request) + assert original_request.headers == to_explicit_port_redirect_request.headers # For default ports, whether the port is explicit or implicit does not # affect the outcome, it is still the same origin. @@ -168,10 +161,8 @@ class Base: to_implicit_port_redirect_request = self.mw.process_response( original_request, to_implicit_port_response, self.spider ) - self.assertIsInstance(to_implicit_port_redirect_request, Request) - self.assertEqual( - original_request.headers, to_implicit_port_redirect_request.headers - ) + assert isinstance(to_implicit_port_redirect_request, Request) + assert original_request.headers == to_implicit_port_redirect_request.headers # A port change drops the Authorization header because the origin # changes, but keeps the Cookie header because the domain remains the @@ -182,11 +173,11 @@ class Base: different_port_redirect_request = self.mw.process_response( original_request, different_port_response, self.spider ) - self.assertIsInstance(different_port_redirect_request, Request) - self.assertEqual( - {**safe_headers, **cookie_header}, - different_port_redirect_request.headers.to_unicode_dict(), - ) + assert isinstance(different_port_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == different_port_redirect_request.headers.to_unicode_dict() # A domain change drops both the Authorization and the Cookie header. external_response = self.get_response( @@ -195,10 +186,8 @@ class Base: external_redirect_request = self.mw.process_response( original_request, external_response, self.spider ) - self.assertIsInstance(external_redirect_request, Request) - self.assertEqual( - safe_headers, external_redirect_request.headers.to_unicode_dict() - ) + assert isinstance(external_redirect_request, Request) + assert safe_headers == external_redirect_request.headers.to_unicode_dict() # A scheme upgrade (http → https) drops the Authorization header # because the origin changes, but keeps the Cookie header because the @@ -207,11 +196,11 @@ class Base: upgrade_redirect_request = self.mw.process_response( http_request, upgrade_response, self.spider ) - self.assertIsInstance(upgrade_redirect_request, Request) - self.assertEqual( - {**safe_headers, **cookie_header}, - upgrade_redirect_request.headers.to_unicode_dict(), - ) + assert isinstance(upgrade_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == upgrade_redirect_request.headers.to_unicode_dict() # A scheme downgrade (https → http) drops the Authorization header # because the origin changes, and the Cookie header because its value @@ -228,11 +217,8 @@ class Base: downgrade_redirect_request = self.mw.process_response( original_request, downgrade_response, self.spider ) - self.assertIsInstance(downgrade_redirect_request, Request) - self.assertEqual( - safe_headers, - downgrade_redirect_request.headers.to_unicode_dict(), - ) + assert isinstance(downgrade_redirect_request, Request) + assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() def test_meta_proxy_http_absolute(self): crawler = get_crawler() @@ -244,37 +230,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_meta_proxy_http_relative(self): crawler = get_crawler() @@ -286,37 +272,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "/a") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "/a") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_meta_proxy_https_absolute(self): crawler = get_crawler() @@ -328,37 +314,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_meta_proxy_https_relative(self): crawler = get_crawler() @@ -370,37 +356,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "/a") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "/a") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_meta_proxy_http_to_https(self): crawler = get_crawler() @@ -412,37 +398,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_meta_proxy_https_to_http(self): crawler = get_crawler() @@ -454,37 +440,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_http_absolute(self): crawler = get_crawler() @@ -499,37 +485,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_http_relative(self): crawler = get_crawler() @@ -544,37 +530,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "/a") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "/a") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_https_absolute(self): crawler = get_crawler() @@ -589,37 +575,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_https_relative(self): crawler = get_crawler() @@ -634,37 +620,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "/a") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "/a") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_proxied_http_to_proxied_https(self): crawler = get_crawler() @@ -680,37 +666,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic Yjo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://b.example") - self.assertEqual(request2.meta["proxy"], "https://b.example") + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_proxied_http_to_unproxied_https(self): crawler = get_crawler() @@ -725,37 +711,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request1.meta["proxy"], "https://a.example") + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request3.meta["proxy"], "https://a.example") + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" def test_system_proxy_unproxied_http_to_proxied_https(self): crawler = get_crawler() @@ -770,37 +756,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertNotIn("Proxy-Authorization", request1.headers) - self.assertNotIn("_auth_proxy", request1.meta) - self.assertNotIn("proxy", request1.meta) + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic Yjo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://b.example") - self.assertEqual(request2.meta["proxy"], "https://b.example") + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta def test_system_proxy_unproxied_http_to_unproxied_https(self): crawler = get_crawler() @@ -811,37 +797,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertNotIn("Proxy-Authorization", request1.headers) - self.assertNotIn("_auth_proxy", request1.meta) - self.assertNotIn("proxy", request1.meta) + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta response1 = self.get_response(request1, "https://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta response2 = self.get_response(request2, "http://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta def test_system_proxy_proxied_https_to_proxied_http(self): crawler = get_crawler() @@ -857,37 +843,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic Yjo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://b.example") - self.assertEqual(request1.meta["proxy"], "https://b.example") + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic Yjo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://b.example") - self.assertEqual(request3.meta["proxy"], "https://b.example") + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" def test_system_proxy_proxied_https_to_unproxied_http(self): crawler = get_crawler() @@ -902,37 +888,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertEqual(request1.headers["Proxy-Authorization"], b"Basic Yjo=") - self.assertEqual(request1.meta["_auth_proxy"], "https://b.example") - self.assertEqual(request1.meta["proxy"], "https://b.example") + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertEqual(request3.headers["Proxy-Authorization"], b"Basic Yjo=") - self.assertEqual(request3.meta["_auth_proxy"], "https://b.example") - self.assertEqual(request3.meta["proxy"], "https://b.example") + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" def test_system_proxy_unproxied_https_to_proxied_http(self): crawler = get_crawler() @@ -947,37 +933,37 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertNotIn("Proxy-Authorization", request1.headers) - self.assertNotIn("_auth_proxy", request1.meta) - self.assertNotIn("proxy", request1.meta) + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertEqual(request2.headers["Proxy-Authorization"], b"Basic YTo=") - self.assertEqual(request2.meta["_auth_proxy"], "https://a.example") - self.assertEqual(request2.meta["proxy"], "https://a.example") + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta def test_system_proxy_unproxied_https_to_unproxied_http(self): crawler = get_crawler() @@ -988,44 +974,44 @@ class Base: spider = None proxy_mw.process_request(request1, spider) - self.assertNotIn("Proxy-Authorization", request1.headers) - self.assertNotIn("_auth_proxy", request1.meta) - self.assertNotIn("proxy", request1.meta) + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta response1 = self.get_response(request1, "http://example.com") request2 = redirect_mw.process_response(request1, response1, spider) - self.assertIsInstance(request2, Request) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta proxy_mw.process_request(request2, spider) - self.assertNotIn("Proxy-Authorization", request2.headers) - self.assertNotIn("_auth_proxy", request2.meta) - self.assertNotIn("proxy", request2.meta) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta response2 = self.get_response(request2, "https://example.com") request3 = redirect_mw.process_response(request2, response2, spider) - self.assertIsInstance(request3, Request) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta proxy_mw.process_request(request3, spider) - self.assertNotIn("Proxy-Authorization", request3.headers) - self.assertNotIn("_auth_proxy", request3.meta) - self.assertNotIn("proxy", request3.meta) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta -class RedirectMiddlewareTest(Base.Test): +class TestRedirectMiddleware(Base.Test): mwcls = RedirectMiddleware reason = 302 - def setUp(self): + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") self.mw = self.mwcls.from_crawler(self.crawler) @@ -1043,8 +1029,8 @@ class RedirectMiddlewareTest(Base.Test): req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, method) + assert req2.url == url2 + assert req2.method == method # response without Location header but with status code is 3XX should be ignored del rsp.headers["Location"] @@ -1070,8 +1056,8 @@ class RedirectMiddlewareTest(Base.Test): req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) - self.assertEqual(req2.method, "HEAD") + assert req2.url == url2 + assert req2.method == "HEAD" def test_redirect_302_relative(self): url = "http://www.example.com/302" @@ -1082,8 +1068,8 @@ class RedirectMiddlewareTest(Base.Test): req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, url3) - self.assertEqual(req2.method, "HEAD") + assert req2.url == url3 + assert req2.method == "HEAD" def test_spider_handling(self): smartspider = self.crawler._create_spider("smarty") @@ -1093,7 +1079,7 @@ class RedirectMiddlewareTest(Base.Test): req = Request(url) rsp = Response(url, headers={"Location": url2}, status=301) r = self.mw.process_response(req, rsp, smartspider) - self.assertIs(r, rsp) + assert r is rsp def test_request_meta_handling(self): url = "http://www.example.com/301" @@ -1102,7 +1088,7 @@ class RedirectMiddlewareTest(Base.Test): def _test_passthrough(req): rsp = Response(url, headers={"Location": url2}, status=301, request=req) r = self.mw.process_response(req, rsp, self.spider) - self.assertIs(r, rsp) + assert r is rsp _test_passthrough( Request(url, meta={"handle_httpstatus_list": [404, 301, 302]}) @@ -1119,7 +1105,7 @@ class RedirectMiddlewareTest(Base.Test): ) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = "http://scrapytest.org/a%E7%E3o" - self.assertEqual(perc_encoded_utf8_url, req_result.url) + assert perc_encoded_utf8_url == req_result.url def test_utf8_location(self): req = Request("http://scrapytest.org/first") @@ -1131,7 +1117,7 @@ class RedirectMiddlewareTest(Base.Test): ) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = "http://scrapytest.org/a%C3%A7%C3%A3o" - self.assertEqual(perc_encoded_utf8_url, req_result.url) + assert perc_encoded_utf8_url == req_result.url def test_no_location(self): request = Request("https://example.com") @@ -1197,11 +1183,11 @@ def meta_refresh_body(url, interval=5): return html.encode("utf-8") -class MetaRefreshMiddlewareTest(Base.Test): +class TestMetaRefreshMiddleware(Base.Test): mwcls = MetaRefreshMiddleware reason = "meta refresh" - def setUp(self): + def setup_method(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider("foo") self.mw = self.mwcls.from_crawler(crawler) @@ -1217,7 +1203,7 @@ class MetaRefreshMiddlewareTest(Base.Test): rsp = HtmlResponse(req.url, body=self._body()) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, "http://example.org/newpage") + assert req2.url == "http://example.org/newpage" def test_meta_refresh_with_high_interval(self): # meta-refresh with high intervals don't trigger redirects @@ -1239,8 +1225,8 @@ class MetaRefreshMiddlewareTest(Base.Test): req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) - self.assertEqual(req2.url, "http://example.org/newpage") - self.assertEqual(req2.method, "GET") + assert req2.url == "http://example.org/newpage" + assert req2.method == "GET" assert "Content-Type" not in req2.headers, ( "Content-Type header must not be present in redirected request" ) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 36f48db69..ffdcdf49e 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,5 +1,4 @@ import logging -import unittest import pytest from testfixtures import LogCapture @@ -21,8 +20,8 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class RetryTest(unittest.TestCase): - def setUp(self): +class TestRetry: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") self.mw = RetryMiddleware.from_crawler(self.crawler) @@ -70,12 +69,12 @@ class RetryTest(unittest.TestCase): # first retry req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 1) + assert req.meta["retry_times"] == 1 # second retry req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 2) + assert req.meta["retry_times"] == 2 # discard it assert self.mw.process_response(req, rsp, self.spider) is rsp @@ -129,19 +128,19 @@ class RetryTest(unittest.TestCase): # first retry req = mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 1) + assert req.meta["retry_times"] == 1 # second retry req = mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) - self.assertEqual(req.meta["retry_times"], 2) + assert req.meta["retry_times"] == 2 # discard it req = mw.process_exception(req, exception, self.spider) - self.assertEqual(req, None) + assert req is None -class MaxRetryTimesTest(unittest.TestCase): +class TestMaxRetryTimes: invalid_url = "http://www.scrapytest.org/invalid_url" def get_spider_and_middleware(self, settings=None): @@ -272,10 +271,10 @@ class MaxRetryTimesTest(unittest.TestCase): # discard it req = middleware.process_exception(req, exception, spider) - self.assertEqual(req, None) + assert req is None -class GetRetryRequestTest(unittest.TestCase): +class TestGetRetryRequest: def get_spider(self, settings=None): crawler = get_crawler(Spider, settings or {}) return crawler._create_spider("foo") @@ -288,15 +287,15 @@ class GetRetryRequestTest(unittest.TestCase): request, spider=spider, ) - self.assertIsInstance(new_request, Request) - self.assertNotEqual(new_request, request) - self.assertEqual(new_request.dont_filter, True) + assert isinstance(new_request, Request) + assert new_request != request + assert new_request.dont_filter expected_retry_times = 1 - self.assertEqual(new_request.meta["retry_times"], expected_retry_times) - self.assertEqual(new_request.priority, -1) + assert new_request.meta["retry_times"] == expected_retry_times + assert new_request.priority == -1 expected_reason = "unspecified" for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): - self.assertEqual(spider.crawler.stats.get_value(stat), 1) + assert spider.crawler.stats.get_value(stat) == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -316,8 +315,8 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=max_retry_times, ) - self.assertEqual(new_request, None) - self.assertEqual(spider.crawler.stats.get_value("retry/max_reached"), 1) + assert new_request is None + assert spider.crawler.stats.get_value("retry/max_reached") == 1 failure_count = max_retry_times + 1 expected_reason = "unspecified" log.check_present( @@ -338,15 +337,15 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=1, ) - self.assertIsInstance(new_request, Request) - self.assertNotEqual(new_request, request) - self.assertEqual(new_request.dont_filter, True) + assert isinstance(new_request, Request) + assert new_request != request + assert new_request.dont_filter expected_retry_times = 1 - self.assertEqual(new_request.meta["retry_times"], expected_retry_times) - self.assertEqual(new_request.priority, -1) + assert new_request.meta["retry_times"] == expected_retry_times + assert new_request.priority == -1 expected_reason = "unspecified" for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): - self.assertEqual(spider.crawler.stats.get_value(stat), 1) + assert spider.crawler.stats.get_value(stat) == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -368,16 +367,16 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=max_retry_times, ) - self.assertIsInstance(new_request, Request) - self.assertNotEqual(new_request, request) - self.assertEqual(new_request.dont_filter, True) + assert isinstance(new_request, Request) + assert new_request != request + assert new_request.dont_filter expected_retry_times = index + 1 - self.assertEqual(new_request.meta["retry_times"], expected_retry_times) - self.assertEqual(new_request.priority, -expected_retry_times) + assert new_request.meta["retry_times"] == expected_retry_times + assert new_request.priority == -expected_retry_times expected_reason = "unspecified" for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): value = spider.crawler.stats.get_value(stat) - self.assertEqual(value, expected_retry_times) + assert value == expected_retry_times log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -393,8 +392,8 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=max_retry_times, ) - self.assertEqual(new_request, None) - self.assertEqual(spider.crawler.stats.get_value("retry/max_reached"), 1) + assert new_request is None + assert spider.crawler.stats.get_value("retry/max_reached") == 1 failure_count = max_retry_times + 1 expected_reason = "unspecified" log.check_present( @@ -419,7 +418,7 @@ class GetRetryRequestTest(unittest.TestCase): request, spider=spider, ) - self.assertEqual(new_request, None) + assert new_request is None def test_max_retry_times_meta(self): max_retry_times = 0 @@ -430,7 +429,7 @@ class GetRetryRequestTest(unittest.TestCase): request, spider=spider, ) - self.assertEqual(new_request, None) + assert new_request is None def test_max_retry_times_argument(self): max_retry_times = 0 @@ -442,7 +441,7 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, max_retry_times=max_retry_times, ) - self.assertEqual(new_request, None) + assert new_request is None def test_priority_adjust_setting(self): priority_adjust = 1 @@ -452,7 +451,7 @@ class GetRetryRequestTest(unittest.TestCase): request, spider=spider, ) - self.assertEqual(new_request.priority, priority_adjust) + assert new_request.priority == priority_adjust def test_priority_adjust_argument(self): priority_adjust = 1 @@ -463,7 +462,7 @@ class GetRetryRequestTest(unittest.TestCase): spider=spider, priority_adjust=priority_adjust, ) - self.assertEqual(new_request.priority, priority_adjust) + assert new_request.priority == priority_adjust def test_log_extra_retry_success(self): request = Request("https://example.com") @@ -498,7 +497,7 @@ class GetRetryRequestTest(unittest.TestCase): ) expected_retry_times = 1 for stat in ("retry/count", f"retry/reason_count/{expected_reason}"): - self.assertEqual(spider.crawler.stats.get_value(stat), 1) + assert spider.crawler.stats.get_value(stat) == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -523,7 +522,7 @@ class GetRetryRequestTest(unittest.TestCase): stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) - self.assertEqual(stat, 1) + assert stat == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -548,7 +547,7 @@ class GetRetryRequestTest(unittest.TestCase): stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) - self.assertEqual(stat, 1) + assert stat == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -573,7 +572,7 @@ class GetRetryRequestTest(unittest.TestCase): stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) - self.assertEqual(stat, 1) + assert stat == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -598,7 +597,7 @@ class GetRetryRequestTest(unittest.TestCase): stat = spider.crawler.stats.get_value( f"retry/reason_count/{expected_reason_string}" ) - self.assertEqual(stat, 1) + assert stat == 1 log.check_present( ( "scrapy.downloadermiddlewares.retry", @@ -643,4 +642,4 @@ class GetRetryRequestTest(unittest.TestCase): f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}", ): - self.assertEqual(spider.crawler.stats.get_value(stat), 1) + assert spider.crawler.stats.get_value(stat) == 1 diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 9b95400fd..38f0333bb 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -15,7 +15,7 @@ from scrapy.settings import Settings from tests.test_robotstxt_interface import rerp_available -class RobotsTxtMiddlewareTest(unittest.TestCase): +class TestRobotsTxtMiddleware(unittest.TestCase): def setUp(self): self.crawler = mock.MagicMock() self.crawler.settings = Settings() @@ -242,11 +242,11 @@ Disallow: /some/randome/page.html def assertRobotsTxtRequested(self, base_url): calls = self.crawler.engine.download.call_args_list request = calls[0][0][0] - self.assertEqual(request.url, f"{base_url}/robots.txt") - self.assertEqual(request.callback, NO_CALLBACK) + assert request.url == f"{base_url}/robots.txt" + assert request.callback == NO_CALLBACK -class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): +class TestRobotsTxtMiddlewareWithRerp(TestRobotsTxtMiddleware): if not rerp_available(): skip = "Rerp parser is not installed" diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 5b7181848..748ef7d76 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.http import Request, Response from scrapy.spiders import Spider @@ -10,8 +8,8 @@ class MyException(Exception): pass -class TestDownloaderStats(TestCase): - def setUp(self): +class TestDownloaderStats: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("scrapytest.org") self.mw = DownloaderStats(self.crawler.stats) @@ -22,10 +20,8 @@ class TestDownloaderStats(TestCase): self.res = Response("scrapytest.org", status=400) def assertStatsEqual(self, key, value): - self.assertEqual( - self.crawler.stats.get_value(key, spider=self.spider), - value, - str(self.crawler.stats.get_stats(self.spider)), + assert self.crawler.stats.get_value(key, spider=self.spider) == value, str( + self.crawler.stats.get_stats(self.spider) ) def test_process_request(self): @@ -44,5 +40,5 @@ class TestDownloaderStats(TestCase): 1, ) - def tearDown(self): + def teardown_method(self): self.crawler.stats.close_spider(self.spider, "") diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index cad3dea5c..1497f8c67 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -1,12 +1,10 @@ -from unittest import TestCase - from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class UserAgentMiddlewareTest(TestCase): +class TestUserAgentMiddleware: def get_spider_and_mw(self, default_useragent): crawler = get_crawler(Spider, {"USER_AGENT": default_useragent}) spider = crawler._create_spider("foo") @@ -16,7 +14,7 @@ class UserAgentMiddlewareTest(TestCase): spider, mw = self.get_spider_and_mw("default_useragent") req = Request("http://scrapytest.org/") assert mw.process_request(req, spider) is None - self.assertEqual(req.headers["User-Agent"], b"default_useragent") + assert req.headers["User-Agent"] == b"default_useragent" def test_remove_agent(self): # settings USER_AGENT to None should remove the user agent @@ -33,7 +31,7 @@ class UserAgentMiddlewareTest(TestCase): mw.spider_opened(spider) req = Request("http://scrapytest.org/") assert mw.process_request(req, spider) is None - self.assertEqual(req.headers["User-Agent"], b"spider_useragent") + assert req.headers["User-Agent"] == b"spider_useragent" def test_header_agent(self): spider, mw = self.get_spider_and_mw("default_useragent") @@ -43,7 +41,7 @@ class UserAgentMiddlewareTest(TestCase): "http://scrapytest.org/", headers={"User-Agent": "header_useragent"} ) assert mw.process_request(req, spider) is None - self.assertEqual(req.headers["User-Agent"], b"header_useragent") + assert req.headers["User-Agent"] == b"header_useragent" def test_no_agent(self): spider, mw = self.get_spider_and_mw(None) From 5a605969bdc102e0193ad15ccc571dc6164e5d26 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 6 Mar 2025 23:52:41 +0400 Subject: [PATCH 1673/2083] Converting tests to plain asserts, part 2. (#6699) --- tests/test_addons.py | 48 +-- tests/test_closespider.py | 28 +- tests/test_cmdline/__init__.py | 30 +- .../__init__.py | 7 +- tests/test_command_check.py | 14 +- tests/test_command_fetch.py | 10 +- tests/test_command_parse.py | 81 ++-- tests/test_command_shell.py | 28 +- tests/test_command_version.py | 36 +- tests/test_commands.py | 371 +++++++++--------- tests/test_contracts.py | 112 +++--- tests/test_core_downloader.py | 25 +- tests/test_crawl.py | 288 +++++++------- tests/test_crawler.py | 351 +++++++++-------- tests/test_dependencies.py | 7 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_dupefilters.py | 21 +- tests/test_engine.py | 137 ++++--- tests/test_engine_stop_download_bytes.py | 13 +- tests/test_engine_stop_download_headers.py | 12 +- tests/test_webclient.py | 40 +- 21 files changed, 796 insertions(+), 865 deletions(-) diff --git a/tests/test_addons.py b/tests/test_addons.py index a0caa3511..686bf9952 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -39,7 +39,7 @@ class CreateInstanceAddon: settings.update(self.config, "addon") -class AddonTest(unittest.TestCase): +class TestAddon: def test_update_settings(self): settings = BaseSettings() settings.set("KEY1", "default", priority="default") @@ -47,19 +47,19 @@ class AddonTest(unittest.TestCase): addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"} testaddon = get_addon_cls(addon_config)() testaddon.update_settings(settings) - self.assertEqual(settings["KEY1"], "addon") - self.assertEqual(settings["KEY2"], "project") - self.assertEqual(settings["KEY3"], "addon") + assert settings["KEY1"] == "addon" + assert settings["KEY2"] == "project" + assert settings["KEY3"] == "addon" -class AddonManagerTest(unittest.TestCase): +class TestAddonManager(unittest.TestCase): def test_load_settings(self): settings_dict = { "ADDONS": {"tests.test_addons.SimpleAddon": 0}, } crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons - self.assertIsInstance(manager.addons[0], SimpleAddon) + assert isinstance(manager.addons[0], SimpleAddon) def test_notconfigured(self): class NotConfiguredAddon: @@ -71,7 +71,7 @@ class AddonManagerTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons - self.assertFalse(manager.addons) + assert not manager.addons def test_load_settings_order(self): # Get three addons with different settings @@ -86,8 +86,8 @@ class AddonManagerTest(unittest.TestCase): settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} crawler = get_crawler(settings_dict=settings) manager = crawler.addons - self.assertEqual([a.number for a in manager.addons], expected_order) - self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) + assert [a.number for a in manager.addons] == expected_order + assert crawler.settings.getint("KEY1") == expected_order[-1] def test_build_from_crawler(self): settings_dict = { @@ -96,8 +96,8 @@ class AddonManagerTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons - self.assertIsInstance(manager.addons[0], CreateInstanceAddon) - self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") + assert isinstance(manager.addons[0], CreateInstanceAddon) + assert crawler.settings.get("MYADDON_KEY") == "val" def test_settings_priority(self): config = { @@ -107,14 +107,14 @@ class AddonManagerTest(unittest.TestCase): "ADDONS": {get_addon_cls(config): 1}, } crawler = get_crawler(settings_dict=settings_dict) - self.assertEqual(crawler.settings.getint("KEY"), 15) + assert crawler.settings.getint("KEY") == 15 settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") runner = CrawlerRunner(settings) crawler = runner.create_crawler(Spider) crawler._apply_settings() - self.assertEqual(crawler.settings.getint("KEY"), 15) + assert crawler.settings.getint("KEY") == 15 settings_dict = { "KEY": 20, # priority=project @@ -124,7 +124,7 @@ class AddonManagerTest(unittest.TestCase): settings.set("KEY", 0, priority="default") runner = CrawlerRunner(settings) crawler = runner.create_crawler(Spider) - self.assertEqual(crawler.settings.getint("KEY"), 20) + assert crawler.settings.getint("KEY") == 20 def test_fallback_workflow(self): FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" @@ -143,12 +143,12 @@ class AddonManagerTest(unittest.TestCase): "ADDONS": {AddonWithFallback: 1}, } crawler = get_crawler(settings_dict=settings_dict) - self.assertEqual( - crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + assert ( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"] == "AddonHandler" ) - self.assertEqual( - crawler.settings.get(FALLBACK_SETTING), - "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + assert ( + crawler.settings.get(FALLBACK_SETTING) + == "scrapy.core.downloader.handlers.http.HTTPDownloadHandler" ) settings_dict = { @@ -156,10 +156,10 @@ class AddonManagerTest(unittest.TestCase): "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, } crawler = get_crawler(settings_dict=settings_dict) - self.assertEqual( - crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + assert ( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"] == "AddonHandler" ) - self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") + assert crawler.settings.get(FALLBACK_SETTING) == "UserHandler" def test_logging_message(self): class LoggedAddon: @@ -199,6 +199,6 @@ class AddonManagerTest(unittest.TestCase): settings.set("KEY", "default", priority="default") runner = CrawlerRunner(settings) crawler = runner.create_crawler(MySpider) - self.assertEqual(crawler.settings.get("KEY"), "default") + assert crawler.settings.get("KEY") == "default" yield crawler.crawl() - self.assertEqual(crawler.settings.get("KEY"), "addon") + assert crawler.settings.get("KEY") == "addon" diff --git a/tests/test_closespider.py b/tests/test_closespider.py index ecde301d1..476662789 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -28,9 +28,9 @@ class TestCloseSpider(TestCase): crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_itemcount") + assert reason == "closespider_itemcount" itemcount = crawler.stats.get_value("item_scraped_count") - self.assertTrue(itemcount >= close_on) + assert itemcount >= close_on @defer.inlineCallbacks def test_closespider_pagecount(self): @@ -38,9 +38,9 @@ class TestCloseSpider(TestCase): crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on}) yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_pagecount") + assert reason == "closespider_pagecount" pagecount = crawler.stats.get_value("response_received_count") - self.assertTrue(pagecount >= close_on) + assert pagecount >= close_on @defer.inlineCallbacks def test_closespider_pagecount_no_item(self): @@ -57,10 +57,10 @@ class TestCloseSpider(TestCase): max_items=max_items, max_requests=max_requests, mockserver=self.mockserver ) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_pagecount_no_item") + assert reason == "closespider_pagecount_no_item" pagecount = crawler.stats.get_value("response_received_count") itemcount = crawler.stats.get_value("item_scraped_count") - self.assertLessEqual(pagecount, close_on + itemcount) + assert pagecount <= close_on + itemcount @defer.inlineCallbacks def test_closespider_pagecount_no_item_with_pagecount(self): @@ -75,9 +75,9 @@ class TestCloseSpider(TestCase): ) yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_pagecount_no_item") + assert reason == "closespider_pagecount_no_item" pagecount = crawler.stats.get_value("response_received_count") - self.assertLess(pagecount, close_on_pagecount) + assert pagecount < close_on_pagecount @defer.inlineCallbacks def test_closespider_errorcount(self): @@ -85,10 +85,10 @@ class TestCloseSpider(TestCase): crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_errorcount") + assert reason == "closespider_errorcount" key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}" errorcount = crawler.stats.get_value(key) - self.assertTrue(errorcount >= close_on) + assert errorcount >= close_on @defer.inlineCallbacks def test_closespider_timeout(self): @@ -96,9 +96,9 @@ class TestCloseSpider(TestCase): crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on}) yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_timeout") + assert reason == "closespider_timeout" total_seconds = crawler.stats.get_value("elapsed_time_seconds") - self.assertTrue(total_seconds >= close_on) + assert total_seconds >= close_on @defer.inlineCallbacks def test_closespider_timeout_no_item(self): @@ -106,6 +106,6 @@ class TestCloseSpider(TestCase): crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) yield crawler.crawl(n=3, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] - self.assertEqual(reason, "closespider_timeout_no_item") + assert reason == "closespider_timeout_no_item" total_seconds = crawler.stats.get_value("elapsed_time_seconds") - self.assertTrue(total_seconds >= timeout) + assert total_seconds >= timeout diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index acd524ea4..98a85bc17 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -4,7 +4,6 @@ import pstats import shutil import sys import tempfile -import unittest from io import StringIO from pathlib import Path from subprocess import PIPE, Popen @@ -12,8 +11,8 @@ from subprocess import PIPE, Popen from scrapy.utils.test import get_testenv -class CmdlineTest(unittest.TestCase): - def setUp(self): +class TestCmdline: + def setup_method(self): self.env = get_testenv() tests_path = Path(__file__).parent.parent self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent) @@ -27,12 +26,12 @@ class CmdlineTest(unittest.TestCase): return comm.decode(encoding) def test_default_settings(self): - self.assertEqual(self._execute("settings", "--get", "TEST1"), "default") + assert self._execute("settings", "--get", "TEST1") == "default" def test_override_settings_using_set_arg(self): - self.assertEqual( - self._execute("settings", "--get", "TEST1", "-s", "TEST1=override"), - "override", + assert ( + self._execute("settings", "--get", "TEST1", "-s", "TEST1=override") + == "override" ) def test_profiling(self): @@ -40,14 +39,14 @@ class CmdlineTest(unittest.TestCase): filename = path / "res.prof" try: self._execute("version", "--profile", str(filename)) - self.assertTrue(filename.exists()) + assert filename.exists() out = StringIO() stats = pstats.Stats(str(filename), stream=out) stats.print_stats() out.seek(0) stats = out.read() - self.assertIn(str(Path("scrapy", "commands", "version.py")), stats) - self.assertIn("tottime", stats) + assert str(Path("scrapy", "commands", "version.py")) in stats + assert "tottime" in stats finally: shutil.rmtree(path) @@ -62,15 +61,14 @@ class CmdlineTest(unittest.TestCase): "EXTENSIONS=" + json.dumps(EXTENSIONS), ) # XXX: There's gotta be a smarter way to do this... - self.assertNotIn("...", settingsstr) + assert "..." not in settingsstr for char in ("'", "<", ">"): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) - self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) - self.assertEqual(200, settingsdict[EXT_PATH]) + assert set(settingsdict.keys()) == set(EXTENSIONS.keys()) + assert settingsdict[EXT_PATH] == 200 def test_pathlib_path_as_feeds_key(self): - self.assertEqual( - self._execute("settings", "--get", "FEEDS"), - json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}}), + assert self._execute("settings", "--get", "FEEDS") == json.dumps( + {"items.csv": {"format": "csv", "fields": ["price", "name"]}} ) diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index 5cb09b5c0..5228f6abd 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -1,10 +1,9 @@ import sys -import unittest from pathlib import Path from subprocess import PIPE, Popen -class CmdlineCrawlPipelineTest(unittest.TestCase): +class TestCmdlineCrawlPipeline: def _execute(self, spname): args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname) cwd = Path(__file__).resolve().parent @@ -13,7 +12,7 @@ class CmdlineCrawlPipelineTest(unittest.TestCase): return proc.returncode def test_open_spider_normally_in_pipeline(self): - self.assertEqual(self._execute("normal"), 0) + assert self._execute("normal") == 0 def test_exception_at_open_spider_in_pipeline(self): - self.assertEqual(self._execute("exception"), 1) + assert self._execute("exception") == 1 diff --git a/tests/test_command_check.py b/tests/test_command_check.py index b0f1cd38a..975f31dfe 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -3,10 +3,10 @@ from io import StringIO from unittest.mock import Mock, PropertyMock, call, patch from scrapy.commands.check import Command, TextTestResult -from tests.test_commands import CommandTest +from tests.test_commands import TestCommandBase -class CheckCommandTest(CommandTest): +class TestCheckCommand(TestCommandBase): command = "check" def setUp(self): @@ -36,9 +36,9 @@ class CheckSpider(scrapy.Spider): def _test_contract(self, contracts="", parse_def="pass"): self._write_contract(contracts, parse_def) p, out, err = self.proc("check") - self.assertNotIn("F", out) - self.assertIn("OK", err) - self.assertEqual(p.returncode, 0) + assert "F" not in out + assert "OK" in err + assert p.returncode == 0 def test_check_returns_requests_contract(self): contracts = """ @@ -171,9 +171,7 @@ class CheckSpider(scrapy.Spider): cmd.run([spider_name], Mock(list=True)) - self.assertEqual( - "FakeSpider\n * fakeMethod1\n * fakeMethod2\n", output.getvalue() - ) + assert output.getvalue() == "FakeSpider\n * fakeMethod1\n * fakeMethod2\n" sys.stdout = sys.__stdout__ @patch("scrapy.commands.check.ContractsManager") diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index a4d7fdd30..a31cada85 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -5,18 +5,18 @@ from tests.utils.testproc import ProcessTest from tests.utils.testsite import SiteTest -class FetchTest(ProcessTest, SiteTest, unittest.TestCase): +class TestFetchCommand(ProcessTest, SiteTest, unittest.TestCase): command = "fetch" @defer.inlineCallbacks def test_output(self): _, out, _ = yield self.execute([self.url("/text")]) - self.assertEqual(out.strip(), b"Works") + assert out.strip() == b"Works" @defer.inlineCallbacks def test_redirect_default(self): _, out, _ = yield self.execute([self.url("/redirect")]) - self.assertEqual(out.strip(), b"Redirected here") + assert out.strip() == b"Redirected here" @defer.inlineCallbacks def test_redirect_disabled(self): @@ -24,8 +24,8 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): ["--no-redirect", self.url("/redirect-no-meta-refresh")] ) err = err.strip() - self.assertIn(b"downloader/response_status_count/302", err, err) - self.assertNotIn(b"downloader/response_status_count/200", err, err) + assert b"downloader/response_status_count/302" in err, err + assert b"downloader/response_status_count/200" not in err, err @defer.inlineCallbacks def test_headers(self): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 9f2c7fa13..9e66d319c 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,5 +1,6 @@ import argparse import os +import re from pathlib import Path from twisted.internet import defer @@ -7,18 +8,18 @@ from twisted.internet import defer from scrapy.commands import parse from scrapy.settings import Settings from scrapy.utils.python import to_unicode -from tests.test_commands import CommandTest +from tests.test_commands import TestCommandBase from tests.utils.testproc import ProcessTest from tests.utils.testsite import SiteTest -def _textmode(bstr): +def _textmode(bstr: bytes) -> str: """Normalize input the same as writing to a file and reading from it in text mode""" return to_unicode(bstr).replace(os.linesep, "\n") -class ParseCommandTest(ProcessTest, SiteTest, CommandTest): +class TestParseCommand(ProcessTest, SiteTest, TestCommandBase): command = "parse" def setUp(self): @@ -184,7 +185,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + assert "DEBUG: It Works!" in _textmode(stderr) @defer.inlineCallbacks def test_request_with_meta(self): @@ -201,7 +202,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + assert "DEBUG: It Works!" in _textmode(stderr) _, _, stderr = yield self.execute( [ @@ -215,7 +216,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + assert "DEBUG: It Works!" in _textmode(stderr) @defer.inlineCallbacks def test_request_with_cb_kwargs(self): @@ -233,9 +234,9 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ] ) log = _textmode(stderr) - self.assertIn("DEBUG: It Works!", log) - self.assertIn( - "DEBUG: request.callback signature: (response, foo=None, key=None)", log + assert "DEBUG: It Works!" in log + assert ( + "DEBUG: request.callback signature: (response, foo=None, key=None)" in log ) @defer.inlineCallbacks @@ -250,7 +251,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + assert "DEBUG: It Works!" in _textmode(stderr) @defer.inlineCallbacks def test_pipelines(self): @@ -265,7 +266,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("INFO: It Works!", _textmode(stderr)) + assert "INFO: It Works!" in _textmode(stderr) @defer.inlineCallbacks def test_async_def_asyncio_parse_items_list(self): @@ -278,9 +279,9 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("INFO: Got response 200", _textmode(stderr)) - self.assertIn("{'id': 1}", _textmode(out)) - self.assertIn("{'id': 2}", _textmode(out)) + assert "INFO: Got response 200" in _textmode(stderr) + assert "{'id': 1}" in _textmode(out) + assert "{'id': 2}" in _textmode(out) @defer.inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): @@ -293,8 +294,8 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("INFO: Got response 200", _textmode(stderr)) - self.assertIn("{'foo': 42}", _textmode(out)) + assert "INFO: Got response 200" in _textmode(stderr) + assert "{'foo': 42}" in _textmode(out) @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): @@ -307,9 +308,9 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("INFO: Got response 200", _textmode(stderr)) + assert "INFO: Got response 200" in _textmode(stderr) for i in range(10): - self.assertIn(f"{{'foo': {i}}}", _textmode(out)) + assert f"{{'foo': {i}}}" in _textmode(out) @defer.inlineCallbacks def test_async_def_asyncgen_parse_exc(self): @@ -322,9 +323,9 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("ValueError", _textmode(stderr)) + assert "ValueError" in _textmode(stderr) for i in range(7): - self.assertIn(f"{{'foo': {i}}}", _textmode(out)) + assert f"{{'foo': {i}}}" in _textmode(out) @defer.inlineCallbacks def test_async_def_asyncio_parse(self): @@ -337,29 +338,29 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: Got response 200", _textmode(stderr)) + assert "DEBUG: Got response 200" in _textmode(stderr) @defer.inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, "-c", "parse", self.url("/html")] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + assert "[{}, {'foo': 'bar'}]" in _textmode(out) @defer.inlineCallbacks def test_parse_items_no_callback_passed(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, self.url("/html")] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + assert "[{}, {'foo': 'bar'}]" in _textmode(out) @defer.inlineCallbacks def test_wrong_callback_passed(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, "-c", "dummy", self.url("/html")] ) - self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") - self.assertIn("""Cannot find callback""", _textmode(stderr)) + assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) + assert "Cannot find callback" in _textmode(stderr) @defer.inlineCallbacks def test_crawlspider_matching_rule_callback_set(self): @@ -367,7 +368,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} status, out, stderr = yield self.execute( ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/html")] ) - self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + assert "[{}, {'foo': 'bar'}]" in _textmode(out) @defer.inlineCallbacks def test_crawlspider_matching_rule_default_callback(self): @@ -375,7 +376,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} status, out, stderr = yield self.execute( ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/text")] ) - self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out)) + assert "[{}, {'nomatch': 'default'}]" in _textmode(out) @defer.inlineCallbacks def test_spider_with_no_rules_attribute(self): @@ -383,15 +384,15 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} status, out, stderr = yield self.execute( ["--spider", self.spider_name, "-r", self.url("/html")] ) - self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") - self.assertIn("""No CrawlSpider rules found""", _textmode(stderr)) + assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) + assert "No CrawlSpider rules found" in _textmode(stderr) @defer.inlineCallbacks def test_crawlspider_missing_callback(self): status, out, stderr = yield self.execute( ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/html")] ) - self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") + assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) @defer.inlineCallbacks def test_crawlspider_no_matching_rule(self): @@ -399,13 +400,13 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} status, out, stderr = yield self.execute( ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/enc-gb18030")] ) - self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") - self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) + assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) + assert "Cannot find a rule that matches" in _textmode(stderr) @defer.inlineCallbacks def test_crawlspider_not_exists_with_not_matched_url(self): status, out, stderr = yield self.execute([self.url("/invalid_url")]) - self.assertEqual(status, 0) + assert status == 0 @defer.inlineCallbacks def test_output_flag(self): @@ -426,11 +427,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ] ) - self.assertTrue(file_path.exists()) - self.assertTrue(file_path.is_file()) + assert file_path.exists() + assert file_path.is_file() content = '[\n{},\n{"foo": "bar"}\n]' - self.assertEqual(file_path.read_text(encoding="utf-8"), content) + assert file_path.read_text(encoding="utf-8") == content def test_parse_add_options(self): command = parse.Command() @@ -445,7 +446,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} namespace = parser.parse_args( ["--verbose", "--nolinks", "-d", "2", "--spider", self.spider_name] ) - self.assertTrue(namespace.nolinks) - self.assertEqual(namespace.depth, 2) - self.assertEqual(namespace.spider, self.spider_name) - self.assertTrue(namespace.verbose) + assert namespace.nolinks + assert namespace.depth == 2 + assert namespace.spider == self.spider_name + assert namespace.verbose diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 9ca5e05dc..0f45a7ee8 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -13,7 +13,7 @@ from tests.utils.testproc import ProcessTest from tests.utils.testsite import SiteTest -class ShellTest(ProcessTest, SiteTest, unittest.TestCase): +class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): command = "shell" @defer.inlineCallbacks @@ -40,14 +40,14 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): def test_response_selector_html(self): xpath = "response.xpath(\"//p[@class='one']/text()\").get()" _, out, _ = yield self.execute([self.url("/html"), "-c", xpath]) - self.assertEqual(out.strip(), b"Works") + assert out.strip() == b"Works" @defer.inlineCallbacks def test_response_encoding_gb18030(self): _, out, _ = yield self.execute( [self.url("/enc-gb18030"), "-c", "response.encoding"] ) - self.assertEqual(out.strip(), b"gb18030") + assert out.strip() == b"gb18030" @defer.inlineCallbacks def test_redirect(self): @@ -79,7 +79,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): url = self.url("/redirect-no-meta-refresh") code = f"fetch('{url}')" errcode, out, errout = yield self.execute(["-c", code]) - self.assertEqual(errcode, 0, out) + assert errcode == 0, out assert b"Redirecting (302)" in errout assert b"Crawled (200)" in errout @@ -89,7 +89,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): url = self.url("/redirect-no-meta-refresh") code = f"fetch('{url}', redirect=False)" errcode, out, errout = yield self.execute(["-c", code]) - self.assertEqual(errcode, 0, out) + assert errcode == 0, out assert b"Crawled (302)" in errout @defer.inlineCallbacks @@ -97,14 +97,14 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): url = self.url("/text") code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" errcode, out, _ = yield self.execute(["-c", code]) - self.assertEqual(errcode, 0, out) + assert errcode == 0, out @defer.inlineCallbacks def test_scrapy_import(self): url = self.url("/text") code = f"fetch(scrapy.Request('{url}'))" errcode, out, _ = yield self.execute(["-c", code]) - self.assertEqual(errcode, 0, out) + assert errcode == 0, out @defer.inlineCallbacks def test_local_file(self): @@ -118,8 +118,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): errcode, out, err = yield self.execute( [filepath, "-c", "item"], check_code=False ) - self.assertEqual(errcode, 1, out or err) - self.assertIn(b"No such file or directory", err) + assert errcode == 1, out or err + assert b"No such file or directory" in err @defer.inlineCallbacks def test_dns_failures(self): @@ -127,8 +127,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): raise unittest.SkipTest("Non-existing hosts are resolvable") url = "www.somedomainthatdoesntexi.st" errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False) - self.assertEqual(errcode, 1, out or err) - self.assertIn(b"DNS lookup failed", err) + assert errcode == 1, out or err + assert b"DNS lookup failed" in err @defer.inlineCallbacks def test_shell_fetch_async(self): @@ -137,10 +137,10 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): code = f"fetch('{url}')" args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] _, _, err = yield self.execute(args, check_code=True) - self.assertNotIn(b"RuntimeError: There is no current event loop in thread", err) + assert b"RuntimeError: There is no current event loop in thread" not in err -class InteractiveShellTest(unittest.TestCase): +class TestInteractiveShell: def test_fetch(self): args = ( sys.executable, @@ -161,4 +161,4 @@ class InteractiveShellTest(unittest.TestCase): p.sendeof() p.wait() logfile.seek(0) - self.assertNotIn("Traceback", logfile.read().decode()) + assert "Traceback" not in logfile.read().decode() diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 917f457cb..a61a6a32b 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -7,17 +7,14 @@ import scrapy from tests.utils.testproc import ProcessTest -class VersionTest(ProcessTest, unittest.TestCase): +class TestVersionCommand(ProcessTest, unittest.TestCase): command = "version" @defer.inlineCallbacks def test_output(self): encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute([]) - self.assertEqual( - out.strip().decode(encoding), - f"Scrapy {scrapy.__version__}", - ) + assert out.strip().decode(encoding) == f"Scrapy {scrapy.__version__}" @defer.inlineCallbacks def test_verbose_output(self): @@ -27,19 +24,16 @@ class VersionTest(ProcessTest, unittest.TestCase): line.partition(":")[0].strip() for line in out.strip().decode(encoding).splitlines() ] - self.assertEqual( - headers, - [ - "Scrapy", - "lxml", - "libxml2", - "cssselect", - "parsel", - "w3lib", - "Twisted", - "Python", - "pyOpenSSL", - "cryptography", - "Platform", - ], - ) + assert headers == [ + "Scrapy", + "lxml", + "libxml2", + "cssselect", + "parsel", + "w3lib", + "Twisted", + "Python", + "pyOpenSSL", + "cryptography", + "Platform", + ] diff --git a/tests/test_commands.py b/tests/test_commands.py index 1a0db1e03..f63e05628 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -14,7 +14,7 @@ from itertools import chain from pathlib import Path from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION -from tempfile import TemporaryFile, mkdtemp +from tempfile import TemporaryDirectory, TemporaryFile, mkdtemp from threading import Timer from typing import TYPE_CHECKING from unittest import mock, skipIf @@ -35,8 +35,8 @@ if TYPE_CHECKING: from collections.abc import Iterator -class CommandSettings(unittest.TestCase): - def setUp(self): +class TestCommandSettings: + def setup_method(self): self.command = ScrapyCommand() self.command.settings = Settings() self.parser = argparse.ArgumentParser( @@ -50,10 +50,8 @@ class CommandSettings(unittest.TestCase): args=["-s", f"FEEDS={feeds_json}", "spider.py"] ) self.command.process_options(args, opts) - self.assertIsInstance( - self.command.settings["FEEDS"], scrapy.settings.BaseSettings - ) - self.assertEqual(dict(self.command.settings["FEEDS"]), json.loads(feeds_json)) + assert isinstance(self.command.settings["FEEDS"], scrapy.settings.BaseSettings) + assert dict(self.command.settings["FEEDS"]) == json.loads(feeds_json) def test_help_formatter(self): formatter = ScrapyHelpFormatter(prog="scrapy") @@ -64,17 +62,14 @@ class CommandSettings(unittest.TestCase): "\n", "Global Options:\n", ] - self.assertEqual( - formatter._join_parts(part_strings), - ( - "Usage\n=====\n scrapy genspider [options] <name> <domain>\n\n\n" - "Optional Arguments\n==================\n\n" - "Global Options\n--------------\n" - ), + assert formatter._join_parts(part_strings) == ( + "Usage\n=====\n scrapy genspider [options] <name> <domain>\n\n\n" + "Optional Arguments\n==================\n\n" + "Global Options\n--------------\n" ) -class ProjectTest(unittest.TestCase): +class TestProjectBase(unittest.TestCase): project_name = "testproject" def setUp(self): @@ -130,12 +125,12 @@ class ProjectTest(unittest.TestCase): return None -class StartprojectTest(ProjectTest): +class TestStartprojectCommand(TestProjectBase): def test_startproject(self): p, out, err = self.proc("startproject", self.project_name) print(out) print(err, file=sys.stderr) - self.assertEqual(p.returncode, 0) + assert p.returncode == 0 assert Path(self.proj_path, "scrapy.cfg").exists() assert Path(self.proj_path, "testproject").exists() @@ -145,13 +140,13 @@ class StartprojectTest(ProjectTest): assert Path(self.proj_mod_path, "settings.py").exists() assert Path(self.proj_mod_path, "spiders", "__init__.py").exists() - self.assertEqual(1, self.call("startproject", self.project_name)) - self.assertEqual(1, self.call("startproject", "wrong---project---name")) - self.assertEqual(1, self.call("startproject", "sys")) + assert self.call("startproject", self.project_name) == 1 + assert self.call("startproject", "wrong---project---name") == 1 + assert self.call("startproject", "sys") == 1 def test_startproject_with_project_dir(self): project_dir = mkdtemp() - self.assertEqual(0, self.call("startproject", self.project_name, project_dir)) + assert self.call("startproject", self.project_name, project_dir) == 0 assert Path(project_dir, "scrapy.cfg").exists() assert Path(project_dir, "testproject").exists() @@ -161,20 +156,16 @@ class StartprojectTest(ProjectTest): assert Path(project_dir, self.project_name, "settings.py").exists() assert Path(project_dir, self.project_name, "spiders", "__init__.py").exists() - self.assertEqual( - 0, self.call("startproject", self.project_name, project_dir + "2") - ) + assert self.call("startproject", self.project_name, project_dir + "2") == 0 - self.assertEqual(1, self.call("startproject", self.project_name, project_dir)) - self.assertEqual( - 1, self.call("startproject", self.project_name + "2", project_dir) - ) - self.assertEqual(1, self.call("startproject", "wrong---project---name")) - self.assertEqual(1, self.call("startproject", "sys")) - self.assertEqual(2, self.call("startproject")) - self.assertEqual( - 2, - self.call("startproject", self.project_name, project_dir, "another_params"), + assert self.call("startproject", self.project_name, project_dir) == 1 + assert self.call("startproject", self.project_name + "2", project_dir) == 1 + assert self.call("startproject", "wrong---project---name") == 1 + assert self.call("startproject", "sys") == 1 + assert self.call("startproject") == 2 + assert ( + self.call("startproject", self.project_name, project_dir, "another_params") + == 2 ) def test_existing_project_dir(self): @@ -186,7 +177,7 @@ class StartprojectTest(ProjectTest): p, out, err = self.proc("startproject", project_name, cwd=project_dir) print(out) print(err, file=sys.stderr) - self.assertEqual(p.returncode, 0) + assert p.returncode == 0 assert Path(project_path, "scrapy.cfg").exists() assert Path(project_path, project_name).exists() @@ -224,7 +215,7 @@ def get_permissions_dict( return permissions_dict -class StartprojectTemplatesTest(ProjectTest): +class TestStartprojectTemplates(TestProjectBase): maxDiff = None def setUp(self): @@ -239,11 +230,10 @@ class StartprojectTemplatesTest(ProjectTest): args = ["--set", f"TEMPLATES_DIR={self.tmpl}"] p, out, err = self.proc("startproject", self.project_name, *args) - self.assertIn( - f"New Scrapy project '{self.project_name}', using template directory", - out, + assert ( + f"New Scrapy project '{self.project_name}', using template directory" in out ) - self.assertIn(self.tmpl_proj, out) + assert self.tmpl_proj in out assert Path(self.proj_path, "root_template").exists() def test_startproject_permissions_from_writable(self): @@ -280,7 +270,7 @@ class StartprojectTemplatesTest(ProjectTest): project_dir = Path(destination, project_name) actual_permissions = get_permissions_dict(project_dir) - self.assertEqual(actual_permissions, expected_permissions) + assert actual_permissions == expected_permissions def test_startproject_permissions_from_read_only(self): """Check that generated files have the right permissions when the @@ -333,7 +323,7 @@ class StartprojectTemplatesTest(ProjectTest): project_dir = Path(destination, project_name) actual_permissions = get_permissions_dict(project_dir) - self.assertEqual(actual_permissions, expected_permissions) + assert actual_permissions == expected_permissions def test_startproject_permissions_unchanged_in_destination(self): """Check that preexisting folders and files in the destination folder @@ -391,7 +381,7 @@ class StartprojectTemplatesTest(ProjectTest): actual_permissions = get_permissions_dict(project_dir) - self.assertEqual(actual_permissions, expected_permissions) + assert actual_permissions == expected_permissions def test_startproject_permissions_umask_022(self): """Check that generated files have the right permissions when the @@ -435,10 +425,10 @@ class StartprojectTemplatesTest(ProjectTest): project_dir = Path(destination, project_name) actual_permissions = get_permissions_dict(project_dir) - self.assertEqual(actual_permissions, expected_permissions) + assert actual_permissions == expected_permissions -class CommandTest(ProjectTest): +class TestCommandBase(TestProjectBase): def setUp(self): super().setUp() self.call("startproject", self.project_name) @@ -446,13 +436,13 @@ class CommandTest(ProjectTest): self.env["SCRAPY_SETTINGS_MODULE"] = f"{self.project_name}.settings" -class GenspiderCommandTest(CommandTest): +class TestGenspiderCommand(TestCommandBase): def test_arguments(self): # only pass one argument. spider script shouldn't be created - self.assertEqual(2, self.call("genspider", "test_name")) + assert self.call("genspider", "test_name") == 2 assert not Path(self.proj_mod_path, "spiders", "test_name.py").exists() # pass two arguments <name> <domain>. spider script should be created - self.assertEqual(0, self.call("genspider", "test_name", "test.com")) + assert self.call("genspider", "test_name", "test.com") == 0 assert Path(self.proj_mod_path, "spiders", "test_name.py").exists() def test_template(self, tplname="crawl"): @@ -460,20 +450,20 @@ class GenspiderCommandTest(CommandTest): spname = "test_spider" spmodule = f"{self.project_name}.spiders.{spname}" p, out, err = self.proc("genspider", spname, "test.com", *args) - self.assertIn( - f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", - out, + assert ( + f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}" + in out ) - self.assertTrue(Path(self.proj_mod_path, "spiders", "test_spider.py").exists()) + assert Path(self.proj_mod_path, "spiders", "test_spider.py").exists() modify_time_before = ( Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime ) p, out, err = self.proc("genspider", spname, "test.com", *args) - self.assertIn(f"Spider {spname!r} already exists in module", out) + assert f"Spider {spname!r} already exists in module" in out modify_time_after = ( Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime ) - self.assertEqual(modify_time_after, modify_time_before) + assert modify_time_after == modify_time_before def test_template_basic(self): self.test_template("basic") @@ -485,14 +475,14 @@ class GenspiderCommandTest(CommandTest): self.test_template("xmlfeed") def test_list(self): - self.assertEqual(0, self.call("genspider", "--list")) + assert self.call("genspider", "--list") == 0 def test_dump(self): - self.assertEqual(0, self.call("genspider", "--dump=basic")) - self.assertEqual(0, self.call("genspider", "-d", "basic")) + assert self.call("genspider", "--dump=basic") == 0 + assert self.call("genspider", "-d", "basic") == 0 def test_same_name_as_project(self): - self.assertEqual(2, self.call("genspider", self.project_name)) + assert self.call("genspider", self.project_name) == 2 assert not Path( self.proj_mod_path, "spiders", f"{self.project_name}.py" ).exists() @@ -500,7 +490,7 @@ class GenspiderCommandTest(CommandTest): def test_same_filename_as_existing_spider(self, force=False): file_name = "example" file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py") - self.assertEqual(0, self.call("genspider", file_name, "example.com")) + assert self.call("genspider", file_name, "example.com") == 0 assert file_path.exists() # change name of spider but not its file name @@ -515,39 +505,39 @@ class GenspiderCommandTest(CommandTest): if force: p, out, err = self.proc("genspider", "--force", file_name, "example.com") - self.assertIn( - f"Created spider {file_name!r} using template 'basic' in module", out + assert ( + f"Created spider {file_name!r} using template 'basic' in module" in out ) modify_time_after = file_path.stat().st_mtime - self.assertNotEqual(modify_time_after, modify_time_before) + assert modify_time_after != modify_time_before file_contents_after = file_path.read_text(encoding="utf-8") - self.assertNotEqual(file_contents_after, file_contents_before) + assert file_contents_after != file_contents_before else: p, out, err = self.proc("genspider", file_name, "example.com") - self.assertIn(f"{file_path.resolve()} already exists", out) + assert f"{file_path.resolve()} already exists" in out modify_time_after = file_path.stat().st_mtime - self.assertEqual(modify_time_after, modify_time_before) + assert modify_time_after == modify_time_before file_contents_after = file_path.read_text(encoding="utf-8") - self.assertEqual(file_contents_after, file_contents_before) + assert file_contents_after == file_contents_before def test_same_filename_as_existing_spider_force(self): self.test_same_filename_as_existing_spider(force=True) def test_url(self, url="test.com", domain="test.com"): - self.assertEqual(0, self.call("genspider", "--force", "test_name", url)) - self.assertEqual( - domain, + assert self.call("genspider", "--force", "test_name", url) == 0 + assert ( self.find_in_file( Path(self.proj_mod_path, "spiders", "test_name.py"), r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", - ).group(1), + ).group(1) + == domain ) - self.assertEqual( - f"https://{domain}", + assert ( self.find_in_file( Path(self.proj_mod_path, "spiders", "test_name.py"), r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1), + ).group(1) + == f"https://{domain}" ) def test_url_schema(self): @@ -556,15 +546,13 @@ class GenspiderCommandTest(CommandTest): def test_template_start_urls( self, url="test.com", expected="https://test.com", template="basic" ): - self.assertEqual( - 0, self.call("genspider", "-t", template, "--force", "test_name", url) - ) - self.assertEqual( - expected, + assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0 + assert ( self.find_in_file( Path(self.proj_mod_path, "spiders", "test_name.py"), r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1), + ).group(1) + == expected ) def test_genspider_basic_start_urls(self): @@ -611,7 +599,7 @@ class GenspiderCommandTest(CommandTest): ) -class GenspiderStandaloneCommandTest(ProjectTest): +class TestGenspiderStandaloneCommand(TestProjectBase): def test_generate_standalone_spider(self): self.call("genspider", "example", "example.com") assert Path(self.temp_path, "example.py").exists() @@ -620,7 +608,7 @@ class GenspiderStandaloneCommandTest(ProjectTest): file_name = "example" file_path = Path(self.temp_path, file_name + ".py") p, out, err = self.proc("genspider", file_name, "example.com") - self.assertIn(f"Created spider {file_name!r} using template 'basic' ", out) + assert f"Created spider {file_name!r} using template 'basic' " in out assert file_path.exists() modify_time_before = file_path.stat().st_mtime file_contents_before = file_path.read_text(encoding="utf-8") @@ -630,29 +618,29 @@ class GenspiderStandaloneCommandTest(ProjectTest): p, out, err = self.proc( "genspider", "--force", "-t", "crawl", file_name, "example.com" ) - self.assertIn(f"Created spider {file_name!r} using template 'crawl' ", out) + assert f"Created spider {file_name!r} using template 'crawl' " in out modify_time_after = file_path.stat().st_mtime - self.assertNotEqual(modify_time_after, modify_time_before) + assert modify_time_after != modify_time_before file_contents_after = file_path.read_text(encoding="utf-8") - self.assertNotEqual(file_contents_after, file_contents_before) + assert file_contents_after != file_contents_before else: p, out, err = self.proc("genspider", file_name, "example.com") - self.assertIn( - f"{Path(self.temp_path, file_name + '.py').resolve()} already exists", - out, + assert ( + f"{Path(self.temp_path, file_name + '.py').resolve()} already exists" + in out ) modify_time_after = file_path.stat().st_mtime - self.assertEqual(modify_time_after, modify_time_before) + assert modify_time_after == modify_time_before file_contents_after = file_path.read_text(encoding="utf-8") - self.assertEqual(file_contents_after, file_contents_before) + assert file_contents_after == file_contents_before def test_same_name_as_existing_file_force(self): self.test_same_name_as_existing_file(force=True) -class MiscCommandsTest(CommandTest): +class TestMiscCommands(TestCommandBase): def test_list(self): - self.assertEqual(0, self.call("list")) + assert self.call("list") == 0 def test_command_not_found(self): na_msg = """ @@ -670,10 +658,10 @@ Unknown command: abc for cmdname, inproject, message in params: with mock.patch("sys.stdout", new=StringIO()) as out: _print_unknown_command_msg(Settings(), cmdname, inproject) - self.assertEqual(out.getvalue().strip(), message.strip()) + assert out.getvalue().strip() == message.strip() -class RunSpiderCommandTest(CommandTest): +class TestRunSpiderCommand(TestCommandBase): spider_filename = "myspider.py" debug_log_spider = """ @@ -697,18 +685,14 @@ class BadSpider(scrapy.Spider): """ @contextmanager - def _create_file(self, content, name=None) -> Iterator[str]: - tmpdir = Path(self.mktemp()) - tmpdir.mkdir() - if name: - fname = (tmpdir / name).resolve() - else: - fname = (tmpdir / self.spider_filename).resolve() - fname.write_text(content, encoding="utf-8") - try: + def _create_file(self, content: str, name: str | None = None) -> Iterator[str]: + with TemporaryDirectory() as tmpdir: + if name: + fname = Path(tmpdir, name).resolve() + else: + fname = Path(tmpdir, self.spider_filename).resolve() + fname.write_text(content, encoding="utf-8") yield str(fname) - finally: - rmtree(tmpdir) def runspider(self, code, name=None, args=()): with self._create_file(code, name) as fname: @@ -720,29 +704,29 @@ class BadSpider(scrapy.Spider): def test_runspider(self): log = self.get_log(self.debug_log_spider) - self.assertIn("DEBUG: It Works!", log) - self.assertIn("INFO: Spider opened", log) - self.assertIn("INFO: Closing spider (finished)", log) - self.assertIn("INFO: Spider closed (finished)", log) + assert "DEBUG: It Works!" in log + assert "INFO: Spider opened" in log + assert "INFO: Closing spider (finished)" in log + assert "INFO: Spider closed (finished)" in log def test_run_fail_spider(self): proc, _, _ = self.runspider( "import scrapy\n" + inspect.getsource(ExceptionSpider) ) ret = proc.returncode - self.assertNotEqual(ret, 0) + assert ret != 0 def test_run_good_spider(self): proc, _, _ = self.runspider( "import scrapy\n" + inspect.getsource(NoRequestsSpider) ) ret = proc.returncode - self.assertEqual(ret, 0) + assert ret == 0 def test_runspider_log_level(self): log = self.get_log(self.debug_log_spider, args=("-s", "LOG_LEVEL=INFO")) - self.assertNotIn("DEBUG: It Works!", log) - self.assertIn("INFO: Spider opened", log) + assert "DEBUG: It Works!" not in log + assert "INFO: Spider opened" in log def test_runspider_dnscache_disabled(self): # see https://github.com/scrapy/scrapy/issues/2811 @@ -761,36 +745,36 @@ class MySpider(scrapy.Spider): return {'test': 'value'} """ log = self.get_log(dnscache_spider, args=("-s", "DNSCACHE_ENABLED=False")) - self.assertNotIn("DNSLookupError", log) - self.assertIn("INFO: Spider opened", log) + assert "DNSLookupError" not in log + assert "INFO: Spider opened" in log def test_runspider_log_short_names(self): log1 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=1")) - self.assertIn("[myspider] DEBUG: It Works!", log1) - self.assertIn("[scrapy]", log1) - self.assertNotIn("[scrapy.core.engine]", log1) + assert "[myspider] DEBUG: It Works!" in log1 + assert "[scrapy]" in log1 + assert "[scrapy.core.engine]" not in log1 log2 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=0")) - self.assertIn("[myspider] DEBUG: It Works!", log2) - self.assertNotIn("[scrapy]", log2) - self.assertIn("[scrapy.core.engine]", log2) + assert "[myspider] DEBUG: It Works!" in log2 + assert "[scrapy]" not in log2 + assert "[scrapy.core.engine]" in log2 def test_runspider_no_spider_found(self): log = self.get_log("from scrapy.spiders import Spider\n") - self.assertIn("No spider found in file", log) + assert "No spider found in file" in log def test_runspider_file_not_found(self): _, _, log = self.proc("runspider", "some_non_existent_file") - self.assertIn("File not found: some_non_existent_file", log) + assert "File not found: some_non_existent_file" in log def test_runspider_unable_to_load(self): log = self.get_log("", name="myspider.txt") - self.assertIn("Unable to load", log) + assert "Unable to load" in log def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.py") - self.assertIn("start_requests", log) - self.assertIn("badspider.py", log) + assert "start_requests" in log + assert "badspider.py" in log def test_asyncio_enabled_true(self): log = self.get_log( @@ -800,14 +784,16 @@ class MySpider(scrapy.Spider): "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", ], ) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) def test_asyncio_enabled_default(self): log = self.get_log(self.debug_log_spider, args=[]) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) def test_asyncio_enabled_false(self): @@ -815,11 +801,10 @@ class MySpider(scrapy.Spider): self.debug_log_spider, args=["-s", "TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor"], ) - self.assertIn( - "Using reactor: twisted.internet.selectreactor.SelectReactor", log - ) - self.assertNotIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log ) @pytest.mark.requires_uvloop @@ -833,7 +818,7 @@ class MySpider(scrapy.Spider): "ASYNCIO_EVENT_LOOP=uvloop.Loop", ], ) - self.assertIn("Using asyncio event loop: uvloop.Loop", log) + assert "Using asyncio event loop: uvloop.Loop" in log def test_custom_asyncio_loop_enabled_false(self): log = self.get_log( @@ -849,9 +834,9 @@ class MySpider(scrapy.Spider): loop = asyncio.new_event_loop() else: loop = asyncio.SelectorEventLoop() - self.assertIn( - f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", - log, + assert ( + f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}" + in log ) def test_output(self): @@ -867,9 +852,7 @@ class MySpider(scrapy.Spider): """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log - ) + assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log def test_overwrite_output(self): spider_code = """ @@ -890,13 +873,13 @@ class MySpider(scrapy.Spider): Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ["-O", "example.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', - log, + assert ( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + in log ) with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: first_line = f2.readline() - self.assertNotEqual(first_line, "not empty") + assert first_line != "not empty" def test_output_and_overwrite_output(self): spider_code = """ @@ -910,8 +893,8 @@ class MySpider(scrapy.Spider): """ args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - "error: Please use only one of -o/--output and -O/--overwrite-output", log + assert ( + "error: Please use only one of -o/--output and -O/--overwrite-output" in log ) def test_output_stdout(self): @@ -927,7 +910,7 @@ class MySpider(scrapy.Spider): """ args = ["-o", "-:json"] log = self.get_log(spider_code, args=args) - self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log) + assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log @skipIf(platform.system() == "Windows", reason="Linux only") def test_absolute_path_linux(self): @@ -946,16 +929,16 @@ class MySpider(scrapy.Spider): args = ["-o", f"{temp_dir}/output1.json:json"] log = self.get_log(spider_code, args=args) - self.assertIn( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json", - log, + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json" + in log ) args = ["-o", f"{temp_dir}/output2.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json", - log, + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json" + in log ) @skipIf(platform.system() != "Windows", reason="Windows only") @@ -975,16 +958,16 @@ class MySpider(scrapy.Spider): args = ["-o", f"{temp_dir}\\output1.json:json"] log = self.get_log(spider_code, args=args) - self.assertIn( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json", - log, + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json" + in log ) args = ["-o", f"{temp_dir}\\output2.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json", - log, + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json" + in log ) def test_args_change_settings(self): @@ -1006,11 +989,11 @@ class MySpider(scrapy.Spider): """ args = ["-a", "foo=42"] log = self.get_log(spider_code, args=args) - self.assertIn("Spider closed (finished)", log) - self.assertIn("The value of FOO is 42", log) + assert "Spider closed (finished)" in log + assert "The value of FOO is 42" in log -class WindowsRunSpiderCommandTest(RunSpiderCommandTest): +class TestWindowsRunSpiderCommand(TestRunSpiderCommand): spider_filename = "myspider.pyw" def setUp(self): @@ -1020,24 +1003,24 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") - self.assertIn("start_requests", log) - self.assertIn("badspider.pyw", log) + assert "start_requests" in log + assert "badspider.pyw" in log def test_runspider_unable_to_load(self): raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") -class BenchCommandTest(CommandTest): +class TestBenchCommand(TestCommandBase): def test_run(self): _, _, log = self.proc( "bench", "-s", "LOGSTATS_INTERVAL=0.001", "-s", "CLOSESPIDER_TIMEOUT=0.01" ) - self.assertIn("INFO: Crawled", log) - self.assertNotIn("Unhandled Error", log) - self.assertNotIn("log_count/ERROR", log) + assert "INFO: Crawled" in log + assert "Unhandled Error" not in log + assert "log_count/ERROR" not in log -class ViewCommandTest(CommandTest): +class TestViewCommand(TestCommandBase): def test_methods(self): command = view.Command() command.settings = Settings() @@ -1048,13 +1031,11 @@ class ViewCommandTest(CommandTest): conflict_handler="resolve", ) command.add_options(parser) - self.assertEqual(command.short_desc(), "Open URL in browser, as seen by Scrapy") - self.assertIn( - "URL using the Scrapy downloader and show its", command.long_desc() - ) + assert command.short_desc() == "Open URL in browser, as seen by Scrapy" + assert "URL using the Scrapy downloader and show its" in command.long_desc() -class CrawlCommandTest(CommandTest): +class TestCrawlCommand(TestCommandBase): def crawl(self, code, args=()): Path(self.proj_mod_path, "spiders", "myspider.py").write_text( code, encoding="utf-8" @@ -1077,7 +1058,7 @@ class MySpider(scrapy.Spider): return [] """ log = self.get_log(spider_code) - self.assertIn("[myspider] DEBUG: It works!", log) + assert "[myspider] DEBUG: It works!" in log def test_output(self): spider_code = """ @@ -1092,9 +1073,7 @@ class MySpider(scrapy.Spider): """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log - ) + assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log def test_overwrite_output(self): spider_code = """ @@ -1115,13 +1094,13 @@ class MySpider(scrapy.Spider): Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ["-O", "example.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', - log, + assert ( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + in log ) with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: first_line = f2.readline() - self.assertNotEqual(first_line, "not empty") + assert first_line != "not empty" def test_output_and_overwrite_output(self): spider_code = """ @@ -1135,12 +1114,12 @@ class MySpider(scrapy.Spider): """ args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) - self.assertIn( - "error: Please use only one of -o/--output and -O/--overwrite-output", log + assert ( + "error: Please use only one of -o/--output and -O/--overwrite-output" in log ) -class HelpMessageTest(CommandTest): +class TestHelpMessage(TestCommandBase): def setUp(self): super().setUp() self.commands = [ @@ -1163,30 +1142,30 @@ class HelpMessageTest(CommandTest): def test_help_messages(self): for command in self.commands: _, out, _ = self.proc(command, "-h") - self.assertIn("Usage", out) + assert "Usage" in out -class PopCommandNameTest(unittest.TestCase): +class TestPopCommandName: def test_valid_command(self): argv = ["scrapy", "crawl", "my_spider"] command = _pop_command_name(argv) - self.assertEqual(command, "crawl") - self.assertEqual(argv, ["scrapy", "my_spider"]) + assert command == "crawl" + assert argv == ["scrapy", "my_spider"] def test_no_command(self): argv = ["scrapy"] command = _pop_command_name(argv) - self.assertIsNone(command) - self.assertEqual(argv, ["scrapy"]) + assert command is None + assert argv == ["scrapy"] def test_option_before_command(self): argv = ["scrapy", "-h", "crawl"] command = _pop_command_name(argv) - self.assertEqual(command, "crawl") - self.assertEqual(argv, ["scrapy", "-h"]) + assert command == "crawl" + assert argv == ["scrapy", "-h"] def test_option_after_command(self): argv = ["scrapy", "crawl", "-h"] command = _pop_command_name(argv) - self.assertEqual(command, "crawl") - self.assertEqual(argv, ["scrapy", "-h"]) + assert command == "crawl" + assert argv == ["scrapy", "-h"] diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 0f7d7b54c..fb961ace2 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,5 +1,6 @@ from unittest import TextTestResult +import pytest from twisted.internet import defer from twisted.python import failure from twisted.trial import unittest @@ -246,7 +247,7 @@ class InheritsDemoSpider(DemoSpider): name = "inherits_demo_spider" -class ContractsManagerTest(unittest.TestCase): +class TestContractsManager(unittest.TestCase): contracts = [ UrlContract, CallbackKeywordArgumentsContract, @@ -263,34 +264,33 @@ class ContractsManagerTest(unittest.TestCase): self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) def should_succeed(self): - self.assertFalse(self.results.failures) - self.assertFalse(self.results.errors) + assert not self.results.failures + assert not self.results.errors def should_fail(self): - self.assertTrue(self.results.failures) - self.assertFalse(self.results.errors) + assert self.results.failures + assert not self.results.errors def should_error(self): - self.assertTrue(self.results.errors) + assert self.results.errors def test_contracts(self): spider = DemoSpider() # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request) - self.assertEqual(len(contracts), 2) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, ReturnsContract]), + assert len(contracts) == 2 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, ReturnsContract] ) # returns request for valid method request = self.conman.from_method(spider.returns_request, self.results) - self.assertNotEqual(request, None) + assert request is not None # no request for missing url request = self.conman.from_method(spider.parse_no_url, self.results) - self.assertEqual(request, None) + assert request is None def test_cb_kwargs(self): spider = DemoSpider() @@ -298,35 +298,31 @@ class ContractsManagerTest(unittest.TestCase): # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request_cb_kwargs) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, CallbackKeywordArgumentsContract, ReturnsContract] ) contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, CallbackKeywordArgumentsContract, ReturnsContract] ) contracts = self.conman.extract_contracts( spider.returns_item_cb_kwargs_error_unexpected_keyword ) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, CallbackKeywordArgumentsContract, ReturnsContract] ) contracts = self.conman.extract_contracts( spider.returns_item_cb_kwargs_error_missing_argument ) - self.assertEqual(len(contracts), 2) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, ReturnsContract]), + assert len(contracts) == 2 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, ReturnsContract] ) # returns_request @@ -360,17 +356,15 @@ class ContractsManagerTest(unittest.TestCase): # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request_meta) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, MetadataContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, MetadataContract, ReturnsContract] ) contracts = self.conman.extract_contracts(spider.returns_item_meta) - self.assertEqual(len(contracts), 3) - self.assertEqual( - frozenset(type(x) for x in contracts), - frozenset([UrlContract, MetadataContract, ReturnsContract]), + assert len(contracts) == 3 + assert frozenset(type(x) for x in contracts) == frozenset( + [UrlContract, MetadataContract, ReturnsContract] ) response = ResponseMetaMock() @@ -505,8 +499,8 @@ class ContractsManagerTest(unittest.TestCase): request = self.conman.from_method(spider.returns_request, self.results) request.errback(failure_mock) - self.assertFalse(self.results.failures) - self.assertTrue(self.results.errors) + assert not self.results.failures + assert self.results.errors @defer.inlineCallbacks def test_same_url(self): @@ -537,19 +531,19 @@ class ContractsManagerTest(unittest.TestCase): crawler = get_crawler(TestSameUrlSpider) yield crawler.crawl() - self.assertEqual(crawler.spider.visited, 2) + assert crawler.spider.visited == 2 def test_form_contract(self): spider = DemoSpider() request = self.conman.from_method(spider.custom_form, self.results) - self.assertEqual(request.method, "POST") - self.assertIsInstance(request, FormRequest) + assert request.method == "POST" + assert isinstance(request, FormRequest) def test_inherited_contracts(self): spider = InheritsDemoSpider() requests = self.conman.from_spider(spider, self.results) - self.assertTrue(requests) + assert requests class CustomFailContractPreProcess(Contract): @@ -566,8 +560,8 @@ class CustomFailContractPostProcess(Contract): raise KeyboardInterrupt("Post-process exception") -class CustomContractPrePostProcess(unittest.TestCase): - def setUp(self): +class TestCustomContractPrePostProcess: + def setup_method(self): self.results = TextTestResult(stream=None, descriptions=False, verbosity=0) def test_pre_hook_keyboard_interrupt(self): @@ -576,18 +570,13 @@ class CustomContractPrePostProcess(unittest.TestCase): contract = CustomFailContractPreProcess(spider.returns_request) conman = ContractsManager([contract]) - try: - request = conman.from_method(spider.returns_request, self.results) - contract.add_pre_hook(request, self.results) - # Expect this to raise a KeyboardInterrupt + request = conman.from_method(spider.returns_request, self.results) + contract.add_pre_hook(request, self.results) + with pytest.raises(KeyboardInterrupt, match="Pre-process exception"): request.callback(response, **request.cb_kwargs) - except KeyboardInterrupt as e: - self.assertEqual(str(e), "Pre-process exception") - else: - self.fail("KeyboardInterrupt not raised") - self.assertFalse(self.results.failures) - self.assertFalse(self.results.errors) + assert not self.results.failures + assert not self.results.errors def test_post_hook_keyboard_interrupt(self): spider = DemoSpider() @@ -595,15 +584,10 @@ class CustomContractPrePostProcess(unittest.TestCase): contract = CustomFailContractPostProcess(spider.returns_request) conman = ContractsManager([contract]) - try: - request = conman.from_method(spider.returns_request, self.results) - contract.add_post_hook(request, self.results) - # Expect this to raise a KeyboardInterrupt + request = conman.from_method(spider.returns_request, self.results) + contract.add_post_hook(request, self.results) + with pytest.raises(KeyboardInterrupt, match="Post-process exception"): request.callback(response, **request.cb_kwargs) - except KeyboardInterrupt as e: - self.assertEqual(str(e), "Post-process exception") - else: - self.fail("KeyboardInterrupt not raised") - self.assertFalse(self.results.failures) - self.assertFalse(self.results.errors) + assert not self.results.failures + assert not self.results.errors diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index dffba303f..1bffd69ed 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -30,16 +30,13 @@ from scrapy.utils.test import get_crawler from tests.mockserver import PayloadResource, ssl_context_factory -class SlotTest(unittest.TestCase): +class TestSlot: def test_repr(self): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) - self.assertEqual( - repr(slot), - "Slot(concurrency=8, delay=0.10, randomize_delay=True)", - ) + assert repr(slot) == "Slot(concurrency=8, delay=0.10, randomize_delay=True)" -class ContextFactoryBaseTestCase(unittest.TestCase): +class TestContextFactoryBase(unittest.TestCase): context_factory = None def _listen(self, site): @@ -90,7 +87,7 @@ class ContextFactoryBaseTestCase(unittest.TestCase): return await maybe_deferred_to_future(d) -class ContextFactoryTestCase(ContextFactoryBaseTestCase): +class TestContextFactory(TestContextFactoryBase): @deferred_f_from_coro_f async def testPayload(self): s = "0123456789" * 10 @@ -100,7 +97,7 @@ class ContextFactoryTestCase(ContextFactoryBaseTestCase): body = await self.get_page( self.getURL("payload"), client_context_factory, body=s ) - self.assertEqual(body, to_bytes(s)) + assert body == to_bytes(s) def test_override_getContext(self): class MyFactory(ScrapyClientContextFactory): @@ -112,14 +109,14 @@ class ContextFactoryTestCase(ContextFactoryBaseTestCase): with warnings.catch_warnings(record=True) as w: MyFactory() - self.assertEqual(len(w), 1) - self.assertIn( - "Overriding ScrapyClientContextFactory.getContext() is deprecated", - str(w[0].message), + assert len(w) == 1 + assert ( + "Overriding ScrapyClientContextFactory.getContext() is deprecated" + in str(w[0].message) ) -class ContextFactoryTLSMethodTestCase(ContextFactoryBaseTestCase): +class TestContextFactoryTLSMethod(TestContextFactoryBase): async def _assert_factory_works( self, client_context_factory: ScrapyClientContextFactory ) -> None: @@ -127,7 +124,7 @@ class ContextFactoryTLSMethodTestCase(ContextFactoryBaseTestCase): body = await self.get_page( self.getURL("payload"), client_context_factory, body=s ) - self.assertEqual(body, to_bytes(s)) + assert body == to_bytes(s) @deferred_f_from_coro_f async def test_setting_default(self): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3aca2bbce..5766f9313 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -55,7 +55,7 @@ from tests.spiders import ( ) -class CrawlTestCase(TestCase): +class TestCrawl(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -69,7 +69,7 @@ class CrawlTestCase(TestCase): def test_follow_all(self): crawler = get_crawler(FollowAllSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url + assert len(crawler.spider.urls_visited) == 11 # 10 + start_url @defer.inlineCallbacks def test_fixed_delay(self): @@ -94,9 +94,7 @@ class CrawlTestCase(TestCase): times = crawler.spider.times total_time = times[-1] - times[0] average = total_time / (len(times) - 1) - self.assertTrue( - average > delay * tolerance, f"download delay too small: {average}" - ) + assert average > delay * tolerance, f"download delay too small: {average}" # Ensure that the same test parameters would cause a failure if no # download delay is set. Otherwise, it means we are using a combination @@ -108,34 +106,32 @@ class CrawlTestCase(TestCase): times = crawler.spider.times total_time = times[-1] - times[0] average = total_time / (len(times) - 1) - self.assertFalse( - average > delay / tolerance, "test total or delay values are too small" - ) + assert average <= delay / tolerance, "test total or delay values are too small" @defer.inlineCallbacks def test_timeout_success(self): crawler = get_crawler(DelaySpider) yield crawler.crawl(n=0.5, mockserver=self.mockserver) - self.assertTrue(crawler.spider.t1 > 0) - self.assertTrue(crawler.spider.t2 > 0) - self.assertTrue(crawler.spider.t2 > crawler.spider.t1) + assert crawler.spider.t1 > 0 + assert crawler.spider.t2 > 0 + assert crawler.spider.t2 > crawler.spider.t1 @defer.inlineCallbacks def test_timeout_failure(self): crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) yield crawler.crawl(n=0.5, mockserver=self.mockserver) - self.assertTrue(crawler.spider.t1 > 0) - self.assertTrue(crawler.spider.t2 == 0) - self.assertTrue(crawler.spider.t2_err > 0) - self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) + assert crawler.spider.t1 > 0 + assert crawler.spider.t2 == 0 + assert crawler.spider.t2_err > 0 + assert crawler.spider.t2_err > crawler.spider.t1 # server hangs after receiving response headers crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) - self.assertTrue(crawler.spider.t1 > 0) - self.assertTrue(crawler.spider.t2 == 0) - self.assertTrue(crawler.spider.t2_err > 0) - self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) + assert crawler.spider.t1 > 0 + assert crawler.spider.t2 == 0 + assert crawler.spider.t2_err > 0 + assert crawler.spider.t2_err > crawler.spider.t1 @defer.inlineCallbacks def test_retry_503(self): @@ -173,10 +169,10 @@ class CrawlTestCase(TestCase): crawler = get_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) - self.assertEqual(len(log.records), 1) + assert len(log.records) == 1 record = log.records[0] - self.assertIsNotNone(record.exc_info) - self.assertIs(record.exc_info[0], ZeroDivisionError) + assert record.exc_info is not None + assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks def test_start_requests_bug_yielding(self): @@ -184,10 +180,10 @@ class CrawlTestCase(TestCase): crawler = get_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) - self.assertEqual(len(log.records), 1) + assert len(log.records) == 1 record = log.records[0] - self.assertIsNotNone(record.exc_info) - self.assertIs(record.exc_info[0], ZeroDivisionError) + assert record.exc_info is not None + assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks def test_start_requests_items(self): @@ -195,7 +191,7 @@ class CrawlTestCase(TestCase): crawler = get_crawler(StartRequestsItemSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(log.records), 0) + assert len(log.records) == 0 @defer.inlineCallbacks def test_start_requests_unsupported_output(self): @@ -203,23 +199,18 @@ class CrawlTestCase(TestCase): crawler = get_crawler(StartRequestsGoodAndBadOutput) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(log.records), 2) - self.assertEqual( - log.records[0].msg, - ( - "Got 'data:,b' among start requests. Only requests and items " - "are supported. It will be ignored." - ), + assert len(log.records) == 2 + assert log.records[0].msg == ( + "Got 'data:,b' among start requests. Only requests and items " + "are supported. It will be ignored." ) - self.assertTrue( - re.match( - ( - r"^Got <object object at 0x[0-9a-fA-F]+> among start " - r"requests\. Only requests and items are supported\. It " - r"will be ignored\.$" - ), - log.records[1].msg, - ) + assert re.match( + ( + r"^Got <object object at 0x[0-9a-fA-F]+> among start " + r"requests\. Only requests and items are supported\. It " + r"will be ignored\.$" + ), + log.records[1].msg, ) @defer.inlineCallbacks @@ -227,10 +218,9 @@ class CrawlTestCase(TestCase): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(BrokenStartRequestsSpider, settings) yield crawler.crawl(mockserver=self.mockserver) - self.assertTrue( - crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), - crawler.spider.seedsseen, - ) + assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index( + 99 + ), crawler.spider.seedsseen @defer.inlineCallbacks def test_start_requests_dupes(self): @@ -239,7 +229,7 @@ class CrawlTestCase(TestCase): yield crawler.crawl( dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver ) - self.assertEqual(crawler.spider.visited, 6) + assert crawler.spider.visited == 6 crawler = get_crawler(DuplicateStartRequestsSpider, settings) yield crawler.crawl( @@ -248,7 +238,7 @@ class CrawlTestCase(TestCase): dupe_factor=4, mockserver=self.mockserver, ) - self.assertEqual(crawler.spider.visited, 3) + assert crawler.spider.visited == 3 @defer.inlineCallbacks def test_unbounded_response(self): @@ -282,7 +272,7 @@ with multiples lines yield crawler.crawl( self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver ) - self.assertEqual(str(log).count("Got response 200"), 1) + assert str(log).count("Got response 200") == 1 @defer.inlineCallbacks def test_retry_conn_lost(self): @@ -305,8 +295,8 @@ with multiples lines self._assert_retried(log) def _assert_retried(self, log): - self.assertEqual(str(log).count("Retrying"), 2) - self.assertEqual(str(log).count("Gave up retrying"), 1) + assert str(log).count("Retrying") == 2 + assert str(log).count("Gave up retrying") == 1 @defer.inlineCallbacks def test_referer_header(self): @@ -321,20 +311,20 @@ with multiples lines crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=req0, mockserver=self.mockserver) # basic asserts in case of weird communication errors - self.assertIn("responses", crawler.spider.meta) - self.assertNotIn("failures", crawler.spider.meta) + assert "responses" in crawler.spider.meta + assert "failures" not in crawler.spider.meta # start requests doesn't set Referer header echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) - self.assertNotIn("Referer", echo0["headers"]) + assert "Referer" not in echo0["headers"] # following request sets Referer to start request url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) - self.assertEqual(echo1["headers"].get("Referer"), [req0.url]) + assert echo1["headers"].get("Referer") == [req0.url] # next request avoids Referer header echo2 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) - self.assertNotIn("Referer", echo2["headers"]) + assert "Referer" not in echo2["headers"] # last request explicitly sets a Referer header echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body)) - self.assertEqual(echo3["headers"].get("Referer"), ["http://example.com"]) + assert echo3["headers"].get("Referer") == ["http://example.com"] @defer.inlineCallbacks def test_engine_status(self): @@ -349,10 +339,10 @@ with multiples lines yield crawler.crawl( seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver ) - self.assertEqual(len(est), 1, est) + assert len(est) == 1, est s = dict(est[0]) - self.assertEqual(s["engine.spider.name"], crawler.spider.name) - self.assertEqual(s["len(engine.scraper.slot.active)"], 1) + assert s["engine.spider.name"] == crawler.spider.name + assert s["len(engine.scraper.slot.active)"] == 1 @defer.inlineCallbacks def test_format_engine_status(self): @@ -367,7 +357,7 @@ with multiples lines yield crawler.crawl( seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver ) - self.assertEqual(len(est), 1, est) + assert len(est) == 1, est est = est[0].split("\n")[2:-2] # remove header & footer # convert to dict est = [x.split(":") for x in est] @@ -376,8 +366,8 @@ with multiples lines it = iter(est) s = dict(zip(it, it)) - self.assertEqual(s["engine.spider.name"], crawler.spider.name) - self.assertEqual(s["len(engine.scraper.slot.active)"], "1") + assert s["engine.spider.name"] == crawler.spider.name + assert s["len(engine.scraper.slot.active)"] == "1" @defer.inlineCallbacks def test_graceful_crawl_error_handling(self): @@ -398,7 +388,7 @@ with multiples lines crawler = get_crawler(FaultySpider) yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) - self.assertFalse(crawler.crawling) + assert not crawler.crawling @defer.inlineCallbacks def test_open_spider_error_on_faulty_pipeline(self): @@ -414,7 +404,7 @@ with multiples lines ), ZeroDivisionError, ) - self.assertFalse(crawler.crawling) + assert not crawler.crawling @defer.inlineCallbacks def test_crawlerrunner_accepts_crawler(self): @@ -426,7 +416,7 @@ with multiples lines self.mockserver.url("/status?n=200"), mockserver=self.mockserver, ) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in str(log) @defer.inlineCallbacks def test_crawl_multiple(self): @@ -446,10 +436,10 @@ with multiples lines yield runner.join() self._assert_retried(log) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in str(log) -class CrawlSpiderTestCase(TestCase): +class TestCrawlSpider(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -480,9 +470,9 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("[parse] status 200 (foo: None)", str(log)) - self.assertIn("[parse] status 201 (foo: None)", str(log)) - self.assertIn("[parse] status 202 (foo: bar)", str(log)) + assert "[parse] status 200 (foo: None)" in str(log) + assert "[parse] status 201 (foo: None)" in str(log) + assert "[parse] status 202 (foo: bar)" in str(log) @defer.inlineCallbacks def test_crawlspider_with_async_callback(self): @@ -490,9 +480,9 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("[parse_async] status 200 (foo: None)", str(log)) - self.assertIn("[parse_async] status 201 (foo: None)", str(log)) - self.assertIn("[parse_async] status 202 (foo: bar)", str(log)) + assert "[parse_async] status 200 (foo: None)" in str(log) + assert "[parse_async] status 201 (foo: None)" in str(log) + assert "[parse_async] status 202 (foo: bar)" in str(log) @defer.inlineCallbacks def test_crawlspider_with_async_generator_callback(self): @@ -500,9 +490,9 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log)) - self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log)) - self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log)) + assert "[parse_async_gen] status 200 (foo: None)" in str(log) + assert "[parse_async_gen] status 201 (foo: None)" in str(log) + assert "[parse_async_gen] status 202 (foo: bar)" in str(log) @defer.inlineCallbacks def test_crawlspider_with_errback(self): @@ -510,12 +500,12 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("[parse] status 200 (foo: None)", str(log)) - self.assertIn("[parse] status 201 (foo: None)", str(log)) - self.assertIn("[parse] status 202 (foo: bar)", str(log)) - self.assertIn("[errback] status 404", str(log)) - self.assertIn("[errback] status 500", str(log)) - self.assertIn("[errback] status 501", str(log)) + assert "[parse] status 200 (foo: None)" in str(log) + assert "[parse] status 201 (foo: None)" in str(log) + assert "[parse] status 202 (foo: bar)" in str(log) + assert "[errback] status 404" in str(log) + assert "[errback] status 500" in str(log) + assert "[errback] status 501" in str(log) @defer.inlineCallbacks def test_crawlspider_process_request_cb_kwargs(self): @@ -523,9 +513,9 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("[parse] status 200 (foo: process_request)", str(log)) - self.assertIn("[parse] status 201 (foo: process_request)", str(log)) - self.assertIn("[parse] status 202 (foo: bar)", str(log)) + assert "[parse] status 200 (foo: process_request)" in str(log) + assert "[parse] status 201 (foo: process_request)" in str(log) + assert "[parse] status 202 (foo: bar)" in str(log) @defer.inlineCallbacks def test_async_def_parse(self): @@ -534,7 +524,7 @@ class CrawlSpiderTestCase(TestCase): yield crawler.crawl( self.mockserver.url("/status?n=200"), mockserver=self.mockserver ) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in str(log) @pytest.mark.only_asyncio @defer.inlineCallbacks @@ -549,15 +539,15 @@ class CrawlSpiderTestCase(TestCase): yield crawler.crawl( self.mockserver.url("/status?n=200"), mockserver=self.mockserver ) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in str(log) @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_items_list(self): log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) - self.assertIn("Got response 200", str(log)) - self.assertIn({"id": 1}, items) - self.assertIn({"id": 2}, items) + assert "Got response 200" in str(log) + assert {"id": 1} in items + assert {"id": 2} in items @pytest.mark.only_asyncio @defer.inlineCallbacks @@ -573,81 +563,81 @@ class CrawlSpiderTestCase(TestCase): yield crawler.crawl( self.mockserver.url("/status?n=200"), mockserver=self.mockserver ) - self.assertIn("Got response 200", str(log)) - self.assertIn({"foo": 42}, items) + assert "Got response 200" in str(log) + assert {"foo": 42} in items @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in str(log) itemcount = stats.get_value("item_scraped_count") - self.assertEqual(itemcount, 1) + assert itemcount == 1 @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) - self.assertIn("Got response 200", str(log)) + assert "Got response 200" in str(log) itemcount = stats.get_value("item_scraped_count") - self.assertEqual(itemcount, 10) + assert itemcount == 10 for i in range(10): - self.assertIn({"foo": i}, items) + assert {"foo": i} in items @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_exc(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) log = str(log) - self.assertIn("Spider error processing", log) - self.assertIn("ValueError", log) + assert "Spider error processing" in log + assert "ValueError" in log itemcount = stats.get_value("item_scraped_count") - self.assertEqual(itemcount, 7) + assert itemcount == 7 for i in range(7): - self.assertIn({"foo": i}, items) + assert {"foo": i} in items @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) itemcount = stats.get_value("item_scraped_count") - self.assertEqual(itemcount, 156) + assert itemcount == 156 # some random items for i in [1, 4, 21, 22, 207, 311]: - self.assertIn({"index": i}, items) + assert {"index": i} in items for i in [10, 30, 122]: - self.assertIn({"index2": i}, items) + assert {"index2": i} in items @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): - self.assertIn(f"Got response 200, req_id {req_id}", str(log)) + assert f"Got response 200, req_id {req_id}" in str(log) @pytest.mark.only_not_asyncio @defer.inlineCallbacks def test_async_def_deferred_direct(self): _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) - self.assertEqual(items, [{"code": 200}]) + assert items == [{"code": 200}] @pytest.mark.only_asyncio @defer.inlineCallbacks def test_async_def_deferred_wrapped(self): log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) - self.assertEqual(items, [{"code": 200}]) + assert items == [{"code": 200}] @defer.inlineCallbacks def test_async_def_deferred_maybe_wrapped(self): _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider) - self.assertEqual(items, [{"code": 200}]) + assert items == [{"code": 200}] @defer.inlineCallbacks def test_response_ssl_certificate_none(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=False) yield crawler.crawl(seed=url, mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta["responses"][0].certificate) + assert crawler.spider.meta["responses"][0].certificate is None @defer.inlineCallbacks def test_response_ssl_certificate(self): @@ -655,9 +645,9 @@ class CrawlSpiderTestCase(TestCase): url = self.mockserver.url("/echo?body=test", is_secure=True) yield crawler.crawl(seed=url, mockserver=self.mockserver) cert = crawler.spider.meta["responses"][0].certificate - self.assertIsInstance(cert, Certificate) - self.assertEqual(cert.getSubject().commonName, b"localhost") - self.assertEqual(cert.getIssuer().commonName, b"localhost") + assert isinstance(cert, Certificate) + assert cert.getSubject().commonName == b"localhost" + assert cert.getIssuer().commonName == b"localhost" @pytest.mark.xfail( reason="Responses with no body return early and contain no certificate" @@ -668,9 +658,9 @@ class CrawlSpiderTestCase(TestCase): url = self.mockserver.url("/status?n=200", is_secure=True) yield crawler.crawl(seed=url, mockserver=self.mockserver) cert = crawler.spider.meta["responses"][0].certificate - self.assertIsInstance(cert, Certificate) - self.assertEqual(cert.getSubject().commonName, b"localhost") - self.assertEqual(cert.getIssuer().commonName, b"localhost") + assert isinstance(cert, Certificate) + assert cert.getSubject().commonName == b"localhost" + assert cert.getIssuer().commonName == b"localhost" @defer.inlineCallbacks def test_dns_server_ip_address_none(self): @@ -678,7 +668,7 @@ class CrawlSpiderTestCase(TestCase): url = self.mockserver.url("/status?n=200") yield crawler.crawl(seed=url, mockserver=self.mockserver) ip_address = crawler.spider.meta["responses"][0].ip_address - self.assertIsNone(ip_address) + assert ip_address is None @defer.inlineCallbacks def test_dns_server_ip_address(self): @@ -687,61 +677,57 @@ class CrawlSpiderTestCase(TestCase): expected_netloc, _ = urlparse(url).netloc.split(":") yield crawler.crawl(seed=url, mockserver=self.mockserver) ip_address = crawler.spider.meta["responses"][0].ip_address - self.assertIsInstance(ip_address, IPv4Address) - self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) + assert isinstance(ip_address, IPv4Address) + assert str(ip_address) == gethostbyname(expected_netloc) @defer.inlineCallbacks def test_bytes_received_stop_download_callback(self): crawler = get_crawler(BytesReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta.get("failure")) - self.assertIsInstance(crawler.spider.meta["response"], Response) - self.assertEqual( - crawler.spider.meta["response"].body, - crawler.spider.meta.get("bytes_received"), + assert crawler.spider.meta.get("failure") is None + assert isinstance(crawler.spider.meta["response"], Response) + assert crawler.spider.meta["response"].body == crawler.spider.meta.get( + "bytes_received" ) - self.assertLess( - len(crawler.spider.meta["response"].body), - crawler.spider.full_response_length, + assert ( + len(crawler.spider.meta["response"].body) + < crawler.spider.full_response_length ) @defer.inlineCallbacks def test_bytes_received_stop_download_errback(self): crawler = get_crawler(BytesReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta.get("response")) - self.assertIsInstance(crawler.spider.meta["failure"], Failure) - self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) - self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) - self.assertEqual( - crawler.spider.meta["failure"].value.response.body, - crawler.spider.meta.get("bytes_received"), - ) - self.assertLess( - len(crawler.spider.meta["failure"].value.response.body), - crawler.spider.full_response_length, + assert crawler.spider.meta.get("response") is None + assert isinstance(crawler.spider.meta["failure"], Failure) + assert isinstance(crawler.spider.meta["failure"].value, StopDownload) + assert isinstance(crawler.spider.meta["failure"].value.response, Response) + assert crawler.spider.meta[ + "failure" + ].value.response.body == crawler.spider.meta.get("bytes_received") + assert ( + len(crawler.spider.meta["failure"].value.response.body) + < crawler.spider.full_response_length ) @defer.inlineCallbacks def test_headers_received_stop_download_callback(self): crawler = get_crawler(HeadersReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta.get("failure")) - self.assertIsInstance(crawler.spider.meta["response"], Response) - self.assertEqual( - crawler.spider.meta["response"].headers, - crawler.spider.meta.get("headers_received"), + assert crawler.spider.meta.get("failure") is None + assert isinstance(crawler.spider.meta["response"], Response) + assert crawler.spider.meta["response"].headers == crawler.spider.meta.get( + "headers_received" ) @defer.inlineCallbacks def test_headers_received_stop_download_errback(self): crawler = get_crawler(HeadersReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) - self.assertIsNone(crawler.spider.meta.get("response")) - self.assertIsInstance(crawler.spider.meta["failure"], Failure) - self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) - self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) - self.assertEqual( - crawler.spider.meta["failure"].value.response.headers, - crawler.spider.meta.get("headers_received"), - ) + assert crawler.spider.meta.get("response") is None + assert isinstance(crawler.spider.meta["failure"], Failure) + assert isinstance(crawler.spider.meta["failure"].value, StopDownload) + assert isinstance(crawler.spider.meta["failure"].value.response, Response) + assert crawler.spider.meta[ + "failure" + ].value.response.headers == crawler.spider.meta.get("headers_received") diff --git a/tests/test_crawler.py b/tests/test_crawler.py index df5ebfa7b..0bbcc0843 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -39,13 +39,13 @@ def get_raw_crawler(spidercls=None, settings_dict=None): return Crawler(spidercls or DefaultSpider, settings) -class BaseCrawlerTest(unittest.TestCase): +class TestBaseCrawler(unittest.TestCase): def assertOptionIsDefault(self, settings, key): - self.assertIsInstance(settings, Settings) - self.assertEqual(settings[key], getattr(default_settings, key)) + assert isinstance(settings, Settings) + assert settings[key] == getattr(default_settings, key) -class CrawlerTestCase(BaseCrawlerTest): +class TestCrawler(TestBaseCrawler): def test_populate_spidercls_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"} @@ -58,16 +58,16 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = Crawler(CustomSettingsSpider, settings) crawler._apply_settings() - self.assertEqual(crawler.settings.get("TEST1"), "spider") - self.assertEqual(crawler.settings.get("TEST2"), "spider") - self.assertEqual(crawler.settings.get("TEST3"), "project") + assert crawler.settings.get("TEST1") == "spider" + assert crawler.settings.get("TEST2") == "spider" + assert crawler.settings.get("TEST3") == "project" - self.assertFalse(settings.frozen) - self.assertTrue(crawler.settings.frozen) + assert not settings.frozen + assert crawler.settings.frozen def test_crawler_accepts_dict(self): crawler = get_crawler(DefaultSpider, {"foo": "bar"}) - self.assertEqual(crawler.settings["foo"], "bar") + assert crawler.settings["foo"] == "bar" self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") def test_crawler_accepts_None(self): @@ -107,23 +107,23 @@ class CrawlerTestCase(BaseCrawlerTest): }, } crawler = get_crawler(settings_dict=settings) - self.assertEqual(len(TrackingAddon.instances), 1) + assert len(TrackingAddon.instances) == 1 expected = TrackingAddon.instances[-1] addon = crawler.get_addon(TrackingAddon) - self.assertEqual(addon, expected) + assert addon == expected addon = crawler.get_addon(DefaultSpider) - self.assertIsNone(addon) + assert addon is None addon = crawler.get_addon(ParentAddon) - self.assertEqual(addon, expected) + assert addon == expected class ChildAddon(TrackingAddon): pass addon = crawler.get_addon(ChildAddon) - self.assertIsNone(addon) + assert addon is None @inlineCallbacks def test_get_downloader_middleware(self): @@ -162,18 +162,18 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingDownloaderMiddleware yield crawler.crawl() - self.assertEqual(len(TrackingDownloaderMiddleware.instances), 1) - self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1]) + assert len(TrackingDownloaderMiddleware.instances) == 1 + assert MySpider.result == TrackingDownloaderMiddleware.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentDownloaderMiddleware yield crawler.crawl() - self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1]) + assert MySpider.result == TrackingDownloaderMiddleware.instances[-1] class ChildDownloaderMiddleware(TrackingDownloaderMiddleware): pass @@ -181,7 +181,7 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildDownloaderMiddleware yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None def test_get_downloader_middleware_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) @@ -242,18 +242,18 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingExtension yield crawler.crawl() - self.assertEqual(len(TrackingExtension.instances), 1) - self.assertEqual(MySpider.result, TrackingExtension.instances[-1]) + assert len(TrackingExtension.instances) == 1 + assert MySpider.result == TrackingExtension.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentExtension yield crawler.crawl() - self.assertEqual(MySpider.result, TrackingExtension.instances[-1]) + assert MySpider.result == TrackingExtension.instances[-1] class ChildExtension(TrackingExtension): pass @@ -261,7 +261,7 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildExtension yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None def test_get_extension_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) @@ -322,18 +322,18 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingItemPipeline yield crawler.crawl() - self.assertEqual(len(TrackingItemPipeline.instances), 1) - self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1]) + assert len(TrackingItemPipeline.instances) == 1 + assert MySpider.result == TrackingItemPipeline.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentItemPipeline yield crawler.crawl() - self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1]) + assert MySpider.result == TrackingItemPipeline.instances[-1] class ChildItemPipeline(TrackingItemPipeline): pass @@ -341,7 +341,7 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildItemPipeline yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None def test_get_item_pipeline_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) @@ -402,18 +402,18 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingSpiderMiddleware yield crawler.crawl() - self.assertEqual(len(TrackingSpiderMiddleware.instances), 1) - self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1]) + assert len(TrackingSpiderMiddleware.instances) == 1 + assert MySpider.result == TrackingSpiderMiddleware.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentSpiderMiddleware yield crawler.crawl() - self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1]) + assert MySpider.result == TrackingSpiderMiddleware.instances[-1] class ChildSpiderMiddleware(TrackingSpiderMiddleware): pass @@ -421,7 +421,7 @@ class CrawlerTestCase(BaseCrawlerTest): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildSpiderMiddleware yield crawler.crawl() - self.assertIsNone(MySpider.result) + assert MySpider.result is None def test_get_spider_middleware_not_crawling(self): crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) @@ -446,7 +446,7 @@ class CrawlerTestCase(BaseCrawlerTest): yield crawler.crawl() -class SpiderSettingsTestCase(unittest.TestCase): +class TestSpiderSettings: def test_spider_custom_settings(self): class MySpider(scrapy.Spider): name = "spider" @@ -454,10 +454,10 @@ class SpiderSettingsTestCase(unittest.TestCase): crawler = get_crawler(MySpider) enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] - self.assertIn(AutoThrottle, enabled_exts) + assert AutoThrottle in enabled_exts -class CrawlerLoggingTestCase(unittest.TestCase): +class TestCrawlerLogging: def test_no_root_handler_installed(self): handler = get_scrapy_root_handler() if handler is not None: @@ -469,8 +469,8 @@ class CrawlerLoggingTestCase(unittest.TestCase): get_crawler(MySpider) assert get_scrapy_root_handler() is None - def test_spider_custom_settings_log_level(self): - log_file = Path(self.mktemp()) + def test_spider_custom_settings_log_level(self, tmp_path): + log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") class MySpider(scrapy.Spider): @@ -481,9 +481,9 @@ class CrawlerLoggingTestCase(unittest.TestCase): } configure_logging() - self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) + assert get_scrapy_root_handler().level == logging.DEBUG crawler = get_crawler(MySpider) - self.assertEqual(get_scrapy_root_handler().level, logging.INFO) + assert get_scrapy_root_handler().level == logging.INFO info_count = crawler.stats.get_value("log_count/INFO") logging.debug("debug message") logging.info("info message") @@ -492,18 +492,18 @@ class CrawlerLoggingTestCase(unittest.TestCase): logged = log_file.read_text(encoding="utf-8") - self.assertIn("previous message", logged) - self.assertNotIn("debug message", logged) - self.assertIn("info message", logged) - self.assertIn("warning message", logged) - self.assertIn("error message", logged) - self.assertEqual(crawler.stats.get_value("log_count/ERROR"), 1) - self.assertEqual(crawler.stats.get_value("log_count/WARNING"), 1) - self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1) - self.assertEqual(crawler.stats.get_value("log_count/DEBUG", 0), 0) + assert "previous message" in logged + assert "debug message" not in logged + assert "info message" in logged + assert "warning message" in logged + assert "error message" in logged + assert crawler.stats.get_value("log_count/ERROR") == 1 + assert crawler.stats.get_value("log_count/WARNING") == 1 + assert crawler.stats.get_value("log_count/INFO") - info_count == 1 + assert crawler.stats.get_value("log_count/DEBUG", 0) == 0 - def test_spider_custom_settings_log_append(self): - log_file = Path(self.mktemp()) + def test_spider_custom_settings_log_append(self, tmp_path): + log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") class MySpider(scrapy.Spider): @@ -519,8 +519,8 @@ class CrawlerLoggingTestCase(unittest.TestCase): logged = log_file.read_text(encoding="utf-8") - self.assertNotIn("previous message", logged) - self.assertIn("debug message", logged) + assert "previous message" not in logged + assert "debug message" in logged class SpiderLoaderWithWrongInterface: @@ -532,7 +532,7 @@ class CustomSpiderLoader(SpiderLoader): pass -class CrawlerRunnerTestCase(BaseCrawlerTest): +class TestCrawlerRunner(TestBaseCrawler): def test_spider_manager_verify_interface(self): settings = Settings( { @@ -544,7 +544,7 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): def test_crawler_runner_accepts_dict(self): runner = CrawlerRunner({"foo": "bar"}) - self.assertEqual(runner.settings["foo"], "bar") + assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") def test_crawler_runner_accepts_None(self): @@ -552,10 +552,10 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") -class CrawlerProcessTest(BaseCrawlerTest): +class TestCrawlerProcess(TestBaseCrawler): def test_crawler_process_accepts_dict(self): runner = CrawlerProcess({"foo": "bar"}) - self.assertEqual(runner.settings["foo"], "bar") + assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") def test_crawler_process_accepts_None(self): @@ -579,7 +579,7 @@ class NoRequestsSpider(scrapy.Spider): @pytest.mark.usefixtures("reactor_pytest") -class CrawlerRunnerHasSpider(unittest.TestCase): +class TestCrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): return CrawlerRunner() @@ -587,14 +587,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def test_crawler_runner_bootstrap_successful(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) - self.assertFalse(runner.bootstrap_failed) + assert not runner.bootstrap_failed @inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) yield runner.crawl(NoRequestsSpider) - self.assertFalse(runner.bootstrap_failed) + assert not runner.bootstrap_failed @inlineCallbacks def test_crawler_runner_bootstrap_failed(self): @@ -605,9 +605,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase): except ValueError: pass else: - self.fail("Exception should be raised from spider") + pytest.fail("Exception should be raised from spider") - self.assertTrue(runner.bootstrap_failed) + assert runner.bootstrap_failed @inlineCallbacks def test_crawler_runner_bootstrap_failed_for_several(self): @@ -618,11 +618,11 @@ class CrawlerRunnerHasSpider(unittest.TestCase): except ValueError: pass else: - self.fail("Exception should be raised from spider") + pytest.fail("Exception should be raised from spider") yield runner.crawl(NoRequestsSpider) - self.assertTrue(runner.bootstrap_failed) + assert runner.bootstrap_failed @inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): @@ -664,31 +664,34 @@ class ScriptRunnerMixin: return stderr.decode("utf-8") -class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): +class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = Path(__file__).parent.resolve() / "CrawlerProcess" def test_simple(self): log = self.run_script("simple.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log ) def test_multi(self): log = self.run_script("multi.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log ) - self.assertNotIn("ReactorAlreadyInstalledError", log) + assert "ReactorAlreadyInstalledError" not in log def test_reactor_default(self): log = self.run_script("reactor_default.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log ) - self.assertNotIn("ReactorAlreadyInstalledError", log) + assert "ReactorAlreadyInstalledError" not in log def test_reactor_default_twisted_reactor_select(self): log = self.run_script("reactor_default_twisted_reactor_select.py") @@ -703,50 +706,46 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): # If that ever becomes the case on more platforms (i.e. if Linux # also starts using the select reactor by default in a future # version of Twisted), then we will need to rethink this test. - self.assertIn("Spider closed (finished)", log) + assert "Spider closed (finished)" in log else: - self.assertNotIn("Spider closed (finished)", log) - self.assertIn( - ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ), - log, - ) + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log def test_reactor_select(self): log = self.run_script("reactor_select.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn("ReactorAlreadyInstalledError", log) + assert "Spider closed (finished)" in log + assert "ReactorAlreadyInstalledError" not in log def test_reactor_select_twisted_reactor_select(self): log = self.run_script("reactor_select_twisted_reactor_select.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn("ReactorAlreadyInstalledError", log) + assert "Spider closed (finished)" in log + assert "ReactorAlreadyInstalledError" not in log def test_reactor_select_subclass_twisted_reactor_select(self): log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") - self.assertNotIn("Spider closed (finished)", log) - self.assertIn( - ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ), - log, - ) + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log def test_asyncio_enabled_no_reactor(self): log = self.run_script("asyncio_enabled_no_reactor.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) def test_asyncio_enabled_reactor(self): log = self.run_script("asyncio_enabled_reactor.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) @pytest.mark.skipif( @@ -755,129 +754,129 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) def test_ipv6_default_name_resolver(self): log = self.run_script("default_name_resolver.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", - log, + assert "Spider closed (finished)" in log + assert ( + "'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1," + in log ) - self.assertIn( - "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", - log, + assert ( + "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1." + in log ) def test_caching_hostname_resolver_ipv6(self): log = self.run_script("caching_hostname_resolver_ipv6.py") - self.assertIn("Spider closed (finished)", log) - self.assertNotIn("twisted.internet.error.DNSLookupError", log) + assert "Spider closed (finished)" in log + assert "twisted.internet.error.DNSLookupError" not in log def test_caching_hostname_resolver_finite_execution(self): with MockServer() as mock_server: http_address = mock_server.http_address.replace("0.0.0.0", "127.0.0.1") log = self.run_script("caching_hostname_resolver.py", http_address) - self.assertIn("Spider closed (finished)", log) - self.assertNotIn("ERROR: Error downloading", log) - self.assertNotIn("TimeoutError", log) - self.assertNotIn("twisted.internet.error.DNSLookupError", log) + assert "Spider closed (finished)" in log + assert "ERROR: Error downloading" not in log + assert "TimeoutError" not in log + assert "twisted.internet.error.DNSLookupError" not in log def test_twisted_reactor_select(self): log = self.run_script("twisted_reactor_select.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.selectreactor.SelectReactor", log - ) + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log @pytest.mark.skipif( platform.system() == "Windows", reason="PollReactor is not supported on Windows" ) def test_twisted_reactor_poll(self): log = self.run_script("twisted_reactor_poll.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log def test_twisted_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) def test_twisted_reactor_asyncio_custom_settings(self): log = self.run_script("twisted_reactor_custom_settings.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) def test_twisted_reactor_asyncio_custom_settings_same(self): log = self.run_script("twisted_reactor_custom_settings_same.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) def test_twisted_reactor_asyncio_custom_settings_conflict(self): log = self.run_script("twisted_reactor_custom_settings_conflict.py") - self.assertIn( - "Using reactor: twisted.internet.selectreactor.SelectReactor", log - ) - self.assertIn( - "(twisted.internet.selectreactor.SelectReactor) does not match the requested one", - log, + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" + in log ) @pytest.mark.requires_uvloop def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) - self.assertIn("Using asyncio event loop: uvloop.Loop", log) + assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) - self.assertIn("Using asyncio event loop: uvloop.Loop", log) - self.assertIn("async pipeline opened!", log) + assert "Using asyncio event loop: uvloop.Loop" in log + assert "async pipeline opened!" in log @pytest.mark.requires_uvloop def test_asyncio_enabled_reactor_same_loop(self): log = self.run_script("asyncio_enabled_reactor_same_loop.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) - self.assertIn("Using asyncio event loop: uvloop.Loop", log) + assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop def test_asyncio_enabled_reactor_different_loop(self): log = self.run_script("asyncio_enabled_reactor_different_loop.py") - self.assertNotIn("Spider closed (finished)", log) - self.assertIn( - ( - "does not match the one specified in the ASYNCIO_EVENT_LOOP " - "setting (uvloop.Loop)" - ), - log, - ) + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) - self.assertNotIn("Using asyncio event loop: uvloop.Loop", log) - self.assertIn("async pipeline opened!", log) + assert "Using asyncio event loop: uvloop.Loop" not in log + assert "async pipeline opened!" in log def test_args_change_settings(self): log = self.run_script("args_settings.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn("The value of FOO is 42", log) + assert "Spider closed (finished)" in log + assert "The value of FOO is 42" in log def test_shutdown_graceful(self): sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK @@ -910,23 +909,23 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.wait() -class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): +class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" def test_response_ip_address(self): log = self.run_script("ip_address.py") - self.assertIn("INFO: Spider closed (finished)", log) - self.assertIn("INFO: Host: not.a.real.domain", log) - self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log) - self.assertIn("INFO: IP address: 127.0.0.1", log) + assert "INFO: Spider closed (finished)" in log + assert "INFO: Host: not.a.real.domain" in log + assert "INFO: Type: <class 'ipaddress.IPv4Address'>" in log + assert "INFO: IP address: 127.0.0.1" in log def test_change_default_reactor(self): log = self.run_script("change_reactor.py") - self.assertIn( - "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", - log, + assert ( + "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log ) - self.assertIn("DEBUG: Using asyncio event loop", log) + assert "DEBUG: Using asyncio event loop" in log @pytest.mark.parametrize( diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index a39ed0694..162747581 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -4,11 +4,12 @@ from configparser import ConfigParser from importlib import import_module from pathlib import Path +import pytest from twisted import version as twisted_version from twisted.trial import unittest -class ScrapyUtilsTest(unittest.TestCase): +class TestScrapyUtils: def test_required_openssl_version(self): try: module = import_module("OpenSSL") @@ -27,7 +28,7 @@ class ScrapyUtilsTest(unittest.TestCase): See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 """ if not os.environ.get("_SCRAPY_PINNED", None): - self.skipTest("Not in a pinned environment") + pytest.skip("Not in a pinned environment") tox_config_file_path = Path(__file__).parent / ".." / "tox.ini" config_parser = ConfigParser() @@ -36,4 +37,4 @@ class ScrapyUtilsTest(unittest.TestCase): match = re.search(pattern, config_parser["pinned"]["deps"]) pinned_twisted_version_string = match[1] - self.assertEqual(twisted_version.short(), pinned_twisted_version_string) + assert twisted_version.short() == pinned_twisted_version_string diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 4f8b005d7..15b3ad5af 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -76,7 +76,7 @@ class CrawlTestCase(TestCase): for k, v in slots.items() } - self.assertTrue(max(list(error_delta.values())) < tolerance) + assert max(list(error_delta.values())) < tolerance def test_params(): diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 703c23529..d5e1b37f7 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -2,7 +2,6 @@ import hashlib import shutil import sys import tempfile -import unittest from pathlib import Path from warnings import catch_warnings @@ -39,7 +38,7 @@ class DirectDupeFilter: method = "n/a" -class RFPDupeFilterTest(unittest.TestCase): +class TestRFPDupeFilter: def test_df_from_crawler_scheduler(self): settings = { "DUPEFILTER_DEBUG": True, @@ -47,8 +46,8 @@ class RFPDupeFilterTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) - self.assertTrue(scheduler.df.debug) - self.assertEqual(scheduler.df.method, "from_crawler") + assert scheduler.df.debug + assert scheduler.df.method == "from_crawler" def test_df_direct_scheduler(self): settings = { @@ -56,7 +55,7 @@ class RFPDupeFilterTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) - self.assertEqual(scheduler.df.method, "n/a") + assert scheduler.df.method == "n/a" def test_filter(self): dupefilter = _get_dupefilter() @@ -256,16 +255,16 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close("finished") -class BaseDupeFilterTestCase(unittest.TestCase): +class TestBaseDupeFilter: def test_log_deprecation(self): dupefilter = _get_dupefilter( settings={"DUPEFILTER_CLASS": BaseDupeFilter}, ) with catch_warnings(record=True) as warning_list: dupefilter.log(None, None) - self.assertEqual(len(warning_list), 1) - self.assertEqual( - str(warning_list[0].message), - "Calling BaseDupeFilter.log() is deprecated.", + assert len(warning_list) == 1 + assert ( + str(warning_list[0].message) + == "Calling BaseDupeFilter.log() is deprecated." ) - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + assert warning_list[0].category == ScrapyDeprecationWarning diff --git a/tests/test_engine.py b/tests/test_engine.py index e9470493f..4bac8d273 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -243,8 +243,9 @@ class CrawlerRun: self.signals_caught[sig] = signalargs -class EngineTestBase(unittest.TestCase): - def _assert_visited_urls(self, run: CrawlerRun): +class TestEngineBase(unittest.TestCase): + @staticmethod + def _assert_visited_urls(run: CrawlerRun) -> None: must_be_visited = [ "/", "/redirect", @@ -259,8 +260,9 @@ class EngineTestBase(unittest.TestCase): f"URLs not visited: {list(urls_expected - urls_visited)}" ) - def _assert_scheduled_requests(self, run: CrawlerRun, count=None): - self.assertEqual(count, len(run.reqplug)) + @staticmethod + def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None: + assert len(run.reqplug) == count paths_expected = ["/item999.html", "/item2.html", "/item1.html"] @@ -270,101 +272,104 @@ class EngineTestBase(unittest.TestCase): scheduled_requests_count = len(run.reqplug) dropped_requests_count = len(run.reqdropped) responses_count = len(run.respplug) - self.assertEqual( - scheduled_requests_count, dropped_requests_count + responses_count - ) - self.assertEqual(len(run.reqreached), responses_count) + assert scheduled_requests_count == dropped_requests_count + responses_count + assert len(run.reqreached) == responses_count - def _assert_dropped_requests(self, run: CrawlerRun): - self.assertEqual(len(run.reqdropped), 1) + @staticmethod + def _assert_dropped_requests(run: CrawlerRun) -> None: + assert len(run.reqdropped) == 1 - def _assert_downloaded_responses(self, run: CrawlerRun, count): + @staticmethod + def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None: # response tests - self.assertEqual(count, len(run.respplug)) - self.assertEqual(count, len(run.reqreached)) + assert len(run.respplug) == count + assert len(run.reqreached) == count for response, _ in run.respplug: if run.getpath(response.url) == "/item999.html": - self.assertEqual(404, response.status) + assert response.status == 404 if run.getpath(response.url) == "/redirect": - self.assertEqual(302, response.status) + assert response.status == 302 - def _assert_items_error(self, run: CrawlerRun): - self.assertEqual(2, len(run.itemerror)) + @staticmethod + def _assert_items_error(run: CrawlerRun) -> None: + assert len(run.itemerror) == 2 for item, response, spider, failure in run.itemerror: - self.assertEqual(failure.value.__class__, ZeroDivisionError) - self.assertEqual(spider, run.spider) + assert failure.value.__class__ is ZeroDivisionError + assert spider == run.spider - self.assertEqual(item["url"], response.url) + assert item["url"] == response.url if "item1.html" in item["url"]: - self.assertEqual("Item 1 name", item["name"]) - self.assertEqual("100", item["price"]) + assert item["name"] == "Item 1 name" + assert item["price"] == "100" if "item2.html" in item["url"]: - self.assertEqual("Item 2 name", item["name"]) - self.assertEqual("200", item["price"]) + assert item["name"] == "Item 2 name" + assert item["price"] == "200" - def _assert_scraped_items(self, run: CrawlerRun): - self.assertEqual(2, len(run.itemresp)) + @staticmethod + def _assert_scraped_items(run: CrawlerRun) -> None: + assert len(run.itemresp) == 2 for item, response in run.itemresp: item = ItemAdapter(item) - self.assertEqual(item["url"], response.url) + assert item["url"] == response.url if "item1.html" in item["url"]: - self.assertEqual("Item 1 name", item["name"]) - self.assertEqual("100", item["price"]) + assert item["name"] == "Item 1 name" + assert item["price"] == "100" if "item2.html" in item["url"]: - self.assertEqual("Item 2 name", item["name"]) - self.assertEqual("200", item["price"]) + assert item["name"] == "Item 2 name" + assert item["price"] == "200" - def _assert_headers_received(self, run: CrawlerRun): + @staticmethod + def _assert_headers_received(run: CrawlerRun) -> None: for headers in run.headers.values(): - self.assertIn(b"Server", headers) - self.assertIn(b"TwistedWeb", headers[b"Server"]) - self.assertIn(b"Date", headers) - self.assertIn(b"Content-Type", headers) + assert b"Server" in headers + assert b"TwistedWeb" in headers[b"Server"] + assert b"Date" in headers + assert b"Content-Type" in headers - def _assert_bytes_received(self, run: CrawlerRun): - self.assertEqual(9, len(run.bytes)) + @staticmethod + def _assert_bytes_received(run: CrawlerRun) -> None: + assert len(run.bytes) == 9 for request, data in run.bytes.items(): joined_data = b"".join(data) if run.getpath(request.url) == "/": - self.assertEqual(joined_data, get_testdata("test_site", "index.html")) + assert joined_data == get_testdata("test_site", "index.html") elif run.getpath(request.url) == "/item1.html": - self.assertEqual(joined_data, get_testdata("test_site", "item1.html")) + assert joined_data == get_testdata("test_site", "item1.html") elif run.getpath(request.url) == "/item2.html": - self.assertEqual(joined_data, get_testdata("test_site", "item2.html")) + assert joined_data == get_testdata("test_site", "item2.html") elif run.getpath(request.url) == "/redirected": - self.assertEqual(joined_data, b"Redirected here") + assert joined_data == b"Redirected here" elif run.getpath(request.url) == "/redirect": - self.assertEqual( - joined_data, - b"\n<html>\n" + assert ( + joined_data == b"\n<html>\n" b" <head>\n" b' <meta http-equiv="refresh" content="0;URL=/redirected">\n' b" </head>\n" b' <body bgcolor="#FFFFFF" text="#000000">\n' b' <a href="/redirected">click here</a>\n' b" </body>\n" - b"</html>\n", + b"</html>\n" ) elif run.getpath(request.url) == "/tem999.html": - self.assertEqual( - joined_data, - b"\n<html>\n" + assert ( + joined_data == b"\n<html>\n" b" <head><title>404 - No Such Resource\n" b" \n" b"

No Such Resource

\n" b"

File not found.

\n" b" \n" - b"\n", + b"\n" ) elif run.getpath(request.url) == "/numbers": # signal was fired multiple times - self.assertTrue(len(data) > 1) + assert len(data) > 1 # bytes were received in order numbers = [str(x).encode("utf8") for x in range(2**18)] - self.assertEqual(joined_data, b"".join(numbers)) + assert joined_data == b"".join(numbers) - def _assert_signals_caught(self, run: CrawlerRun): + @staticmethod + def _assert_signals_caught(run: CrawlerRun) -> None: assert signals.engine_started in run.signals_caught assert signals.engine_stopped in run.signals_caught assert signals.spider_opened in run.signals_caught @@ -372,19 +377,14 @@ class EngineTestBase(unittest.TestCase): assert signals.spider_closed in run.signals_caught assert signals.headers_received in run.signals_caught - self.assertEqual( - {"spider": run.spider}, run.signals_caught[signals.spider_opened] - ) - self.assertEqual( - {"spider": run.spider}, run.signals_caught[signals.spider_idle] - ) - self.assertEqual( - {"spider": run.spider, "reason": "finished"}, - run.signals_caught[signals.spider_closed], - ) + assert {"spider": run.spider} == run.signals_caught[signals.spider_opened] + assert {"spider": run.spider} == run.signals_caught[signals.spider_idle] + assert {"spider": run.spider, "reason": "finished"} == run.signals_caught[ + signals.spider_closed + ] -class EngineTest(EngineTestBase): +class TestEngine(TestEngineBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( @@ -419,10 +419,9 @@ class EngineTest(EngineTestBase): def test_crawler_change_close_reason_on_idle(self): run = CrawlerRun(ChangeCloseReasonSpider) yield run.run() - self.assertEqual( - {"spider": run.spider, "reason": "custom_reason"}, - run.signals_caught[signals.spider_closed], - ) + assert {"spider": run.spider, "reason": "custom_reason"} == run.signals_caught[ + signals.spider_closed + ] @defer.inlineCallbacks def test_close_downloader(self): @@ -470,7 +469,7 @@ class EngineTest(EngineTestBase): finally: timer.cancel() - self.assertNotIn(b"Traceback", stderr) + assert b"Traceback" not in stderr def test_request_scheduled_signal(caplog): diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 5dd04c310..f09b0e091 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -7,8 +7,8 @@ from tests.test_engine import ( CrawlerRun, DataClassItemsSpider, DictItemsSpider, - EngineTestBase, MySpider, + TestEngineBase, ) @@ -18,7 +18,7 @@ class BytesReceivedCrawlerRun(CrawlerRun): raise StopDownload(fail=False) -class BytesReceivedEngineTest(EngineTestBase): +class TestBytesReceivedEngine(TestEngineBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( @@ -61,14 +61,15 @@ class BytesReceivedEngineTest(EngineTestBase): self._assert_headers_received(run) self._assert_bytes_received(run) - def _assert_bytes_received(self, run: CrawlerRun): - self.assertEqual(9, len(run.bytes)) + @staticmethod + def _assert_bytes_received(run: CrawlerRun) -> None: + assert len(run.bytes) == 9 for request, data in run.bytes.items(): joined_data = b"".join(data) - self.assertTrue(len(data) == 1) # signal was fired only once + assert len(data) == 1 # signal was fired only once if run.getpath(request.url) == "/numbers": # Received bytes are not the complete response. The exact amount depends # on the buffer size, which can vary, so we only check that the amount # of received bytes is strictly less than the full response. numbers = [str(x).encode("utf8") for x in range(2**18)] - self.assertTrue(len(joined_data) < len(b"".join(numbers))) + assert len(joined_data) < len(b"".join(numbers)) diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 06929d1e4..dbb0ea0d2 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -7,8 +7,8 @@ from tests.test_engine import ( CrawlerRun, DataClassItemsSpider, DictItemsSpider, - EngineTestBase, MySpider, + TestEngineBase, ) @@ -18,7 +18,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun): raise StopDownload(fail=False) -class HeadersReceivedEngineTest(EngineTestBase): +class TestHeadersReceivedEngine(TestEngineBase): @defer.inlineCallbacks def test_crawler(self): for spider in ( @@ -60,10 +60,12 @@ class HeadersReceivedEngineTest(EngineTestBase): self._assert_bytes_received(run) self._assert_headers_received(run) - def _assert_bytes_received(self, run: CrawlerRun): - self.assertEqual(0, len(run.bytes)) + @staticmethod + def _assert_bytes_received(run: CrawlerRun) -> None: + assert len(run.bytes) == 0 - def _assert_visited_urls(self, run: CrawlerRun): + @staticmethod + def _assert_visited_urls(run: CrawlerRun) -> None: must_be_visited = ["/", "/redirect", "/redirected"] urls_visited = {rp[0].url for rp in run.respplug} urls_expected = {run.geturl(p) for p in must_be_visited} diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 1b4ad2f2f..c3c03d6c3 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -36,7 +36,7 @@ from tests.mockserver import ( PayloadResource, ssl_context_factory, ) -from tests.test_core_downloader import ContextFactoryBaseTestCase +from tests.test_core_downloader import TestContextFactoryBase def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): @@ -63,7 +63,7 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class ScrapyHTTPPageGetterTests(unittest.TestCase): +class TestScrapyHTTPPageGetter: def test_earlyHeaders(self): # basic test stolen from twisted HTTPageGetter factory = client.ScrapyHTTPClientFactory( @@ -177,9 +177,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): protocol = client.ScrapyHTTPPageGetter() protocol.factory = factory protocol.makeConnection(transport) - self.assertEqual( - set(transport.value().splitlines()), set(testvalue.splitlines()) - ) + assert set(transport.value().splitlines()) == set(testvalue.splitlines()) return testvalue def test_non_standard_line_endings(self): @@ -192,9 +190,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): protocol.dataReceived(b"Hello: World\n") protocol.dataReceived(b"Foo: Bar\n") protocol.dataReceived(b"\n") - self.assertEqual( - protocol.headers, Headers({"Hello": ["World"], "Foo": ["Bar"]}) - ) + assert protocol.headers == Headers({"Hello": ["World"], "Foo": ["Bar"]}) class EncodingResource(resource.Resource): @@ -207,7 +203,7 @@ class EncodingResource(resource.Resource): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class WebClientTestCase(unittest.TestCase): +class TestWebClient(unittest.TestCase): def _listen(self, site): return reactor.listenTCP(0, site, interface="127.0.0.1") @@ -319,7 +315,7 @@ class WebClientTestCase(unittest.TestCase): return getPage(self.getURL("notsuchfile")).addCallback(self._cbNoSuchFile) def _cbNoSuchFile(self, pageData): - self.assertIn(b"404 - No Such Resource", pageData) + assert b"404 - No Such Resource" in pageData def testFactoryInfo(self): url = self.getURL("file") @@ -329,20 +325,20 @@ class WebClientTestCase(unittest.TestCase): return factory.deferred.addCallback(self._cbFactoryInfo, factory) def _cbFactoryInfo(self, ignoredResult, factory): - self.assertEqual(factory.status, b"200") - self.assertTrue(factory.version.startswith(b"HTTP/")) - self.assertEqual(factory.message, b"OK") - self.assertEqual(factory.response_headers[b"content-length"], b"10") + assert factory.status == b"200" + assert factory.version.startswith(b"HTTP/") + assert factory.message == b"OK" + assert factory.response_headers[b"content-length"] == b"10" def testRedirect(self): return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) def _cbRedirect(self, pageData): - self.assertEqual( - pageData, - b'\n\n \n \n' + assert ( + pageData + == b'\n\n \n \n' b' \n \n ' - b'click here\n \n\n', + b'click here\n \n\n' ) def test_encoding(self): @@ -356,14 +352,12 @@ class WebClientTestCase(unittest.TestCase): def _check_Encoding(self, response, original_body): content_encoding = to_unicode(response.headers[b"Content-Encoding"]) - self.assertEqual(content_encoding, EncodingResource.out_encoding) - self.assertEqual( - response.body.decode(content_encoding), to_unicode(original_body) - ) + assert content_encoding == EncodingResource.out_encoding + assert response.body.decode(content_encoding) == to_unicode(original_body) @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class WebClientSSLTestCase(ContextFactoryBaseTestCase): +class WebClientSSLTestCase(TestContextFactoryBase): def testPayload(self): s = "0123456789" * 10 return getPage(self.getURL("payload"), body=s).addCallback( From d2e5486d5a0ddfa9c202e39f5af98257a230d4f3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Mar 2025 13:20:42 +0500 Subject: [PATCH 1674/2083] Remove the Splash recommendation. --- docs/topics/dynamic-content.rst | 35 +++++++-------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 801f6d06d..65270433f 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -14,7 +14,7 @@ from it. If you fail to do that, and you can nonetheless access the desired data through the :ref:`DOM ` from your web browser, see -:ref:`topics-javascript-rendering`. +:ref:`topics-headless-browsing`. .. _topics-finding-data-source: @@ -97,7 +97,7 @@ it `. You can reproduce any request with Scrapy. However, some times reproducing all necessary requests may not seem efficient in developer time. If that is your case, and crawling speed is not a major concern for you, you can alternatively -consider :ref:`JavaScript pre-rendering `. +consider :ref:`using a headless browser `. If you get the expected response `sometimes`, but not always, the issue is probably not your request, but the target server. The target server might be @@ -220,9 +220,9 @@ data from it: >>> selector.css('var[name="data"]').get() 'value' -.. _topics-javascript-rendering: +.. _topics-headless-browsing: -Pre-rendering JavaScript +Using a headless browser ======================== On webpages that fetch data from additional requests, reproducing those @@ -232,29 +232,10 @@ network transfer. However, sometimes it can be really hard to reproduce certain requests. Or you may need something that no request can give you, such as a screenshot of a -webpage as seen in a web browser. +webpage as seen in a web browser. In this case using a `headless browser`_ will +help. -In these cases use the Splash_ JavaScript-rendering service, along with -`scrapy-splash`_ for seamless integration. - -Splash returns as HTML the :ref:`DOM ` of a webpage, so that -you can parse it with :ref:`selectors `. It provides great -flexibility through configuration_ or scripting_. - -If you need something beyond what Splash offers, such as interacting with the -DOM on-the-fly from Python code instead of using a previously-written script, -or handling multiple web browser windows, you might need to -:ref:`use a headless browser ` instead. - -.. _configuration: https://splash.readthedocs.io/en/stable/api.html -.. _scripting: https://splash.readthedocs.io/en/stable/scripting-tutorial.html - -.. _topics-headless-browsing: - -Using a headless browser -======================== - -A `headless browser`_ is a special web browser that provides an API for +A headless browser is a special web browser that provides an API for automation. By installing the :ref:`asyncio reactor `, it is possible to integrate ``asyncio``-based libraries which handle headless browsers. @@ -287,7 +268,6 @@ We recommend using `scrapy-playwright`_ for a better integration. .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29 .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets .. _JavaScript: https://en.wikipedia.org/wiki/JavaScript -.. _Splash: https://github.com/scrapinghub/splash .. _chompjs: https://github.com/Nykakin/chompjs .. _curl: https://curl.se/ .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser @@ -297,7 +277,6 @@ We recommend using `scrapy-playwright`_ for a better integration. .. _pyppeteer: https://pyppeteer.github.io/pyppeteer/ .. _pytesseract: https://github.com/madmaze/pytesseract .. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright -.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash .. _tabula-py: https://github.com/chezou/tabula-py .. _wget: https://www.gnu.org/software/wget/ .. _wgrep: https://github.com/stav/wgrep From 3ded1dfe31510f00e14a70811b7c01dae8b5a641 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Mar 2025 20:25:15 +0400 Subject: [PATCH 1675/2083] Converting tests to plain asserts, part 4. (#6702) --- tests/test_spider.py | 344 +++++++++----------- tests/test_spiderloader/__init__.py | 113 ++++--- tests/test_spidermiddleware.py | 105 +++--- tests/test_spidermiddleware_depth.py | 16 +- tests/test_spidermiddleware_httperror.py | 98 +++--- tests/test_spidermiddleware_offsite.py | 11 +- tests/test_spidermiddleware_output_chain.py | 112 +++---- tests/test_spidermiddleware_referer.py | 31 +- tests/test_spidermiddleware_urllength.py | 12 +- tests/test_spiderstate.py | 7 +- tests/test_squeues.py | 45 +-- tests/test_squeues_request.py | 148 ++++----- tests/test_stats.py | 72 ++-- 13 files changed, 529 insertions(+), 585 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index af29872a8..05f1c59d0 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -31,7 +31,7 @@ from scrapy.utils.test import get_crawler from tests import get_testdata, tests_datadir -class SpiderTest(unittest.TestCase): +class TestSpider(unittest.TestCase): spider_class = Spider def setUp(self): @@ -42,19 +42,19 @@ class SpiderTest(unittest.TestCase): def test_base_spider(self): spider = self.spider_class("example.com") - self.assertEqual(spider.name, "example.com") - self.assertEqual(spider.start_urls, []) + assert spider.name == "example.com" + assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison def test_start_requests(self): spider = self.spider_class("example.com") start_requests = spider.start_requests() - self.assertTrue(inspect.isgenerator(start_requests)) - self.assertEqual(list(start_requests), []) + assert inspect.isgenerator(start_requests) + assert not list(start_requests) def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" spider = self.spider_class("example.com", foo="bar") - self.assertEqual(spider.foo, "bar") + assert spider.foo == "bar" def test_spider_without_name(self): """``__init__`` method arguments are assigned to spider attributes""" @@ -67,10 +67,10 @@ class SpiderTest(unittest.TestCase): def test_from_crawler_crawler_and_settings_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "crawler")) - self.assertIs(spider.crawler, crawler) - self.assertTrue(hasattr(spider, "settings")) - self.assertIs(spider.settings, crawler.settings) + assert hasattr(spider, "crawler") + assert spider.crawler is crawler + assert hasattr(spider, "settings") + assert spider.settings is crawler.settings def test_from_crawler_init_call(self): with mock.patch.object( @@ -92,7 +92,7 @@ class SpiderTest(unittest.TestCase): crawler.signals.send_catch_log( signal=signals.spider_closed, spider=spider, reason=None ) - self.assertTrue(spider.closed_called) + assert spider.closed_called def test_update_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} @@ -101,9 +101,9 @@ class SpiderTest(unittest.TestCase): settings = Settings(project_settings, priority="project") self.spider_class.update_settings(settings) - self.assertEqual(settings.get("TEST1"), "spider") - self.assertEqual(settings.get("TEST2"), "spider") - self.assertEqual(settings.get("TEST3"), "project") + assert settings.get("TEST1") == "spider" + assert settings.get("TEST2") == "spider" + assert settings.get("TEST3") == "project" @inlineCallbacks def test_settings_in_from_crawler(self): @@ -121,11 +121,11 @@ class SpiderTest(unittest.TestCase): return spider crawler = Crawler(TestSpider, project_settings) - self.assertEqual(crawler.settings.get("TEST1"), "spider") - self.assertEqual(crawler.settings.get("TEST2"), "spider") - self.assertEqual(crawler.settings.get("TEST3"), "project") + assert crawler.settings.get("TEST1") == "spider" + assert crawler.settings.get("TEST2") == "spider" + assert crawler.settings.get("TEST3") == "project" yield crawler.crawl() - self.assertEqual(crawler.settings.get("TEST1"), "spider_instance") + assert crawler.settings.get("TEST1") == "spider_instance" def test_logger(self): spider = self.spider_class("example.com") @@ -134,8 +134,8 @@ class SpiderTest(unittest.TestCase): lc.check(("example.com", "INFO", "test log msg")) record = lc.records[0] - self.assertIn("spider", record.__dict__) - self.assertIs(record.spider, spider) + assert "spider" in record.__dict__ + assert record.spider is spider def test_log(self): spider = self.spider_class("example.com") @@ -144,11 +144,11 @@ class SpiderTest(unittest.TestCase): mock_logger.log.assert_called_once_with("INFO", "test log msg") -class InitSpiderTest(SpiderTest): +class TestInitSpider(TestSpider): spider_class = InitSpider -class XMLFeedSpiderTest(SpiderTest): +class TestXMLFeedSpider(TestSpider): spider_class = XMLFeedSpider def test_register_namespace(self): @@ -180,28 +180,24 @@ class XMLFeedSpiderTest(SpiderTest): for iterator in ("iternodes", "xml"): spider = _XMLSpider("example", iterator=iterator) output = list(spider._parse(response)) - self.assertEqual(len(output), 2, iterator) - self.assertEqual( - output, - [ - { - "loc": ["http://www.example.com/Special-Offers.html"], - "updated": ["2009-08-16"], - "custom": ["fuu"], - "other": ["bar"], - }, - { - "loc": [], - "updated": ["2009-08-16"], - "other": ["foo"], - "custom": [], - }, - ], - iterator, - ) + assert len(output) == 2, iterator + assert output == [ + { + "loc": ["http://www.example.com/Special-Offers.html"], + "updated": ["2009-08-16"], + "custom": ["fuu"], + "other": ["bar"], + }, + { + "loc": [], + "updated": ["2009-08-16"], + "other": ["foo"], + "custom": [], + }, + ], iterator -class CSVFeedSpiderTest(SpiderTest): +class TestCSVFeedSpider(TestSpider): spider_class = CSVFeedSpider def test_parse_rows(self): @@ -222,7 +218,7 @@ class CSVFeedSpiderTest(SpiderTest): assert len(rows) == 4 -class CrawlSpiderTest(SpiderTest): +class TestCrawlSpider(TestSpider): test_body = b"""Page title<title> <body> <p><a href="item/12.html">Item 12</a></p> @@ -247,16 +243,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] def test_process_links(self): response = HtmlResponse( @@ -273,16 +266,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] def test_process_links_filter(self): response = HtmlResponse( @@ -302,15 +292,12 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 2) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - ], - ) + assert len(output) == 2 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + ] def test_process_links_generator(self): response = HtmlResponse( @@ -327,16 +314,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] def test_process_request(self): response = HtmlResponse( @@ -355,16 +339,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.com/somepage/item/12.html", - "http://example.com/about.html", - "http://example.com/nofollow.html", - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.com/somepage/item/12.html", + "http://example.com/about.html", + "http://example.com/nofollow.html", + ] def test_process_request_with_response(self): response = HtmlResponse( @@ -386,20 +367,18 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - self.assertEqual( - [r.meta["response_class"] for r in output], - ["HtmlResponse", "HtmlResponse", "HtmlResponse"], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] def test_process_request_instance_method(self): response = HtmlResponse( @@ -416,16 +395,13 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), - safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), - safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), - ], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), + safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), + safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), + ] def test_process_request_instance_method_with_response(self): response = HtmlResponse( @@ -448,32 +424,30 @@ class CrawlSpiderTest(SpiderTest): spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(isinstance(r, Request) for r in output)) - self.assertEqual( - [r.url for r in output], - [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ], - ) - self.assertEqual( - [r.meta["response_class"] for r in output], - ["HtmlResponse", "HtmlResponse", "HtmlResponse"], - ) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] def test_follow_links_attribute_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "_follow_links")) - self.assertTrue(spider._follow_links) + assert hasattr(spider, "_follow_links") + assert spider._follow_links settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False} crawler = get_crawler(settings_dict=settings_dict) spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertTrue(hasattr(spider, "_follow_links")) - self.assertFalse(spider._follow_links) + assert hasattr(spider, "_follow_links") + assert not spider._follow_links def test_start_url(self): spider = self.spider_class("example.com") @@ -483,7 +457,7 @@ class CrawlSpiderTest(SpiderTest): list(spider.start_requests()) -class SitemapSpiderTest(SpiderTest): +class TestSitemapSpider(TestSpider): spider_class = SitemapSpider BODY = b"SITEMAP" @@ -496,7 +470,7 @@ class SitemapSpiderTest(SpiderTest): def assertSitemapBody(self, response, body): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") - self.assertEqual(spider._get_sitemap_body(response), body) + assert spider._get_sitemap_body(response) == body def test_get_sitemap_body(self): r = XmlResponse(url="http://www.example.com/", body=self.BODY) @@ -543,15 +517,12 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/robots.txt", body=robots) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://example.com/sitemap.xml", - "http://example.com/sitemap-product-index.xml", - "http://example.com/sitemap-uppercase.xml", - "http://www.example.com/sitemap-relative-url.xml", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://example.com/sitemap.xml", + "http://example.com/sitemap-product-index.xml", + "http://example.com/sitemap-uppercase.xml", + "http://www.example.com/sitemap-relative-url.xml", + ] def test_alternate_url_locs(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -570,21 +541,17 @@ Sitemap: /sitemap-relative-url.xml </urlset>""" r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] spider.sitemap_alternate_links = True - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/english/", - "http://www.example.com/deutsch/", - "http://www.example.com/schweiz-deutsch/", - "http://www.example.com/italiano/", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/deutsch/", + "http://www.example.com/schweiz-deutsch/", + "http://www.example.com/italiano/", + ] def test_sitemap_filter(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -611,16 +578,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/", "http://www.example.com/portuguese/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/portuguese/", + ] spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/english/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] def test_sitemap_filter_with_alternate_links(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -649,19 +615,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/english/article_1/", - "http://www.example.com/english/article_2/", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/article_1/", + "http://www.example.com/english/article_2/", + ] spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/deutsch/article_1/"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/deutsch/article_1/" + ] def test_sitemapindex_filter(self): sitemap = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -689,19 +651,15 @@ Sitemap: /sitemap-relative-url.xml r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - [ - "http://www.example.com/sitemap1.xml", - "http://www.example.com/sitemap2.xml", - ], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap1.xml", + "http://www.example.com/sitemap2.xml", + ] spider = FilteredSitemapSpider("example.com") - self.assertEqual( - [req.url for req in spider._parse_sitemap(r)], - ["http://www.example.com/sitemap2.xml"], - ) + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap2.xml" + ] def test_compression_bomb_setting(self): settings = {"DOWNLOAD_MAXSIZE": 10_000_000} @@ -711,7 +669,7 @@ Sitemap: /sitemap-relative-url.xml body = body_path.read_bytes() request = Request(url="https://example.com") response = Response(url="https://example.com", body=body, request=request) - self.assertIsNone(spider._get_sitemap_body(response)) + assert spider._get_sitemap_body(response) is None def test_compression_bomb_spider_attr(self): class DownloadMaxSizeSpider(self.spider_class): @@ -723,7 +681,7 @@ Sitemap: /sitemap-relative-url.xml body = body_path.read_bytes() request = Request(url="https://example.com") response = Response(url="https://example.com", body=body, request=request) - self.assertIsNone(spider._get_sitemap_body(response)) + assert spider._get_sitemap_body(response) is None def test_compression_bomb_request_meta(self): crawler = get_crawler() @@ -734,7 +692,7 @@ Sitemap: /sitemap-relative-url.xml url="https://example.com", meta={"download_maxsize": 10_000_000} ) response = Response(url="https://example.com", body=body, request=request) - self.assertIsNone(spider._get_sitemap_body(response)) + assert spider._get_sitemap_body(response) is None def test_download_warnsize_setting(self): settings = {"DOWNLOAD_WARNSIZE": 10_000_000} @@ -814,13 +772,13 @@ Sitemap: /sitemap-relative-url.xml ) -class DeprecationTest(unittest.TestCase): +class TestDeprecation: def test_crawl_spider(self): assert issubclass(CrawlSpider, Spider) assert isinstance(CrawlSpider(name="foo"), Spider) -class NoParseMethodSpiderTest(unittest.TestCase): +class TestNoParseMethodSpider: spider_class = Spider def test_undefined_parse_method(self): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index b103e9ed0..476487a04 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -8,7 +8,6 @@ from tempfile import mkdtemp from unittest import mock import pytest -from twisted.trial import unittest from zope.interface.verify import verifyObject # ugly hack to avoid cyclic imports of scrapy.spiders when running this test @@ -28,8 +27,8 @@ def _copytree(source: Path, target: Path): shutil.copytree(source, target) -class SpiderLoaderTest(unittest.TestCase): - def setUp(self): +class TestSpiderLoader: + def setup_method(self): orig_spiders_dir = module_dir / "test_spiders" self.tmpdir = Path(tempfile.mkdtemp()) self.spiders_dir = self.tmpdir / "test_spiders_xxx" @@ -38,7 +37,7 @@ class SpiderLoaderTest(unittest.TestCase): settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) self.spider_loader = SpiderLoader.from_settings(settings) - def tearDown(self): + def teardown_method(self): del self.spider_loader del sys.modules["test_spiders_xxx"] sys.path.remove(str(self.tmpdir)) @@ -47,37 +46,35 @@ class SpiderLoaderTest(unittest.TestCase): verifyObject(ISpiderLoader, self.spider_loader) def test_list(self): - self.assertEqual( - set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"} - ) + assert set(self.spider_loader.list()) == { + "spider1", + "spider2", + "spider3", + "spider4", + } def test_load(self): spider1 = self.spider_loader.load("spider1") - self.assertEqual(spider1.__name__, "Spider1") + assert spider1.__name__ == "Spider1" def test_find_by_request(self): - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy1.org/test")), - ["spider1"], - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy2.org/test")), - ["spider2"], - ) - self.assertEqual( - set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))), - {"spider1", "spider2"}, - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), [] - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://spider3.com")), [] - ) - self.assertEqual( - self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")), - ["spider3"], + assert self.spider_loader.find_by_request( + Request("http://scrapy1.org/test") + ) == ["spider1"] + assert self.spider_loader.find_by_request( + Request("http://scrapy2.org/test") + ) == ["spider2"] + assert set( + self.spider_loader.find_by_request(Request("http://scrapy3.org/test")) + ) == {"spider1", "spider2"} + assert ( + self.spider_loader.find_by_request(Request("http://scrapy999.org/test")) + == [] ) + assert self.spider_loader.find_by_request(Request("http://spider3.com")) == [] + assert self.spider_loader.find_by_request( + Request("http://spider3.com/onlythis") + ) == ["spider3"] def test_load_spider_module(self): module = "tests.test_spiderloader.test_spiders.spider1" @@ -113,9 +110,9 @@ class SpiderLoaderTest(unittest.TestCase): runner = CrawlerRunner({"ADDONS": {SpiderModuleAddon: 1}}) crawler = runner.create_crawler("spider_from_addon") - self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) - self.assertEqual(crawler.spidercls.name, "spider_from_addon") - self.assertTrue(len(crawler.settings["SPIDER_MODULES"]) == 1) + assert issubclass(crawler.spidercls, scrapy.Spider) + assert crawler.spidercls.name == "spider_from_addon" + assert len(crawler.settings["SPIDER_MODULES"]) == 1 def test_crawler_runner_loading(self): module = "tests.test_spiderloader.test_spiders.spider1" @@ -129,8 +126,8 @@ class SpiderLoaderTest(unittest.TestCase): runner.create_crawler("spider2") crawler = runner.create_crawler("spider1") - self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) - self.assertEqual(crawler.spidercls.name, "spider1") + assert issubclass(crawler.spidercls, scrapy.Spider) + assert crawler.spidercls.name == "spider1" def test_bad_spider_modules_exception(self): module = "tests.test_spiderloader.test_spiders.doesnotexist" @@ -150,10 +147,10 @@ class SpiderLoaderTest(unittest.TestCase): # at least until all six versions we can import (including botocore.vendored.six) # are updated to 1.16.0+ w.pop(0) - self.assertIn("Could not load spiders from module", str(w[0].message)) + assert "Could not load spiders from module" in str(w[0].message) spiders = spider_loader.list() - self.assertEqual(spiders, []) + assert not spiders def test_syntax_error_exception(self): module = "tests.test_spiderloader.test_spiders.spider1" @@ -179,14 +176,14 @@ class SpiderLoaderTest(unittest.TestCase): # at least until all six versions we can import (including botocore.vendored.six) # are updated to 1.16.0+ w.pop(0) - self.assertIn("Could not load spiders from module", str(w[0].message)) + assert "Could not load spiders from module" in str(w[0].message) spiders = spider_loader.list() - self.assertEqual(spiders, []) + assert not spiders -class DuplicateSpiderNameLoaderTest(unittest.TestCase): - def setUp(self): +class TestDuplicateSpiderNameLoader: + def setup_method(self): orig_spiders_dir = module_dir / "test_spiders" self.tmpdir = Path(mkdtemp()) self.spiders_dir = self.tmpdir / "test_spiders_xxx" @@ -194,7 +191,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): sys.path.append(str(self.tmpdir)) self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) - def tearDown(self): + def teardown_method(self): del sys.modules["test_spiders_xxx"] sys.path.remove(str(self.tmpdir)) @@ -208,18 +205,18 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) - self.assertEqual(len(w), 1) + assert len(w) == 1 msg = str(w[0].message) - self.assertIn("several spiders with the same name", msg) - self.assertIn("'spider3'", msg) - self.assertTrue(msg.count("'spider3'") == 2) + assert "several spiders with the same name" in msg + assert "'spider3'" in msg + assert msg.count("'spider3'") == 2 - self.assertNotIn("'spider1'", msg) - self.assertNotIn("'spider2'", msg) - self.assertNotIn("'spider4'", msg) + assert "'spider1'" not in msg + assert "'spider2'" not in msg + assert "'spider4'" not in msg spiders = set(spider_loader.list()) - self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"}) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name @@ -236,17 +233,17 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) - self.assertEqual(len(w), 1) + assert len(w) == 1 msg = str(w[0].message) - self.assertIn("several spiders with the same name", msg) - self.assertIn("'spider1'", msg) - self.assertTrue(msg.count("'spider1'") == 2) + assert "several spiders with the same name" in msg + assert "'spider1'" in msg + assert msg.count("'spider1'") == 2 - self.assertIn("'spider2'", msg) - self.assertTrue(msg.count("'spider2'") == 2) + assert "'spider2'" in msg + assert msg.count("'spider2'") == 2 - self.assertNotIn("'spider3'", msg) - self.assertNotIn("'spider4'", msg) + assert "'spider3'" not in msg + assert "'spider4'" not in msg spiders = set(spider_loader.list()) - self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"}) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index a9f3876bb..ddc9b5206 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -18,7 +18,7 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.test import get_crawler -class SpiderMiddlewareTestCase(TestCase): +class TestSpiderMiddleware(TestCase): def setUp(self): self.request = Request("http://example.com/index.html") self.response = Response(self.request.url, request=self.request) @@ -41,7 +41,7 @@ class SpiderMiddlewareTestCase(TestCase): return results[0] -class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_input method""" def test_invalid_process_spider_input(self): @@ -51,11 +51,11 @@ class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + assert isinstance(result, Failure) + assert isinstance(result.value, _InvalidOutput) -class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_output method""" def test_invalid_process_spider_output(self): @@ -65,11 +65,11 @@ class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase): self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + assert isinstance(result, Failure) + assert isinstance(result.value, _InvalidOutput) -class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): +class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_exception method""" def test_invalid_process_spider_exception(self): @@ -84,11 +84,11 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, _InvalidOutput) + assert isinstance(result, Failure) + assert isinstance(result.value, _InvalidOutput) -class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): +class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): """Re raise the exception by returning None""" def test_process_spider_exception_return_none(self): @@ -103,11 +103,11 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) result = self._scrape_response() - self.assertIsInstance(result, Failure) - self.assertIsInstance(result.value, ZeroDivisionError) + assert isinstance(result, Failure) + assert isinstance(result.value, ZeroDivisionError) -class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): +class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): """Helpers for testing sync, async and mixed middlewares. Should work for process_spider_output and, when it's supported, process_start_requests. @@ -148,14 +148,13 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, Iterable) + assert isinstance(result, Iterable) result_list = list(result) - self.assertEqual(len(result_list), self.RESULT_COUNT) - self.assertIsInstance(result_list[0], self.ITEM_TYPE) - self.assertEqual("downgraded to a non-async" in str(log), downgrade) - self.assertEqual( - "doesn't support asynchronous spider output" in str(log), - ProcessSpiderOutputSimpleMiddleware in mw_classes, + assert len(result_list) == self.RESULT_COUNT + assert isinstance(result_list[0], self.ITEM_TYPE) + assert ("downgraded to a non-async" in str(log)) == downgrade + assert ("doesn't support asynchronous spider output" in str(log)) == ( + ProcessSpiderOutputSimpleMiddleware in mw_classes ) @defer.inlineCallbacks @@ -166,11 +165,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self._get_middleware_result( *mw_classes, start_index=start_index ) - self.assertIsInstance(result, AsyncIterator) + assert isinstance(result, AsyncIterator) result_list = yield deferred_from_coro(collect_asyncgen(result)) - self.assertEqual(len(result_list), self.RESULT_COUNT) - self.assertIsInstance(result_list[0], self.ITEM_TYPE) - self.assertEqual("downgraded to a non-async" in str(log), downgrade) + assert len(result_list) == self.RESULT_COUNT + assert isinstance(result_list[0], self.ITEM_TYPE) + assert ("downgraded to a non-async" in str(log)) == downgrade class ProcessSpiderOutputSimpleMiddleware: @@ -212,7 +211,7 @@ class ProcessSpiderExceptionAsyncIterableMiddleware: yield {"foo": 3} -class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware): """process_spider_output tests for simple callbacks""" ITEM_TYPE = dict @@ -257,7 +256,7 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) -class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): +class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): """process_spider_output tests for async generator callbacks""" async def _scrape_func(self, *args, **kwargs): @@ -297,7 +296,7 @@ class ProcessSpiderOutputCoroutineMiddleware: return result -class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): @defer.inlineCallbacks def test_non_iterable(self): with pytest.raises( @@ -324,7 +323,7 @@ class ProcessStartRequestsSimpleMiddleware: yield from start_requests -class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessStartRequestsSimple(TestBaseAsyncSpiderMiddleware): """process_start_requests tests for simple start_requests""" ITEM_TYPE = (Request, dict) @@ -373,67 +372,65 @@ class UniversalMiddlewareBothAsync: yield -class UniversalMiddlewareManagerTest(TestCase): - def setUp(self): +class TestUniversalMiddlewareManager: + def setup_method(self): self.mwman = SpiderMiddlewareManager() def test_simple_mw(self): mw = ProcessSpiderOutputSimpleMiddleware() self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) def test_async_mw(self): mw = ProcessSpiderOutputAsyncGenMiddleware() self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) def test_universal_mw(self): mw = ProcessSpiderOutputUniversalMiddleware() self.mwman._add_middleware(mw) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], - (mw.process_spider_output, mw.process_spider_output_async), + assert self.mwman.methods["process_spider_output"][0] == ( + mw.process_spider_output, + mw.process_spider_output_async, ) def test_universal_mw_no_sync(self): with LogCapture() as log: self.mwman._add_middleware(UniversalMiddlewareNoSync()) - self.assertIn( + assert ( "UniversalMiddlewareNoSync has process_spider_output_async" - " without process_spider_output", - str(log), + " without process_spider_output" in str(log) ) - self.assertEqual(self.mwman.methods["process_spider_output"][0], None) + assert self.mwman.methods["process_spider_output"][0] is None def test_universal_mw_both_sync(self): mw = UniversalMiddlewareBothSync() with LogCapture() as log: self.mwman._add_middleware(mw) - self.assertIn( + assert ( "UniversalMiddlewareBothSync.process_spider_output_async " - "is not an async generator function", - str(log), + "is not an async generator function" in str(log) ) - self.assertEqual( - self.mwman.methods["process_spider_output"][0], mw.process_spider_output + assert ( + self.mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable ) def test_universal_mw_both_async(self): with LogCapture() as log: self.mwman._add_middleware(UniversalMiddlewareBothAsync()) - self.assertIn( + assert ( "UniversalMiddlewareBothAsync.process_spider_output " - "is an async generator function while process_spider_output_async exists", - str(log), + "is an async generator function while process_spider_output_async exists" + in str(log) ) - self.assertEqual(self.mwman.methods["process_spider_output"][0], None) + assert self.mwman.methods["process_spider_output"][0] is None -class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): +class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware @@ -474,7 +471,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): return self._test_simple_base(self.MW_UNIVERSAL) -class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): +class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): async def _scrape_func(self, *args, **kwargs): for item in super()._scrape_func(): yield item @@ -503,7 +500,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): return self._test_asyncgen_base(self.MW_UNIVERSAL) -class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase): +class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index e359d9cfc..dfcc141c3 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.spiders import Spider @@ -7,8 +5,8 @@ from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler -class TestDepthMiddleware(TestCase): - def setUp(self): +class TestDepthMiddleware: + def setup_method(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider("scrapytest.org") @@ -24,18 +22,18 @@ class TestDepthMiddleware(TestCase): result = [Request("http://scrapytest.org")] out = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEqual(out, result) + assert out == result rdc = self.stats.get_value("request_depth_count/1", spider=self.spider) - self.assertEqual(rdc, 1) + assert rdc == 1 req.meta["depth"] = 1 out2 = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEqual(out2, []) + assert not out2 rdm = self.stats.get_value("request_depth_max", spider=self.spider) - self.assertEqual(rdm, 1) + assert rdm == 1 - def tearDown(self): + def teardown_method(self): self.stats.close_spider(self.spider, "") diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index f9eb93d6b..e306579fa 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,10 +1,9 @@ import logging -from unittest import TestCase import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.trial.unittest import TestCase as TrialTestCase +from twisted.trial.unittest import TestCase from scrapy.http import Request, Response from scrapy.settings import Settings @@ -59,8 +58,8 @@ def _responses(request, status_codes): return responses -class TestHttpErrorMiddleware(TestCase): - def setUp(self): +class TestHttpErrorMiddleware: + def setup_method(self): crawler = get_crawler(Spider) self.spider = Spider.from_crawler(crawler, name="foo") self.mw = HttpErrorMiddleware(Settings({})) @@ -68,19 +67,20 @@ class TestHttpErrorMiddleware(TestCase): self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + assert self.mw.process_spider_input(self.res200, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(self.res404, self.spider) def test_process_spider_exception(self): - self.assertEqual( - [], + assert ( self.mw.process_spider_exception( self.res404, HttpError(self.res404), self.spider - ), + ) + == [] ) - self.assertIsNone( + assert ( self.mw.process_spider_exception(self.res404, Exception(), self.spider) + is None ) def test_handle_httpstatus_list(self): @@ -88,26 +88,26 @@ class TestHttpErrorMiddleware(TestCase): res.request = Request( "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} ) - self.assertIsNone(self.mw.process_spider_input(res, self.spider)) + assert self.mw.process_spider_input(res, self.spider) is None self.spider.handle_httpstatus_list = [404] - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert self.mw.process_spider_input(self.res404, self.spider) is None -class TestHttpErrorMiddlewareSettings(TestCase): +class TestHttpErrorMiddlewareSettings: """Similar test, but with settings""" - def setUp(self): + def setup_method(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)})) self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + assert self.mw.process_spider_input(self.res200, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(self.res404, self.spider) - self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) + assert self.mw.process_spider_input(self.res402, self.spider) is None def test_meta_overrides_settings(self): request = Request( @@ -118,27 +118,27 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402 = self.res402.copy() res402.request = request - self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + assert self.mw.process_spider_input(res404, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert self.mw.process_spider_input(self.res404, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(self.res402, self.spider) -class TestHttpErrorMiddlewareHandleAll(TestCase): - def setUp(self): +class TestHttpErrorMiddlewareHandleAll: + def setup_method(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True})) self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) - self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + assert self.mw.process_spider_input(self.res200, self.spider) is None + assert self.mw.process_spider_input(self.res404, self.spider) is None def test_meta_overrides_settings(self): request = Request( @@ -149,7 +149,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request - self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + assert self.mw.process_spider_input(res404, self.spider) is None with pytest.raises(HttpError): self.mw.process_spider_input(res402, self.spider) @@ -169,10 +169,10 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): with pytest.raises(HttpError): mw.process_spider_input(res404, self.spider) - self.assertIsNone(mw.process_spider_input(res402, self.spider)) + assert mw.process_spider_input(res402, self.spider) is None -class TestHttpErrorMiddlewareIntegrational(TrialTestCase): +class TestHttpErrorMiddlewareIntegrational(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -187,28 +187,28 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): crawler = get_crawler(_HttpErrorSpider) yield crawler.crawl(mockserver=self.mockserver) assert not crawler.spider.skipped, crawler.spider.skipped - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} get_value = crawler.stats.get_value - self.assertEqual(get_value("httperror/response_ignored_count"), 3) - self.assertEqual(get_value("httperror/response_ignored_status_count/404"), 1) - self.assertEqual(get_value("httperror/response_ignored_status_count/402"), 1) - self.assertEqual(get_value("httperror/response_ignored_status_count/500"), 1) + assert get_value("httperror/response_ignored_count") == 3 + assert get_value("httperror/response_ignored_status_count/404") == 1 + assert get_value("httperror/response_ignored_status_count/402") == 1 + assert get_value("httperror/response_ignored_status_count/500") == 1 @defer.inlineCallbacks def test_logging(self): crawler = get_crawler(_HttpErrorSpider) with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) - self.assertEqual(crawler.spider.parsed, {"200", "402"}) - self.assertEqual(crawler.spider.skipped, {"402"}) - self.assertEqual(crawler.spider.failed, {"404", "500"}) + assert crawler.spider.parsed == {"200", "402"} + assert crawler.spider.skipped == {"402"} + assert crawler.spider.failed == {"404", "500"} - self.assertIn("Ignoring response <404", str(log)) - self.assertIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) - self.assertNotIn("Ignoring response <402", str(log)) + assert "Ignoring response <404" in str(log) + assert "Ignoring response <500" in str(log) + assert "Ignoring response <200" not in str(log) + assert "Ignoring response <402" not in str(log) @defer.inlineCallbacks def test_logging_level(self): @@ -216,22 +216,22 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.INFO) as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} - self.assertIn("Ignoring response <402", str(log)) - self.assertIn("Ignoring response <404", str(log)) - self.assertIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) + assert "Ignoring response <402" in str(log) + assert "Ignoring response <404" in str(log) + assert "Ignoring response <500" in str(log) + assert "Ignoring response <200" not in str(log) # with level WARNING, we shouldn't capture anything from HttpError crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.WARNING) as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(crawler.spider.parsed, {"200"}) - self.assertEqual(crawler.spider.failed, {"404", "402", "500"}) + assert crawler.spider.parsed == {"200"} + assert crawler.spider.failed == {"404", "402", "500"} - self.assertNotIn("Ignoring response <402", str(log)) - self.assertNotIn("Ignoring response <404", str(log)) - self.assertNotIn("Ignoring response <500", str(log)) - self.assertNotIn("Ignoring response <200", str(log)) + assert "Ignoring response <402" not in str(log) + assert "Ignoring response <404" not in str(log) + assert "Ignoring response <500" not in str(log) + assert "Ignoring response <200" not in str(log) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 906928e01..f4563a0a4 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -1,5 +1,4 @@ import warnings -from unittest import TestCase from urllib.parse import urlparse from scrapy.http import Request, Response @@ -8,8 +7,8 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestOffsiteMiddleware(TestCase): - def setUp(self): +class TestOffsiteMiddleware: + def setup_method(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider(**self._get_spiderargs()) self.mw = OffsiteMiddleware.from_crawler(crawler) @@ -46,7 +45,7 @@ class TestOffsiteMiddleware(TestCase): reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, onsite_reqs) + assert out == onsite_reqs class TestOffsiteMiddleware2(TestOffsiteMiddleware): @@ -57,7 +56,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): res = Response("http://scrapytest.org") reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, reqs) + assert out == reqs class TestOffsiteMiddleware3(TestOffsiteMiddleware2): @@ -77,7 +76,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): res = Response("http://scrapytest.org") reqs = [Request("http://scrapytest.org/1")] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, reqs) + assert out == reqs class TestOffsiteMiddleware5(TestOffsiteMiddleware4): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index e51957497..6e26a85ea 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -324,9 +324,9 @@ class TestSpiderMiddleware(TestCase): was enqueued from the recovery middleware) """ log = yield self.crawl_log(RecoverySpider) - self.assertIn("Middleware: TabError exception caught", str(log)) - self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) - self.assertIn("'item_scraped_count': 3", str(log)) + assert "Middleware: TabError exception caught" in str(log) + assert str(log).count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in str(log) @defer.inlineCallbacks def test_recovery_asyncgen(self): @@ -334,9 +334,9 @@ class TestSpiderMiddleware(TestCase): Same as test_recovery but with an async callback. """ log = yield self.crawl_log(RecoveryAsyncGenSpider) - self.assertIn("Middleware: TabError exception caught", str(log)) - self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) - self.assertIn("'item_scraped_count': 3", str(log)) + assert "Middleware: TabError exception caught" in str(log) + assert str(log).count("Middleware: TabError exception caught") == 1 + assert "'item_scraped_count': 3" in str(log) @defer.inlineCallbacks def test_process_spider_input_without_errback(self): @@ -345,8 +345,8 @@ class TestSpiderMiddleware(TestCase): process_spider_exception chain from the start if the Request has no errback """ log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) - self.assertIn("Middleware: will raise IndexError", str(log1)) - self.assertIn("Middleware: IndexError exception caught", str(log1)) + assert "Middleware: will raise IndexError" in str(log1) + assert "Middleware: IndexError exception caught" in str(log1) @defer.inlineCallbacks def test_process_spider_input_with_errback(self): @@ -355,12 +355,12 @@ class TestSpiderMiddleware(TestCase): process_spider_exception chain if the Request has an errback """ log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback) - self.assertNotIn("Middleware: IndexError exception caught", str(log1)) - self.assertIn("Middleware: will raise IndexError", str(log1)) - self.assertIn("Got a Failure on the Request errback", str(log1)) - self.assertIn("{'from': 'errback'}", str(log1)) - self.assertNotIn("{'from': 'callback'}", str(log1)) - self.assertIn("'item_scraped_count': 1", str(log1)) + assert "Middleware: IndexError exception caught" not in str(log1) + assert "Middleware: will raise IndexError" in str(log1) + assert "Got a Failure on the Request errback" in str(log1) + assert "{'from': 'errback'}" in str(log1) + assert "{'from': 'callback'}" not in str(log1) + assert "'item_scraped_count': 1" in str(log1) @defer.inlineCallbacks def test_generator_callback(self): @@ -370,8 +370,8 @@ class TestSpiderMiddleware(TestCase): exception is raised should be processed normally. """ log2 = yield self.crawl_log(GeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + assert "Middleware: ImportError exception caught" in str(log2) + assert "'item_scraped_count': 2" in str(log2) @defer.inlineCallbacks def test_async_generator_callback(self): @@ -379,8 +379,8 @@ class TestSpiderMiddleware(TestCase): Same as test_generator_callback but with an async callback. """ log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + assert "Middleware: ImportError exception caught" in str(log2) + assert "'item_scraped_count': 2" in str(log2) @defer.inlineCallbacks def test_generator_callback_right_after_callback(self): @@ -389,8 +389,8 @@ class TestSpiderMiddleware(TestCase): even if the middleware is placed right after the spider """ log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) - self.assertIn("Middleware: ImportError exception caught", str(log21)) - self.assertIn("'item_scraped_count': 2", str(log21)) + assert "Middleware: ImportError exception caught" in str(log21) + assert "'item_scraped_count': 2" in str(log21) @defer.inlineCallbacks def test_not_a_generator_callback(self): @@ -399,8 +399,8 @@ class TestSpiderMiddleware(TestCase): be caught by the process_spider_exception chain. No items should be processed. """ log3 = yield self.crawl_log(NotGeneratorCallbackSpider) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) - self.assertNotIn("item_scraped_count", str(log3)) + assert "Middleware: ZeroDivisionError exception caught" in str(log3) + assert "item_scraped_count" not in str(log3) @defer.inlineCallbacks def test_not_a_generator_callback_right_after_callback(self): @@ -411,8 +411,8 @@ class TestSpiderMiddleware(TestCase): log31 = yield self.crawl_log( NotGeneratorCallbackSpiderMiddlewareRightAfterSpider ) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31)) - self.assertNotIn("item_scraped_count", str(log31)) + assert "Middleware: ZeroDivisionError exception caught" in str(log31) + assert "item_scraped_count" not in str(log31) @defer.inlineCallbacks def test_generator_output_chain(self): @@ -425,22 +425,22 @@ class TestSpiderMiddleware(TestCase): process_spider_exception chain) """ log4 = yield self.crawl_log(GeneratorOutputChainSpider) - self.assertIn("'item_scraped_count': 2", str(log4)) - self.assertIn( - "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert "'item_scraped_count': 2" in str(log4) + assert ( + "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught" + in str(log4) ) - self.assertIn( - "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught" + in str(log4) ) - self.assertNotIn( - "GeneratorFailMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorFailMiddleware.process_spider_exception: LookupError caught" + not in str(log4) ) - self.assertNotIn( - "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", - str(log4), + assert ( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught" + not in str(log4) ) item_from_callback = { "processed": [ @@ -457,9 +457,9 @@ class TestSpiderMiddleware(TestCase): "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", ] } - self.assertIn(str(item_from_callback), str(log4)) - self.assertIn(str(item_recovered), str(log4)) - self.assertNotIn("parse-second-item", str(log4)) + assert str(item_from_callback) in str(log4) + assert str(item_recovered) in str(log4) + assert "parse-second-item" not in str(log4) @defer.inlineCallbacks def test_not_a_generator_output_chain(self): @@ -472,22 +472,22 @@ class TestSpiderMiddleware(TestCase): from the spider callback are lost) """ log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) - self.assertIn("'item_scraped_count': 1", str(log5)) - self.assertIn( - "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert "'item_scraped_count': 1" in str(log5) + assert ( + "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught" + in str(log5) ) - self.assertIn( - "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert ( + "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught" + in str(log5) ) - self.assertNotIn( - "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert ( + "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught" + not in str(log5) ) - self.assertNotIn( - "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", - str(log5), + assert ( + "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught" + not in str(log5) ) item_recovered = { "processed": [ @@ -495,6 +495,6 @@ class TestSpiderMiddleware(TestCase): "NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", ] } - self.assertIn(str(item_recovered), str(log5)) - self.assertNotIn("parse-first-item", str(log5)) - self.assertNotIn("parse-second-item", str(log5)) + assert str(item_recovered) in str(log5) + assert "parse-first-item" not in str(log5) + assert "parse-second-item" not in str(log5) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 01a87c645..300a40c13 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -2,7 +2,6 @@ from __future__ import annotations import warnings from typing import Any -from unittest import TestCase from urllib.parse import urlparse import pytest @@ -35,7 +34,7 @@ from scrapy.spidermiddlewares.referer import ( from scrapy.spiders import Spider -class TestRefererMiddleware(TestCase): +class TestRefererMiddleware: req_meta: dict[str, Any] = {} resp_headers: dict[str, str] = {} settings: dict[str, Any] = {} @@ -43,7 +42,7 @@ class TestRefererMiddleware(TestCase): ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"), ] - def setUp(self): + def setup_method(self): self.spider = Spider("foo") settings = Settings(self.settings) self.mw = RefererMiddleware(settings) @@ -59,7 +58,7 @@ class TestRefererMiddleware(TestCase): response = self.get_response(origin) request = self.get_request(target) out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEqual(out[0].headers.get("Referer"), referrer) + assert out[0].headers.get("Referer") == referrer class MixinDefault: @@ -773,7 +772,7 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {"referrer_policy": POLICY_UNSAFE_URL} -class TestRequestMetaSettingFallback(TestCase): +class TestRequestMetaSettingFallback: params = [ ( # When an unknown policy is referenced in Request.meta @@ -844,14 +843,14 @@ class TestRequestMetaSettingFallback(TestCase): with warnings.catch_warnings(record=True) as w: policy = mw.policy(response, request) - self.assertIsInstance(policy, policy_class) + assert isinstance(policy, policy_class) if check_warning: - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, RuntimeWarning, w[0].message) + assert len(w) == 1 + assert w[0].category is RuntimeWarning, w[0].message -class TestSettingsPolicyByName(TestCase): +class TestSettingsPolicyByName: def test_valid_name(self): for s, p in [ (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), @@ -866,7 +865,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({"REFERRER_POLICY": s}) mw = RefererMiddleware(settings) - self.assertEqual(mw.default_policy, p) + assert mw.default_policy == p def test_valid_name_casevariants(self): for s, p in [ @@ -882,7 +881,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({"REFERRER_POLICY": s.upper()}) mw = RefererMiddleware(settings) - self.assertEqual(mw.default_policy, p) + assert mw.default_policy == p def test_invalid_name(self): settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"}) @@ -902,7 +901,7 @@ class TestSettingsPolicyByName(TestCase): } ) mw1 = RefererMiddleware(settings1) - self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy) + assert mw1.default_policy == StrictOriginWhenCrossOriginPolicy # test parsing with space(s) after the comma settings2 = Settings( @@ -915,7 +914,7 @@ class TestSettingsPolicyByName(TestCase): } ) mw2 = RefererMiddleware(settings2) - self.assertEqual(mw2.default_policy, UnsafeUrlPolicy) + assert mw2.default_policy == UnsafeUrlPolicy def test_multiple_policy_tokens_all_invalid(self): settings = Settings( @@ -1003,7 +1002,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): ), ] - def setUp(self): + def setup_method(self): self.spider = Spider("foo") settings = Settings(self.settings) self.referrermw = RefererMiddleware(settings) @@ -1023,7 +1022,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): out = list( self.referrermw.process_spider_output(response, [request], self.spider) ) - self.assertEqual(out[0].headers.get("Referer"), init_referrer) + assert out[0].headers.get("Referer") == init_referrer for status, url in redirections: response = Response( @@ -1035,7 +1034,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): self.referrermw.request_scheduled(request, self.spider) assert isinstance(request, Request) - self.assertEqual(request.headers.get("Referer"), final_referrer) + assert request.headers.get("Referer") == final_referrer class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 1a0f2e223..5cc3cdc6c 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,5 +1,3 @@ -from unittest import TestCase - from testfixtures import LogCapture from scrapy.http import Request, Response @@ -8,8 +6,8 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestUrlLengthMiddleware(TestCase): - def setUp(self): +class TestUrlLengthMiddleware: + def setup_method(self): self.maxlength = 25 crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": self.maxlength}) self.spider = crawler._create_spider("foo") @@ -27,7 +25,7 @@ class TestUrlLengthMiddleware(TestCase): ) def test_middleware_works(self): - self.assertEqual(self.process_spider_output(), [self.short_url_req]) + assert self.process_spider_output() == [self.short_url_req] def test_logging(self): with LogCapture() as log: @@ -36,6 +34,6 @@ class TestUrlLengthMiddleware(TestCase): ric = self.stats.get_value( "urllength/request_ignored_count", spider=self.spider ) - self.assertEqual(ric, 1) + assert ric == 1 - self.assertIn(f"Ignoring link (url length > {self.maxlength})", str(log)) + assert f"Ignoring link (url length > {self.maxlength})" in str(log) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 72692afab..cd31891a0 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -3,7 +3,6 @@ from datetime import datetime, timezone from tempfile import mkdtemp import pytest -from twisted.trial import unittest from scrapy.exceptions import NotConfigured from scrapy.extensions.spiderstate import SpiderState @@ -11,7 +10,7 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class SpiderStateTest(unittest.TestCase): +class TestSpiderState: def test_store_load(self): jobdir = mkdtemp() try: @@ -27,7 +26,7 @@ class SpiderStateTest(unittest.TestCase): spider2 = Spider(name="default") ss2 = SpiderState(jobdir) ss2.spider_opened(spider2) - self.assertEqual(spider.state, {"one": 1, "dt": dt}) + assert spider.state == {"one": 1, "dt": dt} ss2.spider_closed(spider2) finally: shutil.rmtree(jobdir) @@ -38,7 +37,7 @@ class SpiderStateTest(unittest.TestCase): spider = Spider(name="default") ss = SpiderState() ss.spider_opened(spider) - self.assertEqual(spider.state, {}) + assert spider.state == {} ss.spider_closed(spider) def test_not_configured(self): diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 8556b75dd..6283b9ad6 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -50,9 +50,9 @@ class FifoDiskQueueTestMixin: q.push("a") q.push(123) q.push({"a": "dict"}) - self.assertEqual(q.pop(), "a") - self.assertEqual(q.pop(), 123) - self.assertEqual(q.pop(), {"a": "dict"}) + assert q.pop() == "a" + assert q.pop() == 123 + assert q.pop() == {"a": "dict"} test_nonserializable_object = nonserializable_object_test @@ -92,7 +92,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): q.push(i) i2 = q.pop() assert isinstance(i2, MyItem) - self.assertEqual(i, i2) + assert i == i2 def test_serialize_loader(self): q = self.queue() @@ -101,7 +101,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): loader2 = q.pop() assert isinstance(loader2, MyLoader) assert loader2.default_item_class is MyItem - self.assertEqual(loader2.name_out("x"), "xx") + assert loader2.name_out("x") == "xx" def test_serialize_request_recursive(self): q = self.queue() @@ -110,23 +110,26 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): q.push(r) r2 = q.pop() assert isinstance(r2, Request) - self.assertEqual(r.url, r2.url) + assert r.url == r2.url assert r2.meta["request"] is r2 def test_non_pickable_object(self): q = self.queue() - try: + with pytest.raises( + ValueError, + match="Can't (get|pickle) local object|Can't pickle .*: it's not found as", + ) as exc_info: q.push(lambda x: x) - except ValueError as exc: - if hasattr(sys, "pypy_version_info"): - self.assertIsInstance(exc.__context__, pickle.PicklingError) - else: - self.assertIsInstance(exc.__context__, AttributeError) + if hasattr(sys, "pypy_version_info"): + assert isinstance(exc_info.value.__context__, pickle.PicklingError) + else: + assert isinstance(exc_info.value.__context__, AttributeError) sel = Selector(text="<html><body><p>some text</p></body></html>") - try: + with pytest.raises( + ValueError, match="can't pickle Selector objects" + ) as exc_info: q.push(sel) - except ValueError as exc: - self.assertIsInstance(exc.__context__, TypeError) + assert isinstance(exc_info.value.__context__, TypeError) class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): @@ -151,9 +154,9 @@ class LifoDiskQueueTestMixin: q.push("a") q.push(123) q.push({"a": "dict"}) - self.assertEqual(q.pop(), {"a": "dict"}) - self.assertEqual(q.pop(), 123) - self.assertEqual(q.pop(), "a") + assert q.pop() == {"a": "dict"} + assert q.pop() == 123 + assert q.pop() == "a" test_nonserializable_object = nonserializable_object_test @@ -173,7 +176,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): q.push(i) i2 = q.pop() assert isinstance(i2, MyItem) - self.assertEqual(i, i2) + assert i == i2 def test_serialize_loader(self): q = self.queue() @@ -182,7 +185,7 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): loader2 = q.pop() assert isinstance(loader2, MyLoader) assert loader2.default_item_class is MyItem - self.assertEqual(loader2.name_out("x"), "xx") + assert loader2.name_out("x") == "xx" def test_serialize_request_recursive(self): q = self.queue() @@ -191,5 +194,5 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): q.push(r) r2 = q.pop() assert isinstance(r2, Request) - self.assertEqual(r.url, r2.url) + assert r.url == r2.url assert r2.meta["request"] is r2 diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 88f6657d8..6c153f40e 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -22,14 +22,14 @@ from scrapy.squeues import ( from scrapy.utils.test import get_crawler -class BaseQueueTestCase(unittest.TestCase): - def setUp(self): +class TestBaseQueue: + def setup_method(self): self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") self.qpath = self.tempfilename() self.qdir = tempfile.mkdtemp() self.crawler = get_crawler(Spider) - def tearDown(self): + def teardown_method(self): shutil.rmtree(self.tmpdir) def tempfilename(self): @@ -48,36 +48,36 @@ class RequestQueueTestMixin: if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None req = Request("http://www.example.com") q.push(req) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req.url) - self.assertEqual(q.pop().url, req.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 1 + assert q.peek().url == req.url + assert q.pop().url == req.url + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None q.close() def test_one_element_without_peek(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.pop() is None req = Request("http://www.example.com") q.push(req) - self.assertEqual(len(q), 1) + assert len(q) == 1 with pytest.raises( NotImplementedError, match="The underlying queue class does not implement 'peek'", ): q.peek() - self.assertEqual(q.pop().url, req.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert q.pop().url == req.url + assert len(q) == 0 + assert q.pop() is None q.close() @@ -86,35 +86,35 @@ class FifoQueueMixin(RequestQueueTestMixin): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") q.push(req1) q.push(req2) q.push(req3) - self.assertEqual(len(q), 3) - self.assertEqual(q.peek().url, req1.url) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.peek().url, req2.url) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req3.url) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.peek().url == req1.url + assert q.pop().url == req1.url + assert len(q) == 2 + assert q.peek().url == req2.url + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.peek().url == req3.url + assert q.pop().url == req3.url + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None q.close() def test_fifo_without_peek(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") @@ -126,14 +126,14 @@ class FifoQueueMixin(RequestQueueTestMixin): match="The underlying queue class does not implement 'peek'", ): q.peek() - self.assertEqual(len(q), 3) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.pop().url == req1.url + assert len(q) == 2 + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.pop().url == req3.url + assert len(q) == 0 + assert q.pop() is None q.close() @@ -142,35 +142,35 @@ class LifoQueueMixin(RequestQueueTestMixin): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") q.push(req1) q.push(req2) q.push(req3) - self.assertEqual(len(q), 3) - self.assertEqual(q.peek().url, req3.url) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.peek().url, req2.url) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.peek().url, req1.url) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.peek()) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.peek().url == req3.url + assert q.pop().url == req3.url + assert len(q) == 2 + assert q.peek().url == req2.url + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.peek().url == req1.url + assert q.pop().url == req1.url + assert len(q) == 0 + assert q.peek() is None + assert q.pop() is None q.close() def test_lifo_without_peek(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): raise unittest.SkipTest("The queuelib queues do not define peek") q = self.queue() - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 0 + assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") @@ -182,46 +182,46 @@ class LifoQueueMixin(RequestQueueTestMixin): match="The underlying queue class does not implement 'peek'", ): q.peek() - self.assertEqual(len(q), 3) - self.assertEqual(q.pop().url, req3.url) - self.assertEqual(len(q), 2) - self.assertEqual(q.pop().url, req2.url) - self.assertEqual(len(q), 1) - self.assertEqual(q.pop().url, req1.url) - self.assertEqual(len(q), 0) - self.assertIsNone(q.pop()) + assert len(q) == 3 + assert q.pop().url == req3.url + assert len(q) == 2 + assert q.pop().url == req2.url + assert len(q) == 1 + assert q.pop().url == req1.url + assert len(q) == 0 + assert q.pop() is None q.close() -class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): +class TestPickleFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): def queue(self): return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo") -class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): +class TestPickleLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): def queue(self): return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo") -class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): +class TestMarshalFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): def queue(self): return MarshalFifoDiskQueue.from_crawler( crawler=self.crawler, key="marshal/fifo" ) -class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): +class TestMarshalLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): def queue(self): return MarshalLifoDiskQueue.from_crawler( crawler=self.crawler, key="marshal/lifo" ) -class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): +class TestFifoMemoryQueueRequest(FifoQueueMixin, TestBaseQueue): def queue(self): return FifoMemoryQueue.from_crawler(crawler=self.crawler) -class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): +class TestLifoMemoryQueueRequest(LifoQueueMixin, TestBaseQueue): def queue(self): return LifoMemoryQueue.from_crawler(crawler=self.crawler) diff --git a/tests/test_stats.py b/tests/test_stats.py index 3d4c7e88e..537614364 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,4 +1,3 @@ -import unittest from datetime import datetime from unittest import mock @@ -8,8 +7,8 @@ from scrapy.statscollectors import DummyStatsCollector, StatsCollector from scrapy.utils.test import get_crawler -class CoreStatsExtensionTest(unittest.TestCase): - def setUp(self): +class TestCoreStatsExtension: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") @@ -24,19 +23,16 @@ class CoreStatsExtensionTest(unittest.TestCase): ext.response_received(self.spider) ext.item_dropped({}, self.spider, ZeroDivisionError()) ext.spider_closed(self.spider, "finished") - self.assertEqual( - ext.stats._stats, - { - "start_time": fixed_datetime, - "finish_time": fixed_datetime, - "item_scraped_count": 1, - "response_received_count": 1, - "item_dropped_count": 1, - "item_dropped_reasons_count/ZeroDivisionError": 1, - "finish_reason": "finished", - "elapsed_time_seconds": 0.0, - }, - ) + assert ext.stats._stats == { + "start_time": fixed_datetime, + "finish_time": fixed_datetime, + "item_scraped_count": 1, + "response_received_count": 1, + "item_dropped_count": 1, + "item_dropped_reasons_count/ZeroDivisionError": 1, + "finish_reason": "finished", + "elapsed_time_seconds": 0.0, + } def test_core_stats_dummy_stats_collector(self): self.crawler.stats = DummyStatsCollector(self.crawler) @@ -46,51 +42,51 @@ class CoreStatsExtensionTest(unittest.TestCase): ext.response_received(self.spider) ext.item_dropped({}, self.spider, ZeroDivisionError()) ext.spider_closed(self.spider, "finished") - self.assertEqual(ext.stats._stats, {}) + assert ext.stats._stats == {} -class StatsCollectorTest(unittest.TestCase): - def setUp(self): +class TestStatsCollector: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") def test_collector(self): stats = StatsCollector(self.crawler) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_value("anything"), None) - self.assertEqual(stats.get_value("anything", "default"), "default") + assert stats.get_stats() == {} + assert stats.get_value("anything") is None + assert stats.get_value("anything", "default") == "default" stats.set_value("test", "value") - self.assertEqual(stats.get_stats(), {"test": "value"}) + assert stats.get_stats() == {"test": "value"} stats.set_value("test2", 23) - self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23}) - self.assertEqual(stats.get_value("test2"), 23) + assert stats.get_stats() == {"test": "value", "test2": 23} + assert stats.get_value("test2") == 23 stats.inc_value("test2") - self.assertEqual(stats.get_value("test2"), 24) + assert stats.get_value("test2") == 24 stats.inc_value("test2", 6) - self.assertEqual(stats.get_value("test2"), 30) + assert stats.get_value("test2") == 30 stats.max_value("test2", 6) - self.assertEqual(stats.get_value("test2"), 30) + assert stats.get_value("test2") == 30 stats.max_value("test2", 40) - self.assertEqual(stats.get_value("test2"), 40) + assert stats.get_value("test2") == 40 stats.max_value("test3", 1) - self.assertEqual(stats.get_value("test3"), 1) + assert stats.get_value("test3") == 1 stats.min_value("test2", 60) - self.assertEqual(stats.get_value("test2"), 40) + assert stats.get_value("test2") == 40 stats.min_value("test2", 35) - self.assertEqual(stats.get_value("test2"), 35) + assert stats.get_value("test2") == 35 stats.min_value("test4", 7) - self.assertEqual(stats.get_value("test4"), 7) + assert stats.get_value("test4") == 7 def test_dummy_collector(self): stats = DummyStatsCollector(self.crawler) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_value("anything"), None) - self.assertEqual(stats.get_value("anything", "default"), "default") + assert stats.get_stats() == {} + assert stats.get_value("anything") is None + assert stats.get_value("anything", "default") == "default" stats.set_value("test", "value") stats.inc_value("v1") stats.max_value("v2", 100) stats.min_value("v3", 100) stats.open_spider("a") stats.set_value("test", "value", spider=self.spider) - self.assertEqual(stats.get_stats(), {}) - self.assertEqual(stats.get_stats("a"), {}) + assert stats.get_stats() == {} + assert stats.get_stats("a") == {} From 40833afc86d45543a521b0f147387f7f770a8adc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 7 Mar 2025 22:33:41 +0400 Subject: [PATCH 1676/2083] Work around a queuelib test file close problem. (#6703) --- tests/test_squeues.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 6283b9ad6..0b6ed8e11 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -130,6 +130,9 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): ) as exc_info: q.push(sel) assert isinstance(exc_info.value.__context__, TypeError) + # This seems to help with https://github.com/scrapy/queuelib/issues/70. + # It will need to remain under a queuelib version check after that bug is fixed. + del exc_info class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): From 1469b2739ea566a57e0b5f8e6bb104fd19460d24 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sat, 8 Mar 2025 20:50:54 +0400 Subject: [PATCH 1677/2083] Drop tests/test_loader_deprecated.py. (#6704) --- tests/test_loader_deprecated.py | 750 -------------------------------- 1 file changed, 750 deletions(-) delete mode 100644 tests/test_loader_deprecated.py diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py deleted file mode 100644 index 0d7921b1d..000000000 --- a/tests/test_loader_deprecated.py +++ /dev/null @@ -1,750 +0,0 @@ -""" -These tests are kept as references from the ones that were ported to a itemloaders library. -Once we remove the references from scrapy, we can remove these tests. -""" - -import unittest -from functools import partial - -import pytest -from itemloaders.processors import ( - Compose, - Identity, - Join, - MapCompose, - SelectJmes, - TakeFirst, -) - -from scrapy.item import Field, Item -from scrapy.loader import ItemLoader - - -# test items -class NameItem(Item): - name = Field() - - -class SummaryItem(NameItem): - url = Field() - summary = Field() - - -# test item loaders -class NameItemLoader(ItemLoader): - default_item_class = SummaryItem - - -class ProcessorItemLoader(NameItemLoader): - name_in = MapCompose(lambda v: v.title()) - - -class DefaultedItemLoader(NameItemLoader): - default_input_processor = MapCompose(lambda v: v[:-1]) - - -# test processors -def processor_with_args(value, other=None, loader_context=None): - if "key" in loader_context: - return loader_context["key"] - return value - - -class BasicItemLoaderTest(unittest.TestCase): - def test_load_item_using_default_loader(self): - i = SummaryItem() - i["summary"] = "lala" - il = ItemLoader(item=i) - il.add_value("name", "marta") - item = il.load_item() - assert item is i - self.assertEqual(item["summary"], ["lala"]) - self.assertEqual(item["name"], ["marta"]) - - def test_load_item_using_custom_loader(self): - il = ProcessorItemLoader() - il.add_value("name", "marta") - item = il.load_item() - self.assertEqual(item["name"], ["Marta"]) - - def test_load_item_ignore_none_field_values(self): - def validate_sku(value): - # Let's assume a SKU is only digits. - return value if value.isdigit() else None - - class MyLoader(ItemLoader): - name_out = Compose(lambda vs: vs[0]) # take first which allows empty values - price_out = Compose(TakeFirst(), float) - sku_out = Compose(TakeFirst(), validate_sku) - - valid_fragment = "SKU: 1234" - invalid_fragment = "SKU: not available" - sku_re = "SKU: (.+)" - - il = MyLoader(item={}) - # Should not return "sku: None". - il.add_value("sku", [invalid_fragment], re=sku_re) - # Should not ignore empty values. - il.add_value("name", "") - il.add_value("price", ["0"]) - self.assertEqual( - il.load_item(), - { - "name": "", - "price": 0.0, - }, - ) - - il.replace_value("sku", [valid_fragment], re=sku_re) - self.assertEqual(il.load_item()["sku"], "1234") - - def test_self_referencing_loader(self): - class MyLoader(ItemLoader): - url_out = TakeFirst() - - def img_url_out(self, values): - return (self.get_output_value("url") or "") + values[0] - - il = MyLoader(item={}) - il.add_value("url", "http://example.com/") - il.add_value("img_url", "1234.png") - self.assertEqual( - il.load_item(), - { - "url": "http://example.com/", - "img_url": "http://example.com/1234.png", - }, - ) - - il = MyLoader(item={}) - il.add_value("img_url", "1234.png") - self.assertEqual( - il.load_item(), - { - "img_url": "1234.png", - }, - ) - - def test_add_value(self): - il = ProcessorItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["Marta"]) - self.assertEqual(il.get_output_value("name"), ["Marta"]) - il.add_value("name", "pepe") - self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe"]) - self.assertEqual(il.get_output_value("name"), ["Marta", "Pepe"]) - - # test add object value - il.add_value("summary", {"key": 1}) - self.assertEqual(il.get_collected_values("summary"), [{"key": 1}]) - - il.add_value(None, "Jim", lambda x: {"name": x}) - self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe", "Jim"]) - - def test_add_zero(self): - il = NameItemLoader() - il.add_value("name", 0) - self.assertEqual(il.get_collected_values("name"), [0]) - - def test_replace_value(self): - il = ProcessorItemLoader() - il.replace_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["Marta"]) - self.assertEqual(il.get_output_value("name"), ["Marta"]) - il.replace_value("name", "pepe") - self.assertEqual(il.get_collected_values("name"), ["Pepe"]) - self.assertEqual(il.get_output_value("name"), ["Pepe"]) - - il.replace_value(None, "Jim", lambda x: {"name": x}) - self.assertEqual(il.get_collected_values("name"), ["Jim"]) - - def test_get_value(self): - il = NameItemLoader() - self.assertEqual("FOO", il.get_value(["foo", "bar"], TakeFirst(), str.upper)) - self.assertEqual( - ["foo", "bar"], il.get_value(["name:foo", "name:bar"], re="name:(.*)$") - ) - self.assertEqual( - "foo", il.get_value(["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$") - ) - - il.add_value("name", ["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$") - self.assertEqual(["foo"], il.get_collected_values("name")) - il.replace_value("name", "name:bar", re="name:(.*)$") - self.assertEqual(["bar"], il.get_collected_values("name")) - - def test_iter_on_input_processor_input(self): - class NameFirstItemLoader(NameItemLoader): - name_in = TakeFirst() - - il = NameFirstItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["marta"]) - il = NameFirstItemLoader() - il.add_value("name", ["marta", "jose"]) - self.assertEqual(il.get_collected_values("name"), ["marta"]) - - il = NameFirstItemLoader() - il.replace_value("name", "marta") - self.assertEqual(il.get_collected_values("name"), ["marta"]) - il = NameFirstItemLoader() - il.replace_value("name", ["marta", "jose"]) - self.assertEqual(il.get_collected_values("name"), ["marta"]) - - il = NameFirstItemLoader() - il.add_value("name", "marta") - il.add_value("name", ["jose", "pedro"]) - self.assertEqual(il.get_collected_values("name"), ["marta", "jose"]) - - def test_map_compose_filter(self): - def filter_world(x): - return None if x == "world" else x - - proc = MapCompose(filter_world, str.upper) - self.assertEqual( - proc(["hello", "world", "this", "is", "scrapy"]), - ["HELLO", "THIS", "IS", "SCRAPY"], - ) - - def test_map_compose_filter_multil(self): - class TestItemLoader(NameItemLoader): - name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) - - il = TestItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["Mart"]) - item = il.load_item() - self.assertEqual(item["name"], ["Mart"]) - - def test_default_input_processor(self): - il = DefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["mart"]) - - def test_inherited_default_input_processor(self): - class InheritDefaultedItemLoader(DefaultedItemLoader): - pass - - il = InheritDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["mart"]) - - def test_input_processor_inheritance(self): - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(lambda v: v.lower()) - - il = ChildItemLoader() - il.add_value("url", "HTTP://scrapy.ORG") - self.assertEqual(il.get_output_value("url"), ["http://scrapy.org"]) - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["Marta"]) - - class ChildChildItemLoader(ChildItemLoader): - url_in = MapCompose(lambda v: v.upper()) - summary_in = MapCompose(lambda v: v) - - il = ChildChildItemLoader() - il.add_value("url", "http://scrapy.org") - self.assertEqual(il.get_output_value("url"), ["HTTP://SCRAPY.ORG"]) - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["Marta"]) - - def test_empty_map_compose(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose() - - il = IdentityDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["marta"]) - - def test_identity_input_processor(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = Identity() - - il = IdentityDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["marta"]) - - def test_extend_custom_input_processors(self): - class ChildItemLoader(ProcessorItemLoader): - name_in = MapCompose(ProcessorItemLoader.name_in, str.swapcase) - - il = ChildItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["mARTA"]) - - def test_extend_default_input_processors(self): - class ChildDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose( - DefaultedItemLoader.default_input_processor, str.swapcase - ) - - il = ChildDefaultedItemLoader() - il.add_value("name", "marta") - self.assertEqual(il.get_output_value("name"), ["MART"]) - - def test_output_processor_using_function(self): - il = ProcessorItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - class TakeFirstItemLoader(ProcessorItemLoader): - name_out = " ".join - - il = TakeFirstItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), "Mar Ta") - - def test_output_processor_error(self): - class TestItemLoader(ItemLoader): - default_item_class = SummaryItem - name_out = MapCompose(float) - - il = TestItemLoader() - il.add_value("name", ["$10"]) - try: - float("$10") - except Exception as e: - expected_exc_str = str(e) - - exc = None - try: - il.load_item() - except Exception as e: - exc = e - assert isinstance(exc, ValueError) - s = str(exc) - assert "name" in s, s - assert "$10" in s, s - assert "ValueError" in s, s - assert expected_exc_str in s, s - - def test_output_processor_using_classes(self): - il = ProcessorItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - class TakeFirstItemLoader(ProcessorItemLoader): - name_out = Join() - - il = TakeFirstItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), "Mar Ta") - - class TakeFirstItemLoader2(ProcessorItemLoader): - name_out = Join("<br>") - - il = TakeFirstItemLoader2() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), "Mar<br>Ta") - - def test_default_output_processor(self): - il = ProcessorItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - class LalaItemLoader(ProcessorItemLoader): - default_output_processor = Identity() - - il = LalaItemLoader() - il.add_value("name", ["mar", "ta"]) - self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"]) - - def test_loader_context_on_declaration(self): - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(processor_with_args, key="val") - - il = ChildItemLoader() - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["val"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["val"]) - - def test_loader_context_on_instantiation(self): - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(processor_with_args) - - il = ChildItemLoader(key="val") - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["val"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["val"]) - - def test_loader_context_on_assign(self): - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(processor_with_args) - - il = ChildItemLoader() - il.context["key"] = "val" - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["val"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["val"]) - - def test_item_passed_to_input_processor_functions(self): - def processor(value, loader_context): - return loader_context["item"]["name"] - - class ChildItemLoader(ProcessorItemLoader): - url_in = MapCompose(processor) - - it = SummaryItem(name="marta") - il = ChildItemLoader(item=it) - il.add_value("url", "text") - self.assertEqual(il.get_output_value("url"), ["marta"]) - il.replace_value("url", "text2") - self.assertEqual(il.get_output_value("url"), ["marta"]) - - def test_compose_processor(self): - class TestItemLoader(NameItemLoader): - name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) - - il = TestItemLoader() - il.add_value("name", ["marta", "other"]) - self.assertEqual(il.get_output_value("name"), "Mart") - item = il.load_item() - self.assertEqual(item["name"], "Mart") - - def test_partial_processor(self): - def join(values, sep=None, loader_context=None, ignored=None): - if sep is not None: - return sep.join(values) - if loader_context and "sep" in loader_context: - return loader_context["sep"].join(values) - return "".join(values) - - class TestItemLoader(NameItemLoader): - name_out = Compose(partial(join, sep="+")) - url_out = Compose(partial(join, loader_context={"sep": "."})) - summary_out = Compose(partial(join, ignored="foo")) - - il = TestItemLoader() - il.add_value("name", ["rabbit", "hole"]) - il.add_value("url", ["rabbit", "hole"]) - il.add_value("summary", ["rabbit", "hole"]) - item = il.load_item() - self.assertEqual(item["name"], "rabbit+hole") - self.assertEqual(item["url"], "rabbit.hole") - self.assertEqual(item["summary"], "rabbithole") - - def test_error_input_processor(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_in = MapCompose(float) - - il = TestItemLoader() - with pytest.raises( - ValueError, - match="Error with input processor MapCompose: .* " - "error='ValueError: Error in MapCompose .* " - "error='ValueError: could not convert", - ): - il.add_value("name", ["marta", "other"]) - - def test_error_output_processor(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_out = Compose(Join(), float) - - il = TestItemLoader() - il.add_value("name", "marta") - with pytest.raises( - ValueError, - match="Error with output processor: .* " - "error='ValueError: Error in Compose .* " - "error='ValueError: could not convert", - ): - il.load_item() - - def test_error_processor_as_argument(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - - il = TestItemLoader() - with pytest.raises( - ValueError, - match=r"Error with processor Compose .* " - r"error='ValueError: Error in Compose .* " - r"error='TypeError: float\(\) argument", - ): - il.add_value("name", ["marta", "other"], Compose(float)) - - -class InitializationFromDictTest(unittest.TestCase): - item_class = dict - - def test_keep_single_value(self): - """Loaded item should contain values from the initial item""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo"]}) - - def test_keep_list(self): - """Loaded item should contain values from the initial item""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]}) - - def test_add_value_singlevalue_singlevalue(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - il.add_value("name", "bar") - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]}) - - def test_add_value_singlevalue_list(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - il.add_value("name", ["item", "loader"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "item", "loader"]}) - - def test_add_value_list_singlevalue(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - il.add_value("name", "qwerty") - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "qwerty"]}) - - def test_add_value_list_list(self): - """Values added after initialization should be appended""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - il.add_value("name", ["item", "loader"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "item", "loader"]}) - - def test_get_output_value_singlevalue(self): - """Getting output value must not remove value from item""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, {"name": ["foo"]}) - - def test_get_output_value_list(self): - """Getting output value must not remove value from item""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) - loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(loaded_item, {"name": ["foo", "bar"]}) - - def test_values_single(self): - """Values from initial item must be added to loader._values""" - input_item = self.item_class(name="foo") - il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo"]) - - def test_values_list(self): - """Values from initial item must be added to loader._values""" - input_item = self.item_class(name=["foo", "bar"]) - il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo", "bar"]) - - -class BaseNoInputReprocessingLoader(ItemLoader): - title_in = MapCompose(str.upper) - title_out = TakeFirst() - - -class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader): - default_item_class = dict - - -class NoInputReprocessingFromDictTest(unittest.TestCase): - """ - Loaders initialized from loaded items must not reprocess fields (dict instances) - """ - - def test_avoid_reprocessing_with_initial_values_single(self): - il = NoInputReprocessingDictLoader(item={"title": "foo"}) - il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} - ) - - def test_avoid_reprocessing_with_initial_values_list(self): - il = NoInputReprocessingDictLoader(item={"title": ["foo", "bar"]}) - il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} - ) - - def test_avoid_reprocessing_without_initial_values_single(self): - il = NoInputReprocessingDictLoader() - il.add_value("title", "foo") - il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) - - def test_avoid_reprocessing_without_initial_values_list(self): - il = NoInputReprocessingDictLoader() - il.add_value("title", ["foo", "bar"]) - il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) - - -class TestOutputProcessorDict(unittest.TestCase): - def test_output_processor(self): - class TempDict(dict): - def __init__(self, *args, **kwargs): - super().__init__(self, *args, **kwargs) - self.setdefault("temp", 0.3) - - class TempLoader(ItemLoader): - default_item_class = TempDict - default_input_processor = Identity() - default_output_processor = Compose(TakeFirst()) - - loader = TempLoader() - item = loader.load_item() - self.assertIsInstance(item, TempDict) - self.assertEqual(dict(item), {"temp": 0.3}) - - -class ProcessorsTest(unittest.TestCase): - def test_take_first(self): - proc = TakeFirst() - self.assertEqual(proc([None, "", "hello", "world"]), "hello") - self.assertEqual(proc([None, "", 0, "hello", "world"]), 0) - - def test_identity(self): - proc = Identity() - self.assertEqual( - proc([None, "", "hello", "world"]), [None, "", "hello", "world"] - ) - - def test_join(self): - proc = Join() - with pytest.raises(TypeError): - proc([None, "", "hello", "world"]) - self.assertEqual(proc(["", "hello", "world"]), " hello world") - self.assertEqual(proc(["hello", "world"]), "hello world") - self.assertIsInstance(proc(["hello", "world"]), str) - - def test_compose(self): - proc = Compose(lambda v: v[0], str.upper) - self.assertEqual(proc(["hello", "world"]), "HELLO") - proc = Compose(str.upper) - self.assertEqual(proc(None), None) - proc = Compose(str.upper, stop_on_none=False) - with pytest.raises( - ValueError, - match="Error in Compose with .* error='TypeError: (descriptor 'upper'|'str' object expected)", - ): - proc(None) - proc = Compose(str.upper, lambda x: x + 1) - with pytest.raises( - ValueError, - match="Error in Compose with .* error='TypeError: (can only|unsupported operand)", - ): - proc("hello") - - def test_mapcompose(self): - def filter_world(x): - return None if x == "world" else x - - proc = MapCompose(filter_world, str.upper) - self.assertEqual( - proc(["hello", "world", "this", "is", "scrapy"]), - ["HELLO", "THIS", "IS", "SCRAPY"], - ) - proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc(None), []) - proc = MapCompose(filter_world, str.upper) - with pytest.raises( - ValueError, - match="Error in MapCompose with .* error='TypeError: (descriptor 'upper'|'str' object expected)", - ): - proc([1]) - proc = MapCompose(filter_world, lambda x: x + 1) - with pytest.raises( - ValueError, - match="Error in MapCompose with .* error='TypeError: (can only|unsupported operand)", - ): - proc("hello") - - -class SelectJmesTestCase(unittest.TestCase): - test_list_equals = { - "simple": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"), - "invalid": ("foo.bar.baz", {"foo": {"bar": "baz"}}, None), - "top_level": ("foo", {"foo": {"bar": "baz"}}, {"bar": "baz"}), - "double_vs_single_quote_string": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"), - "dict": ( - "foo.bar[*].name", - {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ["one", "two"], - ), - "list": ("[1]", [1, 2], 2), - } - - def test_output(self): - for k, v in self.test_list_equals.items(): - expr, test_list, expected = v - test = SelectJmes(expr)(test_list) - self.assertEqual( - test, expected, msg=f'test "{k}" got {test} expected {expected}' - ) - - -# Functions as processors - - -def function_processor_strip(iterable): - return [x.strip() for x in iterable] - - -def function_processor_upper(iterable): - return [x.upper() for x in iterable] - - -class FunctionProcessorItem(Item): - foo = Field( - input_processor=function_processor_strip, - output_processor=function_processor_upper, - ) - - -class FunctionProcessorDictLoader(ItemLoader): - default_item_class = dict - foo_in = function_processor_strip - foo_out = function_processor_upper - - -class FunctionProcessorTestCase(unittest.TestCase): - def test_processor_defined_in_item_loader(self): - lo = FunctionProcessorDictLoader() - lo.add_value("foo", " bar ") - lo.add_value("foo", [" asdf ", " qwerty "]) - self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) From 044c3f69edd1bf926408649361ada7f2146db04e Mon Sep 17 00:00:00 2001 From: Mehraz Hossain Rumman <59512321+MehrazRumman@users.noreply.github.com> Date: Mon, 10 Mar 2025 01:18:57 +0600 Subject: [PATCH 1678/2083] Deprecate InitSpider (#6714) --- scrapy/spiders/init.py | 17 ++++++++++++++++- tests/test_spider.py | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 4ec2919f7..a7dba989e 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,9 +1,11 @@ from __future__ import annotations +import warnings from collections.abc import Iterable from typing import TYPE_CHECKING, Any, cast from scrapy import Request +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output @@ -12,7 +14,20 @@ if TYPE_CHECKING: class InitSpider(Spider): - """Base Spider with initialization facilities""" + """Base Spider with initialization facilities + + .. warning:: This class is deprecated. Copy its code into your project if needed. + It will be removed in a future Scrapy version. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + warnings.warn( + "InitSpider is deprecated. Copy its code from Scrapy's source if needed. " + "Will be removed in a future version.", + ScrapyDeprecationWarning, + stacklevel=2, + ) def start_requests(self) -> Iterable[Request]: self._postinit_reqs: Iterable[Request] = super().start_requests() diff --git a/tests/test_spider.py b/tests/test_spider.py index 05f1c59d0..4e8330c06 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -144,6 +144,7 @@ class TestSpider(unittest.TestCase): mock_logger.log.assert_called_once_with("INFO", "test log msg") +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestInitSpider(TestSpider): spider_class = InitSpider From 02ed71d8877d1f3f270a9085c3cdb7fc7e917b8a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 9 Mar 2025 23:20:24 +0400 Subject: [PATCH 1679/2083] Converting tests to plain asserts, part 6. (#6709) --- tests/test_item.py | 97 ++-- tests/test_link.py | 12 +- tests/test_linkextractors.py | 937 +++++++++++++++------------------- tests/test_pipeline_crawl.py | 46 +- tests/test_pipeline_files.py | 213 ++++---- tests/test_pipeline_images.py | 204 ++++---- tests/test_pipeline_media.py | 149 +++--- 7 files changed, 730 insertions(+), 928 deletions(-) diff --git a/tests/test_item.py b/tests/test_item.py index 47c5c3db6..bf51eb398 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,4 +1,3 @@ -import unittest from abc import ABCMeta from unittest import mock @@ -7,9 +6,9 @@ import pytest from scrapy.item import Field, Item, ItemMeta -class ItemTest(unittest.TestCase): +class TestItem: def assertSortedEqual(self, first, second, msg=None): - return self.assertEqual(sorted(first), sorted(second), msg) + assert sorted(first) == sorted(second), msg def test_simple(self): class TestItem(Item): @@ -17,7 +16,7 @@ class ItemTest(unittest.TestCase): i = TestItem() i["name"] = "name" - self.assertEqual(i["name"], "name") + assert i["name"] == "name" def test_init(self): class TestItem(Item): @@ -28,13 +27,13 @@ class ItemTest(unittest.TestCase): i["name"] i2 = TestItem(name="john doe") - self.assertEqual(i2["name"], "john doe") + assert i2["name"] == "john doe" i3 = TestItem({"name": "john doe"}) - self.assertEqual(i3["name"], "john doe") + assert i3["name"] == "john doe" i4 = TestItem(i3) - self.assertEqual(i4["name"], "john doe") + assert i4["name"] == "john doe" with pytest.raises(KeyError): TestItem({"name": "john doe", "other": "foo"}) @@ -59,11 +58,11 @@ class ItemTest(unittest.TestCase): i["number"] = 123 itemrepr = repr(i) - self.assertEqual(itemrepr, "{'name': 'John Doe', 'number': 123}") + assert itemrepr == "{'name': 'John Doe', 'number': 123}" i2 = eval(itemrepr) # pylint: disable=eval-used - self.assertEqual(i2["name"], "John Doe") - self.assertEqual(i2["number"], 123) + assert i2["name"] == "John Doe" + assert i2["number"] == 123 def test_private_attr(self): class TestItem(Item): @@ -71,7 +70,7 @@ class ItemTest(unittest.TestCase): i = TestItem() i._private = "test" - self.assertEqual(i._private, "test") + assert i._private == "test" def test_raise_getattr(self): class TestItem(Item): @@ -103,9 +102,9 @@ class ItemTest(unittest.TestCase): with pytest.raises(KeyError): i.get_name() i["name"] = "lala" - self.assertEqual(i.get_name(), "lala") + assert i.get_name() == "lala" i.change_name("other") - self.assertEqual(i.get_name(), "other") + assert i.get_name() == "other" def test_metaclass(self): class TestItem(Item): @@ -115,8 +114,8 @@ class ItemTest(unittest.TestCase): i = TestItem() i["name"] = "John" - self.assertEqual(list(i.keys()), ["name"]) - self.assertEqual(list(i.values()), ["John"]) + assert list(i.keys()) == ["name"] + assert list(i.values()) == ["John"] i["keys"] = "Keys" i["values"] = "Values" @@ -142,8 +141,8 @@ class ItemTest(unittest.TestCase): i = TestItem() i["keys"] = 3 - self.assertEqual(list(i.keys()), ["keys"]) - self.assertEqual(list(i.values()), [3]) + assert list(i.keys()) == ["keys"] + assert list(i.values()) == [3] def test_metaclass_multiple_inheritance_simple(self): class A(Item): @@ -161,17 +160,17 @@ class ItemTest(unittest.TestCase): pass item = D(save="X", load="Y") - self.assertEqual(item["save"], "X") - self.assertEqual(item["load"], "Y") - self.assertEqual(D.fields, {"load": {"default": "A"}, "save": {"default": "A"}}) + assert item["save"] == "X" + assert item["load"] == "Y" + assert D.fields == {"load": {"default": "A"}, "save": {"default": "A"}} # D class inverted class E(C, B): pass - self.assertEqual(E(save="X")["save"], "X") - self.assertEqual(E(load="X")["load"], "X") - self.assertEqual(E.fields, {"load": {"default": "C"}, "save": {"default": "C"}}) + assert E(save="X")["save"] == "X" + assert E(load="X")["load"] == "X" + assert E.fields == {"load": {"default": "C"}, "save": {"default": "C"}} def test_metaclass_multiple_inheritance_diamond(self): class A(Item): @@ -190,31 +189,25 @@ class ItemTest(unittest.TestCase): fields = {"update": Field(default="D")} load = Field(default="D") - self.assertEqual(D(save="X")["save"], "X") - self.assertEqual(D(load="X")["load"], "X") - self.assertEqual( - D.fields, - { - "save": {"default": "C"}, - "load": {"default": "D"}, - "update": {"default": "D"}, - }, - ) + assert D(save="X")["save"] == "X" + assert D(load="X")["load"] == "X" + assert D.fields == { + "save": {"default": "C"}, + "load": {"default": "D"}, + "update": {"default": "D"}, + } # D class inverted class E(C, B): load = Field(default="E") - self.assertEqual(E(save="X")["save"], "X") - self.assertEqual(E(load="X")["load"], "X") - self.assertEqual( - E.fields, - { - "save": {"default": "C"}, - "load": {"default": "E"}, - "update": {"default": "C"}, - }, - ) + assert E(save="X")["save"] == "X" + assert E(load="X")["load"] == "X" + assert E.fields == { + "save": {"default": "C"}, + "load": {"default": "E"}, + "update": {"default": "C"}, + } def test_metaclass_multiple_inheritance_without_metaclass(self): class A(Item): @@ -234,8 +227,8 @@ class ItemTest(unittest.TestCase): with pytest.raises(KeyError): D(not_allowed="value") - self.assertEqual(D(save="X")["save"], "X") - self.assertEqual(D.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) + assert D(save="X")["save"] == "X" + assert D.fields == {"save": {"default": "A"}, "load": {"default": "A"}} # D class inverted class E(C, B): @@ -243,8 +236,8 @@ class ItemTest(unittest.TestCase): with pytest.raises(KeyError): E(not_allowed="value") - self.assertEqual(E(save="X")["save"], "X") - self.assertEqual(E.fields, {"save": {"default": "A"}, "load": {"default": "A"}}) + assert E(save="X")["save"] == "X" + assert E.fields == {"save": {"default": "A"}, "load": {"default": "A"}} def test_to_dict(self): class TestItem(Item): @@ -252,7 +245,7 @@ class ItemTest(unittest.TestCase): i = TestItem() i["name"] = "John" - self.assertEqual(dict(i), {"name": "John"}) + assert dict(i) == {"name": "John"} def test_copy(self): class TestItem(Item): @@ -260,9 +253,9 @@ class ItemTest(unittest.TestCase): item = TestItem({"name": "lower"}) copied_item = item.copy() - self.assertNotEqual(id(item), id(copied_item)) + assert id(item) != id(copied_item) copied_item["name"] = copied_item["name"].upper() - self.assertNotEqual(item["name"], copied_item["name"]) + assert item["name"] != copied_item["name"] def test_deepcopy(self): class TestItem(Item): @@ -274,7 +267,7 @@ class ItemTest(unittest.TestCase): assert item["tags"] != copied_item["tags"] -class ItemMetaTest(unittest.TestCase): +class TestItemMeta: def test_new_method_propagates_classcell(self): new_mock = mock.Mock(side_effect=ABCMeta.__new__) base = ItemMeta.__bases__[0] @@ -297,7 +290,7 @@ class ItemMetaTest(unittest.TestCase): assert "__classcell__" in attrs -class ItemMetaClassCellRegression(unittest.TestCase): +class TestItemMetaClassCellRegression: def test_item_meta_classcell_regression(self): class MyItem(Item, metaclass=ItemMeta): def __init__(self, *args, **kwargs): # pylint: disable=useless-parent-delegation diff --git a/tests/test_link.py b/tests/test_link.py index ed9d27a37..f96961075 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -1,18 +1,16 @@ -import unittest - import pytest from scrapy.link import Link -class LinkTest(unittest.TestCase): +class TestLink: def _assert_same_links(self, link1, link2): - self.assertEqual(link1, link2) - self.assertEqual(hash(link1), hash(link2)) + assert link1 == link2 + assert hash(link1) == hash(link2) def _assert_different_links(self, link1, link2): - self.assertNotEqual(link1, link2) - self.assertNotEqual(hash(link1), hash(link2)) + assert link1 != link2 + assert hash(link1) != hash(link2) def test_eq_and_hash(self): l1 = Link("http://www.example.com") diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index e751e0a63..1bff369af 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -2,7 +2,6 @@ from __future__ import annotations import pickle import re -import unittest import pytest from packaging.version import Version @@ -16,175 +15,139 @@ from tests import get_testdata # a hack to skip base class tests in pytest class Base: - class LinkExtractorTestCase(unittest.TestCase): + class TestLinkExtractorBase: extractor_cls: type | None = None - def setUp(self): + def setup_method(self): body = get_testdata("link_extractor", "linkextractor.html") self.response = HtmlResponse(url="http://example.com/index", body=body) def test_urls_type(self): """Test that the resulting urls are str objects""" lx = self.extractor_cls() - self.assertTrue( - all( - isinstance(link.url, str) - for link in lx.extract_links(self.response) - ) + assert all( + isinstance(link.url, str) for link in lx.extract_links(self.response) ) def test_extract_all_links(self): lx = self.extractor_cls() page4_url = "http://example.com/page%204.html" - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - Link(url="http://www.google.com/something", text=""), - Link(url="http://example.com/innertag.html", text="inner tag"), - Link(url=page4_url, text="href with whitespaces"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + Link(url="http://www.google.com/something", text=""), + Link(url="http://example.com/innertag.html", text="inner tag"), + Link(url=page4_url, text="href with whitespaces"), + ] def test_extract_filter_allow(self): lx = self.extractor_cls(allow=("sample",)) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + ] def test_extract_filter_allow_with_duplicates(self): lx = self.extractor_cls(allow=("sample",), unique=False) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition", - ), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition", - ), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + ] def test_extract_filter_allow_with_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=False, canonicalize=True) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition", - ), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition", - ), - Link( - url="http://example.com/sample3.html", - text="sample 3 repetition with fragment", - ), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition", + ), + Link( + url="http://example.com/sample3.html", + text="sample 3 repetition with fragment", + ), + ] def test_extract_filter_allow_no_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=True, canonicalize=True) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + ] def test_extract_filter_allow_and_deny(self): lx = self.extractor_cls(allow=("sample",), deny=("3",)) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ] def test_extract_filter_allowed_domains(self): lx = self.extractor_cls(allow_domains=("google.com",)) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://www.google.com/something", text=""), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://www.google.com/something", text=""), + ] def test_extraction_using_single_values(self): """Test the extractor's behaviour among different situations""" lx = self.extractor_cls(allow="sample") - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + ] lx = self.extractor_cls(allow="sample", deny="3") - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ] lx = self.extractor_cls(allow_domains="google.com") - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://www.google.com/something", text=""), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://www.google.com/something", text=""), + ] lx = self.extractor_cls(deny_domains="example.com") - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://www.google.com/something", text=""), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://www.google.com/something", text=""), + ] def test_nofollow(self): """Test the extractor's behaviour for links with rel='nofollow'""" @@ -210,47 +173,44 @@ class Base: response = HtmlResponse("http://example.org/somepage/index.html", body=html) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.org/about.html", text="About us"), - Link(url="http://example.org/follow.html", text="Follow this link"), - Link( - url="http://example.org/nofollow.html", - text="Dont follow this one", - nofollow=True, - ), - Link( - url="http://example.org/nofollow2.html", - text="Choose to follow or not", - ), - Link( - url="http://google.com/something", - text="External link not to follow", - nofollow=True, - ), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/about.html", text="About us"), + Link(url="http://example.org/follow.html", text="Follow this link"), + Link( + url="http://example.org/nofollow.html", + text="Dont follow this one", + nofollow=True, + ), + Link( + url="http://example.org/nofollow2.html", + text="Choose to follow or not", + ), + Link( + url="http://google.com/something", + text="External link not to follow", + nofollow=True, + ), + ] def test_matches(self): url1 = "http://lotsofstuff.com/stuff1/index" url2 = "http://evenmorestuff.com/uglystuff/index" lx = self.extractor_cls(allow=(r"stuff1",)) - self.assertTrue(lx.matches(url1)) - self.assertFalse(lx.matches(url2)) + assert lx.matches(url1) + assert not lx.matches(url2) lx = self.extractor_cls(deny=(r"uglystuff",)) - self.assertTrue(lx.matches(url1)) - self.assertFalse(lx.matches(url2)) + assert lx.matches(url1) + assert not lx.matches(url2) lx = self.extractor_cls(allow_domains=("evenmorestuff.com",)) - self.assertFalse(lx.matches(url1)) - self.assertTrue(lx.matches(url2)) + assert not lx.matches(url1) + assert lx.matches(url2) lx = self.extractor_cls(deny_domains=("lotsofstuff.com",)) - self.assertFalse(lx.matches(url1)) - self.assertTrue(lx.matches(url2)) + assert not lx.matches(url1) + assert lx.matches(url2) lx = self.extractor_cls( allow=["blah1"], @@ -258,20 +218,17 @@ class Base: allow_domains=["blah1.com"], deny_domains=["blah2.com"], ) - self.assertTrue(lx.matches("http://blah1.com/blah1")) - self.assertFalse(lx.matches("http://blah1.com/blah2")) - self.assertFalse(lx.matches("http://blah2.com/blah1")) - self.assertFalse(lx.matches("http://blah2.com/blah2")) + assert lx.matches("http://blah1.com/blah1") + assert not lx.matches("http://blah1.com/blah2") + assert not lx.matches("http://blah2.com/blah1") + assert not lx.matches("http://blah2.com/blah2") def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',)) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ] def test_restrict_xpaths_encoding(self): """Test restrict_xpaths with encodings""" @@ -291,10 +248,9 @@ class Base: ) lx = self.extractor_cls(restrict_xpaths="//div[@class='links']") - self.assertEqual( - lx.extract_links(response), - [Link(url="http://example.org/about.html", text="About us\xa3")], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/about.html", text="About us\xa3") + ] def test_restrict_xpaths_with_html_entities(self): html = b'<html><body><p><a href="/♥/you?c=€">text</a></p></body></html>' @@ -304,47 +260,40 @@ class Base: encoding="iso8859-15", ) links = self.extractor_cls(restrict_xpaths="//p").extract_links(response) - self.assertEqual( - links, [Link(url="http://example.org/%E2%99%A5/you?c=%A4", text="text")] - ) + assert links == [ + Link(url="http://example.org/%E2%99%A5/you?c=%A4", text="text") + ] def test_restrict_xpaths_concat_in_handle_data(self): """html entities cause SGMLParser to call handle_data hook twice""" body = b"""<html><body><div><a href="/foo">>\xbe\xa9<\xb6\xab</a></body></html>""" response = HtmlResponse("http://example.org", body=body, encoding="gb18030") lx = self.extractor_cls(restrict_xpaths="//div") - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.org/foo", - text=">\u4eac<\u4e1c", - fragment="", - nofollow=False, - ) - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.org/foo", + text=">\u4eac<\u4e1c", + fragment="", + nofollow=False, + ) + ] def test_restrict_css(self): lx = self.extractor_cls(restrict_css=("#subwrapper a",)) - self.assertEqual( - lx.extract_links(self.response), - [Link(url="http://example.com/sample2.html", text="sample 2")], - ) + assert lx.extract_links(self.response) == [ + Link(url="http://example.com/sample2.html", text="sample 2") + ] def test_restrict_css_and_restrict_xpaths_together(self): lx = self.extractor_cls( restrict_xpaths=('//div[@id="subwrapper"]',), restrict_css=("#subwrapper + a",), ) - self.assertEqual( - list(lx.extract_links(self.response)), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - ], - ) + assert list(lx.extract_links(self.response)) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + ] def test_area_tag_with_unicode_present(self): body = b"""<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>""" @@ -353,17 +302,14 @@ class Base: lx.extract_links(response) lx.extract_links(response) lx.extract_links(response) - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.org/foo", - text="", - fragment="", - nofollow=False, - ) - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.org/foo", + text="", + fragment="", + nofollow=False, + ) + ] def test_encoded_url(self): body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>""" @@ -371,17 +317,14 @@ class Base: "http://known.fm/AC%2FDC/", body=body, encoding="utf8" ) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://known.fm/AC%2FDC/?page=2", - text="BinB", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://known.fm/AC%2FDC/?page=2", + text="BinB", + fragment="", + nofollow=False, + ), + ] def test_encoded_url_in_restricted_xpath(self): body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>""" @@ -389,38 +332,29 @@ class Base: "http://known.fm/AC%2FDC/", body=body, encoding="utf8" ) lx = self.extractor_cls(restrict_xpaths="//div") - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://known.fm/AC%2FDC/?page=2", - text="BinB", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://known.fm/AC%2FDC/?page=2", + text="BinB", + fragment="", + nofollow=False, + ), + ] def test_ignored_extensions(self): # jpg is ignored by default html = b"""<a href="page.html">asd</a> and <a href="photo.jpg">""" response = HtmlResponse("http://example.org/", body=html) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.org/page.html", text="asd"), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/page.html", text="asd"), + ] # override denied extensions lx = self.extractor_cls(deny_extensions=["html"]) - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.org/photo.jpg"), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/photo.jpg"), + ] def test_process_value(self): """Test restrict_xpaths with encodings""" @@ -439,10 +373,9 @@ class Base: return m.group(1) if m else None lx = self.extractor_cls(process_value=process_value) - self.assertEqual( - lx.extract_links(response), - [Link(url="http://example.org/other/page.html", text="Text")], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.org/other/page.html", text="Text") + ] def test_base_url_with_restrict_xpaths(self): html = b"""<html><head><title>Page title<title><base href="http://otherdomain.com/base/" /> @@ -450,53 +383,46 @@ class Base: </body></html>""" response = HtmlResponse("http://example.org/somepage/index.html", body=html) lx = self.extractor_cls(restrict_xpaths="//p") - self.assertEqual( - lx.extract_links(response), - [Link(url="http://otherdomain.com/base/item/12.html", text="Item 12")], - ) + assert lx.extract_links(response) == [ + Link(url="http://otherdomain.com/base/item/12.html", text="Item 12") + ] def test_attrs(self): lx = self.extractor_cls(attrs="href") page4_url = "http://example.com/page%204.html" - self.assertEqual( - lx.extract_links(self.response), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - Link(url="http://www.google.com/something", text=""), - Link(url="http://example.com/innertag.html", text="inner tag"), - Link(url=page4_url, text="href with whitespaces"), - ], - ) + assert lx.extract_links(self.response) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + Link(url="http://www.google.com/something", text=""), + Link(url="http://example.com/innertag.html", text="inner tag"), + Link(url=page4_url, text="href with whitespaces"), + ] lx = self.extractor_cls( attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=() ) - self.assertEqual( - lx.extract_links(self.response), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample2.jpg", text=""), - Link(url="http://example.com/sample3.html", text="sample 3 text"), - Link( - url="http://example.com/sample3.html#foo", - text="sample 3 repetition with fragment", - ), - Link(url="http://www.google.com/something", text=""), - Link(url="http://example.com/innertag.html", text="inner tag"), - Link(url=page4_url, text="href with whitespaces"), - ], - ) + assert lx.extract_links(self.response) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample2.jpg", text=""), + Link(url="http://example.com/sample3.html", text="sample 3 text"), + Link( + url="http://example.com/sample3.html#foo", + text="sample 3 repetition with fragment", + ), + Link(url="http://www.google.com/something", text=""), + Link(url="http://example.com/innertag.html", text="inner tag"), + Link(url=page4_url, text="href with whitespaces"), + ] lx = self.extractor_cls(attrs=None) - self.assertEqual(lx.extract_links(self.response), []) + assert lx.extract_links(self.response) == [] def test_tags(self): html = ( @@ -506,43 +432,31 @@ class Base: response = HtmlResponse("http://example.com/index.html", body=html) lx = self.extractor_cls(tags=None) - self.assertEqual(lx.extract_links(response), []) + assert lx.extract_links(response) == [] lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.com/sample1.html", text=""), - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.com/sample1.html", text=""), + Link(url="http://example.com/sample2.html", text="sample 2"), + ] lx = self.extractor_cls(tags="area") - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.com/sample1.html", text=""), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.com/sample1.html", text=""), + ] lx = self.extractor_cls(tags="a") - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.com/sample2.html", text="sample 2"), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.com/sample2.html", text="sample 2"), + ] lx = self.extractor_cls( tags=("a", "img"), attrs=("href", "src"), deny_extensions=() ) - self.assertEqual( - lx.extract_links(response), - [ - Link(url="http://example.com/sample2.html", text="sample 2"), - Link(url="http://example.com/sample2.jpg", text=""), - ], - ) + assert lx.extract_links(response) == [ + Link(url="http://example.com/sample2.html", text="sample 2"), + Link(url="http://example.com/sample2.jpg", text=""), + ] def test_tags_attrs(self): html = b""" @@ -554,42 +468,36 @@ class Base: response = HtmlResponse("http://example.com/index.html", body=html) lx = self.extractor_cls(tags="div", attrs="data-url") - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.com/get?id=1", - text="Item 1", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/get?id=2", - text="Item 2", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.com/get?id=1", + text="Item 1", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/get?id=2", + text="Item 2", + fragment="", + nofollow=False, + ), + ] lx = self.extractor_cls(tags=("div",), attrs=("data-url",)) - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.com/get?id=1", - text="Item 1", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/get?id=2", - text="Item 2", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.com/get?id=1", + text="Item 1", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/get?id=2", + text="Item 2", + fragment="", + nofollow=False, + ), + ] def test_xhtml(self): xhtml = b""" @@ -623,78 +531,72 @@ class Base: response = HtmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.com/about.html", - text="About us", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/follow.html", - text="Follow this link", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/nofollow.html", - text="Dont follow this one", - fragment="", - nofollow=True, - ), - Link( - url="http://example.com/nofollow2.html", - text="Choose to follow or not", - fragment="", - nofollow=False, - ), - Link( - url="http://google.com/something", - text="External link not to follow", - nofollow=True, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.com/about.html", + text="About us", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/follow.html", + text="Follow this link", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/nofollow.html", + text="Dont follow this one", + fragment="", + nofollow=True, + ), + Link( + url="http://example.com/nofollow2.html", + text="Choose to follow or not", + fragment="", + nofollow=False, + ), + Link( + url="http://google.com/something", + text="External link not to follow", + nofollow=True, + ), + ] response = XmlResponse("http://example.com/index.xhtml", body=xhtml) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="http://example.com/about.html", - text="About us", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/follow.html", - text="Follow this link", - fragment="", - nofollow=False, - ), - Link( - url="http://example.com/nofollow.html", - text="Dont follow this one", - fragment="", - nofollow=True, - ), - Link( - url="http://example.com/nofollow2.html", - text="Choose to follow or not", - fragment="", - nofollow=False, - ), - Link( - url="http://google.com/something", - text="External link not to follow", - nofollow=True, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="http://example.com/about.html", + text="About us", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/follow.html", + text="Follow this link", + fragment="", + nofollow=False, + ), + Link( + url="http://example.com/nofollow.html", + text="Dont follow this one", + fragment="", + nofollow=True, + ), + Link( + url="http://example.com/nofollow2.html", + text="Choose to follow or not", + fragment="", + nofollow=False, + ), + Link( + url="http://google.com/something", + text="External link not to follow", + nofollow=True, + ), + ] def test_link_wrong_href(self): html = b""" @@ -704,21 +606,18 @@ class Base: """ response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() - self.assertEqual( - list(lx.extract_links(response)), - [ - Link( - url="http://example.org/item1.html", - text="Item 1", - nofollow=False, - ), - Link( - url="http://example.org/item3.html", - text="Item 3", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item1.html", + text="Item 1", + nofollow=False, + ), + Link( + url="http://example.org/item3.html", + text="Item 3", + nofollow=False, + ), + ] def test_ftp_links(self): body = b""" @@ -729,21 +628,18 @@ class Base: "http://www.example.com/index.html", body=body, encoding="utf8" ) lx = self.extractor_cls() - self.assertEqual( - lx.extract_links(response), - [ - Link( - url="ftp://www.external.com/", - text="An Item", - fragment="", - nofollow=False, - ), - ], - ) + assert lx.extract_links(response) == [ + Link( + url="ftp://www.external.com/", + text="An Item", + fragment="", + nofollow=False, + ), + ] def test_pickle_extractor(self): lx = self.extractor_cls() - self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) + assert isinstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) def test_link_extractor_aggregation(self): """When a parameter like restrict_css is used, the underlying @@ -770,14 +666,11 @@ class Base: """, ) actual = lx.extract_links(response) - self.assertEqual( - actual, - [ - Link(url="https://example.com/a", text="a1"), - Link(url="https://example.com/b?a=1&b=2", text="b1"), - Link(url="https://example.com/b?b=2&a=1", text="b2"), - ], - ) + assert actual == [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/b?b=2&a=1", text="b2"), + ] # unique=True (default), canonicalize=True lx = self.extractor_cls(restrict_css=("div",), canonicalize=True) @@ -795,13 +688,10 @@ class Base: """, ) actual = lx.extract_links(response) - self.assertEqual( - actual, - [ - Link(url="https://example.com/a", text="a1"), - Link(url="https://example.com/b?a=1&b=2", text="b1"), - ], - ) + assert actual == [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + ] # unique=False, canonicalize=False (default) lx = self.extractor_cls(restrict_css=("div",), unique=False) @@ -819,15 +709,12 @@ class Base: """, ) actual = lx.extract_links(response) - self.assertEqual( - actual, - [ - Link(url="https://example.com/a", text="a1"), - Link(url="https://example.com/b?a=1&b=2", text="b1"), - Link(url="https://example.com/a", text="a2"), - Link(url="https://example.com/b?b=2&a=1", text="b2"), - ], - ) + assert actual == [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/a", text="a2"), + Link(url="https://example.com/b?b=2&a=1", text="b2"), + ] # unique=False, canonicalize=True lx = self.extractor_cls( @@ -847,18 +734,15 @@ class Base: """, ) actual = lx.extract_links(response) - self.assertEqual( - actual, - [ - Link(url="https://example.com/a", text="a1"), - Link(url="https://example.com/b?a=1&b=2", text="b1"), - Link(url="https://example.com/a", text="a2"), - Link(url="https://example.com/b?a=1&b=2", text="b2"), - ], - ) + assert actual == [ + Link(url="https://example.com/a", text="a1"), + Link(url="https://example.com/b?a=1&b=2", text="b1"), + Link(url="https://example.com/a", text="a2"), + Link(url="https://example.com/b?a=1&b=2", text="b2"), + ] -class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): +class TestLxmlLinkExtractor(Base.TestLinkExtractorBase): extractor_cls = LxmlLinkExtractor def test_link_wrong_href(self): @@ -869,17 +753,10 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): """ response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() - self.assertEqual( - list(lx.extract_links(response)), - [ - Link( - url="http://example.org/item1.html", text="Item 1", nofollow=False - ), - Link( - url="http://example.org/item3.html", text="Item 3", nofollow=False - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link(url="http://example.org/item1.html", text="Item 1", nofollow=False), + Link(url="http://example.org/item3.html", text="Item 3", nofollow=False), + ] def test_link_restrict_text(self): html = b""" @@ -890,45 +767,36 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) # Simple text inclusion test lx = self.extractor_cls(restrict_text="dog") - self.assertEqual( - list(lx.extract_links(response)), - [ - Link( - url="http://example.org/item2.html", - text="Pic of a dog", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item2.html", + text="Pic of a dog", + nofollow=False, + ), + ] # Unique regex test lx = self.extractor_cls(restrict_text=r"of.*dog") - self.assertEqual( - list(lx.extract_links(response)), - [ - Link( - url="http://example.org/item2.html", - text="Pic of a dog", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item2.html", + text="Pic of a dog", + nofollow=False, + ), + ] # Multiple regex test lx = self.extractor_cls(restrict_text=[r"of.*dog", r"of.*cat"]) - self.assertEqual( - list(lx.extract_links(response)), - [ - Link( - url="http://example.org/item1.html", - text="Pic of a cat", - nofollow=False, - ), - Link( - url="http://example.org/item2.html", - text="Pic of a dog", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item1.html", + text="Pic of a cat", + nofollow=False, + ), + Link( + url="http://example.org/item2.html", + text="Pic of a dog", + nofollow=False, + ), + ] @pytest.mark.skipif( Version(w3lib_version) < Version("2.0.0"), @@ -945,30 +813,27 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): """ response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() - self.assertEqual( - list(lx.extract_links(response)), - [ - Link( - url="http://example.org/item2.html", - text="Good Link", - nofollow=False, - ), - Link( - url="http://example.org/item3.html", - text="Good Link 2", - nofollow=False, - ), - ], - ) + assert list(lx.extract_links(response)) == [ + Link( + url="http://example.org/item2.html", + text="Good Link", + nofollow=False, + ), + Link( + url="http://example.org/item3.html", + text="Good Link 2", + nofollow=False, + ), + ] def test_link_allowed_is_false_with_empty_url(self): bad_link = Link("") - self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) + assert not LxmlLinkExtractor()._link_allowed(bad_link) def test_link_allowed_is_false_with_bad_url_prefix(self): bad_link = Link("htp://should_be_http.example") - self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) + assert not LxmlLinkExtractor()._link_allowed(bad_link) def test_link_allowed_is_false_with_missing_url_prefix(self): bad_link = Link("should_have_prefix.example") - self.assertFalse(LxmlLinkExtractor()._link_allowed(bad_link)) + assert not LxmlLinkExtractor()._link_allowed(bad_link) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 84d714e5c..162dfdaf4 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -53,7 +53,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): ) -class FileDownloadCrawlTestCase(TestCase): +class TestFileDownloadCrawl(TestCase): pipeline_class = "scrapy.pipelines.files.FilesPipeline" store_setting_key = "FILES_STORE" media_key = "files" @@ -98,52 +98,46 @@ class FileDownloadCrawlTestCase(TestCase): return crawler def _assert_files_downloaded(self, items, logs): - self.assertEqual(len(items), 1) - self.assertIn(self.media_key, items[0]) + assert len(items) == 1 + assert self.media_key in items[0] # check that logs show the expected number of successful file downloads file_dl_success = "File (downloaded): Downloaded file from" - self.assertEqual(logs.count(file_dl_success), 3) + assert logs.count(file_dl_success) == 3 # check that the images/files status is `downloaded` for item in items: for i in item[self.media_key]: - self.assertEqual(i["status"], "downloaded") + assert i["status"] == "downloaded" # check that the images/files checksums are what we know they should be if self.expected_checksums is not None: checksums = {i["checksum"] for item in items for i in item[self.media_key]} - self.assertEqual(checksums, self.expected_checksums) + assert checksums == self.expected_checksums # check that the image files where actually written to the media store for item in items: for i in item[self.media_key]: - self.assertTrue((self.tmpmediastore / i["path"]).exists()) + assert (self.tmpmediastore / i["path"]).exists() def _assert_files_download_failure(self, crawler, items, code, logs): # check that the item does NOT have the "images/files" field populated - self.assertEqual(len(items), 1) - self.assertIn(self.media_key, items[0]) - self.assertFalse(items[0][self.media_key]) + assert len(items) == 1 + assert self.media_key in items[0] + assert not items[0][self.media_key] # check that there was 1 successful fetch and 3 other responses with non-200 code - self.assertEqual( - crawler.stats.get_value("downloader/request_method_count/GET"), 4 - ) - self.assertEqual(crawler.stats.get_value("downloader/response_count"), 4) - self.assertEqual( - crawler.stats.get_value("downloader/response_status_count/200"), 1 - ) - self.assertEqual( - crawler.stats.get_value(f"downloader/response_status_count/{code}"), 3 - ) + assert crawler.stats.get_value("downloader/request_method_count/GET") == 4 + assert crawler.stats.get_value("downloader/response_count") == 4 + assert crawler.stats.get_value("downloader/response_status_count/200") == 1 + assert crawler.stats.get_value(f"downloader/response_status_count/{code}") == 3 # check that logs do show the failure on the file downloads file_dl_failure = f"File (code: {code}): Error downloading file from" - self.assertEqual(logs.count(file_dl_failure), 3) + assert logs.count(file_dl_failure) == 3 # check that no files were written to the media store - self.assertEqual(list(self.tmpmediastore.iterdir()), []) + assert not list(self.tmpmediastore.iterdir()) @defer.inlineCallbacks def test_download_media(self): @@ -193,9 +187,7 @@ class FileDownloadCrawlTestCase(TestCase): mockserver=self.mockserver, ) self._assert_files_downloaded(self.items, str(log)) - self.assertEqual( - crawler.stats.get_value("downloader/response_status_count/302"), 3 - ) + assert crawler.stats.get_value("downloader/response_status_count/302") == 3 @defer.inlineCallbacks def test_download_media_file_path_error(self): @@ -218,7 +210,7 @@ class FileDownloadCrawlTestCase(TestCase): media_urls_key=self.media_urls_key, mockserver=self.mockserver, ) - self.assertIn("ZeroDivisionError", str(log)) + assert "ZeroDivisionError" in str(log) skip_pillow: str | None @@ -230,7 +222,7 @@ else: skip_pillow = None -class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): +class ImageDownloadCrawlTestCase(TestFileDownloadCrawl): skip = skip_pillow pipeline_class = "scrapy.pipelines.images.ImagesPipeline" diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 05fd17207..e515c16a0 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -77,7 +77,7 @@ def get_ftp_content_and_delete( return b"".join(ftp_data) -class FilesPipelineTestCase(unittest.TestCase): +class TestFilesPipeline(unittest.TestCase): def setUp(self): self.tempdir = mkdtemp() settings_dict = {"FILES_STORE": self.tempdir} @@ -91,73 +91,73 @@ class FilesPipelineTestCase(unittest.TestCase): def test_file_path(self): file_path = self.pipeline.file_path - self.assertEqual( - file_path(Request("https://dev.mydeco.com/mydeco.pdf")), - "full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf", + assert ( + file_path(Request("https://dev.mydeco.com/mydeco.pdf")) + == "full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf" ) - self.assertEqual( + assert ( file_path( Request( "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt" ) - ), - "full/4ce274dd83db0368bafd7e406f382ae088e39219.txt", + ) + == "full/4ce274dd83db0368bafd7e406f382ae088e39219.txt" ) - self.assertEqual( + assert ( file_path( Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc") - ), - "full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc", + ) + == "full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc" ) - self.assertEqual( + assert ( file_path( Request( "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg" ) - ), - "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg", + ) + == "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg" ) - self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2", + assert ( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")) + == "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2" ) - self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1", + assert ( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532")) + == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1" ) - self.assertEqual( + assert ( file_path( Request("http://www.dorma.co.uk/images/product_details/2532"), response=Response("http://www.dorma.co.uk/images/product_details/2532"), info=object(), - ), - "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1", + ) + == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1" ) - self.assertEqual( + assert ( file_path( Request( "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha" ) - ), - "full/76c00cef2ef669ae65052661f68d451162829507", + ) + == "full/76c00cef2ef669ae65052661f68d451162829507" ) - self.assertEqual( + assert ( file_path( Request( "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y" ) - ), - "full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png", + ) + == "full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png" ) def test_fs_store(self): assert isinstance(self.pipeline.store, FSFilesStore) - self.assertEqual(self.pipeline.store.basedir, self.tempdir) + assert self.pipeline.store.basedir == self.tempdir path = "some/image/key.jpg" fullpath = Path(self.tempdir, "some", "image", "key.jpg") - self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath) + assert self.pipeline.store._get_filesystem_path(path) == fullpath @defer.inlineCallbacks def test_file_not_expired(self): @@ -180,8 +180,8 @@ class FilesPipelineTestCase(unittest.TestCase): p.start() result = yield self.pipeline.process_item(item, None) - self.assertEqual(result["files"][0]["checksum"], "abc") - self.assertEqual(result["files"][0]["status"], "uptodate") + assert result["files"][0]["checksum"] == "abc" + assert result["files"][0]["status"] == "uptodate" for p in patchers: p.stop() @@ -211,8 +211,8 @@ class FilesPipelineTestCase(unittest.TestCase): p.start() result = yield self.pipeline.process_item(item, None) - self.assertNotEqual(result["files"][0]["checksum"], "abc") - self.assertEqual(result["files"][0]["status"], "downloaded") + assert result["files"][0]["checksum"] != "abc" + assert result["files"][0]["status"] == "downloaded" for p in patchers: p.stop() @@ -242,8 +242,8 @@ class FilesPipelineTestCase(unittest.TestCase): p.start() result = yield self.pipeline.process_item(item, None) - self.assertNotEqual(result["files"][0]["checksum"], "abc") - self.assertEqual(result["files"][0]["status"], "cached") + assert result["files"][0]["checksum"] != "abc" + assert result["files"][0]["status"] == "cached" for p in patchers: p.stop() @@ -262,14 +262,14 @@ class FilesPipelineTestCase(unittest.TestCase): ).file_path item = {"path": "path-to-store-file"} request = Request("http://example.com") - self.assertEqual(file_path(request, item=item), "full/path-to-store-file") + assert file_path(request, item=item) == "full/path-to-store-file" class FilesPipelineTestCaseFieldsMixin: - def setUp(self): + def setup_method(self): self.tempdir = mkdtemp() - def tearDown(self): + def teardown_method(self): rmtree(self.tempdir) def test_item_fields_default(self): @@ -279,12 +279,12 @@ class FilesPipelineTestCaseFieldsMixin: get_crawler(None, {"FILES_STORE": self.tempdir}) ) requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) + assert requests[0].url == url results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) files = ItemAdapter(item).get("files") - self.assertEqual(files, [results[0][1]]) - self.assertIsInstance(item, self.item_class) + assert files == [results[0][1]] + assert isinstance(item, self.item_class) def test_item_fields_override_settings(self): url = "http://www.example.com/files/1.txt" @@ -300,17 +300,15 @@ class FilesPipelineTestCaseFieldsMixin: ) ) requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) + assert requests[0].url == url results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) custom_files = ItemAdapter(item).get("custom_files") - self.assertEqual(custom_files, [results[0][1]]) - self.assertIsInstance(item, self.item_class) + assert custom_files == [results[0][1]] + assert isinstance(item, self.item_class) -class FilesPipelineTestCaseFieldsDict( - FilesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestFilesPipelineFieldsDict(FilesPipelineTestCaseFieldsMixin): item_class = dict @@ -324,9 +322,7 @@ class FilesPipelineTestItem(Item): custom_files = Field() -class FilesPipelineTestCaseFieldsItem( - FilesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestFilesPipelineFieldsItem(FilesPipelineTestCaseFieldsMixin): item_class = FilesPipelineTestItem @@ -341,9 +337,7 @@ class FilesPipelineTestDataClass: custom_files: list = dataclasses.field(default_factory=list) -class FilesPipelineTestCaseFieldsDataClass( - FilesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestFilesPipelineFieldsDataClass(FilesPipelineTestCaseFieldsMixin): item_class = FilesPipelineTestDataClass @@ -358,13 +352,11 @@ class FilesPipelineTestAttrsItem: custom_files: list[dict[str, str]] = attr.ib(default=list) -class FilesPipelineTestCaseFieldsAttrsItem( - FilesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestFilesPipelineFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin): item_class = FilesPipelineTestAttrsItem -class FilesPipelineTestCaseCustomSettings(unittest.TestCase): +class TestFilesPipelineCustomSettings: default_cls_settings = { "EXPIRES": 90, "FILES_URLS_FIELD": "file_urls", @@ -376,10 +368,10 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"), } - def setUp(self): + def setup_method(self): self.tempdir = mkdtemp() - def tearDown(self): + def teardown_method(self): rmtree(self.tempdir) def _generate_fake_settings(self, prefix=None): @@ -420,10 +412,10 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): one_pipeline = FilesPipeline(self.tempdir, crawler=get_crawler(None)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: default_value = self.default_cls_settings[pipe_attr] - self.assertEqual(getattr(one_pipeline, pipe_attr), default_value) + assert getattr(one_pipeline, pipe_attr) == default_value custom_value = custom_settings[settings_attr] - self.assertNotEqual(default_value, custom_value) - self.assertEqual(getattr(another_pipeline, pipe_ins_attr), custom_value) + assert default_value != custom_value + assert getattr(another_pipeline, pipe_ins_attr) == custom_value def test_subclass_attributes_preserved_if_no_settings(self): """ @@ -433,8 +425,8 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": self.tempdir})) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: custom_value = getattr(pipe, pipe_ins_attr) - self.assertNotEqual(custom_value, self.default_cls_settings[pipe_attr]) - self.assertEqual(getattr(pipe, pipe_ins_attr), getattr(pipe, pipe_attr)) + assert custom_value != self.default_cls_settings[pipe_attr] + assert getattr(pipe, pipe_ins_attr) == getattr(pipe, pipe_attr) def test_subclass_attrs_preserved_custom_settings(self): """ @@ -447,8 +439,8 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: value = getattr(pipeline, pipe_ins_attr) setting_value = settings.get(settings_attr) - self.assertNotEqual(value, self.default_cls_settings[pipe_attr]) - self.assertEqual(value, setting_value) + assert value != self.default_cls_settings[pipe_attr] + assert value == setting_value def test_no_custom_settings_for_subclasses(self): """ @@ -465,7 +457,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_cls_settings.get(pipe_attr.upper()) - self.assertEqual(getattr(user_pipeline, pipe_ins_attr), custom_value) + assert getattr(user_pipeline, pipe_ins_attr) == custom_value def test_custom_settings_for_subclasses(self): """ @@ -484,8 +476,8 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = settings.get(prefix + "_" + settings_attr) - self.assertNotEqual(custom_value, self.default_cls_settings[pipe_attr]) - self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value) + assert custom_value != self.default_cls_settings[pipe_attr] + assert getattr(user_pipeline, pipe_inst_attr) == custom_value def test_custom_settings_and_class_attrs_for_subclasses(self): """ @@ -502,8 +494,8 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): pipe_inst_attr, ) in self.file_cls_attr_settings_map: custom_value = settings.get(prefix + "_" + settings_attr) - self.assertNotEqual(custom_value, self.default_cls_settings[pipe_cls_attr]) - self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value) + assert custom_value != self.default_cls_settings[pipe_cls_attr] + assert getattr(user_pipeline, pipe_inst_attr) == custom_value def test_cls_attrs_with_DEFAULT_prefix(self): class UserDefinedFilesPipeline(FilesPipeline): @@ -513,12 +505,13 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline = UserDefinedFilesPipeline.from_crawler( get_crawler(None, {"FILES_STORE": self.tempdir}) ) - self.assertEqual( - pipeline.files_result_field, - UserDefinedFilesPipeline.DEFAULT_FILES_RESULT_FIELD, + assert ( + pipeline.files_result_field + == UserDefinedFilesPipeline.DEFAULT_FILES_RESULT_FIELD ) - self.assertEqual( - pipeline.files_urls_field, UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD + assert ( + pipeline.files_urls_field + == UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD ) def test_user_defined_subclass_default_key_names(self): @@ -535,7 +528,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: expected_value = settings.get(settings_attr) - self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value) + assert getattr(pipeline_cls, pipe_inst_attr) == expected_value def test_file_pipeline_using_pathlike_objects(self): class CustomFilesPipelineWithPathLikeDir(FilesPipeline): @@ -546,12 +539,12 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): get_crawler(None, {"FILES_STORE": Path("./Temp")}) ) request = Request("http://example.com/image01.jpg") - self.assertEqual(pipeline.file_path(request), Path("subdir/image01.jpg")) + assert pipeline.file_path(request) == Path("subdir/image01.jpg") def test_files_store_constructor_with_pathlike_object(self): path = Path("./FileDir") fs_store = FSFilesStore(path) - self.assertEqual(fs_store.basedir, str(path)) + assert fs_store.basedir == str(path) @pytest.mark.requires_botocore @@ -593,13 +586,8 @@ class TestS3FilesStore(unittest.TestCase): ) stub.assert_no_pending_responses() - self.assertEqual( - buffer.method_calls, - [ - mock.call.seek(0), - # The call to read does not happen with Stubber - ], - ) + # The call to read does not happen with Stubber + assert buffer.method_calls == [mock.call.seek(0)] @defer.inlineCallbacks def test_stat(self): @@ -626,13 +614,10 @@ class TestS3FilesStore(unittest.TestCase): ) file_stats = yield store.stat_file("", info=None) - self.assertEqual( - file_stats, - { - "checksum": checksum, - "last_modified": last_modified.timestamp(), - }, - ) + assert file_stats == { + "checksum": checksum, + "last_modified": last_modified.timestamp(), + } stub.assert_no_pending_responses() @@ -655,16 +640,16 @@ class TestGCSFilesStore(unittest.TestCase): expected_policy = {"role": "READER", "entity": "allAuthenticatedUsers"} yield store.persist_file(path, buf, info=None, meta=meta, headers=None) s = yield store.stat_file(path, info=None) - self.assertIn("last_modified", s) - self.assertIn("checksum", s) - self.assertEqual(s["checksum"], "cdcda85605e46d0af6110752770dce3c") + assert "last_modified" in s + assert "checksum" in s + assert s["checksum"] == "cdcda85605e46d0af6110752770dce3c" u = urlparse(uri) content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path) - self.assertEqual(content, data) - self.assertEqual(blob.metadata, {"foo": "bar"}) - self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) - self.assertEqual(blob.content_type, "application/octet-stream") - self.assertIn(expected_policy, acl) + assert content == data + assert blob.metadata == {"foo": "bar"} + assert blob.cache_control == GCSFilesStore.CACHE_CONTROL + assert blob.content_type == "application/octet-stream" + assert expected_policy in acl @defer.inlineCallbacks def test_blob_path_consistency(self): @@ -702,12 +687,12 @@ class TestFTPFileStore(unittest.TestCase): with MockFTPServer() as ftp_server: store = FTPFilesStore(ftp_server.url("/")) empty_dict = yield store.stat_file(path, info=None) - self.assertEqual(empty_dict, {}) + assert empty_dict == {} yield store.persist_file(path, buf, info=None, meta=meta, headers=None) stat = yield store.stat_file(path, info=None) - self.assertIn("last_modified", stat) - self.assertIn("checksum", stat) - self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6") + assert "last_modified" in stat + assert "checksum" in stat + assert stat["checksum"] == "d113d66b2ec7258724a268bd88eef6b6" path = f"{store.basedir}/{path}" content = get_ftp_content_and_delete( path, @@ -717,7 +702,7 @@ class TestFTPFileStore(unittest.TestCase): store.password, store.USE_ACTIVE_MODE, ) - self.assertEqual(data, content) + assert data == content class ItemWithFiles(Item): @@ -739,12 +724,12 @@ def _prepare_request_object(item_url, flags=None): # this is separate from the one in test_pipeline_media.py to specifically test FilesPipeline subclasses -class BuildFromCrawlerTestCase(unittest.TestCase): - def setUp(self): +class TestBuildFromCrawler: + def setup_method(self): self.tempdir = mkdtemp() self.crawler = get_crawler(None, {"FILES_STORE": self.tempdir}) - def tearDown(self): + def teardown_method(self): rmtree(self.tempdir) def test_simple(self): @@ -755,7 +740,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + assert len(w) == 0 assert pipe.store def test_has_old_init(self): @@ -768,7 +753,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 2) + assert len(w) == 2 assert pipe._init_called def test_has_from_settings(self): @@ -785,7 +770,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 3) + assert len(w) == 3 assert pipe.store assert pipe._from_settings_called @@ -805,6 +790,6 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + assert len(w) == 0 assert pipe.store assert pipe._from_crawler_called diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 1d89e44ce..fef6bbbe9 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -9,109 +9,106 @@ from tempfile import mkdtemp import attr import pytest from itemadapter import ItemAdapter -from twisted.trial import unittest from scrapy.http import Request, Response from scrapy.item import Field, Item from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.utils.test import get_crawler -skip_pillow: str | None try: from PIL import Image except ImportError: - skip_pillow = "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow" + pytest.skip( + "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow", + allow_module_level=True, + ) else: encoders = {"jpeg_encoder", "jpeg_decoder"} if not encoders.issubset(set(Image.core.__dict__)): # type: ignore[attr-defined] - skip_pillow = "Missing JPEG encoders" - else: - skip_pillow = None + pytest.skip("Missing JPEG encoders", allow_module_level=True) -class ImagesPipelineTestCase(unittest.TestCase): - skip = skip_pillow - - def setUp(self): +class TestImagesPipeline: + def setup_method(self): self.tempdir = mkdtemp() crawler = get_crawler() self.pipeline = ImagesPipeline(self.tempdir, crawler=crawler) - def tearDown(self): + def teardown_method(self): rmtree(self.tempdir) def test_file_path(self): file_path = self.pipeline.file_path - self.assertEqual( - file_path(Request("https://dev.mydeco.com/mydeco.gif")), - "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg", + assert ( + file_path(Request("https://dev.mydeco.com/mydeco.gif")) + == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" ) - self.assertEqual( + assert ( file_path( Request( "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg" ) - ), - "full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg", + ) + == "full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg" ) - self.assertEqual( + assert ( file_path( Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif") - ), - "full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg", + ) + == "full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg" ) - self.assertEqual( + assert ( file_path( Request( "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg" ) - ), - "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg", + ) + == "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg" ) - self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")), - "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg", + assert ( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")) + == "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg" ) - self.assertEqual( - file_path(Request("http://www.dorma.co.uk/images/product_details/2532")), - "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg", + assert ( + file_path(Request("http://www.dorma.co.uk/images/product_details/2532")) + == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg" ) - self.assertEqual( + assert ( file_path( Request("http://www.dorma.co.uk/images/product_details/2532"), response=Response("http://www.dorma.co.uk/images/product_details/2532"), info=object(), - ), - "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg", + ) + == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg" ) def test_thumbnail_name(self): thumb_path = self.pipeline.thumb_path name = "50" - self.assertEqual( - thumb_path(Request("file:///tmp/foo.jpg"), name), - "thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg", + assert ( + thumb_path(Request("file:///tmp/foo.jpg"), name) + == "thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg" ) - self.assertEqual( - thumb_path(Request("file://foo.png"), name), - "thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg", + assert ( + thumb_path(Request("file://foo.png"), name) + == "thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg" ) - self.assertEqual( - thumb_path(Request("file:///tmp/foo"), name), - "thumbs/50/0329ad83ebb8e93ea7c7906d46e9ed55f7349a50.jpg", + assert ( + thumb_path(Request("file:///tmp/foo"), name) + == "thumbs/50/0329ad83ebb8e93ea7c7906d46e9ed55f7349a50.jpg" ) - self.assertEqual( - thumb_path(Request("file:///tmp/some.name/foo"), name), - "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg", + assert ( + thumb_path(Request("file:///tmp/some.name/foo"), name) + == "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg" ) - self.assertEqual( + assert ( thumb_path( Request("file:///tmp/some.name/foo"), name, response=Response("file:///tmp/some.name/foo"), info=object(), - ), - "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg", + ) + == "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg" ) def test_thumbnail_name_from_item(self): @@ -130,8 +127,8 @@ class ImagesPipelineTestCase(unittest.TestCase): ).thumb_path item = {"path": "path-to-store-file"} request = Request("http://example.com") - self.assertEqual( - thumb_path(request, "small", item=item), "thumb/small/path-to-store-file" + assert ( + thumb_path(request, "small", item=item) == "thumb/small/path-to-store-file" ) def test_get_images_exception(self): @@ -169,16 +166,13 @@ class ImagesPipelineTestCase(unittest.TestCase): ) path, new_im, new_buf = next(get_images_gen) - self.assertEqual(path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg") - self.assertEqual(orig_im, new_im) - self.assertEqual(buf.getvalue(), new_buf.getvalue()) + assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + assert orig_im == new_im + assert buf.getvalue() == new_buf.getvalue() thumb_path, thumb_img, thumb_buf = next(get_images_gen) - self.assertEqual( - thumb_path, "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" - ) - self.assertEqual(thumb_img, thumb_img) - self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue()) + assert thumb_path == "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + assert orig_thumb_buf.getvalue() == thumb_buf.getvalue() def test_convert_image(self): SIZE = (100, 100) @@ -186,37 +180,35 @@ class ImagesPipelineTestCase(unittest.TestCase): COLOUR = (0, 127, 255) im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + assert converted.mode == "RGB" + assert converted.getcolors() == [(10000, COLOUR)] # check that we don't convert JPEGs again - self.assertEqual(converted_buf, buf) + assert converted_buf == buf # check that thumbnail keep image ratio thumbnail, _ = self.pipeline.convert_image( converted, size=(10, 25), response_body=converted_buf ) - self.assertEqual(thumbnail.mode, "RGB") - self.assertEqual(thumbnail.size, (10, 10)) + assert thumbnail.mode == "RGB" + assert thumbnail.size == (10, 10) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + assert converted.mode == "RGB" + assert converted.getcolors() == [(10000, (205, 230, 255))] # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR) im = im.convert("P") converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, "RGB") - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + assert converted.mode == "RGB" + assert converted.getcolors() == [(10000, (205, 230, 255))] class ImagesPipelineTestCaseFieldsMixin: - skip = skip_pillow - def test_item_fields_default(self): url = "http://www.example.com/images/1.jpg" item = self.item_class(name="item1", image_urls=[url]) @@ -224,12 +216,12 @@ class ImagesPipelineTestCaseFieldsMixin: get_crawler(None, {"IMAGES_STORE": "s3://example/images/"}) ) requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) + assert requests[0].url == url results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) images = ItemAdapter(item).get("images") - self.assertEqual(images, [results[0][1]]) - self.assertIsInstance(item, self.item_class) + assert images == [results[0][1]] + assert isinstance(item, self.item_class) def test_item_fields_override_settings(self): url = "http://www.example.com/images/1.jpg" @@ -245,17 +237,15 @@ class ImagesPipelineTestCaseFieldsMixin: ) ) requests = list(pipeline.get_media_requests(item, None)) - self.assertEqual(requests[0].url, url) + assert requests[0].url == url results = [(True, {"url": url})] item = pipeline.item_completed(results, item, None) custom_images = ItemAdapter(item).get("custom_images") - self.assertEqual(custom_images, [results[0][1]]) - self.assertIsInstance(item, self.item_class) + assert custom_images == [results[0][1]] + assert isinstance(item, self.item_class) -class ImagesPipelineTestCaseFieldsDict( - ImagesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestImagesPipelineFieldsDict(ImagesPipelineTestCaseFieldsMixin): item_class = dict @@ -269,9 +259,7 @@ class ImagesPipelineTestItem(Item): custom_images = Field() -class ImagesPipelineTestCaseFieldsItem( - ImagesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestImagesPipelineFieldsItem(ImagesPipelineTestCaseFieldsMixin): item_class = ImagesPipelineTestItem @@ -286,9 +274,7 @@ class ImagesPipelineTestDataClass: custom_images: list = dataclasses.field(default_factory=list) -class ImagesPipelineTestCaseFieldsDataClass( - ImagesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestImagesPipelineFieldsDataClass(ImagesPipelineTestCaseFieldsMixin): item_class = ImagesPipelineTestDataClass @@ -303,15 +289,11 @@ class ImagesPipelineTestAttrsItem: custom_images: list[dict[str, str]] = attr.ib(default=list) -class ImagesPipelineTestCaseFieldsAttrsItem( - ImagesPipelineTestCaseFieldsMixin, unittest.TestCase -): +class TestImagesPipelineFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin): item_class = ImagesPipelineTestAttrsItem -class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): - skip = skip_pillow - +class TestImagesPipelineCustomSettings: img_cls_attribute_names = [ # Pipeline attribute names with corresponding setting names. ("EXPIRES", "IMAGES_EXPIRES"), @@ -332,10 +314,10 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): "IMAGES_RESULT_FIELD": "images", } - def setUp(self): + def setup_method(self): self.tempdir = mkdtemp() - def tearDown(self): + def teardown_method(self): rmtree(self.tempdir) def _generate_fake_settings(self, prefix=None): @@ -397,11 +379,11 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_default_value = self.default_pipeline_settings.get(pipe_attr) custom_value = custom_settings.get(settings_attr) - self.assertNotEqual(expected_default_value, custom_value) - self.assertEqual( - getattr(default_sts_pipe, pipe_attr.lower()), expected_default_value + assert expected_default_value != custom_value + assert ( + getattr(default_sts_pipe, pipe_attr.lower()) == expected_default_value ) - self.assertEqual(getattr(user_sts_pipe, pipe_attr.lower()), custom_value) + assert getattr(user_sts_pipe, pipe_attr.lower()) == custom_value def test_subclass_attrs_preserved_default_settings(self): """ @@ -415,8 +397,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to class attribute (uppercase). attr_value = getattr(pipeline, pipe_attr.lower()) - self.assertNotEqual(attr_value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(attr_value, getattr(pipeline, pipe_attr)) + assert attr_value != self.default_pipeline_settings[pipe_attr] + assert attr_value == getattr(pipeline, pipe_attr) def test_subclass_attrs_preserved_custom_settings(self): """ @@ -430,9 +412,9 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): # Instance attribute (lowercase) must be equal to # value defined in settings. value = getattr(pipeline, pipe_attr.lower()) - self.assertNotEqual(value, self.default_pipeline_settings[pipe_attr]) + assert value != self.default_pipeline_settings[pipe_attr] setings_value = settings.get(settings_attr) - self.assertEqual(value, setings_value) + assert value == setings_value def test_no_custom_settings_for_subclasses(self): """ @@ -449,7 +431,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_pipeline_settings.get(pipe_attr.upper()) - self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value) + assert getattr(user_pipeline, pipe_attr.lower()) == custom_value def test_custom_settings_for_subclasses(self): """ @@ -468,8 +450,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = settings.get(prefix + "_" + settings_attr) - self.assertNotEqual(custom_value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value) + assert custom_value != self.default_pipeline_settings[pipe_attr] + assert getattr(user_pipeline, pipe_attr.lower()) == custom_value def test_custom_settings_and_class_attrs_for_subclasses(self): """ @@ -482,8 +464,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: custom_value = settings.get(prefix + "_" + settings_attr) - self.assertNotEqual(custom_value, self.default_pipeline_settings[pipe_attr]) - self.assertEqual(getattr(user_pipeline, pipe_attr.lower()), custom_value) + assert custom_value != self.default_pipeline_settings[pipe_attr] + assert getattr(user_pipeline, pipe_attr.lower()) == custom_value def test_cls_attrs_with_DEFAULT_prefix(self): class UserDefinedImagePipeline(ImagesPipeline): @@ -493,13 +475,13 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline = UserDefinedImagePipeline.from_crawler( get_crawler(None, {"IMAGES_STORE": self.tempdir}) ) - self.assertEqual( - pipeline.images_result_field, - UserDefinedImagePipeline.DEFAULT_IMAGES_RESULT_FIELD, + assert ( + pipeline.images_result_field + == UserDefinedImagePipeline.DEFAULT_IMAGES_RESULT_FIELD ) - self.assertEqual( - pipeline.images_urls_field, - UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD, + assert ( + pipeline.images_urls_field + == UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD ) def test_user_defined_subclass_default_key_names(self): @@ -516,7 +498,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_value = settings.get(settings_attr) - self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value) + assert getattr(pipeline_cls, pipe_attr.lower()) == expected_value def _create_image(format, *a, **kw): diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index c6fdd3767..d915fc2a3 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -2,6 +2,7 @@ from __future__ import annotations import warnings +import pytest from testfixtures import LogCapture from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks @@ -18,15 +19,6 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler -try: - from PIL import Image # noqa: F401 -except ImportError: - skip_pillow: str | None = ( - "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow" - ) -else: - skip_pillow = None - def _mocked_download_func(request, info): assert request.callback is NO_CALLBACK @@ -51,7 +43,7 @@ class UserDefinedPipeline(MediaPipeline): return "" -class BaseMediaPipelineTestCase(unittest.TestCase): +class TestBaseMediaPipeline(unittest.TestCase): pipeline_class = UserDefinedPipeline settings = None @@ -123,9 +115,9 @@ class BaseMediaPipelineTestCase(unittest.TestCase): failure = Failure(file_exc) # The Failure should encapsulate a FileException ... - self.assertEqual(failure.value, file_exc) + assert failure.value == file_exc # ... and it should have the StopIteration exception set as its context - self.assertEqual(failure.value.__context__, def_gen_return_exc) + assert failure.value.__context__ == def_gen_return_exc # Let's calculate the request fingerprint and fake some runtime data... fp = self.fingerprint(request) @@ -136,12 +128,12 @@ class BaseMediaPipelineTestCase(unittest.TestCase): # When calling the method that caches the Request's result ... self.pipe._cache_result_and_execute_waiters(failure, fp, info) # ... it should store the Twisted Failure ... - self.assertEqual(info.downloaded[fp], failure) + assert info.downloaded[fp] == failure # ... encapsulating the original FileException ... - self.assertEqual(info.downloaded[fp].value, file_exc) + assert info.downloaded[fp].value == file_exc # ... but it should not store the StopIteration exception on its context context = getattr(info.downloaded[fp].value, "__context__", None) - self.assertIsNone(context) + assert context is None def test_default_item_completed(self): item = {"name": "name"} @@ -158,7 +150,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert len(log.records) == 1 record = log.records[0] assert record.levelname == "ERROR" - self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) + assert record.exc_info == failure_to_exc_info(fail) # disable failure logging and check again self.pipe.LOG_FAILED_RESULTS = False @@ -208,7 +200,7 @@ class MockedMediaPipeline(UserDefinedPipeline): return item -class MediaPipelineTestCase(BaseMediaPipelineTestCase): +class TestMediaPipeline(TestBaseMediaPipeline): pipeline_class = MockedMediaPipeline def _errback(self, result): @@ -225,16 +217,13 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ) item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, {})]) - self.assertEqual( - self.pipe._mockcalled, - [ - "get_media_requests", - "media_to_download", - "media_downloaded", - "item_completed", - ], - ) + assert new_item["results"] == [(True, {})] + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "media_downloaded", + "item_completed", + ] @inlineCallbacks def test_result_failure(self): @@ -247,17 +236,14 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ) item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(False, fail)]) - self.assertEqual( - self.pipe._mockcalled, - [ - "get_media_requests", - "media_to_download", - "media_failed", - "request_errback", - "item_completed", - ], - ) + assert new_item["results"] == [(False, fail)] + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "media_failed", + "request_errback", + "item_completed", + ] @inlineCallbacks def test_mix_of_success_and_failure(self): @@ -268,18 +254,18 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req2 = Request("http://url2", meta={"response": fail}) item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, {}), (False, fail)]) + assert new_item["results"] == [(True, {}), (False, fail)] m = self.pipe._mockcalled # only once - self.assertEqual(m[0], "get_media_requests") # first hook called - self.assertEqual(m.count("get_media_requests"), 1) - self.assertEqual(m.count("item_completed"), 1) - self.assertEqual(m[-1], "item_completed") # last hook called + assert m[0] == "get_media_requests" # first hook called + assert m.count("get_media_requests") == 1 + assert m.count("item_completed") == 1 + assert m[-1] == "item_completed" # last hook called # twice, one per request - self.assertEqual(m.count("media_to_download"), 2) + assert m.count("media_to_download") == 2 # one to handle success and other for failure - self.assertEqual(m.count("media_downloaded"), 1) - self.assertEqual(m.count("media_failed"), 1) + assert m.count("media_downloaded") == 1 + assert m.count("media_failed") == 1 @inlineCallbacks def test_get_media_requests(self): @@ -288,7 +274,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = {"requests": req} # pass a single item new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item - self.assertIn(self.fingerprint(req), self.info.downloaded) + assert self.fingerprint(req) in self.info.downloaded # returns iterable of Requests req1 = Request("http://url1") @@ -305,8 +291,8 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): req1 = Request("http://url1", meta={"response": rsp1}) item = {"requests": req1} new_item = yield self.pipe.process_item(item, self.spider) - self.assertTrue(new_item is item) - self.assertEqual(new_item["results"], [(True, {})]) + assert new_item is item + assert new_item["results"] == [(True, {})] # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same req2 = Request( @@ -314,9 +300,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ) item = {"requests": req2} new_item = yield self.pipe.process_item(item, self.spider) - self.assertTrue(new_item is item) - self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) - self.assertEqual(new_item["results"], [(True, {})]) + assert new_item is item + assert self.fingerprint(req1) == self.fingerprint(req2) + assert new_item["results"] == [(True, {})] @inlineCallbacks def test_results_are_cached_for_requests_of_single_item(self): @@ -327,17 +313,17 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ) item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) - self.assertTrue(new_item is item) - self.assertEqual(new_item["results"], [(True, {}), (True, {})]) + assert new_item is item + assert new_item["results"] == [(True, {}), (True, {})] @inlineCallbacks def test_wait_if_request_is_downloading(self): def _check_downloading(response): fp = self.fingerprint(req1) - self.assertTrue(fp in self.info.downloading) - self.assertTrue(fp in self.info.waiting) - self.assertTrue(fp not in self.info.downloaded) - self.assertEqual(len(self.info.waiting[fp]), 2) + assert fp in self.info.downloading + assert fp in self.info.waiting + assert fp not in self.info.downloaded + assert len(self.info.waiting[fp]) == 2 return response rsp1 = Response("http://url") @@ -348,39 +334,40 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): return dfd def rsp2_func(): - self.fail("it must cache rsp1 result and must not try to redownload") + pytest.fail("it must cache rsp1 result and must not try to redownload") req1 = Request("http://url", meta={"response": rsp1_func}) req2 = Request(req1.url, meta={"response": rsp2_func}) item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, {}), (True, {})]) + assert new_item["results"] == [(True, {}), (True, {})] @inlineCallbacks def test_use_media_to_download_result(self): req = Request("http://url", meta={"result": "ITSME", "response": self.fail}) item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) - self.assertEqual(new_item["results"], [(True, "ITSME")]) - self.assertEqual( - self.pipe._mockcalled, - ["get_media_requests", "media_to_download", "item_completed"], - ) + assert new_item["results"] == [(True, "ITSME")] + assert self.pipe._mockcalled == [ + "get_media_requests", + "media_to_download", + "item_completed", + ] def test_key_for_pipe(self): - self.assertEqual( - self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline"), - "MOCKEDMEDIAPIPELINE_IMAGES", + assert ( + self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline") + == "MOCKEDMEDIAPIPELINE_IMAGES" ) -class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): +class TestMediaPipelineAllowRedirectSettings: def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(crawler=get_crawler(None, settings)) request = Request("http://url") pipe._modify_media_request(request) - self.assertIn("handle_httpstatus_list", request.meta) + assert "handle_httpstatus_list" in request.meta for status, check in [ (200, True), # These are the status codes we want @@ -396,9 +383,9 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): (500, True), ]: if check: - self.assertIn(status, request.meta["handle_httpstatus_list"]) + assert status in request.meta["handle_httpstatus_list"] else: - self.assertNotIn(status, request.meta["handle_httpstatus_list"]) + assert status not in request.meta["handle_httpstatus_list"] def test_subclass_standard_setting(self): self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True}) @@ -409,8 +396,8 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): ) -class BuildFromCrawlerTestCase(unittest.TestCase): - def setUp(self): +class TestBuildFromCrawler: + def setup_method(self): self.crawler = get_crawler(None, {"FILES_STORE": "/foo"}) def test_simple(self): @@ -421,7 +408,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + assert len(w) == 0 def test_has_old_init(self): class Pipeline(UserDefinedPipeline): @@ -433,7 +420,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 2) + assert len(w) == 2 assert pipe._init_called def test_has_from_settings(self): @@ -450,7 +437,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 2) + assert len(w) == 2 assert pipe._from_settings_called def test_has_from_settings_and_from_crawler(self): @@ -474,7 +461,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 2) + assert len(w) == 2 assert pipe._from_settings_called assert pipe._from_crawler_called @@ -497,7 +484,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 2) + assert len(w) == 2 assert pipe._from_settings_called assert pipe._init_called @@ -521,7 +508,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + assert len(w) == 0 assert pipe._from_crawler_called assert pipe._init_called @@ -542,5 +529,5 @@ class BuildFromCrawlerTestCase(unittest.TestCase): # this and the next assert will fail as MediaPipeline.from_crawler() wasn't called assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 0) + assert len(w) == 0 assert pipe._from_crawler_called From 380c2279b92f1aa7386e79fc43109499a057e8cf Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 9 Mar 2025 23:23:51 +0400 Subject: [PATCH 1680/2083] Converting tests to plain asserts, part 7. (#6710) --- tests/test_downloader_handlers.py | 243 ++++++++-------- tests/test_downloader_handlers_http2.py | 41 ++- tests/test_exporters.py | 168 +++++------ tests/test_feedexport.py | 367 +++++++++++------------- tests/test_http2_client_protocol.py | 123 ++++---- tests/test_http_cookies.py | 52 ++-- tests/test_http_headers.py | 87 +++--- 7 files changed, 512 insertions(+), 569 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 323a51002..19bd02498 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -69,46 +69,46 @@ class OffDH: return cls(crawler) -class LoadTestCase(unittest.TestCase): +class TestLoad: def test_enabled_handler(self): handlers = {"scheme": DummyDH} crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertIn("scheme", dh._schemes) - self.assertIn("scheme", dh._handlers) - self.assertNotIn("scheme", dh._notconfigured) + assert "scheme" in dh._schemes + assert "scheme" in dh._handlers + assert "scheme" not in dh._notconfigured def test_not_configured_handler(self): handlers = {"scheme": OffDH} crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertIn("scheme", dh._schemes) - self.assertNotIn("scheme", dh._handlers) - self.assertIn("scheme", dh._notconfigured) + assert "scheme" in dh._schemes + assert "scheme" not in dh._handlers + assert "scheme" in dh._notconfigured def test_disabled_handler(self): handlers = {"scheme": None} crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertNotIn("scheme", dh._schemes) + assert "scheme" not in dh._schemes for scheme in handlers: # force load handlers dh._get_handler(scheme) - self.assertNotIn("scheme", dh._handlers) - self.assertIn("scheme", dh._notconfigured) + assert "scheme" not in dh._handlers + assert "scheme" in dh._notconfigured def test_lazy_handlers(self): handlers = {"scheme": DummyLazyDH} crawler = get_crawler(settings_dict={"DOWNLOAD_HANDLERS": handlers}) dh = DownloadHandlers(crawler) - self.assertIn("scheme", dh._schemes) - self.assertNotIn("scheme", dh._handlers) + assert "scheme" in dh._schemes + assert "scheme" not in dh._handlers for scheme in handlers: # force load lazy handler dh._get_handler(scheme) - self.assertIn("scheme", dh._handlers) - self.assertNotIn("scheme", dh._notconfigured) + assert "scheme" in dh._handlers + assert "scheme" not in dh._notconfigured -class FileTestCase(unittest.TestCase): +class TestFile(unittest.TestCase): def setUp(self): # add a special char to check that they are handled correctly self.fd, self.tmpname = mkstemp(suffix="^") @@ -122,10 +122,10 @@ class FileTestCase(unittest.TestCase): def test_download(self): def _test(response): - self.assertEqual(response.url, request.url) - self.assertEqual(response.status, 200) - self.assertEqual(response.body, b"0123456789") - self.assertEqual(response.protocol, None) + assert response.url == request.url + assert response.status == 200 + assert response.body == b"0123456789" + assert response.protocol is None request = Request(path_to_file_uri(self.tmpname)) assert request.url.upper().endswith("%5E") @@ -217,7 +217,7 @@ class DuplicateHeaderResource(resource.Resource): return b"" -class HttpTestCase(unittest.TestCase, ABC): +class TestHttp(unittest.TestCase, ABC): scheme = "http" # only used for HTTPS tests @@ -336,8 +336,8 @@ class HttpTestCase(unittest.TestCase, ABC): def test_host_header_not_in_request_headers(self): def _test(response): - self.assertEqual(response.body, to_bytes(f"{self.host}:{self.portno}")) - self.assertEqual(request.headers, {}) + assert response.body == to_bytes(f"{self.host}:{self.portno}") + assert not request.headers request = Request(self.getURL("host")) return self.download_request(request, Spider("foo")).addCallback(_test) @@ -346,8 +346,8 @@ class HttpTestCase(unittest.TestCase, ABC): host = self.host + ":" + str(self.portno) def _test(response): - self.assertEqual(response.body, host.encode()) - self.assertEqual(request.headers.get("Host"), host.encode()) + assert response.body == host.encode() + assert request.headers.get("Host") == host.encode() request = Request(self.getURL("host"), headers={"Host": host}) return self.download_request(request, Spider("foo")).addCallback(_test) @@ -365,7 +365,7 @@ class HttpTestCase(unittest.TestCase, ABC): """ def _test(response): - self.assertEqual(response.body, b"0") + assert response.body == b"0" request = Request(self.getURL("contentlength"), method="POST") return self.download_request(request, Spider("foo")).addCallback(_test) @@ -376,8 +376,8 @@ class HttpTestCase(unittest.TestCase, ABC): headers = Headers(json.loads(response.text)["headers"]) contentlengths = headers.getlist("Content-Length") - self.assertEqual(len(contentlengths), 1) - self.assertEqual(contentlengths, [b"0"]) + assert len(contentlengths) == 1 + assert contentlengths == [b"0"] request = Request(self.getURL("echo"), method="POST") return self.download_request(request, Spider("foo")).addCallback(_test) @@ -399,7 +399,7 @@ class HttpTestCase(unittest.TestCase, ABC): def _test_response_class(self, filename, body, response_class): def _test(response): - self.assertEqual(type(response), response_class) + assert type(response) is response_class # pylint: disable=unidiomatic-typecheck request = Request(self.getURL(filename), body=body) return self.download_request(request, Spider("foo")).addCallback(_test) @@ -416,17 +416,14 @@ class HttpTestCase(unittest.TestCase, ABC): def test_get_duplicate_header(self): def _test(response): - self.assertEqual( - response.headers.getlist(b"Set-Cookie"), - [b"a=b", b"c=d"], - ) + assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] request = Request(self.getURL("duplicate-header")) return self.download_request(request, Spider("foo")).addCallback(_test) @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class Http10TestCase(HttpTestCase): +class TestHttp10(TestHttp): """HTTP 1.0 test case""" @property @@ -441,11 +438,11 @@ class Http10TestCase(HttpTestCase): return d -class Https10TestCase(Http10TestCase): +class TestHttps10(TestHttp10): scheme = "https" -class Http11TestCase(HttpTestCase): +class TestHttp11(TestHttp): """HTTP 1.1 test case""" @property @@ -466,7 +463,7 @@ class Http11TestCase(HttpTestCase): body = b"Some plain text\ndata with tabs\t and null bytes\0" def _test_type(response): - self.assertEqual(type(response), TextResponse) + assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck request = Request(self.getURL("nocontenttype"), body=body) d = self.download_request(request, Spider("foo")) @@ -583,7 +580,7 @@ class Http11TestCase(HttpTestCase): return d -class Https11TestCase(Http11TestCase): +class TestHttps11(TestHttp11): scheme = "https" tls_log_message = ( @@ -611,7 +608,7 @@ class Https11TestCase(Http11TestCase): yield download_handler.close() -class SimpleHttpsTest(unittest.TestCase): +class TestSimpleHttps(unittest.TestCase): """Base class for special cases tested with just one simple request""" keyfile = "keys/localhost.key" @@ -663,7 +660,7 @@ class SimpleHttpsTest(unittest.TestCase): return d -class Https11WrongHostnameTestCase(SimpleHttpsTest): +class TestHttps11WrongHostname(TestSimpleHttps): # above tests use a server certificate for "localhost", # client connection to "localhost" too. # here we test that even if the server certificate is for another domain, @@ -673,7 +670,7 @@ class Https11WrongHostnameTestCase(SimpleHttpsTest): certfile = "keys/example-com.cert.pem" -class Https11InvalidDNSId(SimpleHttpsTest): +class TestHttps11InvalidDNSId(TestSimpleHttps): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" def setUp(self): @@ -681,18 +678,18 @@ class Https11InvalidDNSId(SimpleHttpsTest): self.host = "127.0.0.1" -class Https11InvalidDNSPattern(SimpleHttpsTest): +class TestHttps11InvalidDNSPattern(TestSimpleHttps): """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" keyfile = "keys/localhost.ip.key" certfile = "keys/localhost.ip.crt" -class Https11CustomCiphers(SimpleHttpsTest): +class TestHttps11CustomCiphers(TestSimpleHttps): cipher_string = "CAMELLIA256-SHA" -class Http11MockServerTestCase(unittest.TestCase): +class TestHttp11MockServer(unittest.TestCase): """HTTP 1.1 test case with MockServer""" settings_dict: dict | None = None @@ -719,7 +716,7 @@ class Http11MockServerTestCase(unittest.TestCase): ) ) failure = crawler.spider.meta["failure"] - self.assertIsInstance(failure.value, defer.CancelledError) + assert isinstance(failure.value, defer.CancelledError) @defer.inlineCallbacks def test_download(self): @@ -728,9 +725,9 @@ class Http11MockServerTestCase(unittest.TestCase): seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) ) failure = crawler.spider.meta.get("failure") - self.assertTrue(failure is None) + assert failure is None reason = crawler.spider.meta["close_reason"] - self.assertTrue(reason, "finished") + assert reason == "finished" class UriResource(resource.Resource): @@ -748,7 +745,7 @@ class UriResource(resource.Resource): return b"" -class HttpProxyTestCase(unittest.TestCase, ABC): +class TestHttpProxy(unittest.TestCase, ABC): expected_http_proxy_request_body = b"http://example.com" @property @@ -777,9 +774,9 @@ class HttpProxyTestCase(unittest.TestCase, ABC): def test_download_with_proxy(self): def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, self.expected_http_proxy_request_body) + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body http_proxy = self.getURL("") request = Request("http://example.com", meta={"proxy": http_proxy}) @@ -787,22 +784,22 @@ class HttpProxyTestCase(unittest.TestCase, ABC): def test_download_without_proxy(self): def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b"/path/to/resource") + assert response.status == 200 + assert response.url == request.url + assert response.body == b"/path/to/resource" request = Request(self.getURL("path/to/resource")) return self.download_request(request, Spider("foo")).addCallback(_test) @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class Http10ProxyTestCase(HttpProxyTestCase): +class TestHttp10Proxy(TestHttpProxy): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return HTTP10DownloadHandler -class Http11ProxyTestCase(HttpProxyTestCase): +class TestHttp11Proxy(TestHttpProxy): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return HTTP11DownloadHandler @@ -817,13 +814,13 @@ class Http11ProxyTestCase(HttpProxyTestCase): request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) d = self.download_request(request, Spider("foo")) timeout = yield self.assertFailure(d, error.TimeoutError) - self.assertIn(domain, timeout.osError) + assert domain in timeout.osError def test_download_with_proxy_without_http_scheme(self): def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, self.expected_http_proxy_request_body) + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body http_proxy = self.getURL("").replace("http://", "") request = Request("http://example.com", meta={"proxy": http_proxy}) @@ -839,8 +836,8 @@ class HttpDownloadHandlerMock: @pytest.mark.requires_botocore -class S3AnonTestCase(unittest.TestCase): - def setUp(self): +class TestS3Anon: + def setup_method(self): crawler = get_crawler() self.s3reqh = build_from_crawler( S3DownloadHandler, @@ -854,13 +851,13 @@ class S3AnonTestCase(unittest.TestCase): def test_anon_request(self): req = Request("s3://aws-publicdatasets/") httpreq = self.download_request(req, self.spider) - self.assertEqual(hasattr(self.s3reqh, "anon"), True) - self.assertEqual(self.s3reqh.anon, True) - self.assertEqual(httpreq.url, "http://aws-publicdatasets.s3.amazonaws.com/") + assert hasattr(self.s3reqh, "anon") + assert self.s3reqh.anon + assert httpreq.url == "http://aws-publicdatasets.s3.amazonaws.com/" @pytest.mark.requires_botocore -class S3TestCase(unittest.TestCase): +class TestS3: download_handler_cls: type = S3DownloadHandler # test use same example keys than amazon developer guide @@ -870,7 +867,7 @@ class S3TestCase(unittest.TestCase): AWS_ACCESS_KEY_ID = "0PN5J17HBGZHT7JJ3X82" AWS_SECRET_ACCESS_KEY = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o" - def setUp(self): + def setup_method(self): crawler = get_crawler() s3reqh = build_from_crawler( S3DownloadHandler, @@ -897,17 +894,13 @@ class S3TestCase(unittest.TestCase): yield def test_extra_kw(self): - try: - crawler = get_crawler() + crawler = get_crawler() + with pytest.raises((TypeError, NotConfigured)): build_from_crawler( S3DownloadHandler, crawler, extra_kw=True, ) - except Exception as e: - self.assertIsInstance(e, (TypeError, NotConfigured)) - else: - raise AssertionError def test_request_signing1(self): # gets an object from the johnsmith bucket. @@ -915,9 +908,9 @@ class S3TestCase(unittest.TestCase): req = Request("s3://johnsmith/photos/puppy.jpg", headers={"Date": date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual( - httpreq.headers["Authorization"], - b"AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=", + assert ( + httpreq.headers["Authorization"] + == b"AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=" ) def test_request_signing2(self): @@ -934,9 +927,9 @@ class S3TestCase(unittest.TestCase): ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual( - httpreq.headers["Authorization"], - b"AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=", + assert ( + httpreq.headers["Authorization"] + == b"AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=" ) def test_request_signing3(self): @@ -952,9 +945,9 @@ class S3TestCase(unittest.TestCase): ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual( - httpreq.headers["Authorization"], - b"AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=", + assert ( + httpreq.headers["Authorization"] + == b"AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=" ) def test_request_signing4(self): @@ -963,9 +956,9 @@ class S3TestCase(unittest.TestCase): req = Request("s3://johnsmith/?acl", method="GET", headers={"Date": date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual( - httpreq.headers["Authorization"], - b"AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=", + assert ( + httpreq.headers["Authorization"] + == b"AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=" ) def test_request_signing6(self): @@ -991,9 +984,9 @@ class S3TestCase(unittest.TestCase): ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual( - httpreq.headers["Authorization"], - b"AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=", + assert ( + httpreq.headers["Authorization"] + == b"AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=" ) def test_request_signing7(self): @@ -1006,13 +999,13 @@ class S3TestCase(unittest.TestCase): ) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual( - httpreq.headers["Authorization"], - b"AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=", + assert ( + httpreq.headers["Authorization"] + == b"AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=" ) -class BaseFTPTestCase(unittest.TestCase): +class TestFTPBase(unittest.TestCase): username = "scrapy" password = "passwd" req_meta = {"ftp_user": username, "ftp_password": password} @@ -1068,10 +1061,10 @@ class BaseFTPTestCase(unittest.TestCase): d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(r.status, 200) - self.assertEqual(r.body, b"I have the power!") - self.assertEqual(r.headers, {b"Local Filename": [b""], b"Size": [b"17"]}) - self.assertIsNone(r.protocol) + assert r.status == 200 + assert r.body == b"I have the power!" + assert r.headers == {b"Local Filename": [b""], b"Size": [b"17"]} + assert r.protocol is None return self._add_test_callbacks(d, _test) @@ -1083,9 +1076,9 @@ class BaseFTPTestCase(unittest.TestCase): d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(r.status, 200) - self.assertEqual(r.body, b"Moooooooooo power!") - self.assertEqual(r.headers, {b"Local Filename": [b""], b"Size": [b"18"]}) + assert r.status == 200 + assert r.body == b"Moooooooooo power!" + assert r.headers == {b"Local Filename": [b""], b"Size": [b"18"]} return self._add_test_callbacks(d, _test) @@ -1096,7 +1089,7 @@ class BaseFTPTestCase(unittest.TestCase): d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(r.status, 404) + assert r.status == 404 return self._add_test_callbacks(d, _test) @@ -1111,12 +1104,10 @@ class BaseFTPTestCase(unittest.TestCase): d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(r.body, fname_bytes) - self.assertEqual( - r.headers, {b"Local Filename": [fname_bytes], b"Size": [b"17"]} - ) - self.assertTrue(local_fname.exists()) - self.assertEqual(local_fname.read_bytes(), b"I have the power!") + assert r.body == fname_bytes + assert r.headers == {b"Local Filename": [fname_bytes], b"Size": [b"17"]} + assert local_fname.exists() + assert local_fname.read_bytes() == b"I have the power!" local_fname.unlink() return self._add_test_callbacks(d, _test) @@ -1131,7 +1122,7 @@ class BaseFTPTestCase(unittest.TestCase): d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(type(r), response_class) + assert type(r) is response_class # pylint: disable=unidiomatic-typecheck local_fname.unlink() return self._add_test_callbacks(d, _test) @@ -1143,7 +1134,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._test_response_class("html-file-without-extension", HtmlResponse) -class FTPTestCase(BaseFTPTestCase): +class TestFTP(TestFTPBase): def test_invalid_credentials(self): if self.reactor_pytest == "asyncio" and sys.platform == "win32": raise unittest.SkipTest( @@ -1157,12 +1148,12 @@ class FTPTestCase(BaseFTPTestCase): d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(r.type, ConnectionLost) + assert r.type == ConnectionLost return self._add_test_callbacks(d, errback=_test) -class AnonymousFTPTestCase(BaseFTPTestCase): +class TestAnonymousFTP(TestFTPBase): username = "anonymous" req_meta = {} @@ -1188,7 +1179,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase): shutil.rmtree(self.directory) -class DataURITestCase(unittest.TestCase): +class TestDataURI(unittest.TestCase): def setUp(self): crawler = get_crawler() self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler) @@ -1199,44 +1190,44 @@ class DataURITestCase(unittest.TestCase): uri = "data:,A%20brief%20note" def _test(response): - self.assertEqual(response.url, uri) - self.assertFalse(response.headers) + assert response.url == uri + assert not response.headers request = Request(uri) return self.download_request(request, self.spider).addCallback(_test) def test_default_mediatype_encoding(self): def _test(response): - self.assertEqual(response.text, "A brief note") - self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) - self.assertEqual(response.encoding, "US-ASCII") + assert response.text == "A brief note" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "US-ASCII" request = Request("data:,A%20brief%20note") return self.download_request(request, self.spider).addCallback(_test) def test_default_mediatype(self): def _test(response): - self.assertEqual(response.text, "\u038e\u03a3\u038e") - self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) - self.assertEqual(response.encoding, "iso-8859-7") + assert response.text == "\u038e\u03a3\u038e" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "iso-8859-7" request = Request("data:;charset=iso-8859-7,%be%d3%be") return self.download_request(request, self.spider).addCallback(_test) def test_text_charset(self): def _test(response): - self.assertEqual(response.text, "\u038e\u03a3\u038e") - self.assertEqual(response.body, b"\xbe\xd3\xbe") - self.assertEqual(response.encoding, "iso-8859-7") + assert response.text == "\u038e\u03a3\u038e" + assert response.body == b"\xbe\xd3\xbe" + assert response.encoding == "iso-8859-7" request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") return self.download_request(request, self.spider).addCallback(_test) def test_mediatype_parameters(self): def _test(response): - self.assertEqual(response.text, "\u038e\u03a3\u038e") - self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) - self.assertEqual(response.encoding, "utf-8") + assert response.text == "\u038e\u03a3\u038e" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "utf-8" request = Request( "data:text/plain;foo=%22foo;bar%5C%22%22;" @@ -1247,14 +1238,14 @@ class DataURITestCase(unittest.TestCase): def test_base64(self): def _test(response): - self.assertEqual(response.text, "Hello, world.") + assert response.text == "Hello, world." request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D") return self.download_request(request, self.spider).addCallback(_test) def test_protocol(self): def _test(response): - self.assertIsNone(response.protocol) + assert response.protocol is None request = Request("data:,") return self.download_request(request, self.spider).addCallback(_test) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 17d5c2d0a..c74c09cbb 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -4,7 +4,6 @@ from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer, error, reactor -from twisted.trial import unittest from twisted.web import server from twisted.web.error import SchemeNotSupported from twisted.web.http import H2_ENABLED @@ -28,25 +27,25 @@ class BaseTestClasses: # A hack to prevent tests from the imported classes to run here too. # See https://stackoverflow.com/q/1323455/113586 for other ways. from tests.test_downloader_handlers import ( - Http11MockServerTestCase as Http11MockServerTestCase, + TestHttp11MockServer as TestHttp11MockServer, ) from tests.test_downloader_handlers import ( - Http11ProxyTestCase as Http11ProxyTestCase, + TestHttp11Proxy as TestHttp11Proxy, ) from tests.test_downloader_handlers import ( - Https11CustomCiphers as Https11CustomCiphers, + TestHttps11 as TestHttps11, ) from tests.test_downloader_handlers import ( - Https11InvalidDNSId as Https11InvalidDNSId, + TestHttps11CustomCiphers as TestHttps11CustomCiphers, ) from tests.test_downloader_handlers import ( - Https11InvalidDNSPattern as Https11InvalidDNSPattern, + TestHttps11InvalidDNSId as TestHttps11InvalidDNSId, ) from tests.test_downloader_handlers import ( - Https11TestCase as Https11TestCase, + TestHttps11InvalidDNSPattern as TestHttps11InvalidDNSPattern, ) from tests.test_downloader_handlers import ( - Https11WrongHostnameTestCase as Https11WrongHostnameTestCase, + TestHttps11WrongHostname as TestHttps11WrongHostname, ) @@ -56,7 +55,7 @@ def _get_dh() -> type[DownloadHandlerProtocol]: return H2DownloadHandler -class Https2TestCase(BaseTestClasses.Https11TestCase): +class TestHttps2(BaseTestClasses.TestHttps11): scheme = "https" HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" @@ -97,22 +96,22 @@ class Https2TestCase(BaseTestClasses.Https11TestCase): yield self.assertFailure(d, SchemeNotSupported) def test_download_broken_content_cause_data_loss(self, url="broken"): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_chunked_content_cause_data_loss(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_content_allow_data_loss(self, url="broken"): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_chunked_content_allow_data_loss(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) def test_download_broken_chunked_content_allow_data_loss_via_setting(self): - raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) def test_concurrent_requests_same_domain(self): spider = Spider("foo") @@ -180,31 +179,31 @@ class Https2TestCase(BaseTestClasses.Https11TestCase): return d -class Https2WrongHostnameTestCase(BaseTestClasses.Https11WrongHostnameTestCase): +class Https2WrongHostnameTestCase(BaseTestClasses.TestHttps11WrongHostname): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return _get_dh() -class Https2InvalidDNSId(BaseTestClasses.Https11InvalidDNSId): +class Https2InvalidDNSId(BaseTestClasses.TestHttps11InvalidDNSId): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return _get_dh() -class Https2InvalidDNSPattern(BaseTestClasses.Https11InvalidDNSPattern): +class Https2InvalidDNSPattern(BaseTestClasses.TestHttps11InvalidDNSPattern): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return _get_dh() -class Https2CustomCiphers(BaseTestClasses.Https11CustomCiphers): +class Https2CustomCiphers(BaseTestClasses.TestHttps11CustomCiphers): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return _get_dh() -class Http2MockServerTestCase(BaseTestClasses.Http11MockServerTestCase): +class Http2MockServerTestCase(BaseTestClasses.TestHttp11MockServer): """HTTP 2.0 test case with MockServer""" settings_dict = { @@ -215,7 +214,7 @@ class Http2MockServerTestCase(BaseTestClasses.Http11MockServerTestCase): is_secure = True -class Https2ProxyTestCase(BaseTestClasses.Http11ProxyTestCase): +class Https2ProxyTestCase(BaseTestClasses.TestHttp11Proxy): # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 48728e078..f55cb6c97 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -54,11 +54,11 @@ class CustomFieldDataclass: age: int = dataclasses.field(metadata={"serializer": custom_serializer}) -class BaseItemExporterTest(unittest.TestCase): +class TestBaseItemExporter: item_class: type = MyItem custom_field_item_class: type = CustomFieldItem - def setUp(self): + def setup_method(self): self.i = self.item_class(name="John\xa3", age="22") self.output = BytesIO() self.ie = self._get_exporter() @@ -72,7 +72,7 @@ class BaseItemExporterTest(unittest.TestCase): def _assert_expected_item(self, exported_dict): for k, v in exported_dict.items(): exported_dict[k] = to_unicode(v) - self.assertEqual(self.i, self.item_class(**exported_dict)) + assert self.i == self.item_class(**exported_dict) def _get_nonstring_types_item(self): return { @@ -105,45 +105,40 @@ class BaseItemExporterTest(unittest.TestCase): def test_serialize_field(self): a = ItemAdapter(self.i) res = self.ie.serialize_field(a.get_field_meta("name"), "name", a["name"]) - self.assertEqual(res, "John\xa3") + assert res == "John\xa3" res = self.ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) - self.assertEqual(res, "22") + assert res == "22" def test_fields_to_export(self): ie = self._get_exporter(fields_to_export=["name"]) - self.assertEqual( - list(ie._get_serialized_fields(self.i)), [("name", "John\xa3")] - ) + assert list(ie._get_serialized_fields(self.i)) == [("name", "John\xa3")] ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1") _, name = next(iter(ie._get_serialized_fields(self.i))) assert isinstance(name, str) - self.assertEqual(name, "John\xa3") + assert name == "John\xa3" ie = self._get_exporter(fields_to_export={"name": "å稱"}) - self.assertEqual( - list(ie._get_serialized_fields(self.i)), [("å稱", "John\xa3")] - ) + assert list(ie._get_serialized_fields(self.i)) == [("å稱", "John\xa3")] def test_field_custom_serializer(self): i = self.custom_field_item_class(name="John\xa3", age="22") a = ItemAdapter(i) ie = self._get_exporter() - self.assertEqual( - ie.serialize_field(a.get_field_meta("name"), "name", a["name"]), "John\xa3" - ) - self.assertEqual( - ie.serialize_field(a.get_field_meta("age"), "age", a["age"]), "24" + assert ( + ie.serialize_field(a.get_field_meta("name"), "name", a["name"]) + == "John\xa3" ) + assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == "24" -class BaseItemExporterDataclassTest(BaseItemExporterTest): +class TestBaseItemExporterDataclass(TestBaseItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class PythonItemExporterTest(BaseItemExporterTest): +class TestPythonItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return PythonItemExporter(**kwargs) @@ -157,16 +152,13 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = self.item_class(name="Jesus", age=i2) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual(type(exported), dict) - self.assertEqual( - exported, - { - "age": {"age": {"age": "22", "name": "Joseph"}, "name": "Maria"}, - "name": "Jesus", - }, - ) - self.assertEqual(type(exported["age"]), dict) - self.assertEqual(type(exported["age"]["age"]), dict) + assert isinstance(exported, dict) + assert exported == { + "age": {"age": {"age": "22", "name": "Joseph"}, "name": "Maria"}, + "name": "Jesus", + } + assert isinstance(exported["age"], dict) + assert isinstance(exported["age"]["age"], dict) def test_export_list(self): i1 = self.item_class(name="Joseph", age="22") @@ -174,15 +166,12 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = self.item_class(name="Jesus", age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual( - exported, - { - "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], - "name": "Jesus", - }, - ) - self.assertEqual(type(exported["age"][0]), dict) - self.assertEqual(type(exported["age"][0]["age"][0]), dict) + assert exported == { + "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], + "name": "Jesus", + } + assert isinstance(exported["age"][0], dict) + assert isinstance(exported["age"][0]["age"][0], dict) def test_export_item_dict_list(self): i1 = self.item_class(name="Joseph", age="22") @@ -190,29 +179,26 @@ class PythonItemExporterTest(BaseItemExporterTest): i3 = self.item_class(name="Jesus", age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) - self.assertEqual( - exported, - { - "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], - "name": "Jesus", - }, - ) - self.assertEqual(type(exported["age"][0]), dict) - self.assertEqual(type(exported["age"][0]["age"][0]), dict) + assert exported == { + "age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}], + "name": "Jesus", + } + assert isinstance(exported["age"][0], dict) + assert isinstance(exported["age"][0]["age"][0], dict) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() ie = self._get_exporter() exported = ie.export_item(item) - self.assertEqual(exported, item) + assert exported == item -class PythonItemExporterDataclassTest(PythonItemExporterTest): +class TestPythonItemExporterDataclass(TestPythonItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class PprintItemExporterTest(BaseItemExporterTest): +class TestPprintItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return PprintItemExporter(self.output, **kwargs) @@ -222,12 +208,12 @@ class PprintItemExporterTest(BaseItemExporterTest): ) -class PprintItemExporterDataclassTest(PprintItemExporterTest): +class TestPprintItemExporterDataclass(TestPprintItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class PickleItemExporterTest(BaseItemExporterTest): +class TestPickleItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return PickleItemExporter(self.output, **kwargs) @@ -245,8 +231,8 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.finish_exporting() del ie # See the first “del self.ie†in this file for context. f.seek(0) - self.assertEqual(self.item_class(**pickle.load(f)), i1) - self.assertEqual(self.item_class(**pickle.load(f)), i2) + assert self.item_class(**pickle.load(f)) == i1 + assert self.item_class(**pickle.load(f)) == i2 def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -256,15 +242,15 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.export_item(item) ie.finish_exporting() del ie # See the first “del self.ie†in this file for context. - self.assertEqual(pickle.loads(fp.getvalue()), item) + assert pickle.loads(fp.getvalue()) == item -class PickleItemExporterDataclassTest(PickleItemExporterTest): +class TestPickleItemExporterDataclass(TestPickleItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class MarshalItemExporterTest(BaseItemExporterTest): +class TestMarshalItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): self.output = tempfile.TemporaryFile() return MarshalItemExporter(self.output, **kwargs) @@ -283,15 +269,15 @@ class MarshalItemExporterTest(BaseItemExporterTest): ie.finish_exporting() del ie # See the first “del self.ie†in this file for context. fp.seek(0) - self.assertEqual(marshal.load(fp), item) + assert marshal.load(fp) == item -class MarshalItemExporterDataclassTest(MarshalItemExporterTest): +class TestMarshalItemExporterDataclass(TestMarshalItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class CsvItemExporterTest(BaseItemExporterTest): +class TestCsvItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): self.output = tempfile.TemporaryFile() return CsvItemExporter(self.output, **kwargs) @@ -303,7 +289,7 @@ class CsvItemExporterTest(BaseItemExporterTest): for line in to_unicode(csv).splitlines(True) ] - return self.assertEqual(split_csv(first), split_csv(second), msg=msg) + assert split_csv(first) == split_csv(second), msg def _check_output(self): self.output.seek(0) @@ -406,12 +392,12 @@ class CsvItemExporterTest(BaseItemExporterTest): ) -class CsvItemExporterDataclassTest(CsvItemExporterTest): +class TestCsvItemExporterDataclass(TestCsvItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class XmlItemExporterTest(BaseItemExporterTest): +class TestXmlItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return XmlItemExporter(self.output, **kwargs) @@ -426,7 +412,7 @@ class XmlItemExporterTest(BaseItemExporterTest): doc = lxml.etree.fromstring(xmlcontent) return xmltuple(doc) - return self.assertEqual(xmlsplit(first), xmlsplit(second), msg) + assert xmlsplit(first) == xmlsplit(second), msg def assertExportResult(self, item, expected_value): fp = BytesIO() @@ -517,12 +503,12 @@ class XmlItemExporterTest(BaseItemExporterTest): ) -class XmlItemExporterDataclassTest(XmlItemExporterTest): +class TestXmlItemExporterDataclass(TestXmlItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class JsonLinesItemExporterTest(BaseItemExporterTest): +class TestJsonLinesItemExporter(TestBaseItemExporter): _expected_nested: Any = { "name": "Jesus", "age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}}, @@ -533,7 +519,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, ItemAdapter(self.i).asdict()) + assert exported == ItemAdapter(self.i).asdict() def test_nested_item(self): i1 = self.item_class(name="Joseph", age="22") @@ -544,7 +530,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.finish_exporting() del self.ie # See the first “del self.ie†in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual(exported, self._expected_nested) + assert exported == self._expected_nested def test_extra_keywords(self): self.ie = self._get_exporter(sort_keys=True) @@ -561,23 +547,23 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): del self.ie # See the first “del self.ie†in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item["time"] = str(item["time"]) - self.assertEqual(exported, item) + assert exported == item -class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest): +class TestJsonLinesItemExporterDataclass(TestJsonLinesItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class JsonItemExporterTest(JsonLinesItemExporterTest): - _expected_nested = [JsonLinesItemExporterTest._expected_nested] +class TestJsonItemExporter(TestJsonLinesItemExporter): + _expected_nested = [TestJsonLinesItemExporter._expected_nested] def _get_exporter(self, **kwargs): return JsonItemExporter(self.output, **kwargs) def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, [ItemAdapter(self.i).asdict()]) + assert exported == [ItemAdapter(self.i).asdict()] def assertTwoItemsExported(self, item): self.ie.start_exporting() @@ -586,9 +572,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.finish_exporting() del self.ie # See the first “del self.ie†in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual( - exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()] - ) + assert exported == [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()] def test_two_items(self): self.assertTwoItemsExported(self.i) @@ -609,7 +593,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual(exported, [dict(i1), dict(i3)]) + assert exported == [dict(i1), dict(i3)] def test_nested_item(self): i1 = self.item_class(name="Joseph\xa3", age="22") @@ -624,7 +608,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): "name": "Jesus", "age": {"name": "Maria", "age": ItemAdapter(i1).asdict()}, } - self.assertEqual(exported, [expected]) + assert exported == [expected] def test_nested_dict_item(self): i1 = {"name": "Joseph\xa3", "age": "22"} @@ -636,7 +620,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): del self.ie # See the first “del self.ie†in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) expected = {"name": "Jesus", "age": {"name": "Maria", "age": i1}} - self.assertEqual(exported, [expected]) + assert exported == [expected] def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -646,10 +630,10 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): del self.ie # See the first “del self.ie†in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item["time"] = str(item["time"]) - self.assertEqual(exported, [item]) + assert exported == [item] -class JsonItemExporterToBytesTest(BaseItemExporterTest): +class TestJsonItemExporterToBytes(TestBaseItemExporter): def _get_exporter(self, **kwargs): kwargs["encoding"] = "latin" return JsonItemExporter(self.output, **kwargs) @@ -665,18 +649,18 @@ class JsonItemExporterToBytesTest(BaseItemExporterTest): self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin")) - self.assertEqual(exported, [dict(i1), dict(i3)]) + assert exported == [dict(i1), dict(i3)] -class JsonItemExporterDataclassTest(JsonItemExporterTest): +class TestJsonItemExporterDataclass(TestJsonItemExporter): item_class = MyDataClass custom_field_item_class = CustomFieldDataclass -class CustomExporterItemTest(unittest.TestCase): +class TestCustomExporterItem: item_class: type = MyItem - def setUp(self): + def setup_method(self): if self.item_class is None: raise unittest.SkipTest("item class is None") @@ -691,17 +675,13 @@ class CustomExporterItemTest(unittest.TestCase): a = ItemAdapter(i) ie = CustomItemExporter() - self.assertEqual( - ie.serialize_field(a.get_field_meta("name"), "name", a["name"]), "John" - ) - self.assertEqual( - ie.serialize_field(a.get_field_meta("age"), "age", a["age"]), "23" - ) + assert ie.serialize_field(a.get_field_meta("name"), "name", a["name"]) == "John" + assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == "23" i2 = {"name": "John", "age": "22"} - self.assertEqual(ie.serialize_field({}, "name", i2["name"]), "John") - self.assertEqual(ie.serialize_field({}, "age", i2["age"]), "23") + assert ie.serialize_field({}, "name", i2["name"]) == "John" + assert ie.serialize_field({}, "age", i2["age"]) == "23" -class CustomExporterDataclassTest(CustomExporterItemTest): +class TestCustomExporterDataclass(TestCustomExporterItem): item_class = MyDataClass diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8e008ab98..44cd10ec3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -88,7 +88,7 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: return (client_mock, bucket_mock, blob_mock) -class FileFeedStorageTest(unittest.TestCase): +class TestFileFeedStorage(unittest.TestCase): def test_store_file_uri(self): path = Path(self.mktemp()).resolve() uri = path_to_file_uri(str(path)) @@ -137,14 +137,14 @@ class FileFeedStorageTest(unittest.TestCase): file = storage.open(spider) file.write(b"content") yield storage.store(file) - self.assertTrue(path.exists()) + assert path.exists() try: - self.assertEqual(path.read_bytes(), expected_content) + assert path.read_bytes() == expected_content finally: path.unlink() -class FTPFeedStorageTest(unittest.TestCase): +class TestFTPFeedStorage(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): name = "test_spider" @@ -166,9 +166,9 @@ class FTPFeedStorageTest(unittest.TestCase): return storage.store(file) def _assert_stored(self, path: Path, content): - self.assertTrue(path.exists()) + assert path.exists() try: - self.assertEqual(path.read_bytes(), content) + assert path.read_bytes() == content finally: path.unlink() @@ -216,10 +216,10 @@ class FTPFeedStorageTest(unittest.TestCase): # RFC3986: 3.2.1. User Information pw_quoted = quote(string.punctuation, safe="") st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) - self.assertEqual(st.password, string.punctuation) + assert st.password == string.punctuation -class BlockingFeedStorageTest(unittest.TestCase): +class TestBlockingFeedStorage: def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): name = "test_spider" @@ -232,7 +232,7 @@ class BlockingFeedStorageTest(unittest.TestCase): tmp = b.open(self.get_test_spider()) tmp_path = Path(tmp.name).parent - self.assertEqual(str(tmp_path), tempfile.gettempdir()) + assert str(tmp_path) == tempfile.gettempdir() def test_temp_file(self): b = BlockingFeedStorage() @@ -241,7 +241,7 @@ class BlockingFeedStorageTest(unittest.TestCase): spider = self.get_test_spider({"FEED_TEMPDIR": str(tests_path)}) tmp = b.open(spider) tmp_path = Path(tmp.name).parent - self.assertEqual(tmp_path, tests_path) + assert tmp_path == tests_path def test_invalid_folder(self): b = BlockingFeedStorage() @@ -255,7 +255,7 @@ class BlockingFeedStorageTest(unittest.TestCase): @pytest.mark.requires_boto3 -class S3FeedStorageTest(unittest.TestCase): +class TestS3FeedStorage(unittest.TestCase): def test_parse_credentials(self): aws_credentials = { "AWS_ACCESS_KEY_ID": "settings_key", @@ -268,9 +268,9 @@ class S3FeedStorageTest(unittest.TestCase): crawler, "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, "settings_key") - self.assertEqual(storage.secret_key, "settings_secret") - self.assertEqual(storage.session_token, "settings_token") + assert storage.access_key == "settings_key" + assert storage.secret_key == "settings_secret" + assert storage.session_token == "settings_token" # Instantiate directly storage = S3FeedStorage( "s3://mybucket/export.csv", @@ -278,17 +278,17 @@ class S3FeedStorageTest(unittest.TestCase): aws_credentials["AWS_SECRET_ACCESS_KEY"], session_token=aws_credentials["AWS_SESSION_TOKEN"], ) - self.assertEqual(storage.access_key, "settings_key") - self.assertEqual(storage.secret_key, "settings_secret") - self.assertEqual(storage.session_token, "settings_token") + assert storage.access_key == "settings_key" + assert storage.secret_key == "settings_secret" + assert storage.session_token == "settings_token" # URI priority > settings priority storage = S3FeedStorage( "s3://uri_key:uri_secret@mybucket/export.csv", aws_credentials["AWS_ACCESS_KEY_ID"], aws_credentials["AWS_SECRET_ACCESS_KEY"], ) - self.assertEqual(storage.access_key, "uri_key") - self.assertEqual(storage.secret_key, "uri_secret") + assert storage.access_key == "uri_key" + assert storage.secret_key == "uri_secret" @defer.inlineCallbacks def test_store(self): @@ -306,24 +306,23 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(file) - self.assertEqual( - storage.s3_client.upload_fileobj.call_args, - mock.call(Bucket=bucket, Key=key, Fileobj=file), + assert storage.s3_client.upload_fileobj.call_args == mock.call( + Bucket=bucket, Key=key, Fileobj=file ) def test_init_without_acl(self): storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, None) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None def test_init_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, "custom-acl") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" def test_init_with_endpoint_url(self): storage = S3FeedStorage( @@ -332,9 +331,9 @@ class S3FeedStorageTest(unittest.TestCase): "secret_key", endpoint_url="https://example.com", ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.endpoint_url, "https://example.com") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url == "https://example.com" def test_init_with_region_name(self): region_name = "ap-east-1" @@ -344,10 +343,10 @@ class S3FeedStorageTest(unittest.TestCase): "secret_key", region_name=region_name, ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.region_name, region_name) - self.assertEqual(storage.s3_client._client_config.region_name, region_name) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.region_name == region_name + assert storage.s3_client._client_config.region_name == region_name def test_from_crawler_without_acl(self): settings = { @@ -359,9 +358,9 @@ class S3FeedStorageTest(unittest.TestCase): crawler, "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, None) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None def test_without_endpoint_url(self): settings = { @@ -373,9 +372,9 @@ class S3FeedStorageTest(unittest.TestCase): crawler, "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.endpoint_url, None) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url is None def test_without_region_name(self): settings = { @@ -387,9 +386,9 @@ class S3FeedStorageTest(unittest.TestCase): crawler, "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.s3_client._client_config.region_name, "us-east-1") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.s3_client._client_config.region_name == "us-east-1" def test_from_crawler_with_acl(self): settings = { @@ -402,9 +401,9 @@ class S3FeedStorageTest(unittest.TestCase): crawler, "s3://mybucket/export.csv", ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, "custom-acl") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" def test_from_crawler_with_endpoint_url(self): settings = { @@ -414,9 +413,9 @@ class S3FeedStorageTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.endpoint_url, "https://example.com") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url == "https://example.com" def test_from_crawler_with_region_name(self): region_name = "ap-east-1" @@ -427,10 +426,10 @@ class S3FeedStorageTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.region_name, region_name) - self.assertEqual(storage.s3_client._client_config.region_name, region_name) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.region_name == region_name + assert storage.s3_client._client_config.region_name == region_name @defer.inlineCallbacks def test_store_without_acl(self): @@ -439,9 +438,9 @@ class S3FeedStorageTest(unittest.TestCase): "access_key", "secret_key", ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, None) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) @@ -450,28 +449,28 @@ class S3FeedStorageTest(unittest.TestCase): .get("ExtraArgs", {}) .get("ACL") ) - self.assertIsNone(acl) + assert acl is None @defer.inlineCallbacks def test_store_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) - self.assertEqual(storage.access_key, "access_key") - self.assertEqual(storage.secret_key, "secret_key") - self.assertEqual(storage.acl, "custom-acl") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] - self.assertEqual(acl, "custom-acl") + assert acl == "custom-acl" def test_overwrite_default(self): with LogCapture() as log: S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) - self.assertNotIn("S3 does not support appending to files", str(log)) + assert "S3 does not support appending to files" not in str(log) def test_overwrite_false(self): with LogCapture() as log: @@ -482,10 +481,10 @@ class S3FeedStorageTest(unittest.TestCase): "custom-acl", feed_options={"overwrite": False}, ) - self.assertIn("S3 does not support appending to files", str(log)) + assert "S3 does not support appending to files" in str(log) -class GCSFeedStorageTest(unittest.TestCase): +class TestGCSFeedStorage(unittest.TestCase): def test_parse_settings(self): try: from google.cloud.storage import Client # noqa: F401 @@ -543,7 +542,7 @@ class GCSFeedStorageTest(unittest.TestCase): def test_overwrite_default(self): with LogCapture() as log: GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl") - self.assertNotIn("GCS does not support appending to files", str(log)) + assert "GCS does not support appending to files" not in str(log) def test_overwrite_false(self): with LogCapture() as log: @@ -553,10 +552,10 @@ class GCSFeedStorageTest(unittest.TestCase): "custom-acl", feed_options={"overwrite": False}, ) - self.assertIn("GCS does not support appending to files", str(log)) + assert "GCS does not support appending to files" in str(log) -class StdoutFeedStorageTest(unittest.TestCase): +class TestStdoutFeedStorage(unittest.TestCase): @defer.inlineCallbacks def test_store(self): out = BytesIO() @@ -564,20 +563,21 @@ class StdoutFeedStorageTest(unittest.TestCase): file = storage.open(scrapy.Spider("default")) file.write(b"content") yield storage.store(file) - self.assertEqual(out.getvalue(), b"content") + assert out.getvalue() == b"content" def test_overwrite_default(self): with LogCapture() as log: StdoutFeedStorage("stdout:") - self.assertNotIn( - "Standard output (stdout) storage does not support overwriting", str(log) + assert ( + "Standard output (stdout) storage does not support overwriting" + not in str(log) ) def test_overwrite_true(self): with LogCapture() as log: StdoutFeedStorage("stdout:", feed_options={"overwrite": True}) - self.assertIn( - "Standard output (stdout) storage does not support overwriting", str(log) + assert "Standard output (stdout) storage does not support overwriting" in str( + log ) @@ -639,7 +639,7 @@ class LogOnStoreFileStorage: file.close() -class FeedExportTestBase(ABC, unittest.TestCase): +class TestFeedExportBase(ABC, unittest.TestCase): class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() @@ -769,7 +769,7 @@ class ExceptionJsonItemExporter(JsonItemExporter): raise RuntimeError("foo") -class FeedExportTest(FeedExportTestBase): +class TestFeedExport(TestFeedExportBase): @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data.""" @@ -812,8 +812,8 @@ class FeedExportTest(FeedExportTestBase): ) data = yield self.exported_data(items, settings) reader = csv.DictReader(to_unicode(data["csv"]).splitlines()) - self.assertEqual(reader.fieldnames, list(header)) - self.assertEqual(rows, list(reader)) + assert reader.fieldnames == list(header) + assert rows == list(reader) @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): @@ -828,7 +828,7 @@ class FeedExportTest(FeedExportTestBase): data = yield self.exported_data(items, settings) parsed = [json.loads(to_unicode(line)) for line in data["jl"].splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] - self.assertEqual(rows, parsed) + assert rows == parsed @defer.inlineCallbacks def assertExportedXml(self, items, rows, settings=None): @@ -844,7 +844,7 @@ class FeedExportTest(FeedExportTestBase): rows = [{k: v for k, v in row.items() if v} for row in rows] root = lxml.etree.fromstring(data["xml"]) got_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] - self.assertEqual(rows, got_rows) + assert rows == got_rows @defer.inlineCallbacks def assertExportedMultiple(self, items, rows, settings=None): @@ -862,10 +862,10 @@ class FeedExportTest(FeedExportTestBase): # XML root = lxml.etree.fromstring(data["xml"]) xml_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] - self.assertEqual(rows, xml_rows) + assert rows == xml_rows # JSON json_rows = json.loads(to_unicode(data["json"])) - self.assertEqual(rows, json_rows) + assert rows == json_rows @defer.inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): @@ -882,7 +882,7 @@ class FeedExportTest(FeedExportTestBase): import pickle result = self._load_until_eof(data["pickle"], load_func=pickle.load) - self.assertEqual(expected, result) + assert result == expected @defer.inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): @@ -899,7 +899,7 @@ class FeedExportTest(FeedExportTestBase): import marshal result = self._load_until_eof(data["marshal"], load_func=marshal.load) - self.assertEqual(expected, result) + assert result == expected @defer.inlineCallbacks def test_stats_file_success(self): @@ -912,12 +912,8 @@ class FeedExportTest(FeedExportTestBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(mockserver=self.mockserver) - self.assertIn( - "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() - ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1 - ) + assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 @defer.inlineCallbacks def test_stats_file_failed(self): @@ -934,12 +930,8 @@ class FeedExportTest(FeedExportTestBase): side_effect=KeyError("foo"), ): yield crawler.crawl(mockserver=self.mockserver) - self.assertIn( - "feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats() - ) - self.assertEqual( - crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1 - ) + assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1 @defer.inlineCallbacks def test_stats_multiple_file(self): @@ -956,17 +948,11 @@ class FeedExportTest(FeedExportTestBase): crawler = get_crawler(ItemSpider, settings) with mock.patch.object(S3FeedStorage, "store"): yield crawler.crawl(mockserver=self.mockserver) - self.assertIn( - "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() - ) - self.assertIn( - "feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats() - ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1 - ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1 + assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() + assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 + assert ( + crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1 ) @defer.inlineCallbacks @@ -993,7 +979,7 @@ class FeedExportTest(FeedExportTestBase): "FEED_STORE_EMPTY": False, } data = yield self.exported_no_data(settings) - self.assertEqual(None, data[fmt]) + assert data[fmt] is None @defer.inlineCallbacks def test_start_finish_exporting_items(self): @@ -1012,8 +998,8 @@ class FeedExportTest(FeedExportTestBase): with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) - self.assertFalse(listener.start_without_finish) - self.assertFalse(listener.finish_without_start) + assert not listener.start_without_finish + assert not listener.finish_without_start @defer.inlineCallbacks def test_start_finish_exporting_no_items(self): @@ -1030,8 +1016,8 @@ class FeedExportTest(FeedExportTestBase): with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) - self.assertFalse(listener.start_without_finish) - self.assertFalse(listener.finish_without_start) + assert not listener.start_without_finish + assert not listener.finish_without_start @defer.inlineCallbacks def test_start_finish_exporting_items_exception(self): @@ -1051,8 +1037,8 @@ class FeedExportTest(FeedExportTestBase): with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) - self.assertFalse(listener.start_without_finish) - self.assertFalse(listener.finish_without_start) + assert not listener.start_without_finish + assert not listener.finish_without_start @defer.inlineCallbacks def test_start_finish_exporting_no_items_exception(self): @@ -1070,8 +1056,8 @@ class FeedExportTest(FeedExportTestBase): with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) - self.assertFalse(listener.start_without_finish) - self.assertFalse(listener.finish_without_start) + assert not listener.start_without_finish + assert not listener.finish_without_start @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -1091,7 +1077,7 @@ class FeedExportTest(FeedExportTestBase): "FEED_EXPORT_INDENT": None, } data = yield self.exported_no_data(settings) - self.assertEqual(expctd, data[fmt]) + assert expctd == data[fmt] @defer.inlineCallbacks def test_export_no_items_multiple_feeds(self): @@ -1109,7 +1095,7 @@ class FeedExportTest(FeedExportTestBase): with LogCapture() as log: yield self.exported_no_data(settings) - self.assertEqual(str(log).count("Storage.store is called"), 0) + assert str(log).count("Storage.store is called") == 0 @defer.inlineCallbacks def test_export_multiple_item_classes(self): @@ -1238,7 +1224,7 @@ class FeedExportTest(FeedExportTestBase): data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected @defer.inlineCallbacks def test_export_based_on_custom_filters(self): @@ -1297,7 +1283,7 @@ class FeedExportTest(FeedExportTestBase): data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected @defer.inlineCallbacks def test_export_dicts(self): @@ -1371,7 +1357,7 @@ class FeedExportTest(FeedExportTestBase): "FEED_EXPORT_INDENT": None, } data = yield self.exported_data(items, settings) - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected formats = { "json": b'[{"foo": "Test\xd6"}]', @@ -1392,7 +1378,7 @@ class FeedExportTest(FeedExportTestBase): "FEED_EXPORT_ENCODING": "latin-1", } data = yield self.exported_data(items, settings) - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected @defer.inlineCallbacks def test_export_multiple_configs(self): @@ -1432,7 +1418,7 @@ class FeedExportTest(FeedExportTestBase): data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): - self.assertEqual(expected, data[fmt]) + assert data[fmt] == expected @defer.inlineCallbacks def test_export_indentation(self): @@ -1588,7 +1574,7 @@ class FeedExportTest(FeedExportTestBase): }, } data = yield self.exported_data(items, settings) - self.assertEqual(row["expected"], data[row["format"]]) + assert data[row["format"]] == row["expected"] @defer.inlineCallbacks def test_init_exporters_storages_with_crawler(self): @@ -1600,8 +1586,8 @@ class FeedExportTest(FeedExportTestBase): }, } yield self.exported_data(items=[], settings=settings) - self.assertTrue(FromCrawlerCsvItemExporter.init_with_crawler) - self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler) + assert FromCrawlerCsvItemExporter.init_with_crawler + assert FromCrawlerFileFeedStorage.init_with_crawler @defer.inlineCallbacks def test_str_uri(self): @@ -1610,7 +1596,7 @@ class FeedExportTest(FeedExportTestBase): "FEEDS": {str(self._random_temp_filename()): {"format": "csv"}}, } data = yield self.exported_no_data(settings) - self.assertEqual(data["csv"], b"") + assert data["csv"] == b"" @defer.inlineCallbacks def test_multiple_feeds_success_logs_blocking_feed_storage(self): @@ -1631,7 +1617,7 @@ class FeedExportTest(FeedExportTestBase): print(log) for fmt in ["json", "xml", "csv"]: - self.assertIn(f"Stored {fmt} feed (2 items)", str(log)) + assert f"Stored {fmt} feed (2 items)" in str(log) @defer.inlineCallbacks def test_multiple_feeds_failing_logs_blocking_feed_storage(self): @@ -1652,7 +1638,7 @@ class FeedExportTest(FeedExportTestBase): print(log) for fmt in ["json", "xml", "csv"]: - self.assertIn(f"Error storing {fmt} feed (2 items)", str(log)) + assert f"Error storing {fmt} feed (2 items)" in str(log) @defer.inlineCallbacks def test_extend_kwargs(self): @@ -1689,7 +1675,7 @@ class FeedExportTest(FeedExportTestBase): } data = yield self.exported_data(items, settings) - self.assertEqual(row["expected"], data[feed_options["format"]]) + assert data[feed_options["format"]] == row["expected"] @defer.inlineCallbacks def test_storage_file_no_postprocessing(self): @@ -1711,7 +1697,7 @@ class FeedExportTest(FeedExportTestBase): "FEED_STORAGES": {"file": Storage}, } yield self.exported_no_data(settings) - self.assertIs(Storage.open_file, Storage.store_file) + assert Storage.open_file is Storage.store_file @defer.inlineCallbacks def test_storage_file_postprocessing(self): @@ -1741,11 +1727,11 @@ class FeedExportTest(FeedExportTestBase): "FEED_STORAGES": {"file": Storage}, } yield self.exported_no_data(settings) - self.assertIs(Storage.open_file, Storage.store_file) - self.assertFalse(Storage.file_was_closed) + assert Storage.open_file is Storage.store_file + assert not Storage.file_was_closed -class FeedPostProcessedExportsTest(FeedExportTestBase): +class TestFeedPostProcessedExports(TestFeedExportBase): items = [{"foo": "bar"}] expected = b"foo\r\nbar\r\n" @@ -1827,7 +1813,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): try: gzip.decompress(data[filename]) except OSError: - self.fail("Received invalid gzip data.") + pytest.fail("Received invalid gzip data.") @defer.inlineCallbacks def test_gzip_plugin_compresslevel(self): @@ -1863,8 +1849,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = gzip.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_gzip_plugin_mtime(self): @@ -1898,8 +1884,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = gzip.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_gzip_plugin_filename(self): @@ -1933,8 +1919,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = gzip.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_lzma_plugin(self): @@ -1953,7 +1939,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): try: lzma.decompress(data[filename]) except lzma.LZMAError: - self.fail("Received invalid lzma data.") + pytest.fail("Received invalid lzma data.") @defer.inlineCallbacks def test_lzma_plugin_format(self): @@ -1985,8 +1971,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = lzma.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_lzma_plugin_check(self): @@ -2018,8 +2004,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = lzma.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_lzma_plugin_preset(self): @@ -2051,8 +2037,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = lzma.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_lzma_plugin_filters(self): @@ -2075,9 +2061,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): } data = yield self.exported_data(self.items, settings) - self.assertEqual(compressed, data[filename]) + assert compressed == data[filename] result = lzma.decompress(data[filename]) - self.assertEqual(self.expected, result) + assert result == self.expected @defer.inlineCallbacks def test_bz2_plugin(self): @@ -2096,7 +2082,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): try: bz2.decompress(data[filename]) except OSError: - self.fail("Received invalid bz2 data.") + pytest.fail("Received invalid bz2 data.") @defer.inlineCallbacks def test_bz2_plugin_compresslevel(self): @@ -2128,8 +2114,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, compressed in filename_to_compressed.items(): result = bz2.decompress(data[filename]) - self.assertEqual(compressed, data[filename]) - self.assertEqual(self.expected, result) + assert compressed == data[filename] + assert result == self.expected @defer.inlineCallbacks def test_custom_plugin(self): @@ -2145,7 +2131,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): } data = yield self.exported_data(self.items, settings) - self.assertEqual(self.expected, data[filename]) + assert data[filename] == self.expected @defer.inlineCallbacks def test_custom_plugin_with_parameter(self): @@ -2163,7 +2149,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): } data = yield self.exported_data(self.items, settings) - self.assertEqual(expected, data[filename]) + assert data[filename] == expected @defer.inlineCallbacks def test_custom_plugin_with_compression(self): @@ -2208,7 +2194,7 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): for filename, decompressor in filename_to_decompressor.items(): result = decompressor(data[filename]) - self.assertEqual(expected, result) + assert result == expected @defer.inlineCallbacks def test_exports_compatibility_with_postproc(self): @@ -2262,10 +2248,10 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): expected, result = self.items[0], marshal.loads(result) else: expected = filename_to_expected[filename] - self.assertEqual(expected, result) + assert result == expected -class BatchDeliveriesTest(FeedExportTestBase): +class TestBatchDeliveries(TestFeedExportBase): _file_mark = "_%(batch_time)s_#%(batch_id)02d_" @defer.inlineCallbacks @@ -2310,7 +2296,7 @@ class BatchDeliveriesTest(FeedExportTestBase): json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines() ] expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None): @@ -2328,9 +2314,9 @@ class BatchDeliveriesTest(FeedExportTestBase): data = yield self.exported_data(items, settings) for batch in data["csv"]: got_batch = csv.DictReader(to_unicode(batch).splitlines()) - self.assertEqual(list(header), got_batch.fieldnames) + assert list(header) == got_batch.fieldnames expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, list(got_batch)) + assert list(got_batch) == expected_batch @defer.inlineCallbacks def assertExportedXml(self, items, rows, settings=None): @@ -2351,7 +2337,7 @@ class BatchDeliveriesTest(FeedExportTestBase): root = lxml.etree.fromstring(batch) got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def assertExportedMultiple(self, items, rows, settings=None): @@ -2377,13 +2363,13 @@ class BatchDeliveriesTest(FeedExportTestBase): root = lxml.etree.fromstring(batch) got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch # JSON json_rows = rows.copy() for batch in data["json"]: got_batch = json.loads(batch.decode("utf-8")) expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): @@ -2405,7 +2391,7 @@ class BatchDeliveriesTest(FeedExportTestBase): for batch in data["pickle"]: got_batch = self._load_until_eof(batch, load_func=pickle.load) expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): @@ -2427,7 +2413,7 @@ class BatchDeliveriesTest(FeedExportTestBase): for batch in data["marshal"]: got_batch = self._load_until_eof(batch, load_func=marshal.load) expected_batch, rows = rows[:batch_size], rows[batch_size:] - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def test_export_items(self): @@ -2472,7 +2458,7 @@ class BatchDeliveriesTest(FeedExportTestBase): } data = yield self.exported_no_data(settings) data = dict(data) - self.assertEqual(0, len(data[fmt])) + assert len(data[fmt]) == 0 @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -2496,7 +2482,7 @@ class BatchDeliveriesTest(FeedExportTestBase): } data = yield self.exported_no_data(settings) data = dict(data) - self.assertEqual(expctd, data[fmt][0]) + assert data[fmt][0] == expctd @defer.inlineCallbacks def test_export_multiple_configs(self): @@ -2552,7 +2538,7 @@ class BatchDeliveriesTest(FeedExportTestBase): data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): for expected_batch, got_batch in zip(expected, data[fmt]): - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): @@ -2576,7 +2562,7 @@ class BatchDeliveriesTest(FeedExportTestBase): data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): for expected_batch, got_batch in zip(expected, data[fmt]): - self.assertEqual(expected_batch, got_batch) + assert got_batch == expected_batch @defer.inlineCallbacks def test_batch_path_differ(self): @@ -2598,7 +2584,7 @@ class BatchDeliveriesTest(FeedExportTestBase): "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } data = yield self.exported_data(items, settings) - self.assertEqual(len(items), len(data["json"])) + assert len(items) == len(data["json"]) @defer.inlineCallbacks def test_stats_batch_file_success(self): @@ -2614,12 +2600,8 @@ class BatchDeliveriesTest(FeedExportTestBase): } crawler = get_crawler(ItemSpider, settings) yield crawler.crawl(total=2, mockserver=self.mockserver) - self.assertIn( - "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() - ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12 - ) + assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 @pytest.mark.requires_boto3 @defer.inlineCallbacks @@ -2687,13 +2669,13 @@ class BatchDeliveriesTest(FeedExportTestBase): crawler = get_crawler(TestSpider, settings) yield crawler.crawl() - self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)) + assert len(CustomS3FeedStorage.stubs) == len(items) for stub in CustomS3FeedStorage.stubs[:-1]: stub.assert_no_pending_responses() # Test that the FeedExporer sends the feed_exporter_closed and feed_slot_closed signals -class FeedExporterSignalsTest(unittest.TestCase): +class TestFeedExporterSignals: items = [ {"foo": "bar1", "egg": "spam1"}, {"foo": "bar2", "egg": "spam2", "baz": "quux2"}, @@ -2754,8 +2736,8 @@ class FeedExporterSignalsTest(unittest.TestCase): self.feed_exporter_closed_signal_handler, self.feed_slot_closed_signal_handler, ) - self.assertTrue(self.feed_slot_closed_received) - self.assertTrue(self.feed_exporter_closed_received) + assert self.feed_slot_closed_received + assert self.feed_exporter_closed_received def test_feed_exporter_signals_sent_deferred(self): self.feed_exporter_closed_received = False @@ -2765,11 +2747,11 @@ class FeedExporterSignalsTest(unittest.TestCase): self.feed_exporter_closed_signal_handler_deferred, self.feed_slot_closed_signal_handler_deferred, ) - self.assertTrue(self.feed_slot_closed_received) - self.assertTrue(self.feed_exporter_closed_received) + assert self.feed_slot_closed_received + assert self.feed_exporter_closed_received -class FeedExportInitTest(unittest.TestCase): +class TestFeedExportInit: def test_unsupported_storage(self): settings = { "FEEDS": { @@ -2803,7 +2785,7 @@ class FeedExportInitTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) exporter = FeedExporter.from_crawler(crawler) - self.assertIsInstance(exporter, FeedExporter) + assert isinstance(exporter, FeedExporter) def test_relative_pathlib_as_uri(self): settings = { @@ -2815,13 +2797,14 @@ class FeedExportInitTest(unittest.TestCase): } crawler = get_crawler(settings_dict=settings) exporter = FeedExporter.from_crawler(crawler) - self.assertIsInstance(exporter, FeedExporter) + assert isinstance(exporter, FeedExporter) -class URIParamsTest: +class TestURIParams(ABC): spider_name = "uri_params_spider" deprecated_options = False + @abstractmethod def build_settings(self, uri="file:///tmp/foobar", uri_params=None): raise NotImplementedError @@ -2850,7 +2833,7 @@ class URIParamsTest: warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" def test_none(self): def uri_params(params, spider): @@ -2866,7 +2849,7 @@ class URIParamsTest: feed_exporter.open_spider(spider) - self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" def test_empty_dict(self): def uri_params(params, spider): @@ -2900,7 +2883,7 @@ class URIParamsTest: warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" def test_custom_param(self): def uri_params(params, spider): @@ -2917,10 +2900,10 @@ class URIParamsTest: warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" -class URIParamsSettingTest(URIParamsTest, unittest.TestCase): +class TestURIParamsSetting(TestURIParams): deprecated_options = True def build_settings(self, uri="file:///tmp/foobar", uri_params=None): @@ -2933,7 +2916,7 @@ class URIParamsSettingTest(URIParamsTest, unittest.TestCase): } -class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase): +class TestURIParamsFeedOption(TestURIParams): deprecated_options = False def build_settings(self, uri="file:///tmp/foobar", uri_params=None): diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0881bbeca..7c1b38877 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -185,7 +185,7 @@ def get_client_certificate( @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") -class Https2ClientProtocolTestCase(TestCase): +class TestHttps2ClientProtocol(TestCase): scheme = "https" key_file = Path(__file__).parent / "keys" / "localhost.key" certificate_file = Path(__file__).parent / "keys" / "localhost.crt" @@ -277,14 +277,14 @@ class Https2ClientProtocolTestCase(TestCase): def _check_GET(self, request: Request, expected_body, expected_status): def check_response(response: Response): - self.assertEqual(response.status, expected_status) - self.assertEqual(response.body, expected_body) - self.assertEqual(response.request, request) + assert response.status == expected_status + assert response.body == expected_body + assert response.request == request content_length_header = response.headers.get("Content-Length") assert content_length_header is not None content_length = int(content_length_header) - self.assertEqual(len(response.body), content_length) + assert len(response.body) == content_length d = self.make_request(request) d.addCallback(check_response) @@ -325,35 +325,35 @@ class Https2ClientProtocolTestCase(TestCase): d = self.make_request(request) def assert_response(response: Response): - self.assertEqual(response.status, expected_status) - self.assertEqual(response.request, request) + assert response.status == expected_status + assert response.request == request content_length_header = response.headers.get("Content-Length") assert content_length_header is not None content_length = int(content_length_header) - self.assertEqual(len(response.body), content_length) + assert len(response.body) == content_length # Parse the body content_encoding_header = response.headers[b"Content-Encoding"] assert content_encoding_header is not None content_encoding = str(content_encoding_header, "utf-8") body = json.loads(str(response.body, content_encoding)) - self.assertIn("request-body", body) - self.assertIn("extra-data", body) - self.assertIn("request-headers", body) + assert "request-body" in body + assert "extra-data" in body + assert "request-headers" in body request_body = body["request-body"] - self.assertEqual(request_body, expected_request_body) + assert request_body == expected_request_body extra_data = body["extra-data"] - self.assertEqual(extra_data, expected_extra_data) + assert extra_data == expected_extra_data # Check if headers were sent successfully request_headers = body["request-headers"] for k, v in request.headers.items(): k_str = str(k, "utf-8") - self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], str(v[0], "utf-8")) + assert k_str in request_headers + assert request_headers[k_str] == str(v[0], "utf-8") d.addCallback(assert_response) d.addErrback(self.fail) @@ -414,8 +414,8 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(url=self.get_url("/get-data-html-large")) def assert_response(response: Response): - self.assertEqual(response.status, 499) - self.assertEqual(response.request, request) + assert response.status == 499 + assert response.request == request d = self.make_request(request) d.addCallback(assert_response) @@ -430,12 +430,12 @@ class Https2ClientProtocolTestCase(TestCase): ) def assert_cancelled_error(failure): - self.assertIsInstance(failure.value, CancelledError) + assert isinstance(failure.value, CancelledError) error_pattern = re.compile( rf"Cancelling download of {request.url}: received response " rf"size \(\d*\) larger than download max size \(1000\)" ) - self.assertEqual(len(re.findall(error_pattern, str(failure.value))), 1) + assert len(re.findall(error_pattern, str(failure.value))) == 1 d = self.make_request(request) d.addCallback(self.fail) @@ -448,14 +448,12 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(url=self.get_url("/dataloss")) def assert_failure(failure: Failure): - self.assertTrue(len(failure.value.reasons) > 0) + assert len(failure.value.reasons) > 0 from h2.exceptions import InvalidBodyLengthError - self.assertTrue( - any( - isinstance(error, InvalidBodyLengthError) - for error in failure.value.reasons - ) + assert any( + isinstance(error, InvalidBodyLengthError) + for error in failure.value.reasons ) d = self.make_request(request) @@ -467,10 +465,10 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(url=self.get_url("/no-content-length-header")) def assert_content_length(response: Response): - self.assertEqual(response.status, 200) - self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) - self.assertEqual(response.request, request) - self.assertNotIn("Content-Length", response.headers) + assert response.status == 200 + assert response.body == Data.NO_CONTENT_LENGTH + assert response.request == request + assert "Content-Length" not in response.headers d = self.make_request(request) d.addCallback(assert_content_length) @@ -481,14 +479,12 @@ class Https2ClientProtocolTestCase(TestCase): def _check_log_warnsize(self, request, warn_pattern, expected_body): with self.assertLogs("scrapy.core.http2.stream", level="WARNING") as cm: response = yield self.make_request(request) - self.assertEqual(response.status, 200) - self.assertEqual(response.request, request) - self.assertEqual(response.body, expected_body) + assert response.status == 200 + assert response.request == request + assert response.body == expected_body # Check the warning is raised only once for this request - self.assertEqual( - sum(len(re.findall(warn_pattern, log)) for log in cm.output), 1 - ) + assert sum(len(re.findall(warn_pattern, log)) for log in cm.output) == 1 @inlineCallbacks def test_log_expected_warnsize(self): @@ -534,11 +530,11 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [] def assert_inactive_stream(failure): - self.assertIsNotNone(failure.check(ResponseFailed)) + assert failure.check(ResponseFailed) is not None from scrapy.core.http2.stream import InactiveStreamClosed - self.assertTrue( - any(isinstance(e, InactiveStreamClosed) for e in failure.value.reasons) + assert any( + isinstance(e, InactiveStreamClosed) for e in failure.value.reasons ) # Send 100 request (we do not check the result) @@ -578,7 +574,7 @@ class Https2ClientProtocolTestCase(TestCase): assert content_encoding_header is not None content_encoding = str(content_encoding_header, "utf-8") data = json.loads(str(response.body, content_encoding)) - self.assertEqual(data, params) + assert data == params d = self.make_request(request) d.addCallback(assert_query_params) @@ -588,7 +584,7 @@ class Https2ClientProtocolTestCase(TestCase): def test_status_codes(self): def assert_response_status(response: Response, expected_status: int): - self.assertEqual(response.status, expected_status) + assert response.status == expected_status d_list = [] for status in [200, 404]: @@ -604,21 +600,18 @@ class Https2ClientProtocolTestCase(TestCase): request = Request(self.get_url("/status?n=200")) def assert_metadata(response: Response): - self.assertEqual(response.request, request) - self.assertIsInstance(response.certificate, Certificate) - assert response.certificate # typing - self.assertIsNotNone(response.certificate.original) - self.assertEqual( - response.certificate.getIssuer(), self.client_certificate.getIssuer() + assert response.request == request + assert isinstance(response.certificate, Certificate) + assert response.certificate.original is not None + assert ( + response.certificate.getIssuer() == self.client_certificate.getIssuer() ) - self.assertTrue( - response.certificate.getPublicKey().matches( - self.client_certificate.getPublicKey() - ) + assert response.certificate.getPublicKey().matches( + self.client_certificate.getPublicKey() ) - self.assertIsInstance(response.ip_address, IPv4Address) - self.assertEqual(str(response.ip_address), "127.0.0.1") + assert isinstance(response.ip_address, IPv4Address) + assert str(response.ip_address) == "127.0.0.1" d = self.make_request(request) d.addCallback(assert_metadata) @@ -632,11 +625,11 @@ class Https2ClientProtocolTestCase(TestCase): def assert_invalid_hostname(failure: Failure): from scrapy.core.http2.stream import InvalidHostname - self.assertIsNotNone(failure.check(InvalidHostname)) + assert failure.check(InvalidHostname) is not None error_msg = str(failure.value) - self.assertIn("localhost", error_msg) - self.assertIn("127.0.0.1", error_msg) - self.assertIn(str(request), error_msg) + assert "localhost" in error_msg + assert "127.0.0.1" in error_msg + assert str(request) in error_msg d = self.make_request(request) d.addCallback(self.fail) @@ -672,13 +665,13 @@ class Https2ClientProtocolTestCase(TestCase): from scrapy.core.http2.protocol import H2ClientProtocol if isinstance(err, TimeoutError): - self.assertIn( - f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", - str(err), + assert ( + f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s" + in str(err) ) break else: - self.fail() + pytest.fail("No TimeoutError raised.") d.addCallback(self.fail) d.addErrback(assert_timeout_error) @@ -692,15 +685,15 @@ class Https2ClientProtocolTestCase(TestCase): d = self.make_request(request) def assert_request_headers(response: Response): - self.assertEqual(response.status, 200) - self.assertEqual(response.request, request) + assert response.status == 200 + assert response.request == request response_headers = json.loads(str(response.body, "utf-8")) - self.assertIsInstance(response_headers, dict) + assert isinstance(response_headers, dict) for k, v in request.headers.items(): k, v = str(k, "utf-8"), str(v[0], "utf-8") - self.assertIn(k, response_headers) - self.assertEqual(v, response_headers[k]) + assert k in response_headers + assert v == response_headers[k] d.addErrback(self.fail) d.addCallback(assert_request_headers) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 932644320..660b76d08 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,74 +1,72 @@ -from unittest import TestCase - from scrapy.http import Request, Response from scrapy.http.cookies import WrappedRequest, WrappedResponse from scrapy.utils.httpobj import urlparse_cached -class WrappedRequestTest(TestCase): - def setUp(self): +class TestWrappedRequest: + def setup_method(self): self.request = Request( "http://www.example.com/page.html", headers={"Content-Type": "text/html"} ) self.wrapped = WrappedRequest(self.request) def test_get_full_url(self): - self.assertEqual(self.wrapped.get_full_url(), self.request.url) - self.assertEqual(self.wrapped.full_url, self.request.url) + assert self.wrapped.get_full_url() == self.request.url + assert self.wrapped.full_url == self.request.url def test_get_host(self): - self.assertEqual(self.wrapped.get_host(), urlparse_cached(self.request).netloc) - self.assertEqual(self.wrapped.host, urlparse_cached(self.request).netloc) + assert self.wrapped.get_host() == urlparse_cached(self.request).netloc + assert self.wrapped.host == urlparse_cached(self.request).netloc def test_get_type(self): - self.assertEqual(self.wrapped.get_type(), urlparse_cached(self.request).scheme) - self.assertEqual(self.wrapped.type, urlparse_cached(self.request).scheme) + assert self.wrapped.get_type() == urlparse_cached(self.request).scheme + assert self.wrapped.type == urlparse_cached(self.request).scheme def test_is_unverifiable(self): - self.assertFalse(self.wrapped.is_unverifiable()) - self.assertFalse(self.wrapped.unverifiable) + assert not self.wrapped.is_unverifiable() + assert not self.wrapped.unverifiable def test_is_unverifiable2(self): self.request.meta["is_unverifiable"] = True - self.assertTrue(self.wrapped.is_unverifiable()) - self.assertTrue(self.wrapped.unverifiable) + assert self.wrapped.is_unverifiable() + assert self.wrapped.unverifiable def test_get_origin_req_host(self): - self.assertEqual(self.wrapped.origin_req_host, "www.example.com") + assert self.wrapped.origin_req_host == "www.example.com" def test_has_header(self): - self.assertTrue(self.wrapped.has_header("content-type")) - self.assertFalse(self.wrapped.has_header("xxxxx")) + assert self.wrapped.has_header("content-type") + assert not self.wrapped.has_header("xxxxx") def test_get_header(self): - self.assertEqual(self.wrapped.get_header("content-type"), "text/html") - self.assertEqual(self.wrapped.get_header("xxxxx", "def"), "def") - self.assertEqual(self.wrapped.get_header("xxxxx"), None) + assert self.wrapped.get_header("content-type") == "text/html" + assert self.wrapped.get_header("xxxxx", "def") == "def" + assert self.wrapped.get_header("xxxxx") is None wrapped = WrappedRequest( Request( "http://www.example.com/page.html", headers={"empty-binary-header": b""} ) ) - self.assertEqual(wrapped.get_header("empty-binary-header"), "") + assert wrapped.get_header("empty-binary-header") == "" def test_header_items(self): - self.assertEqual(self.wrapped.header_items(), [("Content-Type", ["text/html"])]) + assert self.wrapped.header_items() == [("Content-Type", ["text/html"])] def test_add_unredirected_header(self): self.wrapped.add_unredirected_header("hello", "world") - self.assertEqual(self.request.headers["hello"], b"world") + assert self.request.headers["hello"] == b"world" -class WrappedResponseTest(TestCase): - def setUp(self): +class TestWrappedResponse: + def setup_method(self): self.response = Response( "http://www.example.com/page.html", headers={"Content-TYpe": "text/html"} ) self.wrapped = WrappedResponse(self.response) def test_info(self): - self.assertIs(self.wrapped.info(), self.wrapped) + assert self.wrapped.info() is self.wrapped def test_get_all(self): # get_all result must be native string - self.assertEqual(self.wrapped.get_all("content-type"), ["text/html"]) + assert self.wrapped.get_all("content-type") == ["text/html"] diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 0bbbcda46..2fcf9e83c 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -1,14 +1,13 @@ import copy -import unittest import pytest from scrapy.http import Headers -class HeadersTest(unittest.TestCase): +class TestHeaders: def assertSortedEqual(self, first, second, msg=None): - return self.assertEqual(sorted(first), sorted(second), msg) + assert sorted(first) == sorted(second), msg def test_basics(self): h = Headers({"Content-Type": "text/html", "Content-Length": 1234}) @@ -17,53 +16,53 @@ class HeadersTest(unittest.TestCase): with pytest.raises(KeyError): h["Accept"] - self.assertEqual(h.get("Accept"), None) - self.assertEqual(h.getlist("Accept"), []) + assert h.get("Accept") is None + assert h.getlist("Accept") == [] - self.assertEqual(h.get("Accept", "*/*"), b"*/*") - self.assertEqual(h.getlist("Accept", "*/*"), [b"*/*"]) - self.assertEqual( - h.getlist("Accept", ["text/html", "images/jpeg"]), - [b"text/html", b"images/jpeg"], - ) + assert h.get("Accept", "*/*") == b"*/*" + assert h.getlist("Accept", "*/*") == [b"*/*"] + assert h.getlist("Accept", ["text/html", "images/jpeg"]) == [ + b"text/html", + b"images/jpeg", + ] def test_single_value(self): h = Headers() h["Content-Type"] = "text/html" - self.assertEqual(h["Content-Type"], b"text/html") - self.assertEqual(h.get("Content-Type"), b"text/html") - self.assertEqual(h.getlist("Content-Type"), [b"text/html"]) + assert h["Content-Type"] == b"text/html" + assert h.get("Content-Type") == b"text/html" + assert h.getlist("Content-Type") == [b"text/html"] def test_multivalue(self): h = Headers() h["X-Forwarded-For"] = hlist = ["ip1", "ip2"] - self.assertEqual(h["X-Forwarded-For"], b"ip2") - self.assertEqual(h.get("X-Forwarded-For"), b"ip2") - self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"]) + assert h["X-Forwarded-For"] == b"ip2" + assert h.get("X-Forwarded-For") == b"ip2" + assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"] assert h.getlist("X-Forwarded-For") is not hlist def test_multivalue_for_one_header(self): h = Headers((("a", "b"), ("a", "c"))) - self.assertEqual(h["a"], b"c") - self.assertEqual(h.get("a"), b"c") - self.assertEqual(h.getlist("a"), [b"b", b"c"]) + assert h["a"] == b"c" + assert h.get("a") == b"c" + assert h.getlist("a") == [b"b", b"c"] def test_encode_utf8(self): h = Headers({"key": "\xa3"}, encoding="utf-8") key, val = dict(h).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] - self.assertEqual(val[0], b"\xc2\xa3") + assert val[0] == b"\xc2\xa3" def test_encode_latin1(self): h = Headers({"key": "\xa3"}, encoding="latin1") key, val = dict(h).popitem() - self.assertEqual(val[0], b"\xa3") + assert val[0] == b"\xa3" def test_encode_multiple(self): h = Headers({"key": ["\xa3"]}, encoding="utf-8") key, val = dict(h).popitem() - self.assertEqual(val[0], b"\xc2\xa3") + assert val[0] == b"\xc2\xa3" def test_delete_and_contains(self): h = Headers() @@ -81,17 +80,17 @@ class HeadersTest(unittest.TestCase): h = Headers() olist = h.setdefault("X-Forwarded-For", "ip1") - self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1"]) + assert h.getlist("X-Forwarded-For") == [b"ip1"] assert h.getlist("X-Forwarded-For") is olist def test_iterables(self): idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]} h = Headers(idict) - self.assertDictEqual( - dict(h), - {b"Content-Type": [b"text/html"], b"X-Forwarded-For": [b"ip1", b"ip2"]}, - ) + assert dict(h) == { + b"Content-Type": [b"text/html"], + b"X-Forwarded-For": [b"ip1", b"ip2"], + } self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"]) self.assertSortedEqual( h.items(), @@ -102,57 +101,57 @@ class HeadersTest(unittest.TestCase): def test_update(self): h = Headers() h.update({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}) - self.assertEqual(h.getlist("Content-Type"), [b"text/html"]) - self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"]) + assert h.getlist("Content-Type") == [b"text/html"] + assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"] def test_copy(self): h1 = Headers({"header1": ["value1", "value2"]}) h2 = copy.copy(h1) - self.assertEqual(h1, h2) - self.assertEqual(h1.getlist("header1"), h2.getlist("header1")) + assert h1 == h2 + assert h1.getlist("header1") == h2.getlist("header1") assert h1.getlist("header1") is not h2.getlist("header1") assert isinstance(h2, Headers) def test_appendlist(self): h1 = Headers({"header1": "value1"}) h1.appendlist("header1", "value3") - self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"]) + assert h1.getlist("header1") == [b"value1", b"value3"] h1 = Headers() h1.appendlist("header1", "value1") h1.appendlist("header1", "value3") - self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"]) + assert h1.getlist("header1") == [b"value1", b"value3"] def test_setlist(self): h1 = Headers({"header1": "value1"}) - self.assertEqual(h1.getlist("header1"), [b"value1"]) + assert h1.getlist("header1") == [b"value1"] h1.setlist("header1", [b"value2", b"value3"]) - self.assertEqual(h1.getlist("header1"), [b"value2", b"value3"]) + assert h1.getlist("header1") == [b"value2", b"value3"] def test_setlistdefault(self): h1 = Headers({"header1": "value1"}) h1.setlistdefault("header1", ["value2", "value3"]) h1.setlistdefault("header2", ["value2", "value3"]) - self.assertEqual(h1.getlist("header1"), [b"value1"]) - self.assertEqual(h1.getlist("header2"), [b"value2", b"value3"]) + assert h1.getlist("header1") == [b"value1"] + assert h1.getlist("header2") == [b"value2", b"value3"] def test_none_value(self): h1 = Headers() h1["foo"] = "bar" h1["foo"] = None h1.setdefault("foo", "bar") - self.assertEqual(h1.get("foo"), None) - self.assertEqual(h1.getlist("foo"), []) + assert h1.get("foo") is None + assert h1.getlist("foo") == [] def test_int_value(self): h1 = Headers({"hey": 5}) h1["foo"] = 1 h1.setdefault("bar", 2) h1.setlist("buz", [1, "dos", 3]) - self.assertEqual(h1.getlist("foo"), [b"1"]) - self.assertEqual(h1.getlist("bar"), [b"2"]) - self.assertEqual(h1.getlist("buz"), [b"1", b"dos", b"3"]) - self.assertEqual(h1.getlist("hey"), [b"5"]) + assert h1.getlist("foo") == [b"1"] + assert h1.getlist("bar") == [b"2"] + assert h1.getlist("buz") == [b"1", b"dos", b"3"] + assert h1.getlist("hey") == [b"5"] def test_invalid_value(self): with pytest.raises(TypeError, match="Unsupported value type"): From d442227fa74e414f4c7ac6baea6c3c4a1d938219 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Sun, 9 Mar 2025 23:24:12 +0400 Subject: [PATCH 1681/2083] Converting tests to plain asserts, part 8. (#6711) --- tests/test_http_request.py | 608 +++++++++++++++----------------- tests/test_http_response.py | 316 ++++++++--------- tests/test_loader.py | 280 +++++++-------- tests/test_settings/__init__.py | 309 ++++++++-------- 4 files changed, 718 insertions(+), 795 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index e5291157d..6bf0b8e3f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,6 +1,5 @@ import json import re -import unittest import warnings import xmlrpc.client from typing import Any @@ -22,7 +21,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode -class RequestTest(unittest.TestCase): +class TestRequest: request_class = Request default_method = "GET" default_headers: dict[bytes, list[bytes]] = {} @@ -40,12 +39,12 @@ class RequestTest(unittest.TestCase): r = self.request_class("http://www.example.com") assert isinstance(r.url, str) - self.assertEqual(r.url, "http://www.example.com") - self.assertEqual(r.method, self.default_method) + assert r.url == "http://www.example.com" + assert r.method == self.default_method assert isinstance(r.headers, Headers) - self.assertEqual(r.headers, self.default_headers) - self.assertEqual(r.meta, self.default_meta) + assert r.headers == self.default_headers + assert r.meta == self.default_meta meta = {"lala": "lolo"} headers = {b"caca": b"coco"} @@ -54,9 +53,9 @@ class RequestTest(unittest.TestCase): ) assert r.meta is not meta - self.assertEqual(r.meta, meta) + assert r.meta == meta assert r.headers is not headers - self.assertEqual(r.headers[b"caca"], b"coco") + assert r.headers[b"caca"] == b"coco" def test_url_scheme(self): # This test passes by not raising any (ValueError) exception @@ -83,61 +82,61 @@ class RequestTest(unittest.TestCase): r = self.request_class(url=url, headers=headers) p = self.request_class(url=url, headers=r.headers) - self.assertEqual(r.headers, p.headers) - self.assertFalse(r.headers is headers) - self.assertFalse(p.headers is r.headers) + assert r.headers == p.headers + assert r.headers is not headers + assert p.headers is not r.headers # headers must not be unicode h = Headers({"key1": "val1", "key2": "val2"}) h["newkey"] = "newval" for k, v in h.items(): - self.assertIsInstance(k, bytes) + assert isinstance(k, bytes) for s in v: - self.assertIsInstance(s, bytes) + assert isinstance(s, bytes) def test_eq(self): url = "http://www.scrapy.org" r1 = self.request_class(url=url) r2 = self.request_class(url=url) - self.assertNotEqual(r1, r2) + assert r1 != r2 set_ = set() set_.add(r1) set_.add(r2) - self.assertEqual(len(set_), 2) + assert len(set_) == 2 def test_url(self): r = self.request_class(url="http://www.scrapy.org/path") - self.assertEqual(r.url, "http://www.scrapy.org/path") + assert r.url == "http://www.scrapy.org/path" def test_url_quoting(self): r = self.request_class(url="http://www.scrapy.org/blank%20space") - self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") + assert r.url == "http://www.scrapy.org/blank%20space" r = self.request_class(url="http://www.scrapy.org/blank space") - self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") + assert r.url == "http://www.scrapy.org/blank%20space" def test_url_encoding(self): r = self.request_class(url="http://www.scrapy.org/price/£") - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + assert r.url == "http://www.scrapy.org/price/%C2%A3" def test_url_encoding_other(self): # encoding affects only query part of URI, not path # path part should always be UTF-8 encoded before percent-escaping r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + assert r.url == "http://www.scrapy.org/price/%C2%A3" r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + assert r.url == "http://www.scrapy.org/price/%C2%A3" def test_url_encoding_query(self): r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") - self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") + assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" # should be same as above r2 = self.request_class( url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8" ) - self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") + assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" def test_url_encoding_query_latin1(self): # encoding is used for encoding query-string before percent-escaping; @@ -145,7 +144,7 @@ class RequestTest(unittest.TestCase): r3 = self.request_class( url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1" ) - self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3") + assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3" def test_url_encoding_nonutf8_untouched(self): # percent-escaping sequences that do not match valid UTF-8 sequences @@ -164,16 +163,16 @@ class RequestTest(unittest.TestCase): # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different # URI from "http://www.example.org/r%E9sum%E9.html". r1 = self.request_class(url="http://www.scrapy.org/price/%a3") - self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3") + assert r1.url == "http://www.scrapy.org/price/%a3" r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") - self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") - self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html") + assert r4.url == "http://www.example.org/r%E9sum%E9.html" def test_body(self): r1 = self.request_class(url="http://www.example.com/") @@ -181,19 +180,19 @@ class RequestTest(unittest.TestCase): r2 = self.request_class(url="http://www.example.com/", body=b"") assert isinstance(r2.body, bytes) - self.assertEqual(r2.encoding, "utf-8") # default encoding + assert r2.encoding == "utf-8" # default encoding r3 = self.request_class( url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8" ) assert isinstance(r3.body, bytes) - self.assertEqual(r3.body, b"Price: \xc2\xa3100") + assert r3.body == b"Price: \xc2\xa3100" r4 = self.request_class( url="http://www.example.com/", body="Price: \xa3100", encoding="latin1" ) assert isinstance(r4.body, bytes) - self.assertEqual(r4.body, b"Price: \xa3100") + assert r4.body == b"Price: \xa3100" def test_copy(self): """Test Request copy""" @@ -219,25 +218,25 @@ class RequestTest(unittest.TestCase): # make sure flags list is shallow copied assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - self.assertEqual(r1.flags, r2.flags) + assert r1.flags == r2.flags # make sure cb_kwargs dict is shallow copied assert r1.cb_kwargs is not r2.cb_kwargs, ( "cb_kwargs must be a shallow copy, not identical" ) - self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) + assert r1.cb_kwargs == r2.cb_kwargs # make sure meta dict is shallow copied assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" - self.assertEqual(r1.meta, r2.meta) + assert r1.meta == r2.meta # make sure headers attribute is shallow copied assert r1.headers is not r2.headers, ( "headers must be a shallow copy, not identical" ) - self.assertEqual(r1.headers, r2.headers) - self.assertEqual(r1.encoding, r2.encoding) - self.assertEqual(r1.dont_filter, r2.dont_filter) + assert r1.headers == r2.headers + assert r1.encoding == r2.encoding + assert r1.dont_filter == r2.dont_filter # Request.body can be identical since it's an immutable object (str) @@ -258,10 +257,10 @@ class RequestTest(unittest.TestCase): hdrs = Headers(r1.headers) hdrs[b"key"] = b"value" r2 = r1.replace(method="POST", body="New body", headers=hdrs) - self.assertEqual(r1.url, r2.url) - self.assertEqual((r1.method, r2.method), ("GET", "POST")) - self.assertEqual((r1.body, r2.body), (b"", b"New body")) - self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs)) + assert r1.url == r2.url + assert (r1.method, r2.method) == ("GET", "POST") + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == (self.default_headers, hdrs) # Empty attributes (which may fail if not compared properly) r3 = self.request_class( @@ -270,9 +269,9 @@ class RequestTest(unittest.TestCase): r4 = r3.replace( url="http://www.example.com/2", body=b"", meta={}, dont_filter=False ) - self.assertEqual(r4.url, "http://www.example.com/2") - self.assertEqual(r4.body, b"") - self.assertEqual(r4.meta, {}) + assert r4.url == "http://www.example.com/2" + assert r4.body == b"" + assert r4.meta == {} assert r4.dont_filter is False def test_method_always_str(self): @@ -291,32 +290,32 @@ class RequestTest(unittest.TestCase): pass r1 = self.request_class("http://example.com") - self.assertIsNone(r1.callback) - self.assertIsNone(r1.errback) + assert r1.callback is None + assert r1.errback is None r2 = self.request_class("http://example.com", callback=a_function) - self.assertIs(r2.callback, a_function) - self.assertIsNone(r2.errback) + assert r2.callback is a_function + assert r2.errback is None r3 = self.request_class("http://example.com", errback=a_function) - self.assertIsNone(r3.callback) - self.assertIs(r3.errback, a_function) + assert r3.callback is None + assert r3.errback is a_function r4 = self.request_class( url="http://example.com", callback=a_function, errback=a_function, ) - self.assertIs(r4.callback, a_function) - self.assertIs(r4.errback, a_function) + assert r4.callback is a_function + assert r4.errback is a_function r5 = self.request_class( url="http://example.com", callback=NO_CALLBACK, errback=NO_CALLBACK, ) - self.assertIs(r5.callback, NO_CALLBACK) - self.assertIs(r5.errback, NO_CALLBACK) + assert r5.callback is NO_CALLBACK + assert r5.errback is NO_CALLBACK def test_callback_and_errback_type(self): with pytest.raises(TypeError): @@ -354,53 +353,46 @@ class RequestTest(unittest.TestCase): "2%3A15&comments=' --compressed" ) r = self.request_class.from_curl(curl_command) - self.assertEqual(r.method, "POST") - self.assertEqual(r.url, "http://httpbin.org/post") - self.assertEqual( - r.body, - b"custname=John+Smith&custtel=500&custemail=jsmith%40" + assert r.method == "POST" + assert r.url == "http://httpbin.org/post" + assert ( + r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40" b"example.org&size=small&topping=cheese&topping=onion" - b"&delivery=12%3A15&comments=", - ) - self.assertEqual( - r.cookies, - { - "_gauges_unique_year": "1", - "_gauges_unique": "1", - "_gauges_unique_month": "1", - "_gauges_unique_hour": "1", - "_gauges_unique_day": "1", - }, - ) - self.assertEqual( - r.headers, - { - b"Origin": [b"http://httpbin.org"], - b"Accept-Encoding": [b"gzip, deflate"], - b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], - b"Upgrade-Insecure-Requests": [b"1"], - b"User-Agent": [ - b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." - b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" - b".75 Chrome/62.0.3202.75 Safari/537.36" - ], - b"Content-Type": [b"application /x-www-form-urlencoded"], - b"Accept": [ - b"text/html,application/xhtml+xml,application/xml;q=0." - b"9,image/webp,image/apng,*/*;q=0.8" - ], - b"Cache-Control": [b"max-age=0"], - b"Referer": [b"http://httpbin.org/forms/post"], - b"Connection": [b"keep-alive"], - }, + b"&delivery=12%3A15&comments=" ) + assert r.cookies == { + "_gauges_unique_year": "1", + "_gauges_unique": "1", + "_gauges_unique_month": "1", + "_gauges_unique_hour": "1", + "_gauges_unique_day": "1", + } + assert r.headers == { + b"Origin": [b"http://httpbin.org"], + b"Accept-Encoding": [b"gzip, deflate"], + b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], + b"Upgrade-Insecure-Requests": [b"1"], + b"User-Agent": [ + b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." + b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" + b".75 Chrome/62.0.3202.75 Safari/537.36" + ], + b"Content-Type": [b"application /x-www-form-urlencoded"], + b"Accept": [ + b"text/html,application/xhtml+xml,application/xml;q=0." + b"9,image/webp,image/apng,*/*;q=0.8" + ], + b"Cache-Control": [b"max-age=0"], + b"Referer": [b"http://httpbin.org/forms/post"], + b"Connection": [b"keep-alive"], + } def test_from_curl_with_kwargs(self): r = self.request_class.from_curl( 'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"} ) - self.assertEqual(r.method, "POST") - self.assertEqual(r.meta, {"key": "value"}) + assert r.method == "POST" + assert r.meta == {"key": "value"} def test_from_curl_ignore_unknown_options(self): # By default: it works and ignores the unknown options: --foo and -z @@ -409,7 +401,7 @@ class RequestTest(unittest.TestCase): r = self.request_class.from_curl( 'curl -X DELETE "http://example.org" --foo -z', ) - self.assertEqual(r.method, "DELETE") + assert r.method == "DELETE" # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z @@ -420,17 +412,17 @@ class RequestTest(unittest.TestCase): ) -class FormRequestTest(RequestTest): +class TestFormRequest(TestRequest): request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): first = to_unicode(first).split("&") second = to_unicode(second).split("&") - return self.assertEqual(sorted(first), sorted(second), msg) + assert sorted(first) == sorted(second), msg def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) - self.assertEqual(r1.body, b"") + assert r1.body == b"" def test_formdata_overrides_querystring(self): data = (("a", "one"), ("a", "two"), ("b", "2")) @@ -438,69 +430,61 @@ class FormRequestTest(RequestTest): "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data ).url.split("#", maxsplit=1)[0] fs = _qs(self.request_class(url, method="GET", formdata=data)) - self.assertEqual(set(fs[b"a"]), {b"one", b"two"}) - self.assertEqual(fs[b"b"], [b"2"]) - self.assertIsNone(fs.get(b"c")) + assert set(fs[b"a"]) == {b"one", b"two"} + assert fs[b"b"] == [b"2"] + assert fs.get(b"c") is None data = {"a": "1", "b": "2"} fs = _qs( self.request_class("http://www.example.com/", method="GET", formdata=data) ) - self.assertEqual(fs[b"a"], [b"1"]) - self.assertEqual(fs[b"b"], [b"2"]) + assert fs[b"a"] == [b"1"] + assert fs[b"b"] == [b"2"] def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b"one": b"two", b"price": b"\xc2\xa3 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "utf-8") + assert r2.method == "POST" + assert r2.encoding == "utf-8" self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_textual_data(self): # using default encoding (utf-8) data = {"µ one": "two", "price": "£ 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "utf-8") + assert r2.method == "POST" + assert r2.encoding == "utf-8" self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_mixed_data(self): # using default encoding (utf-8) data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "utf-8") + assert r2.method == "POST" + assert r2.encoding == "utf-8" self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_bytes(self): data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} r2 = self.request_class( "http://www.example.com", formdata=data, encoding="latin1" ) - self.assertEqual(r2.method, "POST") - self.assertEqual(r2.encoding, "latin1") + assert r2.method == "POST" + assert r2.encoding == "latin1" self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") - self.assertEqual( - r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" - ) + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_textual_data(self): data = {"price": "£ 100"} r3 = self.request_class( "http://www.example.com", formdata=data, encoding="latin1" ) - self.assertEqual(r3.encoding, "latin1") - self.assertEqual(r3.body, b"price=%A3+100") + assert r3.encoding == "latin1" + assert r3.body == b"price=%A3+100" def test_multi_key_values(self): # using multiples values for a single key @@ -523,16 +507,14 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) - self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) - self.assertEqual(fs[b"test2"], [b"xxx"]) - self.assertEqual(fs[b"six"], [b"seven"]) + assert set(fs[b"test"]) == {b"val1", b"val2"} + assert set(fs[b"one"]) == {b"two", b"three"} + assert fs[b"test2"] == [b"xxx"] + assert fs[b"six"] == [b"seven"] def test_from_response_post_nonascii_bytes_utf8(self): response = _buildresponse( @@ -547,16 +529,14 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs["test £"]), {"val1", "val2"}) - self.assertEqual(set(fs["one"]), {"two", "three"}) - self.assertEqual(fs["test2"], ["xxx µ"]) - self.assertEqual(fs["six"], ["seven"]) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] def test_from_response_post_nonascii_bytes_latin1(self): response = _buildresponse( @@ -572,16 +552,14 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req, to_unicode=True, encoding="latin1") - self.assertEqual(set(fs["test £"]), {"val1", "val2"}) - self.assertEqual(set(fs["one"]), {"two", "three"}) - self.assertEqual(fs["test2"], ["xxx µ"]) - self.assertEqual(fs["six"], ["seven"]) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] def test_from_response_post_nonascii_unicode(self): response = _buildresponse( @@ -596,16 +574,14 @@ class FormRequestTest(RequestTest): response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers[b"Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs["test £"]), {"val1", "val2"}) - self.assertEqual(set(fs["one"]), {"two", "three"}) - self.assertEqual(fs["test2"], ["xxx µ"]) - self.assertEqual(fs["six"], ["seven"]) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] def test_from_response_duplicate_form_key(self): response = _buildresponse("<form></form>", url="http://www.example.com") @@ -614,8 +590,8 @@ class FormRequestTest(RequestTest): method="GET", formdata=(("foo", "bar"), ("foo", "baz")), ) - self.assertEqual(urlparse_cached(req).hostname, "www.example.com") - self.assertEqual(urlparse_cached(req).query, "foo=bar&foo=baz") + assert urlparse_cached(req).hostname == "www.example.com" + assert urlparse_cached(req).query == "foo=bar&foo=baz" def test_from_response_override_duplicate_form_key(self): response = _buildresponse( @@ -628,8 +604,8 @@ class FormRequestTest(RequestTest): response, formdata=(("two", "2"), ("two", "4")) ) fs = _qs(req) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2", b"4"]) + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2", b"4"] def test_from_response_extra_headers(self): response = _buildresponse( @@ -644,11 +620,9 @@ class FormRequestTest(RequestTest): formdata={"one": ["two", "three"], "six": "seven"}, headers={"Accept-Encoding": "gzip,deflate"}, ) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.headers["Accept-Encoding"], b"gzip,deflate") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.headers["Accept-Encoding"] == b"gzip,deflate" def test_from_response_get(self): response = _buildresponse( @@ -662,14 +636,14 @@ class FormRequestTest(RequestTest): r1 = self.request_class.from_response( response, formdata={"one": ["two", "three"], "six": "seven"} ) - self.assertEqual(r1.method, "GET") - self.assertEqual(urlparse_cached(r1).hostname, "www.example.com") - self.assertEqual(urlparse_cached(r1).path, "/this/get.php") + assert r1.method == "GET" + assert urlparse_cached(r1).hostname == "www.example.com" + assert urlparse_cached(r1).path == "/this/get.php" fs = _qs(r1) - self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) - self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) - self.assertEqual(fs[b"test2"], [b"xxx"]) - self.assertEqual(fs[b"six"], [b"seven"]) + assert set(fs[b"test"]) == {b"val1", b"val2"} + assert set(fs[b"one"]) == {b"two", b"three"} + assert fs[b"test2"] == [b"xxx"] + assert fs[b"six"] == [b"seven"] def test_from_response_override_params(self): response = _buildresponse( @@ -680,8 +654,8 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, formdata={"two": "2"}) fs = _qs(req) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2"]) + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] def test_from_response_drop_params(self): response = _buildresponse( @@ -692,8 +666,8 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, formdata={"two": None}) fs = _qs(req) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertNotIn(b"two", fs) + assert fs[b"one"] == [b"1"] + assert b"two" not in fs def test_from_response_override_method(self): response = _buildresponse( @@ -702,9 +676,9 @@ class FormRequestTest(RequestTest): </body></html>""" ) request = FormRequest.from_response(response) - self.assertEqual(request.method, "GET") + assert request.method == "GET" request = FormRequest.from_response(response, method="POST") - self.assertEqual(request.method, "POST") + assert request.method == "POST" def test_from_response_override_url(self): response = _buildresponse( @@ -713,11 +687,11 @@ class FormRequestTest(RequestTest): </body></html>""" ) request = FormRequest.from_response(response) - self.assertEqual(request.url, "http://example.com/app") + assert request.url == "http://example.com/app" request = FormRequest.from_response(response, url="http://foo.bar/absolute") - self.assertEqual(request.url, "http://foo.bar/absolute") + assert request.url == "http://foo.bar/absolute" request = FormRequest.from_response(response, url="/relative") - self.assertEqual(request.url, "http://example.com/relative") + assert request.url == "http://example.com/relative" def test_from_response_case_insensitive(self): response = _buildresponse( @@ -729,9 +703,9 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response) fs = _qs(req) - self.assertEqual(fs[b"clickable1"], [b"clicked1"]) - self.assertFalse(b"i1" in fs, fs) # xpath in _get_inputs() - self.assertFalse(b"clickable2" in fs, fs) # xpath in _get_clickable() + assert fs[b"clickable1"] == [b"clicked1"] + assert b"i1" not in fs, fs # xpath in _get_inputs() + assert b"clickable2" not in fs, fs # xpath in _get_clickable() def test_from_response_submit_first_clickable(self): response = _buildresponse( @@ -744,10 +718,10 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, formdata={"two": "2"}) fs = _qs(req) - self.assertEqual(fs[b"clickable1"], [b"clicked1"]) - self.assertFalse(b"clickable2" in fs, fs) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2"]) + assert fs[b"clickable1"] == [b"clicked1"] + assert b"clickable2" not in fs, fs + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] def test_from_response_submit_not_first_clickable(self): response = _buildresponse( @@ -762,10 +736,10 @@ class FormRequestTest(RequestTest): response, formdata={"two": "2"}, clickdata={"name": "clickable2"} ) fs = _qs(req) - self.assertEqual(fs[b"clickable2"], [b"clicked2"]) - self.assertFalse(b"clickable1" in fs, fs) - self.assertEqual(fs[b"one"], [b"1"]) - self.assertEqual(fs[b"two"], [b"2"]) + assert fs[b"clickable2"] == [b"clicked2"] + assert b"clickable1" not in fs, fs + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] def test_from_response_dont_submit_image_as_input(self): response = _buildresponse( @@ -777,7 +751,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"]}) + assert fs == {b"i1": [b"i1v"]} def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( @@ -790,7 +764,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]}) + assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} def test_from_response_clickdata_does_not_ignore_image(self): response = _buildresponse( @@ -801,7 +775,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]}) + assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} def test_from_response_multiple_clickdata(self): response = _buildresponse( @@ -816,9 +790,9 @@ class FormRequestTest(RequestTest): response, clickdata={"name": "clickable", "value": "clicked2"} ) fs = _qs(req) - self.assertEqual(fs[b"clickable"], [b"clicked2"]) - self.assertEqual(fs[b"one"], [b"clicked1"]) - self.assertEqual(fs[b"two"], [b"clicked2"]) + assert fs[b"clickable"] == [b"clicked2"] + assert fs[b"one"] == [b"clicked1"] + assert fs[b"two"] == [b"clicked2"] def test_from_response_unicode_clickdata(self): response = _buildresponse( @@ -833,7 +807,7 @@ class FormRequestTest(RequestTest): response, clickdata={"name": "price in \u00a3"} ) fs = _qs(req, to_unicode=True) - self.assertTrue(fs["price in \u00a3"]) + assert fs["price in \u00a3"] def test_from_response_unicode_clickdata_latin1(self): response = _buildresponse( @@ -849,7 +823,7 @@ class FormRequestTest(RequestTest): response, clickdata={"name": "price in \u00a5"} ) fs = _qs(req, to_unicode=True, encoding="latin1") - self.assertTrue(fs["price in \u00a5"]) + assert fs["price in \u00a5"] def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( @@ -867,9 +841,9 @@ class FormRequestTest(RequestTest): response, formname="form2", clickdata={"name": "clickable"} ) fs = _qs(req) - self.assertEqual(fs[b"clickable"], [b"clicked2"]) - self.assertEqual(fs[b"field2"], [b"value2"]) - self.assertFalse(b"field1" in fs, fs) + assert fs[b"clickable"] == [b"clicked2"] + assert fs[b"field2"] == [b"value2"] + assert b"field1" not in fs, fs def test_from_response_override_clickable(self): response = _buildresponse( @@ -879,7 +853,7 @@ class FormRequestTest(RequestTest): response, formdata={"clickme": "two"}, clickdata={"name": "clickme"} ) fs = _qs(req) - self.assertEqual(fs[b"clickme"], [b"two"]) + assert fs[b"clickme"] == [b"two"] def test_from_response_dont_click(self): response = _buildresponse( @@ -892,8 +866,8 @@ class FormRequestTest(RequestTest): ) r1 = self.request_class.from_response(response, dont_click=True) fs = _qs(r1) - self.assertFalse(b"clickable1" in fs, fs) - self.assertFalse(b"clickable2" in fs, fs) + assert b"clickable1" not in fs, fs + assert b"clickable2" not in fs, fs def test_from_response_ambiguous_clickdata(self): response = _buildresponse( @@ -934,8 +908,8 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, clickdata={"nr": 1}) fs = _qs(req) - self.assertIn(b"clickable2", fs) - self.assertNotIn(b"clickable1", fs) + assert b"clickable2" in fs + assert b"clickable1" not in fs def test_from_response_invalid_nr_index_clickdata(self): response = _buildresponse( @@ -962,7 +936,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(response, formdata={"bar": "buz"}) fs = _qs(req) - self.assertEqual(fs, {b"foo": [b"xxx"], b"bar": [b"buz"]}) + assert fs == {b"foo": [b"xxx"], b"bar": [b"buz"]} def test_from_response_errors_formnumber(self): response = _buildresponse( @@ -983,12 +957,10 @@ class FormRequestTest(RequestTest): </form>""" ) r1 = self.request_class.from_response(response, formdata={"two": "3"}) - self.assertEqual(r1.method, "POST") - self.assertEqual( - r1.headers["Content-type"], b"application/x-www-form-urlencoded" - ) + assert r1.method == "POST" + assert r1.headers["Content-type"] == b"application/x-www-form-urlencoded" fs = _qs(r1) - self.assertEqual(fs, {b"one": [b"1"], b"two": [b"3"]}) + assert fs == {b"one": [b"1"], b"two": [b"3"]} def test_from_response_formname_exists(self): response = _buildresponse( @@ -1002,9 +974,9 @@ class FormRequestTest(RequestTest): </form>""" ) r1 = self.request_class.from_response(response, formname="form2") - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) + assert fs == {b"four": [b"4"], b"three": [b"3"]} def test_from_response_formname_nonexistent(self): response = _buildresponse( @@ -1016,9 +988,9 @@ class FormRequestTest(RequestTest): </form>""" ) r1 = self.request_class.from_response(response, formname="form3") - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"one": [b"1"]}) + assert fs == {b"one": [b"1"]} def test_from_response_formname_errors_formnumber(self): response = _buildresponse( @@ -1044,9 +1016,9 @@ class FormRequestTest(RequestTest): </form>""" ) r1 = self.request_class.from_response(response, formid="form2") - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) + assert fs == {b"four": [b"4"], b"three": [b"3"]} def test_from_response_formname_nonexistent_fallback_formid(self): response = _buildresponse( @@ -1062,9 +1034,9 @@ class FormRequestTest(RequestTest): r1 = self.request_class.from_response( response, formname="form3", formid="form2" ) - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) + assert fs == {b"four": [b"4"], b"three": [b"3"]} def test_from_response_formid_nonexistent(self): response = _buildresponse( @@ -1076,9 +1048,9 @@ class FormRequestTest(RequestTest): </form>""" ) r1 = self.request_class.from_response(response, formid="form3") - self.assertEqual(r1.method, "POST") + assert r1.method == "POST" fs = _qs(r1) - self.assertEqual(fs, {b"one": [b"1"]}) + assert fs == {b"one": [b"1"]} def test_from_response_formid_errors_formnumber(self): response = _buildresponse( @@ -1122,7 +1094,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req, to_unicode=True) - self.assertEqual(fs, {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]}) + assert fs == {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]} def test_from_response_radio(self): res = _buildresponse( @@ -1139,7 +1111,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]}) + assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} def test_from_response_checkbox(self): res = _buildresponse( @@ -1156,7 +1128,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]}) + assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} def test_from_response_input_text(self): res = _buildresponse( @@ -1170,7 +1142,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]}) + assert fs == {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]} def test_from_response_input_hidden(self): res = _buildresponse( @@ -1183,7 +1155,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""]}) + assert fs == {b"i1": [b"i1v1"], b"i2": [b""]} def test_from_response_input_textarea(self): res = _buildresponse( @@ -1196,7 +1168,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]}) + assert fs == {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]} def test_from_response_descendants(self): res = _buildresponse( @@ -1218,7 +1190,7 @@ class FormRequestTest(RequestTest): ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(set(fs), {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"}) + assert set(fs) == {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"} def test_from_response_xpath(self): response = _buildresponse( @@ -1235,13 +1207,13 @@ class FormRequestTest(RequestTest): response, formxpath="//form[@action='post.php']" ) fs = _qs(r1) - self.assertEqual(fs[b"one"], [b"1"]) + assert fs[b"one"] == [b"1"] r1 = self.request_class.from_response( response, formxpath="//form/input[@name='four']" ) fs = _qs(r1) - self.assertEqual(fs[b"three"], [b"3"]) + assert fs[b"three"] == [b"3"] with pytest.raises(ValueError, match="No <form> element found with"): self.request_class.from_response( @@ -1254,7 +1226,7 @@ class FormRequestTest(RequestTest): response, formxpath="//form[@name='\u044a']" ) fs = _qs(r) - self.assertEqual(fs, {}) + assert not fs xpath = "//form[@name='\u03b1']" with pytest.raises(ValueError, match=re.escape(xpath)): @@ -1270,15 +1242,13 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", ) req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b"submit1"]) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b"submit1"] def test_from_response_button_notype(self): response = _buildresponse( @@ -1290,15 +1260,13 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", ) req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b"submit1"]) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b"submit1"] def test_from_response_submit_novalue(self): response = _buildresponse( @@ -1310,15 +1278,13 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", ) req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b""]) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b""] def test_from_response_button_novalue(self): response = _buildresponse( @@ -1330,15 +1296,13 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", ) req = self.request_class.from_response(response) - self.assertEqual(req.method, "POST") - self.assertEqual( - req.headers["Content-type"], b"application/x-www-form-urlencoded" - ) - self.assertEqual(req.url, "http://www.example.com/this/post.php") + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" fs = _qs(req) - self.assertEqual(fs[b"test1"], [b"val1"]) - self.assertEqual(fs[b"test2"], [b"val2"]) - self.assertEqual(fs[b"button1"], [b""]) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b""] def test_html_base_form_action(self): response = _buildresponse( @@ -1356,12 +1320,12 @@ class FormRequestTest(RequestTest): url="http://a.com/", ) req = self.request_class.from_response(response) - self.assertEqual(req.url, "http://b.com/test_form") + assert req.url == "http://b.com/test_form" def test_spaces_in_action(self): resp = _buildresponse('<body><form action=" path\n"></form></body>') req = self.request_class.from_response(resp) - self.assertEqual(req.url, "http://example.com/path") + assert req.url == "http://example.com/path" def test_from_response_css(self): response = _buildresponse( @@ -1378,11 +1342,11 @@ class FormRequestTest(RequestTest): response, formcss="form[action='post.php']" ) fs = _qs(r1) - self.assertEqual(fs[b"one"], [b"1"]) + assert fs[b"one"] == [b"1"] r1 = self.request_class.from_response(response, formcss="input[name='four']") fs = _qs(r1) - self.assertEqual(fs[b"three"], [b"3"]) + assert fs[b"three"] == [b"3"] with pytest.raises(ValueError, match="No <form> element found with"): self.request_class.from_response(response, formcss="input[name='abc']") @@ -1400,7 +1364,7 @@ class FormRequestTest(RequestTest): "</form>" ) r = self.request_class.from_response(response) - self.assertEqual(r.method, expected) + assert r.method == expected def test_form_response_with_invalid_formdata_type_error(self): """Test that a ValueError is raised for non-iterable and non-dict formdata input""" @@ -1464,23 +1428,20 @@ def _qs(req, encoding="utf-8", to_unicode=False): return parse_qs(uqs, True) -class XmlRpcRequestTest(RequestTest): +class TestXmlRpcRequest(TestRequest): request_class = XmlRpcRequest default_method = "POST" default_headers = {b"Content-Type": [b"text/xml"]} def _test_request(self, **kwargs): r = self.request_class("http://scrapytest.org/rpc2", **kwargs) - self.assertEqual(r.headers[b"Content-Type"], b"text/xml") - self.assertEqual( - r.body, - to_bytes( - xmlrpc.client.dumps(**kwargs), encoding=kwargs.get("encoding", "utf-8") - ), + assert r.headers[b"Content-Type"] == b"text/xml" + assert r.body == to_bytes( + xmlrpc.client.dumps(**kwargs), encoding=kwargs.get("encoding", "utf-8") ) - self.assertEqual(r.method, "POST") - self.assertEqual(r.encoding, kwargs.get("encoding", "utf-8")) - self.assertTrue(r.dont_filter, True) + assert r.method == "POST" + assert r.encoding == kwargs.get("encoding", "utf-8") + assert r.dont_filter, True def test_xmlrpc_dumps(self): self._test_request(params=("value",)) @@ -1497,7 +1458,7 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=("pas£",), encoding="latin1") -class JsonRequestTest(RequestTest): +class TestJsonRequest(TestRequest): request_class = JsonRequest default_method = "GET" default_headers = { @@ -1505,49 +1466,51 @@ class JsonRequestTest(RequestTest): b"Accept": [b"application/json, text/javascript, */*; q=0.01"], } - def setUp(self): + def setup_method(self): warnings.simplefilter("always") - super().setUp() + + def teardown_method(self): + warnings.resetwarnings() def test_data(self): r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.body, b"") + assert r1.body == b"" body = b"body" r2 = self.request_class(url="http://www.example.com/", body=body) - self.assertEqual(r2.body, body) + assert r2.body == body data = { "name": "value", } r3 = self.request_class(url="http://www.example.com/", data=data) - self.assertEqual(r3.body, to_bytes(json.dumps(data))) + assert r3.body == to_bytes(json.dumps(data)) # empty data r4 = self.request_class(url="http://www.example.com/", data=[]) - self.assertEqual(r4.body, to_bytes(json.dumps([]))) + assert r4.body == to_bytes(json.dumps([])) def test_data_method(self): # data is not passed r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.method, "GET") + assert r1.method == "GET" body = b"body" r2 = self.request_class(url="http://www.example.com/", body=body) - self.assertEqual(r2.method, "GET") + assert r2.method == "GET" data = { "name": "value", } r3 = self.request_class(url="http://www.example.com/", data=data) - self.assertEqual(r3.method, "POST") + assert r3.method == "POST" # method passed explicitly r4 = self.request_class(url="http://www.example.com/", data=data, method="GET") - self.assertEqual(r4.method, "GET") + assert r4.method == "GET" r5 = self.request_class(url="http://www.example.com/", data=[]) - self.assertEqual(r5.method, "POST") + assert r5.method == "POST" def test_body_data(self): """passing both body and data should result a warning""" @@ -1557,10 +1520,10 @@ class JsonRequestTest(RequestTest): } with warnings.catch_warnings(record=True) as _warnings: r5 = self.request_class(url="http://www.example.com/", body=body, data=data) - self.assertEqual(r5.body, body) - self.assertEqual(r5.method, "GET") - self.assertEqual(len(_warnings), 1) - self.assertIn("data will be ignored", str(_warnings[0].message)) + assert r5.body == body + assert r5.method == "GET" + assert len(_warnings) == 1 + assert "data will be ignored" in str(_warnings[0].message) def test_empty_body_data(self): """passing any body value and data should result a warning""" @@ -1569,10 +1532,10 @@ class JsonRequestTest(RequestTest): } with warnings.catch_warnings(record=True) as _warnings: r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) - self.assertEqual(r6.body, b"") - self.assertEqual(r6.method, "GET") - self.assertEqual(len(_warnings), 1) - self.assertIn("data will be ignored", str(_warnings[0].message)) + assert r6.body == b"" + assert r6.method == "GET" + assert len(_warnings) == 1 + assert "data will be ignored" in str(_warnings[0].message) def test_body_none_data(self): data = { @@ -1580,15 +1543,15 @@ class JsonRequestTest(RequestTest): } with warnings.catch_warnings(record=True) as _warnings: r7 = self.request_class(url="http://www.example.com/", body=None, data=data) - self.assertEqual(r7.body, to_bytes(json.dumps(data))) - self.assertEqual(r7.method, "POST") - self.assertEqual(len(_warnings), 0) + assert r7.body == to_bytes(json.dumps(data)) + assert r7.method == "POST" + assert len(_warnings) == 0 def test_body_data_none(self): with warnings.catch_warnings(record=True) as _warnings: r8 = self.request_class(url="http://www.example.com/", body=None, data=None) - self.assertEqual(r8.method, "GET") - self.assertEqual(len(_warnings), 0) + assert r8.method == "GET" + assert len(_warnings) == 0 def test_dumps_sort_keys(self): """Test that sort_keys=True is passed to json.dumps by default""" @@ -1598,7 +1561,7 @@ class JsonRequestTest(RequestTest): with mock.patch("json.dumps", return_value=b"") as mock_dumps: self.request_class(url="http://www.example.com/", data=data) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["sort_keys"], True) + assert kwargs["sort_keys"] is True def test_dumps_kwargs(self): """Test that dumps_kwargs are passed to json.dumps""" @@ -1614,8 +1577,8 @@ class JsonRequestTest(RequestTest): url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs ) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["ensure_ascii"], True) - self.assertEqual(kwargs["allow_nan"], True) + assert kwargs["ensure_ascii"] is True + assert kwargs["allow_nan"] is True def test_replace_data(self): data1 = { @@ -1626,7 +1589,7 @@ class JsonRequestTest(RequestTest): } r1 = self.request_class(url="http://www.example.com/", data=data1) r2 = r1.replace(data=data2) - self.assertEqual(r2.body, to_bytes(json.dumps(data2))) + assert r2.body == to_bytes(json.dumps(data2)) def test_replace_sort_keys(self): """Test that replace provides sort_keys=True to json.dumps""" @@ -1640,7 +1603,7 @@ class JsonRequestTest(RequestTest): with mock.patch("json.dumps", return_value=b"") as mock_dumps: r1.replace(data=data2) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["sort_keys"], True) + assert kwargs["sort_keys"] is True def test_replace_dumps_kwargs(self): """Test that dumps_kwargs are provided to json.dumps when replace is called""" @@ -1660,8 +1623,8 @@ class JsonRequestTest(RequestTest): with mock.patch("json.dumps", return_value=b"") as mock_dumps: r1.replace(data=data2) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs["ensure_ascii"], True) - self.assertEqual(kwargs["allow_nan"], True) + assert kwargs["ensure_ascii"] is True + assert kwargs["allow_nan"] is True def test_replacement_both_body_and_data_warns(self): """Test that we get a warning if both body and data are passed""" @@ -1677,11 +1640,6 @@ class JsonRequestTest(RequestTest): with warnings.catch_warnings(record=True) as _warnings: r1.replace(data=data2, body=body2) - self.assertIn( - "Both body and data passed. data will be ignored", - str(_warnings[0].message), + assert "Both body and data passed. data will be ignored" in str( + _warnings[0].message ) - - def tearDown(self): - warnings.resetwarnings() - super().tearDown() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 5a943f084..fdef5adea 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,5 +1,4 @@ import codecs -import unittest from unittest import mock import pytest @@ -22,62 +21,56 @@ from scrapy.utils.python import to_unicode from tests import get_testdata -class BaseResponseTest(unittest.TestCase): +class TestResponseBase: response_class = Response def test_init(self): # Response requires url in the constructor with pytest.raises(TypeError): self.response_class() - self.assertTrue( - isinstance(self.response_class("http://example.com/"), self.response_class) + assert isinstance( + self.response_class("http://example.com/"), self.response_class ) with pytest.raises(TypeError): self.response_class(b"http://example.com") with pytest.raises(TypeError): self.response_class(url="http://example.com", body={}) # body can be str or None - self.assertTrue( - isinstance( - self.response_class("http://example.com/", body=b""), - self.response_class, - ) + assert isinstance( + self.response_class("http://example.com/", body=b""), + self.response_class, ) - self.assertTrue( - isinstance( - self.response_class("http://example.com/", body=b"body"), - self.response_class, - ) + assert isinstance( + self.response_class("http://example.com/", body=b"body"), + self.response_class, ) # test presence of all optional parameters - self.assertTrue( - isinstance( - self.response_class( - "http://example.com/", body=b"", headers={}, status=200 - ), - self.response_class, - ) + assert isinstance( + self.response_class( + "http://example.com/", body=b"", headers={}, status=200 + ), + self.response_class, ) r = self.response_class("http://www.example.com") assert isinstance(r.url, str) - self.assertEqual(r.url, "http://www.example.com") - self.assertEqual(r.status, 200) + assert r.url == "http://www.example.com" + assert r.status == 200 assert isinstance(r.headers, Headers) - self.assertEqual(r.headers, {}) + assert not r.headers headers = {"foo": "bar"} body = b"a body" r = self.response_class("http://www.example.com", headers=headers, body=body) assert r.headers is not headers - self.assertEqual(r.headers[b"foo"], b"bar") + assert r.headers[b"foo"] == b"bar" r = self.response_class("http://www.example.com", status=301) - self.assertEqual(r.status, 301) + assert r.status == 301 r = self.response_class("http://www.example.com", status="301") - self.assertEqual(r.status, 301) + assert r.status == 301 with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): self.response_class("http://example.com", status="lala200") @@ -88,18 +81,18 @@ class BaseResponseTest(unittest.TestCase): r1.flags.append("cached") r2 = r1.copy() - self.assertEqual(r1.status, r2.status) - self.assertEqual(r1.body, r2.body) + assert r1.status == r2.status + assert r1.body == r2.body # make sure flags list is shallow copied assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - self.assertEqual(r1.flags, r2.flags) + assert r1.flags == r2.flags # make sure headers attribute is shallow copied assert r1.headers is not r2.headers, ( "headers must be a shallow copy, not identical" ) - self.assertEqual(r1.headers, r2.headers) + assert r1.headers == r2.headers def test_copy_meta(self): req = Request("http://www.example.com") @@ -144,16 +137,16 @@ class BaseResponseTest(unittest.TestCase): r1 = self.response_class("http://www.example.com") r2 = r1.replace(status=301, body=b"New body", headers=hdrs) assert r1.body == b"" - self.assertEqual(r1.url, r2.url) - self.assertEqual((r1.status, r2.status), (200, 301)) - self.assertEqual((r1.body, r2.body), (b"", b"New body")) - self.assertEqual((r1.headers, r2.headers), ({}, hdrs)) + assert r1.url == r2.url + assert (r1.status, r2.status) == (200, 301) + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == ({}, hdrs) # Empty attributes (which may fail if not compared properly) r3 = self.response_class("http://www.example.com", flags=["cached"]) r4 = r3.replace(body=b"", flags=[]) - self.assertEqual(r4.body, b"") - self.assertEqual(r4.flags, []) + assert r4.body == b"" + assert not r4.flags def _assert_response_values(self, response, encoding, body): if isinstance(body, str): @@ -166,11 +159,11 @@ class BaseResponseTest(unittest.TestCase): assert isinstance(response.body, bytes) assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) - self.assertEqual(response.body, body_bytes) - self.assertEqual(response.text, body_unicode) + assert response.body == body_bytes + assert response.text == body_unicode def _assert_response_encoding(self, response, encoding): - self.assertEqual(response.encoding, resolve_encoding(encoding)) + assert response.encoding == resolve_encoding(encoding) def test_immutable_attributes(self): r = self.response_class("http://example.com") @@ -183,7 +176,7 @@ class BaseResponseTest(unittest.TestCase): """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" joined = self.response_class("http://www.example.com").urljoin("/test") absolute = "http://www.example.com/test" - self.assertEqual(joined, absolute) + assert joined == absolute def test_shortcut_attributes(self): r = self.response_class("http://example.com", body=b"hello") @@ -241,7 +234,7 @@ class BaseResponseTest(unittest.TestCase): def test_follow_flags(self): res = self.response_class("http://example.com/") fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - self.assertEqual(fol.flags, ["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] # Response.follow_all @@ -276,7 +269,7 @@ class BaseResponseTest(unittest.TestCase): def test_follow_all_empty(self): r = self.response_class("http://example.com") - self.assertEqual([], list(r.follow_all([]))) + assert not list(r.follow_all([])) def test_follow_all_invalid(self): r = self.response_class("http://example.com") @@ -327,13 +320,13 @@ class BaseResponseTest(unittest.TestCase): ] fol = re.follow_all(urls, flags=["cached", "allowed"]) for req in fol: - self.assertEqual(req.flags, ["cached", "allowed"]) + assert req.flags == ["cached", "allowed"] def _assert_followed_url(self, follow_obj, target_url, response=None): if response is None: response = self._links_response() req = response.follow(follow_obj) - self.assertEqual(req.url, target_url) + assert req.url == target_url return req def _assert_followed_all_urls(self, follow_obj, target_urls, response=None): @@ -341,7 +334,7 @@ class BaseResponseTest(unittest.TestCase): response = self._links_response() followed = response.follow_all(follow_obj) for req, target in zip(followed, target_urls): - self.assertEqual(req.url, target) + assert req.url == target yield req def _links_response(self): @@ -353,7 +346,7 @@ class BaseResponseTest(unittest.TestCase): return self.response_class("http://example.com/index", body=body) -class TextResponseTest(BaseResponseTest): +class TestTextResponse(TestResponseBase): response_class = TextResponse def test_replace(self): @@ -365,10 +358,10 @@ class TextResponseTest(BaseResponseTest): r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") assert isinstance(r2, self.response_class) - self.assertEqual(r2.url, "http://www.example.com/other") + assert r2.url == "http://www.example.com/other" self._assert_response_encoding(r2, "cp852") - self.assertEqual(r3.url, "http://www.example.com/other") - self.assertEqual(r3._declared_encoding(), "latin1") + assert r3.url == "http://www.example.com/other" + assert r3._declared_encoding() == "latin1" def test_unicode_url(self): # instantiate with unicode url without encoding (should set default encoding) @@ -382,21 +375,21 @@ class TextResponseTest(BaseResponseTest): resp = self.response_class( url="http://www.example.com/price/\xa3", encoding="utf-8" ) - self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")) + assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") resp = self.response_class( url="http://www.example.com/price/\xa3", encoding="latin-1" ) - self.assertEqual(resp.url, "http://www.example.com/price/\xa3") + assert resp.url == "http://www.example.com/price/\xa3" resp = self.response_class( "http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}, ) - self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")) + assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") resp = self.response_class( "http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, ) - self.assertEqual(resp.url, "http://www.example.com/price/\xa3") + assert resp.url == "http://www.example.com/price/\xa3" def test_unicode_body(self): unicode_string = ( @@ -412,8 +405,8 @@ class TextResponseTest(BaseResponseTest): ) # check response.text - self.assertTrue(isinstance(r1.text, str)) - self.assertEqual(r1.text, unicode_string) + assert isinstance(r1.text, str) + assert r1.text == unicode_string def test_encoding(self): r1 = self.response_class( @@ -458,18 +451,18 @@ class TextResponseTest(BaseResponseTest): }, ) - self.assertEqual(r1._headers_encoding(), "utf-8") - self.assertEqual(r2._headers_encoding(), None) - self.assertEqual(r2._declared_encoding(), "utf-8") + assert r1._headers_encoding() == "utf-8" + assert r2._headers_encoding() is None + assert r2._declared_encoding() == "utf-8" self._assert_response_encoding(r2, "utf-8") - self.assertEqual(r3._headers_encoding(), "cp1252") - self.assertEqual(r3._declared_encoding(), "cp1252") - self.assertEqual(r4._headers_encoding(), None) - self.assertEqual(r5._headers_encoding(), None) - self.assertEqual(r8._headers_encoding(), "cp1251") - self.assertEqual(r9._headers_encoding(), None) - self.assertEqual(r8._declared_encoding(), "utf-8") - self.assertEqual(r9._declared_encoding(), None) + assert r3._headers_encoding() == "cp1252" + assert r3._declared_encoding() == "cp1252" + assert r4._headers_encoding() is None + assert r5._headers_encoding() is None + assert r8._headers_encoding() == "cp1251" + assert r9._headers_encoding() is None + assert r8._declared_encoding() == "utf-8" + assert r9._declared_encoding() is None self._assert_response_encoding(r5, "utf-8") self._assert_response_encoding(r8, "utf-8") self._assert_response_encoding(r9, "cp1252") @@ -493,7 +486,7 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=UNKNOWN"]}, body=b"\xc2\xa3", ) - self.assertEqual(r._declared_encoding(), None) + assert r._declared_encoding() is None self._assert_response_values(r, "utf-8", "\xa3") def test_utf16(self): @@ -511,14 +504,11 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xef\xbb\xbfWORD\xe3\xab", ) - self.assertEqual(r6.encoding, "utf-8") - self.assertIn( - r6.text, - { - "WORD\ufffd\ufffd", # w3lib < 1.19.0 - "WORD\ufffd", # w3lib >= 1.19.0 - }, - ) + assert r6.encoding == "utf-8" + assert r6.text in { + "WORD\ufffd\ufffd", # w3lib < 1.19.0 + "WORD\ufffd", # w3lib >= 1.19.0 + } def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, @@ -532,21 +522,21 @@ class TextResponseTest(BaseResponseTest): # Test response without content-type and BOM encoding response = self.response_class(url, body=body) - self.assertEqual(response.encoding, "utf-8") - self.assertEqual(response.text, "WORD") + assert response.encoding == "utf-8" + assert response.text == "WORD" response = self.response_class(url, body=body) - self.assertEqual(response.text, "WORD") - self.assertEqual(response.encoding, "utf-8") + assert response.text == "WORD" + assert response.encoding == "utf-8" # Body caching sideeffect isn't triggered when encoding is declared in # content-type header but BOM still need to be removed from decoded # body response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.encoding, "utf-8") - self.assertEqual(response.text, "WORD") + assert response.encoding == "utf-8" + assert response.text == "WORD" response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.text, "WORD") - self.assertEqual(response.encoding, "utf-8") + assert response.text == "WORD" + assert response.encoding == "utf-8" def test_replace_wrong_encoding(self): """Test invalid chars are replaced properly""" @@ -577,49 +567,47 @@ class TextResponseTest(BaseResponseTest): body = b"<html><head><title>Some page" response = self.response_class("http://www.example.com", body=body) - self.assertIsInstance(response.selector, Selector) - self.assertEqual(response.selector.type, "html") - self.assertIs(response.selector, response.selector) # property is cached - self.assertIs(response.selector.response, response) + assert isinstance(response.selector, Selector) + assert response.selector.type == "html" + assert response.selector is response.selector # property is cached + assert response.selector.response is response - self.assertEqual( - response.selector.xpath("//title/text()").getall(), ["Some page"] - ) - self.assertEqual(response.selector.css("title::text").getall(), ["Some page"]) - self.assertEqual(response.selector.re("Some (.*)"), ["page"]) + assert response.selector.xpath("//title/text()").getall() == ["Some page"] + assert response.selector.css("title::text").getall() == ["Some page"] + assert response.selector.re("Some (.*)") == ["page"] def test_selector_shortcuts(self): body = b"Some page" response = self.response_class("http://www.example.com", body=body) - self.assertEqual( - response.xpath("//title/text()").getall(), - response.selector.xpath("//title/text()").getall(), + assert ( + response.xpath("//title/text()").getall() + == response.selector.xpath("//title/text()").getall() ) - self.assertEqual( - response.css("title::text").getall(), - response.selector.css("title::text").getall(), + assert ( + response.css("title::text").getall() + == response.selector.css("title::text").getall() ) def test_selector_shortcuts_kwargs(self): body = b'Some page

A nice paragraph.

' response = self.response_class("http://www.example.com", body=body) - self.assertEqual( + assert ( response.xpath( "normalize-space(//p[@class=$pclass])", pclass="content" - ).getall(), - response.xpath('normalize-space(//p[@class="content"])').getall(), + ).getall() + == response.xpath('normalize-space(//p[@class="content"])').getall() ) - self.assertEqual( + assert ( response.xpath( "//title[count(following::p[@class=$pclass])=$pcount]/text()", pclass="content", pcount=1, - ).getall(), - response.xpath( + ).getall() + == response.xpath( '//title[count(following::p[@class="content"])=1]/text()' - ).getall(), + ).getall() ) def test_urljoin_with_base_url(self): @@ -629,21 +617,21 @@ class TextResponseTest(BaseResponseTest): "/test" ) absolute = "https://example.net/test" - self.assertEqual(joined, absolute) + assert joined == absolute body = b'' joined = self.response_class("http://www.example.com", body=body).urljoin( "test" ) absolute = "http://www.example.com/test" - self.assertEqual(joined, absolute) + assert joined == absolute body = b'' joined = self.response_class("http://www.example.com", body=body).urljoin( "test" ) absolute = "http://www.example.com/elsewhere/test" - self.assertEqual(joined, absolute) + assert joined == absolute def test_follow_selector(self): resp = self._links_response() @@ -728,7 +716,7 @@ class TextResponseTest(BaseResponseTest): "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82", response=resp1, ) - self.assertEqual(req.encoding, "utf8") + assert req.encoding == "utf8" resp2 = self.response_class( "http://example.com", @@ -742,12 +730,12 @@ class TextResponseTest(BaseResponseTest): "http://example.com/foo?%EF%F0%E8%E2%E5%F2", response=resp2, ) - self.assertEqual(req.encoding, "cp1251") + assert req.encoding == "cp1251" def test_follow_flags(self): res = self.response_class("http://example.com/") fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - self.assertEqual(fol.flags, ["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] def test_follow_all_flags(self): re = self.response_class("http://www.example.com/") @@ -758,7 +746,7 @@ class TextResponseTest(BaseResponseTest): ] fol = re.follow_all(urls, flags=["cached", "allowed"]) for req in fol: - self.assertEqual(req.flags, ["cached", "allowed"]) + assert req.flags == ["cached", "allowed"] def test_follow_all_css(self): expected = [ @@ -767,7 +755,7 @@ class TextResponseTest(BaseResponseTest): ] response = self._links_response() extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')] - self.assertEqual(expected, extracted) + assert expected == extracted def test_follow_all_css_skip_invalid(self): expected = [ @@ -777,9 +765,9 @@ class TextResponseTest(BaseResponseTest): ] response = self._links_response_no_href() extracted1 = [r.url for r in response.follow_all(css=".pagination a")] - self.assertEqual(expected, extracted1) + assert expected == extracted1 extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))] - self.assertEqual(expected, extracted2) + assert expected == extracted2 def test_follow_all_xpath(self): expected = [ @@ -788,7 +776,7 @@ class TextResponseTest(BaseResponseTest): ] response = self._links_response() extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]') - self.assertEqual(expected, [r.url for r in extracted]) + assert expected == [r.url for r in extracted] def test_follow_all_xpath_skip_invalid(self): expected = [ @@ -800,12 +788,12 @@ class TextResponseTest(BaseResponseTest): extracted1 = [ r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a') ] - self.assertEqual(expected, extracted1) + assert expected == extracted1 extracted2 = [ r.url for r in response.follow_all(response.xpath('//div[@id="pagination"]/a')) ] - self.assertEqual(expected, extracted2) + assert expected == extracted2 def test_follow_all_too_many_arguments(self): response = self._links_response() @@ -820,7 +808,7 @@ class TextResponseTest(BaseResponseTest): def test_json_response(self): json_body = b"""{"ip": "109.187.217.200"}""" json_response = self.response_class("http://www.example.com", body=json_body) - self.assertEqual(json_response.json(), {"ip": "109.187.217.200"}) + assert json_response.json() == {"ip": "109.187.217.200"} text_body = b"""text""" text_response = self.response_class("http://www.example.com", body=text_body) @@ -842,7 +830,7 @@ class TextResponseTest(BaseResponseTest): mock_json.assert_called_once_with(json_body) -class HtmlResponseTest(TextResponseTest): +class TestHtmlResponse(TestTextResponse): response_class = HtmlResponse def test_html_encoding(self): @@ -883,7 +871,7 @@ class HtmlResponseTest(TextResponseTest): self._assert_response_values(r1, "gb2312", body) -class XmlResponseTest(TextResponseTest): +class TestXmlResponse(TestTextResponse): response_class = XmlResponse def test_xml_encoding(self): @@ -917,20 +905,20 @@ class XmlResponseTest(TextResponseTest): body = b'value' response = self.response_class("http://www.example.com", body=body) - self.assertIsInstance(response.selector, Selector) - self.assertEqual(response.selector.type, "xml") - self.assertIs(response.selector, response.selector) # property is cached - self.assertIs(response.selector.response, response) + assert isinstance(response.selector, Selector) + assert response.selector.type == "xml" + assert response.selector is response.selector # property is cached + assert response.selector.response is response - self.assertEqual(response.selector.xpath("//elem/text()").getall(), ["value"]) + assert response.selector.xpath("//elem/text()").getall() == ["value"] def test_selector_shortcuts(self): body = b'value' response = self.response_class("http://www.example.com", body=body) - self.assertEqual( - response.xpath("//elem/text()").getall(), - response.selector.xpath("//elem/text()").getall(), + assert ( + response.xpath("//elem/text()").getall() + == response.selector.xpath("//elem/text()").getall() ) def test_selector_shortcuts_kwargs(self): @@ -940,21 +928,21 @@ class XmlResponseTest(TextResponseTest): """ response = self.response_class("http://www.example.com", body=body) - self.assertEqual( + assert ( response.xpath( "//s:elem/text()", namespaces={"s": "http://scrapy.org"} - ).getall(), - response.selector.xpath( + ).getall() + == response.selector.xpath( "//s:elem/text()", namespaces={"s": "http://scrapy.org"} - ).getall(), + ).getall() ) response.selector.register_namespace("s2", "http://scrapy.org") - self.assertEqual( + assert ( response.xpath( "//s1:elem/text()", namespaces={"s1": "http://scrapy.org"} - ).getall(), - response.selector.xpath("//s2:elem/text()").getall(), + ).getall() + == response.selector.xpath("//s2:elem/text()").getall() ) @@ -968,7 +956,7 @@ class CustomResponse(TextResponse): super().__init__(*args, **kwargs) -class CustomResponseTest(TextResponseTest): +class TestCustomResponse(TestTextResponse): response_class = CustomResponse def test_copy(self): @@ -981,11 +969,11 @@ class CustomResponseTest(TextResponseTest): lost="lost", ) r2 = r1.copy() - self.assertIsInstance(r2, self.response_class) - self.assertEqual(r1.foo, r2.foo) - self.assertEqual(r1.bar, r2.bar) - self.assertEqual(r1.lost, "lost") - self.assertIsNone(r2.lost) + assert isinstance(r2, self.response_class) + assert r1.foo == r2.foo + assert r1.bar == r2.bar + assert r1.lost == "lost" + assert r2.lost is None def test_replace(self): super().test_replace() @@ -998,31 +986,31 @@ class CustomResponseTest(TextResponseTest): ) r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") - self.assertIsInstance(r2, self.response_class) - self.assertEqual(r1.foo, "foo") - self.assertEqual(r1.bar, "bar") - self.assertEqual(r1.lost, "lost") - self.assertEqual(r2.foo, "new-foo") - self.assertEqual(r2.bar, "new-bar") - self.assertEqual(r2.lost, "new-lost") + assert isinstance(r2, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r2.foo == "new-foo" + assert r2.bar == "new-bar" + assert r2.lost == "new-lost" r3 = r1.replace(foo="new-foo", bar="new-bar") - self.assertIsInstance(r3, self.response_class) - self.assertEqual(r1.foo, "foo") - self.assertEqual(r1.bar, "bar") - self.assertEqual(r1.lost, "lost") - self.assertEqual(r3.foo, "new-foo") - self.assertEqual(r3.bar, "new-bar") - self.assertIsNone(r3.lost) + assert isinstance(r3, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r3.foo == "new-foo" + assert r3.bar == "new-bar" + assert r3.lost is None r4 = r1.replace(foo="new-foo") - self.assertIsInstance(r4, self.response_class) - self.assertEqual(r1.foo, "foo") - self.assertEqual(r1.bar, "bar") - self.assertEqual(r1.lost, "lost") - self.assertEqual(r4.foo, "new-foo") - self.assertEqual(r4.bar, "bar") - self.assertIsNone(r4.lost) + assert isinstance(r4, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r4.foo == "new-foo" + assert r4.bar == "bar" + assert r4.lost is None with pytest.raises( TypeError, diff --git a/tests/test_loader.py b/tests/test_loader.py index 1a933bb8d..224158e7f 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,7 +1,6 @@ from __future__ import annotations import dataclasses -import unittest import attr import pytest @@ -67,7 +66,7 @@ def processor_with_args(value, other=None, loader_context=None): return value -class BasicItemLoaderTest(unittest.TestCase): +class TestBasicItemLoader: def test_add_value_on_unknown_field(self): il = ProcessorItemLoader() with pytest.raises(KeyError): @@ -80,14 +79,14 @@ class BasicItemLoaderTest(unittest.TestCase): il.add_value("name", "marta") item = il.load_item() assert item is i - self.assertEqual(item["summary"], ["lala"]) - self.assertEqual(item["name"], ["marta"]) + assert item["summary"] == ["lala"] + assert item["name"] == ["marta"] def test_load_item_using_custom_loader(self): il = ProcessorItemLoader() il.add_value("name", "marta") item = il.load_item() - self.assertEqual(item["name"], ["Marta"]) + assert item["name"] == ["Marta"] class InitializationTestMixin: @@ -98,16 +97,16 @@ class InitializationTestMixin: input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo"]} def test_keep_list(self): """Loaded item should contain values from the initial item""" input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]} def test_add_value_singlevalue_singlevalue(self): """Values added after initialization should be appended""" @@ -115,8 +114,8 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", "bar") loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]} def test_add_value_singlevalue_list(self): """Values added after initialization should be appended""" @@ -124,10 +123,8 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", ["item", "loader"]) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), {"name": ["foo", "item", "loader"]} - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "item", "loader"]} def test_add_value_list_singlevalue(self): """Values added after initialization should be appended""" @@ -135,10 +132,8 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", "qwerty") loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar", "qwerty"]} - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar", "qwerty"]} def test_add_value_list_list(self): """Values added after initialization should be appended""" @@ -146,56 +141,55 @@ class InitializationTestMixin: il = ItemLoader(item=input_item) il.add_value("name", ["item", "loader"]) loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual( - ItemAdapter(loaded_item).asdict(), - {"name": ["foo", "bar", "item", "loader"]}, - ) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == { + "name": ["foo", "bar", "item", "loader"] + } def test_get_output_value_singlevalue(self): """Getting output value must not remove value from item""" input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo"]) + assert il.get_output_value("name") == ["foo"] loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo"]} def test_get_output_value_list(self): """Getting output value must not remove value from item""" input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - self.assertEqual(il.get_output_value("name"), ["foo", "bar"]) + assert il.get_output_value("name") == ["foo", "bar"] loaded_item = il.load_item() - self.assertIsInstance(loaded_item, self.item_class) - self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]}) + assert isinstance(loaded_item, self.item_class) + assert ItemAdapter(loaded_item).asdict() == {"name": ["foo", "bar"]} def test_values_single(self): """Values from initial item must be added to loader._values""" input_item = self.item_class(name="foo") il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo"]) + assert il._values.get("name") == ["foo"] def test_values_list(self): """Values from initial item must be added to loader._values""" input_item = self.item_class(name=["foo", "bar"]) il = ItemLoader(item=input_item) - self.assertEqual(il._values.get("name"), ["foo", "bar"]) + assert il._values.get("name") == ["foo", "bar"] -class InitializationFromDictTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromDict(InitializationTestMixin): item_class = dict -class InitializationFromItemTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromItem(InitializationTestMixin): item_class = NameItem -class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromAttrsItem(InitializationTestMixin): item_class = AttrsNameItem -class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase): +class TestInitializationFromDataClass(InitializationTestMixin): item_class = NameDataClass @@ -212,7 +206,7 @@ class NoInputReprocessingItemLoader(BaseNoInputReprocessingLoader): default_item_class = NoInputReprocessingItem -class NoInputReprocessingFromItemTest(unittest.TestCase): +class TestNoInputReprocessingFromItem: """ Loaders initialized from loaded items must not reprocess fields (Item instances) """ @@ -220,41 +214,41 @@ class NoInputReprocessingFromItemTest(unittest.TestCase): def test_avoid_reprocessing_with_initial_values_single(self): il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title="foo")) il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"} - ) + assert il_loaded == {"title": "foo"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "foo" + } def test_avoid_reprocessing_with_initial_values_list(self): il = NoInputReprocessingItemLoader( item=NoInputReprocessingItem(title=["foo", "bar"]) ) il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "foo"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"} - ) + assert il_loaded == {"title": "foo"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "foo" + } def test_avoid_reprocessing_without_initial_values_single(self): il = NoInputReprocessingItemLoader() il.add_value("title", "FOO") il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) + assert il_loaded == {"title": "FOO"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "FOO" + } def test_avoid_reprocessing_without_initial_values_list(self): il = NoInputReprocessingItemLoader() il.add_value("title", ["foo", "bar"]) il_loaded = il.load_item() - self.assertEqual(il_loaded, {"title": "FOO"}) - self.assertEqual( - NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"} - ) + assert il_loaded == {"title": "FOO"} + assert NoInputReprocessingItemLoader(item=il_loaded).load_item() == { + "title": "FOO" + } -class TestOutputProcessorItem(unittest.TestCase): +class TestOutputProcessorItem: def test_output_processor(self): class TempItem(Item): temp = Field() @@ -270,11 +264,11 @@ class TestOutputProcessorItem(unittest.TestCase): loader = TempLoader() item = loader.load_item() - self.assertIsInstance(item, TempItem) - self.assertEqual(dict(item), {"temp": 0.3}) + assert isinstance(item, TempItem) + assert dict(item) == {"temp": 0.3} -class SelectortemLoaderTest(unittest.TestCase): +class TestSelectortemLoader: response = HtmlResponse( url="", encoding="utf-8", @@ -292,7 +286,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method(self): l = ProcessorItemLoader() - self.assertEqual(l.selector, None) + assert l.selector is None def test_init_method_errors(self): l = ProcessorItemLoader() @@ -312,150 +306,149 @@ class SelectortemLoaderTest(unittest.TestCase): def test_init_method_with_selector(self): sel = Selector(text="
marta
") l = ProcessorItemLoader(selector=sel) - self.assertIs(l.selector, sel) + assert l.selector is sel l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] def test_init_method_with_selector_css(self): sel = Selector(text="
marta
") l = ProcessorItemLoader(selector=sel) - self.assertIs(l.selector, sel) + assert l.selector is sel l.add_css("name", "div::text") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] def test_init_method_with_base_response(self): """Selector should be None after initialization""" response = Response("https://scrapy.org") l = ProcessorItemLoader(response=response) - self.assertIs(l.selector, None) + assert l.selector is None def test_init_method_with_response(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] def test_init_method_with_response_css(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_css("name", "div::text") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.add_css("url", "a::attr(href)") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] # combining/accumulating CSS selectors and XPath expressions l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta", "Marta"]) + assert l.get_output_value("name") == ["Marta", "Marta"] l.add_xpath("url", "//img/@src") - self.assertEqual( - l.get_output_value("url"), ["http://www.scrapy.org", "/images/logo.png"] - ) + assert l.get_output_value("url") == [ + "http://www.scrapy.org", + "/images/logo.png", + ] def test_add_xpath_re(self): l = ProcessorItemLoader(response=self.response) l.add_xpath("name", "//div/text()", re="ma") - self.assertEqual(l.get_output_value("name"), ["Ma"]) + assert l.get_output_value("name") == ["Ma"] def test_replace_xpath(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_xpath("name", "//p/text()") - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] l.replace_xpath("name", ["//p/text()", "//div/text()"]) - self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) + assert l.get_output_value("name") == ["Paragraph", "Marta"] def test_get_xpath(self): l = ProcessorItemLoader(response=self.response) - self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"]) - self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph") - self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa") + assert l.get_xpath("//p/text()") == ["paragraph"] + assert l.get_xpath("//p/text()", TakeFirst()) == "paragraph" + assert l.get_xpath("//p/text()", TakeFirst(), re="pa") == "pa" - self.assertEqual( - l.get_xpath(["//p/text()", "//div/text()"]), ["paragraph", "marta"] - ) + assert l.get_xpath(["//p/text()", "//div/text()"]) == ["paragraph", "marta"] def test_replace_xpath_multi_fields(self): l = ProcessorItemLoader(response=self.response) l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] def test_replace_xpath_re(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_xpath("name", "//div/text()") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_xpath("name", "//div/text()", re="ma") - self.assertEqual(l.get_output_value("name"), ["Ma"]) + assert l.get_output_value("name") == ["Ma"] def test_add_css_re(self): l = ProcessorItemLoader(response=self.response) l.add_css("name", "div::text", re="ma") - self.assertEqual(l.get_output_value("name"), ["Ma"]) + assert l.get_output_value("name") == ["Ma"] l.add_css("url", "a::attr(href)", re="http://(.+)") - self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) + assert l.get_output_value("url") == ["www.scrapy.org"] def test_replace_css(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_css("name", "div::text") - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_css("name", "p::text") - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] l.replace_css("name", ["p::text", "div::text"]) - self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"]) + assert l.get_output_value("name") == ["Paragraph", "Marta"] l.add_css("url", "a::attr(href)", re="http://(.+)") - self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"]) + assert l.get_output_value("url") == ["www.scrapy.org"] l.replace_css("url", "img::attr(src)") - self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + assert l.get_output_value("url") == ["/images/logo.png"] def test_get_css(self): l = ProcessorItemLoader(response=self.response) - self.assertEqual(l.get_css("p::text"), ["paragraph"]) - self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph") - self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa") + assert l.get_css("p::text") == ["paragraph"] + assert l.get_css("p::text", TakeFirst()) == "paragraph" + assert l.get_css("p::text", TakeFirst(), re="pa") == "pa" - self.assertEqual(l.get_css(["p::text", "div::text"]), ["paragraph", "marta"]) - self.assertEqual( - l.get_css(["a::attr(href)", "img::attr(src)"]), - ["http://www.scrapy.org", "/images/logo.png"], - ) + assert l.get_css(["p::text", "div::text"]) == ["paragraph", "marta"] + assert l.get_css(["a::attr(href)", "img::attr(src)"]) == [ + "http://www.scrapy.org", + "/images/logo.png", + ] def test_replace_css_multi_fields(self): l = ProcessorItemLoader(response=self.response) l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Marta"]) + assert l.get_output_value("name") == ["Marta"] l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x}) - self.assertEqual(l.get_output_value("name"), ["Paragraph"]) + assert l.get_output_value("name") == ["Paragraph"] l.add_css(None, "a::attr(href)", TakeFirst(), lambda x: {"url": x}) - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] l.replace_css(None, "img::attr(src)", TakeFirst(), lambda x: {"url": x}) - self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + assert l.get_output_value("url") == ["/images/logo.png"] def test_replace_css_re(self): l = ProcessorItemLoader(response=self.response) - self.assertTrue(l.selector) + assert l.selector l.add_css("url", "a::attr(href)") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] l.replace_css("url", "a::attr(href)", re=r"http://www\.(.+)") - self.assertEqual(l.get_output_value("url"), ["scrapy.org"]) + assert l.get_output_value("url") == ["scrapy.org"] -class SubselectorLoaderTest(unittest.TestCase): +class TestSubselectorLoader: response = HtmlResponse( url="", encoding="utf-8", @@ -483,17 +476,13 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css("name_div", "#id") nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value("name"), ["marta"]) - self.assertEqual(l.get_output_value("name_div"), ['
marta
']) - self.assertEqual(l.get_output_value("name_value"), ["marta"]) + assert l.get_output_value("name") == ["marta"] + assert l.get_output_value("name_div") == ['
marta
'] + assert l.get_output_value("name_value") == ["marta"] - self.assertEqual(l.get_output_value("name"), nl.get_output_value("name")) - self.assertEqual( - l.get_output_value("name_div"), nl.get_output_value("name_div") - ) - self.assertEqual( - l.get_output_value("name_value"), nl.get_output_value("name_value") - ) + assert l.get_output_value("name") == nl.get_output_value("name") + assert l.get_output_value("name_div") == nl.get_output_value("name_div") + assert l.get_output_value("name_value") == nl.get_output_value("name_value") def test_nested_css(self): l = NestedItemLoader(response=self.response) @@ -502,17 +491,13 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css("name_div", "#id") nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value("name"), ["marta"]) - self.assertEqual(l.get_output_value("name_div"), ['
marta
']) - self.assertEqual(l.get_output_value("name_value"), ["marta"]) + assert l.get_output_value("name") == ["marta"] + assert l.get_output_value("name_div") == ['
marta
'] + assert l.get_output_value("name_value") == ["marta"] - self.assertEqual(l.get_output_value("name"), nl.get_output_value("name")) - self.assertEqual( - l.get_output_value("name_div"), nl.get_output_value("name_div") - ) - self.assertEqual( - l.get_output_value("name_value"), nl.get_output_value("name_value") - ) + assert l.get_output_value("name") == nl.get_output_value("name") + assert l.get_output_value("name_div") == nl.get_output_value("name_div") + assert l.get_output_value("name_value") == nl.get_output_value("name_value") def test_nested_replace(self): l = NestedItemLoader(response=self.response) @@ -520,11 +505,11 @@ class SubselectorLoaderTest(unittest.TestCase): nl2 = nl1.nested_xpath("a") l.add_xpath("url", "//footer/a/@href") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] nl1.replace_xpath("url", "img/@src") - self.assertEqual(l.get_output_value("url"), ["/images/logo.png"]) + assert l.get_output_value("url") == ["/images/logo.png"] nl2.replace_xpath("url", "@href") - self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"]) + assert l.get_output_value("url") == ["http://www.scrapy.org"] def test_nested_ordering(self): l = NestedItemLoader(response=self.response) @@ -536,15 +521,12 @@ class SubselectorLoaderTest(unittest.TestCase): nl2.add_xpath("url", "text()") l.add_xpath("url", "//footer/a/@href") - self.assertEqual( - l.get_output_value("url"), - [ - "/images/logo.png", - "http://www.scrapy.org", - "homepage", - "http://www.scrapy.org", - ], - ) + assert l.get_output_value("url") == [ + "/images/logo.png", + "http://www.scrapy.org", + "homepage", + "http://www.scrapy.org", + ] def test_nested_load_item(self): l = NestedItemLoader(response=self.response) @@ -561,9 +543,9 @@ class SubselectorLoaderTest(unittest.TestCase): assert item is nl1.item assert item is nl2.item - self.assertEqual(item["name"], ["marta"]) - self.assertEqual(item["url"], ["http://www.scrapy.org"]) - self.assertEqual(item["image"], ["/images/logo.png"]) + assert item["name"] == ["marta"] + assert item["url"] == ["http://www.scrapy.org"] + assert item["image"] == ["/images/logo.png"] # Functions as processors @@ -588,9 +570,9 @@ class FunctionProcessorItemLoader(ItemLoader): default_item_class = FunctionProcessorItem -class FunctionProcessorTestCase(unittest.TestCase): +class TestFunctionProcessor: def test_processor_defined_in_item(self): lo = FunctionProcessorItemLoader() lo.add_value("foo", " bar ") lo.add_value("foo", [" asdf ", " qwerty "]) - self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]}) + assert dict(lo.load_item()) == {"foo": ["BAR", "ASDF", "QWERTY"]} diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 909b365a9..d7d900546 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,4 +1,6 @@ -import unittest +# pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison +# (too many false positives) + from unittest import mock import pytest @@ -14,31 +16,31 @@ from scrapy.settings import ( from . import default_settings -class SettingsGlobalFuncsTest(unittest.TestCase): +class TestSettingsGlobalFuncs: def test_get_settings_priority(self): for prio_str, prio_num in SETTINGS_PRIORITIES.items(): - self.assertEqual(get_settings_priority(prio_str), prio_num) - self.assertEqual(get_settings_priority(99), 99) + assert get_settings_priority(prio_str) == prio_num + assert get_settings_priority(99) == 99 -class SettingsAttributeTest(unittest.TestCase): - def setUp(self): +class TestSettingsAttribute: + def setup_method(self): self.attribute = SettingsAttribute("value", 10) def test_set_greater_priority(self): self.attribute.set("value2", 20) - self.assertEqual(self.attribute.value, "value2") - self.assertEqual(self.attribute.priority, 20) + assert self.attribute.value == "value2" + assert self.attribute.priority == 20 def test_set_equal_priority(self): self.attribute.set("value2", 10) - self.assertEqual(self.attribute.value, "value2") - self.assertEqual(self.attribute.priority, 10) + assert self.attribute.value == "value2" + assert self.attribute.priority == 10 def test_set_less_priority(self): self.attribute.set("value2", 0) - self.assertEqual(self.attribute.value, "value") - self.assertEqual(self.attribute.priority, 10) + assert self.attribute.value == "value" + assert self.attribute.priority == 10 def test_overwrite_basesettings(self): original_dict = {"one": 10, "two": 20} @@ -47,61 +49,59 @@ class SettingsAttributeTest(unittest.TestCase): new_dict = {"three": 11, "four": 21} attribute.set(new_dict, 10) - self.assertIsInstance(attribute.value, BaseSettings) - self.assertCountEqual(attribute.value, new_dict) - self.assertCountEqual(original_settings, original_dict) + assert isinstance(attribute.value, BaseSettings) + assert set(attribute.value) == set(new_dict) + assert set(original_settings) == set(original_dict) new_settings = BaseSettings({"five": 12}, 0) attribute.set(new_settings, 0) # Insufficient priority - self.assertCountEqual(attribute.value, new_dict) + assert set(attribute.value) == set(new_dict) attribute.set(new_settings, 10) - self.assertCountEqual(attribute.value, new_settings) + assert set(attribute.value) == set(new_settings) def test_repr(self): - self.assertEqual( - repr(self.attribute), "" - ) + assert repr(self.attribute) == "" -class BaseSettingsTest(unittest.TestCase): - def setUp(self): +class TestBaseSettings: + def setup_method(self): self.settings = BaseSettings() def test_setdefault_not_existing_value(self): settings = BaseSettings() value = settings.setdefault("TEST_OPTION", "value") - self.assertEqual(settings["TEST_OPTION"], "value") - self.assertEqual(value, "value") - self.assertIsNotNone(value) + assert settings["TEST_OPTION"] == "value" + assert value == "value" + assert value is not None def test_setdefault_existing_value(self): settings = BaseSettings({"TEST_OPTION": "value"}) value = settings.setdefault("TEST_OPTION", None) - self.assertEqual(settings["TEST_OPTION"], "value") - self.assertEqual(value, "value") + assert settings["TEST_OPTION"] == "value" + assert value == "value" def test_set_new_attribute(self): self.settings.set("TEST_OPTION", "value", 0) - self.assertIn("TEST_OPTION", self.settings.attributes) + assert "TEST_OPTION" in self.settings.attributes attr = self.settings.attributes["TEST_OPTION"] - self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, "value") - self.assertEqual(attr.priority, 0) + assert isinstance(attr, SettingsAttribute) + assert attr.value == "value" + assert attr.priority == 0 def test_set_settingsattribute(self): myattr = SettingsAttribute(0, 30) # Note priority 30 self.settings.set("TEST_ATTR", myattr, 10) - self.assertEqual(self.settings.get("TEST_ATTR"), 0) - self.assertEqual(self.settings.getpriority("TEST_ATTR"), 30) + assert self.settings.get("TEST_ATTR") == 0 + assert self.settings.getpriority("TEST_ATTR") == 30 def test_set_instance_identity_on_update(self): attr = SettingsAttribute("value", 0) self.settings.attributes = {"TEST_OPTION": attr} self.settings.set("TEST_OPTION", "othervalue", 10) - self.assertIn("TEST_OPTION", self.settings.attributes) - self.assertIs(attr, self.settings.attributes["TEST_OPTION"]) + assert "TEST_OPTION" in self.settings.attributes + assert attr is self.settings.attributes["TEST_OPTION"] def test_set_calls_settings_attributes_methods_on_update(self): attr = SettingsAttribute("value", 10) @@ -114,7 +114,7 @@ class BaseSettingsTest(unittest.TestCase): for priority in (0, 10, 20): self.settings.set("TEST_OPTION", "othervalue", priority) mock_set.assert_called_once_with("othervalue", priority) - self.assertFalse(mock_setattr.called) + assert not mock_setattr.called mock_set.reset_mock() mock_setattr.reset_mock() @@ -122,19 +122,19 @@ class BaseSettingsTest(unittest.TestCase): settings = BaseSettings() settings.set("key", "a", "default") settings["key"] = "b" - self.assertEqual(settings["key"], "b") - self.assertEqual(settings.getpriority("key"), 20) + assert settings["key"] == "b" + assert settings.getpriority("key") == 20 settings["key"] = "c" - self.assertEqual(settings["key"], "c") + assert settings["key"] == "c" settings["key2"] = "x" - self.assertIn("key2", settings) - self.assertEqual(settings["key2"], "x") - self.assertEqual(settings.getpriority("key2"), 20) + assert "key2" in settings + assert settings["key2"] == "x" + assert settings.getpriority("key2") == 20 def test_setdict_alias(self): with mock.patch.object(self.settings, "set") as mock_set: self.settings.setdict({"TEST_1": "value1", "TEST_2": "value2"}, 10) - self.assertEqual(mock_set.call_count, 2) + assert mock_set.call_count == 2 calls = [ mock.call("TEST_1", "value1", 10), mock.call("TEST_2", "value2", 10), @@ -149,10 +149,10 @@ class BaseSettingsTest(unittest.TestCase): self.settings.attributes = {} self.settings.setmodule(ModuleMock(), 10) - self.assertIn("UPPERCASE_VAR", self.settings.attributes) - self.assertNotIn("MIXEDcase_VAR", self.settings.attributes) - self.assertNotIn("lowercase_var", self.settings.attributes) - self.assertEqual(len(self.settings.attributes), 1) + assert "UPPERCASE_VAR" in self.settings.attributes + assert "MIXEDcase_VAR" not in self.settings.attributes + assert "lowercase_var" not in self.settings.attributes + assert len(self.settings.attributes) == 1 def test_setmodule_alias(self): with mock.patch.object(self.settings, "set") as mock_set: @@ -168,13 +168,13 @@ class BaseSettingsTest(unittest.TestCase): self.settings.attributes = {} self.settings.setmodule("tests.test_settings.default_settings", 10) - self.assertCountEqual(self.settings.attributes.keys(), ctrl_attributes.keys()) + assert set(self.settings.attributes) == set(ctrl_attributes) for key in ctrl_attributes: attr = self.settings.attributes[key] ctrl_attr = ctrl_attributes[key] - self.assertEqual(attr.value, ctrl_attr.value) - self.assertEqual(attr.priority, ctrl_attr.priority) + assert attr.value == ctrl_attr.value + assert attr.priority == ctrl_attr.priority def test_update(self): settings = BaseSettings({"key_lowprio": 0}, priority=0) @@ -186,21 +186,21 @@ class BaseSettingsTest(unittest.TestCase): custom_dict = {"key_lowprio": 2, "key_highprio": 12, "newkey_two": None} settings.update(custom_dict, priority=20) - self.assertEqual(settings["key_lowprio"], 2) - self.assertEqual(settings.getpriority("key_lowprio"), 20) - self.assertEqual(settings["key_highprio"], 10) - self.assertIn("newkey_two", settings) - self.assertEqual(settings.getpriority("newkey_two"), 20) + assert settings["key_lowprio"] == 2 + assert settings.getpriority("key_lowprio") == 20 + assert settings["key_highprio"] == 10 + assert "newkey_two" in settings + assert settings.getpriority("newkey_two") == 20 settings.update(custom_settings) - self.assertEqual(settings["key_lowprio"], 1) - self.assertEqual(settings.getpriority("key_lowprio"), 30) - self.assertEqual(settings["key_highprio"], 10) - self.assertIn("newkey_one", settings) - self.assertEqual(settings.getpriority("newkey_one"), 50) + assert settings["key_lowprio"] == 1 + assert settings.getpriority("key_lowprio") == 30 + assert settings["key_highprio"] == 10 + assert "newkey_one" in settings + assert settings.getpriority("newkey_one") == 50 settings.update({"key_lowprio": 3}, priority=20) - self.assertEqual(settings["key_lowprio"], 1) + assert settings["key_lowprio"] == 1 @pytest.mark.xfail( raises=TypeError, reason="BaseSettings.update doesn't support kwargs input" @@ -220,21 +220,21 @@ class BaseSettingsTest(unittest.TestCase): def test_update_jsonstring(self): settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})}) settings.update('{"number": 1, "newnumber": 2}') - self.assertEqual(settings["number"], 1) - self.assertEqual(settings["newnumber"], 2) + assert settings["number"] == 1 + assert settings["newnumber"] == 2 settings.set("dict", '{"key": "newval", "newkey": "newval2"}') - self.assertEqual(settings["dict"]["key"], "newval") - self.assertEqual(settings["dict"]["newkey"], "newval2") + assert settings["dict"]["key"] == "newval" + assert settings["dict"]["newkey"] == "newval2" def test_delete(self): settings = BaseSettings({"key": None}) settings.set("key_highprio", None, priority=50) settings.delete("key") settings.delete("key_highprio") - self.assertNotIn("key", settings) - self.assertIn("key_highprio", settings) + assert "key" not in settings + assert "key_highprio" in settings del settings["key_highprio"] - self.assertNotIn("key_highprio", settings) + assert "key_highprio" not in settings with pytest.raises(KeyError): settings.delete("notkey") with pytest.raises(KeyError): @@ -271,40 +271,40 @@ class BaseSettingsTest(unittest.TestCase): for key, value in test_configuration.items() } - self.assertTrue(settings.getbool("TEST_ENABLED1")) - self.assertTrue(settings.getbool("TEST_ENABLED2")) - self.assertTrue(settings.getbool("TEST_ENABLED3")) - self.assertTrue(settings.getbool("TEST_ENABLED4")) - self.assertTrue(settings.getbool("TEST_ENABLED5")) - self.assertFalse(settings.getbool("TEST_ENABLEDx")) - self.assertTrue(settings.getbool("TEST_ENABLEDx", True)) - self.assertFalse(settings.getbool("TEST_DISABLED1")) - self.assertFalse(settings.getbool("TEST_DISABLED2")) - self.assertFalse(settings.getbool("TEST_DISABLED3")) - self.assertFalse(settings.getbool("TEST_DISABLED4")) - self.assertFalse(settings.getbool("TEST_DISABLED5")) - self.assertEqual(settings.getint("TEST_INT1"), 123) - self.assertEqual(settings.getint("TEST_INT2"), 123) - self.assertEqual(settings.getint("TEST_INTx"), 0) - self.assertEqual(settings.getint("TEST_INTx", 45), 45) - self.assertEqual(settings.getfloat("TEST_FLOAT1"), 123.45) - self.assertEqual(settings.getfloat("TEST_FLOAT2"), 123.45) - self.assertEqual(settings.getfloat("TEST_FLOATx"), 0.0) - self.assertEqual(settings.getfloat("TEST_FLOATx", 55.0), 55.0) - self.assertEqual(settings.getlist("TEST_LIST1"), ["one", "two"]) - self.assertEqual(settings.getlist("TEST_LIST2"), ["one", "two"]) - self.assertEqual(settings.getlist("TEST_LIST3"), []) - self.assertEqual(settings.getlist("TEST_LISTx"), []) - self.assertEqual(settings.getlist("TEST_LISTx", ["default"]), ["default"]) - self.assertEqual(settings["TEST_STR"], "value") - self.assertEqual(settings.get("TEST_STR"), "value") - self.assertEqual(settings["TEST_STRx"], None) - self.assertEqual(settings.get("TEST_STRx"), None) - self.assertEqual(settings.get("TEST_STRx", "default"), "default") - self.assertEqual(settings.getdict("TEST_DICT1"), {"key1": "val1", "ke2": 3}) - self.assertEqual(settings.getdict("TEST_DICT2"), {"key1": "val1", "ke2": 3}) - self.assertEqual(settings.getdict("TEST_DICT3"), {}) - self.assertEqual(settings.getdict("TEST_DICT3", {"key1": 5}), {"key1": 5}) + assert settings.getbool("TEST_ENABLED1") + assert settings.getbool("TEST_ENABLED2") + assert settings.getbool("TEST_ENABLED3") + assert settings.getbool("TEST_ENABLED4") + assert settings.getbool("TEST_ENABLED5") + assert not settings.getbool("TEST_ENABLEDx") + assert settings.getbool("TEST_ENABLEDx", True) + assert not settings.getbool("TEST_DISABLED1") + assert not settings.getbool("TEST_DISABLED2") + assert not settings.getbool("TEST_DISABLED3") + assert not settings.getbool("TEST_DISABLED4") + assert not settings.getbool("TEST_DISABLED5") + assert settings.getint("TEST_INT1") == 123 + assert settings.getint("TEST_INT2") == 123 + assert settings.getint("TEST_INTx") == 0 + assert settings.getint("TEST_INTx", 45) == 45 + assert settings.getfloat("TEST_FLOAT1") == 123.45 + assert settings.getfloat("TEST_FLOAT2") == 123.45 + assert settings.getfloat("TEST_FLOATx") == 0.0 + assert settings.getfloat("TEST_FLOATx", 55.0) == 55.0 + assert settings.getlist("TEST_LIST1") == ["one", "two"] + assert settings.getlist("TEST_LIST2") == ["one", "two"] + assert settings.getlist("TEST_LIST3") == [] + assert settings.getlist("TEST_LISTx") == [] + assert settings.getlist("TEST_LISTx", ["default"]) == ["default"] + assert settings["TEST_STR"] == "value" + assert settings.get("TEST_STR") == "value" + assert settings["TEST_STRx"] is None + assert settings.get("TEST_STRx") is None + assert settings.get("TEST_STRx", "default") == "default" + assert settings.getdict("TEST_DICT1") == {"key1": "val1", "ke2": 3} + assert settings.getdict("TEST_DICT2") == {"key1": "val1", "ke2": 3} + assert settings.getdict("TEST_DICT3") == {} + assert settings.getdict("TEST_DICT3", {"key1": 5}) == {"key1": 5} with pytest.raises( ValueError, match="dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected", @@ -321,8 +321,8 @@ class BaseSettingsTest(unittest.TestCase): def test_getpriority(self): settings = BaseSettings({"key": "value"}, priority=99) - self.assertEqual(settings.getpriority("key"), 99) - self.assertEqual(settings.getpriority("nonexistentkey"), None) + assert settings.getpriority("key") == 99 + assert settings.getpriority("nonexistentkey") is None def test_getwithbase(self): s = BaseSettings( @@ -333,16 +333,16 @@ class BaseSettingsTest(unittest.TestCase): } ) s["TEST"].set(2, 200, "cmdline") - self.assertCountEqual(s.getwithbase("TEST"), {1: 1, 2: 200, 3: 30}) - self.assertCountEqual(s.getwithbase("HASNOBASE"), s["HASNOBASE"]) - self.assertEqual(s.getwithbase("NONEXISTENT"), {}) + assert set(s.getwithbase("TEST")) == {1, 2, 3} + assert set(s.getwithbase("HASNOBASE")) == set(s["HASNOBASE"]) + assert s.getwithbase("NONEXISTENT") == {} def test_maxpriority(self): # Empty settings should return 'default' - self.assertEqual(self.settings.maxpriority(), 0) + assert self.settings.maxpriority() == 0 self.settings.set("A", 0, 10) self.settings.set("B", 0, 30) - self.assertEqual(self.settings.maxpriority(), 30) + assert self.settings.maxpriority() == 30 def test_copy(self): values = { @@ -356,17 +356,15 @@ class BaseSettingsTest(unittest.TestCase): self.settings.setdict(values) copy = self.settings.copy() self.settings.set("TEST_BOOL", False) - self.assertTrue(copy.get("TEST_BOOL")) + assert copy.get("TEST_BOOL") test_list = self.settings.get("TEST_LIST") test_list.append("three") - self.assertListEqual(copy.get("TEST_LIST"), ["one", "two"]) + assert copy.get("TEST_LIST") == ["one", "two"] test_list_of_lists = self.settings.get("TEST_LIST_OF_LISTS") test_list_of_lists[0].append("first_three") - self.assertListEqual( - copy.get("TEST_LIST_OF_LISTS")[0], ["first_one", "first_two"] - ) + assert copy.get("TEST_LIST_OF_LISTS")[0] == ["first_one", "first_two"] def test_copy_to_dict(self): s = BaseSettings( @@ -379,17 +377,14 @@ class BaseSettingsTest(unittest.TestCase): "HASNOBASE": BaseSettings({3: 3000}, "default"), } ) - self.assertDictEqual( - s.copy_to_dict(), - { - "HASNOBASE": {3: 3000}, - "TEST": {1: 10, 3: 30}, - "TEST_BASE": {1: 1, 2: 2}, - "TEST_LIST": [1, 2], - "TEST_BOOLEAN": False, - "TEST_STRING": "a string", - }, - ) + assert s.copy_to_dict() == { + "HASNOBASE": {3: 3000}, + "TEST": {1: 10, 3: 30}, + "TEST_BASE": {1: 1, 2: 2}, + "TEST_LIST": [1, 2], + "TEST_BOOLEAN": False, + "TEST_STRING": "a string", + } def test_freeze(self): self.settings.freeze() @@ -400,55 +395,55 @@ class BaseSettingsTest(unittest.TestCase): def test_frozencopy(self): frozencopy = self.settings.frozencopy() - self.assertTrue(frozencopy.frozen) - self.assertIsNot(frozencopy, self.settings) + assert frozencopy.frozen + assert frozencopy is not self.settings -class SettingsTest(unittest.TestCase): - def setUp(self): +class TestSettings: + def setup_method(self): self.settings = Settings() @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {"default": 10}) @mock.patch("scrapy.settings.default_settings", default_settings) def test_initial_defaults(self): settings = Settings() - self.assertEqual(len(settings.attributes), 2) - self.assertIn("TEST_DEFAULT", settings.attributes) + assert len(settings.attributes) == 2 + assert "TEST_DEFAULT" in settings.attributes attr = settings.attributes["TEST_DEFAULT"] - self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, "defvalue") - self.assertEqual(attr.priority, 10) + assert isinstance(attr, SettingsAttribute) + assert attr.value == "defvalue" + assert attr.priority == 10 @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {}) @mock.patch("scrapy.settings.default_settings", {}) def test_initial_values(self): settings = Settings({"TEST_OPTION": "value"}, 10) - self.assertEqual(len(settings.attributes), 1) - self.assertIn("TEST_OPTION", settings.attributes) + assert len(settings.attributes) == 1 + assert "TEST_OPTION" in settings.attributes attr = settings.attributes["TEST_OPTION"] - self.assertIsInstance(attr, SettingsAttribute) - self.assertEqual(attr.value, "value") - self.assertEqual(attr.priority, 10) + assert isinstance(attr, SettingsAttribute) + assert attr.value == "value" + assert attr.priority == 10 @mock.patch("scrapy.settings.default_settings", default_settings) def test_autopromote_dicts(self): settings = Settings() mydict = settings.get("TEST_DICT") - self.assertIsInstance(mydict, BaseSettings) - self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") # pylint: disable=unsubscriptable-object - self.assertEqual(mydict.getpriority("key"), 0) + assert isinstance(mydict, BaseSettings) + assert "key" in mydict + assert mydict["key"] == "val" + assert mydict.getpriority("key") == 0 @mock.patch("scrapy.settings.default_settings", default_settings) def test_getdict_autodegrade_basesettings(self): settings = Settings() mydict = settings.getdict("TEST_DICT") - self.assertIsInstance(mydict, dict) - self.assertEqual(len(mydict), 1) - self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") + assert isinstance(mydict, dict) + assert len(mydict) == 1 + assert "key" in mydict + assert mydict["key"] == "val" def test_passing_objects_as_values(self): from scrapy.core.downloader.handlers.file import FileDownloadHandler @@ -470,19 +465,19 @@ class SettingsTest(unittest.TestCase): } ) - self.assertIn("ITEM_PIPELINES", settings.attributes) + assert "ITEM_PIPELINES" in settings.attributes mypipeline, priority = settings.getdict("ITEM_PIPELINES").popitem() - self.assertEqual(priority, 800) - self.assertEqual(mypipeline, TestPipeline) - self.assertIsInstance(mypipeline(), TestPipeline) - self.assertEqual(mypipeline().process_item("item", None), "item") + assert priority == 800 + assert mypipeline == TestPipeline + assert isinstance(mypipeline(), TestPipeline) + assert mypipeline().process_item("item", None) == "item" myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp") - self.assertEqual(myhandler, FileDownloadHandler) + assert myhandler == FileDownloadHandler myhandler_instance = build_from_crawler(myhandler, get_crawler()) - self.assertIsInstance(myhandler_instance, FileDownloadHandler) - self.assertTrue(hasattr(myhandler_instance, "download_request")) + assert isinstance(myhandler_instance, FileDownloadHandler) + assert hasattr(myhandler_instance, "download_request") def test_pop_item_with_default_value(self): settings = Settings() @@ -491,14 +486,14 @@ class SettingsTest(unittest.TestCase): settings.pop("DUMMY_CONFIG") dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value") - self.assertEqual(dummy_config_value, "dummy_value") + assert dummy_config_value == "dummy_value" def test_pop_item_with_immutable_settings(self): settings = Settings( {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} ) - self.assertEqual(settings.pop("DUMMY_CONFIG"), "dummy_value") + assert settings.pop("DUMMY_CONFIG") == "dummy_value" settings.freeze() From 7bbe775040d5d695bc3b48a73de5d6fa99312b4c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 9 Mar 2025 23:24:45 +0400 Subject: [PATCH 1682/2083] Converting tests to plain asserts, part 5. (#6712) --- tests/test_logformatter.py | 93 +++++++-------- tests/test_logstats.py | 45 ++++---- tests/test_mail.py | 71 ++++++------ tests/test_middleware.py | 30 ++--- tests/test_pipelines.py | 16 +-- tests/test_pqueues.py | 113 +++++++++--------- tests/test_proxy_connect.py | 11 +- tests/test_request_attribute_binding.py | 26 ++--- tests/test_request_cb_kwargs.py | 30 ++--- tests/test_request_dict.py | 50 ++++---- tests/test_request_left.py | 8 +- tests/test_responsetypes.py | 8 +- tests/test_robotstxt_interface.py | 106 +++++++---------- tests/test_scheduler.py | 119 +++++++++---------- tests/test_scheduler_base.py | 59 +++++----- tests/test_selector.py | 145 +++++++++++------------- tests/test_signals.py | 6 +- tests/test_toplevel.py | 20 ++-- tests/test_urlparse_monkeypatches.py | 11 +- 19 files changed, 446 insertions(+), 521 deletions(-) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 962692a31..3c9f97631 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -1,11 +1,10 @@ import logging -import unittest import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.python.failure import Failure -from twisted.trial.unittest import TestCase as TwistedTestCase +from twisted.trial.unittest import TestCase from scrapy.exceptions import DropItem from scrapy.http import Request, Response @@ -24,8 +23,8 @@ class CustomItem(Item): return f"name: {self['name']}" -class LogFormatterTestCase(unittest.TestCase): - def setUp(self): +class TestLogFormatter: + def setup_method(self): self.formatter = LogFormatter() self.spider = Spider("default") self.spider.crawler = get_crawler() @@ -35,9 +34,7 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, "Crawled (200) (referer: None)" - ) + assert logline == "Crawled (200) (referer: None)" def test_crawled_without_referer(self): req = Request( @@ -46,9 +43,9 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com", flags=["cached"]) logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Crawled (200) (referer: http://example.com) ['cached']", + assert ( + logline + == "Crawled (200) (referer: http://example.com) ['cached']" ) def test_flags_in_request(self): @@ -56,9 +53,9 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Crawled (200) ['test', 'flag'] (referer: None)", + assert ( + logline + == "Crawled (200) ['test', 'flag'] (referer: None)" ) def test_dropped(self): @@ -69,7 +66,7 @@ class LogFormatterTestCase(unittest.TestCase): logline = logkws["msg"] % logkws["args"] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, ["Dropped: \u2018", "{}"]) + assert lines == ["Dropped: \u2018", "{}"] def test_dropitem_default_log_level(self): item = {} @@ -79,38 +76,38 @@ class LogFormatterTestCase(unittest.TestCase): spider.crawler = get_crawler(Spider) logkws = self.formatter.dropped(item, exception, response, spider) - self.assertEqual(logkws["level"], logging.WARNING) + assert logkws["level"] == logging.WARNING spider.crawler.settings.frozen = False spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = logging.INFO spider.crawler.settings.frozen = True logkws = self.formatter.dropped(item, exception, response, spider) - self.assertEqual(logkws["level"], logging.INFO) + assert logkws["level"] == logging.INFO spider.crawler.settings.frozen = False spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = "INFO" spider.crawler.settings.frozen = True logkws = self.formatter.dropped(item, exception, response, spider) - self.assertEqual(logkws["level"], logging.INFO) + assert logkws["level"] == logging.INFO spider.crawler.settings.frozen = False spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 10 spider.crawler.settings.frozen = True logkws = self.formatter.dropped(item, exception, response, spider) - self.assertEqual(logkws["level"], logging.DEBUG) + assert logkws["level"] == logging.DEBUG spider.crawler.settings.frozen = False spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = 0 spider.crawler.settings.frozen = True logkws = self.formatter.dropped(item, exception, response, spider) - self.assertEqual(logkws["level"], logging.NOTSET) + assert logkws["level"] == logging.NOTSET unsupported_value = object() spider.crawler.settings.frozen = False spider.crawler.settings["DEFAULT_DROPITEM_LOG_LEVEL"] = unsupported_value spider.crawler.settings.frozen = True logkws = self.formatter.dropped(item, exception, response, spider) - self.assertEqual(logkws["level"], unsupported_value) + assert logkws["level"] == unsupported_value with pytest.raises(TypeError): logging.log(logkws["level"], "message") @@ -121,11 +118,11 @@ class LogFormatterTestCase(unittest.TestCase): exception = DropItem("Test drop", log_level="INFO") logkws = self.formatter.dropped(item, exception, response, self.spider) - self.assertEqual(logkws["level"], logging.INFO) + assert logkws["level"] == logging.INFO exception = DropItem("Test drop", log_level="ERROR") logkws = self.formatter.dropped(item, exception, response, self.spider) - self.assertEqual(logkws["level"], logging.ERROR) + assert logkws["level"] == logging.ERROR def test_item_error(self): # In practice, the complete traceback is shown by passing the @@ -135,7 +132,7 @@ class LogFormatterTestCase(unittest.TestCase): response = Response("http://www.example.com") logkws = self.formatter.item_error(item, exception, response, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual(logline, "Error processing {'key': 'value'}") + assert logline == "Error processing {'key': 'value'}" def test_spider_error(self): # In practice, the complete traceback is shown by passing the @@ -147,9 +144,9 @@ class LogFormatterTestCase(unittest.TestCase): response = Response("http://www.example.com", request=request) logkws = self.formatter.spider_error(failure, request, response, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Spider error processing (referer: http://example.org)", + assert ( + logline + == "Spider error processing (referer: http://example.org)" ) def test_download_error_short(self): @@ -159,7 +156,7 @@ class LogFormatterTestCase(unittest.TestCase): request = Request("http://www.example.com") logkws = self.formatter.download_error(failure, request, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual(logline, "Error downloading ") + assert logline == "Error downloading " def test_download_error_long(self): # In practice, the complete traceback is shown by passing the @@ -170,9 +167,7 @@ class LogFormatterTestCase(unittest.TestCase): failure, request, self.spider, "Some message" ) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, "Error downloading : Some message" - ) + assert logline == "Error downloading : Some message" def test_scraped(self): item = CustomItem() @@ -182,9 +177,7 @@ class LogFormatterTestCase(unittest.TestCase): logline = logkws["msg"] % logkws["args"] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual( - lines, ["Scraped from <200 http://www.example.com>", "name: \xa3"] - ) + assert lines == ["Scraped from <200 http://www.example.com>", "name: \xa3"] class LogFormatterSubclass(LogFormatter): @@ -200,8 +193,8 @@ class LogFormatterSubclass(LogFormatter): } -class LogformatterSubclassTest(LogFormatterTestCase): - def setUp(self): +class TestLogformatterSubclass(TestLogFormatter): + def setup_method(self): self.formatter = LogFormatterSubclass() self.spider = Spider("default") self.spider.crawler = get_crawler(Spider) @@ -211,8 +204,8 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, "Crawled (200) (referer: None) []" + assert ( + logline == "Crawled (200) (referer: None) []" ) def test_crawled_without_referer(self): @@ -224,9 +217,9 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Crawled (200) (referer: http://example.com) ['cached']", + assert ( + logline + == "Crawled (200) (referer: http://example.com) ['cached']" ) def test_flags_in_request(self): @@ -234,9 +227,9 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] - self.assertEqual( - logline, - "Crawled (200) (referer: None) ['test', 'flag']", + assert ( + logline + == "Crawled (200) (referer: None) ['test', 'flag']" ) @@ -261,7 +254,7 @@ class DropSomeItemsPipeline: self.drop = True -class ShowOrSkipMessagesTestCase(TwistedTestCase): +class TestShowOrSkipMessages(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -284,9 +277,9 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): crawler = get_crawler(ItemSpider, self.base_settings) with LogCapture() as lc: yield crawler.crawl(mockserver=self.mockserver) - self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc)) - self.assertIn("Crawled (200) body

") - self.assertEqual(msg.get("Content-Type"), "text/html") + assert msg.get_payload() == "

body

" + assert msg.get("Content-Type") == "text/html" def test_send_attach(self): attach = BytesIO() @@ -70,22 +69,22 @@ class MailSenderTest(unittest.TestCase): ) assert self.catched_msg - self.assertEqual(self.catched_msg["to"], ["test@scrapy.org"]) - self.assertEqual(self.catched_msg["subject"], "subject") - self.assertEqual(self.catched_msg["body"], "body") + assert self.catched_msg["to"] == ["test@scrapy.org"] + assert self.catched_msg["subject"] == "subject" + assert self.catched_msg["body"] == "body" msg = self.catched_msg["msg"] - self.assertEqual(msg["to"], "test@scrapy.org") - self.assertEqual(msg["subject"], "subject") + assert msg["to"] == "test@scrapy.org" + assert msg["subject"] == "subject" payload = msg.get_payload() assert isinstance(payload, list) - self.assertEqual(len(payload), 2) + assert len(payload) == 2 text, attach = payload - self.assertEqual(text.get_payload(decode=True), b"body") - self.assertEqual(text.get_charset(), Charset("us-ascii")) - self.assertEqual(attach.get_payload(decode=True), b"content") + assert text.get_payload(decode=True) == b"body" + assert text.get_charset() == Charset("us-ascii") + assert attach.get_payload(decode=True) == b"content" def _catch_mail_sent(self, **kwargs): self.catched_msg = {**kwargs} @@ -103,14 +102,14 @@ class MailSenderTest(unittest.TestCase): ) assert self.catched_msg - self.assertEqual(self.catched_msg["subject"], subject) - self.assertEqual(self.catched_msg["body"], body) + assert self.catched_msg["subject"] == subject + assert self.catched_msg["body"] == body msg = self.catched_msg["msg"] - self.assertEqual(msg["subject"], subject) - self.assertEqual(msg.get_payload(decode=True).decode("utf-8"), body) - self.assertEqual(msg.get_charset(), Charset("utf-8")) - self.assertEqual(msg.get("Content-Type"), 'text/plain; charset="utf-8"') + assert msg["subject"] == subject + assert msg.get_payload(decode=True).decode("utf-8") == body + assert msg.get_charset() == Charset("utf-8") + assert msg.get("Content-Type") == 'text/plain; charset="utf-8"' def test_send_attach_utf8(self): subject = "sübjèçt" @@ -131,22 +130,22 @@ class MailSenderTest(unittest.TestCase): ) assert self.catched_msg - self.assertEqual(self.catched_msg["subject"], subject) - self.assertEqual(self.catched_msg["body"], body) + assert self.catched_msg["subject"] == subject + assert self.catched_msg["body"] == body msg = self.catched_msg["msg"] - self.assertEqual(msg["subject"], subject) - self.assertEqual(msg.get_charset(), Charset("utf-8")) - self.assertEqual(msg.get("Content-Type"), 'multipart/mixed; charset="utf-8"') + assert msg["subject"] == subject + assert msg.get_charset() == Charset("utf-8") + assert msg.get("Content-Type") == 'multipart/mixed; charset="utf-8"' payload = msg.get_payload() assert isinstance(payload, list) - self.assertEqual(len(payload), 2) + assert len(payload) == 2 text, attach = payload - self.assertEqual(text.get_payload(decode=True).decode("utf-8"), body) - self.assertEqual(text.get_charset(), Charset("utf-8")) - self.assertEqual(attach.get_payload(decode=True).decode("utf-8"), body) + assert text.get_payload(decode=True).decode("utf-8") == body + assert text.get_charset() == Charset("utf-8") + assert attach.get_payload(decode=True).decode("utf-8") == body def test_create_sender_factory_with_host(self): mailsender = MailSender(debug=False, smtphost="smtp.testhost.com") @@ -156,4 +155,4 @@ class MailSenderTest(unittest.TestCase): ) context = factory.buildProtocol("test@scrapy.org").context - self.assertIsInstance(context, ClientTLSOptions) + assert isinstance(context, ClientTLSOptions) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 0cc532570..d004d4d93 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -1,5 +1,3 @@ -from twisted.trial import unittest - from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager from scrapy.utils.test import get_crawler @@ -51,37 +49,27 @@ class MyMiddlewareManager(MiddlewareManager): self.methods["process"].append(mw.process) -class MiddlewareManagerTest(unittest.TestCase): +class TestMiddlewareManager: def test_init(self): m1, m2, m3 = M1(), M2(), M3() mwman = MyMiddlewareManager(m1, m2, m3) - self.assertEqual( - list(mwman.methods["open_spider"]), [m1.open_spider, m2.open_spider] - ) - self.assertEqual( - list(mwman.methods["close_spider"]), [m2.close_spider, m1.close_spider] - ) - self.assertEqual(list(mwman.methods["process"]), [m1.process, m3.process]) + assert list(mwman.methods["open_spider"]) == [m1.open_spider, m2.open_spider] + assert list(mwman.methods["close_spider"]) == [m2.close_spider, m1.close_spider] + assert list(mwman.methods["process"]) == [m1.process, m3.process] def test_methods(self): mwman = MyMiddlewareManager(M1(), M2(), M3()) - self.assertEqual( - [x.__self__.__class__ for x in mwman.methods["open_spider"]], [M1, M2] - ) - self.assertEqual( - [x.__self__.__class__ for x in mwman.methods["close_spider"]], [M2, M1] - ) - self.assertEqual( - [x.__self__.__class__ for x in mwman.methods["process"]], [M1, M3] - ) + assert [x.__self__.__class__ for x in mwman.methods["open_spider"]] == [M1, M2] + assert [x.__self__.__class__ for x in mwman.methods["close_spider"]] == [M2, M1] + assert [x.__self__.__class__ for x in mwman.methods["process"]] == [M1, M3] def test_enabled(self): m1, m2, m3 = M1(), M2(), M3() mwman = MiddlewareManager(m1, m2, m3) - self.assertEqual(mwman.middlewares, (m1, m2, m3)) + assert mwman.middlewares == (m1, m2, m3) def test_enabled_from_settings(self): crawler = get_crawler() mwman = MyMiddlewareManager.from_crawler(crawler) classes = [x.__class__ for x in mwman.middlewares] - self.assertEqual(classes, [M1, M3]) + assert classes == [M1, M3] diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 0ae86235c..743d9774b 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -76,7 +76,7 @@ class ItemSpider(Spider): return {"field": 42} -class PipelineTestCase(unittest.TestCase): +class TestPipeline(unittest.TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -87,8 +87,8 @@ class PipelineTestCase(unittest.TestCase): cls.mockserver.__exit__(None, None, None) def _on_item_scraped(self, item): - self.assertIsInstance(item, dict) - self.assertTrue(item.get("pipeline_passed")) + assert isinstance(item, dict) + assert item.get("pipeline_passed") self.items.append(item) def _create_crawler(self, pipeline_class): @@ -104,30 +104,30 @@ class PipelineTestCase(unittest.TestCase): def test_simple_pipeline(self): crawler = self._create_crawler(SimplePipeline) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + assert len(self.items) == 1 @defer.inlineCallbacks def test_deferred_pipeline(self): crawler = self._create_crawler(DeferredPipeline) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + assert len(self.items) == 1 @defer.inlineCallbacks def test_asyncdef_pipeline(self): crawler = self._create_crawler(AsyncDefPipeline) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + assert len(self.items) == 1 @pytest.mark.only_asyncio @defer.inlineCallbacks def test_asyncdef_asyncio_pipeline(self): crawler = self._create_crawler(AsyncDefAsyncioPipeline) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + assert len(self.items) == 1 @pytest.mark.only_not_asyncio @defer.inlineCallbacks def test_asyncdef_not_asyncio_pipeline(self): crawler = self._create_crawler(AsyncDefNotAsyncioPipeline) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 1) + assert len(self.items) == 1 diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index c223c4562..d5c710ed2 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -1,5 +1,4 @@ import tempfile -import unittest import pytest import queuelib @@ -12,8 +11,8 @@ from scrapy.utils.test import get_crawler from tests.test_scheduler import MockDownloader, MockEngine -class PriorityQueueTest(unittest.TestCase): - def setUp(self): +class TestPriorityQueue: + def setup_method(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider("foo") @@ -22,20 +21,20 @@ class PriorityQueueTest(unittest.TestCase): queue = ScrapyPriorityQueue.from_crawler( self.crawler, FifoMemoryQueue, temp_dir ) - self.assertIsNone(queue.pop()) - self.assertEqual(len(queue), 0) + assert queue.pop() is None + assert len(queue) == 0 req1 = Request("https://example.org/1", priority=1) queue.push(req1) - self.assertEqual(len(queue), 1) + assert len(queue) == 1 dequeued = queue.pop() - self.assertEqual(len(queue), 0) - self.assertEqual(dequeued.url, req1.url) - self.assertEqual(dequeued.priority, req1.priority) - self.assertEqual(queue.close(), []) + assert len(queue) == 0 + assert dequeued.url == req1.url + assert dequeued.priority == req1.priority + assert not queue.close() def test_no_peek_raises(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + pytest.skip("queuelib.queue.FifoMemoryQueue.peek is defined") temp_dir = tempfile.mkdtemp() queue = ScrapyPriorityQueue.from_crawler( self.crawler, FifoMemoryQueue, temp_dir @@ -50,53 +49,53 @@ class PriorityQueueTest(unittest.TestCase): def test_peek(self): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") + pytest.skip("queuelib.queue.FifoMemoryQueue.peek is undefined") temp_dir = tempfile.mkdtemp() queue = ScrapyPriorityQueue.from_crawler( self.crawler, FifoMemoryQueue, temp_dir ) - self.assertEqual(len(queue), 0) - self.assertIsNone(queue.peek()) + assert len(queue) == 0 + assert queue.peek() is None req1 = Request("https://example.org/1") req2 = Request("https://example.org/2") req3 = Request("https://example.org/3") queue.push(req1) queue.push(req2) queue.push(req3) - self.assertEqual(len(queue), 3) - self.assertEqual(queue.peek().url, req1.url) - self.assertEqual(queue.pop().url, req1.url) - self.assertEqual(len(queue), 2) - self.assertEqual(queue.peek().url, req2.url) - self.assertEqual(queue.pop().url, req2.url) - self.assertEqual(len(queue), 1) - self.assertEqual(queue.peek().url, req3.url) - self.assertEqual(queue.pop().url, req3.url) - self.assertEqual(queue.close(), []) + assert len(queue) == 3 + assert queue.peek().url == req1.url + assert queue.pop().url == req1.url + assert len(queue) == 2 + assert queue.peek().url == req2.url + assert queue.pop().url == req2.url + assert len(queue) == 1 + assert queue.peek().url == req3.url + assert queue.pop().url == req3.url + assert not queue.close() def test_queue_push_pop_priorities(self): temp_dir = tempfile.mkdtemp() queue = ScrapyPriorityQueue.from_crawler( self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3] ) - self.assertIsNone(queue.pop()) - self.assertEqual(len(queue), 0) + assert queue.pop() is None + assert len(queue) == 0 req1 = Request("https://example.org/1", priority=1) req2 = Request("https://example.org/2", priority=2) req3 = Request("https://example.org/3", priority=3) queue.push(req1) queue.push(req2) queue.push(req3) - self.assertEqual(len(queue), 3) + assert len(queue) == 3 dequeued = queue.pop() - self.assertEqual(len(queue), 2) - self.assertEqual(dequeued.url, req3.url) - self.assertEqual(dequeued.priority, req3.priority) - self.assertEqual(queue.close(), [-1, -2]) + assert len(queue) == 2 + assert dequeued.url == req3.url + assert dequeued.priority == req3.priority + assert queue.close() == [-1, -2] -class DownloaderAwarePriorityQueueTest(unittest.TestCase): - def setUp(self): +class TestDownloaderAwarePriorityQueue: + def setup_method(self): crawler = get_crawler(Spider) crawler.engine = MockEngine(downloader=MockDownloader()) self.queue = DownloaderAwarePriorityQueue.from_crawler( @@ -105,30 +104,30 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase): key="foo/bar", ) - def tearDown(self): + def teardown_method(self): self.queue.close() def test_push_pop(self): - self.assertEqual(len(self.queue), 0) - self.assertIsNone(self.queue.pop()) + assert len(self.queue) == 0 + assert self.queue.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") self.queue.push(req1) self.queue.push(req2) self.queue.push(req3) - self.assertEqual(len(self.queue), 3) - self.assertEqual(self.queue.pop().url, req1.url) - self.assertEqual(len(self.queue), 2) - self.assertEqual(self.queue.pop().url, req2.url) - self.assertEqual(len(self.queue), 1) - self.assertEqual(self.queue.pop().url, req3.url) - self.assertEqual(len(self.queue), 0) - self.assertIsNone(self.queue.pop()) + assert len(self.queue) == 3 + assert self.queue.pop().url == req1.url + assert len(self.queue) == 2 + assert self.queue.pop().url == req2.url + assert len(self.queue) == 1 + assert self.queue.pop().url == req3.url + assert len(self.queue) == 0 + assert self.queue.pop() is None def test_no_peek_raises(self): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + pytest.skip("queuelib.queue.FifoMemoryQueue.peek is defined") self.queue.push(Request("https://example.org")) with pytest.raises( NotImplementedError, @@ -138,21 +137,21 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase): def test_peek(self): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") - self.assertEqual(len(self.queue), 0) + pytest.skip("queuelib.queue.FifoMemoryQueue.peek is undefined") + assert len(self.queue) == 0 req1 = Request("https://example.org/1") req2 = Request("https://example.org/2") req3 = Request("https://example.org/3") self.queue.push(req1) self.queue.push(req2) self.queue.push(req3) - self.assertEqual(len(self.queue), 3) - self.assertEqual(self.queue.peek().url, req1.url) - self.assertEqual(self.queue.pop().url, req1.url) - self.assertEqual(len(self.queue), 2) - self.assertEqual(self.queue.peek().url, req2.url) - self.assertEqual(self.queue.pop().url, req2.url) - self.assertEqual(len(self.queue), 1) - self.assertEqual(self.queue.peek().url, req3.url) - self.assertEqual(self.queue.pop().url, req3.url) - self.assertIsNone(self.queue.peek()) + assert len(self.queue) == 3 + assert self.queue.peek().url == req1.url + assert self.queue.pop().url == req1.url + assert len(self.queue) == 2 + assert self.queue.peek().url == req2.url + assert self.queue.pop().url == req2.url + assert len(self.queue) == 1 + assert self.queue.peek().url == req3.url + assert self.queue.pop().url == req3.url + assert self.queue.peek() is None diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 6ed7e93a6..885b7b7ae 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -6,6 +6,7 @@ from pathlib import Path from subprocess import PIPE, Popen from urllib.parse import urlsplit, urlunsplit +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -61,7 +62,7 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) -class ProxyConnectTestCase(TestCase): +class TestProxyConnect(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -75,7 +76,7 @@ class ProxyConnectTestCase(TestCase): try: import mitmproxy # noqa: F401 except ImportError: - self.skipTest("mitmproxy is not installed") + pytest.skip("mitmproxy is not installed") self._oldenv = os.environ.copy() @@ -113,12 +114,12 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(seed=request) self._assert_got_response_code(200, log) echo = json.loads(crawler.spider.meta["responses"][0].text) - self.assertTrue("Proxy-Authorization" not in echo["headers"]) + assert "Proxy-Authorization" not in echo["headers"] def _assert_got_response_code(self, code, log): print(log) - self.assertEqual(str(log).count(f"Crawled ({code})"), 1) + assert str(log).count(f"Crawled ({code})") == 1 def _assert_got_tunnel_error(self, log): print(log) - self.assertIn("TunnelError", str(log)) + assert "TunnelError" in str(log) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 0072660a7..9b42fd6c7 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -56,7 +56,7 @@ class AlternativeCallbacksMiddleware: return response.replace(request=new_request) -class CrawlTestCase(TestCase): +class TestCrawl(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -72,7 +72,7 @@ class CrawlTestCase(TestCase): crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.request.url, url) + assert response.request.url == url @defer.inlineCallbacks def test_response_error(self): @@ -82,8 +82,8 @@ class CrawlTestCase(TestCase): yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] response = failure.value.response - self.assertEqual(failure.request.url, url) - self.assertEqual(response.request.url, url) + assert failure.request.url == url + assert response.request.url == url @defer.inlineCallbacks def test_downloader_middleware_raise_exception(self): @@ -98,8 +98,8 @@ class CrawlTestCase(TestCase): ) yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] - self.assertEqual(failure.request.url, url) - self.assertIsInstance(failure.value, ZeroDivisionError) + assert failure.request.url == url + assert isinstance(failure.value, ZeroDivisionError) @defer.inlineCallbacks def test_downloader_middleware_override_request_in_process_response(self): @@ -131,10 +131,10 @@ class CrawlTestCase(TestCase): yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.request.url, OVERRIDDEN_URL) + assert response.request.url == OVERRIDDEN_URL - self.assertEqual(signal_params["response"].url, url) - self.assertEqual(signal_params["request"].url, OVERRIDDEN_URL) + assert signal_params["response"].url == url + assert signal_params["request"].url == OVERRIDDEN_URL log.check_present( ( @@ -164,8 +164,8 @@ class CrawlTestCase(TestCase): ) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.body, b"Caught ZeroDivisionError") - self.assertEqual(response.request.url, OVERRIDDEN_URL) + assert response.body == b"Caught ZeroDivisionError" + assert response.request.url == OVERRIDDEN_URL @defer.inlineCallbacks def test_downloader_middleware_do_not_override_in_process_exception(self): @@ -187,8 +187,8 @@ class CrawlTestCase(TestCase): ) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] - self.assertEqual(response.body, b"Caught ZeroDivisionError") - self.assertEqual(response.request.url, url) + assert response.body == b"Caught ZeroDivisionError" + assert response.request.url == url @defer.inlineCallbacks def test_downloader_middleware_alternative_callback(self): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index a21cb43ff..ab6baa5f0 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -151,7 +151,7 @@ class KeywordArgumentsSpider(MockServerSpider): self.crawler.stats.inc_value("boolean_checks", 1) -class CallbackKeywordArgumentsTestCase(TestCase): +class TestCallbackKeywordArguments(TestCase): maxDiff = None @classmethod @@ -168,27 +168,19 @@ class CallbackKeywordArgumentsTestCase(TestCase): crawler = get_crawler(KeywordArgumentsSpider) with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) - self.assertTrue(all(crawler.spider.checks)) - self.assertEqual( - len(crawler.spider.checks), crawler.stats.get_value("boolean_checks") - ) + assert all(crawler.spider.checks) + assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks") # check exceptions for argument mismatch exceptions = {} for line in log.records: for key in ("takes_less", "takes_more"): if key in line.getMessage(): exceptions[key] = line - self.assertEqual(exceptions["takes_less"].exc_info[0], TypeError) - self.assertTrue( - str(exceptions["takes_less"].exc_info[1]).endswith( - "parse_takes_less() got an unexpected keyword argument 'number'" - ), - msg="Exception message: " + str(exceptions["takes_less"].exc_info[1]), - ) - self.assertEqual(exceptions["takes_more"].exc_info[0], TypeError) - self.assertTrue( - str(exceptions["takes_more"].exc_info[1]).endswith( - "parse_takes_more() missing 1 required positional argument: 'other'" - ), - msg="Exception message: " + str(exceptions["takes_more"].exc_info[1]), - ) + assert exceptions["takes_less"].exc_info[0] is TypeError + assert str(exceptions["takes_less"].exc_info[1]).endswith( + "parse_takes_less() got an unexpected keyword argument 'number'" + ), "Exception message: " + str(exceptions["takes_less"].exc_info[1]) + assert exceptions["takes_more"].exc_info[0] is TypeError + assert str(exceptions["takes_more"].exc_info[1]).endswith( + "parse_takes_more() missing 1 required positional argument: 'other'" + ), "Exception message: " + str(exceptions["takes_more"].exc_info[1]) diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 2c605a015..ea7018541 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,5 +1,3 @@ -import unittest - import pytest from scrapy import Request, Spider @@ -11,8 +9,8 @@ class CustomRequest(Request): pass -class RequestSerializationTest(unittest.TestCase): - def setUp(self): +class TestRequestSerialization: + def setup_method(self): self.spider = MethodsSpider() def test_basic(self): @@ -50,23 +48,23 @@ class RequestSerializationTest(unittest.TestCase): self._assert_same_request(request, request2) def _assert_same_request(self, r1, r2): - self.assertEqual(r1.__class__, r2.__class__) - self.assertEqual(r1.url, r2.url) - self.assertEqual(r1.callback, r2.callback) - self.assertEqual(r1.errback, r2.errback) - self.assertEqual(r1.method, r2.method) - self.assertEqual(r1.body, r2.body) - self.assertEqual(r1.headers, r2.headers) - self.assertEqual(r1.cookies, r2.cookies) - self.assertEqual(r1.meta, r2.meta) - self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) - self.assertEqual(r1.encoding, r2.encoding) - self.assertEqual(r1._encoding, r2._encoding) - self.assertEqual(r1.priority, r2.priority) - self.assertEqual(r1.dont_filter, r2.dont_filter) - self.assertEqual(r1.flags, r2.flags) + assert r1.__class__ == r2.__class__ + assert r1.url == r2.url + assert r1.callback == r2.callback + assert r1.errback == r2.errback + assert r1.method == r2.method + assert r1.body == r2.body + assert r1.headers == r2.headers + assert r1.cookies == r2.cookies + assert r1.meta == r2.meta + assert r1.cb_kwargs == r2.cb_kwargs + assert r1.encoding == r2.encoding + assert r1._encoding == r2._encoding + assert r1.priority == r2.priority + assert r1.dont_filter == r2.dont_filter + assert r1.flags == r2.flags if isinstance(r1, JsonRequest): - self.assertEqual(r1.dumps_kwargs, r2.dumps_kwargs) + assert r1.dumps_kwargs == r2.dumps_kwargs def test_request_class(self): r1 = FormRequest("http://www.example.com") @@ -92,8 +90,8 @@ class RequestSerializationTest(unittest.TestCase): ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) - self.assertEqual(request_dict["callback"], "parse_item_reference") - self.assertEqual(request_dict["errback"], "handle_error_reference") + assert request_dict["callback"] == "parse_item_reference" + assert request_dict["errback"] == "handle_error_reference" def test_private_reference_callback_serialization(self): r = Request( @@ -103,12 +101,8 @@ class RequestSerializationTest(unittest.TestCase): ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) - self.assertEqual( - request_dict["callback"], "_MethodsSpider__parse_item_reference" - ) - self.assertEqual( - request_dict["errback"], "_MethodsSpider__handle_error_reference" - ) + assert request_dict["callback"] == "_MethodsSpider__parse_item_reference" + assert request_dict["errback"] == "_MethodsSpider__handle_error_reference" def test_private_callback_serialization(self): r = Request( diff --git a/tests/test_request_left.py b/tests/test_request_left.py index cf4c8a2d5..d55905f9c 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -38,22 +38,22 @@ class TestCatching(TestCase): def test_success(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl(self.mockserver.url("/status?n=200")) - self.assertEqual(crawler.spider.caught_times, 1) + assert crawler.spider.caught_times == 1 @defer.inlineCallbacks def test_timeout(self): crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1}) yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) - self.assertEqual(crawler.spider.caught_times, 1) + assert crawler.spider.caught_times == 1 @defer.inlineCallbacks def test_disconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl(self.mockserver.url("/drop")) - self.assertEqual(crawler.spider.caught_times, 1) + assert crawler.spider.caught_times == 1 @defer.inlineCallbacks def test_noconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl("http://thereisdefinetelynosuchdomain.com") - self.assertEqual(crawler.spider.caught_times, 1) + assert crawler.spider.caught_times == 1 diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index f9f56ff97..5b04c7436 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -1,5 +1,3 @@ -import unittest - from scrapy.http import ( Headers, HtmlResponse, @@ -11,7 +9,7 @@ from scrapy.http import ( from scrapy.responsetypes import responsetypes -class ResponseTypesTest(unittest.TestCase): +class TestResponseTypes: def test_from_filename(self): mappings = [ ("data.bin", Response), @@ -123,6 +121,4 @@ class ResponseTypesTest(unittest.TestCase): def test_custom_mime_types_loaded(self): # check that mime.types files shipped with scrapy are loaded - self.assertEqual( - responsetypes.mimetypes.guess_type("x.scrapytest")[0], "x-scrapy/test" - ) + assert responsetypes.mimetypes.guess_type("x.scrapytest")[0] == "x-scrapy/test" diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 0d00ff660..221ccabe6 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,4 +1,4 @@ -from twisted.trial import unittest +import pytest from scrapy.robotstxt import decode_robotstxt @@ -32,8 +32,8 @@ class BaseRobotParserTest: rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://www.site.local/allowed", "*")) - self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*")) + assert rp.allowed("https://www.site.local/allowed", "*") + assert not rp.allowed("https://www.site.local/disallowed", "*") def test_allowed_wildcards(self): robotstxt_robotstxt_body = b"""User-agent: first @@ -47,42 +47,36 @@ class BaseRobotParserTest: crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://www.site.local/disallowed", "first")) - self.assertFalse( - rp.allowed("https://www.site.local/disallowed/xyz/end", "first") - ) - self.assertFalse( - rp.allowed("https://www.site.local/disallowed/abc/end", "first") - ) - self.assertTrue( - rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first") - ) + assert rp.allowed("https://www.site.local/disallowed", "first") + assert not rp.allowed("https://www.site.local/disallowed/xyz/end", "first") + assert not rp.allowed("https://www.site.local/disallowed/abc/end", "first") + assert rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first") - self.assertTrue(rp.allowed("https://www.site.local/allowed", "second")) - self.assertTrue(rp.allowed("https://www.site.local/is_still_allowed", "second")) - self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second")) + assert rp.allowed("https://www.site.local/allowed", "second") + assert rp.allowed("https://www.site.local/is_still_allowed", "second") + assert rp.allowed("https://www.site.local/is_allowed_too", "second") def test_length_based_precedence(self): robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://www.site.local/page", "*")) + assert rp.allowed("https://www.site.local/page", "*") def test_order_based_precedence(self): robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertFalse(rp.allowed("https://www.site.local/page", "*")) + assert not rp.allowed("https://www.site.local/page", "*") def test_empty_response(self): """empty response should equal 'allow all'""" rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"") - self.assertTrue(rp.allowed("https://site.local/", "*")) - self.assertTrue(rp.allowed("https://site.local/", "chrome")) - self.assertTrue(rp.allowed("https://site.local/index.html", "*")) - self.assertTrue(rp.allowed("https://site.local/disallowed", "*")) + assert rp.allowed("https://site.local/", "*") + assert rp.allowed("https://site.local/", "chrome") + assert rp.allowed("https://site.local/index.html", "*") + assert rp.allowed("https://site.local/disallowed", "*") def test_garbage_response(self): """garbage response should be discarded, equal 'allow all'""" @@ -90,10 +84,10 @@ class BaseRobotParserTest: rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://site.local/", "*")) - self.assertTrue(rp.allowed("https://site.local/", "chrome")) - self.assertTrue(rp.allowed("https://site.local/index.html", "*")) - self.assertTrue(rp.allowed("https://site.local/disallowed", "*")) + assert rp.allowed("https://site.local/", "*") + assert rp.allowed("https://site.local/", "chrome") + assert rp.allowed("https://site.local/index.html", "*") + assert rp.allowed("https://site.local/disallowed", "*") def test_unicode_url_and_useragent(self): robotstxt_robotstxt_body = """ @@ -109,79 +103,67 @@ class BaseRobotParserTest: rp = self.parser_cls.from_crawler( crawler=None, robotstxt_body=robotstxt_robotstxt_body ) - self.assertTrue(rp.allowed("https://site.local/", "*")) - self.assertFalse(rp.allowed("https://site.local/admin/", "*")) - self.assertFalse(rp.allowed("https://site.local/static/", "*")) - self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt")) - self.assertFalse( - rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*") - ) - self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*")) - self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*")) - self.assertFalse( - rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt") - ) + assert rp.allowed("https://site.local/", "*") + assert not rp.allowed("https://site.local/admin/", "*") + assert not rp.allowed("https://site.local/static/", "*") + assert rp.allowed("https://site.local/admin/", "UnicödeBöt") + assert not rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*") + assert not rp.allowed("https://site.local/wiki/Käyttäjä:", "*") + assert rp.allowed("https://site.local/some/randome/page.html", "*") + assert not rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt") -class DecodeRobotsTxtTest(unittest.TestCase): +class TestDecodeRobotsTxt: def test_native_string_conversion(self): robotstxt_body = b"User-agent: *\nDisallow: /\n" decoded_content = decode_robotstxt( robotstxt_body, spider=None, to_native_str_type=True ) - self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + assert decoded_content == "User-agent: *\nDisallow: /\n" def test_decode_utf8(self): robotstxt_body = b"User-agent: *\nDisallow: /\n" decoded_content = decode_robotstxt(robotstxt_body, spider=None) - self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + assert decoded_content == "User-agent: *\nDisallow: /\n" def test_decode_non_utf8(self): robotstxt_body = b"User-agent: *\n\xffDisallow: /\n" decoded_content = decode_robotstxt(robotstxt_body, spider=None) - self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n") + assert decoded_content == "User-agent: *\nDisallow: /\n" -class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): - def setUp(self): +class TestPythonRobotParser(BaseRobotParserTest): + def setup_method(self): from scrapy.robotstxt import PythonRobotParser super()._setUp(PythonRobotParser) def test_length_based_precedence(self): - raise unittest.SkipTest( + pytest.skip( "RobotFileParser does not support length based directives precedence." ) def test_allowed_wildcards(self): - raise unittest.SkipTest("RobotFileParser does not support wildcards.") + pytest.skip("RobotFileParser does not support wildcards.") -class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): - if not rerp_available(): - skip = "Rerp parser is not installed" - - def setUp(self): +@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed") +class TestRerpRobotParser(BaseRobotParserTest): + def setup_method(self): from scrapy.robotstxt import RerpRobotParser super()._setUp(RerpRobotParser) def test_length_based_precedence(self): - raise unittest.SkipTest( - "Rerp does not support length based directives precedence." - ) + pytest.skip("Rerp does not support length based directives precedence.") -class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase): - if not protego_available(): - skip = "Protego parser is not installed" - - def setUp(self): +@pytest.mark.skipif(not protego_available(), reason="Protego parser is not installed") +class TestProtegoRobotParser(BaseRobotParserTest): + def setup_method(self): from scrapy.robotstxt import ProtegoRobotParser super()._setUp(ProtegoRobotParser) def test_order_based_precedence(self): - raise unittest.SkipTest( - "Protego does not support order based directives precedence." - ) + pytest.skip("Protego does not support order based directives precedence.") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index f2f8b96cd..1d6992a32 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2,7 +2,7 @@ from __future__ import annotations import shutil import tempfile -import unittest +from abc import ABC, abstractmethod from typing import Any, NamedTuple import pytest @@ -65,10 +65,14 @@ class MockCrawler(Crawler): self.stats = load_object(self.settings["STATS_CLASS"])(self) -class SchedulerHandler: - priority_queue_cls: str | None = None +class SchedulerHandler(ABC): jobdir = None + @property + @abstractmethod + def priority_queue_cls(self) -> str: + raise NotImplementedError + def create_scheduler(self): self.mock_crawler = MockCrawler(self.priority_queue_cls, self.jobdir) self.scheduler = Scheduler.from_crawler(self.mock_crawler) @@ -80,10 +84,10 @@ class SchedulerHandler: self.mock_crawler.stop() self.mock_crawler.engine.downloader.close() - def setUp(self): + def setup_method(self): self.create_scheduler() - def tearDown(self): + def teardown_method(self): self.close_scheduler() @@ -99,16 +103,16 @@ _PRIORITIES = [ _URLS = {"http://foo.com/a", "http://foo.com/b", "http://foo.com/c"} -class BaseSchedulerInMemoryTester(SchedulerHandler): +class TestSchedulerInMemoryBase(SchedulerHandler): def test_length(self): - self.assertFalse(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), 0) + assert not self.scheduler.has_pending_requests() + assert len(self.scheduler) == 0 for url in _URLS: self.scheduler.enqueue_request(Request(url)) - self.assertTrue(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), len(_URLS)) + assert self.scheduler.has_pending_requests() + assert len(self.scheduler) == len(_URLS) def test_dequeue(self): for url in _URLS: @@ -118,7 +122,7 @@ class BaseSchedulerInMemoryTester(SchedulerHandler): while self.scheduler.has_pending_requests(): urls.add(self.scheduler.next_request().url) - self.assertEqual(urls, _URLS) + assert urls == _URLS def test_dequeue_priorities(self): for url, priority in _PRIORITIES: @@ -128,25 +132,23 @@ class BaseSchedulerInMemoryTester(SchedulerHandler): while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) - self.assertEqual( - priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) - ) + assert priorities == sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) -class BaseSchedulerOnDiskTester(SchedulerHandler): - def setUp(self): +class TestSchedulerOnDiskBase(SchedulerHandler): + def setup_method(self): self.jobdir = tempfile.mkdtemp() self.create_scheduler() - def tearDown(self): + def teardown_method(self): self.close_scheduler() shutil.rmtree(self.jobdir) self.jobdir = None def test_length(self): - self.assertFalse(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), 0) + assert not self.scheduler.has_pending_requests() + assert len(self.scheduler) == 0 for url in _URLS: self.scheduler.enqueue_request(Request(url)) @@ -154,8 +156,8 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): self.close_scheduler() self.create_scheduler() - self.assertTrue(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), len(_URLS)) + assert self.scheduler.has_pending_requests() + assert len(self.scheduler) == len(_URLS) def test_dequeue(self): for url in _URLS: @@ -168,7 +170,7 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): while self.scheduler.has_pending_requests(): urls.add(self.scheduler.next_request().url) - self.assertEqual(urls, _URLS) + assert urls == _URLS def test_dequeue_priorities(self): for url, priority in _PRIORITIES: @@ -181,17 +183,19 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) - self.assertEqual( - priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) - ) + assert priorities == sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) -class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): - priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue" +class TestSchedulerInMemory(TestSchedulerInMemoryBase): + @property + def priority_queue_cls(self) -> str: + return "scrapy.pqueues.ScrapyPriorityQueue" -class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): - priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue" +class TestSchedulerOnDisk(TestSchedulerOnDiskBase): + @property + def priority_queue_cls(self) -> str: + return "scrapy.pqueues.ScrapyPriorityQueue" _URLS_WITH_SLOTS = [ @@ -204,37 +208,34 @@ _URLS_WITH_SLOTS = [ ] -class TestMigration(unittest.TestCase): - def setUp(self): - self.tmpdir = tempfile.mkdtemp() +class TestMigration: + def test_migration(self, tmpdir): + class PrevSchedulerHandler(SchedulerHandler): + jobdir = tmpdir - def tearDown(self): - shutil.rmtree(self.tmpdir) + @property + def priority_queue_cls(self) -> str: + return "scrapy.pqueues.ScrapyPriorityQueue" - def _migration(self, tmp_dir): - prev_scheduler_handler = SchedulerHandler() - prev_scheduler_handler.priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue" - prev_scheduler_handler.jobdir = tmp_dir + class NextSchedulerHandler(SchedulerHandler): + jobdir = tmpdir + @property + def priority_queue_cls(self) -> str: + return "scrapy.pqueues.DownloaderAwarePriorityQueue" + + prev_scheduler_handler = PrevSchedulerHandler() prev_scheduler_handler.create_scheduler() for url in _URLS: prev_scheduler_handler.scheduler.enqueue_request(Request(url)) prev_scheduler_handler.close_scheduler() - next_scheduler_handler = SchedulerHandler() - next_scheduler_handler.priority_queue_cls = ( - "scrapy.pqueues.DownloaderAwarePriorityQueue" - ) - next_scheduler_handler.jobdir = tmp_dir - - next_scheduler_handler.create_scheduler() - - def test_migration(self): + next_scheduler_handler = NextSchedulerHandler() with pytest.raises( ValueError, match="DownloaderAwarePriorityQueue accepts ``slot_startprios`` as a dict", ): - self._migration(self.tmpdir) + next_scheduler_handler.create_scheduler() def _is_scheduling_fair(enqueued_slots, dequeued_slots): @@ -263,9 +264,12 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): class DownloaderAwareSchedulerTestMixin: - priority_queue_cls: str | None = "scrapy.pqueues.DownloaderAwarePriorityQueue" reopen = False + @property + def priority_queue_cls(self) -> str: + return "scrapy.pqueues.DownloaderAwarePriorityQueue" + def test_logic(self): for url, slot in _URLS_WITH_SLOTS: request = Request(url) @@ -290,20 +294,18 @@ class DownloaderAwareSchedulerTestMixin: slot = downloader.get_slot_key(request) downloader.decrement(slot) - self.assertTrue( - _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots) - ) - self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0) + assert _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots) + assert sum(len(s.active) for s in downloader.slots.values()) == 0 class TestSchedulerWithDownloaderAwareInMemory( - DownloaderAwareSchedulerTestMixin, BaseSchedulerInMemoryTester, unittest.TestCase + DownloaderAwareSchedulerTestMixin, TestSchedulerInMemoryBase ): pass class TestSchedulerWithDownloaderAwareOnDisk( - DownloaderAwareSchedulerTestMixin, BaseSchedulerOnDiskTester, unittest.TestCase + DownloaderAwareSchedulerTestMixin, TestSchedulerOnDiskBase ): reopen = True @@ -337,13 +339,12 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase): url = mockserver.url("/status?n=200", is_secure=False) start_urls = [url] * 6 yield self.crawler.crawl(start_urls) - self.assertEqual( - self.crawler.stats.get_value("downloader/response_count"), - len(start_urls), + assert self.crawler.stats.get_value("downloader/response_count") == len( + start_urls ) -class TestIncompatibility(unittest.TestCase): +class TestIncompatibility: def _incompatible(self): settings = { "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue", diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index c2bb8cec5..4a36d3cdb 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -1,12 +1,11 @@ from __future__ import annotations -from unittest import TestCase from urllib.parse import urljoin import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.trial.unittest import TestCase as TwistedTestCase +from twisted.trial.unittest import TestCase from scrapy.core.scheduler import BaseScheduler from scrapy.http import Request @@ -65,17 +64,17 @@ class PathsSpider(Spider): class InterfaceCheckMixin: def test_scheduler_class(self): - self.assertTrue(isinstance(self.scheduler, BaseScheduler)) - self.assertTrue(issubclass(self.scheduler.__class__, BaseScheduler)) + assert isinstance(self.scheduler, BaseScheduler) + assert issubclass(self.scheduler.__class__, BaseScheduler) -class BaseSchedulerTest(TestCase, InterfaceCheckMixin): - def setUp(self): +class TestBaseScheduler(InterfaceCheckMixin): + def setup_method(self): self.scheduler = BaseScheduler() def test_methods(self): - self.assertIsNone(self.scheduler.open(Spider("foo"))) - self.assertIsNone(self.scheduler.close("finished")) + assert self.scheduler.open(Spider("foo")) is None + assert self.scheduler.close("finished") is None with pytest.raises(NotImplementedError): self.scheduler.has_pending_requests() with pytest.raises(NotImplementedError): @@ -84,8 +83,8 @@ class BaseSchedulerTest(TestCase, InterfaceCheckMixin): self.scheduler.next_request() -class MinimalSchedulerTest(TestCase, InterfaceCheckMixin): - def setUp(self): +class TestMinimalScheduler(InterfaceCheckMixin): + def setup_method(self): self.scheduler = MinimalScheduler() def test_open_close(self): @@ -101,51 +100,51 @@ class MinimalSchedulerTest(TestCase, InterfaceCheckMixin): len(self.scheduler) def test_enqueue_dequeue(self): - self.assertFalse(self.scheduler.has_pending_requests()) + assert not self.scheduler.has_pending_requests() for url in URLS: - self.assertTrue(self.scheduler.enqueue_request(Request(url))) - self.assertFalse(self.scheduler.enqueue_request(Request(url))) - self.assertTrue(self.scheduler.has_pending_requests) + assert self.scheduler.enqueue_request(Request(url)) + assert not self.scheduler.enqueue_request(Request(url)) + assert self.scheduler.has_pending_requests dequeued = [] while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() dequeued.append(request.url) - self.assertEqual(set(dequeued), set(URLS)) - self.assertFalse(self.scheduler.has_pending_requests()) + assert set(dequeued) == set(URLS) + assert not self.scheduler.has_pending_requests() -class SimpleSchedulerTest(TwistedTestCase, InterfaceCheckMixin): +class SimpleSchedulerTest(TestCase, InterfaceCheckMixin): def setUp(self): self.scheduler = SimpleScheduler() @defer.inlineCallbacks def test_enqueue_dequeue(self): open_result = yield self.scheduler.open(Spider("foo")) - self.assertEqual(open_result, "open") - self.assertFalse(self.scheduler.has_pending_requests()) + assert open_result == "open" + assert not self.scheduler.has_pending_requests() for url in URLS: - self.assertTrue(self.scheduler.enqueue_request(Request(url))) - self.assertFalse(self.scheduler.enqueue_request(Request(url))) + assert self.scheduler.enqueue_request(Request(url)) + assert not self.scheduler.enqueue_request(Request(url)) - self.assertTrue(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), len(URLS)) + assert self.scheduler.has_pending_requests() + assert len(self.scheduler) == len(URLS) dequeued = [] while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() dequeued.append(request.url) - self.assertEqual(set(dequeued), set(URLS)) + assert set(dequeued) == set(URLS) - self.assertFalse(self.scheduler.has_pending_requests()) - self.assertEqual(len(self.scheduler), 0) + assert not self.scheduler.has_pending_requests() + assert len(self.scheduler) == 0 close_result = yield self.scheduler.close("") - self.assertEqual(close_result, "close") + assert close_result == "close" -class MinimalSchedulerCrawlTest(TwistedTestCase): +class MinimalSchedulerCrawlTest(TestCase): scheduler_cls = MinimalScheduler @defer.inlineCallbacks @@ -158,8 +157,8 @@ class MinimalSchedulerCrawlTest(TwistedTestCase): crawler = get_crawler(PathsSpider, settings) yield crawler.crawl(mockserver) for path in PATHS: - self.assertIn(f"{{'path': '{path}'}}", str(log)) - self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log)) + assert f"{{'path': '{path}'}}" in str(log) + assert f"'item_scraped_count': {len(PATHS)}" in str(log) class SimpleSchedulerCrawlTest(MinimalSchedulerCrawlTest): diff --git a/tests/test_selector.py b/tests/test_selector.py index 2d7a1442e..5c8eadf0b 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -3,7 +3,6 @@ import weakref import parsel import pytest from packaging import version -from twisted.trial import unittest from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.selector import Selector @@ -12,7 +11,7 @@ PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0") -class SelectorTestCase(unittest.TestCase): +class TestSelector: def test_simple_selection(self): """Simple selector tests""" body = b"

" @@ -20,57 +19,46 @@ class SelectorTestCase(unittest.TestCase): sel = Selector(response) xl = sel.xpath("//input") - self.assertEqual(2, len(xl)) + assert len(xl) == 2 for x in xl: assert isinstance(x, Selector) - self.assertEqual( - sel.xpath("//input").getall(), [x.get() for x in sel.xpath("//input")] - ) - self.assertEqual( - [x.get() for x in sel.xpath("//input[@name='a']/@name")], ["a"] - ) - self.assertEqual( - [ - x.get() - for x in sel.xpath( - "number(concat(//input[@name='a']/@value, //input[@name='b']/@value))" - ) - ], - ["12.0"], - ) - self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(), ["xpathrules"]) - self.assertEqual( - [ - x.get() - for x in sel.xpath( - "concat(//input[@name='a']/@value, //input[@name='b']/@value)" - ) - ], - ["12"], - ) + assert sel.xpath("//input").getall() == [x.get() for x in sel.xpath("//input")] + assert [x.get() for x in sel.xpath("//input[@name='a']/@name")] == ["a"] + assert [ + x.get() + for x in sel.xpath( + "number(concat(//input[@name='a']/@value, //input[@name='b']/@value))" + ) + ] == ["12.0"] + assert sel.xpath("concat('xpath', 'rules')").getall() == ["xpathrules"] + assert [ + x.get() + for x in sel.xpath( + "concat(//input[@name='a']/@value, //input[@name='b']/@value)" + ) + ] == ["12"] def test_root_base_url(self): body = b'' url = "http://example.com" response = TextResponse(url=url, body=body, encoding="utf-8") sel = Selector(response) - self.assertEqual(url, sel.root.base) + assert url == sel.root.base def test_flavor_detection(self): text = b'

Hello

' sel = Selector(XmlResponse("http://example.com", body=text, encoding="utf-8")) - self.assertEqual(sel.type, "xml") - self.assertEqual( - sel.xpath("//div").getall(), - ['

Hello

'], - ) + assert sel.type == "xml" + assert sel.xpath("//div").getall() == [ + '

Hello

' + ] sel = Selector(HtmlResponse("http://example.com", body=text, encoding="utf-8")) - self.assertEqual(sel.type, "html") - self.assertEqual( - sel.xpath("//div").getall(), ['

Hello

'] - ) + assert sel.type == "html" + assert sel.xpath("//div").getall() == [ + '

Hello

' + ] def test_http_header_encoding_precedence(self): # '\xa3' = pound symbol in unicode @@ -92,7 +80,7 @@ class SelectorTestCase(unittest.TestCase): url="http://example.com", headers=headers, body=html_utf8 ) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ["\xa3"]) + assert x.xpath("//span[@id='blank']/text()").getall() == ["\xa3"] def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence @@ -116,7 +104,7 @@ class SelectorTestCase(unittest.TestCase): Selector(TextResponse(url="http://example.com", body=b""), text="") -class JMESPathTestCase(unittest.TestCase): +class TestJMESPath: @pytest.mark.skipif( not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" ) @@ -149,16 +137,13 @@ class JMESPathTestCase(unittest.TestCase): } """ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") - self.assertEqual( - resp.jmespath("html").get(), - "
def
", + assert ( + resp.jmespath("html").get() + == "
def
" ) - self.assertEqual( - resp.jmespath("html").xpath("//div/a/text()").getall(), - ["a", "b", "d"], - ) - self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["f"]) - self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18") + assert resp.jmespath("html").xpath("//div/a/text()").getall() == ["a", "b", "d"] + assert resp.jmespath("html").css("div > b").getall() == ["f"] + assert resp.jmespath("content").jmespath("name.age").get() == "18" @pytest.mark.skipif( not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" @@ -194,15 +179,19 @@ class JMESPathTestCase(unittest.TestCase): """ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") - self.assertEqual( - resp.xpath("//div/content/text()").jmespath("user[*].name").getall(), - ["A", "B", "C", "D"], - ) - self.assertEqual( - resp.xpath("//div/content").jmespath("user[*].name").getall(), - ["A", "B", "C", "D"], - ) - self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4") + assert resp.xpath("//div/content/text()").jmespath("user[*].name").getall() == [ + "A", + "B", + "C", + "D", + ] + assert resp.xpath("//div/content").jmespath("user[*].name").getall() == [ + "A", + "B", + "C", + "D", + ] + assert resp.xpath("//div/content").jmespath("total").get() == "4" @pytest.mark.skipif( not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" @@ -238,30 +227,26 @@ class JMESPathTestCase(unittest.TestCase): """ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") - self.assertEqual( - resp.xpath("//div/content/text()").jmespath("user[*].name").re(r"(\w+)"), - ["A", "B", "C", "D"], - ) - self.assertEqual( - resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"), - ["A", "B", "C", "D"], + assert resp.xpath("//div/content/text()").jmespath("user[*].name").re( + r"(\w+)" + ) == ["A", "B", "C", "D"] + assert resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)") == [ + "A", + "B", + "C", + "D", + ] + + assert resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)") == [] + + assert ( + resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)") + is None ) - self.assertEqual( - resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), [] - ) - - self.assertEqual( - resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)"), - None, - ) - - self.assertEqual( - resp.xpath("//div/content") - .jmespath("user[*].age.to_string(@)") - .re(r"(\d+)"), - ["18", "32", "22", "25"], - ) + assert resp.xpath("//div/content").jmespath("user[*].age.to_string(@)").re( + r"(\d+)" + ) == ["18", "32", "22", "25"] @pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath") def test_jmespath_not_available(self) -> None: diff --git a/tests/test_signals.py b/tests/test_signals.py index a508eb41a..f5075fb60 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -20,7 +20,7 @@ class ItemSpider(Spider): return {"index": response.meta["index"]} -class AsyncSignalTestCase(unittest.TestCase): +class TestAsyncSignal(unittest.TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -43,6 +43,6 @@ class AsyncSignalTestCase(unittest.TestCase): crawler = get_crawler(ItemSpider) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) - self.assertEqual(len(self.items), 10) + assert len(self.items) == 10 for index in range(10): - self.assertIn({"index": index}, self.items) + assert {"index": index} in self.items diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index d272101b8..a4f31096e 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -1,33 +1,31 @@ -from unittest import TestCase - import scrapy -class ToplevelTestCase(TestCase): +class TestToplevel: def test_version(self): - self.assertIs(type(scrapy.__version__), str) + assert isinstance(scrapy.__version__, str) def test_version_info(self): - self.assertIs(type(scrapy.version_info), tuple) + assert isinstance(scrapy.version_info, tuple) def test_request_shortcut(self): from scrapy.http import FormRequest, Request - self.assertIs(scrapy.Request, Request) - self.assertIs(scrapy.FormRequest, FormRequest) + assert scrapy.Request is Request + assert scrapy.FormRequest is FormRequest def test_spider_shortcut(self): from scrapy.spiders import Spider - self.assertIs(scrapy.Spider, Spider) + assert scrapy.Spider is Spider def test_selector_shortcut(self): from scrapy.selector import Selector - self.assertIs(scrapy.Selector, Selector) + assert scrapy.Selector is Selector def test_item_shortcut(self): from scrapy.item import Field, Item - self.assertIs(scrapy.Item, Item) - self.assertIs(scrapy.Field, Field) + assert scrapy.Item is Item + assert scrapy.Field is Field diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py index c695968d7..0e1e89e81 100644 --- a/tests/test_urlparse_monkeypatches.py +++ b/tests/test_urlparse_monkeypatches.py @@ -1,11 +1,10 @@ -import unittest from urllib.parse import urlparse -class UrlparseTestCase(unittest.TestCase): +class TestUrlparse: def test_s3_url(self): p = urlparse("s3://bucket/key/name?param=value") - self.assertEqual(p.scheme, "s3") - self.assertEqual(p.hostname, "bucket") - self.assertEqual(p.path, "/key/name") - self.assertEqual(p.query, "param=value") + assert p.scheme == "s3" + assert p.hostname == "bucket" + assert p.path == "/key/name" + assert p.query == "param=value" From 0bbfca6c1d1327a9919f2c0efc0c75aadb5b0033 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Mar 2025 13:15:28 +0500 Subject: [PATCH 1683/2083] Better fix for test_non_pickable_object on Windows. --- tests/test_squeues.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 0b6ed8e11..21bbeece2 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -130,9 +130,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): ) as exc_info: q.push(sel) assert isinstance(exc_info.value.__context__, TypeError) - # This seems to help with https://github.com/scrapy/queuelib/issues/70. - # It will need to remain under a queuelib version check after that bug is fixed. - del exc_info + q.close() class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): From bee74fb753afa373ecdb1dbf272af9a108b14b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 11:56:44 +0100 Subject: [PATCH 1684/2083] Remove trailing whitespace --- .github/workflows/publish.yml | 2 +- .pre-commit-config.yaml | 4 + README.rst | 2 +- docs/_tests/quotes.html | 138 +++++++++++++++---------------- docs/_tests/quotes1.html | 138 +++++++++++++++---------------- docs/faq.rst | 2 +- docs/intro/install.rst | 4 +- docs/intro/tutorial.rst | 6 +- docs/news.rst | 8 +- docs/topics/api.rst | 2 +- docs/topics/architecture.rst | 4 +- docs/topics/exporters.rst | 2 +- docs/topics/extensions.rst | 16 ++-- docs/topics/feed-exports.rst | 10 +-- docs/topics/logging.rst | 10 +-- docs/topics/media-pipeline.rst | 14 ++-- docs/topics/practices.rst | 2 +- docs/topics/request-response.rst | 6 +- docs/topics/selectors.rst | 6 +- docs/topics/spiders.rst | 4 +- docs/topics/stats.rst | 2 +- sep/sep-004.rst | 2 +- sep/sep-007.rst | 2 +- sep/sep-008.rst | 4 +- sep/sep-014.rst | 4 +- sep/sep-018.rst | 4 +- 26 files changed, 201 insertions(+), 197 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8e01ffd88..d1589f4f7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.13" - - run: | + - run: | python -m pip install --upgrade build python -m build - name: Publish to PyPI diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18402b908..0d1a76247 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,3 +11,7 @@ repos: - id: blacken-docs additional_dependencies: - black==24.10.0 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace diff --git a/README.rst b/README.rst index 3f468953e..cf7c6043c 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ .. image:: https://scrapy.org/img/scrapylogo.png :target: https://scrapy.org/ - + ====== Scrapy ====== diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html index f4002ecd1..d1cfd9020 100644 --- a/docs/_tests/quotes.html +++ b/docs/_tests/quotes.html @@ -16,13 +16,13 @@

- + Login - +

- +
@@ -34,16 +34,16 @@
Tags: - - + + change - + deep-thoughts - + thinking - + world - +
@@ -54,12 +54,12 @@
Tags: - - + + abilities - + choices - +
@@ -70,18 +70,18 @@
Tags: - - + + inspirational - + life - + live - + miracle - + miracles - +
@@ -92,16 +92,16 @@
Tags: - - + + aliteracy - + books - + classic - + humor - +
@@ -112,12 +112,12 @@
Tags: - - + + be-yourself - + inspirational - +
@@ -128,14 +128,14 @@
Tags: - - + + adulthood - + success - + value - +
@@ -146,12 +146,12 @@
Tags: - - + + life - + love - +
@@ -162,16 +162,16 @@
Tags: - - + + edison - + failure - + inspirational - + paraphrased - +
@@ -182,10 +182,10 @@
Tags: - - + + misattributed-eleanor-roosevelt - +
@@ -196,73 +196,73 @@
Tags: - - + + humor - + obvious - + simile - +
- +

Top Ten tags

- + love - + inspirational - + life - + humor - + books - + reading - + friendship - + friends - + truth - + simile - - + +
diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html index f4002ecd1..d1cfd9020 100644 --- a/docs/_tests/quotes1.html +++ b/docs/_tests/quotes1.html @@ -16,13 +16,13 @@

- + Login - +

- +
@@ -34,16 +34,16 @@
Tags: - - + + change - + deep-thoughts - + thinking - + world - +
@@ -54,12 +54,12 @@
Tags: - - + + abilities - + choices - +
@@ -70,18 +70,18 @@
Tags: - - + + inspirational - + life - + live - + miracle - + miracles - +
@@ -92,16 +92,16 @@
Tags: - - + + aliteracy - + books - + classic - + humor - +
@@ -112,12 +112,12 @@
Tags: - - + + be-yourself - + inspirational - +
@@ -128,14 +128,14 @@
Tags: - - + + adulthood - + success - + value - +
@@ -146,12 +146,12 @@
Tags: - - + + life - + love - +
@@ -162,16 +162,16 @@
Tags: - - + + edison - + failure - + inspirational - + paraphrased - +
@@ -182,10 +182,10 @@
Tags: - - + + misattributed-eleanor-roosevelt - +
@@ -196,73 +196,73 @@
Tags: - - + + humor - + obvious - + simile - +
- +

Top Ten tags

- + love - + inspirational - + life - + humor - + books - + reading - + friendship - + friends - + truth - + simile - - + +
diff --git a/docs/faq.rst b/docs/faq.rst index f81ec3601..cef3e69f3 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -410,7 +410,7 @@ How can I make a blank request? ------------------------------- .. code-block:: python - + from scrapy import Request diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 82a0e18c5..488a66f36 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -111,7 +111,7 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: To install Scrapy on Windows using ``pip``: .. warning:: - This installation method requires “Microsoft Visual C++†for installing some + This installation method requires “Microsoft Visual C++†for installing some Scrapy dependencies, which demands significantly more disk space than Anaconda. #. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. @@ -123,7 +123,7 @@ To install Scrapy on Windows using ``pip``: #. Check the installation details and make sure following packages are selected as optional components: * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) - + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) #. Install the Visual Studio Build Tools. diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6e6caebf1..5041b49ea 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -292,7 +292,7 @@ As an alternative, you could've written: >>> response.css("title::text")[0].get() 'Quotes to Scrape' -Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will +Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will raise an :exc:`IndexError` exception if there are no results: .. code-block:: pycon @@ -302,8 +302,8 @@ raise an :exc:`IndexError` exception if there are no results: ... IndexError: list index out of range -You might want to use ``.get()`` directly on the -:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` +You might want to use ``.get()`` directly on the +:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` if there are no results: .. code-block:: pycon diff --git a/docs/news.rst b/docs/news.rst index 8230c3aef..9a68f8852 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -934,10 +934,10 @@ Modified requirements Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` - instead of ``False``. In earlier Scrapy versions empty files were created - even when this setting was ``False`` (which was a bug that is now fixed), - so the new default should keep the old behavior. (:issue:`872`, +- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` + instead of ``False``. In earlier Scrapy versions empty files were created + even when this setting was ``False`` (which was a bug that is now fixed), + so the new default should keep the old behavior. (:issue:`872`, :issue:`5847`) Deprecation removals diff --git a/docs/topics/api.rst b/docs/topics/api.rst index f7cffb61b..edc625be8 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -88,7 +88,7 @@ how you :ref:`configure the downloader middlewares The execution engine, which coordinates the core crawling logic between the scheduler, downloader and spiders. - Some extension may want to access the Scrapy engine, to inspect or + Some extension may want to access the Scrapy engine, to inspect or modify the downloader and scheduler behaviour, although this is an advanced use and this API is not yet stable. diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 0370dc538..4e53b6e3d 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -87,8 +87,8 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The :ref:`scheduler ` receives requests from the engine and -enqueues them for feeding them later (also to the engine) when the engine +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine requests them. .. _component-downloader: diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 7a85c099b..5c078568b 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -224,7 +224,7 @@ BaseItemExporter .. [1] Not all exporters respect the specified field order. .. [2] When using :ref:`item objects ` that do not expose all their possible fields, exporters that do not support exporting - a different subset of fields per item will only export the fields + a different subset of fields per item will only export the fields found in the first item exported. .. attribute:: export_empty_fields diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index c47a3226a..23bbcfcb5 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -256,14 +256,14 @@ Spider state extension Manages spider state data by loading it before a crawl and saving it after. Give a value to the :setting:`JOBDIR` setting to enable this extension. -When enabled, this extension manages the :attr:`~scrapy.Spider.state` +When enabled, this extension manages the :attr:`~scrapy.Spider.state` attribute of your :class:`~scrapy.Spider` instance: - -- When your spider closes (:signal:`spider_closed`), the contents of its - :attr:`~scrapy.Spider.state` attribute are serialized into a file named + +- When your spider closes (:signal:`spider_closed`), the contents of its + :attr:`~scrapy.Spider.state` attribute are serialized into a file named ``spider.state`` in the :setting:`JOBDIR` folder. -- When your spider opens (:signal:`spider_opened`), if a previously-generated - ``spider.state`` file exists in the :setting:`JOBDIR` folder, it is loaded +- When your spider opens (:signal:`spider_opened`), if a previously-generated + ``spider.state`` file exists in the :setting:`JOBDIR` folder, it is loaded into the :attr:`~scrapy.Spider.state` attribute. @@ -291,8 +291,8 @@ settings: .. note:: - When a certain closing condition is met, requests which are - currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` + When a certain closing condition is met, requests which are + currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` requests) are still processed. .. setting:: CLOSESPIDER_TIMEOUT diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 07a3f3678..7f401f0c7 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -180,7 +180,7 @@ FTP supports two different connection modes: `active or passive mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. -The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. .. caution:: The value ``True`` in ``overwrite`` will cause you to lose the @@ -222,7 +222,7 @@ feeds using these settings: - :setting:`AWS_ENDPOINT_URL` - :setting:`AWS_REGION_NAME` -The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. .. caution:: The value ``True`` in ``overwrite`` will cause you to lose the @@ -255,7 +255,7 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following - :setting:`FEED_STORAGE_GCS_ACL` - :setting:`GCS_PROJECT_ID` -The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. .. caution:: The value ``True`` in ``overwrite`` will cause you to lose the @@ -587,8 +587,8 @@ FEED_STORE_EMPTY Default: ``True`` Whether to export empty feeds (i.e. feeds with no items). -If ``False``, and there are no items to export, no new files are created and -existing files are not modified, even if the :ref:`overwrite feed option +If ``False``, and there are no items to export, no new files are created and +existing files are not modified, even if the :ref:`overwrite feed option ` is enabled. .. setting:: FEED_STORAGES diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index fe1c4d162..a398d6c83 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -266,9 +266,9 @@ e.g. in the spider's ``__init__`` method: If you run this spider again then INFO messages from ``scrapy.spidermiddlewares.httperror`` logger will be gone. -You can also filter log records by :class:`~logging.LogRecord` data. For +You can also filter log records by :class:`~logging.LogRecord` data. For example, you can filter log records by message content using a substring or -a regular expression. Create a :class:`logging.Filter` subclass +a regular expression. Create a :class:`logging.Filter` subclass and equip it with a regular expression pattern to filter out unwanted messages: @@ -284,8 +284,8 @@ filter out unwanted messages: if match: return False -A project-level filter may be attached to the root -handler created by Scrapy, this is a wieldy way to +A project-level filter may be attached to the root +handler created by Scrapy, this is a wieldy way to filter all loggers in different parts of the project (middlewares, spider, etc.): @@ -301,7 +301,7 @@ filter all loggers in different parts of the project for handler in logging.root.handlers: handler.addFilter(ContentFilter()) -Alternatively, you may choose a specific logger +Alternatively, you may choose a specific logger and hide it without affecting other loggers: .. code-block:: python diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index f086a943e..cc1fe8703 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -414,7 +414,7 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key: and pipeline class MyPipeline will have expiration time set to 180. -The last modified time from the file is used to determine the age of the file in days, +The last modified time from the file is used to determine the age of the file in days, which is then compared to the set expiration time to determine if the file is expired. .. _topics-images-thumbnails: @@ -519,7 +519,7 @@ See here the methods that you can override in your custom Files Pipeline: In addition to ``response``, this method receives the original :class:`request `, - :class:`info ` and + :class:`info ` and :class:`item ` You can override this method to customize the download path of each file. @@ -541,9 +541,9 @@ See here the methods that you can override in your custom Files Pipeline: def file_path(self, request, response=None, info=None, *, item=None): return "files/" + PurePosixPath(urlparse_cached(request).path).name - Similarly, you can use the ``item`` to determine the file path based on some item + Similarly, you can use the ``item`` to determine the file path based on some item property. - + By default the :meth:`file_path` method returns ``full/.``. @@ -677,7 +677,7 @@ See here the methods that you can override in your custom Images Pipeline: In addition to ``response``, this method receives the original :class:`request `, - :class:`info ` and + :class:`info ` and :class:`item ` You can override this method to customize the download path of each file. @@ -699,9 +699,9 @@ See here the methods that you can override in your custom Images Pipeline: def file_path(self, request, response=None, info=None, *, item=None): return "files/" + PurePosixPath(urlparse_cached(request).path).name - Similarly, you can use the ``item`` to determine the file path based on some item + Similarly, you can use the ``item`` to determine the file path based on some item property. - + By default the :meth:`file_path` method returns ``full/.``. diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 5f6798601..db91cd073 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -309,7 +309,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. * use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy - plugin `__ and additional + plugin `__ and additional features, like `AI web scraping `__ If you are still unable to prevent your bot getting banned, consider contacting diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index b187f3aaf..55e8518a5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1309,7 +1309,7 @@ JsonResponse objects .. class:: JsonResponse(url[, ...]) - The :class:`JsonResponse` class is a subclass of :class:`TextResponse` - that is used when the response has a `JSON MIME type - `_ in its `Content-Type` + The :class:`JsonResponse` class is a subclass of :class:`TextResponse` + that is used when the response has a `JSON MIME type + `_ in its `Content-Type` header. diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index b95e6eab3..dbef07b73 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -559,7 +559,7 @@ For example, suppose you want to extract all ``

`` elements inside ``

`` elements. First, you would get all ``
`` elements: .. code-block:: pycon - + >>> divs = response.xpath("//div") At first, you may be tempted to use the following approach, which is wrong, as @@ -610,7 +610,7 @@ As it turns out, Scrapy selectors allow you to chain selectors, so most of the t you can just select by class using CSS and then switch to XPath when needed: .. code-block:: pycon - + >>> from scrapy import Selector >>> sel = Selector( ... text='
' @@ -1032,7 +1032,7 @@ whereas the CSS lookup is translated into XPath and thus runs more efficiently, so performance-wise its uses are limited to situations that are not easily described with CSS selectors. -Parsel also simplifies adding your own XPath extensions with +Parsel also simplifies adding your own XPath extensions with :func:`~parsel.xpathfuncs.set_xpathfunc`. .. _topics-selectors-ref: diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e1b1c5ad6..0a67240d6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -379,8 +379,8 @@ The above example can also be written as follows: def start_requests(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") -If you are :ref:`running Scrapy from a script `, you can -specify spider arguments when calling +If you are :ref:`running Scrapy from a script `, you can +specify spider arguments when calling :class:`CrawlerProcess.crawl ` or :class:`CrawlerRunner.crawl `: diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index be8ecb7a5..9572a3785 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -86,7 +86,7 @@ Available Stats Collectors Besides the basic :class:`StatsCollector` there are other Stats Collectors available in Scrapy which extend the basic Stats Collector. You can select which Stats Collector to use through the :setting:`STATS_CLASS` setting. The -default Stats Collector used is the :class:`MemoryStatsCollector`. +default Stats Collector used is the :class:`MemoryStatsCollector`. .. currentmodule:: scrapy.statscollectors diff --git a/sep/sep-004.rst b/sep/sep-004.rst index b1cef2600..7a4ebe886 100644 --- a/sep/sep-004.rst +++ b/sep/sep-004.rst @@ -11,7 +11,7 @@ SEP-004: Library API ==================== .. note:: the library API has been implemented, but slightly different from proposed in this SEP. You can run a Scrapy crawler inside a Twisted - reactor, but not outside it. + reactor, but not outside it. Introduction ============ diff --git a/sep/sep-007.rst b/sep/sep-007.rst index 0ca2036ce..73ce0d338 100644 --- a/sep/sep-007.rst +++ b/sep/sep-007.rst @@ -96,7 +96,7 @@ specified, else utf-8 is used) and returns a new unicode object. E.g: ``clean_spaces`` ---------------- - + Converts multispaces into single spaces for the given string. E.g: :: diff --git a/sep/sep-008.rst b/sep/sep-008.rst index be5987e39..1c38b1c40 100644 --- a/sep/sep-008.rst +++ b/sep/sep-008.rst @@ -73,8 +73,8 @@ Alternative Public API Proposal - ``ItemLoader.get_stored_values()`` or ``ItemLoader.get_values()`` *(returns the ``ItemLoader values)* - ``ItemLoader.get_output_value()`` -- ``ItemLoader.get_input_processor()`` or ``ItemLoader.get_in_processor()`` *(short version)* -- ``ItemLoader.get_output_processor()`` or ``ItemLoader.get_out_processor()`` *(short version)* +- ``ItemLoader.get_input_processor()`` or ``ItemLoader.get_in_processor()`` *(short version)* +- ``ItemLoader.get_output_processor()`` or ``ItemLoader.get_out_processor()`` *(short version)* - ``ItemLoader.context`` diff --git a/sep/sep-014.rst b/sep/sep-014.rst index e03a2b0f6..0a2e6b51e 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -21,7 +21,7 @@ Current flaws and inconsistencies 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) 3. Isn't possible to crawl an url directly from command line because the Spider - does not know which callback use. + does not know which callback use. These flaws will be corrected by the changes proposed in this SEP. @@ -55,7 +55,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. This is an enhancement to ``LinkExtractors`` which returns urls (links), -Request Extractors return Request objects. +Request Extractors return Request objects. Request Processors ------------------ diff --git a/sep/sep-018.rst b/sep/sep-018.rst index 13ab501ed..e6d601fe1 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -200,7 +200,7 @@ the same spider: # extract item from response return item -The Spider Middleware that implements spider code +The Spider Middleware that implements spider code ================================================= There's gonna be one middleware that will take care of calling the proper @@ -625,7 +625,7 @@ Resolved: not the original one (think of redirections), but it does carry the ``meta`` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be - the deserialized request from the persistent scheduler queue. + the deserialized request from the persistent scheduler queue. - No - this would make implementation more complex and we're not sure it's really needed From 9b7db1a068895254aae1618fb684baf9cb0c2784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 12:43:50 +0100 Subject: [PATCH 1685/2083] Move some reference docs of Request to the code (#6721) --- docs/topics/request-response.rst | 75 +++++++++++++------------------- scrapy/http/request/__init__.py | 49 +++++++++++++++++++++ 2 files changed, 80 insertions(+), 44 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 55e8518a5..0375e0ff1 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -31,23 +31,12 @@ Request objects If the URL is invalid, a :exc:`ValueError` exception is raised. :type url: str - :param callback: the function that will be called with the response of this - request (once it's downloaded) as its first parameter. + :param callback: sets :attr:`callback`, defaults to ``None``. - In addition to a function, the following values are supported: - - - ``None`` (default), which indicates that the spider's - :meth:`~scrapy.Spider.parse` method must be used. - - - :func:`~scrapy.http.request.NO_CALLBACK` - - For more information, see - :ref:`topics-request-response-ref-request-callback-arguments`. - - .. note:: If exceptions are raised during processing, ``errback`` is - called instead. - - :type callback: collections.abc.Callable + .. versionchanged:: 2.0 + The *callback* parameter is no longer required when the *errback* + parameter is specified. + :type callback: Callable[Concatenate[Response, ...], Any] | None :param method: the HTTP method of this request. Defaults to ``'GET'``. :type method: str @@ -144,23 +133,15 @@ Request objects Negative values are allowed in order to indicate relatively low-priority. :type priority: int - :param dont_filter: indicates that this request should not be filtered by - the scheduler or some middlewares. This is used when you want to perform - an identical request multiple times, to ignore the duplicates filter. - Use it with care, or you will get into crawling loops. Default to ``False``. + :param dont_filter: sets :attr:`dont_filter`, defaults to ``False``. :type dont_filter: bool - :param errback: a function that will be called if any exception was - raised while processing the request. This includes pages that failed - with 404 HTTP errors and such. It receives a - :exc:`~twisted.python.failure.Failure` as first parameter. - For more information, - see :ref:`topics-request-response-ref-errbacks` below. + :param errback: sets :attr:`errback`, defaults to ``None``. - .. versionchanged:: 2.0 - The *callback* parameter is no longer required when the *errback* - parameter is specified. - :type errback: collections.abc.Callable + .. versionchanged:: 2.0 + The *callback* parameter is no longer required when the *errback* + parameter is specified. + :type errback: Callable[[Failure], Any] | None :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list @@ -194,6 +175,25 @@ Request objects This attribute is read-only. To change the body of a Request use :meth:`replace`. + .. autoattribute:: callback + + .. autoattribute:: errback + + .. attribute:: Request.cb_kwargs + + A dictionary that contains arbitrary metadata for this request. Its contents + will be passed to the Request's callback as keyword arguments. It is empty + for new Requests, which means by default callbacks only get a + :class:`~scrapy.http.Response` object as argument. + + This dict is :doc:`shallow copied ` when the request is + cloned using the ``copy()`` or ``replace()`` methods, and can also be + accessed, in your spider, from the ``response.cb_kwargs`` attribute. + + In case of a failure to process the request, this dict can be accessed as + ``failure.request.cb_kwargs`` in the request's errback. For more information, + see :ref:`errback-cb_kwargs`. + .. attribute:: Request.meta :value: {} @@ -237,20 +237,7 @@ Request objects Also mind that the :meth:`copy` and :meth:`replace` request methods :doc:`shallow-copy ` request metadata. - .. attribute:: Request.cb_kwargs - - A dictionary that contains arbitrary metadata for this request. Its contents - will be passed to the Request's callback as keyword arguments. It is empty - for new Requests, which means by default callbacks only get a - :class:`~scrapy.http.Response` object as argument. - - This dict is :doc:`shallow copied ` when the request is - cloned using the ``copy()`` or ``replace()`` methods, and can also be - accessed, in your spider, from the ``response.cb_kwargs`` attribute. - - In case of a failure to process the request, this dict can be accessed as - ``failure.request.cb_kwargs`` in the request's errback. For more information, - see :ref:`errback-cb_kwargs`. + .. autoattribute:: dont_filter .. autoattribute:: Request.attributes diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index e24f6874d..6d3b7a926 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -138,11 +138,60 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") + + #: :class:`~collections.abc.Callable` to parse the + #: :class:`~scrapy.http.Response` to this request once received. + #: + #: The callable must expect the response as its first parameter, and + #: support any additional keyword arguments set through + #: :attr:`cb_kwargs`. + #: + #: In addition to an arbitrary callable, the following values are also + #: supported: + #: + #: - ``None`` (default), which indicates that the + #: :meth:`~scrapy.Spider.parse` method of the spider must be used. + #: + #: - :func:`~scrapy.http.request.NO_CALLBACK`. + #: + #: If an unhandled exception is raised during request or response + #: processing, i.e. by a :ref:`spider middleware + #: `, :ref:`downloader middleware + #: ` or download handler + #: (:setting:`DOWNLOAD_HANDLERS`), :attr:`errback` is called instead. + #: + #: .. tip:: + #: :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware` + #: raises exceptions for non-2xx responses by default, sending them + #: to the :attr:`errback` instead. + #: + #: .. seealso:: + #: :ref:`topics-request-response-ref-request-callback-arguments` self.callback: CallbackT | None = callback + + #: :class:`~collections.abc.Callable` to handle exceptions raised + #: during request or response processing. + #: + #: The callable must expect a :exc:`~twisted.python.failure.Failure` as + #: its first parameter. + #: + #: .. seealso:: :ref:`topics-request-response-ref-errbacks` self.errback: Callable[[Failure], Any] | None = errback self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) + + #: Whether this request may be filtered out by :ref:`components + #: ` that support filtering out requests (``False``, + #: default), or those components should not filter out this request + #: (``True``). + #: + #: This attribute is commonly set to ``True`` to prevent duplicate + #: requests from being filtered out. + #: + #: When defining the start URLs of a spider through + #: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by + #: default. See :meth:`~scrapy.Spider.start_requests`. self.dont_filter: bool = dont_filter self._meta: dict[str, Any] | None = dict(meta) if meta else None From 26ecc93228bc3dbb9d62daa419cb344ba6f88caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 14:12:11 +0100 Subject: [PATCH 1686/2083] Run CI only on the main branch, on release branches and on PRs (#6720) --- .github/workflows/checks.yml | 7 ++++++- .github/workflows/tests-macos.yml | 7 ++++++- .github/workflows/tests-ubuntu.yml | 7 ++++++- .github/workflows/tests-windows.yml | 7 ++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a064bf5b2..312af3b2e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,5 +1,10 @@ name: Checks -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index c28a99982..ce0e1a6c2 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,5 +1,10 @@ name: macOS -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 6c7842217..f74575ee1 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,5 +1,10 @@ name: Ubuntu -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 45e4ca157..21d621240 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,5 +1,10 @@ name: Windows -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} From 5a0690c89d718b33bd63c1cd724c50c9ceb809e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 14:52:48 +0100 Subject: [PATCH 1687/2083] Remove or post-pone the use of itemadapter.is_item, as a potentially expensive call (#6719) --- docs/faq.rst | 14 +++++++------ docs/topics/items.rst | 5 ++--- scrapy/commands/parse.py | 8 ++++---- scrapy/core/engine.py | 9 +-------- scrapy/core/scraper.py | 10 +--------- scrapy/exporters.py | 4 ++-- .../project/module/middlewares.py.tmpl | 2 +- scrapy/utils/serialize.py | 4 ++-- tests/test_crawl.py | 20 ++++++------------- 9 files changed, 27 insertions(+), 49 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index cef3e69f3..da255f29e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -361,16 +361,18 @@ method for this purpose. For example: from copy import deepcopy - from itemadapter import is_item, ItemAdapter + from itemadapter import ItemAdapter + from scrapy import Request class MultiplyItemsMiddleware: def process_spider_output(self, response, result, spider): - for item in result: - if is_item(item): - adapter = ItemAdapter(item) - for _ in range(adapter["multiply_by"]): - yield deepcopy(item) + for item_or_request in result: + if isinstance(item_or_request, Request): + continue + adapter = ItemAdapter(item) + for _ in range(adapter["multiply_by"]): + yield deepcopy(item) Does Scrapy support IPv6 addresses? ----------------------------------- diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 7cc476863..0365c95b3 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -384,9 +384,8 @@ Supporting All Item Types In code that receives an item, such as methods of :ref:`item pipelines ` or :ref:`spider middlewares `, it is a good practice to use the -:class:`~itemadapter.ItemAdapter` class and the -:func:`~itemadapter.is_item` function to write code that works for -any supported item type. +:class:`~itemadapter.ItemAdapter` class to write code that works for any +supported item type. Other classes related to items ============================== diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 61aea3ee4..c6ed20b3b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -6,7 +6,7 @@ import json import logging from typing import TYPE_CHECKING, Any, TypeVar, overload -from itemadapter import ItemAdapter, is_item +from itemadapter import ItemAdapter from twisted.internet.defer import Deferred, maybeDeferred from w3lib.url import is_url @@ -211,10 +211,10 @@ class Command(BaseRunSpiderCommand): ) -> tuple[list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT]: items, requests = [], [] for x in spider_output: - if is_item(x): - items.append(x) - elif isinstance(x, Request): + if isinstance(x, Request): requests.append(x) + else: + items.append(x) return items, requests, opts, depth, spider, callback def run_callback( diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 61f444e31..b7a73700b 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -11,7 +11,6 @@ import logging from time import time from typing import TYPE_CHECKING, Any, TypeVar, cast -from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -194,14 +193,8 @@ class ExecutionEngine: else: if isinstance(request_or_item, Request): self.crawl(request_or_item) - elif is_item(request_or_item): - self.scraper.start_itemproc(request_or_item, response=None) else: - logger.error( - f"Got {request_or_item!r} among start requests. Only " - f"requests and items are supported. It will be " - f"ignored." - ) + self.scraper.start_itemproc(request_or_item, response=None) if self.spider_is_idle() and self.slot.close_if_idle: self._spider_idle() diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 03301717d..b664b61f6 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -8,7 +8,6 @@ from collections import deque from collections.abc import AsyncIterable, Iterator from typing import TYPE_CHECKING, Any, TypeVar, Union, cast -from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -298,17 +297,10 @@ class Scraper: if isinstance(output, Request): assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) - elif is_item(output): - return self.start_itemproc(output, response=response) elif output is None: pass else: - typename = type(output).__name__ - logger.error( - "Spider must return request, item, or None, got %(typename)r in %(request)s", - {"request": request, "typename": typename}, - extra={"spider": spider}, - ) + return self.start_itemproc(output, response=response) return None def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[Any]: diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 46c6aa3fa..0a641752e 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -356,12 +356,12 @@ class PythonItemExporter(BaseItemExporter): def _serialize_value(self, value: Any) -> Any: if isinstance(value, Item): return self.export_item(value) + if isinstance(value, (str, bytes)): + return to_unicode(value, encoding=self.encoding) if is_item(value): return dict(self._serialize_item(value)) if is_listlike(value): return [self._serialize_value(v) for v in value] - if isinstance(value, (str, bytes)): - return to_unicode(value, encoding=self.encoding) return value def _serialize_item(self, item: Any) -> Iterable[tuple[str | bytes, Any]]: diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 8c9a86dce..dcb2d63de 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -6,7 +6,7 @@ from scrapy import signals # useful for handling different item types with a single interface -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter class ${ProjectName}SpiderMiddleware: diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 308e351c6..bcfae0c00 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -28,12 +28,12 @@ class ScrapyJSONEncoder(json.JSONEncoder): return str(o) if isinstance(o, defer.Deferred): return str(o) - if is_item(o): - return ItemAdapter(o).asdict() if isinstance(o, Request): return f"<{type(o).__name__} {o.method} {o.url}>" if isinstance(o, Response): return f"<{type(o).__name__} {o.status} {o.url}>" + if is_item(o): + return ItemAdapter(o).asdict() return super().default(o) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 5766f9313..6f4045fc8 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,5 @@ import json import logging -import re import unittest from ipaddress import IPv4Address from socket import gethostbyname @@ -195,23 +194,16 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_requests_unsupported_output(self): + """Anything that is not a request is assumed to be an item, avoiding a + potentially expensive call to itemadapter.is_item, and letting instead + things fail when ItemAdapter is actually used on the corresponding + non-item object.""" + with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(StartRequestsGoodAndBadOutput) yield crawler.crawl(mockserver=self.mockserver) - assert len(log.records) == 2 - assert log.records[0].msg == ( - "Got 'data:,b' among start requests. Only requests and items " - "are supported. It will be ignored." - ) - assert re.match( - ( - r"^Got among start " - r"requests\. Only requests and items are supported\. It " - r"will be ignored\.$" - ), - log.records[1].msg, - ) + assert len(log.records) == 0 @defer.inlineCallbacks def test_start_requests_laziness(self): From ba28d96d3ef2488e9c5c86d4a5c28a1aac269a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 14:53:04 +0100 Subject: [PATCH 1688/2083] Centralize from_crawler docs (and somewhat related changes) (#6723) --- docs/topics/addons.rst | 17 +----- docs/topics/api.rst | 9 +-- docs/topics/components.rst | 80 +++++++++++++++++++++++++-- docs/topics/downloader-middleware.rst | 19 +------ docs/topics/email.rst | 14 +---- docs/topics/extensions.rst | 80 ++++++--------------------- docs/topics/item-pipeline.rst | 18 ++---- docs/topics/request-response.rst | 24 +------- docs/topics/settings.rst | 39 ++++--------- docs/topics/spider-middleware.rst | 69 +++++++++-------------- 10 files changed, 150 insertions(+), 219 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 8ec7b0295..17e3c177a 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -32,7 +32,8 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are Python classes that include one or both of the following methods: +Add-ons are :ref:`components ` that include one or both of +the following methods: .. method:: update_settings(settings) @@ -54,20 +55,6 @@ Add-ons are Python classes that include one or both of the following methods: :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.BaseSettings` -They can also have the following method: - -.. classmethod:: from_crawler(cls, crawler) - :noindex: - - If present, this class method is called to create an add-on instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the add-on. The crawler object provides access to all Scrapy core - components like settings and signals; it is a way for the add-on to access - them and hook its functionality into Scrapy. - - :param crawler: The crawler that uses this add-on - :type crawler: :class:`~scrapy.crawler.Crawler` - The settings set by the add-on should use the ``addon`` priority (see :ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: diff --git a/docs/topics/api.rst b/docs/topics/api.rst index edc625be8..5a00fd570 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -12,10 +12,11 @@ extensions and middlewares. Crawler API =========== -The main entry point to Scrapy API is the :class:`~scrapy.crawler.Crawler` -object, passed to extensions through the ``from_crawler`` class method. This -object provides access to all Scrapy core components, and it's the only way for -extensions to access them and hook their functionality into Scrapy. +The main entry point to the Scrapy API is the :class:`~scrapy.crawler.Crawler` +object, which :ref:`components ` can :ref:`get for +initialization `. It provides access to all Scrapy core +components, and it is the only way for components to access them and hook their +functionality into Scrapy. .. module:: scrapy.crawler :synopsis: The Scrapy crawler diff --git a/docs/topics/components.rst b/docs/topics/components.rst index d34b3884b..3a7644379 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -9,6 +9,8 @@ A Scrapy component is any class whose objects are built using That includes the classes that you may assign to the following settings: +- :setting:`ADDONS` + - :setting:`DNS_RESOLVER` - :setting:`DOWNLOAD_HANDLERS` @@ -41,10 +43,80 @@ Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to modify their behavior. +.. _from-crawler: + +Initializing from the crawler +============================= + +Any Scrapy component may optionally define the following class method: + +.. classmethod:: from_crawler(cls, crawler: scrapy.crawler.Crawler, *args, **kwargs) + + Return an instance of the component based on *crawler*. + + *args* and *kwargs* are component-specific arguments that some components + receive. However, most components do not get any arguments, and instead + :ref:`use settings `. + + If a component class defines this method, this class method is called to + create any instance of the component. + + The *crawler* object provides access to all Scrapy core components like + :ref:`settings ` and :ref:`signals `, + allowing the component to access them and hook its functionality into + Scrapy. + +.. _component-settings: + +Settings +======== + +Components can be configured through :ref:`settings `. + +Components can read any setting from the +:attr:`~scrapy.crawler.Crawler.settings` attribute of the +:class:`~scrapy.crawler.Crawler` object they can :ref:`get for initialization +`. That includes both built-in and custom settings. + +For example: + +.. code-block:: python + + class MyExtension: + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + return cls(settings.getbool("LOG_ENABLED")) + + def __init__(self, log_is_enabled=False): + if log_is_enabled: + print("log is enabled!") + +Components do not need to declare their custom settings programmatically. +However, they should document them, so that users know they exist and how to +use them. + +It is a good practice to prefix custom settings with the name of the component, +to avoid collisions with custom settings of other existing (or future) +components. For example, an extension called ``WarcCaching`` could prefix its +custom settings with ``WARC_CACHING_``. + +Another good practice, mainly for components meant for :ref:`component priority +dictionaries `, is to provide a boolean setting +called ``_ENABLED`` (e.g. ``WARC_CACHING_ENABLED``) to allow toggling +that component on and off without changing the component priority dictionary +setting. You can usually check the value of such a setting during +initialization, and if ``False``, raise +:exc:`~scrapy.exceptions.NotConfigured`. + +When choosing a name for a custom setting, it is also a good idea to have a +look at the names of :ref:`built-in settings `, to try to +maintain consistency with them. + .. _enforce-component-requirements: -Enforcing component requirements -================================ +Enforcing requirements +====================== Sometimes, your components may only be intended to work under certain conditions. For example, they may require a minimum version of Scrapy to work as @@ -58,8 +130,8 @@ In the case of :ref:`downloader middlewares `, :ref:`extensions `, :ref:`item pipelines `, and :ref:`spider middlewares `, you should raise -:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a -parameter to the exception so that it is printed in the logs, for the user to +:exc:`~scrapy.exceptions.NotConfigured`, passing a description of the issue as +a parameter to the exception so that it is printed in the logs, for the user to see. For other components, feel free to raise whatever other exception feels right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy version mismatch, while :exc:`ValueError` may be better if the issue is the diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index ab7e6a0ec..60b6aab78 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -61,12 +61,8 @@ particular setting. See each middleware documentation for more info. Writing your own downloader middleware ====================================== -Each downloader middleware is a Python class that defines one or more of the -methods defined below. - -The main entry point is the ``from_crawler`` class method, which receives a -:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` -object gives you access, for example, to the :ref:`settings `. +Each downloader middleware is a :ref:`component ` that +defines one or more of these methods: .. module:: scrapy.downloadermiddlewares @@ -167,17 +163,6 @@ object gives you access, for example, to the :ref:`settings `. :param spider: the spider for which this request is intended :type spider: :class:`~scrapy.Spider` object - .. method:: from_crawler(cls, crawler) - - If present, this classmethod is called to create a middleware instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the middleware. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for middleware to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this middleware - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. _topics-downloader-middleware-ref: Built-in downloader middleware reference diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 8f7a2357a..1d7bad787 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -50,9 +50,9 @@ And here is how to use it to send an e-mail (without attachments): MailSender class reference ========================== -MailSender is the preferred class to use for sending emails from Scrapy, as it -uses :doc:`Twisted non-blocking IO `, like the -rest of the framework. +The MailSender :ref:`components ` is the preferred class to +use for sending emails from Scrapy, as it uses :doc:`Twisted non-blocking IO +`, like the rest of the framework. .. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) @@ -81,14 +81,6 @@ rest of the framework. :param smtpssl: enforce using a secure SSL connection :type smtpssl: bool - .. classmethod:: from_crawler(crawler) - - Instantiate using a :class:`scrapy.Crawler` instance, which will - respect :ref:`these Scrapy settings `. - - :param crawler: the crawler - :type settings: :class:`scrapy.Crawler` object - .. method:: send(to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None) Send email to the given recipients. diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 23bbcfcb5..e1e3dd6b4 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -4,34 +4,21 @@ Extensions ========== -The extensions framework provides a mechanism for inserting your own -custom functionality into Scrapy. +Extensions are :ref:`components ` that allow inserting your +own custom functionality into Scrapy. -Extensions are just regular classes. +Unlike other components, extensions do not have a specific role in Scrapy. They +are “wildcard†components that can be used for anything that does not fit the +role of any other type of component. -Extension settings -================== +Loading and activating extensions +================================= -Extensions use the :ref:`Scrapy settings ` to manage their -settings, just like any other Scrapy code. +Extensions are loaded at startup by creating a single instance of the extension +class per spider being run. -It is customary for extensions to prefix their settings with their own name, to -avoid collision with existing (and future) extensions. For example, a -hypothetical extension to handle `Google Sitemaps`_ would use settings like -``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on. - -.. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps - -Loading & activating extensions -=============================== - -Extensions are loaded and activated at startup by instantiating a single -instance of the extension class per spider being run. All the extension -initialization code must be performed in the class ``__init__`` method. - -To make an extension available, add it to the :setting:`EXTENSIONS` setting in -your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented -by a string: the full Python path to the extension's class name. For example: +To enable an extension, add it to the :setting:`EXTENSIONS` setting. For +example: .. code-block:: python @@ -40,55 +27,24 @@ by a string: the full Python path to the extension's class name. For example: "scrapy.extensions.telnet.TelnetConsole": 500, } - -As you can see, the :setting:`EXTENSIONS` setting is a dict where the keys are -the extension paths, and their values are the orders, which define the -extension *loading* order. The :setting:`EXTENSIONS` setting is merged with the -:setting:`EXTENSIONS_BASE` setting defined in Scrapy (and not meant to be -overridden) and then sorted by order to get the final sorted list of enabled -extensions. +:setting:`EXTENSIONS` is merged with :setting:`EXTENSIONS_BASE` (not meant to +be overridden), and the priorities in the resulting value determine the +*loading* order. As extensions typically do not depend on each other, their loading order is irrelevant in most cases. This is why the :setting:`EXTENSIONS_BASE` setting -defines all extensions with the same order (``0``). However, this feature can -be exploited if you need to add an extension which depends on other extensions -already loaded. - -Available, enabled and disabled extensions -========================================== - -Not all available extensions will be enabled. Some of them usually depend on a -particular setting. For example, the HTTP Cache extension is available by default -but disabled unless the :setting:`HTTPCACHE_ENABLED` setting is set. - -Disabling an extension -====================== - -In order to disable an extension that comes enabled by default (i.e. those -included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to -``None``. For example: - -.. code-block:: python - - EXTENSIONS = { - "scrapy.extensions.corestats.CoreStats": None, - } +defines all extensions with the same order (``0``). However, you may need to +carefully use priorities if you add an extension that depends on other +extensions being already loaded. Writing your own extension ========================== -Each extension is a Python class. The main entry point for a Scrapy extension -(this also includes middlewares and pipelines) is the ``from_crawler`` -class method which receives a ``Crawler`` instance. Through the Crawler object -you can access settings, signals, stats, and also control the crawling behaviour. +Each extension is a :ref:`component `. Typically, extensions connect to :ref:`signals ` and perform tasks triggered by them. -Finally, if the ``from_crawler`` method raises the -:exc:`~scrapy.exceptions.NotConfigured` exception, the extension will be -disabled. Otherwise, the extension will be enabled. - Sample extension ---------------- diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 310f153e8..dc27ce6ca 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -23,7 +23,8 @@ Typical uses of item pipelines are: Writing your own item pipeline ============================== -Each item pipeline component is a Python class that must implement the following method: +Each item pipeline is a :ref:`component ` that must +implement the following method: .. method:: process_item(self, item, spider) @@ -60,17 +61,6 @@ Additionally, they may also implement the following methods: :param spider: the spider which was closed :type spider: :class:`~scrapy.Spider` object -.. classmethod:: from_crawler(cls, crawler) - - If present, this class method is called to create a pipeline instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the pipeline. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for pipeline to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this pipeline - :type crawler: :class:`~scrapy.crawler.Crawler` object - Item pipeline example ===================== @@ -139,8 +129,8 @@ In this example we'll write items to MongoDB_ using pymongo_. MongoDB address and database name are specified in Scrapy settings; MongoDB collection is named after item class. -The main point of this example is to show how to use :meth:`from_crawler` -method and how to clean up the resources properly. +The main point of this example is to show how to :ref:`get the crawler +` and how to clean up the resources properly. .. skip: next .. code-block:: python diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 0375e0ff1..77837378e 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -463,35 +463,17 @@ import path. Writing your own request fingerprinter ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A request fingerprinter is a class that must implement the following method: +A request fingerprinter is a :ref:`component ` that must +implement the following method: .. currentmodule:: None -.. method:: fingerprint(self, request) +.. method:: fingerprint(self, request: scrapy.Request) Return a :class:`bytes` object that uniquely identifies *request*. See also :ref:`request-fingerprint-restrictions`. - :param request: request to fingerprint - :type request: scrapy.Request - -Additionally, it may also implement the following method: - -.. classmethod:: from_crawler(cls, crawler) - :noindex: - - If present, this class method is called to create a request fingerprinter - instance from a :class:`~scrapy.crawler.Crawler` object. It must return a - new instance of the request fingerprinter. - - *crawler* provides access to all Scrapy core components like settings and - signals; it is a way for the request fingerprinter to access them and hook - its functionality into Scrapy. - - :param crawler: crawler that uses this request fingerprinter - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. currentmodule:: scrapy.http The :meth:`fingerprint` method of the default request fingerprinter, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7646aca4f..b2bb7148f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -204,7 +204,7 @@ How to access settings .. highlight:: python -In a spider, the settings are available through ``self.settings``: +In a spider, settings are available through ``self.settings``: .. code-block:: python @@ -217,37 +217,17 @@ In a spider, the settings are available through ``self.settings``: .. note:: The ``settings`` attribute is set in the base Spider class after the spider - is initialized. If you want to use the settings before the initialization + is initialized. If you want to use settings before the initialization (e.g., in your spider's ``__init__()`` method), you'll need to override the :meth:`~scrapy.Spider.from_crawler` method. -Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` -attribute of the Crawler that is passed to ``from_crawler`` method in -extensions, middlewares and item pipelines: +:ref:`Components ` can also :ref:`access settings +`. -.. code-block:: python - - class MyExtension: - def __init__(self, log_is_enabled=False): - if log_is_enabled: - print("log is enabled!") - - @classmethod - def from_crawler(cls, crawler): - settings = crawler.settings - return cls(settings.getbool("LOG_ENABLED")) - -The settings object can be used like a dict (e.g., -``settings['LOG_ENABLED']``), but it's usually preferred to extract the setting -in the format you need it to avoid type errors, using one of the methods -provided by the :class:`~scrapy.settings.Settings` API. - -Rationale for setting names -=========================== - -Setting names are usually prefixed with the component that they configure. For -example, proper setting names for a fictional robots.txt extension would be -``ROBOTSTXT_ENABLED``, ``ROBOTSTXT_OBEY``, ``ROBOTSTXT_CACHEDIR``, etc. +The ``settings`` object can be used like a :class:`dict` (e.g. +``settings["LOG_ENABLED"]``). However, to support non-string setting values, +which may be passed from the command line as strings, it is recommended to use +one of the methods provided by the :class:`~scrapy.settings.Settings` API. .. _component-priority-dictionaries: @@ -1211,7 +1191,8 @@ EXTENSIONS Default:: ``{}`` -A dict containing the extensions enabled in your project, and their orders. +:ref:`Component priority dictionary ` of +enabled extensions. See :ref:`topics-extensions`. .. setting:: EXTENSIONS_BASE diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b59cabe1..567a875b6 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -63,17 +63,38 @@ particular setting. See each middleware documentation for more info. Writing your own spider middleware ================================== -Each spider middleware is a Python class that defines one or more of the -methods defined below. - -The main entry point is the ``from_crawler`` class method, which receives a -:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` -object gives you access, for example, to the :ref:`settings `. +Each spider middleware is a :ref:`component ` that defines +one or more of these methods: .. module:: scrapy.spidermiddlewares .. class:: SpiderMiddleware + .. method:: process_start_requests(start_requests, spider) + + This method is called with the start requests of the spider, and works + similarly to the :meth:`process_spider_output` method, except that it + doesn't have a response associated and must return only requests (not + items). + + It receives an iterable (in the ``start_requests`` parameter) and must + return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects `. + + .. note:: When implementing this method in your spider middleware, you + should always return an iterable (that follows the input one) and + not consume all ``start_requests`` iterator because it can be very + large (or even unbounded) and cause a memory overflow. The Scrapy + engine is designed to pull start requests while it has capacity to + process them, so the start requests iterator can be effectively + endless where there is some other condition for stopping the spider + (like a time limit or item/page count). + + :param start_requests: the start requests + :type start_requests: an iterable of :class:`~scrapy.Request` + + :param spider: the spider to whom the start requests belong + :type spider: :class:`~scrapy.Spider` object + .. method:: process_spider_input(response, spider) This method is called for each response that goes through the spider @@ -168,42 +189,6 @@ object gives you access, for example, to the :ref:`settings `. :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object - .. method:: process_start_requests(start_requests, spider) - - This method is called with the start requests of the spider, and works - similarly to the :meth:`process_spider_output` method, except that it - doesn't have a response associated and must return only requests (not - items). - - It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects `. - - .. note:: When implementing this method in your spider middleware, you - should always return an iterable (that follows the input one) and - not consume all ``start_requests`` iterator because it can be very - large (or even unbounded) and cause a memory overflow. The Scrapy - engine is designed to pull start requests while it has capacity to - process them, so the start requests iterator can be effectively - endless where there is some other condition for stopping the spider - (like a time limit or item/page count). - - :param start_requests: the start requests - :type start_requests: an iterable of :class:`~scrapy.Request` - - :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.Spider` object - - .. method:: from_crawler(cls, crawler) - - If present, this classmethod is called to create a middleware instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the middleware. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for middleware to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this middleware - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. _topics-spider-middleware-ref: Built-in spider middleware reference From 803b4f258d85ab4f85c7c230e6c025f54e836269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 14:53:42 +0100 Subject: [PATCH 1689/2083] tox: move to posargs pytest parameters that can be too noisy when running specific tests (#6724) --- tox.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 041fcffca..70c841603 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml {posargs:--durations=10 docs scrapy tests} --doctest-modules + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --durations=10 docs scrapy tests --doctest-modules} install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} @@ -58,7 +58,7 @@ deps = pytest >= 8.2.0 w3lib >= 2.2.0 commands = - mypy {posargs: scrapy tests} + mypy {posargs:scrapy tests} [testenv:typing-tests] basepython = python3.9 @@ -67,7 +67,7 @@ deps = {[testenv:typing]deps} pytest-mypy-testing==0.1.3 commands = - pytest {posargs: tests_typing} + pytest {posargs:tests_typing} [testenv:pre-commit] basepython = python3 @@ -119,7 +119,7 @@ install_command = python -I -m pip install {opts} {packages} commands = ; tests for docs fail with parsel < 1.8.0 - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --durations=10 scrapy tests} [testenv:pinned] basepython = {[pinned]basepython} @@ -266,7 +266,7 @@ deps = {[testenv]deps} botocore>=1.4.87 commands = - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests -m requires_botocore} [testenv:botocore-pinned] basepython = {[pinned]basepython} @@ -277,4 +277,4 @@ install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests -m requires_botocore} From eb654aa1a8d2ef6433957fcc1361420b6141094e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Mar 2025 21:00:36 +0400 Subject: [PATCH 1690/2083] Convert remaining unittest assert* calls, use the tmp_path fixture. (#6725) --- pyproject.toml | 4 -- scrapy/utils/test.py | 2 +- tests/test_downloadermiddleware_robotstxt.py | 13 ++++- tests/test_engine.py | 8 ++- tests/test_pipeline_files.py | 58 ++++++++------------ tests/test_pipeline_images.py | 44 +++++++-------- tests/test_utils_template.py | 16 +----- 7 files changed, 62 insertions(+), 83 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 82d8056f6..84bf41a94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -374,10 +374,6 @@ ignore = [ "B904", # Use capitalized environment variable "SIM112", - - # Temporarily silenced PT rules - # Use a regular `assert` instead of unittest-style `assertEqual` - "PT009", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index db1f5c419..b69f43438 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -156,7 +156,7 @@ def assert_samelines( category=ScrapyDeprecationWarning, stacklevel=2, ) - testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) + testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) # noqa: PT009 def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 38f0333bb..ad335f852 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,3 +1,4 @@ +from typing import Any from unittest import mock import pytest @@ -171,7 +172,11 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self.crawler) middleware._logerror = mock.MagicMock(side_effect=middleware._logerror) deferred = middleware.process_request(Request("http://site.local"), None) - deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called)) + + def check_called(_: Any) -> None: + assert middleware._logerror.called + + deferred.addCallback(check_called) return deferred def test_robotstxt_immediate_error(self): @@ -202,7 +207,11 @@ Disallow: /some/randome/page.html mw_module_logger.error = mock.MagicMock() d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called)) + + def check_not_called(_: Any) -> None: + assert not mw_module_logger.error.called # type: ignore[attr-defined] + + d.addCallback(check_not_called) return d def test_robotstxt_user_agent_setting(self): diff --git a/tests/test_engine.py b/tests/test_engine.py index 4bac8d273..ba4c6dc40 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -433,10 +433,12 @@ class TestEngine(TestEngineBase): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.open_spider(MySpider(), []) e.start() + + def cb(exc: BaseException) -> None: + assert str(exc), "Engine already running" + try: - yield self.assertFailure(e.start(), RuntimeError).addBoth( - lambda exc: self.assertEqual(str(exc), "Engine already running") - ) + yield self.assertFailure(e.start(), RuntimeError).addBoth(cb) finally: yield e.stop() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e515c16a0..9a582e4b7 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -266,17 +266,11 @@ class TestFilesPipeline(unittest.TestCase): class FilesPipelineTestCaseFieldsMixin: - def setup_method(self): - self.tempdir = mkdtemp() - - def teardown_method(self): - rmtree(self.tempdir) - - def test_item_fields_default(self): + def test_item_fields_default(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) pipeline = FilesPipeline.from_crawler( - get_crawler(None, {"FILES_STORE": self.tempdir}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) requests = list(pipeline.get_media_requests(item, None)) assert requests[0].url == url @@ -286,14 +280,14 @@ class FilesPipelineTestCaseFieldsMixin: assert files == [results[0][1]] assert isinstance(item, self.item_class) - def test_item_fields_override_settings(self): + def test_item_fields_override_settings(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", custom_file_urls=[url]) pipeline = FilesPipeline.from_crawler( get_crawler( None, { - "FILES_STORE": self.tempdir, + "FILES_STORE": tmp_path, "FILES_URLS_FIELD": "custom_file_urls", "FILES_RESULT_FIELD": "custom_files", }, @@ -368,13 +362,7 @@ class TestFilesPipelineCustomSettings: ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"), } - def setup_method(self): - self.tempdir = mkdtemp() - - def teardown_method(self): - rmtree(self.tempdir) - - def _generate_fake_settings(self, prefix=None): + def _generate_fake_settings(self, tmp_path, prefix=None): def random_string(): return "".join([chr(random.randint(97, 123)) for _ in range(10)]) @@ -382,7 +370,7 @@ class TestFilesPipelineCustomSettings: "FILES_EXPIRES": random.randint(100, 1000), "FILES_URLS_FIELD": random_string(), "FILES_RESULT_FIELD": random_string(), - "FILES_STORE": self.tempdir, + "FILES_STORE": tmp_path, } if not prefix: return settings @@ -400,16 +388,16 @@ class TestFilesPipelineCustomSettings: return UserDefinedFilePipeline - def test_different_settings_for_different_instances(self): + def test_different_settings_for_different_instances(self, tmp_path): """ If there are different instances with different settings they should keep different settings. """ - custom_settings = self._generate_fake_settings() + custom_settings = self._generate_fake_settings(tmp_path) another_pipeline = FilesPipeline.from_crawler( get_crawler(None, custom_settings) ) - one_pipeline = FilesPipeline(self.tempdir, crawler=get_crawler(None)) + one_pipeline = FilesPipeline(tmp_path, crawler=get_crawler(None)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: default_value = self.default_cls_settings[pipe_attr] assert getattr(one_pipeline, pipe_attr) == default_value @@ -417,24 +405,24 @@ class TestFilesPipelineCustomSettings: assert default_value != custom_value assert getattr(another_pipeline, pipe_ins_attr) == custom_value - def test_subclass_attributes_preserved_if_no_settings(self): + def test_subclass_attributes_preserved_if_no_settings(self, tmp_path): """ If subclasses override class attributes and there are no special settings those values should be kept. """ pipe_cls = self._generate_fake_pipeline() - pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": self.tempdir})) + pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path})) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: custom_value = getattr(pipe, pipe_ins_attr) assert custom_value != self.default_cls_settings[pipe_attr] assert getattr(pipe, pipe_ins_attr) == getattr(pipe, pipe_attr) - def test_subclass_attrs_preserved_custom_settings(self): + def test_subclass_attrs_preserved_custom_settings(self, tmp_path): """ If file settings are defined but they are not defined for subclass settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline() - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: value = getattr(pipeline, pipe_ins_attr) @@ -442,7 +430,7 @@ class TestFilesPipelineCustomSettings: assert value != self.default_cls_settings[pipe_attr] assert value == setting_value - def test_no_custom_settings_for_subclasses(self): + def test_no_custom_settings_for_subclasses(self, tmp_path): """ If there are no settings for subclass and no subclass attributes, pipeline should use attributes of base class. @@ -452,14 +440,14 @@ class TestFilesPipelineCustomSettings: pass user_pipeline = UserDefinedFilesPipeline.from_crawler( - get_crawler(None, {"FILES_STORE": self.tempdir}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_cls_settings.get(pipe_attr.upper()) assert getattr(user_pipeline, pipe_ins_attr) == custom_value - def test_custom_settings_for_subclasses(self): + def test_custom_settings_for_subclasses(self, tmp_path): """ If there are custom settings for subclass and NO class attributes, pipeline should use custom settings. @@ -469,7 +457,7 @@ class TestFilesPipelineCustomSettings: pass prefix = UserDefinedFilesPipeline.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = UserDefinedFilesPipeline.from_crawler( get_crawler(None, settings) ) @@ -479,14 +467,14 @@ class TestFilesPipelineCustomSettings: assert custom_value != self.default_cls_settings[pipe_attr] assert getattr(user_pipeline, pipe_inst_attr) == custom_value - def test_custom_settings_and_class_attrs_for_subclasses(self): + def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path): """ If there are custom settings for subclass AND class attributes setting keys are preferred and override attributes. """ pipeline_cls = self._generate_fake_pipeline() prefix = pipeline_cls.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for ( pipe_cls_attr, @@ -497,13 +485,13 @@ class TestFilesPipelineCustomSettings: assert custom_value != self.default_cls_settings[pipe_cls_attr] assert getattr(user_pipeline, pipe_inst_attr) == custom_value - def test_cls_attrs_with_DEFAULT_prefix(self): + def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path): class UserDefinedFilesPipeline(FilesPipeline): DEFAULT_FILES_RESULT_FIELD = "this" DEFAULT_FILES_URLS_FIELD = "that" pipeline = UserDefinedFilesPipeline.from_crawler( - get_crawler(None, {"FILES_STORE": self.tempdir}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) assert ( pipeline.files_result_field @@ -514,12 +502,12 @@ class TestFilesPipelineCustomSettings: == UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD ) - def test_user_defined_subclass_default_key_names(self): + def test_user_defined_subclass_default_key_names(self, tmp_path): """Test situation when user defines subclass of FilesPipeline, but uses attribute names for default pipeline (without prefixing them with pipeline class name). """ - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) class UserPipe(FilesPipeline): pass diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index fef6bbbe9..f2ee18bd9 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -314,13 +314,7 @@ class TestImagesPipelineCustomSettings: "IMAGES_RESULT_FIELD": "images", } - def setup_method(self): - self.tempdir = mkdtemp() - - def teardown_method(self): - rmtree(self.tempdir) - - def _generate_fake_settings(self, prefix=None): + def _generate_fake_settings(self, tmp_path, prefix=None): """ :param prefix: string for setting keys :return: dictionary of image pipeline settings @@ -331,7 +325,7 @@ class TestImagesPipelineCustomSettings: settings = { "IMAGES_EXPIRES": random.randint(100, 1000), - "IMAGES_STORE": self.tempdir, + "IMAGES_STORE": tmp_path, "IMAGES_RESULT_FIELD": random_string(), "IMAGES_URLS_FIELD": random_string(), "IMAGES_MIN_WIDTH": random.randint(1, 1000), @@ -368,13 +362,13 @@ class TestImagesPipelineCustomSettings: return UserDefinedImagePipeline - def test_different_settings_for_different_instances(self): + def test_different_settings_for_different_instances(self, tmp_path): """ If there are two instances of ImagesPipeline class with different settings, they should have different settings. """ - custom_settings = self._generate_fake_settings() - default_sts_pipe = ImagesPipeline(self.tempdir, crawler=get_crawler(None)) + custom_settings = self._generate_fake_settings(tmp_path) + default_sts_pipe = ImagesPipeline(tmp_path, crawler=get_crawler(None)) user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_default_value = self.default_pipeline_settings.get(pipe_attr) @@ -385,14 +379,14 @@ class TestImagesPipelineCustomSettings: ) assert getattr(user_sts_pipe, pipe_attr.lower()) == custom_value - def test_subclass_attrs_preserved_default_settings(self): + def test_subclass_attrs_preserved_default_settings(self, tmp_path): """ If image settings are not defined at all subclass of ImagePipeline takes values from class attributes. """ pipeline_cls = self._generate_fake_pipeline_subclass() pipeline = pipeline_cls.from_crawler( - get_crawler(None, {"IMAGES_STORE": self.tempdir}) + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to class attribute (uppercase). @@ -400,13 +394,13 @@ class TestImagesPipelineCustomSettings: assert attr_value != self.default_pipeline_settings[pipe_attr] assert attr_value == getattr(pipeline, pipe_attr) - def test_subclass_attrs_preserved_custom_settings(self): + def test_subclass_attrs_preserved_custom_settings(self, tmp_path): """ If image settings are defined but they are not defined for subclass default values taken from settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline_subclass() - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to @@ -416,7 +410,7 @@ class TestImagesPipelineCustomSettings: setings_value = settings.get(settings_attr) assert value == setings_value - def test_no_custom_settings_for_subclasses(self): + def test_no_custom_settings_for_subclasses(self, tmp_path): """ If there are no settings for subclass and no subclass attributes, pipeline should use attributes of base class. @@ -426,14 +420,14 @@ class TestImagesPipelineCustomSettings: pass user_pipeline = UserDefinedImagePipeline.from_crawler( - get_crawler(None, {"IMAGES_STORE": self.tempdir}) + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_pipeline_settings.get(pipe_attr.upper()) assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_custom_settings_for_subclasses(self): + def test_custom_settings_for_subclasses(self, tmp_path): """ If there are custom settings for subclass and NO class attributes, pipeline should use custom settings. @@ -443,7 +437,7 @@ class TestImagesPipelineCustomSettings: pass prefix = UserDefinedImagePipeline.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = UserDefinedImagePipeline.from_crawler( get_crawler(None, settings) ) @@ -453,27 +447,27 @@ class TestImagesPipelineCustomSettings: assert custom_value != self.default_pipeline_settings[pipe_attr] assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_custom_settings_and_class_attrs_for_subclasses(self): + def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path): """ If there are custom settings for subclass AND class attributes setting keys are preferred and override attributes. """ pipeline_cls = self._generate_fake_pipeline_subclass() prefix = pipeline_cls.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: custom_value = settings.get(prefix + "_" + settings_attr) assert custom_value != self.default_pipeline_settings[pipe_attr] assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_cls_attrs_with_DEFAULT_prefix(self): + def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path): class UserDefinedImagePipeline(ImagesPipeline): DEFAULT_IMAGES_URLS_FIELD = "something" DEFAULT_IMAGES_RESULT_FIELD = "something_else" pipeline = UserDefinedImagePipeline.from_crawler( - get_crawler(None, {"IMAGES_STORE": self.tempdir}) + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) assert ( pipeline.images_result_field @@ -484,12 +478,12 @@ class TestImagesPipelineCustomSettings: == UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD ) - def test_user_defined_subclass_default_key_names(self): + def test_user_defined_subclass_default_key_names(self, tmp_path): """Test situation when user defines subclass of ImagePipeline, but uses attribute names for default pipeline (without prefixing them with pipeline class name). """ - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) class UserPipe(ImagesPipeline): pass diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 0b845fdb0..41d9b8933 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,24 +1,14 @@ -from pathlib import Path -from shutil import rmtree -from tempfile import mkdtemp - from scrapy.utils.template import render_templatefile class TestUtilsRenderTemplateFile: - def setup_method(self): - self.tmp_path = mkdtemp() - - def teardown_method(self): - rmtree(self.tmp_path) - - def test_simple_render(self): + def test_simple_render(self, tmp_path): context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"} template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" - template_path = Path(self.tmp_path, "templ.py.tmpl") - render_path = Path(self.tmp_path, "templ.py") + template_path = tmp_path / "templ.py.tmpl" + render_path = tmp_path / "templ.py" template_path.write_text(template, encoding="utf8") assert template_path.is_file() # Failure of test itself From d0dabbc09706b082e2250790cd7a00c033ad8021 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 12 Mar 2025 00:18:30 +0400 Subject: [PATCH 1691/2083] Enable AsyncioSelectorReactor by default. (#6713) * Enable AsyncioSelectorReactor by default. * Improve get_crawler(), switch more tests to it. * Fix the remaining default-reactor test failures. * Address documentation feedback. * Make pinned envs more consistent. --- .github/workflows/tests-ubuntu.yml | 14 ++-- .github/workflows/tests-windows.yml | 16 ++++- conftest.py | 12 ++-- docs/topics/asyncio.rst | 23 +++++-- docs/topics/media-pipeline.rst | 2 +- docs/topics/settings.rst | 14 ++-- scrapy/pipelines/images.py | 2 +- scrapy/settings/default_settings.py | 2 +- .../templates/project/module/settings.py.tmpl | 1 - scrapy/utils/log.py | 4 +- scrapy/utils/test.py | 23 ++++++- .../CrawlerProcess/asyncio_enabled_reactor.py | 14 ++-- tests/CrawlerProcess/reactor_default.py | 4 +- tests/CrawlerProcess/reactor_select.py | 4 +- tests/CrawlerRunner/ip_address.py | 6 ++ tests/test_addons.py | 5 +- tests/test_crawl.py | 4 +- tests/test_crawler.py | 46 +++++++------ tests/test_dependencies.py | 2 +- tests/test_downloader_handlers.py | 6 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_extension_periodic_log.py | 19 +++--- tests/test_pipeline_crawl.py | 31 +++++---- tests/test_spider.py | 8 ++- tests/test_utils_asyncio.py | 5 +- tox.ini | 68 ++++++++----------- 26 files changed, 197 insertions(+), 140 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f74575ee1..444aa3557 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -34,25 +34,25 @@ jobs: TOXENV: py - python-version: "3.13" env: - TOXENV: asyncio + TOXENV: default-reactor - python-version: pypy3.10 env: TOXENV: pypy3 # pinned deps - - python-version: 3.9.19 + - python-version: "3.9.21" env: TOXENV: pinned - - python-version: 3.9.19 + - python-version: "3.9.21" env: - TOXENV: asyncio-pinned + TOXENV: default-reactor-pinned - python-version: pypy3.10 env: TOXENV: pypy3-pinned - - python-version: 3.9.19 + - python-version: "3.9.21" env: TOXENV: extra-deps-pinned - - python-version: 3.9.19 + - python-version: "3.9.21" env: TOXENV: botocore-pinned @@ -78,7 +78,7 @@ jobs: if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update - sudo apt-get install libxml2-dev libxslt-dev libjpeg-dev + sudo apt-get install libxml2-dev libxslt-dev - name: Run tests env: ${{ matrix.env }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 21d621240..537a01e29 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -19,7 +19,7 @@ jobs: include: - python-version: "3.9" env: - TOXENV: windows-pinned + TOXENV: py - python-version: "3.10" env: TOXENV: py @@ -34,7 +34,19 @@ jobs: TOXENV: py - python-version: "3.13" env: - TOXENV: asyncio + TOXENV: default-reactor + + # pinned deps + - python-version: "3.9.13" + env: + TOXENV: pinned + - python-version: "3.9.13" + env: + TOXENV: extra-deps-pinned + + - python-version: "3.13" + env: + TOXENV: extra-deps steps: - uses: actions/checkout@v4 diff --git a/conftest.py b/conftest.py index f33ffb1a4..9999e41d2 100644 --- a/conftest.py +++ b/conftest.py @@ -51,7 +51,7 @@ def chdir(tmpdir): def pytest_addoption(parser): parser.addoption( "--reactor", - default="default", + default="asyncio", choices=["default", "asyncio"], ) @@ -67,17 +67,17 @@ def reactor_pytest(request): @pytest.fixture(autouse=True) def only_asyncio(request, reactor_pytest): - if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio": - pytest.skip("This test is only run with --reactor=asyncio") + if request.node.get_closest_marker("only_asyncio") and reactor_pytest == "default": + pytest.skip("This test is only run without --reactor=default") @pytest.fixture(autouse=True) def only_not_asyncio(request, reactor_pytest): if ( request.node.get_closest_marker("only_not_asyncio") - and reactor_pytest == "asyncio" + and reactor_pytest != "default" ): - pytest.skip("This test is only run without --reactor=asyncio") + pytest.skip("This test is only run with --reactor=default") @pytest.fixture(autouse=True) @@ -117,7 +117,7 @@ def requires_boto3(request): def pytest_configure(config): - if config.getoption("--reactor") == "asyncio": + if config.getoption("--reactor") != "default": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 07baea071..35afdc11b 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -16,15 +16,19 @@ asyncio reactor `, you may use :mod:`asyncio` and Installing the asyncio reactor ============================== -To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to -``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``. +To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs +to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, +which is the default value. If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` reactor manually. You can do that using -:func:`~scrapy.utils.reactor.install_reactor`:: +:func:`~scrapy.utils.reactor.install_reactor`: - install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') +.. skip: next +.. code-block:: python + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") .. _asyncio-preinstalled-reactor: @@ -144,3 +148,14 @@ Using custom asyncio loops You can also use custom asyncio event loops with the asyncio reactor. Set the :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to use it instead of the default asyncio event loop. + + +.. _disable-asyncio: + +Switching to a non-asyncio reactor +================================== + +If for some reason your code doesn't work with the asyncio reactor, you can use +a different reactor by setting the :setting:`TWISTED_REACTOR` setting to its +import path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to +``None``, which will use the default reactor for your platform. diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index cc1fe8703..01da53342 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for +The Images Pipeline requires Pillow_ 8.0.0 or greater. It is used for thumbnailing and normalizing images to JPEG/RGB format. .. _Pillow: https://github.com/python-pillow/Pillow diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index b2bb7148f..ca0af569f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1911,7 +1911,7 @@ TWISTED_REACTOR .. versionadded:: 2.0 -Default: ``None`` +Default: ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"`` Import path of a given :mod:`~twisted.internet.reactor`. @@ -1996,17 +1996,19 @@ which raises :exc:`Exception`, becomes: self.crawler.engine.close_spider(self, "timeout") -The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will use the existing reactor if one is already installed, or -install the default reactor defined by Twisted for the current platform. This -is to maintain backward compatibility and avoid possible problems caused by -using a non-default reactor. +If this setting is set ``None``, Scrapy will use the existing reactor if one is +already installed, or install the default reactor defined by Twisted for the +current platform. .. versionchanged:: 2.7 The :command:`startproject` command now sets this setting to ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated ``settings.py`` file. +.. versionchanged:: VERSION + The default value was changed from ``None`` to + ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. + For additional information, see :doc:`core/howto/choosing-reactor`. diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 29dc13f0a..63c6908dc 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -68,7 +68,7 @@ class ImagesPipeline(FilesPipeline): self._Image = Image except ImportError: raise NotConfigured( - "ImagesPipeline requires installing Pillow 4.0.0 or later" + "ImagesPipeline requires installing Pillow 8.0.0 or later" ) super().__init__( diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c473b369c..645e50301 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -341,7 +341,7 @@ TELNETCONSOLE_HOST = "127.0.0.1" TELNETCONSOLE_USERNAME = "scrapy" TELNETCONSOLE_PASSWORD = None -TWISTED_REACTOR = None +TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 0bb31ffaa..db7400af8 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -90,5 +90,4 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # Set settings whose default value is deprecated to a future-proof value -TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index b865cf48d..24e17ecb6 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -182,11 +182,9 @@ def log_scrapy_info(settings: Settings) -> None: def log_reactor_info() -> None: - from twisted.internet import reactor + from twisted.internet import asyncioreactor, reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) - from twisted.internet import asyncioreactor - if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor): logger.debug( "Using asyncio event loop: %s.%s", diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index b69f43438..2da526cd8 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -18,6 +18,7 @@ from twisted.trial.unittest import SkipTest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available from scrapy.utils.deprecate import create_deprecated_class +from scrapy.utils.reactor import is_asyncio_reactor_installed from scrapy.utils.spider import DefaultSpider if TYPE_CHECKING: @@ -109,6 +110,19 @@ def get_ftp_content_and_delete( TestSpider = create_deprecated_class("TestSpider", DefaultSpider) +def get_reactor_settings() -> dict[str, Any]: + """Return a settings dict that works with the installed reactor. + + ``Crawler._apply_settings()`` checks that the installed reactor matches the + settings, so tests that run the crawler in the current process may need to + pass a correct ``"TWISTED_REACTOR"`` setting value when creating it. + """ + settings: dict[str, Any] = {} + if not is_asyncio_reactor_installed(): + settings["TWISTED_REACTOR"] = None + return settings + + def get_crawler( spidercls: type[Spider] | None = None, settings_dict: dict[str, Any] | None = None, @@ -120,9 +134,12 @@ def get_crawler( """ from scrapy.crawler import CrawlerRunner - # Set by default settings that prevent deprecation warnings. - settings: dict[str, Any] = {} - settings.update(settings_dict or {}) + # When needed, useful settings can be added here, e.g. ones that prevent + # deprecation warnings. + settings: dict[str, Any] = { + **get_reactor_settings(), + **(settings_dict or {}), + } runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or DefaultSpider) crawler._apply_settings() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index f013eed27..0c380610d 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,14 +1,8 @@ -import asyncio -import sys +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.utils.reactor import install_reactor -from twisted.internet import asyncioreactor - -if sys.platform == "win32": - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncioreactor.install(asyncio.get_event_loop()) - -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 078cb72cb..e2933338b 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,4 +1,5 @@ from twisted.internet import reactor # noqa: F401 +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -13,5 +14,6 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess(settings={}) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index 814a2a46d..b61e52625 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -1,4 +1,5 @@ from twisted.internet import selectreactor +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -15,5 +16,6 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess(settings={}) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 2f1bb7713..b1b297777 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,3 +1,9 @@ +# ruff: noqa: E402 + +from scrapy.utils.reactor import install_reactor + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + from urllib.parse import urlparse from twisted.internet import reactor diff --git a/tests/test_addons.py b/tests/test_addons.py index 686bf9952..b4294c815 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -9,7 +9,7 @@ from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings class SimpleAddon: @@ -105,6 +105,7 @@ class TestAddonManager(unittest.TestCase): } settings_dict = { "ADDONS": {get_addon_cls(config): 1}, + **get_reactor_settings(), } crawler = get_crawler(settings_dict=settings_dict) assert crawler.settings.getint("KEY") == 15 @@ -119,6 +120,7 @@ class TestAddonManager(unittest.TestCase): settings_dict = { "KEY": 20, # priority=project "ADDONS": {get_addon_cls(config): 1}, + **get_reactor_settings(), } settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") @@ -196,6 +198,7 @@ class TestAddonManager(unittest.TestCase): return spider settings = Settings() + settings.setdict(get_reactor_settings()) settings.set("KEY", "default", priority="default") runner = CrawlerRunner(settings) crawler = runner.create_crawler(MySpider) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 6f4045fc8..f49deac1f 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -18,7 +18,7 @@ from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import MockServer from tests.spiders import ( @@ -412,7 +412,7 @@ with multiples lines @defer.inlineCallbacks def test_crawl_multiple(self): - runner = CrawlerRunner() + runner = CrawlerRunner(get_reactor_settings()) runner.crawl( SimpleSpider, self.mockserver.url("/status?n=200"), diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0bbcc0843..98352b66e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -25,7 +25,7 @@ from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests.mockserver import MockServer, get_mockserver_env BASE_SETTINGS: dict[str, Any] = {} @@ -35,6 +35,7 @@ def get_raw_crawler(spidercls=None, settings_dict=None): """get_crawler alternative that only calls the __init__ method of the crawler.""" settings = Settings() + settings.setdict(get_reactor_settings()) settings.setdict(settings_dict or {}) return Crawler(spidercls or DefaultSpider, settings) @@ -48,7 +49,12 @@ class TestBaseCrawler(unittest.TestCase): class TestCrawler(TestBaseCrawler): def test_populate_spidercls_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"} + project_settings = { + **BASE_SETTINGS, + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } class CustomSettingsSpider(DefaultSpider): custom_settings = spider_settings @@ -581,7 +587,7 @@ class NoRequestsSpider(scrapy.Spider): @pytest.mark.usefixtures("reactor_pytest") class TestCrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): - return CrawlerRunner() + return CrawlerRunner(get_reactor_settings()) @inlineCallbacks def test_crawler_runner_bootstrap_successful(self): @@ -626,13 +632,7 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): @inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): - if self.reactor_pytest == "asyncio": - CrawlerRunner( - settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - } - ) - else: + if self.reactor_pytest == "default": runner = CrawlerRunner( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", @@ -643,6 +643,12 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", ): yield runner.crawl(NoRequestsSpider) + else: + CrawlerRunner( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + ) class ScriptRunnerMixin: @@ -672,7 +678,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): assert "Spider closed (finished)" in log assert ( "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log + in log ) def test_multi(self): @@ -680,18 +686,17 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): assert "Spider closed (finished)" in log assert ( "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log + in log ) assert "ReactorAlreadyInstalledError" not in log def test_reactor_default(self): log = self.run_script("reactor_default.py") - assert "Spider closed (finished)" in log + assert "Spider closed (finished)" not in log assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log - ) - assert "ReactorAlreadyInstalledError" not in log + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log def test_reactor_default_twisted_reactor_select(self): log = self.run_script("reactor_default_twisted_reactor_select.py") @@ -716,8 +721,11 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_reactor_select(self): log = self.run_script("reactor_select.py") - assert "Spider closed (finished)" in log - assert "ReactorAlreadyInstalledError" not in log + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log def test_reactor_select_twisted_reactor_select(self): log = self.run_script("reactor_select_twisted_reactor_select.py") diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 162747581..c2df67c66 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -33,7 +33,7 @@ class TestScrapyUtils: tox_config_file_path = Path(__file__).parent / ".." / "tox.ini" config_parser = ConfigParser() config_parser.read(tox_config_file_path) - pattern = r"Twisted\[http2\]==([\d.]+)" + pattern = r"Twisted==([\d.]+)" match = re.search(pattern, config_parser["pinned"]["deps"]) pinned_twisted_version_string = match[1] diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 19bd02498..bc18e76e1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -307,7 +307,7 @@ class TestHttp(unittest.TestCase, ABC): @defer.inlineCallbacks def test_timeout_download_from_spider_nodata_rcvd(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": + if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( "This test produces DirtyReactorAggregateError on Windows with asyncio" @@ -322,7 +322,7 @@ class TestHttp(unittest.TestCase, ABC): @defer.inlineCallbacks def test_timeout_download_from_spider_server_hangs(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": + if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( "This test produces DirtyReactorAggregateError on Windows with asyncio" @@ -1136,7 +1136,7 @@ class TestFTPBase(unittest.TestCase): class TestFTP(TestFTPBase): def test_invalid_credentials(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": + if self.reactor_pytest != "default" and sys.platform == "win32": raise unittest.SkipTest( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 15b3ad5af..4fca9eefb 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -64,7 +64,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_delay(self): - crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider) + crawler = get_crawler(DownloaderSlotsSettingsTestSpider) yield crawler.crawl(mockserver=self.mockserver) slots = crawler.engine.downloader.slots times = crawler.spider.times diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index ca5ffdc26..85bd42857 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,9 +1,11 @@ -import datetime -import typing -import unittest +from __future__ import annotations + +import datetime +import unittest +from typing import Any, Callable -from scrapy.crawler import Crawler from scrapy.extensions.periodic_log import PeriodicLog +from scrapy.utils.test import get_crawler from .spiders import MetaSpider @@ -59,9 +61,8 @@ class CustomPeriodicLog(PeriodicLog): self.stats._stats = stats_dump_2 -def extension(settings=None): - crawler = Crawler(MetaSpider, settings=settings) - crawler._apply_settings() +def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog: + crawler = get_crawler(MetaSpider, settings) return CustomPeriodicLog.from_crawler(crawler) @@ -94,7 +95,7 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict, condition: typing.Callable): + def check(settings: dict[str, Any], condition: Callable) -> None: ext, a, b = emulate(settings) assert list(a["delta"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) @@ -151,7 +152,7 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict, condition: typing.Callable): + def check(settings: dict[str, Any], condition: Callable) -> None: ext, a, b = emulate(settings) assert list(a["stats"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 162dfdaf4..c5f1b6321 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -3,18 +3,22 @@ from __future__ import annotations import shutil from pathlib import Path from tempfile import mkdtemp +from typing import TYPE_CHECKING, Any from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase from w3lib.url import add_or_replace_parameter -from scrapy import signals -from scrapy.crawler import CrawlerRunner +from scrapy import Spider, signals from scrapy.utils.misc import load_object +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import SimpleSpider +if TYPE_CHECKING: + from scrapy.crawler import Crawler + class MediaDownloadSpider(SimpleSpider): name = "mediadownload" @@ -80,7 +84,6 @@ class TestFileDownloadCrawl(TestCase): "ITEM_PIPELINES": {self.pipeline_class: 1}, self.store_setting_key: str(self.tmpmediastore), } - self.runner = CrawlerRunner(self.settings) self.items = [] def tearDown(self): @@ -90,10 +93,12 @@ class TestFileDownloadCrawl(TestCase): def _on_item_scraped(self, item): self.items.append(item) - def _create_crawler(self, spider_class, runner=None, **kwargs): - if runner is None: - runner = self.runner - crawler = runner.create_crawler(spider_class, **kwargs) + def _create_crawler( + self, spider_class: type[Spider], settings: dict[str, Any] | None = None + ) -> Crawler: + if settings is None: + settings = self.settings + crawler = get_crawler(spider_class, settings) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) return crawler @@ -175,10 +180,11 @@ class TestFileDownloadCrawl(TestCase): @defer.inlineCallbacks def test_download_media_redirected_allowed(self): - settings = dict(self.settings) - settings.update({"MEDIA_ALLOW_REDIRECTS": True}) - runner = CrawlerRunner(settings) - crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner) + settings = { + **self.settings, + "MEDIA_ALLOW_REDIRECTS": True, + } + crawler = self._create_crawler(RedirectedMediaDownloadSpider, settings) with LogCapture() as log: yield crawler.crawl( self.mockserver.url("/files/images/"), @@ -201,8 +207,7 @@ class TestFileDownloadCrawl(TestCase): **self.settings, "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1}, } - runner = CrawlerRunner(settings) - crawler = self._create_crawler(MediaDownloadSpider, runner=runner) + crawler = self._create_crawler(MediaDownloadSpider, settings) with LogCapture() as log: yield crawler.crawl( self.mockserver.url("/files/images/"), diff --git a/tests/test_spider.py b/tests/test_spider.py index 4e8330c06..aaf72390d 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -27,7 +27,7 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.spiders.init import InitSpider -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata, tests_datadir @@ -108,7 +108,11 @@ class TestSpider(unittest.TestCase): @inlineCallbacks def test_settings_in_from_crawler(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {"TEST1": "project", "TEST3": "project"} + project_settings = { + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } class TestSpider(self.spider_class): name = "test" diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a65a36219..901e03d59 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -2,6 +2,7 @@ import asyncio import warnings import pytest +from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( @@ -12,10 +13,10 @@ from scrapy.utils.reactor import ( @pytest.mark.usefixtures("reactor_pytest") -class TestAsyncio: +class TestAsyncio(TestCase): def test_is_asyncio_reactor_installed(self): # the result should depend only on the pytest --reactor argument - assert is_asyncio_reactor_installed() == (self.reactor_pytest == "asyncio") + assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default") def test_install_asyncio_reactor(self): from twisted.internet import reactor as original_reactor diff --git a/tox.ini b/tox.ini index 70c841603..eb084f0f5 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ deps = {[test-requirements]deps} # mitmproxy does not support PyPy - mitmproxy; implementation_name != 'pypy' + mitmproxy; implementation_name != "pypy" setenv = COVERAGE_CORE=sysmon passenv = @@ -96,19 +96,18 @@ commands = [pinned] basepython = python3.9 deps = + Protego==0.1.15 + Twisted==21.7.0 cryptography==37.0.0 cssselect==0.9.1 - h2==3.0 itemadapter==0.1.0 + lxml==4.6.0 parsel==1.5.0 - Protego==0.1.15 pyOpenSSL==22.0.0 queuelib==1.4.2 service_identity==18.1.0 - Twisted[http2]==21.7.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.6.0 {[test-requirements]deps} # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies @@ -131,60 +130,50 @@ setenv = {[pinned]setenv} commands = {[pinned]commands} -[testenv:windows-pinned] -basepython = {[pinned]basepython} -deps = - {[pinned]deps} - PyDispatcher==2.0.5 -install_command = {[pinned]install_command} -setenv = - {[pinned]setenv} -commands = {[pinned]commands} - [testenv:extra-deps] basepython = python3 deps = {[testenv]deps} - boto3 - google-cloud-storage - robotexclusionrulesparser Pillow Twisted[http2] - uvloop; platform_system != "Windows" + boto3 bpython # optional for shell wrapper tests - brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests - brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests - zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + brotli; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests + brotlicffi; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests + google-cloud-storage ipython + robotexclusionrulesparser + uvloop; platform_system != "Windows" + zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests [testenv:extra-deps-pinned] basepython = {[pinned]basepython} deps = {[pinned]deps} + Pillow==8.0.0 boto3==1.20.0 - google-cloud-storage==1.29.0 - Pillow==7.1.0 - robotexclusionrulesparser==1.6.2 - brotlipy - uvloop==0.14.0; platform_system != "Windows" bpython==0.7.1 - zstandard==0.1; implementation_name != 'pypy' + brotli==0.5.2; implementation_name != "pypy" + brotlicffi==0.8.0; implementation_name == "pypy" + brotlipy + google-cloud-storage==1.29.0 ipython==2.0.0 - brotli==0.5.2; implementation_name != 'pypy' - brotlicffi==0.8.0; implementation_name == 'pypy' + robotexclusionrulesparser==1.6.2 + uvloop==0.14.0; platform_system != "Windows" + zstandard==0.1; implementation_name != "pypy" install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = {[pinned]commands} -[testenv:asyncio] +[testenv:default-reactor] commands = - {[testenv]commands} --reactor=asyncio + {[testenv]commands} --reactor=default -[testenv:asyncio-pinned] +[testenv:default-reactor-pinned] basepython = {[pinned]basepython} deps = {[testenv:pinned]deps} -commands = {[pinned]commands} --reactor=asyncio +commands = {[pinned]commands} --reactor=default install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -204,21 +193,20 @@ commands = {[testenv:pypy3]commands} [testenv:pypy3-pinned] basepython = pypy3.10 deps = + PyPyDispatcher==2.1.0 + {[test-requirements]deps} + Protego==0.1.15 + Twisted==21.7.0 cryptography==41.0.5 cssselect==0.9.1 - h2==3.1 itemadapter==0.1.0 + lxml==4.6.0 parsel==1.5.0 - Protego==0.1.15 pyOpenSSL==23.3.0 queuelib==1.4.2 service_identity==18.1.0 - Twisted[http2]==21.7.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.6.0 - {[test-requirements]deps} - PyPyDispatcher==2.1.0 commands = ; disabling both coverage and docs tests pytest {posargs:--durations=10 scrapy tests} From fc566a7ff9913be5fce2b961a6599f861e1ef59a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Mar 2025 23:46:17 +0400 Subject: [PATCH 1692/2083] Don't install the reactor in is_asyncio_reactor_installed(). (#6732) --- docs/topics/asyncio.rst | 2 ++ scrapy/utils/reactor.py | 13 ++++++++ .../asyncio_enabled_no_reactor.py | 8 +++++ .../CrawlerProcess/asyncio_enabled_reactor.py | 33 ++++++++++++++++++- tests/test_crawler.py | 2 ++ 5 files changed, 57 insertions(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 35afdc11b..b61a6e4a8 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -116,6 +116,8 @@ example: f"of Scrapy for more information." ) +.. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed + .. _asyncio-windows: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 679e38206..099c81f0e 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -175,7 +175,20 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None: ) +def is_reactor_installed() -> bool: + return "twisted.internet.reactor" in sys.modules + + def is_asyncio_reactor_installed() -> bool: + """Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + + Raise a :exc:`RuntimeError` if no reactor is installed. + """ + if not is_reactor_installed(): + raise RuntimeError( + "is_asyncio_reactor_installed() called without an installed reactor." + ) + from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index 6df6d76fa..6f82cf589 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -1,5 +1,12 @@ import scrapy from scrapy.crawler import CrawlerProcess +from scrapy.utils.reactor import is_asyncio_reactor_installed + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") class NoRequestsSpider(scrapy.Spider): @@ -12,6 +19,7 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, } ) process.crawl(NoRequestsSpider) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 0c380610d..a8bf1bc3c 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,9 +1,39 @@ import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.utils.reactor import install_reactor +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + is_reactor_installed, +) + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed before is_asyncio_reactor_installed()." + ) + +try: + is_asyncio_reactor_installed() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.") + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed after is_asyncio_reactor_installed()." + ) install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +if not is_asyncio_reactor_installed(): + raise RuntimeError("Wrong reactor installed after install_reactor().") + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + class NoRequestsSpider(scrapy.Spider): name = "no_request" @@ -15,6 +45,7 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, } ) process.crawl(NoRequestsSpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 98352b66e..6c465f000 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -747,6 +747,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" in log ) + assert "RuntimeError" not in log def test_asyncio_enabled_reactor(self): log = self.run_script("asyncio_enabled_reactor.py") @@ -755,6 +756,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" in log ) + assert "RuntimeError" not in log @pytest.mark.skipif( parse_version(w3lib_version) >= parse_version("2.0.0"), From 9057bf4e1e08dccac4fa6d9f0f191d1f4708a43a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 15 Mar 2025 14:47:16 +0400 Subject: [PATCH 1693/2083] More docs about Deferred<->Future interoperability. (#6734) --- docs/topics/asyncio.rst | 24 ++++++++++++++++-------- docs/topics/coroutines.rst | 24 ++++++++++++++++++++++++ scrapy/utils/defer.py | 9 +++++---- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index b61a6e4a8..0490129b3 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -72,24 +72,32 @@ those imports happen. .. _asyncio-await-dfd: -Awaiting on Deferreds -===================== +Integrating Deferred code and asyncio code +========================================== -When the asyncio reactor isn't installed, you can await on Deferreds in the -coroutines directly. When it is installed, this is not possible anymore, due to -specifics of the Scrapy coroutine integration (the coroutines are wrapped into -:class:`asyncio.Future` objects, not into -:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into -Futures. Scrapy provides two helpers for this: +Coroutine functions can await on Deferreds by wrapping them into +:class:`asyncio.Future` objects. Scrapy provides two helpers for this: .. autofunction:: scrapy.utils.defer.deferred_to_future .. autofunction:: scrapy.utils.defer.maybe_deferred_to_future + +.. tip:: If you don't need to support reactors other than the default + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, you + can use :func:`~scrapy.utils.defer.deferred_to_future`, otherwise you + should use :func:`~scrapy.utils.defer.maybe_deferred_to_future`. + .. tip:: If you need to use these functions in code that aims to be compatible with lower versions of Scrapy that do not provide these functions, down to Scrapy 2.0 (earlier versions do not support :mod:`asyncio`), you can copy the implementation of these functions into your own code. +Coroutines and futures can be wrapped into Deferreds (for example, when a +Scrapy API requires passing a Deferred to it) using the following helpers: + +.. autofunction:: scrapy.utils.defer.deferred_from_coro +.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f + .. _enforce-asyncio-requirement: diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 57aa3a62d..1c80857f6 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -9,6 +9,7 @@ Coroutines Scrapy has :ref:`partial support ` for the :ref:`coroutine syntax `. + .. _coroutine-support: Supported callables @@ -51,6 +52,29 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): .. versionadded:: 2.7 + +.. _coroutine-deferred-apis: + +Using Deferred-based APIs +========================= + +In addition to native coroutine APIs Scrapy has some APIs that return a +:class:`~twisted.internet.defer.Deferred` object or take a user-supplied +function that returns a :class:`~twisted.internet.defer.Deferred` object. These +APIs are also asynchronous but don't yet support native ``async def`` syntax. +For example: + +- The :meth:`ExecutionEngine.download` method returns a + :class:`~twisted.internet.defer.Deferred` object. +- A custom download handler needs to define a ``download_request()`` method that + returns a :class:`~twisted.internet.defer.Deferred` object. + +In most cases you can use these APIs in code that otherwise uses coroutines, by +wrapping a :class:`~twisted.internet.defer.Deferred` object into a +:class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for +more information about this. + + General usage ============= diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 8f52836c4..42ad28d8d 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -362,7 +362,8 @@ def deferred_from_coro(o: _T) -> _T: ... def deferred_from_coro(o: _T) -> Deferred | _T: - """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" + """Converts a coroutine or other awaitable object into a Deferred, + or returns the object as is if it isn't a coroutine.""" if isinstance(o, Deferred): return o if asyncio.isfuture(o) or inspect.isawaitable(o): @@ -442,12 +443,12 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: What you can await in Scrapy callables defined as coroutines depends on the value of :setting:`TWISTED_REACTOR`: - - When not using the asyncio reactor, you can only await on - :class:`~twisted.internet.defer.Deferred` objects. - - When :ref:`using the asyncio reactor `, you can only await on :class:`asyncio.Future` objects. + - When not using the asyncio reactor, you can only await on + :class:`~twisted.internet.defer.Deferred` objects. + If you want to write code that uses ``Deferred`` objects but works with any reactor, use this function on all ``Deferred`` objects:: From 872924721344502cca92a72ce68d5f78a9b9e5bc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 15 Mar 2025 23:33:10 +0400 Subject: [PATCH 1694/2083] Fix running simple tests with --reactor=default. (#6735) --- conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conftest.py b/conftest.py index 9999e41d2..8e0c429a0 100644 --- a/conftest.py +++ b/conftest.py @@ -119,6 +119,9 @@ def requires_boto3(request): def pytest_configure(config): if config.getoption("--reactor") != "default": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + else: + # install the reactor explicitly + from twisted.internet import reactor # noqa: F401 # Generate localhost certificate files, needed by some tests From 2ee01efe496db9ff8506ca37fe2571f5f4ac2849 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 20 Mar 2025 07:49:44 -0300 Subject: [PATCH 1695/2083] feat: Add count to spider_exceptions stats (#6740) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add overall exception_count to spider_exceptions stats * Remove variable * Update test_closespider.py * Update test_closespider.py * Rename exception_count → count --- scrapy/core/scraper.py | 1 + tests/test_closespider.py | 1 + 2 files changed, 2 insertions(+) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index b664b61f6..496adb500 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -246,6 +246,7 @@ class Scraper: spider=spider, ) assert self.crawler.stats + self.crawler.stats.inc_value("spider_exceptions/count", spider=spider) self.crawler.stats.inc_value( f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider ) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 476662789..4a17b254b 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -88,6 +88,7 @@ class TestCloseSpider(TestCase): assert reason == "closespider_errorcount" key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}" errorcount = crawler.stats.get_value(key) + assert crawler.stats.get_value("spider_exceptions/count") >= close_on assert errorcount >= close_on @defer.inlineCallbacks From 3ca882fba86750199c2f41ef24b5495a4afa7988 Mon Sep 17 00:00:00 2001 From: Mehraz Hossain Rumman <59512321+MehrazRumman@users.noreply.github.com> Date: Thu, 20 Mar 2025 18:02:10 +0600 Subject: [PATCH 1696/2083] Syntax Error Fixed (#6738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Syntax error fix issue #6731 * test case added * extra logic removed * mock spider fixture * Update scrapy/utils/misc.py Co-authored-by: Adrián Chaves * settings.rst updated * settings.rst updated * settings.rst updated --------- Co-authored-by: Adrián Chaves --- docs/topics/settings.rst | 15 +++ scrapy/settings/default_settings.py | 2 + scrapy/utils/misc.py | 2 + ...t_return_with_argument_inside_generator.py | 110 +++++++++++++----- 4 files changed, 98 insertions(+), 31 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ca0af569f..a59a61050 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2047,6 +2047,21 @@ also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and there is no overriding User-Agent header specified for the request. +.. setting:: WARN_ON_GENERATOR_RETURN_VALUE + +WARN_ON_GENERATOR_RETURN_VALUE +------------------------------ + +Default: ``True`` + +When enabled, Scrapy will warn if generator-based callback methods (like +``parse``) contain return statements with non-``None`` values. This helps detect +potential mistakes in spider development. + +Disable this setting to prevent syntax errors that may occur when dynamically +modifying generator function source code during runtime, skip AST parsing of +callback functions, or improve performance in auto-reloading development +environments. Settings documented elsewhere: ------------------------------ diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 645e50301..680fded7a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -351,3 +351,5 @@ SPIDER_CONTRACTS_BASE = { "scrapy.contracts.default.ReturnsContract": 2, "scrapy.contracts.default.ScrapesContract": 3, } + +WARN_ON_GENERATOR_RETURN_VALUE = True diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d319e7950..b7b436260 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -286,6 +286,8 @@ def warn_on_generator_with_return_value( Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None """ + if not spider.settings.getbool("WARN_ON_GENERATOR_RETURN_VALUE"): + return try: if is_generator_with_return_value(callable): warnings.warn( diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 81a83c3d7..ad31e5185 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -2,6 +2,8 @@ import warnings from functools import partial from unittest import mock +import pytest + from scrapy.utils.misc import ( is_generator_with_return_value, warn_on_generator_with_return_value, @@ -40,7 +42,24 @@ def generator_that_returns_stuff(): class TestUtilsMisc: - def test_generators_return_something(self): + @pytest.fixture + def mock_spider(self): + class MockSettings: + def __init__(self, settings_dict=None): + self.settings_dict = settings_dict or { + "WARN_ON_GENERATOR_RETURN_VALUE": True + } + + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) + + class MockSpider: + def __init__(self): + self.settings = MockSettings() + + return MockSpider() + + def test_generators_return_something(self, mock_spider): def f1(): yield 1 return 2 @@ -75,30 +94,30 @@ https://example.org assert is_generator_with_return_value(i1) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_something) + warn_on_generator_with_return_value(mock_spider, top_level_return_something) assert len(w) == 1 assert ( - 'The "NoneType.top_level_return_something" method is a generator' + 'The "MockSpider.top_level_return_something" method is a generator' in str(w[0].message) ) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, f1) + warn_on_generator_with_return_value(mock_spider, f1) assert len(w) == 1 - assert 'The "NoneType.f1" method is a generator' in str(w[0].message) + assert 'The "MockSpider.f1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, g1) + warn_on_generator_with_return_value(mock_spider, g1) assert len(w) == 1 - assert 'The "NoneType.g1" method is a generator' in str(w[0].message) + assert 'The "MockSpider.g1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, h1) + warn_on_generator_with_return_value(mock_spider, h1) assert len(w) == 1 - assert 'The "NoneType.h1" method is a generator' in str(w[0].message) + assert 'The "MockSpider.h1" method is a generator' in str(w[0].message) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, i1) + warn_on_generator_with_return_value(mock_spider, i1) assert len(w) == 1 - assert 'The "NoneType.i1" method is a generator' in str(w[0].message) + assert 'The "MockSpider.i1" method is a generator' in str(w[0].message) - def test_generators_return_none(self): + def test_generators_return_none(self, mock_spider): def f2(): yield 1 @@ -142,31 +161,31 @@ https://example.org assert not is_generator_with_return_value(l2) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, f2) + warn_on_generator_with_return_value(mock_spider, f2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, g2) + warn_on_generator_with_return_value(mock_spider, g2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, h2) + warn_on_generator_with_return_value(mock_spider, h2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, i2) + warn_on_generator_with_return_value(mock_spider, i2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, j2) + warn_on_generator_with_return_value(mock_spider, j2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, k2) + warn_on_generator_with_return_value(mock_spider, k2) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, l2) + warn_on_generator_with_return_value(mock_spider, l2) assert len(w) == 0 - def test_generators_return_none_with_decorator(self): + def test_generators_return_none_with_decorator(self, mock_spider): def decorator(func): def inner_func(): func() @@ -223,36 +242,36 @@ https://example.org assert not is_generator_with_return_value(l3) with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, f3) + warn_on_generator_with_return_value(mock_spider, f3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, g3) + warn_on_generator_with_return_value(mock_spider, g3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, h3) + warn_on_generator_with_return_value(mock_spider, h3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, i3) + warn_on_generator_with_return_value(mock_spider, i3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, j3) + warn_on_generator_with_return_value(mock_spider, j3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, k3) + warn_on_generator_with_return_value(mock_spider, k3) assert len(w) == 0 with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, l3) + warn_on_generator_with_return_value(mock_spider, l3) assert len(w) == 0 @mock.patch( "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error ) - def test_indentation_error(self): + def test_indentation_error(self, mock_spider): with warnings.catch_warnings(record=True) as w: - warn_on_generator_with_return_value(None, top_level_return_none) + warn_on_generator_with_return_value(mock_spider, top_level_return_none) assert len(w) == 1 assert "Unable to determine" in str(w[0].message) @@ -262,3 +281,32 @@ https://example.org partial_cb = partial(cb, arg1=42) assert not is_generator_with_return_value(partial_cb) + + def test_warn_on_generator_with_return_value_settings_disabled(self): + class MockSettings: + def __init__(self, settings_dict=None): + self.settings_dict = settings_dict or {} + + def getbool(self, name, default=False): + return self.settings_dict.get(name, default) + + class MockSpider: + def __init__(self): + self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False}) + + spider = MockSpider() + + def gen_with_return(): + yield 1 + return "value" + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(spider, gen_with_return) + assert len(w) == 0 + + spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(spider, gen_with_return) + assert len(w) == 1 + assert "is a generator" in str(w[0].message) From e50914e0f5b98ee4c9cb1f182ec2cd684fdf9900 Mon Sep 17 00:00:00 2001 From: Suejung Shin Date: Fri, 21 Mar 2025 04:28:47 -0700 Subject: [PATCH 1697/2083] Codecov: Add test analytics (#6741) --- .github/workflows/tests-macos.yml | 4 ++++ .github/workflows/tests-ubuntu.yml | 4 ++++ .github/workflows/tests-windows.yml | 4 ++++ .gitignore | 1 + tox.ini | 8 ++++---- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index ce0e1a6c2..d740808cc 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -33,3 +33,7 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 444aa3557..34819f227 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -88,3 +88,7 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 537a01e29..bbbb704e5 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -64,3 +64,7 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 diff --git a/.gitignore b/.gitignore index 6c5c50e08..0a3f0ac1c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ htmlcov/ .pytest_cache/ .coverage.* coverage.* +*.junit.xml test-output.* .cache/ .mypy_cache/ diff --git a/tox.ini b/tox.ini index eb084f0f5..59572442d 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --durations=10 docs scrapy tests --doctest-modules} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 docs scrapy tests --doctest-modules} install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} @@ -118,7 +118,7 @@ install_command = python -I -m pip install {opts} {packages} commands = ; tests for docs fail with parsel < 1.8.0 - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --durations=10 scrapy tests} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=pinned.junit.xml -o junit_family=legacy --durations=10 scrapy tests} [testenv:pinned] basepython = {[pinned]basepython} @@ -254,7 +254,7 @@ deps = {[testenv]deps} botocore>=1.4.87 commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy -m requires_botocore} [testenv:botocore-pinned] basepython = {[pinned]basepython} @@ -265,4 +265,4 @@ install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy -m requires_botocore} From 9f99da8f865efff11f6c8736567b8fbd4413091c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 24 Mar 2025 13:26:25 +0500 Subject: [PATCH 1698/2083] Convert test_downloadermiddleware_robotstxt.py from callbacks to awaits. (#6743) --- tests/test_downloadermiddleware_robotstxt.py | 168 +++++++++---------- 1 file changed, 76 insertions(+), 92 deletions(-) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index ad335f852..9518f1835 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,9 +1,11 @@ -from typing import Any +from __future__ import annotations + +from typing import TYPE_CHECKING from unittest import mock import pytest from twisted.internet import error, reactor -from twisted.internet.defer import Deferred, DeferredList, maybeDeferred +from twisted.internet.defer import Deferred, maybeDeferred from twisted.python import failure from twisted.trial import unittest @@ -13,8 +15,12 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from tests.test_robotstxt_interface import rerp_available +if TYPE_CHECKING: + from scrapy.crawler import Crawler + class TestRobotsTxtMiddleware(unittest.TestCase): def setUp(self): @@ -31,7 +37,7 @@ class TestRobotsTxtMiddleware(unittest.TestCase): with pytest.raises(NotConfigured): RobotsTxtMiddleware(self.crawler) - def _get_successful_crawler(self): + def _get_successful_crawler(self) -> Crawler: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) ROBOTS = """ @@ -54,54 +60,41 @@ Disallow: /some/randome/page.html crawler.engine.download.side_effect = return_response return crawler - def test_robotstxt(self): + @deferred_f_from_coro_f + async def test_robotstxt(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) - return DeferredList( - [ - self.assertNotIgnored(Request("http://site.local/allowed"), middleware), - maybeDeferred(self.assertRobotsTxtRequested, "http://site.local"), - self.assertIgnored(Request("http://site.local/admin/main"), middleware), - self.assertIgnored(Request("http://site.local/static/"), middleware), - self.assertIgnored( - Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"), - middleware, - ), - self.assertIgnored( - Request("http://site.local/wiki/Käyttäjä:"), middleware - ), - ], - fireOnOneErrback=True, + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + self.assertRobotsTxtRequested("http://site.local") + await self.assertIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertIgnored(Request("http://site.local/static/"), middleware) + await self.assertIgnored( + Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"), middleware + ) + await self.assertIgnored( + Request("http://site.local/wiki/Käyttäjä:"), middleware ) - def test_robotstxt_ready_parser(self): + @deferred_f_from_coro_f + async def test_robotstxt_ready_parser(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) - d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - d.addCallback( - lambda _: self.assertNotIgnored( - Request("http://site.local/allowed"), middleware - ) - ) - return d + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - def test_robotstxt_meta(self): + @deferred_f_from_coro_f + async def test_robotstxt_meta(self): middleware = RobotsTxtMiddleware(self._get_successful_crawler()) meta = {"dont_obey_robotstxt": True} - return DeferredList( - [ - self.assertNotIgnored( - Request("http://site.local/allowed", meta=meta), middleware - ), - self.assertNotIgnored( - Request("http://site.local/admin/main", meta=meta), middleware - ), - self.assertNotIgnored( - Request("http://site.local/static/", meta=meta), middleware - ), - ], - fireOnOneErrback=True, + await self.assertNotIgnored( + Request("http://site.local/allowed", meta=meta), middleware + ) + await self.assertNotIgnored( + Request("http://site.local/admin/main", meta=meta), middleware + ) + await self.assertNotIgnored( + Request("http://site.local/static/", meta=meta), middleware ) - def _get_garbage_crawler(self): + def _get_garbage_crawler(self) -> Crawler: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response( @@ -116,22 +109,16 @@ Disallow: /some/randome/page.html crawler.engine.download.side_effect = return_response return crawler - def test_robotstxt_garbage(self): + @deferred_f_from_coro_f + async def test_robotstxt_garbage(self): # garbage response should be discarded, equal 'allow all' middleware = RobotsTxtMiddleware(self._get_garbage_crawler()) - return DeferredList( - [ - self.assertNotIgnored(Request("http://site.local"), middleware), - self.assertNotIgnored(Request("http://site.local/allowed"), middleware), - self.assertNotIgnored( - Request("http://site.local/admin/main"), middleware - ), - self.assertNotIgnored(Request("http://site.local/static/"), middleware), - ], - fireOnOneErrback=True, - ) + await self.assertNotIgnored(Request("http://site.local"), middleware) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - def _get_emptybody_crawler(self): + def _get_emptybody_crawler(self) -> Crawler: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response("http://site.local/robots.txt") @@ -144,21 +131,16 @@ Disallow: /some/randome/page.html crawler.engine.download.side_effect = return_response return crawler - def test_robotstxt_empty_response(self): + @deferred_f_from_coro_f + async def test_robotstxt_empty_response(self): # empty response should equal 'allow all' middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) - return DeferredList( - [ - self.assertNotIgnored(Request("http://site.local/allowed"), middleware), - self.assertNotIgnored( - Request("http://site.local/admin/main"), middleware - ), - self.assertNotIgnored(Request("http://site.local/static/"), middleware), - ], - fireOnOneErrback=True, - ) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - def test_robotstxt_error(self): + @deferred_f_from_coro_f + async def test_robotstxt_error(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = error.DNSLookupError("Robotstxt address not found") @@ -171,15 +153,13 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self.crawler) middleware._logerror = mock.MagicMock(side_effect=middleware._logerror) - deferred = middleware.process_request(Request("http://site.local"), None) + await maybe_deferred_to_future( + middleware.process_request(Request("http://site.local"), None) + ) + assert middleware._logerror.called - def check_called(_: Any) -> None: - assert middleware._logerror.called - - deferred.addCallback(check_called) - return deferred - - def test_robotstxt_immediate_error(self): + @deferred_f_from_coro_f + async def test_robotstxt_immediate_error(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = error.DNSLookupError("Robotstxt address not found") @@ -191,9 +171,10 @@ Disallow: /some/randome/page.html self.crawler.engine.download.side_effect = immediate_failure middleware = RobotsTxtMiddleware(self.crawler) - return self.assertNotIgnored(Request("http://site.local"), middleware) + await self.assertNotIgnored(Request("http://site.local"), middleware) - def test_ignore_robotstxt_request(self): + @deferred_f_from_coro_f + async def test_ignore_robotstxt_request(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) def ignore_request(request): @@ -206,13 +187,8 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self.crawler) mw_module_logger.error = mock.MagicMock() - d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - - def check_not_called(_: Any) -> None: - assert not mw_module_logger.error.called # type: ignore[attr-defined] - - d.addCallback(check_not_called) - return d + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + assert not mw_module_logger.error.called # type: ignore[attr-defined] def test_robotstxt_user_agent_setting(self): crawler = self._get_successful_crawler() @@ -236,19 +212,27 @@ Disallow: /some/randome/page.html Deferred, ) - def assertNotIgnored(self, request, middleware): + async def assertNotIgnored( + self, request: Request, middleware: RobotsTxtMiddleware + ) -> None: spider = None # not actually used - dfd = maybeDeferred(middleware.process_request, request, spider) - dfd.addCallback(self.assertIsNone) - return dfd + result = await maybe_deferred_to_future( + maybeDeferred(middleware.process_request, request, spider) # type: ignore[call-overload] + ) + assert result is None - def assertIgnored(self, request, middleware): + async def assertIgnored( + self, request: Request, middleware: RobotsTxtMiddleware + ) -> None: spider = None # not actually used - return self.assertFailure( - maybeDeferred(middleware.process_request, request, spider), IgnoreRequest + await maybe_deferred_to_future( + self.assertFailure( + middleware.process_request(request, spider), # type: ignore[arg-type] + IgnoreRequest, + ) ) - def assertRobotsTxtRequested(self, base_url): + def assertRobotsTxtRequested(self, base_url: str) -> None: calls = self.crawler.engine.download.call_args_list request = calls[0][0][0] assert request.url == f"{base_url}/robots.txt" From daf9db72b2dc4ff8a6344fb9a0a4817fb953fdd4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 23 Apr 2025 18:29:04 +0400 Subject: [PATCH 1699/2083] Base class for universal spider middlewares (#6693) * Initial BaseSpiderMiddleware. * Rename the new methods. * Remove the spider argument from new BaseSpiderMiddleware methods. * Add docs for BaseSpiderMiddleware. * Silence pylint. * Add BaseSpiderMiddleware tests. * Add a release note. --- docs/news.rst | 17 ++++ docs/topics/spider-middleware.rst | 13 +++ scrapy/spidermiddlewares/base.py | 97 ++++++++++++++++++++ scrapy/spidermiddlewares/depth.py | 38 ++++---- scrapy/spidermiddlewares/offsite.py | 45 ++++------ scrapy/spidermiddlewares/referer.py | 33 +++---- scrapy/spidermiddlewares/urllength.py | 77 ++++++---------- tests/test_spidermiddleware_base.py | 120 +++++++++++++++++++++++++ tests/test_spidermiddleware_depth.py | 7 +- tests/test_spidermiddleware_offsite.py | 2 +- 10 files changed, 329 insertions(+), 120 deletions(-) create mode 100644 scrapy/spidermiddlewares/base.py create mode 100644 tests/test_spidermiddleware_base.py diff --git a/docs/news.rst b/docs/news.rst index 9a68f8852..9f476ee21 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,23 @@ Release notes ============= +.. _release-VERSION: + +Scrapy VERSION (unreleased) +--------------------------- + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The ``from_settings()`` method of + :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is removed + without a deprecation period (this was needed because after the + introduction of the + :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and + switching built-in spider middlewares to it those middlewares need the + :class:`~scrapy.crawler.Crawler` instance at run time). Please use + ``from_crawler()`` instead. + .. _release-2.12.0: Scrapy 2.12.0 (2024-11-18) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 567a875b6..2211a822f 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -189,6 +189,19 @@ one or more of these methods: :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object +Base class for custom spider middlewares +---------------------------------------- + +Scrapy provides a base class for custom spider middlewares. It's not required +to use it but it can help with simplifying middleware implementations and +reducing the amount of boilerplate code in :ref:`universal middlewares +`. + +.. module:: scrapy.spidermiddlewares.base + +.. autoclass:: BaseSpiderMiddleware + :members: + .. _topics-spider-middleware-ref: Built-in spider middleware reference diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py new file mode 100644 index 000000000..650192095 --- /dev/null +++ b/scrapy/spidermiddlewares/base.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy import Request, Spider + +if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http import Response + + +class BaseSpiderMiddleware: + """Optional base class for spider middlewares. + + This class provides helper methods for asynchronous ``process_spider_output`` + methods. Middlewares that don't have a ``process_spider_output`` method don't need + to use it. + + You can override the + :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_request` + method to add processing code for requests and the + :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_item` + method to add processing code for items. These methods take a single + request or item from the spider output iterable and return a request or + item (the same or a new one), or ``None`` to remove this request or item + from the processing. + """ + + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + + def process_spider_output( + self, response: Response, result: Iterable[Any], spider: Spider + ) -> Iterable[Any]: + for o in result: + if isinstance(o, Request): + o = self.get_processed_request(o, response) + else: + o = self.get_processed_item(o, response) + if o is not None: + yield o + + async def process_spider_output_async( + self, response: Response, result: AsyncIterable[Any], spider: Spider + ) -> AsyncIterable[Any]: + async for o in result: + if isinstance(o, Request): + o = self.get_processed_request(o, response) + else: + o = self.get_processed_item(o, response) + if o is not None: + yield o + + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + """Return a processed request from the spider output. + + This method is called with a single request from the spider output. + It should return the same or a different request, or ``None`` to + ignore it. + + :param request: the input request + :type request: :class:`~scrapy.Request` object + + :param response: the response being processed + :type response: :class:`~scrapy.http.Response` object + + :return: the processed request or ``None`` + """ + return request + + def get_processed_item(self, item: Any, response: Response) -> Any: + """Return a processed item from the spider output. + + This method is called with a single item from the spider output. + It should return the same or a different item, or ``None`` to + ignore it. + + :param item: the input item + :type item: item object + + :param response: the response being processed + :type response: :class:`~scrapy.http.Response` object + + :return: the processed item or ``None`` + """ + return item diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 3164c1c03..65905f483 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -9,7 +9,7 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any -from scrapy.http import Request, Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware if TYPE_CHECKING: from collections.abc import AsyncIterable, Iterable @@ -19,14 +19,17 @@ if TYPE_CHECKING: from scrapy import Spider from scrapy.crawler import Crawler + from scrapy.http import Request, Response from scrapy.statscollectors import StatsCollector logger = logging.getLogger(__name__) -class DepthMiddleware: - def __init__( +class DepthMiddleware(BaseSpiderMiddleware): + crawler: Crawler + + def __init__( # pylint: disable=super-init-not-called self, maxdepth: int, stats: StatsCollector, @@ -45,21 +48,22 @@ class DepthMiddleware: verbose = settings.getbool("DEPTH_STATS_VERBOSE") prio = settings.getint("DEPTH_PRIORITY") assert crawler.stats - return cls(maxdepth, crawler.stats, verbose, prio) + o = cls(maxdepth, crawler.stats, verbose, prio) + o.crawler = crawler + return o def process_spider_output( self, response: Response, result: Iterable[Any], spider: Spider ) -> Iterable[Any]: self._init_depth(response, spider) - return (r for r in result if self._filter(r, response, spider)) + yield from super().process_spider_output(response, result, spider) async def process_spider_output_async( self, response: Response, result: AsyncIterable[Any], spider: Spider ) -> AsyncIterable[Any]: self._init_depth(response, spider) - async for r in result: - if self._filter(r, response, spider): - yield r + async for o in super().process_spider_output_async(response, result, spider): + yield o def _init_depth(self, response: Response, spider: Spider) -> None: # base case (depth=0) @@ -68,9 +72,9 @@ class DepthMiddleware: if self.verbose_stats: self.stats.inc_value("request_depth_count/0", spider=spider) - def _filter(self, request: Any, response: Response, spider: Spider) -> bool: - if not isinstance(request, Request): - return True + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: depth = response.meta["depth"] + 1 request.meta["depth"] = depth if self.prio: @@ -79,10 +83,12 @@ class DepthMiddleware: logger.debug( "Ignoring link (depth > %(maxdepth)d): %(requrl)s ", {"maxdepth": self.maxdepth, "requrl": request.url}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) - return False + return None if self.verbose_stats: - self.stats.inc_value(f"request_depth_count/{depth}", spider=spider) - self.stats.max_value("request_depth_max", depth, spider=spider) - return True + self.stats.inc_value( + f"request_depth_count/{depth}", spider=self.crawler.spider + ) + self.stats.max_value("request_depth_max", depth, spider=self.crawler.spider) + return request diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 646beb911..0918c9fac 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -9,11 +9,11 @@ from __future__ import annotations import logging import re import warnings -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from scrapy import Spider, signals from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.http import Request, Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware from scrapy.utils.httpobj import urlparse_cached warnings.warn( @@ -23,61 +23,52 @@ warnings.warn( ) if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable - # typing.Self requires Python 3.11 from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http import Request, Response from scrapy.statscollectors import StatsCollector logger = logging.getLogger(__name__) -class OffsiteMiddleware: - def __init__(self, stats: StatsCollector): +class OffsiteMiddleware(BaseSpiderMiddleware): + crawler: Crawler + + def __init__(self, stats: StatsCollector): # pylint: disable=super-init-not-called self.stats: StatsCollector = stats @classmethod def from_crawler(cls, crawler: Crawler) -> Self: assert crawler.stats o = cls(crawler.stats) + o.crawler = crawler crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - return (r for r in result if self._filter(r, spider)) - - async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - async for r in result: - if self._filter(r, spider): - yield r - - def _filter(self, request: Any, spider: Spider) -> bool: - if not isinstance(request, Request): - return True + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + assert self.crawler.spider if ( request.dont_filter or request.meta.get("allow_offsite") - or self.should_follow(request, spider) + or self.should_follow(request, self.crawler.spider) ): - return True + return request domain = urlparse_cached(request).hostname if domain and domain not in self.domains_seen: self.domains_seen.add(domain) logger.debug( "Filtered offsite request to %(domain)r: %(request)s", {"domain": domain, "request": request}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) - self.stats.inc_value("offsite/domains", spider=spider) - self.stats.inc_value("offsite/filtered", spider=spider) - return False + self.stats.inc_value("offsite/domains", spider=self.crawler.spider) + self.stats.inc_value("offsite/filtered", spider=self.crawler.spider) + return None def should_follow(self, request: Request, spider: Spider) -> bool: regex = self.host_regex diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a3a1e5b92..b2ba8ba8c 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -6,7 +6,7 @@ originated it. from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, cast from urllib.parse import urlparse from w3lib.url import safe_url_string @@ -14,13 +14,12 @@ from w3lib.url import safe_url_string from scrapy import Spider, signals from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable - # typing.Self requires Python 3.11 from typing_extensions import Self @@ -327,8 +326,8 @@ def _load_policy_class( return None -class RefererMiddleware: - def __init__(self, settings: BaseSettings | None = None): +class RefererMiddleware(BaseSpiderMiddleware): + def __init__(self, settings: BaseSettings | None = None): # pylint: disable=super-init-not-called self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy if settings is not None: settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) @@ -370,23 +369,13 @@ class RefererMiddleware: cls = _load_policy_class(policy_name, warning_only=True) return cls() if cls else self.default_policy() - def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - return (self._set_referer(r, response) for r in result) - - async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - async for r in result: - yield self._set_referer(r, response) - - def _set_referer(self, r: Any, response: Response) -> Any: - if isinstance(r, Request): - referrer = self.policy(response, r).referrer(response.url, r.url) - if referrer is not None: - r.headers.setdefault("Referer", referrer) - return r + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + referrer = self.policy(response, request).referrer(response.url, request.url) + if referrer is not None: + request.headers.setdefault("Referer", referrer) + return request def request_scheduled(self, request: Request, spider: Spider) -> None: # check redirected request to patch "Referer" header if necessary diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index a1cd1bb7c..177c19e1b 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -7,72 +7,49 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -import warnings -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Request, Response +from scrapy.exceptions import NotConfigured +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable - # typing.Self requires Python 3.11 from typing_extensions import Self - from scrapy import Spider from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings + from scrapy.http import Request, Response logger = logging.getLogger(__name__) -class UrlLengthMiddleware: - def __init__(self, maxlength: int): +class UrlLengthMiddleware(BaseSpiderMiddleware): + crawler: Crawler + + def __init__(self, maxlength: int): # pylint: disable=super-init-not-called self.maxlength: int = maxlength - @classmethod - def from_settings(cls, settings: BaseSettings) -> Self: - warnings.warn( - f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return cls._from_settings(settings) - @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - return cls._from_settings(crawler.settings) - - @classmethod - def _from_settings(cls, settings: BaseSettings) -> Self: - maxlength = settings.getint("URLLENGTH_LIMIT") + maxlength = crawler.settings.getint("URLLENGTH_LIMIT") if not maxlength: raise NotConfigured - return cls(maxlength) + o = cls(maxlength) + o.crawler = crawler + return o - def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - return (r for r in result if self._filter(r, spider)) - - async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - async for r in result: - if self._filter(r, spider): - yield r - - def _filter(self, request: Any, spider: Spider) -> bool: - if isinstance(request, Request) and len(request.url) > self.maxlength: - logger.info( - "Ignoring link (url length > %(maxlength)d): %(url)s ", - {"maxlength": self.maxlength, "url": request.url}, - extra={"spider": spider}, - ) - assert spider.crawler.stats - spider.crawler.stats.inc_value( - "urllength/request_ignored_count", spider=spider - ) - return False - return True + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + if len(request.url) <= self.maxlength: + return request + logger.info( + "Ignoring link (url length > %(maxlength)d): %(url)s ", + {"maxlength": self.maxlength, "url": request.url}, + extra={"spider": self.crawler.spider}, + ) + assert self.crawler.stats + self.crawler.stats.inc_value( + "urllength/request_ignored_count", spider=self.crawler.spider + ) + return None diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py new file mode 100644 index 000000000..46be879f3 --- /dev/null +++ b/tests/test_spidermiddleware_base.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy import Request, Spider +from scrapy.http import Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + + +@pytest.fixture +def crawler() -> Crawler: + return get_crawler(Spider) + + +def test_trivial(crawler): + class TrivialSpiderMiddleware(BaseSpiderMiddleware): + pass + + mw = TrivialSpiderMiddleware.from_crawler(crawler) + assert hasattr(mw, "crawler") + assert mw.crawler is crawler + test_req = Request("data:,") + spider_output = [test_req, {"foo": "bar"}] + processed = list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ) + assert processed == [test_req, {"foo": "bar"}] + + +def test_processed_request(crawler): + class ProcessReqSpiderMiddleware(BaseSpiderMiddleware): + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + if request.url == "data:2,": + return None + if request.url == "data:3,": + return Request("data:30,") + return request + + mw = ProcessReqSpiderMiddleware.from_crawler(crawler) + test_req1 = Request("data:1,") + test_req2 = Request("data:2,") + test_req3 = Request("data:3,") + spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] + processed = list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ) + assert len(processed) == 3 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": "bar"} + assert isinstance(processed[2], Request) + assert processed[2].url == "data:30," + + +def test_processed_item(crawler): + class ProcessItemSpiderMiddleware(BaseSpiderMiddleware): + def get_processed_item(self, item: Any, response: Response) -> Any: + if item["foo"] == 2: + return None + if item["foo"] == 3: + item["foo"] = 30 + return item + + mw = ProcessItemSpiderMiddleware.from_crawler(crawler) + test_req = Request("data:,") + spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}] + processed = list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ) + assert processed == [{"foo": 1}, test_req, {"foo": 30}] + + +def test_processed_both(crawler): + class ProcessBothSpiderMiddleware(BaseSpiderMiddleware): + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + if request.url == "data:2,": + return None + if request.url == "data:3,": + return Request("data:30,") + return request + + def get_processed_item(self, item: Any, response: Response) -> Any: + if item["foo"] == 2: + return None + if item["foo"] == 3: + item["foo"] = 30 + return item + + mw = ProcessBothSpiderMiddleware.from_crawler(crawler) + test_req1 = Request("data:1,") + test_req2 = Request("data:2,") + test_req3 = Request("data:3,") + spider_output = [ + test_req1, + {"foo": 1}, + {"foo": 2}, + test_req2, + {"foo": 3}, + test_req3, + ] + processed = list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ) + assert len(processed) == 4 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": 1} + assert processed[2] == {"foo": 30} + assert isinstance(processed[3], Request) + assert processed[3].url == "data:30," diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index dfcc141c3..9b4aa624c 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -1,19 +1,18 @@ from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler class TestDepthMiddleware: def setup_method(self): - crawler = get_crawler(Spider) + crawler = get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True}) self.spider = crawler._create_spider("scrapytest.org") - self.stats = StatsCollector(crawler) + self.stats = crawler.stats self.stats.open_spider(self.spider) - self.mw = DepthMiddleware(1, self.stats, True) + self.mw = DepthMiddleware.from_crawler(crawler) def test_process_spider_output(self): req = Request("http://scrapytest.org") diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index f4563a0a4..e4f4b8f9b 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -10,7 +10,7 @@ from scrapy.utils.test import get_crawler class TestOffsiteMiddleware: def setup_method(self): crawler = get_crawler(Spider) - self.spider = crawler._create_spider(**self._get_spiderargs()) + self.spider = crawler.spider = crawler._create_spider(**self._get_spiderargs()) self.mw = OffsiteMiddleware.from_crawler(crawler) self.mw.spider_opened(self.spider) From b1f85b5a173f48bad2881465efbab5c22aa0327d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 24 Apr 2025 20:03:36 +0500 Subject: [PATCH 1700/2083] Release notes for 2.13.0, up to b4c253102139e842859a9abf1455e62504cc9511. --- docs/news.rst | 198 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 193 insertions(+), 5 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 9f476ee21..b9b5ce320 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,21 +5,209 @@ Release notes .. _release-VERSION: -Scrapy VERSION (unreleased) ---------------------------- +Scrapy 2.13.0 (unreleased) +-------------------------- + +Highlights: + +- Added the :reqmeta:`allow_offsite` request meta key + +- HTTP/1.0 support is deprecated + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for PyPy 3.9. + (:issue:`6613`) Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The ``from_settings()`` method of - :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is removed - without a deprecation period (this was needed because after the - introduction of the + :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, + deprecated in 2.12.0, is removed earlier than the usual deprecation period + (this was needed because after the introduction of the :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and switching built-in spider middlewares to it those middlewares need the :class:`~scrapy.crawler.Crawler` instance at run time). Please use ``from_crawler()`` instead. +Deprecations +~~~~~~~~~~~~ + +- Functions that were imported from :mod:`w3lib.url` and re-exported in + :mod:`scrapy.utils.url` are now deprecated, you should import them from + ``w3lib.url`` directly. They are: + + - ``scrapy.utils.url.add_or_replace_parameter()`` + + - ``scrapy.utils.url.add_or_replace_parameters()`` + + - ``scrapy.utils.url.any_to_uri()`` + + - ``scrapy.utils.url.canonicalize_url()`` + + - ``scrapy.utils.url.file_uri_to_path()`` + + - ``scrapy.utils.url.is_url()`` + + - ``scrapy.utils.url.parse_data_uri()`` + + - ``scrapy.utils.url.parse_url()`` + + - ``scrapy.utils.url.path_to_file_uri()`` + + - ``scrapy.utils.url.safe_download_url()`` + + - ``scrapy.utils.url.safe_url_string()`` + + - ``scrapy.utils.url.url_query_cleaner()`` + + - ``scrapy.utils.url.url_query_parameter()`` + + - ``scrapy.utils.url._unquotepath()`` + + - ``scrapy.utils.url._safe_chars`` attribute + + (:issue:`4577`, :issue:`6583`, :issue:`6586`) + +- HTTP/1.0 support code is deprecated. It was disabled by default and + couldn't be used together with HTTP/1.1. If you still need it, you should + write your own download handler or copy the code from Scrapy. The + deprecations include: + + - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler`` + + - ``scrapy.core.downloader.webclient.ScrapyHTTPClientFactory`` + + - ``scrapy.core.downloader.webclient.ScrapyHTTPPageGetter`` + + - Overriding + ``scrapy.core.downloader.contextfactory.ScrapyClientContextFactory.getContext()`` + + (:issue:`6634`) + +- ``scrapy.utils.versions.scrapy_components_versions()`` is deprecated, use + :func:`scrapy.utils.versions.get_versions()` instead. + (:issue:`6582`) + +- ``BaseDupeFilter.log()`` is deprecated. It does nothing and shouldn't be + called. + (:issue:`4151`) + +New features +~~~~~~~~~~~~ + +- Added the :reqmeta:`allow_offsite` request meta key that can be used + instead of the more general :attr:`~scrapy.Request.dont_filter` request + attribute to skip processing of the request by + :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` (but not + by other code that checks :attr:`~scrapy.Request.dont_filter`). + (:issue:`3690`, :issue:`6151`, :issue:`6366`) + +- :ref:`Scrapy add-ons ` can now define a class method called + ``update_pre_crawler_settings()`` to update :ref:`pre-crawler settings + `. + (:issue:`6544`, :issue:`6568`) + +- Added the :setting:`DEFAULT_DROPITEM_LOG_LEVEL` setting and the + :attr:`scrapy.exceptions.DropItem.log_level` attribute that allow + customizing the log level of the message that is logged when an item is + dropped. + (:issue:`6603`, :issue:`6608`) + +- Added the :setting:`LOG_VERSIONS` setting that allows customizing the + list of software which versions are logged when the spider starts. + (:issue:`6582`) + +Improvements +~~~~~~~~~~~~ + +- Improved the error message when running a ``scrapy`` command that requires + a project (such as ``scrapy crawl``) outside of a project directory. + (:issue:`2349`, :issue:`3426`) + +- An empty :setting:`ADDONS` setting added to the ``settings.py`` template + for new projects. + (:issue:`6587`) + +Bug fixes +~~~~~~~~~ + +- Fixed calculation of ``items_per_minute`` and ``responses_per_minute`` + stats. + (:issue:`6599`) + +- Fixed an error initializing + :class:`scrapy.extensions.feedexport.GCSFeedStorage`. + (:issue:`6617`, :issue:`6628`) + +- Fixed an error running ``scrapy bench``. + (:issue:`6632`, :issue:`6633`) + +Documentation +~~~~~~~~~~~~~ + +- Improved the contribution docs. + (:issue:`6561`, :issue:`6575`) + +- Other documentation improvements and fixes. + (:issue:`4151`, + :issue:`6526`, + :issue:`6620`, + :issue:`6621`, + :issue:`6622`, + :issue:`6623`, + :issue:`6624`) + +Packaging +~~~~~~~~~ + +- Switched from ``setup.py`` to ``pyproject.toml``. + (:issue:`6514`, :issue:`6547`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Replaced most linters with ``ruff``. + (:issue:`6565`, :issue:`6576`, :issue:`6577`, :issue:`6581`, :issue:`6584`, + :issue:`6595`, :issue:`6601`, :issue:`6631`) + +- Improved accuracy and performance of collecting test coverage. + (:issue:`6567`) + +- Fixed an error that prevented running tests from directories other than the + top level source directory. + (:issue:`6567`) + +- Reduced the amount of ``mockserver`` calls in tests to improve the overall + test run time. + (:issue:`6637`, :issue:`6648`) + +- Fixed tests that were running the same test code more than once. + (:issue:`6646`) + +- Type hints improvements and fixes. + (:issue:`6578`, :issue:`6579`, :issue:`6593`, :issue:`6605`) + +- CI and test improvements and fixes. + (:issue:`5360`, + :issue:`6271`, + :issue:`6547`, + :issue:`6560`, + :issue:`6602`, + :issue:`6607`, + :issue:`6609`, + :issue:`6613`, + :issue:`6619`, + :issue:`6626`) + +- Code cleanups. + (:issue:`6600`, + :issue:`6606`, + :issue:`6635`) + + .. _release-2.12.0: Scrapy 2.12.0 (2024-11-18) From 095140f134745960751c8c4b34da7de6b91e4a82 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 25 Apr 2025 19:43:29 +0500 Subject: [PATCH 1701/2083] Cover the current master in the release notes (up to daf9db7). --- docs/news.rst | 223 +++++++++++++++++++++++++++---- docs/topics/addons.rst | 2 + docs/topics/coroutines.rst | 6 + docs/topics/settings.rst | 2 +- scrapy/spidermiddlewares/base.py | 2 + 5 files changed, 210 insertions(+), 25 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index b9b5ce320..74a26c383 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,16 +3,22 @@ Release notes ============= -.. _release-VERSION: +.. _release-2.13.0: Scrapy 2.13.0 (unreleased) -------------------------- Highlights: +- The asyncio reactor is now enabled by default + - Added the :reqmeta:`allow_offsite` request meta key -- HTTP/1.0 support is deprecated +- :ref:`Spider middlewares that don't support asynchronous spider output + ` are deprecated + +- Added a base class for :ref:`universal spider middlewares + ` Modified requirements ~~~~~~~~~~~~~~~~~~~~~ @@ -23,21 +29,62 @@ Modified requirements Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- The default value of the :setting:`TWISTED_REACTOR` setting was changed + from ``None`` to + ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. This value + was used in newly generated projects since Scrapy 2.7.0 but now existing + projects that don't explicitly set this setting will also use the asyncio + reactor. You can :ref:`change this setting in your project + ` to use a different reactor. + (:issue:`6659`, :issue:`6713`) + - The ``from_settings()`` method of :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, - deprecated in 2.12.0, is removed earlier than the usual deprecation period - (this was needed because after the introduction of the + deprecated in Scrapy 2.12.0, is removed earlier than the usual deprecation + period (this was needed because after the introduction of the :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and switching built-in spider middlewares to it those middlewares need the :class:`~scrapy.crawler.Crawler` instance at run time). Please use ``from_crawler()`` instead. + (:issue:`6693`) + +- ``scrapy.utils.url.escape_ajax()`` is no longer called when a + :class:`~scrapy.Request` instance is created. It was only useful for + websites supporting the ``_escaped_fragment_`` feature which most modern + websites don't support. If you still need this you can modify the URLs + before passing them to :class:`~scrapy.Request`. + (:issue:`6523`, :issue:`6651`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed old deprecated name aliases for some signals: + + - ``stats_spider_opened`` (use ``spider_opened`` instead) + + - ``stats_spider_closing`` and ``stats_spider_closed`` (use + ``spider_closed`` instead) + + - ``item_passed`` (use ``item_scraped`` instead) + + - ``request_received`` (use ``request_scheduled`` instead) + + (:issue:`6654`, :issue:`6655`) Deprecations ~~~~~~~~~~~~ +- :ref:`Spider middlewares that don't support asynchronous spider output + ` are deprecated. The async iterable + downgrading feature, needed for using such middlewares with asynchronous + callbacks and with other spider middlewares that produce asynchronous + iterables, is also deprecated. Please update all such middlewares to + support asynchronous spider output. + (:issue:`6664`) + - Functions that were imported from :mod:`w3lib.url` and re-exported in :mod:`scrapy.utils.url` are now deprecated, you should import them from - ``w3lib.url`` directly. They are: + :mod:`w3lib.url` directly. They are: - ``scrapy.utils.url.add_or_replace_parameter()`` @@ -65,10 +112,6 @@ Deprecations - ``scrapy.utils.url.url_query_parameter()`` - - ``scrapy.utils.url._unquotepath()`` - - - ``scrapy.utils.url._safe_chars`` attribute - (:issue:`4577`, :issue:`6583`, :issue:`6586`) - HTTP/1.0 support code is deprecated. It was disabled by default and @@ -87,6 +130,37 @@ Deprecations (:issue:`6634`) +- The following modules and functions used only in tests are deprecated: + + - the ``scrapy/utils/testproc`` module + + - the ``scrapy/utils/testsite`` module + + - ``scrapy.utils.test.assert_gcs_environ()`` + + - ``scrapy.utils.test.get_ftp_content_and_delete()`` + + - ``scrapy.utils.test.get_gcs_content_and_delete()`` + + - ``scrapy.utils.test.mock_google_cloud_storage()`` + + - ``scrapy.utils.test.skip_if_no_boto()`` + + If you need to use them in your tests or code, you can copy the code from Scrapy. + (:issue:`6696`) + +- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` is + deprecated. It was disabled by default and isn't useful for most of the + existing websites. + (:issue:`6523`, :issue:`6651`, :issue:`6656`) + +- ``scrapy.utils.url.escape_ajax()`` is deprecated. + (:issue:`6523`, :issue:`6651`) + +- ``scrapy.spiders.init.InitSpider`` is deprecated. If you find it useful, + you can copy its code from Scrapy. + (:issue:`6708`, :issue:`6714`) + - ``scrapy.utils.versions.scrapy_components_versions()`` is deprecated, use :func:`scrapy.utils.versions.get_versions()` instead. (:issue:`6582`) @@ -105,29 +179,63 @@ New features by other code that checks :attr:`~scrapy.Request.dont_filter`). (:issue:`3690`, :issue:`6151`, :issue:`6366`) +- Added an optional base class for spider middlewares, + :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which can be + helpful for writing :ref:`universal spider middlewares + ` without boilerplate and code duplication. + The built-in spider middlewares now inherit from this class. + (:issue:`6693`) + - :ref:`Scrapy add-ons ` can now define a class method called ``update_pre_crawler_settings()`` to update :ref:`pre-crawler settings `. (:issue:`6544`, :issue:`6568`) +- Added :ref:`helpers ` for modifying :ref:`component + priority dictionary ` settings. + (:issue:`6614`) + +- Responses that use an unknown/unsupported encoding now produce a warning. + If Scrapy knows that installing an additional package (such as brotli_) + will allow decoding the response, that will be mentioned in the warning. + (:issue:`4697`, :issue:`6618`) + +- Added the ``spider_exceptions/count`` stat which tracks the total count of + exceptions (tracked also by per-type ``spider_exceptions/*`` stats). + (:issue:`6739`, :issue:`6740`) + - Added the :setting:`DEFAULT_DROPITEM_LOG_LEVEL` setting and the :attr:`scrapy.exceptions.DropItem.log_level` attribute that allow customizing the log level of the message that is logged when an item is dropped. (:issue:`6603`, :issue:`6608`) +- Added support for the ``-b, --cookie`` curl argument to + :meth:`scrapy.Request.from_curl`. + (:issue:`6684`) + - Added the :setting:`LOG_VERSIONS` setting that allows customizing the - list of software which versions are logged when the spider starts. + list of software whose versions are logged when the spider starts. (:issue:`6582`) +- Added the :setting:`WARN_ON_GENERATOR_RETURN_VALUE` setting that allows + disabling run time analysis of callback code used to warn about incorrect + ``return`` statements in generator-based callbacks. You may need to disable + this setting if this analysis breaks on your callback code. + (:issue:`6731`, :issue:`6738`) + Improvements ~~~~~~~~~~~~ +- Removed or postponed some calls of :func:`itemadapter.is_item` to increase + performance. + (:issue:`6719`) + - Improved the error message when running a ``scrapy`` command that requires a project (such as ``scrapy crawl``) outside of a project directory. (:issue:`2349`, :issue:`3426`) -- An empty :setting:`ADDONS` setting added to the ``settings.py`` template +- Added an empty :setting:`ADDONS` setting to the ``settings.py`` template for new projects. (:issue:`6587`) @@ -145,12 +253,46 @@ Bug fixes - Fixed an error running ``scrapy bench``. (:issue:`6632`, :issue:`6633`) +- Fixed duplicated log messages about the reactor and the event loop. + (:issue:`6636`, :issue:`6657`) + +- Fixed resolving type annotations of ``SitemapSpider._parse_sitemap()`` at + run time, required by tools such as scrapy-poet_. + (:issue:`6665`, :issue:`6671`) + + .. _scrapy-poet: https://github.com/scrapinghub/scrapy-poet + +- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without + an installed reactor now raises an exception instead of installing a + reactor. + (:issue:`6732`, :issue:`6735`) + +- Restored support for the ``x-gzip`` content encoding. + (:issue:`6618`) + Documentation ~~~~~~~~~~~~~ -- Improved the contribution docs. +- Improved the :ref:`docs ` about asynchronous + iterable support in spider middlewares. + (:issue:`6688`) + +- Improved the :ref:`docs ` about using + :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based + code. + (:issue:`6734`) + +- Improved the :ref:`contribution docs `. (:issue:`6561`, :issue:`6575`) +- Removed the ``Splash`` recommendation from the :ref:`headless browser + ` suggestion. We no longer recommend using + ``Splash`` and recommend using other headless browser solutions instead. + (:issue:`6642`, :issue:`6701`) + +- Added the dark mode to the HTML documentation. + (:issue:`6653`) + - Other documentation improvements and fixes. (:issue:`4151`, :issue:`6526`, @@ -158,7 +300,9 @@ Documentation :issue:`6621`, :issue:`6622`, :issue:`6623`, - :issue:`6624`) + :issue:`6624`, + :issue:`6721`, + :issue:`6723`) Packaging ~~~~~~~~~ @@ -169,12 +313,20 @@ Packaging Quality assurance ~~~~~~~~~~~~~~~~~ -- Replaced most linters with ``ruff``. - (:issue:`6565`, :issue:`6576`, :issue:`6577`, :issue:`6581`, :issue:`6584`, - :issue:`6595`, :issue:`6601`, :issue:`6631`) +- Replaced most linters with ruff_. + (:issue:`6565`, + :issue:`6576`, + :issue:`6577`, + :issue:`6581`, + :issue:`6584`, + :issue:`6595`, + :issue:`6601`, + :issue:`6631`) + + .. _ruff: https://docs.astral.sh/ruff/ - Improved accuracy and performance of collecting test coverage. - (:issue:`6567`) + (:issue:`6255`, :issue:`6610`) - Fixed an error that prevented running tests from directories other than the top level source directory. @@ -185,10 +337,28 @@ Quality assurance (:issue:`6637`, :issue:`6648`) - Fixed tests that were running the same test code more than once. - (:issue:`6646`) + (:issue:`6646`, :issue:`6647`, :issue:`6650`) + +- Refactored tests to use more ``pytest`` features instead of ``unittest`` + ones where possible. + (:issue:`6678`, + :issue:`6680`, + :issue:`6695`, + :issue:`6699`, + :issue:`6700`, + :issue:`6702`, + :issue:`6709`, + :issue:`6710`, + :issue:`6711`, + :issue:`6712`, + :issue:`6725`) - Type hints improvements and fixes. - (:issue:`6578`, :issue:`6579`, :issue:`6593`, :issue:`6605`) + (:issue:`6578`, + :issue:`6579`, + :issue:`6593`, + :issue:`6605`, + :issue:`6694`) - CI and test improvements and fixes. (:issue:`5360`, @@ -200,7 +370,16 @@ Quality assurance :issue:`6609`, :issue:`6613`, :issue:`6619`, - :issue:`6626`) + :issue:`6626`, + :issue:`6679`, + :issue:`6703`, + :issue:`6704`, + :issue:`6716`, + :issue:`6720`, + :issue:`6722`, + :issue:`6724`, + :issue:`6741`, + :issue:`6743`) - Code cleanups. (:issue:`6600`, @@ -806,8 +985,6 @@ Bug fixes - Restored support for brotlipy_, which had been dropped in Scrapy 2.11.1 in favor of brotli_. (:issue:`6261`) - .. _brotli: https://github.com/google/brotli - .. note:: brotlipy is deprecated, both in Scrapy and upstream. Use brotli instead if you can. @@ -2391,8 +2568,6 @@ Scrapy 2.5.1 (2021-10-05) need to upgrade scrapy-splash to a greater version for it to continue to work. -.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash - .. _release-2.5.0: diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 17e3c177a..815501e66 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -124,6 +124,8 @@ Set some basic configuration: "ITEM_PIPELINES", MyPipeline, 200 ) +.. _priority-dict-helpers: + .. tip:: When editing a :ref:`component priority dictionary ` setting, like :setting:`ITEM_PIPELINES`, consider using setting methods like diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 1c80857f6..1a84f893c 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -344,3 +344,9 @@ For example: feature will be removed, and all spider middlewares will be expected to define their ``process_spider_output`` method as an asynchronous generator. + +Since 2.13.0, Scrapy provides a base class, +:class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which implements +the ``process_spider_output()`` and ``process_spider_output_async()`` methods, +so instead of duplicating the processing code you can override the +``get_processed_request()`` and/or the ``get_processed_item()`` method. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a59a61050..73ac36646 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2005,7 +2005,7 @@ current platform. ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated ``settings.py`` file. -.. versionchanged:: VERSION +.. versionchanged:: 2.13 The default value was changed from ``None`` to ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index 650192095..5e4370d45 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -17,6 +17,8 @@ if TYPE_CHECKING: class BaseSpiderMiddleware: """Optional base class for spider middlewares. + .. versionadded:: 2.13 + This class provides helper methods for asynchronous ``process_spider_output`` methods. Middlewares that don't have a ``process_spider_output`` method don't need to use it. From 4aba7e5f6675703159220ee22bba6953c5685ef6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 25 Apr 2025 20:16:26 +0500 Subject: [PATCH 1702/2083] Mention the deprecation of TestSpider. --- docs/news.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 74a26c383..7bb25e6b6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -149,6 +149,11 @@ Deprecations If you need to use them in your tests or code, you can copy the code from Scrapy. (:issue:`6696`) +- ``scrapy.utils.test.TestSpider`` is deprecated. If you need an empty spider + class you can use :class:`scrapy.utils.spider.DefaultSpider` or create your + own subclass of :class:`scrapy.Spider`. + (:issue:`6678`) + - ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` is deprecated. It was disabled by default and isn't useful for most of the existing websites. From eced5ca2d3c85c36b18f0da3d5d888e7e66a7014 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 30 Apr 2025 00:51:23 +0500 Subject: [PATCH 1703/2083] Remove the unnecessary spider argument from Scraper and related code. --- scrapy/core/engine.py | 12 +-- scrapy/core/scraper.py | 178 +++++++++++++++++++++++++++------------- scrapy/core/spidermw.py | 6 +- tests/test_engine.py | 7 +- 4 files changed, 131 insertions(+), 72 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b7a73700b..653e5e05c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -265,7 +265,7 @@ class ExecutionEngine: self.crawl(result) return None - d = self.scraper.enqueue_scrape(result, request, self.spider) + d = self.scraper.enqueue_scrape(result, request) d.addErrback( lambda f: logger.error( "Error while enqueuing downloader output", @@ -290,14 +290,14 @@ class ExecutionEngine: """Inject the request into the spider <-> downloader pipeline""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - self._schedule_request(request, self.spider) + self._schedule_request(request) self.slot.nextcall.schedule() # type: ignore[union-attr] - def _schedule_request(self, request: Request, spider: Spider) -> None: + def _schedule_request(self, request: Request) -> None: request_scheduled_result = self.signals.send_catch_log( signals.request_scheduled, request=request, - spider=spider, + spider=self.spider, dont_log=IgnoreRequest, ) for handler, result in request_scheduled_result: @@ -305,7 +305,7 @@ class ExecutionEngine: return if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( - signals.request_dropped, request=request, spider=spider + signals.request_dropped, request=request, spider=self.spider ) def download(self, request: Request) -> Deferred[Response]: @@ -438,7 +438,7 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.downloader.close()) dfd.addErrback(log_failure("Downloader close failure")) - dfd.addBoth(lambda _: self.scraper.close_spider(spider)) + dfd.addBoth(lambda _: self.scraper.close_spider()) dfd.addErrback(log_failure("Scraper close failure")) if hasattr(self.slot.scheduler, "close"): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 496adb500..2942dfa58 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -4,6 +4,7 @@ extracts information from them""" from __future__ import annotations import logging +import warnings from collections import deque from collections.abc import AsyncIterable, Iterator from typing import TYPE_CHECKING, Any, TypeVar, Union, cast @@ -13,7 +14,12 @@ from twisted.python.failure import Failure from scrapy import Spider, signals from scrapy.core.spidermw import SpiderMiddlewareManager -from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest +from scrapy.exceptions import ( + CloseSpider, + DropItem, + IgnoreRequest, + ScrapyDeprecationWarning, +) from scrapy.http import Request, Response from scrapy.utils.defer import ( aiter_errback, @@ -110,27 +116,43 @@ class Scraper: self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) yield self.itemproc.open_spider(spider) - def close_spider(self, spider: Spider) -> Deferred[Spider]: + def close_spider(self, spider: Spider | None = None) -> Deferred[Spider]: """Close a spider being scraped and release its resources""" + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.close_spider() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: raise RuntimeError("Scraper slot not assigned") self.slot.closing = Deferred() self.slot.closing.addCallback(self.itemproc.close_spider) - self._check_if_closing(spider) + self._check_if_closing() return self.slot.closing def is_idle(self) -> bool: """Return True if there isn't any more spiders to process""" return not self.slot - def _check_if_closing(self, spider: Spider) -> None: + def _check_if_closing(self) -> None: assert self.slot is not None # typing + assert self.crawler.spider if self.slot.closing and self.slot.is_idle(): - self.slot.closing.callback(spider) + assert self.crawler.spider + self.slot.closing.callback(self.crawler.spider) def enqueue_scrape( - self, result: Response | Failure, request: Request, spider: Spider + self, result: Response | Failure, request: Request, spider: Spider | None = None ) -> _HandleOutputDeferred: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.enqueue_scrape() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) @@ -138,8 +160,8 @@ class Scraper: def finish_scraping(_: _T) -> _T: assert self.slot is not None self.slot.finish_response(result, request) - self._check_if_closing(spider) - self._scrape_next(spider) + self._check_if_closing() + self._scrape_next() return _ dfd.addBoth(finish_scraping) @@ -148,20 +170,20 @@ class Scraper: "Scraper bug processing %(request)s", {"request": request}, exc_info=failure_to_exc_info(f), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) ) - self._scrape_next(spider) + self._scrape_next() return dfd - def _scrape_next(self, spider: Spider) -> None: + def _scrape_next(self) -> None: assert self.slot is not None # typing while self.slot.queue: response, request, deferred = self.slot.next_response_request_deferred() - self._scrape(response, request, spider).chainDeferred(deferred) + self._scrape(response, request).chainDeferred(deferred) def _scrape( - self, result: Response | Failure, request: Request, spider: Spider + self, result: Response | Failure, request: Request ) -> _HandleOutputDeferred: """ Handle the downloaded response or failure through the spider callback/errback @@ -171,40 +193,49 @@ class Scraper: f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) dfd: Deferred[Iterable[Any] | AsyncIterable[Any]] = self._scrape2( - result, request, spider + result, request ) # returns spider's processed output - dfd.addErrback(self.handle_spider_error, request, result, spider) + dfd.addErrback(self.handle_spider_error, request, result) dfd2: _HandleOutputDeferred = dfd.addCallback( - self.handle_spider_output, request, cast(Response, result), spider + self.handle_spider_output, request, cast(Response, result) ) return dfd2 def _scrape2( - self, result: Response | Failure, request: Request, spider: Spider + self, result: Response | Failure, request: Request ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: """ Handle the different cases of request's result been a Response or a Failure """ if isinstance(result, Response): # Deferreds are invariant so Mutable*Chain isn't matched to *Iterable + assert self.crawler.spider return self.spidermw.scrape_response( # type: ignore[return-value] - self.call_spider, result, request, spider + self.call_spider, result, request, self.crawler.spider ) # else result is a Failure - dfd = self.call_spider(result, request, spider) - dfd.addErrback(self._log_download_errors, result, request, spider) + dfd = self.call_spider(result, request) + dfd.addErrback(self._log_download_errors, result, request) return dfd def call_spider( - self, result: Response | Failure, request: Request, spider: Spider + self, result: Response | Failure, request: Request, spider: Spider | None = None ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.call_spider() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + assert self.crawler.spider dfd: Deferred[Any] if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request assert result.request - callback = result.request.callback or spider._parse - warn_on_generator_with_return_value(spider, callback) + callback = result.request.callback or self.crawler.spider._parse + warn_on_generator_with_return_value(self.crawler.spider, callback) dfd = defer_succeed(result) dfd.addCallbacks( callback=callback, callbackKeywords=result.request.cb_kwargs @@ -214,7 +245,9 @@ class Scraper: result.request = request # type: ignore[attr-defined] dfd = defer_fail(result) if request.errback: - warn_on_generator_with_return_value(spider, request.errback) + warn_on_generator_with_return_value( + self.crawler.spider, request.errback + ) dfd.addErrback(request.errback) dfd2: Deferred[Iterable[Any] | AsyncIterable[Any]] = dfd.addCallback( iterate_spider_output @@ -226,29 +259,44 @@ class Scraper: _failure: Failure, request: Request, response: Response | Failure, - spider: Spider, + spider: Spider | None = None, ) -> None: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.handle_spider_error() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + assert self.crawler.spider exc = _failure.value if isinstance(exc, CloseSpider): assert self.crawler.engine is not None # typing - self.crawler.engine.close_spider(spider, exc.reason or "cancelled") + self.crawler.engine.close_spider( + self.crawler.spider, exc.reason or "cancelled" + ) return - logkws = self.logformatter.spider_error(_failure, request, response, spider) + logkws = self.logformatter.spider_error( + _failure, request, response, self.crawler.spider + ) logger.log( *logformatter_adapter(logkws), exc_info=failure_to_exc_info(_failure), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) self.signals.send_catch_log( signal=signals.spider_error, failure=_failure, response=response, - spider=spider, + spider=self.crawler.spider, ) assert self.crawler.stats - self.crawler.stats.inc_value("spider_exceptions/count", spider=spider) self.crawler.stats.inc_value( - f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider + "spider_exceptions/count", spider=self.crawler.spider + ) + self.crawler.stats.inc_value( + f"spider_exceptions/{_failure.value.__class__.__name__}", + spider=self.crawler.spider, ) def handle_spider_output( @@ -256,41 +304,40 @@ class Scraper: result: Iterable[_T] | AsyncIterable[_T], request: Request, response: Response, - spider: Spider, + spider: Spider | None = None, ) -> _HandleOutputDeferred: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.handle_spider_output() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if not result: return defer_succeed(None) it: Iterable[_T] | AsyncIterable[_T] dfd: Deferred[_ParallelResult] if isinstance(result, AsyncIterable): - it = aiter_errback( - result, self.handle_spider_error, request, response, spider - ) + it = aiter_errback(result, self.handle_spider_error, request, response) dfd = parallel_async( it, self.concurrent_items, self._process_spidermw_output, - request, response, - spider, ) else: - it = iter_errback( - result, self.handle_spider_error, request, response, spider - ) + it = iter_errback(result, self.handle_spider_error, request, response) dfd = parallel( it, self.concurrent_items, self._process_spidermw_output, - request, response, - spider, ) # returning Deferred[_ParallelResult] instead of Deferred[Union[_ParallelResult, None]] return dfd # type: ignore[return-value] def _process_spidermw_output( - self, output: Any, request: Request, response: Response, spider: Spider + self, output: Any, response: Response ) -> Deferred[Any] | None: """Process each Request/Item (given in the output parameter) returned from the given spider @@ -314,7 +361,7 @@ class Scraper: assert self.crawler.spider is not None # typing self.slot.itemproc_size += 1 dfd = self.itemproc.process_item(item, self.crawler.spider) - dfd.addBoth(self._itemproc_finished, item, response, self.crawler.spider) + dfd.addBoth(self._itemproc_finished, item, response) return dfd def _log_download_errors( @@ -322,7 +369,6 @@ class Scraper: spider_failure: Failure, download_failure: Failure, request: Request, - spider: Spider, ) -> Failure | None: """Log and silence errors that come from the engine (typically download errors that got propagated thru here). @@ -332,24 +378,25 @@ class Scraper: ExecutionEngine._handle_downloader_output() as "result" """ if not download_failure.check(IgnoreRequest): + assert self.crawler.spider if download_failure.frames: logkws = self.logformatter.download_error( - download_failure, request, spider + download_failure, request, self.crawler.spider ) logger.log( *logformatter_adapter(logkws), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, exc_info=failure_to_exc_info(download_failure), ) else: errmsg = download_failure.getErrorMessage() if errmsg: logkws = self.logformatter.download_error( - download_failure, request, spider, errmsg + download_failure, request, self.crawler.spider, errmsg ) logger.log( *logformatter_adapter(logkws), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) if spider_failure is not download_failure: @@ -357,41 +404,54 @@ class Scraper: return None def _itemproc_finished( - self, output: Any, item: Any, response: Response | None, spider: Spider + self, output: Any, item: Any, response: Response | None ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing + assert self.crawler.spider self.slot.itemproc_size -= 1 if isinstance(output, Failure): ex = output.value if isinstance(ex, DropItem): - logkws = self.logformatter.dropped(item, ex, response, spider) + logkws = self.logformatter.dropped( + item, ex, response, self.crawler.spider + ) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) + logger.log( + *logformatter_adapter(logkws), + extra={"spider": self.crawler.spider}, + ) return self.signals.send_catch_log_deferred( signal=signals.item_dropped, item=item, response=response, - spider=spider, + spider=self.crawler.spider, exception=output.value, ) assert ex - logkws = self.logformatter.item_error(item, ex, response, spider) + logkws = self.logformatter.item_error( + item, ex, response, self.crawler.spider + ) logger.log( *logformatter_adapter(logkws), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, exc_info=failure_to_exc_info(output), ) return self.signals.send_catch_log_deferred( signal=signals.item_error, item=item, response=response, - spider=spider, + spider=self.crawler.spider, failure=output, ) - logkws = self.logformatter.scraped(output, response, spider) + logkws = self.logformatter.scraped(output, response, self.crawler.spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} + ) return self.signals.send_catch_log_deferred( - signal=signals.item_scraped, item=output, response=response, spider=spider + signal=signals.item_scraped, + item=output, + response=response, + spider=self.crawler.spider, ) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 85a3b5895..b8b0aec44 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -40,7 +40,7 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") ScrapeFunc = Callable[ - [Union[Response, Failure], Request, Spider], Union[Iterable[_T], AsyncIterable[_T]] + [Union[Response, Failure], Request], Union[Iterable[_T], AsyncIterable[_T]] ] @@ -86,8 +86,8 @@ class SpiderMiddlewareManager(MiddlewareManager): except _InvalidOutput: raise except Exception: - return scrape_func(Failure(), request, spider) - return scrape_func(response, request, spider) + return scrape_func(Failure(), request) + return scrape_func(response, request) def _evaluate_iterable( self, diff --git a/tests/test_engine.py b/tests/test_engine.py index ba4c6dc40..8928e4daf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -487,18 +487,17 @@ def test_request_scheduled_signal(caplog): if "drop" in request.url: raise IgnoreRequest - spider = MySpider() - crawler = get_crawler(spider.__class__) + crawler = get_crawler(MySpider) engine = ExecutionEngine(crawler, lambda _: None) engine.downloader._slot_gc_loop.stop() scheduler = TestScheduler() engine.slot = Slot((), None, Mock(), scheduler) crawler.signals.connect(signal_handler, request_scheduled) keep_request = Request("https://keep.example") - engine._schedule_request(keep_request, spider) + engine._schedule_request(keep_request) drop_request = Request("https://drop.example") caplog.set_level(DEBUG) - engine._schedule_request(drop_request, spider) + engine._schedule_request(drop_request) assert scheduler.enqueued == [keep_request], ( f"{scheduler.enqueued!r} != [{keep_request!r}]" ) From 23c206af35a8a7772d63cd63fdccee085dc38e40 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 1 May 2025 22:59:18 +0500 Subject: [PATCH 1704/2083] Improve test coverage of Scraper. --- scrapy/core/scraper.py | 32 +++++++----------- tests/test_crawl.py | 77 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 20 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 2942dfa58..6f69d668e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -374,32 +374,24 @@ class Scraper: errors that got propagated thru here). spider_failure: the value passed into the errback of self.call_spider() + (likely raised in the request errback) + download_failure: the value passed into _scrape2() from ExecutionEngine._handle_downloader_output() as "result" + (likely raised in the download handler or a downloader middleware) """ if not download_failure.check(IgnoreRequest): assert self.crawler.spider - if download_failure.frames: - logkws = self.logformatter.download_error( - download_failure, request, self.crawler.spider - ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": self.crawler.spider}, - exc_info=failure_to_exc_info(download_failure), - ) - else: - errmsg = download_failure.getErrorMessage() - if errmsg: - logkws = self.logformatter.download_error( - download_failure, request, self.crawler.spider, errmsg - ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": self.crawler.spider}, - ) - + logkws = self.logformatter.download_error( + download_failure, request, self.crawler.spider + ) + logger.log( + *logformatter_adapter(logkws), + extra={"spider": self.crawler.spider}, + exc_info=failure_to_exc_info(download_failure), + ) if spider_failure is not download_failure: + # a request errback raised a different exception, it needs to be handled later return spider_failure return None diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f49deac1f..a8174d537 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import logging import unittest @@ -723,3 +725,78 @@ class TestCrawlSpider(TestCase): assert crawler.spider.meta[ "failure" ].value.response.headers == crawler.spider.meta.get("headers_received") + + @defer.inlineCallbacks + def test_spider_errback(self): + failures = [] + + def eb(failure: Failure) -> Failure: + failures.append(failure) + return failure + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert len(failures) == 1 + assert "HTTP status code is not handled or not allowed" in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_spider_errback_silence(self): + failures = [] + + def eb(failure: Failure) -> None: + failures.append(failure) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert len(failures) == 1 + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_spider_errback_exception(self): + def eb(failure: Failure) -> None: + raise ValueError("foo") + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "Spider error processing" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_downloader_error(self): + failures = [] + + def eb(failure: Failure) -> Failure: + failures.append(failure) + return failure + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert len(failures) == 1 + assert "Error downloading" in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_spider_errback_exception_downloader_error(self): + def eb(failure: Failure) -> None: + raise ValueError("foo") + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "Error downloading" in str(log) + assert "Spider error processing" in str(log) From da9078c4bb942be8f55495d9a44ea522f3cdcbc4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 1 May 2025 23:12:39 +0500 Subject: [PATCH 1705/2083] Add tests for raising CloseSpider in callbacks. --- tests/test_crawl.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index a8174d537..b85f56909 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -16,7 +16,7 @@ from twisted.trial.unittest import TestCase from scrapy import signals from scrapy.crawler import CrawlerRunner -from scrapy.exceptions import StopDownload +from scrapy.exceptions import CloseSpider, StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import to_unicode @@ -800,3 +800,25 @@ class TestCrawlSpider(TestCase): ) assert "Error downloading" in str(log) assert "Spider error processing" in str(log) + + @defer.inlineCallbacks + def test_raise_closespider(self): + def cb(response): + raise CloseSpider + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) + assert "Closing spider (cancelled)" in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_raise_closespider_reason(self): + def cb(response): + raise CloseSpider("my_reason") + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) + assert "Closing spider (my_reason)" in str(log) + assert "Spider error processing" not in str(log) From 5dfe7cd7b87ffc8bb287934fa3d6ffbcd63da332 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 11:36:52 +0400 Subject: [PATCH 1706/2083] Improve tests for start items. (#6770) --- tests/test_crawl.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b85f56909..b7a8a9628 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -188,11 +188,18 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_requests_items(self): + items = [] + + def _on_item_scraped(item): + items.append(item) + with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(StartRequestsItemSpider) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) assert len(log.records) == 0 + assert items == [{"name": "test item"}] @defer.inlineCallbacks def test_start_requests_unsupported_output(self): @@ -201,11 +208,19 @@ class TestCrawl(TestCase): things fail when ItemAdapter is actually used on the corresponding non-item object.""" + items = [] + + def _on_item_scraped(item): + items.append(item) + with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(StartRequestsGoodAndBadOutput) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) assert len(log.records) == 0 + assert len(items) == 3 + assert not any(isinstance(item, Request) for item in items) @defer.inlineCallbacks def test_start_requests_laziness(self): From ff1ac75c9ef538b49212dbb1d4112b3653efab12 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 11:37:38 +0400 Subject: [PATCH 1707/2083] Fix shutdown tests. (#6772) --- tests/CrawlerProcess/sleeping.py | 4 +++- tests/test_crawler.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py index 45479ea4f..cb8f869e1 100644 --- a/tests/CrawlerProcess/sleeping.py +++ b/tests/CrawlerProcess/sleeping.py @@ -1,3 +1,5 @@ +import sys + from twisted.internet.defer import Deferred import scrapy @@ -14,7 +16,7 @@ class SleepingSpider(scrapy.Spider): from twisted.internet import reactor d = Deferred() - reactor.callLater(int(self.sleep), d.callback, None) + reactor.callLater(int(sys.argv[1]), d.callback, None) await maybe_deferred_to_future(d) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6c465f000..efb346dde 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -890,7 +890,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_shutdown_graceful(self): sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py", "-a", "sleep=3") + args = self.get_script_args("sleeping.py", "3") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") @@ -904,7 +904,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): from twisted.internet import reactor sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py", "-a", "sleep=10") + args = self.get_script_args("sleeping.py", "10") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") From 2a1edbd473e47b15183e57d975d71db0cb3a2197 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 11:44:17 +0400 Subject: [PATCH 1708/2083] Remove usages of TestCase._wait(). (#6773) --- tests/test_downloadermiddleware.py | 146 +++++++++++++---------------- tests/test_spidermiddleware.py | 53 ++++++----- 2 files changed, 90 insertions(+), 109 deletions(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 8e718ad5b..408160ccb 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,17 +1,18 @@ +from __future__ import annotations + import asyncio from gzip import BadGzipFile from unittest import mock import pytest -from twisted.internet import defer -from twisted.internet.defer import Deferred -from twisted.python.failure import Failure +from twisted.internet.defer import Deferred, succeed from twisted.trial.unittest import TestCase from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue @@ -29,38 +30,36 @@ class TestManagerBase(TestCase): def tearDown(self): return self.crawler.engine.close_spider(self.spider) - def _download(self, request, response=None): + async def _download( + self, request: Request, response: Response | None = None + ) -> Response | Request: """Executes downloader mw manager's download method and returns - the result (Request or Response) or raise exception in case of + the result (Request or Response) or raises exception in case of failure. """ if not response: response = Response(request.url) - def download_func(request, spider): - return response + def download_func(request: Request, spider: Spider) -> Deferred[Response]: + return succeed(response) - dfd = self.mwman.download(download_func, request, self.spider) - # catch deferred result and return the value - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - ret = results[0] - if isinstance(ret, Failure): - ret.raiseException() - return ret + return await maybe_deferred_to_future( + self.mwman.download(download_func, request, self.spider) + ) class TestDefaults(TestManagerBase): """Tests default behavior with default settings""" - def test_request_response(self): + @deferred_f_from_coro_f + async def test_request_response(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) - ret = self._download(req, resp) + ret = await self._download(req, resp) assert isinstance(ret, Response), "Non-response returned" - def test_3xx_and_invalid_gzipped_body_must_redirect(self): + @deferred_f_from_coro_f + async def test_3xx_and_invalid_gzipped_body_must_redirect(self): """Regression test for a failure when redirecting a compressed request. @@ -85,13 +84,14 @@ class TestDefaults(TestManagerBase): "Location": "http://example.com/login", }, ) - ret = self._download(request=req, response=resp) + ret = await self._download(req, resp) assert isinstance(ret, Request), f"Not redirected: {ret!r}" assert to_bytes(ret.url) == resp.headers["Location"], ( "Not redirected to location header" ) - def test_200_and_invalid_gzipped_body_must_fail(self): + @deferred_f_from_coro_f + async def test_200_and_invalid_gzipped_body_must_fail(self): req = Request("http://example.com") body = b"

You are being redirected

" resp = Response( @@ -106,13 +106,14 @@ class TestDefaults(TestManagerBase): }, ) with pytest.raises(BadGzipFile): - self._download(request=req, response=resp) + await self._download(req, resp) class TestResponseFromProcessRequest(TestManagerBase): """Tests middleware returning a response from process_request.""" - def test_download_func_not_called(self): + @deferred_f_from_coro_f + async def test_download_func_not_called(self): resp = Response("http://example.com/index.html") class ResponseMiddleware: @@ -123,19 +124,17 @@ class TestResponseFromProcessRequest(TestManagerBase): req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called -class TestProcessRequestInvalidOutput(TestManagerBase): - """Invalid return value for process_request method should raise an exception""" - - def test_invalid_process_request(self): +class TestInvalidOutput(TestManagerBase): + @deferred_f_from_coro_f + async def test_invalid_process_request(self): + """Invalid return value for process_request method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessRequestMiddleware: @@ -143,18 +142,12 @@ class TestProcessRequestInvalidOutput(TestManagerBase): return 1 self.mwman._add_middleware(InvalidProcessRequestMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - assert isinstance(results[0], Failure) - assert isinstance(results[0].value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._download(req) - -class TestProcessResponseInvalidOutput(TestManagerBase): - """Invalid return value for process_response method should raise an exception""" - - def test_invalid_process_response(self): + @deferred_f_from_coro_f + async def test_invalid_process_response(self): + """Invalid return value for process_response method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessResponseMiddleware: @@ -162,18 +155,12 @@ class TestProcessResponseInvalidOutput(TestManagerBase): return 1 self.mwman._add_middleware(InvalidProcessResponseMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - assert isinstance(results[0], Failure) - assert isinstance(results[0].value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._download(req) - -class TestProcessExceptionInvalidOutput(TestManagerBase): - """Invalid return value for process_exception method should raise an exception""" - - def test_invalid_process_exception(self): + @deferred_f_from_coro_f + async def test_invalid_process_exception(self): + """Invalid return value for process_exception method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessExceptionMiddleware: @@ -184,18 +171,15 @@ class TestProcessExceptionInvalidOutput(TestManagerBase): return 1 self.mwman._add_middleware(InvalidProcessExceptionMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - assert isinstance(results[0], Failure) - assert isinstance(results[0].value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._download(req) class TestMiddlewareUsingDeferreds(TestManagerBase): """Middlewares using Deferreds should work""" - def test_deferred(self): + @deferred_f_from_coro_f + async def test_deferred(self): resp = Response("http://example.com/index.html") class DeferredMiddleware: @@ -211,12 +195,10 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): self.mwman._add_middleware(DeferredMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called @@ -224,27 +206,27 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): class TestMiddlewareUsingCoro(TestManagerBase): """Middlewares using asyncio coroutines should work""" - def test_asyncdef(self): + @deferred_f_from_coro_f + async def test_asyncdef(self): resp = Response("http://example.com/index.html") class CoroMiddleware: async def process_request(self, request, spider): - await defer.succeed(42) + await succeed(42) return resp self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called @pytest.mark.only_asyncio - def test_asyncdef_asyncio(self): + @deferred_f_from_coro_f + async def test_asyncdef_asyncio(self): resp = Response("http://example.com/index.html") class CoroMiddleware: @@ -255,10 +237,8 @@ class TestMiddlewareUsingCoro(TestManagerBase): self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index ddc9b5206..1d671134e 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,12 +1,12 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterable +from typing import Any from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy.core.spidermw import SpiderMiddlewareManager @@ -14,7 +14,11 @@ from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen -from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_from_coro, + maybe_deferred_to_future, +) from scrapy.utils.test import get_crawler @@ -26,53 +30,51 @@ class TestSpiderMiddleware(TestCase): self.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - def _scrape_response(self): + async def _scrape_response(self) -> Any: """Execute spider mw manager's scrape_response method and return the result. Raise exception in case of failure. """ scrape_func = mock.MagicMock() - dfd = self.mwman.scrape_response( - scrape_func, self.response, self.request, self.spider + return await maybe_deferred_to_future( + self.mwman.scrape_response( + scrape_func, self.response, self.request, self.spider + ) ) - # catch deferred result and return the value - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - return results[0] class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_input method""" - def test_invalid_process_spider_input(self): + @deferred_f_from_coro_f + async def test_invalid_process_spider_input(self): class InvalidProcessSpiderInputMiddleware: def process_spider_input(self, response, spider): return 1 self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_output method""" - def test_invalid_process_spider_output(self): + @deferred_f_from_coro_f + async def test_invalid_process_spider_output(self): class InvalidProcessSpiderOutputMiddleware: def process_spider_output(self, response, result, spider): return 1 self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_exception method""" - def test_invalid_process_spider_exception(self): + @deferred_f_from_coro_f + async def test_invalid_process_spider_exception(self): class InvalidProcessSpiderOutputExceptionMiddleware: def process_spider_exception(self, response, exception, spider): return 1 @@ -83,15 +85,15 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): """Re raise the exception by returning None""" - def test_process_spider_exception_return_none(self): + @deferred_f_from_coro_f + async def test_process_spider_exception_return_none(self): class ProcessSpiderExceptionReturnNoneMiddleware: def process_spider_exception(self, response, exception, spider): return None @@ -102,9 +104,8 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, ZeroDivisionError) + with pytest.raises(ZeroDivisionError): + await self._scrape_response() class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): From 509b572efc85ac8ef96224d560a8851322a3f606 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 11:51:53 +0400 Subject: [PATCH 1709/2083] Migrate the build system to hatchling. (#6771) --- MANIFEST.in | 22 ---------------------- pyproject.toml | 45 ++++++++++++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 37 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 7700ae7bd..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,22 +0,0 @@ -include CODE_OF_CONDUCT.md -include CONTRIBUTING.md -include INSTALL.md -include NEWS -include SECURITY.md - -include scrapy/VERSION -include scrapy/mime.types -include scrapy/py.typed - -include codecov.yml -include conftest.py -include tox.ini - -recursive-include scrapy/templates * -recursive-include docs * -prune docs/build - -recursive-include extras * -recursive-include tests * - -global-exclude __pycache__ *.py[cod] diff --git a/pyproject.toml b/pyproject.toml index 84bf41a94..e14efdd17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -requires = ["setuptools >= 61.0"] -build-backend = "setuptools.build_meta" +requires = ["hatchling>=1.27.0"] +build-backend = "hatchling.build" [project] name = "Scrapy" @@ -10,29 +10,28 @@ dependencies = [ "Twisted>=21.7.0", "cryptography>=37.0.0", "cssselect>=0.9.1", + "defusedxml>=0.7.1", + "itemadapter>=0.1.0", "itemloaders>=1.0.1", + "lxml>=4.6.0", + "packaging", "parsel>=1.5.0", + "protego>=0.1.15", "pyOpenSSL>=22.0.0", "queuelib>=1.4.2", "service_identity>=18.1.0", + "tldextract", "w3lib>=1.17.0", "zope.interface>=5.1.0", - "protego>=0.1.15", - "itemadapter>=0.1.0", - "packaging", - "tldextract", - "lxml>=4.6.0", - "defusedxml>=0.7.1", # Platform-specific dependencies 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', ] classifiers = [ - "Framework :: Scrapy", "Development Status :: 5 - Production/Stable", "Environment :: Console", + "Framework :: Scrapy", "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", @@ -47,6 +46,8 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ] +license = "BSD-3-Clause" +license-files = ["LICENSE", "AUTHORS"] readme = "README.rst" requires-python = ">=3.9" authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }] @@ -63,12 +64,26 @@ releasenotes = "https://docs.scrapy.org/en/latest/news.html" [project.scripts] scrapy = "scrapy.cmdline:execute" -[tool.setuptools.packages.find] -where = ["."] -include = ["scrapy", "scrapy.*",] +[tool.hatch.build.targets.sdist] +include = [ + "/docs", + "/extras", + "/scrapy", + "/tests", + "/tests_typing", + "/CODE_OF_CONDUCT.md", + "/CONTRIBUTING.md", + "/INSTALL.md", + "/NEWS", + "/SECURITY.md", + "/codecov.yml", + "/conftest.py", + "/tox.ini", +] -[tool.setuptools.dynamic] -version = {file = "./scrapy/VERSION"} +[tool.hatch.version] +path = "scrapy/VERSION" +pattern = "^(?P.+)$" [tool.mypy] ignore_missing_imports = true From b93290f28affdc0bdd780672ec1adb9d7def4940 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 19:38:04 +0500 Subject: [PATCH 1710/2083] Add a list of Deferred-only APIs. --- docs/topics/coroutines.rst | 95 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 1c80857f6..439474310 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -62,18 +62,103 @@ In addition to native coroutine APIs Scrapy has some APIs that return a :class:`~twisted.internet.defer.Deferred` object or take a user-supplied function that returns a :class:`~twisted.internet.defer.Deferred` object. These APIs are also asynchronous but don't yet support native ``async def`` syntax. -For example: +In the future we plan to add support for the ``async def`` syntax to these APIs +or replace them with other APIs where changing the existing ones is +possible. -- The :meth:`ExecutionEngine.download` method returns a - :class:`~twisted.internet.defer.Deferred` object. -- A custom download handler needs to define a ``download_request()`` method that - returns a :class:`~twisted.internet.defer.Deferred` object. +The following Scrapy methods return :class:`~twisted.internet.defer.Deferred` +objects (this list is not complete as it only includes methods that we think +may be useful for user code): + +- :class:`scrapy.crawler.Crawler`: + + - :meth:`~scrapy.crawler.Crawler.crawl` + + - :meth:`~scrapy.crawler.Crawler.stop` + +- :class:`scrapy.crawler.CrawlerRunner` (also inherited by + :class:`scrapy.crawler.CrawlerProcess`): + + - :meth:`~scrapy.crawler.CrawlerRunner.crawl` + + - :meth:`~scrapy.crawler.CrawlerRunner.stop` + + - :meth:`~scrapy.crawler.CrawlerRunner.join` + +- :class:`scrapy.core.engine.ExecutionEngine`: + + - :meth:`~scrapy.core.engine.ExecutionEngine.download` + +- :class:`scrapy.signalmanager.SignalManager`: + + - :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_deferred` + +- :class:`~scrapy.mail.MailSender` + + - :meth:`~scrapy.mail.MailSender.send` + +The following user-supplied methods can return +:class:`~twisted.internet.defer.Deferred` objects (the methods that can also +return coroutines are listed in :ref:`coroutine-support`): + +- Custom download handlers (see :setting:`DOWNLOAD_HANDLERS`): + + - ``download_request()`` + + - ``close()`` + +- Custom downloader implementations (see :setting:`DOWNLOADER`): + + - ``fetch()`` + +- Custom scheduler implementations (see :setting:`SCHEDULER`): + + - :meth:`~scrapy.core.scheduler.BaseScheduler.open` + + - :meth:`~scrapy.core.scheduler.BaseScheduler.close` + +- Custom dupefilters (see :setting:`DUPEFILTER_CLASS`): + + - ``open()`` + + - ``close()`` + +- Custom feed storages (see :setting:`FEED_STORAGES`): + + - ``store()`` + +- Subclasses of :class:`scrapy.pipelines.media.MediaPipeline`: + + - ``media_to_download()`` + + - ``item_completed()`` + +- Custom storages used by subclasses of + :class:`scrapy.pipelines.files.FilesPipeline`: + + - ``persist_file()`` + + - ``stat_file()`` In most cases you can use these APIs in code that otherwise uses coroutines, by wrapping a :class:`~twisted.internet.defer.Deferred` object into a :class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for more information about this. +For example: + +- The :meth:`ExecutionEngine.download() + ` method returns a + :class:`~twisted.internet.defer.Deferred` object that fires with the + downloaded response. You can use this object directly in Deferred-based + code or convert it into a :class:`~asyncio.Future` object with + :func:`~scrapy.utils.defer.maybe_deferred_to_future`. +- A custom download handler needs to define a ``download_request()`` method + that returns a :class:`~twisted.internet.defer.Deferred` object. You can + write a method that works with Deferreds and returns one directly, or you + can write a coroutine and convert it into a functions that returns a + Deferred with :func:`~scrapy.utils.defer.deferred_f_from_coro_f`. + General usage ============= From 523fc25c4d7550d721e8160f4f09cb61f94d53d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 5 May 2025 18:51:15 +0200 Subject: [PATCH 1711/2083] Document default values set by startproject (#6775) --- docs/topics/feed-exports.rst | 10 +++++----- docs/topics/settings.rst | 23 +++++++++++++++-------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7f401f0c7..2184f2d0e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -539,18 +539,18 @@ as a fallback value if that key is not provided for a specific feed definition: FEED_EXPORT_ENCODING -------------------- -Default: ``None`` +Default: ``"utf-8"`` (:ref:`fallback `: ``None``) The encoding to be used for the feed. -If unset or set to ``None`` (default) it uses UTF-8 for everything except JSON output, -which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons. +If set to ``None``, it uses UTF-8 for everything except JSON output, which uses +safe numeric encoding (``\uXXXX`` sequences) for historic reasons. -Use ``utf-8`` if you want UTF-8 for JSON too. +Use ``"utf-8"`` if you want UTF-8 for JSON too. .. versionchanged:: 2.8 The :command:`startproject` command now sets this setting to - ``utf-8`` in the generated ``settings.py`` file. + ``"utf-8"`` in the generated ``settings.py`` file. .. setting:: FEED_EXPORT_FIELDS diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a59a61050..3a61306d6 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -162,8 +162,17 @@ Those command-specific default settings are specified in the 6. Default global settings -------------------------- -The global defaults are located in the ``scrapy.settings.default_settings`` -module and documented in the :ref:`topics-settings-ref` section. +The ``scrapy.settings.default_settings`` module defines global default values +for some :ref:`built-in settings `. + +.. note:: :command:`startproject` generates a ``settings.py`` file that sets + some settings to different values. + + The reference documentation of settings indicates the default value if one + exists. If :command:`startproject` sets a value, that value is documented + as default, and the value from ``scrapy.settings.default_settings`` is + documented as “fallbackâ€. + Compatibility with pickle ========================= @@ -461,7 +470,7 @@ Note that the event loop class must inherit from :class:`asyncio.AbstractEventLo BOT_NAME -------- -Default: ``'scrapybot'`` +Default: ```` (:ref:`fallback `: ``'scrapybot'``) The name of the bot implemented by this Scrapy project (also known as the project name). This name will be used for the logging too. @@ -1563,7 +1572,7 @@ email notifying about it. If zero, no warning will be produced. NEWSPIDER_MODULE ---------------- -Default: ``''`` +Default: ``".spiders"`` (:ref:`fallback `: ``""``) Module where to create new spiders using the :command:`genspider` command. @@ -1622,9 +1631,7 @@ Adjust redirect request priority relative to original request: ROBOTSTXT_OBEY -------------- -Default: ``False`` - -Scope: ``scrapy.downloadermiddlewares.robotstxt`` +Default: ``True`` (:ref:`fallback `: ``False``) If enabled, Scrapy will respect robots.txt policies. For more information see :ref:`topics-dlmw-robots`. @@ -1838,7 +1845,7 @@ the spider. For more info see :ref:`topics-spider-middleware-setting`. SPIDER_MODULES -------------- -Default: ``[]`` +Default: ``[".spiders"]`` (:ref:`fallback `: ``[]``) A list of modules where Scrapy will look for spiders. From acb5f895cd0b0f63f3dafdd3025b314830ab4a67 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 22:28:36 +0500 Subject: [PATCH 1712/2083] Update docs/topics/coroutines.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/coroutines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 439474310..8af4ce71d 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -156,7 +156,7 @@ For example: - A custom download handler needs to define a ``download_request()`` method that returns a :class:`~twisted.internet.defer.Deferred` object. You can write a method that works with Deferreds and returns one directly, or you - can write a coroutine and convert it into a functions that returns a + can write a coroutine and convert it into a function that returns a Deferred with :func:`~scrapy.utils.defer.deferred_f_from_coro_f`. From 4899d416e701c4d405fdc77fa2ba31327541f292 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 6 May 2025 14:31:28 +0400 Subject: [PATCH 1713/2083] Add PyPy 3.11 to CI. (#6697) --- .github/workflows/tests-ubuntu.yml | 5 ++++- tox.ini | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 34819f227..06da46ca1 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -38,6 +38,9 @@ jobs: - python-version: pypy3.10 env: TOXENV: pypy3 + - python-version: pypy3.11 + env: + TOXENV: pypy3 # pinned deps - python-version: "3.9.21" @@ -59,7 +62,7 @@ jobs: - python-version: "3.13" env: TOXENV: extra-deps - - python-version: pypy3.10 + - python-version: pypy3.11 env: TOXENV: pypy3-extra-deps - python-version: "3.13" diff --git a/tox.ini b/tox.ini index 59572442d..e63e44189 100644 --- a/tox.ini +++ b/tox.ini @@ -143,7 +143,7 @@ deps = google-cloud-storage ipython robotexclusionrulesparser - uvloop; platform_system != "Windows" + uvloop; platform_system != "Windows" and implementation_name != "pypy" zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests [testenv:extra-deps-pinned] @@ -159,7 +159,7 @@ deps = google-cloud-storage==1.29.0 ipython==2.0.0 robotexclusionrulesparser==1.6.2 - uvloop==0.14.0; platform_system != "Windows" + uvloop==0.14.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.1; implementation_name != "pypy" install_command = {[pinned]install_command} setenv = From 373e501f78703e9fae2b9e970071052a13b1a18e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 7 May 2025 16:11:22 +0200 Subject: [PATCH 1714/2083] Link to scrapy.org from the docs (#6780) --- docs/_templates/layout.html | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docs/_templates/layout.html diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 000000000..6ec565e24 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,23 @@ +{% extends "!layout.html" %} + +{# Overriden to include a link to scrapy.org, not just to the docs root #} +{%- block sidebartitle %} + +{# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #} +{# the master_doc variable was renamed to root_doc in Sphinx 4 (master_doc still exists in later Sphinx versions) #} +{%- set _logo_url = logo_url|default(pathto('_static/' + (logo or ""), 1)) %} +{%- set _root_doc = root_doc|default(master_doc) %} +scrapy.org / docs + +{%- if READTHEDOCS or DEBUG %} + {%- if theme_version_selector or theme_language_selector %} +
+
+
+
+ {%- endif %} +{%- endif %} + +{%- include "searchbox.html" %} + +{%- endblock %} From 036f3e562716aaf67a4d0ff1c8011281394ef240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 7 May 2025 19:04:03 +0200 Subject: [PATCH 1715/2083] Support asynchronous start requests (#6729) --- docs/faq.rst | 24 +- docs/intro/tutorial.rst | 39 +- docs/news.rst | 133 ++++++- docs/topics/api.rst | 6 + docs/topics/architecture.rst | 2 +- docs/topics/components.rst | 4 + docs/topics/coroutines.rst | 28 +- docs/topics/jobs.rst | 6 +- docs/topics/request-response.rst | 10 +- docs/topics/scheduler.rst | 8 +- docs/topics/settings.rst | 62 ++- docs/topics/signals.rst | 19 +- docs/topics/spider-middleware.rst | 46 ++- docs/topics/spiders.rst | 119 +++--- docs/topics/telnetconsole.rst | 12 +- extras/qpsclient.py | 4 + pyproject.toml | 4 +- scrapy/commands/bench.py | 8 +- scrapy/commands/check.py | 10 +- scrapy/commands/fetch.py | 8 +- scrapy/commands/parse.py | 6 +- scrapy/commands/shell.py | 4 +- scrapy/core/engine.py | 248 +++++++----- scrapy/core/scheduler.py | 287 ++++++++++---- scrapy/core/scraper.py | 333 ++++++++-------- scrapy/core/spidermw.py | 203 ++++++++-- scrapy/crawler.py | 8 +- scrapy/extensions/telnet.py | 1 - scrapy/http/request/__init__.py | 12 +- scrapy/logformatter.py | 2 +- scrapy/pqueues.py | 126 ++++-- scrapy/settings/default_settings.py | 3 + scrapy/shell.py | 23 +- scrapy/signalmanager.py | 21 +- scrapy/signals.py | 1 + scrapy/spidermiddlewares/base.py | 63 +-- scrapy/spidermiddlewares/depth.py | 11 +- scrapy/spidermiddlewares/offsite.py | 5 +- scrapy/spidermiddlewares/referer.py | 5 +- scrapy/spidermiddlewares/start.py | 31 ++ scrapy/spidermiddlewares/urllength.py | 2 +- scrapy/spiders/__init__.py | 79 +++- scrapy/spiders/crawl.py | 6 +- scrapy/spiders/init.py | 10 +- scrapy/spiders/sitemap.py | 6 +- .../project/module/middlewares.py.tmpl | 13 +- scrapy/utils/asyncgen.py | 8 +- scrapy/utils/defer.py | 34 +- scrapy/utils/engine.py | 8 +- scrapy/utils/python.py | 17 +- scrapy/utils/reactor.py | 25 +- sep/sep-018.rst | 2 +- tests/CrawlerProcess/args_settings.py | 5 +- tests/CrawlerProcess/asyncio_custom_loop.py | 5 +- .../asyncio_enabled_no_reactor.py | 5 +- .../CrawlerProcess/asyncio_enabled_reactor.py | 5 +- .../asyncio_enabled_reactor_different_loop.py | 5 +- .../asyncio_enabled_reactor_same_loop.py | 5 +- .../caching_hostname_resolver.py | 2 +- tests/CrawlerProcess/multi.py | 5 +- tests/CrawlerProcess/reactor_default.py | 5 +- .../reactor_default_twisted_reactor_select.py | 5 +- tests/CrawlerProcess/reactor_select.py | 5 +- ..._select_subclass_twisted_reactor_select.py | 5 +- .../reactor_select_twisted_reactor_select.py | 5 +- tests/CrawlerProcess/simple.py | 5 +- tests/CrawlerRunner/change_reactor.py | 5 +- tests/CrawlerRunner/ip_address.py | 2 +- tests/__init__.py | 6 + tests/spiders.py | 36 +- .../__init__.py | 17 +- tests/test_commands.py | 63 +-- tests/test_contracts.py | 5 +- tests/test_crawl.py | 70 +--- tests/test_crawler.py | 13 +- tests/test_downloadermiddleware.py | 2 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_engine.py | 39 +- tests/test_engine_loop.py | 364 ++++++++++++++++++ tests/test_pipelines.py | 2 +- tests/test_request_cb_kwargs.py | 14 +- tests/test_scheduler.py | 35 +- tests/test_signals.py | 21 +- tests/test_spider.py | 57 ++- tests/test_spider_start.py | 186 +++++++++ tests/test_spidermiddleware.py | 53 +-- tests/test_spidermiddleware_base.py | 74 ++-- tests/test_spidermiddleware_httperror.py | 2 +- tests/test_spidermiddleware_output_chain.py | 16 +- tests/test_spidermiddleware_process_start.py | 352 +++++++++++++++++ tests/test_spidermiddleware_start.py | 44 +++ tests/utils/__init__.py | 9 + tox.ini | 2 +- 93 files changed, 2777 insertions(+), 936 deletions(-) create mode 100644 scrapy/spidermiddlewares/start.py create mode 100644 tests/test_engine_loop.py create mode 100644 tests/test_spider_start.py create mode 100644 tests/test_spidermiddleware_process_start.py create mode 100644 tests/test_spidermiddleware_start.py diff --git a/docs/faq.rst b/docs/faq.rst index da255f29e..1d09a0e63 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -96,30 +96,13 @@ How can I simulate a user login in my spider? See :ref:`topics-request-response-ref-request-userlogin`. + .. _faq-bfo-dfo: Does Scrapy crawl in breadth-first or depth-first order? -------------------------------------------------------- -By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which -basically means that it crawls in `DFO order`_. This order is more convenient -in most cases. - -If you do want to crawl in true `BFO order`_, you can do it by -setting the following settings: - -.. code-block:: python - - DEPTH_PRIORITY = 1 - SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" - SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" - -While pending requests are below the configured values of -:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent -concurrently. As a result, the first few requests of a crawl rarely follow the -desired order. Lowering those settings to ``1`` enforces the desired order, but -it significantly slows down the crawl as a whole. +:ref:`DFO by default, but other orders are possible `. My Scrapy crawler has memory leaks. What can I do? @@ -436,6 +419,3 @@ See :issue:`2680`. .. _Python standard library modules: https://docs.python.org/3/py-modindex.html .. _Python package: https://pypi.org/ .. _user agents: https://en.wikipedia.org/wiki/User_agent -.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) -.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 5041b49ea..c4e04364b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -94,7 +94,7 @@ This is the code for our first Spider. Save it in a file named class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def start(self): urls = [ "https://quotes.toscrape.com/page/1/", "https://quotes.toscrape.com/page/2/", @@ -116,10 +116,10 @@ and defines some attributes and methods: unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.Spider.start_requests`: must return an iterable of - Requests (you can return a list of requests or write a generator function) - which the Spider will begin to crawl from. Subsequent requests will be - generated successively from these initial requests. +* :meth:`~scrapy.Spider.start`: must be an asynchronous generator that + yields requests (and, optionally, items) for the spider to start crawling. + Subsequent requests will be generated successively from these initial + requests. * :meth:`~scrapy.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter @@ -164,21 +164,22 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy schedules the :class:`scrapy.Request ` objects -returned by the ``start_requests`` method of the Spider. Upon receiving a -response for each one, it instantiates :class:`~scrapy.http.Response` objects -and calls the callback method associated with the request (in this case, the -``parse`` method) passing the response as an argument. +Scrapy sends the first :class:`scrapy.Request ` objects yielded +by the :meth:`~scrapy.Spider.start` spider method. Upon receiving a +response for each one, Scrapy calls the callback method associated with the +request (in this case, the ``parse`` method) with a +:class:`~scrapy.http.Response` object. -A shortcut to the start_requests method ---------------------------------------- -Instead of implementing a :meth:`~scrapy.Spider.start_requests` method -that generates :class:`scrapy.Request ` objects from URLs, -you can just define a :attr:`~scrapy.Spider.start_urls` class attribute -with a list of URLs. This list will then be used by the default implementation -of :meth:`~scrapy.Spider.start_requests` to create the initial requests -for your spider. +A shortcut to the ``start`` method +---------------------------------- + +Instead of implementing a :meth:`~scrapy.Spider.start` method that yields +:class:`~scrapy.Request` objects from URLs, you can define a +:attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This +list will then be used by the default implementation of +:meth:`~scrapy.Spider.start` to create the initial requests for your +spider. .. code-block:: python @@ -794,7 +795,7 @@ with a specific tag, building the URL based on the argument: class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def start(self): url = "https://quotes.toscrape.com/" tag = getattr(self, "tag", None) if tag is not None: diff --git a/docs/news.rst b/docs/news.rst index 9f476ee21..64a3ad2b1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,6 +8,11 @@ Release notes Scrapy VERSION (unreleased) --------------------------- +Highlights: + +- Replaced ``start_requests()`` (sync) with :meth:`~scrapy.Spider.start` + (async) and changed how it is iterated. + Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -20,6 +25,116 @@ Backward-incompatible changes :class:`~scrapy.crawler.Crawler` instance at run time). Please use ``from_crawler()`` instead. +- The iteration of start requests and items no longer stops once there are + requests in the scheduler, and instead runs continuously until all start + requests have been scheduled. + + To reproduce the previous behavior, see :ref:`start-requests-lazy`. + +- An unhandled exception from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.open_spider` method of a + :ref:`spider middleware ` no longer stops the + crawl. + +- In ``scrapy.core.engine.ExecutionEngine``: + + - The second parameter of ``open_spider()``, ``start_requests``, has been + removed. The start requests are determined by the ``spider`` parameter + instead (see :meth:`~scrapy.Spider.start`). + + - The ``slot`` attribute has been renamed to ``_slot`` and should not be + used. + +- In ``scrapy.core.engine``, the ``Slot`` class has been renamed to ``_Slot`` + and should not be used. + +- The ``slot`` :ref:`telnet variable ` has been removed. + +- In ``scrapy.core.spidermw.SpiderMiddlewareManager``, + ``process_start_requests()`` has been replaced by ``process_start()``. + +- The now-deprecated ``start_requests()`` method, when it returns an iterable + instead of being defined as a generator, is now executed *after* the + :ref:`scheduler ` instance has been created. + +- When using :setting:`JOBDIR`, :ref:`start requests ` are + now serialized into their own, ``s``-suffixed priority folders. You can set + :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` or ``""`` to change that, + but the side effects may be undesirable. See + :setting:`SCHEDULER_START_DISK_QUEUE` for details. + +Deprecations +~~~~~~~~~~~~ + +- The ``start_requests()`` method of :class:`~scrapy.Spider` is deprecated, + use :meth:`~scrapy.Spider.start` instead, or both to maintain support for + lower Scrapy versions. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The ``process_start_requests()`` method of :ref:`spider middlewares + ` is deprecated, use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The ``__init__`` method of priority queue classes (see + :setting:`SCHEDULER_PRIORITY_QUEUE`) should now support a keyword-only + ``start_queue_cls`` parameter. + + (:issue:`6752`) + +New features +~~~~~~~~~~~~ + +- You can now yield the start requests and items of a spider from the + :meth:`~scrapy.Spider.start` spider method and from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` spider + middleware method, both :term:`asynchronous generators `. + + This makes it possible to use asynchronous code to generate those start + requests and items, e.g. reading them from a queue service or database + using an asynchronous client, without workarounds. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- Start requests are now :ref:`scheduled ` as soon as + possible. + + As a result, their :attr:`~scrapy.Request.priority` is now taken into + account as soon as :setting:`CONCURRENT_REQUESTS` is reached. + + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- :class:`Crawler.signals ` has a new + :meth:`~scrapy.signalmanager.SignalManager.wait_for` method. + +- Added a new :signal:`scheduler_empty` signal. + +- Added new settings: :setting:`SCHEDULER_START_DISK_QUEUE` and + :setting:`SCHEDULER_START_MEMORY_QUEUE`. + +- Added :class:`~scrapy.spidermiddlewares.start.StartSpiderMiddleware`, which + sets :reqmeta:`is_start_request` to ``True`` on :ref:`start requests + `. + +- Exposed a new method of :class:`Crawler.engine + `: + :meth:`~scrapy.core.engine.ExecutionEngine.needs_backout`. + +Bug fixes +~~~~~~~~~ + +- Yielding an item from :meth:`Spider.start ` or from + :meth:`SpiderMiddleware.process_start + ` no longer delays + the next iteration of starting requests and items by up to 5 seconds. + + (:issue:`6729`) + + .. _release-2.12.0: Scrapy 2.12.0 (2024-11-18) @@ -29,7 +144,7 @@ Highlights: - Dropped support for Python 3.8, added support for Python 3.13 -- :meth:`~scrapy.Spider.start_requests` can now yield items +- ``scrapy.Spider.start_requests()`` can now yield items - Added :class:`~scrapy.http.JsonResponse` @@ -320,9 +435,13 @@ Deprecations New features ~~~~~~~~~~~~ -- :meth:`~scrapy.Spider.start_requests` can now yield items. +- ``scrapy.Spider.start_requests()`` can now yield items. (:issue:`5289`, :issue:`6417`) + .. note:: Some spider middlewares may need to be updated for Scrapy 2.12 + support before you can use them in combination with the ability to + yield items from ``start_requests()``. + - Added a new :class:`~scrapy.http.Response` subclass, :class:`~scrapy.http.JsonResponse`, for responses with a `JSON MIME type `_. @@ -812,7 +931,7 @@ Backward-incompatible changes in :meth:`scrapy.Spider.from_crawler`. If you want to access the final setting values and the initialized :class:`~scrapy.crawler.Crawler` attributes in the spider code as early as possible you can do this in - :meth:`~scrapy.Spider.start_requests` or in a handler of the + ``scrapy.Spider.start_requests()`` or in a handler of the :signal:`engine_started` signal. (:issue:`6038`) - The :meth:`TextResponse.json ` method now @@ -3388,7 +3507,7 @@ New features * :class:`~scrapy.spiders.Spider` objects now raise an :exc:`AttributeError` exception if they do not have a :class:`~scrapy.spiders.Spider.start_urls` - attribute nor reimplement :class:`~scrapy.spiders.Spider.start_requests`, + attribute nor reimplement ``scrapy.spiders.Spider.start_requests()``, but have a ``start_url`` attribute (:issue:`4133`, :issue:`4170`) * :class:`~scrapy.exporters.BaseItemExporter` subclasses may now use @@ -6309,7 +6428,7 @@ Scrapy 0.18.4 (released 2013-10-10) - IPython refuses to update the namespace. fix #396 (:commit:`3d32c4f`) - Fix AlreadyCalledError replacing a request in shell command. closes #407 (:commit:`b1d8919`) -- Fix start_requests laziness and early hangs (:commit:`89faf52`) +- Fix ``start_requests()`` laziness and early hangs (:commit:`89faf52`) Scrapy 0.18.3 (released 2013-10-03) ----------------------------------- @@ -6502,7 +6621,7 @@ Scrapy changes: - added options ``-o`` and ``-t`` to the :command:`runspider` command - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. -- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares +- added a ``process_start_requests()`` method to spider middlewares - dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` @@ -6565,7 +6684,7 @@ Scrapy 0.14.2 - fixed bug in MemoryUsage extension: get_engine_status() takes exactly 1 argument (0 given) (:commit:`11133e9`) - fixed struct.error on http compression middleware. closes #87 (:commit:`1423140`) - ajax crawling wasn't expanding for unicode urls (:commit:`0de3fb4`) -- Catch start_requests iterator errors. refs #83 (:commit:`454a21d`) +- Catch ``start_requests()`` iterator errors. refs #83 (:commit:`454a21d`) - Speed-up libxml2 XPathSelector (:commit:`2fbd662`) - updated versioning doc according to recent changes (:commit:`0a070f5`) - scrapyd: fixed documentation link (:commit:`2b4e4c3`) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 5a00fd570..8e8f3a0c9 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -280,3 +280,9 @@ class (which they all inherit from). Close the given spider. After this is called, no more specific stats can be accessed or collected. + +Engine API +========== + +.. autoclass:: scrapy.core.engine.ExecutionEngine() + :members: needs_backout diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 4e53b6e3d..e8c510ea5 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -150,7 +150,7 @@ requests). Use a Spider middleware if you need to * post-process output of spider callbacks - change/add/remove requests or items; -* post-process start_requests; +* post-process start requests or items; * handle spider exceptions; * call errback instead of callback for some of the requests based on response content. diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 3a7644379..56f8c6498 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -37,6 +37,10 @@ That includes the classes that you may assign to the following settings: - :setting:`SCHEDULER_PRIORITY_QUEUE` +- :setting:`SCHEDULER_START_DISK_QUEUE` + +- :setting:`SCHEDULER_START_MEMORY_QUEUE` + - :setting:`SPIDER_MIDDLEWARES` Third-party Scrapy components may also let you define additional Scrapy diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 8af4ce71d..448bf07e7 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -6,8 +6,8 @@ Coroutines .. versionadded:: 2.0 -Scrapy has :ref:`partial support ` for the -:ref:`coroutine syntax `. +Scrapy :ref:`supports ` the :ref:`coroutine syntax ` +(i.e. ``async def``). .. _coroutine-support: @@ -18,6 +18,11 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): +- The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be + defined as an :term:`asynchronous generator`. + + .. versionadded: VERSION + - :class:`~scrapy.Request` callbacks. If you are using any custom or third-party :ref:`spider middleware @@ -38,20 +43,26 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): methods of :ref:`downloader middlewares `. -- :ref:`Signal handlers that support deferreds `. - - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method of :ref:`spider middlewares `. - It must be defined as an :term:`asynchronous generator`. The input - ``result`` parameter is an :term:`asynchronous iterable`. + If defined as a coroutine, it must be an :term:`asynchronous generator`. + The input ``result`` parameter is an :term:`asynchronous iterable`. See also :ref:`sync-async-spider-middleware` and :ref:`universal-spider-middleware`. .. versionadded:: 2.7 +- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method + of :ref:`spider middlewares `, which *must* be + defined as an :term:`asynchronous generator`. + + .. versionadded:: VERSION + +- :ref:`Signal handlers that support deferreds `. + .. _coroutine-deferred-apis: @@ -232,8 +243,9 @@ This means you can use many useful Python libraries providing such code: Common use cases for asynchronous code include: -* requesting data from websites, databases and other services (in callbacks, - pipelines and middlewares); +* requesting data from websites, databases and other services (in + :meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and + middlewares); * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the :signal:`spider_opened` handler); diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 0e705dc64..50bcaa6d6 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -46,9 +46,9 @@ Keeping persistent state between batches Sometimes you'll want to keep some persistent spider state between pause/resume batches. You can use the ``spider.state`` attribute for that, which should be a -dict. There's :ref:`a built-in extension ` that takes care of serializing, storing and -loading that attribute from the job directory, when the spider starts and -stops. +dict. There's :ref:`a built-in extension ` +that takes care of serializing, storing and loading that attribute from the job +directory, when the spider starts and stops. Here's an example of a callback that uses the spider state (other spider code is omitted for brevity): diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 77837378e..6ca0973d8 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -127,10 +127,7 @@ Request objects body to bytes (if given as a string). :type encoding: str - :param priority: the priority of this request (defaults to ``0``). - The priority is used by the scheduler to define the order used to process - requests. Requests with a higher priority value will execute earlier. - Negative values are allowed in order to indicate relatively low-priority. + :param priority: sets :attr:`priority`, defaults to ``0``. :type priority: int :param dont_filter: sets :attr:`dont_filter`, defaults to ``False``. @@ -179,6 +176,8 @@ Request objects .. autoattribute:: errback + .. autoattribute:: priority + .. attribute:: Request.cb_kwargs A dictionary that contains arbitrary metadata for this request. Its contents @@ -353,7 +352,7 @@ errors if needed: "https://example.invalid/", # DNS error expected ] - def start_requests(self): + async def start(self): for u in self.start_urls: yield scrapy.Request( u, @@ -647,6 +646,7 @@ Those are: * ``ftp_user`` (See :setting:`FTP_USER` for more info) * :reqmeta:`handle_httpstatus_all` * :reqmeta:`handle_httpstatus_list` +* :reqmeta:`is_start_request` * :reqmeta:`max_retry_times` * :reqmeta:`proxy` * :reqmeta:`redirect_reasons` diff --git a/docs/topics/scheduler.rst b/docs/topics/scheduler.rst index 57c24b76a..b6e54ebd7 100644 --- a/docs/topics/scheduler.rst +++ b/docs/topics/scheduler.rst @@ -26,9 +26,9 @@ Minimal scheduler interface :members: -Default Scrapy scheduler -======================== +Default scheduler +================= -.. autoclass:: Scheduler +.. autoclass:: Scheduler() :members: - :special-members: __len__ + :special-members: __init__, __len__ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 3a61306d6..537e51e40 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1326,6 +1326,7 @@ Default: ``{}`` A dict containing the pipelines enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. + .. setting:: JOBDIR JOBDIR @@ -1336,6 +1337,7 @@ Default: ``None`` A string indicating the directory for storing the state of a crawl when :ref:`pausing and resuming crawls `. + .. setting:: LOG_ENABLED LOG_ENABLED @@ -1700,23 +1702,28 @@ SCHEDULER_DISK_QUEUE Default: ``'scrapy.squeues.PickleLifoDiskQueue'`` -Type of disk queue that will be used by scheduler. Other available types are -``scrapy.squeues.PickleFifoDiskQueue``, ``scrapy.squeues.MarshalFifoDiskQueue``, +Type of disk queue that will be used by the scheduler. Other available types +are ``scrapy.squeues.PickleFifoDiskQueue``, +``scrapy.squeues.MarshalFifoDiskQueue``, ``scrapy.squeues.MarshalLifoDiskQueue``. + .. setting:: SCHEDULER_MEMORY_QUEUE SCHEDULER_MEMORY_QUEUE ---------------------- + Default: ``'scrapy.squeues.LifoMemoryQueue'`` -Type of in-memory queue used by scheduler. Other available type is: +Type of in-memory queue used by the scheduler. Other available type is: ``scrapy.squeues.FifoMemoryQueue``. + .. setting:: SCHEDULER_PRIORITY_QUEUE SCHEDULER_PRIORITY_QUEUE ------------------------ + Default: ``'scrapy.pqueues.ScrapyPriorityQueue'`` Type of priority queue used by the scheduler. Another available type is @@ -1726,6 +1733,51 @@ Type of priority queue used by the scheduler. Another available type is domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue`` does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. + +.. setting:: SCHEDULER_START_DISK_QUEUE + +SCHEDULER_START_DISK_QUEUE +-------------------------- + +Default: ``'scrapy.squeues.PickleFifoDiskQueue'`` + +Type of disk queue (see :setting:`JOBDIR`) that the :ref:`scheduler +` uses for :ref:`start requests `. + +For available choices, see :setting:`SCHEDULER_DISK_QUEUE`. + +.. queue-common-starts + +Use ``None`` or ``""`` to disable these separate queues entirely, and instead +have start requests share the same queues as other requests. + +.. note:: + + Disabling separate start request queues makes :ref:`start request order + ` unintuitive: start requests will be sent in order + only until :setting:`CONCURRENT_REQUESTS` is reached, then remaining start + requests will be sent in reverse order. + +.. queue-common-ends + + +.. setting:: SCHEDULER_START_MEMORY_QUEUE + +SCHEDULER_START_MEMORY_QUEUE +---------------------------- + +Default: ``'scrapy.squeues.FifoMemoryQueue'`` + +Type of in-memory queue that the :ref:`scheduler ` uses for +:ref:`start requests `. + +For available choices, see :setting:`SCHEDULER_MEMORY_QUEUE`. + +.. include:: settings.rst + :start-after: queue-common-starts + :end-before: queue-common-ends + + .. setting:: SCRAPER_SLOT_MAX_ACTIVE_SIZE SCRAPER_SLOT_MAX_ACTIVE_SIZE @@ -1957,7 +2009,7 @@ In order to use the reactor installed by Scrapy: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - def start_requests(self): + async def start(self): reactor.callLater(self.timeout, self.stop) urls = ["https://quotes.toscrape.com/page/1"] @@ -1986,7 +2038,7 @@ which raises :exc:`Exception`, becomes: self.timeout = int(kwargs.pop("timeout", "60")) super(QuotesSpider, self).__init__(*args, **kwargs) - def start_requests(self): + async def start(self): from twisted.internet import reactor reactor.callLater(self.timeout, self.stop) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index b45b12540..66cb87fc5 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -131,6 +131,19 @@ engine_stopped This signal supports returning deferreds from its handlers. +scheduler_empty +~~~~~~~~~~~~~~~ + +.. signal:: scheduler_empty +.. function:: scheduler_empty() + + Sent whenever the engine asks for a pending request from the + :ref:`scheduler ` (i.e. calls its + :meth:`~scrapy.core.scheduler.BaseScheduler.next_request` method) and the + scheduler returns none. + + See :ref:`start-requests-lazy` for an example. + Item signals ------------ @@ -160,7 +173,7 @@ item_scraped :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was scraped, or ``None`` - if it was yielded from :meth:`~scrapy.Spider.start_requests`. + if it was yielded from :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` item_dropped @@ -181,7 +194,7 @@ item_dropped :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was dropped, or ``None`` - if it was yielded from :meth:`~scrapy.Spider.start_requests`. + if it was yielded from :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` :param exception: the exception (which must be a @@ -205,7 +218,7 @@ item_error :param response: the response being processed when the exception was raised, or ``None`` if it was yielded from - :meth:`~scrapy.Spider.start_requests`. + :meth:`~scrapy.Spider.start`. :type response: :class:`~scrapy.http.Response` | ``None`` :param spider: the spider which raised the exception diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2211a822f..638035e64 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -70,30 +70,29 @@ one or more of these methods: .. class:: SpiderMiddleware - .. method:: process_start_requests(start_requests, spider) + .. method:: process_start(start: AsyncIterator[Any], /) -> AsyncIterator[Any] + :async: - This method is called with the start requests of the spider, and works - similarly to the :meth:`process_spider_output` method, except that it - doesn't have a response associated and must return only requests (not - items). + Iterate over the output of :meth:`~scrapy.Spider.start` or that + of the :meth:`process_start` method of an earlier spider middleware, + overriding it. For example: - It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects `. + .. code-block:: python - .. note:: When implementing this method in your spider middleware, you - should always return an iterable (that follows the input one) and - not consume all ``start_requests`` iterator because it can be very - large (or even unbounded) and cause a memory overflow. The Scrapy - engine is designed to pull start requests while it has capacity to - process them, so the start requests iterator can be effectively - endless where there is some other condition for stopping the spider - (like a time limit or item/page count). + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request - :param start_requests: the start requests - :type start_requests: an iterable of :class:`~scrapy.Request` + You may yield the same type of objects as :meth:`~scrapy.Spider.start`. - :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.Spider` object + To write spider middlewares that work on Scrapy versions lower than + VERSION, define also a synchronous ``process_start_requests()`` method + that returns an iterable. For example: + + .. code-block:: python + + def process_start_requests(self, start, spider): + yield from start .. method:: process_spider_input(response, spider) @@ -154,6 +153,7 @@ one or more of these methods: :type spider: :class:`~scrapy.Spider` object .. method:: process_spider_output_async(response, result, spider) + :async: .. versionadded:: 2.7 @@ -417,6 +417,14 @@ String value Class name (as a string) .. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url +StartSpiderMiddleware +--------------------- + +.. module:: scrapy.spidermiddlewares.start + +.. autoclass:: StartSpiderMiddleware + + UrlLengthMiddleware ------------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 0a67240d6..891c4da05 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -12,16 +12,16 @@ parsing pages for a particular site (or, in some cases, a group of sites). For spiders, the scraping cycle goes through something like this: -1. You start by generating the initial Requests to crawl the first URLs, and +1. You start by generating the initial requests to crawl the first URLs, and specify a callback function to be called with the response downloaded from those requests. - The first requests to perform are obtained by calling the - :meth:`~scrapy.Spider.start_requests` method which (by default) - generates :class:`~scrapy.Request` for the URLs specified in the - :attr:`~scrapy.Spider.start_urls` and the - :attr:`~scrapy.Spider.parse` method as callback function for the - Requests. + The first requests to perform are obtained by iterating the + :meth:`~scrapy.Spider.start` method, which by default yields a + :class:`~scrapy.Request` object for each URL in the + :attr:`~scrapy.Spider.start_urls` spider attribute, with the + :attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback` + function to handle each :class:`~scrapy.http.Response`. 2. In the callback function, you parse the response (web page) and return :ref:`item objects `, @@ -48,14 +48,7 @@ scrapy.Spider ============= .. class:: scrapy.spiders.Spider -.. class:: scrapy.Spider() - - This is the simplest spider, and the one from which every other spider - must inherit (including spiders that come bundled with Scrapy, as well as spiders - that you write yourself). It doesn't provide any special functionality. It just - provides a default :meth:`start_requests` implementation which sends requests from - the :attr:`start_urls` spider attribute and calls the spider's method ``parse`` - for each of the resulting responses. +.. autoclass:: scrapy.Spider .. attribute:: name @@ -81,12 +74,7 @@ scrapy.Spider Let's say your target url is ``https://www.example.com/1.html``, then add ``'example.com'`` to the list. - .. attribute:: start_urls - - A list of URLs where the spider will begin to crawl from, when no - particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data - contained in the start URLs. + .. autoattribute:: start_urls .. attribute:: custom_settings @@ -149,7 +137,7 @@ scrapy.Spider The final settings and the initialized :class:`~scrapy.crawler.Crawler` attributes are available in the - :meth:`start_requests` method, handlers of the + :meth:`start` method, handlers of the :signal:`engine_started` signal and later. :param crawler: crawler to which the spider will be bound @@ -201,42 +189,7 @@ scrapy.Spider super().update_settings(settings) settings.setdefault("FEEDS", {}).update(cls.custom_feed) - .. method:: start_requests() - - This method must return an iterable with the first Requests to crawl and/or with :ref:`item objects - ` for - this spider. It is called by Scrapy when the spider is opened for - scraping. Scrapy calls it only once, so it is safe to implement - :meth:`start_requests` as a generator. - - The default implementation generates ``Request(url, dont_filter=True)`` - for each url in :attr:`start_urls`. - - If you want to change the Requests used to start scraping a domain, this is - the method to override. For example, if you need to start by logging in using - a POST request, you could do: - - .. code-block:: python - - import scrapy - - - class MySpider(scrapy.Spider): - name = "myspider" - - def start_requests(self): - return [ - scrapy.FormRequest( - "http://www.example.com/login", - formdata={"user": "john", "pass": "secret"}, - callback=self.logged_in, - ) - ] - - def logged_in(self, response): - # here you would extract links to follow and return Requests for - # each of them, with another callback - pass + .. automethod:: start .. method:: parse(response) @@ -308,8 +261,9 @@ Return multiple Requests and items from a single callback: for href in response.xpath("//a/@href").getall(): yield scrapy.Request(response.urljoin(href), self.parse) -Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; -to give data more structure you can use :class:`~scrapy.Item` objects: +Instead of :attr:`~.start_urls` you can use :meth:`~scrapy.Spider.start` +directly; to give data more structure you can use :class:`~scrapy.Item` +objects: .. skip: next .. code-block:: python @@ -322,7 +276,7 @@ to give data more structure you can use :class:`~scrapy.Item` objects: name = "example.com" allowed_domains = ["example.com"] - def start_requests(self): + async def start(self): yield scrapy.Request("http://www.example.com/1.html", self.parse) yield scrapy.Request("http://www.example.com/2.html", self.parse) yield scrapy.Request("http://www.example.com/3.html", self.parse) @@ -376,7 +330,7 @@ The above example can also be written as follows: class MySpider(scrapy.Spider): name = "myspider" - def start_requests(self): + async def start(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") If you are :ref:`running Scrapy from a script `, you can @@ -410,6 +364,38 @@ used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. +.. _start-requests: + +Start requests +============== + +**Start requests** are :class:`~scrapy.Request` objects yielded from the +:meth:`~scrapy.Spider.start` method of a spider or from the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method of a +:ref:`spider middleware `. + +.. seealso:: :ref:`start-request-order` + +.. _start-requests-lazy: + +Delaying start request iteration +-------------------------------- + +You can override the :meth:`~scrapy.Spider.start` method as follows to pause +its iteration whenever there are scheduled requests: + +.. code-block:: python + + async def start(self): + async for item_or_request in super().start(): + if self.crawler.engine.needs_backoff(): + await self.crawler.signals.wait_for(signals.scheduler_empty) + yield item_or_request + +This can help minimize the number of requests in the scheduler at any given +time, to minimize resource usage (memory or disk, depending on +:setting:`JOBDIR`). + .. _builtin-spiders: Generic Spiders @@ -940,10 +926,11 @@ Combine SitemapSpider with other sources of urls: other_urls = ["http://www.example.com/about"] - def start_requests(self): - requests = list(super(MySpider, self).start_requests()) - requests += [scrapy.Request(x, self.parse_other) for x in self.other_urls] - return requests + async def start(self): + async for item_or_request in super().start(): + yield item_or_request + for url in self.other_urls: + yield Request(url, self.parse_other) def parse_shop(self, response): pass # ... scrape shop here ... diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 0e4a8fa6c..3e9bbe56e 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -59,6 +59,8 @@ Default Username and Password can be overridden by the settings You need the telnet program which comes installed by default in Windows, and most Linux distros. +.. _telnet-vars: + Available variables in the telnet console ========================================= @@ -77,8 +79,6 @@ convenience: +----------------+-------------------------------------------------------------------+ | ``spider`` | the active spider | +----------------+-------------------------------------------------------------------+ -| ``slot`` | the engine slot | -+----------------+-------------------------------------------------------------------+ | ``extensions`` | the Extension Manager (Crawler.extensions attribute) | +----------------+-------------------------------------------------------------------+ | ``stats`` | the Stats Collector (Crawler.stats attribute) | @@ -114,10 +114,10 @@ using the telnet console:: engine.scraper.is_idle() : False engine.spider.name : followall engine.spider_is_idle() : False - engine.slot.closing : False - len(engine.slot.inprogress) : 16 - len(engine.slot.scheduler.dqs or []) : 0 - len(engine.slot.scheduler.mqs) : 92 + engine._slot.closing : False + len(engine._slot.inprogress) : 16 + len(engine._slot.scheduler.dqs or []) : 0 + len(engine._slot.scheduler.mqs) : 92 len(engine.scraper.slot.queue) : 0 len(engine.scraper.slot.active) : 0 engine.scraper.slot.active_size : 0 diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 119dfdabb..269b27336 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -34,6 +34,10 @@ class QPSSpider(Spider): elif self.download_delay is not None: self.download_delay = float(self.download_delay) + async def start(self): + for item_or_request in self.start_requests(): + yield item_or_request + def start_requests(self): url = self.benchurl if self.latency is not None: diff --git a/pyproject.toml b/pyproject.toml index e14efdd17..187587eb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -238,7 +238,9 @@ markers = [ "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", ] -filterwarnings = [] +filterwarnings = [ + "ignore::DeprecationWarning:twisted.web.static" +] [tool.ruff.lint] extend-select = [ diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 16dae6ac4..96bb1ae84 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -13,9 +13,7 @@ from scrapy.linkextractors import LinkExtractor if TYPE_CHECKING: import argparse - from collections.abc import Iterable - - from scrapy import Request + from collections.abc import AsyncIterator class Command(ScrapyCommand): @@ -61,10 +59,10 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - def start_requests(self) -> Iterable[Request]: + async def start(self) -> AsyncIterator[Any]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" - return [scrapy.Request(url, dont_filter=True)] + yield scrapy.Request(url, dont_filter=True) def parse(self, response: Response) -> Any: assert isinstance(response, TextResponse) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 1ce155da7..56dc1ea55 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -80,10 +80,14 @@ class Command(ScrapyCommand): assert self.crawler_process spider_loader = self.crawler_process.spider_loader + async def start(self): + for request in conman.from_spider(self, result): + yield request + with set_environ(SCRAPY_CHECK="true"): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) - spidercls.start_requests = lambda s: conman.from_spider(s, result) # type: ignore[assignment,method-assign,return-value] + spidercls.start = start # type: ignore[assignment,method-assign,return-value] tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: @@ -101,10 +105,10 @@ class Command(ScrapyCommand): for method in sorted(methods): print(f" * {method}") else: - start = time.time() + start_time = time.time() self.crawler_process.start() stop = time.time() result.printErrors() - result.printSummary(start, stop) + result.printSummary(start_time, stop) self.exitcode = int(not result.wasSuccessful()) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 8a8d04ff6..ef6e13de2 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -89,5 +89,11 @@ class Command(ScrapyCommand): spidercls = spider_loader.load(opts.spider) else: spidercls = spidercls_for_request(spider_loader, request, spidercls) - self.crawler_process.crawl(spidercls, start_requests=lambda: [request]) + + async def start(self): + yield request + + spidercls.start = start # type: ignore[method-assign,attr-defined] + + self.crawler_process.crawl(spidercls) self.crawler_process.start() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index c6ed20b3b..0dd9954cb 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -22,7 +22,7 @@ from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: import argparse - from collections.abc import AsyncGenerator, Coroutine, Iterable + from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Iterable from twisted.python.failure import Failure @@ -258,11 +258,11 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - def _start_requests(spider: Spider) -> Iterable[Request]: + async def start(spider: Spider) -> AsyncIterator[Any]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: - self.spidercls.start_requests = _start_requests # type: ignore[assignment,method-assign] + self.spidercls.start = start # type: ignore[assignment,method-assign] def start_parsing(self, url: str, opts: argparse.Namespace) -> None: assert self.crawler_process diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 3047ae396..9dabfcd9c 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -24,9 +24,9 @@ if TYPE_CHECKING: class Command(ScrapyCommand): requires_project = False default_settings = { + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", "KEEP_ALIVE": True, "LOGSTATS_INTERVAL": 0, - "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", } def syntax(self) -> str: @@ -85,7 +85,7 @@ class Command(ScrapyCommand): crawler._apply_settings() # The Shell class needs a persistent engine in the crawler crawler.engine = crawler._create_engine() - crawler.engine.start() + crawler.engine.start(_start_request_processing=False) self._start_crawler_thread() diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 653e5e05c..7f5dd0405 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -9,6 +9,7 @@ from __future__ import annotations import logging from time import time +from traceback import format_exc from typing import TYPE_CHECKING, Any, TypeVar, cast from twisted.internet.defer import Deferred, inlineCallbacks, succeed @@ -16,15 +17,19 @@ from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals -from scrapy.core.scraper import Scraper, _HandleOutputDeferred +from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + maybe_deferred_to_future, +) from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: - from collections.abc import Callable, Generator, Iterable, Iterator + from collections.abc import AsyncIterator, Callable, Generator from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler @@ -40,17 +45,15 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -class Slot: +class _Slot: def __init__( self, - start_requests: Iterable[Request], close_if_idle: bool, nextcall: CallLaterOnce[None], scheduler: BaseScheduler, ) -> None: self.closing: Deferred[None] | None = None self.inprogress: set[Request] = set() - self.start_requests: Iterator[Request] | None = iter(start_requests) self.close_if_idle: bool = close_if_idle self.nextcall: CallLaterOnce[None] = nextcall self.scheduler: BaseScheduler = scheduler @@ -78,6 +81,8 @@ class Slot: class ExecutionEngine: + _SLOT_HEARTBEAT_INTERVAL: float = 5.0 + def __init__( self, crawler: Crawler, @@ -88,20 +93,25 @@ class ExecutionEngine: self.signals: SignalManager = crawler.signals assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter - self.slot: Slot | None = None + self._slot: _Slot | None = None self.spider: Spider | None = None self.running: bool = False self.paused: bool = False - self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( - crawler.settings - ) - downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) - self.downloader: Downloader = downloader_cls(crawler) - self.scraper: Scraper = Scraper(crawler) self._spider_closed_callback: Callable[[Spider], Deferred[None] | None] = ( spider_closed_callback ) self.start_time: float | None = None + self._start: AsyncIterator[Any] | None = None + downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) + try: + self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( + crawler.settings + ) + self.downloader: Downloader = downloader_cls(crawler) + self.scraper: Scraper = Scraper(crawler) + except Exception: + self.close() + raise def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler @@ -114,22 +124,28 @@ class ExecutionEngine: ) return scheduler_cls - @inlineCallbacks - def start(self) -> Generator[Deferred[Any], Any, None]: + @deferred_f_from_coro_f + async def start(self, _start_request_processing=True) -> None: if self.running: raise RuntimeError("Engine already running") self.start_time = time() - yield self.signals.send_catch_log_deferred(signal=signals.engine_started) + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred(signal=signals.engine_started) + ) self.running = True self._closewait: Deferred[None] = Deferred() - yield self._closewait + if _start_request_processing: + self._start_request_processing() + await maybe_deferred_to_future(self._closewait) def stop(self) -> Deferred[None]: """Gracefully stop the execution engine""" - @inlineCallbacks - def _finish_stopping_engine(_: Any) -> Generator[Deferred[Any], Any, None]: - yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + @deferred_f_from_coro_f + async def _finish_stopping_engine(_: Any) -> None: + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + ) self._closewait.callback(None) if not self.running: @@ -163,59 +179,85 @@ class ExecutionEngine: def unpause(self) -> None: self.paused = False - def _next_request(self) -> None: - if self.slot is None: - return + async def _process_start_next(self): + """Processes the next item or request from Spider.start(). - assert self.spider is not None # typing - - if self.paused: - return - - while ( - not self._needs_backout() - and self._next_request_from_scheduler() is not None - ): - pass - - if self.slot.start_requests is not None and not self._needs_backout(): - try: - request_or_item = next(self.slot.start_requests) - except StopIteration: - self.slot.start_requests = None - except Exception: - self.slot.start_requests = None - logger.error( - "Error while obtaining start requests", - exc_info=True, - extra={"spider": self.spider}, - ) + If a request, it is scheduled. If an item, it is sent to item + pipelines. + """ + try: + item_or_request = await self._start.__anext__() + except StopAsyncIteration: + self._start = None + except Exception as exception: + self._start = None + exception_traceback = format_exc() + logger.error( + f"Error while reading start items and requests: {exception}.\n{exception_traceback}", + exc_info=True, + ) + else: + if not self.spider: + return # spider already closed + if isinstance(item_or_request, Request): + self.crawl(item_or_request) else: - if isinstance(request_or_item, Request): - self.crawl(request_or_item) - else: - self.scraper.start_itemproc(request_or_item, response=None) + self.scraper.start_itemproc(item_or_request, response=None) + self._slot.nextcall.schedule() - if self.spider_is_idle() and self.slot.close_if_idle: + @deferred_f_from_coro_f + async def _start_request_processing(self) -> None: + """Starts consuming Spider.start() output and sending scheduled + requests.""" + # Starts the processing of scheduled requests, as well as a periodic + # call to that processing method for scenarios where the scheduler + # reports having pending requests but returns none. + assert self._slot is not None # typing + self._slot.nextcall.schedule() + self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) + + while self._start and self.spider: + await self._process_start_next() + if not self.needs_backout(): + # Give room for the outcome of self._process_start_next() to be + # processed before continuing with the next iteration. + self._slot.nextcall.schedule() + await self._slot.nextcall.wait() + + def _start_scheduled_requests(self) -> None: + if self._slot is None or self._slot.closing is not None or self.paused: + return + + while not self.needs_backout(): + if not self._start_scheduled_request(): + break + + if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() - def _needs_backout(self) -> bool: - assert self.slot is not None # typing + def needs_backout(self) -> bool: + """Returns ``True`` if no more requests can be sent at the moment, or + ``False`` otherwise. + + See :ref:`start-requests-lazy` for an example. + """ + assert self._slot is not None # typing assert self.scraper.slot is not None # typing return ( not self.running - or bool(self.slot.closing) + or bool(self._slot.closing) or self.downloader.needs_backout() or self.scraper.slot.needs_backout() ) - def _next_request_from_scheduler(self) -> Deferred[None] | None: - assert self.slot is not None # typing + def _start_scheduled_request(self) -> bool: + assert self._slot is not None # typing assert self.spider is not None # typing - request = self.slot.scheduler.next_request() + request = self._slot.scheduler.next_request() if request is None: - return None + self.signals.send_catch_log(signals.scheduler_empty) + return False d: Deferred[Response | Request] = self._download(request) d.addBoth(self._handle_downloader_output, request) @@ -228,8 +270,8 @@ class ExecutionEngine: ) def _remove_request(_: Any) -> None: - assert self.slot - self.slot.remove_request(request) + assert self._slot + self._slot.remove_request(request) d2: Deferred[None] = d.addBoth(_remove_request) d2.addErrback( @@ -239,7 +281,7 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - slot = self.slot + slot = self._slot d2.addBoth(lambda _: slot.nextcall.schedule()) d2.addErrback( lambda f: logger.info( @@ -248,13 +290,12 @@ class ExecutionEngine: extra={"spider": self.spider}, ) ) - return d2 + return True + @inlineCallbacks def _handle_downloader_output( self, result: Request | Response | Failure, request: Request - ) -> _HandleOutputDeferred | None: - assert self.spider is not None # typing - + ) -> Generator[Deferred[Any], Any, None]: if not isinstance(result, (Request, Response, Failure)): raise TypeError( f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}" @@ -263,35 +304,35 @@ class ExecutionEngine: # downloader middleware can return requests (for example, redirects) if isinstance(result, Request): self.crawl(result) - return None + return - d = self.scraper.enqueue_scrape(result, request) - d.addErrback( - lambda f: logger.error( - "Error while enqueuing downloader output", - exc_info=failure_to_exc_info(f), + try: + yield self.scraper.enqueue_scrape(result, request) + except Exception: + assert self.spider is not None + logger.error( + "Error while enqueuing scrape", + exc_info=True, extra={"spider": self.spider}, ) - ) - return d def spider_is_idle(self) -> bool: - if self.slot is None: + if self._slot is None: raise RuntimeError("Engine slot not assigned") if not self.scraper.slot.is_idle(): # type: ignore[union-attr] return False if self.downloader.active: # downloader has pending requests return False - if self.slot.start_requests is not None: # not all start requests are handled + if self._start is not None: # not all start requests are handled return False - return not self.slot.scheduler.has_pending_requests() + return not self._slot.scheduler.has_pending_requests() def crawl(self, request: Request) -> None: """Inject the request into the spider <-> downloader pipeline""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") self._schedule_request(request) - self.slot.nextcall.schedule() # type: ignore[union-attr] + self._slot.nextcall.schedule() # type: ignore[union-attr] def _schedule_request(self, request: Request) -> None: request_scheduled_result = self.signals.send_catch_log( @@ -303,7 +344,7 @@ class ExecutionEngine: for handler, result in request_scheduled_result: if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): return - if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] + if not self._slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( signals.request_dropped, request=request, spider=self.spider ) @@ -320,14 +361,14 @@ class ExecutionEngine: def _downloaded( self, result: Response | Request | Failure, request: Request ) -> Deferred[Response] | Response | Failure: - assert self.slot is not None # typing - self.slot.remove_request(request) + assert self._slot is not None # typing + self._slot.remove_request(request) return self.download(result) if isinstance(result, Request) else result def _download(self, request: Request) -> Deferred[Response | Request]: - assert self.slot is not None # typing + assert self._slot is not None # typing - self.slot.add_request(request) + self._slot.add_request(request) def _on_success(result: Response | Request) -> Response | Request: if not isinstance(result, (Response, Request)): @@ -352,8 +393,8 @@ class ExecutionEngine: return result def _on_complete(_: _T) -> _T: - assert self.slot is not None - self.slot.nextcall.schedule() + assert self._slot is not None + self._slot.nextcall.schedule() return _ assert self.spider is not None @@ -362,31 +403,28 @@ class ExecutionEngine: dwld.addBoth(_on_complete) return dwld - @inlineCallbacks - def open_spider( + @deferred_f_from_coro_f + async def open_spider( self, spider: Spider, - start_requests: Iterable[Request] = (), close_if_idle: bool = True, - ) -> Generator[Deferred[Any], Any, None]: - if self.slot is not None: + ) -> None: + if self._slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) - nextcall = CallLaterOnce(self._next_request) - scheduler = build_from_crawler(self.scheduler_cls, self.crawler) - start_requests = yield self.scraper.spidermw.process_start_requests( - start_requests, spider - ) - self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider + nextcall = CallLaterOnce(self._start_scheduled_requests) + scheduler = build_from_crawler(self.scheduler_cls, self.crawler) + self._slot = _Slot(close_if_idle, nextcall, scheduler) + self._start = await self.scraper.spidermw.process_start(spider) if hasattr(scheduler, "open") and (d := scheduler.open(spider)): - yield d - yield self.scraper.open_spider(spider) + await maybe_deferred_to_future(d) + await maybe_deferred_to_future(self.scraper.open_spider(spider)) assert self.crawler.stats self.crawler.stats.open_spider(spider) - yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) - self.slot.nextcall.schedule() - self.slot.heartbeat.start(5) + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) + ) def _spider_idle(self) -> None: """ @@ -415,17 +453,17 @@ class ExecutionEngine: def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred[None]: """Close (cancel) spider and clear all its outstanding requests""" - if self.slot is None: + if self._slot is None: raise RuntimeError("Engine slot not assigned") - if self.slot.closing is not None: - return self.slot.closing + if self._slot.closing is not None: + return self._slot.closing logger.info( "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider} ) - dfd = self.slot.close() + dfd = self._slot.close() def log_failure(msg: str) -> Callable[[Failure], None]: def errback(failure: Failure) -> None: @@ -441,8 +479,8 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.scraper.close_spider()) dfd.addErrback(log_failure("Scraper close failure")) - if hasattr(self.slot.scheduler, "close"): - dfd.addBoth(lambda _: cast(Slot, self.slot).scheduler.close(reason)) + if hasattr(self._slot.scheduler, "close"): + dfd.addBoth(lambda _: cast(_Slot, self._slot).scheduler.close(reason)) dfd.addErrback(log_failure("Scheduler close failure")) dfd.addBoth( diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 4bb143dfd..57d27b7cf 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -5,13 +5,16 @@ import logging from abc import abstractmethod from pathlib import Path from typing import TYPE_CHECKING, Any, cast +from warnings import warn # working around https://github.com/sphinx-doc/sphinx/issues/10400 from twisted.internet.defer import Deferred # noqa: TC002 +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.spiders import Spider # noqa: TC001 from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.python import global_object_name if TYPE_CHECKING: # requires queuelib >= 1.6.2 @@ -50,18 +53,17 @@ class BaseSchedulerMeta(type): class BaseScheduler(metaclass=BaseSchedulerMeta): - """ - The scheduler component is responsible for storing requests received from - the engine, and feeding them back upon request (also to the engine). + """The scheduler component is responsible for storing requests received + from the engine, and feeding them back upon request (also to the engine). The original sources of said requests are: - * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks + * Spider: ``start`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods The order in which the scheduler returns its stored requests (via the ``next_request`` method) - plays a great part in determining the order in which those requests are downloaded. + plays a great part in determining the order in which those requests are downloaded. See :ref:`request-order`. The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with. """ @@ -126,55 +128,112 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): class Scheduler(BaseScheduler): + """Default scheduler. + + Requests are stored into priority queues + (:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by + :attr:`~scrapy.http.Request.priority`. + + By default, a single, memory-based priority queue is used for all requests. + When using :setting:`JOBDIR`, a disk-based priority queue is also created, + and only unserializable requests are stored in the memory-based priority + queue. For a given priority value, requests in memory take precedence over + requests in disk. + + Each priority queue stores requests in separate internal queues, one per + priority value. The memory priority queue uses + :setting:`SCHEDULER_MEMORY_QUEUE` queues, while the disk priority queue + uses :setting:`SCHEDULER_DISK_QUEUE` queues. The internal queues determine + :ref:`request order ` when requests have the same priority. + :ref:`Start requests ` are stored into separate internal + queues by default, and :ref:`ordered differently `. + + Duplicate requests are filtered out with an instance of + :setting:`DUPEFILTER_CLASS`. + + .. _request-order: + + Request order + ============= + + With default settings, pending requests are stored in a LIFO_ queue + (:ref:`except for start requests `). As a result, + crawling happens in `DFO order`_, which is usually the most convenient + crawl order. However, you can enforce :ref:`BFO ` or :ref:`a custom + order ` (:ref:`except for the first few requests + `). + + .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) + .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search + + .. _start-request-order: + + Start request order + ------------------- + + :ref:`Start requests ` are sent in the order they are + yielded from :meth:`~scrapy.Spider.start`, and given the same + :attr:`~scrapy.http.Request.priority`, start requests take precedence over + other requests. + + You can set :setting:`SCHEDULER_START_MEMORY_QUEUE` and + :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` to handle start requests + the same as other requests when it comes to order and priority. + + + .. _bfo: + + Crawling in BFO order + --------------------- + + If you do want to crawl in `BFO order`_, you can do it by setting the + following :ref:`settings `: + + | :setting:`DEPTH_PRIORITY` = ``1`` + | :setting:`SCHEDULER_DISK_QUEUE` = ``"scrapy.squeues.PickleFifoDiskQueue"`` + | :setting:`SCHEDULER_MEMORY_QUEUE` = ``"scrapy.squeues.FifoMemoryQueue"`` + + .. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search + + + .. _custom-request-order: + + Crawling in a custom order + -------------------------- + + You can manually set :attr:`~scrapy.http.Request.priority` on requests to + force a specific request order. + + + .. _concurrency-v-order: + + Concurrency affects order + ------------------------- + + While pending requests are below the configured values of + :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` + or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent + concurrently. + + As a result, the first few requests of a crawl may not follow the desired + order. Lowering those settings to ``1`` enforces the desired order except + for the very first request, but it significantly slows down the crawl as a + whole. """ - Default Scrapy scheduler. This implementation also handles duplication - filtering via the :setting:`dupefilter `. - This scheduler stores requests into several priority queues (defined by the - :setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues - are backed by either memory or disk based queues (respectively defined by the - :setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings). - - Request prioritization is almost entirely delegated to the priority queue. The only - prioritization performed by this scheduler is using the disk-based queue if present - (i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based - queue if a serialization error occurs. If the disk queue is not present, the memory one - is used directly. - - :param dupefilter: An object responsible for checking and filtering duplicate requests. - The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. - :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: - any class that implements the `BaseDupeFilter` interface - - :param jobdir: The path of a directory to be used for persisting the crawl's state. - The value for the :setting:`JOBDIR` setting is used by default. - See :ref:`topics-jobs`. - :type jobdir: :class:`str` or ``None`` - - :param dqclass: A class to be used as persistent request queue. - The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. - :type dqclass: class - - :param mqclass: A class to be used as non-persistent request queue. - The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. - :type mqclass: class - - :param logunser: A boolean that indicates whether or not unserializable requests should be logged. - The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. - :type logunser: bool - - :param stats: A stats collector object to record stats about the request scheduling process. - The value for the :setting:`STATS_CLASS` setting is used by default. - :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: - any class that implements the `StatsCollector` interface - - :param pqclass: A class to be used as priority queue for requests. - The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. - :type pqclass: class - - :param crawler: The crawler object corresponding to the current crawl. - :type crawler: :class:`scrapy.crawler.Crawler` - """ + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"]) + return cls( + dupefilter=build_from_crawler(dupefilter_cls, crawler), + jobdir=job_dir(crawler.settings), + dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]), + mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]), + logunser=crawler.settings.getbool("SCHEDULER_DEBUG"), + stats=crawler.stats, + pqclass=load_object(crawler.settings["SCHEDULER_PRIORITY_QUEUE"]), + crawler=crawler, + ) def __init__( self, @@ -187,6 +246,42 @@ class Scheduler(BaseScheduler): pqclass: type[ScrapyPriorityQueue] | None = None, crawler: Crawler | None = None, ): + """Initialize the scheduler. + + :param dupefilter: An object responsible for checking and filtering duplicate requests. + The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. + :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: + any class that implements the `BaseDupeFilter` interface + + :param jobdir: The path of a directory to be used for persisting the crawl's state. + The value for the :setting:`JOBDIR` setting is used by default. + See :ref:`topics-jobs`. + :type jobdir: :class:`str` or ``None`` + + :param dqclass: A class to be used as persistent request queue. + The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. + :type dqclass: class + + :param mqclass: A class to be used as non-persistent request queue. + The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. + :type mqclass: class + + :param logunser: A boolean that indicates whether or not unserializable requests should be logged. + The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. + :type logunser: bool + + :param stats: A stats collector object to record stats about the request scheduling process. + The value for the :setting:`STATS_CLASS` setting is used by default. + :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: + any class that implements the `StatsCollector` interface + + :param pqclass: A class to be used as priority queue for requests. + The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. + :type pqclass: class + + :param crawler: The crawler object corresponding to the current crawl. + :type crawler: :class:`scrapy.crawler.Crawler` + """ self.df: BaseDupeFilter = dupefilter self.dqdir: str | None = self._dqdir(jobdir) self.pqclass: type[ScrapyPriorityQueue] | None = pqclass @@ -195,23 +290,22 @@ class Scheduler(BaseScheduler): self.logunser: bool = logunser self.stats: StatsCollector | None = stats self.crawler: Crawler | None = crawler - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - """ - Factory method, initializes the scheduler with arguments taken from the crawl settings - """ - dupefilter_cls = load_object(crawler.settings["DUPEFILTER_CLASS"]) - return cls( - dupefilter=build_from_crawler(dupefilter_cls, crawler), - jobdir=job_dir(crawler.settings), - dqclass=load_object(crawler.settings["SCHEDULER_DISK_QUEUE"]), - mqclass=load_object(crawler.settings["SCHEDULER_MEMORY_QUEUE"]), - logunser=crawler.settings.getbool("SCHEDULER_DEBUG"), - stats=crawler.stats, - pqclass=load_object(crawler.settings["SCHEDULER_PRIORITY_QUEUE"]), - crawler=crawler, + self._sdqclass: type[BaseQueue] | None = self._get_start_queue_cls( + crawler, "DISK" ) + self._smqclass: type[BaseQueue] | None = self._get_start_queue_cls( + crawler, "MEMORY" + ) + + def _get_start_queue_cls( + self, crawler: Crawler | None, queue: str + ) -> type[BaseQueue] | None: + if crawler is None: + return None + cls = crawler.settings[f"SCHEDULER_START_{queue}_QUEUE"] + if not cls: + return None + return load_object(cls) def has_pending_requests(self) -> bool: return len(self) > 0 @@ -324,12 +418,27 @@ class Scheduler(BaseScheduler): """Create a new priority queue instance, with in-memory storage""" assert self.crawler assert self.pqclass - return build_from_crawler( - self.pqclass, - self.crawler, - downstream_queue_cls=self.mqclass, - key="", - ) + try: + return build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.mqclass, + key="", + start_queue_cls=self._smqclass, + ) + except TypeError: + warn( + f"The __init__ method of {global_object_name(self.pqclass)} " + f"does not support a `start_queue_cls` keyword-only " + f"parameter.", + ScrapyDeprecationWarning, + ) + return build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.mqclass, + key="", + ) def _dq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with disk storage""" @@ -337,13 +446,29 @@ class Scheduler(BaseScheduler): assert self.dqdir assert self.pqclass state = self._read_dqs_state(self.dqdir) - q = build_from_crawler( - self.pqclass, - self.crawler, - downstream_queue_cls=self.dqclass, - key=self.dqdir, - startprios=state, - ) + try: + q = build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.dqclass, + key=self.dqdir, + startprios=state, + start_queue_cls=self._sdqclass, + ) + except TypeError: + warn( + f"The __init__ method of {global_object_name(self.pqclass)} " + f"does not support a `start_queue_cls` keyword-only " + f"parameter.", + ScrapyDeprecationWarning, + ) + q = build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.dqclass, + key=self.dqdir, + startprios=state, + ) if q: logger.info( "Resuming crawl (%(queuesize)d requests scheduled)", diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 6f69d668e..9378f2651 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -6,10 +6,10 @@ from __future__ import annotations import logging import warnings from collections import deque -from collections.abc import AsyncIterable, Iterator -from typing import TYPE_CHECKING, Any, TypeVar, Union, cast +from collections.abc import AsyncIterator +from typing import TYPE_CHECKING, Any, TypeVar, Union -from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.internet.defer import Deferred, inlineCallbacks, maybeDeferred from twisted.python.failure import Failure from scrapy import Spider, signals @@ -22,10 +22,12 @@ from scrapy.exceptions import ( ) from scrapy.http import Request, Response from scrapy.utils.defer import ( + _defer_sleep, aiter_errback, - defer_fail, - defer_succeed, + deferred_f_from_coro_f, + deferred_from_coro, iter_errback, + maybe_deferred_to_future, parallel, parallel_async, ) @@ -46,9 +48,7 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -_ParallelResult = list[tuple[bool, Iterator[Any]]] -_HandleOutputDeferred = Deferred[Union[_ParallelResult, None]] -QueueTuple = tuple[Union[Response, Failure], Request, _HandleOutputDeferred] +QueueTuple = tuple[Union[Response, Failure], Request, Deferred[None]] class Slot: @@ -66,8 +66,9 @@ class Slot: def add_response_request( self, result: Response | Failure, request: Request - ) -> _HandleOutputDeferred: - deferred: _HandleOutputDeferred = Deferred() + ) -> Deferred[None]: + # this Deferred will be awaited in enqueue_scrape() + deferred: Deferred[None] = Deferred() self.queue.append((result, request, deferred)) if isinstance(result, Response): self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) @@ -76,9 +77,9 @@ class Slot: return deferred def next_response_request_deferred(self) -> QueueTuple: - response, request, deferred = self.queue.popleft() + result, request, deferred = self.queue.popleft() self.active.add(request) - return response, request, deferred + return result, request, deferred def finish_response(self, result: Response | Failure, request: Request) -> None: self.active.remove(request) @@ -143,9 +144,10 @@ class Scraper: assert self.crawler.spider self.slot.closing.callback(self.crawler.spider) + @inlineCallbacks def enqueue_scrape( self, result: Response | Failure, request: Request, spider: Spider | None = None - ) -> _HandleOutputDeferred: + ) -> Generator[Deferred[Any], Any, None]: if spider is not None: warnings.warn( "Passing a 'spider' argument to Scraper.enqueue_scrape() is deprecated.", @@ -156,103 +158,106 @@ class Scraper: if self.slot is None: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) - - def finish_scraping(_: _T) -> _T: - assert self.slot is not None + self._scrape_next() + try: + yield dfd + except Exception: + logger.error( + "Scraper bug processing %(request)s", + {"request": request}, + exc_info=True, + extra={"spider": self.crawler.spider}, + ) + finally: self.slot.finish_response(result, request) self._check_if_closing() self._scrape_next() - return _ - - dfd.addBoth(finish_scraping) - dfd.addErrback( - lambda f: logger.error( - "Scraper bug processing %(request)s", - {"request": request}, - exc_info=failure_to_exc_info(f), - extra={"spider": self.crawler.spider}, - ) - ) - self._scrape_next() - return dfd def _scrape_next(self) -> None: assert self.slot is not None # typing while self.slot.queue: - response, request, deferred = self.slot.next_response_request_deferred() - self._scrape(response, request).chainDeferred(deferred) + result, request, deferred = self.slot.next_response_request_deferred() + self._scrape(result, request).chainDeferred(deferred) - def _scrape( - self, result: Response | Failure, request: Request - ) -> _HandleOutputDeferred: - """ - Handle the downloaded response or failure through the spider callback/errback - """ + @deferred_f_from_coro_f + async def _scrape(self, result: Response | Failure, request: Request) -> None: + """Handle the downloaded response or failure through the spider callback/errback.""" if not isinstance(result, (Response, Failure)): raise TypeError( f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) - dfd: Deferred[Iterable[Any] | AsyncIterable[Any]] = self._scrape2( - result, request - ) # returns spider's processed output - dfd.addErrback(self.handle_spider_error, request, result) - dfd2: _HandleOutputDeferred = dfd.addCallback( - self.handle_spider_output, request, cast(Response, result) - ) - return dfd2 - def _scrape2( - self, result: Response | Failure, request: Request - ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: - """ - Handle the different cases of request's result been a Response or a Failure - """ + assert self.crawler.spider if isinstance(result, Response): - # Deferreds are invariant so Mutable*Chain isn't matched to *Iterable - assert self.crawler.spider - return self.spidermw.scrape_response( # type: ignore[return-value] - self.call_spider, result, request, self.crawler.spider - ) - # else result is a Failure - dfd = self.call_spider(result, request) - dfd.addErrback(self._log_download_errors, result, request) - return dfd + try: + # call the spider middlewares and the request callback with the response + output = await maybe_deferred_to_future( + self.spidermw.scrape_response( + self.call_spider, result, request, self.crawler.spider + ) + ) + except Exception: + self.handle_spider_error(Failure(), request, result) + else: + await self.handle_spider_output_async(output, request, result) + return + + try: + # call the request errback with the downloader error + await self.call_spider_async(result, request) + except Exception as spider_exc: + # the errback didn't silence the exception + if not result.check(IgnoreRequest): + logkws = self.logformatter.download_error( + result, request, self.crawler.spider + ) + logger.log( + *logformatter_adapter(logkws), + extra={"spider": self.crawler.spider}, + exc_info=failure_to_exc_info(result), + ) + if spider_exc is not result.value: + # the errback raised a different exception, handle it + self.handle_spider_error(Failure(), request, result) def call_spider( self, result: Response | Failure, request: Request, spider: Spider | None = None - ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: + ) -> Deferred[Iterable[Any] | AsyncIterator[Any]]: if spider is not None: warnings.warn( "Passing a 'spider' argument to Scraper.call_spider() is deprecated.", category=ScrapyDeprecationWarning, stacklevel=2, ) + return deferred_from_coro(self.call_spider_async(result, request)) + async def call_spider_async( + self, result: Response | Failure, request: Request + ) -> Iterable[Any] | AsyncIterator[Any]: + """Call the request callback or errback with the response or failure.""" + await maybe_deferred_to_future(_defer_sleep()) assert self.crawler.spider - dfd: Deferred[Any] if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request assert result.request callback = result.request.callback or self.crawler.spider._parse warn_on_generator_with_return_value(self.crawler.spider, callback) - dfd = defer_succeed(result) - dfd.addCallbacks( - callback=callback, callbackKeywords=result.request.cb_kwargs - ) + output = callback(result, **result.request.cb_kwargs) else: # result is a Failure # TODO: properly type adding this attribute to a Failure result.request = request # type: ignore[attr-defined] - dfd = defer_fail(result) - if request.errback: - warn_on_generator_with_return_value( - self.crawler.spider, request.errback - ) - dfd.addErrback(request.errback) - dfd2: Deferred[Iterable[Any] | AsyncIterable[Any]] = dfd.addCallback( - iterate_spider_output + if not request.errback: + result.raiseException() + warn_on_generator_with_return_value(self.crawler.spider, request.errback) + output = request.errback(result) + if isinstance(output, Failure): + output.raiseException() + # else the errback returned actual output (like a callback), + # which needs to be passed to iterate_spider_output() + return await maybe_deferred_to_future( + maybeDeferred(iterate_spider_output, output) ) - return dfd2 def handle_spider_error( self, @@ -261,6 +266,7 @@ class Scraper: response: Response | Failure, spider: Spider | None = None, ) -> None: + """Handle an exception raised by a spider callback or errback.""" if spider is not None: warnings.warn( "Passing a 'spider' argument to Scraper.handle_spider_error() is deprecated.", @@ -301,57 +307,68 @@ class Scraper: def handle_spider_output( self, - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], request: Request, response: Response, spider: Spider | None = None, - ) -> _HandleOutputDeferred: + ) -> Deferred[None]: + """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" if spider is not None: warnings.warn( "Passing a 'spider' argument to Scraper.handle_spider_output() is deprecated.", category=ScrapyDeprecationWarning, stacklevel=2, ) + return deferred_from_coro( + self.handle_spider_output_async(result, request, response) + ) - if not result: - return defer_succeed(None) - it: Iterable[_T] | AsyncIterable[_T] - dfd: Deferred[_ParallelResult] - if isinstance(result, AsyncIterable): - it = aiter_errback(result, self.handle_spider_error, request, response) - dfd = parallel_async( + async def handle_spider_output_async( + self, + result: Iterable[_T] | AsyncIterator[_T], + request: Request, + response: Response, + ) -> None: + """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" + if isinstance(result, AsyncIterator): + ait = aiter_errback(result, self.handle_spider_error, request, response) + await maybe_deferred_to_future( + parallel_async( + ait, + self.concurrent_items, + self._process_spidermw_output, + response, + ) + ) + return + it = iter_errback(result, self.handle_spider_error, request, response) + await maybe_deferred_to_future( + parallel( it, self.concurrent_items, self._process_spidermw_output, response, ) - else: - it = iter_errback(result, self.handle_spider_error, request, response) - dfd = parallel( - it, - self.concurrent_items, - self._process_spidermw_output, - response, - ) - # returning Deferred[_ParallelResult] instead of Deferred[Union[_ParallelResult, None]] - return dfd # type: ignore[return-value] + ) - def _process_spidermw_output( - self, output: Any, response: Response - ) -> Deferred[Any] | None: + @deferred_f_from_coro_f + async def _process_spidermw_output(self, output: Any, response: Response) -> None: """Process each Request/Item (given in the output parameter) returned - from the given spider + from the given spider. + + Items are sent to the item pipelines, requests are scheduled. """ if isinstance(output, Request): assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) - elif output is None: - pass - else: - return self.start_itemproc(output, response=response) - return None + return + if output is not None: + await maybe_deferred_to_future( + self.start_itemproc(output, response=response) + ) - def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[Any]: + @deferred_f_from_coro_f + async def start_itemproc(self, item: Any, *, response: Response | None) -> None: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come @@ -360,90 +377,56 @@ class Scraper: assert self.slot is not None # typing assert self.crawler.spider is not None # typing self.slot.itemproc_size += 1 - dfd = self.itemproc.process_item(item, self.crawler.spider) - dfd.addBoth(self._itemproc_finished, item, response) - return dfd - - def _log_download_errors( - self, - spider_failure: Failure, - download_failure: Failure, - request: Request, - ) -> Failure | None: - """Log and silence errors that come from the engine (typically download - errors that got propagated thru here). - - spider_failure: the value passed into the errback of self.call_spider() - (likely raised in the request errback) - - download_failure: the value passed into _scrape2() from - ExecutionEngine._handle_downloader_output() as "result" - (likely raised in the download handler or a downloader middleware) - """ - if not download_failure.check(IgnoreRequest): - assert self.crawler.spider - logkws = self.logformatter.download_error( - download_failure, request, self.crawler.spider + try: + output = await maybe_deferred_to_future( + self.itemproc.process_item(item, self.crawler.spider) ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": self.crawler.spider}, - exc_info=failure_to_exc_info(download_failure), - ) - if spider_failure is not download_failure: - # a request errback raised a different exception, it needs to be handled later - return spider_failure - return None - - def _itemproc_finished( - self, output: Any, item: Any, response: Response | None - ) -> Deferred[Any]: - """ItemProcessor finished for the given ``item`` and returned ``output``""" - assert self.slot is not None # typing - assert self.crawler.spider - self.slot.itemproc_size -= 1 - if isinstance(output, Failure): - ex = output.value - if isinstance(ex, DropItem): - logkws = self.logformatter.dropped( - item, ex, response, self.crawler.spider + except DropItem as ex: + logkws = self.logformatter.dropped(item, ex, response, self.crawler.spider) + if logkws is not None: + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} ) - if logkws is not None: - logger.log( - *logformatter_adapter(logkws), - extra={"spider": self.crawler.spider}, - ) - return self.signals.send_catch_log_deferred( + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred( signal=signals.item_dropped, item=item, response=response, spider=self.crawler.spider, - exception=output.value, + exception=ex, ) - assert ex + ) + except Exception as ex: logkws = self.logformatter.item_error( item, ex, response, self.crawler.spider ) logger.log( *logformatter_adapter(logkws), extra={"spider": self.crawler.spider}, - exc_info=failure_to_exc_info(output), + exc_info=True, ) - return self.signals.send_catch_log_deferred( - signal=signals.item_error, - item=item, - response=response, - spider=self.crawler.spider, - failure=output, + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred( + signal=signals.item_error, + item=item, + response=response, + spider=self.crawler.spider, + failure=Failure(), + ) ) - logkws = self.logformatter.scraped(output, response, self.crawler.spider) - if logkws is not None: - logger.log( - *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} + else: + logkws = self.logformatter.scraped(output, response, self.crawler.spider) + if logkws is not None: + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} + ) + await maybe_deferred_to_future( + self.signals.send_catch_log_deferred( + signal=signals.item_scraped, + item=output, + response=response, + spider=self.crawler.spider, + ) ) - return self.signals.send_catch_log_deferred( - signal=signals.item_scraped, - item=output, - response=response, - spider=self.crawler.spider, - ) + finally: + self.slot.itemproc_size -= 1 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index b8b0aec44..4a0cd9464 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -7,16 +7,17 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from collections.abc import AsyncIterable, Callable, Iterable +from collections.abc import AsyncIterator, Callable, Iterable from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast +from warnings import warn from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import Request, Spider -from scrapy.exceptions import _InvalidOutput +from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen @@ -40,12 +41,13 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") ScrapeFunc = Callable[ - [Union[Response, Failure], Request], Union[Iterable[_T], AsyncIterable[_T]] + [Union[Response, Failure], Request], + Deferred[Union[Iterable[_T], AsyncIterator[_T]]], ] def _isiterable(o: Any) -> bool: - return isinstance(o, (Iterable, AsyncIterable)) + return isinstance(o, (Iterable, AsyncIterator)) class SpiderMiddlewareManager(MiddlewareManager): @@ -55,12 +57,75 @@ class SpiderMiddlewareManager(MiddlewareManager): def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) + def __init__(self, *middlewares: Any) -> None: + self._check_deprecated_process_start_requests_use(middlewares) + super().__init__(*middlewares) + + def _check_deprecated_process_start_requests_use( + self, middlewares: tuple[Any] + ) -> None: + deprecated_middlewares = [ + middleware + for middleware in middlewares + if hasattr(middleware, "process_start_requests") + and not hasattr(middleware, "process_start") + ] + modern_middlewares = [ + middleware + for middleware in middlewares + if not hasattr(middleware, "process_start_requests") + and hasattr(middleware, "process_start") + ] + if deprecated_middlewares and modern_middlewares: + raise ValueError( + "You are trying to combine spider middlewares that only " + "define the deprecated process_start_requests() method () " + "with spider middlewares that only define the " + "process_start() method (). This is not possible. You must " + "either disable or make universal 1 of those 2 sets of " + "spider middlewares. Making a spider middleware universal " + "means having it define both methods. See the release notes " + "of Scrapy VERSION for details: " + "https://docs.scrapy.org/en/VERSION/news.html" + ) + + self._use_start_requests = bool(deprecated_middlewares) + if self._use_start_requests: + deprecated_middleware_list = ", ".join( + global_object_name(middleware.__class__) + for middleware in deprecated_middlewares + ) + warn( + f"The following enabled spider middlewares, directly or " + f"through their parent classes, define the deprecated " + f"process_start_requests() method: " + f"{deprecated_middleware_list}. process_start_requests() has " + f"been deprecated in favor of a new method, process_start(), " + f"to support asynchronous code execution. " + f"process_start_requests() will stop being called in a future " + f"version of Scrapy. If you use Scrapy VERSION or higher " + f"only, replace process_start_requests() with " + f"process_start(); note that process_start() is a coroutine " + f"(async def). If you need to maintain compatibility with " + f"lower Scrapy versions, when defining " + f"process_start_requests() in a spider middleware class, " + f"define process_start() as well. See the release notes of " + f"Scrapy VERSION for details: " + f"https://docs.scrapy.org/en/VERSION/news.html", + ScrapyDeprecationWarning, + ) + def _add_middleware(self, mw: Any) -> None: super()._add_middleware(mw) if hasattr(mw, "process_spider_input"): self.methods["process_spider_input"].append(mw.process_spider_input) - if hasattr(mw, "process_start_requests"): - self.methods["process_start_requests"].appendleft(mw.process_start_requests) + if self._use_start_requests: + if hasattr(mw, "process_start_requests"): + self.methods["process_start_requests"].appendleft( + mw.process_start_requests + ) + elif hasattr(mw, "process_start"): + self.methods["process_start"].appendleft(mw.process_start) process_spider_output = self._get_async_method_pair(mw, "process_spider_output") self.methods["process_spider_output"].appendleft(process_spider_output) process_spider_exception = getattr(mw, "process_spider_exception", None) @@ -72,7 +137,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Iterable[_T] | AsyncIterable[_T]: + ) -> Deferred[Iterable[_T] | AsyncIterator[_T]]: for method in self.methods["process_spider_input"]: method = cast(Callable, method) try: @@ -93,10 +158,10 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - iterable: Iterable[_T] | AsyncIterable[_T], + iterable: Iterable[_T] | AsyncIterator[_T], exception_processor_index: int, recover_to: MutableChain[_T] | MutableAsyncChain[_T], - ) -> Iterable[_T] | AsyncIterable[_T]: + ) -> Iterable[_T] | AsyncIterator[_T]: def process_sync(iterable: Iterable[_T]) -> Iterable[_T]: try: yield from iterable @@ -112,7 +177,7 @@ class SpiderMiddlewareManager(MiddlewareManager): assert isinstance(recover_to, MutableChain) recover_to.extend(exception_result) - async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]: + async def process_async(iterable: AsyncIterator[_T]) -> AsyncIterator[_T]: try: async for r in iterable: yield r @@ -128,7 +193,7 @@ class SpiderMiddlewareManager(MiddlewareManager): assert isinstance(recover_to, MutableAsyncChain) recover_to.extend(exception_result) - if isinstance(iterable, AsyncIterable): + if isinstance(iterable, AsyncIterator): return process_async(iterable) return process_sync(iterable) @@ -187,13 +252,13 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], start_index: int = 0, ) -> Generator[Deferred[Any], Any, MutableChain[_T] | MutableAsyncChain[_T]]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered: MutableChain[_T] | MutableAsyncChain[_T] - last_result_is_async = isinstance(result, AsyncIterable) + last_result_is_async = isinstance(result, AsyncIterator) recovered = MutableAsyncChain() if last_result_is_async else MutableChain() # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async. @@ -220,7 +285,7 @@ class SpiderMiddlewareManager(MiddlewareManager): need_downgrade = True try: if need_upgrade: - # Iterable -> AsyncIterable + # Iterable -> AsyncIterator result = as_async_generator(result) elif need_downgrade: logger.warning( @@ -230,10 +295,10 @@ class SpiderMiddlewareManager(MiddlewareManager): f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users" f" for more information." ) - assert isinstance(result, AsyncIterable) - # AsyncIterable -> Iterable + assert isinstance(result, AsyncIterator) + # AsyncIterator -> Iterable result = yield deferred_from_coro(collect_asyncgen(result)) - if isinstance(recovered, AsyncIterable): + if isinstance(recovered, AsyncIterator): recovered_collected = yield deferred_from_coro( collect_asyncgen(recovered) ) @@ -266,7 +331,7 @@ class SpiderMiddlewareManager(MiddlewareManager): f"{type(result)}" ) raise _InvalidOutput(msg) - last_result_is_async = isinstance(result, AsyncIterable) + last_result_is_async = isinstance(result, AsyncIterator) if last_result_is_async: return MutableAsyncChain(result, recovered) @@ -276,23 +341,23 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], ) -> MutableChain[_T] | MutableAsyncChain[_T]: recovered: MutableChain[_T] | MutableAsyncChain[_T] - if isinstance(result, AsyncIterable): + if isinstance(result, AsyncIterator): recovered = MutableAsyncChain() else: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) result = await maybe_deferred_to_future( cast( - "Deferred[Iterable[_T] | AsyncIterable[_T]]", + "Deferred[Iterable[_T] | AsyncIterator[_T]]", self._process_spider_output(response, spider, result), ) ) - if isinstance(result, AsyncIterable): + if isinstance(result, AsyncIterator): return MutableAsyncChain(result, recovered) - if isinstance(recovered, AsyncIterable): + if isinstance(recovered, AsyncIterator): recovered_collected = await collect_asyncgen(recovered) recovered = MutableChain(recovered_collected) return MutableChain(result, recovered) @@ -305,7 +370,7 @@ class SpiderMiddlewareManager(MiddlewareManager): spider: Spider, ) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]: async def process_callback_output( - result: Iterable[_T] | AsyncIterable[_T], + result: Iterable[_T] | AsyncIterator[_T], ) -> MutableChain[_T] | MutableAsyncChain[_T]: return await self._process_callback_output(response, spider, result) @@ -314,7 +379,7 @@ class SpiderMiddlewareManager(MiddlewareManager): ) -> Failure | MutableChain[_T] | MutableAsyncChain[_T]: return self._process_spider_exception(response, spider, _failure) - dfd: Deferred[Iterable[_T] | AsyncIterable[_T]] = mustbe_deferred( + dfd: Deferred[Iterable[_T] | AsyncIterator[_T]] = mustbe_deferred( self._process_spider_input, scrape_func, response, request, spider ) dfd2: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = dfd.addCallback( @@ -323,10 +388,90 @@ class SpiderMiddlewareManager(MiddlewareManager): dfd2.addErrback(process_spider_exception) return dfd2 - def process_start_requests( - self, start_requests: Iterable[Request], spider: Spider - ) -> Deferred[Iterable[Request]]: - return self._process_chain("process_start_requests", start_requests, spider) + async def process_start(self, spider: Spider) -> AsyncIterator[Any] | None: + self._check_deprecated_start_requests_use(spider) + if self._use_start_requests: + sync_start = iter(spider.start_requests()) + sync_start = await maybe_deferred_to_future( + self._process_chain("process_start_requests", sync_start, spider) + ) + start: AsyncIterator[Any] = as_async_generator(sync_start) + else: + start = spider.start() + start = await maybe_deferred_to_future( + self._process_chain("process_start", start) + ) + return start + + def _check_deprecated_start_requests_use(self, spider: Spider): + start_requests_cls = None + start_cls = None + spidercls = spider.__class__ + mro = spidercls.__mro__ + + for cls in mro: + cls_dict = cls.__dict__ + if start_requests_cls is None and "start_requests" in cls_dict: + start_requests_cls = cls + if start_cls is None and "start" in cls_dict: + start_cls = cls + if start_requests_cls is not None and start_cls is not None: + break + + # Spider defines both, start_requests and start. + assert start_requests_cls is not None + assert start_cls is not None + + if ( + start_requests_cls is not Spider + and start_cls is not start_requests_cls + and mro.index(start_requests_cls) < mro.index(start_cls) + ): + src = global_object_name(start_requests_cls) + if start_requests_cls is not spidercls: + src += f" (inherited by {global_object_name(spidercls)})" + warn( + f"{src} defines the deprecated start_requests() method. " + f"start_requests() has been deprecated in favor of a new " + f"method, start(), to support asynchronous code " + f"execution. start_requests() will stop being called in a " + f"future version of Scrapy. If you use Scrapy VERSION or " + f"higher only, replace start_requests() with start(); " + f"note that start() is a coroutine (async def). If you " + f"need to maintain compatibility with lower Scrapy versions, " + f"when overriding start_requests() in a spider class, " + f"override start() as well; you can use super() to " + f"reuse the inherited start() implementation without " + f"copy-pasting. See the release notes of Scrapy VERSION for " + f"details: https://docs.scrapy.org/en/VERSION/news.html", + ScrapyDeprecationWarning, + ) + + if ( + self._use_start_requests + and start_cls is not Spider + and start_requests_cls is not start_cls + and mro.index(start_cls) < mro.index(start_requests_cls) + ): + src = global_object_name(start_cls) + if start_cls is not spidercls: + src += f" (inherited by {global_object_name(spidercls)})" + raise ValueError( + f"{src} does not define the deprecated start_requests() " + f"method. However, one or more of your enabled spider " + f"middlewares (reported in an earlier deprecation warning) " + f"define the process_start_requests() method, and not the " + f"process_start() method, making them only compatible with " + f"(deprecated) spiders that define the start_requests() " + f"method. To solve this issue, disable the offending spider " + f"middlewares, upgrade them as described in that earlier " + f"deprecation warning, or make your spider compatible with " + f"deprecated spider middlewares (and earlier Scrapy versions) " + f"by defining a sync start_requests() method that works " + f"similarly to its existing start() method. See the " + f"release notes of Scrapy VERSION for details: " + f"https://docs.scrapy.org/en/VERSION/news.html" + ) # This method is only needed until _async compatibility methods are removed. @staticmethod diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1ec1e31dc..749096db5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -136,6 +136,9 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + # Cannot use @deferred_f_from_coro_f because that relies on the reactor + # being installed already, which is done within _apply_settings(), inside + # this method. @inlineCallbacks def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]: if self.crawling: @@ -151,9 +154,8 @@ class Crawler: self._apply_settings() self._update_root_log_handler() self.engine = self._create_engine() - start_requests = iter(self.spider.start_requests()) - yield self.engine.open_spider(self.spider, start_requests) - yield maybeDeferred(self.engine.start) + yield self.engine.open_spider(self.spider) + yield self.engine.start() except Exception: self.crawling = False if self.engine is not None: diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index ac832e025..bacee8f0a 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -104,7 +104,6 @@ class TelnetConsole(protocol.ServerFactory): telnet_vars: dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, - "slot": self.crawler.engine.slot, "crawler": self.crawler, "extensions": self.crawler.extensions, "stats": self.crawler.stats, diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 6d3b7a926..2b8d0ab84 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -130,6 +130,16 @@ class Request(object_ref): self._set_body(body) if not isinstance(priority, int): raise TypeError(f"Request priority not an integer: {priority!r}") + + #: Default: ``0`` + #: + #: Value that the :ref:`scheduler ` may use for + #: request prioritization. + #: + #: Built-in schedulers prioritize requests with a higher priority + #: value. + #: + #: Negative values are allowed. self.priority: int = priority if not (callable(callback) or callback is None): @@ -191,7 +201,7 @@ class Request(object_ref): #: #: When defining the start URLs of a spider through #: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by - #: default. See :meth:`~scrapy.Spider.start_requests`. + #: default. See :meth:`~scrapy.Spider.start`. self.dont_filter: bool = dont_filter self._meta: dict[str, Any] | None = dict(meta) if meta else None diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index f10e91beb..4f08918ae 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -98,7 +98,7 @@ class LogFormatter: """Logs a message when an item is scraped by a spider.""" src: Any if response is None: - src = f"{global_object_name(spider.__class__)}.start_requests" + src = f"{global_object_name(spider.__class__)}.start" elif isinstance(response, Failure): src = response.getErrorMessage() else: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 324a9b955..e6c6b8bf1 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -72,7 +72,6 @@ class ScrapyPriorityQueue: startprios is a sequence of priorities to start with. If the queue was previously closed leaving some priority buckets non-empty, those priorities should be passed in startprios. - """ @classmethod @@ -82,8 +81,16 @@ class ScrapyPriorityQueue: downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), + *, + start_queue_cls: type[QueueProtocol] | None = None, ) -> Self: - return cls(crawler, downstream_queue_cls, key, startprios) + return cls( + crawler, + downstream_queue_cls, + key, + startprios, + start_queue_cls=start_queue_cls, + ) def __init__( self, @@ -91,11 +98,15 @@ class ScrapyPriorityQueue: downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), + *, + start_queue_cls: type[QueueProtocol] | None = None, ): self.crawler: Crawler = crawler self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls + self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key self.queues: dict[int, QueueProtocol] = {} + self._start_queues: dict[int, QueueProtocol] = {} self.curprio: int | None = None self.init_prios(startprios) @@ -104,7 +115,13 @@ class ScrapyPriorityQueue: return for priority in startprios: - self.queues[priority] = self.qfactory(priority) + q = self.qfactory(priority) + if q: + self.queues[priority] = q + if self._start_queue_cls: + q = self._sqfactory(priority) + if q: + self._start_queues[priority] = q self.curprio = min(startprios) @@ -115,29 +132,66 @@ class ScrapyPriorityQueue: self.key + "/" + str(key), ) + def _sqfactory(self, key: int) -> QueueProtocol: + assert self._start_queue_cls is not None + return build_from_crawler( + self._start_queue_cls, + self.crawler, + f"{self.key}/{key}s", + ) + def priority(self, request: Request) -> int: return -request.priority def push(self, request: Request) -> None: priority = self.priority(request) - if priority not in self.queues: - self.queues[priority] = self.qfactory(priority) - q = self.queues[priority] + is_start_request = request.meta.get("is_start_request", False) + if is_start_request and self._start_queue_cls: + if priority not in self._start_queues: + self._start_queues[priority] = self._sqfactory(priority) + q = self._start_queues[priority] + else: + if priority not in self.queues: + self.queues[priority] = self.qfactory(priority) + q = self.queues[priority] q.push(request) # this may fail (eg. serialization error) if self.curprio is None or priority < self.curprio: self.curprio = priority def pop(self) -> Request | None: - if self.curprio is None: - return None - q = self.queues[self.curprio] - m = q.pop() - if not q: - del self.queues[self.curprio] - q.close() - prios = [p for p, q in self.queues.items() if q] - self.curprio = min(prios) if prios else None - return m + while self.curprio is not None: + if self._start_queues: + try: + q = self._start_queues[self.curprio] + except KeyError: + pass + else: + m = q.pop() + if not q: + del self._start_queues[self.curprio] + q.close() + return m + try: + q = self.queues[self.curprio] + except KeyError: + self._update_curprio() + else: + m = q.pop() + if not q: + del self.queues[self.curprio] + q.close() + self._update_curprio() + return m + return None + + def _update_curprio(self) -> None: + prios = { + p + for queues in (self.queues, self._start_queues) + for p, q in queues.items() + if q + } + self.curprio = min(prios) if prios else None def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, @@ -148,19 +202,31 @@ class ScrapyPriorityQueue: """ if self.curprio is None: return None - queue = self.queues[self.curprio] + try: + queue = self._start_queues[self.curprio] + except KeyError: + queue = self.queues[self.curprio] # Protocols can't declare optional members return cast(Request, queue.peek()) # type: ignore[attr-defined] def close(self) -> list[int]: - active: list[int] = [] - for p, q in self.queues.items(): - active.append(p) - q.close() - return active + active: set[int] = set() + for queues in (self.queues, self._start_queues): + for p, q in queues.items(): + active.add(p) + q.close() + return list(active) def __len__(self) -> int: - return sum(len(x) for x in self.queues.values()) if self.queues else 0 + return ( + sum( + len(x) + for queues in (self.queues, self._start_queues) + for x in queues.values() + ) + if self.queues or self._start_queues + else 0 + ) class DownloaderInterface: @@ -194,8 +260,16 @@ class DownloaderAwarePriorityQueue: downstream_queue_cls: type[QueueProtocol], key: str, startprios: dict[str, Iterable[int]] | None = None, + *, + start_queue_cls: type[QueueProtocol] | None = None, ) -> Self: - return cls(crawler, downstream_queue_cls, key, startprios) + return cls( + crawler, + downstream_queue_cls, + key, + startprios, + start_queue_cls=start_queue_cls, + ) def __init__( self, @@ -203,6 +277,8 @@ class DownloaderAwarePriorityQueue: downstream_queue_cls: type[QueueProtocol], key: str, slot_startprios: dict[str, Iterable[int]] | None = None, + *, + start_queue_cls: type[QueueProtocol] | None = None, ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( @@ -222,6 +298,7 @@ class DownloaderAwarePriorityQueue: self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls + self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key self.crawler: Crawler = crawler @@ -237,6 +314,7 @@ class DownloaderAwarePriorityQueue: self.downstream_queue_cls, self.key + "/" + _path_safe(slot), startprios, + start_queue_cls=self._start_queue_cls, ) def pop(self) -> Request | None: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 680fded7a..01443fa17 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -305,6 +305,8 @@ SCHEDULER = "scrapy.core.scheduler.Scheduler" SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue" SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue" SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" +SCHEDULER_START_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" +SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 @@ -315,6 +317,7 @@ SPIDER_MIDDLEWARES = {} SPIDER_MIDDLEWARES_BASE = { # Engine side + "scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25, "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, diff --git a/scrapy/shell.py b/scrapy/shell.py index 5e5e57a9a..bb39eccc3 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -24,6 +24,7 @@ from scrapy.spiders import Spider from scrapy.utils.conf import get_config from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.misc import load_object from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser @@ -102,25 +103,33 @@ class Shell: # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] set_asyncio_event_loop(event_loop_path) - spider = self._open_spider(request, spider) + + def crawl_request(_): + assert self.crawler.engine is not None + self.crawler.engine.crawl(request) + + d2 = self._open_spider(request, spider) + d2.addCallback(crawl_request) + d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) - assert self.crawler.engine - self.crawler.engine.crawl(request) return d - def _open_spider(self, request: Request, spider: Spider | None) -> Spider: + @deferred_f_from_coro_f + async def _open_spider(self, request: Request, spider: Spider | None) -> None: if self.spider: - return self.spider + return if spider is None: spider = self.crawler.spider or self.crawler._create_spider() self.crawler.spider = spider assert self.crawler.engine - self.crawler.engine.open_spider(spider, close_if_idle=False) + await maybe_deferred_to_future( + self.crawler.engine.open_spider(spider, close_if_idle=False) + ) + self.crawler.engine._start_request_processing() self.spider = spider - return spider def fetch( self, diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index e106418d6..f8c50b5e3 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,13 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import Any from pydispatch import dispatcher +from twisted.internet.defer import Deferred from scrapy.utils import signal as _signal - -if TYPE_CHECKING: - from twisted.internet.defer import Deferred +from scrapy.utils.defer import maybe_deferred_to_future class SignalManager: @@ -75,3 +74,17 @@ class SignalManager: """ kwargs.setdefault("sender", self.sender) _signal.disconnect_all(signal, **kwargs) + + async def wait_for(self, signal): + """Await the next *signal*. + + See :ref:`start-requests-lazy` for an example. + """ + d = Deferred() + + def handle(): + self.disconnect(handle, signal) + d.callback(None) + + self.connect(handle, signal) + await maybe_deferred_to_future(d) diff --git a/scrapy/signals.py b/scrapy/signals.py index 8ef0f34f0..bdeec1ba0 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -7,6 +7,7 @@ signals here without documenting them there. engine_started = object() engine_stopped = object() +scheduler_empty = object() spider_opened = object() spider_idle = object() spider_closed = object() diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index 650192095..cfb50c599 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable + from collections.abc import AsyncIterator, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -17,9 +17,9 @@ if TYPE_CHECKING: class BaseSpiderMiddleware: """Optional base class for spider middlewares. - This class provides helper methods for asynchronous ``process_spider_output`` - methods. Middlewares that don't have a ``process_spider_output`` method don't need - to use it. + This class provides helper methods for asynchronous + ``process_spider_output()`` and ``process_start()`` methods. Middlewares + that don't have either of these methods don't need to use this class. You can override the :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_request` @@ -38,59 +38,70 @@ class BaseSpiderMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) + def process_start_requests( + self, start: Iterable[Any], spider: Spider + ) -> Iterable[Any]: + for o in start: + if (o := self._get_processed(o, None)) is not None: + yield o + + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: + async for o in start: + if (o := self._get_processed(o, None)) is not None: + yield o + def process_spider_output( self, response: Response, result: Iterable[Any], spider: Spider ) -> Iterable[Any]: for o in result: - if isinstance(o, Request): - o = self.get_processed_request(o, response) - else: - o = self.get_processed_item(o, response) - if o is not None: + if (o := self._get_processed(o, response)) is not None: yield o async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: + self, response: Response, result: AsyncIterator[Any], spider: Spider + ) -> AsyncIterator[Any]: async for o in result: - if isinstance(o, Request): - o = self.get_processed_request(o, response) - else: - o = self.get_processed_item(o, response) - if o is not None: + if (o := self._get_processed(o, response)) is not None: yield o + def _get_processed(self, o: Any, response: Response | None) -> Any: + if isinstance(o, Request): + return self.get_processed_request(o, response) + return self.get_processed_item(o, response) + def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: """Return a processed request from the spider output. - This method is called with a single request from the spider output. - It should return the same or a different request, or ``None`` to - ignore it. + This method is called with a single request from the start seeds or the + spider output. It should return the same or a different request, or + ``None`` to ignore it. :param request: the input request :type request: :class:`~scrapy.Request` object :param response: the response being processed - :type response: :class:`~scrapy.http.Response` object + :type response: :class:`~scrapy.http.Response` object or ``None`` for + start seeds :return: the processed request or ``None`` """ return request - def get_processed_item(self, item: Any, response: Response) -> Any: + def get_processed_item(self, item: Any, response: Response | None) -> Any: """Return a processed item from the spider output. - This method is called with a single item from the spider output. - It should return the same or a different item, or ``None`` to - ignore it. + This method is called with a single item from the start seeds or the + spider output. It should return the same or a different item, or + ``None`` to ignore it. :param item: the input item :type item: item object :param response: the response being processed - :type response: :class:`~scrapy.http.Response` object + :type response: :class:`~scrapy.http.Response` object or ``None`` for + start seeds :return: the processed item or ``None`` """ diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 65905f483..6b115ebe6 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Any from scrapy.spidermiddlewares.base import BaseSpiderMiddleware if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable + from collections.abc import AsyncIterator, Iterable # typing.Self requires Python 3.11 from typing_extensions import Self @@ -59,8 +59,8 @@ class DepthMiddleware(BaseSpiderMiddleware): yield from super().process_spider_output(response, result, spider) async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: + self, response: Response, result: AsyncIterator[Any], spider: Spider + ) -> AsyncIterator[Any]: self._init_depth(response, spider) async for o in super().process_spider_output_async(response, result, spider): yield o @@ -73,8 +73,11 @@ class DepthMiddleware(BaseSpiderMiddleware): self.stats.inc_value("request_depth_count/0", spider=spider) def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: + if response is None: + # start requests + return request depth = response.meta["depth"] + 1 request.meta["depth"] = depth if self.prio: diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 0918c9fac..2463275d5 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -49,8 +49,11 @@ class OffsiteMiddleware(BaseSpiderMiddleware): return o def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: + if response is None: + # skip start requests for backward compatibility + return request assert self.crawler.spider if ( request.dont_filter diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index b2ba8ba8c..f5d406c13 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -370,8 +370,11 @@ class RefererMiddleware(BaseSpiderMiddleware): return cls() if cls else self.default_policy() def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: + if response is None: + # start requests + return request referrer = self.policy(response, request).referrer(response.url, request.url) if referrer is not None: request.headers.setdefault("Referer", referrer) diff --git a/scrapy/spidermiddlewares/start.py b/scrapy/spidermiddlewares/start.py new file mode 100644 index 000000000..5d76b60d2 --- /dev/null +++ b/scrapy/spidermiddlewares/start.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .base import BaseSpiderMiddleware + +if TYPE_CHECKING: + from scrapy.http import Request + from scrapy.http.response import Response + + +class StartSpiderMiddleware(BaseSpiderMiddleware): + """Set :reqmeta:`is_start_request`. + + .. reqmeta:: is_start_request + + is_start_request + ---------------- + + :attr:`~scrapy.Request.meta` key that is set to ``True`` in :ref:`start + requests `, allowing you to tell start requests apart from + other requests, e.g. in :ref:`downloader middlewares + `. + """ + + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if response is None: + request.meta.setdefault("is_start_request", True) + return request diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 177c19e1b..5590165a5 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -39,7 +39,7 @@ class UrlLengthMiddleware(BaseSpiderMiddleware): return o def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: if len(request.url) <= self.maxlength: return request diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e255e91cc..0a1d85ae6 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,15 +7,17 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging +import warnings from typing import TYPE_CHECKING, Any, cast from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import AsyncIterator, Iterable from twisted.internet.defer import Deferred @@ -29,13 +31,19 @@ if TYPE_CHECKING: class Spider(object_ref): - """Base class for scrapy spiders. All spiders must inherit from this - class. + """Base class that any spider must subclass. + + It provides a default :meth:`start` implementation that sends + requests based on the :attr:`start_urls` class attribute and calls the + :meth:`parse` method for each response. """ name: str custom_settings: dict[_SettingsKeyT, Any] | None = None + #: Start URLs. See :meth:`start`. + start_urls: list[str] + def __init__(self, name: str | None = None, **kwargs: Any): if name is not None: self.name: str = name @@ -72,7 +80,70 @@ class Spider(object_ref): self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - def start_requests(self) -> Iterable[Request]: + async def start(self) -> AsyncIterator[Any]: + """Yield the initial :class:`~scrapy.Request` objects to send. + + .. versionadded:: VERSION + + For example: + + .. code-block:: python + + from scrapy import Request, Spider + + + class MySpider(Spider): + name = "myspider" + + async def start(self): + yield Request("https://toscrape.com/") + + The default implementation reads URLs from :attr:`start_urls` and + yields a request for each with :attr:`~scrapy.Request.dont_filter` + enabled. It is functionally equivalent to: + + .. code-block:: python + + async def start(self): + for url in self.start_urls: + yield Request(url, dont_filter=True) + + You can also yield :ref:`items `. For example: + + .. code-block:: python + + async def start(self): + yield {"foo": "bar"} + + To write spiders that work on Scrapy versions lower than VERSION, + define also a synchronous ``start_requests()`` method that returns an + iterable. For example: + + .. code-block:: python + + def start_requests(self): + yield Request("https://toscrape.com/") + + .. seealso:: :ref:`start-requests` + """ + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" + ) + for item_or_request in self.start_requests(): + yield item_or_request + + def start_requests(self) -> Iterable[Any]: + warnings.warn( + ( + "The Spider.start_requests() method is deprecated, use " + "Spider.start() instead. If you are calling " + "super().start_requests() from a Spider.start() override, " + "iterate super().start() instead." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 087049425..171d8479c 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,7 +8,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import copy -from collections.abc import AsyncIterable, Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast from twisted.python.failure import Failure @@ -156,10 +156,10 @@ class CrawlSpider(Spider): callback: CallbackT | None, cb_kwargs: dict[str, Any], follow: bool = True, - ) -> AsyncIterable[Any]: + ) -> AsyncIterator[Any]: if callback: cb_res = callback(response, **cb_kwargs) or () - if isinstance(cb_res, AsyncIterable): + if isinstance(cb_res, AsyncIterator): cb_res = await collect_asyncgen(cb_res) elif isinstance(cb_res, Awaitable): cb_res = await cb_res diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index a7dba989e..e5548b9fa 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from collections.abc import Iterable +from collections.abc import AsyncIterator, Iterable from typing import TYPE_CHECKING, Any, cast from scrapy import Request @@ -29,6 +29,14 @@ class InitSpider(Spider): stacklevel=2, ) + async def start(self) -> AsyncIterator[Any]: + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" + ) + for item_or_request in self.start_requests(): + yield item_or_request + def start_requests(self) -> Iterable[Request]: self._postinit_reqs: Iterable[Request] = super().start_requests() return cast(Iterable[Request], iterate_spider_output(self.init_request())) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 39033ac3c..2813a32a0 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -4,7 +4,7 @@ import logging import re # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation -from collections.abc import Iterable, Sequence # noqa: TC003 +from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse @@ -53,6 +53,10 @@ class SitemapSpider(Spider): self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] + async def start(self) -> AsyncIterator[Any]: + for item_or_request in self.start_requests(): + yield item_or_request + def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index dcb2d63de..3f0239832 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -43,14 +43,11 @@ class ${ProjectName}SpiderMiddleware: # Should return either None or an iterable of Request or item objects. pass - def process_start_requests(self, start_requests, spider): - # Called with the start requests of the spider, and works - # similarly to the process_spider_output() method, except - # that it doesn’t have a response associated. - - # Must return only requests (not items). - for r in start_requests: - yield r + async def process_start(self, start): + # Called with an async iterator over the spider start() method or the + # maching method of an earlier spider middleware. + async for item_or_request in start: + yield item_or_request def spider_opened(self, spider): spider.logger.info("Spider opened: %s" % spider.name) diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 237bd8331..6d96a41f5 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,20 +1,20 @@ from __future__ import annotations -from collections.abc import AsyncGenerator, AsyncIterable, Iterable +from collections.abc import AsyncGenerator, AsyncIterator, Iterable from typing import TypeVar _T = TypeVar("_T") -async def collect_asyncgen(result: AsyncIterable[_T]) -> list[_T]: +async def collect_asyncgen(result: AsyncIterator[_T]) -> list[_T]: return [x async for x in result] async def as_async_generator( - it: Iterable[_T] | AsyncIterable[_T], + it: Iterable[_T] | AsyncIterator[_T], ) -> AsyncGenerator[_T]: """Wraps an iterable (sync or async) into an async generator.""" - if isinstance(it, AsyncIterable): + if isinstance(it, AsyncIterator): async for r in it: yield r else: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 42ad28d8d..6e1687f3e 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -14,7 +14,11 @@ from types import CoroutineType from typing import TYPE_CHECKING, Any, Generic, TypeVar, Union, cast, overload from twisted.internet import defer -from twisted.internet.defer import Deferred, DeferredList, ensureDeferred +from twisted.internet.defer import ( + Deferred, + DeferredList, + ensureDeferred, +) from twisted.internet.task import Cooperator from twisted.python import failure @@ -22,7 +26,7 @@ from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed if TYPE_CHECKING: - from collections.abc import AsyncIterable, AsyncIterator, Callable + from collections.abc import AsyncIterator, Callable from twisted.python.failure import Failure @@ -36,6 +40,9 @@ _T = TypeVar("_T") _T2 = TypeVar("_T2") +_DEFER_DELAY = 0.1 + + def defer_fail(_failure: Failure) -> Deferred[Any]: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -46,7 +53,7 @@ def defer_fail(_failure: Failure) -> Deferred[Any]: from twisted.internet import reactor d: Deferred[Any] = Deferred() - reactor.callLater(0.1, d.errback, _failure) + reactor.callLater(_DEFER_DELAY, d.errback, _failure) return d @@ -60,7 +67,16 @@ def defer_succeed(result: _T) -> Deferred[_T]: from twisted.internet import reactor d: Deferred[_T] = Deferred() - reactor.callLater(0.1, d.callback, result) + reactor.callLater(_DEFER_DELAY, d.callback, result) + return d + + +def _defer_sleep() -> Deferred[None]: + """Like ``defer_succeed`` and ``defer_fail`` but doesn't call any real callbacks.""" + from twisted.internet import reactor + + d: Deferred[None] = Deferred() + reactor.callLater(_DEFER_DELAY, d.callback, None) return d @@ -177,7 +193,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def __init__( self, - aiterable: AsyncIterable[_T], + aiterable: AsyncIterator[_T], callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], *callable_args: _P.args, **callable_kwargs: _P.kwargs, @@ -234,7 +250,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def parallel_async( - async_iterable: AsyncIterable[_T], + async_iterable: AsyncIterator[_T], count: int, callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], *args: _P.args, @@ -332,13 +348,13 @@ def iter_errback( async def aiter_errback( - aiterable: AsyncIterable[_T], + aiterable: AsyncIterator[_T], errback: Callable[Concatenate[Failure, _P], Any], *a: _P.args, **kw: _P.kwargs, -) -> AsyncIterable[_T]: +) -> AsyncIterator[_T]: """Wraps an async iterable calling an errback if an error is caught while - iterating it. Similar to scrapy.utils.defer.iter_errback() + iterating it. Similar to :func:`scrapy.utils.defer.iter_errback`. """ it = aiterable.__aiter__() while True: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 52f29e22c..1e0c53212 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -18,10 +18,10 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]: "engine.scraper.is_idle()", "engine.spider.name", "engine.spider_is_idle()", - "engine.slot.closing", - "len(engine.slot.inprogress)", - "len(engine.slot.scheduler.dqs or [])", - "len(engine.slot.scheduler.mqs)", + "engine._slot.closing", + "len(engine._slot.inprogress)", + "len(engine._slot.scheduler.dqs or [])", + "len(engine._slot.scheduler.mqs)", "len(engine.scraper.slot.queue)", "len(engine.scraper.slot.active)", "engine.scraper.slot.active_size", diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 2e6869779..c859fbc2a 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -10,7 +10,7 @@ import re import sys import warnings import weakref -from collections.abc import AsyncIterable, Iterable, Mapping +from collections.abc import AsyncIterator, Iterable, Mapping from functools import partial, wraps from itertools import chain from typing import TYPE_CHECKING, Any, TypeVar, overload @@ -19,11 +19,12 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Iterator + from collections.abc import Callable, Iterator from re import Pattern # typing.Concatenate and typing.ParamSpec require Python 3.10 - from typing_extensions import Concatenate, ParamSpec + # typing.Self requires Python 3.11 + from typing_extensions import Concatenate, ParamSpec, Self _P = ParamSpec("_P") @@ -369,25 +370,25 @@ class MutableChain(Iterable[_T]): async def _async_chain( - *iterables: Iterable[_T] | AsyncIterable[_T], + *iterables: Iterable[_T] | AsyncIterator[_T], ) -> AsyncIterator[_T]: for it in iterables: async for o in as_async_generator(it): yield o -class MutableAsyncChain(AsyncIterable[_T]): +class MutableAsyncChain(AsyncIterator[_T]): """ Similar to MutableChain but for async iterables """ - def __init__(self, *args: Iterable[_T] | AsyncIterable[_T]): + def __init__(self, *args: Iterable[_T] | AsyncIterator[_T]): self.data: AsyncIterator[_T] = _async_chain(*args) - def extend(self, *iterables: Iterable[_T] | AsyncIterable[_T]) -> None: + def extend(self, *iterables: Iterable[_T] | AsyncIterator[_T]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) - def __aiter__(self) -> AsyncIterator[_T]: + def __aiter__(self) -> Self: return self async def __anext__(self) -> _T: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 099c81f0e..9c2754394 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar from warnings import catch_warnings, filterwarnings from twisted.internet import asyncioreactor, error +from twisted.internet.defer import Deferred from scrapy.utils.misc import load_object @@ -54,6 +55,7 @@ class CallLaterOnce(Generic[_T]): self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw self._call: DelayedCall | None = None + self._deferreds: list[Deferred] = [] def schedule(self, delay: float = 0) -> None: from twisted.internet import reactor @@ -66,8 +68,23 @@ class CallLaterOnce(Generic[_T]): self._call.cancel() def __call__(self) -> _T: + from twisted.internet import reactor + self._call = None - return self._func(*self._a, **self._kw) + result = self._func(*self._a, **self._kw) + + for d in self._deferreds: + reactor.callLater(0, d.callback, None) + self._deferreds = [] + + return result + + async def wait(self): + from scrapy.utils.defer import maybe_deferred_to_future + + d = Deferred() + self._deferreds.append(d) + await maybe_deferred_to_future(d) def set_asyncio_event_loop_policy() -> None: @@ -114,8 +131,10 @@ def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: event_loop_class: type[AbstractEventLoop] = load_object(event_loop_path) - event_loop = event_loop_class() - asyncio.set_event_loop(event_loop) + event_loop = _get_asyncio_event_loop() + if not isinstance(event_loop, event_loop_class): + event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) else: try: with catch_warnings(): diff --git a/sep/sep-018.rst b/sep/sep-018.rst index e6d601fe1..29b1f860e 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -619,7 +619,7 @@ Resolved: ``manager.scraper.process_request()`` instead of ``manager.engine.crawl()`` - should we support adding additional start requests from a spider middleware? - - Yes - there is a spider middleware method (``start_requests``) for that + - Yes - there is a spider middleware method (``start_requests()``) for that - should ``process_response()`` receive a ``request`` argument with the ``request`` that originated it?. ``response.request`` is the latest request, not the original one (think of redirections), but it does carry the ``meta`` diff --git a/tests/CrawlerProcess/args_settings.py b/tests/CrawlerProcess/args_settings.py index a46a8806b..c8a3d0a5b 100644 --- a/tests/CrawlerProcess/args_settings.py +++ b/tests/CrawlerProcess/args_settings.py @@ -13,9 +13,10 @@ class NoRequestsSpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - def start_requests(self): + async def start(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return [] + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py index 5e72aa6d4..bd78a0de7 100644 --- a/tests/CrawlerProcess/asyncio_custom_loop.py +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -5,8 +5,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index 6f82cf589..6bb6fb3c6 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -12,8 +12,9 @@ class ReactorCheckExtension: class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index a8bf1bc3c..f3dab12fe 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -38,8 +38,9 @@ class ReactorCheckExtension: class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index e9d6d8875..d8c467f40 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -15,8 +15,9 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index c72a0a17c..e7d3ca9cc 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -16,8 +16,9 @@ from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py index 7b0497bde..53d427061 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver.py +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -11,7 +11,7 @@ class CachingHostnameResolverSpider(scrapy.Spider): name = "caching_hostname_resolver_spider" - def start_requests(self): + async def start(self): yield scrapy.Request(self.url) def parse(self, response): diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py index 9f7eaf2ae..0058896b5 100644 --- a/tests/CrawlerProcess/multi.py +++ b/tests/CrawlerProcess/multi.py @@ -5,8 +5,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index e2933338b..8f59c035c 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -8,8 +8,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index eee808c32..9901dd634 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -8,8 +8,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index b61e52625..53941568a 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -10,8 +10,9 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 38ca4c4f1..5739d77ae 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -17,8 +17,9 @@ installReactor(reactor) class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index b397608ec..c488f7526 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -9,8 +9,9 @@ selectreactor.install() class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess( diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py index 2d8769501..9e4ad70d9 100644 --- a/tests/CrawlerProcess/simple.py +++ b/tests/CrawlerProcess/simple.py @@ -5,8 +5,9 @@ from scrapy.crawler import CrawlerProcess class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield process = CrawlerProcess(settings={}) diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py index de76e13e8..6c0102241 100644 --- a/tests/CrawlerRunner/change_reactor.py +++ b/tests/CrawlerRunner/change_reactor.py @@ -10,8 +10,9 @@ class NoRequestsSpider(Spider): "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } - def start_requests(self): - return [] + async def start(self): + return + yield configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"}) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index b1b297777..5e2184afb 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -32,7 +32,7 @@ def createResolver(servers=None, resolvconf=None, hosts=None): class LocalhostSpider(Spider): name = "localhost_spider" - def start_requests(self): + async def start(self): yield Request(self.url) def parse(self, response): diff --git a/tests/__init__.py b/tests/__init__.py index cd52ade58..ccfabb0da 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,6 +8,9 @@ import os import socket from pathlib import Path +from twisted import version as TWISTED_VERSION +from twisted.python.versions import Version + # ignore system-wide proxies for tests # which would send requests to a totally unsuspecting server # (e.g. because urllib does not fully understand the proxy spec) @@ -30,3 +33,6 @@ except socket.gaierror: def get_testdata(*paths: str) -> bytes: """Return test data""" return Path(tests_datadir, *paths).read_bytes() + + +TWISTED_KEEPS_TRACEBACKS = TWISTED_VERSION >= Version("twisted", 24, 10, 0) diff --git a/tests/spiders.py b/tests/spiders.py index da923de6e..c47f2bd2b 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -68,7 +68,7 @@ class DelaySpider(MetaSpider): self.b = b self.t1 = self.t2 = self.t2_err = 0 - def start_requests(self): + async def start(self): self.t1 = time.time() url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) @@ -105,7 +105,7 @@ class LogSpider(MetaSpider): class SlowSpider(DelaySpider): name = "slow" - def start_requests(self): + async def start(self): # 1st response is fast url = self.mockserver.url("/delay?n=0&b=0") yield Request(url, callback=self.parse, errback=self.errback) @@ -255,7 +255,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): callback=cb, ) - def start_requests(self): + async def start(self): for i in range(1, self.initial_reqs + 1): yield self._get_req(i) @@ -319,7 +319,7 @@ class ErrorSpider(FollowAllSpider): self.raise_exception() -class BrokenStartRequestsSpider(FollowAllSpider): +class BrokenStartSpider(FollowAllSpider): fail_before_yield = False fail_yielding = False @@ -327,7 +327,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): super().__init__(*a, **kw) self.seedsseen = [] - def start_requests(self): + async def start(self): if self.fail_before_yield: 1 / 0 @@ -338,22 +338,20 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_yielding: 2 / 0 - assert self.seedsseen, ( - "All start requests consumed before any download happened" - ) + assert self.seedsseen, "All seeds consumed before any download happened" def parse(self, response): self.seedsseen.append(response.meta.get("seed")) yield from super().parse(response) -class StartRequestsItemSpider(FollowAllSpider): - def start_requests(self): +class StartItemSpider(FollowAllSpider): + async def start(self): yield {"name": "test item"} -class StartRequestsGoodAndBadOutput(FollowAllSpider): - def start_requests(self): +class StartGoodAndBadOutput(FollowAllSpider): + async def start(self): yield {"a": "a"} yield Request("data:,a") yield "data:,b" @@ -365,7 +363,7 @@ class SingleRequestSpider(MetaSpider): callback_func = None errback_func = None - def start_requests(self): + async def start(self): if isinstance(self.seed, Request): yield self.seed.replace(callback=self.parse, errback=self.on_error) else: @@ -386,13 +384,13 @@ class SingleRequestSpider(MetaSpider): return None -class DuplicateStartRequestsSpider(MockServerSpider): +class DuplicateStartSpider(MockServerSpider): dont_filter = True name = "duplicatestartrequests" distinct_urls = 2 dupe_factor = 3 - def start_requests(self): + async def start(self): for i in range(self.distinct_urls): for j in range(self.dupe_factor): url = self.mockserver.url(f"/echo?headers=1&body=test{i}") @@ -417,7 +415,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): } rules = (Rule(LinkExtractor(), callback="parse", follow=True),) - def start_requests(self): + async def start(self): test_body = b""" Page title<title></head> @@ -471,7 +469,7 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = "crawl_spider_with_errback" rules = (Rule(LinkExtractor(), callback="parse", errback="errback", follow=True),) - def start_requests(self): + async def start(self): test_body = b""" <html> <head><title>Page title<title></head> @@ -516,7 +514,7 @@ class BytesReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.bytes_received, signals.bytes_received) return spider - def start_requests(self): + async def start(self): body = b"a" * self.full_response_length url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=body, errback=self.errback) @@ -545,7 +543,7 @@ class HeadersReceivedCallbackSpider(MetaSpider): crawler.signals.connect(spider.headers_received, signals.headers_received) return spider - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status"), errback=self.errback) def parse(self, response): diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index 5228f6abd..5006e3689 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -2,17 +2,26 @@ import sys from pathlib import Path from subprocess import PIPE, Popen +from .. import TWISTED_KEEPS_TRACEBACKS + class TestCmdlineCrawlPipeline: def _execute(self, spname): args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname) cwd = Path(__file__).resolve().parent proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd) - proc.communicate() - return proc.returncode + _, stderr = proc.communicate() + return proc.returncode, stderr def test_open_spider_normally_in_pipeline(self): - assert self._execute("normal") == 0 + returncode, stderr = self._execute("normal") + assert returncode == 0 def test_exception_at_open_spider_in_pipeline(self): - assert self._execute("exception") == 1 + returncode, stderr = self._execute("exception") + # An unhandled exception in a pipeline should not stop the crawl + assert returncode == 0 + if TWISTED_KEEPS_TRACEBACKS: + assert b'RuntimeError("exception")' in stderr + else: + assert b"RuntimeError: exception" in stderr diff --git a/tests/test_commands.py b/tests/test_commands.py index f63e05628..16af97842 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -670,9 +670,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug("It Works!") - return [] + return + yield """ badspider = """ @@ -680,8 +681,9 @@ import scrapy class BadSpider(scrapy.Spider): name = "bad" - def start_requests(self): + async def start(self): raise Exception("oops!") + yield """ @contextmanager @@ -771,10 +773,10 @@ class MySpider(scrapy.Spider): log = self.get_log("", name="myspider.txt") assert "Unable to load" in log - def test_start_requests_errors(self): + def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.py") - assert "start_requests" in log - assert "badspider.py" in log + assert "start" in log + assert "badspider.py" in log, log def test_asyncio_enabled_true(self): log = self.get_log( @@ -846,9 +848,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] + return + yield """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) @@ -862,13 +865,14 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug( 'FEEDS: {}'.format( json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) ) ) - return [] + return + yield """ Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ["-O", "example.json"] @@ -888,8 +892,9 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): - return [] + async def start(self): + return + yield """ args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) @@ -904,9 +909,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] + return + yield """ args = ["-o", "-:json"] log = self.get_log(spider_code, args=args) @@ -983,9 +989,10 @@ class MySpider(scrapy.Spider): spider.settings.set("FOO", kwargs.get("foo")) return spider - def start_requests(self): + async def start(self): self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return [] + return + yield """ args = ["-a", "foo=42"] log = self.get_log(spider_code, args=args) @@ -1001,9 +1008,9 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand): raise unittest.SkipTest("Windows required for .pyw files") return super().setUp() - def test_start_requests_errors(self): + def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") - assert "start_requests" in log + assert "start" in log assert "badspider.pyw" in log def test_runspider_unable_to_load(self): @@ -1053,9 +1060,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug('It works!') - return [] + return + yield """ log = self.get_log(spider_code) assert "[myspider] DEBUG: It works!" in log @@ -1067,9 +1075,10 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return [] + return + yield """ args = ["-o", "example.json"] log = self.get_log(spider_code, args=args) @@ -1083,13 +1092,14 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): + async def start(self): self.logger.debug( 'FEEDS: {}'.format( json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) ) ) - return [] + return + yield """ Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ["-O", "example.json"] @@ -1109,8 +1119,9 @@ import scrapy class MySpider(scrapy.Spider): name = 'myspider' - def start_requests(self): - return [] + async def start(self): + return + yield """ args = ["-o", "example1.json", "-O", "example2.json"] log = self.get_log(spider_code, args=args) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index fb961ace2..26b16a1d4 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -511,8 +511,9 @@ class TestContractsManager(unittest.TestCase): super().__init__(*args, **kwargs) self.visited = 0 - def start_requests(self_): # pylint: disable=no-self-argument - return self.conman.from_spider(self_, self.results) + async def start(self_): # pylint: disable=no-self-argument + for item_or_request in self.conman.from_spider(self_, self.results): + yield item_or_request def parse_first(self, response): self.visited += 1 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b7a8a9628..b90706027 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -36,7 +36,7 @@ from tests.spiders import ( AsyncDefDeferredMaybeWrappedSpider, AsyncDefDeferredWrappedSpider, AsyncDefSpider, - BrokenStartRequestsSpider, + BrokenStartSpider, BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, @@ -45,14 +45,14 @@ from tests.spiders import ( CrawlSpiderWithParseMethod, CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, - DuplicateStartRequestsSpider, + DuplicateStartSpider, FollowAllSpider, HeadersReceivedCallbackSpider, HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, - StartRequestsGoodAndBadOutput, - StartRequestsItemSpider, + StartGoodAndBadOutput, + StartItemSpider, ) @@ -165,9 +165,9 @@ class TestCrawl(TestCase): self._assert_retried(log) @defer.inlineCallbacks - def test_start_requests_bug_before_yield(self): + def test_start_bug_before_yield(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenStartSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) assert len(log.records) == 1 @@ -176,9 +176,9 @@ class TestCrawl(TestCase): assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks - def test_start_requests_bug_yielding(self): + def test_start_bug_yielding(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenStartSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) assert len(log.records) == 1 @@ -187,14 +187,14 @@ class TestCrawl(TestCase): assert record.exc_info[0] is ZeroDivisionError @defer.inlineCallbacks - def test_start_requests_items(self): + def test_start_items(self): items = [] def _on_item_scraped(item): items.append(item) with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(StartRequestsItemSpider) + crawler = get_crawler(StartItemSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) @@ -202,11 +202,11 @@ class TestCrawl(TestCase): assert items == [{"name": "test item"}] @defer.inlineCallbacks - def test_start_requests_unsupported_output(self): + def test_start_unsupported_output(self): """Anything that is not a request is assumed to be an item, avoiding a - potentially expensive call to itemadapter.is_item, and letting instead - things fail when ItemAdapter is actually used on the corresponding - non-item object.""" + potentially expensive call to itemadapter.is_item(), and letting + instead things fail when ItemAdapter is actually used on the + corresponding non-item object.""" items = [] @@ -214,7 +214,7 @@ class TestCrawl(TestCase): items.append(item) with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(StartRequestsGoodAndBadOutput) + crawler = get_crawler(StartGoodAndBadOutput) crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) @@ -223,24 +223,15 @@ class TestCrawl(TestCase): assert not any(isinstance(item, Request) for item in items) @defer.inlineCallbacks - def test_start_requests_laziness(self): + def test_start_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(BrokenStartRequestsSpider, settings) - yield crawler.crawl(mockserver=self.mockserver) - assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index( - 99 - ), crawler.spider.seedsseen - - @defer.inlineCallbacks - def test_start_requests_dupes(self): - settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(DuplicateStartRequestsSpider, settings) + crawler = get_crawler(DuplicateStartSpider, settings) yield crawler.crawl( dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver ) assert crawler.spider.visited == 6 - crawler = get_crawler(DuplicateStartRequestsSpider, settings) + crawler = get_crawler(DuplicateStartSpider, settings) yield crawler.crawl( dont_filter=False, distinct_urls=3, @@ -322,10 +313,10 @@ with multiples lines # basic asserts in case of weird communication errors assert "responses" in crawler.spider.meta assert "failures" not in crawler.spider.meta - # start requests doesn't set Referer header + # start() doesn't set Referer header echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) assert "Referer" not in echo0["headers"] - # following request sets Referer to start request url + # following request sets Referer to the source request url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) assert echo1["headers"].get("Referer") == [req0.url] # next request avoids Referer header @@ -378,27 +369,6 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" - @defer.inlineCallbacks - def test_graceful_crawl_error_handling(self): - """ - Test whether errors happening anywhere in Crawler.crawl() are properly - reported (and not somehow swallowed) after a graceful engine shutdown. - The errors should not come from within Scrapy's core but from within - spiders/middlewares/etc., e.g. raised in Spider.start_requests(), - SpiderMiddleware.process_start_requests(), etc. - """ - - class TestError(Exception): - pass - - class FaultySpider(SimpleSpider): - def start_requests(self): - raise TestError - - crawler = get_crawler(FaultySpider) - yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) - assert not crawler.crawling - @defer.inlineCallbacks def test_open_spider_error_on_faulty_pipeline(self): settings = { diff --git a/tests/test_crawler.py b/tests/test_crawler.py index efb346dde..7a3d562e5 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -153,7 +153,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_downloader_middleware(MySpider.cls) return yield @@ -233,7 +233,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_extension(MySpider.cls) return yield @@ -313,7 +313,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_item_pipeline(MySpider.cls) return yield @@ -393,7 +393,7 @@ class TestCrawler(TestBaseCrawler): super().__init__(**kwargs) self.crawler = crawler - def start_requests(self): + async def start(self): MySpider.result = crawler.get_spider_middleware(MySpider.cls) return yield @@ -580,8 +580,9 @@ class ExceptionSpider(scrapy.Spider): class NoRequestsSpider(scrapy.Spider): name = "no_request" - def start_requests(self): - return [] + async def start(self): + return + yield @pytest.mark.usefixtures("reactor_pytest") diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 408160ccb..8ae160f8a 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -25,7 +25,7 @@ class TestManagerBase(TestCase): self.spider = self.crawler._create_spider("foo") self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) self.crawler.engine = self.crawler._create_engine() - return self.crawler.engine.open_spider(self.spider, start_requests=()) + return self.crawler.engine.open_spider(self.spider) def tearDown(self): return self.crawler.engine.close_spider(self.spider) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 4fca9eefb..78c83ea83 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -28,7 +28,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): }, } - def start_requests(self): + async def start(self): self.times = {None: []} slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None] diff --git a/tests/test_engine.py b/tests/test_engine.py index 8928e4daf..b60b510b2 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -29,7 +29,7 @@ from twisted.trial import unittest from twisted.web import server, static, util from scrapy import signals -from scrapy.core.engine import ExecutionEngine, Slot +from scrapy.core.engine import ExecutionEngine, _Slot from scrapy.core.scheduler import BaseScheduler from scrapy.exceptions import CloseSpider, IgnoreRequest from scrapy.http import Request @@ -92,8 +92,9 @@ class MySpider(Spider): class DupeFilterSpider(MySpider): - def start_requests(self): - return (Request(url) for url in self.start_urls) # no dont_filter=True + async def start(self): + for url in self.start_urls: + yield Request(url) # no dont_filter=True class DictItemsSpider(MySpider): @@ -149,7 +150,6 @@ class CrawlerRun: """A class to run the crawler and keep track of events occurred""" def __init__(self, spider_class): - self.spider = None self.respplug = [] self.reqplug = [] self.reqdropped = [] @@ -190,7 +190,6 @@ class CrawlerRun: self.response_downloaded, signals.response_downloaded ) self.crawler.crawl(start_urls=start_urls) - self.spider = self.crawler.spider self.deferred = defer.Deferred() dispatcher.connect(self.stop, signals.engine_stopped) @@ -296,7 +295,7 @@ class TestEngineBase(unittest.TestCase): assert len(run.itemerror) == 2 for item, response, spider, failure in run.itemerror: assert failure.value.__class__ is ZeroDivisionError - assert spider == run.spider + assert spider == run.crawler.spider assert item["url"] == response.url if "item1.html" in item["url"]: @@ -377,11 +376,14 @@ class TestEngineBase(unittest.TestCase): assert signals.spider_closed in run.signals_caught assert signals.headers_received in run.signals_caught - assert {"spider": run.spider} == run.signals_caught[signals.spider_opened] - assert {"spider": run.spider} == run.signals_caught[signals.spider_idle] - assert {"spider": run.spider, "reason": "finished"} == run.signals_caught[ - signals.spider_closed + assert {"spider": run.crawler.spider} == run.signals_caught[ + signals.spider_opened ] + assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle] + assert { + "spider": run.crawler.spider, + "reason": "finished", + } == run.signals_caught[signals.spider_closed] class TestEngine(TestEngineBase): @@ -419,9 +421,10 @@ class TestEngine(TestEngineBase): def test_crawler_change_close_reason_on_idle(self): run = CrawlerRun(ChangeCloseReasonSpider) yield run.run() - assert {"spider": run.spider, "reason": "custom_reason"} == run.signals_caught[ - signals.spider_closed - ] + assert { + "spider": run.crawler.spider, + "reason": "custom_reason", + } == run.signals_caught[signals.spider_closed] @defer.inlineCallbacks def test_close_downloader(self): @@ -471,7 +474,7 @@ class TestEngine(TestEngineBase): finally: timer.cancel() - assert b"Traceback" not in stderr + assert b"Traceback" not in stderr, stderr def test_request_scheduled_signal(caplog): @@ -491,7 +494,13 @@ def test_request_scheduled_signal(caplog): engine = ExecutionEngine(crawler, lambda _: None) engine.downloader._slot_gc_loop.stop() scheduler = TestScheduler() - engine.slot = Slot((), None, Mock(), scheduler) + + async def start(): + return + yield + + engine._start = start() + engine._slot = _Slot(False, Mock(), scheduler) crawler.signals.connect(signal_handler, request_scheduled) keep_request = Request("https://keep.example") engine._schedule_request(keep_request) diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py new file mode 100644 index 000000000..90af10f0e --- /dev/null +++ b/tests/test_engine_loop.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +from collections import deque +from logging import ERROR +from typing import TYPE_CHECKING + +from testfixtures import LogCapture +from twisted.internet.defer import Deferred +from twisted.trial.unittest import TestCase + +from scrapy import Request, Spider, signals +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.test import get_crawler + +from .mockserver import MockServer +from .test_scheduler import MemoryScheduler + +if TYPE_CHECKING: + from scrapy.http import Response + + +async def sleep(seconds: float = 0.001) -> None: + from twisted.internet import reactor + + deferred: Deferred[None] = Deferred() + reactor.callLater(seconds, deferred.callback, None) + await maybe_deferred_to_future(deferred) + + +class MainTestCase(TestCase): + @deferred_f_from_coro_f + async def test_sleep(self): + """Neither asynchronous sleeps on Spider.start() nor the equivalent on + the scheduler (returning no requests while also returning True from + the has_pending_requests() method) should cause the spider to miss the + processing of any later requests.""" + seconds = 2 + + class TestSpider(Spider): + name = "test" + + async def start(self): + from twisted.internet import reactor + + yield Request("data:,a") + + await sleep(seconds) + + self.crawler.engine._slot.scheduler.pause() + self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) + + # During this time, the scheduler reports having requests but + # returns None. + await sleep(seconds) + + self.crawler.engine._slot.scheduler.unpause() + + # The scheduler request is processed. + await sleep(seconds) + + yield Request("data:,c") + + await sleep(seconds) + + self.crawler.engine._slot.scheduler.pause() + self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) + + # The last start request is processed during the time until the + # delayed call below, proving that the start iteration can + # finish before a scheduler “sleep†without causing the + # scheduler to finish. + reactor.callLater(seconds, self.crawler.engine._slot.scheduler.unpause) + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": MemoryScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + @deferred_f_from_coro_f + async def test_close_during_start_iteration(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + assert self.crawler.engine is not None + await maybe_deferred_to_future(self.crawler.engine.close()) + yield Request("data:,a") + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": MemoryScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + + with LogCapture(level=ERROR) as log: + await maybe_deferred_to_future(crawler.crawl()) + + assert not log.records, f"{log.records=}" + finish_reason = crawler.stats.get_value("finish_reason") + assert finish_reason == "shutdown", f"{finish_reason=}" + expected_urls = [] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + +class RequestSendOrderTestCase(TestCase): + seconds = 0.1 # increase if flaky + + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) # increase if flaky + + def request(self, num, response_seconds, download_slots, priority=0): + url = self.mockserver.url(f"/delay?n={response_seconds}&{num}") + meta = {"download_slot": str(num % download_slots)} + return Request(url, meta=meta, priority=priority) + + def get_num(self, request_or_response: Request | Response): + return int(request_or_response.url.rsplit("&", maxsplit=1)[1]) + + @deferred_f_from_coro_f + async def _test_request_order( + self, + start_nums, + cb_nums=None, + settings=None, + response_seconds=None, + download_slots=1, + start_fn=None, + parse_fn=None, + ): + cb_nums = cb_nums or [] + settings = settings or {} + response_seconds = response_seconds or self.seconds + + cb_requests = deque( + [self.request(num, response_seconds, download_slots) for num in cb_nums] + ) + + if start_fn is None: + + async def start_fn(spider): + for num in start_nums: + yield self.request(num, response_seconds, download_slots) + + if parse_fn is None: + + def parse_fn(spider, response): + while cb_requests: + yield cb_requests.popleft() + + class TestSpider(Spider): + name = "test" + start = start_fn + parse = parse_fn + + actual_nums = [] + + def track_num(request, spider): + actual_nums.append(self.get_num(request)) + + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_num, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_nums = sorted(start_nums + cb_nums) + assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" + + @deferred_f_from_coro_f + async def test_default(self): + """By default, start requests take priority over callback requests and + are sent in order. Priority matters, but given the same priority, a + start request takes precedence.""" + nums = [1, 2, 3, 4, 5, 6] + response_seconds = 0 + download_slots = 1 + + def _request(num, priority=0): + return self.request( + num, response_seconds, download_slots, priority=priority + ) + + async def start(spider): + # The first CONCURRENT_REQUESTS start requests are sent + # immediately. + yield _request(1) + + for request in ( + _request(4, priority=1), + _request(6), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + yield _request(5) + yield _request(2, priority=1) + yield _request(3, priority=1) + + def parse(spider, response): + return + yield + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=nums, + settings={"CONCURRENT_REQUESTS": 1}, + response_seconds=response_seconds, + start_fn=start, + parse_fn=parse, + ) + ) + + @deferred_f_from_coro_f + async def test_lifo_start(self): + """Changing the queues of start requests to LIFO, matching the queues + of non-start requests, does not cause all requests to be stored in the + same queue objects, it only affects the order of start requests.""" + nums = [1, 2, 3, 4, 5, 6] + response_seconds = 0 + download_slots = 1 + + def _request(num, priority=0): + return self.request( + num, response_seconds, download_slots, priority=priority + ) + + async def start(spider): + # The first CONCURRENT_REQUESTS start requests are sent + # immediately. + yield _request(1) + + for request in ( + _request(4, priority=1), + _request(6), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + yield _request(5) + yield _request(3, priority=1) + yield _request(2, priority=1) + + def parse(spider, response): + return + yield + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=nums, + settings={ + "CONCURRENT_REQUESTS": 1, + "SCHEDULER_START_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue", + }, + response_seconds=response_seconds, + start_fn=start, + parse_fn=parse, + ) + ) + + @deferred_f_from_coro_f + async def test_shared_queues(self): + """If SCHEDULER_START_*_QUEUE is falsy, start requests and other + requests share the same queue, i.e. start requests are not priorized + over other requests if their priority matches.""" + nums = list(range(1, 14)) + response_seconds = 0 + download_slots = 1 + + def _request(num, priority=0): + return self.request( + num, response_seconds, download_slots, priority=priority + ) + + async def start(spider): + # The first CONCURRENT_REQUESTS start requests are sent + # immediately. + yield _request(1) + + # Below, priority 1 requests are sent first, and requests are sent + # in LIFO order. + + for request in ( + _request(7, priority=1), + _request(6, priority=1), + _request(13), + _request(12), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + + yield _request(11) + yield _request(10) + yield _request(5, priority=1) + yield _request(4, priority=1) + + for request in ( + _request(3, priority=1), + _request(2, priority=1), + _request(9), + _request(8), + ): + spider.crawler.engine._slot.scheduler.enqueue_request(request) + + def parse(spider, response): + return + yield + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=nums, + settings={ + "CONCURRENT_REQUESTS": 1, + "SCHEDULER_START_MEMORY_QUEUE": None, + }, + response_seconds=response_seconds, + start_fn=start, + parse_fn=parse, + ) + ) + + # Examples from the “Start requests†section of the documentation about + # spiders. + + @deferred_f_from_coro_f + async def test_lazy(self): + start_nums = [1, 2, 4] + cb_nums = [3] + response_seconds = self.seconds * 2**1 # increase if flaky + download_slots = 1 + + async def start(spider): + for num in start_nums: + if spider.crawler.engine.needs_backout(): + await spider.crawler.signals.wait_for(signals.scheduler_empty) + request = self.request(num, response_seconds, download_slots) + yield request + + await maybe_deferred_to_future( + self._test_request_order( + start_nums=start_nums, + cb_nums=cb_nums, + settings={ + "CONCURRENT_REQUESTS": 1, + }, + response_seconds=response_seconds, + start_fn=start, + ) + ) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 743d9774b..d658d1526 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -69,7 +69,7 @@ class AsyncDefNotAsyncioPipeline: class ItemSpider(Spider): name = "itemspider" - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index ab6baa5f0..79b53b33b 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -28,10 +28,10 @@ class InjectArgumentsSpiderMiddleware: Make sure spider middlewares are able to update the keyword arguments """ - def process_start_requests(self, start_requests, spider): - for request in start_requests: + async def process_start(self, start): + async for request in start: if request.callback.__name__ == "parse_spider_mw": - request.cb_kwargs["from_process_start_requests"] = True + request.cb_kwargs["from_process_start"] = True yield request def process_spider_input(self, response, spider): @@ -62,7 +62,7 @@ class KeywordArgumentsSpider(MockServerSpider): checks: list[bool] = [] - def start_requests(self): + async def start(self): data = {"key": "value", "number": 123, "callback": "some_callback"} yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data) yield Request( @@ -138,11 +138,9 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(bool(from_process_response)) self.crawler.stats.inc_value("boolean_checks", 2) - def parse_spider_mw( - self, response, from_process_spider_input, from_process_start_requests - ): + def parse_spider_mw(self, response, from_process_spider_input, from_process_start): self.checks.append(bool(from_process_spider_input)) - self.checks.append(bool(from_process_start_requests)) + self.checks.append(bool(from_process_start)) self.crawler.stats.inc_value("boolean_checks", 2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 1d6992a32..f90293dd3 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -3,6 +3,7 @@ from __future__ import annotations import shutil import tempfile from abc import ABC, abstractmethod +from collections import deque from typing import Any, NamedTuple import pytest @@ -10,7 +11,7 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.core.downloader import Downloader -from scrapy.core.scheduler import Scheduler +from scrapy.core.scheduler import BaseScheduler, Scheduler from scrapy.crawler import Crawler from scrapy.http import Request from scrapy.spiders import Spider @@ -20,6 +21,38 @@ from scrapy.utils.test import get_crawler from tests.mockserver import MockServer +class MemoryScheduler(BaseScheduler): + paused = False + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.queue = deque( + Request(value) if isinstance(value, str) else value + for value in getattr(self, "queue", []) + ) + + def enqueue_request(self, request: Request) -> bool: + self.queue.append(request) + return True + + def has_pending_requests(self) -> bool: + return self.paused or bool(self.queue) + + def next_request(self) -> Request | None: + if self.paused: + return None + try: + return self.queue.pop() + except IndexError: + return None + + def pause(self) -> None: + self.paused = True + + def unpause(self) -> None: + self.paused = False + + class MockEngine(NamedTuple): downloader: MockDownloader diff --git a/tests/test_signals.py b/tests/test_signals.py index f5075fb60..663e912b7 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,8 +1,9 @@ import pytest from twisted.internet import defer -from twisted.trial import unittest +from twisted.trial.unittest import TestCase from scrapy import Request, Spider, signals +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver import MockServer @@ -10,7 +11,7 @@ from tests.mockserver import MockServer class ItemSpider(Spider): name = "itemspider" - def start_requests(self): + async def start(self): for index in range(10): yield Request( self.mockserver.url(f"/status?n=200&id={index}"), meta={"index": index} @@ -20,7 +21,21 @@ class ItemSpider(Spider): return {"index": response.meta["index"]} -class TestAsyncSignal(unittest.TestCase): +class MainTestCase(TestCase): + @deferred_f_from_coro_f + async def test_scheduler_empty(self): + crawler = get_crawler() + calls = [] + + def track_call(): + calls.append(object()) + + crawler.signals.connect(track_call, signals.scheduler_empty) + await maybe_deferred_to_future(crawler.crawl()) + assert len(calls) >= 1 + + +class MockServerTestCase(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() diff --git a/tests/test_spider.py b/tests/test_spider.py index aaf72390d..b4aa649a3 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,8 +1,7 @@ import gzip -import inspect import warnings from io import BytesIO -from logging import WARNING +from logging import ERROR, WARNING from pathlib import Path from typing import Any from unittest import mock @@ -27,6 +26,7 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.spiders.init import InitSpider +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata, tests_datadir @@ -45,12 +45,6 @@ class TestSpider(unittest.TestCase): assert spider.name == "example.com" assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison - def test_start_requests(self): - spider = self.spider_class("example.com") - start_requests = spider.start_requests() - assert inspect.isgenerator(start_requests) - assert not list(start_requests) - def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" spider = self.spider_class("example.com", foo="bar") @@ -152,6 +146,22 @@ class TestSpider(unittest.TestCase): class TestInitSpider(TestSpider): spider_class = InitSpider + @deferred_f_from_coro_f + async def test_start_urls(self): + responses = [] + + class TestSpider(self.spider_class): + name = "test" + start_urls = ["data:,"] + + async def parse(self, response): + responses.append(response) + + crawler = get_crawler(TestSpider) + await maybe_deferred_to_future(crawler.crawl()) + assert len(responses) == 1 + assert responses[0].url == "data:," + class TestXMLFeedSpider(TestSpider): spider_class = XMLFeedSpider @@ -454,12 +464,17 @@ class TestCrawlSpider(TestSpider): assert hasattr(spider, "_follow_links") assert not spider._follow_links + @inlineCallbacks def test_start_url(self): - spider = self.spider_class("example.com") - spider.start_url = "https://www.example.com" + class TestSpider(self.spider_class): + name = "test" + start_url = "https://www.example.com" - with pytest.raises(AttributeError, match=r"^Crawling could not start.*$"): - list(spider.start_requests()) + crawler = get_crawler(TestSpider) + with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log: + yield crawler.crawl() + assert "Error while reading start items and requests" in str(log) + assert "did you miss an 's'?" in str(log) class TestSitemapSpider(TestSpider): @@ -776,6 +791,24 @@ Sitemap: /sitemap-relative-url.xml ), ) + @deferred_f_from_coro_f + async def test_sitemap_urls(self): + class TestSpider(self.spider_class): + name = "test" + sitemap_urls = ["https://toscrape.com/sitemap.xml"] + + crawler = get_crawler(TestSpider) + spider = TestSpider.from_crawler(crawler) + with warnings.catch_warnings(): + warnings.simplefilter("error") + requests = [request async for request in spider.start()] + + assert len(requests) == 1 + request = requests[0] + assert request.url == "https://toscrape.com/sitemap.xml" + assert request.dont_filter is False + assert request.callback == spider._parse_sitemap + class TestDeprecation: def test_crawl_spider(self): diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py new file mode 100644 index 000000000..1815aad76 --- /dev/null +++ b/tests/test_spider_start.py @@ -0,0 +1,186 @@ +import warnings +from asyncio import sleep + +import pytest +from testfixtures import LogCapture +from twisted.trial.unittest import TestCase + +from scrapy import Spider, signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.test import get_crawler + +from .utils import twisted_sleep + +SLEEP_SECONDS = 0.1 + +ITEM_A = {"id": "a"} +ITEM_B = {"id": "b"} + + +class MainTestCase(TestCase): + async def _test_spider(self, spider, expected_items=None): + actual_items = [] + expected_items = [] if expected_items is None else expected_items + + def track_item(item, response, spider): + actual_items.append(item) + + crawler = get_crawler(spider) + crawler.signals.connect(track_item, signals.item_scraped) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + assert actual_items == expected_items + + @deferred_f_from_coro_f + async def test_start_urls(self): + class TestSpider(Spider): + name = "test" + start_urls = ["data:,"] + + async def parse(self, response): + yield ITEM_A + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_start(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_A + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_start_subclass(self): + class BaseSpider(Spider): + async def start(self): + yield ITEM_A + + class TestSpider(BaseSpider): + name = "test" + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_deprecated(self): + class TestSpider(Spider): + name = "test" + + def start_requests(self): + yield ITEM_A + + with pytest.warns(ScrapyDeprecationWarning): + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_deprecated_subclass(self): + class BaseSpider(Spider): + def start_requests(self): + yield ITEM_A + + class TestSpider(BaseSpider): + name = "test" + + # The warning must be about the base class and not the subclass. + with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"): + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_universal(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_A + + def start_requests(self): + yield ITEM_B + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_universal_subclass(self): + class BaseSpider(Spider): + async def start(self): + yield ITEM_A + + def start_requests(self): + yield ITEM_B + + class TestSpider(BaseSpider): + name = "test" + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_spider(TestSpider, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_start_deprecated_super(self): + class TestSpider(Spider): + name = "test" + + async def start(self): + for item_or_request in super().start_requests(): + yield item_or_request + + with pytest.warns( + ScrapyDeprecationWarning, match=r"use Spider\.start\(\) instead" + ) as messages: + await self._test_spider(TestSpider, []) + assert messages[0].filename.endswith("test_spider_start.py") + + async def _test_start(self, start_, expected_items=None): + class TestSpider(Spider): + name = "test" + start = start_ + + await self._test_spider(TestSpider, expected_items) + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_asyncio_delayed(self): + async def start(spider): + await sleep(SLEEP_SECONDS) + yield ITEM_A + + await self._test_start(start, [ITEM_A]) + + @deferred_f_from_coro_f + async def test_twisted_delayed(self): + async def start(spider): + await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) + yield ITEM_A + + await self._test_start(start, [ITEM_A]) + + # Exceptions + + @deferred_f_from_coro_f + async def test_deprecated_non_generator_exception(self): + class TestSpider(Spider): + name = "test" + + def start_requests(self): + raise RuntimeError + + with ( + LogCapture() as log, + pytest.warns( + ScrapyDeprecationWarning, + match=r"defines the deprecated start_requests\(\) method", + ), + ): + await self._test_spider(TestSpider, []) + + assert "in start_requests\n raise RuntimeError" in str(log) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 1d671134e..db46be7dd 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterable +from inspect import isasyncgen from typing import Any from unittest import mock @@ -111,7 +112,7 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): """Helpers for testing sync, async and mixed middlewares. - Should work for process_spider_output and, when it's supported, process_start_requests. + Should work for process_spider_output and, when it's supported, process_start. """ ITEM_TYPE: type | tuple @@ -200,7 +201,7 @@ class ProcessSpiderExceptionSimpleIterableMiddleware: yield {"foo": 3} -class ProcessSpiderExceptionAsyncIterableMiddleware: +class ProcessSpiderExceptionAsyncIteratorMiddleware: async def process_spider_exception(self, response, exception, spider): yield {"foo": 1} d = defer.Deferred() @@ -319,37 +320,43 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): ) -class ProcessStartRequestsSimpleMiddleware: - def process_start_requests(self, start_requests, spider): - yield from start_requests +class ProcessStartSimpleMiddleware: + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request -class TestProcessStartRequestsSimple(TestBaseAsyncSpiderMiddleware): - """process_start_requests tests for simple start_requests""" +class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware): + """process_start tests for simple start""" ITEM_TYPE = (Request, dict) - MW_SIMPLE = ProcessStartRequestsSimpleMiddleware + MW_SIMPLE = ProcessStartSimpleMiddleware - def _start_requests(self): - for i in range(2): - yield Request(f"https://example.com/{i}", dont_filter=True) - yield {"name": "test item"} + async def _get_processed_start(self, *mw_classes): + class TestSpider(Spider): + name = "test" - @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: int | None = None): - setting = self._construct_mw_setting(*mw_classes, start_index=start_index) + async def start(self): + for i in range(2): + yield Request(f"https://example.com/{i}", dont_filter=True) + yield {"name": "test item"} + + setting = self._construct_mw_setting(*mw_classes) self.crawler = get_crawler( - Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} + TestSpider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} ) - self.spider = self.crawler._create_spider("foo") + self.spider = self.crawler._create_spider() self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - start_requests = iter(self._start_requests()) - results = yield self.mwman.process_start_requests(start_requests, self.spider) - return results + return await self.mwman.process_start(self.spider) - def test_simple(self): + @deferred_f_from_coro_f + async def test_simple(self): """Simple mw""" - return self._test_simple_base(self.MW_SIMPLE) + start = await self._get_processed_start(self.MW_SIMPLE) + assert isasyncgen(start) + start_list = await collect_asyncgen(start) + assert len(start_list) == self.RESULT_COUNT + assert isinstance(start_list[0], self.ITEM_TYPE) class UniversalMiddlewareNoSync: @@ -507,7 +514,7 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware - MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIterableMiddleware + MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIteratorMiddleware def _scrape_func(self, *args, **kwargs): 1 / 0 diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py index 46be879f3..77d055d50 100644 --- a/tests/test_spidermiddleware_base.py +++ b/tests/test_spidermiddleware_base.py @@ -27,16 +27,19 @@ def test_trivial(crawler): assert mw.crawler is crawler test_req = Request("data:,") spider_output = [test_req, {"foo": "bar"}] - processed = list( - mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) - ) - assert processed == [test_req, {"foo": "bar"}] + for processed in [ + list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ), + list(mw.process_start_requests(spider_output, crawler.spider)), + ]: + assert processed == [test_req, {"foo": "bar"}] def test_processed_request(crawler): class ProcessReqSpiderMiddleware(BaseSpiderMiddleware): def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: if request.url == "data:2,": return None @@ -49,20 +52,23 @@ def test_processed_request(crawler): test_req2 = Request("data:2,") test_req3 = Request("data:3,") spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] - processed = list( - mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) - ) - assert len(processed) == 3 - assert isinstance(processed[0], Request) - assert processed[0].url == "data:1," - assert processed[1] == {"foo": "bar"} - assert isinstance(processed[2], Request) - assert processed[2].url == "data:30," + for processed in [ + list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ), + list(mw.process_start_requests(spider_output, crawler.spider)), + ]: + assert len(processed) == 3 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": "bar"} + assert isinstance(processed[2], Request) + assert processed[2].url == "data:30," def test_processed_item(crawler): class ProcessItemSpiderMiddleware(BaseSpiderMiddleware): - def get_processed_item(self, item: Any, response: Response) -> Any: + def get_processed_item(self, item: Any, response: Response | None) -> Any: if item["foo"] == 2: return None if item["foo"] == 3: @@ -72,16 +78,19 @@ def test_processed_item(crawler): mw = ProcessItemSpiderMiddleware.from_crawler(crawler) test_req = Request("data:,") spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}] - processed = list( - mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) - ) - assert processed == [{"foo": 1}, test_req, {"foo": 30}] + for processed in [ + list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ), + list(mw.process_start_requests(spider_output, crawler.spider)), + ]: + assert processed == [{"foo": 1}, test_req, {"foo": 30}] def test_processed_both(crawler): class ProcessBothSpiderMiddleware(BaseSpiderMiddleware): def get_processed_request( - self, request: Request, response: Response + self, request: Request, response: Response | None ) -> Request | None: if request.url == "data:2,": return None @@ -89,7 +98,7 @@ def test_processed_both(crawler): return Request("data:30,") return request - def get_processed_item(self, item: Any, response: Response) -> Any: + def get_processed_item(self, item: Any, response: Response | None) -> Any: if item["foo"] == 2: return None if item["foo"] == 3: @@ -108,13 +117,16 @@ def test_processed_both(crawler): {"foo": 3}, test_req3, ] - processed = list( - mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) - ) - assert len(processed) == 4 - assert isinstance(processed[0], Request) - assert processed[0].url == "data:1," - assert processed[1] == {"foo": 1} - assert processed[2] == {"foo": 30} - assert isinstance(processed[3], Request) - assert processed[3].url == "data:30," + for processed in [ + list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ), + list(mw.process_start_requests(spider_output, crawler.spider)), + ]: + assert len(processed) == 4 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": 1} + assert processed[2] == {"foo": 30} + assert isinstance(processed[3], Request) + assert processed[3].url == "data:30," diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e306579fa..fd2fc3581 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -30,7 +30,7 @@ class _HttpErrorSpider(MockServerSpider): self.skipped = set() self.parsed = set() - def start_requests(self): + async def start(self): for url in self.start_urls: yield Request(url, self.parse, errback=self.on_error) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 6e26a85ea..20efac543 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -36,7 +36,7 @@ class RecoverySpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -73,7 +73,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): } } - def start_requests(self): + async def start(self): yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse) def parse(self, response): @@ -83,7 +83,7 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback): name = "ProcessSpiderInputSpiderWithErrback" - def start_requests(self): + async def start(self): yield Request( self.mockserver.url("/status?n=200"), self.parse, errback=self.errback ) @@ -103,7 +103,7 @@ class GeneratorCallbackSpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -140,7 +140,7 @@ class NotGeneratorCallbackSpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -215,7 +215,7 @@ class GeneratorOutputChainSpider(Spider): }, } - def start_requests(self): + async def start(self): yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): @@ -287,8 +287,8 @@ class NotGeneratorOutputChainSpider(Spider): }, } - def start_requests(self): - return [Request(self.mockserver.url("/status?n=200"))] + async def start(self): + yield Request(self.mockserver.url("/status?n=200")) def parse(self, response): return [ diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py new file mode 100644 index 000000000..725833a49 --- /dev/null +++ b/tests/test_spidermiddleware_process_start.py @@ -0,0 +1,352 @@ +import warnings +from asyncio import sleep + +import pytest +from twisted.trial.unittest import TestCase + +from scrapy import Spider, signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.test import get_crawler +from tests.test_spider_start import SLEEP_SECONDS + +from .utils import twisted_sleep + +ITEM_A = {"id": "a"} +ITEM_B = {"id": "b"} +ITEM_C = {"id": "c"} +ITEM_D = {"id": "d"} + + +class AsyncioSleepSpiderMiddleware: + async def process_start(self, start): + await sleep(SLEEP_SECONDS) + async for item_or_request in start: + yield item_or_request + + +class NoOpSpiderMiddleware: + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request + + +class TwistedSleepSpiderMiddleware: + async def process_start(self, start): + await maybe_deferred_to_future(twisted_sleep(SLEEP_SECONDS)) + async for item_or_request in start: + yield item_or_request + + +class UniversalSpiderMiddleware: + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request + + def process_start_requests(self, start_requests, spider): + raise NotImplementedError + + +# Spiders and spider middlewares for MainTestCase._test_wrap + + +class ModernWrapSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_B + + +class ModernWrapSpiderSubclass(ModernWrapSpider): + name = "test" + + +class UniversalWrapSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_B + + def start_requests(self): + yield ITEM_D + + +class DeprecatedWrapSpider(Spider): + name = "test" + + def start_requests(self): + yield ITEM_B + + +class ModernWrapSpiderMiddleware: + async def process_start(self, start): + yield ITEM_A + async for item_or_request in start: + yield item_or_request + yield ITEM_C + + +class UniversalWrapSpiderMiddleware: + async def process_start(self, start): + yield ITEM_A + async for item_or_request in start: + yield item_or_request + yield ITEM_C + + def process_start_requests(self, start, spider): + yield ITEM_A + yield from start + yield ITEM_C + + +class DeprecatedWrapSpiderMiddleware: + def process_start_requests(self, start, spider): + yield ITEM_A + yield from start + yield ITEM_C + + +class MainTestCase(TestCase): + async def _test(self, spider_middlewares, spider_cls, expected_items): + actual_items = [] + + def track_item(item, response, spider): + actual_items.append(item) + + settings = { + "SPIDER_MIDDLEWARES": {cls: n for n, cls in enumerate(spider_middlewares)}, + } + crawler = get_crawler(spider_cls, settings_dict=settings) + crawler.signals.connect(track_item, signals.item_scraped) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + assert actual_items == expected_items, f"{actual_items=} != {expected_items=}" + + async def _test_wrap(self, spider_middleware, spider_cls, expected_items=None): + expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C] + await self._test([spider_middleware], spider_cls, expected_items) + + async def _test_douple_wrap(self, smw1, smw2, spider_cls, expected_items=None): + expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C] + await self._test([smw1, smw2], spider_cls, expected_items) + + @deferred_f_from_coro_f + async def test_modern_mw_modern_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ): + await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider) + + @deferred_f_from_coro_f + async def test_universal_mw_modern_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider) + + @deferred_f_from_coro_f + async def test_universal_mw_universal_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ): + await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider) + + @deferred_f_from_coro_f + async def test_deprecated_mw_modern_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.raises( + ValueError, match=r"only compatible with \(deprecated\) spiders" + ), + ): + await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider) + + @deferred_f_from_coro_f + async def test_deprecated_mw_modern_spider_subclass(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.raises( + ValueError, + match=r"^\S+?\.ModernWrapSpider \(inherited by \S+?.ModernWrapSpiderSubclass\) .*? only compatible with \(deprecated\) spiders", + ), + ): + await self._test_wrap( + DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass + ) + + @deferred_f_from_coro_f + async def test_deprecated_mw_universal_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ): + await self._test_wrap( + DeprecatedWrapSpiderMiddleware, + UniversalWrapSpider, + [ITEM_A, ITEM_D, ITEM_C], + ) + + @deferred_f_from_coro_f + async def test_deprecated_mw_deprecated_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ), + ): + await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_mw_modern_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + UniversalWrapSpiderMiddleware, + ModernWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_mw_modern_spider(self): + with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + ModernWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_mw_modern_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.raises( + ValueError, match=r"only compatible with \(deprecated\) spiders" + ), + ): + await self._test_douple_wrap( + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + ModernWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_mw_universal_spider(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + UniversalWrapSpiderMiddleware, + UniversalWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_mw_universal_spider(self): + with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + UniversalWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_mw_universal_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ): + await self._test_douple_wrap( + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + UniversalWrapSpider, + [ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C], + ) + + @deferred_f_from_coro_f + async def test_modern_mw_universal_mw_deprecated_spider(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_modern_mw_deprecated_mw_deprecated_spider(self): + with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): + await self._test_douple_wrap( + ModernWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + DeprecatedWrapSpider, + ) + + @deferred_f_from_coro_f + async def test_universal_mw_deprecated_mw_deprecated_spider(self): + with ( + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" + ), + pytest.warns( + ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" + ), + ): + await self._test_douple_wrap( + UniversalWrapSpiderMiddleware, + DeprecatedWrapSpiderMiddleware, + DeprecatedWrapSpider, + ) + + async def _test_sleep(self, spider_middlewares): + class TestSpider(Spider): + name = "test" + + async def start(self): + yield ITEM_A + + await self._test(spider_middlewares, TestSpider, [ITEM_A]) + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_asyncio_sleep_single(self): + await self._test_sleep([AsyncioSleepSpiderMiddleware]) + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_asyncio_sleep_multiple(self): + await self._test_sleep( + [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware] + ) + + @deferred_f_from_coro_f + async def test_twisted_sleep_single(self): + await self._test_sleep([TwistedSleepSpiderMiddleware]) + + @deferred_f_from_coro_f + async def test_twisted_sleep_multiple(self): + await self._test_sleep( + [NoOpSpiderMiddleware, TwistedSleepSpiderMiddleware, NoOpSpiderMiddleware] + ) diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py new file mode 100644 index 000000000..295b10ea8 --- /dev/null +++ b/tests/test_spidermiddleware_start.py @@ -0,0 +1,44 @@ +from twisted.trial.unittest import TestCase + +from scrapy.http import Request +from scrapy.spidermiddlewares.start import StartSpiderMiddleware +from scrapy.spiders import Spider +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler + + +class TestMiddleware(TestCase): + @deferred_f_from_coro_f + async def test_async(self): + crawler = get_crawler(Spider) + mw = build_from_crawler(StartSpiderMiddleware, crawler) + + async def start(): + yield Request("data:,1") + yield Request("data:,2", meta={"is_start_request": True}) + yield Request("data:,2", meta={"is_start_request": False}) + yield Request("data:,2", meta={"is_start_request": "foo"}) + + result = [ + request.meta["is_start_request"] + async for request in mw.process_start(start()) + ] + assert result == [True, True, False, "foo"] + + @deferred_f_from_coro_f + async def test_sync(self): + crawler = get_crawler(Spider) + mw = build_from_crawler(StartSpiderMiddleware, crawler) + + def start(): + yield Request("data:,1") + yield Request("data:,2", meta={"is_start_request": True}) + yield Request("data:,2", meta={"is_start_request": False}) + yield Request("data:,2", meta={"is_start_request": "foo"}) + + result = [ + request.meta["is_start_request"] + for request in mw.process_start_requests(start(), Spider("test")) + ] + assert result == [True, True, False, "foo"] diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index e69de29bb..e5e56f414 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -0,0 +1,9 @@ +from twisted.internet.defer import Deferred + + +def twisted_sleep(seconds): + from twisted.internet import reactor + + d = Deferred() + reactor.callLater(seconds, d.callback, None) + return d diff --git a/tox.ini b/tox.ini index e63e44189..92cfc3794 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,7 @@ install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} [testenv:typing] -basepython = python3 +basepython = python3.9 deps = mypy==1.14.0 typing-extensions==4.12.2 From 5f6d1b464b81e9673a9639fb8f742d53831c98dc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 5 May 2025 18:08:54 +0500 Subject: [PATCH 1716/2083] Cover up to 373e501 in the release notes. --- docs/news.rst | 46 ++++++++++++++++++++++++++++++++++------ docs/topics/settings.rst | 3 ++- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 7bb25e6b6..d9bc572be 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -26,6 +26,9 @@ Modified requirements - Dropped support for PyPy 3.9. (:issue:`6613`) +- Added support for PyPy 3.11. + (:issue:`6697`) + Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -38,6 +41,10 @@ Backward-incompatible changes <disable-asyncio>` to use a different reactor. (:issue:`6659`, :issue:`6713`) +- The URL length limit, set by the :setting:`URLLENGTH_LIMIT` setting, is now + also enforced for start requests. + (:issue:`6777`) + - The ``from_settings()`` method of :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, deprecated in Scrapy 2.12.0, is removed earlier than the usual deprecation @@ -174,6 +181,19 @@ Deprecations called. (:issue:`4151`) +- Passing the ``spider`` argument to the following methods of + :class:`~scrapy.core.scraper.Scraper` is deprecated: + + - ``close_spider()`` + + - ``enqueue_scrape()`` + + - ``handle_spider_error()`` + + - ``handle_spider_output()`` + + (:issue:`6764`) + New features ~~~~~~~~~~~~ @@ -189,7 +209,7 @@ New features helpful for writing :ref:`universal spider middlewares <universal-spider-middleware>` without boilerplate and code duplication. The built-in spider middlewares now inherit from this class. - (:issue:`6693`) + (:issue:`6693`, :issue:`6777`) - :ref:`Scrapy add-ons <topics-addons>` can now define a class method called ``update_pre_crawler_settings()`` to update :ref:`pre-crawler settings @@ -278,14 +298,17 @@ Bug fixes Documentation ~~~~~~~~~~~~~ +- Documented the setting values set in the default project template. + (:issue:`6762`, :issue:`6775`) + - Improved the :ref:`docs <sync-async-spider-middleware>` about asynchronous iterable support in spider middlewares. (:issue:`6688`) - Improved the :ref:`docs <coroutine-deferred-apis>` about using :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based - code. - (:issue:`6734`) + code and included a list of such APIs. + (:issue:`6677`, :issue:`6734`, :issue:`6776`) - Improved the :ref:`contribution docs <topics-contributing>`. (:issue:`6561`, :issue:`6575`) @@ -307,7 +330,8 @@ Documentation :issue:`6623`, :issue:`6624`, :issue:`6721`, - :issue:`6723`) + :issue:`6723`, + :issue:`6780`) Packaging ~~~~~~~~~ @@ -315,6 +339,11 @@ Packaging - Switched from ``setup.py`` to ``pyproject.toml``. (:issue:`6514`, :issue:`6547`) +- Switched the build backend from setuptools_ to hatchling_. + (:issue:`6771`) + + .. _hatchling: https://pypi.org/project/hatchling/ + Quality assurance ~~~~~~~~~~~~~~~~~ @@ -384,12 +413,17 @@ Quality assurance :issue:`6722`, :issue:`6724`, :issue:`6741`, - :issue:`6743`) + :issue:`6743`, + :issue:`6766`, + :issue:`6770`, + :issue:`6772`, + :issue:`6773`) - Code cleanups. (:issue:`6600`, :issue:`6606`, - :issue:`6635`) + :issue:`6635`, + :issue:`6764`) .. _release-2.12.0: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 73ac36646..fec82f8e3 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -701,7 +701,8 @@ connections (for ``HTTP10DownloadHandler``). .. note:: - HTTP/1.0 is rarely used nowadays so you can safely ignore this setting, + HTTP/1.0 is rarely used nowadays and its Scrapy support is deprecated, + so you can safely ignore this setting, unless you really want to use HTTP/1.0 and override :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly, i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. From 0ce693dfa91a4ebf8418fd04c5595a570939e480 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 8 May 2025 13:23:19 +0500 Subject: [PATCH 1717/2083] Update VERSION strings. --- docs/topics/coroutines.rst | 4 ++-- docs/topics/spider-middleware.rst | 2 +- scrapy/core/spidermw.py | 20 ++++++++++---------- scrapy/spiders/__init__.py | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 62cbc3d49..00812ed7f 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -21,7 +21,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be defined as an :term:`asynchronous generator`. - .. versionadded: VERSION + .. versionadded: 2.13 - :class:`~scrapy.Request` callbacks. @@ -59,7 +59,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): of :ref:`spider middlewares <custom-spider-middleware>`, which *must* be defined as an :term:`asynchronous generator`. - .. versionadded:: VERSION + .. versionadded:: 2.13 - :ref:`Signal handlers that support deferreds <signal-deferred>`. diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 638035e64..67178b8fd 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -86,7 +86,7 @@ one or more of these methods: You may yield the same type of objects as :meth:`~scrapy.Spider.start`. To write spider middlewares that work on Scrapy versions lower than - VERSION, define also a synchronous ``process_start_requests()`` method + 2.13, define also a synchronous ``process_start_requests()`` method that returns an iterable. For example: .. code-block:: python diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 4a0cd9464..310abb9b7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -85,8 +85,8 @@ class SpiderMiddlewareManager(MiddlewareManager): "either disable or make universal 1 of those 2 sets of " "spider middlewares. Making a spider middleware universal " "means having it define both methods. See the release notes " - "of Scrapy VERSION for details: " - "https://docs.scrapy.org/en/VERSION/news.html" + "of Scrapy 2.13 for details: " + "https://docs.scrapy.org/en/2.13/news.html" ) self._use_start_requests = bool(deprecated_middlewares) @@ -103,15 +103,15 @@ class SpiderMiddlewareManager(MiddlewareManager): f"been deprecated in favor of a new method, process_start(), " f"to support asynchronous code execution. " f"process_start_requests() will stop being called in a future " - f"version of Scrapy. If you use Scrapy VERSION or higher " + f"version of Scrapy. If you use Scrapy 2.13 or higher " f"only, replace process_start_requests() with " f"process_start(); note that process_start() is a coroutine " f"(async def). If you need to maintain compatibility with " f"lower Scrapy versions, when defining " f"process_start_requests() in a spider middleware class, " f"define process_start() as well. See the release notes of " - f"Scrapy VERSION for details: " - f"https://docs.scrapy.org/en/VERSION/news.html", + f"Scrapy 2.13 for details: " + f"https://docs.scrapy.org/en/2.13/news.html", ScrapyDeprecationWarning, ) @@ -435,15 +435,15 @@ class SpiderMiddlewareManager(MiddlewareManager): f"start_requests() has been deprecated in favor of a new " f"method, start(), to support asynchronous code " f"execution. start_requests() will stop being called in a " - f"future version of Scrapy. If you use Scrapy VERSION or " + f"future version of Scrapy. If you use Scrapy 2.13 or " f"higher only, replace start_requests() with start(); " f"note that start() is a coroutine (async def). If you " f"need to maintain compatibility with lower Scrapy versions, " f"when overriding start_requests() in a spider class, " f"override start() as well; you can use super() to " f"reuse the inherited start() implementation without " - f"copy-pasting. See the release notes of Scrapy VERSION for " - f"details: https://docs.scrapy.org/en/VERSION/news.html", + f"copy-pasting. See the release notes of Scrapy 2.13 for " + f"details: https://docs.scrapy.org/en/2.13/news.html", ScrapyDeprecationWarning, ) @@ -469,8 +469,8 @@ class SpiderMiddlewareManager(MiddlewareManager): f"deprecated spider middlewares (and earlier Scrapy versions) " f"by defining a sync start_requests() method that works " f"similarly to its existing start() method. See the " - f"release notes of Scrapy VERSION for details: " - f"https://docs.scrapy.org/en/VERSION/news.html" + f"release notes of Scrapy 2.13 for details: " + f"https://docs.scrapy.org/en/2.13/news.html" ) # This method is only needed until _async compatibility methods are removed. diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 0a1d85ae6..a722dd83b 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -83,7 +83,7 @@ class Spider(object_ref): async def start(self) -> AsyncIterator[Any]: """Yield the initial :class:`~scrapy.Request` objects to send. - .. versionadded:: VERSION + .. versionadded:: 2.13 For example: @@ -115,7 +115,7 @@ class Spider(object_ref): async def start(self): yield {"foo": "bar"} - To write spiders that work on Scrapy versions lower than VERSION, + To write spiders that work on Scrapy versions lower than 2.13, define also a synchronous ``start_requests()`` method that returns an iterable. For example: From 82a32451583967a828c2e80a31930f64dbc136ac Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 8 May 2025 14:43:34 +0500 Subject: [PATCH 1718/2083] =?UTF-8?q?Bump=20version:=202.12.0=20=E2=86=92?= =?UTF-8?q?=202.13.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- SECURITY.md | 4 ++-- docs/news.rst | 2 +- pyproject.toml | 2 +- scrapy/VERSION | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index bc64dec7b..a5a5c7fb3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,8 +4,8 @@ | Version | Supported | | ------- | ------------------ | -| 2.12.x | :white_check_mark: | -| < 2.12.x | :x: | +| 2.13.x | :white_check_mark: | +| < 2.13.x | :x: | ## Reporting a Vulnerability diff --git a/docs/news.rst b/docs/news.rst index d574317bb..cf1c35893 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.13.0: -Scrapy 2.13.0 (unreleased) +Scrapy 2.13.0 (2025-05-08) -------------------------- Highlights: diff --git a/pyproject.toml b/pyproject.toml index 187587eb1..85fba0f92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,7 +116,7 @@ module = "twisted" implicit_reexport = true [tool.bumpversion] -current_version = "2.12.0" +current_version = "2.13.0" commit = true tag = true tag_name = "{new_version}" diff --git a/scrapy/VERSION b/scrapy/VERSION index d8b698973..fb2c0766b 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.12.0 +2.13.0 From 128cb551eb493601d4a4cd6d7a087e09a07d7092 Mon Sep 17 00:00:00 2001 From: Felipe Benevolo <77981110+fbenevolo@users.noreply.github.com> Date: Mon, 12 May 2025 08:04:34 -0300 Subject: [PATCH 1719/2083] refactor tests/test_downloadermiddleware_httpcache.py (#6769) --- tests/test_downloadermiddleware_httpcache.py | 114 ++++++++++++------- 1 file changed, 72 insertions(+), 42 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 5fac88ed7..02f4f488e 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -15,8 +15,7 @@ from scrapy.utils.test import get_crawler class TestBase: - storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" - policy_class = "scrapy.extensions.httpcache.RFC2616Policy" + """Base class with common setup and helper methods.""" def setup_method(self): self.yesterday = email.utils.formatdate(time.time() - 86400) @@ -90,23 +89,10 @@ class TestBase: ) assert request1.body == request2.body - def test_dont_cache(self): - with self._middleware() as mw: - self.request.meta["dont_cache"] = True - mw.process_response(self.request, self.response, self.spider) - assert mw.storage.retrieve_response(self.spider, self.request) is None - with self._middleware() as mw: - self.request.meta["dont_cache"] = False - mw.process_response(self.request, self.response, self.spider) - if mw.policy.should_cache_response(self.response, self.request): - assert isinstance( - mw.storage.retrieve_response(self.spider, self.request), - self.response.__class__, - ) +class StorageTestMixin: + """Mixin containing storage-specific test methods.""" - -class TestDefaultStorage(TestBase): def test_storage(self): with self._storage() as storage: request2 = self.request.copy() @@ -143,31 +129,27 @@ class TestDefaultStorage(TestBase): self.assertEqualResponse(response, cached_response) -class TestDbmStorage(TestDefaultStorage): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" +class PolicyTestMixin: + """Mixin containing policy-specific test methods.""" + + def test_dont_cache(self): + with self._middleware() as mw: + self.request.meta["dont_cache"] = True + mw.process_response(self.request, self.response, self.spider) + assert mw.storage.retrieve_response(self.spider, self.request) is None + + with self._middleware() as mw: + self.request.meta["dont_cache"] = False + mw.process_response(self.request, self.response, self.spider) + if mw.policy.should_cache_response(self.response, self.request): + assert isinstance( + mw.storage.retrieve_response(self.spider, self.request), + self.response.__class__, + ) -class TestDbmStorageWithCustomDbmModule(TestDbmStorage): - dbm_module = "tests.mocks.dummydbm" - - def _get_settings(self, **new_settings): - new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module) - return super()._get_settings(**new_settings) - - def test_custom_dbm_module_loaded(self): - # make sure our dbm module has been loaded - with self._storage() as storage: - assert storage.dbmodule.__name__ == self.dbm_module - - -class TestFilesystemStorageGzip(TestDefaultStorage): - def _get_settings(self, **new_settings): - new_settings.setdefault("HTTPCACHE_GZIP", True) - return super()._get_settings(**new_settings) - - -class TestDummyPolicy(TestBase): - policy_class = "scrapy.extensions.httpcache.DummyPolicy" +class DummyPolicyTestMixin(PolicyTestMixin): + """Mixin containing dummy policy specific test methods.""" def test_middleware(self): with self._middleware() as mw: @@ -258,8 +240,8 @@ class TestDummyPolicy(TestBase): assert "cached" in response.flags -class TestRFC2616Policy(TestDefaultStorage): - policy_class = "scrapy.extensions.httpcache.RFC2616Policy" +class RFC2616PolicyTestMixin(PolicyTestMixin): + """Mixin containing RFC2616 policy specific test methods.""" def _process_requestresponse(self, mw, request, response): result = None @@ -562,3 +544,51 @@ class TestRFC2616Policy(TestDefaultStorage): res2 = self._process_requestresponse(mw, req0, None) self.assertEqualResponse(res1, res2) assert "cached" in res2.flags + + +# Concrete test classes that combine storage and policy mixins + + +class TestFilesystemStorageWithDummyPolicy( + TestBase, StorageTestMixin, DummyPolicyTestMixin +): + storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" + policy_class = "scrapy.extensions.httpcache.DummyPolicy" + + +class TestFilesystemStorageWithRFC2616Policy( + TestBase, StorageTestMixin, RFC2616PolicyTestMixin +): + storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" + policy_class = "scrapy.extensions.httpcache.RFC2616Policy" + + +class TestDbmStorageWithDummyPolicy(TestBase, StorageTestMixin, DummyPolicyTestMixin): + storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + policy_class = "scrapy.extensions.httpcache.DummyPolicy" + + +class TestDbmStorageWithRFC2616Policy( + TestBase, StorageTestMixin, RFC2616PolicyTestMixin +): + storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + policy_class = "scrapy.extensions.httpcache.RFC2616Policy" + + +class TestDbmStorageWithCustomDbmModule(TestDbmStorageWithDummyPolicy): + dbm_module = "tests.mocks.dummydbm" + + def _get_settings(self, **new_settings): + new_settings.setdefault("HTTPCACHE_DBM_MODULE", self.dbm_module) + return super()._get_settings(**new_settings) + + def test_custom_dbm_module_loaded(self): + # make sure our dbm module has been loaded + with self._storage() as storage: + assert storage.dbmodule.__name__ == self.dbm_module + + +class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPolicy): + def _get_settings(self, **new_settings): + new_settings.setdefault("HTTPCACHE_GZIP", True) + return super()._get_settings(**new_settings) From 2442536d0f5cdd16b836aafbe2da2c668c6dc96f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 13 May 2025 12:22:28 +0400 Subject: [PATCH 1720/2083] Add a deepwiki badge, update other badges. (#6793) --- README.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index cf7c6043c..29488d825 100644 --- a/README.rst +++ b/README.rst @@ -17,19 +17,14 @@ Scrapy :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - .. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - .. :alt: macOS - +.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS .. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows :alt: Windows -.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg - :target: https://pypi.org/pypi/Scrapy - :alt: Wheel Status - .. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report @@ -38,6 +33,10 @@ Scrapy :target: https://anaconda.org/conda-forge/scrapy :alt: Conda Version +.. image:: https://deepwiki.com/badge.svg + :target: https://deepwiki.com/scrapy/scrapy + :alt: Ask DeepWiki + Overview ======== From b86f00327a3d113a97525f68d38f77529d090f30 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 13 May 2025 22:47:57 +0400 Subject: [PATCH 1721/2083] Refactor more Deferred functions. (#6795) --- scrapy/commands/parse.py | 1 + scrapy/core/downloader/__init__.py | 96 +++++++++++++--------------- scrapy/core/downloader/middleware.py | 25 ++++---- scrapy/core/engine.py | 49 ++++++-------- scrapy/core/scraper.py | 29 +++++---- scrapy/core/spidermw.py | 50 +++++++++------ scrapy/crawler.py | 38 +++++------ tests/test_downloadermiddleware.py | 33 ++++++++++ 8 files changed, 179 insertions(+), 142 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 0dd9954cb..c4b3d2af9 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -282,6 +282,7 @@ class Command(BaseRunSpiderCommand): ) -> list[Any]: items, requests, opts, depth, spider, callback = args if opts.pipelines: + assert self.pcrawler.engine itemproc = self.pcrawler.engine.scraper.itemproc for item in items: itemproc.process_item(item, spider) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 78dc16df6..5468398aa 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -5,29 +5,32 @@ import warnings from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, TypeVar, cast +from typing import TYPE_CHECKING, Any, cast from twisted.internet import task -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, inlineCallbacks from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.resolver import dnscache -from scrapy.utils.defer import mustbe_deferred +from scrapy.utils.defer import ( + deferred_from_coro, + maybe_deferred_to_future, + mustbe_deferred, +) from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: + from collections.abc import Generator + from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.signalmanager import SignalManager -_T = TypeVar("_T") - - class Slot: """Downloader slot""" @@ -114,16 +117,17 @@ class Downloader: "DOWNLOAD_SLOTS", {} ) - def fetch(self, request: Request, spider: Spider) -> Deferred[Response | Request]: - def _deactivate(response: _T) -> _T: - self.active.remove(request) - return response - + @inlineCallbacks + def fetch( + self, request: Request, spider: Spider + ) -> Generator[Deferred[Any], Any, Response | Request]: self.active.add(request) - dfd: Deferred[Response | Request] = self.middleware.download( - self._enqueue_request, request, spider - ) - return dfd.addBoth(_deactivate) + try: + return ( + yield self.middleware.download(self._enqueue_request, request, spider) + ) + finally: + self.active.remove(request) def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency @@ -164,22 +168,23 @@ class Downloader: ) return self.get_slot_key(request) - def _enqueue_request(self, request: Request, spider: Spider) -> Deferred[Response]: + @inlineCallbacks + def _enqueue_request( + self, request: Request, spider: Spider + ) -> Generator[Deferred[Any], Any, Response]: key, slot = self._get_slot(request, spider) request.meta[self.DOWNLOAD_SLOT] = key - - def _deactivate(response: Response) -> Response: - slot.active.remove(request) - return response - slot.active.add(request) self.signals.send_catch_log( signal=signals.request_reached_downloader, request=request, spider=spider ) - deferred: Deferred[Response] = Deferred().addBoth(_deactivate) - slot.queue.append((request, deferred)) + d: Deferred[Response] = Deferred() + slot.queue.append((request, d)) self._process_queue(spider, slot) - return deferred + try: + return (yield d) + finally: + slot.active.remove(request) def _process_queue(self, spider: Spider, slot: Slot) -> None: from twisted.internet import reactor @@ -202,26 +207,23 @@ class Downloader: while slot.queue and slot.free_transfer_slots() > 0: slot.lastseen = now request, deferred = slot.queue.popleft() - dfd = self._download(slot, request, spider) + dfd = deferred_from_coro(self._download(slot, request, spider)) dfd.chainDeferred(deferred) # prevent burst if inter-request delays were configured if delay: self._process_queue(spider, slot) break - def _download( - self, slot: Slot, request: Request, spider: Spider - ) -> Deferred[Response]: - # The order is very important for the following deferreds. Do not change! - - # 1. Create the download deferred - dfd: Deferred[Response] = mustbe_deferred( - self.handlers.download_request, request, spider - ) - - # 2. Notify response_downloaded listeners about the recent download - # before querying queue for next request - def _downloaded(response: Response) -> Response: + async def _download(self, slot: Slot, request: Request, spider: Spider) -> Response: + # The order is very important for the following logic. Do not change! + slot.transferring.add(request) + try: + # 1. Download the response + response: Response = await maybe_deferred_to_future( + mustbe_deferred(self.handlers.download_request, request, spider) + ) + # 2. Notify response_downloaded listeners about the recent download + # before querying queue for next request self.signals.send_catch_log( signal=signals.response_downloaded, response=response, @@ -229,24 +231,16 @@ class Downloader: spider=spider, ) return response - - dfd.addCallback(_downloaded) - - # 3. After response arrives, remove the request from transferring - # state to free up the transferring slot so it can be used by the - # following requests (perhaps those which came from the downloader - # middleware itself) - slot.transferring.add(request) - - def finish_transferring(_: _T) -> _T: + finally: + # 3. After response arrives, remove the request from transferring + # state to free up the transferring slot so it can be used by the + # following requests (perhaps those which came from the downloader + # middleware itself) slot.transferring.remove(request) self._process_queue(spider, slot) self.signals.send_catch_log( signal=signals.request_left_downloader, request=request, spider=spider ) - return _ - - return dfd.addBoth(finish_transferring) def close(self) -> None: self._slot_gc_loop.stop() diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index db4191385..a4055849d 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -20,8 +20,6 @@ from scrapy.utils.defer import deferred_from_coro, mustbe_deferred if TYPE_CHECKING: from collections.abc import Generator - from twisted.python.failure import Failure - from scrapy import Spider from scrapy.settings import BaseSettings @@ -41,12 +39,13 @@ class DownloaderMiddlewareManager(MiddlewareManager): if hasattr(mw, "process_exception"): self.methods["process_exception"].appendleft(mw.process_exception) + @inlineCallbacks def download( self, download_func: Callable[[Request, Spider], Deferred[Response]], request: Request, spider: Spider, - ) -> Deferred[Response | Request]: + ) -> Generator[Deferred[Any], Any, Response | Request]: @inlineCallbacks def process_request( request: Request, @@ -92,9 +91,8 @@ class DownloaderMiddlewareManager(MiddlewareManager): @inlineCallbacks def process_exception( - failure: Failure, - ) -> Generator[Deferred[Any], Any, Failure | Response | Request]: - exception = failure.value + exception: Exception, + ) -> Generator[Deferred[Any], Any, Response | Request]: for method in self.methods["process_exception"]: method = cast(Callable, method) response = yield deferred_from_coro( @@ -109,11 +107,12 @@ class DownloaderMiddlewareManager(MiddlewareManager): ) if response: return response - return failure + raise exception - deferred: Deferred[Response | Request] = mustbe_deferred( - process_request, request - ) - deferred.addErrback(process_exception) - deferred.addCallback(process_response) - return deferred + try: + result: Response | Request = yield mustbe_deferred(process_request, request) + except Exception as ex: + # either returns a request or response (which we pass to process_response()) + # or reraises the exception + result = yield process_exception(ex) + return (yield process_response(result)) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7f5dd0405..658f6e774 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -10,7 +10,7 @@ from __future__ import annotations import logging from time import time from traceback import format_exc -from typing import TYPE_CHECKING, Any, TypeVar, cast +from typing import TYPE_CHECKING, Any, cast from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall @@ -42,8 +42,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -_T = TypeVar("_T") - class _Slot: def __init__( @@ -349,28 +347,32 @@ class ExecutionEngine: signals.request_dropped, request=request, spider=self.spider ) - def download(self, request: Request) -> Deferred[Response]: + @inlineCallbacks + def download(self, request: Request) -> Generator[Deferred[Any], Any, Response]: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - d: Deferred[Response | Request] = self._download(request) - # Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type - d2: Deferred[Response] = d.addBoth(self._downloaded, request) # type: ignore[call-overload] - return d2 + try: + response_or_request = yield self._download(request) + finally: + assert self._slot is not None + self._slot.remove_request(request) + if isinstance(response_or_request, Request): + return (yield self.download(response_or_request)) + return response_or_request - def _downloaded( - self, result: Response | Request | Failure, request: Request - ) -> Deferred[Response] | Response | Failure: - assert self._slot is not None # typing - self._slot.remove_request(request) - return self.download(result) if isinstance(result, Request) else result - - def _download(self, request: Request) -> Deferred[Response | Request]: + @inlineCallbacks + def _download( + self, request: Request + ) -> Generator[Deferred[Any], Any, Response | Request]: assert self._slot is not None # typing + assert self.spider is not None self._slot.add_request(request) - - def _on_success(result: Response | Request) -> Response | Request: + try: + result: Response | Request = yield self.downloader.fetch( + request, self.spider + ) if not isinstance(result, (Response, Request)): raise TypeError( f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}" @@ -391,17 +393,8 @@ class ExecutionEngine: spider=self.spider, ) return result - - def _on_complete(_: _T) -> _T: - assert self._slot is not None + finally: self._slot.nextcall.schedule() - return _ - - assert self.spider is not None - dwld: Deferred[Response | Request] = self.downloader.fetch(request, self.spider) - dwld.addCallback(_on_success) - dwld.addBoth(_on_complete) - return dwld @deferred_f_from_coro_f async def open_spider( diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 9378f2651..2c48a9a81 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -111,11 +111,11 @@ class Scraper: assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter - @inlineCallbacks - def open_spider(self, spider: Spider) -> Generator[Deferred[Any], Any, None]: + @deferred_f_from_coro_f + async def open_spider(self, spider: Spider) -> None: """Open the given spider for scraping and allocate resources for it""" self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) - yield self.itemproc.open_spider(spider) + await maybe_deferred_to_future(self.itemproc.open_spider(spider)) def close_spider(self, spider: Spider | None = None) -> Deferred[Spider]: """Close a spider being scraped and release its resources""" @@ -191,10 +191,8 @@ class Scraper: if isinstance(result, Response): try: # call the spider middlewares and the request callback with the response - output = await maybe_deferred_to_future( - self.spidermw.scrape_response( - self.call_spider, result, request, self.crawler.spider - ) + output = await self.spidermw.scrape_response_async( + self.call_spider, result, request, self.crawler.spider ) except Exception: self.handle_spider_error(Failure(), request, result) @@ -363,12 +361,19 @@ class Scraper: self.crawler.engine.crawl(request=output) return if output is not None: - await maybe_deferred_to_future( - self.start_itemproc(output, response=response) - ) + await self.start_itemproc_async(output, response=response) - @deferred_f_from_coro_f - async def start_itemproc(self, item: Any, *, response: Response | None) -> None: + def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[None]: + """Send *item* to the item pipelines for processing. + + *response* is the source of the item data. If the item does not come + from response data, e.g. it was hard-coded, set it to ``None``. + """ + return deferred_from_coro(self.start_itemproc_async(item, response=response)) + + async def start_itemproc_async( + self, item: Any, *, response: Response | None + ) -> None: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 310abb9b7..10aad7858 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -23,7 +23,6 @@ from scrapy.middleware import MiddlewareManager from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( - deferred_f_from_coro_f, deferred_from_coro, maybe_deferred_to_future, mustbe_deferred, @@ -169,7 +168,7 @@ class SpiderMiddlewareManager(MiddlewareManager): exception_result = cast( Union[Failure, MutableChain[_T]], self._process_spider_exception( - response, spider, Failure(ex), exception_processor_index + response, spider, ex, exception_processor_index ), ) if isinstance(exception_result, Failure): @@ -185,7 +184,7 @@ class SpiderMiddlewareManager(MiddlewareManager): exception_result = cast( Union[Failure, MutableAsyncChain[_T]], self._process_spider_exception( - response, spider, Failure(ex), exception_processor_index + response, spider, ex, exception_processor_index ), ) if isinstance(exception_result, Failure): @@ -201,13 +200,12 @@ class SpiderMiddlewareManager(MiddlewareManager): self, response: Response, spider: Spider, - _failure: Failure, + exception: Exception, start_index: int = 0, - ) -> Failure | MutableChain[_T] | MutableAsyncChain[_T]: - exception = _failure.value + ) -> MutableChain[_T] | MutableAsyncChain[_T]: # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): - return _failure + raise exception method_list = islice( self.methods["process_spider_exception"], start_index, None ) @@ -242,7 +240,7 @@ class SpiderMiddlewareManager(MiddlewareManager): f"or an iterable, got {type(result)}" ) raise _InvalidOutput(msg) - return _failure + raise exception # This method cannot be made async def, as _process_spider_exception relies on the Deferred result # being available immediately which doesn't work when it's a wrapped coroutine. @@ -308,7 +306,7 @@ class SpiderMiddlewareManager(MiddlewareManager): except Exception as ex: exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = ( self._process_spider_exception( - response, spider, Failure(ex), method_index + 1 + response, spider, ex, method_index + 1 ) ) if isinstance(exception_result, Failure): @@ -369,24 +367,36 @@ class SpiderMiddlewareManager(MiddlewareManager): request: Request, spider: Spider, ) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]: + return deferred_from_coro( + self.scrape_response_async(scrape_func, response, request, spider) + ) + + async def scrape_response_async( + self, + scrape_func: ScrapeFunc[_T], + response: Response, + request: Request, + spider: Spider, + ) -> MutableChain[_T] | MutableAsyncChain[_T]: async def process_callback_output( result: Iterable[_T] | AsyncIterator[_T], ) -> MutableChain[_T] | MutableAsyncChain[_T]: return await self._process_callback_output(response, spider, result) def process_spider_exception( - _failure: Failure, - ) -> Failure | MutableChain[_T] | MutableAsyncChain[_T]: - return self._process_spider_exception(response, spider, _failure) + exception: Exception, + ) -> MutableChain[_T] | MutableAsyncChain[_T]: + return self._process_spider_exception(response, spider, exception) - dfd: Deferred[Iterable[_T] | AsyncIterator[_T]] = mustbe_deferred( - self._process_spider_input, scrape_func, response, request, spider - ) - dfd2: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = dfd.addCallback( - deferred_f_from_coro_f(process_callback_output) - ) - dfd2.addErrback(process_spider_exception) - return dfd2 + try: + it: Iterable[_T] | AsyncIterator[_T] = await maybe_deferred_to_future( + mustbe_deferred( + self._process_spider_input, scrape_func, response, request, spider + ) + ) + return await process_callback_output(it) + except Exception as ex: + return process_spider_exception(ex) async def process_start(self, spider: Spider) -> AsyncIterator[Any] | None: self._check_deprecated_start_requests_use(spider) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 749096db5..5dbee6537 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -10,7 +10,6 @@ from twisted.internet.defer import ( Deferred, DeferredList, inlineCallbacks, - maybeDeferred, ) from zope.interface.verify import verifyClass @@ -175,7 +174,7 @@ class Crawler: if self.crawling: self.crawling = False assert self.engine - yield maybeDeferred(self.engine.stop) + yield self.engine.stop() @staticmethod def _get_component( @@ -277,12 +276,6 @@ class CrawlerRunner: process. See :ref:`run-from-script` for an example. """ - crawlers = property( - lambda self: self._crawlers, - doc="Set of :class:`crawlers <scrapy.crawler.Crawler>` started by " - ":meth:`crawl` and managed by this class.", - ) - @staticmethod def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: """Get SpiderLoader instance from settings""" @@ -303,6 +296,12 @@ class CrawlerRunner: self._active: set[Deferred[None]] = set() self.bootstrap_failed = False + @property + def crawlers(self) -> set[Crawler]: + """Set of :class:`crawlers <scrapy.crawler.Crawler>` started by + :meth:`crawl` and managed by this class.""" + return self._crawlers + def crawl( self, crawler_or_spidercls: type[Spider] | str | Crawler, @@ -338,18 +337,19 @@ class CrawlerRunner: crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred[None]: + @inlineCallbacks + def _crawl( + self, crawler: Crawler, *args: Any, **kwargs: Any + ) -> Generator[Deferred[Any], Any, None]: self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) - - def _done(result: _T) -> _T: + try: + yield d + finally: self.crawlers.discard(crawler) self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) - return result - - return d.addBoth(_done) def create_crawler( self, crawler_or_spidercls: type[Spider] | str | Crawler @@ -501,10 +501,12 @@ class CrawlerProcess(CrawlerRunner): ) reactor.run(installSignalHandlers=install_signal_handlers) # blocking call - def _graceful_stop_reactor(self) -> Deferred[Any]: - d = self.stop() - d.addBoth(self._stop_reactor) - return d + @inlineCallbacks + def _graceful_stop_reactor(self) -> Generator[Deferred[Any], Any, None]: + try: + yield self.stop() + finally: + self._stop_reactor() def _stop_reactor(self, _: Any = None) -> None: from twisted.internet import reactor diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 8ae160f8a..61a5a7df5 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -131,6 +131,39 @@ class TestResponseFromProcessRequest(TestManagerBase): assert not download_func.called +class TestResponseFromProcessException(TestManagerBase): + """Tests middleware returning a response from process_exception.""" + + @deferred_f_from_coro_f + async def test_process_response_called(self): + resp = Response("http://example.com/index.html") + calls = [] + + def download_func(request, spider): + raise ValueError("test") + + class ResponseMiddleware: + def process_response(self, request, response, spider): + calls.append("process_response") + return resp + + def process_exception(self, request, exception, spider): + calls.append("process_exception") + return resp + + self.mwman._add_middleware(ResponseMiddleware()) + + req = Request("http://example.com/index.html") + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp + assert calls == [ + "process_exception", + "process_response", + ] + + class TestInvalidOutput(TestManagerBase): @deferred_f_from_coro_f async def test_invalid_process_request(self): From 82acef30517496d622a80f24adb5b3599e63f64a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 14 May 2025 18:21:18 +0400 Subject: [PATCH 1722/2083] Add AsyncCrawlerRunner. (#6796) --- conftest.py | 2 + docs/topics/api.rst | 3 + docs/topics/practices.rst | 178 +++++++++-------- scrapy/crawler.py | 188 ++++++++++++++---- tests/AsyncCrawlerRunner/multi_parallel.py | 28 +++ tests/AsyncCrawlerRunner/multi_seq.py | 27 +++ tests/AsyncCrawlerRunner/simple.py | 26 +++ .../simple_default_reactor.py | 24 +++ .../CrawlerRunner/explicit_default_reactor.py | 28 +++ tests/CrawlerRunner/multi_parallel.py | 26 +++ tests/CrawlerRunner/multi_seq.py | 27 +++ tests/CrawlerRunner/simple.py | 24 +++ tests/test_crawler.py | 143 ++++++++++++- 13 files changed, 589 insertions(+), 135 deletions(-) create mode 100644 tests/AsyncCrawlerRunner/multi_parallel.py create mode 100644 tests/AsyncCrawlerRunner/multi_seq.py create mode 100644 tests/AsyncCrawlerRunner/simple.py create mode 100644 tests/AsyncCrawlerRunner/simple_default_reactor.py create mode 100644 tests/CrawlerRunner/explicit_default_reactor.py create mode 100644 tests/CrawlerRunner/multi_parallel.py create mode 100644 tests/CrawlerRunner/multi_seq.py create mode 100644 tests/CrawlerRunner/simple.py diff --git a/conftest.py b/conftest.py index 8e0c429a0..18132b7e6 100644 --- a/conftest.py +++ b/conftest.py @@ -19,6 +19,8 @@ collect_ignore = [ "tests/mockserver.py", "tests/pipelines.py", "tests/spiders.py", + # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess + *_py_files("tests/AsyncCrawlerRunner"), # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 8e8f3a0c9..3e7bc45c5 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -109,6 +109,9 @@ how you :ref:`configure the downloader middlewares .. automethod:: stop +.. autoclass:: AsyncCrawlerRunner + :members: + .. autoclass:: CrawlerRunner :members: diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index db91cd073..18005aaf2 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -73,72 +73,87 @@ project as example. process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling -process: :class:`scrapy.crawler.CrawlerRunner`. This class is a thin wrapper -that encapsulates some simple helpers to run multiple crawlers, but it won't -start or interfere with existing reactors in any way. +process: :class:`scrapy.crawler.AsyncCrawlerRunner` and +:class:`scrapy.crawler.CrawlerRunner`. These classes are thin wrappers +that encapsulate some simple helpers to run multiple crawlers, but they won't +start or interfere with existing reactors in any way. They have similar +functionality, differing in their asynchronous API style: +:class:`~scrapy.crawler.AsyncCrawlerRunner` returns coroutines from its +asynchronous methods while :class:`~scrapy.crawler.CrawlerRunner` returns +:class:`~twisted.internet.defer.Deferred` objects. -Using this class the reactor should be explicitly run after scheduling your -spiders. It's recommended you use :class:`~scrapy.crawler.CrawlerRunner` -instead of :class:`~scrapy.crawler.CrawlerProcess` if your application is -already using Twisted and you want to run Scrapy in the same reactor. +When using these classes the reactor should be explicitly run after scheduling +your spiders. It's recommended that you use +:class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.CrawlerRunner` instead of +:class:`~scrapy.crawler.CrawlerProcess` if your application is already using +Twisted and you want to run Scrapy in the same reactor. -Note that you will also have to shutdown the Twisted reactor yourself after the -spider is finished. This can be achieved by adding callbacks to the deferred -returned by the :meth:`CrawlerRunner.crawl -<scrapy.crawler.CrawlerRunner.crawl>` method. +If you want to stop the reactor or run any other code right after the spider +finishes you can do that after the :meth:`AsyncCrawlerRunner.crawl() +<scrapy.crawler.AsyncCrawlerRunner.crawl>` coroutine completes (or the Deferred +returned from :meth:`CrawlerRunner.crawl() +<scrapy.crawler.CrawlerRunner.crawl>` fires). In the simplest case you can also +use :func:`twisted.internet.task.react` to start and stop the reactor, though +it may be easier to just use :class:`~scrapy.crawler.CrawlerProcess` instead. -Here's an example of its usage, along with a callback to manually stop the -reactor after ``MySpider`` has finished running. +Here's an example of using :class:`~scrapy.crawler.AsyncCrawlerRunner` together +with simple reactor management code: .. code-block:: python import scrapy - from scrapy.crawler import CrawlerRunner + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.log import configure_logging - - - class MySpider(scrapy.Spider): - # Your spider definition - ... - - - configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) - runner = CrawlerRunner() - - d = runner.crawl(MySpider) - - from twisted.internet import reactor - - d.addBoth(lambda _: reactor.stop()) - reactor.run() # the script will block here until the crawling is finished - -Same example but using a non-default reactor, it's only necessary call -``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. - -.. code-block:: python - - import scrapy - from scrapy.crawler import CrawlerRunner - from scrapy.utils.log import configure_logging - - - class MySpider(scrapy.Spider): - # Your spider definition - ... - - - configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) - from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + async def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner() + await runner.crawl(MySpider) # completes when the spider finishes + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - runner = CrawlerRunner() - d = runner.crawl(MySpider) + react(deferred_f_from_coro_f(crawl)) - from twisted.internet import reactor +Same example but using :class:`~scrapy.crawler.CrawlerRunner` and a +different reactor (:class:`~scrapy.crawler.AsyncCrawlerRunner` only works +with :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`): - d.addBoth(lambda _: reactor.stop()) - reactor.run() # the script will block here until the crawling is finished +.. code-block:: python + + import scrapy + from scrapy.crawler import CrawlerRunner + from scrapy.utils.log import configure_logging + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react + + + class MySpider(scrapy.Spider): + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.epollreactor.EPollReactor", + } + # Your spider definition + ... + + + def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = CrawlerRunner() + d = runner.crawl(MySpider) + return d # this Deferred fires when the spider finishes + + + install_reactor("twisted.internet.epollreactor.EPollReactor") + react(crawl) .. seealso:: :doc:`twisted:core/howto/reactor-basics` @@ -176,14 +191,16 @@ Here is an example that runs multiple spiders simultaneously: process.crawl(MySpider2) process.start() # the script will block here until all crawling jobs are finished -Same example using :class:`~scrapy.crawler.CrawlerRunner`: +Same example using :class:`~scrapy.crawler.AsyncCrawlerRunner`: .. code-block:: python import scrapy - from scrapy.crawler import CrawlerRunner + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.log import configure_logging - from scrapy.utils.project import get_project_settings + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react class MySpider1(scrapy.Spider): @@ -196,27 +213,29 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: ... - configure_logging() - settings = get_project_settings() - runner = CrawlerRunner(settings) - runner.crawl(MySpider1) - runner.crawl(MySpider2) - d = runner.join() + async def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner() + runner.crawl(MySpider1) + runner.crawl(MySpider2) + await runner.join() # completes when both spiders finish - from twisted.internet import reactor - d.addBoth(lambda _: reactor.stop()) + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + react(deferred_f_from_coro_f(crawl)) - reactor.run() # the script will block here until all crawling jobs are finished -Same example but running the spiders sequentially by chaining the deferreds: +Same example but running the spiders sequentially by awaiting until each one +finishes before starting the next one: .. code-block:: python - from twisted.internet import defer - from scrapy.crawler import CrawlerRunner + import scrapy + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.log import configure_logging - from scrapy.utils.project import get_project_settings + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react class MySpider1(scrapy.Spider): @@ -229,22 +248,15 @@ Same example but running the spiders sequentially by chaining the deferreds: ... - settings = get_project_settings() - configure_logging(settings) - runner = CrawlerRunner(settings) + async def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner() + await runner.crawl(MySpider1) + await runner.crawl(MySpider2) - @defer.inlineCallbacks - def crawl(): - yield runner.crawl(MySpider1) - yield runner.crawl(MySpider2) - reactor.stop() - - - from twisted.internet import reactor - - crawl() - reactor.run() # the script will block here until the last crawl call is finished + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + react(deferred_f_from_coro_f(crawl)) .. note:: When running multiple spiders in the same process, :ref:`reactor settings <reactor-settings>` should not have a different value per spider. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 5dbee6537..1d6532fa9 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import contextlib import logging import pprint @@ -20,6 +21,7 @@ from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager +from scrapy.utils.defer import deferred_to_future from scrapy.utils.log import ( LogCounterHandler, configure_logging, @@ -263,19 +265,7 @@ class Crawler: return self._get_component(cls, self.engine.scraper.spidermw.middlewares) -class CrawlerRunner: - """ - This is a convenient helper class that keeps track of, manages and runs - crawlers inside an already setup :mod:`~twisted.internet.reactor`. - - The CrawlerRunner object must be instantiated with a - :class:`~scrapy.settings.Settings` object. - - This class shouldn't be needed (since Scrapy is responsible of using it - accordingly) unless writing scripts that manually handle the crawling - process. See :ref:`run-from-script` for an example. - """ - +class CrawlerRunnerBase: @staticmethod def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: """Get SpiderLoader instance from settings""" @@ -293,7 +283,6 @@ class CrawlerRunner: self.settings: Settings = settings self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings) self._crawlers: set[Crawler] = set() - self._active: set[Deferred[None]] = set() self.bootstrap_failed = False @property @@ -302,6 +291,57 @@ class CrawlerRunner: :meth:`crawl` and managed by this class.""" return self._crawlers + def create_crawler( + self, crawler_or_spidercls: type[Spider] | str | Crawler + ) -> Crawler: + """ + Return a :class:`~scrapy.crawler.Crawler` object. + + * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. + * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler + is constructed for it. + * If ``crawler_or_spidercls`` is a string, this function finds + a spider with this name in a Scrapy project (using spider loader), + then creates a Crawler instance for it. + """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + "The crawler_or_spidercls argument cannot be a spider object, " + "it must be a spider class (or a Crawler object)" + ) + if isinstance(crawler_or_spidercls, Crawler): + return crawler_or_spidercls + return self._create_crawler(crawler_or_spidercls) + + def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler: + if isinstance(spidercls, str): + spidercls = self.spider_loader.load(spidercls) + return Crawler(spidercls, self.settings) + + def _stop(self) -> Deferred[Any]: + return DeferredList([c.stop() for c in list(self.crawlers)]) + + +class CrawlerRunner(CrawlerRunnerBase): + """ + This is a convenient helper class that keeps track of, manages and runs + crawlers inside an already setup :mod:`~twisted.internet.reactor`. + + The CrawlerRunner object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides Deferred-based APIs. Use :class:`AsyncCrawlerRunner` + for modern coroutine APIs. + """ + + def __init__(self, settings: dict[str, Any] | Settings | None = None): + super().__init__(settings) + self._active: set[Deferred[None]] = set() + def crawl( self, crawler_or_spidercls: type[Spider] | str | Crawler, @@ -351,40 +391,13 @@ class CrawlerRunner: self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) - def create_crawler( - self, crawler_or_spidercls: type[Spider] | str | Crawler - ) -> Crawler: - """ - Return a :class:`~scrapy.crawler.Crawler` object. - - * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. - * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler - is constructed for it. - * If ``crawler_or_spidercls`` is a string, this function finds - a spider with this name in a Scrapy project (using spider loader), - then creates a Crawler instance for it. - """ - if isinstance(crawler_or_spidercls, Spider): - raise ValueError( - "The crawler_or_spidercls argument cannot be a spider object, " - "it must be a spider class (or a Crawler object)" - ) - if isinstance(crawler_or_spidercls, Crawler): - return crawler_or_spidercls - return self._create_crawler(crawler_or_spidercls) - - def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler: - if isinstance(spidercls, str): - spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings) - def stop(self) -> Deferred[Any]: """ Stops simultaneously all the crawling jobs taking place. Returns a deferred that is fired when they all have ended. """ - return DeferredList([c.stop() for c in list(self.crawlers)]) + return self._stop() @inlineCallbacks def join(self) -> Generator[Deferred[Any], Any, None]: @@ -398,6 +411,96 @@ class CrawlerRunner: yield DeferredList(self._active) +class AsyncCrawlerRunner(CrawlerRunnerBase): + """ + This is a convenient helper class that keeps track of, manages and runs + crawlers inside an already setup :mod:`~twisted.internet.reactor`. + + The AsyncCrawlerRunner object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides coroutine APIs. It requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + """ + + def __init__(self, settings: dict[str, Any] | Settings | None = None): + super().__init__(settings) + self._active: set[asyncio.Future[None]] = set() + + def crawl( + self, + crawler_or_spidercls: type[Spider] | str | Crawler, + *args: Any, + **kwargs: Any, + ) -> asyncio.Future[None]: + """ + Run a crawler with the provided arguments. + + It will call the given Crawler's :meth:`~Crawler.crawl` method, while + keeping track of it so it can be stopped later. + + If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` + instance, this method will try to create one using this parameter as + the spider class given to it. + + Returns a :class:`~asyncio.Future` object which completes when the + crawling is finished. + + :param crawler_or_spidercls: already created crawler, or a spider class + or spider's name inside the project to create it + :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance, + :class:`~scrapy.spiders.Spider` subclass or string + + :param args: arguments to initialize the spider + + :param kwargs: keyword arguments to initialize the spider + """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + "The crawler_or_spidercls argument cannot be a spider object, " + "it must be a spider class (or a Crawler object)" + ) + if not is_asyncio_reactor_installed(): + raise RuntimeError("AsyncCrawlerRunner requires AsyncioSelectorReactor.") + crawler = self.create_crawler(crawler_or_spidercls) + return self._crawl(crawler, *args, **kwargs) + + def _crawl( + self, crawler: Crawler, *args: Any, **kwargs: Any + ) -> asyncio.Future[None]: + self.crawlers.add(crawler) + future = deferred_to_future(crawler.crawl(*args, **kwargs)) + self._active.add(future) + + def _done(_: asyncio.Future[None]) -> None: + self.crawlers.discard(crawler) + self._active.discard(future) + self.bootstrap_failed |= not getattr(crawler, "spider", None) + + future.add_done_callback(_done) + return future + + async def stop(self) -> None: + """ + Stops simultaneously all the crawling jobs taking place. + + Completes when they all have ended. + """ + await deferred_to_future(self._stop()) + + async def join(self) -> None: + """ + Completes when all managed :attr:`crawlers` have completed their + executions. + """ + while self._active: + await asyncio.gather(*self._active) + + class CrawlerProcess(CrawlerRunner): """ A class to run multiple scrapy crawlers in a process simultaneously. @@ -458,7 +561,6 @@ class CrawlerProcess(CrawlerRunner): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True - # temporary cast until self.spider_loader is typed return Crawler(spidercls, self.settings, init_reactor=init_reactor) def start( diff --git a/tests/AsyncCrawlerRunner/multi_parallel.py b/tests/AsyncCrawlerRunner/multi_parallel.py new file mode 100644 index 000000000..f1af9f794 --- /dev/null +++ b/tests/AsyncCrawlerRunner/multi_parallel.py @@ -0,0 +1,28 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + runner.crawl(NoRequestsSpider) + runner.crawl(NoRequestsSpider) + await runner.join() + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/multi_seq.py b/tests/AsyncCrawlerRunner/multi_seq.py new file mode 100644 index 000000000..987f7a514 --- /dev/null +++ b/tests/AsyncCrawlerRunner/multi_seq.py @@ -0,0 +1,27 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/simple.py b/tests/AsyncCrawlerRunner/simple.py new file mode 100644 index 000000000..140777b4f --- /dev/null +++ b/tests/AsyncCrawlerRunner/simple.py @@ -0,0 +1,26 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/simple_default_reactor.py b/tests/AsyncCrawlerRunner/simple_default_reactor.py new file mode 100644 index 000000000..ae052f188 --- /dev/null +++ b/tests/AsyncCrawlerRunner/simple_default_reactor.py @@ -0,0 +1,24 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +react(main) diff --git a/tests/CrawlerRunner/explicit_default_reactor.py b/tests/CrawlerRunner/explicit_default_reactor.py new file mode 100644 index 000000000..9eb8a39bb --- /dev/null +++ b/tests/CrawlerRunner/explicit_default_reactor.py @@ -0,0 +1,28 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": None, + } + + async def start(self): + return + yield + + +def main(reactor): + configure_logging( + {"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"} + ) + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +react(main) diff --git a/tests/CrawlerRunner/multi_parallel.py b/tests/CrawlerRunner/multi_parallel.py new file mode 100644 index 000000000..51feccd0a --- /dev/null +++ b/tests/CrawlerRunner/multi_parallel.py @@ -0,0 +1,26 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + runner.crawl(NoRequestsSpider) + runner.crawl(NoRequestsSpider) + return runner.join() + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/CrawlerRunner/multi_seq.py b/tests/CrawlerRunner/multi_seq.py new file mode 100644 index 000000000..f6549be9b --- /dev/null +++ b/tests/CrawlerRunner/multi_seq.py @@ -0,0 +1,27 @@ +from twisted.internet.defer import inlineCallbacks +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@inlineCallbacks +def main(reactor): + configure_logging() + runner = CrawlerRunner() + yield runner.crawl(NoRequestsSpider) + yield runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/CrawlerRunner/simple.py b/tests/CrawlerRunner/simple.py new file mode 100644 index 000000000..d154dcde4 --- /dev/null +++ b/tests/CrawlerRunner/simple.py @@ -0,0 +1,24 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 7a3d562e5..a1d3c02fb 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -18,11 +18,12 @@ from zope.interface.exceptions import MultipleInvalid import scrapy from scrapy import Spider -from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner +from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader +from scrapy.utils.defer import deferred_from_coro from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings @@ -558,6 +559,26 @@ class TestCrawlerRunner(TestBaseCrawler): self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") +class TestAsyncCrawlerRunner(TestBaseCrawler): + def test_spider_manager_verify_interface(self): + settings = Settings( + { + "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, + } + ) + with pytest.raises(MultipleInvalid): + AsyncCrawlerRunner(settings) + + def test_crawler_runner_accepts_dict(self): + runner = AsyncCrawlerRunner({"foo": "bar"}) + assert runner.settings["foo"] == "bar" + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + def test_crawler_runner_accepts_None(self): + runner = AsyncCrawlerRunner() + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + class TestCrawlerProcess(TestBaseCrawler): def test_crawler_process_accepts_dict(self): runner = CrawlerProcess({"foo": "bar"}) @@ -587,20 +608,25 @@ class NoRequestsSpider(scrapy.Spider): @pytest.mark.usefixtures("reactor_pytest") class TestCrawlerRunnerHasSpider(unittest.TestCase): - def _runner(self): + @staticmethod + def _runner(): return CrawlerRunner(get_reactor_settings()) + @staticmethod + def _crawl(runner, spider): + return runner.crawl(spider) + @inlineCallbacks def test_crawler_runner_bootstrap_successful(self): runner = self._runner() - yield runner.crawl(NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed @inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() - yield runner.crawl(NoRequestsSpider) - yield runner.crawl(NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed @inlineCallbacks @@ -608,7 +634,7 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): runner = self._runner() try: - yield runner.crawl(ExceptionSpider) + yield self._crawl(runner, ExceptionSpider) except ValueError: pass else: @@ -621,13 +647,13 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): runner = self._runner() try: - yield runner.crawl(ExceptionSpider) + yield self._crawl(runner, ExceptionSpider) except ValueError: pass else: pytest.fail("Exception should be raised from spider") - yield runner.crawl(NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) assert runner.bootstrap_failed @@ -643,7 +669,7 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): Exception, match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", ): - yield runner.crawl(NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) else: CrawlerRunner( settings={ @@ -652,6 +678,20 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): ) +@pytest.mark.only_asyncio +class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): + @staticmethod + def _runner(): + return AsyncCrawlerRunner(get_reactor_settings()) + + @staticmethod + def _crawl(runner, spider): + return deferred_from_coro(runner.crawl(spider)) + + def test_crawler_runner_asyncio_enabled_true(self): + pytest.skip("This test is only for CrawlerRunner") + + class ScriptRunnerMixin: script_dir: Path @@ -923,6 +963,48 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" + def test_simple(self): + log = self.run_script("simple.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_explicit_default_reactor(self): + log = self.run_script("explicit_default_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + + def test_multi_parallel(self): + log = self.run_script("multi_parallel.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Spider opened.+Closing spider.+Closing spider", + log, + re.DOTALL, + ) + + def test_multi_seq(self): + log = self.run_script("multi_seq.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Closing spider.+Spider opened.+Closing spider", + log, + re.DOTALL, + ) + def test_response_ip_address(self): log = self.run_script("ip_address.py") assert "INFO: Spider closed (finished)" in log @@ -939,6 +1021,49 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): assert "DEBUG: Using asyncio event loop" in log +class TestAsyncCrawlerRunnerSubprocess(ScriptRunnerMixin): + script_dir = Path(__file__).parent.resolve() / "AsyncCrawlerRunner" + + def test_simple(self): + log = self.run_script("simple.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_simple_default_reactor(self): + log = self.run_script("simple_default_reactor.py") + assert "Spider closed (finished)" not in log + assert "RuntimeError: AsyncCrawlerRunner requires AsyncioSelectorReactor" in log + + def test_multi_parallel(self): + log = self.run_script("multi_parallel.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Spider opened.+Closing spider.+Closing spider", + log, + re.DOTALL, + ) + + def test_multi_seq(self): + log = self.run_script("multi_seq.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Closing spider.+Spider opened.+Closing spider", + log, + re.DOTALL, + ) + + @pytest.mark.parametrize( ("settings", "items"), [ From 1ddcb568e27cda10db4d0640aa42d020d4624a30 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 15 May 2025 14:18:01 +0500 Subject: [PATCH 1723/2083] Add send_catch_log_async(). --- docs/topics/signals.rst | 22 +++++++++--------- scrapy/core/engine.py | 12 +++------- scrapy/core/scraper.py | 40 ++++++++++++++------------------- scrapy/extensions/feedexport.py | 4 +--- scrapy/signalmanager.py | 20 ++++++++++++++--- scrapy/utils/signal.py | 38 +++++++++++++++++++++---------- tests/test_utils_signal.py | 39 +++++++++++++++++++++++++++++++- 7 files changed, 113 insertions(+), 62 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 66cb87fc5..59742ffeb 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -46,8 +46,8 @@ Here is a simple example showing how you can catch signals and perform some acti .. _signal-deferred: -Deferred signal handlers -======================== +Asynchronous signal handlers +============================ Some signals support returning :class:`~twisted.internet.defer.Deferred` or :term:`awaitable objects <awaitable>` from their handlers, allowing @@ -114,7 +114,7 @@ engine_started Sent when the Scrapy engine has started crawling. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. .. note:: This signal may be fired *after* the :signal:`spider_opened` signal, depending on how the spider was started. So **don't** rely on this signal @@ -129,7 +129,7 @@ engine_stopped Sent when the Scrapy engine is stopped (for example, when a crawling process has finished). - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. scheduler_empty ~~~~~~~~~~~~~~~ @@ -164,7 +164,7 @@ item_scraped Sent when an item has been scraped, after it has passed all the :ref:`topics-item-pipeline` stages (without being dropped). - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. :param item: the scraped item :type item: :ref:`item object <item-types>` @@ -185,7 +185,7 @@ item_dropped Sent after an item has been dropped from the :ref:`topics-item-pipeline` when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: :ref:`item object <item-types>` @@ -211,7 +211,7 @@ item_error Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises an exception), except :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. :param item: the item that caused the error in the :ref:`topics-item-pipeline` :type item: :ref:`item object <item-types>` @@ -239,7 +239,7 @@ spider_closed Sent after a spider has been closed. This can be used to release per-spider resources reserved on :signal:`spider_opened`. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. :param spider: the spider which has been closed :type spider: :class:`~scrapy.Spider` object @@ -263,7 +263,7 @@ spider_opened reserve per-spider resources, but can be used for any task that needs to be performed when a spider is opened. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. :param spider: the spider which has been opened :type spider: :class:`~scrapy.Spider` object @@ -332,7 +332,7 @@ feed_slot_closed Sent when a :ref:`feed exports <topics-feed-exports>` slot is closed. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. :param slot: the slot closed :type slot: scrapy.extensions.feedexport.FeedSlot @@ -348,7 +348,7 @@ feed_exporter_closed during the handling of the :signal:`spider_closed` signal by the extension, after all feed exporting has been handled. - This signal supports returning deferreds from its handlers. + This signal supports asynchronous handlers. Request signals diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 658f6e774..b0d9a5452 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -127,9 +127,7 @@ class ExecutionEngine: if self.running: raise RuntimeError("Engine already running") self.start_time = time() - await maybe_deferred_to_future( - self.signals.send_catch_log_deferred(signal=signals.engine_started) - ) + await self.signals.send_catch_log_async(signal=signals.engine_started) self.running = True self._closewait: Deferred[None] = Deferred() if _start_request_processing: @@ -141,9 +139,7 @@ class ExecutionEngine: @deferred_f_from_coro_f async def _finish_stopping_engine(_: Any) -> None: - await maybe_deferred_to_future( - self.signals.send_catch_log_deferred(signal=signals.engine_stopped) - ) + await self.signals.send_catch_log_async(signal=signals.engine_stopped) self._closewait.callback(None) if not self.running: @@ -415,9 +411,7 @@ class ExecutionEngine: await maybe_deferred_to_future(self.scraper.open_spider(spider)) assert self.crawler.stats self.crawler.stats.open_spider(spider) - await maybe_deferred_to_future( - self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) - ) + await self.signals.send_catch_log_async(signals.spider_opened, spider=spider) def _spider_idle(self) -> None: """ diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 2c48a9a81..9fc1d20ed 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -392,14 +392,12 @@ class Scraper: logger.log( *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} ) - await maybe_deferred_to_future( - self.signals.send_catch_log_deferred( - signal=signals.item_dropped, - item=item, - response=response, - spider=self.crawler.spider, - exception=ex, - ) + await self.signals.send_catch_log_async( + signal=signals.item_dropped, + item=item, + response=response, + spider=self.crawler.spider, + exception=ex, ) except Exception as ex: logkws = self.logformatter.item_error( @@ -410,14 +408,12 @@ class Scraper: extra={"spider": self.crawler.spider}, exc_info=True, ) - await maybe_deferred_to_future( - self.signals.send_catch_log_deferred( - signal=signals.item_error, - item=item, - response=response, - spider=self.crawler.spider, - failure=Failure(), - ) + await self.signals.send_catch_log_async( + signal=signals.item_error, + item=item, + response=response, + spider=self.crawler.spider, + failure=Failure(), ) else: logkws = self.logformatter.scraped(output, response, self.crawler.spider) @@ -425,13 +421,11 @@ class Scraper: logger.log( *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} ) - await maybe_deferred_to_future( - self.signals.send_catch_log_deferred( - signal=signals.item_scraped, - item=output, - response=response, - spider=self.crawler.spider, - ) + await self.signals.send_catch_log_async( + signal=signals.item_scraped, + item=output, + response=response, + spider=self.crawler.spider, ) finally: self.slot.itemproc_size -= 1 diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 8bcd4e40d..c39a9c92e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -531,9 +531,7 @@ class FeedExporter: await maybe_deferred_to_future(DeferredList(self._pending_deferreds)) # Send FEED_EXPORTER_CLOSED signal - await maybe_deferred_to_future( - self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) - ) + await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed) def _close_slot(self, slot: FeedSlot, spider: Spider) -> Deferred[None] | None: def get_file(slot_: FeedSlot) -> IO[bytes]: diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index f8c50b5e3..7fd172535 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -53,11 +53,10 @@ class SignalManager: self, signal: Any, **kwargs: Any ) -> Deferred[list[tuple[Any, Any]]]: """ - Like :meth:`send_catch_log` but supports returning - :class:`~twisted.internet.defer.Deferred` objects from signal handlers. + Like :meth:`send_catch_log` but supports asynchronous signal handlers. Returns a Deferred that gets fired once all signal handlers - deferreds were fired. Send a signal, catch exceptions and log them. + have finished. Send a signal, catch exceptions and log them. The keyword arguments are passed to the signal handlers (connected through the :meth:`connect` method). @@ -65,6 +64,21 @@ class SignalManager: kwargs.setdefault("sender", self.sender) return _signal.send_catch_log_deferred(signal, **kwargs) + async def send_catch_log_async( + self, signal: Any, **kwargs: Any + ) -> list[tuple[Any, Any]]: + """ + Like :meth:`send_catch_log` but supports asynchronous signal handlers. + + Returns a coroutine that completes once all signal handlers + have finished. Send a signal, catch exceptions and log them. + + The keyword arguments are passed to the signal handlers (connected + through the :meth:`connect` method). + """ + kwargs.setdefault("sender", self.sender) + return await _signal.send_catch_log_async(signal, **kwargs) + def disconnect_all(self, signal: Any, **kwargs: Any) -> None: """ Disconnect all receivers from the given signal. diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 5fd176a3f..d6b0a671b 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging -from collections.abc import Sequence +from collections.abc import Generator, Sequence from typing import Any as TypingAny from pydispatch.dispatcher import ( @@ -14,11 +14,11 @@ from pydispatch.dispatcher import ( liveReceivers, ) from pydispatch.robustapply import robustApply -from twisted.internet.defer import Deferred, DeferredList +from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks from twisted.python.failure import Failure from scrapy.exceptions import StopDownload -from scrapy.utils.defer import maybeDeferred_coro +from scrapy.utils.defer import maybe_deferred_to_future, maybeDeferred_coro from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) @@ -66,18 +66,19 @@ def send_catch_log( return responses +@inlineCallbacks def send_catch_log_deferred( signal: TypingAny = Any, sender: TypingAny = Anonymous, *arguments: TypingAny, **named: TypingAny, -) -> Deferred[list[tuple[TypingAny, TypingAny]]]: - """Like send_catch_log but supports returning deferreds on signal handlers. - Returns a deferred that gets fired once all signal handlers deferreds were - fired. +) -> Generator[Deferred[TypingAny], TypingAny, list[tuple[TypingAny, TypingAny]]]: + """Like send_catch_log but supports asynchronous signal handlers. + + Returns a deferred that gets fired once all signal handlers have finished. """ - def logerror(failure: Failure, recv: Any) -> Failure: + def logerror(failure: Failure, recv: TypingAny) -> Failure: if dont_log is None or not isinstance(failure.value, dont_log): logger.error( "Error caught on signal handler: %(receiver)s", @@ -103,11 +104,24 @@ def send_catch_log_deferred( ) ) dfds.append(d2) - dl = DeferredList(dfds) - d3: Deferred[list[tuple[TypingAny, TypingAny]]] = dl.addCallback( - lambda out: [x[1] for x in out] + + results = yield DeferredList(dfds) + return [result[1] for result in results] + + +async def send_catch_log_async( + signal: TypingAny = Any, + sender: TypingAny = Anonymous, + *arguments: TypingAny, + **named: TypingAny, +) -> list[tuple[TypingAny, TypingAny]]: + """Like send_catch_log but supports asynchronous signal handlers. + + Returns a coroutine that completes once all signal handlers have finished. + """ + return await maybe_deferred_to_future( + send_catch_log_deferred(signal, sender, *arguments, **named) ) - return d3 def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 751a77031..6dff321da 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -7,7 +7,12 @@ from twisted.internet import defer, reactor from twisted.python.failure import Failure from twisted.trial import unittest -from scrapy.utils.signal import send_catch_log, send_catch_log_deferred +from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.signal import ( + send_catch_log, + send_catch_log_async, + send_catch_log_deferred, +) from scrapy.utils.test import get_from_asyncio_queue @@ -85,6 +90,38 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): return await get_from_asyncio_queue("OK") +class SendCatchLogAsyncTest(TestSendCatchLog): + def _get_result(self, signal, *a, **kw): + return deferred_from_coro(send_catch_log_async(signal, *a, **kw)) + + +class SendCatchLogAsyncTest2(SendCatchLogAsyncTest): + def ok_handler(self, arg, handlers_called): + handlers_called.add(self.ok_handler) + assert arg == "test" + d = defer.Deferred() + reactor.callLater(0, d.callback, "OK") + return d + + +@pytest.mark.usefixtures("reactor_pytest") +class SendCatchLogAsyncAsyncDefTest(SendCatchLogAsyncTest): + async def ok_handler(self, arg, handlers_called): + handlers_called.add(self.ok_handler) + assert arg == "test" + await defer.succeed(42) + return "OK" + + +@pytest.mark.only_asyncio +class SendCatchLogAsyncAsyncioTest(SendCatchLogAsyncTest): + async def ok_handler(self, arg, handlers_called): + handlers_called.add(self.ok_handler) + assert arg == "test" + await asyncio.sleep(0.2) + return await get_from_asyncio_queue("OK") + + class TestSendCatchLog2: def test_error_logged_if_deferred_not_supported(self): def test_handler(): From bf1bfaaa3e584b085f78e9f89ab03c22cfbb3e59 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 15 May 2025 20:02:38 +0500 Subject: [PATCH 1724/2083] Slight improvements for the signal docs. --- docs/topics/signals.rst | 66 ++++++++++++++++++++++------------------- scrapy/signalmanager.py | 6 ++-- scrapy/utils/signal.py | 10 ++++--- 3 files changed, 46 insertions(+), 36 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 59742ffeb..a815ffb43 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -60,6 +60,7 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`: .. code-block:: python import scrapy + import treq class SignalSpider(scrapy.Spider): @@ -103,6 +104,7 @@ Built-in signals reference Here's the list of Scrapy built-in signals and their meaning. + Engine signals -------------- @@ -114,7 +116,7 @@ engine_started Sent when the Scrapy engine has started crawling. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. .. note:: This signal may be fired *after* the :signal:`spider_opened` signal, depending on how the spider was started. So **don't** rely on this signal @@ -129,7 +131,7 @@ engine_stopped Sent when the Scrapy engine is stopped (for example, when a crawling process has finished). - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. scheduler_empty ~~~~~~~~~~~~~~~ @@ -144,6 +146,9 @@ scheduler_empty See :ref:`start-requests-lazy` for an example. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. + + Item signals ------------ @@ -164,7 +169,7 @@ item_scraped Sent when an item has been scraped, after it has passed all the :ref:`topics-item-pipeline` stages (without being dropped). - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. :param item: the scraped item :type item: :ref:`item object <item-types>` @@ -185,7 +190,7 @@ item_dropped Sent after an item has been dropped from the :ref:`topics-item-pipeline` when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: :ref:`item object <item-types>` @@ -211,7 +216,7 @@ item_error Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises an exception), except :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. :param item: the item that caused the error in the :ref:`topics-item-pipeline` :type item: :ref:`item object <item-types>` @@ -227,6 +232,7 @@ item_error :param failure: the exception raised :type failure: twisted.python.failure.Failure + Spider signals -------------- @@ -239,7 +245,7 @@ spider_closed Sent after a spider has been closed. This can be used to release per-spider resources reserved on :signal:`spider_opened`. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. :param spider: the spider which has been closed :type spider: :class:`~scrapy.Spider` object @@ -263,7 +269,7 @@ spider_opened reserve per-spider resources, but can be used for any task that needs to be performed when a spider is opened. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. :param spider: the spider which has been opened :type spider: :class:`~scrapy.Spider` object @@ -294,16 +300,16 @@ spider_idle accordingly (e.g. setting it to 'too_few_results' instead of 'finished'). - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param spider: the spider which has gone idle :type spider: :class:`~scrapy.Spider` object -.. note:: Scheduling some requests in your :signal:`spider_idle` handler does - **not** guarantee that it can prevent the spider from being closed, - although it sometimes can. That's because the spider may still remain idle - if all the scheduled requests are rejected by the scheduler (e.g. filtered - due to duplication). + .. note:: Scheduling some requests in your :signal:`spider_idle` handler does + **not** guarantee that it can prevent the spider from being closed, + although it sometimes can. That's because the spider may still remain idle + if all the scheduled requests are rejected by the scheduler (e.g. filtered + due to duplication). spider_error ~~~~~~~~~~~~ @@ -313,7 +319,7 @@ spider_error Sent when a spider callback generates an error (i.e. raises an exception). - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param failure: the exception raised :type failure: twisted.python.failure.Failure @@ -332,12 +338,11 @@ feed_slot_closed Sent when a :ref:`feed exports <topics-feed-exports>` slot is closed. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. :param slot: the slot closed :type slot: scrapy.extensions.feedexport.FeedSlot - feed_exporter_closed ~~~~~~~~~~~~~~~~~~~~ @@ -348,7 +353,7 @@ feed_exporter_closed during the handling of the :signal:`spider_closed` signal by the extension, after all feed exporting has been handled. - This signal supports asynchronous handlers. + This signal supports :ref:`asynchronous handlers <signal-deferred>`. Request signals @@ -367,7 +372,7 @@ request_scheduled Raise :exc:`~scrapy.exceptions.IgnoreRequest` to drop a request before it reaches the scheduler. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. .. versionadded:: 2.11.2 Allow dropping requests with :exc:`~scrapy.exceptions.IgnoreRequest`. @@ -387,7 +392,7 @@ request_dropped Sent when a :class:`~scrapy.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param request: the request that reached the scheduler :type request: :class:`~scrapy.Request` object @@ -403,7 +408,7 @@ request_reached_downloader Sent when a :class:`~scrapy.Request` reached downloader. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param request: the request that reached downloader :type request: :class:`~scrapy.Request` object @@ -422,7 +427,7 @@ request_left_downloader Sent when a :class:`~scrapy.Request` leaves the downloader, even in case of failure. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param request: the request that reached the downloader :type request: :class:`~scrapy.Request` object @@ -433,11 +438,11 @@ request_left_downloader bytes_received ~~~~~~~~~~~~~~ -.. versionadded:: 2.2 - .. signal:: bytes_received .. function:: bytes_received(data, request, spider) + .. versionadded:: 2.2 + Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is received for a specific request. This signal might be fired multiple times for the same request, with partial data each time. For instance, @@ -449,7 +454,7 @@ bytes_received exception. Please refer to the :ref:`topics-stop-response-download` topic for additional information and examples. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param data: the data received by the download handler :type data: :class:`bytes` object @@ -463,11 +468,11 @@ bytes_received headers_received ~~~~~~~~~~~~~~~~ -.. versionadded:: 2.5 - .. signal:: headers_received .. function:: headers_received(headers, body_length, request, spider) + .. versionadded:: 2.5 + Sent by the HTTP 1.1 and S3 download handlers when the response headers are available for a given request, before downloading any additional content. @@ -476,7 +481,7 @@ headers_received exception. Please refer to the :ref:`topics-stop-response-download` topic for additional information and examples. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param headers: the headers received by the download handler :type headers: :class:`scrapy.http.headers.Headers` object @@ -490,6 +495,7 @@ headers_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.Spider` object + Response signals ---------------- @@ -502,7 +508,7 @@ response_received Sent when the engine receives a new :class:`~scrapy.http.Response` from the downloader. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param response: the response received :type response: :class:`~scrapy.http.Response` object @@ -524,9 +530,9 @@ response_downloaded .. signal:: response_downloaded .. function:: response_downloaded(response, request, spider) - Sent by the downloader right after a ``HTTPResponse`` is downloaded. + Sent by the downloader right after a :class:`~scrapy.http.Response` is downloaded. - This signal does not support returning deferreds from its handlers. + This signal does not support :ref:`asynchronous handlers <signal-deferred>`. :param response: the response downloaded :type response: :class:`~scrapy.http.Response` object diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 7fd172535..283060074 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -53,7 +53,8 @@ class SignalManager: self, signal: Any, **kwargs: Any ) -> Deferred[list[tuple[Any, Any]]]: """ - Like :meth:`send_catch_log` but supports asynchronous signal handlers. + Like :meth:`send_catch_log` but supports :ref:`asynchronous signal + handlers <signal-deferred>`. Returns a Deferred that gets fired once all signal handlers have finished. Send a signal, catch exceptions and log them. @@ -68,7 +69,8 @@ class SignalManager: self, signal: Any, **kwargs: Any ) -> list[tuple[Any, Any]]: """ - Like :meth:`send_catch_log` but supports asynchronous signal handlers. + Like :meth:`send_catch_log` but supports :ref:`asynchronous signal + handlers <signal-deferred>`. Returns a coroutine that completes once all signal handlers have finished. Send a signal, catch exceptions and log them. diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index d6b0a671b..552fbaa90 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -30,7 +30,7 @@ def send_catch_log( *arguments: TypingAny, **named: TypingAny, ) -> list[tuple[TypingAny, TypingAny]]: - """Like pydispatcher.robust.sendRobust but it also logs errors and returns + """Like ``pydispatcher.robust.sendRobust()`` but it also logs errors and returns Failures instead of exceptions. """ dont_log = named.pop("dont_log", ()) @@ -73,7 +73,8 @@ def send_catch_log_deferred( *arguments: TypingAny, **named: TypingAny, ) -> Generator[Deferred[TypingAny], TypingAny, list[tuple[TypingAny, TypingAny]]]: - """Like send_catch_log but supports asynchronous signal handlers. + """Like :func:`send_catch_log` but supports :ref:`asynchronous signal handlers + <signal-deferred>`. Returns a deferred that gets fired once all signal handlers have finished. """ @@ -115,7 +116,8 @@ async def send_catch_log_async( *arguments: TypingAny, **named: TypingAny, ) -> list[tuple[TypingAny, TypingAny]]: - """Like send_catch_log but supports asynchronous signal handlers. + """Like :func:`send_catch_log` but supports :ref:`asynchronous signal handlers + <signal-deferred>`. Returns a coroutine that completes once all signal handlers have finished. """ @@ -126,7 +128,7 @@ async def send_catch_log_async( def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: """Disconnect all signal handlers. Useful for cleaning up after running - tests + tests. """ for receiver in liveReceivers(getAllReceivers(sender, signal)): disconnect(receiver, signal=signal, sender=sender) From 3c2cd53abb0651f3ac093e71ad340626f72ad0a3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 15 May 2025 22:17:37 +0500 Subject: [PATCH 1725/2083] Skip the doctest. --- docs/topics/signals.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index a815ffb43..aa27e62dd 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -57,6 +57,7 @@ operation to finish. Let's take an example using :ref:`coroutines <topics-coroutines>`: +.. skip: next .. code-block:: python import scrapy From b9caaf8a63bc3280645dca2788a8c4ed1a556769 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 16 May 2025 17:13:52 +0500 Subject: [PATCH 1726/2083] Simplify deferred_from_coro(), add more tests. --- scrapy/utils/defer.py | 34 +++--- tests/test_utils_defer.py | 219 ++++++++++++++++++++++++++++++++++---- 2 files changed, 212 insertions(+), 41 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 6e1687f3e..d06397f50 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -10,14 +10,13 @@ import warnings from asyncio import Future from collections.abc import Awaitable, Coroutine, Iterable, Iterator from functools import wraps -from types import CoroutineType -from typing import TYPE_CHECKING, Any, Generic, TypeVar, Union, cast, overload +from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast, overload -from twisted.internet import defer from twisted.internet.defer import ( Deferred, DeferredList, - ensureDeferred, + fail, + succeed, ) from twisted.internet.task import Cooperator from twisted.python import failure @@ -315,7 +314,7 @@ def process_parallel( """Return a Deferred with the output of all successful calls to the given callbacks """ - dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] + dfds = [succeed(input).addCallback(x, *a, **kw) for x in callbacks] d: Deferred[list[tuple[bool, _T2]]] = DeferredList( dfds, fireOnOneErrback=True, consumeErrors=True ) @@ -366,27 +365,24 @@ async def aiter_errback( errback(failure.Failure(), *a, **kw) -_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future]) +@overload +def deferred_from_coro(o: Awaitable[_T]) -> Deferred[_T]: ... @overload -def deferred_from_coro(o: _CT) -> Deferred: ... +def deferred_from_coro(o: _T2) -> _T2: ... -@overload -def deferred_from_coro(o: _T) -> _T: ... - - -def deferred_from_coro(o: _T) -> Deferred | _T: +def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2: """Converts a coroutine or other awaitable object into a Deferred, or returns the object as is if it isn't a coroutine.""" if isinstance(o, Deferred): return o - if asyncio.isfuture(o) or inspect.isawaitable(o): + if inspect.isawaitable(o): if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" - return ensureDeferred(cast(Coroutine[Deferred, Any, Any], o)) + return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o)) # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor event_loop = _get_asyncio_event_loop() return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) @@ -394,7 +390,7 @@ def deferred_from_coro(o: _T) -> Deferred | _T: def deferred_f_from_coro_f( - coro_f: Callable[_P, Coroutine[Any, Any, _T]], + coro_f: Callable[_P, Awaitable[_T]], ) -> Callable[_P, Deferred[_T]]: """Converts a coroutine function into a function that returns a Deferred. @@ -403,7 +399,7 @@ def deferred_f_from_coro_f( """ @wraps(coro_f) - def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Any: + def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Deferred[_T]: return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) return f @@ -416,15 +412,15 @@ def maybeDeferred_coro( try: result = f(*args, **kw) except: # noqa: E722 # pylint: disable=bare-except - return defer.fail(failure.Failure(captureVars=Deferred.debug)) + return fail(failure.Failure(captureVars=Deferred.debug)) if isinstance(result, Deferred): return result if asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) if isinstance(result, failure.Failure): - return defer.fail(result) - return defer.succeed(result) + return fail(result) + return succeed(result) def deferred_to_future(d: Deferred[_T]) -> Future[_T]: diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 36bd8ced9..29cd5fbf2 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,7 +1,12 @@ +from __future__ import annotations + +import asyncio import random +from asyncio import Future +from typing import TYPE_CHECKING, Any import pytest -from twisted.internet import defer, reactor +from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.python.failure import Failure from twisted.trial import unittest @@ -9,6 +14,8 @@ from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import ( aiter_errback, deferred_f_from_coro_f, + deferred_from_coro, + deferred_to_future, iter_errback, maybe_deferred_to_future, mustbe_deferred, @@ -17,12 +24,15 @@ from scrapy.utils.defer import ( process_parallel, ) +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, Awaitable, Callable, Generator + class TestMustbeDeferred(unittest.TestCase): - def test_success_function(self): - steps = [] + def test_success_function(self) -> Deferred[list[int]]: + steps: list[int] = [] - def _append(v): + def _append(v: int) -> list[int]: steps.append(v) return steps @@ -31,12 +41,14 @@ class TestMustbeDeferred(unittest.TestCase): steps.append(2) # add another value, that should be caught by assertEqual return dfd - def test_unfired_deferred(self): - steps = [] + def test_unfired_deferred(self) -> Deferred[list[int]]: + steps: list[int] = [] + + def _append(v: int) -> Deferred[list[int]]: + from twisted.internet import reactor - def _append(v): steps.append(v) - dfd = defer.Deferred() + dfd: Deferred[list[int]] = Deferred() reactor.callLater(0, dfd.callback, steps) return dfd @@ -51,7 +63,7 @@ def cb1(value, arg1, arg2): def cb2(value, arg1, arg2): - return defer.succeed(f"(cb2 {value} {arg1} {arg2})") + return succeed(f"(cb2 {value} {arg1} {arg2})") def cb3(value, arg1, arg2): @@ -67,7 +79,7 @@ def eb1(failure, arg1, arg2): class TestDeferUtils(unittest.TestCase): - @defer.inlineCallbacks + @inlineCallbacks def test_process_chain(self): x = yield process_chain([cb1, cb2, cb3], "res", "v1", "v2") assert x == "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)" @@ -75,7 +87,7 @@ class TestDeferUtils(unittest.TestCase): with pytest.raises(TypeError): yield process_chain([cb1, cb_fail, cb3], "res", "v1", "v2") - @defer.inlineCallbacks + @inlineCallbacks def test_process_parallel(self): x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"] @@ -88,7 +100,7 @@ class TestDeferUtils(unittest.TestCase): class TestIterErrback: def test_iter_errback_good(self): - def itergood(): + def itergood() -> Generator[int, None, None]: yield from range(10) errors = [] @@ -97,7 +109,7 @@ class TestIterErrback: assert not errors def test_iter_errback_bad(self): - def iterbad(): + def iterbad() -> Generator[int, None, None]: for x in range(10): if x == 5: 1 / 0 @@ -113,7 +125,7 @@ class TestIterErrback: class TestAiterErrback(unittest.TestCase): @deferred_f_from_coro_f async def test_aiter_errback_good(self): - async def itergood(): + async def itergood() -> AsyncGenerator[int, None]: for x in range(10): yield x @@ -124,7 +136,7 @@ class TestAiterErrback(unittest.TestCase): @deferred_f_from_coro_f async def test_iter_errback_bad(self): - async def iterbad(): + async def iterbad() -> AsyncGenerator[int, None]: for x in range(10): if x == 5: 1 / 0 @@ -168,10 +180,12 @@ class TestAsyncCooperator(unittest.TestCase): CONCURRENT_ITEMS = 50 @staticmethod - def callable(o, results): + def callable(o: int, results: list[int]) -> Deferred[None] | None: + from twisted.internet import reactor + if random.random() < 0.4: # simulate async processing - dfd = defer.Deferred() + dfd: Deferred[None] = Deferred() dfd.addCallback(lambda _: results.append(o)) delay = random.random() / 8 reactor.callLater(delay, dfd.callback, None) @@ -181,22 +195,24 @@ class TestAsyncCooperator(unittest.TestCase): return None @staticmethod - def get_async_iterable(length): + def get_async_iterable(length: int) -> AsyncGenerator[int, None]: # simulate a simple callback without delays between results return as_async_generator(range(length)) @staticmethod - async def get_async_iterable_with_delays(length): + async def get_async_iterable_with_delays(length: int) -> AsyncGenerator[int, None]: # simulate a callback with delays between some of the results + from twisted.internet import reactor + for i in range(length): if random.random() < 0.1: - dfd = defer.Deferred() + dfd: Deferred[None] = Deferred() delay = random.random() / 20 reactor.callLater(delay, dfd.callback, None) await maybe_deferred_to_future(dfd) yield i - @defer.inlineCallbacks + @inlineCallbacks def test_simple(self): for length in [20, 50, 100]: results = [] @@ -205,7 +221,7 @@ class TestAsyncCooperator(unittest.TestCase): yield dl assert list(range(length)) == sorted(results) - @defer.inlineCallbacks + @inlineCallbacks def test_delays(self): for length in [20, 50, 100]: results = [] @@ -213,3 +229,162 @@ class TestAsyncCooperator(unittest.TestCase): dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) yield dl assert list(range(length)) == sorted(results) + + +class TestDeferredFromCoro(unittest.TestCase): + def test_deferred(self): + d = Deferred() + result = deferred_from_coro(d) + assert isinstance(result, Deferred) + assert result is d + + def test_object(self): + result = deferred_from_coro(42) + assert result == 42 + + @inlineCallbacks + def test_coroutine(self): + async def coroutine() -> int: + return 42 + + result = deferred_from_coro(coroutine()) + assert isinstance(result, Deferred) + coro_result = yield result + assert coro_result == 42 + + @pytest.mark.only_asyncio + @inlineCallbacks + def test_coroutine_asyncio(self): + async def coroutine() -> int: + await asyncio.sleep(0) + return 42 + + result = deferred_from_coro(coroutine()) + assert isinstance(result, Deferred) + coro_result = yield result + assert coro_result == 42 + + @pytest.mark.only_asyncio + @inlineCallbacks + def test_future(self): + future = Future() + result = deferred_from_coro(future) + assert isinstance(result, Deferred) + future.set_result(42) + future_result = yield result + assert future_result == 42 + + +class TestDeferredFFromCoroF(unittest.TestCase): + @inlineCallbacks + def _assert_result( + self, c_f: Callable[[], Awaitable[int]] + ) -> Generator[Deferred[Any], Any, None]: + d_f = deferred_f_from_coro_f(c_f) + d = d_f() + assert isinstance(d, Deferred) + result = yield d + assert result == 42 + + @inlineCallbacks + def test_coroutine(self): + async def c_f() -> int: + return 42 + + yield self._assert_result(c_f) + + @inlineCallbacks + def test_coroutine_asyncio(self): + async def c_f() -> int: + return 42 + + yield self._assert_result(c_f) + + @pytest.mark.only_asyncio + @inlineCallbacks + def test_future(self): + def c_f() -> Future[int]: + f: Future[int] = Future() + f.set_result(42) + return f + + yield self._assert_result(c_f) + + +class TestDeferredToFuture(unittest.TestCase): + @deferred_f_from_coro_f + async def test_deferred(self): + d = Deferred() + result = deferred_to_future(d) + assert isinstance(result, Future) + d.callback(42) + future_result = await result + assert future_result == 42 + + @deferred_f_from_coro_f + async def test_wrapped_coroutine(self): + async def c_f() -> int: + return 42 + + d = deferred_from_coro(c_f()) + result = deferred_to_future(d) + assert isinstance(result, Future) + future_result = await result + assert future_result == 42 + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_wrapped_coroutine_asyncio(self): + async def c_f() -> int: + await asyncio.sleep(0) + return 42 + + d = deferred_from_coro(c_f()) + result = maybe_deferred_to_future(d) + assert isinstance(result, Future) + future_result = await result + assert future_result == 42 + + +@pytest.mark.only_asyncio +class TestMaybeDeferredToFutureAsyncio(unittest.TestCase): + @deferred_f_from_coro_f + async def test_deferred(self): + d = Deferred() + result = maybe_deferred_to_future(d) + assert isinstance(result, Future) + d.callback(42) + future_result = await result + assert future_result == 42 + + @deferred_f_from_coro_f + async def test_wrapped_coroutine(self): + async def c_f() -> int: + return 42 + + d = deferred_from_coro(c_f()) + result = maybe_deferred_to_future(d) + assert isinstance(result, Future) + future_result = await result + assert future_result == 42 + + @deferred_f_from_coro_f + async def test_wrapped_coroutine_asyncio(self): + async def c_f() -> int: + await asyncio.sleep(0) + return 42 + + d = deferred_from_coro(c_f()) + result = maybe_deferred_to_future(d) + assert isinstance(result, Future) + future_result = await result + assert future_result == 42 + + +@pytest.mark.only_not_asyncio +class TestMaybeDeferredToFutureNotAsyncio: + def test_deferred(self): + d = Deferred() + result = maybe_deferred_to_future(d) + assert isinstance(result, Deferred) + assert result is d From ff7d29654a975c12bfcf6d1b7719bd861ed6d8be Mon Sep 17 00:00:00 2001 From: Keval Sakhiya <37344767+kevalsakhiya@users.noreply.github.com> Date: Tue, 20 May 2025 11:21:31 +0530 Subject: [PATCH 1727/2083] Fix typo in documentation and code: 'needs_backoff' -> 'needs_backout' (#6815) Corrected the typo in the code and documentation where 'needs_backoff' was incorrectly used instead of 'needs_backout'. --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 891c4da05..8240d5d4b 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -388,7 +388,7 @@ its iteration whenever there are scheduled requests: async def start(self): async for item_or_request in super().start(): - if self.crawler.engine.needs_backoff(): + if self.crawler.engine.needs_backout(): await self.crawler.signals.wait_for(signals.scheduler_empty) yield item_or_request From f2fc177f1fb954480922301749cf00ee79eebb97 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 23 May 2025 14:06:33 +0500 Subject: [PATCH 1728/2083] Fix a wrong versionadded usage. (#6822) --- docs/topics/coroutines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 00812ed7f..2c0df5e0f 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -21,7 +21,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be defined as an :term:`asynchronous generator`. - .. versionadded: 2.13 + .. versionadded:: 2.13 - :class:`~scrapy.Request` callbacks. From 816d23da306e9fce0e55a933002fa7737f06fa64 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 27 May 2025 00:31:28 +0500 Subject: [PATCH 1729/2083] Make the release notes work better on PyPI. (#6826) --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 85fba0f92..47707e061 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,8 +58,7 @@ Homepage = "https://scrapy.org/" Documentation = "https://docs.scrapy.org/" Source = "https://github.com/scrapy/scrapy" Tracker = "https://github.com/scrapy/scrapy/issues" -Changelog = "https://github.com/scrapy/scrapy/commits/master/" -releasenotes = "https://docs.scrapy.org/en/latest/news.html" +"Release notes" = "https://docs.scrapy.org/en/latest/news.html" [project.scripts] scrapy = "scrapy.cmdline:execute" From 9d92d16510b8c82d529abf1662bce3e16ee293ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 27 May 2025 08:32:24 +0200 Subject: [PATCH 1730/2083] Prioritize other requests over start requests --- scrapy/core/scheduler.py | 4 ++-- scrapy/pqueues.py | 30 +++++++++++++++++------------- tests/test_engine_loop.py | 20 ++++++++++---------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 57d27b7cf..9ac447289 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -173,8 +173,8 @@ class Scheduler(BaseScheduler): :ref:`Start requests <start-requests>` are sent in the order they are yielded from :meth:`~scrapy.Spider.start`, and given the same - :attr:`~scrapy.http.Request.priority`, start requests take precedence over - other requests. + :attr:`~scrapy.http.Request.priority`, other requests take precedence over + start requests. You can set :setting:`SCHEDULER_START_MEMORY_QUEUE` and :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` to handle start requests diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index e6c6b8bf1..34b235d83 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -160,28 +160,32 @@ class ScrapyPriorityQueue: def pop(self) -> Request | None: while self.curprio is not None: - if self._start_queues: - try: - q = self._start_queues[self.curprio] - except KeyError: - pass - else: - m = q.pop() - if not q: - del self._start_queues[self.curprio] - q.close() - return m try: q = self.queues[self.curprio] except KeyError: - self._update_curprio() + pass else: m = q.pop() if not q: del self.queues[self.curprio] q.close() - self._update_curprio() + if not self._start_queues: + self._update_curprio() return m + if self._start_queues: + try: + q = self._start_queues[self.curprio] + except KeyError: + self._update_curprio() + else: + m = q.pop() + if not q: + del self._start_queues[self.curprio] + q.close() + self._update_curprio() + return m + else: + self._update_curprio() return None def _update_curprio(self) -> None: diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 90af10f0e..c7dbc82d4 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -189,9 +189,9 @@ class RequestSendOrderTestCase(TestCase): @deferred_f_from_coro_f async def test_default(self): - """By default, start requests take priority over callback requests and + """By default, callback requests take priority over start requests and are sent in order. Priority matters, but given the same priority, a - start request takes precedence.""" + callback request takes precedence.""" nums = [1, 2, 3, 4, 5, 6] response_seconds = 0 download_slots = 1 @@ -207,13 +207,13 @@ class RequestSendOrderTestCase(TestCase): yield _request(1) for request in ( - _request(4, priority=1), - _request(6), + _request(2, priority=1), + _request(5), ): spider.crawler.engine._slot.scheduler.enqueue_request(request) - yield _request(5) - yield _request(2, priority=1) + yield _request(6) yield _request(3, priority=1) + yield _request(4, priority=1) def parse(spider, response): return @@ -249,13 +249,13 @@ class RequestSendOrderTestCase(TestCase): yield _request(1) for request in ( - _request(4, priority=1), - _request(6), + _request(2, priority=1), + _request(5), ): spider.crawler.engine._slot.scheduler.enqueue_request(request) - yield _request(5) + yield _request(6) + yield _request(4, priority=1) yield _request(3, priority=1) - yield _request(2, priority=1) def parse(spider, response): return From 05529f3017a6327ba4553c2af422f2e5f02c7d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 27 May 2025 08:44:18 +0200 Subject: [PATCH 1731/2083] Release notes for Scrapy 2.13.1 --- docs/news.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index cf1c35893..eb5370b6e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,22 @@ Release notes ============= +.. _release-2.13.1: + +Scrapy 2.13.1 (unreleased) +-------------------------- + +- Give callback requests precedence over start requests when priority values + are the same. + + This makes changes from 2.13.0 to start request handling more intuitive and + backward compatible. For scenarios where all requests have the same + priorities, in 2.13.0 all start requests were sent before the first + callback request. In 2.13.1, same as in 2.12 and lower, start requests are + only sent when there are not enough pending callback requests to reach + concurrency limits. + + .. _release-2.13.0: Scrapy 2.13.0 (2025-05-08) From f28be27423d720a59dcd7194df26c935fcc7e416 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 13 May 2025 12:22:28 +0400 Subject: [PATCH 1732/2083] Add a deepwiki badge, update other badges. (#6793) --- README.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index cf7c6043c..29488d825 100644 --- a/README.rst +++ b/README.rst @@ -17,19 +17,14 @@ Scrapy :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - .. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - .. :alt: macOS - +.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS .. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows :alt: Windows -.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg - :target: https://pypi.org/pypi/Scrapy - :alt: Wheel Status - .. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report @@ -38,6 +33,10 @@ Scrapy :target: https://anaconda.org/conda-forge/scrapy :alt: Conda Version +.. image:: https://deepwiki.com/badge.svg + :target: https://deepwiki.com/scrapy/scrapy + :alt: Ask DeepWiki + Overview ======== From 43087fe1df5b3209bcc65dabd1831067d1a29711 Mon Sep 17 00:00:00 2001 From: Keval Sakhiya <37344767+kevalsakhiya@users.noreply.github.com> Date: Tue, 20 May 2025 11:21:31 +0530 Subject: [PATCH 1733/2083] Fix typo in documentation and code: 'needs_backoff' -> 'needs_backout' (#6815) Corrected the typo in the code and documentation where 'needs_backoff' was incorrectly used instead of 'needs_backout'. --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 891c4da05..8240d5d4b 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -388,7 +388,7 @@ its iteration whenever there are scheduled requests: async def start(self): async for item_or_request in super().start(): - if self.crawler.engine.needs_backoff(): + if self.crawler.engine.needs_backout(): await self.crawler.signals.wait_for(signals.scheduler_empty) yield item_or_request From 06dec081254e19950eb00fcb6979fe8b7c342ee8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 23 May 2025 14:06:33 +0500 Subject: [PATCH 1734/2083] Fix a wrong versionadded usage. (#6822) --- docs/topics/coroutines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 00812ed7f..2c0df5e0f 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -21,7 +21,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be defined as an :term:`asynchronous generator`. - .. versionadded: 2.13 + .. versionadded:: 2.13 - :class:`~scrapy.Request` callbacks. From 597320856776c2b6e44fc3a45a97d6524c64d464 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 27 May 2025 00:31:28 +0500 Subject: [PATCH 1735/2083] Make the release notes work better on PyPI. (#6826) --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 85fba0f92..47707e061 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,8 +58,7 @@ Homepage = "https://scrapy.org/" Documentation = "https://docs.scrapy.org/" Source = "https://github.com/scrapy/scrapy" Tracker = "https://github.com/scrapy/scrapy/issues" -Changelog = "https://github.com/scrapy/scrapy/commits/master/" -releasenotes = "https://docs.scrapy.org/en/latest/news.html" +"Release notes" = "https://docs.scrapy.org/en/latest/news.html" [project.scripts] scrapy = "scrapy.cmdline:execute" From e3f82afaf1ab12ac8f5915dfb0b926391bc81f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 27 May 2025 10:01:00 +0200 Subject: [PATCH 1736/2083] Add a test for ScrapyPriorityQueue pop order --- tests/test_pqueues.py | 54 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index d5c710ed2..b65f1b7e7 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -7,6 +7,7 @@ from scrapy.http.request import Request from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler from tests.test_scheduler import MockDownloader, MockEngine @@ -155,3 +156,56 @@ class TestDownloaderAwarePriorityQueue: assert self.queue.peek().url == req3.url assert self.queue.pop().url == req3.url assert self.queue.peek() is None + + +@pytest.mark.parametrize( + ("input", "output"), + [ + # By default, start requests are FIFO, other requests are LIFO. + ([{}, {}], [2, 1]), + ([{"start": True}, {"start": True}], [1, 2]), + # Priority matters. + ([{"priority": 1}, {"start": True}], [1, 2]), + ([{}, {"start": True, "priority": 1}], [2, 1]), + # For the same priority, start requests pop last. + ([{}, {"start": True}], [1, 2]), + ([{"start": True}, {}], [2, 1]), + ], +) +def test_pop_order(input, output): + def make_url(index): + return f"https://toscrape.com/{index}" + + def make_request(index, data): + meta = {} + if data.get("start", False): + meta["is_start_request"] = True + return Request( + url=make_url(index), + priority=data.get("priority", 0), + meta=meta, + ) + + input_requests = [ + make_request(index, data) for index, data in enumerate(input, start=1) + ] + expected_output_urls = [make_url(index) for index in output] + + crawler = get_crawler(Spider) + settings = crawler.settings + queue = build_from_crawler( + ScrapyPriorityQueue, + crawler, + downstream_queue_cls=load_object(settings["SCHEDULER_MEMORY_QUEUE"]), + key="", + start_queue_cls=load_object(settings["SCHEDULER_START_MEMORY_QUEUE"]), + ) + + for request in input_requests: + queue.push(request) + + actual_output_urls = [] + while request := queue.pop(): + actual_output_urls.append(request.url) + + assert actual_output_urls == expected_output_urls From b41aea4873319df15ccb9145a4940ff1702d123a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 27 May 2025 18:19:47 +0500 Subject: [PATCH 1737/2083] Restructure download handler tests. (#6821) * Restructure download handler tests. * Typo. * Use mixins to reduce boilerplate. --- .../test_downloader_handler_twisted_http10.py | 46 ++ .../test_downloader_handler_twisted_http11.py | 69 ++ ... test_downloader_handler_twisted_http2.py} | 108 ++- tests/test_downloader_handlers.py | 719 +----------------- tests/test_downloader_handlers_http_base.py | 698 +++++++++++++++++ 5 files changed, 862 insertions(+), 778 deletions(-) create mode 100644 tests/test_downloader_handler_twisted_http10.py create mode 100644 tests/test_downloader_handler_twisted_http11.py rename tests/{test_downloader_handlers_http2.py => test_downloader_handler_twisted_http2.py} (73%) create mode 100644 tests/test_downloader_handlers_http_base.py diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py new file mode 100644 index 000000000..807c8c4cb --- /dev/null +++ b/tests/test_downloader_handler_twisted_http10.py @@ -0,0 +1,46 @@ +"""Tests for scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler +from scrapy.http import Request +from scrapy.spiders import Spider +from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + +class HTTP10DownloadHandlerMixin: + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP10DownloadHandler + + +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase): + """HTTP 1.0 test case""" + + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.0") + return d + + +class TestHttps10(TestHttp10): + scheme = "https" + + +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestHttp10Proxy(HTTP10DownloadHandlerMixin, TestHttpProxyBase): + def test_download_with_proxy_https_timeout(self): + pytest.skip("Not implemented") + + def test_download_with_proxy_without_http_scheme(self): + pytest.skip("Not implemented") diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py new file mode 100644 index 000000000..70f55e787 --- /dev/null +++ b/tests/test_downloader_handler_twisted_http11.py @@ -0,0 +1,69 @@ +"""Tests for scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler +from tests.test_downloader_handlers_http_base import ( + TestHttp11Base, + TestHttpMockServerBase, + TestHttpProxyBase, + TestHttps11Base, + TestHttpsCustomCiphersBase, + TestHttpsInvalidDNSIdBase, + TestHttpsInvalidDNSPatternBase, + TestHttpsWrongHostnameBase, + TestSimpleHttpsBase, +) + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + +class HTTP11DownloadHandlerMixin: + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return HTTP11DownloadHandler + + +class TestHttp11(HTTP11DownloadHandlerMixin, TestHttp11Base): + pass + + +class TestHttps11(HTTP11DownloadHandlerMixin, TestHttps11Base): + pass + + +class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase): + pass + + +class Https11WrongHostnameTestCase( + HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase +): + pass + + +class Https11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass + + +class Https11InvalidDNSPattern( + HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase +): + pass + + +class Https11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): + pass + + +class TestHttp11MockServer(TestHttpMockServerBase): + @property + def settings_dict(self) -> dict[str, Any] | None: + return None # default handler settings + + +class TestHttp11Proxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): + pass diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handler_twisted_http2.py similarity index 73% rename from tests/test_downloader_handlers_http2.py rename to tests/test_downloader_handler_twisted_http2.py index c74c09cbb..46322a747 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -1,4 +1,9 @@ +"""Tests for scrapy.core.downloader.handlers.http2.H2DownloadHandler.""" + +from __future__ import annotations + import json +from typing import TYPE_CHECKING, Any from unittest import mock import pytest @@ -8,60 +13,42 @@ from twisted.web import server from twisted.web.error import SchemeNotSupported from twisted.web.http import H2_ENABLED -from scrapy.core.downloader.handlers import DownloadHandlerProtocol from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory -from tests.test_downloader_handlers import ( +from tests.test_downloader_handlers_http_base import ( + TestHttpMockServerBase, + TestHttpProxyBase, + TestHttps11Base, + TestHttpsCustomCiphersBase, + TestHttpsInvalidDNSIdBase, + TestHttpsInvalidDNSPatternBase, + TestHttpsWrongHostnameBase, UriResource, ) +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + pytestmark = pytest.mark.skipif( not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" ) -class BaseTestClasses: - # A hack to prevent tests from the imported classes to run here too. - # See https://stackoverflow.com/q/1323455/113586 for other ways. - from tests.test_downloader_handlers import ( - TestHttp11MockServer as TestHttp11MockServer, - ) - from tests.test_downloader_handlers import ( - TestHttp11Proxy as TestHttp11Proxy, - ) - from tests.test_downloader_handlers import ( - TestHttps11 as TestHttps11, - ) - from tests.test_downloader_handlers import ( - TestHttps11CustomCiphers as TestHttps11CustomCiphers, - ) - from tests.test_downloader_handlers import ( - TestHttps11InvalidDNSId as TestHttps11InvalidDNSId, - ) - from tests.test_downloader_handlers import ( - TestHttps11InvalidDNSPattern as TestHttps11InvalidDNSPattern, - ) - from tests.test_downloader_handlers import ( - TestHttps11WrongHostname as TestHttps11WrongHostname, - ) - - -def _get_dh() -> type[DownloadHandlerProtocol]: - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - return H2DownloadHandler - - -class TestHttps2(BaseTestClasses.TestHttps11): - scheme = "https" - HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - +class H2DownloadHandlerMixin: @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() + # the import can fail when H2_ENABLED is False + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + + return H2DownloadHandler + + +class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): + HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -179,42 +166,37 @@ class TestHttps2(BaseTestClasses.TestHttps11): return d -class Https2WrongHostnameTestCase(BaseTestClasses.TestHttps11WrongHostname): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() +class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): + pass -class Https2InvalidDNSId(BaseTestClasses.TestHttps11InvalidDNSId): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() +class Https2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass -class Https2InvalidDNSPattern(BaseTestClasses.TestHttps11InvalidDNSPattern): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() +class Https2InvalidDNSPattern(H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase): + pass -class Https2CustomCiphers(BaseTestClasses.TestHttps11CustomCiphers): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() +class Https2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): + pass -class Http2MockServerTestCase(BaseTestClasses.TestHttp11MockServer): +class Http2MockServerTestCase(TestHttpMockServerBase): """HTTP 2.0 test case with MockServer""" - settings_dict = { - "DOWNLOAD_HANDLERS": { - "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler" + @property + def settings_dict(self) -> dict[str, Any] | None: + return { + "DOWNLOAD_HANDLERS": { + "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler" + } } - } + is_secure = True -class Https2ProxyTestCase(BaseTestClasses.TestHttp11Proxy): +class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase): # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" @@ -224,10 +206,6 @@ class Https2ProxyTestCase(BaseTestClasses.TestHttp11Proxy): expected_http_proxy_request_body = b"/" - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return _get_dh() - def setUp(self): site = server.Site(UriResource(), timeout=None) self.port = reactor.listenSSL( diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index bc18e76e1..fc6ac5aee 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,52 +1,35 @@ +"""Tests for DownloadHandlers and for specific non-HTTP download handlers.""" + from __future__ import annotations import contextlib import os import shutil import sys -from abc import ABC, abstractmethod from pathlib import Path from tempfile import mkdtemp, mkstemp -from unittest import SkipTest, mock +from unittest import mock import pytest -from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal -from twisted.internet import defer, error, reactor +from twisted.internet import reactor from twisted.protocols.ftp import FTPFactory, FTPRealm -from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest -from twisted.web import resource, server, static, util -from twisted.web.client import ResponseFailed -from twisted.web.http import _DataLoss from w3lib.url import path_to_file_uri -from scrapy.core.downloader.handlers import DownloadHandlerProtocol, DownloadHandlers +from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler -from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler -from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.exceptions import NotConfigured -from scrapy.http import Headers, HtmlResponse, Request +from scrapy.http import HtmlResponse, Request from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler -from tests import NON_EXISTING_RESOLVABLE -from tests.mockserver import ( - Echo, - ForeverTakingResource, - HostHeaderResource, - MockServer, - NoLengthResource, - PayloadResource, - ssl_context_factory, -) -from tests.spiders import SingleRequestSpider class DummyDH: @@ -137,696 +120,6 @@ class TestFile(unittest.TestCase): return self.assertFailure(d, OSError) -class ContentLengthHeaderResource(resource.Resource): - """ - A testing resource which renders itself as the value of the Content-Length - header from the request. - """ - - def render(self, request): - return request.requestHeaders.getRawHeaders(b"content-length")[0] - - -class ChunkedResource(resource.Resource): - def render(self, request): - def response(): - request.write(b"chunked ") - request.write(b"content\n") - request.finish() - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -class BrokenChunkedResource(resource.Resource): - def render(self, request): - def response(): - request.write(b"chunked ") - request.write(b"content\n") - # Disable terminating chunk on finish. - request.chunked = False - closeConnection(request) - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -class BrokenDownloadResource(resource.Resource): - def render(self, request): - def response(): - request.setHeader(b"Content-Length", b"20") - request.write(b"partial") - closeConnection(request) - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -def closeConnection(request): - # We have to force a disconnection for HTTP/1.1 clients. Otherwise - # client keeps the connection open waiting for more data. - request.channel.loseConnection() - request.finish() - - -class EmptyContentTypeHeaderResource(resource.Resource): - """ - A testing resource which renders itself as the value of request body - without content-type header in response. - """ - - def render(self, request): - request.setHeader("content-type", "") - return request.content.read() - - -class LargeChunkedFileResource(resource.Resource): - def render(self, request): - def response(): - for i in range(1024): - request.write(b"x" * 1024) - request.finish() - - reactor.callLater(0, response) - return server.NOT_DONE_YET - - -class DuplicateHeaderResource(resource.Resource): - def render(self, request): - request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) - return b"" - - -class TestHttp(unittest.TestCase, ABC): - scheme = "http" - - # only used for HTTPS tests - keyfile = "keys/localhost.key" - certfile = "keys/localhost.crt" - - @property - @abstractmethod - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - raise NotImplementedError - - def setUp(self): - self.tmpname = Path(mkdtemp()) - (self.tmpname / "file").write_bytes(b"0123456789") - r = static.File(str(self.tmpname)) - r.putChild(b"redirect", util.Redirect(b"/file")) - r.putChild(b"wait", ForeverTakingResource()) - r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) - r.putChild(b"nolength", NoLengthResource()) - r.putChild(b"host", HostHeaderResource()) - r.putChild(b"payload", PayloadResource()) - r.putChild(b"broken", BrokenDownloadResource()) - r.putChild(b"chunked", ChunkedResource()) - r.putChild(b"broken-chunked", BrokenChunkedResource()) - r.putChild(b"contentlength", ContentLengthHeaderResource()) - r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) - r.putChild(b"largechunkedfile", LargeChunkedFileResource()) - r.putChild(b"duplicate-header", DuplicateHeaderResource()) - r.putChild(b"echo", Echo()) - self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) - self.host = "localhost" - if self.scheme == "https": - # Using WrappingFactory do not enable HTTP/2 failing all the - # tests with H2DownloadHandler - self.port = reactor.listenSSL( - 0, - self.site, - ssl_context_factory(self.keyfile, self.certfile), - interface=self.host, - ) - else: - self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) - self.portno = self.port.getHost().port - self.download_handler = build_from_crawler( - self.download_handler_cls, get_crawler() - ) - self.download_request = self.download_handler.download_request - - @defer.inlineCallbacks - def tearDown(self): - yield self.port.stopListening() - if hasattr(self.download_handler, "close"): - yield self.download_handler.close() - shutil.rmtree(self.tmpname) - - def getURL(self, path): - return f"{self.scheme}://{self.host}:{self.portno}/{path}" - - def test_download(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - def test_download_head(self): - request = Request(self.getURL("file"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"") - return d - - def test_redirect_status(self): - request = Request(self.getURL("redirect")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d - - def test_redirect_status_head(self): - request = Request(self.getURL("redirect"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d - - @defer.inlineCallbacks - def test_timeout_download_from_spider_nodata_rcvd(self): - if self.reactor_pytest != "default" and sys.platform == "win32": - # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( - "This test produces DirtyReactorAggregateError on Windows with asyncio" - ) - - # client connects but no data is received - spider = Spider("foo") - meta = {"download_timeout": 0.5} - request = Request(self.getURL("wait"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) - - @defer.inlineCallbacks - def test_timeout_download_from_spider_server_hangs(self): - if self.reactor_pytest != "default" and sys.platform == "win32": - # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( - "This test produces DirtyReactorAggregateError on Windows with asyncio" - ) - # client connects, server send headers and some body bytes but hangs - spider = Spider("foo") - meta = {"download_timeout": 0.5} - request = Request(self.getURL("hang-after-headers"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) - - def test_host_header_not_in_request_headers(self): - def _test(response): - assert response.body == to_bytes(f"{self.host}:{self.portno}") - assert not request.headers - - request = Request(self.getURL("host")) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_host_header_seted_in_request_headers(self): - host = self.host + ":" + str(self.portno) - - def _test(response): - assert response.body == host.encode() - assert request.headers.get("Host") == host.encode() - - request = Request(self.getURL("host"), headers={"Host": host}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_content_length_zero_bodyless_post_request_headers(self): - """Tests if "Content-Length: 0" is sent for bodyless POST requests. - - This is not strictly required by HTTP RFCs but can cause trouble - for some web servers. - See: - https://github.com/scrapy/scrapy/issues/823 - https://issues.apache.org/jira/browse/TS-2902 - https://github.com/kennethreitz/requests/issues/405 - https://bugs.python.org/issue14721 - """ - - def _test(response): - assert response.body == b"0" - - request = Request(self.getURL("contentlength"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_content_length_zero_bodyless_post_only_one(self): - def _test(response): - import json - - headers = Headers(json.loads(response.text)["headers"]) - contentlengths = headers.getlist("Content-Length") - assert len(contentlengths) == 1 - assert contentlengths == [b"0"] - - request = Request(self.getURL("echo"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_payload(self): - body = b"1" * 100 # PayloadResource requires body length to be 100 - request = Request(self.getURL("payload"), method="POST", body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, body) - return d - - def test_response_header_content_length(self): - request = Request(self.getURL("file"), method=b"GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.headers[b"content-length"]) - d.addCallback(self.assertEqual, b"159") - return d - - def _test_response_class(self, filename, body, response_class): - def _test(response): - assert type(response) is response_class # pylint: disable=unidiomatic-typecheck - - request = Request(self.getURL(filename), body=body) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_response_class_from_url(self): - return self._test_response_class("foo.html", b"", HtmlResponse) - - def test_response_class_from_body(self): - return self._test_response_class( - "foo", - b"<!DOCTYPE html>\n<title>.", - HtmlResponse, - ) - - def test_get_duplicate_header(self): - def _test(response): - assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] - - request = Request(self.getURL("duplicate-header")) - return self.download_request(request, Spider("foo")).addCallback(_test) - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestHttp10(TestHttp): - """HTTP 1.0 test case""" - - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP10DownloadHandler - - def test_protocol(self): - request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.0") - return d - - -class TestHttps10(TestHttp10): - scheme = "https" - - -class TestHttp11(TestHttp): - """HTTP 1.1 test case""" - - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP11DownloadHandler - - def test_download_without_maxsize_limit(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - def test_response_class_choosing_request(self): - """Tests choosing of correct response type - in case of Content-Type is empty but body contains text. - """ - body = b"Some plain text\ndata with tabs\t and null bytes\0" - - def _test_type(response): - assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck - - request = Request(self.getURL("nocontenttype"), body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(_test_type) - return d - - @defer.inlineCallbacks - def test_download_with_maxsize(self): - request = Request(self.getURL("file")) - - # 10 is minimal size for this request and the limit is only counted on - # response body. (regardless of headers) - d = self.download_request(request, Spider("foo", download_maxsize=10)) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - yield d - - d = self.download_request(request, Spider("foo", download_maxsize=9)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): - with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger: - request = Request(self.getURL("largechunkedfile")) - - def check(logger): - logger.warning.assert_called_once_with(mock.ANY, mock.ANY) - - d = self.download_request(request, Spider("foo", download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - # As the error message is logged in the dataReceived callback, we - # have to give a bit of time to the reactor to process the queue - # after closing the connection. - d = defer.Deferred() - d.addCallback(check) - reactor.callLater(0.1, d.callback, logger) - yield d - - @defer.inlineCallbacks - def test_download_with_maxsize_per_req(self): - meta = {"download_maxsize": 2} - request = Request(self.getURL("file"), meta=meta) - d = self.download_request(request, Spider("foo")) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - @defer.inlineCallbacks - def test_download_with_small_maxsize_per_spider(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo", download_maxsize=2)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) - - def test_download_with_large_maxsize_per_spider(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo", download_maxsize=100)) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - def test_download_chunked_content(self): - request = Request(self.getURL("chunked")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"chunked content\n") - return d - - def test_download_broken_content_cause_data_loss(self, url="broken"): - request = Request(self.getURL(url)) - d = self.download_request(request, Spider("foo")) - - def checkDataLoss(failure): - if failure.check(ResponseFailed) and any( - r.check(_DataLoss) for r in failure.value.reasons - ): - return None - return failure - - d.addCallback(lambda _: self.fail("No DataLoss exception")) - d.addErrback(checkDataLoss) - return d - - def test_download_broken_chunked_content_cause_data_loss(self): - return self.test_download_broken_content_cause_data_loss("broken-chunked") - - def test_download_broken_content_allow_data_loss(self, url="broken"): - request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False}) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ["dataloss"]) - return d - - def test_download_broken_chunked_content_allow_data_loss(self): - return self.test_download_broken_content_allow_data_loss("broken-chunked") - - def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): - crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) - download_handler = build_from_crawler(self.download_handler_cls, crawler) - request = Request(self.getURL(url)) - d = download_handler.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ["dataloss"]) - return d - - def test_download_broken_chunked_content_allow_data_loss_via_setting(self): - return self.test_download_broken_content_allow_data_loss_via_setting( - "broken-chunked" - ) - - def test_protocol(self): - request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.1") - return d - - -class TestHttps11(TestHttp11): - scheme = "https" - - tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' - 'subject "/C=IE/O=Scrapy/CN=localhost"' - ) - - @defer.inlineCallbacks - def test_tls_logging(self): - crawler = get_crawler( - settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} - ) - download_handler = build_from_crawler(self.download_handler_cls, crawler) - try: - with LogCapture() as log_capture: - request = Request(self.getURL("file")) - d = download_handler.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - yield d - log_capture.check_present( - ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message) - ) - finally: - yield download_handler.close() - - -class TestSimpleHttps(unittest.TestCase): - """Base class for special cases tested with just one simple request""" - - keyfile = "keys/localhost.key" - certfile = "keys/localhost.crt" - cipher_string: str | None = None - - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP11DownloadHandler - - def setUp(self): - self.tmpname = Path(mkdtemp()) - (self.tmpname / "file").write_bytes(b"0123456789") - r = static.File(str(self.tmpname)) - self.site = server.Site(r, timeout=None) - self.host = "localhost" - self.port = reactor.listenSSL( - 0, - self.site, - ssl_context_factory( - self.keyfile, self.certfile, cipher_string=self.cipher_string - ), - interface=self.host, - ) - self.portno = self.port.getHost().port - if self.cipher_string is not None: - settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string} - else: - settings_dict = None - crawler = get_crawler(settings_dict=settings_dict) - self.download_handler = build_from_crawler(self.download_handler_cls, crawler) - self.download_request = self.download_handler.download_request - - @defer.inlineCallbacks - def tearDown(self): - yield self.port.stopListening() - if hasattr(self.download_handler, "close"): - yield self.download_handler.close() - shutil.rmtree(self.tmpname) - - def getURL(self, path): - return f"https://{self.host}:{self.portno}/{path}" - - def test_download(self): - request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d - - -class TestHttps11WrongHostname(TestSimpleHttps): - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = "keys/example-com.key.pem" - certfile = "keys/example-com.cert.pem" - - -class TestHttps11InvalidDNSId(TestSimpleHttps): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super().setUp() - self.host = "127.0.0.1" - - -class TestHttps11InvalidDNSPattern(TestSimpleHttps): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = "keys/localhost.ip.key" - certfile = "keys/localhost.ip.crt" - - -class TestHttps11CustomCiphers(TestSimpleHttps): - cipher_string = "CAMELLIA256-SHA" - - -class TestHttp11MockServer(unittest.TestCase): - """HTTP 1.1 test case with MockServer""" - - settings_dict: dict | None = None - is_secure = False - - @classmethod - def setUpClass(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def tearDownClass(cls): - cls.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def test_download_with_content_length(self): - crawler = get_crawler(SingleRequestSpider, self.settings_dict) - # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid - # download it - yield crawler.crawl( - seed=Request( - url=self.mockserver.url("/partial", is_secure=self.is_secure), - meta={"download_maxsize": 1000}, - ) - ) - failure = crawler.spider.meta["failure"] - assert isinstance(failure.value, defer.CancelledError) - - @defer.inlineCallbacks - def test_download(self): - crawler = get_crawler(SingleRequestSpider, self.settings_dict) - yield crawler.crawl( - seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) - ) - failure = crawler.spider.meta.get("failure") - assert failure is None - reason = crawler.spider.meta["close_reason"] - assert reason == "finished" - - -class UriResource(resource.Resource): - """Return the full uri that was requested""" - - def getChild(self, path, request): - return self - - def render(self, request): - # Note: this is an ugly hack for CONNECT request timeout test. - # Returning some data here fail SSL/TLS handshake - # ToDo: implement proper HTTPS proxy tests, not faking them. - if request.method != b"CONNECT": - return request.uri - return b"" - - -class TestHttpProxy(unittest.TestCase, ABC): - expected_http_proxy_request_body = b"http://example.com" - - @property - @abstractmethod - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - raise NotImplementedError - - def setUp(self): - site = server.Site(UriResource(), timeout=None) - wrapper = WrappingFactory(site) - self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") - self.portno = self.port.getHost().port - self.download_handler = build_from_crawler( - self.download_handler_cls, get_crawler() - ) - self.download_request = self.download_handler.download_request - - @defer.inlineCallbacks - def tearDown(self): - yield self.port.stopListening() - if hasattr(self.download_handler, "close"): - yield self.download_handler.close() - - def getURL(self, path): - return f"http://127.0.0.1:{self.portno}/{path}" - - def test_download_with_proxy(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == self.expected_http_proxy_request_body - - http_proxy = self.getURL("") - request = Request("http://example.com", meta={"proxy": http_proxy}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_download_without_proxy(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == b"/path/to/resource" - - request = Request(self.getURL("path/to/resource")) - return self.download_request(request, Spider("foo")).addCallback(_test) - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestHttp10Proxy(TestHttpProxy): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP10DownloadHandler - - -class TestHttp11Proxy(TestHttpProxy): - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP11DownloadHandler - - @defer.inlineCallbacks - def test_download_with_proxy_https_timeout(self): - """Test TunnelingTCP4ClientEndpoint""" - if NON_EXISTING_RESOLVABLE: - raise SkipTest("Non-existing hosts are resolvable") - http_proxy = self.getURL("") - domain = "https://no-such-domain.nosuch" - request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) - d = self.download_request(request, Spider("foo")) - timeout = yield self.assertFailure(d, error.TimeoutError) - assert domain in timeout.osError - - def test_download_with_proxy_without_http_scheme(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == self.expected_http_proxy_request_body - - http_proxy = self.getURL("").replace("http://", "") - request = Request("http://example.com", meta={"proxy": http_proxy}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - class HttpDownloadHandlerMock: def __init__(self, *args, **kwargs): pass diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py new file mode 100644 index 000000000..46e5972f7 --- /dev/null +++ b/tests/test_downloader_handlers_http_base.py @@ -0,0 +1,698 @@ +"""Base classes for HTTP download handler tests.""" + +from __future__ import annotations + +import shutil +import sys +from abc import ABC, abstractmethod +from pathlib import Path +from tempfile import mkdtemp +from typing import TYPE_CHECKING, Any +from unittest import mock + +import pytest +from testfixtures import LogCapture +from twisted.internet import defer, error, reactor +from twisted.protocols.policies import WrappingFactory +from twisted.trial import unittest +from twisted.web import resource, server, static, util +from twisted.web._newclient import ResponseFailed +from twisted.web.http import _DataLoss + +from scrapy.http import Headers, HtmlResponse, Request, TextResponse +from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.python import to_bytes +from scrapy.utils.test import get_crawler +from tests import NON_EXISTING_RESOLVABLE +from tests.mockserver import ( + Echo, + ForeverTakingResource, + HostHeaderResource, + MockServer, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) +from tests.spiders import SingleRequestSpider + +if TYPE_CHECKING: + from scrapy.core.downloader.handlers import DownloadHandlerProtocol + + +class ContentLengthHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of the Content-Length + header from the request. + """ + + def render(self, request): + return request.requestHeaders.getRawHeaders(b"content-length")[0] + + +class ChunkedResource(resource.Resource): + def render(self, request): + def response(): + request.write(b"chunked ") + request.write(b"content\n") + request.finish() + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenChunkedResource(resource.Resource): + def render(self, request): + def response(): + request.write(b"chunked ") + request.write(b"content\n") + # Disable terminating chunk on finish. + request.chunked = False + closeConnection(request) + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenDownloadResource(resource.Resource): + def render(self, request): + def response(): + request.setHeader(b"Content-Length", b"20") + request.write(b"partial") + closeConnection(request) + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +def closeConnection(request): + # We have to force a disconnection for HTTP/1.1 clients. Otherwise + # client keeps the connection open waiting for more data. + request.channel.loseConnection() + request.finish() + + +class EmptyContentTypeHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of request body + without content-type header in response. + """ + + def render(self, request): + request.setHeader("content-type", "") + return request.content.read() + + +class LargeChunkedFileResource(resource.Resource): + def render(self, request): + def response(): + for i in range(1024): + request.write(b"x" * 1024) + request.finish() + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class DuplicateHeaderResource(resource.Resource): + def render(self, request): + request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"]) + return b"" + + +class TestHttpBase(unittest.TestCase, ABC): + scheme = "http" + + # only used for HTTPS tests + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + def setUp(self): + self.tmpname = Path(mkdtemp()) + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) + r.putChild(b"redirect", util.Redirect(b"/file")) + r.putChild(b"wait", ForeverTakingResource()) + r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) + r.putChild(b"nolength", NoLengthResource()) + r.putChild(b"host", HostHeaderResource()) + r.putChild(b"payload", PayloadResource()) + r.putChild(b"broken", BrokenDownloadResource()) + r.putChild(b"chunked", ChunkedResource()) + r.putChild(b"broken-chunked", BrokenChunkedResource()) + r.putChild(b"contentlength", ContentLengthHeaderResource()) + r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) + r.putChild(b"largechunkedfile", LargeChunkedFileResource()) + r.putChild(b"duplicate-header", DuplicateHeaderResource()) + r.putChild(b"echo", Echo()) + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.host = "localhost" + if self.scheme == "https": + # Using WrappingFactory do not enable HTTP/2 failing all the + # tests with H2DownloadHandler + self.port = reactor.listenSSL( + 0, + self.site, + ssl_context_factory(self.keyfile, self.certfile), + interface=self.host, + ) + else: + self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) + self.portno = self.port.getHost().port + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() + ) + self.download_request = self.download_handler.download_request + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + if hasattr(self.download_handler, "close"): + yield self.download_handler.close() + shutil.rmtree(self.tmpname) + + def getURL(self, path): + return f"{self.scheme}://{self.host}:{self.portno}/{path}" + + def test_download(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + def test_download_head(self): + request = Request(self.getURL("file"), method="HEAD") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"") + return d + + def test_redirect_status(self): + request = Request(self.getURL("redirect")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.status) + d.addCallback(self.assertEqual, 302) + return d + + def test_redirect_status_head(self): + request = Request(self.getURL("redirect"), method="HEAD") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.status) + d.addCallback(self.assertEqual, 302) + return d + + @defer.inlineCallbacks + def test_timeout_download_from_spider_nodata_rcvd(self): + if self.reactor_pytest != "default" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + + # client connects but no data is received + spider = Spider("foo") + meta = {"download_timeout": 0.5} + request = Request(self.getURL("wait"), meta=meta) + d = self.download_request(request, spider) + yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) + + @defer.inlineCallbacks + def test_timeout_download_from_spider_server_hangs(self): + if self.reactor_pytest != "default" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + # client connects, server send headers and some body bytes but hangs + spider = Spider("foo") + meta = {"download_timeout": 0.5} + request = Request(self.getURL("hang-after-headers"), meta=meta) + d = self.download_request(request, spider) + yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) + + def test_host_header_not_in_request_headers(self): + def _test(response): + assert response.body == to_bytes(f"{self.host}:{self.portno}") + assert not request.headers + + request = Request(self.getURL("host")) + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_host_header_seted_in_request_headers(self): + host = self.host + ":" + str(self.portno) + + def _test(response): + assert response.body == host.encode() + assert request.headers.get("Host") == host.encode() + + request = Request(self.getURL("host"), headers={"Host": host}) + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_content_length_zero_bodyless_post_request_headers(self): + """Tests if "Content-Length: 0" is sent for bodyless POST requests. + + This is not strictly required by HTTP RFCs but can cause trouble + for some web servers. + See: + https://github.com/scrapy/scrapy/issues/823 + https://issues.apache.org/jira/browse/TS-2902 + https://github.com/kennethreitz/requests/issues/405 + https://bugs.python.org/issue14721 + """ + + def _test(response): + assert response.body == b"0" + + request = Request(self.getURL("contentlength"), method="POST") + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_content_length_zero_bodyless_post_only_one(self): + def _test(response): + import json + + headers = Headers(json.loads(response.text)["headers"]) + contentlengths = headers.getlist("Content-Length") + assert len(contentlengths) == 1 + assert contentlengths == [b"0"] + + request = Request(self.getURL("echo"), method="POST") + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_payload(self): + body = b"1" * 100 # PayloadResource requires body length to be 100 + request = Request(self.getURL("payload"), method="POST", body=body) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, body) + return d + + def test_response_header_content_length(self): + request = Request(self.getURL("file"), method=b"GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.headers[b"content-length"]) + d.addCallback(self.assertEqual, b"159") + return d + + def _test_response_class(self, filename, body, response_class): + def _test(response): + assert type(response) is response_class # pylint: disable=unidiomatic-typecheck + + request = Request(self.getURL(filename), body=body) + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_response_class_from_url(self): + return self._test_response_class("foo.html", b"", HtmlResponse) + + def test_response_class_from_body(self): + return self._test_response_class( + "foo", + b"\n.", + HtmlResponse, + ) + + def test_get_duplicate_header(self): + def _test(response): + assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] + + request = Request(self.getURL("duplicate-header")) + return self.download_request(request, Spider("foo")).addCallback(_test) + + +class TestHttp11Base(TestHttpBase): + """HTTP 1.1 test case""" + + def test_download_without_maxsize_limit(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + def test_response_class_choosing_request(self): + """Tests choosing of correct response type + in case of Content-Type is empty but body contains text. + """ + body = b"Some plain text\ndata with tabs\t and null bytes\0" + + def _test_type(response): + assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck + + request = Request(self.getURL("nocontenttype"), body=body) + d = self.download_request(request, Spider("foo")) + d.addCallback(_test_type) + return d + + @defer.inlineCallbacks + def test_download_with_maxsize(self): + request = Request(self.getURL("file")) + + # 10 is minimal size for this request and the limit is only counted on + # response body. (regardless of headers) + d = self.download_request(request, Spider("foo", download_maxsize=10)) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + yield d + + d = self.download_request(request, Spider("foo", download_maxsize=9)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + @defer.inlineCallbacks + def test_download_with_maxsize_very_large_file(self): + with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger: + request = Request(self.getURL("largechunkedfile")) + + def check(logger): + logger.warning.assert_called_once_with(mock.ANY, mock.ANY) + + d = self.download_request(request, Spider("foo", download_maxsize=1500)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + # As the error message is logged in the dataReceived callback, we + # have to give a bit of time to the reactor to process the queue + # after closing the connection. + d = defer.Deferred() + d.addCallback(check) + reactor.callLater(0.1, d.callback, logger) + yield d + + @defer.inlineCallbacks + def test_download_with_maxsize_per_req(self): + meta = {"download_maxsize": 2} + request = Request(self.getURL("file"), meta=meta) + d = self.download_request(request, Spider("foo")) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + @defer.inlineCallbacks + def test_download_with_small_maxsize_per_spider(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo", download_maxsize=2)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + def test_download_with_large_maxsize_per_spider(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo", download_maxsize=100)) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + def test_download_chunked_content(self): + request = Request(self.getURL("chunked")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"chunked content\n") + return d + + def test_download_broken_content_cause_data_loss(self, url="broken"): + # TODO: this one checks for Twisted-specific exceptions + request = Request(self.getURL(url)) + d = self.download_request(request, Spider("foo")) + + def checkDataLoss(failure): + if failure.check(ResponseFailed) and any( + r.check(_DataLoss) for r in failure.value.reasons + ): + return None + return failure + + d.addCallback(lambda _: self.fail("No DataLoss exception")) + d.addErrback(checkDataLoss) + return d + + def test_download_broken_chunked_content_cause_data_loss(self): + return self.test_download_broken_content_cause_data_loss("broken-chunked") + + def test_download_broken_content_allow_data_loss(self, url="broken"): + request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False}) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.flags) + d.addCallback(self.assertEqual, ["dataloss"]) + return d + + def test_download_broken_chunked_content_allow_data_loss(self): + return self.test_download_broken_content_allow_data_loss("broken-chunked") + + def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): + crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) + download_handler = build_from_crawler(self.download_handler_cls, crawler) + request = Request(self.getURL(url)) + d = download_handler.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.flags) + d.addCallback(self.assertEqual, ["dataloss"]) + return d + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + return self.test_download_broken_content_allow_data_loss_via_setting( + "broken-chunked" + ) + + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.1") + return d + + +class TestHttps11Base(TestHttp11Base): + scheme = "https" + + tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' + 'subject "/C=IE/O=Scrapy/CN=localhost"' + ) + + @defer.inlineCallbacks + def test_tls_logging(self): + crawler = get_crawler( + settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} + ) + download_handler = build_from_crawler(self.download_handler_cls, crawler) + try: + with LogCapture() as log_capture: + request = Request(self.getURL("file")) + d = download_handler.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + yield d + log_capture.check_present( + ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message) + ) + finally: + yield download_handler.close() + + +class TestSimpleHttpsBase(unittest.TestCase, ABC): + """Base class for special cases tested with just one simple request""" + + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" + cipher_string: str | None = None + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + def setUp(self): + self.tmpname = Path(mkdtemp()) + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) + self.site = server.Site(r, timeout=None) + self.host = "localhost" + self.port = reactor.listenSSL( + 0, + self.site, + ssl_context_factory( + self.keyfile, self.certfile, cipher_string=self.cipher_string + ), + interface=self.host, + ) + self.portno = self.port.getHost().port + if self.cipher_string is not None: + settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string} + else: + settings_dict = None + crawler = get_crawler(settings_dict=settings_dict) + self.download_handler = build_from_crawler(self.download_handler_cls, crawler) + self.download_request = self.download_handler.download_request + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + if hasattr(self.download_handler, "close"): + yield self.download_handler.close() + shutil.rmtree(self.tmpname) + + def getURL(self, path): + return f"https://{self.host}:{self.portno}/{path}" + + def test_download(self): + request = Request(self.getURL("file")) + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + +class TestHttpsWrongHostnameBase(TestSimpleHttpsBase): + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = "keys/example-com.key.pem" + certfile = "keys/example-com.cert.pem" + + +class TestHttpsInvalidDNSIdBase(TestSimpleHttpsBase): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super().setUp() + self.host = "127.0.0.1" + + +class TestHttpsInvalidDNSPatternBase(TestSimpleHttpsBase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = "keys/localhost.ip.key" + certfile = "keys/localhost.ip.crt" + + +class TestHttpsCustomCiphersBase(TestSimpleHttpsBase): + cipher_string = "CAMELLIA256-SHA" + + +class TestHttpMockServerBase(unittest.TestCase, ABC): + """HTTP 1.1 test case with MockServer""" + + @property + @abstractmethod + def settings_dict(self) -> dict[str, Any] | None: + raise NotImplementedError + + is_secure = False + + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_download_with_content_length(self): + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid + # download it + yield crawler.crawl( + seed=Request( + url=self.mockserver.url("/partial", is_secure=self.is_secure), + meta={"download_maxsize": 1000}, + ) + ) + failure = crawler.spider.meta["failure"] + assert isinstance(failure.value, defer.CancelledError) + + @defer.inlineCallbacks + def test_download(self): + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + yield crawler.crawl( + seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) + ) + failure = crawler.spider.meta.get("failure") + assert failure is None + reason = crawler.spider.meta["close_reason"] + assert reason == "finished" + + +class UriResource(resource.Resource): + """Return the full uri that was requested""" + + def getChild(self, path, request): + return self + + def render(self, request): + # Note: this is an ugly hack for CONNECT request timeout test. + # Returning some data here fail SSL/TLS handshake + # ToDo: implement proper HTTPS proxy tests, not faking them. + if request.method != b"CONNECT": + return request.uri + return b"" + + +class TestHttpProxyBase(unittest.TestCase, ABC): + expected_http_proxy_request_body = b"http://example.com" + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + def setUp(self): + site = server.Site(UriResource(), timeout=None) + wrapper = WrappingFactory(site) + self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") + self.portno = self.port.getHost().port + self.download_handler = build_from_crawler( + self.download_handler_cls, get_crawler() + ) + self.download_request = self.download_handler.download_request + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + if hasattr(self.download_handler, "close"): + yield self.download_handler.close() + + def getURL(self, path): + return f"http://127.0.0.1:{self.portno}/{path}" + + def test_download_with_proxy(self): + def _test(response): + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body + + http_proxy = self.getURL("") + request = Request("http://example.com", meta={"proxy": http_proxy}) + return self.download_request(request, Spider("foo")).addCallback(_test) + + def test_download_without_proxy(self): + def _test(response): + assert response.status == 200 + assert response.url == request.url + assert response.body == b"/path/to/resource" + + request = Request(self.getURL("path/to/resource")) + return self.download_request(request, Spider("foo")).addCallback(_test) + + @defer.inlineCallbacks + def test_download_with_proxy_https_timeout(self): + if NON_EXISTING_RESOLVABLE: + pytest.skip("Non-existing hosts are resolvable") + http_proxy = self.getURL("") + domain = "https://no-such-domain.nosuch" + request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) + d = self.download_request(request, Spider("foo")) + timeout = yield self.assertFailure(d, error.TimeoutError) + assert domain in timeout.osError + + def test_download_with_proxy_without_http_scheme(self): + def _test(response): + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body + + http_proxy = self.getURL("").replace("http://", "") + request = Request("http://example.com", meta={"proxy": http_proxy}) + return self.download_request(request, Spider("foo")).addCallback(_test) From 0cdb971f636fc75ed08ed13642adfa04fcc235ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 10:06:04 +0200 Subject: [PATCH 1738/2083] Complete the release notes --- docs/news.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index eb5370b6e..76e195feb 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -18,6 +18,21 @@ Scrapy 2.13.1 (unreleased) only sent when there are not enough pending callback requests to reach concurrency limits. + (:issue:`6828`) + +- Added a deepwiki_ badge to the README. (:issue:`6793`) + + .. _deepwiki: https://deepwiki.com/scrapy/scrapy + +- Fixed a typo in the code example of :ref:`start-requests-lazy`. + (:issue:`6812`, :issue:`6815`) + +- Fixed a typo in the :ref:`coroutine-support` section of the documentation. + (:issue:`6822`) + +- Made this page more prominently listed in PyPI project links. + (:issue:`6826`) + .. _release-2.13.0: From 08ee88456f850f247117aa602f7b066356b66419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 10:12:37 +0200 Subject: [PATCH 1739/2083] bumpversion: set the release date automatically --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 47707e061..3b8174afe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,6 +120,12 @@ commit = true tag = true tag_name = "{new_version}" +[[tool.bumpversion.files]] +filename = "docs/news.rst" +search = "\\(unreleased\\)$" +replace = "({now:%Y-%m-%d})" +regex = true + [[tool.bumpversion.files]] filename = "scrapy/VERSION" From 7fe7f1734aba5625f8c9c405bb92ef1da18cd983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 10:12:46 +0200 Subject: [PATCH 1740/2083] =?UTF-8?q?Bump=20version:=202.13.0=20=E2=86=92?= =?UTF-8?q?=202.13.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/news.rst | 2 +- pyproject.toml | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 76e195feb..ef3b549e7 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.13.1: -Scrapy 2.13.1 (unreleased) +Scrapy 2.13.1 (2025-05-28) -------------------------- - Give callback requests precedence over start requests when priority values diff --git a/pyproject.toml b/pyproject.toml index 3b8174afe..68c1e07bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,7 +115,7 @@ module = "twisted" implicit_reexport = true [tool.bumpversion] -current_version = "2.13.0" +current_version = "2.13.1" commit = true tag = true tag_name = "{new_version}" diff --git a/scrapy/VERSION b/scrapy/VERSION index fb2c0766b..94f15e9cc 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.13.0 +2.13.1 From 7b4cf06b6e381b6210fe43bc94a1eb65b3db0583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 10:54:36 +0200 Subject: [PATCH 1741/2083] Feature the new logo in the README (#6831) --- README.rst | 22 ++++++++++------------ docs/_static/logo.svg | 1 + 2 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 docs/_static/logo.svg diff --git a/README.rst b/README.rst index 29488d825..30001e4b0 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,10 @@ -.. image:: https://scrapy.org/img/scrapylogo.png - :target: https://scrapy.org/ +.. raw:: html -====== -Scrapy -====== +

+ + Scrapy + +

.. image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy @@ -37,13 +38,10 @@ Scrapy :target: https://deepwiki.com/scrapy/scrapy :alt: Ask DeepWiki - -Overview -======== - -Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to -crawl websites and extract structured data from their pages. It can be used for -a wide range of purposes, from data mining to monitoring and automated testing. +Scrapy is a BSD-licensed fast high-level web crawling and web scraping +framework, used to crawl websites and extract structured data from their pages. +It can be used for a wide range of purposes, from data mining to monitoring and +automated testing. Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other contributors`_. diff --git a/docs/_static/logo.svg b/docs/_static/logo.svg new file mode 100644 index 000000000..04b2d18a7 --- /dev/null +++ b/docs/_static/logo.svg @@ -0,0 +1 @@ + From f98ffc71d25ae74351df70ae03ef5299a2ab3813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 10:58:34 +0200 Subject: [PATCH 1742/2083] 2.13.1 (#6832) --- docs/news.rst | 31 ++++++++++++++++++++++ pyproject.toml | 8 +++++- scrapy/VERSION | 2 +- scrapy/core/scheduler.py | 4 +-- scrapy/pqueues.py | 30 ++++++++++++---------- tests/test_engine_loop.py | 20 +++++++-------- tests/test_pqueues.py | 54 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 122 insertions(+), 27 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index cf1c35893..ef3b549e7 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,37 @@ Release notes ============= +.. _release-2.13.1: + +Scrapy 2.13.1 (2025-05-28) +-------------------------- + +- Give callback requests precedence over start requests when priority values + are the same. + + This makes changes from 2.13.0 to start request handling more intuitive and + backward compatible. For scenarios where all requests have the same + priorities, in 2.13.0 all start requests were sent before the first + callback request. In 2.13.1, same as in 2.12 and lower, start requests are + only sent when there are not enough pending callback requests to reach + concurrency limits. + + (:issue:`6828`) + +- Added a deepwiki_ badge to the README. (:issue:`6793`) + + .. _deepwiki: https://deepwiki.com/scrapy/scrapy + +- Fixed a typo in the code example of :ref:`start-requests-lazy`. + (:issue:`6812`, :issue:`6815`) + +- Fixed a typo in the :ref:`coroutine-support` section of the documentation. + (:issue:`6822`) + +- Made this page more prominently listed in PyPI project links. + (:issue:`6826`) + + .. _release-2.13.0: Scrapy 2.13.0 (2025-05-08) diff --git a/pyproject.toml b/pyproject.toml index 47707e061..68c1e07bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,11 +115,17 @@ module = "twisted" implicit_reexport = true [tool.bumpversion] -current_version = "2.13.0" +current_version = "2.13.1" commit = true tag = true tag_name = "{new_version}" +[[tool.bumpversion.files]] +filename = "docs/news.rst" +search = "\\(unreleased\\)$" +replace = "({now:%Y-%m-%d})" +regex = true + [[tool.bumpversion.files]] filename = "scrapy/VERSION" diff --git a/scrapy/VERSION b/scrapy/VERSION index fb2c0766b..94f15e9cc 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.13.0 +2.13.1 diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 57d27b7cf..9ac447289 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -173,8 +173,8 @@ class Scheduler(BaseScheduler): :ref:`Start requests ` are sent in the order they are yielded from :meth:`~scrapy.Spider.start`, and given the same - :attr:`~scrapy.http.Request.priority`, start requests take precedence over - other requests. + :attr:`~scrapy.http.Request.priority`, other requests take precedence over + start requests. You can set :setting:`SCHEDULER_START_MEMORY_QUEUE` and :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` to handle start requests diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index e6c6b8bf1..34b235d83 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -160,28 +160,32 @@ class ScrapyPriorityQueue: def pop(self) -> Request | None: while self.curprio is not None: - if self._start_queues: - try: - q = self._start_queues[self.curprio] - except KeyError: - pass - else: - m = q.pop() - if not q: - del self._start_queues[self.curprio] - q.close() - return m try: q = self.queues[self.curprio] except KeyError: - self._update_curprio() + pass else: m = q.pop() if not q: del self.queues[self.curprio] q.close() - self._update_curprio() + if not self._start_queues: + self._update_curprio() return m + if self._start_queues: + try: + q = self._start_queues[self.curprio] + except KeyError: + self._update_curprio() + else: + m = q.pop() + if not q: + del self._start_queues[self.curprio] + q.close() + self._update_curprio() + return m + else: + self._update_curprio() return None def _update_curprio(self) -> None: diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 90af10f0e..c7dbc82d4 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -189,9 +189,9 @@ class RequestSendOrderTestCase(TestCase): @deferred_f_from_coro_f async def test_default(self): - """By default, start requests take priority over callback requests and + """By default, callback requests take priority over start requests and are sent in order. Priority matters, but given the same priority, a - start request takes precedence.""" + callback request takes precedence.""" nums = [1, 2, 3, 4, 5, 6] response_seconds = 0 download_slots = 1 @@ -207,13 +207,13 @@ class RequestSendOrderTestCase(TestCase): yield _request(1) for request in ( - _request(4, priority=1), - _request(6), + _request(2, priority=1), + _request(5), ): spider.crawler.engine._slot.scheduler.enqueue_request(request) - yield _request(5) - yield _request(2, priority=1) + yield _request(6) yield _request(3, priority=1) + yield _request(4, priority=1) def parse(spider, response): return @@ -249,13 +249,13 @@ class RequestSendOrderTestCase(TestCase): yield _request(1) for request in ( - _request(4, priority=1), - _request(6), + _request(2, priority=1), + _request(5), ): spider.crawler.engine._slot.scheduler.enqueue_request(request) - yield _request(5) + yield _request(6) + yield _request(4, priority=1) yield _request(3, priority=1) - yield _request(2, priority=1) def parse(spider, response): return diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index d5c710ed2..b65f1b7e7 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -7,6 +7,7 @@ from scrapy.http.request import Request from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue +from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler from tests.test_scheduler import MockDownloader, MockEngine @@ -155,3 +156,56 @@ class TestDownloaderAwarePriorityQueue: assert self.queue.peek().url == req3.url assert self.queue.pop().url == req3.url assert self.queue.peek() is None + + +@pytest.mark.parametrize( + ("input", "output"), + [ + # By default, start requests are FIFO, other requests are LIFO. + ([{}, {}], [2, 1]), + ([{"start": True}, {"start": True}], [1, 2]), + # Priority matters. + ([{"priority": 1}, {"start": True}], [1, 2]), + ([{}, {"start": True, "priority": 1}], [2, 1]), + # For the same priority, start requests pop last. + ([{}, {"start": True}], [1, 2]), + ([{"start": True}, {}], [2, 1]), + ], +) +def test_pop_order(input, output): + def make_url(index): + return f"https://toscrape.com/{index}" + + def make_request(index, data): + meta = {} + if data.get("start", False): + meta["is_start_request"] = True + return Request( + url=make_url(index), + priority=data.get("priority", 0), + meta=meta, + ) + + input_requests = [ + make_request(index, data) for index, data in enumerate(input, start=1) + ] + expected_output_urls = [make_url(index) for index in output] + + crawler = get_crawler(Spider) + settings = crawler.settings + queue = build_from_crawler( + ScrapyPriorityQueue, + crawler, + downstream_queue_cls=load_object(settings["SCHEDULER_MEMORY_QUEUE"]), + key="", + start_queue_cls=load_object(settings["SCHEDULER_START_MEMORY_QUEUE"]), + ) + + for request in input_requests: + queue.push(request) + + actual_output_urls = [] + while request := queue.pop(): + actual_output_urls.append(request.url) + + assert actual_output_urls == expected_output_urls From c480c77f54e1b417468847ab5437458cf5beffe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 11:35:18 +0200 Subject: [PATCH 1743/2083] Shorten the README and remove broken links to scrapy.org (#6833) --- README.rst | 68 ++++++------------------------------------- docs/contributing.rst | 9 ++++-- 2 files changed, 16 insertions(+), 61 deletions(-) diff --git a/README.rst b/README.rst index 30001e4b0..5dc994570 100644 --- a/README.rst +++ b/README.rst @@ -38,74 +38,24 @@ :target: https://deepwiki.com/scrapy/scrapy :alt: Ask DeepWiki -Scrapy is a BSD-licensed fast high-level web crawling and web scraping -framework, used to crawl websites and extract structured data from their pages. -It can be used for a wide range of purposes, from data mining to monitoring and -automated testing. - -Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other -contributors`_. +Scrapy_ is a web scraping framework to extract structured data from websites. +It is cross-platform, and requires Python 3.9+. It is maintained by Zyte_ +(formerly Scrapinghub) and `many other contributors`_. .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapy: https://scrapy.org/ .. _Zyte: https://www.zyte.com/ -Check the Scrapy homepage at https://scrapy.org for more information, -including a list of features. - - -Requirements -============ - -* Python 3.9+ -* Works on Linux, Windows, macOS, BSD - -Install -======= - -The quick way: +Install with: .. code:: bash pip install scrapy -See the install section in the documentation at -https://docs.scrapy.org/en/latest/intro/install.html for more details. +And follow the documentation_ to learn how to use it. -Documentation -============= +.. _documentation: https://docs.scrapy.org/en/latest/ -Documentation is available online at https://docs.scrapy.org/ and in the ``docs`` -directory. +If you wish to contribute, see Contributing_. -Releases -======== - -You can check https://docs.scrapy.org/en/latest/news.html for the release notes. - -Community (blog, twitter, mail list, IRC) -========================================= - -See https://scrapy.org/community/ for details. - -Contributing -============ - -See https://docs.scrapy.org/en/master/contributing.html for details. - -Code of Conduct ---------------- - -Please note that this project is released with a Contributor `Code of Conduct `_. - -By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@zyte.com. - -Companies using Scrapy -====================== - -See https://scrapy.org/companies/ for a list. - -Commercial Support -================== - -See https://scrapy.org/support/ for details. +.. _Contributing: https://docs.scrapy.org/en/master/contributing.html diff --git a/docs/contributing.rst b/docs/contributing.rst index f5c1c74b8..0172887d6 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -6,8 +6,13 @@ Contributing to Scrapy .. important:: - Double check that you are reading the most recent version of this document at - https://docs.scrapy.org/en/master/contributing.html + Double check that you are reading the most recent version of this document + at https://docs.scrapy.org/en/master/contributing.html + + By participating in this project you agree to abide by the terms of our + `Code of Conduct + `_. Please + report unacceptable behavior to opensource@zyte.com. There are many ways to contribute to Scrapy. Here are some of them: From dceb85bf3e41d06dfac81c037c69fd0f1ab61156 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 May 2025 14:46:39 +0500 Subject: [PATCH 1744/2083] Add is_asyncio_available(). (#6827) * Add is_asyncio_available(). * Print unexpected warnings in test_install_asyncio_reactor(). * Fix printing warnings. * Fix printing warnings - 2. * Skip TestDeferredToFuture on non-asyncio. * Test the is_asyncio_available() exception. --- docs/topics/asyncio.rst | 11 +++--- scrapy/utils/asyncio.py | 38 +++++++++++++++++++ scrapy/utils/defer.py | 7 ++-- scrapy/utils/reactor.py | 6 +++ .../CrawlerProcess/asyncio_enabled_reactor.py | 10 +++++ tests/test_utils_asyncio.py | 33 ++-------------- tests/test_utils_defer.py | 4 +- tests/test_utils_deprecate.py | 2 +- tests/test_utils_reactor.py | 35 +++++++++++++++++ 9 files changed, 106 insertions(+), 40 deletions(-) create mode 100644 scrapy/utils/asyncio.py create mode 100644 tests/test_utils_reactor.py diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 0490129b3..473ef7bfa 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -105,25 +105,26 @@ Enforcing asyncio as a requirement ================================== If you are writing a :ref:`component ` that requires asyncio -to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to +to work, use :func:`scrapy.utils.asyncio.is_asyncio_available` to :ref:`enforce it as a requirement `. For example: .. code-block:: python - from scrapy.utils.reactor import is_asyncio_reactor_installed + from scrapy.utils.asyncio import is_asyncio_available class MyComponent: def __init__(self): - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): raise ValueError( - f"{MyComponent.__qualname__} requires the asyncio Twisted " - f"reactor. Make sure you have it configured in the " + f"{MyComponent.__qualname__} requires the asyncio support. " + f"Make sure you have configured the asyncio reactor in the " f"TWISTED_REACTOR setting. See the asyncio documentation " f"of Scrapy for more information." ) +.. autofunction:: scrapy.utils.asyncio.is_asyncio_available .. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py new file mode 100644 index 000000000..4469369fa --- /dev/null +++ b/scrapy/utils/asyncio.py @@ -0,0 +1,38 @@ +"""Utilities related to asyncio and its support in Scrapy.""" + +from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed + + +def is_asyncio_available() -> bool: + """Check if it's possible to call asyncio code that relies on the asyncio event loop. + + .. versionadded:: VERSION + + Currently this function is identical to + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed`: it returns + ``True`` if the Twisted reactor that is installed is + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, returns + ``False`` if a different reactor is installed, and raises a + :exc:`RuntimeError` if no reactor is installed. In a future Scrapy version, + when Scrapy supports running without a Twisted reactor, this function will + also return ``True`` when running in that mode, so code that doesn't + directly require a Twisted reactor should use this function instead of + :func:`~scrapy.utils.reactor.is_asyncio_reactor_installed`. + + When this returns ``True``, an asyncio loop is installed and used by + Scrapy. It's possible to call functions that require it, such as + :func:`asyncio.sleep`, and await on :class:`asyncio.Future` objects in + Scrapy-related code. + + When this returns ``False``, a non-asyncio Twisted reactor is installed. + It's not possible to use asyncio features that require an asyncio event + loop or await on :class:`asyncio.Future` objects in Scrapy-related code, + but it's possible to await on :class:`~twisted.internet.defer.Deferred` + objects. + """ + if not is_reactor_installed(): + raise RuntimeError( + "is_asyncio_available() called without an installed reactor." + ) + + return is_asyncio_reactor_installed() diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d06397f50..4649c4daa 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -22,7 +22,8 @@ from twisted.internet.task import Cooperator from twisted.python import failure from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning -from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.reactor import _get_asyncio_event_loop if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable @@ -379,7 +380,7 @@ def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2: if isinstance(o, Deferred): return o if inspect.isawaitable(o): - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o)) @@ -471,6 +472,6 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: deferred = self.crawler.engine.download(additional_request) additional_response = await maybe_deferred_to_future(deferred) """ - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): return d return deferred_to_future(d) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 9c2754394..5e76da37b 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -202,6 +202,12 @@ def is_asyncio_reactor_installed() -> bool: """Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. Raise a :exc:`RuntimeError` if no reactor is installed. + + In a future Scrapy version, when Scrapy supports running without a Twisted + reactor, this function won't be useful for checking if it's possible to use + asyncio features, so the code that that doesn't directly require a Twisted + reactor should use :func:`scrapy.utils.asyncio.is_asyncio_available` + instead of this function. """ if not is_reactor_installed(): raise RuntimeError( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index f3dab12fe..4e8d3db12 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,5 +1,6 @@ import scrapy from scrapy.crawler import CrawlerProcess +from scrapy.utils.asyncio import is_asyncio_available from scrapy.utils.reactor import ( install_reactor, is_asyncio_reactor_installed, @@ -18,6 +19,13 @@ except RuntimeError: else: raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.") +try: + is_asyncio_available() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_available() did not raise RuntimeError.") + if is_reactor_installed(): raise RuntimeError( "Reactor already installed after is_asyncio_reactor_installed()." @@ -33,6 +41,8 @@ class ReactorCheckExtension: def __init__(self): if not is_asyncio_reactor_installed(): raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + if not is_asyncio_available(): + raise RuntimeError("ReactorCheckExtension requires asyncio support.") class NoRequestsSpider(scrapy.Spider): diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 901e03d59..fe44748f9 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,35 +1,10 @@ -import asyncio -import warnings - import pytest -from twisted.trial.unittest import TestCase -from scrapy.utils.defer import deferred_f_from_coro_f -from scrapy.utils.reactor import ( - install_reactor, - is_asyncio_reactor_installed, - set_asyncio_event_loop, -) +from scrapy.utils.asyncio import is_asyncio_available @pytest.mark.usefixtures("reactor_pytest") -class TestAsyncio(TestCase): - def test_is_asyncio_reactor_installed(self): +class TestAsyncio: + def test_is_asyncio_available(self): # the result should depend only on the pytest --reactor argument - assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default") - - def test_install_asyncio_reactor(self): - from twisted.internet import reactor as original_reactor - - with warnings.catch_warnings(record=True) as w: - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - assert len(w) == 0 - from twisted.internet import reactor # pylint: disable=reimported - - assert original_reactor == reactor - - @pytest.mark.only_asyncio - @deferred_f_from_coro_f - async def test_set_asyncio_event_loop(self): - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - assert set_asyncio_event_loop(None) is asyncio.get_running_loop() + assert is_asyncio_available() == (self.reactor_pytest != "default") diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 29cd5fbf2..c565c1c4e 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -311,6 +311,7 @@ class TestDeferredFFromCoroF(unittest.TestCase): yield self._assert_result(c_f) +@pytest.mark.only_asyncio class TestDeferredToFuture(unittest.TestCase): @deferred_f_from_coro_f async def test_deferred(self): @@ -332,7 +333,6 @@ class TestDeferredToFuture(unittest.TestCase): future_result = await result assert future_result == 42 - @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_wrapped_coroutine_asyncio(self): async def c_f() -> int: @@ -340,7 +340,7 @@ class TestDeferredToFuture(unittest.TestCase): return 42 d = deferred_from_coro(c_f()) - result = maybe_deferred_to_future(d) + result = deferred_to_future(d) assert isinstance(result, Future) future_result = await result assert future_result == 42 diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 52c165bb4..662de0dc3 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -243,7 +243,7 @@ class TestWarnWhenSubclassed: ) w = self._mywarnings(w) - assert len(w) == 0, str(map(str, w)) + assert len(w) == 0, [str(warning) for warning in w] with warnings.catch_warnings(record=True) as w: AlsoDeprecated() diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py new file mode 100644 index 000000000..99f175c60 --- /dev/null +++ b/tests/test_utils_reactor.py @@ -0,0 +1,35 @@ +import asyncio +import warnings + +import pytest +from twisted.trial.unittest import TestCase + +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + set_asyncio_event_loop, +) + + +@pytest.mark.usefixtures("reactor_pytest") +class TestAsyncio(TestCase): + def test_is_asyncio_reactor_installed(self): + # the result should depend only on the pytest --reactor argument + assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default") + + def test_install_asyncio_reactor(self): + from twisted.internet import reactor as original_reactor + + with warnings.catch_warnings(record=True) as w: + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + assert len(w) == 0, [str(warning) for warning in w] + from twisted.internet import reactor # pylint: disable=reimported + + assert original_reactor == reactor + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_set_asyncio_event_loop(self): + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + assert set_asyncio_event_loop(None) is asyncio.get_running_loop() From 916fe509744c39be63b35c859e50a35a2ac4333d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 May 2025 15:53:52 +0500 Subject: [PATCH 1745/2083] Fix and ban the top-level twisted.internet.reactor imports. (#6835) --- extras/qps-bench-server.py | 2 +- pyproject.toml | 7 +++++++ tests/CrawlerProcess/reactor_default.py | 2 +- .../reactor_default_twisted_reactor_select.py | 2 +- tests/CrawlerRunner/change_reactor.py | 2 +- tests/CrawlerRunner/ip_address.py | 3 ++- tests/mockserver.py | 6 +++++- .../__init__.py | 2 +- tests/test_core_downloader.py | 5 ++++- .../test_downloader_handler_twisted_http2.py | 6 +++++- tests/test_downloader_handlers.py | 5 ++++- tests/test_downloader_handlers_http_base.py | 19 ++++++++++++++++++- tests/test_downloadermiddleware_robotstxt.py | 12 +++++++++++- tests/test_engine.py | 6 +++++- tests/test_http2_client_protocol.py | 3 ++- tests/test_pipeline_media.py | 3 ++- tests/test_utils_signal.py | 6 +++++- tests/test_webclient.py | 6 +++++- 18 files changed, 80 insertions(+), 17 deletions(-) diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index 70c9003e5..734614aa5 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -2,7 +2,7 @@ from collections import deque from time import time -from twisted.internet import reactor +from twisted.internet import reactor # noqa: TID253 from twisted.web.resource import Resource from twisted.web.server import NOT_DONE_YET, Site diff --git a/pyproject.toml b/pyproject.toml index 68c1e07bb..871da8020 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -299,6 +299,8 @@ extend-select = [ "T10", # flake8-type-checking "TC", + # flake8-tidy-imports + "TID", # pyupgrade "UP", # pycodestyle warnings @@ -398,6 +400,11 @@ ignore = [ "SIM112", ] +[tool.ruff.lint.flake8-tidy-imports] +banned-module-level-imports = [ + "twisted.internet.reactor", +] + [tool.ruff.lint.per-file-ignores] # Circular import workarounds "scrapy/linkextractors/__init__.py" = ["E402"] diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 8f59c035c..cbe6427ea 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,4 +1,4 @@ -from twisted.internet import reactor # noqa: F401 +from twisted.internet import reactor # noqa: F401,TID253 from twisted.python import log import scrapy diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index 9901dd634..f7802fbc6 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -1,4 +1,4 @@ -from twisted.internet import reactor # noqa: F401 +from twisted.internet import reactor # noqa: F401,TID253 from twisted.python import log import scrapy diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py index 6c0102241..c275e0583 100644 --- a/tests/CrawlerRunner/change_reactor.py +++ b/tests/CrawlerRunner/change_reactor.py @@ -26,7 +26,7 @@ runner = CrawlerRunner() d = runner.crawl(NoRequestsSpider) -from twisted.internet import reactor # noqa: E402 +from twisted.internet import reactor # noqa: E402,TID253 d.addBoth(callback=lambda _: reactor.stop()) reactor.run() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 5e2184afb..207fc86ad 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -6,7 +6,6 @@ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") from urllib.parse import urlparse -from twisted.internet import reactor from twisted.names import cache, resolve from twisted.names import hosts as hostsModule from twisted.names.client import Resolver @@ -44,6 +43,8 @@ class LocalhostSpider(Spider): if __name__ == "__main__": + from twisted.internet import reactor + with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: port = urlparse(mock_http_server.http_address).port url = f"http://not.a.real.domain:{port}/echo" diff --git a/tests/mockserver.py b/tests/mockserver.py index f5c12787a..e0ac127f2 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING from urllib.parse import urlencode from OpenSSL import SSL -from twisted.internet import defer, reactor, ssl +from twisted.internet import defer, ssl from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory @@ -114,6 +114,8 @@ class LeafResource(resource.Resource): isLeaf = True def deferRequest(self, request, delay, f, *a, **kw): + from twisted.internet import reactor + def _cancelrequest(_): # silence CancelledError d.addErrback(lambda _: None) @@ -378,6 +380,8 @@ def ssl_context_factory( if __name__ == "__main__": + from twisted.internet import reactor + parser = argparse.ArgumentParser() parser.add_argument( "-t", "--type", type=str, choices=("http", "dns"), default="http" diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index 5006e3689..c6fdb13ea 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -2,7 +2,7 @@ import sys from pathlib import Path from subprocess import PIPE, Popen -from .. import TWISTED_KEEPS_TRACEBACKS +from tests import TWISTED_KEEPS_TRACEBACKS class TestCmdlineCrawlPipeline: diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 1bffd69ed..ef77f7843 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -8,7 +8,6 @@ from typing import Any import OpenSSL.SSL import pytest -from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest @@ -40,6 +39,8 @@ class TestContextFactoryBase(unittest.TestCase): context_factory = None def _listen(self, site): + from twisted.internet import reactor + return reactor.listenSSL( 0, site, @@ -71,6 +72,8 @@ class TestContextFactoryBase(unittest.TestCase): client_context_factory: BrowserLikePolicyForHTTPS, body: str | None = None, ) -> bytes: + from twisted.internet import reactor + agent = Agent(reactor, contextFactory=client_context_factory) body_producer = _RequestBodyProducer(body.encode()) if body else None response: TxResponse = await maybe_deferred_to_future( diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 46322a747..159f403d0 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -8,7 +8,7 @@ from unittest import mock import pytest from testfixtures import LogCapture -from twisted.internet import defer, error, reactor +from twisted.internet import defer, error from twisted.web import server from twisted.web.error import SchemeNotSupported from twisted.web.http import H2_ENABLED @@ -59,6 +59,8 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): @defer.inlineCallbacks def test_download_with_maxsize_very_large_file(self): + from twisted.internet import reactor + with mock.patch("scrapy.core.http2.stream.logger") as logger: request = Request(self.getURL("largechunkedfile")) @@ -207,6 +209,8 @@ class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase): expected_http_proxy_request_body = b"/" def setUp(self): + from twisted.internet import reactor + site = server.Site(UriResource(), timeout=None) self.port = reactor.listenSSL( 0, diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index fc6ac5aee..dacadb075 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -12,7 +12,6 @@ from unittest import mock import pytest from twisted.cred import checkers, credentials, portal -from twisted.internet import reactor from twisted.protocols.ftp import FTPFactory, FTPRealm from twisted.trial import unittest from w3lib.url import path_to_file_uri @@ -310,6 +309,8 @@ class TestFTPBase(unittest.TestCase): ) def setUp(self): + from twisted.internet import reactor + # setup dirs and test file self.directory = Path(mkdtemp()) userdir = self.directory / self.username @@ -451,6 +452,8 @@ class TestAnonymousFTP(TestFTPBase): req_meta = {} def setUp(self): + from twisted.internet import reactor + # setup dir and test file self.directory = Path(mkdtemp()) for filename, content in self.test_files: diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 46e5972f7..5eaf66966 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -12,7 +12,7 @@ from unittest import mock import pytest from testfixtures import LogCapture -from twisted.internet import defer, error, reactor +from twisted.internet import defer, error from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util @@ -52,6 +52,8 @@ class ContentLengthHeaderResource(resource.Resource): class ChunkedResource(resource.Resource): def render(self, request): + from twisted.internet import reactor + def response(): request.write(b"chunked ") request.write(b"content\n") @@ -63,6 +65,8 @@ class ChunkedResource(resource.Resource): class BrokenChunkedResource(resource.Resource): def render(self, request): + from twisted.internet import reactor + def response(): request.write(b"chunked ") request.write(b"content\n") @@ -76,6 +80,8 @@ class BrokenChunkedResource(resource.Resource): class BrokenDownloadResource(resource.Resource): def render(self, request): + from twisted.internet import reactor + def response(): request.setHeader(b"Content-Length", b"20") request.write(b"partial") @@ -105,6 +111,8 @@ class EmptyContentTypeHeaderResource(resource.Resource): class LargeChunkedFileResource(resource.Resource): def render(self, request): + from twisted.internet import reactor + def response(): for i in range(1024): request.write(b"x" * 1024) @@ -133,6 +141,8 @@ class TestHttpBase(unittest.TestCase, ABC): raise NotImplementedError def setUp(self): + from twisted.internet import reactor + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) @@ -365,6 +375,9 @@ class TestHttp11Base(TestHttpBase): @defer.inlineCallbacks def test_download_with_maxsize_very_large_file(self): + from twisted.internet import reactor + + # TODO: the logger check is specific to scrapy.core.downloader.handlers.http11 with mock.patch("scrapy.core.downloader.handlers.http11.logger") as logger: request = Request(self.getURL("largechunkedfile")) @@ -501,6 +514,8 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC): raise NotImplementedError def setUp(self): + from twisted.internet import reactor + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) @@ -639,6 +654,8 @@ class TestHttpProxyBase(unittest.TestCase, ABC): raise NotImplementedError def setUp(self): + from twisted.internet import reactor + site = server.Site(UriResource(), timeout=None) wrapper = WrappingFactory(site) self.port = reactor.listenTCP(0, wrapper, interface="127.0.0.1") diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 9518f1835..04800896c 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING from unittest import mock import pytest -from twisted.internet import error, reactor +from twisted.internet import error from twisted.internet.defer import Deferred, maybeDeferred from twisted.python import failure from twisted.trial import unittest @@ -53,6 +53,8 @@ Disallow: /some/randome/page.html response = TextResponse("http://site.local/robots.txt", body=ROBOTS) def return_response(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -102,6 +104,8 @@ Disallow: /some/randome/page.html ) def return_response(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -124,6 +128,8 @@ Disallow: /some/randome/page.html response = Response("http://site.local/robots.txt") def return_response(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -145,6 +151,8 @@ Disallow: /some/randome/page.html err = error.DNSLookupError("Robotstxt address not found") def return_failure(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(err)) return deferred @@ -178,6 +186,8 @@ Disallow: /some/randome/page.html self.crawler.settings.set("ROBOTSTXT_OBEY", True) def ignore_request(request): + from twisted.internet import reactor + deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest())) return deferred diff --git a/tests/test_engine.py b/tests/test_engine.py index b60b510b2..1f79a081d 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -24,7 +24,7 @@ from urllib.parse import urlparse import attr from itemadapter import ItemAdapter from pydispatch import dispatcher -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.trial import unittest from twisted.web import server, static, util @@ -130,6 +130,8 @@ class ChangeCloseReasonSpider(MySpider): def start_test_site(debug=False): + from twisted.internet import reactor + root_dir = Path(tests_datadir, "test_site") r = static.File(str(root_dir)) r.putChild(b"redirect", util.Redirect(b"/redirected")) @@ -514,6 +516,8 @@ def test_request_scheduled_signal(caplog): if __name__ == "__main__": + from twisted.internet import reactor # pylint: disable=ungrouped-imports + if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) reactor.run() diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 7c1b38877..0605c2438 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -13,7 +13,6 @@ from unittest import mock, skipIf from urllib.parse import urlencode import pytest -from twisted.internet import reactor from twisted.internet.defer import ( CancelledError, Deferred, @@ -209,6 +208,8 @@ class TestHttps2ClientProtocol(TestCase): @inlineCallbacks def setUp(self): + from twisted.internet import reactor + # Initialize resource tree root = self._init_resource() self.site = Site(root, timeout=None) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d915fc2a3..2d0db6e25 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -4,7 +4,6 @@ import warnings import pytest from testfixtures import LogCapture -from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from twisted.trial import unittest @@ -329,6 +328,8 @@ class TestMediaPipeline(TestBaseMediaPipeline): rsp1 = Response("http://url") def rsp1_func(): + from twisted.internet import reactor + dfd = Deferred().addCallback(_check_downloading) reactor.callLater(0.1, dfd.callback, rsp1) return dfd diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 6dff321da..9b3ce6b0b 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -3,7 +3,7 @@ import asyncio import pytest from pydispatch import dispatcher from testfixtures import LogCapture -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.python.failure import Failure from twisted.trial import unittest @@ -65,6 +65,8 @@ class SendCatchLogDeferredTest(TestSendCatchLog): class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): def ok_handler(self, arg, handlers_called): + from twisted.internet import reactor + handlers_called.add(self.ok_handler) assert arg == "test" d = defer.Deferred() @@ -97,6 +99,8 @@ class SendCatchLogAsyncTest(TestSendCatchLog): class SendCatchLogAsyncTest2(SendCatchLogAsyncTest): def ok_handler(self, arg, handlers_called): + from twisted.internet import reactor + handlers_called.add(self.ok_handler) assert arg == "test" d = defer.Deferred() diff --git a/tests/test_webclient.py b/tests/test_webclient.py index c3c03d6c3..e580d51ca 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -12,7 +12,7 @@ from urllib.parse import urlparse import OpenSSL.SSL import pytest -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.defer import inlineCallbacks from twisted.internet.testing import StringTransport from twisted.protocols.policies import WrappingFactory @@ -205,6 +205,8 @@ class EncodingResource(resource.Resource): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestWebClient(unittest.TestCase): def _listen(self, site): + from twisted.internet import reactor + return reactor.listenTCP(0, site, interface="127.0.0.1") def setUp(self): @@ -318,6 +320,8 @@ class TestWebClient(unittest.TestCase): assert b"404 - No Such Resource" in pageData def testFactoryInfo(self): + from twisted.internet import reactor + url = self.getURL("file") parsed = urlparse(url) factory = client.ScrapyHTTPClientFactory(Request(url)) From 05b3b205ce296c72063e05d15f5cae8047476ca2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 May 2025 18:55:44 +0500 Subject: [PATCH 1746/2083] Add `AsyncCrawlerProcess` and `Crawler.crawl_async()` (#6817) * Add a basic Crawler.crawl_async(). * Add custom loop tests for *CrawlerRunner. * Add AsyncCrawlerProcess. * Update related docs. * Update practices.rst. * Address test failures. * Add a note about AsyncCrawler* to the docs about switching reactors. * Address feedback. * Update for TID253. * Simplify test_crawler_crawl_async_twice_parallel_unsupported(). --- conftest.py | 2 + docs/topics/api.rst | 13 +- docs/topics/asyncio.rst | 9 +- docs/topics/practices.rst | 55 +-- docs/topics/settings.rst | 32 +- scrapy/core/engine.py | 16 +- scrapy/crawler.py | 301 ++++++++++++---- scrapy/utils/defer.py | 17 +- scrapy/utils/reactor.py | 35 +- tests/AsyncCrawlerProcess/args_settings.py | 25 ++ .../asyncio_custom_loop.py | 20 ++ ...o_custom_loop_custom_settings_different.py | 23 ++ ...syncio_custom_loop_custom_settings_same.py | 23 ++ .../asyncio_deferred_signal.py | 48 +++ .../asyncio_enabled_no_reactor.py | 27 ++ .../asyncio_enabled_reactor.py | 53 +++ .../asyncio_enabled_reactor_different_loop.py | 29 ++ .../asyncio_enabled_reactor_same_loop.py | 31 ++ .../caching_hostname_resolver.py | 35 ++ .../caching_hostname_resolver_ipv6.py | 22 ++ .../default_name_resolver.py | 18 + tests/AsyncCrawlerProcess/multi.py | 17 + tests/AsyncCrawlerProcess/reactor_default.py | 18 + tests/AsyncCrawlerProcess/simple.py | 16 + tests/AsyncCrawlerProcess/sleeping.py | 20 ++ .../twisted_reactor_asyncio.py | 15 + .../twisted_reactor_custom_settings.py | 14 + .../twisted_reactor_custom_settings_same.py | 22 ++ .../twisted_reactor_custom_settings_select.py | 30 ++ .../custom_loop_different.py | 31 ++ tests/AsyncCrawlerRunner/custom_loop_same.py | 31 ++ .../asyncio_enabled_reactor_different_loop.py | 8 +- .../asyncio_enabled_reactor_same_loop.py | 8 +- tests/CrawlerRunner/custom_loop_different.py | 29 ++ tests/CrawlerRunner/custom_loop_same.py | 29 ++ tests/test_crawler.py | 323 +++++++++++------- tests/test_utils_reactor.py | 5 +- 37 files changed, 1201 insertions(+), 249 deletions(-) create mode 100644 tests/AsyncCrawlerProcess/args_settings.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_custom_loop.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_deferred_signal.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py create mode 100644 tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py create mode 100644 tests/AsyncCrawlerProcess/caching_hostname_resolver.py create mode 100644 tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py create mode 100644 tests/AsyncCrawlerProcess/default_name_resolver.py create mode 100644 tests/AsyncCrawlerProcess/multi.py create mode 100644 tests/AsyncCrawlerProcess/reactor_default.py create mode 100644 tests/AsyncCrawlerProcess/simple.py create mode 100644 tests/AsyncCrawlerProcess/sleeping.py create mode 100644 tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py create mode 100644 tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py create mode 100644 tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py create mode 100644 tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py create mode 100644 tests/AsyncCrawlerRunner/custom_loop_different.py create mode 100644 tests/AsyncCrawlerRunner/custom_loop_same.py create mode 100644 tests/CrawlerRunner/custom_loop_different.py create mode 100644 tests/CrawlerRunner/custom_loop_same.py diff --git a/conftest.py b/conftest.py index 18132b7e6..ed7d14166 100644 --- a/conftest.py +++ b/conftest.py @@ -19,6 +19,8 @@ collect_ignore = [ "tests/mockserver.py", "tests/pipelines.py", "tests/spiders.py", + # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess + *_py_files("tests/AsyncCrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess *_py_files("tests/AsyncCrawlerRunner"), # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 3e7bc45c5..b11de2914 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -99,13 +99,11 @@ how you :ref:`configure the downloader middlewares provided while constructing the crawler, and it is created after the arguments given in the :meth:`crawl` method. - .. method:: crawl(*args, **kwargs) + .. automethod:: crawl_async - Starts the crawler by instantiating its spider class with the given - ``args`` and ``kwargs`` arguments, while setting the execution engine in - motion. Should be called only once. + .. automethod:: crawl - Returns a deferred that is fired when the crawl is finished. + .. automethod:: stop_async .. automethod:: stop @@ -115,6 +113,11 @@ how you :ref:`configure the downloader middlewares .. autoclass:: CrawlerRunner :members: +.. autoclass:: AsyncCrawlerProcess + :show-inheritance: + :members: + :inherited-members: + .. autoclass:: CrawlerProcess :show-inheritance: :members: diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 473ef7bfa..ad5c71fbf 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -20,7 +20,8 @@ To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, which is the default value. -If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to +If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.CrawlerRunner`, you also need to install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`: @@ -169,4 +170,8 @@ Switching to a non-asyncio reactor If for some reason your code doesn't work with the asyncio reactor, you can use a different reactor by setting the :setting:`TWISTED_REACTOR` setting to its import path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to -``None``, which will use the default reactor for your platform. +``None``, which will use the default reactor for your platform. If you are +using :class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.AsyncCrawlerProcess` you also need to switch to their +Deferred-based counterparts: :class:`~scrapy.crawler.CrawlerRunner` or +:class:`~scrapy.crawler.CrawlerProcess` respectively. diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 18005aaf2..56177ba4e 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -21,16 +21,21 @@ Remember that Scrapy is built on top of the Twisted asynchronous networking library, so you need to run it inside the Twisted reactor. The first utility you can use to run your spiders is -:class:`scrapy.crawler.CrawlerProcess`. This class will start a Twisted reactor -for you, configuring the logging and setting shutdown handlers. This class is -the one used by all Scrapy commands. +:class:`scrapy.crawler.AsyncCrawlerProcess` or +:class:`scrapy.crawler.CrawlerProcess`. These classes will start a Twisted +reactor for you, configuring the logging and setting shutdown handlers. These +classes are the ones used by all Scrapy commands. They have similar +functionality, differing in their asynchronous API style: +:class:`~scrapy.crawler.AsyncCrawlerProcess` returns coroutines from its +asynchronous methods while :class:`~scrapy.crawler.CrawlerProcess` returns +:class:`~twisted.internet.defer.Deferred` objects. Here's an example showing how to run a single spider with it. .. code-block:: python import scrapy - from scrapy.crawler import CrawlerProcess + from scrapy.crawler import AsyncCrawlerProcess class MySpider(scrapy.Spider): @@ -38,7 +43,7 @@ Here's an example showing how to run a single spider with it. ... - process = CrawlerProcess( + process = AsyncCrawlerProcess( settings={ "FEEDS": { "items.json": {"format": "json"}, @@ -49,53 +54,57 @@ Here's an example showing how to run a single spider with it. process.crawl(MySpider) process.start() # the script will block here until the crawling is finished -Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess` +You can define :ref:`settings ` within the dictionary passed +to :class:`~scrapy.crawler.AsyncCrawlerProcess`. Make sure to check the +:class:`~scrapy.crawler.AsyncCrawlerProcess` documentation to get acquainted with its usage details. If you are inside a Scrapy project there are some additional helpers you can use to import those components within the project. You can automatically import -your spiders passing their name to :class:`~scrapy.crawler.CrawlerProcess`, and -use ``get_project_settings`` to get a :class:`~scrapy.settings.Settings` -instance with your project settings. +your spiders passing their name to +:class:`~scrapy.crawler.AsyncCrawlerProcess`, and use +:func:`scrapy.utils.project.get_project_settings` to get a +:class:`~scrapy.settings.Settings` instance with your project settings. What follows is a working example of how to do that, using the `testspiders`_ project as example. .. code-block:: python - from scrapy.crawler import CrawlerProcess + from scrapy.crawler import AsyncCrawlerProcess from scrapy.utils.project import get_project_settings - process = CrawlerProcess(get_project_settings()) + process = AsyncCrawlerProcess(get_project_settings()) # 'followall' is the name of one of the spiders of the project. process.crawl("followall", domain="scrapy.org") process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling -process: :class:`scrapy.crawler.AsyncCrawlerRunner` and +process: :class:`scrapy.crawler.AsyncCrawlerRunner` or :class:`scrapy.crawler.CrawlerRunner`. These classes are thin wrappers that encapsulate some simple helpers to run multiple crawlers, but they won't -start or interfere with existing reactors in any way. They have similar -functionality, differing in their asynchronous API style: -:class:`~scrapy.crawler.AsyncCrawlerRunner` returns coroutines from its -asynchronous methods while :class:`~scrapy.crawler.CrawlerRunner` returns -:class:`~twisted.internet.defer.Deferred` objects. +start or interfere with existing reactors in any way. Just like +:class:`scrapy.crawler.AsyncCrawlerProcess` and +:class:`scrapy.crawler.CrawlerProcess` they differ in their asynchronous API +style. When using these classes the reactor should be explicitly run after scheduling your spiders. It's recommended that you use :class:`~scrapy.crawler.AsyncCrawlerRunner` or :class:`~scrapy.crawler.CrawlerRunner` instead of +:class:`~scrapy.crawler.AsyncCrawlerProcess` or :class:`~scrapy.crawler.CrawlerProcess` if your application is already using Twisted and you want to run Scrapy in the same reactor. If you want to stop the reactor or run any other code right after the spider -finishes you can do that after the :meth:`AsyncCrawlerRunner.crawl() -` coroutine completes (or the Deferred -returned from :meth:`CrawlerRunner.crawl() +finishes you can do that after the task returned from +:meth:`AsyncCrawlerRunner.crawl() ` +completes (or the Deferred returned from :meth:`CrawlerRunner.crawl() ` fires). In the simplest case you can also use :func:`twisted.internet.task.react` to start and stop the reactor, though -it may be easier to just use :class:`~scrapy.crawler.CrawlerProcess` instead. +it may be easier to just use :class:`~scrapy.crawler.AsyncCrawlerProcess` or +:class:`~scrapy.crawler.CrawlerProcess` instead. Here's an example of using :class:`~scrapy.crawler.AsyncCrawlerRunner` together with simple reactor management code: @@ -171,7 +180,7 @@ Here is an example that runs multiple spiders simultaneously: .. code-block:: python import scrapy - from scrapy.crawler import CrawlerProcess + from scrapy.crawler import AsyncCrawlerProcess from scrapy.utils.project import get_project_settings @@ -186,7 +195,7 @@ Here is an example that runs multiple spiders simultaneously: settings = get_project_settings() - process = CrawlerProcess(settings) + process = AsyncCrawlerProcess(settings) process.crawl(MySpider1) process.crawl(MySpider2) process.start() # the script will block here until all crawling jobs are finished diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0f81a0c0a..68c5079cf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -330,7 +330,8 @@ exception is raised. These settings are: -- :setting:`ASYNCIO_EVENT_LOOP` +- :setting:`ASYNCIO_EVENT_LOOP` (not possible to set per-spider when using + :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) - :setting:`DNS_RESOLVER` and settings used by the corresponding component, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE` @@ -338,12 +339,25 @@ These settings are: - :setting:`REACTOR_THREADPOOL_MAXSIZE` -- :setting:`TWISTED_REACTOR` +- :setting:`TWISTED_REACTOR` (ignored when using + :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) :setting:`ASYNCIO_EVENT_LOOP` and :setting:`TWISTED_REACTOR` are used upon installing the reactor. The rest of the settings are applied when starting the reactor. +There is an additional restriction for :setting:`TWISTED_REACTOR` and +:setting:`ASYNCIO_EVENT_LOOP` when using +:class:`~scrapy.crawler.AsyncCrawlerProcess`: when this class is instantiated, +it installs :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, +ignoring the value of :setting:`TWISTED_REACTOR` and using the value of +:setting:`ASYNCIO_EVENT_LOOP` that was passed to +:meth:`AsyncCrawlerProcess.__init__() +`. If a different value for +:setting:`TWISTED_REACTOR` or :setting:`ASYNCIO_EVENT_LOOP` is provided later, +e.g. in :ref:`per-spider settings `, an exception will be +raised. + .. _topics-settings-ref: @@ -1977,9 +1991,11 @@ Import path of a given :mod:`~twisted.internet.reactor`. Scrapy will install this reactor if no other reactor is installed yet, such as when the ``scrapy`` CLI program is invoked or when using the +:class:`~scrapy.crawler.AsyncCrawlerProcess` class or the :class:`~scrapy.crawler.CrawlerProcess` class. -If you are using the :class:`~scrapy.crawler.CrawlerRunner` class, you also +If you are using the :class:`~scrapy.crawler.AsyncCrawlerRunner` class or the +:class:`~scrapy.crawler.CrawlerRunner` class, you also need to install the correct reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`: @@ -1988,12 +2004,12 @@ need to install the correct reactor manually. You can do that using If a reactor is already installed, :func:`~scrapy.utils.reactor.install_reactor` has no effect. -:meth:`CrawlerRunner.__init__ ` raises -:exc:`Exception` if the installed reactor does not match the +:class:`~scrapy.crawler.AsyncCrawlerRunner` and other similar classes raise an +exception if the installed reactor does not match the :setting:`TWISTED_REACTOR` setting; therefore, having top-level :mod:`~twisted.internet.reactor` imports in project files and imported -third-party libraries will make Scrapy raise :exc:`Exception` when -it checks which reactor is installed. +third-party libraries will make Scrapy raise an exception when it checks which +reactor is installed. In order to use the reactor installed by Scrapy: @@ -2025,7 +2041,7 @@ In order to use the reactor installed by Scrapy: self.crawler.engine.close_spider(self, "timeout") -which raises :exc:`Exception`, becomes: +which raises an exception, becomes: .. code-block:: python diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b0d9a5452..721c81d81 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -22,6 +22,7 @@ from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response from scrapy.utils.defer import ( deferred_f_from_coro_f, + deferred_from_coro, maybe_deferred_to_future, ) from scrapy.utils.log import failure_to_exc_info, logformatter_adapter @@ -122,8 +123,10 @@ class ExecutionEngine: ) return scheduler_cls - @deferred_f_from_coro_f - async def start(self, _start_request_processing=True) -> None: + def start(self, _start_request_processing=True) -> Deferred[None]: + return deferred_from_coro(self.start_async(_start_request_processing)) + + async def start_async(self, _start_request_processing=True) -> None: if self.running: raise RuntimeError("Engine already running") self.start_time = time() @@ -392,10 +395,15 @@ class ExecutionEngine: finally: self._slot.nextcall.schedule() - @deferred_f_from_coro_f - async def open_spider( + def open_spider(self, spider: Spider, close_if_idle: bool = True) -> Deferred[None]: + return deferred_from_coro( + self.open_spider_async(spider, close_if_idle=close_if_idle) + ) + + async def open_spider_async( self, spider: Spider, + *, close_if_idle: bool = True, ) -> None: if self._slot is not None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1d6532fa9..c22b8603b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -21,7 +21,8 @@ from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager -from scrapy.utils.defer import deferred_to_future +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.defer import deferred_from_coro, deferred_to_future from scrapy.utils.log import ( LogCounterHandler, configure_logging, @@ -33,8 +34,10 @@ from scrapy.utils.log import ( from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.reactor import ( + _asyncio_reactor_path, install_reactor, is_asyncio_reactor_installed, + is_reactor_installed, verify_installed_asyncio_event_loop, verify_installed_reactor, ) @@ -142,6 +145,12 @@ class Crawler: # this method. @inlineCallbacks def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]: + """Start the crawler by instantiating its spider class with the given + *args* and *kwargs* arguments, while setting the execution engine in + motion. Should be called only once. + + Return a deferred that is fired when the crawl is finished. + """ if self.crawling: raise RuntimeError("Crawling already taking place") if self._started: @@ -163,6 +172,42 @@ class Crawler: yield self.engine.close() raise + async def crawl_async(self, *args: Any, **kwargs: Any) -> None: + """Start the crawler by instantiating its spider class with the given + *args* and *kwargs* arguments, while setting the execution engine in + motion. Should be called only once. + + .. versionadded:: VERSION + + Complete when the crawl is finished. + + This function requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be + installed. + """ + if not is_asyncio_available(): + raise RuntimeError("Crawler.crawl_async() requires AsyncioSelectorReactor.") + if self.crawling: + raise RuntimeError("Crawling already taking place") + if self._started: + raise RuntimeError( + "Cannot run Crawler.crawl_async() more than once on the same instance." + ) + self.crawling = self._started = True + + try: + self.spider = self._create_spider(*args, **kwargs) + self._apply_settings() + self._update_root_log_handler() + self.engine = self._create_engine() + await self.engine.open_spider_async(self.spider) + await self.engine.start_async() + except Exception: + self.crawling = False + if self.engine is not None: + await deferred_to_future(self.engine.close()) + raise + def _create_spider(self, *args: Any, **kwargs: Any) -> Spider: return self.spidercls.from_crawler(self, *args, **kwargs) @@ -171,13 +216,26 @@ class Crawler: @inlineCallbacks def stop(self) -> Generator[Deferred[Any], Any, None]: - """Starts a graceful stop of the crawler and returns a deferred that is + """Start a graceful stop of the crawler and return a deferred that is fired when the crawler is stopped.""" if self.crawling: self.crawling = False assert self.engine yield self.engine.stop() + async def stop_async(self) -> None: + """Start a graceful stop of the crawler and complete when the crawler is stopped. + + .. versionadded:: VERSION + + This function requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be + installed. + """ + if not is_asyncio_available(): + raise RuntimeError("Crawler.stop_async() requires AsyncioSelectorReactor.") + await deferred_to_future(self.stop()) + @staticmethod def _get_component( component_class: type[_T], components: Iterable[Any] @@ -318,9 +376,6 @@ class CrawlerRunnerBase: spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) - def _stop(self) -> Deferred[Any]: - return DeferredList([c.stop() for c in list(self.crawlers)]) - class CrawlerRunner(CrawlerRunnerBase): """ @@ -397,7 +452,7 @@ class CrawlerRunner(CrawlerRunnerBase): Returns a deferred that is fired when they all have ended. """ - return self._stop() + return DeferredList(c.stop() for c in self.crawlers) @inlineCallbacks def join(self) -> Generator[Deferred[Any], Any, None]: @@ -429,14 +484,14 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): def __init__(self, settings: dict[str, Any] | Settings | None = None): super().__init__(settings) - self._active: set[asyncio.Future[None]] = set() + self._active: set[asyncio.Task[None]] = set() def crawl( self, crawler_or_spidercls: type[Spider] | str | Crawler, *args: Any, **kwargs: Any, - ) -> asyncio.Future[None]: + ) -> asyncio.Task[None]: """ Run a crawler with the provided arguments. @@ -447,7 +502,7 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): instance, this method will try to create one using this parameter as the spider class given to it. - Returns a :class:`~asyncio.Future` object which completes when the + Returns a :class:`~asyncio.Task` object which completes when the crawling is finished. :param crawler_or_spidercls: already created crawler, or a spider class @@ -465,24 +520,27 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): "it must be a spider class (or a Crawler object)" ) if not is_asyncio_reactor_installed(): - raise RuntimeError("AsyncCrawlerRunner requires AsyncioSelectorReactor.") + raise RuntimeError( + f"{type(self).__name__} requires AsyncioSelectorReactor." + ) crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl( - self, crawler: Crawler, *args: Any, **kwargs: Any - ) -> asyncio.Future[None]: + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]: + # At this point the asyncio loop has been installed either by the user + # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()). + loop = asyncio.get_event_loop() self.crawlers.add(crawler) - future = deferred_to_future(crawler.crawl(*args, **kwargs)) - self._active.add(future) + task = loop.create_task(crawler.crawl_async(*args, **kwargs)) + self._active.add(task) - def _done(_: asyncio.Future[None]) -> None: + def _done(_: asyncio.Task[None]) -> None: self.crawlers.discard(crawler) - self._active.discard(future) + self._active.discard(task) self.bootstrap_failed |= not getattr(crawler, "spider", None) - future.add_done_callback(_done) - return future + task.add_done_callback(_done) + return task async def stop(self) -> None: """ @@ -490,7 +548,10 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): Completes when they all have ended. """ - await deferred_to_future(self._stop()) + if self.crawlers: + await asyncio.wait( + [asyncio.create_task(c.stop_async()) for c in self.crawlers] + ) async def join(self) -> None: """ @@ -498,33 +559,10 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): executions. """ while self._active: - await asyncio.gather(*self._active) + await asyncio.wait(self._active) -class CrawlerProcess(CrawlerRunner): - """ - A class to run multiple scrapy crawlers in a process simultaneously. - - This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support - for starting a :mod:`~twisted.internet.reactor` and handling shutdown - signals, like the keyboard interrupt command Ctrl-C. It also configures - top-level logging. - - This utility should be a better fit than - :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another - :mod:`~twisted.internet.reactor` within your application. - - The CrawlerProcess object must be instantiated with a - :class:`~scrapy.settings.Settings` object. - - :param install_root_handler: whether to install root logging handler - (default: True) - - This class shouldn't be needed (since Scrapy is responsible of using it - accordingly) unless writing scripts that manually handle the crawling - process. See :ref:`run-from-script` for an example. - """ - +class CrawlerProcessBase(CrawlerRunnerBase): def __init__( self, settings: dict[str, Any] | Settings | None = None, @@ -533,7 +571,6 @@ class CrawlerProcess(CrawlerRunner): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) - self._initialized_reactor: bool = False def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor @@ -556,6 +593,75 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._stop_reactor) + def _setup_reactor(self, install_signal_handlers: bool) -> None: + from twisted.internet import reactor + + resolver_class = load_object(self.settings["DNS_RESOLVER"]) + # We pass self, which is CrawlerProcess, instead of Crawler here, + # which works because the default resolvers only use crawler.settings. + resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type] + resolver.install_on_reactor() + tp = reactor.getThreadPool() + tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) + reactor.addSystemEventTrigger("before", "shutdown", self._stop_dfd) + if install_signal_handlers: + reactor.addSystemEventTrigger( + "after", "startup", install_shutdown_handlers, self._signal_shutdown + ) + + def _stop_dfd(self) -> Deferred[Any]: + raise NotImplementedError + + @inlineCallbacks + def _graceful_stop_reactor(self) -> Generator[Deferred[Any], Any, None]: + try: + yield self._stop_dfd() + finally: + self._stop_reactor() + + def _stop_reactor(self, _: Any = None) -> None: + from twisted.internet import reactor + + # raised if already stopped or in shutdown stage + with contextlib.suppress(RuntimeError): + reactor.stop() + + +class CrawlerProcess(CrawlerProcessBase, CrawlerRunner): + """ + A class to run multiple scrapy crawlers in a process simultaneously. + + This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support + for starting a :mod:`~twisted.internet.reactor` and handling shutdown + signals, like the keyboard interrupt command Ctrl-C. It also configures + top-level logging. + + This utility should be a better fit than + :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another + :mod:`~twisted.internet.reactor` within your application. + + The CrawlerProcess object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + :param install_root_handler: whether to install root logging handler + (default: True) + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides Deferred-based APIs. Use :class:`AsyncCrawlerProcess` + for modern coroutine APIs. + """ + + def __init__( + self, + settings: dict[str, Any] | Settings | None = None, + install_root_handler: bool = True, + ): + super().__init__(settings, install_root_handler) + self._initialized_reactor: bool = False + def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) @@ -563,6 +669,9 @@ class CrawlerProcess(CrawlerRunner): self._initialized_reactor = True return Crawler(spidercls, self.settings, init_reactor=init_reactor) + def _stop_dfd(self) -> Deferred[Any]: + return self.stop() + def start( self, stop_after_crawl: bool = True, install_signal_handlers: bool = True ) -> None: @@ -589,30 +698,86 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) - resolver_class = load_object(self.settings["DNS_RESOLVER"]) - # We pass self, which is CrawlerProcess, instead of Crawler here, - # which works because the default resolvers only use crawler.settings. - resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type] - resolver.install_on_reactor() - tp = reactor.getThreadPool() - tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) - reactor.addSystemEventTrigger("before", "shutdown", self.stop) - if install_signal_handlers: - reactor.addSystemEventTrigger( - "after", "startup", install_shutdown_handlers, self._signal_shutdown - ) + self._setup_reactor(install_signal_handlers) reactor.run(installSignalHandlers=install_signal_handlers) # blocking call - @inlineCallbacks - def _graceful_stop_reactor(self) -> Generator[Deferred[Any], Any, None]: - try: - yield self.stop() - finally: - self._stop_reactor() - def _stop_reactor(self, _: Any = None) -> None: +class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): + """ + A class to run multiple scrapy crawlers in a process simultaneously. + + This class extends :class:`~scrapy.crawler.AsyncCrawlerRunner` by adding support + for starting a :mod:`~twisted.internet.reactor` and handling shutdown + signals, like the keyboard interrupt command Ctrl-C. It also configures + top-level logging. + + This utility should be a better fit than + :class:`~scrapy.crawler.AsyncCrawlerRunner` if you aren't running another + :mod:`~twisted.internet.reactor` within your application. + + The AsyncCrawlerProcess object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + :param install_root_handler: whether to install root logging handler + (default: True) + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides coroutine APIs. It requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + """ + + def __init__( + self, + settings: dict[str, Any] | Settings | None = None, + install_root_handler: bool = True, + ): + super().__init__(settings, install_root_handler) + # We want the asyncio event loop to be installed early, so that it's + # always the correct one. And as we do that, we can also install the + # reactor here. + # The ASYNCIO_EVENT_LOOP setting cannot be overridden by add-ons and + # spiders when using AsyncCrawlerProcess. + loop_path = self.settings["ASYNCIO_EVENT_LOOP"] + if is_reactor_installed(): + # The user could install a reactor before this class is instantiated. + # We need to make sure the reactor is the correct one and the loop + # type matches the setting. + verify_installed_reactor(_asyncio_reactor_path) + if loop_path: + verify_installed_asyncio_event_loop(loop_path) + else: + install_reactor(_asyncio_reactor_path, loop_path) + self._initialized_reactor = True + + def _stop_dfd(self) -> Deferred[Any]: + return deferred_from_coro(self.stop()) + + def start( + self, stop_after_crawl: bool = True, install_signal_handlers: bool = True + ) -> None: + """ + This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool + size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache + based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. + + If ``stop_after_crawl`` is True, the reactor will be stopped after all + crawlers have finished, using :meth:`join`. + + :param bool stop_after_crawl: stop or not the reactor when all + crawlers have finished + + :param bool install_signal_handlers: whether to install the OS signal + handlers from Twisted and Scrapy (default: True) + """ from twisted.internet import reactor - # raised if already stopped or in shutdown stage - with contextlib.suppress(RuntimeError): - reactor.stop() + if stop_after_crawl: + loop = asyncio.get_event_loop() + join_task = loop.create_task(self.join()) + join_task.add_done_callback(self._stop_reactor) + + self._setup_reactor(install_signal_handlers) + reactor.run(installSignalHandlers=install_signal_handlers) # blocking call diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 4649c4daa..2df82c8f2 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -23,7 +23,6 @@ from twisted.python import failure from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.asyncio import is_asyncio_available -from scrapy.utils.reactor import _get_asyncio_event_loop if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable @@ -385,8 +384,7 @@ def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2: # that use asyncio, e.g. "await asyncio.sleep(1)" return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o)) # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - event_loop = _get_asyncio_event_loop() - return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) + return Deferred.fromFuture(asyncio.ensure_future(o)) return o @@ -430,6 +428,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]: Return an :class:`asyncio.Future` object that wraps *d*. + This function requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be + installed. + When :ref:`using the asyncio reactor `, you cannot await on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy callables defined as coroutines `, you can only await on @@ -442,8 +444,15 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]: additional_request = scrapy.Request('https://example.org/price') deferred = self.crawler.engine.download(additional_request) additional_response = await deferred_to_future(deferred) + + .. versionchanged:: VERSION + This function no longer installs an asyncio loop if called before the + Twisted asyncio reactor is installed. A :exc:`RuntimeError` is raised + in this case. """ - return d.asFuture(_get_asyncio_event_loop()) + if not is_asyncio_available(): + raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.") + return d.asFuture(asyncio.get_event_loop()) def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 5e76da37b..2fb1e0ce7 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -10,6 +10,7 @@ from twisted.internet import asyncioreactor, error from twisted.internet.defer import Deferred from scrapy.utils.misc import load_object +from scrapy.utils.python import global_object_name if TYPE_CHECKING: from asyncio import AbstractEventLoop, AbstractEventLoopPolicy @@ -87,6 +88,9 @@ class CallLaterOnce(Generic[_T]): await maybe_deferred_to_future(d) +_asyncio_reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + + def set_asyncio_event_loop_policy() -> None: """The policy functions from asyncio often behave unexpectedly, so we restrict their use to the absolutely essential case. @@ -161,21 +165,34 @@ def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop: def verify_installed_reactor(reactor_path: str) -> None: - """Raises :exc:`Exception` if the installed + """Raise :exc:`RuntimeError` if the installed :mod:`~twisted.internet.reactor` does not match the specified import - path.""" + path or if no reactor is installed.""" + if not is_reactor_installed(): + raise RuntimeError( + "verify_installed_reactor() called without an installed reactor." + ) + from twisted.internet import reactor - reactor_class = load_object(reactor_path) - if not reactor.__class__ == reactor_class: + expected_reactor_type = load_object(reactor_path) + reactor_type = type(reactor) + if not reactor_type == expected_reactor_type: raise RuntimeError( - "The installed reactor " - f"({reactor.__module__}.{reactor.__class__.__name__}) does not " - f"match the requested one ({reactor_path})" + f"The installed reactor ({global_object_name(reactor_type)}) " + f"does not match the requested one ({reactor_path})" ) def verify_installed_asyncio_event_loop(loop_path: str) -> None: + """Raise :exc:`RuntimeError` if the even loop of the installed + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` + does not match the specified import path or if no reactor is installed.""" + if not is_reactor_installed(): + raise RuntimeError( + "verify_installed_asyncio_event_loop() called without an installed reactor." + ) + from twisted.internet import reactor loop_class = load_object(loop_path) @@ -185,16 +202,16 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None: f"{reactor._asyncioEventloop.__class__.__module__}" f".{reactor._asyncioEventloop.__class__.__qualname__}" ) - specified = f"{loop_class.__module__}.{loop_class.__qualname__}" raise RuntimeError( "Scrapy found an asyncio Twisted reactor already " f"installed, and its event loop class ({installed}) does " "not match the one specified in the ASYNCIO_EVENT_LOOP " - f"setting ({specified})" + f"setting ({global_object_name(loop_class)})" ) def is_reactor_installed() -> bool: + """Check whether a :mod:`~twisted.internet.reactor` is installed.""" return "twisted.internet.reactor" in sys.modules diff --git a/tests/AsyncCrawlerProcess/args_settings.py b/tests/AsyncCrawlerProcess/args_settings.py new file mode 100644 index 000000000..5e162e782 --- /dev/null +++ b/tests/AsyncCrawlerProcess/args_settings.py @@ -0,0 +1,25 @@ +from typing import Any + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess, Crawler + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("FOO", kwargs.get("foo")) + return spider + + async def start(self): + self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider, foo=42) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop.py new file mode 100644 index 000000000..172e36b7b --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop.py @@ -0,0 +1,20 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py new file mode 100644 index 000000000..d76da51a1 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py @@ -0,0 +1,23 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + custom_settings = { + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": None, + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py new file mode 100644 index 000000000..bd4a99e14 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py @@ -0,0 +1,23 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + custom_settings = { + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py new file mode 100644 index 000000000..c32aaf37d --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import asyncio +import sys + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.defer import deferred_from_coro + + +class UppercasePipeline: + async def _open_spider(self, spider): + spider.logger.info("async pipeline opened!") + await asyncio.sleep(0.1) + + def open_spider(self, spider): + return deferred_from_coro(self._open_spider(spider)) + + def process_item(self, item, spider): + return {"url": item["url"].upper()} + + +class UrlSpider(Spider): + name = "url_spider" + start_urls = ["data:,"] + custom_settings = { + "ITEM_PIPELINES": {UppercasePipeline: 100}, + } + + def parse(self, response): + yield {"url": response.url} + + +if __name__ == "__main__": + ASYNCIO_EVENT_LOOP: str | None + try: + ASYNCIO_EVENT_LOOP = sys.argv[1] + except IndexError: + ASYNCIO_EVENT_LOOP = None + + process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP, + } + ) + process.crawl(UrlSpider) + process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py new file mode 100644 index 000000000..3c47eb826 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py @@ -0,0 +1,27 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactor import is_asyncio_reactor_installed + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py new file mode 100644 index 000000000..e025e17d1 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py @@ -0,0 +1,53 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + is_reactor_installed, +) + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed before is_asyncio_reactor_installed()." + ) + +try: + is_asyncio_reactor_installed() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.") + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed after is_asyncio_reactor_installed()." + ) + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +if not is_asyncio_reactor_installed(): + raise RuntimeError("Wrong reactor installed after install_reactor().") + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py new file mode 100644 index 000000000..4257bc0ac --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -0,0 +1,29 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncioreactor.install(asyncio.get_event_loop()) + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py new file mode 100644 index 000000000..9c6fd090b --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -0,0 +1,31 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor +from uvloop import Loop + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncio.set_event_loop(Loop()) +asyncioreactor.install(asyncio.get_event_loop()) + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py new file mode 100644 index 000000000..5f75d5e17 --- /dev/null +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py @@ -0,0 +1,35 @@ +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution) + """ + + name = "caching_hostname_resolver_spider" + + async def start(self): + yield scrapy.Request(self.url) + + def parse(self, response): + for _ in range(10): + yield scrapy.Request( + response.url, dont_filter=True, callback=self.ignore_response + ) + + def ignore_response(self, response): + self.logger.info(repr(response.ip_address)) + + +if __name__ == "__main__": + process = AsyncCrawlerProcess( + settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + } + ) + process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) + process.start() diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py new file mode 100644 index 000000000..c43f0a9c2 --- /dev/null +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes without a twisted.internet.error.DNSLookupError exception + """ + + name = "caching_hostname_resolver_spider" + start_urls = ["http://[::1]"] + + +if __name__ == "__main__": + process = AsyncCrawlerProcess( + settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + } + ) + process.crawl(CachingHostnameResolverSpider) + process.start() diff --git a/tests/AsyncCrawlerProcess/default_name_resolver.py b/tests/AsyncCrawlerProcess/default_name_resolver.py new file mode 100644 index 000000000..af56ccd01 --- /dev/null +++ b/tests/AsyncCrawlerProcess/default_name_resolver.py @@ -0,0 +1,18 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class IPv6Spider(scrapy.Spider): + """ + Raises a twisted.internet.error.DNSLookupError: + the default name resolver does not handle IPv6 addresses. + """ + + name = "ipv6_spider" + start_urls = ["http://[::1]"] + + +if __name__ == "__main__": + process = AsyncCrawlerProcess(settings={"RETRY_ENABLED": False}) + process.crawl(IPv6Spider) + process.start() diff --git a/tests/AsyncCrawlerProcess/multi.py b/tests/AsyncCrawlerProcess/multi.py new file mode 100644 index 000000000..2eede5471 --- /dev/null +++ b/tests/AsyncCrawlerProcess/multi.py @@ -0,0 +1,17 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactor_default.py b/tests/AsyncCrawlerProcess/reactor_default.py new file mode 100644 index 000000000..9638652bd --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactor_default.py @@ -0,0 +1,18 @@ +from twisted.internet import reactor # noqa: F401,TID253 + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +d = process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/simple.py b/tests/AsyncCrawlerProcess/simple.py new file mode 100644 index 000000000..d24b4f193 --- /dev/null +++ b/tests/AsyncCrawlerProcess/simple.py @@ -0,0 +1,16 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/sleeping.py b/tests/AsyncCrawlerProcess/sleeping.py new file mode 100644 index 000000000..88caf5032 --- /dev/null +++ b/tests/AsyncCrawlerProcess/sleeping.py @@ -0,0 +1,20 @@ +import asyncio +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class SleepingSpider(scrapy.Spider): + name = "sleeping" + + start_urls = ["data:,;"] + + async def parse(self, response): + await asyncio.sleep(int(sys.argv[1])) + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(SleepingSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py b/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py new file mode 100644 index 000000000..dc820ea3a --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py @@ -0,0 +1,15 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = "asyncio_reactor" + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } +) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py new file mode 100644 index 000000000..5fd48274a --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py @@ -0,0 +1,14 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = "asyncio_reactor" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = AsyncCrawlerProcess() +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py new file mode 100644 index 000000000..c205c3cd2 --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class AsyncioReactorSpider1(scrapy.Spider): + name = "asyncio_reactor1" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +class AsyncioReactorSpider2(scrapy.Spider): + name = "asyncio_reactor2" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = AsyncCrawlerProcess() +process.crawl(AsyncioReactorSpider1) +process.crawl(AsyncioReactorSpider2) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py new file mode 100644 index 000000000..68239e651 --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if TYPE_CHECKING: + from asyncio import Task + + +class AsyncioReactorSpider(scrapy.Spider): + name = "asyncio_reactor" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", + } + + +def log_task_exception(task: Task) -> None: + try: + task.result() + except Exception: + logging.exception("Crawl task failed") + + +process = AsyncCrawlerProcess() +task = process.crawl(AsyncioReactorSpider) +task.add_done_callback(log_task_exception) +process.start() diff --git a/tests/AsyncCrawlerRunner/custom_loop_different.py b/tests/AsyncCrawlerRunner/custom_loop_different.py new file mode 100644 index 000000000..89cf0e536 --- /dev/null +++ b/tests/AsyncCrawlerRunner/custom_loop_different.py @@ -0,0 +1,31 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/custom_loop_same.py b/tests/AsyncCrawlerRunner/custom_loop_same.py new file mode 100644 index 000000000..43d0dc053 --- /dev/null +++ b/tests/AsyncCrawlerRunner/custom_loop_same.py @@ -0,0 +1,31 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor", "uvloop.Loop") +react(main) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index d8c467f40..7c50277b9 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -4,12 +4,12 @@ import sys from twisted.internet import asyncioreactor from twisted.python import log +import scrapy +from scrapy.crawler import CrawlerProcess + if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncioreactor.install(asyncio.get_event_loop()) - -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +asyncioreactor.install() class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index e7d3ca9cc..578e0029d 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -4,13 +4,13 @@ import sys from twisted.internet import asyncioreactor from uvloop import Loop +import scrapy +from scrapy.crawler import CrawlerProcess + if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.set_event_loop(Loop()) -asyncioreactor.install(asyncio.get_event_loop()) - -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +asyncioreactor.install() class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerRunner/custom_loop_different.py b/tests/CrawlerRunner/custom_loop_different.py new file mode 100644 index 000000000..86ba1ed47 --- /dev/null +++ b/tests/CrawlerRunner/custom_loop_different.py @@ -0,0 +1,29 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/CrawlerRunner/custom_loop_same.py b/tests/CrawlerRunner/custom_loop_same.py new file mode 100644 index 000000000..98b8dde87 --- /dev/null +++ b/tests/CrawlerRunner/custom_loop_same.py @@ -0,0 +1,29 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor", "uvloop.Loop") +react(main) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index a1d3c02fb..ce5963cd7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,3 +1,4 @@ +import asyncio import logging import platform import re @@ -5,6 +6,7 @@ import signal import subprocess import sys import warnings +from abc import ABC, abstractmethod from pathlib import Path from typing import Any @@ -18,12 +20,18 @@ from zope.interface.exceptions import MultipleInvalid import scrapy from scrapy import Spider -from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerProcess, CrawlerRunner +from scrapy.crawler import ( + AsyncCrawlerProcess, + AsyncCrawlerRunner, + Crawler, + CrawlerProcess, + CrawlerRunner, +) from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader -from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_coro from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings @@ -88,12 +96,39 @@ class TestCrawler(TestBaseCrawler): Crawler(DefaultSpider()) @inlineCallbacks - def test_crawler_crawl_twice_unsupported(self): + def test_crawler_crawl_twice_seq_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) yield crawler.crawl() with pytest.raises(RuntimeError, match="more than once on the same instance"): yield crawler.crawl() + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_crawler_crawl_async_twice_seq_unsupported(self): + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + await crawler.crawl_async() + with pytest.raises(RuntimeError, match="more than once on the same instance"): + await crawler.crawl_async() + + @inlineCallbacks + def test_crawler_crawl_twice_parallel_unsupported(self): + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + d1 = crawler.crawl() + d2 = crawler.crawl() + yield d1 + with pytest.raises(RuntimeError, match="Crawling already taking place"): + yield d2 + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_crawler_crawl_async_twice_parallel_unsupported(self): + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + t1 = asyncio.create_task(crawler.crawl_async()) + t2 = asyncio.create_task(crawler.crawl_async()) + await t1 + with pytest.raises(RuntimeError, match="Crawling already taking place"): + await t2 + def test_get_addon(self): class ParentAddon: pass @@ -590,6 +625,18 @@ class TestCrawlerProcess(TestBaseCrawler): self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") +@pytest.mark.only_asyncio +class TestAsyncCrawlerProcess(TestBaseCrawler): + def test_crawler_process_accepts_dict(self): + runner = AsyncCrawlerProcess({"foo": "bar"}) + assert runner.settings["foo"] == "bar" + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + def test_crawler_process_accepts_None(self): + runner = AsyncCrawlerProcess() + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + class ExceptionSpider(scrapy.Spider): name = "exception" @@ -692,8 +739,15 @@ class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): pytest.skip("This test is only for CrawlerRunner") -class ScriptRunnerMixin: - script_dir: Path +class ScriptRunnerMixin(ABC): + @property + @abstractmethod + def script_dir(self) -> Path: + raise NotImplementedError + + @staticmethod + def get_script_dir(name: str) -> Path: + return Path(__file__).parent.resolve() / name def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name @@ -711,8 +765,10 @@ class ScriptRunnerMixin: return stderr.decode("utf-8") -class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): - script_dir = Path(__file__).parent.resolve() / "CrawlerProcess" +class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin, unittest.TestCase): + """Common tests between CrawlerProcess and AsyncCrawlerProcess, + with the same file names and expectations. + """ def test_simple(self): log = self.run_script("simple.py") @@ -739,48 +795,6 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" ) in log - def test_reactor_default_twisted_reactor_select(self): - log = self.run_script("reactor_default_twisted_reactor_select.py") - if platform.system() in ["Windows", "Darwin"]: - # The goal of this test function is to test that, when a reactor is - # installed (the default one here) and a different reactor is - # configured (select here), an error raises. - # - # In Windows the default reactor is the select reactor, so that - # error does not raise. - # - # If that ever becomes the case on more platforms (i.e. if Linux - # also starts using the select reactor by default in a future - # version of Twisted), then we will need to rethink this test. - assert "Spider closed (finished)" in log - else: - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ) in log - - def test_reactor_select(self): - log = self.run_script("reactor_select.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" - ) in log - - def test_reactor_select_twisted_reactor_select(self): - log = self.run_script("reactor_select_twisted_reactor_select.py") - assert "Spider closed (finished)" in log - assert "ReactorAlreadyInstalledError" not in log - - def test_reactor_select_subclass_twisted_reactor_select(self): - log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ) in log - def test_asyncio_enabled_no_reactor(self): log = self.run_script("asyncio_enabled_no_reactor.py") assert "Spider closed (finished)" in log @@ -829,19 +843,6 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): assert "TimeoutError" not in log assert "twisted.internet.error.DNSLookupError" not in log - def test_twisted_reactor_select(self): - log = self.run_script("twisted_reactor_select.py") - assert "Spider closed (finished)" in log - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - - @pytest.mark.skipif( - platform.system() == "Windows", reason="PollReactor is not supported on Windows" - ) - def test_twisted_reactor_poll(self): - log = self.run_script("twisted_reactor_poll.py") - assert "Spider closed (finished)" in log - assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log - def test_twisted_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") assert "Spider closed (finished)" in log @@ -866,14 +867,6 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): in log ) - def test_twisted_reactor_asyncio_custom_settings_conflict(self): - log = self.run_script("twisted_reactor_custom_settings_conflict.py") - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - assert ( - "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" - in log - ) - @pytest.mark.requires_uvloop def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") @@ -960,8 +953,113 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.wait() -class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): - script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" +class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("CrawlerProcess") + + def test_reactor_default_twisted_reactor_select(self): + log = self.run_script("reactor_default_twisted_reactor_select.py") + if platform.system() in ["Windows", "Darwin"]: + # The goal of this test function is to test that, when a reactor is + # installed (the default one here) and a different reactor is + # configured (select here), an error raises. + # + # In Windows the default reactor is the select reactor, so that + # error does not raise. + # + # If that ever becomes the case on more platforms (i.e. if Linux + # also starts using the select reactor by default in a future + # version of Twisted), then we will need to rethink this test. + assert "Spider closed (finished)" in log + else: + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + def test_reactor_select(self): + log = self.run_script("reactor_select.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log + + def test_reactor_select_twisted_reactor_select(self): + log = self.run_script("reactor_select_twisted_reactor_select.py") + assert "Spider closed (finished)" in log + assert "ReactorAlreadyInstalledError" not in log + + def test_reactor_select_subclass_twisted_reactor_select(self): + log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + def test_twisted_reactor_select(self): + log = self.run_script("twisted_reactor_select.py") + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + + @pytest.mark.skipif( + platform.system() == "Windows", reason="PollReactor is not supported on Windows" + ) + def test_twisted_reactor_poll(self): + log = self.run_script("twisted_reactor_poll.py") + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log + + def test_twisted_reactor_asyncio_custom_settings_conflict(self): + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" + in log + ) + + +class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("AsyncCrawlerProcess") + + def test_twisted_reactor_custom_settings_select(self): + log = self.run_script("twisted_reactor_custom_settings_select.py") + assert "Spider closed (finished)" not in log + assert ( + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor) " + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_same_loop(self): + log = self.run_script("asyncio_custom_loop_custom_settings_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_different_loop(self): + log = self.run_script("asyncio_custom_loop_custom_settings_different.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + +class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): + """Common tests between CrawlerRunner and AsyncCrawlerRunner, + with the same file names and expectations. + """ def test_simple(self): log = self.run_script("simple.py") @@ -971,14 +1069,6 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): in log ) - def test_explicit_default_reactor(self): - log = self.run_script("explicit_default_reactor.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log - ) - def test_multi_parallel(self): log = self.run_script("multi_parallel.py") assert "Spider closed (finished)" in log @@ -1005,6 +1095,39 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): re.DOTALL, ) + @pytest.mark.requires_uvloop + def test_custom_loop_same(self): + log = self.run_script("custom_loop_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_custom_loop_different(self): + log = self.run_script("custom_loop_different.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + +class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("CrawlerRunner") + + def test_explicit_default_reactor(self): + log = self.run_script("explicit_default_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + def test_response_ip_address(self): log = self.run_script("ip_address.py") assert "INFO: Spider closed (finished)" in log @@ -1021,48 +1144,16 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): assert "DEBUG: Using asyncio event loop" in log -class TestAsyncCrawlerRunnerSubprocess(ScriptRunnerMixin): - script_dir = Path(__file__).parent.resolve() / "AsyncCrawlerRunner" - - def test_simple(self): - log = self.run_script("simple.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) +class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("AsyncCrawlerRunner") def test_simple_default_reactor(self): log = self.run_script("simple_default_reactor.py") assert "Spider closed (finished)" not in log assert "RuntimeError: AsyncCrawlerRunner requires AsyncioSelectorReactor" in log - def test_multi_parallel(self): - log = self.run_script("multi_parallel.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert re.search( - r"Spider opened.+Spider opened.+Closing spider.+Closing spider", - log, - re.DOTALL, - ) - - def test_multi_seq(self): - log = self.run_script("multi_seq.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert re.search( - r"Spider opened.+Closing spider.+Spider opened.+Closing spider", - log, - re.DOTALL, - ) - @pytest.mark.parametrize( ("settings", "items"), diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py index 99f175c60..eb00ab193 100644 --- a/tests/test_utils_reactor.py +++ b/tests/test_utils_reactor.py @@ -6,6 +6,7 @@ from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( + _asyncio_reactor_path, install_reactor, is_asyncio_reactor_installed, set_asyncio_event_loop, @@ -22,7 +23,7 @@ class TestAsyncio(TestCase): from twisted.internet import reactor as original_reactor with warnings.catch_warnings(record=True) as w: - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + install_reactor(_asyncio_reactor_path) assert len(w) == 0, [str(warning) for warning in w] from twisted.internet import reactor # pylint: disable=reimported @@ -31,5 +32,5 @@ class TestAsyncio(TestCase): @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_set_asyncio_event_loop(self): - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + install_reactor(_asyncio_reactor_path) assert set_asyncio_event_loop(None) is asyncio.get_running_loop() From e0b9f2d8f6f0feec9626e314166d5ae320d83be1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 May 2025 19:57:33 +0500 Subject: [PATCH 1747/2083] Don't use CrawlerProcess in the commands that don't need it. (#6824) * Don't use CrawlerProcess in the commands that don't need it. * Use a dummy spider loader in runspider. --- docs/topics/api.rst | 2 + docs/topics/settings.rst | 9 -- scrapy/cmdline.py | 3 +- scrapy/commands/__init__.py | 9 +- scrapy/commands/check.py | 1 + scrapy/commands/edit.py | 7 +- scrapy/commands/genspider.py | 14 ++- scrapy/commands/list.py | 7 +- scrapy/commands/runspider.py | 3 +- scrapy/commands/settings.py | 7 +- scrapy/commands/startproject.py | 4 +- scrapy/commands/version.py | 3 +- scrapy/crawler.py | 20 +--- scrapy/spiderloader.py | 31 +++++- tests/test_crawler.py | 5 - tests/test_spiderloader/__init__.py | 145 ++++++++++++++-------------- 16 files changed, 150 insertions(+), 120 deletions(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index b11de2914..d90eb0bad 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -213,6 +213,8 @@ SpiderLoader API :param request: queried request :type request: :class:`~scrapy.Request` instance +.. autoclass:: DummySpiderLoader + .. _topics-api-signals: Signals API diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 68c5079cf..65f2e5ebd 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1868,15 +1868,6 @@ it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception But you can choose to silence this exception and turn it into a simple warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. -.. note:: - Some :ref:`scrapy commands ` run with this setting to ``True`` - already (i.e. they will only issue a warning and will not fail) - since they do not actually need to load spider classes to work: - :command:`scrapy runspider `, - :command:`scrapy settings `, - :command:`scrapy startproject `, - :command:`scrapy version `. - .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index b08fd3409..81e507a4e 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -201,7 +201,8 @@ def execute(argv: list[str] | None = None, settings: Settings | None = None) -> opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) - cmd.crawler_process = CrawlerProcess(settings) + if cmd.requires_crawler_process: + cmd.crawler_process = CrawlerProcess(settings) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 56199cc01..2818ead77 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -19,11 +19,13 @@ if TYPE_CHECKING: from collections.abc import Iterable from scrapy.crawler import Crawler, CrawlerProcess + from scrapy.settings import Settings class ScrapyCommand: requires_project: bool = False - crawler_process: CrawlerProcess | None = None + requires_crawler_process: bool = True + crawler_process: CrawlerProcess | None = None # set in scrapy.cmdline # default settings to be used for this command instead of global defaults default_settings: dict[str, Any] = {} @@ -31,7 +33,7 @@ class ScrapyCommand: exitcode: int = 0 def __init__(self) -> None: - self.settings: Any = None # set in scrapy.cmdline + self.settings: Settings | None = None # set in scrapy.cmdline def set_crawler(self, crawler: Crawler) -> None: if hasattr(self, "_crawler"): @@ -68,6 +70,7 @@ class ScrapyCommand: """ Populate option parse with options available for this command """ + assert self.settings is not None group = parser.add_argument_group(title="Global Options") group.add_argument( "--logfile", metavar="FILE", help="log file. if omitted stderr will be used" @@ -100,6 +103,7 @@ class ScrapyCommand: group.add_argument("--pdb", action="store_true", help="enable pdb on failure") def process_options(self, args: list[str], opts: argparse.Namespace) -> None: + assert self.settings is not None try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: @@ -170,6 +174,7 @@ class BaseRunSpiderCommand(ScrapyCommand): except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output or opts.overwrite_output: + assert self.settings is not None feeds = feed_process_params_from_cli( self.settings, opts.output, diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 56dc1ea55..e9ada0fb6 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -69,6 +69,7 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: # load contracts + assert self.settings is not None contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) conman = ContractsManager(load_object(c) for c in contracts) runner = TextTestRunner(verbosity=2 if opts.verbose else 1) diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index d153a5271..f2d52673a 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -4,10 +4,12 @@ import sys from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError +from scrapy.spiderloader import get_spider_loader class Command(ScrapyCommand): requires_project = True + requires_crawler_process = False default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: @@ -30,10 +32,11 @@ class Command(ScrapyCommand): if len(args) != 1: raise UsageError + assert self.settings is not None editor = self.settings["EDITOR"] - assert self.crawler_process + spider_loader = get_spider_loader(self.settings) try: - spidercls = self.crawler_process.spider_loader.load(args[0]) + spidercls = spider_loader.load(args[0]) except KeyError: self._err(f"Spider not found: {args[0]}") return diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 6d4aec3d8..c4abfc4c9 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -11,6 +11,7 @@ from urllib.parse import urlparse import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError +from scrapy.spiderloader import get_spider_loader from scrapy.utils.template import render_templatefile, string_camelcase if TYPE_CHECKING: @@ -46,6 +47,7 @@ def verify_url_scheme(url: str) -> str: class Command(ScrapyCommand): requires_project = False + requires_crawler_process = False default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: @@ -92,6 +94,7 @@ class Command(ScrapyCommand): ) def run(self, args: list[str], opts: argparse.Namespace) -> None: + assert self.settings is not None if opts.list: self._list_templates() return @@ -127,6 +130,7 @@ class Command(ScrapyCommand): url: str, template_name: str, ) -> dict[str, Any]: + assert self.settings is not None capitalized_module = "".join(s.capitalize() for s in module.split("_")) return { "project_name": self.settings.get("BOT_NAME"), @@ -147,6 +151,7 @@ class Command(ScrapyCommand): template_file: str | os.PathLike, ) -> None: """Generate the spider module, based on the given template""" + assert self.settings is not None tvars = self._generate_template_variables(module, name, url, template_name) if self.settings.get("NEWSPIDER_MODULE"): spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) @@ -180,6 +185,7 @@ class Command(ScrapyCommand): print(f" {file.stem}") def _spider_exists(self, name: str) -> bool: + assert self.settings is not None if not self.settings.get("NEWSPIDER_MODULE"): # if run as a standalone command and file with same filename already exists path = Path(name + ".py") @@ -188,12 +194,9 @@ class Command(ScrapyCommand): return True return False - assert self.crawler_process is not None, ( - "crawler_process must be set before calling run" - ) - + spider_loader = get_spider_loader(self.settings) try: - spidercls = self.crawler_process.spider_loader.load(name) + spidercls = spider_loader.load(name) except KeyError: pass else: @@ -215,6 +218,7 @@ class Command(ScrapyCommand): @property def templates_dir(self) -> str: + assert self.settings is not None return str( Path( self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"), diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 3b2f127c2..b4dc97f3d 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -3,6 +3,7 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy.commands import ScrapyCommand +from scrapy.spiderloader import get_spider_loader if TYPE_CHECKING: import argparse @@ -10,12 +11,14 @@ if TYPE_CHECKING: class Command(ScrapyCommand): requires_project = True + requires_crawler_process = False default_settings = {"LOG_ENABLED": False} def short_desc(self) -> str: return "List available spiders" def run(self, args: list[str], opts: argparse.Namespace) -> None: - assert self.crawler_process - for s in sorted(self.crawler_process.spider_loader.list()): + assert self.settings is not None + spider_loader = get_spider_loader(self.settings) + for s in sorted(spider_loader.list()): print(s) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 357ca8b37..3e826456e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError +from scrapy.spiderloader import DummySpiderLoader from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: @@ -30,7 +31,7 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType: class Command(BaseRunSpiderCommand): requires_project = False - default_settings = {"SPIDER_LOADER_WARN_ONLY": True} + default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader} def syntax(self) -> str: return "[options] " diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 59f86b9a7..e63031f2d 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): requires_project = False - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} + requires_crawler_process = False + default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: return "[options]" @@ -46,8 +47,8 @@ class Command(ScrapyCommand): ) def run(self, args: list[str], opts: argparse.Namespace) -> None: - assert self.crawler_process - settings = self.crawler_process.settings + assert self.settings is not None + settings = self.settings if opts.get: s = settings.get(opts.get) if isinstance(s, BaseSettings): diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 1adc1530f..323979193 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -34,7 +34,8 @@ def _make_writable(path: Path) -> None: class Command(ScrapyCommand): requires_project = False - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} + requires_crawler_process = False + default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: return " [project_dir]" @@ -132,6 +133,7 @@ class Command(ScrapyCommand): @property def templates_dir(self) -> str: + assert self.settings is not None return str( Path( self.settings["TEMPLATES_DIR"] or Path(scrapy.__path__[0], "templates"), diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 713a78ad9..30b0e9fd7 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -6,7 +6,8 @@ from scrapy.utils.versions import get_versions class Command(ScrapyCommand): - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} + requires_crawler_process = False + default_settings = {"LOG_ENABLED": False} def syntax(self) -> str: return "[-v]" diff --git a/scrapy/crawler.py b/scrapy/crawler.py index c22b8603b..8e3223a5c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -5,22 +5,21 @@ import contextlib import logging import pprint import signal -from typing import TYPE_CHECKING, Any, TypeVar, cast +from typing import TYPE_CHECKING, Any, TypeVar from twisted.internet.defer import ( Deferred, DeferredList, inlineCallbacks, ) -from zope.interface.verify import verifyClass from scrapy import Spider, signals from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.extension import ExtensionManager -from scrapy.interfaces import ISpiderLoader -from scrapy.settings import BaseSettings, Settings, overridden_settings +from scrapy.settings import Settings, overridden_settings from scrapy.signalmanager import SignalManager +from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader from scrapy.utils.asyncio import is_asyncio_available from scrapy.utils.defer import deferred_from_coro, deferred_to_future from scrapy.utils.log import ( @@ -46,7 +45,6 @@ if TYPE_CHECKING: from collections.abc import Generator, Iterable from scrapy.logformatter import LogFormatter - from scrapy.spiderloader import SpiderLoaderProtocol from scrapy.statscollectors import StatsCollector from scrapy.utils.request import RequestFingerprinterProtocol @@ -324,22 +322,12 @@ class Crawler: class CrawlerRunnerBase: - @staticmethod - def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: - """Get SpiderLoader instance from settings""" - cls_path = settings.get("SPIDER_LOADER_CLASS") - loader_cls = load_object(cls_path) - verifyClass(ISpiderLoader, loader_cls) - return cast( - "SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()) - ) - def __init__(self, settings: dict[str, Any] | Settings | None = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) AddonManager.load_pre_crawler_settings(settings) self.settings: Settings = settings - self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings) + self.spider_loader: SpiderLoaderProtocol = get_spider_loader(settings) self._crawlers: set[Crawler] = set() self.bootstrap_failed = False diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index f537e0593..8eac188c8 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -3,12 +3,13 @@ from __future__ import annotations import traceback import warnings from collections import defaultdict -from typing import TYPE_CHECKING, Protocol +from typing import TYPE_CHECKING, Protocol, cast from zope.interface import implementer +from zope.interface.verify import verifyClass from scrapy.interfaces import ISpiderLoader -from scrapy.utils.misc import walk_modules +from scrapy.utils.misc import load_object, walk_modules from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: @@ -21,6 +22,14 @@ if TYPE_CHECKING: from scrapy.settings import BaseSettings +def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: + """Get SpiderLoader instance from settings""" + cls_path = settings.get("SPIDER_LOADER_CLASS") + loader_cls = load_object(cls_path) + verifyClass(ISpiderLoader, loader_cls) + return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy())) + + class SpiderLoaderProtocol(Protocol): @classmethod def from_settings(cls, settings: BaseSettings) -> Self: @@ -120,3 +129,21 @@ class SpiderLoader: Return a list with the names of all spiders available in the project. """ return list(self._spiders.keys()) + + +@implementer(ISpiderLoader) +class DummySpiderLoader: + """A dummy spider loader that does not load any spiders.""" + + @classmethod + def from_settings(cls, settings: BaseSettings) -> Self: + return cls() + + def load(self, spider_name: str) -> type[Spider]: + raise KeyError("DummySpiderLoader doesn't load any spiders") + + def list(self) -> list[str]: + return [] + + def find_by_request(self, request: Request) -> __builtins__.list[str]: + return [] diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ce5963cd7..56cb21650 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -30,7 +30,6 @@ from scrapy.crawler import ( from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings -from scrapy.spiderloader import SpiderLoader from scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_coro from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider @@ -570,10 +569,6 @@ class SpiderLoaderWithWrongInterface: pass -class CustomSpiderLoader(SpiderLoader): - pass - - class TestCrawlerRunner(TestBaseCrawler): def test_spider_manager_verify_interface(self): settings = Settings( diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 476487a04..245507c0b 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,10 +1,8 @@ import contextlib import shutil import sys -import tempfile import warnings from pathlib import Path -from tempfile import mkdtemp from unittest import mock import pytest @@ -17,7 +15,7 @@ from scrapy.crawler import CrawlerRunner from scrapy.http import Request from scrapy.interfaces import ISpiderLoader from scrapy.settings import Settings -from scrapy.spiderloader import SpiderLoader +from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader module_dir = Path(__file__).resolve().parent @@ -27,73 +25,76 @@ def _copytree(source: Path, target: Path): shutil.copytree(source, target) +@pytest.fixture +def spider_loader_env(tmp_path): + orig_spiders_dir = module_dir / "test_spiders" + spiders_dir = tmp_path / "test_spiders_xxx" + _copytree(orig_spiders_dir, spiders_dir) + sys.path.append(str(tmp_path)) + settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) + + yield settings, spiders_dir + + sys.modules.pop("test_spiders_xxx", None) + sys.path.remove(str(tmp_path)) + + +@pytest.fixture +def spider_loader(spider_loader_env): + settings, _ = spider_loader_env + return SpiderLoader.from_settings(settings) + + class TestSpiderLoader: - def setup_method(self): - orig_spiders_dir = module_dir / "test_spiders" - self.tmpdir = Path(tempfile.mkdtemp()) - self.spiders_dir = self.tmpdir / "test_spiders_xxx" - _copytree(orig_spiders_dir, self.spiders_dir) - sys.path.append(str(self.tmpdir)) - settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) - self.spider_loader = SpiderLoader.from_settings(settings) + def test_interface(self, spider_loader): + verifyObject(ISpiderLoader, spider_loader) - def teardown_method(self): - del self.spider_loader - del sys.modules["test_spiders_xxx"] - sys.path.remove(str(self.tmpdir)) - - def test_interface(self): - verifyObject(ISpiderLoader, self.spider_loader) - - def test_list(self): - assert set(self.spider_loader.list()) == { + def test_list(self, spider_loader): + assert set(spider_loader.list()) == { "spider1", "spider2", "spider3", "spider4", } - def test_load(self): - spider1 = self.spider_loader.load("spider1") + def test_load(self, spider_loader): + spider1 = spider_loader.load("spider1") assert spider1.__name__ == "Spider1" - def test_find_by_request(self): - assert self.spider_loader.find_by_request( - Request("http://scrapy1.org/test") - ) == ["spider1"] - assert self.spider_loader.find_by_request( - Request("http://scrapy2.org/test") - ) == ["spider2"] + def test_find_by_request(self, spider_loader): + assert spider_loader.find_by_request(Request("http://scrapy1.org/test")) == [ + "spider1" + ] + assert spider_loader.find_by_request(Request("http://scrapy2.org/test")) == [ + "spider2" + ] assert set( - self.spider_loader.find_by_request(Request("http://scrapy3.org/test")) + spider_loader.find_by_request(Request("http://scrapy3.org/test")) ) == {"spider1", "spider2"} - assert ( - self.spider_loader.find_by_request(Request("http://scrapy999.org/test")) - == [] - ) - assert self.spider_loader.find_by_request(Request("http://spider3.com")) == [] - assert self.spider_loader.find_by_request( + assert spider_loader.find_by_request(Request("http://scrapy999.org/test")) == [] + assert spider_loader.find_by_request(Request("http://spider3.com")) == [] + assert spider_loader.find_by_request( Request("http://spider3.com/onlythis") ) == ["spider3"] def test_load_spider_module(self): module = "tests.test_spiderloader.test_spiders.spider1" settings = Settings({"SPIDER_MODULES": [module]}) - self.spider_loader = SpiderLoader.from_settings(settings) - assert len(self.spider_loader._spiders) == 1 + spider_loader = SpiderLoader.from_settings(settings) + assert len(spider_loader._spiders) == 1 def test_load_spider_module_multiple(self): prefix = "tests.test_spiderloader.test_spiders." module = ",".join(prefix + s for s in ("spider1", "spider2")) settings = Settings({"SPIDER_MODULES": module}) - self.spider_loader = SpiderLoader.from_settings(settings) - assert len(self.spider_loader._spiders) == 2 + spider_loader = SpiderLoader.from_settings(settings) + assert len(spider_loader._spiders) == 2 def test_load_base_spider(self): module = "tests.test_spiderloader.test_spiders.spider0" settings = Settings({"SPIDER_MODULES": [module]}) - self.spider_loader = SpiderLoader.from_settings(settings) - assert len(self.spider_loader._spiders) == 0 + spider_loader = SpiderLoader.from_settings(settings) + assert len(spider_loader._spiders) == 0 def test_load_spider_module_from_addons(self): module = "tests.test_spiderloader.spiders_from_addons.spider0" @@ -183,27 +184,14 @@ class TestSpiderLoader: class TestDuplicateSpiderNameLoader: - def setup_method(self): - orig_spiders_dir = module_dir / "test_spiders" - self.tmpdir = Path(mkdtemp()) - self.spiders_dir = self.tmpdir / "test_spiders_xxx" - _copytree(orig_spiders_dir, self.spiders_dir) - sys.path.append(str(self.tmpdir)) - self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]}) + def test_dupename_warning(self, spider_loader_env): + settings, spiders_dir = spider_loader_env - def teardown_method(self): - del sys.modules["test_spiders_xxx"] - sys.path.remove(str(self.tmpdir)) - - def test_dupename_warning(self): # copy 1 spider module so as to have duplicate spider name - shutil.copyfile( - self.tmpdir / "test_spiders_xxx" / "spider3.py", - self.tmpdir / "test_spiders_xxx" / "spider3dupe.py", - ) + shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py") with warnings.catch_warnings(record=True) as w: - spider_loader = SpiderLoader.from_settings(self.settings) + spider_loader = SpiderLoader.from_settings(settings) assert len(w) == 1 msg = str(w[0].message) @@ -218,20 +206,15 @@ class TestDuplicateSpiderNameLoader: spiders = set(spider_loader.list()) assert spiders == {"spider1", "spider2", "spider3", "spider4"} - def test_multiple_dupename_warning(self): + def test_multiple_dupename_warning(self, spider_loader_env): + settings, spiders_dir = spider_loader_env # copy 2 spider modules so as to have duplicate spider name # This should issue 2 warning, 1 for each duplicate spider name - shutil.copyfile( - self.tmpdir / "test_spiders_xxx" / "spider1.py", - self.tmpdir / "test_spiders_xxx" / "spider1dupe.py", - ) - shutil.copyfile( - self.tmpdir / "test_spiders_xxx" / "spider2.py", - self.tmpdir / "test_spiders_xxx" / "spider2dupe.py", - ) + shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py") + shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py") with warnings.catch_warnings(record=True) as w: - spider_loader = SpiderLoader.from_settings(self.settings) + spider_loader = SpiderLoader.from_settings(settings) assert len(w) == 1 msg = str(w[0].message) @@ -247,3 +230,25 @@ class TestDuplicateSpiderNameLoader: spiders = set(spider_loader.list()) assert spiders == {"spider1", "spider2", "spider3", "spider4"} + + +class CustomSpiderLoader(SpiderLoader): + pass + + +def test_custom_spider_loader(): + settings = Settings( + { + "SPIDER_LOADER_CLASS": CustomSpiderLoader, + } + ) + spider_loader = get_spider_loader(settings) + assert isinstance(spider_loader, CustomSpiderLoader) + + +def test_dummy_spider_loader(spider_loader_env): + settings, _ = spider_loader_env + spider_loader = DummySpiderLoader.from_settings(settings) + assert not spider_loader.list() + with pytest.raises(KeyError): + spider_loader.load("spider1") From a724541a715bb9fc5428ba630f24e1036ca0a896 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 May 2025 00:46:04 +0500 Subject: [PATCH 1748/2083] Split tests/test_commands.py. (#6836) --- tests/test_command_crawl.py | 93 +++ tests/test_command_genspider.py | 208 +++++++ tests/test_command_runspider.py | 375 +++++++++++ tests/test_command_startproject.py | 318 ++++++++++ tests/test_commands.py | 966 +---------------------------- 5 files changed, 998 insertions(+), 962 deletions(-) create mode 100644 tests/test_command_crawl.py create mode 100644 tests/test_command_genspider.py create mode 100644 tests/test_command_runspider.py create mode 100644 tests/test_command_startproject.py diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py new file mode 100644 index 000000000..3d5e17977 --- /dev/null +++ b/tests/test_command_crawl.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from pathlib import Path + +from tests.test_commands import TestCommandBase + + +class TestCrawlCommand(TestCommandBase): + def crawl(self, code, args=()): + Path(self.proj_mod_path, "spiders", "myspider.py").write_text( + code, encoding="utf-8" + ) + return self.proc("crawl", "myspider", *args) + + def get_log(self, code, args=()): + _, _, stderr = self.crawl(code, args=args) + return stderr + + def test_no_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('It works!') + return + yield +""" + log = self.get_log(spider_code) + assert "[myspider] DEBUG: It works!" in log + + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return + yield +""" + args = ["-o", "example.json"] + log = self.get_log(spider_code, args=args) + assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return + yield +""" + Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") + args = ["-O", "example.json"] + log = self.get_log(spider_code, args=args) + assert ( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + in log + ) + with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: + first_line = f2.readline() + assert first_line != "not empty" + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + return + yield +""" + args = ["-o", "example1.json", "-O", "example2.json"] + log = self.get_log(spider_code, args=args) + assert ( + "error: Please use only one of -o/--output and -O/--overwrite-output" in log + ) diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py new file mode 100644 index 000000000..18ec81fed --- /dev/null +++ b/tests/test_command_genspider.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import os +from pathlib import Path + +from tests.test_commands import TestCommandBase, TestProjectBase + + +class TestGenspiderCommand(TestCommandBase): + def test_arguments(self): + # only pass one argument. spider script shouldn't be created + assert self.call("genspider", "test_name") == 2 + assert not Path(self.proj_mod_path, "spiders", "test_name.py").exists() + # pass two arguments . spider script should be created + assert self.call("genspider", "test_name", "test.com") == 0 + assert Path(self.proj_mod_path, "spiders", "test_name.py").exists() + + def test_template(self, tplname="crawl"): + args = [f"--template={tplname}"] if tplname else [] + spname = "test_spider" + spmodule = f"{self.project_name}.spiders.{spname}" + p, out, err = self.proc("genspider", spname, "test.com", *args) + assert ( + f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}" + in out + ) + assert Path(self.proj_mod_path, "spiders", "test_spider.py").exists() + modify_time_before = ( + Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime + ) + p, out, err = self.proc("genspider", spname, "test.com", *args) + assert f"Spider {spname!r} already exists in module" in out + modify_time_after = ( + Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime + ) + assert modify_time_after == modify_time_before + + def test_template_basic(self): + self.test_template("basic") + + def test_template_csvfeed(self): + self.test_template("csvfeed") + + def test_template_xmlfeed(self): + self.test_template("xmlfeed") + + def test_list(self): + assert self.call("genspider", "--list") == 0 + + def test_dump(self): + assert self.call("genspider", "--dump=basic") == 0 + assert self.call("genspider", "-d", "basic") == 0 + + def test_same_name_as_project(self): + assert self.call("genspider", self.project_name) == 2 + assert not Path( + self.proj_mod_path, "spiders", f"{self.project_name}.py" + ).exists() + + def test_same_filename_as_existing_spider(self, force=False): + file_name = "example" + file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py") + assert self.call("genspider", file_name, "example.com") == 0 + assert file_path.exists() + + # change name of spider but not its file name + with file_path.open("r+", encoding="utf-8") as spider_file: + file_data = spider_file.read() + file_data = file_data.replace('name = "example"', 'name = "renamed"') + spider_file.seek(0) + spider_file.write(file_data) + spider_file.truncate() + modify_time_before = file_path.stat().st_mtime + file_contents_before = file_data + + if force: + p, out, err = self.proc("genspider", "--force", file_name, "example.com") + assert ( + f"Created spider {file_name!r} using template 'basic' in module" in out + ) + modify_time_after = file_path.stat().st_mtime + assert modify_time_after != modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after != file_contents_before + else: + p, out, err = self.proc("genspider", file_name, "example.com") + assert f"{file_path.resolve()} already exists" in out + modify_time_after = file_path.stat().st_mtime + assert modify_time_after == modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after == file_contents_before + + def test_same_filename_as_existing_spider_force(self): + self.test_same_filename_as_existing_spider(force=True) + + def test_url(self, url="test.com", domain="test.com"): + assert self.call("genspider", "--force", "test_name", url) == 0 + assert ( + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", + ).group(1) + == domain + ) + assert ( + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"start_urls\s*=\s*\[['\"](.+)['\"]\]", + ).group(1) + == f"https://{domain}" + ) + + def test_url_schema(self): + self.test_url("https://test.com", "test.com") + + def test_template_start_urls( + self, url="test.com", expected="https://test.com", template="basic" + ): + assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0 + assert ( + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"start_urls\s*=\s*\[['\"](.+)['\"]\]", + ).group(1) + == expected + ) + + def test_genspider_basic_start_urls(self): + self.test_template_start_urls("https://test.com", "https://test.com", "basic") + self.test_template_start_urls("http://test.com", "http://test.com", "basic") + self.test_template_start_urls( + "http://test.com/other/path", "http://test.com/other/path", "basic" + ) + self.test_template_start_urls( + "test.com/other/path", "https://test.com/other/path", "basic" + ) + + def test_genspider_crawl_start_urls(self): + self.test_template_start_urls("https://test.com", "https://test.com", "crawl") + self.test_template_start_urls("http://test.com", "http://test.com", "crawl") + self.test_template_start_urls( + "http://test.com/other/path", "http://test.com/other/path", "crawl" + ) + self.test_template_start_urls( + "test.com/other/path", "https://test.com/other/path", "crawl" + ) + self.test_template_start_urls("test.com", "https://test.com", "crawl") + + def test_genspider_xmlfeed_start_urls(self): + self.test_template_start_urls( + "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" + ) + self.test_template_start_urls( + "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed" + ) + self.test_template_start_urls( + "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" + ) + + def test_genspider_csvfeed_start_urls(self): + self.test_template_start_urls( + "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" + ) + self.test_template_start_urls( + "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed" + ) + self.test_template_start_urls( + "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" + ) + + +class TestGenspiderStandaloneCommand(TestProjectBase): + def test_generate_standalone_spider(self): + self.call("genspider", "example", "example.com") + assert Path(self.temp_path, "example.py").exists() + + def test_same_name_as_existing_file(self, force=False): + file_name = "example" + file_path = Path(self.temp_path, file_name + ".py") + p, out, err = self.proc("genspider", file_name, "example.com") + assert f"Created spider {file_name!r} using template 'basic' " in out + assert file_path.exists() + modify_time_before = file_path.stat().st_mtime + file_contents_before = file_path.read_text(encoding="utf-8") + + if force: + # use different template to ensure contents were changed + p, out, err = self.proc( + "genspider", "--force", "-t", "crawl", file_name, "example.com" + ) + assert f"Created spider {file_name!r} using template 'crawl' " in out + modify_time_after = file_path.stat().st_mtime + assert modify_time_after != modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after != file_contents_before + else: + p, out, err = self.proc("genspider", file_name, "example.com") + assert ( + f"{Path(self.temp_path, file_name + '.py').resolve()} already exists" + in out + ) + modify_time_after = file_path.stat().st_mtime + assert modify_time_after == modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after == file_contents_before + + def test_same_name_as_existing_file_force(self): + self.test_same_name_as_existing_file(force=True) diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py new file mode 100644 index 000000000..664de16f8 --- /dev/null +++ b/tests/test_command_runspider.py @@ -0,0 +1,375 @@ +from __future__ import annotations + +import inspect +import platform +import sys +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory, mkdtemp +from typing import TYPE_CHECKING +from unittest import skipIf + +import pytest +from twisted.trial import unittest + +from tests.test_commands import TestCommandBase +from tests.test_crawler import ExceptionSpider, NoRequestsSpider + +if TYPE_CHECKING: + from collections.abc import Iterator + + +class TestRunSpiderCommand(TestCommandBase): + spider_filename = "myspider.py" + + debug_log_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug("It Works!") + return + yield +""" + + badspider = """ +import scrapy + +class BadSpider(scrapy.Spider): + name = "bad" + async def start(self): + raise Exception("oops!") + yield + """ + + @contextmanager + def _create_file(self, content: str, name: str | None = None) -> Iterator[str]: + with TemporaryDirectory() as tmpdir: + if name: + fname = Path(tmpdir, name).resolve() + else: + fname = Path(tmpdir, self.spider_filename).resolve() + fname.write_text(content, encoding="utf-8") + yield str(fname) + + def runspider(self, code, name=None, args=()): + with self._create_file(code, name) as fname: + return self.proc("runspider", fname, *args) + + def get_log(self, code, name=None, args=()): + p, stdout, stderr = self.runspider(code, name, args=args) + return stderr + + def test_runspider(self): + log = self.get_log(self.debug_log_spider) + assert "DEBUG: It Works!" in log + assert "INFO: Spider opened" in log + assert "INFO: Closing spider (finished)" in log + assert "INFO: Spider closed (finished)" in log + + def test_run_fail_spider(self): + proc, _, _ = self.runspider( + "import scrapy\n" + inspect.getsource(ExceptionSpider) + ) + ret = proc.returncode + assert ret != 0 + + def test_run_good_spider(self): + proc, _, _ = self.runspider( + "import scrapy\n" + inspect.getsource(NoRequestsSpider) + ) + ret = proc.returncode + assert ret == 0 + + def test_runspider_log_level(self): + log = self.get_log(self.debug_log_spider, args=("-s", "LOG_LEVEL=INFO")) + assert "DEBUG: It Works!" not in log + assert "INFO: Spider opened" in log + + def test_runspider_dnscache_disabled(self): + # see https://github.com/scrapy/scrapy/issues/2811 + # The spider below should not be able to connect to localhost:12345, + # which is intended, + # but this should not be because of DNS lookup error + # assumption: localhost will resolve in all cases (true?) + dnscache_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + start_urls = ['http://localhost:12345'] + + def parse(self, response): + return {'test': 'value'} +""" + log = self.get_log(dnscache_spider, args=("-s", "DNSCACHE_ENABLED=False")) + assert "DNSLookupError" not in log + assert "INFO: Spider opened" in log + + def test_runspider_log_short_names(self): + log1 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=1")) + assert "[myspider] DEBUG: It Works!" in log1 + assert "[scrapy]" in log1 + assert "[scrapy.core.engine]" not in log1 + + log2 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=0")) + assert "[myspider] DEBUG: It Works!" in log2 + assert "[scrapy]" not in log2 + assert "[scrapy.core.engine]" in log2 + + def test_runspider_no_spider_found(self): + log = self.get_log("from scrapy.spiders import Spider\n") + assert "No spider found in file" in log + + def test_runspider_file_not_found(self): + _, _, log = self.proc("runspider", "some_non_existent_file") + assert "File not found: some_non_existent_file" in log + + def test_runspider_unable_to_load(self): + log = self.get_log("", name="myspider.txt") + assert "Unable to load" in log + + def test_start_errors(self): + log = self.get_log(self.badspider, name="badspider.py") + assert "start" in log + assert "badspider.py" in log, log + + def test_asyncio_enabled_true(self): + log = self.get_log( + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + ], + ) + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_asyncio_enabled_default(self): + log = self.get_log(self.debug_log_spider, args=[]) + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_asyncio_enabled_false(self): + log = self.get_log( + self.debug_log_spider, + args=["-s", "TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor"], + ) + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + + @pytest.mark.requires_uvloop + def test_custom_asyncio_loop_enabled_true(self): + log = self.get_log( + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "-s", + "ASYNCIO_EVENT_LOOP=uvloop.Loop", + ], + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + def test_custom_asyncio_loop_enabled_false(self): + log = self.get_log( + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + ], + ) + import asyncio + + if sys.platform != "win32": + loop = asyncio.new_event_loop() + else: + loop = asyncio.SelectorEventLoop() + assert ( + f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}" + in log + ) + + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return + yield +""" + args = ["-o", "example.json"] + log = self.get_log(spider_code, args=args) + assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return + yield +""" + Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") + args = ["-O", "example.json"] + log = self.get_log(spider_code, args=args) + assert ( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + in log + ) + with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: + first_line = f2.readline() + assert first_line != "not empty" + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + return + yield +""" + args = ["-o", "example1.json", "-O", "example2.json"] + log = self.get_log(spider_code, args=args) + assert ( + "error: Please use only one of -o/--output and -O/--overwrite-output" in log + ) + + def test_output_stdout(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return + yield +""" + args = ["-o", "-:json"] + log = self.get_log(spider_code, args=args) + assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log + + @skipIf(platform.system() == "Windows", reason="Linux only") + def test_absolute_path_linux(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}/output1.json:json"] + log = self.get_log(spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json" + in log + ) + + args = ["-o", f"{temp_dir}/output2.json"] + log = self.get_log(spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json" + in log + ) + + @skipIf(platform.system() != "Windows", reason="Windows only") + def test_absolute_path_windows(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}\\output1.json:json"] + log = self.get_log(spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json" + in log + ) + + args = ["-o", f"{temp_dir}\\output2.json"] + log = self.get_log(spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json" + in log + ) + + def test_args_change_settings(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("FOO", kwargs.get("foo")) + return spider + + async def start(self): + self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") + return + yield +""" + args = ["-a", "foo=42"] + log = self.get_log(spider_code, args=args) + assert "Spider closed (finished)" in log + assert "The value of FOO is 42" in log + + +class TestWindowsRunSpiderCommand(TestRunSpiderCommand): + spider_filename = "myspider.pyw" + + def setUp(self): + if platform.system() != "Windows": + raise unittest.SkipTest("Windows required for .pyw files") + return super().setUp() + + def test_start_errors(self): + log = self.get_log(self.badspider, name="badspider.pyw") + assert "start" in log + assert "badspider.pyw" in log + + def test_runspider_unable_to_load(self): + raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py new file mode 100644 index 000000000..08bf9b0fd --- /dev/null +++ b/tests/test_command_startproject.py @@ -0,0 +1,318 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from contextlib import contextmanager +from itertools import chain +from pathlib import Path +from shutil import copytree +from stat import S_IWRITE as ANYONE_WRITE_PERMISSION +from tempfile import mkdtemp + +import scrapy +from scrapy.commands.startproject import IGNORE +from tests.test_commands import TestProjectBase + + +class TestStartprojectCommand(TestProjectBase): + def test_startproject(self): + p, out, err = self.proc("startproject", self.project_name) + print(out) + print(err, file=sys.stderr) + assert p.returncode == 0 + + assert Path(self.proj_path, "scrapy.cfg").exists() + assert Path(self.proj_path, "testproject").exists() + assert Path(self.proj_mod_path, "__init__.py").exists() + assert Path(self.proj_mod_path, "items.py").exists() + assert Path(self.proj_mod_path, "pipelines.py").exists() + assert Path(self.proj_mod_path, "settings.py").exists() + assert Path(self.proj_mod_path, "spiders", "__init__.py").exists() + + assert self.call("startproject", self.project_name) == 1 + assert self.call("startproject", "wrong---project---name") == 1 + assert self.call("startproject", "sys") == 1 + + def test_startproject_with_project_dir(self): + project_dir = mkdtemp() + assert self.call("startproject", self.project_name, project_dir) == 0 + + assert Path(project_dir, "scrapy.cfg").exists() + assert Path(project_dir, "testproject").exists() + assert Path(project_dir, self.project_name, "__init__.py").exists() + assert Path(project_dir, self.project_name, "items.py").exists() + assert Path(project_dir, self.project_name, "pipelines.py").exists() + assert Path(project_dir, self.project_name, "settings.py").exists() + assert Path(project_dir, self.project_name, "spiders", "__init__.py").exists() + + assert self.call("startproject", self.project_name, project_dir + "2") == 0 + + assert self.call("startproject", self.project_name, project_dir) == 1 + assert self.call("startproject", self.project_name + "2", project_dir) == 1 + assert self.call("startproject", "wrong---project---name") == 1 + assert self.call("startproject", "sys") == 1 + assert self.call("startproject") == 2 + assert ( + self.call("startproject", self.project_name, project_dir, "another_params") + == 2 + ) + + def test_existing_project_dir(self): + project_dir = mkdtemp() + project_name = self.project_name + "_existing" + project_path = Path(project_dir, project_name) + project_path.mkdir() + + p, out, err = self.proc("startproject", project_name, cwd=project_dir) + print(out) + print(err, file=sys.stderr) + assert p.returncode == 0 + + assert Path(project_path, "scrapy.cfg").exists() + assert Path(project_path, project_name).exists() + assert Path(project_path, project_name, "__init__.py").exists() + assert Path(project_path, project_name, "items.py").exists() + assert Path(project_path, project_name, "pipelines.py").exists() + assert Path(project_path, project_name, "settings.py").exists() + assert Path(project_path, project_name, "spiders", "__init__.py").exists() + + +def get_permissions_dict( + path: str | os.PathLike, renamings=None, ignore=None +) -> dict[str, str]: + def get_permissions(path: Path) -> str: + return oct(path.stat().st_mode) + + path_obj = Path(path) + + renamings = renamings or () + permissions_dict = { + ".": get_permissions(path_obj), + } + for root, dirs, files in os.walk(path_obj): + nodes = list(chain(dirs, files)) + if ignore: + ignored_names = ignore(root, nodes) + nodes = [node for node in nodes if node not in ignored_names] + for node in nodes: + absolute_path = Path(root, node) + relative_path = str(absolute_path.relative_to(path)) + for search_string, replacement in renamings: + relative_path = relative_path.replace(search_string, replacement) + permissions = get_permissions(absolute_path) + permissions_dict[relative_path] = permissions + return permissions_dict + + +class TestStartprojectTemplates(TestProjectBase): + maxDiff = None + + def setUp(self): + super().setUp() + self.tmpl = str(Path(self.temp_path, "templates")) + self.tmpl_proj = str(Path(self.tmpl, "project")) + + def test_startproject_template_override(self): + copytree(Path(scrapy.__path__[0], "templates"), self.tmpl) + Path(self.tmpl_proj, "root_template").write_bytes(b"") + assert Path(self.tmpl_proj, "root_template").exists() + + args = ["--set", f"TEMPLATES_DIR={self.tmpl}"] + p, out, err = self.proc("startproject", self.project_name, *args) + assert ( + f"New Scrapy project '{self.project_name}', using template directory" in out + ) + assert self.tmpl_proj in out + assert Path(self.proj_path, "root_template").exists() + + def test_startproject_permissions_from_writable(self): + """Check that generated files have the right permissions when the + template folder has the same permissions as in the project, i.e. + everything is writable.""" + scrapy_path = scrapy.__path__[0] + project_template = Path(scrapy_path, "templates", "project") + project_name = "startproject1" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = Path(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions + + def test_startproject_permissions_from_read_only(self): + """Check that generated files have the right permissions when the + template folder has been made read-only, which is something that some + systems do. + + See https://github.com/scrapy/scrapy/pull/4604 + """ + scrapy_path = scrapy.__path__[0] + templates_dir = Path(scrapy_path, "templates") + project_template = Path(templates_dir, "project") + project_name = "startproject2" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + def _make_read_only(path: Path): + current_permissions = path.stat().st_mode + path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION) + + read_only_templates_dir = str(Path(mkdtemp()) / "templates") + copytree(templates_dir, read_only_templates_dir) + + for root, dirs, files in os.walk(read_only_templates_dir): + for node in chain(dirs, files): + _make_read_only(Path(root, node)) + + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + "--set", + f"TEMPLATES_DIR={read_only_templates_dir}", + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = Path(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions + + def test_startproject_permissions_unchanged_in_destination(self): + """Check that preexisting folders and files in the destination folder + do not see their permissions modified.""" + scrapy_path = scrapy.__path__[0] + project_template = Path(scrapy_path, "templates", "project") + project_name = "startproject3" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + destination = mkdtemp() + project_dir = Path(destination, project_name) + + existing_nodes = { + oct(permissions)[2:] + extension: permissions + for extension in ("", ".d") + for permissions in ( + 0o444, + 0o555, + 0o644, + 0o666, + 0o755, + 0o777, + ) + } + project_dir.mkdir() + for node, permissions in existing_nodes.items(): + path = project_dir / node + if node.endswith(".d"): + path.mkdir(mode=permissions) + else: + path.touch(mode=permissions) + expected_permissions[node] = oct(path.stat().st_mode) + + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + ".", + ), + cwd=project_dir, + env=self.env, + ) + process.wait() + + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions + + def test_startproject_permissions_umask_022(self): + """Check that generated files have the right permissions when the + system uses a umask value that causes new files to have different + permissions than those from the template folder.""" + + @contextmanager + def umask(new_mask): + cur_mask = os.umask(new_mask) + yield + os.umask(cur_mask) + + scrapy_path = scrapy.__path__[0] + project_template = Path(scrapy_path, "templates", "project") + project_name = "umaskproject" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + with umask(0o002): + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = Path(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions diff --git a/tests/test_commands.py b/tests/test_commands.py index 16af97842..6e59f561d 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,38 +1,29 @@ from __future__ import annotations import argparse -import inspect import json -import os -import platform import re import subprocess import sys -from contextlib import contextmanager from io import StringIO -from itertools import chain from pathlib import Path -from shutil import copytree, rmtree -from stat import S_IWRITE as ANYONE_WRITE_PERMISSION -from tempfile import TemporaryDirectory, TemporaryFile, mkdtemp +from shutil import rmtree +from tempfile import TemporaryFile, mkdtemp from threading import Timer from typing import TYPE_CHECKING -from unittest import mock, skipIf +from unittest import mock -import pytest from twisted.trial import unittest import scrapy from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view -from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv -from tests.test_crawler import ExceptionSpider, NoRequestsSpider if TYPE_CHECKING: - from collections.abc import Iterator + import os class TestCommandSettings: @@ -125,309 +116,6 @@ class TestProjectBase(unittest.TestCase): return None -class TestStartprojectCommand(TestProjectBase): - def test_startproject(self): - p, out, err = self.proc("startproject", self.project_name) - print(out) - print(err, file=sys.stderr) - assert p.returncode == 0 - - assert Path(self.proj_path, "scrapy.cfg").exists() - assert Path(self.proj_path, "testproject").exists() - assert Path(self.proj_mod_path, "__init__.py").exists() - assert Path(self.proj_mod_path, "items.py").exists() - assert Path(self.proj_mod_path, "pipelines.py").exists() - assert Path(self.proj_mod_path, "settings.py").exists() - assert Path(self.proj_mod_path, "spiders", "__init__.py").exists() - - assert self.call("startproject", self.project_name) == 1 - assert self.call("startproject", "wrong---project---name") == 1 - assert self.call("startproject", "sys") == 1 - - def test_startproject_with_project_dir(self): - project_dir = mkdtemp() - assert self.call("startproject", self.project_name, project_dir) == 0 - - assert Path(project_dir, "scrapy.cfg").exists() - assert Path(project_dir, "testproject").exists() - assert Path(project_dir, self.project_name, "__init__.py").exists() - assert Path(project_dir, self.project_name, "items.py").exists() - assert Path(project_dir, self.project_name, "pipelines.py").exists() - assert Path(project_dir, self.project_name, "settings.py").exists() - assert Path(project_dir, self.project_name, "spiders", "__init__.py").exists() - - assert self.call("startproject", self.project_name, project_dir + "2") == 0 - - assert self.call("startproject", self.project_name, project_dir) == 1 - assert self.call("startproject", self.project_name + "2", project_dir) == 1 - assert self.call("startproject", "wrong---project---name") == 1 - assert self.call("startproject", "sys") == 1 - assert self.call("startproject") == 2 - assert ( - self.call("startproject", self.project_name, project_dir, "another_params") - == 2 - ) - - def test_existing_project_dir(self): - project_dir = mkdtemp() - project_name = self.project_name + "_existing" - project_path = Path(project_dir, project_name) - project_path.mkdir() - - p, out, err = self.proc("startproject", project_name, cwd=project_dir) - print(out) - print(err, file=sys.stderr) - assert p.returncode == 0 - - assert Path(project_path, "scrapy.cfg").exists() - assert Path(project_path, project_name).exists() - assert Path(project_path, project_name, "__init__.py").exists() - assert Path(project_path, project_name, "items.py").exists() - assert Path(project_path, project_name, "pipelines.py").exists() - assert Path(project_path, project_name, "settings.py").exists() - assert Path(project_path, project_name, "spiders", "__init__.py").exists() - - -def get_permissions_dict( - path: str | os.PathLike, renamings=None, ignore=None -) -> dict[str, str]: - def get_permissions(path: Path) -> str: - return oct(path.stat().st_mode) - - path_obj = Path(path) - - renamings = renamings or () - permissions_dict = { - ".": get_permissions(path_obj), - } - for root, dirs, files in os.walk(path_obj): - nodes = list(chain(dirs, files)) - if ignore: - ignored_names = ignore(root, nodes) - nodes = [node for node in nodes if node not in ignored_names] - for node in nodes: - absolute_path = Path(root, node) - relative_path = str(absolute_path.relative_to(path)) - for search_string, replacement in renamings: - relative_path = relative_path.replace(search_string, replacement) - permissions = get_permissions(absolute_path) - permissions_dict[relative_path] = permissions - return permissions_dict - - -class TestStartprojectTemplates(TestProjectBase): - maxDiff = None - - def setUp(self): - super().setUp() - self.tmpl = str(Path(self.temp_path, "templates")) - self.tmpl_proj = str(Path(self.tmpl, "project")) - - def test_startproject_template_override(self): - copytree(Path(scrapy.__path__[0], "templates"), self.tmpl) - Path(self.tmpl_proj, "root_template").write_bytes(b"") - assert Path(self.tmpl_proj, "root_template").exists() - - args = ["--set", f"TEMPLATES_DIR={self.tmpl}"] - p, out, err = self.proc("startproject", self.project_name, *args) - assert ( - f"New Scrapy project '{self.project_name}', using template directory" in out - ) - assert self.tmpl_proj in out - assert Path(self.proj_path, "root_template").exists() - - def test_startproject_permissions_from_writable(self): - """Check that generated files have the right permissions when the - template folder has the same permissions as in the project, i.e. - everything is writable.""" - scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, "templates", "project") - project_name = "startproject1" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - destination = mkdtemp() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - ), - cwd=destination, - env=self.env, - ) - process.wait() - - project_dir = Path(destination, project_name) - actual_permissions = get_permissions_dict(project_dir) - - assert actual_permissions == expected_permissions - - def test_startproject_permissions_from_read_only(self): - """Check that generated files have the right permissions when the - template folder has been made read-only, which is something that some - systems do. - - See https://github.com/scrapy/scrapy/pull/4604 - """ - scrapy_path = scrapy.__path__[0] - templates_dir = Path(scrapy_path, "templates") - project_template = Path(templates_dir, "project") - project_name = "startproject2" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - def _make_read_only(path: Path): - current_permissions = path.stat().st_mode - path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION) - - read_only_templates_dir = str(Path(mkdtemp()) / "templates") - copytree(templates_dir, read_only_templates_dir) - - for root, dirs, files in os.walk(read_only_templates_dir): - for node in chain(dirs, files): - _make_read_only(Path(root, node)) - - destination = mkdtemp() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - "--set", - f"TEMPLATES_DIR={read_only_templates_dir}", - ), - cwd=destination, - env=self.env, - ) - process.wait() - - project_dir = Path(destination, project_name) - actual_permissions = get_permissions_dict(project_dir) - - assert actual_permissions == expected_permissions - - def test_startproject_permissions_unchanged_in_destination(self): - """Check that preexisting folders and files in the destination folder - do not see their permissions modified.""" - scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, "templates", "project") - project_name = "startproject3" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - destination = mkdtemp() - project_dir = Path(destination, project_name) - - existing_nodes = { - oct(permissions)[2:] + extension: permissions - for extension in ("", ".d") - for permissions in ( - 0o444, - 0o555, - 0o644, - 0o666, - 0o755, - 0o777, - ) - } - project_dir.mkdir() - for node, permissions in existing_nodes.items(): - path = project_dir / node - if node.endswith(".d"): - path.mkdir(mode=permissions) - else: - path.touch(mode=permissions) - expected_permissions[node] = oct(path.stat().st_mode) - - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - ".", - ), - cwd=project_dir, - env=self.env, - ) - process.wait() - - actual_permissions = get_permissions_dict(project_dir) - - assert actual_permissions == expected_permissions - - def test_startproject_permissions_umask_022(self): - """Check that generated files have the right permissions when the - system uses a umask value that causes new files to have different - permissions than those from the template folder.""" - - @contextmanager - def umask(new_mask): - cur_mask = os.umask(new_mask) - yield - os.umask(cur_mask) - - scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, "templates", "project") - project_name = "umaskproject" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - with umask(0o002): - destination = mkdtemp() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - ), - cwd=destination, - env=self.env, - ) - process.wait() - - project_dir = Path(destination, project_name) - actual_permissions = get_permissions_dict(project_dir) - - assert actual_permissions == expected_permissions - - class TestCommandBase(TestProjectBase): def setUp(self): super().setUp() @@ -436,208 +124,6 @@ class TestCommandBase(TestProjectBase): self.env["SCRAPY_SETTINGS_MODULE"] = f"{self.project_name}.settings" -class TestGenspiderCommand(TestCommandBase): - def test_arguments(self): - # only pass one argument. spider script shouldn't be created - assert self.call("genspider", "test_name") == 2 - assert not Path(self.proj_mod_path, "spiders", "test_name.py").exists() - # pass two arguments . spider script should be created - assert self.call("genspider", "test_name", "test.com") == 0 - assert Path(self.proj_mod_path, "spiders", "test_name.py").exists() - - def test_template(self, tplname="crawl"): - args = [f"--template={tplname}"] if tplname else [] - spname = "test_spider" - spmodule = f"{self.project_name}.spiders.{spname}" - p, out, err = self.proc("genspider", spname, "test.com", *args) - assert ( - f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}" - in out - ) - assert Path(self.proj_mod_path, "spiders", "test_spider.py").exists() - modify_time_before = ( - Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime - ) - p, out, err = self.proc("genspider", spname, "test.com", *args) - assert f"Spider {spname!r} already exists in module" in out - modify_time_after = ( - Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime - ) - assert modify_time_after == modify_time_before - - def test_template_basic(self): - self.test_template("basic") - - def test_template_csvfeed(self): - self.test_template("csvfeed") - - def test_template_xmlfeed(self): - self.test_template("xmlfeed") - - def test_list(self): - assert self.call("genspider", "--list") == 0 - - def test_dump(self): - assert self.call("genspider", "--dump=basic") == 0 - assert self.call("genspider", "-d", "basic") == 0 - - def test_same_name_as_project(self): - assert self.call("genspider", self.project_name) == 2 - assert not Path( - self.proj_mod_path, "spiders", f"{self.project_name}.py" - ).exists() - - def test_same_filename_as_existing_spider(self, force=False): - file_name = "example" - file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py") - assert self.call("genspider", file_name, "example.com") == 0 - assert file_path.exists() - - # change name of spider but not its file name - with file_path.open("r+", encoding="utf-8") as spider_file: - file_data = spider_file.read() - file_data = file_data.replace('name = "example"', 'name = "renamed"') - spider_file.seek(0) - spider_file.write(file_data) - spider_file.truncate() - modify_time_before = file_path.stat().st_mtime - file_contents_before = file_data - - if force: - p, out, err = self.proc("genspider", "--force", file_name, "example.com") - assert ( - f"Created spider {file_name!r} using template 'basic' in module" in out - ) - modify_time_after = file_path.stat().st_mtime - assert modify_time_after != modify_time_before - file_contents_after = file_path.read_text(encoding="utf-8") - assert file_contents_after != file_contents_before - else: - p, out, err = self.proc("genspider", file_name, "example.com") - assert f"{file_path.resolve()} already exists" in out - modify_time_after = file_path.stat().st_mtime - assert modify_time_after == modify_time_before - file_contents_after = file_path.read_text(encoding="utf-8") - assert file_contents_after == file_contents_before - - def test_same_filename_as_existing_spider_force(self): - self.test_same_filename_as_existing_spider(force=True) - - def test_url(self, url="test.com", domain="test.com"): - assert self.call("genspider", "--force", "test_name", url) == 0 - assert ( - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", - ).group(1) - == domain - ) - assert ( - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1) - == f"https://{domain}" - ) - - def test_url_schema(self): - self.test_url("https://test.com", "test.com") - - def test_template_start_urls( - self, url="test.com", expected="https://test.com", template="basic" - ): - assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0 - assert ( - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1) - == expected - ) - - def test_genspider_basic_start_urls(self): - self.test_template_start_urls("https://test.com", "https://test.com", "basic") - self.test_template_start_urls("http://test.com", "http://test.com", "basic") - self.test_template_start_urls( - "http://test.com/other/path", "http://test.com/other/path", "basic" - ) - self.test_template_start_urls( - "test.com/other/path", "https://test.com/other/path", "basic" - ) - - def test_genspider_crawl_start_urls(self): - self.test_template_start_urls("https://test.com", "https://test.com", "crawl") - self.test_template_start_urls("http://test.com", "http://test.com", "crawl") - self.test_template_start_urls( - "http://test.com/other/path", "http://test.com/other/path", "crawl" - ) - self.test_template_start_urls( - "test.com/other/path", "https://test.com/other/path", "crawl" - ) - self.test_template_start_urls("test.com", "https://test.com", "crawl") - - def test_genspider_xmlfeed_start_urls(self): - self.test_template_start_urls( - "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" - ) - self.test_template_start_urls( - "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed" - ) - self.test_template_start_urls( - "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" - ) - - def test_genspider_csvfeed_start_urls(self): - self.test_template_start_urls( - "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" - ) - self.test_template_start_urls( - "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed" - ) - self.test_template_start_urls( - "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" - ) - - -class TestGenspiderStandaloneCommand(TestProjectBase): - def test_generate_standalone_spider(self): - self.call("genspider", "example", "example.com") - assert Path(self.temp_path, "example.py").exists() - - def test_same_name_as_existing_file(self, force=False): - file_name = "example" - file_path = Path(self.temp_path, file_name + ".py") - p, out, err = self.proc("genspider", file_name, "example.com") - assert f"Created spider {file_name!r} using template 'basic' " in out - assert file_path.exists() - modify_time_before = file_path.stat().st_mtime - file_contents_before = file_path.read_text(encoding="utf-8") - - if force: - # use different template to ensure contents were changed - p, out, err = self.proc( - "genspider", "--force", "-t", "crawl", file_name, "example.com" - ) - assert f"Created spider {file_name!r} using template 'crawl' " in out - modify_time_after = file_path.stat().st_mtime - assert modify_time_after != modify_time_before - file_contents_after = file_path.read_text(encoding="utf-8") - assert file_contents_after != file_contents_before - else: - p, out, err = self.proc("genspider", file_name, "example.com") - assert ( - f"{Path(self.temp_path, file_name + '.py').resolve()} already exists" - in out - ) - modify_time_after = file_path.stat().st_mtime - assert modify_time_after == modify_time_before - file_contents_after = file_path.read_text(encoding="utf-8") - assert file_contents_after == file_contents_before - - def test_same_name_as_existing_file_force(self): - self.test_same_name_as_existing_file(force=True) - - class TestMiscCommands(TestCommandBase): def test_list(self): assert self.call("list") == 0 @@ -661,362 +147,6 @@ Unknown command: abc assert out.getvalue().strip() == message.strip() -class TestRunSpiderCommand(TestCommandBase): - spider_filename = "myspider.py" - - debug_log_spider = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug("It Works!") - return - yield -""" - - badspider = """ -import scrapy - -class BadSpider(scrapy.Spider): - name = "bad" - async def start(self): - raise Exception("oops!") - yield - """ - - @contextmanager - def _create_file(self, content: str, name: str | None = None) -> Iterator[str]: - with TemporaryDirectory() as tmpdir: - if name: - fname = Path(tmpdir, name).resolve() - else: - fname = Path(tmpdir, self.spider_filename).resolve() - fname.write_text(content, encoding="utf-8") - yield str(fname) - - def runspider(self, code, name=None, args=()): - with self._create_file(code, name) as fname: - return self.proc("runspider", fname, *args) - - def get_log(self, code, name=None, args=()): - p, stdout, stderr = self.runspider(code, name, args=args) - return stderr - - def test_runspider(self): - log = self.get_log(self.debug_log_spider) - assert "DEBUG: It Works!" in log - assert "INFO: Spider opened" in log - assert "INFO: Closing spider (finished)" in log - assert "INFO: Spider closed (finished)" in log - - def test_run_fail_spider(self): - proc, _, _ = self.runspider( - "import scrapy\n" + inspect.getsource(ExceptionSpider) - ) - ret = proc.returncode - assert ret != 0 - - def test_run_good_spider(self): - proc, _, _ = self.runspider( - "import scrapy\n" + inspect.getsource(NoRequestsSpider) - ) - ret = proc.returncode - assert ret == 0 - - def test_runspider_log_level(self): - log = self.get_log(self.debug_log_spider, args=("-s", "LOG_LEVEL=INFO")) - assert "DEBUG: It Works!" not in log - assert "INFO: Spider opened" in log - - def test_runspider_dnscache_disabled(self): - # see https://github.com/scrapy/scrapy/issues/2811 - # The spider below should not be able to connect to localhost:12345, - # which is intended, - # but this should not be because of DNS lookup error - # assumption: localhost will resolve in all cases (true?) - dnscache_spider = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - start_urls = ['http://localhost:12345'] - - def parse(self, response): - return {'test': 'value'} -""" - log = self.get_log(dnscache_spider, args=("-s", "DNSCACHE_ENABLED=False")) - assert "DNSLookupError" not in log - assert "INFO: Spider opened" in log - - def test_runspider_log_short_names(self): - log1 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=1")) - assert "[myspider] DEBUG: It Works!" in log1 - assert "[scrapy]" in log1 - assert "[scrapy.core.engine]" not in log1 - - log2 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=0")) - assert "[myspider] DEBUG: It Works!" in log2 - assert "[scrapy]" not in log2 - assert "[scrapy.core.engine]" in log2 - - def test_runspider_no_spider_found(self): - log = self.get_log("from scrapy.spiders import Spider\n") - assert "No spider found in file" in log - - def test_runspider_file_not_found(self): - _, _, log = self.proc("runspider", "some_non_existent_file") - assert "File not found: some_non_existent_file" in log - - def test_runspider_unable_to_load(self): - log = self.get_log("", name="myspider.txt") - assert "Unable to load" in log - - def test_start_errors(self): - log = self.get_log(self.badspider, name="badspider.py") - assert "start" in log - assert "badspider.py" in log, log - - def test_asyncio_enabled_true(self): - log = self.get_log( - self.debug_log_spider, - args=[ - "-s", - "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", - ], - ) - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - - def test_asyncio_enabled_default(self): - log = self.get_log(self.debug_log_spider, args=[]) - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - - def test_asyncio_enabled_false(self): - log = self.get_log( - self.debug_log_spider, - args=["-s", "TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor"], - ) - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log - ) - - @pytest.mark.requires_uvloop - def test_custom_asyncio_loop_enabled_true(self): - log = self.get_log( - self.debug_log_spider, - args=[ - "-s", - "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "-s", - "ASYNCIO_EVENT_LOOP=uvloop.Loop", - ], - ) - assert "Using asyncio event loop: uvloop.Loop" in log - - def test_custom_asyncio_loop_enabled_false(self): - log = self.get_log( - self.debug_log_spider, - args=[ - "-s", - "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", - ], - ) - import asyncio - - if sys.platform != "win32": - loop = asyncio.new_event_loop() - else: - loop = asyncio.SelectorEventLoop() - assert ( - f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}" - in log - ) - - def test_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return - yield -""" - args = ["-o", "example.json"] - log = self.get_log(spider_code, args=args) - assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log - - def test_overwrite_output(self): - spider_code = """ -import json -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug( - 'FEEDS: {}'.format( - json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) - ) - ) - return - yield -""" - Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") - args = ["-O", "example.json"] - log = self.get_log(spider_code, args=args) - assert ( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' - in log - ) - with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: - first_line = f2.readline() - assert first_line != "not empty" - - def test_output_and_overwrite_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - return - yield -""" - args = ["-o", "example1.json", "-O", "example2.json"] - log = self.get_log(spider_code, args=args) - assert ( - "error: Please use only one of -o/--output and -O/--overwrite-output" in log - ) - - def test_output_stdout(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return - yield -""" - args = ["-o", "-:json"] - log = self.get_log(spider_code, args=args) - assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log - - @skipIf(platform.system() == "Windows", reason="Linux only") - def test_absolute_path_linux(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - start_urls = ["data:,"] - - def parse(self, response): - yield {"hello": "world"} - """ - temp_dir = mkdtemp() - - args = ["-o", f"{temp_dir}/output1.json:json"] - log = self.get_log(spider_code, args=args) - assert ( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json" - in log - ) - - args = ["-o", f"{temp_dir}/output2.json"] - log = self.get_log(spider_code, args=args) - assert ( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json" - in log - ) - - @skipIf(platform.system() != "Windows", reason="Windows only") - def test_absolute_path_windows(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - start_urls = ["data:,"] - - def parse(self, response): - yield {"hello": "world"} - """ - temp_dir = mkdtemp() - - args = ["-o", f"{temp_dir}\\output1.json:json"] - log = self.get_log(spider_code, args=args) - assert ( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json" - in log - ) - - args = ["-o", f"{temp_dir}\\output2.json"] - log = self.get_log(spider_code, args=args) - assert ( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json" - in log - ) - - def test_args_change_settings(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - @classmethod - def from_crawler(cls, crawler, *args, **kwargs): - spider = super().from_crawler(crawler, *args, **kwargs) - spider.settings.set("FOO", kwargs.get("foo")) - return spider - - async def start(self): - self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return - yield -""" - args = ["-a", "foo=42"] - log = self.get_log(spider_code, args=args) - assert "Spider closed (finished)" in log - assert "The value of FOO is 42" in log - - -class TestWindowsRunSpiderCommand(TestRunSpiderCommand): - spider_filename = "myspider.pyw" - - def setUp(self): - if platform.system() != "Windows": - raise unittest.SkipTest("Windows required for .pyw files") - return super().setUp() - - def test_start_errors(self): - log = self.get_log(self.badspider, name="badspider.pyw") - assert "start" in log - assert "badspider.pyw" in log - - def test_runspider_unable_to_load(self): - raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") - - class TestBenchCommand(TestCommandBase): def test_run(self): _, _, log = self.proc( @@ -1042,94 +172,6 @@ class TestViewCommand(TestCommandBase): assert "URL using the Scrapy downloader and show its" in command.long_desc() -class TestCrawlCommand(TestCommandBase): - def crawl(self, code, args=()): - Path(self.proj_mod_path, "spiders", "myspider.py").write_text( - code, encoding="utf-8" - ) - return self.proc("crawl", "myspider", *args) - - def get_log(self, code, args=()): - _, _, stderr = self.crawl(code, args=args) - return stderr - - def test_no_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug('It works!') - return - yield -""" - log = self.get_log(spider_code) - assert "[myspider] DEBUG: It works!" in log - - def test_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return - yield -""" - args = ["-o", "example.json"] - log = self.get_log(spider_code, args=args) - assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log - - def test_overwrite_output(self): - spider_code = """ -import json -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug( - 'FEEDS: {}'.format( - json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) - ) - ) - return - yield -""" - Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") - args = ["-O", "example.json"] - log = self.get_log(spider_code, args=args) - assert ( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' - in log - ) - with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: - first_line = f2.readline() - assert first_line != "not empty" - - def test_output_and_overwrite_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - return - yield -""" - args = ["-o", "example1.json", "-O", "example2.json"] - log = self.get_log(spider_code, args=args) - assert ( - "error: Please use only one of -o/--output and -O/--overwrite-output" in log - ) - - class TestHelpMessage(TestCommandBase): def setUp(self): super().setUp() From 8f92a26636b3cabf64fd29a65ef0eb554bdf4f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 May 2025 09:33:17 +0200 Subject: [PATCH 1749/2083] Avoid raw HTML in the README (#6839) --- README.rst | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 5dc994570..536dec7f0 100644 --- a/README.rst +++ b/README.rst @@ -1,40 +1,41 @@ -.. raw:: html +|logo| -

- - Scrapy - -

+.. |logo| image:: https://raw.githubusercontent.com/scrapy/scrapy/master/docs/_static/logo.svg + :target: https://scrapy.org + :alt: Scrapy + :width: 480px -.. image:: https://img.shields.io/pypi/v/Scrapy.svg +|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki| + +.. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy :alt: PyPI Version -.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg +.. |python_version| image:: https://img.shields.io/pypi/pyversions/Scrapy.svg :target: https://pypi.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg +.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg +.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS :alt: macOS -.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg +.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows :alt: Windows -.. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg +.. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report -.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg +.. |conda| image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg :target: https://anaconda.org/conda-forge/scrapy :alt: Conda Version -.. image:: https://deepwiki.com/badge.svg +.. |deepwiki| image:: https://deepwiki.com/badge.svg :target: https://deepwiki.com/scrapy/scrapy :alt: Ask DeepWiki From 8ae418df44b7a107a4abe1a718721742d7e33fc0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 2 Jun 2025 19:02:08 +0500 Subject: [PATCH 1750/2083] Rewrite download handler tests to coroutines. (#6846) --- .../test_downloader_handler_twisted_http10.py | 10 +- .../test_downloader_handler_twisted_http2.py | 112 +++-- tests/test_downloader_handlers.py | 123 +++--- tests/test_downloader_handlers_http_base.py | 410 +++++++++--------- 4 files changed, 329 insertions(+), 326 deletions(-) diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py index 807c8c4cb..bc306aa07 100644 --- a/tests/test_downloader_handler_twisted_http10.py +++ b/tests/test_downloader_handler_twisted_http10.py @@ -9,6 +9,7 @@ import pytest from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.defer import deferred_f_from_coro_f from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase if TYPE_CHECKING: @@ -25,12 +26,11 @@ class HTTP10DownloadHandlerMixin: class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase): """HTTP 1.0 test case""" - def test_protocol(self): + @deferred_f_from_coro_f + async def test_protocol(self): request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.0") - return d + response = await self.download_request(request, Spider("foo")) + assert response.protocol == "HTTP/1.0" class TestHttps10(TestHttp10): diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 159f403d0..e058cedae 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -15,6 +15,10 @@ from twisted.web.http import H2_ENABLED from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + maybe_deferred_to_future, +) from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests.mockserver import ssl_context_factory @@ -50,15 +54,14 @@ class H2DownloadHandlerMixin: class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - def test_protocol(self): + @deferred_f_from_coro_f + async def test_protocol(self): request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "h2") - return d + response = await self.download_request(request, Spider("foo")) + assert response.protocol == "h2" - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): + @deferred_f_from_coro_f + async def test_download_with_maxsize_very_large_file(self): from twisted.internet import reactor with mock.patch("scrapy.core.http2.stream.logger") as logger: @@ -67,8 +70,10 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): def check(logger): logger.error.assert_called_once_with(mock.ANY) - d = self.download_request(request, Spider("foo", download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request( + request, Spider("foo", download_maxsize=1500) + ) # As the error message is logged in the dataReceived callback, we # have to give a bit of time to the reactor to process the queue @@ -76,13 +81,13 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): d = defer.Deferred() d.addCallback(check) reactor.callLater(0.1, d.callback, logger) - yield d + await maybe_deferred_to_future(d) - @defer.inlineCallbacks - def test_unsupported_scheme(self): + @deferred_f_from_coro_f + async def test_unsupported_scheme(self): request = Request("ftp://unsupported.scheme") - d = self.download_request(request, Spider("foo")) - yield self.assertFailure(d, SchemeNotSupported) + with pytest.raises(SchemeNotSupported): + await self.download_request(request, Spider("foo")) def test_download_broken_content_cause_data_loss(self, url="broken"): pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) @@ -102,70 +107,60 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): def test_download_broken_chunked_content_allow_data_loss_via_setting(self): pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - def test_concurrent_requests_same_domain(self): + @deferred_f_from_coro_f + async def test_concurrent_requests_same_domain(self): spider = Spider("foo") request1 = Request(self.getURL("file")) - d1 = self.download_request(request1, spider) - d1.addCallback(lambda r: r.body) - d1.addCallback(self.assertEqual, b"0123456789") + response1 = await self.download_request(request1, spider) + assert response1.body == b"0123456789" request2 = Request(self.getURL("echo"), method="POST") - d2 = self.download_request(request2, spider) - d2.addCallback(lambda r: r.headers["Content-Length"]) - d2.addCallback(self.assertEqual, b"79") - - return defer.DeferredList([d1, d2]) + response2 = await self.download_request(request2, spider) + assert response2.headers["Content-Length"] == b"79" @pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") - def test_connect_request(self): + @deferred_f_from_coro_f + async def test_connect_request(self): request = Request(self.getURL("file"), method="CONNECT") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"" - def test_custom_content_length_good(self): + @deferred_f_from_coro_f + async def test_custom_content_length_good(self): request = Request(self.getURL("contentlength")) custom_content_length = str(len(request.body)) request.headers["Content-Length"] = custom_content_length - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.text) - d.addCallback(self.assertEqual, custom_content_length) - return d + response = await self.download_request(request, Spider("foo")) + assert response.text == custom_content_length - def test_custom_content_length_bad(self): + @deferred_f_from_coro_f + async def test_custom_content_length_bad(self): request = Request(self.getURL("contentlength")) actual_content_length = str(len(request.body)) bad_content_length = str(len(request.body) + 1) request.headers["Content-Length"] = bad_content_length - log = LogCapture() - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.text) - d.addCallback(self.assertEqual, actual_content_length) - d.addCallback( - lambda _: log.check_present( - ( - "scrapy.core.http2.stream", - "WARNING", - f"Ignoring bad Content-Length header " - f"{bad_content_length!r} of request {request}, sending " - f"{actual_content_length!r} instead", - ) + with LogCapture() as log: + response = await self.download_request(request, Spider("foo")) + assert response.text == actual_content_length + log.check_present( + ( + "scrapy.core.http2.stream", + "WARNING", + f"Ignoring bad Content-Length header " + f"{bad_content_length!r} of request {request}, sending " + f"{actual_content_length!r} instead", ) ) - d.addCallback(lambda _: log.uninstall()) - return d - def test_duplicate_header(self): + @deferred_f_from_coro_f + async def test_duplicate_header(self): request = Request(self.getURL("echo")) header, value1, value2 = "Custom-Header", "foo", "bar" request.headers.appendlist(header, value1) request.headers.appendlist(header, value2) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: json.loads(r.text)["headers"][header]) - d.addCallback(self.assertEqual, [value1, value2]) - return d + response = await self.download_request(request, Spider("foo")) + assert json.loads(response.text)["headers"][header] == [value1, value2] class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): @@ -222,12 +217,13 @@ class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase): self.download_handler = build_from_crawler( self.download_handler_cls, get_crawler() ) - self.download_request = self.download_handler.download_request def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" - @defer.inlineCallbacks - def test_download_with_proxy_https_timeout(self): + @deferred_f_from_coro_f + async def test_download_with_proxy_https_timeout(self): with pytest.raises(NotImplementedError): - yield super().test_download_with_proxy_https_timeout() + await maybe_deferred_to_future( + super().test_download_with_proxy_https_timeout() + ) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index dacadb075..09cdbaf35 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -22,10 +22,14 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.exceptions import NotConfigured -from scrapy.http import HtmlResponse, Request +from scrapy.http import HtmlResponse, Request, Response from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + maybe_deferred_to_future, +) from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler @@ -95,28 +99,33 @@ class TestFile(unittest.TestCase): # add a special char to check that they are handled correctly self.fd, self.tmpname = mkstemp(suffix="^") Path(self.tmpname).write_text("0123456789", encoding="utf-8") - handler = build_from_crawler(FileDownloadHandler, get_crawler()) - self.download_request = handler.download_request + self.download_handler = build_from_crawler(FileDownloadHandler, get_crawler()) def tearDown(self): os.close(self.fd) Path(self.tmpname).unlink() - def test_download(self): - def _test(response): - assert response.url == request.url - assert response.status == 200 - assert response.body == b"0123456789" - assert response.protocol is None + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + @deferred_f_from_coro_f + async def test_download(self): request = Request(path_to_file_uri(self.tmpname)) assert request.url.upper().endswith("%5E") - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.url == request.url + assert response.status == 200 + assert response.body == b"0123456789" + assert response.protocol is None - def test_non_existent(self): + @deferred_f_from_coro_f + async def test_non_existent(self): request = Request(path_to_file_uri(mkdtemp())) - d = self.download_request(request, Spider("foo")) - return self.assertFailure(d, OSError) + # the specific exception differs between platforms + with pytest.raises(OSError): # noqa: PT011 + await self.download_request(request, Spider("foo")) class HttpDownloadHandlerMock: @@ -479,69 +488,65 @@ class TestDataURI(unittest.TestCase): def setUp(self): crawler = get_crawler() self.download_handler = build_from_crawler(DataURIDownloadHandler, crawler) - self.download_request = self.download_handler.download_request self.spider = Spider("foo") - def test_response_attrs(self): + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + + @deferred_f_from_coro_f + async def test_response_attrs(self): uri = "data:,A%20brief%20note" - - def _test(response): - assert response.url == uri - assert not response.headers - request = Request(uri) - return self.download_request(request, self.spider).addCallback(_test) - - def test_default_mediatype_encoding(self): - def _test(response): - assert response.text == "A brief note" - assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck - assert response.encoding == "US-ASCII" + response = await self.download_request(request, self.spider) + assert response.url == uri + assert not response.headers + @deferred_f_from_coro_f + async def test_default_mediatype_encoding(self): request = Request("data:,A%20brief%20note") - return self.download_request(request, self.spider).addCallback(_test) - - def test_default_mediatype(self): - def _test(response): - assert response.text == "\u038e\u03a3\u038e" - assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck - assert response.encoding == "iso-8859-7" + response = await self.download_request(request, self.spider) + assert response.text == "A brief note" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "US-ASCII" + @deferred_f_from_coro_f + async def test_default_mediatype(self): request = Request("data:;charset=iso-8859-7,%be%d3%be") - return self.download_request(request, self.spider).addCallback(_test) - - def test_text_charset(self): - def _test(response): - assert response.text == "\u038e\u03a3\u038e" - assert response.body == b"\xbe\xd3\xbe" - assert response.encoding == "iso-8859-7" + response = await self.download_request(request, self.spider) + assert response.text == "\u038e\u03a3\u038e" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "iso-8859-7" + @deferred_f_from_coro_f + async def test_text_charset(self): request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") - return self.download_request(request, self.spider).addCallback(_test) - - def test_mediatype_parameters(self): - def _test(response): - assert response.text == "\u038e\u03a3\u038e" - assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck - assert response.encoding == "utf-8" + response = await self.download_request(request, self.spider) + assert response.text == "\u038e\u03a3\u038e" + assert response.body == b"\xbe\xd3\xbe" + assert response.encoding == "iso-8859-7" + @deferred_f_from_coro_f + async def test_mediatype_parameters(self): request = Request( "data:text/plain;foo=%22foo;bar%5C%22%22;" "charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22" ",%CE%8E%CE%A3%CE%8E" ) - return self.download_request(request, self.spider).addCallback(_test) - - def test_base64(self): - def _test(response): - assert response.text == "Hello, world." + response = await self.download_request(request, self.spider) + assert response.text == "\u038e\u03a3\u038e" + assert type(response) is responsetypes.from_mimetype("text/plain") # pylint: disable=unidiomatic-typecheck + assert response.encoding == "utf-8" + @deferred_f_from_coro_f + async def test_base64(self): request = Request("data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D") - return self.download_request(request, self.spider).addCallback(_test) - - def test_protocol(self): - def _test(response): - assert response.protocol is None + response = await self.download_request(request, self.spider) + assert response.text == "Hello, world." + @deferred_f_from_coro_f + async def test_protocol(self): request = Request("data:,") - return self.download_request(request, self.spider).addCallback(_test) + response = await self.download_request(request, self.spider) + assert response.protocol is None diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 5eaf66966..14e12a3e6 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import shutil import sys from abc import ABC, abstractmethod @@ -13,14 +14,20 @@ from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer, error +from twisted.internet.defer import maybeDeferred from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util from twisted.web._newclient import ResponseFailed from twisted.web.http import _DataLoss -from scrapy.http import Headers, HtmlResponse, Request, TextResponse +from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_from_coro, + maybe_deferred_to_future, +) from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler @@ -178,7 +185,6 @@ class TestHttpBase(unittest.TestCase, ABC): self.download_handler = build_from_crawler( self.download_handler_cls, get_crawler() ) - self.download_request = self.download_handler.download_request @defer.inlineCallbacks def tearDown(self): @@ -190,36 +196,37 @@ class TestHttpBase(unittest.TestCase, ABC): def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" - def test_download(self): + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + + @deferred_f_from_coro_f + async def test_download(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"0123456789" - def test_download_head(self): + @deferred_f_from_coro_f + async def test_download_head(self): request = Request(self.getURL("file"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"" - def test_redirect_status(self): + @deferred_f_from_coro_f + async def test_redirect_status(self): request = Request(self.getURL("redirect")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d + response = await self.download_request(request, Spider("foo")) + assert response.status == 302 - def test_redirect_status_head(self): + @deferred_f_from_coro_f + async def test_redirect_status_head(self): request = Request(self.getURL("redirect"), method="HEAD") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.status) - d.addCallback(self.assertEqual, 302) - return d + response = await self.download_request(request, Spider("foo")) + assert response.status == 302 - @defer.inlineCallbacks - def test_timeout_download_from_spider_nodata_rcvd(self): + @deferred_f_from_coro_f + async def test_timeout_download_from_spider_nodata_rcvd(self): if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( @@ -230,11 +237,12 @@ class TestHttpBase(unittest.TestCase, ABC): spider = Spider("foo") meta = {"download_timeout": 0.5} request = Request(self.getURL("wait"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) + d = deferred_from_coro(self.download_request(request, spider)) + with pytest.raises((defer.TimeoutError, error.TimeoutError)): + await maybe_deferred_to_future(d) - @defer.inlineCallbacks - def test_timeout_download_from_spider_server_hangs(self): + @deferred_f_from_coro_f + async def test_timeout_download_from_spider_server_hangs(self): if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( @@ -244,28 +252,27 @@ class TestHttpBase(unittest.TestCase, ABC): spider = Spider("foo") meta = {"download_timeout": 0.5} request = Request(self.getURL("hang-after-headers"), meta=meta) - d = self.download_request(request, spider) - yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) - - def test_host_header_not_in_request_headers(self): - def _test(response): - assert response.body == to_bytes(f"{self.host}:{self.portno}") - assert not request.headers + d = deferred_from_coro(self.download_request(request, spider)) + with pytest.raises((defer.TimeoutError, error.TimeoutError)): + await maybe_deferred_to_future(d) + @deferred_f_from_coro_f + async def test_host_header_not_in_request_headers(self): request = Request(self.getURL("host")) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.body == to_bytes(f"{self.host}:{self.portno}") + assert not request.headers - def test_host_header_seted_in_request_headers(self): + @deferred_f_from_coro_f + async def test_host_header_set_in_request_headers(self): host = self.host + ":" + str(self.portno) - - def _test(response): - assert response.body == host.encode() - assert request.headers.get("Host") == host.encode() - request = Request(self.getURL("host"), headers={"Host": host}) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.body == host.encode() + assert request.headers.get("Host") == host.encode() - def test_content_length_zero_bodyless_post_request_headers(self): + @deferred_f_from_coro_f + async def test_content_length_zero_bodyless_post_request_headers(self): """Tests if "Content-Length: 0" is sent for bodyless POST requests. This is not strictly required by HTTP RFCs but can cause trouble @@ -276,105 +283,93 @@ class TestHttpBase(unittest.TestCase, ABC): https://github.com/kennethreitz/requests/issues/405 https://bugs.python.org/issue14721 """ - - def _test(response): - assert response.body == b"0" - request = Request(self.getURL("contentlength"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_content_length_zero_bodyless_post_only_one(self): - def _test(response): - import json - - headers = Headers(json.loads(response.text)["headers"]) - contentlengths = headers.getlist("Content-Length") - assert len(contentlengths) == 1 - assert contentlengths == [b"0"] + response = await self.download_request(request, Spider("foo")) + assert response.body == b"0" + @deferred_f_from_coro_f + async def test_content_length_zero_bodyless_post_only_one(self): request = Request(self.getURL("echo"), method="POST") - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + headers = Headers(json.loads(response.text)["headers"]) + contentlengths = headers.getlist("Content-Length") + assert len(contentlengths) == 1 + assert contentlengths == [b"0"] - def test_payload(self): + @deferred_f_from_coro_f + async def test_payload(self): body = b"1" * 100 # PayloadResource requires body length to be 100 request = Request(self.getURL("payload"), method="POST", body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, body) - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == body - def test_response_header_content_length(self): + @deferred_f_from_coro_f + async def test_response_header_content_length(self): request = Request(self.getURL("file"), method=b"GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.headers[b"content-length"]) - d.addCallback(self.assertEqual, b"159") - return d - - def _test_response_class(self, filename, body, response_class): - def _test(response): - assert type(response) is response_class # pylint: disable=unidiomatic-typecheck + response = await self.download_request(request, Spider("foo")) + assert response.headers[b"content-length"] == b"159" + async def _test_response_class( + self, filename: str, body: bytes, response_class: type[Response] + ) -> None: request = Request(self.getURL(filename), body=body) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert type(response) is response_class # pylint: disable=unidiomatic-typecheck - def test_response_class_from_url(self): - return self._test_response_class("foo.html", b"", HtmlResponse) + @deferred_f_from_coro_f + async def test_response_class_from_url(self): + await self._test_response_class("foo.html", b"", HtmlResponse) - def test_response_class_from_body(self): - return self._test_response_class( + @deferred_f_from_coro_f + async def test_response_class_from_body(self): + await self._test_response_class( "foo", b"\n.", HtmlResponse, ) - def test_get_duplicate_header(self): - def _test(response): - assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] - + @deferred_f_from_coro_f + async def test_get_duplicate_header(self): request = Request(self.getURL("duplicate-header")) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] class TestHttp11Base(TestHttpBase): """HTTP 1.1 test case""" - def test_download_without_maxsize_limit(self): + @deferred_f_from_coro_f + async def test_download_without_maxsize_limit(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"0123456789" - def test_response_class_choosing_request(self): + @deferred_f_from_coro_f + async def test_response_class_choosing_request(self): """Tests choosing of correct response type in case of Content-Type is empty but body contains text. """ body = b"Some plain text\ndata with tabs\t and null bytes\0" - - def _test_type(response): - assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck - request = Request(self.getURL("nocontenttype"), body=body) - d = self.download_request(request, Spider("foo")) - d.addCallback(_test_type) - return d + response = await self.download_request(request, Spider("foo")) + assert type(response) is TextResponse # pylint: disable=unidiomatic-typecheck - @defer.inlineCallbacks - def test_download_with_maxsize(self): + @deferred_f_from_coro_f + async def test_download_with_maxsize(self): request = Request(self.getURL("file")) # 10 is minimal size for this request and the limit is only counted on # response body. (regardless of headers) - d = self.download_request(request, Spider("foo", download_maxsize=10)) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - yield d + response = await self.download_request( + request, Spider("foo", download_maxsize=10) + ) + assert response.body == b"0123456789" - d = self.download_request(request, Spider("foo", download_maxsize=9)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request(request, Spider("foo", download_maxsize=9)) - @defer.inlineCallbacks - def test_download_with_maxsize_very_large_file(self): + @deferred_f_from_coro_f + async def test_download_with_maxsize_very_large_file(self): from twisted.internet import reactor # TODO: the logger check is specific to scrapy.core.downloader.handlers.http11 @@ -384,8 +379,10 @@ class TestHttp11Base(TestHttpBase): def check(logger): logger.warning.assert_called_once_with(mock.ANY, mock.ANY) - d = self.download_request(request, Spider("foo", download_maxsize=1500)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request( + request, Spider("foo", download_maxsize=1500) + ) # As the error message is logged in the dataReceived callback, we # have to give a bit of time to the reactor to process the queue @@ -393,84 +390,81 @@ class TestHttp11Base(TestHttpBase): d = defer.Deferred() d.addCallback(check) reactor.callLater(0.1, d.callback, logger) - yield d + await maybe_deferred_to_future(d) - @defer.inlineCallbacks - def test_download_with_maxsize_per_req(self): + @deferred_f_from_coro_f + async def test_download_with_maxsize_per_req(self): meta = {"download_maxsize": 2} request = Request(self.getURL("file"), meta=meta) - d = self.download_request(request, Spider("foo")) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request(request, Spider("foo")) - @defer.inlineCallbacks - def test_download_with_small_maxsize_per_spider(self): + @deferred_f_from_coro_f + async def test_download_with_small_maxsize_per_spider(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo", download_maxsize=2)) - yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + with pytest.raises((defer.CancelledError, error.ConnectionAborted)): + await self.download_request(request, Spider("foo", download_maxsize=2)) - def test_download_with_large_maxsize_per_spider(self): + @deferred_f_from_coro_f + async def test_download_with_large_maxsize_per_spider(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo", download_maxsize=100)) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d + response = await self.download_request( + request, Spider("foo", download_maxsize=100) + ) + assert response.body == b"0123456789" - def test_download_chunked_content(self): + @deferred_f_from_coro_f + async def test_download_chunked_content(self): request = Request(self.getURL("chunked")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"chunked content\n") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"chunked content\n" - def test_download_broken_content_cause_data_loss(self, url="broken"): + @deferred_f_from_coro_f + async def test_download_broken_content_cause_data_loss( + self, url: str = "broken" + ) -> None: # TODO: this one checks for Twisted-specific exceptions request = Request(self.getURL(url)) - d = self.download_request(request, Spider("foo")) - - def checkDataLoss(failure): - if failure.check(ResponseFailed) and any( - r.check(_DataLoss) for r in failure.value.reasons - ): - return None - return failure - - d.addCallback(lambda _: self.fail("No DataLoss exception")) - d.addErrback(checkDataLoss) - return d + with pytest.raises(ResponseFailed) as exc_info: + await self.download_request(request, Spider("foo")) + assert any(r.check(_DataLoss) for r in exc_info.value.reasons) def test_download_broken_chunked_content_cause_data_loss(self): return self.test_download_broken_content_cause_data_loss("broken-chunked") - def test_download_broken_content_allow_data_loss(self, url="broken"): + @deferred_f_from_coro_f + async def test_download_broken_content_allow_data_loss( + self, url: str = "broken" + ) -> None: request = Request(self.getURL(url), meta={"download_fail_on_dataloss": False}) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ["dataloss"]) - return d + response = await self.download_request(request, Spider("foo")) + assert response.flags == ["dataloss"] def test_download_broken_chunked_content_allow_data_loss(self): return self.test_download_broken_content_allow_data_loss("broken-chunked") - def test_download_broken_content_allow_data_loss_via_setting(self, url="broken"): + @deferred_f_from_coro_f + async def test_download_broken_content_allow_data_loss_via_setting( + self, url: str = "broken" + ) -> None: crawler = get_crawler(settings_dict={"DOWNLOAD_FAIL_ON_DATALOSS": False}) download_handler = build_from_crawler(self.download_handler_cls, crawler) request = Request(self.getURL(url)) - d = download_handler.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.flags) - d.addCallback(self.assertEqual, ["dataloss"]) - return d + response = await maybe_deferred_to_future( + download_handler.download_request(request, Spider("foo")) + ) + assert response.flags == ["dataloss"] def test_download_broken_chunked_content_allow_data_loss_via_setting(self): return self.test_download_broken_content_allow_data_loss_via_setting( "broken-chunked" ) - def test_protocol(self): + @deferred_f_from_coro_f + async def test_protocol(self): request = Request(self.getURL("host"), method="GET") - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.protocol) - d.addCallback(self.assertEqual, "HTTP/1.1") - return d + response = await self.download_request(request, Spider("foo")) + assert response.protocol == "HTTP/1.1" class TestHttps11Base(TestHttp11Base): @@ -481,8 +475,8 @@ class TestHttps11Base(TestHttp11Base): 'subject "/C=IE/O=Scrapy/CN=localhost"' ) - @defer.inlineCallbacks - def test_tls_logging(self): + @deferred_f_from_coro_f + async def test_tls_logging(self): crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True} ) @@ -490,15 +484,15 @@ class TestHttps11Base(TestHttp11Base): try: with LogCapture() as log_capture: request = Request(self.getURL("file")) - d = download_handler.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - yield d + response = await maybe_deferred_to_future( + download_handler.download_request(request, Spider("foo")) + ) + assert response.body == b"0123456789" log_capture.check_present( ("scrapy.core.downloader.tls", "DEBUG", self.tls_log_message) ) finally: - yield download_handler.close() + await maybe_deferred_to_future(maybeDeferred(download_handler.close)) class TestSimpleHttpsBase(unittest.TestCase, ABC): @@ -536,7 +530,6 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC): settings_dict = None crawler = get_crawler(settings_dict=settings_dict) self.download_handler = build_from_crawler(self.download_handler_cls, crawler) - self.download_request = self.download_handler.download_request @defer.inlineCallbacks def tearDown(self): @@ -548,12 +541,16 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC): def getURL(self, path): return f"https://{self.host}:{self.portno}/{path}" - def test_download(self): + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + + @deferred_f_from_coro_f + async def test_download(self): request = Request(self.getURL("file")) - d = self.download_request(request, Spider("foo")) - d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b"0123456789") - return d + response = await self.download_request(request, Spider("foo")) + assert response.body == b"0123456789" class TestHttpsWrongHostnameBase(TestSimpleHttpsBase): @@ -604,25 +601,29 @@ class TestHttpMockServerBase(unittest.TestCase, ABC): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks - def test_download_with_content_length(self): + @deferred_f_from_coro_f + async def test_download_with_content_length(self): crawler = get_crawler(SingleRequestSpider, self.settings_dict) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it - yield crawler.crawl( - seed=Request( - url=self.mockserver.url("/partial", is_secure=self.is_secure), - meta={"download_maxsize": 1000}, + await maybe_deferred_to_future( + crawler.crawl( + seed=Request( + url=self.mockserver.url("/partial", is_secure=self.is_secure), + meta={"download_maxsize": 1000}, + ) ) ) failure = crawler.spider.meta["failure"] assert isinstance(failure.value, defer.CancelledError) - @defer.inlineCallbacks - def test_download(self): + @deferred_f_from_coro_f + async def test_download(self): crawler = get_crawler(SingleRequestSpider, self.settings_dict) - yield crawler.crawl( - seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) + await maybe_deferred_to_future( + crawler.crawl( + seed=Request(url=self.mockserver.url("", is_secure=self.is_secure)) + ) ) failure = crawler.spider.meta.get("failure") assert failure is None @@ -663,7 +664,6 @@ class TestHttpProxyBase(unittest.TestCase, ABC): self.download_handler = build_from_crawler( self.download_handler_cls, get_crawler() ) - self.download_request = self.download_handler.download_request @defer.inlineCallbacks def tearDown(self): @@ -674,42 +674,44 @@ class TestHttpProxyBase(unittest.TestCase, ABC): def getURL(self, path): return f"http://127.0.0.1:{self.portno}/{path}" - def test_download_with_proxy(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == self.expected_http_proxy_request_body + async def download_request(self, request: Request, spider: Spider) -> Response: + return await maybe_deferred_to_future( + self.download_handler.download_request(request, spider) + ) + @deferred_f_from_coro_f + async def test_download_with_proxy(self): http_proxy = self.getURL("") request = Request("http://example.com", meta={"proxy": http_proxy}) - return self.download_request(request, Spider("foo")).addCallback(_test) - - def test_download_without_proxy(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == b"/path/to/resource" + response = await self.download_request(request, Spider("foo")) + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body + @deferred_f_from_coro_f + async def test_download_without_proxy(self): request = Request(self.getURL("path/to/resource")) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.status == 200 + assert response.url == request.url + assert response.body == b"/path/to/resource" - @defer.inlineCallbacks - def test_download_with_proxy_https_timeout(self): + @deferred_f_from_coro_f + async def test_download_with_proxy_https_timeout(self): if NON_EXISTING_RESOLVABLE: pytest.skip("Non-existing hosts are resolvable") http_proxy = self.getURL("") domain = "https://no-such-domain.nosuch" request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) - d = self.download_request(request, Spider("foo")) - timeout = yield self.assertFailure(d, error.TimeoutError) - assert domain in timeout.osError - - def test_download_with_proxy_without_http_scheme(self): - def _test(response): - assert response.status == 200 - assert response.url == request.url - assert response.body == self.expected_http_proxy_request_body + with pytest.raises(error.TimeoutError) as exc_info: + await self.download_request(request, Spider("foo")) + assert domain in exc_info.value.osError + @deferred_f_from_coro_f + async def test_download_with_proxy_without_http_scheme(self): http_proxy = self.getURL("").replace("http://", "") request = Request("http://example.com", meta={"proxy": http_proxy}) - return self.download_request(request, Spider("foo")).addCallback(_test) + response = await self.download_request(request, Spider("foo")) + assert response.status == 200 + assert response.url == request.url + assert response.body == self.expected_http_proxy_request_body From 9cc23641ccc988d1f623902397496db5d9fe499b Mon Sep 17 00:00:00 2001 From: Rodrigosnrocha <83819959+Rodrigosnrocha@users.noreply.github.com> Date: Mon, 2 Jun 2025 17:00:17 +0200 Subject: [PATCH 1751/2083] Deprecate _parse_response and implement parse_with_rules (#6804) --- docs/conf.py | 1 - pyproject.toml | 3 +++ scrapy/spiders/crawl.py | 32 +++++++++++++++++++++++++++--- tests/test_spider.py | 44 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1167ce050..493a62976 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -3,7 +3,6 @@ # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -# pylint: disable=import-error import os import sys from collections.abc import Sequence diff --git a/pyproject.toml b/pyproject.toml index 871da8020..02ab7858d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,6 +161,9 @@ extension-pkg-allow-list=[ enable = [ "useless-suppression", ] +# Make INFO checks like useless-suppression also cause pylint to return a +# non-zero exit code. +fail-on = "I" disable = [ # Ones we want to ignore "attribute-defined-outside-init", diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 171d8479c..f44f70e40 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,6 +8,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import copy +import warnings from collections.abc import AsyncIterator, Awaitable, Callable from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast @@ -18,6 +19,8 @@ from scrapy.link import Link from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen +from scrapy.utils.deprecate import method_is_overridden +from scrapy.utils.python import global_object_name from scrapy.utils.spider import iterate_spider_output if TYPE_CHECKING: @@ -95,9 +98,17 @@ class CrawlSpider(Spider): def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) self._compile_rules() + if method_is_overridden(self.__class__, CrawlSpider, "_parse_response"): + warnings.warn( + f"The CrawlSpider._parse_response method, which the " + f"{global_object_name(self.__class__)} class overrides, is " + f"deprecated: it will be removed in future Scrapy releases. " + f"Please override the CrawlSpider.parse_with_rules method " + f"instead." + ) def _parse(self, response: Response, **kwargs: Any) -> Any: - return self._parse_response( + return self.parse_with_rules( response=response, callback=self.parse_start_url, cb_kwargs=kwargs, @@ -137,7 +148,7 @@ class CrawlSpider(Spider): def _callback(self, response: Response, **cb_kwargs: Any) -> Any: rule = self._rules[cast(int, response.meta["rule"])] - return self._parse_response( + return self.parse_with_rules( response, cast("CallbackT", rule.callback), {**rule.cb_kwargs, **cb_kwargs}, @@ -150,7 +161,7 @@ class CrawlSpider(Spider): failure, cast(Callable[[Failure], Any], rule.errback) ) - async def _parse_response( + async def parse_with_rules( self, response: Response, callback: CallbackT | None, @@ -171,6 +182,21 @@ class CrawlSpider(Spider): for request_or_item in self._requests_to_follow(response): yield request_or_item + def _parse_response( + self, + response: Response, + callback: CallbackT | None, + cb_kwargs: dict[str, Any], + follow: bool = True, + ) -> AsyncIterator[Any]: + warnings.warn( + "The CrawlSpider._parse_response method is deprecated: " + "it will be removed in future Scrapy releases. " + "Please use the CrawlSpider.parse_with_rules method instead.", + stacklevel=2, + ) + return self.parse_with_rules(response, callback, cb_kwargs, follow) + def _handle_failure( self, failure: Failure, errback: Callable[[Failure], Any] | None ) -> Iterable[Any]: diff --git a/tests/test_spider.py b/tests/test_spider.py index b4aa649a3..4e4a99638 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -476,6 +476,50 @@ class TestCrawlSpider(TestSpider): assert "Error while reading start items and requests" in str(log) assert "did you miss an 's'?" in str(log) + def test_parse_response_use(self): + class _CrawlSpider(CrawlSpider): + name = "test" + start_urls = "https://www.example.com" + _follow_links = False + + with warnings.catch_warnings(record=True) as w: + spider = _CrawlSpider() + assert len(w) == 0 + spider._parse_response( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 1 + + def test_parse_response_override(self): + class _CrawlSpider(CrawlSpider): + def _parse_response(self, response, callback, cb_kwargs, follow=True): + pass + + name = "test" + start_urls = "https://www.example.com" + _follow_links = False + + with warnings.catch_warnings(record=True) as w: + assert len(w) == 0 + spider = _CrawlSpider() + assert len(w) == 1 + spider._parse_response( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 1 + + def test_parse_with_rules(self): + class _CrawlSpider(CrawlSpider): + name = "test" + start_urls = "https://www.example.com" + + with warnings.catch_warnings(record=True) as w: + spider = _CrawlSpider() + spider.parse_with_rules( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 0 + class TestSitemapSpider(TestSpider): spider_class = SitemapSpider From d400aa3e2d8c5e3d1a7d8a483fb1a5a6f1c66d50 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Jun 2025 14:19:15 +0500 Subject: [PATCH 1752/2083] Add _parallel_asyncio(). (#6852) --- scrapy/core/scraper.py | 30 ++++++++++-- scrapy/utils/asyncio.py | 54 +++++++++++++++++++++ tests/test_utils_asyncio.py | 93 ++++++++++++++++++++++++++++++++++++- tests/test_utils_defer.py | 49 +++++++++++++++++-- 4 files changed, 217 insertions(+), 9 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 9fc1d20ed..9fd68bce5 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -21,6 +21,7 @@ from scrapy.exceptions import ( ScrapyDeprecationWarning, ) from scrapy.http import Request, Response +from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available from scrapy.utils.defer import ( _defer_sleep, aiter_errback, @@ -328,11 +329,21 @@ class Scraper: response: Response, ) -> None: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" + it: Iterable[_T] | AsyncIterator[_T] + if is_asyncio_available(): + if isinstance(result, AsyncIterator): + it = aiter_errback(result, self.handle_spider_error, request, response) + else: + it = iter_errback(result, self.handle_spider_error, request, response) + await _parallel_asyncio( + it, self.concurrent_items, self._process_spidermw_output_async, response + ) + return if isinstance(result, AsyncIterator): - ait = aiter_errback(result, self.handle_spider_error, request, response) + it = aiter_errback(result, self.handle_spider_error, request, response) await maybe_deferred_to_future( parallel_async( - ait, + it, self.concurrent_items, self._process_spidermw_output, response, @@ -349,8 +360,19 @@ class Scraper: ) ) - @deferred_f_from_coro_f - async def _process_spidermw_output(self, output: Any, response: Response) -> None: + def _process_spidermw_output( + self, output: Any, response: Response + ) -> Deferred[None]: + """Process each Request/Item (given in the output parameter) returned + from the given spider. + + Items are sent to the item pipelines, requests are scheduled. + """ + return deferred_from_coro(self._process_spidermw_output_async(output, response)) + + async def _process_spidermw_output_async( + self, output: Any, response: Response + ) -> None: """Process each Request/Item (given in the output parameter) returned from the given spider. diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 4469369fa..a3f27bcc9 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -1,7 +1,23 @@ """Utilities related to asyncio and its support in Scrapy.""" +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Any, TypeVar + +from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable, Coroutine, Iterable + + # typing.Concatenate and typing.ParamSpec require Python 3.10 + from typing_extensions import Concatenate, ParamSpec + + _P = ParamSpec("_P") + +_T = TypeVar("_T") + def is_asyncio_available() -> bool: """Check if it's possible to call asyncio code that relies on the asyncio event loop. @@ -36,3 +52,41 @@ def is_asyncio_available() -> bool: ) return is_asyncio_reactor_installed() + + +async def _parallel_asyncio( + iterable: Iterable[_T] | AsyncIterator[_T], + count: int, + callable: Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]], + *args: _P.args, + **kwargs: _P.kwargs, +) -> None: + """Execute a callable over the objects in the given iterable, in parallel, + using no more than ``count`` concurrent calls. + + This function is only used in + :meth:`scrapy.core.scraper.Scraper.handle_spider_output_async` and so it + assumes that neither *callable* nor iterating *iterable* will raise an + exception. + """ + queue: asyncio.Queue[_T | None] = asyncio.Queue() + + async def worker() -> None: + while True: + item = await queue.get() + if item is None: + break + try: + await callable(item, *args, **kwargs) + finally: + queue.task_done() + + async def fill_queue() -> None: + async for item in as_async_generator(iterable): + await queue.put(item) + for _ in range(count): + await queue.put(None) + + fill_task = asyncio.create_task(fill_queue()) + work_tasks = [asyncio.create_task(worker()) for _ in range(count)] + await asyncio.wait([fill_task, *work_tasks]) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index fe44748f9..6c47965a3 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,6 +1,18 @@ -import pytest +from __future__ import annotations -from scrapy.utils.asyncio import is_asyncio_available +import asyncio +import random +from typing import TYPE_CHECKING + +import pytest +from twisted.trial import unittest + +from scrapy.utils.asyncgen import as_async_generator +from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available +from scrapy.utils.defer import deferred_f_from_coro_f + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator @pytest.mark.usefixtures("reactor_pytest") @@ -8,3 +20,80 @@ class TestAsyncio: def test_is_asyncio_available(self): # the result should depend only on the pytest --reactor argument assert is_asyncio_available() == (self.reactor_pytest != "default") + + +@pytest.mark.only_asyncio +class TestParallelAsyncio(unittest.TestCase): + """Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync.""" + + CONCURRENT_ITEMS = 50 + + @staticmethod + async def callable(o: int, results: list[int]) -> None: + if random.random() < 0.4: + # simulate async processing + await asyncio.sleep(random.random() / 8) + # simulate trivial sync processing + results.append(o) + + async def callable_wrapped( + self, + o: int, + results: list[int], + parallel_count: list[int], + max_parallel_count: list[int], + ) -> None: + parallel_count[0] += 1 + max_parallel_count[0] = max(max_parallel_count[0], parallel_count[0]) + await self.callable(o, results) + assert parallel_count[0] > 0, parallel_count[0] + parallel_count[0] -= 1 + + @staticmethod + def get_async_iterable(length: int) -> AsyncGenerator[int, None]: + # simulate a simple callback without delays between results + return as_async_generator(range(length)) + + @staticmethod + async def get_async_iterable_with_delays(length: int) -> AsyncGenerator[int, None]: + # simulate a callback with delays between some of the results + for i in range(length): + if random.random() < 0.1: + await asyncio.sleep(random.random() / 20) + yield i + + @deferred_f_from_coro_f + async def test_simple(self): + for length in [20, 50, 100]: + parallel_count = [0] + max_parallel_count = [0] + results = [] + ait = self.get_async_iterable(length) + await _parallel_asyncio( + ait, + self.CONCURRENT_ITEMS, + self.callable_wrapped, + results, + parallel_count, + max_parallel_count, + ) + assert list(range(length)) == sorted(results) + assert max_parallel_count[0] <= self.CONCURRENT_ITEMS + + @deferred_f_from_coro_f + async def test_delays(self): + for length in [20, 50, 100]: + parallel_count = [0] + max_parallel_count = [0] + results = [] + ait = self.get_async_iterable_with_delays(length) + await _parallel_asyncio( + ait, + self.CONCURRENT_ITEMS, + self.callable_wrapped, + results, + parallel_count, + max_parallel_count, + ) + assert list(range(length)) == sorted(results) + assert max_parallel_count[0] <= self.CONCURRENT_ITEMS diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index c565c1c4e..98962f74c 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -164,7 +164,7 @@ class TestAsyncDefTestsuite(unittest.TestCase): raise RuntimeError("This is expected to be raised") -class TestAsyncCooperator(unittest.TestCase): +class TestParallelAsync(unittest.TestCase): """This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage. parallel_async is called with the results of a callback (so an iterable of items, requests and None, @@ -194,6 +194,27 @@ class TestAsyncCooperator(unittest.TestCase): results.append(o) return None + def callable_wrapped( + self, + o: int, + results: list[int], + parallel_count: list[int], + max_parallel_count: list[int], + ) -> Deferred[None] | None: + parallel_count[0] += 1 + max_parallel_count[0] = max(max_parallel_count[0], parallel_count[0]) + dfd = self.callable(o, results) + + def decrement(_: Any = None) -> None: + assert parallel_count[0] > 0, parallel_count[0] + parallel_count[0] -= 1 + + if dfd is not None: + dfd.addBoth(decrement) + else: + decrement() + return dfd + @staticmethod def get_async_iterable(length: int) -> AsyncGenerator[int, None]: # simulate a simple callback without delays between results @@ -215,20 +236,42 @@ class TestAsyncCooperator(unittest.TestCase): @inlineCallbacks def test_simple(self): for length in [20, 50, 100]: + parallel_count = [0] + max_parallel_count = [0] results = [] ait = self.get_async_iterable(length) - dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) + dl = parallel_async( + ait, + self.CONCURRENT_ITEMS, + self.callable_wrapped, + results, + parallel_count, + max_parallel_count, + ) yield dl assert list(range(length)) == sorted(results) + assert parallel_count[0] == 0 + assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0] @inlineCallbacks def test_delays(self): for length in [20, 50, 100]: + parallel_count = [0] + max_parallel_count = [0] results = [] ait = self.get_async_iterable_with_delays(length) - dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) + dl = parallel_async( + ait, + self.CONCURRENT_ITEMS, + self.callable_wrapped, + results, + parallel_count, + max_parallel_count, + ) yield dl assert list(range(length)) == sorted(results) + assert parallel_count[0] == 0 + assert max_parallel_count[0] <= self.CONCURRENT_ITEMS, max_parallel_count[0] class TestDeferredFromCoro(unittest.TestCase): From 3aa5e757871c6288f0a1e7031ad74b48d6321f5c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Jun 2025 17:47:52 +0500 Subject: [PATCH 1753/2083] Use AsyncCrawlerProcess in commands. (#6845) * Use AsyncCrawlerProcess in commands. * Ignore coverage of abstract methods. * Address feedback. --- docs/news.rst | 12 ++ docs/topics/commands.rst | 38 +++++ docs/topics/settings.rst | 20 +++ pyproject.toml | 6 +- scrapy/cmdline.py | 10 +- scrapy/commands/__init__.py | 4 +- scrapy/commands/crawl.py | 20 +-- scrapy/commands/fetch.py | 2 - scrapy/commands/genspider.py | 1 - scrapy/commands/runspider.py | 1 - scrapy/commands/settings.py | 1 - scrapy/commands/shell.py | 1 - scrapy/commands/startproject.py | 1 - scrapy/crawler.py | 21 ++- scrapy/settings/default_settings.py | 2 + tests/test_command_crawl.py | 25 ++++ tests/test_command_runspider.py | 17 ++- tests/test_command_shell.py | 4 +- tests/test_commands.py | 216 ++++++++++++++++++++++++++++ tests/test_utils_defer.py | 6 +- 20 files changed, 370 insertions(+), 38 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index ef3b549e7..d3e6c6774 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,18 @@ Release notes ============= +Scrapy VERSION (unreleased) +--------------------------- + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- If you set the :setting:`TWISTED_REACTOR` setting to a :ref:`non-asyncio + value ` at the :ref:`spider level `, you + may now need to set the :setting:`FORCE_CRAWLER_PROCESS` setting to + ``True`` when running Scrapy via :ref:`its command-line tool + ` to avoid a reactor mismatch exception. + .. _release-2.13.1: Scrapy 2.13.1 (2025-05-28) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 6ffb8ae93..4994fe1d6 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -587,6 +587,44 @@ bench Run a quick benchmark test. :ref:`benchmarking`. +.. _topics-commands-crawlerprocess: + +Commands that run a crawl +========================= + +Many commands need to run a crawl of some kind, running either a user-provided +spider or a special internal one: + +* :command:`bench` +* :command:`check` +* :command:`crawl` +* :command:`fetch` +* :command:`parse` +* :command:`runspider` +* :command:`shell` +* :command:`view` + +They use an internal instance of :class:`scrapy.crawler.AsyncCrawlerProcess` or +:class:`scrapy.crawler.CrawlerProcess` for this. In most cases this detail +shouldn't matter to the user running the command, but when the user :ref:`needs +a non-default Twisted reactor `, it may be important. + +Scrapy decides which of these two classes to use based on the value of the +:setting:`TWISTED_REACTOR` setting. If the setting value is the default one +(``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``), +:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise +:class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings +` are not taken into account when doing this, as they are +loaded after this decision is made. This may cause an error if the +project-level setting is set to :ref:`the asyncio reactor ` +(:ref:`explicitly ` or :ref:`by using the Scrapy default +`) and :ref:`the setting of the spider being run +` is set to :ref:`a different one `, because +:class:`~scrapy.crawler.AsyncCrawlerProcess` only supports the asyncio reactor. +In this case you should set the :setting:`FORCE_CRAWLER_PROCESS` setting to +``True`` (at the project level or via the command line) so that Scrapy uses +:class:`~scrapy.crawler.CrawlerProcess` which supports all reactors. + Custom project commands ======================= diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 65f2e5ebd..2a1be5f88 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1263,6 +1263,26 @@ FEED_STORAGE_GCS_ACL The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_. +.. setting:: FORCE_CRAWLER_PROCESS + +FORCE_CRAWLER_PROCESS +--------------------- + +Default: ``False`` + +If ``False``, :ref:`Scrapy commands that need a CrawlerProcess +` will decide between using +:class:`scrapy.crawler.AsyncCrawlerProcess` and +:class:`scrapy.crawler.CrawlerProcess` based on the value of the +:setting:`TWISTED_REACTOR` setting, but ignoring its value in :ref:`per-spider +settings `. + +If ``True``, these commands will always use +:class:`~scrapy.crawler.CrawlerProcess`. + +Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a +non-default value in :ref:`per-spider settings `. + .. setting:: FTP_PASSIVE_MODE FTP_PASSIVE_MODE diff --git a/pyproject.toml b/pyproject.toml index 02ab7858d..8ec0c1056 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,8 +147,10 @@ source = [ ] [tool.coverage.report] -# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 -exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"] +exclude_also = [ + "if TYPE_CHECKING:", + "@(abc\\.)?abstractmethod", +] [tool.pylint.MASTER] persistent = "no" diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 81e507a4e..3d448532b 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -10,11 +10,12 @@ from typing import TYPE_CHECKING import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter -from scrapy.crawler import CrawlerProcess +from scrapy.crawler import AsyncCrawlerProcess, CrawlerProcess from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect +from scrapy.utils.reactor import _asyncio_reactor_path if TYPE_CHECKING: from collections.abc import Callable, Iterable @@ -202,7 +203,12 @@ def execute(argv: list[str] | None = None, settings: Settings | None = None) -> _run_print_help(parser, cmd.process_options, args, opts) if cmd.requires_crawler_process: - cmd.crawler_process = CrawlerProcess(settings) + if settings[ + "TWISTED_REACTOR" + ] == _asyncio_reactor_path and not settings.getbool("FORCE_CRAWLER_PROCESS"): + cmd.crawler_process = AsyncCrawlerProcess(settings) + else: + cmd.crawler_process = CrawlerProcess(settings) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 2818ead77..4ce070e6e 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -18,14 +18,14 @@ from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli if TYPE_CHECKING: from collections.abc import Iterable - from scrapy.crawler import Crawler, CrawlerProcess + from scrapy.crawler import Crawler, CrawlerProcessBase from scrapy.settings import Settings class ScrapyCommand: requires_project: bool = False requires_crawler_process: bool = True - crawler_process: CrawlerProcess | None = None # set in scrapy.cmdline + crawler_process: CrawlerProcessBase | None = None # set in scrapy.cmdline # default settings to be used for this command instead of global defaults default_settings: dict[str, Any] = {} diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 184bd5ca4..866ba9f6b 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,8 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, cast - -from twisted.python.failure import Failure +from typing import TYPE_CHECKING from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -30,17 +28,7 @@ class Command(BaseRunSpiderCommand): spname = args[0] assert self.crawler_process - crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) - - if getattr(crawl_defer, "result", None) is not None and issubclass( - cast(Failure, crawl_defer.result).type, Exception - ): + self.crawler_process.crawl(spname, **opts.spargs) + self.crawler_process.start() + if self.crawler_process.bootstrap_failed: self.exitcode = 1 - else: - self.crawler_process.start() - - if self.crawler_process.bootstrap_failed or ( - hasattr(self.crawler_process, "has_exception") - and self.crawler_process.has_exception - ): - self.exitcode = 1 diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index ef6e13de2..e5eedffb5 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -18,8 +18,6 @@ if TYPE_CHECKING: class Command(ScrapyCommand): - requires_project = False - def syntax(self) -> str: return "[options] " diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index c4abfc4c9..0e90c3188 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -46,7 +46,6 @@ def verify_url_scheme(url: str) -> str: class Command(ScrapyCommand): - requires_project = False requires_crawler_process = False default_settings = {"LOG_ENABLED": False} diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 3e826456e..eeb1303e2 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -30,7 +30,6 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType: class Command(BaseRunSpiderCommand): - requires_project = False default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader} def syntax(self) -> str: diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index e63031f2d..704cc500d 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -6,7 +6,6 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): - requires_project = False requires_crawler_process = False default_settings = {"LOG_ENABLED": False} diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 9dabfcd9c..eedaeb263 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -22,7 +22,6 @@ if TYPE_CHECKING: class Command(ScrapyCommand): - requires_project = False default_settings = { "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", "KEEP_ALIVE": True, diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 323979193..8f4427580 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -33,7 +33,6 @@ def _make_writable(path: Path) -> None: class Command(ScrapyCommand): - requires_project = False requires_crawler_process = False default_settings = {"LOG_ENABLED": False} diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 8e3223a5c..d6fb9972e 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -5,6 +5,7 @@ import contextlib import logging import pprint import signal +from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, TypeVar from twisted.internet.defer import ( @@ -42,7 +43,7 @@ from scrapy.utils.reactor import ( ) if TYPE_CHECKING: - from collections.abc import Generator, Iterable + from collections.abc import Awaitable, Generator, Iterable from scrapy.logformatter import LogFormatter from scrapy.statscollectors import StatsCollector @@ -321,7 +322,7 @@ class Crawler: return self._get_component(cls, self.engine.scraper.spidermw.middlewares) -class CrawlerRunnerBase: +class CrawlerRunnerBase(ABC): def __init__(self, settings: dict[str, Any] | Settings | None = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -364,6 +365,15 @@ class CrawlerRunnerBase: spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) + @abstractmethod + def crawl( + self, + crawler_or_spidercls: type[Spider] | str | Crawler, + *args: Any, + **kwargs: Any, + ) -> Awaitable[None]: + raise NotImplementedError + class CrawlerRunner(CrawlerRunnerBase): """ @@ -560,6 +570,12 @@ class CrawlerProcessBase(CrawlerRunnerBase): configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) + @abstractmethod + def start( + self, stop_after_crawl: bool = True, install_signal_handlers: bool = True + ) -> None: + raise NotImplementedError + def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor @@ -597,6 +613,7 @@ class CrawlerProcessBase(CrawlerRunnerBase): "after", "startup", install_shutdown_handlers, self._signal_shutdown ) + @abstractmethod def _stop_dfd(self) -> Deferred[Any]: raise NotImplementedError diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 01443fa17..4a27017a6 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -179,6 +179,8 @@ FEED_STORAGE_S3_ACL = "" FILES_STORE_S3_ACL = "private" FILES_STORE_GCS_ACL = "" +FORCE_CRAWLER_PROCESS = False + FTP_USER = "anonymous" FTP_PASSWORD = "guest" # noqa: S105 FTP_PASSIVE_MODE = True diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py index 3d5e17977..0ab0659b2 100644 --- a/tests/test_command_crawl.py +++ b/tests/test_command_crawl.py @@ -30,6 +30,11 @@ class MySpider(scrapy.Spider): """ log = self.get_log(spider_code) assert "[myspider] DEBUG: It works!" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Spider closed (finished)" in log def test_output(self): spider_code = """ @@ -91,3 +96,23 @@ class MySpider(scrapy.Spider): assert ( "error: Please use only one of -o/--output and -O/--overwrite-output" in log ) + + def test_default_reactor(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('It works!') + return + yield +""" + log = self.get_log(spider_code, args=("-s", "TWISTED_REACTOR=")) + assert "[myspider] DEBUG: It works!" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + assert "Spider closed (finished)" in log diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 664de16f8..c57c09249 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -65,8 +65,10 @@ class BadSpider(scrapy.Spider): def test_runspider(self): log = self.get_log(self.debug_log_spider) assert "DEBUG: It Works!" in log - assert "INFO: Spider opened" in log - assert "INFO: Closing spider (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) assert "INFO: Spider closed (finished)" in log def test_run_fail_spider(self): @@ -88,6 +90,17 @@ class BadSpider(scrapy.Spider): assert "DEBUG: It Works!" not in log assert "INFO: Spider opened" in log + def test_runspider_default_reactor(self): + log = self.get_log(self.debug_log_spider, args=("-s", "TWISTED_REACTOR=")) + assert "DEBUG: It Works!" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + assert "INFO: Spider opened" in log + assert "INFO: Closing spider (finished)" in log + assert "INFO: Spider closed (finished)" in log + def test_runspider_dnscache_disabled(self): # see https://github.com/scrapy/scrapy/issues/2811 # The spider below should not be able to connect to localhost:12345, diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 0f45a7ee8..8041e7cb1 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -7,6 +7,7 @@ from pexpect.popen_spawn import PopenSpawn from twisted.internet import defer from twisted.trial import unittest +from scrapy.utils.reactor import _asyncio_reactor_path from tests import NON_EXISTING_RESOLVABLE, tests_datadir from tests.mockserver import MockServer from tests.utils.testproc import ProcessTest @@ -132,10 +133,9 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_shell_fetch_async(self): - reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" url = self.url("/html") code = f"fetch('{url}')" - args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] + args = ["-c", code, "--set", f"TWISTED_REACTOR={_asyncio_reactor_path}"] _, _, err = yield self.execute(args, check_code=True) assert b"RuntimeError: There is no current event loop in thread" not in err diff --git a/tests/test_commands.py b/tests/test_commands.py index 6e59f561d..8ca5d51e5 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -20,6 +20,7 @@ from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view from scrapy.settings import Settings from scrapy.utils.python import to_unicode +from scrapy.utils.reactor import _asyncio_reactor_path from scrapy.utils.test import get_testenv if TYPE_CHECKING: @@ -124,6 +125,221 @@ class TestCommandBase(TestProjectBase): self.env["SCRAPY_SETTINGS_MODULE"] = f"{self.project_name}.settings" +class TestCommandCrawlerProcess(TestCommandBase): + """Test that the command uses the expected kind of *CrawlerProcess + and produces expected errors when needed.""" + + name = "crawltest" + + NORMAL_MSG = "Type of self.crawler_process: " + ASYNC_MSG = ( + "Type of self.crawler_process: " + ) + + def setUp(self): + super().setUp() + (self.cwd / self.project_name / "commands").mkdir(exist_ok=True) + (self.cwd / self.project_name / "commands" / "__init__.py").touch() + (self.cwd / self.project_name / "commands" / f"{self.name}.py").write_text(""" +from scrapy.commands.crawl import Command + +class CrawlerProcessCrawlCommand(Command): + requires_project = True + + def run(self, args, opts): + print(f"Type of self.crawler_process: {type(self.crawler_process)}") + super().run(args, opts) +""") + + self._append_settings(f"COMMANDS_MODULE = '{self.project_name}.commands'\n") + + (self.cwd / self.project_name / "spiders" / "sp.py").write_text(""" +import scrapy + +class MySpider(scrapy.Spider): + name = 'sp' + + custom_settings = {} + + async def start(self): + self.logger.debug('It works!') + return + yield +""") + + (self.cwd / self.project_name / "spiders" / "aiosp.py").write_text(""" +import asyncio + +import scrapy + +class MySpider(scrapy.Spider): + name = 'aiosp' + + custom_settings = {} + + async def start(self): + await asyncio.sleep(0.01) + self.logger.debug('It works!') + return + yield +""") + + def _append_settings(self, text: str) -> None: + """Add text to the end of the project settings.py.""" + with (self.cwd / self.project_name / "settings.py").open( + "a", encoding="utf-8" + ) as f: + f.write(text) + + def _replace_custom_settings(self, spider_name: str, text: str) -> None: + """Replace custom_settings in the given spider file with the given text.""" + spider_path = self.cwd / self.project_name / "spiders" / f"{spider_name}.py" + with spider_path.open("r+", encoding="utf-8") as f: + content = f.read() + content = content.replace( + "custom_settings = {}", f"custom_settings = {text}" + ) + f.seek(0) + f.write(content) + f.truncate() + + def _assert_spider_works(self, msg: str, *args: str) -> None: + """The command uses the expected *CrawlerProcess, the spider works.""" + _, out, err = self.proc(self.name, *args) + assert msg in out, out + assert "It works!" in err, err + assert "Spider closed (finished)" in err, err + + def _assert_spider_asyncio_fail(self, msg: str, *args: str) -> None: + """The command uses the expected *CrawlerProcess, the spider fails to use asyncio.""" + _, out, err = self.proc(self.name, *args) + assert msg in out, out + assert "no running event loop" in err, err + + def test_project_settings(self): + """The reactor is set via the project default settings (to the asyncio value). + + AsyncCrawlerProcess, the asyncio reactor, both spiders work.""" + for spider in ["sp", "aiosp"]: + self._assert_spider_works(self.ASYNC_MSG, spider) + + def test_cmdline_asyncio(self): + """The reactor is set via the command line to the asyncio value. + AsyncCrawlerProcess, the asyncio reactor, both spiders work.""" + for spider in ["sp", "aiosp"]: + self._assert_spider_works( + self.ASYNC_MSG, spider, "-s", f"TWISTED_REACTOR={_asyncio_reactor_path}" + ) + + def test_project_settings_explicit_asyncio(self): + """The reactor explicitly is set via the project settings to the asyncio value. + + AsyncCrawlerProcess, the asyncio reactor, both spiders work.""" + self._append_settings(f"TWISTED_REACTOR = '{_asyncio_reactor_path}'\n") + + for spider in ["sp", "aiosp"]: + self._assert_spider_works(self.ASYNC_MSG, spider) + + def test_cmdline_empty(self): + """The reactor is set via the command line to the empty value. + + CrawlerProcess, the default reactor, only the normal spider works.""" + self._assert_spider_works(self.NORMAL_MSG, "sp", "-s", "TWISTED_REACTOR=") + self._assert_spider_asyncio_fail( + self.NORMAL_MSG, "aiosp", "-s", "TWISTED_REACTOR=" + ) + + def test_project_settings_empty(self): + """The reactor is set via the project settings to the empty value. + + CrawlerProcess, the default reactor, only the normal spider works.""" + self._append_settings("TWISTED_REACTOR = None\n") + + self._assert_spider_works(self.NORMAL_MSG, "sp") + self._assert_spider_asyncio_fail( + self.NORMAL_MSG, "aiosp", "-s", "TWISTED_REACTOR=" + ) + + def test_spider_settings_asyncio(self): + """The reactor is set via the spider settings to the asyncio value. + + AsyncCrawlerProcess, the asyncio reactor, both spiders work.""" + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + spider, f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}" + ) + self._assert_spider_works(self.ASYNC_MSG, spider) + + def test_spider_settings_asyncio_cmdline_empty(self): + """The reactor is set via the spider settings to the asyncio value + and via command line to the empty value. The command line value takes + precedence so the spider settings don't matter. + + CrawlerProcess, the default reactor, only the normal spider works.""" + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + spider, f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}" + ) + + self._assert_spider_works(self.NORMAL_MSG, "sp", "-s", "TWISTED_REACTOR=") + self._assert_spider_asyncio_fail( + self.NORMAL_MSG, "aiosp", "-s", "TWISTED_REACTOR=" + ) + + def test_project_empty_spider_settings_asyncio(self): + """The reactor is set via the project settings to the empty value + and via the spider settings to the asyncio value. CrawlerProcess is + chosen based on the project settings, but the asyncio reactor is chosen + based on the spider settings. + + CrawlerProcess, the asyncio reactor, both spiders work.""" + self._append_settings("TWISTED_REACTOR = None\n") + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + spider, f"{{'TWISTED_REACTOR': '{_asyncio_reactor_path}'}}" + ) + self._assert_spider_works(self.NORMAL_MSG, spider) + + def test_project_asyncio_spider_settings_select(self): + """The reactor is set via the project settings to the asyncio value + and via the spider settings to the select value. AsyncCrawlerProcess + is chosen based on the project settings, and the conflicting reactor + setting in the spider settings causes an exception. + + AsyncCrawlerProcess, the asyncio reactor, both spiders produce a + mismatched reactor exception.""" + self._append_settings(f"TWISTED_REACTOR = '{_asyncio_reactor_path}'\n") + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + spider, + "{'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor'}", + ) + _, out, err = self.proc(self.name, spider) + assert self.ASYNC_MSG in out, out + assert ( + "The installed reactor (twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + " does not match the requested one" + " (twisted.internet.selectreactor.SelectReactor)" + ) in err, err + + def test_project_asyncio_spider_settings_select_forced(self): + """The reactor is set via the project settings to the asyncio value + and via the spider settings to the select value, CrawlerProcess is + forced via the project settings. The reactor is chosen based on the + spider settings. + + CrawlerProcess, the select reactor, only the normal spider works.""" + self._append_settings("FORCE_CRAWLER_PROCESS = True\n") + for spider in ["sp", "aiosp"]: + self._replace_custom_settings( + spider, + "{'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor'}", + ) + + self._assert_spider_works(self.NORMAL_MSG, "sp") + self._assert_spider_asyncio_fail(self.NORMAL_MSG, "aiosp") + + class TestMiscCommands(TestCommandBase): def test_list(self): assert self.call("list") == 0 diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 98962f74c..372213319 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -299,7 +299,7 @@ class TestDeferredFromCoro(unittest.TestCase): @inlineCallbacks def test_coroutine_asyncio(self): async def coroutine() -> int: - await asyncio.sleep(0) + await asyncio.sleep(0.01) return 42 result = deferred_from_coro(coroutine()) @@ -379,7 +379,7 @@ class TestDeferredToFuture(unittest.TestCase): @deferred_f_from_coro_f async def test_wrapped_coroutine_asyncio(self): async def c_f() -> int: - await asyncio.sleep(0) + await asyncio.sleep(0.01) return 42 d = deferred_from_coro(c_f()) @@ -414,7 +414,7 @@ class TestMaybeDeferredToFutureAsyncio(unittest.TestCase): @deferred_f_from_coro_f async def test_wrapped_coroutine_asyncio(self): async def c_f() -> int: - await asyncio.sleep(0) + await asyncio.sleep(0.01) return 42 d = deferred_from_coro(c_f()) From 8fb8d2c6b8a83181ef6564fbc9afa9cfeea2bd05 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Jun 2025 23:21:33 +0500 Subject: [PATCH 1754/2083] Add AsyncioLoopingCall. (#6855) --- scrapy/core/downloader/__init__.py | 8 ++- scrapy/core/engine.py | 11 ++- scrapy/extensions/closespider.py | 5 +- scrapy/extensions/logstats.py | 12 ++-- scrapy/extensions/memusage.py | 16 +++-- scrapy/extensions/periodic_log.py | 14 ++-- scrapy/utils/asyncio.py | 106 ++++++++++++++++++++++++++++- tests/test_utils_asyncio.py | 48 ++++++++++++- 8 files changed, 195 insertions(+), 25 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 5468398aa..501c669ce 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -7,7 +7,6 @@ from datetime import datetime from time import time from typing import TYPE_CHECKING, Any, cast -from twisted.internet import task from twisted.internet.defer import Deferred, inlineCallbacks from scrapy import Request, Spider, signals @@ -15,6 +14,7 @@ from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.resolver import dnscache +from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.defer import ( deferred_from_coro, maybe_deferred_to_future, @@ -25,6 +25,8 @@ from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from collections.abc import Generator + from twisted.internet.task import LoopingCall + from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.settings import BaseSettings @@ -111,7 +113,9 @@ class Downloader: self.middleware: DownloaderMiddlewareManager = ( DownloaderMiddlewareManager.from_crawler(crawler) ) - self._slot_gc_loop: task.LoopingCall = task.LoopingCall(self._slot_gc) + self._slot_gc_loop: AsyncioLoopingCall | LoopingCall = create_looping_call( + self._slot_gc + ) self._slot_gc_loop.start(60) self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( "DOWNLOAD_SLOTS", {} diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 721c81d81..d9361a674 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -13,13 +13,16 @@ from traceback import format_exc from typing import TYPE_CHECKING, Any, cast from twisted.internet.defer import Deferred, inlineCallbacks, succeed -from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + create_looping_call, +) from scrapy.utils.defer import ( deferred_f_from_coro_f, deferred_from_coro, @@ -32,6 +35,8 @@ from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable, Generator + from twisted.internet.task import LoopingCall + from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler from scrapy.crawler import Crawler @@ -56,7 +61,9 @@ class _Slot: self.close_if_idle: bool = close_if_idle self.nextcall: CallLaterOnce[None] = nextcall self.scheduler: BaseScheduler = scheduler - self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule) + self.heartbeat: AsyncioLoopingCall | LoopingCall = create_looping_call( + nextcall.schedule + ) def add_request(self, request: Request) -> None: self.inprogress.add(request) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index dff8bc97e..a649a86e2 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.utils.asyncio import create_looping_call if TYPE_CHECKING: from twisted.python.failure import Failure @@ -118,9 +119,7 @@ class CloseSpider: task_no_item.stop() def spider_opened_no_item(self, spider: Spider) -> None: - from twisted.internet import task - - self.task_no_item = task.LoopingCall(self._count_items_produced, spider) + self.task_no_item = create_looping_call(self._count_items_produced, spider) self.task_no_item.start(self.timeout_no_item, now=False) logger.info( diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index f2e1f57b8..387cfddb3 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -3,12 +3,16 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING -from twisted.internet import task - from scrapy import Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + create_looping_call, +) if TYPE_CHECKING: + from twisted.internet.task import LoopingCall + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -29,7 +33,7 @@ class LogStats: self.stats: StatsCollector = stats self.interval: float = interval self.multiplier: float = 60.0 / self.interval - self.task: task.LoopingCall | None = None + self.task: AsyncioLoopingCall | LoopingCall | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -46,7 +50,7 @@ class LogStats: self.pagesprev: int = 0 self.itemsprev: int = 0 - self.task = task.LoopingCall(self.log, spider) + self.task = create_looping_call(self.log, spider) self.task.start(self.interval) def log(self, spider: Spider) -> None: diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index d7f810107..2ef322f1a 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -13,14 +13,18 @@ from importlib import import_module from pprint import pformat from typing import TYPE_CHECKING -from twisted.internet import task - from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + create_looping_call, +) from scrapy.utils.engine import get_engine_status if TYPE_CHECKING: + from twisted.internet.task import LoopingCall + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -66,16 +70,16 @@ class MemoryUsage: def engine_started(self) -> None: assert self.crawler.stats self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) - self.tasks: list[task.LoopingCall] = [] - tsk = task.LoopingCall(self.update) + self.tasks: list[AsyncioLoopingCall | LoopingCall] = [] + tsk = create_looping_call(self.update) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) if self.limit: - tsk = task.LoopingCall(self._check_limit) + tsk = create_looping_call(self._check_limit) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) if self.warning: - tsk = task.LoopingCall(self._check_warning) + tsk = create_looping_call(self._check_warning) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index f97577442..9158482fa 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -4,16 +4,20 @@ import logging from datetime import datetime, timezone from typing import TYPE_CHECKING, Any -from twisted.internet import task - from scrapy import Spider, signals from scrapy.exceptions import NotConfigured +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + create_looping_call, +) from scrapy.utils.serialize import ScrapyJSONEncoder if TYPE_CHECKING: - # typing.Self requires Python 3.11 from json import JSONEncoder + from twisted.internet.task import LoopingCall + + # typing.Self requires Python 3.11 from typing_extensions import Self from scrapy.crawler import Crawler @@ -37,7 +41,7 @@ class PeriodicLog: self.stats: StatsCollector = stats self.interval: float = interval self.multiplier: float = 60.0 / self.interval - self.task: task.LoopingCall | None = None + self.task: AsyncioLoopingCall | LoopingCall | None = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) self.ext_stats_include: list[str] = ext_stats.get("include", []) @@ -97,7 +101,7 @@ class PeriodicLog: self.delta_prev: dict[str, int | float] = {} self.stats_prev: dict[str, int | float] = {} - self.task = task.LoopingCall(self.log) + self.task = create_looping_call(self.log) self.task.start(self.interval) def log(self) -> None: diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index a3f27bcc9..cae2dc033 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -3,22 +3,30 @@ from __future__ import annotations import asyncio +import logging +import time +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable from typing import TYPE_CHECKING, Any, TypeVar +from twisted.internet.defer import Deferred +from twisted.internet.task import LoopingCall + from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Coroutine, Iterable - # typing.Concatenate and typing.ParamSpec require Python 3.10 from typing_extensions import Concatenate, ParamSpec _P = ParamSpec("_P") + _T = TypeVar("_T") +logger = logging.getLogger(__name__) + + def is_asyncio_available() -> bool: """Check if it's possible to call asyncio code that relies on the asyncio event loop. @@ -90,3 +98,97 @@ async def _parallel_asyncio( fill_task = asyncio.create_task(fill_queue()) work_tasks = [asyncio.create_task(worker()) for _ in range(count)] await asyncio.wait([fill_task, *work_tasks]) + + +class AsyncioLoopingCall: + """A simple implementation of a periodic call using asyncio, keeping + some API and behavior compatibility with the Twisted ``LoopingCall``. + + The function is called every *interval* seconds, independent of the finish + time of the previous call. If the function is still running when it's time + to call it again, calls are skipped until the function finishes. + + The function must not return a coroutine or a ``Deferred``. + """ + + def __init__(self, func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs): + self._func: Callable[_P, _T] = func + self._args: tuple[Any, ...] = args + self._kwargs: dict[str, Any] = kwargs + self._task: asyncio.Task | None = None + self.interval: float | None = None + self._start_time: float | None = None + + @property + def running(self) -> bool: + return self._start_time is not None + + def start(self, interval: float, now: bool = True) -> None: + """Start calling the function every *interval* seconds. + + :param interval: The interval in seconds between calls. + :type interval: float + + :param now: If ``True``, also call the function immediately. + :type now: bool + """ + if self.running: + raise RuntimeError("AsyncioLoopingCall already running") + + if interval <= 0: + raise ValueError("Interval must be greater than 0") + + self.interval = interval + self._start_time = time.time() + if now: + self._call() + loop = asyncio.get_event_loop() + self._task = loop.create_task(self._loop()) + + def _to_sleep(self) -> float: + """Return the time to sleep until the next call.""" + assert self.interval is not None + assert self._start_time is not None + now = time.time() + running_for = now - self._start_time + return self.interval - (running_for % self.interval) + + async def _loop(self) -> None: + """Run an infinite loop that calls the function periodically.""" + while self.running: + await asyncio.sleep(self._to_sleep()) + self._call() + + def stop(self) -> None: + """Stop the periodic calls.""" + self.interval = self._start_time = None + if self._task is not None: + self._task.cancel() + self._task = None + + def _call(self) -> None: + """Execute the function.""" + try: + result = self._func(*self._args, **self._kwargs) + except Exception: + logger.exception("Error calling the AsyncioLoopingCall function") + self.stop() + else: + if isinstance(result, (Coroutine, Deferred)): + self.stop() + raise TypeError( + "The AsyncioLoopingCall function must not return a coroutine or a Deferred" + ) + + +def create_looping_call( + func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs +) -> AsyncioLoopingCall | LoopingCall: + """Create an instance of a looping call class. + + This creates an instance of :class:`AsyncioLoopingCall` or + :class:`LoopingCall`, depending on whether asyncio support is available. + """ + if is_asyncio_available(): + return AsyncioLoopingCall(func, *args, **kwargs) + return LoopingCall(func, *args, **kwargs) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 6c47965a3..a6e52eb26 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -3,12 +3,18 @@ from __future__ import annotations import asyncio import random from typing import TYPE_CHECKING +from unittest import mock import pytest +from twisted.internet.defer import Deferred from twisted.trial import unittest from scrapy.utils.asyncgen import as_async_generator -from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + _parallel_asyncio, + is_asyncio_available, +) from scrapy.utils.defer import deferred_f_from_coro_f if TYPE_CHECKING: @@ -97,3 +103,43 @@ class TestParallelAsyncio(unittest.TestCase): ) assert list(range(length)) == sorted(results) assert max_parallel_count[0] <= self.CONCURRENT_ITEMS + + +@pytest.mark.only_asyncio +class TestAsyncioLoopingCall: + def test_looping_call(self): + func = mock.MagicMock() + looping_call = AsyncioLoopingCall(func) + looping_call.start(1, now=False) + assert looping_call.running + looping_call.stop() + assert not looping_call.running + assert not func.called + + def test_looping_call_now(self): + func = mock.MagicMock() + looping_call = AsyncioLoopingCall(func) + looping_call.start(1) + looping_call.stop() + assert func.called + + def test_looping_call_already_running(self): + looping_call = AsyncioLoopingCall(lambda: None) + looping_call.start(1) + with pytest.raises(RuntimeError): + looping_call.start(1) + looping_call.stop() + + def test_looping_call_interval(self): + looping_call = AsyncioLoopingCall(lambda: None) + with pytest.raises(ValueError, match="Interval must be greater than 0"): + looping_call.start(0) + with pytest.raises(ValueError, match="Interval must be greater than 0"): + looping_call.start(-1) + assert not looping_call.running + + def test_looping_call_bad_function(self): + looping_call = AsyncioLoopingCall(Deferred) + with pytest.raises(TypeError): + looping_call.start(0.1) + assert not looping_call.running From c6698b9fe8d3e1f27f2982b484ba69486a8cbc3d Mon Sep 17 00:00:00 2001 From: Mehraz Hossain Rumman <59512321+MehrazRumman@users.noreply.github.com> Date: Wed, 4 Jun 2025 17:02:29 +0600 Subject: [PATCH 1755/2083] fixing settings order (#6849) * fixing issue #6838 * Reorder some more settings. * Clarify the header. --------- Co-authored-by: Andrey Rakhmatullin --- scrapy/settings/default_settings.py | 146 ++++++++++++++-------------- 1 file changed, 71 insertions(+), 75 deletions(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 4a27017a6..7cd470f11 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -1,16 +1,16 @@ -""" -This module contains the default values for all settings used by Scrapy. +"""This module contains the default values for all settings used by Scrapy. For more information about these settings you can read the settings documentation in docs/topics/settings.rst Scrapy developers, if you add a setting here remember to: -* add it in alphabetical order +* add it in alphabetical order, with the exception that enabling flags and + other high-level settings for a group should come first in their group + and pairs like host/port and user/password should be in the usual order * group similar settings without leaving blank lines * add its documentation to the available settings documentation (docs/topics/settings.rst) - """ import sys @@ -31,10 +31,10 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 BOT_NAME = "scrapybot" -CLOSESPIDER_TIMEOUT = 0 -CLOSESPIDER_PAGECOUNT = 0 -CLOSESPIDER_ITEMCOUNT = 0 CLOSESPIDER_ERRORCOUNT = 0 +CLOSESPIDER_ITEMCOUNT = 0 +CLOSESPIDER_PAGECOUNT = 0 +CLOSESPIDER_TIMEOUT = 0 COMMANDS_MODULE = "" @@ -59,8 +59,8 @@ DEFAULT_REQUEST_HEADERS = { } DEPTH_LIMIT = 0 -DEPTH_STATS_VERBOSE = False DEPTH_PRIORITY = 0 +DEPTH_STATS_VERBOSE = False DNSCACHE_ENABLED = True DNSCACHE_SIZE = 10000 @@ -69,6 +69,8 @@ DNS_TIMEOUT = 60 DOWNLOAD_DELAY = 0 +DOWNLOAD_FAIL_ON_DATALOSS = True + DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { "data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler", @@ -79,18 +81,13 @@ DOWNLOAD_HANDLERS_BASE = { "ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler", } -DOWNLOAD_TIMEOUT = 180 # 3mins - DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m -DOWNLOAD_FAIL_ON_DATALOSS = True +DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOADER = "scrapy.core.downloader.Downloader" -DOWNLOADER_HTTPCLIENTFACTORY = ( - "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory" -) DOWNLOADER_CLIENTCONTEXTFACTORY = ( "scrapy.core.downloader.contextfactory.ScrapyClientContextFactory" ) @@ -99,8 +96,11 @@ DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT" DOWNLOADER_CLIENT_TLS_METHOD = "TLS" DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False -DOWNLOADER_MIDDLEWARES = {} +DOWNLOADER_HTTPCLIENTFACTORY = ( + "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory" +) +DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": 50, @@ -130,7 +130,6 @@ if sys.platform == "win32": EDITOR = "%s -m idlelib.idle" EXTENSIONS = {} - EXTENSIONS_BASE = { "scrapy.extensions.corestats.CoreStats": 0, "scrapy.extensions.telnet.TelnetConsole": 0, @@ -143,22 +142,11 @@ EXTENSIONS_BASE = { "scrapy.extensions.throttle.AutoThrottle": 0, } -FEED_TEMPDIR = None FEEDS = {} -FEED_URI_PARAMS = None # a function to extend uri arguments -FEED_STORE_EMPTY = True +FEED_EXPORT_BATCH_ITEM_COUNT = 0 FEED_EXPORT_ENCODING = None FEED_EXPORT_FIELDS = None -FEED_STORAGES = {} -FEED_STORAGES_BASE = { - "": "scrapy.extensions.feedexport.FileFeedStorage", - "file": "scrapy.extensions.feedexport.FileFeedStorage", - "ftp": "scrapy.extensions.feedexport.FTPFeedStorage", - "gs": "scrapy.extensions.feedexport.GCSFeedStorage", - "s3": "scrapy.extensions.feedexport.S3FeedStorage", - "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage", -} -FEED_EXPORT_BATCH_ITEM_COUNT = 0 +FEED_EXPORT_INDENT = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { "json": "scrapy.exporters.JsonItemExporter", @@ -170,59 +158,69 @@ FEED_EXPORTERS_BASE = { "marshal": "scrapy.exporters.MarshalItemExporter", "pickle": "scrapy.exporters.PickleItemExporter", } -FEED_EXPORT_INDENT = 0 - +FEED_STORE_EMPTY = True +FEED_STORAGES = {} +FEED_STORAGES_BASE = { + "": "scrapy.extensions.feedexport.FileFeedStorage", + "file": "scrapy.extensions.feedexport.FileFeedStorage", + "ftp": "scrapy.extensions.feedexport.FTPFeedStorage", + "gs": "scrapy.extensions.feedexport.GCSFeedStorage", + "s3": "scrapy.extensions.feedexport.S3FeedStorage", + "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage", +} FEED_STORAGE_FTP_ACTIVE = False FEED_STORAGE_GCS_ACL = "" FEED_STORAGE_S3_ACL = "" +FEED_TEMPDIR = None +FEED_URI_PARAMS = None # a function to extend uri arguments -FILES_STORE_S3_ACL = "private" FILES_STORE_GCS_ACL = "" +FILES_STORE_S3_ACL = "private" FORCE_CRAWLER_PROCESS = False +FTP_PASSIVE_MODE = True FTP_USER = "anonymous" FTP_PASSWORD = "guest" # noqa: S105 -FTP_PASSIVE_MODE = True GCS_PROJECT_ID = None HTTPCACHE_ENABLED = False -HTTPCACHE_DIR = "httpcache" -HTTPCACHE_IGNORE_MISSING = False -HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" -HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_ALWAYS_STORE = False -HTTPCACHE_IGNORE_HTTP_CODES = [] -HTTPCACHE_IGNORE_SCHEMES = ["file"] -HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = [] HTTPCACHE_DBM_MODULE = "dbm" -HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy" +HTTPCACHE_DIR = "httpcache" +HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_GZIP = False +HTTPCACHE_IGNORE_HTTP_CODES = [] +HTTPCACHE_IGNORE_MISSING = False +HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = [] +HTTPCACHE_IGNORE_SCHEMES = ["file"] +HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy" +HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = "latin-1" -IMAGES_STORE_S3_ACL = "private" IMAGES_STORE_GCS_ACL = "" - -ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager" +IMAGES_STORE_S3_ACL = "private" ITEM_PIPELINES = {} ITEM_PIPELINES_BASE = {} +ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager" + JOBDIR = None LOG_ENABLED = True -LOG_ENCODING = "utf-8" -LOG_FORMATTER = "scrapy.logformatter.LogFormatter" -LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s" LOG_DATEFORMAT = "%Y-%m-%d %H:%M:%S" -LOG_STDOUT = False -LOG_LEVEL = "DEBUG" +LOG_ENCODING = "utf-8" LOG_FILE = None LOG_FILE_APPEND = True +LOG_FORMAT = "%(asctime)s [%(name)s] %(levelname)s: %(message)s" +LOG_FORMATTER = "scrapy.logformatter.LogFormatter" +LOG_LEVEL = "DEBUG" LOG_SHORT_NAMES = False +LOG_STDOUT = False LOG_VERSIONS = [ "lxml", "libxml2", @@ -236,21 +234,19 @@ LOG_VERSIONS = [ "Platform", ] -SCHEDULER_DEBUG = False - LOGSTATS_INTERVAL = 60.0 +MAIL_FROM = "scrapy@localhost" MAIL_HOST = "localhost" MAIL_PORT = 25 -MAIL_FROM = "scrapy@localhost" -MAIL_PASS = None MAIL_USER = None +MAIL_PASS = None MEMDEBUG_ENABLED = False # enable memory debugging MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown -MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 MEMUSAGE_ENABLED = True +MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 MEMUSAGE_LIMIT_MB = 0 MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_WARNING_MB = 0 @@ -280,9 +276,6 @@ REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL" RETRY_ENABLED = True -RETRY_TIMES = 2 # initial response + 2 retries = 3 requests -RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] -RETRY_PRIORITY_ADJUST = -1 RETRY_EXCEPTIONS = [ "twisted.internet.defer.TimeoutError", "twisted.internet.error.TimeoutError", @@ -298,12 +291,16 @@ RETRY_EXCEPTIONS = [ OSError, "scrapy.core.downloader.handlers.http11.TunnelError", ] +RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] +RETRY_PRIORITY_ADJUST = -1 +RETRY_TIMES = 2 # initial response + 2 retries = 3 requests ROBOTSTXT_OBEY = False ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser" ROBOTSTXT_USER_AGENT = None SCHEDULER = "scrapy.core.scheduler.Scheduler" +SCHEDULER_DEBUG = False SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue" SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue" SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" @@ -312,11 +309,19 @@ SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 +SPIDER_CONTRACTS = {} +SPIDER_CONTRACTS_BASE = { + "scrapy.contracts.default.UrlContract": 1, + "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1, + "scrapy.contracts.default.MetadataContract": 1, + "scrapy.contracts.default.ReturnsContract": 2, + "scrapy.contracts.default.ScrapesContract": 3, +} + SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader" SPIDER_LOADER_WARN_ONLY = False SPIDER_MIDDLEWARES = {} - SPIDER_MIDDLEWARES_BASE = { # Engine side "scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25, @@ -334,27 +339,18 @@ STATS_DUMP = True STATSMAILER_RCPTS = [] +TELNETCONSOLE_ENABLED = 1 +TELNETCONSOLE_HOST = "127.0.0.1" +TELNETCONSOLE_PORT = [6023, 6073] +TELNETCONSOLE_USERNAME = "scrapy" +TELNETCONSOLE_PASSWORD = None + TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) +TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + URLLENGTH_LIMIT = 2083 USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" -TELNETCONSOLE_ENABLED = 1 -TELNETCONSOLE_PORT = [6023, 6073] -TELNETCONSOLE_HOST = "127.0.0.1" -TELNETCONSOLE_USERNAME = "scrapy" -TELNETCONSOLE_PASSWORD = None - -TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" - -SPIDER_CONTRACTS = {} -SPIDER_CONTRACTS_BASE = { - "scrapy.contracts.default.UrlContract": 1, - "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1, - "scrapy.contracts.default.MetadataContract": 1, - "scrapy.contracts.default.ReturnsContract": 2, - "scrapy.contracts.default.ScrapesContract": 3, -} - WARN_ON_GENERATOR_RETURN_VALUE = True From 5902aab25ce2ef0b26e158e0455ee6f0846636bb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 4 Jun 2025 20:37:36 +0500 Subject: [PATCH 1756/2083] Add the call_later() wrapper. (#6858) --- scrapy/core/downloader/__init__.py | 25 +++++++----- scrapy/extensions/closespider.py | 33 ++++++++++------ scrapy/utils/asyncio.py | 63 +++++++++++++++++++++++++++++- scrapy/utils/reactor.py | 13 +++--- 4 files changed, 107 insertions(+), 27 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 501c669ce..9293d7b78 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -14,7 +14,12 @@ from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.resolver import dnscache -from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + CallLaterResult, + call_later, + create_looping_call, +) from scrapy.utils.defer import ( deferred_from_coro, maybe_deferred_to_future, @@ -50,7 +55,7 @@ class Slot: self.queue: deque[tuple[Request, Deferred[Response]]] = deque() self.transferring: set[Request] = set() self.lastseen: float = 0 - self.latercall = None + self.latercall: CallLaterResult | None = None def free_transfer_slots(self) -> int: return self.concurrency - len(self.transferring) @@ -61,8 +66,9 @@ class Slot: return self.delay def close(self) -> None: - if self.latercall and self.latercall.active(): + if self.latercall: self.latercall.cancel() + self.latercall = None def __repr__(self) -> str: cls_name = self.__class__.__name__ @@ -191,9 +197,8 @@ class Downloader: slot.active.remove(request) def _process_queue(self, spider: Spider, slot: Slot) -> None: - from twisted.internet import reactor - - if slot.latercall and slot.latercall.active(): + if slot.latercall: + # block processing until slot.latercall is called return # Delay queue processing if a download_delay is configured @@ -202,9 +207,7 @@ class Downloader: if delay: penalty = delay - now + slot.lastseen if penalty > 0: - slot.latercall = reactor.callLater( - penalty, self._process_queue, spider, slot - ) + slot.latercall = call_later(penalty, self._latercall, spider, slot) return # Process enqueued requests if there are free slots to transfer for this slot @@ -218,6 +221,10 @@ class Downloader: self._process_queue(spider, slot) break + def _latercall(self, spider: Spider, slot: Slot) -> None: + slot.latercall = None + self._process_queue(spider, slot) + async def _download(self, slot: Slot, request: Request, spider: Spider) -> Response: # The order is very important for the following logic. Do not change! slot.transferring.add(request) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index a649a86e2..b4c6c73a0 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -12,9 +12,15 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured -from scrapy.utils.asyncio import create_looping_call +from scrapy.utils.asyncio import ( + AsyncioLoopingCall, + CallLaterResult, + call_later, + create_looping_call, +) if TYPE_CHECKING: + from twisted.internet.task import LoopingCall from twisted.python.failure import Failure # typing.Self requires Python 3.11 @@ -31,6 +37,12 @@ class CloseSpider: def __init__(self, crawler: Crawler): self.crawler: Crawler = crawler + # for CLOSESPIDER_TIMEOUT + self.task: CallLaterResult | None = None + + # for CLOSESPIDER_TIMEOUT_NO_ITEM + self.task_no_item: AsyncioLoopingCall | LoopingCall | None = None + self.close_on: dict[str, Any] = { "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"), "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), @@ -92,14 +104,12 @@ class CloseSpider: self.crawler.engine.close_spider(spider, "closespider_pagecount_no_item") def spider_opened(self, spider: Spider) -> None: - from twisted.internet import reactor - assert self.crawler.engine - self.task = reactor.callLater( + self.task = call_later( self.close_on["timeout"], self.crawler.engine.close_spider, spider, - reason="closespider_timeout", + "closespider_timeout", ) def item_scraped(self, item: Any, spider: Spider) -> None: @@ -110,13 +120,14 @@ class CloseSpider: self.crawler.engine.close_spider(spider, "closespider_itemcount") def spider_closed(self, spider: Spider) -> None: - task = getattr(self, "task", None) - if task and task.active(): - task.cancel() + if self.task: + self.task.cancel() + self.task = None - task_no_item = getattr(self, "task_no_item", None) - if task_no_item and task_no_item.running: - task_no_item.stop() + if self.task_no_item: + if self.task_no_item.running: + self.task_no_item.stop() + self.task_no_item = None def spider_opened_no_item(self, spider: Spider) -> None: self.task_no_item = create_looping_call(self._count_items_produced, spider) diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index cae2dc033..8c5b843cb 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -15,10 +15,14 @@ from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed if TYPE_CHECKING: + from twisted.internet.base import DelayedCall + # typing.Concatenate and typing.ParamSpec require Python 3.10 - from typing_extensions import Concatenate, ParamSpec + # typing.Self, typing.TypeVarTuple and typing.Unpack require Python 3.11 + from typing_extensions import Concatenate, ParamSpec, Self, TypeVarTuple, Unpack _P = ParamSpec("_P") + _Ts = TypeVarTuple("_Ts") _T = TypeVar("_T") @@ -192,3 +196,60 @@ def create_looping_call( if is_asyncio_available(): return AsyncioLoopingCall(func, *args, **kwargs) return LoopingCall(func, *args, **kwargs) + + +def call_later( + delay: float, func: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts] +) -> CallLaterResult: + """Schedule a function to be called after a delay. + + This uses either ``loop.call_later()`` or ``reactor.callLater()``, depending + on whether asyncio support is available. + """ + if is_asyncio_available(): + loop = asyncio.get_event_loop() + return CallLaterResult.from_asyncio(loop.call_later(delay, func, *args)) + + from twisted.internet import reactor + + return CallLaterResult.from_twisted(reactor.callLater(delay, func, *args)) + + +class CallLaterResult: + """An universal result for :func:`call_later`, wrapping either + :class:`asyncio.TimerHandle` or :class:`twisted.internet.base.DelayedCall`. + + The provided API is close to the :class:`asyncio.TimerHandle` one: there is + no ``active()`` (as there is no such public API in + :class:`asyncio.TimerHandle`) but ``cancel()`` can be called on already + called or cancelled instances. + """ + + _timer_handle: asyncio.TimerHandle | None = None + _delayed_call: DelayedCall | None = None + + @classmethod + def from_asyncio(cls, timer_handle: asyncio.TimerHandle) -> Self: + """Create a CallLaterResult from an asyncio TimerHandle.""" + o = cls() + o._timer_handle = timer_handle + return o + + @classmethod + def from_twisted(cls, delayed_call: DelayedCall) -> Self: + """Create a CallLaterResult from a Twisted DelayedCall.""" + o = cls() + o._delayed_call = delayed_call + return o + + def cancel(self) -> None: + """Cancel the underlying delayed call. + + Does nothing if the delayed call was already called or cancelled. + """ + if self._timer_handle: + self._timer_handle.cancel() + self._timer_handle = None + elif self._delayed_call and self._delayed_call.active(): + self._delayed_call.cancel() + self._delayed_call = None diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 2fb1e0ce7..76f42392b 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -16,13 +16,14 @@ if TYPE_CHECKING: from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from collections.abc import Callable - from twisted.internet.base import DelayedCall from twisted.internet.protocol import ServerFactory from twisted.internet.tcp import Port # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec + from scrapy.utils.asyncio import CallLaterResult + _P = ParamSpec("_P") _T = TypeVar("_T") @@ -55,27 +56,27 @@ class CallLaterOnce(Generic[_T]): self._func: Callable[_P, _T] = func self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw - self._call: DelayedCall | None = None + self._call: CallLaterResult | None = None self._deferreds: list[Deferred] = [] def schedule(self, delay: float = 0) -> None: - from twisted.internet import reactor + from scrapy.utils.asyncio import call_later if self._call is None: - self._call = reactor.callLater(delay, self) + self._call = call_later(delay, self) def cancel(self) -> None: if self._call: self._call.cancel() def __call__(self) -> _T: - from twisted.internet import reactor + from scrapy.utils.asyncio import call_later self._call = None result = self._func(*self._a, **self._kw) for d in self._deferreds: - reactor.callLater(0, d.callback, None) + call_later(0, d.callback, None) self._deferreds = [] return result From d602f13e8cd22154936ded9c9356e28fe3be4cd4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 5 Jun 2025 18:02:31 +0500 Subject: [PATCH 1757/2083] Fix a regression in errback result handling. (#6863) --- scrapy/core/scraper.py | 19 ++++++++----- scrapy/logformatter.py | 4 +-- tests/test_crawl.py | 61 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 10 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 9fd68bce5..1f0d57c63 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -189,6 +189,7 @@ class Scraper: ) assert self.crawler.spider + output: Iterable[Any] | AsyncIterator[Any] if isinstance(result, Response): try: # call the spider middlewares and the request callback with the response @@ -203,7 +204,7 @@ class Scraper: try: # call the request errback with the downloader error - await self.call_spider_async(result, request) + output = await self.call_spider_async(result, request) except Exception as spider_exc: # the errback didn't silence the exception if not result.check(IgnoreRequest): @@ -218,6 +219,8 @@ class Scraper: if spider_exc is not result.value: # the errback raised a different exception, handle it self.handle_spider_error(Failure(), request, result) + else: + await self.handle_spider_output_async(output, request, result) def call_spider( self, result: Response | Failure, request: Request, spider: Spider | None = None @@ -308,7 +311,7 @@ class Scraper: self, result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Response, + response: Response | Failure, spider: Spider | None = None, ) -> Deferred[None]: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" @@ -326,7 +329,7 @@ class Scraper: self, result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Response, + response: Response | Failure, ) -> None: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" it: Iterable[_T] | AsyncIterator[_T] @@ -361,7 +364,7 @@ class Scraper: ) def _process_spidermw_output( - self, output: Any, response: Response + self, output: Any, response: Response | Failure ) -> Deferred[None]: """Process each Request/Item (given in the output parameter) returned from the given spider. @@ -371,7 +374,7 @@ class Scraper: return deferred_from_coro(self._process_spidermw_output_async(output, response)) async def _process_spidermw_output_async( - self, output: Any, response: Response + self, output: Any, response: Response | Failure ) -> None: """Process each Request/Item (given in the output parameter) returned from the given spider. @@ -385,7 +388,9 @@ class Scraper: if output is not None: await self.start_itemproc_async(output, response=response) - def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[None]: + def start_itemproc( + self, item: Any, *, response: Response | Failure | None + ) -> Deferred[None]: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come @@ -394,7 +399,7 @@ class Scraper: return deferred_from_coro(self.start_itemproc_async(item, response=response)) async def start_itemproc_async( - self, item: Any, *, response: Response | None + self, item: Any, *, response: Response | Failure | None ) -> None: """Send *item* to the item pipelines for processing. diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 4f08918ae..e81a9ec93 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -116,7 +116,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Response | None, + response: Response | Failure | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" @@ -137,7 +137,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Response | None, + response: Response | Failure | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b90706027..8289b2243 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -5,6 +5,7 @@ import logging import unittest from ipaddress import IPv4Address from socket import gethostbyname +from typing import Any from urllib.parse import urlparse import pytest @@ -419,6 +420,8 @@ with multiples lines class TestCrawlSpider(TestCase): + mockserver: MockServer + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -756,6 +759,34 @@ class TestCrawlSpider(TestCase): ) assert "Spider error processing" in str(log) + @defer.inlineCallbacks + def test_spider_errback_item(self): + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "'item_scraped_count': 1" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_request(self): + def eb(failure: Failure) -> Request: + return Request(self.mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "Crawled (200)" in str(log) + @defer.inlineCallbacks def test_spider_errback_downloader_error(self): failures = [] @@ -774,7 +805,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) @defer.inlineCallbacks - def test_spider_errback_exception_downloader_error(self): + def test_spider_errback_downloader_error_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -786,6 +817,34 @@ class TestCrawlSpider(TestCase): assert "Error downloading" in str(log) assert "Spider error processing" in str(log) + @defer.inlineCallbacks + def test_spider_errback_downloader_error_item(self): + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "'item_scraped_count': 1" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_downloader_error_request(self): + def eb(failure: Failure) -> Request: + return Request(self.mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "Crawled (200)" in str(log) + @defer.inlineCallbacks def test_raise_closespider(self): def cb(response): From 105c0afb6ee12a5d1664b311582caa90bcc6c6bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 10:54:36 +0200 Subject: [PATCH 1758/2083] Feature the new logo in the README (#6831) --- README.rst | 22 ++++++++++------------ docs/_static/logo.svg | 1 + 2 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 docs/_static/logo.svg diff --git a/README.rst b/README.rst index 29488d825..30001e4b0 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,10 @@ -.. image:: https://scrapy.org/img/scrapylogo.png - :target: https://scrapy.org/ +.. raw:: html -====== -Scrapy -====== +

+ + Scrapy + +

.. image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy @@ -37,13 +38,10 @@ Scrapy :target: https://deepwiki.com/scrapy/scrapy :alt: Ask DeepWiki - -Overview -======== - -Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to -crawl websites and extract structured data from their pages. It can be used for -a wide range of purposes, from data mining to monitoring and automated testing. +Scrapy is a BSD-licensed fast high-level web crawling and web scraping +framework, used to crawl websites and extract structured data from their pages. +It can be used for a wide range of purposes, from data mining to monitoring and +automated testing. Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other contributors`_. diff --git a/docs/_static/logo.svg b/docs/_static/logo.svg new file mode 100644 index 000000000..04b2d18a7 --- /dev/null +++ b/docs/_static/logo.svg @@ -0,0 +1 @@ + From b8cd079014f0e31c609d5fd7fd5f52b89283b1c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 28 May 2025 11:35:18 +0200 Subject: [PATCH 1759/2083] Shorten the README and remove broken links to scrapy.org (#6833) --- README.rst | 68 ++++++------------------------------------- docs/contributing.rst | 9 ++++-- 2 files changed, 16 insertions(+), 61 deletions(-) diff --git a/README.rst b/README.rst index 30001e4b0..5dc994570 100644 --- a/README.rst +++ b/README.rst @@ -38,74 +38,24 @@ :target: https://deepwiki.com/scrapy/scrapy :alt: Ask DeepWiki -Scrapy is a BSD-licensed fast high-level web crawling and web scraping -framework, used to crawl websites and extract structured data from their pages. -It can be used for a wide range of purposes, from data mining to monitoring and -automated testing. - -Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other -contributors`_. +Scrapy_ is a web scraping framework to extract structured data from websites. +It is cross-platform, and requires Python 3.9+. It is maintained by Zyte_ +(formerly Scrapinghub) and `many other contributors`_. .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapy: https://scrapy.org/ .. _Zyte: https://www.zyte.com/ -Check the Scrapy homepage at https://scrapy.org for more information, -including a list of features. - - -Requirements -============ - -* Python 3.9+ -* Works on Linux, Windows, macOS, BSD - -Install -======= - -The quick way: +Install with: .. code:: bash pip install scrapy -See the install section in the documentation at -https://docs.scrapy.org/en/latest/intro/install.html for more details. +And follow the documentation_ to learn how to use it. -Documentation -============= +.. _documentation: https://docs.scrapy.org/en/latest/ -Documentation is available online at https://docs.scrapy.org/ and in the ``docs`` -directory. +If you wish to contribute, see Contributing_. -Releases -======== - -You can check https://docs.scrapy.org/en/latest/news.html for the release notes. - -Community (blog, twitter, mail list, IRC) -========================================= - -See https://scrapy.org/community/ for details. - -Contributing -============ - -See https://docs.scrapy.org/en/master/contributing.html for details. - -Code of Conduct ---------------- - -Please note that this project is released with a Contributor `Code of Conduct `_. - -By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@zyte.com. - -Companies using Scrapy -====================== - -See https://scrapy.org/companies/ for a list. - -Commercial Support -================== - -See https://scrapy.org/support/ for details. +.. _Contributing: https://docs.scrapy.org/en/master/contributing.html diff --git a/docs/contributing.rst b/docs/contributing.rst index f5c1c74b8..0172887d6 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -6,8 +6,13 @@ Contributing to Scrapy .. important:: - Double check that you are reading the most recent version of this document at - https://docs.scrapy.org/en/master/contributing.html + Double check that you are reading the most recent version of this document + at https://docs.scrapy.org/en/master/contributing.html + + By participating in this project you agree to abide by the terms of our + `Code of Conduct + `_. Please + report unacceptable behavior to opensource@zyte.com. There are many ways to contribute to Scrapy. Here are some of them: From 3d382aa650735827647093dfb157d5bd2f15efc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 30 May 2025 09:33:17 +0200 Subject: [PATCH 1760/2083] Avoid raw HTML in the README (#6839) --- README.rst | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 5dc994570..536dec7f0 100644 --- a/README.rst +++ b/README.rst @@ -1,40 +1,41 @@ -.. raw:: html +|logo| -

- - Scrapy - -

+.. |logo| image:: https://raw.githubusercontent.com/scrapy/scrapy/master/docs/_static/logo.svg + :target: https://scrapy.org + :alt: Scrapy + :width: 480px -.. image:: https://img.shields.io/pypi/v/Scrapy.svg +|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki| + +.. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy :alt: PyPI Version -.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg +.. |python_version| image:: https://img.shields.io/pypi/pyversions/Scrapy.svg :target: https://pypi.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg +.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg +.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS :alt: macOS -.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg +.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows :alt: Windows -.. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg +.. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report -.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg +.. |conda| image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg :target: https://anaconda.org/conda-forge/scrapy :alt: Conda Version -.. image:: https://deepwiki.com/badge.svg +.. |deepwiki| image:: https://deepwiki.com/badge.svg :target: https://deepwiki.com/scrapy/scrapy :alt: Ask DeepWiki From 54474ceb0d1467d90bf047415d1c1f135263d983 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 5 Jun 2025 18:02:31 +0500 Subject: [PATCH 1761/2083] Fix a regression in errback result handling. (#6863) --- scrapy/core/scraper.py | 17 ++++++++---- scrapy/logformatter.py | 4 +-- tests/test_crawl.py | 61 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 9378f2651..975344103 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -188,6 +188,7 @@ class Scraper: ) assert self.crawler.spider + output: Iterable[Any] | AsyncIterator[Any] if isinstance(result, Response): try: # call the spider middlewares and the request callback with the response @@ -204,7 +205,7 @@ class Scraper: try: # call the request errback with the downloader error - await self.call_spider_async(result, request) + output = await self.call_spider_async(result, request) except Exception as spider_exc: # the errback didn't silence the exception if not result.check(IgnoreRequest): @@ -219,6 +220,8 @@ class Scraper: if spider_exc is not result.value: # the errback raised a different exception, handle it self.handle_spider_error(Failure(), request, result) + else: + await self.handle_spider_output_async(output, request, result) def call_spider( self, result: Response | Failure, request: Request, spider: Spider | None = None @@ -309,7 +312,7 @@ class Scraper: self, result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Response, + response: Response | Failure, spider: Spider | None = None, ) -> Deferred[None]: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" @@ -327,7 +330,7 @@ class Scraper: self, result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Response, + response: Response | Failure, ) -> None: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" if isinstance(result, AsyncIterator): @@ -352,7 +355,9 @@ class Scraper: ) @deferred_f_from_coro_f - async def _process_spidermw_output(self, output: Any, response: Response) -> None: + async def _process_spidermw_output( + self, output: Any, response: Response | Failure + ) -> None: """Process each Request/Item (given in the output parameter) returned from the given spider. @@ -368,7 +373,9 @@ class Scraper: ) @deferred_f_from_coro_f - async def start_itemproc(self, item: Any, *, response: Response | None) -> None: + async def start_itemproc( + self, item: Any, *, response: Response | Failure | None + ) -> None: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 4f08918ae..e81a9ec93 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -116,7 +116,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Response | None, + response: Response | Failure | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" @@ -137,7 +137,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Response | None, + response: Response | Failure | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b90706027..8289b2243 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -5,6 +5,7 @@ import logging import unittest from ipaddress import IPv4Address from socket import gethostbyname +from typing import Any from urllib.parse import urlparse import pytest @@ -419,6 +420,8 @@ with multiples lines class TestCrawlSpider(TestCase): + mockserver: MockServer + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -756,6 +759,34 @@ class TestCrawlSpider(TestCase): ) assert "Spider error processing" in str(log) + @defer.inlineCallbacks + def test_spider_errback_item(self): + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "'item_scraped_count': 1" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_request(self): + def eb(failure: Failure) -> Request: + return Request(self.mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "Crawled (200)" in str(log) + @defer.inlineCallbacks def test_spider_errback_downloader_error(self): failures = [] @@ -774,7 +805,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) @defer.inlineCallbacks - def test_spider_errback_exception_downloader_error(self): + def test_spider_errback_downloader_error_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -786,6 +817,34 @@ class TestCrawlSpider(TestCase): assert "Error downloading" in str(log) assert "Spider error processing" in str(log) + @defer.inlineCallbacks + def test_spider_errback_downloader_error_item(self): + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "'item_scraped_count': 1" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_downloader_error_request(self): + def eb(failure: Failure) -> Request: + return Request(self.mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "Crawled (200)" in str(log) + @defer.inlineCallbacks def test_raise_closespider(self): def cb(response): From b20995c9d8dd00618ae71d0f64cdc53f6d669cf2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 6 Jun 2025 13:16:48 +0500 Subject: [PATCH 1762/2083] Silence a typing error. --- scrapy/core/scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 975344103..ac720e03f 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -193,7 +193,7 @@ class Scraper: try: # call the spider middlewares and the request callback with the response output = await maybe_deferred_to_future( - self.spidermw.scrape_response( + self.spidermw.scrape_response( # type: ignore[arg-type] self.call_spider, result, request, self.crawler.spider ) ) From d99234a33f02f7dd5fb06d167ec78266b7f4dfeb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 6 Jun 2025 14:54:21 +0500 Subject: [PATCH 1763/2083] Install the reactor explicitly in CrawlerRunner examples. (#6865) --- docs/topics/practices.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index db91cd073..b3c881b81 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -95,6 +95,7 @@ reactor after ``MySpider`` has finished running. import scrapy from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.reactor import install_reactor class MySpider(scrapy.Spider): @@ -102,6 +103,7 @@ reactor after ``MySpider`` has finished running. ... + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) runner = CrawlerRunner() @@ -112,26 +114,26 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished -Same example but using a non-default reactor, it's only necessary call -``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. +Same example but using a different reactor. .. code-block:: python import scrapy from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.reactor import install_reactor class MySpider(scrapy.Spider): + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.epollreactor.EPollReactor", + } # Your spider definition ... + install_reactor("twisted.internet.epollreactor.EPollReactor") configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) - - from scrapy.utils.reactor import install_reactor - - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") runner = CrawlerRunner() d = runner.crawl(MySpider) @@ -184,6 +186,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings + from scrapy.utils.reactor import install_reactor class MySpider1(scrapy.Spider): @@ -196,6 +199,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: ... + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") configure_logging() settings = get_project_settings() runner = CrawlerRunner(settings) @@ -217,6 +221,7 @@ Same example but running the spiders sequentially by chaining the deferreds: from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings + from scrapy.utils.reactor import install_reactor class MySpider1(scrapy.Spider): @@ -229,6 +234,7 @@ Same example but running the spiders sequentially by chaining the deferreds: ... + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") settings = get_project_settings() configure_logging(settings) runner = CrawlerRunner(settings) From 405d9bc8a247cfec4d698310c425112c456f134f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 6 Jun 2025 15:59:49 +0500 Subject: [PATCH 1764/2083] More docs for the is_asyncio_reactor_installed() behavior change. (#6866) --- docs/news.rst | 9 +++++++++ scrapy/utils/reactor.py | 5 +++++ scrapy/utils/test.py | 7 ++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index ef3b549e7..8b1d51674 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -126,6 +126,15 @@ Backward-incompatible changes also enforced for start requests. (:issue:`6777`) +- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without + an installed reactor now raises an exception instead of installing a + reactor. This shouldn't affect normal Scrapy use cases, but it may affect + 3rd-party test suites that use Scrapy internals such as + :class:`~scrapy.crawler.Crawler` and don't install a reactor explicitly. If + you are affected by this change, you most likely need to install the + reactor before running Scrapy code that expects it to be installed. + (:issue:`6732`, :issue:`6735`) + - The ``from_settings()`` method of :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, deprecated in Scrapy 2.12.0, is removed earlier than the usual deprecation diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 9c2754394..1b179f988 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -202,6 +202,11 @@ def is_asyncio_reactor_installed() -> bool: """Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. Raise a :exc:`RuntimeError` if no reactor is installed. + + .. versionchanged:: 2.13 + In earlier Scrapy versions this function silently installed the default + reactor if there was no reactor installed. Now it raises an exception to + prevent silent problems in this case. """ if not is_reactor_installed(): raise RuntimeError( diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 2da526cd8..4a732bd72 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -18,7 +18,7 @@ from twisted.trial.unittest import SkipTest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available from scrapy.utils.deprecate import create_deprecated_class -from scrapy.utils.reactor import is_asyncio_reactor_installed +from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed from scrapy.utils.spider import DefaultSpider if TYPE_CHECKING: @@ -117,6 +117,11 @@ def get_reactor_settings() -> dict[str, Any]: settings, so tests that run the crawler in the current process may need to pass a correct ``"TWISTED_REACTOR"`` setting value when creating it. """ + if not is_reactor_installed(): + raise RuntimeError( + "get_reactor_settings() called without an installed reactor," + " you may need to install a reactor explicitly when running your tests." + ) settings: dict[str, Any] = {} if not is_asyncio_reactor_installed(): settings["TWISTED_REACTOR"] = None From 657e6cb2b57d52005740e92543c1270dbaf61ded Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 6 Jun 2025 16:02:15 +0500 Subject: [PATCH 1765/2083] Don't try to close ExecutionEngine.downloader when it doesn't exist. (#6867) --- scrapy/core/engine.py | 3 ++- tests/test_engine.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index d9361a674..fe635dc82 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -174,7 +174,8 @@ class ExecutionEngine: return self.close_spider( self.spider, reason="shutdown" ) # will also close downloader - self.downloader.close() + if hasattr(self, "downloader"): + self.downloader.close() return succeed(None) def pause(self) -> None: diff --git a/tests/test_engine.py b/tests/test_engine.py index 1f79a081d..9f618437c 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -22,6 +22,7 @@ from unittest.mock import Mock from urllib.parse import urlparse import attr +import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer @@ -433,6 +434,19 @@ class TestEngine(TestEngineBase): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.close() + def test_close_without_downloader(self): + class CustomException(Exception): + pass + + class BadDownloader: + def __init__(self, crawler): + raise CustomException + + with pytest.raises(CustomException): + ExecutionEngine( + get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None + ) + @defer.inlineCallbacks def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) From d329eedfefd9a1fa7006e6d0a214e9d5e01a8e0c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 6 Jun 2025 16:02:15 +0500 Subject: [PATCH 1766/2083] Don't try to close ExecutionEngine.downloader when it doesn't exist. (#6867) --- scrapy/core/engine.py | 3 ++- tests/test_engine.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7f5dd0405..0df9ad2b2 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -170,7 +170,8 @@ class ExecutionEngine: return self.close_spider( self.spider, reason="shutdown" ) # will also close downloader - self.downloader.close() + if hasattr(self, "downloader"): + self.downloader.close() return succeed(None) def pause(self) -> None: diff --git a/tests/test_engine.py b/tests/test_engine.py index b60b510b2..b2e436425 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -22,6 +22,7 @@ from unittest.mock import Mock from urllib.parse import urlparse import attr +import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer, reactor @@ -431,6 +432,19 @@ class TestEngine(TestEngineBase): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.close() + def test_close_without_downloader(self): + class CustomException(Exception): + pass + + class BadDownloader: + def __init__(self, crawler): + raise CustomException + + with pytest.raises(CustomException): + ExecutionEngine( + get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None + ) + @defer.inlineCallbacks def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) From 744edb9ba9e293ddccfcfa03e0aef0a7c0da14b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Jun 2025 16:09:51 +0200 Subject: [PATCH 1767/2083] Make scrapy fetch work with scrapy-poet (#6872) --- scrapy/commands/fetch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index ef6e13de2..0aaff6c25 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,6 +1,7 @@ from __future__ import annotations import sys +from argparse import Namespace # noqa: TC003 from typing import TYPE_CHECKING from w3lib.url import is_url @@ -12,7 +13,7 @@ from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import DefaultSpider, spidercls_for_request if TYPE_CHECKING: - from argparse import ArgumentParser, Namespace + from argparse import ArgumentParser from scrapy import Spider From d8251332845d48d2418f0055c27686cee04b5b9a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 7 Jun 2025 01:59:09 +0500 Subject: [PATCH 1768/2083] Reduce deps on unittest, unify inlineCallbacks imports in tests. (#6873) --- tests/test_closespider.py | 16 +- tests/test_command_fetch.py | 10 +- tests/test_command_parse.py | 42 ++-- tests/test_command_runspider.py | 5 +- tests/test_command_shell.py | 39 ++-- tests/test_command_version.py | 6 +- tests/test_contracts.py | 4 +- tests/test_crawl.py | 135 +++++++------ tests/test_dependencies.py | 12 -- tests/test_downloader_handlers.py | 2 +- tests/test_downloader_handlers_http_base.py | 12 +- ...st_downloadermiddleware_httpcompression.py | 35 ++-- tests/test_downloadermiddleware_robotstxt.py | 8 +- tests/test_downloaderslotssettings.py | 4 +- tests/test_engine.py | 24 +-- tests/test_engine_stop_download_bytes.py | 4 +- tests/test_engine_stop_download_headers.py | 4 +- tests/test_exporters.py | 3 +- tests/test_extension_periodic_log.py | 3 +- tests/test_extension_telnet.py | 12 +- tests/test_feedexport.py | 179 +++++++++--------- tests/test_http2_client_protocol.py | 4 +- tests/test_logformatter.py | 6 +- tests/test_pipeline_crawl.py | 12 +- tests/test_pipeline_files.py | 22 +-- tests/test_pipelines.py | 13 +- tests/test_proxy_connect.py | 8 +- tests/test_request_attribute_binding.py | 16 +- tests/test_request_cb_kwargs.py | 4 +- tests/test_request_left.py | 10 +- tests/test_scheduler.py | 6 +- tests/test_scheduler_base.py | 5 +- tests/test_signals.py | 4 +- tests/test_spidermiddleware.py | 15 +- tests/test_spidermiddleware_httperror.py | 8 +- tests/test_spidermiddleware_output_chain.py | 26 +-- tests/test_squeues_request.py | 13 +- tests/test_utils_log.py | 13 +- tests/test_utils_signal.py | 3 +- tests/test_utils_trackref.py | 4 +- tests/test_utils_url.py | 3 +- tests/test_webclient.py | 35 ++-- 42 files changed, 380 insertions(+), 409 deletions(-) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 4a17b254b..c6ec690a1 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,4 +1,4 @@ -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase from scrapy.utils.test import get_crawler @@ -22,7 +22,7 @@ class TestCloseSpider(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_itemcount(self): close_on = 5 crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on}) @@ -32,7 +32,7 @@ class TestCloseSpider(TestCase): itemcount = crawler.stats.get_value("item_scraped_count") assert itemcount >= close_on - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_pagecount(self): close_on = 5 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on}) @@ -42,7 +42,7 @@ class TestCloseSpider(TestCase): pagecount = crawler.stats.get_value("response_received_count") assert pagecount >= close_on - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_pagecount_no_item(self): close_on = 5 max_items = 5 @@ -62,7 +62,7 @@ class TestCloseSpider(TestCase): itemcount = crawler.stats.get_value("item_scraped_count") assert pagecount <= close_on + itemcount - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_pagecount_no_item_with_pagecount(self): close_on_pagecount_no_item = 5 close_on_pagecount = 20 @@ -79,7 +79,7 @@ class TestCloseSpider(TestCase): pagecount = crawler.stats.get_value("response_received_count") assert pagecount < close_on_pagecount - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_errorcount(self): close_on = 5 crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on}) @@ -91,7 +91,7 @@ class TestCloseSpider(TestCase): assert crawler.stats.get_value("spider_exceptions/count") >= close_on assert errorcount >= close_on - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_timeout(self): close_on = 0.1 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on}) @@ -101,7 +101,7 @@ class TestCloseSpider(TestCase): total_seconds = crawler.stats.get_value("elapsed_time_seconds") assert total_seconds >= close_on - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_timeout_no_item(self): timeout = 1 crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index a31cada85..89f664336 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -1,4 +1,4 @@ -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from tests.utils.testproc import ProcessTest @@ -8,17 +8,17 @@ from tests.utils.testsite import SiteTest class TestFetchCommand(ProcessTest, SiteTest, unittest.TestCase): command = "fetch" - @defer.inlineCallbacks + @inlineCallbacks def test_output(self): _, out, _ = yield self.execute([self.url("/text")]) assert out.strip() == b"Works" - @defer.inlineCallbacks + @inlineCallbacks def test_redirect_default(self): _, out, _ = yield self.execute([self.url("/redirect")]) assert out.strip() == b"Redirected here" - @defer.inlineCallbacks + @inlineCallbacks def test_redirect_disabled(self): _, out, err = yield self.execute( ["--no-redirect", self.url("/redirect-no-meta-refresh")] @@ -27,7 +27,7 @@ class TestFetchCommand(ProcessTest, SiteTest, unittest.TestCase): assert b"downloader/response_status_count/302" in err, err assert b"downloader/response_status_count/200" not in err, err - @defer.inlineCallbacks + @inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url("/text"), "--headers"]) out = out.replace(b"\r", b"") # required on win32 diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 9e66d319c..6681aba17 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -3,7 +3,7 @@ import os import re from pathlib import Path -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from scrapy.commands import parse from scrapy.settings import Settings @@ -171,7 +171,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """ ) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_arguments(self): _, _, stderr = yield self.execute( [ @@ -187,7 +187,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: It Works!" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_request_with_meta(self): raw_json_string = '{"foo" : "baz"}' _, _, stderr = yield self.execute( @@ -218,7 +218,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: It Works!" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_request_with_cb_kwargs(self): raw_json_string = '{"foo" : "bar", "key": "value"}' _, _, stderr = yield self.execute( @@ -239,7 +239,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} "DEBUG: request.callback signature: (response, foo=None, key=None)" in log ) - @defer.inlineCallbacks + @inlineCallbacks def test_request_without_meta(self): _, _, stderr = yield self.execute( [ @@ -253,7 +253,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: It Works!" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_pipelines(self): _, _, stderr = yield self.execute( [ @@ -268,7 +268,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "INFO: It Works!" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_items_list(self): status, out, stderr = yield self.execute( [ @@ -283,7 +283,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert "{'id': 1}" in _textmode(out) assert "{'id': 2}" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): status, out, stderr = yield self.execute( [ @@ -297,7 +297,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert "INFO: Got response 200" in _textmode(stderr) assert "{'foo': 42}" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_loop(self): status, out, stderr = yield self.execute( [ @@ -312,7 +312,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} for i in range(10): assert f"{{'foo': {i}}}" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_exc(self): status, out, stderr = yield self.execute( [ @@ -327,7 +327,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} for i in range(7): assert f"{{'foo': {i}}}" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse(self): _, _, stderr = yield self.execute( [ @@ -340,21 +340,21 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: Got response 200" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, "-c", "parse", self.url("/html")] ) assert "[{}, {'foo': 'bar'}]" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_parse_items_no_callback_passed(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, self.url("/html")] ) assert "[{}, {'foo': 'bar'}]" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_wrong_callback_passed(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, "-c", "dummy", self.url("/html")] @@ -362,7 +362,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) assert "Cannot find callback" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_matching_rule_callback_set(self): """If a rule matches the URL, use it's defined callback.""" status, out, stderr = yield self.execute( @@ -370,7 +370,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "[{}, {'foo': 'bar'}]" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_matching_rule_default_callback(self): """If a rule match but it has no callback set, use the 'parse' callback.""" status, out, stderr = yield self.execute( @@ -378,7 +378,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "[{}, {'nomatch': 'default'}]" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_with_no_rules_attribute(self): """Using -r with a spider with no rule should not produce items.""" status, out, stderr = yield self.execute( @@ -387,14 +387,14 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) assert "No CrawlSpider rules found" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_missing_callback(self): status, out, stderr = yield self.execute( ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/html")] ) assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_no_matching_rule(self): """The requested URL has no matching rule, so no items should be scraped""" status, out, stderr = yield self.execute( @@ -403,12 +403,12 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) assert "Cannot find a rule that matches" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_not_exists_with_not_matched_url(self): status, out, stderr = yield self.execute([self.url("/invalid_url")]) assert status == 0 - @defer.inlineCallbacks + @inlineCallbacks def test_output_flag(self): """Checks if a file was created successfully having correct format containing correct data in it. diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index c57c09249..7f8d9fb61 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -10,7 +10,6 @@ from typing import TYPE_CHECKING from unittest import skipIf import pytest -from twisted.trial import unittest from tests.test_commands import TestCommandBase from tests.test_crawler import ExceptionSpider, NoRequestsSpider @@ -376,7 +375,7 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand): def setUp(self): if platform.system() != "Windows": - raise unittest.SkipTest("Windows required for .pyw files") + pytest.skip("Windows required for .pyw files") return super().setUp() def test_start_errors(self): @@ -385,4 +384,4 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand): assert "badspider.pyw" in log def test_runspider_unable_to_load(self): - raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") + pytest.skip("Already Tested in 'RunSpiderCommandTest' ") diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 8041e7cb1..d9f17d76b 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -3,8 +3,9 @@ import sys from io import BytesIO from pathlib import Path +import pytest from pexpect.popen_spawn import PopenSpawn -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from scrapy.utils.reactor import _asyncio_reactor_path @@ -17,52 +18,52 @@ from tests.utils.testsite import SiteTest class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): command = "shell" - @defer.inlineCallbacks + @inlineCallbacks def test_empty(self): _, out, _ = yield self.execute(["-c", "item"]) assert b"{}" in out - @defer.inlineCallbacks + @inlineCallbacks def test_response_body(self): _, out, _ = yield self.execute([self.url("/text"), "-c", "response.body"]) assert b"Works" in out - @defer.inlineCallbacks + @inlineCallbacks def test_response_type_text(self): _, out, _ = yield self.execute([self.url("/text"), "-c", "type(response)"]) assert b"TextResponse" in out - @defer.inlineCallbacks + @inlineCallbacks def test_response_type_html(self): _, out, _ = yield self.execute([self.url("/html"), "-c", "type(response)"]) assert b"HtmlResponse" in out - @defer.inlineCallbacks + @inlineCallbacks def test_response_selector_html(self): xpath = "response.xpath(\"//p[@class='one']/text()\").get()" _, out, _ = yield self.execute([self.url("/html"), "-c", xpath]) assert out.strip() == b"Works" - @defer.inlineCallbacks + @inlineCallbacks def test_response_encoding_gb18030(self): _, out, _ = yield self.execute( [self.url("/enc-gb18030"), "-c", "response.encoding"] ) assert out.strip() == b"gb18030" - @defer.inlineCallbacks + @inlineCallbacks def test_redirect(self): _, out, _ = yield self.execute([self.url("/redirect"), "-c", "response.url"]) assert out.strip().endswith(b"/redirected") - @defer.inlineCallbacks + @inlineCallbacks def test_redirect_follow_302(self): _, out, _ = yield self.execute( [self.url("/redirect-no-meta-refresh"), "-c", "response.status"] ) assert out.strip().endswith(b"200") - @defer.inlineCallbacks + @inlineCallbacks def test_redirect_not_follow_302(self): _, out, _ = yield self.execute( [ @@ -74,7 +75,7 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): ) assert out.strip().endswith(b"302") - @defer.inlineCallbacks + @inlineCallbacks def test_fetch_redirect_follow_302(self): """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" url = self.url("/redirect-no-meta-refresh") @@ -84,7 +85,7 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): assert b"Redirecting (302)" in errout assert b"Crawled (200)" in errout - @defer.inlineCallbacks + @inlineCallbacks def test_fetch_redirect_not_follow_302(self): """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" url = self.url("/redirect-no-meta-refresh") @@ -93,27 +94,27 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): assert errcode == 0, out assert b"Crawled (302)" in errout - @defer.inlineCallbacks + @inlineCallbacks def test_request_replace(self): url = self.url("/text") code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" errcode, out, _ = yield self.execute(["-c", code]) assert errcode == 0, out - @defer.inlineCallbacks + @inlineCallbacks def test_scrapy_import(self): url = self.url("/text") code = f"fetch(scrapy.Request('{url}'))" errcode, out, _ = yield self.execute(["-c", code]) assert errcode == 0, out - @defer.inlineCallbacks + @inlineCallbacks def test_local_file(self): filepath = Path(tests_datadir, "test_site", "index.html") _, out, _ = yield self.execute([str(filepath), "-c", "item"]) assert b"{}" in out - @defer.inlineCallbacks + @inlineCallbacks def test_local_nofile(self): filepath = "file:///tests/sample_data/test_site/nothinghere.html" errcode, out, err = yield self.execute( @@ -122,16 +123,16 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): assert errcode == 1, out or err assert b"No such file or directory" in err - @defer.inlineCallbacks + @inlineCallbacks def test_dns_failures(self): if NON_EXISTING_RESOLVABLE: - raise unittest.SkipTest("Non-existing hosts are resolvable") + pytest.skip("Non-existing hosts are resolvable") url = "www.somedomainthatdoesntexi.st" errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False) assert errcode == 1, out or err assert b"DNS lookup failed" in err - @defer.inlineCallbacks + @inlineCallbacks def test_shell_fetch_async(self): url = self.url("/html") code = f"fetch('{url}')" diff --git a/tests/test_command_version.py b/tests/test_command_version.py index a61a6a32b..87dfb16df 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -1,6 +1,6 @@ import sys -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest import scrapy @@ -10,13 +10,13 @@ from tests.utils.testproc import ProcessTest class TestVersionCommand(ProcessTest, unittest.TestCase): command = "version" - @defer.inlineCallbacks + @inlineCallbacks def test_output(self): encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute([]) assert out.strip().decode(encoding) == f"Scrapy {scrapy.__version__}" - @defer.inlineCallbacks + @inlineCallbacks def test_verbose_output(self): encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute(["-v"]) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 26b16a1d4..ad3efa042 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,7 +1,7 @@ from unittest import TextTestResult import pytest -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.python import failure from twisted.trial import unittest @@ -502,7 +502,7 @@ class TestContractsManager(unittest.TestCase): assert not self.results.failures assert self.results.errors - @defer.inlineCallbacks + @inlineCallbacks def test_same_url(self): class TestSameUrlSpider(Spider): name = "test_same_url" diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 8289b2243..4c1f6216b 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -2,7 +2,6 @@ from __future__ import annotations import json import logging -import unittest from ipaddress import IPv4Address from socket import gethostbyname from typing import Any @@ -10,7 +9,7 @@ from urllib.parse import urlparse import pytest from testfixtures import LogCapture -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase @@ -67,21 +66,21 @@ class TestCrawl(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks + @inlineCallbacks def test_follow_all(self): crawler = get_crawler(FollowAllSpider) yield crawler.crawl(mockserver=self.mockserver) assert len(crawler.spider.urls_visited) == 11 # 10 + start_url - @defer.inlineCallbacks + @inlineCallbacks def test_fixed_delay(self): yield self._test_delay(total=3, delay=0.2) - @defer.inlineCallbacks + @inlineCallbacks def test_randomized_delay(self): yield self._test_delay(total=3, delay=0.1, randomize=True) - @defer.inlineCallbacks + @inlineCallbacks def _test_delay(self, total, delay, randomize=False): crawl_kwargs = { "maxlatency": delay * 2, @@ -110,7 +109,7 @@ class TestCrawl(TestCase): average = total_time / (len(times) - 1) assert average <= delay / tolerance, "test total or delay values are too small" - @defer.inlineCallbacks + @inlineCallbacks def test_timeout_success(self): crawler = get_crawler(DelaySpider) yield crawler.crawl(n=0.5, mockserver=self.mockserver) @@ -118,7 +117,7 @@ class TestCrawl(TestCase): assert crawler.spider.t2 > 0 assert crawler.spider.t2 > crawler.spider.t1 - @defer.inlineCallbacks + @inlineCallbacks def test_timeout_failure(self): crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) yield crawler.crawl(n=0.5, mockserver=self.mockserver) @@ -135,7 +134,7 @@ class TestCrawl(TestCase): assert crawler.spider.t2_err > 0 assert crawler.spider.t2_err > crawler.spider.t1 - @defer.inlineCallbacks + @inlineCallbacks def test_retry_503(self): crawler = get_crawler(SimpleSpider) with LogCapture() as log: @@ -144,7 +143,7 @@ class TestCrawl(TestCase): ) self._assert_retried(log) - @defer.inlineCallbacks + @inlineCallbacks def test_retry_conn_failed(self): crawler = get_crawler(SimpleSpider) with LogCapture() as log: @@ -153,10 +152,10 @@ class TestCrawl(TestCase): ) self._assert_retried(log) - @defer.inlineCallbacks + @inlineCallbacks def test_retry_dns_error(self): if NON_EXISTING_RESOLVABLE: - raise unittest.SkipTest("Non-existing hosts are resolvable") + pytest.skip("Non-existing hosts are resolvable") crawler = get_crawler(SimpleSpider) with LogCapture() as log: # try to fetch the homepage of a nonexistent domain @@ -165,7 +164,7 @@ class TestCrawl(TestCase): ) self._assert_retried(log) - @defer.inlineCallbacks + @inlineCallbacks def test_start_bug_before_yield(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenStartSpider) @@ -176,7 +175,7 @@ class TestCrawl(TestCase): assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @defer.inlineCallbacks + @inlineCallbacks def test_start_bug_yielding(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenStartSpider) @@ -187,7 +186,7 @@ class TestCrawl(TestCase): assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @defer.inlineCallbacks + @inlineCallbacks def test_start_items(self): items = [] @@ -202,7 +201,7 @@ class TestCrawl(TestCase): assert len(log.records) == 0 assert items == [{"name": "test item"}] - @defer.inlineCallbacks + @inlineCallbacks def test_start_unsupported_output(self): """Anything that is not a request is assumed to be an item, avoiding a potentially expensive call to itemadapter.is_item(), and letting @@ -223,7 +222,7 @@ class TestCrawl(TestCase): assert len(items) == 3 assert not any(isinstance(item, Request) for item in items) - @defer.inlineCallbacks + @inlineCallbacks def test_start_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(DuplicateStartSpider, settings) @@ -241,7 +240,7 @@ class TestCrawl(TestCase): ) assert crawler.spider.visited == 3 - @defer.inlineCallbacks + @inlineCallbacks def test_unbounded_response(self): # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" @@ -275,7 +274,7 @@ with multiples lines ) assert str(log).count("Got response 200") == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data crawler = get_crawler(SimpleSpider) @@ -285,7 +284,7 @@ with multiples lines ) self._assert_retried(log) - @defer.inlineCallbacks + @inlineCallbacks def test_retry_conn_aborted(self): # connection lost before receiving data crawler = get_crawler(SimpleSpider) @@ -299,7 +298,7 @@ with multiples lines assert str(log).count("Retrying") == 2 assert str(log).count("Gave up retrying") == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_referer_header(self): """Referer header is set by RefererMiddleware unless it is already set""" req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1) @@ -327,7 +326,7 @@ with multiples lines echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body)) assert echo3["headers"].get("Referer") == ["http://example.com"] - @defer.inlineCallbacks + @inlineCallbacks def test_engine_status(self): from scrapy.utils.engine import get_engine_status @@ -345,7 +344,7 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_format_engine_status(self): from scrapy.utils.engine import format_engine_status @@ -370,7 +369,7 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" - @defer.inlineCallbacks + @inlineCallbacks def test_open_spider_error_on_faulty_pipeline(self): settings = { "ITEM_PIPELINES": { @@ -378,15 +377,13 @@ with multiples lines } } crawler = get_crawler(SimpleSpider, settings) - yield self.assertFailure( - crawler.crawl( + with pytest.raises(ZeroDivisionError): + yield crawler.crawl( self.mockserver.url("/status?n=200"), mockserver=self.mockserver - ), - ZeroDivisionError, - ) + ) assert not crawler.crawling - @defer.inlineCallbacks + @inlineCallbacks def test_crawlerrunner_accepts_crawler(self): crawler = get_crawler(SimpleSpider) runner = CrawlerRunner() @@ -398,7 +395,7 @@ with multiples lines ) assert "Got response 200" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawl_multiple(self): runner = CrawlerRunner(get_reactor_settings()) runner.crawl( @@ -431,7 +428,7 @@ class TestCrawlSpider(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks + @inlineCallbacks def _run_spider(self, spider_cls): items = [] @@ -446,7 +443,7 @@ class TestCrawlSpider(TestCase): ) return log, items, crawler.stats - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_with_parse(self): crawler = get_crawler(CrawlSpiderWithParseMethod) with LogCapture() as log: @@ -456,7 +453,7 @@ class TestCrawlSpider(TestCase): assert "[parse] status 201 (foo: None)" in str(log) assert "[parse] status 202 (foo: bar)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_with_async_callback(self): crawler = get_crawler(CrawlSpiderWithAsyncCallback) with LogCapture() as log: @@ -466,7 +463,7 @@ class TestCrawlSpider(TestCase): assert "[parse_async] status 201 (foo: None)" in str(log) assert "[parse_async] status 202 (foo: bar)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_with_async_generator_callback(self): crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) with LogCapture() as log: @@ -476,7 +473,7 @@ class TestCrawlSpider(TestCase): assert "[parse_async_gen] status 201 (foo: None)" in str(log) assert "[parse_async_gen] status 202 (foo: bar)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback) with LogCapture() as log: @@ -489,7 +486,7 @@ class TestCrawlSpider(TestCase): assert "[errback] status 500" in str(log) assert "[errback] status 501" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_process_request_cb_kwargs(self): crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) with LogCapture() as log: @@ -499,7 +496,7 @@ class TestCrawlSpider(TestCase): assert "[parse] status 201 (foo: process_request)" in str(log) assert "[parse] status 202 (foo: bar)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_parse(self): crawler = get_crawler(AsyncDefSpider) with LogCapture() as log: @@ -509,7 +506,7 @@ class TestCrawlSpider(TestCase): assert "Got response 200" in str(log) @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse(self): crawler = get_crawler( AsyncDefAsyncioSpider, @@ -524,7 +521,7 @@ class TestCrawlSpider(TestCase): assert "Got response 200" in str(log) @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_items_list(self): log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) assert "Got response 200" in str(log) @@ -532,7 +529,7 @@ class TestCrawlSpider(TestCase): assert {"id": 2} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): items = [] @@ -549,7 +546,7 @@ class TestCrawlSpider(TestCase): assert {"foo": 42} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse(self): log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) assert "Got response 200" in str(log) @@ -557,7 +554,7 @@ class TestCrawlSpider(TestCase): assert itemcount == 1 @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_loop(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) assert "Got response 200" in str(log) @@ -567,7 +564,7 @@ class TestCrawlSpider(TestCase): assert {"foo": i} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_exc(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) log = str(log) @@ -579,7 +576,7 @@ class TestCrawlSpider(TestCase): assert {"foo": i} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_complex(self): _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) itemcount = stats.get_value("item_scraped_count") @@ -591,37 +588,37 @@ class TestCrawlSpider(TestCase): assert {"index2": i} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): assert f"Got response 200, req_id {req_id}" in str(log) @pytest.mark.only_not_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_deferred_direct(self): _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) assert items == [{"code": 200}] @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_deferred_wrapped(self): log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) assert items == [{"code": 200}] - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_deferred_maybe_wrapped(self): _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider) assert items == [{"code": 200}] - @defer.inlineCallbacks + @inlineCallbacks def test_response_ssl_certificate_none(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=False) yield crawler.crawl(seed=url, mockserver=self.mockserver) assert crawler.spider.meta["responses"][0].certificate is None - @defer.inlineCallbacks + @inlineCallbacks def test_response_ssl_certificate(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=True) @@ -634,7 +631,7 @@ class TestCrawlSpider(TestCase): @pytest.mark.xfail( reason="Responses with no body return early and contain no certificate" ) - @defer.inlineCallbacks + @inlineCallbacks def test_response_ssl_certificate_empty_response(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/status?n=200", is_secure=True) @@ -644,7 +641,7 @@ class TestCrawlSpider(TestCase): assert cert.getSubject().commonName == b"localhost" assert cert.getIssuer().commonName == b"localhost" - @defer.inlineCallbacks + @inlineCallbacks def test_dns_server_ip_address_none(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/status?n=200") @@ -652,7 +649,7 @@ class TestCrawlSpider(TestCase): ip_address = crawler.spider.meta["responses"][0].ip_address assert ip_address is None - @defer.inlineCallbacks + @inlineCallbacks def test_dns_server_ip_address(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test") @@ -662,7 +659,7 @@ class TestCrawlSpider(TestCase): assert isinstance(ip_address, IPv4Address) assert str(ip_address) == gethostbyname(expected_netloc) - @defer.inlineCallbacks + @inlineCallbacks def test_bytes_received_stop_download_callback(self): crawler = get_crawler(BytesReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -676,7 +673,7 @@ class TestCrawlSpider(TestCase): < crawler.spider.full_response_length ) - @defer.inlineCallbacks + @inlineCallbacks def test_bytes_received_stop_download_errback(self): crawler = get_crawler(BytesReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -692,7 +689,7 @@ class TestCrawlSpider(TestCase): < crawler.spider.full_response_length ) - @defer.inlineCallbacks + @inlineCallbacks def test_headers_received_stop_download_callback(self): crawler = get_crawler(HeadersReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -702,7 +699,7 @@ class TestCrawlSpider(TestCase): "headers_received" ) - @defer.inlineCallbacks + @inlineCallbacks def test_headers_received_stop_download_errback(self): crawler = get_crawler(HeadersReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -714,7 +711,7 @@ class TestCrawlSpider(TestCase): "failure" ].value.response.headers == crawler.spider.meta.get("headers_received") - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback(self): failures = [] @@ -731,7 +728,7 @@ class TestCrawlSpider(TestCase): assert "HTTP status code is not handled or not allowed" in str(log) assert "Spider error processing" not in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_silence(self): failures = [] @@ -747,7 +744,7 @@ class TestCrawlSpider(TestCase): assert "HTTP status code is not handled or not allowed" not in str(log) assert "Spider error processing" not in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -759,7 +756,7 @@ class TestCrawlSpider(TestCase): ) assert "Spider error processing" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_item(self): def eb(failure: Failure) -> Any: return {"foo": "bar"} @@ -773,7 +770,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) assert "'item_scraped_count': 1" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_request(self): def eb(failure: Failure) -> Request: return Request(self.mockserver.url("/")) @@ -787,7 +784,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) assert "Crawled (200)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_downloader_error(self): failures = [] @@ -804,7 +801,7 @@ class TestCrawlSpider(TestCase): assert "Error downloading" in str(log) assert "Spider error processing" not in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_downloader_error_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -817,7 +814,7 @@ class TestCrawlSpider(TestCase): assert "Error downloading" in str(log) assert "Spider error processing" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_downloader_error_item(self): def eb(failure: Failure) -> Any: return {"foo": "bar"} @@ -831,7 +828,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) assert "'item_scraped_count': 1" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_downloader_error_request(self): def eb(failure: Failure) -> Request: return Request(self.mockserver.url("/")) @@ -845,7 +842,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) assert "Crawled (200)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_raise_closespider(self): def cb(response): raise CloseSpider @@ -856,7 +853,7 @@ class TestCrawlSpider(TestCase): assert "Closing spider (cancelled)" in str(log) assert "Spider error processing" not in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_raise_closespider_reason(self): def cb(response): raise CloseSpider("my_reason") diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index c2df67c66..4436efd9b 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,25 +1,13 @@ import os import re from configparser import ConfigParser -from importlib import import_module from pathlib import Path import pytest from twisted import version as twisted_version -from twisted.trial import unittest class TestScrapyUtils: - def test_required_openssl_version(self): - try: - module = import_module("OpenSSL") - except ImportError: - raise unittest.SkipTest("OpenSSL is not available") - - if hasattr(module, "__version__"): - installed_version = [int(x) for x in module.__version__.split(".")[:2]] - assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" - def test_pinned_twisted_version(self): """When running tests within a Tox environment with pinned dependencies, make sure that the version of Twisted is the pinned diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 09cdbaf35..2c8e96040 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -440,7 +440,7 @@ class TestFTPBase(unittest.TestCase): class TestFTP(TestFTPBase): def test_invalid_credentials(self): if self.reactor_pytest != "default" and sys.platform == "win32": - raise unittest.SkipTest( + pytest.skip( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) from twisted.protocols.ftp import ConnectionLost diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 14e12a3e6..9b2c49fd4 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -14,7 +14,7 @@ from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer, error -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import inlineCallbacks, maybeDeferred from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util @@ -186,7 +186,7 @@ class TestHttpBase(unittest.TestCase, ABC): self.download_handler_cls, get_crawler() ) - @defer.inlineCallbacks + @inlineCallbacks def tearDown(self): yield self.port.stopListening() if hasattr(self.download_handler, "close"): @@ -229,7 +229,7 @@ class TestHttpBase(unittest.TestCase, ABC): async def test_timeout_download_from_spider_nodata_rcvd(self): if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( + pytest.skip( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) @@ -245,7 +245,7 @@ class TestHttpBase(unittest.TestCase, ABC): async def test_timeout_download_from_spider_server_hangs(self): if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( + pytest.skip( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) # client connects, server send headers and some body bytes but hangs @@ -531,7 +531,7 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC): crawler = get_crawler(settings_dict=settings_dict) self.download_handler = build_from_crawler(self.download_handler_cls, crawler) - @defer.inlineCallbacks + @inlineCallbacks def tearDown(self): yield self.port.stopListening() if hasattr(self.download_handler, "close"): @@ -665,7 +665,7 @@ class TestHttpProxyBase(unittest.TestCase, ABC): self.download_handler_cls, get_crawler() ) - @defer.inlineCallbacks + @inlineCallbacks def tearDown(self): yield self.port.stopListening() if hasattr(self.download_handler, "close"): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index e7427c5ac..3c26b242f 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -2,7 +2,6 @@ from gzip import GzipFile from io import BytesIO from logging import WARNING from pathlib import Path -from unittest import SkipTest import pytest from testfixtures import LogCapture @@ -130,7 +129,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") response = self._getresponse("br") request = response.request assert response.headers["Content-Encoding"] == b"br" @@ -146,11 +145,11 @@ class TestHttpCompression: try: import brotli # noqa: F401 - raise SkipTest("Requires not having brotli support") + pytest.skip("Requires not having brotli support") except ImportError: import brotlicffi # noqa: F401 - raise SkipTest("Requires not having brotli support") + pytest.skip("Requires not having brotli support") except ImportError: pass response = self._getresponse("br") @@ -180,7 +179,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") raw_content = None for check_key in FORMAT: if not check_key.startswith("zstd-"): @@ -201,7 +200,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 - raise SkipTest("Requires not having zstandard support") + pytest.skip("Requires not having zstandard support") except ImportError: pass response = self._getresponse("zstd-static-content-size") @@ -520,7 +519,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_compression_bomb_setting("br") def test_compression_bomb_setting_deflate(self): @@ -533,7 +532,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_compression_bomb_setting("zstd") def _test_compression_bomb_spider_attr(self, compression_id): @@ -556,7 +555,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_compression_bomb_spider_attr("br") def test_compression_bomb_spider_attr_deflate(self): @@ -569,7 +568,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_compression_bomb_spider_attr("zstd") def _test_compression_bomb_request_meta(self, compression_id): @@ -590,7 +589,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_compression_bomb_request_meta("br") def test_compression_bomb_request_meta_deflate(self): @@ -603,7 +602,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_compression_bomb_request_meta("zstd") def _test_download_warnsize_setting(self, compression_id): @@ -639,7 +638,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_download_warnsize_setting("br") def test_download_warnsize_setting_deflate(self): @@ -652,7 +651,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_download_warnsize_setting("zstd") def _test_download_warnsize_spider_attr(self, compression_id): @@ -690,7 +689,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_download_warnsize_spider_attr("br") def test_download_warnsize_spider_attr_deflate(self): @@ -703,7 +702,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_download_warnsize_spider_attr("zstd") def _test_download_warnsize_request_meta(self, compression_id): @@ -739,7 +738,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_download_warnsize_request_meta("br") def test_download_warnsize_request_meta_deflate(self): @@ -752,5 +751,5 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_download_warnsize_request_meta("zstd") diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 04800896c..146b0057e 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -235,12 +235,10 @@ Disallow: /some/randome/page.html self, request: Request, middleware: RobotsTxtMiddleware ) -> None: spider = None # not actually used - await maybe_deferred_to_future( - self.assertFailure( - middleware.process_request(request, spider), # type: ignore[arg-type] - IgnoreRequest, + with pytest.raises(IgnoreRequest): + await maybe_deferred_to_future( + maybeDeferred(middleware.process_request, request, spider) # type: ignore[call-overload] ) - ) def assertRobotsTxtRequested(self, base_url: str) -> None: calls = self.crawler.engine.download.call_args_list diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 78c83ea83..9b7c09448 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -1,6 +1,6 @@ import time -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase from scrapy import Request @@ -62,7 +62,7 @@ class CrawlTestCase(TestCase): def setUp(self): self.runner = CrawlerRunner() - @defer.inlineCallbacks + @inlineCallbacks def test_delay(self): crawler = get_crawler(DownloaderSlotsSettingsTestSpider) yield crawler.crawl(mockserver=self.mockserver) diff --git a/tests/test_engine.py b/tests/test_engine.py index 9f618437c..e181a36cf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -26,6 +26,7 @@ import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from twisted.web import server, static, util @@ -390,7 +391,7 @@ class TestEngineBase(unittest.TestCase): class TestEngine(TestEngineBase): - @defer.inlineCallbacks + @inlineCallbacks def test_crawler(self): for spider in ( MySpider, @@ -407,20 +408,20 @@ class TestEngine(TestEngineBase): self._assert_signals_caught(run) self._assert_bytes_received(run) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_dupefilter(self): run = CrawlerRun(DupeFilterSpider) yield run.run() self._assert_scheduled_requests(run, count=8) self._assert_dropped_requests(run) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_itemerror(self): run = CrawlerRun(ItemZeroDivisionErrorSpider) yield run.run() self._assert_items_error(run) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_change_close_reason_on_idle(self): run = CrawlerRun(ChangeCloseReasonSpider) yield run.run() @@ -429,7 +430,7 @@ class TestEngine(TestEngineBase): "reason": "custom_reason", } == run.signals_caught[signals.spider_closed] - @defer.inlineCallbacks + @inlineCallbacks def test_close_downloader(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.close() @@ -447,19 +448,14 @@ class TestEngine(TestEngineBase): get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None ) - @defer.inlineCallbacks + @inlineCallbacks def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.open_spider(MySpider(), []) e.start() - - def cb(exc: BaseException) -> None: - assert str(exc), "Engine already running" - - try: - yield self.assertFailure(e.start(), RuntimeError).addBoth(cb) - finally: - yield e.stop() + with pytest.raises(RuntimeError, match="Engine already running"): + yield e.start() + yield e.stop() def test_short_timeout(self): args = ( diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index f09b0e091..2662e45e1 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -1,5 +1,5 @@ from testfixtures import LogCapture -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from scrapy.exceptions import StopDownload from tests.test_engine import ( @@ -19,7 +19,7 @@ class BytesReceivedCrawlerRun(CrawlerRun): class TestBytesReceivedEngine(TestEngineBase): - @defer.inlineCallbacks + @inlineCallbacks def test_crawler(self): for spider in ( MySpider, diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index dbb0ea0d2..142715927 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -1,5 +1,5 @@ from testfixtures import LogCapture -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from scrapy.exceptions import StopDownload from tests.test_engine import ( @@ -19,7 +19,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun): class TestHeadersReceivedEngine(TestEngineBase): - @defer.inlineCallbacks + @inlineCallbacks def test_crawler(self): for spider in ( MySpider, diff --git a/tests/test_exporters.py b/tests/test_exporters.py index f55cb6c97..05e8865bc 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -4,7 +4,6 @@ import marshal import pickle import re import tempfile -import unittest from datetime import datetime from io import BytesIO from typing import Any @@ -662,7 +661,7 @@ class TestCustomExporterItem: def setup_method(self): if self.item_class is None: - raise unittest.SkipTest("item class is None") + pytest.skip("item class is None") def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 85bd42857..b86f3c7f2 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,7 +1,6 @@ from __future__ import annotations import datetime -import unittest from typing import Any, Callable from scrapy.extensions.periodic_log import PeriodicLog @@ -66,7 +65,7 @@ def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog: return CustomPeriodicLog.from_crawler(crawler) -class TestPeriodicLog(unittest.TestCase): +class TestPeriodicLog: def test_extension_enabled(self): # Expected that settings for this extension loaded successfully # And on certain conditions - extension raising NotConfigured diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 8c897c223..2ac4d7830 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -1,6 +1,7 @@ +import pytest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from scrapy.extensions.telnet import TelnetConsole @@ -21,15 +22,16 @@ class TelnetExtensionTest(unittest.TestCase): return console, portal - @defer.inlineCallbacks + @inlineCallbacks def test_bad_credentials(self): console, portal = self._get_console_and_portal() creds = credentials.UsernamePassword(b"username", b"password") d = portal.login(creds, None, ITelnetProtocol) - yield self.assertFailure(d, ValueError) + with pytest.raises(ValueError, match="Invalid credentials"): + yield d console.stop_listening() - @defer.inlineCallbacks + @inlineCallbacks def test_good_credentials(self): console, portal = self._get_console_and_portal() creds = credentials.UsernamePassword( @@ -39,7 +41,7 @@ class TelnetExtensionTest(unittest.TestCase): yield d console.stop_listening() - @defer.inlineCallbacks + @inlineCallbacks def test_custom_credentials(self): settings = { "TELNETCONSOLE_USERNAME": "user", diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 44cd10ec3..cdf03ca76 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -26,6 +26,7 @@ import lxml.etree import pytest from testfixtures import LogCapture from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from w3lib.url import file_uri_to_path, path_to_file_uri from zope.interface import implementer @@ -131,7 +132,7 @@ class TestFileFeedStorage(unittest.TestCase): FileFeedStorage(str(path), feed_options={"overwrite": True}), path ) - @defer.inlineCallbacks + @inlineCallbacks def _assert_stores(self, storage, path: Path, expected_content=b"content"): spider = scrapy.Spider("default") file = storage.open(spider) @@ -172,7 +173,7 @@ class TestFTPFeedStorage(unittest.TestCase): finally: path.unlink() - @defer.inlineCallbacks + @inlineCallbacks def test_append(self): with MockFTPServer() as ftp_server: filename = "file" @@ -182,7 +183,7 @@ class TestFTPFeedStorage(unittest.TestCase): yield self._store(url, b"bar", feed_options=feed_options) self._assert_stored(ftp_server.path / filename, b"foobar") - @defer.inlineCallbacks + @inlineCallbacks def test_overwrite(self): with MockFTPServer() as ftp_server: filename = "file" @@ -191,7 +192,7 @@ class TestFTPFeedStorage(unittest.TestCase): yield self._store(url, b"bar") self._assert_stored(ftp_server.path / filename, b"bar") - @defer.inlineCallbacks + @inlineCallbacks def test_append_active_mode(self): with MockFTPServer() as ftp_server: settings = {"FEED_STORAGE_FTP_ACTIVE": True} @@ -202,7 +203,7 @@ class TestFTPFeedStorage(unittest.TestCase): yield self._store(url, b"bar", feed_options=feed_options, settings=settings) self._assert_stored(ftp_server.path / filename, b"foobar") - @defer.inlineCallbacks + @inlineCallbacks def test_overwrite_active_mode(self): with MockFTPServer() as ftp_server: settings = {"FEED_STORAGE_FTP_ACTIVE": True} @@ -290,7 +291,7 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.access_key == "uri_key" assert storage.secret_key == "uri_secret" - @defer.inlineCallbacks + @inlineCallbacks def test_store(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -431,7 +432,7 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.region_name == region_name assert storage.s3_client._client_config.region_name == region_name - @defer.inlineCallbacks + @inlineCallbacks def test_store_without_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", @@ -451,7 +452,7 @@ class TestS3FeedStorage(unittest.TestCase): ) assert acl is None - @defer.inlineCallbacks + @inlineCallbacks def test_store_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" @@ -489,7 +490,7 @@ class TestGCSFeedStorage(unittest.TestCase): try: from google.cloud.storage import Client # noqa: F401 except ImportError: - raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + pytest.skip("GCSFeedStorage requires google-cloud-storage") settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"} crawler = get_crawler(settings_dict=settings) @@ -503,7 +504,7 @@ class TestGCSFeedStorage(unittest.TestCase): try: from google.cloud.storage import Client # noqa: F401 except ImportError: - raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + pytest.skip("GCSFeedStorage requires google-cloud-storage") settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} crawler = get_crawler(settings_dict=settings) @@ -515,12 +516,12 @@ class TestGCSFeedStorage(unittest.TestCase): storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") assert storage.acl is None - @defer.inlineCallbacks + @inlineCallbacks def test_store(self): try: from google.cloud.storage import Client # noqa: F401 except ImportError: - raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + pytest.skip("GCSFeedStorage requires google-cloud-storage") uri = "gs://mybucket/export.csv" project_id = "myproject-123" @@ -556,7 +557,7 @@ class TestGCSFeedStorage(unittest.TestCase): class TestStdoutFeedStorage(unittest.TestCase): - @defer.inlineCallbacks + @inlineCallbacks def test_store(self): out = BytesIO() storage = StdoutFeedStorage("stdout:", _stdout=out) @@ -669,7 +670,7 @@ class TestFeedExportBase(ABC, unittest.TestCase): def tearDown(self): shutil.rmtree(self.temp_dir, ignore_errors=True) - @defer.inlineCallbacks + @inlineCallbacks def exported_data(self, items, settings): """ Return exported data which a spider yielding ``items`` would return. @@ -684,7 +685,7 @@ class TestFeedExportBase(ABC, unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) return data - @defer.inlineCallbacks + @inlineCallbacks def exported_no_data(self, settings): """ Return exported data which a spider yielding no ``items`` would return. @@ -699,7 +700,7 @@ class TestFeedExportBase(ABC, unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) return data - @defer.inlineCallbacks + @inlineCallbacks def assertExported(self, items, header, rows, settings=None): yield self.assertExportedCsv(items, header, rows, settings) yield self.assertExportedJsonLines(items, rows, settings) @@ -770,7 +771,7 @@ class ExceptionJsonItemExporter(JsonItemExporter): class TestFeedExport(TestFeedExportBase): - @defer.inlineCallbacks + @inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data.""" @@ -800,7 +801,7 @@ class TestFeedExport(TestFeedExportBase): return content - @defer.inlineCallbacks + @inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update( @@ -815,7 +816,7 @@ class TestFeedExport(TestFeedExportBase): assert reader.fieldnames == list(header) assert rows == list(reader) - @defer.inlineCallbacks + @inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -830,7 +831,7 @@ class TestFeedExport(TestFeedExportBase): rows = [{k: v for k, v in row.items() if v} for row in rows] assert rows == parsed - @defer.inlineCallbacks + @inlineCallbacks def assertExportedXml(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -846,7 +847,7 @@ class TestFeedExport(TestFeedExportBase): got_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] assert rows == got_rows - @defer.inlineCallbacks + @inlineCallbacks def assertExportedMultiple(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -867,7 +868,7 @@ class TestFeedExport(TestFeedExportBase): json_rows = json.loads(to_unicode(data["json"])) assert rows == json_rows - @defer.inlineCallbacks + @inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -884,7 +885,7 @@ class TestFeedExport(TestFeedExportBase): result = self._load_until_eof(data["pickle"], load_func=pickle.load) assert result == expected - @defer.inlineCallbacks + @inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -901,7 +902,7 @@ class TestFeedExport(TestFeedExportBase): result = self._load_until_eof(data["marshal"], load_func=marshal.load) assert result == expected - @defer.inlineCallbacks + @inlineCallbacks def test_stats_file_success(self): settings = { "FEEDS": { @@ -915,7 +916,7 @@ class TestFeedExport(TestFeedExportBase): assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_stats_file_failed(self): settings = { "FEEDS": { @@ -933,7 +934,7 @@ class TestFeedExport(TestFeedExportBase): assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_stats_multiple_file(self): settings = { "FEEDS": { @@ -955,7 +956,7 @@ class TestFeedExport(TestFeedExportBase): crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1 ) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items(self): # feed exporters use field names from Item items = [ @@ -969,7 +970,7 @@ class TestFeedExport(TestFeedExportBase): header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows) - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { @@ -981,7 +982,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_no_data(settings) assert data[fmt] is None - @defer.inlineCallbacks + @inlineCallbacks def test_start_finish_exporting_items(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1001,7 +1002,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @defer.inlineCallbacks + @inlineCallbacks def test_start_finish_exporting_no_items(self): items = [] settings = { @@ -1019,7 +1020,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @defer.inlineCallbacks + @inlineCallbacks def test_start_finish_exporting_items_exception(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1040,7 +1041,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @defer.inlineCallbacks + @inlineCallbacks def test_start_finish_exporting_no_items_exception(self): items = [] settings = { @@ -1059,7 +1060,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), @@ -1079,7 +1080,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_no_data(settings) assert expctd == data[fmt] - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_multiple_feeds(self): """Make sure that `storage.store` is called for every feed.""" settings = { @@ -1097,7 +1098,7 @@ class TestFeedExport(TestFeedExportBase): assert str(log).count("Storage.store is called") == 0 - @defer.inlineCallbacks + @inlineCallbacks def test_export_multiple_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1119,7 +1120,7 @@ class TestFeedExport(TestFeedExportBase): yield self.assertExportedCsv(items, header, rows_csv) yield self.assertExportedJsonLines(items, rows_jl) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_empty_field_list(self): # FEED_EXPORT_FIELDS==[] means the same as default None items = [{"foo": "bar"}] @@ -1129,7 +1130,7 @@ class TestFeedExport(TestFeedExportBase): yield self.assertExportedCsv(items, header, rows) yield self.assertExportedJsonLines(items, rows, settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] @@ -1137,7 +1138,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": header} yield self.assertExported(items, header, rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_comma_separated_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] @@ -1145,7 +1146,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": ",".join(header)} yield self.assertExported(items, header, rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_json_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] @@ -1153,7 +1154,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} yield self.assertExported(items, header, rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} @@ -1161,7 +1162,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": header} yield self.assertExported(items, list(header.values()), rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_dict_field_names(self): items = [{"foo": "bar"}] header = { @@ -1172,7 +1173,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": header} yield self.assertExported(items, ["Baz", "Foo"], rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_json_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} @@ -1180,7 +1181,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} yield self.assertExported(items, list(header.values()), rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_based_on_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1226,7 +1227,7 @@ class TestFeedExport(TestFeedExportBase): for fmt, expected in formats.items(): assert data[fmt] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_export_based_on_custom_filters(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1285,7 +1286,7 @@ class TestFeedExport(TestFeedExportBase): for fmt, expected in formats.items(): assert data[fmt] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_export_dicts(self): # When dicts are used, only keys from the first row are used as # a header for CSV, and all fields are used for JSON Lines. @@ -1298,7 +1299,7 @@ class TestFeedExport(TestFeedExportBase): yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) - @defer.inlineCallbacks + @inlineCallbacks def test_export_tuple(self): items = [ {"foo": "bar1", "egg": "spam1"}, @@ -1309,7 +1310,7 @@ class TestFeedExport(TestFeedExportBase): rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}] yield self.assertExported(items, ["foo", "baz"], rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_feed_export_fields(self): # FEED_EXPORT_FIELDS option allows to order export fields # and to select a subset of fields to export, both for Items and dicts. @@ -1335,7 +1336,7 @@ class TestFeedExport(TestFeedExportBase): rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}] yield self.assertExported(items, ["egg", "baz"], rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_encoding(self): items = [{"foo": "Test\xd6"}] @@ -1380,7 +1381,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_data(items, settings) assert data[fmt] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_export_multiple_configs(self): items = [{"foo": "FOO", "bar": "BAR"}] @@ -1420,7 +1421,7 @@ class TestFeedExport(TestFeedExportBase): for fmt, expected in formats.items(): assert data[fmt] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_export_indentation(self): items = [ {"foo": ["bar"]}, @@ -1576,7 +1577,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_data(items, settings) assert data[row["format"]] == row["expected"] - @defer.inlineCallbacks + @inlineCallbacks def test_init_exporters_storages_with_crawler(self): settings = { "FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter}, @@ -1589,7 +1590,7 @@ class TestFeedExport(TestFeedExportBase): assert FromCrawlerCsvItemExporter.init_with_crawler assert FromCrawlerFileFeedStorage.init_with_crawler - @defer.inlineCallbacks + @inlineCallbacks def test_str_uri(self): settings = { "FEED_STORE_EMPTY": True, @@ -1598,7 +1599,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_no_data(settings) assert data["csv"] == b"" - @defer.inlineCallbacks + @inlineCallbacks def test_multiple_feeds_success_logs_blocking_feed_storage(self): settings = { "FEEDS": { @@ -1619,7 +1620,7 @@ class TestFeedExport(TestFeedExportBase): for fmt in ["json", "xml", "csv"]: assert f"Stored {fmt} feed (2 items)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_multiple_feeds_failing_logs_blocking_feed_storage(self): settings = { "FEEDS": { @@ -1640,7 +1641,7 @@ class TestFeedExport(TestFeedExportBase): for fmt in ["json", "xml", "csv"]: assert f"Error storing {fmt} feed (2 items)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_extend_kwargs(self): items = [{"foo": "FOO", "bar": "BAR"}] @@ -1677,7 +1678,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_data(items, settings) assert data[feed_options["format"]] == row["expected"] - @defer.inlineCallbacks + @inlineCallbacks def test_storage_file_no_postprocessing(self): @implementer(IFeedStorage) class Storage: @@ -1699,7 +1700,7 @@ class TestFeedExport(TestFeedExportBase): yield self.exported_no_data(settings) assert Storage.open_file is Storage.store_file - @defer.inlineCallbacks + @inlineCallbacks def test_storage_file_postprocessing(self): @implementer(IFeedStorage) class Storage: @@ -1752,7 +1753,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): def _named_tempfile(self, name) -> str: return str(Path(self.temp_dir, name)) - @defer.inlineCallbacks + @inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data with filename.""" @@ -1796,7 +1797,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data_stream.seek(0) return data_stream.read() - @defer.inlineCallbacks + @inlineCallbacks def test_gzip_plugin(self): filename = self._named_tempfile("gzip_file") @@ -1815,7 +1816,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): except OSError: pytest.fail("Received invalid gzip data.") - @defer.inlineCallbacks + @inlineCallbacks def test_gzip_plugin_compresslevel(self): filename_to_compressed = { self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( @@ -1852,7 +1853,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_gzip_plugin_mtime(self): filename_to_compressed = { self._named_tempfile("mtime_123"): self.get_gzip_compressed( @@ -1887,7 +1888,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_gzip_plugin_filename(self): filename_to_compressed = { self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( @@ -1922,7 +1923,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin(self): filename = self._named_tempfile("lzma_file") @@ -1941,7 +1942,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): except lzma.LZMAError: pytest.fail("Received invalid lzma data.") - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin_format(self): filename_to_compressed = { self._named_tempfile("format_FORMAT_XZ"): lzma.compress( @@ -1974,7 +1975,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin_check(self): filename_to_compressed = { self._named_tempfile("check_CHECK_NONE"): lzma.compress( @@ -2007,7 +2008,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin_preset(self): filename_to_compressed = { self._named_tempfile("preset_PRESET_0"): lzma.compress( @@ -2040,11 +2041,11 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin_filters(self): if "PyPy" in sys.version: # https://foss.heptapod.net/pypy/pypy/-/issues/3527 - raise unittest.SkipTest("lzma filters doesn't work in PyPy") + pytest.skip("lzma filters doesn't work in PyPy") filters = [{"id": lzma.FILTER_LZMA2}] compressed = lzma.compress(self.expected, filters=filters) @@ -2065,7 +2066,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): result = lzma.decompress(data[filename]) assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_bz2_plugin(self): filename = self._named_tempfile("bz2_file") @@ -2084,7 +2085,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): except OSError: pytest.fail("Received invalid bz2 data.") - @defer.inlineCallbacks + @inlineCallbacks def test_bz2_plugin_compresslevel(self): filename_to_compressed = { self._named_tempfile("compresslevel_1"): bz2.compress( @@ -2117,7 +2118,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_custom_plugin(self): filename = self._named_tempfile("csv_file") @@ -2133,7 +2134,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data = yield self.exported_data(self.items, settings) assert data[filename] == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_custom_plugin_with_parameter(self): expected = b"foo\r\n\nbar\r\n\n" filename = self._named_tempfile("newline") @@ -2151,7 +2152,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data = yield self.exported_data(self.items, settings) assert data[filename] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_custom_plugin_with_compression(self): expected = b"foo\r\n\nbar\r\n\n" @@ -2196,7 +2197,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): result = decompressor(data[filename]) assert result == expected - @defer.inlineCallbacks + @inlineCallbacks def test_exports_compatibility_with_postproc(self): import marshal import pickle @@ -2254,7 +2255,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): class TestBatchDeliveries(TestFeedExportBase): _file_mark = "_%(batch_time)s_#%(batch_id)02d_" - @defer.inlineCallbacks + @inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data.""" @@ -2276,7 +2277,7 @@ class TestBatchDeliveries(TestFeedExportBase): content[feed["format"]].append(file.read_bytes()) return content - @defer.inlineCallbacks + @inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2298,7 +2299,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update( @@ -2318,7 +2319,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert list(got_batch) == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedXml(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2339,7 +2340,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedMultiple(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2371,7 +2372,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2393,7 +2394,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2415,7 +2416,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def test_export_items(self): """Test partial deliveries in all supported formats""" items = [ @@ -2444,7 +2445,7 @@ class TestBatchDeliveries(TestFeedExportBase): with pytest.raises(NotConfigured): FeedExporter(crawler) - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { @@ -2460,7 +2461,7 @@ class TestBatchDeliveries(TestFeedExportBase): data = dict(data) assert len(data[fmt]) == 0 - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), @@ -2484,7 +2485,7 @@ class TestBatchDeliveries(TestFeedExportBase): data = dict(data) assert data[fmt][0] == expctd - @defer.inlineCallbacks + @inlineCallbacks def test_export_multiple_configs(self): items = [ {"foo": "FOO", "bar": "BAR"}, @@ -2540,7 +2541,7 @@ class TestBatchDeliveries(TestFeedExportBase): for expected_batch, got_batch in zip(expected, data[fmt]): assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def test_batch_item_count_feeds_setting(self): items = [{"foo": "FOO"}, {"foo": "FOO1"}] formats = { @@ -2564,7 +2565,7 @@ class TestBatchDeliveries(TestFeedExportBase): for expected_batch, got_batch in zip(expected, data[fmt]): assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. @@ -2586,7 +2587,7 @@ class TestBatchDeliveries(TestFeedExportBase): data = yield self.exported_data(items, settings) assert len(items) == len(data["json"]) - @defer.inlineCallbacks + @inlineCallbacks def test_stats_batch_file_success(self): settings = { "FEEDS": { @@ -2604,7 +2605,7 @@ class TestBatchDeliveries(TestFeedExportBase): assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 @pytest.mark.requires_boto3 - @defer.inlineCallbacks + @inlineCallbacks def test_s3_export(self): bucket = "mybucket" items = [ diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0605c2438..ef1806cc0 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -9,7 +9,7 @@ from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp from typing import TYPE_CHECKING -from unittest import mock, skipIf +from unittest import mock from urllib.parse import urlencode import pytest @@ -183,7 +183,7 @@ def get_client_certificate( return PrivateCertificate.loadPEM(pem) -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") +@pytest.mark.skipif(not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled") class TestHttps2ClientProtocol(TestCase): scheme = "https" key_file = Path(__file__).parent / "keys" / "localhost.key" diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 3c9f97631..047f8c610 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -2,7 +2,7 @@ import logging import pytest from testfixtures import LogCapture -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.python.failure import Failure from twisted.trial.unittest import TestCase @@ -272,7 +272,7 @@ class TestShowOrSkipMessages(TestCase): }, } - @defer.inlineCallbacks + @inlineCallbacks def test_show_messages(self): crawler = get_crawler(ItemSpider, self.base_settings) with LogCapture() as lc: @@ -281,7 +281,7 @@ class TestShowOrSkipMessages(TestCase): assert "Crawled (200) Date: Sat, 7 Jun 2025 09:17:36 +0200 Subject: [PATCH 1769/2083] Remove the old artwork (#6874) --- artwork/README.rst | 20 -------------------- artwork/qlassik.zip | Bin 120204 -> 0 bytes artwork/scrapy-blog-logo.xcf | Bin 52428 -> 0 bytes artwork/scrapy-logo.jpg | Bin 23398 -> 0 bytes 4 files changed, 20 deletions(-) delete mode 100644 artwork/README.rst delete mode 100644 artwork/qlassik.zip delete mode 100644 artwork/scrapy-blog-logo.xcf delete mode 100644 artwork/scrapy-logo.jpg diff --git a/artwork/README.rst b/artwork/README.rst deleted file mode 100644 index c1880ef6c..000000000 --- a/artwork/README.rst +++ /dev/null @@ -1,20 +0,0 @@ -============== -Scrapy artwork -============== - -This folder contains the Scrapy artwork resources such as logos and fonts. - -scrapy-logo.jpg ---------------- - -The main Scrapy logo, in JPEG format. - -qlassik.zip ------------ - -The font used for the Scrapy logo. Homepage: https://www.dafont.com/qlassik.font - -scrapy-blog.logo.xcf --------------------- - -The logo used in the Scrapy blog, in Gimp format. diff --git a/artwork/qlassik.zip b/artwork/qlassik.zip deleted file mode 100644 index 2885c06ef4bab2fd9027bf748bd5ad2a69eb857f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 120204 zcmV(+K;6GkO9KQH000080H&bGI*MgI(|wQt0Pr0F01f~E08wmVb8~5HUsOUabaZCy zy?LA@MVU7q5qaO0m06W_SJvHKUENh(-Cfo9HQjUH1H%l*a1Q4TH;5M^iU%U#feN0i zB7(Z2;_HYgq9U&Ax*m8dBI1h2x+-|X)cZtaRZq`=``gd&^ZvO*=Tli(RT=R-@f_dh ziHHPYgir_wkI=g9+jf2LW50Mk!l@?^!s@nN_2!3ub^U%W*@qDl zpF~J_?979wUp)QD+@BEAzl@N)^~@_T_mK^nF7>3W^jL=`^&)t9eS^qWhid*3`9lqZ>7cPje;ywlMm*9Qz+=G{2m3?3R9(YgN zLl^A7^g`iD;VOjfJ#ejj;G#26-}jWi7Gd)dXztpBr(bn3w-f&nK06nF?_YTO!Tn#o z|K{H!9KHsAFI{}mWtR{C$Nm<=9`GaVz4+4o7au}s0Qz$Sq7tC8z-V#J)B2OATGM|- z3il-C=1V*Gd?0xJMXU3f4&v_P-UM$IL{RV#en)=W*^M&XkKyxW?ml*>RsZljy#n;* zNb3Cp{;BXgkPAEK@V)qHU=or0$@TF5hTs+d7&Xz~WJ{2G_qqm=k8DT&#@_pjUwA3I z!{14_4{yTAM>xWo`7n71UVLcgXWg&=EI9D}CdHx2_?{{Q4Wuz88`~~^Jo}9 zg&OFSNWn!^fY%bA70DoKkZHK*ooJlQ zqda~HK6@4P`zy#GeW;CpjSP-MQT%(P;jh8*E$HLf=mNYJ#o;xDKMB&Ne7z2?KZM5d z#mL7G!*L@V5@`J$NCIBl=qqRxeXa8ud;@Bse?e_>C#vHw!trU=Rs(+-zF&j)C&K#~ zXzyV(3Kn6LQ`tQ%{A#4)^FS6&bP0M84FI1T@SmZ-_ahCx1=`;Y#~0yq~t2dZ*B$n+AFhI_a1*HMYw3;q5*jM<|wPIOFefHvssQ)mlW z1Ub7%eZGT^6@EF4)2ai;sf7N$^E)`6fa4F{G4i`du6ItyiH_0ggN;#doL=`l$Lry^ z0)BhL(FgD`guc@GBOJei$`vo4`iO%x(g?Pf(dNnOuT$OlLtJ@V-fs$iN?k@jnUlgxVO@b$Y-# z`6lpAZF~@H4GwBwqBrKdYld4 ze+K%)gOB+*yr(wzMKr?fj@s*iZu<{|%&DzC%>2d#IT6}E8P4B}A|Uf|kjXgc_XNmt zg0*)kdK;dEdzC;|4}$(spNr`~q@%yc9V8>U^DOvq1*nQtpc>ME>Oh}C2GW6=$N*{~ z6KDuooxdX+g@A^U4b(wlpe}Mce?t-E0*#^wP!C0c#(@46#gPXzfnq?DC=N7*5}m(* zZ6<+cPzq=krGfe=)A=*>H48M4e4quC16l<7C$PCZ&@w6jt)L>%K2!pW=tpIsRa5~w zfciR5qZ;Z5I*6)3htL4fI;wT3H>Y$M4FYYVA)q6u-uVM)Q3L208V1@zO`vTw()m3a zN25R|&=}B3)B-vM^mk|)wSmr{aiFtk0_Yr??EDv+M^iu-&@|9RGy`-En(h1+EulG} z%V-|xTC@Ok9nk+o>(L_64QLI}jc5tzCbZo74cd&>0^Nev0o{t$1Kox;bbgJtqm4j! zpiMw`qRl{e0sR#^4s8K?JlYC$H`>yMjrMhZj?O`+0zDU< z2J}31I?(fhK7lSkX8=8b&IEc8odxtlw7>H+bP+lm=*8$9pqHR?fnJKv>pYGwL+1m% z99;nP3UmPIl|X-rUVsh)y$W3j^o8gmpf5rfcOFA8MwbA63Az;MOVMROuSS=5eu7?x zt^j%sx)SKi(F;0{qH7s_1-c68E71#qUWZ=P`7wGGdNI&fqn7}E4SFfi*P^RCKSI}| zmjQhpx(4VC=;c6PkFM?f5Z%b=P3RRsZ$_^K`UZ4e=LhJG=v6@9gkBBw&FD2iZ$Yo^ zd>>-n^+4Z>UI+AT=mwy-0{tGk4ZR-d+tH0c-+^ud`c8Cn=ey`#=nX*Mjot|KJ?Kq9 zZ%1$LdUHZ|C38L+BGgzl8n|(1+0{fqoh2R{?Dt0{Rv7DWLy~?gRQ&^y$vO zqJKm81N|EM3{YqX=-1H$ov)y8pw9vQCVCL)x6tQ1|AHQ2^xNnQK)-{&2=u$?q0X1l z_t2Msejhyy^atq6K!1q-rSmZQ5k!rL(T~x;0(}&H73fdUzjeNZ9%J;UfP=n-9*0Qt zB|wQ^?>vN_K;HoRbM#H1|AD>*^cU!n&KJ>>=-WVliM|8$SLnMye~rG^`2zY4qyLG% z5A?U_2SEP|{jl?S^gHw;pua~y2Koo|DA1?SPdX2xr_p0T{|K@8LG&l|IM6?%pLITm z{=(>A(Gx)bhJFt8@8~}|51?nzFMvLao&PG=wWO?dGPH;@abjn<$d78tKhq9;IoIoS2w^%H^DcLf=_ONFCGUUJPE#c z8gyj_e90Vm+6C~iYrwNEgGXHlo^%6v&`sbuw}8jo2A*;Uc*tGg8IK2#cmjCB2f+`1 z1@sGny}u4@`$n+TTfy$$2exzvSk+x%O?QL+{4-e3hrwPx2G;UXu#bDe7Cr%1aR@Bn zGob(XgNA(%{OGqq_s#*`x(0mAtH9@63|{O9;HzE)I`$*btw+JXLPiE!fIzz*1AY1# zXwXZ*UtI{k_zBRXpTZ~9=UxE*5`q7@0ldSD!ISL=zjrG5(^-hm*Mg2c0KVX4_{Qnr zlg|WS{9W+D=YrQg2>$CP@MJfGM|vZ8qBnyFdMo&xZ-TFRC-{~3fQP&U{Om75H!lP4 zdL`)PufenO9L-O!0vQS*Sp?%&0ol$%e@}znUkyIz4v@h|f#XjDAJ3u=9>$A!7d{$p4S&*K@;{Wz=gN68AI>N8 znfyThwEX_kSN_uZJLzG3)=V@&;@FB zpT!E6nSHziBGP*y)?5O*9BB5hLCb#wKK9SxZT}9Q@)_`%f5B<=Cy?GB@DzFqGS;WT zzf#MhR&_43uk*l?zY8|}pD+^tYZL!#6aQ-yN81E6hiaP+A!YlXk79h|-b1)^-66D; z{uD%f?$mvQhY+s${_=TC_u$ju4XMF}Y7Wl0n!olQuC#XNp2A-LdjI+jXI<~F_0K*1 ztb6zpdxa17U%$8R--CATIS=S@dvf>8?@g|p?%%t267Iv(`|#{O*YAZpoZr2}`RopG zgJqo_jvGl+5kj3*Qn~dJ3;EJulYn zvjlw>t8gL*ciFY)p84cGXz%sc2k#2Gd#<_u`sDS%NB8|9^tod$V07$-`R)af1id3y zT0VrY+0H(?rjSe03x!-E2mRc;1Z^v|O}q9iLr-&i2NA2qQeQ*>G3~^+l0HB$rvjcj zgx1y%p&f7};3(A_!|l;_d2F=R9+}1M!Wf*+lJ-ceJc~!75m%H}pQf?ocqfLcUy?1*4!}fcyDaP`PPA64vmXz~e;@Yv~-X@uv{1WpbRxO=3ae zCwZBNFYr8G3cjEVr-BDGRM|5HS=a*~@8c!mUoAo6i6>^JGLFP!MN>$qoQ?2OwCPzT zKgvnoa77Dv<5$$4PYlF^Ki%$II{b(7 zLOmJohqtGODr;^K6;1F?x$;%lUfO5NqAoi-55D}`7dK;{of)5)T^O6j{LJ{|?EKgi zvxP+GucV*c3f%97eBuzQfCTDrEY$A;-+wI^`ugAu3DYR;O#^p zm^VO{nxN_Bv88IGDVvVQ3*lU9sGO+Jm9y=^vNdpgEjLzly|7r&Oubx9Z$18mVtIPF zFD%n}5R7v8ujp3teaNjWkO&9T5a6(>UV!1#5s$c%$w{KbY`aY@pe<&@qG+3#Xdzdz zLb5H`VJu>fkbA8tA?c{hyM3js8sY>oyOblm?9#SoI?Fgp{s7q=)jtQ?k!d?rQXtDA zZUzUY2Jc=UDG#TGSSUQ%?k}}w;~|%u-hzu8u060j70W0~SHAxOy#L)-ZVq$l$nw|V z-i6LG`uobg#U2l%w9_LQE{*Bf!N|_y8T`SGvqH?WC&%H|GjZD`6WegPtWzvIO_!d%{6{N{}UwVz=;iQhxs2@%VI2w#D`Qi3)ldVdqwaTzQli(46DVX<8C zqR>#K+-gg$s3(b_C%};eQUl=~CYs}5ON&G=S9A&E>lBroA&MbSUE~FOt!(IvLddp- zGi6@AUg1SUBH#vv*Qv64Ig|WhDYYLAsu|U7lc*GS4a0K9x9*G<8SF9T`7E=kCf4^+M z{xmBj;d=`Hu5iVYILur2wfWp@85}}N=Vrhp2O&R~*jPd&p<0sf>Btz>p6DBi5lw~k zoaUy9j5$pLJQ-<5YSn@+tNthi2+dclXd<|00eu@I+=SepMk)jPZFTig4(?CmON&O9 zD5%Jf?CL-@ENY_UR8K2t8nJ)Q2?FTgGOr0lNcFLMmO6g|1a*K_7XV)L?llUXlc)ig zOH+9VW3Q0*nvSPm!J&3;4Bd5j6(3~@r_ z5@C*DjFp5T%Cab&^w^c0sw!9tDVD8-Lpl$BtoqU$-x!R4r-N^UKF)xZ9YQ*bPlr%H z95wLJY1EH&=Aj)XsIghD7pB;_aDW8BWsd}71e(Hh99%FBQzqFU6A=J}0&X>fYty9= z#zH<}<|>VJZbP}SXhMt)w+d^nXz+x{+UV|?++fuy=TrFSq7k-vD-qUW{o}r9h@}lh zx34i-#=Ku9gu|SwSy5~0oSo~6wVJG%l$ToP0emy?vIQ(8koPk1fB+*eGaeYq0E5gk zV*w=?^(e-eC6)4M03-r$#TXbO^gZPYKAOR^T%a$Kh?V(u<%K;XeGN?&K-I!6zcf{G zLE_@p3E7ch95_*k50@b9WTJ)XC=m$g=S z5#Tu1jjv7Myak^`fb;txBM#1}6}HA`Af=27RNtCWfKm~_nbc?F2#lndAHcx?7o658 zRY;K1EbidU%y>Oy=tR!+4G#8)%%Z7Az@xe)oj?q9L`5(ZTPxTCCf;CwUiUXvK_{)E zgF$2UjOom-+qBfOj1b{Th*G-*IZrQ#u4$7L~!Rr+^ z_JP$Aj)>9HSV;*hpr>bDj3Y`cstH)r;sevg_5{QX&=P;^mbEdexMM2gE8Tet?Cx!l z!BM`bUzi5Is%RQ%!8lE0=I}u6V1T4@U~d6L>TyFAl&Ud9iCkB;(?sac$J?OW$$mjt zJXrLyGAGWDMk^V+FT=}%nbK4(meGX#aMtu18=A-M|D}}+A9qd7uf|o)vBP|RIK88l z7)zhE&#g?9#>YVzBIkw^qK3JAb1v6BekOlq;Ik{8uj4n7Q&FCI*ib!)OMt@KU57wB z2t$yg-4~?1IE2@DvkP-xYR8m6XcJkqYjG@t;%k+m;aWk_GYLC8bIaRq8pm*%7qzS@ zkdSYd_g{S3c_$<>M==yf=N0kW$&IK6659+>EU@=996rb~3^t!;nm^DzTV?$eKZ4NO zD>(&-cqDMf(8*}Uje1dU7I%G^(-n6)0H}C(t{88u9c-;1bZjd-?&Xb8C_Xq}mJO-Y zD%xSwD8~Fx+hH)$`dBHPi--0)rL^t1v1Z@I9B!l*221meWOjHqn;mxh3ueZ0hGON} zff(lE{bQNTP}t7ck&k%3?Z&;y;^2T5$sZmy;^l;2bYiL4Vs&&PkbSE2S$qq~ewoRB z7;hOFFlBmV|)2+#V zU(`iC2I0d{5;mw$u9>&v!y6hS%T>L4GJvR=NSpBwUZ#vC+q335knnFdTvok;g4o(&37yT2i{$7cMn@Q*+{b zGB)`1Y$Ad$_QxXN_+VheZZw=u+mQWE_yu1#aU!WZy66^$Q}JrKaQGL=WFnu+j0b%l z=$t`5PR>Rbp_j2fOVDQ*`YccgDFuj&LNJacZ`33!EElOTTn5P>_%m>St+JGdZ-beR z2K-T12>-bek^V`b%PYRUoBfS2AylIDK69Aqek6lQ-5^>-JM;K8m+TmTC?m8_zIyX( zZp)1tKh4qHn4Kq@PG7jUg)dr6yU$tb5?xSKN}F z**n=cn{o!Z@nm!clW5zG%O>wvPr7NpY{@6S=A`-?Z#m)eT>Kk`$ z3jhMKgmR78qUcB*SO{+35AdC4)c(4i)rbz!-^4r@nVfFff*@#u zYzn4eSNh7KNXEdtk~fs_-|x`EYRHj{RK)1tI?~*mmn5E3ZF%m!ciz^DfN(fzZi|9-#^v)JHDOVii&6yt!2Jn>T;U{E+Z(1RL0v__YACM z22+JPg8G@K?*&a@qg!m6W(c8*rLk7C-K6#uTmh4s!eS&cl^I@Ji)6!6NRNf}q!-Fh z?HDT0miLH7ZFm_=^TV1iNo6w>cCuI)tth@gl=BpU0!6}meeLYnlm+iZRf^>5{>09) z1TZ*9xb<5_RcKepA~39s)&!Za0`K7C^PS)0&yYJHhoF{3u{ol75%t@&J|fb<6m;3- zg8B(0YJrodagJt8quEA2!pR~pm`WreTM%l6!t7A$v^~W&HegfNobR7GeYv!||1@%k zX4#S*m$AsJJXN>+(z)wj`Qoq!F3{Fpf6LYTK7UHh5A1vz#^FC<9Ac=C^^RtZVdm$! z6*05?(H?DK8bS+s$qr0*z2aA@&aMOV@&0DCF5*L8GBQyd=-fRuVvTHM#H!XL(THh1{Y)9mT{Rg6G;lcFY`#4e2 zl{c*ezktgrkPk;RN5enK&1Q&>iwS|lUyY?K;preg(|(-?{mL-uDBZF1@*t1$b6a<* zcRV|30Rl}qf8`WSOL^BkKm^rP-U^;4!O6tUjm4m$QjF*D_dRH6=NpzufG2&>lOK@x zp#@e`D%5+~0O-mnHDl6b6`4Q=RZwbfY$Xd`(WIUi1tp?M0$z`aq!mhe4dj58lo_2a z$+kSUXRgpH=1-qnrB{`qlh4VFEEbakvf|nX3E)K?$W-|pQw;*#D%40T@Sb$@|KJ9L`iI!D@ zAOV5VX0pWB%l2^fT0s`Gebjl&g_6ksR+WABPAYQ{J`IcyhvS+|iYK)2hWduI%qz*n zcfV2aB(8YE*x`E{YvB^~YohZkx&`_b)OXC4u>>iO(eT82!D^8-=D?qcmlWELSSg7d zeJyj&#GNNkTr}9lX2j>bz&kN<4A+^uZ02 z16G)Zo~W)H0*4rnf5YE^J}g3f2>hwdB6rXiItGJvTC=|onSgOz=3}S3XCRBAqeg=w z&?ZIJQCiGowStV`v2h!~Gs?6+JQWo5eB{8Gpo*#^$0c47oZ8S(Bvv1&htqE4fZq#zL8sT@yc6T}XgKO(cG^?(OSA(1v7)mXMg9n@M?#8#c@hDFV+C zo~y^AqpeXdk`umIosP*|V52au_*F;26<%nZ)D}V zNvDut$a<42vyh`^~rJS%&NltDxSxi5q;L5F-^ zj@KiE=gn#uunp+CtZ+i50=Xmn1XJHh|KzX}mh6Hp*lJ8Os!Ns@|gh_Kze6Znr>HydsY@^p9Qc8tveMr@WBH}W-~z3wgKGtkOI=Y8EZN-N&iKQVaoGu} ziWn|u+;FBpN^y-UMqM9d`XbSiMG&($ysh2ZP>*s%wk=+?3X=oHX4W8w@4qN+N?dLK zbbxj8om0r)K^84QoDvGWvcSxU)(lyBhQ?GDR#!U9?2tN-Ag>H$RtbVE`D84o(Qzy# z{q42d4nJK>n-e=mVz0b<{?wUFvbk;mi@F<9Y&9{SO%8inT*Hl$68U}=?3xY^qa@~#V&tAyjSHEG+ znTt8f6ZQAs0sJ9y1j(c-A!DnH-Ans=#r$`%l<&YJPS&%lDnM zV+JoO^V2gk1a~m*)lGg6KK~BLm#9`}>p?7cSfTBC=9JldoVdD}_1tOc?@Z52mLQ6{ z*}rN!+03TS*wM? z`QQ)+JMUAj@f<90MB)TZwySB}EDW6s}N5YaI z$cmQUe&VSE=iTTk3gP5zZt*R^&Tx+9}Aq~C6b10=?Z{Z(~KINDDq~>gO9<3IQ>yNiFFMt${6X! zZRd=g+-U3_J#go;q(W{4;6n&ondRE-8(y*Bku*X?kT?9-xL`6qH8-E}rWfW@QmHXk zF9@;Ol>nHxATm&cLwBE zXhDpPBAC(>0IB+2)ltoiZI{BKiW|=9f+|TVQ&xrY8BJ%f)@N}-s6Q^MvK~?GT*xlC z21N*#28kS>IIgj{8*l)Y2rr~3{ldmvdQ;h-ib*`CxNY5s_mp?-*p%T!3eRja(&v$5uO3w-o->WHH*de}cC;d+2S7(hQLc-td-%3i59V-2=^FO{Idw~W0WPBau(W_H zlxf)}kV_rYNp#2{@I|tz$Wa@HRO=t?Qv?Z+p&Cv{WysW>)M5hLnWh+xCu5F4-T)zi zKuQ*2kyD2D%IFW8hyuM{Yw?sE&d@ zQ#?Xz3VuDn<$bKqR6=5RG=no%;iV=<=Q~&$CQr2A4Qzo1NjQ`~pl_I1f$q@!dX3?WCSf;nCh zy?E_~uY2=af_ZE>W{lbee4@#~DJODk-gMJ-ZBZnYw+lK~;Eh}c`mA-wB^Ts5Ue_N` z;fs+JiAI>`fS5>87dV1Zmg>%SOSeByjKt)STeggd2CiDQ%dw0@cuAZ&Kidt$Jc;K^ zT!z=hq)A~v5rpiTFTHmDl%?T_pJW&8h%SP@JsTNUxEG(GH5axr0U&Vo(>C(m6R~eV?$M@SDUQv?25sf7N(~!2MI_vNB{yC zlR(AGII=c#L1s1ouj4f-VF9lgYm1M({eeQG$9vx8dYt3>#aCW2V$fn7V;%Y^b>4;V z>HHA3m zDP|coLrozmu!ZiV#3XfD6bM8_IBAP4kIt7XT`$*N%|&YkUKcCUn!nQ;ox$Zin}er? zO)r4a6PSc{Hg(k)uhJ^Iu46IjhV$UOWxMFc1`Cnm=%gnbl4FMa#6&c#N#cV>YkY@j~n1*PI^lW92Lf8$=57gYnK&_)hW#DJYtZdu>mw^^0(185 z)pHrr8d?#N`?CTUnJVU|Yw=LB;AGck`WMGlKO7k>dgH_HfNkn28}hWFu&rv@{CyUP zXQ-Ovi|tx}votAF{RI!h;&g8E;u6reXi zy-FcIMS@6b%IT~k)Mui_p;&w{M0l(#WGj2EIc4Mf)Re{(1M|C2Y%P^d zO_nV|i%P2QjPBdC{j6jvqUdHIJ6gNE17tUZmRN4ZR>!PT7Lh57xT5enC&#hwm9Zh(*V(tsiMDRBX$jfyN6lexAaYft+h- zo=G}drxiU~#iMfdR>FI^JNX;z zNQa1KO|&fOMrgS`2qj{^kitFg3P++yd5jgnTPxFm5RRD%> z(Z?Hv`d~qmGBerPYnHHm;VDJ8|*mI@K)>VS{>eT?am87wbyTF<`$dyG1RO1ER~gs=x;xMs-(O^bOENf9MZN|!1jP9xzMPbJMkI<6I>vSDU*cWtA(FzD%! zy$cX41xvL(Ox%Ip?TO~({Mc|S@K0^fwbz3mXt9;TYB%2JnHNP}FY3)6TV)PBQyQn( zBHg-jLtB@eX5RwmrPr=|SEAIdwBAC*rS`e>oyhUB})+1xtl zXXi7WY4mnzs|=a9gO1E{(Ta8dQEhhT2D;E?q#2}s&+W$z0>DGV$WDm55Y;mMt#U&_LNyU8(%q_?VeGzGmA&*XxH&`gJy>FqHGI5xtpGL3)Q{VhoZ#B z9n0m6D5hrfao>@1wL#r#O^@5MuGx+iu|+qLG6dVPc>-yWU6}ByStn%MgDz*Zroo9_ zm>6_ZVO`CZ@AaH;Djyn{tHpVBdfi&^k-VT9V$9E3g5!pHqH=H00nYwSP3J37+e@bc zoqig95Og}tP(hsK5%(bhbR{03f}=qZEt0I7=ecPmhA9qsJUS#az8J z6nzC!>>5!axKd%O&FJzdc%AGeqroW`*dU!apo2z#oW}TCkgw5C(&f}Z2<%L?<1Us2 zF;DY)E?fv@a=aqs`;BnP&SU|o<*IRoDA_8dH1T$3!*o;SiQBnE&~n|TM4X8*4Z8+x^@}H&xHKBH@9w3TokPG z3nGTFvTnJ|mM!CVrAOCRPdO_+x_;oZhks}#%Tc$O47CP{TTF$1JGXDnzx-NSUH;SP zx&g--SUY;gPG?|=ty$_c@dxqGKwGBSiVBM7gDiq#dJd(SZC1NyV4JhCAND8-)09!kQZx+%v0B0-;oExGOWTwTWmb8nr_2h6l+4jihM`@4`bnW_K*d&BKqI;V!Oh;9FtLefs1O zbfiiuuvNhVN;01C%9+|d11XPDJz!d}8Ui!kfb$3hZg4jYbFw8D`1Mw55aVz=o1Ys_ z*UH{xL2;A~%Yw9>V|R77fFDBx3>9nC!{?7E=vez01*N%q*CUqmj}2!yh-z$-h%T@6K{*ZH zF3<%=6<$(!K`=x?!8iJcA2b{p@=nmqsk=bHirJN~K=_E3GifcMS2uqO&x1}*vmCQm z)5&&g=Rpa9Cd?~~4!t7_uIYBSW)BJB00{vM0=eEVTjF3p2hc8RcmU?iuruZP$;8l1 zzP?tLWnK$~DiJ{ul3CYtYj&)59DW?|MM_bVG5`uC?HMuC>UR<&eqpo_jyZZtl_dqf z;>Yc*ZRX6#z{uo6z(W-5s!Dd#b+)Ki)1hneF~a??M6NmlI7Dbx{q4qape|HzT^y@FqcnmLyG4 z)J$VK^JhWOHId^KSvb-1NMSBpW)+;mvY(rbSI2l!3i>|Uc^bb3KLN3l`Y>9{ru)fS zpv?u;k6M8b15PCB7NV`Ty{ZgP3xz$zYO=+4N7UygK~z-;bBSStm%0;+$M~~a3;dZX zi%Oa2?GXOF-CzD>z8X_VB!p-Gk!K^ZftkK5hy+SEhIZ_bizQSvkt!S<+SxU?Sy-!UR<(SIxMx3+z z(wr4Cuu$j|6;84>*)p}W)W+P}TxGp4Kn@|RQR}1!FOekzdF=FcCn!3AP@Qz*@1t6&SZ1-bS{C$)Y*ElCVjbnq`tWmj^lxsBoe!8B*s?G_I1zb&Kkzg zP;VgwtJFt(#a*sV!|m+zP2;oKMXwo&_G+cwy9C8!PWq)F%3{iuAvD>JtJ)Dk=&PuL zWm!=fSR#h0gK~wUQBf3mIihKfpvO(sq>CLP61z@_B6MC$rqjmCtspL&p2q8zEu}*t zS_~GGMtpwlhQi!tO;eTV^xXP`>G0+;WpsK}Ap%x0Eno=YP-%Cgu{Bb7p2gcmi%xnr|ZGqpr4fJG|A;(-j$TxfbCa7x#V=xMG*63<* zCf&a78Q3#LQ>zQ=RBc$U#ZfFdg^w00RkA>m8JxxVHaTlMDLJf$ie|XsL$agAG$p&X zWabMwofC3qPH_w~Z)!E$fzYFZ6_BJxjhXHa?n`w!l~H! z#F(pas+^6)r#+|XSt(1Ecuq9Lf&Gn%t0r`2<7#IDUkFInBas|Y0#Uvq)x|L)l|EB?qyev zkFBeQT|-s`y4XVya=8EuV4FVG{}sEB{SL;9-7dOW>#&uMkgV-Sc;t3{Y_mZ6%3-6x;ik5w`r z_~lDFS@a-hf+qH23f=qXQPgbHWgrgy1UDyS6Oo~6GbiXKwqADqNNRav2Nhe#4#fBVB)G7&@kC`FY6C*?Acy_4m(F>L2np(33Ue-yt=z!be4)9)M+swkAiL_|vZ0g9E{PUgf;O~-KfD4LC z%&yI(SWTa+9KDUYkY5KGm5TMTwc9o?1^b~50^wlumJJz8ApJy=Rl2|Vx@#}%3(JD3 zIy(+tbL~r-@nEcb^ID56hSB&%t9!k9zc!l}YIk!xhixlcJ$dCkXQrzi6)k*Zx=Sht zOV>XQpzYY%tXCX8@x_Vw0FlCbyR%;x()krl+KG#VBgf8b;l$AsVWKN($ud#SSeXsG zPjiYQUH-%75}rX~+thQ>)K*J^MhN>EZLuS_E~A=bO#I z*_G)GxEGaWiFs$*?@B$$DQ4;c zW?@Upd8xOuoU4?db5&vyKd08}Rav$)$%-apHsN#SjHFqLlGgyHFNgEK8nAD*1?LRx5MVG_Pz7`&V< z_2*-pP@in+oM~#2>4Zy& zlSsuZ+G(X&9kA8k+w*H@O{_tpA?Nz45FofjaiHSMfFpHGH`~R+w9P(1K~FumO}g`y zIuyP=Bm@tD;JoNl+F)}~g%nW>#nTA~+KfS)nq_iI0oqiawC9(hO;Hcgg}wc(&C)<$ zPL@qg3fi=Lb?n!ZhtVE3a>`M$CfFNxq*|9Z@qb^phQ9z>w}Ls+ZV^1cx(ME$#Q3^y zna{2D`|@)u*b1bAK?ytE2!~7|D1i%6&8#^=3H(k-mya$A{s!ZJRL_Q7S|HtmHZeqo z?}Dnh1ktQp6emm;Pvbj57N@Yh@&wBN=>MVnh5C z$kFvL63z`rIR1ZF-#V8L$ky6HRaes#Weh_^Dy%SIT<2~<#D|_BMtM|M&(8tkN@d@2_sW)VK zK3zl@4G>+UGEh%KLKgOcZ7lZn_m!QjXY>WXw>ob|H=!3m-sG(ErnDj)W_FCb&yxsN z@Y0w*OZU2!2gIx+01h{!37r>tNQ?|O0&Y>*X?le7Bi?YWsL(b1s=w?L-le|V?%aY0 zpe+xLu=$bRYUO0tlISWcn$1-i=dE7;#u}$Bx7xuf`RD7<5dIfKg{YxhZkYB-(?dp? z={ z9Xhpg?DIRYAiL_32YVF#qaN%*z=!JipCA?$ARnb8Pvt+?<%!M(l#cbLbWu3N%Dt=H zijT$rqzyEx3W4AN(Iyrq#E!+IrUu!+llJxTPr zJ=}fd-N*(_X`nUeRCEElirFU3>zAlxyZdHWo*l93xjg?5AL~Qz4j0A>>?uBLG9yKI zgYTycel}BhcH-Cv{%k+yfj>X_ZyxyLuRQSQqbm>mx&5dI{EYOT<0Je(dkhd;H?k2@ z>z_lO(Pq*dsdt}+#fpJ5N0xvE*+sXmN>?Fp-TJF@Bn>*{zum?m1am4{fK-IMOjbvZY*#p1Y%2C*pM<1|7p>$lr+eYV;bf&d0)aY zQkAmBf<|~%*2CFq%_5xE%^l7s6-XMt-QC^1a$l++EFI{g+V9F?3!>apI7rolDppXj7{YgXCM1ofE@Nw`6iJv8AWWM0 z37RTlVtKw3kD0Pc&KGi;q{w)1IA%&pB&^D+N-RHNCrw!lWPC1}MGun?9TSIF?f5>o znFb)@W?~MxvXw|A{3O%QHfiG~xt;kQXO-s(yv|W~SnY9|DY%)Fj=Aq#IU@&0M(| z0|+aPF)#O}6UCSTX``faW(?wv!Yke911*pou8@BP?ds{5)77se=$8TC>FzxWDzeR$ zvf4ZPiZPI5i{avJ7Q+C})F4@vYm7sdTuj700k z9E)(ckPFrW5ya4!Ik1PwQeVE~t_h5B!hS`78Nn@)pS4f?Yk$`jHAW z%{%ETgY|$6nyk*Wv3tg51}Gf%R;q|WO|8JP_wM`PK`QXa9&&*PXmxQUc;(>A2oU^j zdnoBI$3dGzZmT@9DNdJ1i{!3|?}a2z4QZwdT{LOHu$(2wdf~3og3g%s$=K1QF+x0^!Ea8LOtJ zufkgz+sZiY%}|+esM>jql*zpi=ca(;O1CaVy0)8OYjg=pFp11{&p^M#m0gEJxcm5# zU_!Xf>Zr6D*PHb*vExQPhb_+%AjhB&45G6p=LnK2Ng9#TW@7cVzG_<=pZGlOxAv8y6lpt6WJgoX_S6pC-Q} z{{*OTE5n6eR~KABxp~HUseAVPx-hnSevtAH`NA<9drPaHF2&ZV5ziX*{Pk(NX;AMg zw037e8PX?>FYTL3jy3&__4e^8F>e0`G9@h=yW-PCQViuyhtH$is;C%Z0~5L2)IfYa zO?kRnAgDsUS{XXw>g@;LP))>(r8?1YA=%rJEUi6jE|Xb2Z{wD8<`Vw$`5Wl?r#nxP z9r$q=`x61>Q2U;$2lE+TH-5|j?^C^sI^}p}b>I;p1eMC}3Z}7On>Q`QcAXEngkVNE zSYRwPsn(S9Y{xq2NxK^l+TAH^BIqeAyO6N6wNzmuOxL@mOmjHhw?3NKG1Zt)49ym- zBp8&mkf=M3Dao9@XL48ETT?Byi?rY)_aA=CXpZCA{H9QD|1>$BFkD$7**`}<Cf{n_z7&Y)cFDaApR4oqA~DU zS=3vvaUW!w0;@05(Ghy9NFb6PF-e9mAuA6?X?6PUdh9DOK*3{eD$H81fY7UQT9j~! zP$V)twqtF-Waug}B1DKAkx1V2maFASB~;#2)_r$i`aHLk3WdUbeU*@rcjZV}BVj|T z#?re+U%D_=P+VIjj;Yq%^!CQ$NohY9dUDL4ymBgR9)I{rXJG9}yPTezor-%F<2BX! zJ$W1cDH;dgd@>qFJ^c%+YeQ`PR~l6q&xw_@f$kYyE1Lhuy4iaSLU(z>YE4{alJHxm ziK`TG^YbfWZEMooG&HnsIKCWTDBDquTneTH3){PA>RUKzwX?SdHE&K$> zbD~2PZZwFCh0fFXQSwp9m?&bQcDtq?>`&NN51!{S6zosyO}R76q(M0nb-kWau!2r^ zi79nn)2k-Mx=%&oo->`_V!Nf-1o*YY((Fn&k{<0>%z|&4f+m*B0^N&8LSPz>0%YGS(nvEXM4tr@h?H7;f2ihqL9WgqjTH z`sp3Tndy;<6W_bTifbYd>9ZS7Po$?`I7b%<$bw`@BkLlT0S2C^G?M9}rn%9$1(07$ znz6}palGtCjhL;Ars^aET#&}^#fQjS(N1&`+R?S#-Y&7NaBQ#B3fUoaVm+u8(8{4+ zKT@xzJGIZP2fMUs9b`XfYa6nh?o;!aJS$@@`cRu==5j~dO?s6sB=EW$_R6D`GJ6O~ zxB52F-81a62?IBn{SLOw)5!w5R(OovY>F@^^C+hZ7^`wf;3ZMe1zw?Bn-oVDI8F|U zbbl{TxRA@Um2ra3OQtLnTd-wL6ENTeQRcP1STJ0!Ey(O!JmK%+=+2Wn8W+=RY(mhs ztc<0lcrK-R^(Zah2}XV~-zf7$Rzq1^jT=Txu?lLq;&RqtM9!Oh)=mw_6p6!Mdn=3{ zk0tFhSQ8|k=X71N}}zXM3NHzP*tSdnfWD+e%Hdfq2NHLzmh7b~Hkjnb^o z@e}UK$uH>hny&&R9?K^1*5DP<|65)TpNuac^#5@dW6$U9;c{iA`gRN~XIq^oyZZn$ zbQ3o#UIi$i03XvqVG|>*k(S3`d?7*|N^bAAZM#m`w*9!qOlxFmx`oFU_sWJeV*|<+ zL2x|g6rK;sA`dTyO^cR<@H3VP($--tiGKV^C+yjA+6jA?=hrOFFD+y+ZaZ;F=}!`# zZKhB7D!-1y_tQ7yMhMSwa<{hl3y>#$0$OiR2&xBrM7yB5kPAe|9zg}7%IDedwxFH( z1Nc7pJ@qS8>qA{l?_+r|?wWrW=m)io-ZS`u?HiR|{TE`=!`bO9uZEn=OeWDxh#^7J zWLxkT{8T$7NSv&M!G*Q5rX)M8qAtU0?`^mAS2yxnx3Tq1K$+V!+XV~Nx ze19RQ>ar#K^Po3HbUR6tPeO0YV8gvR1vX!`QmY9b{MPP0KCkQcOK$He(iMOt6pwWC zQ^rWwJu+ldEU!6rV*KQVLTT}o@rhH`6sOKTvAS*~ofuwQtF0eP;{>EDoG3!dAsX`k zrS8gO>?*JO{l2@eZ-4jA+qZe!EN|8~i)ZoJ#ybU!V~P!!V1o%zT0$t4s!EfBL{ha$ znv?_;rB0i?xC;O{$Psmy@xzebLdWM7)H)zJC5dF~9%(#@e}qqxl2p z_FZ@=y?E2?^o>jD)WVIkzbS=90y+b4wZ5pU<5`6l4c^oQb?>p-QqgY`J<&Gb#|nRz zVm^I{-k%?>d!s!a4Y+7e>}kn)x(2i-Nc5=9;YmS}6hYBfhlYPjiYgcYMe!%DbovN8tbCAf+21^Y&5M+}M0N zMKU%FX~dPE zrMXvz78_r0E9$-f(%|B1p0yKHFV9h}Y_)OZyq$Ek@<>xFk2JON$hB!@A7$QaP*Goh z72+YZ_N87mWrF=OPBHl6&ejTsFxREh8X5qe?}uJh*;D;P_H{1c;=k`3%nvgRJ%}mB zoZgl#1if!@OZ7oe^x6W1K2i1Z-oZdmus!s-&c5H}>pbvvobokZ9~K?3t0V7|$BkZ2 zyC=7}+RI1t4$p^x=LY09cFWtG9QciTJKR25KH}kAdvb{`x89*Jo*v;65OsuFJn4jc zJJ;$b3HWW_T)s!$qKezxqI#h}PVv7`A1L!`(+3I)q_?Jis4noLi}n+g~FRZ0UG}u^eO3}Cy_cxPb9>ys?$oZ(o07= zQ_f%Lp&b0?+d%(+^49_VRgkhf$C7nUQFlqt(HSQvnAlb$XyxF^xQ$|V+_p!akH}A;b=M?wo@Y`&hOOcZ2k=Be1V?#MJi`<-jkl^JW9hsz8mGACB1Sq;xn5^TjL?% zyP<`0=Ln19))7_|7R9AHqgS%|v0tE{K>Tr>d$OAchY98N5 z@NsRxdaL~fAiIXYhd$mbF5wAo0#5RGQH%-JRYAO)O-@M{guaDVgYk(DkbNh;4M!&= zzglNE+@$parFEUsYS*3Fv2L_>GaQ}hexjEF>BPbJx>^r`;UR)wB|KCev_ir~jz-zt zIVpSk&`xNHiigweMD>F^U1d|&zX9=T)j^FX)?=pbl}%E7AHECW=%`1!sEFVe|Dgl% z_=jLbMl_lW`edCL5zsAggIDkDgaPT&e1gjMIx1O$HPPtjJonc9PABDPpGw_9@)x?Q zC}uc={DWMUyMsnyq$qm?A4Zr%nyfWh#8(Bt!Ex>&cUqL2SyWYuT1s{FQ|DNx&cuK@#nkc zA&kd!z~hM~k26E?*tMG2C%=1p8OQ-S=WD%l62`wcd*cc=jN$-Y-cG9btJ z&DQ*c@%IqKYu7g|?);X;UBlbMHK3OtjAOJ!t+0kcy(C_7LpWJfOZ_p8e?TKUNf1m+ zYesgWp%_(5?g5%HMS|F%}jKEK!2}Esvt=^ zQ%cKK%0CzY{qH4gC}7A{y+t8c2Sx?t9exji*Ix%KMdq&wmd3vN7M9@xi)RO6vBNnY z0O9aSkFz{5>LGfzC4`{;x>wM5`Hz6_Iz!RmTYc9?eX9mbFp(J+`7O><55Fd}8}Rf% zFp2YT_i||883IP}^)3$W(SacrnXRK3dH79=k@p}yE4S8nBIL2$=L12=<6GN_cMCEP zx3?wlB8nb<%)7&ROP+5~c|J<}6qziUTz`F=OfPZwe!$Z0+(x}ya-Qhkj=h^Qf3#a? zx8TVAB9;3&ILV|#@C-=&o|e;m<^z!a(roLQY|mB?9DA;H5;8)HX2J z3uYl=Q~uM7r2M1L~m<3pSOLG=d-aj;Ird~aktLsmlwxTbTn=)k5!jvQc-f_XeV~_ zkQQot0?A@%Y^gD6XR?t9**xc-({E6@ew=Fl-5}9lYmw~TAE0D=r}euf;(Pln1MH&E zmG0F&(w9E&V6`5h`2@M=SgsF-*2;IfpX?vT^;=%Ill@`4UM+SWE#VnPD%e4Sk$S2* z54BOhW|Y*najKzVQ+@3L&yYx}Xt6_Zvr`Udsr53P==zR2=?F2n4XR_;D3va8%h2*k zb`MmBmR0w0GqCwgW*4ce6eFT`xH)?{Mt66$#|rPp$1*`+vT)cALrw*3JyImso3Vw(d|p5Vi^N(?m4xXupKlKiR;~PE6(q-7l9BP7F@58BYW?oR zA74m?WL`}A)R=7Q==@ks)TOXVfOpW5gCatU-hlw-CO5Be{|IsjI=cu-pLWQ50;A1B zY#+&(_Q-O)rZf=hl&|2;{{~v#qO?p?jb^m4&1rc~*vwpVoRMcWrzk2T2fr(~XH+xyhq_n9Z% z*EGnW8XK4^wV%{aWyXMER7;Bnf1gKWdRg&@rwZSkyrl$LZ77dUW|PS15MsjFqS&Dj z3(az;X(U7@EsoW9B8akWdiCty zPkt=Be#h$cfw9nuyXFoOQV-#rr~13JeuL6_xG7n;hpPKY>8;(}#XR4iN_Uq0^}Tus zFvE1kf6J|aZ6g{%kSxi}Q>M4CybU9pO^BwgNJt}k zea{wPt0Li{0b6kd>$0r!64CCEuBd`SH2#1K<2p7P7o^0HFs^j&fXM11cZ561ou&Tq zc%4nm(v1{D%fuVjX_ruJBVKy48_2ewYtx=XS&vNOepEdDtHDrd-s zoiC4$3Rv`OvJU5`RFDkOFJrPVRX}GarlS^UEos=4{K**nC7i28RkGL3P|WC9C9;?z zUE%_KLp1`?gkQqL4T;w<&K34& zHnO2YKndw;&}U#|X{2Yv3`R9MsQB~2%u1rV5>O2+>b#_q-7SQ6F%${2*pA!$U-V=A z4A|*88fB9njV9t0%TnRk1PTNkfeM3Xs1U5W9H9_4ku-Y!<_GqURA-`7)AjK9k@5;s zt)if)*>Y_pyFQz_?m!}!j{Fu<{O5f|{LJLd5ADD3*{N{cN1%#V3c0^aCv*hFUHLP6 z{iCNp`hYI+FX!{Wz&yw5-2a7Mz~2P>aT`Us2-A#=AliqWiNQtZ34CgUx(pw~D2_M{ zGwz!t-N0-i6Wc~bp=nr&tM0<4)5E~CqWSzBEr*d0L67)FS+=4|0M{Csc>HkfbR(87 zW+Syy&M0QXwv{xKX$3?Kp(w-|F)_Kns01~iuQo9j&?CAY`EjgJ4b0YbeD~`>Mzv4rmtI*S^*MvYpAvnZKf|?e%Qx zyR}rF;+Rl+x%a@Ars>-yv*gh_*|Z^Z(E!8qU{8cjXK(Q;@*?ds5CWA8av7+ZbN1j| zPpDb{%iO-WA|ys%ivkVlx7cm&Dd($WnQffu_(= zH#>?R6Q%tP@B%i*c5&Qp?3c){Uj6F3lBOPA9ht2pcvCU(sG5-HDuH|=h;SgD4OFI- z9DEzAuZX4^)fXp2aoQn_?JXwKvI z_znud3!#vtX~$OLGu1e6sK!YxuG|JcKY$VW`Q>T&`6i-2Ux{f;li^YL`8w4zwE6z# zPw5OBLpdrq5sd_Twn5*+yz>NFMv-|KFttV2V%YT2IYiCjwK9RKZmgry-24h!aK3PF zo|?mNXs@V?^QS`ZU^)2D2%*1#-=FrMKDaf;Hg{^@gjxeLsrbNrmgqwuxTkvO?6w)W zB0i18<2S_;C2X5=@^h!WAV+Q*I|H}G!c54dG&(6aH z`a^UQzrdAfHIGyl6LmHRLhdy~a5`PgJkiNh#4^MCVx5-2>PwQHNGISZ#dJj%{h?66 z}Pi&$h;ucDgOc1=?X2PgCbEZ}~}z!^Di?d}AGLy9n6wflW`ihQ18m zED;SDsz_ZIk5h}0%2?DSt_~wLTFYTs^VbvL*@>gGqSF&+!Do?1EZGl!t3ckL4u1t) z`sipmZ-XO+&%yW0;YdiN&JFXB?(^VM4K=V*O{`>sdA|~rwP?^piUQ#R5_3U4qy!3~ z>_%pPA&0TXgYPA-jWm{Gxt0Ql>B|T0sW`aMO5L)iYw1;Rtfm~HvV^P*y@l?F7>Z1a zk>(D5*#fS zf{~1+!CM!VZ-aAN*jF@gCKf4_a-{TJrh%fk* zM|&S1YVAEeg&XK5h`|T@>{DLYg%9lfY{b2aUqQdmpM^|sM+DK@YW*Yc7Has%9ebh? z_Zj>;dX_&iJZ|KV`YVv}Mz1;;;y#9JD2y*V*qqw7NNH!9`h0udi7$I5l9;<4=g}yB zhqJ)Tx%e^$eIK2gMsD|N_P|eibZBt@&-MZ4HouKt#BXu4+&xql#KN7SlO1Vc+kzk3 z%wBRQ8rTYEXY1?24E*^5F>O4%sgz3OyA7hXeA6s8$jn5PSUnKuP1%{XJu_woGXAMp zcs5~1$1PP5!h4eC1{M;zlSgMGmL8RL-ikboqh{vVz6oAPW_i$dg%`Dvd?;9}77g|R zmDO}r*DN_&87Vi=<-*45!c^5)u8N9So7TZSE23BqDQcmBWg(u(?K^&Yd2~5w8b}E@ z?u@0(%<}%58i8PZLPr8`j97`K`AMHIo6Gtt6$mVAQ+fo$am{M*8Prd0uA!&VWyl*z ztVi}t5FBK(MR5;}p`u)g`tG3fMC#EkwWd-z;>5^hGB{Umt2=Gm5rsb8tTBahq{dW> z3#KLTff0ceURbFK>7mq)MN#+ftSRYVH3JCTg})F(+80}kOFMQ&QN3$yzK6b#zs{X( zR+b3vqyic<-@GNIVH!CRYj`0_LpCRjr7wJr_%kQ|09QuhDJKGv96v`7OPGY0G?IBk zfLu$eWN*mog>se51mb||kIrNpPaH$}vIy^Sg@qx!Al0Wo}46B`E^W(G+8`K(~`6u_TAx z%&5%5vw5QYL^LGPTSsqMIaA^ZB*0e$lA%fhmW^Vy;wPPb#UO-A5| zvV=BUn~iOZF~%egcFblBcAVHOAtVqUVT(fy0Ybp+;6OqWKc3@&V+@ZiJmOe0@0_~T zJtJ#D2EXsidoLsP?V8(n`c~CFbIu#!GMP{Pd0E5x<1?&veRA*H}RR@?gDi`{5!XhlFI;WF=0;MDI|dy52SuMr>Jcen~x< zZA>Mbmjy&6^wW) zSQ;(%jTgL=4PT$>c7Mhw2D06aqBk9}yjov6S+zWR&Yom6nIfC0LH`CQvX5DB5kqvh zIZYl3lZlKWRKptqnDAi867o1Er(_x%_~7-Mb~|D;mVK_R&{t_&p+`IrnmDT;AC#47z|Kfg#tuXs?fzy^?lbXlSq8=Wffx0N zX7!ln_qZd17;<}TuI69C)O`FNS*LVypZ#Qz^|_N{#8KBeot|W6Q&8<^opx+m3R{n@ zc2HJvvS(LLc1wcSXZQ4#ZN*e&(*!0A{gcb#SXMWbn?c3Y77Yda4cb_zZV$^N`b*(jVAqq3N9Q@lsB*c3`4@QYx!fB%n9Kezc*gqHJnXb=BoNuUOku zLDG1{t0S>|N|uMru+Q}62hwzG?wh?65iGs!je`;*sb)k|G~m2e%w=kH?lak_*V%JJ z=bxDGrR#)e`0l%Uo3ny2OmuwIynM}2N@S`;6L(}W`RJ%J2pqYMY*9V?fJFoEqFSUk zAMUeQK)UD)1(cM{Dxye~E z%Dz&dk>0s)SI+3^E$OTXy(a2duSYQ?W(USHH(H^m$j5gG#`o7`2OFRBY=E-PJF>G% z^mIn2EE?R|#eS5)JI_7V7ZvE?Lspc6$dSJ zyMSH--A{%7?nGufs;i1FsFnas!>fwAsFnJXg|IHrd=T?N$kt!Tme~hWFGZ3OKdet_T>9X?N`uoocYbczeF&gsS5> z@Ik0e*=6nz7LxWIufpy7cB&LS##s87oHbw(fWpZs33Y8?V33UP-Y2zB+;?*N%<889K3*fXM!>Ub(PrpkMBG#$@xE-BevGCjn%)BGd zGH|HC{d_RnCYQ4~%MD+t6>+bf@F{Z7<#jC|9v(S7 zAgJ=(zXQsoub2HcHB{kV3LMfCNqO(2p5G3}|t3Kwkv+WN1WT@jj8mjqSx

za(*i1v0_81P`mbzcx`NMq-0S_s zqh7C57iE?%7v2PG80Am97HB)uGRxyS$|Y#N1IHLy-(q zcq0IFA)dtL3fv?YrYxte&_(wNkHPBkaL(bYwryF=9@q|E;ua?Qo!S~dRSi?tB)d1C zFWcO#C$U1hh+ z^jCV~-r`h6^J(Q~!Z;F$SfT-VP;^JSV!=W#=aWn&kWQtY=XQxO0R4Xg=BSrxg*Z!r zs$&(v98rG9T(2rFiVAR$(~deoJ(FEK#w204oX}<9Ae9m7TM0IBPp+g|7<% zODm6Bm@`js7rVP$vM!$E9C7T|r|BW#KETars|a)Kfb(otk)|{fX4IACEugM5b8@x( zB_}IayNE_uObK=1(DWcnytS4m-g@klREybEadK}Re(W(dmQ1A{|AJr$?_hZdCU{TVzW8(HFHPF*Oo>iwgtvZ!Oo{Xlf9*7gFFn8E z|C{;XC>fbQP2MWpPZAxZ{VTJOy*N$R`EzppUy~>7zxKi`Y#N{>JO5L*XRg`Fu+oK` z5eEfwRdvAeLg2vSt#aYcwz@L?UU^viV?p)3RSf6*!jXy>?x|U|7;cQ~O*tFmynuWa zwu4So=mXNS>3ES=MzgMSZ-tYc{c;S)0bSiMJle@{AFN)?4vWRxDi+(|&#{@JV|D*e zx2J>Dfe+nIe+2k_4dX&yF2llACsw^xbt=y~!g9)q6|1xVfuIs#k(Bt1CMa%xWQip5 z$fY>vYH=$|Xiy8qa{)ON_C>mfGX5?tq{)6sQJE7XUXj(jE?TjWU)4eZ-RCmBAD3;- z2+4}y%Jd{cTJOk`q!{k1M01U-&Hf0TkDB1g27b}IB_k-N{XXjL?&heDYjX%?^?_Q*7Yr5(2&;Zl3{pGHRC*iSRW~y;R z2xSUh+s?Zv^>hbxLjnCOmGETZKKkhqP`@?F8yAXaDc_X#p0a|9tOhG9>z^N9SC2|k z029+FFt=)<)(F4XvR&1zWji+QI1%Yvgj>k6j`lL_*s^&@f$C_c!FFtpqoOLRqt`MX z+enG+tpJW@`*?QJ#3%x@n)JlAtf`pMp5)+YPeySmu82-mw;n^-maXUP=)me)W4i38 zG~MMJ6?V(Ok^)KF@WQ&ZhSyXFs%u+`jmrl|(T*?3zoFhR4xRjRbr0023GIz!MFn?+BX$2NrYu&mGhiA!sy z2fgJl#Y$WK*79grva}DZ{7bi>DK{L`48ir6@qyQT@9wbB6+8Ve^oN#BU8fHG$&3|7 zj3tt>`Tw8~3vUMvqDf|$j-_(cl4TljmKZ#i!)d|%qKRXu&E$Az91j{+r>3O^M_Ywu zDQ*qchtNC7&%gs1qe?J1mKvDp3#NUl*A?-(;$d%YY-6Q3S=_F4nFG^QT{2*1RJCY% zef~6+n2 zIxPc+VKK))qkjl^vILkV$azJYS%|$P?ToEGC>Bxd;FT8Ol?qm|3GTAkb?Y8Tyx2C|sW%YHpn9wSpeY zZ6nj7QrW4K{MMx?CS@u!IQVMS*Mxtm??h+M8`fIv1(-mU3|&cxT9M#Z!h6&lUpyv_|D( zQI@o{KTycV4cX-lx-EA|)_*!AgZy3RI(--+) zy6o?(ROsRGvem1nD!MF*f+SWWq2^#S9L&m}EssYuQ47F;Gno< z%|tSyzQIgktQ0_Gy?#XVYXBBSMUD;>!kRA?%8!QxkQE{bsATU@rhhc&4n^q^*ZQSH zTSk%|T??8zIwnjDF@vnjl^(QM%cfs5;<@gK6%V)q9+(v)Y|5^%ZAJ_IfxfO@NAG{0 zJ`c3Gj%B_@SzqCFs4+Q*R5rjfWUb*XmMP3JsWA8%VhP}o5wtVFO>(;|)8QQkhg(KR zog|P+W+p;V1lLj7idl)O?eQpqpe<2dE4r?@(!#Z)T}SS1kgm5b>KE)k~lK2N_6bheE7Op)iI{@zB=sGnlYJnk~LNR0&8?2hvm=&Z=GF2st| z^D?^?I0ffnEx`-$FUDBDdtPplxFAz<#%#_1QrHS(?(6Km?B=qRws#(~4J1r$6*;TT zT29_Xj;pEeDK2Z|%eW`R2wO(bh$cNK_wxn13DLb;ENDNXAG_vVr#pGI(;^hMR zFQREJ980)?Eh=sW1c>ZXHIEjp20^E>%05vNR1YW)qFgGOSYcE|^_wjP&FA;2cHWk4 zBVxMCA;D{yewodrDWdbya_b>Wlw2OK@M_tGc369KK<1*&FplJ+aBWCt)R4=Tg1E&Z8s4O@HO;nhgu9eDhMZo+*sFw|I+% zynD%wRm-nS5~ybr_r3f54lh1U|B=y)pL3o(UmF(Cj9f&a-P<++OrEhK8cw~^4(-sp zG<%?YyR0ecZX6qsiz5BV&@$Z)&4|FEKqAihqNXNvT{=itGiuS}v2TApA6CT$vSIOW zKM|6`yIFqYcJ^y#*}gz#@$2cM9(#8=blK^ zw@=eI3lH;b{K#EU->xxT8zUYYi>w6Bs_GPg1a&q^F?yY4JZgnNpcGc5vVeS)0 zqn0Z6?HzZpLXPjfsWQ%AZ1dv9`((I!ZJnn`6m4F7K`7zf7hU#=*(y)d&w#whF?rFp zRbo^H)(NHdd90J6dfRwGy^eL_wnf%PyPTwTX1Wm8FIXwgHX<@T{-^YW@D{R~-M@#+ zYNrpi-(dgn%d}Hic+0Q9D_HCAY?Q`4U8jX7V4gE9LkrSDtlEj-P#u#(ZlayB6Yv}x zDrWqb;SsPP=axlfSlhTG3&>S?;`i?bzE;rheXwW!(HZVny6^3$?>l^w(&I;k%#j=E zjfd%~n{HB)Syhw#lg6z#PF+5pk-|wColBN}`-v4BwBl^Jd$vp6HaWFfyT5m1LESzD z&js^cqYZmCSxOweVyf!k3FPUR^8hmvPE?(|2VSC*ejYv0ZVmDlaC~@R3y%9rJ9B;K z3_MuiTaKCcOzb;FS@t9z;76YZ5Y`hg{DzgX?Sju+3iz@v*-+Jlr5SQ@cirFL*XF{~+2kd~%vc1A6~dBU`Tp(2 z%^NRCi3-A4+X{C33oDYzHN|i>q^pu>xm$19aQtCs?TpSJ7KVg}c%9s~CmHKjC@AL~ zw;LNVJi2@qL6$?npnSqZ3}!owJZOi60pNR777|7DaV536*R>1O0N^$|IBZ3&yelue zBq?BrN0S29mZ6_SG*;MJsqC!&KyU*blN1KW=5BX!z40{tUE%jYYs9E2)ESFaM^7Zs zcbmt0un4~i0s0dP+9_smuX9<$i}E5xxv_ zI!fZiiT_yaA;OAZgU;Nt&@Sk7wmG;ktx2H)OJ(t|i>6rkt2T%|{by~2`?>ZUf37pb zImED-@D?6NXLcFZO)u+O^c(Pqg_+*U=|ME1?~|9sk=+Rwo#Mt_?R}#2awLUd!!D*z|8R6`xNx)46**p}EbS zfqq8TI)0STfZVAvYb+1sDTq@}+0R7aQFA-vmB#duztn z^R=@(Mj`V4BQnZMm{)ryg68E>`=IxLZ7ZMIR{&vRj+x~Wi+X$H% z9+Z(Y9PXx>>utoG{4Ic3CM! zi~IVAi1EAU{c@$Eei?I2J39H;{PT1J{UPA{ zCf`ri#`sQrC3<6UKP8T>O6_qvC9oj2Iz!nxB_ArG%a5t-+&Po?x07dp#u5bP?Kgrx z*FgW8L|@zhQU=QGi?1x?N&sM_yX2TDyHq!5b%lfy77O_SD);BLkSs;1rTAi9kqwn@ zic0zyB~UxD2461ST9Fn+4>PkB(wdZt3v+igh7Gxt5I%+p75*#lt&^(?_fNh+*V5-m z{*2h!9AD{CC(dV#89Xsqv6pFdK{nbF<{%wpDEkt3#Qjygo2AlsQYz`OlqzbHDB$yW z4)CWQFbzYP`$DqMRzyuytPs6d4=9Q%iVm-wn17hQkNz7-m0lM88bPEJR2vGj>Ew~Z z7Q!PMn8*bKB=TxIK!Uz+fTFGWJGy38QdvFq4jQrTm6lQU*@B_mvgQF(R771~HEpV@ zU_M|cFMIf?EqFcAxx48m!=-Oob5q^$s_mRI$AK3hZ6IZYsxM0{+^-$=_G?DCG1-WI z?j-#_xs6=IpD*t2zpUz1IoMfsa`GR8*R54YZ^t+z^tRQij?RYqSqXR|VOm+U)4I0P zI>lSS3&!X|A`<3_NITnaduhj`kfdRUd@9ba!9{Xm!tqovMZE3X9TU=&$rSDhZw+DG zf4J>}Lh=*n-+4{T1=M6DvN^9O6_1%#SQS;p6)?k5)vCs2AWT6?B{2Y8f@;WeMH5L+YH1WyciH7JJ+>%B>rq9KT?(dqi{=#pK5@&69IHoQ#=lQpif$OX zRqCw-z2*LDFxCUKtf_EW$xhE^WUq3=NM6t+N%QHCDqcA=o7Vld3(&U;JV5P<1*`pK zZ}*Oz4oqBDTtEfcRe4R*Y;9Ar;>RdlSqu5z0KaQmepObar0pADMxQ1dIyQg=kAs?Q z7xuH9&1{Sy6~2pikbms#-Djte6Eo#`icX59ir{RE##wKG)QOHI!@VB2A!H)iUP&}P zR&T6OjD!FNskpgSg?@SMNFW`T#H8Q$f>I5nC#bUM$;CaO==y_tBBXg$fF*xwSh8gZ zg4gHi@@ET6&%XMXbK=l0ia1hwNZU`fFUM2!-;}v z^9arhJ^)dMv<8biEjUtMg^3aef!2n+CUyI424tz3jOsG_kI)F@)lx+pH~MN}!3@CC zrDh^%ff^>fKT**^?_hD6etIKGhM68qR{r0-y?jLtS9@K6&N$;?K;OST>7%k0;8aF z`Nx05$=P^*2l>l2Yn#vI9(R1>*Ml6~$g=FA{OjU85d9B`b}>h1o7NBs&%(^QfgYV~ zrro?%8<2C{_kAX>LmjDAItTL_tdH?rI}S5k#5Q3cWPcmRT-y&AD5=O0AA*{m%Kgw)ld|;o2+SqAUQsD`e|5y6ATZwGlrnt z=&w4vRUOpCShW!3euf3mkXHg2sp+U@nAH;}p%JbRUaAL?Urdn?%>;n<1a4;E&3&0XsGnk5iu{ zYKMm8u8<4JO$ijOeUq>XHJ1wC4&Q35d~ElX)HHC;sT#O^oh~X;+`<}ylBi-}R3Pfz zm}rKf6MzSy!^7=5Pf8-7t8aw}NuIWb-1bH2}=dL7i@?!s#J_HyNeb*-UEV+M` zubMt|AusfKJ_C?z>@{kYfMkt z4)lcQKCOt7j;J=Xx|^zV_gO(UC$;%H{bTyiq@Rql=+Z>Bot*^rYmp>uCv&OOsttrK zCu)zImsknN#kCOH8!?zqW9ZC4T{&rk5=#|X40w?UEf4cW#_g`jYJ8v=9||X{?lpFI zF4U;SDqgeuuO(S@4;1M)BV8WZ(Au8GNORNZRj1M+kJosQEBIqyW#i<=>8LgLl;z~0 z%J4lSj_nC8oaO3EtFmJ#;f3jd@d?9Qj;l&!4Hr;iDJ0sg>=U za5O%Jj_-D~IZ)a*0JmYm?I=}4?quL;%-i(Lxlkc%dBRm6T@lFZ%JlNd*S%jH4ab|F_6td-40d20}(wKLe@&g5Kakuwn|mYH?LDw8E=#?K)!a&QJho4!L! z+x~>+GrV1vua=R0VKZXt>6wC+%V%ApoVBvL-)-eAGj8cVT{XRC#si=&xipa_$c6 zD2igypQL~aB(*ER&;m8Ht9$4QQ8bXb$Hud-dw?Er#WbCXg1P4yKPFA4okXjSjiHY}CzU-PnP=rG1Qr_xKq9tH0wAxMD|<|+54E#9acNJNQoSpv zilA}!#k@ma;7wXA1CpHTs>hcz71;=;Caw* zXo)yz{S3=XHb~gH%~X~HE_?#Q4OEu+!#E|;e>Oxm!A~fP#D%S3PJyKDJPQ!CbN?Rb z37vU@zQT$*G$YK?pOBSoJZ)<}?sdrXwZm$tD=(${LyJcbvLv9gM0-VnYHs;OBas8` zk;qyi!N>{_qUH;^WTYuJkmXLl)y(Q19LS4K(1WhHf|Iba7pTl-g(tcV8E#Qh^ zh>i!FvGjnAYHPplf#Fd!a)L0S{@&15-L$$wqnR#uFm{zE0WvagrCnZ+#|=PO)Pi2R z7o?5^;!%+`!GYkWuS;I58O8GYp2_}H(48CZ8?PG1Q<~#rq<;ZpDzHi#S-##!_{eb0 zPqP>=Hy@ebk(q7J8;;C=vBhkWV-dK3>|EK=O_|gM^+g4N11s){WWU{I1!JyIGnT5` zAo)@0=<(rlv-5^08FVM|p0ufks10a2DVvy(t}DjEhT!Kwz*nB_ndndHmItT6PSADX zv?U0BRn?-=jhZp{St~ozJ6_ez{~^g;up~7vy|EMp0=E@gQhe?myLZk65}HZ4FLE=V zk0_rFXC{8LS$GMoF5<|y> zs+962rnrpFG_NdlmAYqkwknlCNL7r1$;?n+Qu&dbK0OI4Fb%%@w6QGYXRGZ#@v`Cie^BM1dFAaV%>DZ z;bg`ap}MW>R!Glgx@O8vjk*AjD>{8d0Mz66=tjWhiduqb`iy8JyQEn18K&Ee>!$8< zr;~wfh`Kz6<_@{am1d?Ga*1NlPX!hmPhX(2@Ga1Mt4zYloG&1Mz=#Yd-UN5@i}@y~ zUW{Im5m;jQ>{B}hVA&FAgL>AZDVj_{m$~~K=LAgvVswu#9dQ9S7lM`+tXJ~txlR(S zFZsX7+l6~O`}Vb}<6YxLHc&Oh-525ZrukNLi6Ehsw9$t?AO&=rUcJ3>|I3A=H=Zn8 zhM5Rtc3rzO8T9U9oAy(~Wbw~EAY$$myuyrdLU=@YUhEUMiMNRlil3DvX`A#3xhcO# z@hPX252;P{LG^PQ(Tdt_`k?+nqiCEkK5k0psQHljtSjca!u2h8!2Mlok9EKGtY?eo zYu=joI`60Ln*C$n?Y{5&*Zbe)|5_jy7!4c=JRe*eygx)j`$GR1elYxtNMGdks1jWk zeLR+nT^9Rnyeobv{*^>u;th#slYfv(rQVeKWjc^vlm2-6N13ZL_hg>SZp+@AeLS}z z_nqAH`A~i^e<=TWSH0`Ot|tq36~0?^6$gtq79T5qr=*nzOZ!UiEj`}7rTbewTY8=^ z=gRl@HhaI-_h8>Q`d9Q{-T(1Qt@8Qmk(yL{WMFjQnfhh*&o!1b-q%btk2Js1%C&A9 zY!2Qv_}tKvp?3^R!}pIEBX^BFKYDER*|FW@WcPk527gl3VhorJE~{>+L~E50;6Jbmx<_hz=uJhSrZl|Ndwf7N$apIW1=dGL}Im)y6uvi8ok zpI>LJTeI$K>tpLbHEYZc&+eOj%j|bHNE@0P?%LS3@xD!&O?Pf~Z64qJrp-@o*|z18 zEiY`{zV*><&eH?|ww*Kfan`{UbR*iqhb(~hrPw(7DEUG~)FgO`8e z3hRm$SG?g$a^+oD{&;8K&PR66T{V8yhpxWu>R;}-@TnWSZ+yoq zrB}Y;mCxKXeA7cW&D~tS`Tm=qy(L2&PDW^sVC{rY0lwb?vTthsZa6Mw$7SreoE=xp zKL^KY*1m)t*Rta}c3jVnv+THm9XHP31;P)AP5yao5Q~0+w?(p+ew()?k`VH|EtB=a6mKgeCfv>2D%mIeA#ZCWC%SoChjx{> zP0}rH!MNX2pTS2TmM2kXgR#IJ{kda&M+y zs}1#KZn)uw3S5ZS!NnU7?K*z^z`@MAy?YLvJiKY|wI>hlI>!IB^OvKUGcOy(uNpb? z+_t^PjvqLBBr{MMs11y5*qo`=1}g9x+kf=LHAjz}f)8qy#xVaQ=WovZhRlKEnO&I^ z$9C=6dwAEegPEiI&K^VMtTShwnAtQvoym0XIeKK*3Fy!byN>Pc*?;21b)(hlnL}q6 zo_QVlBa$H}$bPaH7V2&|9wB?+%r0^eUKxOc2gqS~ogl~Hm;o}}MUL~|t|uqq`V6U) z8vGj~ebByv{j2bAY=58g%^S%f_SeVZZx6z?>)_lTXrF{TY=ZW+(84Q@z38rI-*XiH z`a<6}+UeEE1+Uu%?PKhdqtMR`sE!qQ)!;QoHh{vFVc&sgYy9kfxbg(KhFyP(|3wX6 z4P5$X+-dQBF4Tt%ydP&Xo?&C$1)uDN^EeN9?NPGtqMk$LBCh$>SIoeZn1+8D_}2|R z!Ex?_E1Vg+f%R%H>4AAT0spRpnXbaW3qAFPZ@kd=5rJ_W0z9!!&hbN+z)OIuMBpGQ za0{J$40wYJ_<%(`#0%Wa2WUS4s5t~EFap|P43J;~uvQ9godfN#3$(@}aLI1CU77R( zx%NXBtL(wn;Q=;*-wpy94+GJR!lTCQZj)q+EFsItM`7xwVJIufD!|Wc$R%VgJjC^6 z7RY2H3~)2Zx2-@u&$sObqMg;a`ytQIWi! ze3tw*`3QL*`D5}q@(4VNyUE{?zbF4l?jip~o+ST+{295Iypep6`~mqkd6@h+^4sJM zfEw>2|C9Wf{0{j7`5Tzt_X8<@fDz2+$>ZdUKr&w=Uxx+tP4cJYY4Qa5 zTX-zauazK$ckIZZR9oN z)#N$yDEU6wN&b~QOP(VCOwxe3C9;%U4G(bf&o{`O=p+Zbu3 zq#+un5gMg28m9@0s!Ez>XqM(^o_5g!Ez%P0raiO_N?af9r|5#MQBYoKgEnc44$>hy zOh@P_D1PHC3&0dzLYLBIbU9r?r|Aq`NmtR;bPc_PuBGefdOAxt(2aBx-AuOt-E9R4 zCdri}Cl4L6uG@R;z|lR|>^*W~@3Fmmr0J8#j>;?c9y+m0IlSwdV@HpuCyyMcH3w?W zdu@Qf5AydR=e@DgnRak-?pCd+sVbhIe%a0^HAsg ztn+@?d4KDCJayjhI`4O#_q)#fUFZF-^M2QPzw5l;b>8nf?{}TgUxVMj!SCB@|IY8< z;P-Fv`#1Rg8~pwae*Xr)e}mt@!SCPT_iym~H~9S<{QgaTza}4lllP;^@89J2Z}R&$ z`Td)1I^g$j^7}XW{hR#$O@99-zkie8zs0}5)nd#OUIy^NK0*)eI&lQJCss=&^YiQz zdT{SCJm*}8yRMxW2@PJ46Q_Q$1uH$k@H?V+#O{dP5w|01N6e0h9q~G%b;Rn3)Dfp6 zN=J;22p#b`qI1ONh|FUQnIkSoRF0S&5jo;Ur{;%!9Rh_w-EBhJQrP$;6w{{~P?0|XQR000O8#GuGJY~r*@vUvaikmdjY z5&!@IQEXvzb7^ZrZ){{=R6;IvbY|?lcbFqrl{b9vt(>c*3W}vHb*qEArIysWC-mg; z%y`E3csw|aJ>!HUIDie<#zY%1X2F0LmSisrOBe%Y2QR^7Ff1(DBx5jOSYUU-nC#Ja z?k%ai#|EC~ecpe*Ki;0Hqmopry7$E2IX57T5VC>r2yNN3d*6fad-UZ9{p1A*VRiSu zM)Mtiy8I$Ie+gW5&S}dhoqoyuGe3%ua1@S*&%5-{6+el6Z!1FDD-n{+^RBrnk6gGC zxd|ZT{3|ZFR6Ov8E<&+4!#$pP!6nZ=fBrq)+u{2RLZ8(yJaXvp|4iL{9KO@x`p$*$ zfp`;l9lT$L_vH&Oz3N%7dvErG@cw5Aai4qok!M^c{7iTjLf^OHv+5<6pLd8n@a14_Xs^whUT4p z#WRjvaSWjmXb){Qk0>uN+VBbW+n+t#ntvQA+|OulKC}1quSc(sUe>#}$8#Uypub2# z1Vw-FJM!PX(@@Ag3g2(%KEm#__U||Bli#4bkTm!O{HgFe;0rtF@KN$%B)}&hCbz)* zmqoAm`>2WjBwK}1FZy&tB;@QujcgX*l^xN=C^p`xs4tvb&(Bt$wc+4a4 zN{=s~3a+3e{xI_JXV5tL462e{Ahmx-3A_Qd;Yj08pmF>d8b%+7JCsoxUNgW^6Rr>N zOVJ#>X5gs6p~Eo*hmVhoxHH%V-i`2j^dbzP}C*2g&%o@VhUf3G{W) zfd_l{;yX|SeGPbW548JB(63JekG>A)Uxn*CsD=;2J+}ZKzJkW^tB^`ALMq%t!ncF; zo{2QvL>B^oi_p*Y1Vg(&13GmFJo7a8{ilF89=v+Mr|%*Ke+`bGfet-@%Ak=obRY2N zG-wmO_LHbUZbwb-QF!)Nu+H_sx83Lveg!%WA47Za2)Ynof*QcX^Kl&Q#7EG2ZW0CL z^GG2FQJFJ=$Je6(?%l=T0X}{dFDwCJNGwADMXfxRW--n3*YX_ATz7=@B z<^VYjqi^?q2FLf{_&Jl&Jj$~JWYvF7Q8|$z)MW?AioP-#4dnEizjC}5jvIhqcR>5r z93Y<|^!45!;P??7zh$!d0`%$02bIrPK+bCqDjRyd0L6fZAxZZB1Ts(Jn|qI;U-TZs z4}e@hi{|lndynH6f=aQ!jC}zzJjXwmmq`V zr~pS1pNCFm=fGF;7toI~@EU9fjud$X=-Z9^S=^xV3ALJgAk=%O%^hg1!A{D5HG@v@r zzaRtYKuu%-wU7zaMpp07$U!#H7;=EdQ4FYy;=MnC?YcmHiN2Z0_&PwV{@9YJRTJs+I~ z^a4gNL}&MYf-XYm0KFI;0{V1xF3?NRdA%Q_OVMGVm!Ts-FGuGCy#ig(djwsHE(H1v zMxTi;0(up?7_9SZ^mL%tpi6*Wi!KHFEOc4#hv?bpa-i3tD}Y{)t_1oV^o-ua=(&tO z4?Pp;4d|-g4q=~xc43O zYDQm!UIO$E^irU&MYr_6jb4Xt1^RmQGN5;&mjnGr^orht=nd#Lpl?L41o}_tcA#%U zuj+jZy_wOspnm}RR`hD1Z$q!?eG?-89YF6!uLb(g=ygEfj$YsU2KpCtC(w7Ge+2qY z^ah~+3iRvfUFeNK-;MqW=zGwcfW8;Kx%V}66ukxLJ?O1K--q4?^!@0r-dE8F(A_}q zMgI)+gN*(gdVB9H=otDJpdUi-0D2#KC(wUK|JwUc^kMWapdUf+2KrI-9-tpX@9ljV z-H(m}{W!V@=qDKcBzj-(OXySR{XjpBJ^=JH=w6_oMIY?_2l^cPH=v(K$ACV7J_Pg& z=)T?;(HGIb1N{$1zl1&v^vmcYy)Phm8qlwxj{*HEx*zD*(8qfZfRFzK&~Knm0{tfX z6wq&>Pxn5L9%S^}=rcgSgFXxNyXbSh&!O+3&jbBFdI0D{=nFu9fWFxKEP5FI2hbm) zF9H1#`ZCZ*(0}$mgMQ5DPtaF@{uF%`=%eUsy-%Z`K}7j9`Z@Xr(Emc;1o{i~t=^~5 zF98vJ3jGRV+^5igqwfIyAE2K^|BJp0^w;QnK!1b25A-qgQ127yx9A5zA4d-Z{T=!t z(BGpU^*)aNfF1$*NAzQ$e`55{=qJ7V(O=L{fj)sA1$rF)toJe0Lq7)!NH9A1>-s1$ z1Me`!w}22HU})}Pcu(L{9`K?Fi1IA(|0?)>9sIose%=QE9s|GbfZRj zvR(jRumqlRJ$T5C;2Af8N8AFQa2t5Q9pL#+0*|*FJl)CQ;r4-NI~6?Ie(+=;2S4@z z=m!FOy%lWp6<}$v2fMioY~gOOig$oDyc6{QJ)r&X27P}YX!|{&-yZ;7z8AFm7-;ZE zKz~0B8uB3c!Eb=>TmZUpBlw6HfzP-CywG>R*IWlW@;%UvhrmBl&W=SAQjUjlyR#o$$L2XFEZ;6+{oe&cnZ*>{3(_$TlN&jnxk4A8It0zY{* zc*b9Xeq9Ur>eXE+ku^20+ieYi2cuw(1<$*u+{!Ds3 zrZ-f|^B}pipdZ(RkGdQ9{vP1&ebB!ju&1oYd+<5<3V6zM;VGXX55iO8`9waG&$Fk@ zta*yF@|45uDY(}IzmC}P12`U{%G*1J*8KJE@9pf}%+7A?)q(D&=TGDY-#@(h;f05X z9!@-*`N7#g_}D{mr-#1y&_^D+_o3?^diHli>1)Ed%qGr-0JYIeF12} zC;GSg16URP`wKg0jwxap>)1w5`osK_KY_oq`j!&u`ao~b=5Gf)BEu3KDQizJ6znq!^P|laDyk{B4Kn#{wP=7bNb1rAH8uoeRSROfpnpe-*oi; zJ*OYNe>q(^Z~(3nS6T|M&%G$oZ?gn#7VB^#M|atG`q6diqv*gbw?ywsg`+p#a!dLa z=tuwkF?9cvKEUY7AFS(t0GyzA zWV4xan4f3QPvXb1LB0vtr3=R~WW)1gaE!t+4u=3o0ooHcxzbb>n$(6SHG>dmvB^on zU;Xp~;&`seL2C%sD~=|r3Km3}d-u^_7@C0NARL##aXIDV6mYUV)#^s(!7t`X#Vws?(A}9I&5K zK8^zwvv9Xy@E`BWhufEkK*lE2YFC#OUQ&g`QgL`PQc+pW2QaTym=MNJT6Zd`KwLNzs(1qN{% z<*yx@*~!4KS2B`24A(xBPpE2E=fUViIiKh3lo=0gUgESkf_Pr-JwomvZ-Ts_1gQNa zh#izaP2f)%_*0?U)eH$RsZ<;%_{6tyINFOsx5G>*Fa@b4mMX3v@eVkG*PdS!t9Glo zsj|CuYrb9ejvtLxrYpIrio10iymz02P1`iuoA>NmP8f<}Y9`Sp*6rN2BgD3WN7Csa zmrka_HQtT?`Ih^TM01ti@!> zP-A;9z2TK zPHYDB1!Uiv?Q2bmnOY=pYEI*X>Izi|U<*~0NMBk_YOE4d5zh??y?hb3=W&f|cDn2r zp6laxiem{$iJMA1X%jt_kW8^Wq;h85Rb)*wR3#R#?w@FEok+O(Aum@-jxJVOiH?8;Rd!<|#dKqR zHEfSpt@^3;LZ{*;U2({A%u+3}v}JoyleOj33X&ure`7jdJ++jWs|?M6EhX@8!1sR} zxMBgPxW2cIQE%IKr(Om;3MYvY*hVL^t6IAu6@5jF#WB%sS9Ri=FBib4VgkB(j}s6f z=W`N?4^{GNjFaVsjiAl4DbW!ijM}}eI3<e@XRI9z2io6F3fzJThB&^IESmnGv&I zY<{L*?JlK)BsaeUSGL~p^iwkFys~`FKVN*_o37p!6T)EA1MJ@E-WG7}zXrL}dy5Sk zP2k>O{xpnnOw7=MMQ#DVW!r)ny0N*LdS!YknMjh%WZA>j7kp)1I<0KD^34|@ z1@CLVC;{i_McqzFEG$-Q_i1s5GtI-2-j=l;Adm7LTwRWn%RHsa^cIp*PX6@Z5NJbB4x+Q(z85$jl$)?Ji_V76+crbDO zA3;Rp2w$yWUK6m$hjotiE7SWOpyaDq1qL7}dj^f{tpOOyEY|{zQ^TcHXE6-3m3+hx zv-df?i+l|9h3bk)<5U!(>3^=&iP83S0=z5JPP-$X6xRhk9ws_Z@9>(NERkbp9y@F6 zE4L=cG+rS1UVPaHb%Su2`MIF?8+9I@h%;!g#<_RFv|7zY@MPlzJXbJP z_oj36J4c*zFC5?2OkZjw3U0iRG>sr1A8zBJi(i;`V$Ho5O~2)>r5)GoUVnN!2&T{3 zK7Vr4dph)KuJ;7Kf*b_>FR(sEh%F9B0**HDy@C>`9eG)k^O#$ueiIr4|4m#hfu{8V z4EzVRpe70D9Eb)C45F!|%VvVtK<5SGw_1~N9J+!gf*Ucb)ld>-ESO4iW@=QI9kOfq zIk(=n04@@OZ7MEs7~`a5NE%$vku$z}HLof%R>Ao?iV}AWj!1o(_V8PPf9oLQI)+qe zdku~{9AnUjG^!(&^&yIlT%V!Riv0X0M8s~V*@|#6sMkQj7Qu%>U!~T1Sra*>+?>yk zud6y33#F7kr(%R}7VwIW;d8WddSSV` z@!(W_O2e8Y2Q%4XryK`C2wU26!T`h~3CTvq&sOu(z9cJ>z-wo1Y%G*b9lyw_#LZoc z-BV{mOI3Z%Nt=#eYlk3EnX5x@H6jX{E32tOEl3T=Z9!E7MMw=NYh#{2!fZkB{QSHu`WR;b_6p0S+v|w{gaS<^CD9;7*%?z8ZiZ=w#D}^ah+?7K6WHnNQ$* zz$LLb(ubENFt7GJ>9j#@eTuUm$J1AoER)E&+Su5zZB!jSE)f~}9;;{|DPHtE5!68U zv|(Ss*sT_`T4yVKbD0AElQ%;%zPM%Qa>q0*4$yeePMke<79cdRcv-enBDvgh0b=3{ zHCI)2P191d^Rr1VJ(rGr%n-=?4zR~a-dk9Jhr0GPXD6~J5qqojO+kW6(RUusNrXA`0>$Xe3U5?a`e6Yz7qCm-?PT-q znm=Ao&-zYL;}k^|IFY3Hp0+!|V_7CK4qR-0O9|6#LkudN3gibd!4z?GPd5ydv?r*B zX$JM#V!I6>8gk0S-27b1&S_NVX9bzCe*3-0zJeROzxc8f6*_@qI2@iBl(e}YIv%z@cBk&S4WTpe>6DfjO|-HQ=l1q zCZ3Tr%oSTp#n!0{rE8%t`QBIY8_0R!b0ns{6O%#2c(J%vG6k02Kmtg6QU{dLowq8( zBc5NfQ#+>$lQCYA5@P`d6XCIvuh!jc!?enIk0gc;o_pRzb8i*XtVAn05@&)d=5V<0 ztRoW}o;dOppP-=2*O=IwhkF)2Z0X*+~DapMg>4QOped z>{e?luz3zs&N2aj=1F}Gt44sCbNxP&j?#-WN|n^)w#M|1aW`hAr$>FqPBk_R@e*%V zXDYF{sg*s)m!IPllC~31)Jv_Is^zFzCqLB4=bE^k-7p3zL?+){%7sm@R0)%ABT-u% zOJFWl?}XXOn3=P*nDA>oQBH(KCy@*mtIan4S3k5gSM%G&`p^*5F|YSAkjpt}GmEe^ zvtOWb-|xq+sKL+$&>Wg)c6~SM+Nd_$3kKict4Dt4{L&bPeGBYwH?tEc#eYv%7u8`k!@s1;J>&DZ7)@0et>I6sm5yLY01At|`6*LoPxM2m& zI8O#P=~{911Yfh5%VsON<3H>}9l~%Qy=MXJkKWVjXPtio`PBxR;j;;q#} zwR)^vs#Y2A{NC^J82JaVj~Uj_8bdsAKc)m=0DT|;fs|Hb1{xyK&JGX+w^~4Zv;htu z3u(j2i-IA0n(SmjsVjjQ$hm4gUh5RB`lYm*7F2RceCfil;>dt*wxYe#X90V2L<=<_#y^^2TC&V`0RzrizJrK8`b~ zV9KSaK#&E@d6_V8dNL>oXKWd%P8mAoCH%>PkyLku4B{qE%UfIjBQB) z)8rtW3uY!rdPOs|&6FXVvIKfJp^IMYl%;1~@}|4@T)sI+G=Qk&u9!hvMEsc7@kq4| z;f~5f?EQt<3AmXLwU%*U=nUo!z-n?QiSX`Wm)i;MmQX{;`3zGIbEvdPcSR(Y}M(NZE9Oc_ny+1>~{d!bF zZM2=`cVgd)^9_n*^GJy>`us}QS5P{1K8ohh(S8|lo_4uZ2gK8%;2ro>q9!^3x>$Bp zZ$8`F+VBc-K^HAk&m`^A{Jx3m=IX^tN*)D|70;A?IU$P%Kmgv15hsQDv6AS^e+{jz8m;C@3{yJbKrrN;wJL41!y3GuB9r*3m#t6L zTmEEo!W%oOE+}}x&Q0V~In9%krEs>9CAVXaknw$2ZoJ@?6K={9M1gF~W{l(C=$_mO z1rZwUV=FHQqbBPrK8KMG+ z-=b0aO=isv_v8e}jnCA_c4q;u5(Q+dTYm)eS0sNdv+EcqX}132-2~V30O~ow()c_1 zbs;pFCp>o3w!-oF{&da}2JO2N+85PDjJ|9G#vp3C5Qe%1*0w<-vXGozXi_i5YgkS= z7;%QFWfNC@n25Tg-zr1UlZa86N)QPY3D+|`hrgK0JHlt5#;Np~NzkVslDp7G7X6Cw zOc#!T>J%#wERtr64VN&M)TwE-S4sx~9h4ARt$f9Y1HNg~((wXxQR4N=a9IWG&YAu~ zO?Kp|Gd5JF%cb+yG40b7QCBe}aFzOL=VTfS#h|39uC3!sAjc7CaLK>&RT7NZ%676zxw7c=!nDTm!|SKPl)-r4iMbXhbhm z+s;asMvBy~oqpMm*0Y#p00!|Kh||zre1l?*m#z^NUiS(Ks`To}h%9_lQ->z#J7uJn z$UrN87fTuf&y^0g@uu-@Syj}tsV_fJ4ivt8aO(Jn8k@432yL6_J%RoS+E!%pbynpI zA(G}9@WR^CgoMkbSOH#q%KBx6#Xh+{WQgpQQ_X8f54FO%?9+xCQd}*TpHtjaEUwR; zo2_alJdEml?|*wAMn7bJk@7-ka*cSxbe(4KE4_y?QLSmFXr2$z@eNyAG)Ow}@};}?u=ogK9tkqAOkGaad~(|^JbLK}9V zR=i6=vMl&2J~BG$di9~Q?G`lXooR&yO-<*^kV42< z8Xm@~XW;MD*QF%c$fUj$8;VUHZh-64B+1}3Rl=#c`PsBTyM9Ahp3^jm@ccx=pXxL{ zFD~4Zs5)bs5ar;LpqtNFk;kAOMk^2{>J_OFRuY627Zk*Qx;2`8d4sIrOYGH(v>`9;JfNQd>aMJj=$ClyBlxL@C!DetVp{jkdul0A9U8H!8IT+9-AF`>lNbSJ)a%{V8+4Fm}jY4Vvj zzGPE=L!=MGV57eWeb~*aVzgGf42RcPt<`dn1)BXjk=L)5)6q{_;RByZn#^ggC@hGs zfmdV%*Y!g^b+`SWaM2iI^dIkz6)l2Q#fWDU6jL}-sJUYaBV`V0;Aa#X3kWTCiKdrJ zP`FJ**Y`!ow;O5M01cP$-qPlL@s!f9iyi>Xv5V)9Zy_-vRCd*I&Cqyqy#w2Kw0Lksg<(2dA}&cisaqZ z@;L;>SL|KCX=hOD=GO0i^@c-Bx#z!d)42=T7vtsUKkvp{Hl4dJeE!$R_usJR?H`&r zaKoN=zjxIq+)3U7-qBivTO0IAwEn$XqzCUDJ!YjyPoYxZ=>cP#SWgD+^wi#&u+W-M zseW9y6uX#?+3B*U$7QT5VVDLsUSseIPdO1ZcTab=Hv*0*aYsP9;6y!z|jmGLl7eLTYc+PN_LEhs&|1^;=4_3CgDd-1aLG&uXj=LhenK zb&lg@qm~MH*GKmh!m+%KXLh%Om)uA>mzvx>0(pa@W7mk)ebJIq%|Hu|KWllXTWJ?I z?b^3d$^f)hZPg(}Q#Fg{tIeh(%^tXV!@J*0xxD+@Q<|ER2wX@TAp0YNNM3N`Et}6< z%Ho?}e$x#sPp7fz`=Ixm&_3|V(fIEu<9<{R?lxAvW15YEL41S=Lwj2*RtLpi#(s(u zqx6FM!6u!?F^8M}U{*||=iExF?*Vi0HznJ!Jkttdrl6*Cr5F)Ok(cCB(WP!i)TwF` z8PKGsOo*0v@Sq>&)$F9_<&^*Oi_Ia!mxr{tt3Y~$Z7-#JQ&TMue@BliDmQxJ96l*M z1|W+LRxvMzg)`2{%`}Q~{a~#+t>CTl{Os&3#=pgpF1mmPp8>L7W*T2?MEE353#0vd z{t2@lR>o{jC=0Jnqv*Q;THgBR(z5I*s-^;Hrqdius~*FgmDTg%`R7h;GA&z6F6DnYb8+d}Tn>p@BsB&AX@;yKG1azJyUi|4!TSx3Y?l!L=~Wilhrkq zAcp^bjeiGiX);@<_=jSEVX9eRDi9Rg@u<9FVhRBJ=^7hGSp(B(x{ecLV;Zk27&v3N zA$UZ?%s+j8ZqH;&0G$E2Gpg_qKUy0MnQL7)kF8h^0v3mBhq`av^Ze;*CO^BYk!)rE zePU-$lk8&l`2U36#KK8T52W57SUfm~gUH`qZvfJk$rYCi(A`CqF6ef2+g zLhP5;yrJ+d@J|>iW7!p!jD(_5IjVxa>;=P6eRDvT1M(C|Ho&!^cK0mRhXLUw6r= zzM=r{oZcVEAIa;`+t@5MntKevG17=;sl5=67dN7LB~RwpDzlcbH1M0WY}O2($|nYQ z`Vt8I0ZNQ2H7nP!3)-t#3HO0LgK4zV^As83OCXX;mM$rr?CAg&DE~xJ|2tiY@v+O3GK%=}CyYLlr6deaPYe!rHi}B@82D0aJm6dfWu%mlJ9%r%RS9 zg$7_K&Mxd95lq1y3A`bbmt*5uS(UXwaROVmWyzLQN9Mq(5UjXHEkQVv+Se#djg(@% z0IZYJoh*DN$CTtypahyT7R${9su&^Zg*$g@ajm*pzC-gL{eaxvzH zGI=GP+6s7Hwp2jyseSd*Mo;t2S0;v>aRG9TMDk~n#<_FF^h`Flm<7e%UYl4SEzbtO z@dWhgPKdoDYid)q6-06$N}y3z$D}pi(LqI%ik22FXy_Gaot&n`{YeJ^M8c@Uq}W#( zR%7TaU`UQiwCqr2xSHsio^|mAHr$16yQ?EtcG6XqXf&6Bd|39fLz#ouR|p-a9@@KW zQ~JR|njC~4NwO^3w3Mk6i3?IA&wA-?8))>=9NEf{=+>4DA_S*Qq>Zn*_4yrH=A!>$@F9RxXU<#-~WT&nqqfhKPxZ94>aptv(M0;9=EzDGzZYrtwj(%yas%weFs) zuUp!gwZapaM>rm5Hk4jDG~k{eagWXqdlUY*-cKOk(E9dG`{RLhDInui{7z0il*wtC zeANF0rgPK3hjNV9bk17j6X7 zViMW`=E8uJx!NvYRqTZgbbcZ&ZxP5HoZjR3YWx$l9i6u_Gl*7f>DUaN85E-OpgKZj zOQ$W((x z5^>S3(Hi6HAz2V)$pIIJ`K=p9Gcy?_k?F1=YbZjZ-*9unVoy*h0ktd!Nkb z9hv8~gnaxfN?>h(R%mYMmomhm<7X__n(RG>-$Oo$3TT4O#Gw2NS=BG>^QrsfTHi`G z8AUvnT?~c`r3LDjmC33gc&$QtaXe*b%JKZxTz%O;sHDvJaKUSi+EvRj(+>V)e%x_& zqfmUOZU?PB)2)*iMoix}&1qhUjqjY>vrgBj9`$~MkAO@knBI*v`Z)p}SLx6Bmgrz) zRODvExGOSSsfgnvvaHLtqGt?I811_Gx*tqJ5{gaDn#|UB%oN7LaCojXvqe3E6p|-iU zK<@|b)x3d@Dbp^PWY7bt)9y^Q@$G$`k#j*jrwW?vRK41UiF75G*;);92C%tV1w?g$ z3OK43#?4g8b&D#P_qkLnI1QYpjftz#F;E2nV6wA+{l@c4xlj}?9x(p+y;nG0oRZ}D z$nxa)V%0Wv4C)uyQQscJ*8$(h(1w+H0V!6Yrfg=ezBNES{iz}Y_sIkx)X717Fw>y0v&53iR!j6lMdV4Mz(GjAsJR0ez2tkhC^oD^q7!IbuC&F1jL)k zZZ>Q)Bg?j_ahwznvrk*xc4R4g(gpxc#iVQlCdGKi3_g`#zsyR0f~^2pRflTSAIT8Q zPbHs5r?Q-rRuky_dRph++lXesP=QGzAh<{(61W4}cnIBH`c<)%Sy`z7d3O}yXcE{4 zadMF_?|RXkCA^0^E5N*lCh&4xvC=-IQ6tWf0?1tjamd6h`CyWYT~Ta`fa`I>O8~JA z0J(f+Y9U=1lF3m)5e3L@AglvqmxEepf#Cp6K(fDJH)Y9m<`yRPL+71U6X@hK0LAj0 znX{i=SSS+Hicv&IIJa!$*7}NNMD~t*JiZ%bdL?=;v-dOl_I^3o`xy~BiwXzKq;jiE zDOL@>%4T>&C<>ygIGv9jMN1m>qughzJHWsL#i-J^rAX=F33Xb3tQE!R9`F5rJPaWAz|6(N#nX3?#+a(aY`A=?i9)TqNtk`YULWaP*Kes*ruaWyTI`>1RP zg5-^FZQVY+DJ_8WPNe?FjLW0Bgnj%QR#F#4j_J)v?+Nr4av$>hqoA}}qYXw!7`Sam zra)U=Tb&qmyU3ZG>J?JZJY}MI*5Y7w8GcaD9QwhhemCqJXDXFfvnl4<2wmQvo#}Q@|L1OV(3;1XIG~+ z-2_-`wC6e|z9=>7raeg$P6Yc0<3oYz74GotXPd9n#$R+*{_ zb}FkJ530trVodXWI$MtB`&4_k(MavG z$-xxDMT{GmQ}ju#6AfwxbR2DVkiWCjQpgR=F={vcb=SeSxYr4rHf^GRMv3b4oOE-uidkW;SK1ff1*VNcz^`9#nQ(&i%(QDZ3js`Y&{H|zj@e}y9_{XTj z@OPy~-=<6DHvr>eesy>>4mhq6~SiTLmEDBmqkcjCBR%qI}SYx)J zLFhKTLQ)Iyt9*w${zdRGkT>9=!!!R9HCKIi)8ay=HSZ)~2mD?YS=&M)k>@%!*k!R}_6Cr&j6V+!}78l6iUjdQa& zzoMAk4jnrP)~wKofB>B~*ImWHPKaMC=Hd>mQUDINWz19AhgLRLSwER+4yC74lUcIf zE{E=T-5;{F;>URa3e771kDzSHkOLTUTmw$E_syPlO+K)kI}BI-i!*l8{7D-;$8uW=@9V>J(j!mXY z>sz_;YA|0>;>z}&f^;D<)4Dg5bFEa_b*kDsKs1GY%}LElbWdNm_31@7D`1?n%1KF9 z!KwR=t25&Q^hFqEwKb&d7)9iFM`49g3!?hGSP^ydnfi$Ky2ORLtb~>G&s* z?1h>d(vlD1I3?xkz9HJO=BuerzT7TDJlAr%g6SMgk`0`!39wSn>h5@JegnfBCfHSz zoDMN*usWR1X{dy3Y2aWegW$W(NGj1QRVeJU8g@x z#0F~U%(_T@qHc_HwY6Gyry#0|leB_iou|uF{)$%{pULZxI*DS!6-DBC_!G`>^#l2# zKqipaOgo;ctoS$0B!;3{;NM7MbaD7gnR-mHR6^Idsl7kp=i-OZGFx*+D-qR3v{s4E zHJSlyC8&;OqL|p%{+0hLl~S5S_RFM2sU;3dhpoXX{}sjjcS$A^PiGW>nW%!SYjQde zpmVY%yCW$j$<`-}#Ln{0a0c_1WniJ+fRvFF>3D%;exaP!GG=a5-YJ@H%@KIWW59j_ z2dAb|8*baB(A>k+=AM6sGCnyh*rp8V+c%u5o0(<({Ta}OKJf9tEX3o&K!=$7N5Xx0UbRAGKp#|J63$* zV65>zWJ6RLiN-<(n63Z6&r((p8l-+T>IC~^ilq>Ij2BI14p_zW1Pxyii^iH7lwBq6S>WLyqm&IjrCu@XyhQ8k2|%5J1n`8KrUFW4a5@V zxLU7=4}<;BvoRC8{)o<5SdFc81yYgq{XRf0#Q_%}xX{{fe-%zNS^~j_f&gh{aBYP{ z5RK_N9jZv}SS|f|I`P>WYP8dCGvQYAJ`dU-cP1JW7gTwbPEnHMYVFk0nUMB*xsWqV z{Cyr$44E!&lwxLf`>AJ+V_N&x5^|o#Bu6Mz2Oz>v7}`~y8rNny&`&mGeCNS zcuG-fKywC%&bN9%v;d+|JfHYa04Ut|M4CVoRP60QZvw0~*r$b#IBJ7Y2QFZ>|TeX*pPRT>8P(3RX(Q>B}AL`(iVQ(d|vN)%f= zEj$gaPS@!)2wjsL@SJ3N%Fb3jznRihtw_rO1iWie@W(4bIM$)4j>H16bTC_TT2K!J zPZu(}#6eDpPu)E}ThJ|I3fctunUjy%CgCm-6O$*+E$*Mm$+9T}y0YmUfJg_^y>H=% z$Q__v6fZb^-7ECRU#KE+YvXIsADJ%|H&R#a5xui%ceT7R5OlB)QL~BGTHZ5MTuj@# zTm$U@+X~LS_C@O!udK?tz>Bu#?!WrQ8)u$AmO5c9bT?!Z(enqZM7FRo!!0Oe`ajh_ zV`GL(cy(65>X_$=FyO=iQ~1tjZ`X{R!nxA z(BtTNY_A|Hlfh0wLgR1k4s`zC*azru@ArdHv?tN;@dEiC%76!=vpwi+-bQ0}L^>Mn zXL`P76pCYuNKLx*#HZ8f{_X%3uC0tvVY-V(Ea6#>1MWj0cd~CdckS2}oiD}>#;$S! z2l#QQoY!1CyD>-jn2-PNm{tnu5SzaOk$DHJgUOAkvOzU!U4zvN+4B2nhYWFYuvG?E ztG4^zvj5TQH%!Ue895&}vdK7z+*VShqRffKat0uet`vucZTSQGcrqBKbN`iWr7Vi2 za#pcDT@2Epq4|m={N8A!;=|>PjCIT8w25vLz;tt_qXVkcvDuuP4QvBDOT}=;Aev?J zT5Ebjkte!S8pEAhp<0VsSwnI9b2O5OM~~qL!DHv@| zKb>-Uu~H2c%hRP)p=`^x#0!r@6N>9iqFWX(=PJYHv>>f&2hinrCxHpQ)WRk5O> z3J$VZU84gPt@(6*+yTQk4S@}LaH0^C6}rA<*s;<8nQ(1a)yXPPt@u6yclxWt7uX78 z6dKdPi{|>OMRbvt=P9C-MM0G%70@>Opg$r)Fyq&6@q2+^hggl_AmtHVr~H3BlJit% zQC2^tSV^zjFB^Ayu0 zR${h|2b2&;X#`yr> z3<*#w@ijd|hH_fKkL20!%jkae&v@}kd#J2gbzENALj|LiO;j-0@lo^*{IvdaxdG~- zbCUf&svVh_0<5>>sXkI4atjHg9@&7{y92!r-GD-71A|$=G@iyIzYy(nMEyah zB$ilVd=azN>$Q?p3>49gnO+(gZ3&zTS%C|=0OmP-x|t+=A@Cjx$|3 ziXn3EwYbHeyU1o+4xT*Hk6N9lKAkNbkG5H&Y7|fn>XukRr4=pA?Za4>9F?JQvgpQ5 zk1i9FR8IF}LsZ@%=r`E9NUI=VV~N1>0R`!-=`qObqjvjIyCHqqxWJP6nB%)?T7RKy zOUgme94*tG8|3_QRxnsh{0+p!Z-X3{*-jIKSV(6y(V37G1=9NAbfZ6esTmc#*X|%S z;A?e?4Nk1tU@EeSLDEG)bc7;HmjgPmuW^8(GOA$D>rn^d_FMO8&5vK(&6N840EUkgaV?|^tYjP|U= zOFGg^vk+GGH(8^Z1LwO@4JF#Yj{`?8t@)O)4y`zPi2HBX>IpsY9G-Jr3uKpw(<}lj zg?z$t~cr`iSBBbZmrf@fO({=%^3DZxyN18Bcfm6jhLxBUZC&8`@oebi98Tm`H!m>brs$TN*~#rVH%K}$ znj%H?6Y^kxKR2p}N$Oqt<@9J#C!6BNO6l(x%O0e!SW(ipkS`Y{q8N}+l;uKP4_!i+ zYsGW*F$-{&X(%){93ml(lW+9*qC-sAX*C73r*Ff3WdAM2i|GJM%zPoM+9A=4#Vq-T zYSW=L_%N4M`J@R6zcV(Rf%~3D(%>sU4Zc6y_a0s&x<9SkSV&%}Sg^#sxH7-H6cs2s z-Og8(Fqc)Ngr=x6~d5$FgW z1u0gLk0%qRtdfg`0$pm3$0ie|1bk9uRV7wF<)lqHTCa*OB#Y>?q7YaCKzuHb^&6|;$dTsb2bO{deTVm2GS zz`f5Wvv`CY2V9YSDzfOik7{>~M>xMb=9g0D%xG<9GV2BDbiz-QJBR0-bSYk+ZjT0G z-t!6t+TWeMQS`Ikk0I}pP;}Rz;}7CIEz@-AFwZzCE#kARf|Vah50A$+-KdxCWM~p0 zOH0T@*;F}UaKwkuZYEsS0>_<-__>}GaE*Kcv}>SW@xFegLBDAIP>%E~D(yE{ivS_p z{EmXwcPZI?MiD{5G`l1lDFf2rNWD~5(n(>UU-1d^1nnYx)P?~4fJ@=V8eFDEk+D>W zW~mjxAauLzKCzEiovKUTiMihxVk#?XQYstyC8hUA@_zDKh_X*(mA=8M{lQLbi_n`> z00AztO2--fvuK6llNR)^ls!reO>|GF_t8pCzhFFAX79U{DRlN34dyLofcUGNNbI2? zw>wEnF<=5wT1MRb+1JrQ7QV_wUKU zCyFhL3a;iv!X=uC#AK?Ge|=+1r8wQ4DLNxGUYC0hlTq>kh?C3cWL75{)Z4|rubGCc zvW$CM`)Bc!Xaew!#!LSi?ew?&AJPGW4mn z6JzTxu5Seqi@Yo=F-v1EIG@d}J8yYpM+Ra5-Vr#}cx6jio+wNY`Ntnrwax{dLYr070~xShF59kTV<8AG^#q$ir46jjb@&>I|SYI z4*1hvJK{}Q#Ht~!z6wP9+RbOA++22i|MT`- z@~V0|k<26j&Ela@=!7puJeO28QT3Q8bq!l`edcL3}3Lhr)-)+TkJbCy5_eZidjM|G^t*(ZAw6m zaFN!%D1$rQsVd?0jFp{9;@M_l(WTop<{Yyb*0uzxJu_3w>526f)1m9dI>A^xh|3zU z6h_z3n3lcmPhb|S|)$|UB$ReaCQIqJ*5fP zmLr@U_P&kZhyR2|!9KSj6hyNnPq0YlwwXl+kst=eh8oi~YL9-+9T+Vuu>Ybw*_fwZ|}fYzz>*7W)0hJB5wuHnnygFRKkB-aOKS{Li>%^ zI|MG*aPV40%WvKx1%(#bT2~z-H4_UYOR>yCgwWY!Ol`uco=!wh*RDKxBDPYwHX2*U zS2z_Uv3~PuGZQn$%AqV?e>4k*u`I==Z5%fv82>9QOXcaT9#plYf`y3_^G<5$@#jCi z8i^mxWT@B-2SoFW!Rqnx%cp~tV|CIc1TG(5!!fv^Q`t#wy$k*g{yy_b>cKSX*aVNq z>J1(zGEhqnU2wZUsB-aCcWfLV`|RXX-k^tA0>zMOcztJ_IFWMZ$E}!_W{JScU+N6k~HKd@Yq` z3n5F_VZ2J}nLMi}#CSSBaIbe$xWO5+Kqs0wB=Mw(Q3 zs6@orT(K})vf_Hg5_w)9qITZ~uYhmEFEaNtpJG;;dsTqu!= zPMIhzI(42a=6y<&;sTX^*H@XNgk8dSA z*f43>CdC32jk9!82XX)@JTIUaGDXUD0RfIrSpls9F_c6#q$uRPRqkupPVnG2oYlNl zA-!V=f3qCY1j5PB0+GydN;Wnw8ia%(ir2WzY9>+(3LG}Xq9KY{F0iYjuzFgDtUh8{ zMH8PRD*-N4(L^5%cdCdp^l_Qt>mz~k@_@ozfH zobfYR5k!SRR0XLsz(JD@O+Gl^97^gg)B|3GL&rOj`C{@PEiRtkO z{jPi%e^Yomc6npt{KbtsFZ`-#;2#O7Vg%n@D0h(!8@2FAKefQZoqihc71j9sZne{PLA2YwHHi&>zyi*!ws?m&_W zJM7g|MjE#k8A-(Iumq{a&Npm1fy$<(0^X$!Y$U%=I`fY1=s5a;ykI4r!;TVGCRpCU zQ94jeWaP|1!qlT$GQneMRl@jNQpwq*9#I7J+4QkgAZ3_YLkvif%#n-|Q>IuR%XL^Z zhfFhN>Tx}p!f5h_9LdaOQj2L>7X$R}Wxfpa@EdqXL)d2~@`mazPkG4C9Jw^6GnU=d zDjF^{a{Up~PDmTtBEF~OTWD{HrMPx=?#Sikp~C8wBXd_)3umw0MMiHfZ&X&t62Rsm z5?M(Aii9*FT4Yf^tqv&&VY4gcV#iKIN|Q;aiokC#Z5|)WuWTM$+*~Wi zJ9hQ4_JCUZ_0u>J?#(Q z66+QY-E_?aaUw_{>k^rG-3b$H3)AENAb(nnISBMRA=Q>uGb523Ei8$NR0`i9@BR92;z-Urlhah@?`1y7 zj5VFgMw8@k_>#ZjOa8{bPo`D!*|u4#eV)ZG@xN_682}*UM6K8Tr@f%QOZLAv8H>fz z@ud5S-k~nOPhfm6GLKT3zrDWATzUzkcss@Ovj6mu_q0D`ORW3D#m0D8qcfA_)ectH zpA^awt85aE9bJT~#AnMDowTdS3U~b=q#M2oZC#0qydcX$fEDDBlhLpbz-gKSZr0lH zp*H0DC5-zW%ry`BhFlYt|8p2+m*s!Mf4by7?HBo$vPz3_Z?2;35BQ%4lI3W$m^2nf z!^NbTZ^c@(&5&hTW_T~4v+;1z37E-(Rh!l0h4+5aD(wGmC7fU=m13IY{BCMTK2EL3 zMU3==O|nj`$VK0ZT=cET#eE~b>;CMWvEL{1yG)AEUj<+<2eHe_{CW2(0?FsqJbv7B zeimlAz88jEwo@*CaO&J>yX^gde7f%~cFFcbm)jQmZzIeJ>V?nOn;C8gVz%Aof(<}* zn`rM{^}~P*Uhd>QZSU_syuW+s{_few{ydKebn%D=*z&v>Pg3^-r8MH~1j0?ujuG>n zoqjNY)rK#lr`mfmZvCBLahmeWuhOm&!k@B_6L0T&41mEMZM=$p_6z=br`OOp=a1sO zou=o!wD(;#3c!ttBWeWmHuVAq*xOxVC2G}7o-{_0CVqOw0~HCfiu&5)m#`^N;v$Ew zB@zwRBsaJsvbv#q?^>vAy~q3z>u%1gyOTFa^vMj&w(9C}*`*2|_Nan~tL-(i7lSbT zZYOPoTR+6>eH-gyj#;PcUGIOr{pem}3)PD1vK=dZYFMv5v75cM3+va`Kj8hn%dX)4 zjd*il{562iER+l}tqc z@-wSf=9ACe)!qt$)#so6^yf}Ia4h}&Up8~&Ry(@_{#qZTYh9viB{idTueJJJGz~Jn zCZp47-OtIHTv|a9MU?|(B{ZY~(me;HcJPLf%%){6E@@Rcl9MT=b5O)) z#!-2MV>HSyjPG#5FI@K6f62T6sXf>`c9L)1NU7~@-HCm;&CT42k@)e}5Z=f71`{V4 zc&5S5L+l{o!F40-2`|E?W`pf=1Yn(XB?bxiqAuY37NtvNoscm~f6)Pd_^(_j8j}$x zi=rVYAt$Rz0;_7u&8zKkXXyK>-hnwBj}dHFDYB&Fv|9Jx!qXYqncf!Q#r81Y(Wg>( zK)zRV6m+?H*+=kx9;F^1=`p>d?!H6%GXA($A3YGJ-Wh-Vb)1$3)5NdJPd7?h)WOQ{ zna(#V#*#^TlRNs(n;c-b$-Bx`6czTx`&;9~@i_S{hQlF^T=QA?nwQBnukfUIcLwA6 zHhTp}>#h$&Di%w1mx*_=eFVfMT~#E25HAO?ssT-RuFuV$8I4h5UcOiqSe8=` z%k2%~tS(M`VhIE-N8>mMN|z3Q`P_4}`9x;+>_}|V`PSH(w20Jf>gIoDren)z;TO8a zOzs}<$9vrH_c*?XJ$5sk{qFZ@FGD)u&e>{L150feejCg#&^4Dq1s%2iMz29op)p}F!RiL%Jy@rzRmi2 z%Jm2v#(UQgV^7vsk~gND`>s0EA0=&E1GB+8q9fu6QP2X_a!!;2ng~@fSPcWaD7WW{ zjp)D9gC`bDum-+X6irLVO{kfcAV^5G6(z*EIY^AlX^g`OtnGxQC0}===XDRArG2B* zUI7NhCM7=IMiutTo(o=Vby}dyrHlQ;73M?qtuv(Ra0!20tv5635>7goRoAUOEO}3( z`()Lf>gx*KGxP2@TdvUU+FH{UN}}PtQ}liehso~!iNj=ke=3bI*}p$om;V&LMfp#k zqVnc4pMI0sth?-|n=Y2Ol>M|#5B|XtqAG1V`fmwID!kEQOu)zbpD5PP(Ei;`jPuj= zoout4REi&epO9xe&meeWmim?LIP#WYue2ESo$`N!^6e4%C-3keo!Sh>-j)5D_vEr&jChN+Tg1OwG8hc=SZ_q}dLCf%S%o zc%k8BjX^4IIX$HT=S(l|M5Mlu2bs!EJ67@5b15+AFiMHYS= z|G!kS_)t`{odCeN%coF0lU0GGC{6~)yHGZQ;AJtY;CR8zWX3RHW?|t_ej*z{63M3p zfhnT|MqHFM!Hz~Hd=9ncllVH84O0;D8KIkhjYRzGO_u{^GiD{{DYosKlr}LHsQi+H)wOP^8M1`8n6p-{m5|T$hc#Z zxt&Pj)(H4ebnx z*^$~iN>X>HX+MBGw~!kl*sP(IM!y#yQf(;AjWc9 zR9QabEPe3Q;zcJF5?EHz1EQGJ0~)xyRF?7Hr2wKc1FOe4u+rYEfP^QQ`Fo6o&5Za} zn~vPxZLG%U2~CXRb9{%}#bzDYz5&}Qykg+i|KfGMMb|Y&HJz+$vhK1-7x79X4WDxN zD!YV1EsW%Lk2t3D!n zzKhrOGR3sE^Q`G47Rfp~BwsR38IO01@@KMRr1%`qh84Ul{^2$r@%N>WHJba~=!m^R zRFJc?;Py;X1$+mm7P>su@axPpbBXqrxGdGKMxOQ7HW*uVskg5*2%+`)K0Tp9d9VMm z*X%u+$NLF*9iJEB#h&%&9dh*^%-C(T6ujPzEc^3ql(sX0Hh80>+P19uDAind%_Q$h z7#|UnbNeg1?x3>Aq^r4ApAnX?4BViFC-)cHPP$s^WIO zS=qa+|MV_>-+2}@{(KgdpT-!bI~DFOx-mQx+NJs6_51EWv_74>es%cNbn50?Ml=(w zj_UZoRyL{^b{m5KzVV6F;+3P%K9Me1$A`73Mm{w~h6Fm9BT)N0?ZH zgm5|ITq(My3RSCdAJRgq4^)9Dy zSOv&IIQEf1eIlqxycE!blA=H}n>1~m){K7&PQo{tHD;4)2g!qz_plTwUqYdg?LH9x z@tH0h5(9IS2W+34e6lP5#6FqJ=en_1kat$S3tqu8J;|J=5}mAjwY9$3zey3T!57Z9 zF_V?c)W)3Sn})xVMAK3yECVIz@fca9t?sv+9hEJ z+;?kwb;sGey%14P&I>vykvs<34RzbXuD+suaf_o;=epbI-Qi=d(*}6HRd+v9xpFKxC$=R-M0@BuJN-i zNbV8Dw)%2V4CL{4inMQ5@-OV_pMy+rYYRMr@8EgnUivm*HxJUxR_?NtA2%B;^nRq{$(EnrC~C=R6{3!T|mbulQa z5NYK794h2ko+Pv&joW{=^?%@}@Yk3Wj*=T}di2e_ZVkp$T9cN-T*yKms}Xh_t3V*{ zav26;`+IkP_*lF$VULZEg%6#s&H*hH=OsB;9*(Bw>P{d%nn|P-TYL)0kqZg<|D9b6 zcpTMrzW2^^cOJVlJF~Oew|2D;t+WrlW!aV=vWy?{Tb2#BF|sYNjV*yJ1K~{|#1J5a z0x2o4Q1T*y5DE#UltP-&gaQFVfDn?V2@OfgM_o!&`hCIbxp!t|8ym1IX|L|i&d%J& zx##@n%(>^>R#|uMz*#retY|D`-^kgETK?lhP4~6HBZ|qsqodhZIRKD0WP$gMQ5NMo`WoEOxRmN{ay{Eb+?mclh39R7Pjr5h&r+>GsoZJ^vcSPyauSiXH^S*$pqx`kAZs=>89{^G=J$(cn1(hG zWYT@t#yLB3*?d!DdbrkoYBn6zd5#Ok6-g82pe(}5)3l)N^U0D%LJdMR7EtY|4wFqp z8YfCn4h>c3HU-8u^;U?X+O|C(;pzXH{s5l~^WV<=4Cw6B_~|rmToFb$ih`)|sC-Da^E#e?`7(uXSW2B?rRbC8i?&(lmi z%y%eFLQ{aF;mdeoL1(xn3wS1-FIlxNN#GSZs1JO0k*x5-@;MFd%~4L)WWP$9|!_C zif(VaDdT>W(?a_e^E_VA6>iNqM2t7J*zI#b?gmT+4_vbd8nJ9nw5=E=LEWc~$O%7f zPSn7>6-x!1TIGyC;I&5gHK@JiP(EoB;6t=6)BipFpKR~u2%9(RskIp9O@wh<%;v4v ztHFw9F~g*Onw21cb$6!1-tuO|IJ1|NF6un(3ruT|R|m+^<2{*kw}e))UBP&s;1_Bu zLp|qr6JO*dj7z7Nppu)P3l7iA42I~Q&cMQ!u8oVzKE?47PSAkY`fLempr09YOQT)h z&QQ?GWigZ|Yq2me1P~?F!r7{$JFvXdE!6Hlv^1=^`#YWP4f8OE!$nahNi{?jSysn8 z=a%go z7~4;zNGzdg(6ofEgX(OBd@^tdRnp_KXbO5vARyWr8X`VH@yiDJhTK%^jzkJRH6DXHbg`c%MVx*~)%yQiz73VxA4Qhr~^o-8wvLL!_AH>#Z^&vx~e-}0nh2Cpx|_tW7&X$09ppJ zsW1(cM6m%OXNe&zD7egx6KQ-Nl}26ybfd~&#zc?c z-{Nync_uowQ+4;UvFA^e4mb)4-8IfESUv5`xSe6*oW--x&JWmrInk6i3v;40n{X%> zw88@C*IN`ZoNjikTrO)CEm_x9K@C3&sIhl2r;@ZoqlL#y*+58Qn-*( zsr)`beu=N=PC3b#tp|?dIm(4TyoYpzbmKMT?vuRs$Z>Q6vghb)WAqd9EWUx;iXum& zqYrNb`2F$S`{BzU>93E`=gH6T?cCb`e=OeLpV;#QPW`0k0T@k@61K=|jHA@^2fHZn zX#M_c$13dPwd1Pb1Z^Q1oC98<1CFRigJl^SKhno%%kjK&$Y+jqpkbIWI{i5QA^8R9 zrI#@YLOq~|f!pjwSv1Tf$R%Elj;0q-!Q(Mv)@3(cI?LS9Qr{6%-zsz1CE81`-qY?d z9~=-u4psA+YX=2_E}x=*Fg?tx-uu(>^TblkWFX$t8UtpN&Tzaa0c!|4F%8=t2`y6A z0oswsylA#ch^_=1%~XGTIvh}fEoTzFh%BMpObH))0upu{`P5RQA*eS?Olz7Cc(5> z3`!PhLaZy3w98Bm)A)ZLRDri0Kzc5bxe!J4AT^^@%XR~LhDE$3H7Ch~4k+f5r^J;^ zvDF4iM#4l1L^>LK4h-SO0!_%MXEeYlQb3jnFSa!-E;qI}g?XQ(gcZ@z{SjYNo(E`M zJhec4coipcW`_k5%`f$~0D%&o_s4ve?&AeP*XNo8Wq>iERtrR#-7LbW>MiD2UnToswz>3%Uk@^{=kbg za+;`Sa!oo7UD<4lAggJPCycN2Lz`7jmM;TIPS#N{;1@tEDq2~e=955K1CXYJY=_~- zg)Q5+S2*6u2?CZQ@uaqJywxt4jc|iPR1m*TLwP`7Gt(F2SIHW*i|yE>+$di?9AR>l z`GC+(@Ge7vc4zXmQ`Eb!o2CF}j$(AfL48=_%*g6m6&WeeEtc8(0XN{VJ4iq?o|5a~ zpKb*}3V~tZ{g-4HX9OPsm?|olm>e)1o+wejB5~9xc?%#Kj^@@O!39>@U~hR0@3A6k zY~*P2$(-W{whq;OG!>QA807fD3h~*=8EFLmb$vQ3Gi~o^)8gNX1DBTiKiv@ z+5h|$tM+fZ?B>YQ)39dk8hRDVxAdf@hN=U@@Y#-+^7+-t@j~w0=4`lZ2hPgpseQPN zHj?M?t*8Yw%Vlg_8@zEfP=hzFd9UV1<+0g#F$6=&cP5y&>l4Pny43H*F816kh5EG9 zFN8@fx1bisT(aDotPO$s5|E;U<)(p-RHl1DvSUM7OKG9@7yWU?o||p$Qk{$)O=Hwl zEaXBlP(zg#e8A3kr#X@Hrz^=orA3uP5{@VB;_Q66KNsw*2C_a~eM-)m>1-_ON%LMxLn8?S>4ZL!8Do+xi-U; z@P+vt2EGMaLb92MmCX?r6A9WT;}>Dv-S{GIR%6#Y$kUmRHA3`wx{?#-j&329!@uFh zNXSU=(0wy3GlPVd8b}JVuQBQr0Cz~{9Up2iv@{UJe=jHeMnH3Da%?SFw@x^14nR~# zjaWt!Kr`kJy>fIfcpxsCpN{t91=NK47^0wjfkut)X8m~#uw6G2Ty)pz=`x4y!gOsG zvK}*}T`O=O9{fTk-vw(eBn1?{;6xjFL6I&JgEZ(qFSs4vloA5L;%G)PBoYck&q^kM zUK!sg$NWJ~kolxxMMOjO>Cz^erOYSFS-+vHHn-@|j|+J^Ilu|MZgE}?$;9;gWG@i_ zC()dlIZsB=_^GsqHe1(1Jr=f?XT``G)O5xCkPZtC3LVh-s&qC?1`yI+hyB zwx@CXp*KqXjsP;%j^e8b5k<}i&=iOe8wpkMlS9uYoDhI9mD+Qjo37wt@|L$Vl}1!% z@zCjd_k0TK?M`w`nPEFHK$`bsmJ_X=g@mQTUzBqxBXW+crr3stP1b;Ju)-qTU{2Na!fk7ymwt1nWz4Pd;3y!;WfubRA z;{sf20nE^>4782Tx<+`*P%V*@L?hYQ70DI?Ce@AgMqL1PT)-C)PWD?dRpI?M49YqX z(mbcwn5&fHMw8Jk^9rX2h@@gr-g#asm6Q31ABZU)vS4gDDJVQH=>7uX!cjX&WEu&M z^?xZ~m(PIw4bSMfK6k$W>kTOE(veXR%S&eUFI>`h*~3Z4PR1MTB>v;-HOtnmU2%JV zwK})2t&Q>rv=={zuLQXgK!WxHrT75tSVClqYX;0}FL(eahi>$a*2 z1pla^pu4=Lj4+u1eWk?qQMuQlGWsz@AGrCMd^Wuri`_ii4La}PF3Ed@m@Rl)7+hZ= zsVo@Arsg@Hy`09x=r5fq8Zxo%mxWpVZmf)XojfiTnlC!-{%_N`7hA35Ab*ixw&HKUM`y&_RO9R_KcwgizI->`eRQ}8@j{S zWLITzAQ5;PLo1hBIji4F$+IRpcmr@J7pvkwtiFFWwcuTfg-d-toN2h_&}+9yq#-!< z=gZw}b_v&p07WO<--H&VJUNaS-6Ev0(P{K*wYCllLZMk85S_5;gvBh*7ex95OcKK# zjWyc9ouQN^3xzE;UoqO>J6`=4s4Hpue!NG&4Ezg)XTH{{OE<#;R= z$q!f3e0j7M$Qj|EwX7*=`hBJ0lBT(|wVSNM1LP(cE2{6*bIcqWy)D$|$=mD^53Zf> zrsqXy{^Ot#RGnjRCsDWVC-y(KZQHhO+qONiZQJ%Fnb@|IiEW>p_ujhao=@GqSM~m| zYIjxdy`J^_CN*DDHz$J~K&=ewwF(DSbs|jRZWbQx-MgvB4C#|-?M=mlh z#kGfT&Ra|gXn2!aN6tkthk{A5;Au!Sg~jfWA(4m4i9Swnz^vKLq}o zkp!-Ha#IN|3iD3xmY~t4dpu=`;x&~Fh}q{5DA zs6&p899EUR%Rt1YXZr8_naM(-YA(a3yS!GASXlYmAk*1wasf7usoAfvuBc0P|L^dn zbdbig#fQhUxFia3Dg|)jsw)!l?>>8$V|>1OllO$HgqacP(qA-8L1U89M4uUPSAVf@ zvrNc2|2{gIJGD7o-+F$*evy3veZdhD>Ck-WZIXRyt<(7Gqa9%%KKo+s;O*4(27igC zq>RHamUqhfO89DHFJH~+tsK1ZzFh-fgVIC#QNNfQx82rutaXokD4&>}t9dtk8nk14 zP2AkewKdIiB1npp5zDiwVA;ip71j-L86Py19Z(-YAbA;y??3%5ICQwXQej|C@uKXh zZm76jn{1pc%1CQcd+NMsk|ma1Ve84-$mpVfmisyjQaz!mrps1-QJbUE+n`~ixAbdk zZgjCc?VaDCy`-hqrnBp)J9=F9u=YQ%8}}ZEpk`Oat3s=SUv_JqvhT{j+;Z#r40;O2 zb$7qIigfFl_Z6OTn1SiJ_s+{*;ye%JYq;+2JvLu4!_RK=m|i*l@Vpw`B!72*>f^g^ z_MDm4XZa2VBZfH&m50UASnNCb?1whE$?H`MN{?Dd(=r@XDxs(tOZMmxQidw4hbK7t=_AM4^!#KpvF zu{Ye5uHWUDjZ%%4}1}89T^+qRDYV{_ksfQMY+MF-AoYdMD!K z9)}(A{A_sl)6lMkxBtA4s8fc1C3haR##vTREUnRzoHC)&k)6Z*cO11^XKY-xT4!pU zwOWV!Q<-9=;f;ZMrO7EI1zV#7QR)qO=o8ZWyl?`SU9IwFx&~Q538tG1V$>@_{dg~% zS5NnsiFcqnx}baKM3poH^s}ccCgw?+-WJ58dv_hjDF4BdCkkPyI{K`X(%l&mbP`N=XUH@OyN4^L>8Q8fRydQ+ zY85rb%%hhpYD)V;Pbc^Yybm#Ni*t3Ot&NYk*Tco!L!qv>yR7V$6nykmu>bKDzI5A) z6>rVy{S1E}F!}ub?jftU*z@IvtDD!}-&|blk?Hk5bC>z|{mJLIdGXxE5jy9?goK7M z-!8J@1>2NXt&NMN^Z6DnZ7{ticR3l`JPdw#zH(JAub#SD`^CN!D8r*3hy-8Dby~E_u&aI?edg@*rmA{r2!y zp>MAjZTq?ne!S1))&XLtZO{GgGhxo?g-{%gAT^;g(?QmA^5pz3BgWS&5IY~`)Q`Ax znQ>g^ptX!+v2p?%^W|__E8sg8!cBsyb-~1}Pqq7>d2qZL0cbYn$9c!TMc{RS&hfZT z=}$cT-e>6cV?gVp{%fLCc;J^%$ZkP! zz8hYRZ>nlnT$V%v2O4CK6fj6$*i~v^EHf0AHx8+Ml1fn7iP(tF>M94=k$xJei~ZU4#nXIE4_d}K}NgEW`*+=ebGLD zeIoDze>`WMSh~PZ4nCf+J~%&scLKfK)Xnx4_6f2V(&t0WZJnq-m_FdTz;yx`j#^Im zX9XWPAB-R589JXq;Q0QOABmq}{t);yA8@?zK0!~8z#fRI^YaSri`eHTPRtqT>+=1@ z{loo6PVC{=;fngu|Lx+4y&exlXG1hK;2-`in-x7Vav|&l{lxA>(T%MYNGoWYGd`i6 z1^x>7`t29&7o}8==uF@~9q`J1cTDQS`bP8y`^NId*NKEHkSESw&?mrNq&=%&ura4T zhkGLDLO+^UXMp!fi#AI?TQ}F2zfu32l*TyEZ+i8xHUf8JWVmdYY{cy*_??_QMAX>3 zRBWiHXoH-5xFsUzePjkJXS2_J=H!dgc1iJi8UNAlJK=Z)FLeyjCWRXJw!8th3 z%@mu@+a?jud1mki)9~BWgLiNmR1>_X1w*cYBGd>aQh|c6;3UKdIZ}p#uAn@`2q{v6 z0AfYOZhQP?iNE$NN%`Qj| z1+pvC3=`RWQg;(RGBTdcJ}%alWNc&jSAPY>l9N+lMWDh&(`MsPVwn#@_nJXRFM}ex zH&sC~L{o|NZP;|yN~5GVP9-I?Yhb|m#n}Y^)OnUqIHow$=j8zMYk^Y6(jQOea^1+0 zut`lZWKYT@PunR%y=M)xi8uQ?4Syk^uMl|5!>mkDp-dA<1GJb}OIV9;r-fc>_7KR~ z>@?r9yjC;gDGEmcTUyTiK>w3&#F9a-wqNqmvp;Dz|4BLe&s9xS zNs~ZQRzY4#MNm$KKu%so^gpSW@3$IIK2vVgkm_5jBV~o1( zx!D%Gz3wV_0$I=DotnOwHY=KB7{q75FnH|X@-h7E-N79{YW?e`UYidzF$M0spN`fy z&@y6efmEsqk)QyF%VKzPG~?fd_&IUM>?t|2vF@B*t}zeat2{Q~?X;r~*_|C=ao?_yTn z(NEi~^Ul6hlRS*^nh(J-RSN^D6Yn|HfH z(Y@r&;vmku#s)~FCLBSrX`aN??k0bK+Z#aL-97Q@eh2mGyn?4nk@Xv*dQ++Ru{1$n%6*Xf2qSnu+ zrNRqwS_IM38zPv)*otmVglbwP4?xSU8 zW_k7096g_1TnM52z2&TU;m@Yc!m&7@YbiRE@#@*&Y&M<2Yks?o+t6e_lL57`Py9+s z`RZv}Z}c?z0tVj+c>b7iG-8f2PBM~?OjKHMN{VbWzF+tzUt#_hA6vh)Uj94Gmy(&6 zrUHDLMvht{%d4cMRBy3d{hhs`F!*f~?jo#9BN$6yIVwYf$#);GqLCP4GKdmYEyk3f zlP(oc^b#G~C@&O=e+ZqRw91F3HE83(A1o8@@jAyVVyj zwM3{JMQIgq3;P?cmEz4_<(qs;wEp>quD7D1kFRi?iMg(1?Nc z2GC&greMDZ(t+ii313tSTTkOXVZ<)=FzNR%db7^_#^!H8K!dNbx|rojG?2xj!OuQ7 z*zh6w{I;qw|I%;PgjsQd>o(r|L@8$Gh$2}v^7uLuW?6`@a7ZP+p$o8Ph1vej+Wgu< z{BFAiuR|Q9Jez~qHX3CLzhjIXjXwqww*6Xq`hLeM&?wb_<{gnZAx-{~PXU8p`m;2< zvfeQ~$Qb9Be1pF+z%cF$L+A1vDE@<11WlND+aWU0L?g? z0KXSb6XfJuGZUD_LR*JWHv)PB?t<1!u_pq0LhppG(W-cVfFl7B5g`ys2#b*YW<)bF z?(DzLc)VhCPJsJ}u>}5nT(*E5QCa-LDCZ%p6B=injz}#*TKvSQ_b9}u-=XfItp#ph zFm?=B10fkj3(}#mNYqJuTzp)De7t;u$RV!8{vlPoumoBoikCL?CX}9tRy@ckwL<0CEU?lZUjl&DNa0xH zZ^SAh`o0*?Z}eM2zxJ6$0(-(13Euet7DBIbf6>vlQFjG=>4GhvxDO~R@drqs-$_Rb zK{G_b?^2ubC}_fw5%KuAhYhH;V~QN#_YT8(1pLH(6#ppHsN5(LQ9>i3MSpBuUAfPa z5LXI|3RC9d#1`h7ijKwXW4FXDEh&){{ED`cv&4WHV!YR|3H0Y{4UMq643>! zp!R#^4*Iy`yn~e)rSA(rT@Va2kh!lqPqXf`c^hI1S^BzT@v+1F>tBj^_~p*v>-_Eg z^6|Cu<74x_mN=99X7gbl_x&|{t^a`%`*~`er}z3WS=K4vg0DE*2%EZ+>B*B%>yyKB+RPN<#2`sl@ZSBOUL* zc3sa3*jVVZG2Q{`p$o+N{3!|4<=gqgyuM&qJ##CpED+Y|3;9k3uUmz$lf$khqG=^z zuO}f}6@CYj>4h(=au>(A&W>Ynw3QO(*n~8)3azyYLb(f9od2*Llz$WJVbFLI-9aKTQW2Gw|F#r4Y&dyryw1z|<}^J#Ft+3Wcv+2RY>B)6 ze%BxYc?#1*4&A;(JRuwruET)o*;P5YMicUH#eA#bodt(@E(dO}6>A?y(pg~0ww#G) z?S(vg;z}M8SK!+r@(*?XWhtcui*TtTs?=2P*heG$7wAL2*h7Z(+;C>$1({T&&TiNwlm<+s?RSXIa- zW*fDR>`Fn!OT}JEQpq%qGbGwk7^DcNc&R8}Bqy2^)rs|}PmIERFXil{W~;BPmF#bE zb>Yb{6;26MYHpzQYhj=`C@8zR`y;V=V7@qevio7OdlJc3PRsFDK~PLlOG#2|4nZ$I zBSSqgRWUiaiIkm|7%EIBgHn>SYQae=(do|B(i78isHCGqI8ez_@85-|vl)Egnsb=o z0E34a!5*0GUs5?q$UF_ z8Xy>T(BOzcgB4Z{KA3gT!1gO)o`<7uelxpxK9RO>`r^a2^nsr#Soat(<8sBI?i}u! z%{{rfX*hfgxaV+BAL$t6na2I+c*PL+e?kYFhIs!kB#q23L+_V9`vK8m=yf~zupfRp z1iui4UyA;palH?Qo|pd}V7E^n=-zvs!Y1RkPv7n~^tul_ox(0e1l}shaD$6R}BG)3BQa!GgQeLTAEv59Tno~JZ$zA!Zs$cb0`c`(9HdiDbhZz%> z%jMO7f?C8LyUpfxd=gv~7d?%s%g|}n?AUDF?CfIUa(+@&G!xy+@HT#uW+!H5EfTDJ z&Rn3^cL+V(U%OCn@K;lPPSC4Mi3K~LGw*#04s?&by)x&1qROCUcPTrtMIm}?Aj>nk zO5&e#L^H<9Gx7qz$O=w3?o zHz!_Gx!^jjT@k&*2W@9G)D!1Q7?2XR#eR#a_UR=p&jb$lN1Akx96u-?U$E+{U&IPy z^l#nmY4`PZQWPsNWQB=))o5DZKCexkrf*YJH;dXy45%-=P(QW^3#MKx@p9K5{ykOG zG=;mjj!lxt9!K6&ayBq#iRiLJDp_O15S5KvrS%=Lf~0vP#|7uwgS30K0;1+#%7|9C z?U&y5wDzd3`DZ-y((kXe=XH1Z{$C|GpW)67QmU94&m^UZAbqA~NtLMnv4`^gDl_pZ z8M&;*6WrVEG8#3TllEJ|J3kVqIQ+-Z;!ZRipCX>Y6)oXpsV2K%w+%V3jna?PM)0%} z{5&uydfoCW>wG0f75&aOD}KT_t#3jNI5)qSxfMOFfXIJ@L4LW<3y(HAHDOTeK5TIT zk$`9k;-&LZXs9LHovd6D?P|8F7{Lq!yOc*;w)CyCt*?zD={<1W%ot( z1pzSM-g*QfUYmeN<6(IuVWDweln zbP}@#pv0q_@p`tuN;$anJ*z9nZqkVq1#q4@CCOTc82w_&$J#ZjE~awR&`T@3LMDZ! zKi|LjMv@=-D9|xt1hb zGqTip0DE9!k#5*1{^9e|S?*n^{~ z)0Ce5y8>R+jEl9^VE^5lkN0IeWWk-j+xMuM<(tvPXB~VNs7<+s5T3 zm+yy>E&ajDE7cZ-c|+z@bFGGU;RG}5VIGCGaF1G565#{tkr&KXqMNs?HBcp6pk#t> zCRbNQZ!%LmZ{IHl4%UFkvCRzq* z@%4ad9a9|rZWN_k6DPZLT&9$Cx)V2beaP0hYL=sih(|Y|Upmjlm7VP= zjJtazyG+5n&*L93U|Z?QnUE|#LiCOWJhh^}te|_a|6rNHI~zJQgliw4QTwRe(`hpn zla1N>h_-OrJn@3U`;~S?cmGJPe@*6{R9?VJY$!nCV1#qRa05WZs)=xS;{In9{@3<5 zw|)=GZcllha~E8CH#J$$>>gFA>QX)vldf|Yp+6<<3N^K8&xXWSb&8K)CjXFrXKXO0 z6hq@-Z8cR528vNdi%49Cb#p{iPcodViPsC~JUr52;0ZBulorC2>hmS3YTqK-)9Bss*&d*EE|lN(T9fH*O-2;)hxK z^10Z;6^`#!p;gg}3W3KU=UF6;pk>jasY#~V`H<}Oll6RQ=AQWf##(>y_5*>P6s0r_p6cK zPz(v=6ot0?vYuONcF-F0Kd84RDacT&tD8Zm7R>=Q`WySxb2kDAhzU6odu6d$bxI?>_4e2oVA$9hYQ;j>INyJgvL)C zrO_3-*_gwIS5zkqCD5~MFuWe6F9x+>cszU!rSqDY)G5`hdGQ^UWF1d$cV~Ke!swLP zN=DGns)Nd!8+*Ik3k&-Bc`DlJSxUo~4AbAlr22~5bc7^mEt{>dQh%jd0TNU=$Bdh~O8X}EY~T!Pf2{i|=RF1zS45v5bUK!2ZQxbH zn&}q%IW=;F6d@<>JI@C83w3;DpKco&&RVrdsGBF0aE0B`Hn$cP11oErq5a^uCi1!i zgL%gTS~}Wkzt>ZvW?XFGOv4x6F3Kx24K$X==PRH10=Wn^umDJvp(%hG0KE%}!_Xpt zJ1J904hb%HpKq31KIs?W(|0(~SubfpY)ijX zXR6Zd~3~E1b4-Ru+zzz zEyd(a9mV(}XMpZ_Y9xC*Improiyqe{!WdOOt5Rvkz0dyqq=K9~ z-j>#~(wz3Z9-|;BnRG|8!n~^Q09i*iU;m)4!OmV?k(KT-*n;%m3Ti4@AlQKIIiJn# z9eF6mPuevo7?=g6B5a4zvD*$2`-!! zsXRgA4hcb0p1Cm0gjAgJp-(0J-c_0POqH=w>v3a*G4|D@czoQH4 zk!)6c7TwMsPXArJ!obWg0MZ_PWh!*UFlkB>pl-5SMglt?^L7KyHD~vuvn3F=5 zDLY7-%Vb5ngq6y1A;e0kN+qrM=w~Zw8Z{I$E=Yo-s*_wB(o{scX7lG?ir)Zvh{~J} z!{b<04#6w?ElO0}D`YMmyvUn!btlLw=-V&c9yT)_LsTWwSf?&4Zz^t?rXFHP1eB^R zb8_pu^_4lOHZ4G6K?(!uxL%Y)LC%2HH z1Pv&S*#bz2hxNdciUi`x5{N;`VqOuoWO@een(`KR zc8%xvkdQ7eg-7Wv^8B0UUjn19vDZj8j2S$k(;?KM*L|yl^@G^qoX9Se=Y!%^NoF$I z%x0tcH%V>s%Ul-=rQzZM$#uz6$yCWn62nCIM1Lg@rSsx#$uctC?1%Ycb+R7b(?Q8R z`8T)uxFmF0Jx0$1*V|-Hp}%XOFq{EV-ohikKooH=git4Uwxt)^McnW@HUFY8v%r;N3{DNRQ4%=?j!nVFVp z*U28X=ci5mebigxedGN(4;ns7K2ko~udnVP&z+eaM*m}RZ!W111J?C z(;&Y7cmuf;$rJhZcn8|Ol#ni{I?_#PkLQf7nG4_2s|%?^{##K02NiD`QMUZ2ams);(DoqG&x5@&Nm-rlA5uIo$V$VWdDro+Yp&4kNXL$z zDWa=G@CmT8!Goc;Ar9|Pac_q*VIxwiUPEIh;hAX1$Co(wJvzU78@<8%@al&!jAc72 zBS(x3DYnY}+(gNS5+=%%RE(}69l(TWmr0pNRxsm%0o7{$E1Ow;C@okH;(b7OC#(-J z7wh`l-FjkJcj`dt{J*>5GcRxCGZj6KIj^d+oCX%9>X9jKsImn@sb>B|%Zh&5!0Iqr zIevkWYx?Hnh0<4IbCnX?tw?hSr`S?uIMYj0-J!>nN7WY9laj^G0T&-}r?u42a3R^( zWJ&w!+JTzHDf5znx1_i8Y{^(W|B|IO$#;MLJ9yVLJKqrfrCf=tqQyTBbV_OxLH2hH z9Tw=|bXJnD=RQaj{l^B`{{*xup9mWGKKEnJam(l*=g6P7wr}QPPM&v3V=P=8$UXuL z-*d?;z;-4;e|zKiIW|5T$%nTT=0JyZZ|QZjQg&MohvnREtheTE$z4jUH6CI5jBlat z7I4=!cRQtMeIZ)g5o39MDCBebyIsGj&#m-Bj%f3?Z|v>ss%XoYj~)Z|$f47fa>tLD zY|sFz6TB-B%hpQuv9)P&26wOwh}kOvE$l#_b1TU`$~!9mMA$FFH%@6fFppF2pcuq* z{To88pxK|jPE4=u!I}H*bl|w@K+svbgAW`$&(S>R8$FXX5U6Y#it6-1i49C)=qLos zc+e&#Y)ZymJ87st$3D^yDets&bisHM%w#ckiMN(g6GxG8=J#nT>?Z1%VgphTWu0NxJ?tw{ujHdbXEn{z1V+^oe1$n9=SOI8m6&R@bIOOn)Z*^e}X36Z+l`{O1XsL+4j z0Ug`2{#a#VcM>N`4*`mC8DbSA{yCxtGE^6|zaoIRcwpxsNqpSHiCz9Dgj)cJsecjH zBRVG`QBhax*H9Ltd~Wo2!csuW@l_9SQEXP3+j!Jba0LCuYN8KJbhFB}6_skRM5uT!?T zzUhVEZ<-O0$|+zt>Jl*SIQBhG;xCuFIxOm3RT0pwf6AA5+hp;`)emVBsCZ2Gq!xdi zlb(nRf74Tq=;lig;{S7ZgqXv92R&ujdjG;j_#E%jNhFSt4+D-MNI1YsBj^B4yI=Qk z3+7pCO=Pg6NgcsFF8UJ7wJz{(cYfkS_UdZ99+mQW3u|dgWa=((8t)%eW^&PH1Fs7b zAD|lFy_(x5{zd5XSpY8+0JD4J8SKOGJIl}GYH_}kszRz7$+-kcykxicU)O$@gZQuC zE(7o#C%|4Y-7;R@oWIF_`Mt~k`er(~*W}Zx7i^PA9vrks)ipJ29#Vra9?B^oQ|H}t zyzP$SGrizz1opSunXa*h?*=BzcENSWEBml#?_2b}o43AZ^|2H#3D7|Pvr*kq3u`nH zv>)>L&~NK`kKqYAj>;`H;rO`c&55Pvt$Tv{_6xW)5S~83@(NtOsS7G)kWsTEs4JvZ zg@M#-lWr9(wvo-9(L7>D7dZs^;r>p=vzW#%H4oEeekU+1vl7cz)>GEr*;=?Ati~SB zHE)1l%Ez`coGA+o`Sx!noV*!CV^-vegQv7+8(R%WU41tKEj@z?`2h_5G= zJ-4}2VR8yi)QiPUz8%loyVLHP_qgKAnS#4>DeFhV=JvJK)fj(Q|01)AFJ=2p4)-f9 z`mHM4m1WgJT+#kJ5}&Q@$t{JkZ8H`JFfeRg+LKN2acy1e^Ib*#$qIL@+O0xH zfrA~Vp{FjY8NHJlL0VK7s7D?x^%P_lv|2y`r5 zSs`Mr>vHt73Nmrx6#>VZr+U(mr?W`rKuGJRU#WC8sX>*C^Er$JOj z(#vPw6%g+#J@prfeLVbFGLKlR%0hPpd!|E22GD^CXf>(bHWW*b*h;6>M+;DIYDF8U zA}74eECy*!+NFSQ_d=+@N9`?KWR?vzMHNl6Ji{PiwE|hx=;^A|#qFwJ#LdP@#&P=vfMvOpgO}MeaRum8IQ{Bwprs<~M-Cfq}M^ zcDL)G_z0H(od;_{_u3)sBz^+bK~Kz-vGg}Rd#0w!C>j&|-g8&748h+}s7KcuN6x;b&$PN)prhn;7zb)_W0%+glCStBu6L4H zAnf3ozCW7?rbTW81BeB5_%2~w<{r086=uVMlN!^yCRr3;t32uA*f?nbT0N23Q0`T> zIYGlVNFvNUt!uPCB`TP+tg8yk_=i?dc{_yQCukM@+UNrId3W6~Vh0HxM~r+Xa7y0N zt9LX^;v65qbl-xf2`kmErMKl?jA-yi5T({0cYaHJG)iR`$7dN@QoJzQ0+q)JXOAQp zj5nKHF(&0m4LTy&H?=$P~t8B_KV@FBdwFauXn#Qf@J^ZNAOOtxv zU>Y!P+h6+-Z{Alw)5z&;x4M3*NZI}_24nTinF%aY`vR$X{=ORI-#55O#b!Dyg$_*+ zG9|4euB`OzjbyuL+aAZ}3~OsD$H%h-V;`OH8nC@(r>y2%)}Tjr#KvTXq2g5Fhy{sM1EY4_#woZH_#v;nEn12lr8 z@kQ~9%MCAH<}X)TqI}kB`EN{}8+?M**MHu3s#{GLPJ?TNd3&H!D{wVP%|cU5aA^z- zb8t}GGB=QUw-9!4RXAnYIS+ANhD9E&AH#ORwY9Xl+M5KXZ}y@2E4ohOv4JV%q#ifB zaGIN5gb5fvBbS-JwO8{{^UpP^FiA(72;=I8<`BV}>&J?~@l-N81*i4+_~b_ao3NN8 z`Q%!VI|n+rJj;6X?$}Kz#HaR~@s3e15Iube7FKsaYns}vfuy&|^^+NvAB)mD=BUaR zO9&)Jcacyj!ng?muKnk15L{NZw^jNLy=>GgCsdp zW_9~dTmFaZ2sXph7DeDlZ;)atKD?)pl7>c^@urLAI+K zP@YUgAbNv?0RnQB2b%wa)_1}&k(XbbTyiBUlCotd`&amht2Xd@Z~fJ5y2e=zlC(Vj zdQ?`%#YXba5s6yv=uA;%jaOGJ{V_}jFo5=m^B?54HJ=%$o$yjRGUa>$38v+TzW9FC7GMv|(OziHqzWP`uIs1sL? zQ@z8MjgH>ibZp;hIiPoG4F|FvypQs|YC~Ep)Q4u{toz@eypCE*XO&y<@nKER*!k1@ z=#Ibtg3J1g0=j*Vj;6^ZVGSu{Bwm77?Ew?6AL)};B zJvs5IA-NR4DUe`wSDrGvTWwI%;T#T%^w4iA4T5X1>yDN+T6TvT=C~m~p5A?A{9RPd zIy{lBd7nm^?;PKskAz;dYS6~i8MUV6Dig=;DRp+M%(Xw#elc?!z|4|ArWONJW7u#! zp;)P#FLblTQx>v#REjUj7*6sX=^Sj)G56ekJ*0nTrM4v-3*ChDeCDJw_>)EXrHxu62!m0yudyu2kDQTsf5r;t8?W_s1WZWh8CO*Yn#+r|g}nqU``&J8W66=nJ>|YgmR*ua z6j^vLaGLEnbi=r5z8!>UU*11?4mzEYeiYL5c>mASLZvKK=@f0Lha*%9jzXTnB?cg> zidj9Y{7@u9dZ<2rwGM}qm=-T;N{F;1Yr?5Go>)9Ao&uLOUo?ykY+&ig z*D0XVgz^eRQvOZKV7(&tTT)Cg`7)RMo!%N+5U4;@@e-6q<_d85tEQ};irW$%-sG;ip@sL_p1+vbZCFEJ!?WeTGzEo zBY-=q%+M+x6%&2_S1^(Rkz01VR$5<3DtEHi*kQ8*yHFy%gtKq#y}H9q^&P!w-Hp5pja}jYO-e3V|qy`Q9|+xWg2(x@w37Tu<*ns z`lmzQi@;ww1oeU7X0gGysZG2y7a5LS1fFJf7tYUiFVYISm`PtGWqQJ?gouQ2mY!NP zdEzAphdzIN!vO8XK)yf4s)vUK-teY%epqQ~KCMUaW$)J7S{0Aua36MG1ghHS<)m@~ zpB96}%*mnSuXhr%lB#a3&1oRV0Z4zheP7wO{e;@AIE(FLI|_J_S)hq9*0zU(^&5OE|m-|KiG6{S%U`hqbhKw(N&*eS=`;{ z@lDf!=pL|Z>o8&Sf?geg$7%c_Ktf$X6WLjd1LPb~Z^~ueVLSaoWeuoe&qsr~p6t&s z$mvOo#~l~l^W1sBxd+9jpl3mxi-P70-wF5jq`xj*oRD~wa=4+#QkGwfGtE1r$C%FQ zx^$Hp-4}IF(QJXetmdKd`!(1Ql03oMflXdwS1TwXSZ zS=pZh_%RYHE~M%gz_8`nB#D-M^L^VIx-DTc8W&$80C4!vvh;2|nSZyev$jy5t+Y$? zg5?%UU2rJmf=qPz3+X;i|GE|@-5ygU@*XWAw{e58F4OE9oYt%npM>~vN;`0kqJGxo zuCiCzs_CrqU-MX7{tHf%D|O6AJW@zTdR>1{V@f@p>R8(D6yujxvg*=!7JPRtV=hxB zTecd%Wq+IB6;#IqUwDzt2K)Z#;_Kaf^9qdvV!Soh~B#68g-i$Y9HY&jIPAQC)@ojg{;QT~ zL)2v!0kK&Baiy~txy_%$k;8){bF?7YPYp}9=h7QnI={|7k#&vrlawejCIyXX5aLg*q;KaJgd2KOUF?S017an?4 zlrwm3xK(`67g$v$E*a&{GyVkBSdDXN1$*W2{A{=gr3goGU%!mJ9ppvjDL2!4C-}0SILxciG@;HGQU+`z3vR)Uw44^g*^iBzzm1Gx zha0|pQ!I2rqEa5jb^*J$XPQ8bnYZwsbluXEE@DMhP=hgYLl1>NI!i3GIsY8wl2J>O z6oWC+4E{1c1xK`&Zy?p(g?HU<&c9_Pa?;hypco3OIW-M~G$?{wHlDOhUp8nqc^WL6 zEA!8g8QRVs7yFG2_v_u9uvnpT&+3duXPS{ai6a;()nO1m3Hh-)33a zWEdq=PB1$DieO;t!zUz}Tdt^SxwBqZ-mNl(J^gosjRI67mGsR;-)Kl#CKG0EJgWjk ztZJu5WzDz!Zi8Jph`~WjKYHcvD96iaD7WRz_Z$>heAP&6D4wEgn?=$k6GQ4pwJA@h{ISt8y<4`K@e-U z7*55!+3Xp49Xv%Yf%+#0!~L(tk;|QbHQ~7+Koi8~sfoK~fntBGeC!J3P6Yk+u#{CS z>gdX*VD&R%;H!UI+9IA~S+X*SGwsaRDT-_FDP}`yzuWGLCeL zR73zebz25JRrt`AhtXGIqh)7Qw5invo?ps0+q80z+8{3M0H&XObRsOAynN)k{+*9+ zvJBv0(niIyavo6%;zL%+++Ha?;yS0uyMEh;WKB;TiNB+51Zhf+%!!IiZc5n@-oQFJE~U`^?CrAC+6D#$D>C-L0nBZGIfws^Dz}NI zJZd3>iNaIRliOMDO!V;cKS~&U2t%LG!}D=Yde!!%(fGdFdOH7H1S+ve!I+C|EYth9 zO+?o$m?y&CIkw0fJNUEXCP-O$E(4Mzrs&UY^ZQ;(v*#fvAA4t`$ze6Wd^dpMDVvE3 z5hyLy<-)r8A>K8tNcYBTB%}-HRq)I==4fk zY@WyvLrpn4X@acBK(X>OVGQ1yTvgbPl2kbO`QyZkwpv7SeaxywSfOvpJ}vNm+pzZ! z+*#Tg<(CZQ#)|8(UVR(a${B?`T3aV-X8o2h;BPn=(9|N!@oBd?5c09L@HWR%P1Vh{ zWtZ-(?@hhGiy^uf({}%v-KzhG}%jX(ezDA)QZu<4CAFVD-P+jXuoPX=!-KTu0U^EIwA~n@go<;A^&pE)XLwU2qY{sjtX1mpj z=6ed7%BW|AMbD=4X=%mwYCFycnAVm?(Ho$okR#lP^Z#%Fga<10q|hfGqud|0_Fsh| zxz{6j2)~c#eDoQnE$A|swtt=Q6`k3R?#Zjsdk^>a?*So*Twd7=ecw2dDP=@b1uqa6Ld@G|4kXOkeQ^EiLudX-fAxcAcz@?Oo`dYdC)@*+%^ru-MZcvS{8wkUlV>H` z9pkUADR&(~IhO~<*t{S*M8fNGz=ahAhbZEr($ZJnWeTEozM60we!RRfS)c^7*4yPOzjFQP~LMuZs zU?fhRJ9YNd+0%zlA3k*m4Fuh_HMGa$^0<6ADG1*O=84d@Bgd5DrzQ}Toyy~yBz;y8R zkADLT!OE#$%TLSGsF%kXF!1;%9|9DNdixn@agQ%rC`Zexd>8fztay&Vi{4Vm(nb>49G15H)y9X{0l$l8v z8avA0UyJlkZNEqEl&A|QvJ{>4` z4)R<|n$0Y2HdY(S!uJzfO{koBKmG|!y$sv)w>WG$*fo&1oow9oEH)Y2H!uax9div_ zg>%Q`@dJ4?urI*t1Nj<{==Qk-sM8~PBqIhnLi;in->N1U#$vY9Rn<8rdhnmv+&(hv z=Y&b5ngyu?2iB405~^FVH~lbewK9YM-tBPk4$3YvPAlayxlLXU?Ues87gIlY>nn%B zxEYuTDh4vQlk#)2Y6%vLu>;BkG8Ri5h?xd~yd@HP0>XrWnAxyWj*s+jBG<)|agksg zHfug9Z@!-mL$kFL$;;ri1QJ3wcx@UPkIl!XVfDbDKu*JofPWgf2Iijx{@K_{au-OC z!)B0RR>YJ*La2@5wf!b)x~C7&j@KR-q^s2t`b`U9T)?wD%i_?_IBk^2 z>^1v2bk?>fN&WLol|5MfO6Bc+q+f8*9rF&9#ZDv0mmkwj!MZvSp|z=5z$0nL3ckM5Iv%wayNQge62M- z!Cv<MLAbzc=dtW{EVtrHZb^KJtnaY!txzXHc3v)iMP6(jtJ0NplhGyt%AU*tf zP~M-R)Y{4p@bqo@jhJkVa^L@xX6W-E4Wz#)n+Dw*Esp_*VszNhr$b6z6&VEwse-;p zZ>}=y&3X%Z>d?{?$%N5jvoN%_$Xj}exln#0e}a+OknX^(Cab9guXI|SECo%J3%DEn zje&+h*xwLn@Yhy^ib9#Fx2)7zMC5F-Y=41HI#{@6CH`FVw%%mw(DBQ~w~6al+b$lX zk6mi{?Z-G6oeW}Lc#HD5&a^x5hM-u}i3d23O`_~hD{r!+vT*--&6S!9&0SP?Yin&k z(b;J!ZlXPT&U6^~j44{a?%=B9;4w-aFOsX0#)B6geB9$nd8cw`PNmsh){;EobjwIN zXCDScrCPR%59J7=Q5`4xU%4t&!1@ z3mUtZ@t9#i$FOEAsyE1u@<9s;I^@IUm}~*7OdiVb0amclf%|ODOpsEWKKQ-HY4KRS zgML=>deBf1G=hT;+~bh~b$GzywRkAG6CAd>;fBvou=0$}W~U~-nZt5LNByc79BW3@VXE4_x+4}fUNJX8e!6m`tZ2Hk=NsbEmD*4uiI+v+@r9&gwFFXz+(tw`HQKc<`52wmsM^ux{|D zR?vRuLi^?F4X3w3`_)1FwFry=1EIkmj7cazWJONNEqN)?DfvYAN$!-n(|}$p?H;sW zZL%(zG8LOkm@>}9=>@Y;P?z12T(a>Nl?xa&MUP``9+&tF@#)fFGvB0Az>=wJ3t48-Qx|~cRH@B-Zo6hV_ zYdIjIhqjl3ErLAgavv=C-qXVL4>isH(YeYiXm%vkhQJQdajw2~GTZ4MiY zCngu~PCxAN7f`k$CMRfZu=X-tJnC{foJ8MoPpE_Lt?ln|-9Vw=b$Rd(V`q6Awfl~o z4Mq!qLV*%Irnzvs^~IxYR#O+{tEp?(*PsD?ejtTN5i*$qeRfr9u5AY@kc<^C4CFT! zQavee9=McpCHZ24ewAIz=$J~*?qr?9nNawc>li8qg&lVsxcg`ZHmur@6AqH0rw z`iExu;IEQ&W$BffBDz1X;SaR11ry|HBjpBVwyo4!M!A@JGmIE`X)t`O_tu16G+3yt zw=~;3dGsLJ9Bf;Fsly-_e5e8AL;rfCx>PsP-pDpL;xH5!Je2GGuB+F4wNjX?*A zNm4+P3D>mix{A82ZusB2!Q1}D|9H20@O}Hg?|(FBs=E8u4JX`l zPSrW5J%~5T3~RibTPjPd%S>y%%kg@#($_|rSqJB2<8VzF;644957l5a&}|7Tpo)WAeI1ln^f4kiIMBB%K?99Qi|4`HZxT~z%*pJS zbN~(M>lLGUqFMwzBT~FqU?^!G)T^d0@{{Crql7Qv>l0&n0&*lohNnb1$yb`EIwZb! zh)6S0Z-f#V^cmUN>O~Wgy(KS)yI6+x64C<6pmrlii=kacS4$(o07m2x?&M5ccrT}v z^wSL()WGCwX&_kO&``(*Kv>mDInyEszg($5c%b{Fco`q%$9E7JtvbjD1i!C4l_#RT z#Ygz$@ZvI+^mCPzd&?qZ_TTjJ((mX)zJ1}T5dMT(Qr5h%s)cg3nY^u}wNvo5)5sTd zi0Bn`hgsa)+w1KkM@NJGV>EJMY{<3y4C<5+vlt#yz4s4gh02lD>G=;RCAX^0D~EQ$ zM%6+|$1+MusggrVIr>}GVrix_rt0g7cE#v7ey@afi$SVU>EQ9q?o6J(rov%vqK%z3 z6{aTq-G&thmyu82vqjR}^~IWRP95moIZ4OkecKZ${KfuLCFjVGKivIUkVbBGnE3qZ z)Av!5S$aj^++?(uk}I}O?LWGI&s!%B)fR52%?+)!9Tpr~8}$XHeS2%@RDaiCB8K<$ zbcaXDlVfFNB~{yArghR{sDu^z>8RJ&Pfy3iE;ww|tyGa8d zgj0GsM9G1aiJO&Gw?$IVLz?^gxow(<&u7*4S8oF%;^~!@wzT*hoU4~}g4{G4mUDs* zWpjcC#K?AGmq)aVE`GE78OK@=4!N1&tK|_FWp_Qi5)d|Deb7evodFM0_gqj21>p66 z%nJoWp%{^5qV6CCB=x1vIEAPsxRHH{41Wjiwt8CagbqHN`QZ&@yfVNQ!tKTucN1CB zkvKrZi+@(H)@->_yG(QJ%B`wvxmLbX1AHBpO7E%{46Ogd-u)BXMKtr<;eyO^eINgI ztAUNsAw~`-@cTvN{WTeaULQ3bdQhU7TmK+0y|~WsJr9#mN$8~YaDI;-y^|CB{y)ic zeW0(^@NYp1(|r|a&<6AG*;MNcI%&}dnZM|fd20)}aeqJ%5%s%}_nAE3MVbl1tbetqv3EQ5l8K=s2-wD8AiK@d5ev z4pLW~k1kYq0tf5LqU}(|rvF>%u_u1Dexrm**#3e~E*?AaPfA+B>3=smHbDuhU4zvN z=hfC$Zml2vlGaU}%G{`LZ}&9SAT#YN5T$HDS_7zIVK*;^aUmka;{>c#R|S+hSOoX; zdf)rXrTv?}fWIW*+Vfw2L-DFVy!W`YT3WdIk*AHcf0+^%srtpph&VZ&sD?`_zv|Q* zN57CWgDaoNMw>l09c`vNvha-jx=8EIswH^kCT7ZR`_X59Iwe^Ec)1RlSk;J`ruET%3}Yt5#h}C|NVtXhK|w zW%l0`^aq11{XE>Z2pSi`$~FGHng(N?on9_2e?eNgR$7V|TgHnHlJI#%=Z^3e(MiGE zuaqj0oyK@csaw_3-rF0bA(>gE6hon1X+r18+zT7^fruDO5CiUhYcEyyPSLr7Go`+~ z#%7zblP+De^`&hu;f{8X(L@;irbs)rclo{-4zHYYD_PcJcBPgs}Imkrg`mhJ2fpc=U~T5$5{>i{yk5%DAMY80Gfz;qkTqiS;L`OQOH0=fOFwHm z=A;q=GT?}iS9A!}ny!}kFoA}w`l0L|!$NrVqMj{X4@$_YS9k%rTQ9*~s6(3)Dpnh$ zuj zBJM3})|8f4HfeTD%8C2Ya3q}|f=rZ^^Y$wpRpbl#L-2YR4e#NV1H$M|hL)aIMK15# z|ArseM`Q@_OL^F>s71qiulPl!bu8holk?eWS{7G*`%&l!ts&6e5*7I&z? z3yE+PIUdldX995$CF5Oy576)#HSe_WCbFoeY4z&9#(oE_`w)8IRlRuh;9hc9j~wUI zE;x;Z^n{vQ#msANt2fs9+pIWXje?W129wS~G7{y*IK4k|{sa&25#XjgIGb;Je6KHY7k4&o+qld)my>4T z(gxw+0!WhZXYX8w#Gea3c{obD102l{qJx_UxA;vWA*EE(UT>2Pea6}oX9q%t+{>oYsm zJ8DKMknvH}tdwq1OLIDXn%O&S^1G=Y}b@6%Y}?U_q@=vAThkEjay8jrmh z=bQMELgGq~daGe$so7XZow@Rh3WN1z=6~t0oK?$N(9o;SUa+Nir3Uo^Bbi`Ep?^vv z-LHP`+eJ{Z^iSEKBB!O3vy*1diHvl&;PE?<&I{TCu47Bzn{#sUy>lLvnzvTqE}n~c zDJGa=(quxl)0f_z7$1J~@aHGa436{-M1#0=Ky4B%NMo)sBvgeOIjEs4le17WU!$9^ zIiH!eMDw@?Nl{m3gXrOvehX!HG_e~;u`IX@jgYJ(|^TU05 z^DBk~AB%Jv;C|RE+0+}rtLuqHUTu zc-nxhk%4;*&l#T4tS~J8>koI|rP0BIKUBcq>EYblzLqw|C?D|qUof~q8()n}FWM#5 zGt%RP^e|M+hifcUGeF$cDS86+nh+J{diS7BA_QlC3Lhn|{BK@=>4WodPMF%ruU$vH z4Aye%DQRwG$;m$unSFwO{<7z#M;sJ)1iW$TC^&wUfDhZ>Up1f5J*t6t=JacNKG6~` z2;rHZx-=HfYLm-nbT$xI`lKFcRdeUSg|oz2a9TL>vgQeH8JACGu2)6-cri_HsWw>P z1sKq`HqR3`aQH8|H{fjgKh$r4L%WDbBr2RG{Z^p^iR%t^n~?74Nu;RyiEhgPjHa>+E&dDFvPU-{8fuG(E8DJ>Ay}_L zoiEl;*CBux)#V0Sx7+}4h3!8q)UU`l*St#Cg_Nxz8G z(if_7CQ&;;!lNpQ=)C7vi6&RR)Vs5Q_Ba^3o!Hh@HD;wc_DnerkZ=v0i(YHs%hFtU zzqE$d!Qo5+8bx~O!JDNm4tgE0{FMLV;)5EQspJASZ*cGanmBar65;`B zh*$C{nexdv$0g}7=SQY*lnrn`2IobChu{Jl{y9FM+R)0y?6fPwWW&-6)h-XmdFWU2 z3+C77cUGCUlry;3i|7Yd3b7OgvtjAsG)R$M19CYKIV|076q?9YB_)qNzP)7JO6zXc z=GAF9sn?0ODn=U8lyob6RPXB%LgcPtbE%K!voTI3rbaW8 zHiwgQk)2!%aC7)B$)dNnh`fh}E7Stonxf>PiH=tSG}546H?g>^vVhXf(>$-y^=4*$ zsP6^v@5t|UCv6eR7U}SJa5$%u?l&u?8S0zu^V?QY?zYa378~wuK<>IUSAEDPyApoi z?0{bk1OYgaLYAef9Uybl<#YSIlponS0c6gojzpDkt{RksPHKr&FWsODg@aKsj`zJC z|Lm{C=jy{@WjyMzOa~*smD!YO*UNX=s|vSm*tTglFxGdpPG=)L`PrSFd!-qZ#zcoYraEeQ?+p}yOVJH+@g!} zx%@7lXJYf==l8BFEh<}5{gSDoy~g2Sm^?;p^D`j^X@@wA%n#L=m>7@rcki2cbLUhr z7>R{LcomTD2b7h!A_XCMm44NpEtv-9e5pcLT2h+~Bg*(EXX;%_4 zeKN_W$e}@hcz~XaZSU_M>lqL3_3rW4J{rUjAPnWfS1CWw$TWDmG^mQ3b%7t1snXK>CZA`4fy|I4#_ z#$`v6WapI*2Vfnvhh;rVj`3QZ1gqxM#=x+32MO^MTnOg_;kSI6PQ#g?i);hZ_q-;A zSVD+|Y0;)%55w@NozB+@-+w#)AgaugGnOX?)T zXcwU6=d!*q9}*(71zr>gaj*Z4;Cr~x1xAJlfIQ|9#mfnh-bihzUU1={`(;2SisM_t z^@1}mq>99OAxg)^V89p0Idx<>zB9f%J~=zwADxWuz*)et1mGJ1)yOySZIoc*TYRr5 zfmi`3-XgznT@5^*Y1F%|Y^M<*MQJ2V(oA3`y8*l&P(y6o6{WoL4pL#_Y%&cR`HIbi zWKpl@O12TUsCr*qe#8=q3yCxU@R!|0>MgY4jd$~Wl5Pdk!@MS_+LPv^(P^3>@*X0P z$%4o`N$^pnO;y&)S)BB>pmYn&i|a#CKHiNw`*SzZ1@jP}7=gq9Y8_b56;n=i5ba`- zULcUkM93eaeWEg;hVQGl^TWdg-v@eoh$!NkUKLxyxez&6&Z~?XJRIQ>n4pt#0_h;2 zUPeY`5rw9|D+N*!TK8l&kK|7HUS_@?JPu40V$B19+QQc%L&vQ~8>iiB;YvxjiU}y) zVK$mV^ZUyQIST3UR57tPAW$N)GwMofZT+#bqotZLfgs`iux%17~6_m#^<$v)|R zeQc5%+i{@lu;ZX(PyN6a{37>fE=moA2K-0J5YM;1+Q8lAAO`mA89o?396n$gtH8&$ zb+3JjkbYYJ)T<>_d2N}y+P@`OHDK<>FV5^S51B_C_`!~M_Q1!8uNw{*I_#|1PE}hg z8=4yAh;KW*D(E@8o%4{=L{!Be;E%jcz)#dZxt2?+RpV516`-`R2yEVr81EOxYqahG z@LNEoKsRzeW3d>&hm6`(ToJcvji_lO_*$`O9pUmgSuf>vyZDVkXAc*f&Gn&IKdyTa zemJa8aXs4!$f!^AhxbpjSg9Q#ed%&}o7>S)FWL!@vx99RO{xx+!xap1v;g3GG$G9|9wzGV0_qIR0tWO@I42`W7MVPk`zYdjcV3Y15eq zFJm3)ak<-W(A!VQ*Q0JfZa28H-LAE0E+K>L$B(a7*J;(bvfZwo?zT7EZA$D#-O7vE zh15G*Ofk3VB~fD{YJ?JD9p&^o+;+;zZwxql5gGbeL)xgj zYOu8`zLl(PblOey$LeIjKRQ4}A63_KHOLx4yD!TkedY16F5K&6I;;f47z9_4Q?ii= zh>>6nBq+3(d=02p(c-tsrKpsazDegka>ve4&j`J6yWZR``t3AqRmW3(!EW+!y6y=H zFRL!4b(4!VFjl3Hs-s-jFmX}Meart<`L#L?VqA#wb6mHngt&;94&l*(A^$LW>FreyC+KEr zE7}t{JKA(uhG_S9gdCJR81jZm0DlGgsfRn3uX>a$DT@zQ(e{wrrbNu_a1b@Ah9Ns; z+dk$VBax?}Mz34mtAA_bYxHXyRtzsA3kt2xTabHz!GC7F`ut6;rImE`R@2MgCAfIW<3@lW65Ub!(If>`e>Nu)Mm2nu`L2 znloy^9ZM5jPy|6zd^WXiWGh}g_FBt1^1_jU!xQxQfz*Z12v~Oefk&t->t-(BbQazS z4F}J@ec{MU5+M=AFD|2X&ud^cH0d8O+y@W*P==1uz0jC>RL^9?Vf}tVj;up!PZ^5K z=`Es~6O}1jdDqM2=B<{-Cc3rNR#Qn-Me6%&saSimv&$TI=7F#sP!UJi5p>`@ry8$~ z80(3$=9aRW+HlH9Il?ZR(rIaA`QM~jj&B31kD!FSx%XrOu*mR_B89o+o0Oqo7;Bi1Gt zXHsGU_P%0lR+O__`OaGhH`+-CLSFHS;ygH0v~0O<2>X*`fJF z^E=I7HInv0?Go)OZHd;e?bZ%zcW4i2FK9p1eyY7Z3!9ZY>(*Hh&w6s!>RE-eDrPxn zfmz{M!?X6yIyCFuS-+X})vUkHg1IwuiQKul&*ZMl-Im*&+n(#pWpabLvD{SdQ0|W0 z1Gy)1&*XlX`)TeMxnJl0E%$q!LN`lygO1Qqy1R69bPww0Exp;G)pw?%OpHtqi@q2w z1T`Yy1fDP-GoG#cZ85xYC0xB3{=(~biB17`+W~8%T~6fGA=}YuVd%}$ZIVJduts{^ zYh!Gj1^0H1FbT3R;17#*@?h_|{m&Z$E zg^2=OTBy?MBLS~HLL0lIEfeIi$s_v$w4X5zzHDf6**jWEhcg&<&|SvPo;q?vZAE?? z&4|4%Y0@wH10o&njVF>KuG2$C4#5tE&j)GgMOgIpYJ=q2tPx5^Ts6`E@| zjjQCF#@a98#Hrg55~OF}p7+gP-@F7@!_~_^eX!oJa-Ou7hL6C1>EWW^Nz%=fHOY(0zOVjhgdzl>F|*g zl%P7ZXM^-}i7F^qmtS-kQndC?L&r)_Cll^G-+p$3L3`7!2AyzAe?G)Lvo{<1V*?8` zi!@sOtp+VpRMq;DYPq&&17BT0NJ}3yXj|VD^~XFr*<&0IZ}{^&@ZL+Cc2sxM)}Y02 z_N4-mU^s{m^bGF0NPNETE$Qc_#SBwNSvUvhVeyE;%>afWZdHFJ*GLTyfh76Xp|-_g zns))WleleN!Sfw8qTr-5Lo@lnh!7)>?`~R?^|0M;Vs&Fhqm^Q!wO@|_EP9{xhy0nYOK z@g8uXhX|Mx#mNov;#isJ7@arvaC!}%Ov`t!V?x01i}zpXf2;p&_wntQd*8-+pdt7M zP%o77o54D9D{m5ve4DR8EEX&9Hn9WQ-)7MovQozfgjkUFo_GJ=e%S`Ex5Mil@CLSz ziF)IBQx(IQXpfh5F|N8rRrgmuU~jZ-v)4IVoJ}s1tIOTv9CQviMx6ulgKj2*SA16j z@s@*ltBGUHw2O1ezK5IP>~2D*Pn=s&Fi!J`DlE}%Hgz&o-SFDu8L^ETE@T%%(HUaKQ~+c0#hgqJej+g;by`kXo+M zF1y7b{R|e~VX(pr8Ka(MS*EkTWlQab*`Tc)P^Ky+Xet6!M_p%~wI1h3`H2w%KBMkt zVxAbqiHx8Ocep?q^0ifbA-{1gAt9p&nfrmHBS{Lqz!#x?f*^M9=-b`5XZG%QlwkYo zNHg?C2crEr@P&OLimTx&xpG{3LTwUEK?h}x#W-X`BI9L0ID8UW?FU0dAS8%BLI8eF zq&S~fPzJ4~$PRPb8Lxwu7f}k|dO$OtF-G;JzgzqFhvA&_|2p%}9T&xj5E8<;udYe# zByDcS%g|m&M{RqRxq5cP`leS}wzTePIn;c_GT1rXK4R~7%PEnSbzW~P?aEdp-~0Hl zrJo+VbGd{sm2~Up1d$gR<Nhdjb?PzVR;)Oi2CfvMr6!zX(83=|H&(qGV5+*8)Q z1$V{cY@8gL0O1JD1Hg*_ZK$;Fk1&zBMc+M<9_t$0b$;S>-xMB*M8zl=RoY|+D0xFWN-`sb6^8LxTaBUB6_Ju87 za$aj+Bp&n!a959~f1J?jcMlE@rbqEae`sioc)j`qN&D#Y)m0soN8Kr!gBFSr1Wq7D zIX#o^8J$WVB2V{Jl&xG|@N5I^wtG9hopRVZ@5cEk@;u3pPc^(4qIoxP^Dcb8u;#hD z$l|g{vWeypk{E(@uy!laDRjhblr=GKJwZb9``^MlFRefEXo}YH)CBKJYi;i``ZWfh zTpF}c^OWXF^R)*WI=!S{|KyWfqvaFDba&HG(`a3nsmq)+0XQm5zScw%SVyPJaTs%NA-rCg~_!ldVL zFJdS!@AVLZE$9?joIUyp3w7j2qrA{ZCxv)V5Sgza%0_U`UgIqxpL&27R@482FTz={ z{shGFP}-MDhPZ&=s*#SV8jOO_PTN?<#(EhG+sU-!x|1Ki_wBKZfnYG|_Zz~3KY%=# zfSIT4oxFu~x&TtwOYd7GDW02yBR-nslCC(lzR~M)(tHJ9yoS(z5A|}@@~70F@E7>k zskNMqx7u-sg|)O2TBPm%OW)Y!vbE5I;3S|X*n}rK?S3^?fd(70T$QYvH}P#Xgmg*W zCRqGdN=`0RTKXrQdy(xQ=^mvg2KOF^3gl<(Lw?3UOLd}*td-tF{(#T#pka$Tk?aff zkcZOsk4x&URVB0*z_ULre8|w&OjNcsmwPGB?c&^IU41azO8e~&(aFjgUv`VTwVh~; zoG_gu|M=12Z>HKOiYDor!Gi6LJ$O^MGnppFhI{vQr5I!crrk*{!Qi22us2S4V~qo= zd^j)H9wN2+^T)@&8KixTY3OCcy5`3E0&-L1@QG$R$S|Utw3r&uCNg;c@tO-9uDe$w zJq+)@!?05Gf1h<|U7AkLYX9+hPrtMQVLD&45fpMIl>A1OuX%!9mQQ3ZAzQ~~W7=3N zzUwu6U44sXjfr+3Vmb*&xGlj_$3mYCTqJf^jTS|yHh(i{<8bZ3sc+$&k$3Sx!XJqd z0vF(Xl+Y~K=C>jj+eT~e`zT}6GYn!i)Y0r|@9fI$>JHKBcT9V~d*gE1x$IpSuAdv( zgSObIKB5QShP)tN6&9jFF^=;p5wT>9h)Shub34zQX|0^E*W(W*{Kt?@4*;bFd<$)J z@ov)V^=vd~)k3FW5j$~a{5X4#JgE(L#l8?u;vu9hl6Er>udGpP&8_9_ z6#~AouKN#kL@TY)`vXBa4g3bU5!PmA?*E95I)YZe!RZoNlJ$73F8W^SA&I$i=lZ$4 zWzsAnD#+VSpp!Zi7o%Z%jN5mT&<4n4S5G2LH}BTg7{Wi6CZ-5@w_KW7Et& znPyHaC;R$r(+SCi)=kZBO$;fpNOVw}hy_zZ7Z*n+N>JMtGNx>lD;jsl$U056TRaX3 ztvAxgg~{&T1lsOowW8&FsY||nJqJl`hsM#{l1&pvfDY_1~&*OFMRDH2RGi zt^4>P?`iVHN3qvW({G<2{1Y?~@Y#HLlhmMXF*xOn5N#jP<=8yFfYeGKWK(-T{e#<3 zZ`icx4q7|JcSk}ov=Mjl2}*lpK!|AdJJa2}_FB6tO6-jl99_-@O?ydQWtVyT_U_)X zVeRv`5YnbUmO@NBoKJ3CZ{50u)_R;>G!uz1L2Wo|qIe_O;>aO)kPY&)1AM^m58=V6 zZ#YT-t*Va=+x^rw&cr#9E8E7`H4)ltx#T4^Omn57hTpb<&t?N z(tU*1Z;rP{oI`w%5aJ^~ArkD>`i|qg5BPk4O5 zM~hZirDa4`o6%VeS9Gd2uU7pj|1N);TBj|+oxvCrCAE9;kjo+3NiAn-0WG99v~TAD zl2@JR+VT|5Y5PE5xkg)VHg47OfY%};PO$i_l;~1w)&0Gr-SW_^nHco+yR`I`>s3at zoV}o>Y7E*sP*YAwm(@>b1wqi-J2^ACtw_`&HbRI346mgV0WFsOy*v5eSJR+4g8c%) zqd7;g`*N0G1oj$si(&!xB!b%%Z(t88{((J!9l)+r+>YIbj&~?FV|l1OrLbbZLdV+> z+=t*s1P>rED2B1S(D4?zU%7quTGR*XkIMNBo0EgQWAxqQii_B-IoD%Ppne`xv|*3s z;MmPMH(`%q?_vuTKf9Vyyort`Y++6uyFF(P_CU@PsLv5>LCzxVR>f)bod?bPJeHet zH#T4KXDoNd4D2SwcbHc34uW5!Ic~+O6w9#L==cl80Zg4;7eJq%#ug~*v6~fV5QqrW z*bRyyOoP@sA3Kjdg1vL~sDj5H#m-^#b4IYaic<()%Z~Mc;w)-^A9}w5y?+vocLsYz z!C*hhDa`h9y~2z7*@kfO5Y~Xbj@^saxm*F!e2-vSEQ-cohu{7h5pnGlb)((A*azx~#$~(Y)6w4r2L=d$9^dHCjK5 zZB^WetyGj@i)TEDkvYe)>vL9Pznn1>;j{rWpuXoT-ox(5*@x!+9irLq5uIe3@MsMA z_z&!boM#Z8e}-xPCy;5Sup>JC8z4H}fxUb6ZwS6d@b_ypy7?O1^5eNoCz(e74p|!g zNGI?=fPkO{UCaM>KyiNyDV*9N4)s|f=r(?h|d2GGHqlqV}xQUb|>OJ z4AJ~%h3V=Q?9$aMh`#rucF!m-Ui~kH8=E7yQT!8|jn>bgvDcyFyJ(F*G{0s91&V(n zKKTv8-x<{QOYBk9_HG3CC`#q;&^f|u&J`p-x1;$W*^wc~g^r7_exR6z;10w)ixK=C z;k^rcB0{`81(K0FvvM#zk_LvlR7l3R^e>!ycQFhpxX7$ztR5ct{===6r=@^z_w#Wa(Lj z<|5Cx5#i}Fnj4b4oS&lkRHL{e*4`=00me=ZQOkANf}vH5i56 zhbgLyn`!K>MC6jf!7@@Lb(FfCVm&&eKf1n09Yb1)rNg#8SA`l^5}M(_-R zUm;j>^)J{`bXj9e5v)e=($!z1k*3f{ zQ)r|qG}06rX$p-rg;tW2{e@oa{{T=+0|XQR000O8G%v_HX%^Im1Y7_B9BKdn5&!@I zQEXvzb7^ZrZ){{=R6;IqbY`Tx2bkQ{xj(FR*%1PUmH}G=X)(rhS1@2>1I9Ks#=YR~ zt=`^ecgjv5X;eniXnHT(d$HHOcVg2+O+rsk$R#%k(e9OV`OYlPz5MTezVG=z&o|nm z^PZwp-|~BpziUnmRcZa>R%ve@}aVlg5qyytB-)P0T33Jfs9=HWFw$| z4Z_T_YOOvpd>?4LWEH6Y)9M0!9rV`?peP3hx7?b7>JkWFlK|;D2%2K6t8LVNe+-%r z#?t}H^Prq;)#G0VAqw>|eAGZ;bYD*4fA#+q1l^!>pj$xd z2YM2uf?m8qPljeft8UO!epF{F^bk~agU*F+hWs~Z6|?|K-JoIU4(Pob^i4maZ-(xI zzP>@<0%^e5exPstQU2{e>g$e)@l1wtrrZQl6* zp7NtQQ=xS^U*Di}p}Qx|y+Ny>{gYPSpkZjnq=6gsO+TV_6|?m~ z<&9MZjg94%dCO|6iZ_*%HdPfg+<5T6o?Ddn<4TJrn%P#;&{$qulQ(bnyt(rhZ`hnS zckaB|AS^Dc)fUy(G=qYf`Md4T8Vg6nDMaX{-PuGu$Q{8xKFw7e1U zV)p4q@N6Zhy9}xYImOT>P*w`s%9b|#j~@Sbe~ZAA|8FfV`q5~%fw~P7HEO{a@}PNO z6!SpIJZLcx< zkO$(%iIvQo7=HmMDFOM}IcDqDLi_)()_3;*MV*yk4Of6G4|*8%k{y2mDF0#Rx``f3 zphv*`wBV`(t275(|6eo;uNw0XGesUO+J_RlRPvJ&zq4qD{pSz zqP*pK1NYCkf7XoL8F$V=XW%m)n~^`GFJN~Pe;)z~z zzW@IFub`(vfWH3%gwL}(32f_+e&3(_{?+fD6O`@yN5Or4mi{O4uktTfe!1k!hrgWm z<-K3*`{L~9pry~>{QUIihd(!dZuljxdVlJFTeZC)(Etb36O#F&<^Mt^aivO+68?Gy$^bF z0SlH7@P-rm1M~^>DUgW|pq~SIY=(XXWU~h1AR1yI9%3OkWPk+73yF{ilAs{u12!}O zU4TMR3<^ULC=Nv-3XD7hC4ogvgFUfXYli`U907B-K&QbDo`g<8XP|GPe*-&fg%Z$lz(wbPeCwb;0%>o8{sg4>8z7r{ zz-KU6vwdLA%At>-kAd#|Gg!CDKTQ1l!&xl!zkfM^Wucs$oLhcmXR_%B#%_nk=I4wp zm^8Lv^4L$Nyr4+`vGG44N zq_J@#qqL!xaVkSXC4kYD3{z-r<05d?HIe-PAzD3Y(2=EJF48T)>g#d3t1e8NVK$M}qIXO%& z;61=hKo+379Kb_h!*e`2fC+NEle6mB4FUE8{FW1(4B2u3a{>;|iB5t3Iw`xa02;jn zZOPd*37NEc()W|`$y+CjQ|_3ua>}MD)+zLq^HZUzHB*0^YgXN{Y^LDB;9oI zrZ;c;5?Ot7{mtfEZn@>QTejY^~6>rX#9`;!Ylxvbf!iQRqQ-Sh9>boaKqyYKG5XYM^u-a}1uPW#KW@22mX z{u}fr^bC3)eI0ul>$~^Xdo}mQ?oHme07voP;eW)xC31<)#2dXM8u~KOaCJ*!{q-XRe$1?#!D(mz3`QVsfB-d`kAMtXI4Jb@$Bd#^&-t8Y|)HGk1U$IXyKw~7C*B%`CRpqyrrt8 z3zqgSm7gb{KmWpgFGOGX+lvJ+URYMKEVk_8OZZFXm%d)UZaKC5=<;ut|9!>ND_&T! zZpF40dlg=t{p)x`09Zh$zuv1E9S-(;2tF-z3;oisee%uxRk*{ICX@K8e6oopglD}9 zF3k{1_)5MWKKevIaxxr8lvn;%h{zA{B}k3aVX8sYRbR@xHPn7`KUE3~0Zzo|c#H{R zy@3mdKT^>rJ zq=rmms=mXRv?k~bEUFIox1R0lpqy!(@%lM0=9S!%p9sbMtr64_sZ6hxg0gDb`U+PS z!PL_=Y(2a{pTA)vS{<$&D#jD~c3o$)ga|Ar2&_a09UeMj^N}GNe6l!RUxsck)otCo zFWy#(n+?|eCY1D%0Vf`}95kP%?aHKS*LtNsLhKSd#ZLJ1%=s%Hqi6I7^M`S@Y#l@7 zXPt=hYxy0Ga_^Tc0T_gF5h+Cs|ou3$q>Qi%4&JLTs=1FNEhM|U6{*($QR_T znvXMS#4z{?u6%7pWYyThu@^Oh&CNQ{eYS1+^KfgF5iC*ox!*h{PrZho9cVq&)$M5N z#bfEvfFJGJn|%4?!$##-&nT-Jsn6!c(|Vov7`1!S9@ur^+wLX;kh~H1ZR>+wr<)L#dPabm2MWLFFY> z-5{?U+oO4WRrH{?ljx%|j}d8V?Uu-MX6i*GI{ zS)kmH$~lZiUOf8d`7^_>DNicf3%Bjqw*R91BCdW`{^{7)8mDBD>|%vue%;FrR;%4c z*?6nh?6V|2KT946vdSD*d4N@Jb}~+uV&OWcmNGkdhXZX;o&AjGFB783doW*+7eYkT z%Lj%~pWdhU;x>^K9U?C;6l9wqPlDB-jWuex+<5!x*5lae&f2Z3H!fK=pQtIXEGpY; zYN9QSo(E5O>@a=pEBDvfiD6M_CsIPJ-*W<{v~JE!aM~KS7TaCq<_n3DQysmXy`BB7 z9jUItlbvV1$6?;j2Lk9 z?owF=qtAkqlEip1pGOcqM5gCp^e}cNUQxbo&AwIj1Zk(uj0t9Vpd4|Y&5E>;Nb*7K zy|dcQK7wc5G|lh~gDw_cTQdhMDU2km36~LYCTbE*F^e0q#0ISwF`1Baeknin*{*Yo zQp9jaTSu%7s7h9zq)`x6PaiKtiEc@X0@0W(s{B5;;3vBM;gmlEdpQr|#W{iVuo7&p z<=94o|Ec?p*U>+!`uUMQlp}ct6C4(?1#6;f>_+37O^QY_R4x$>{^qnEb{+qm`4F>k z4l63jE={wFE9DB=Qg~MZFO(8JZ=e3{xp!f%*3H*iII|g5k3nNynvfv*c(3R3H0BlM zDc4IFW2Uuqv-Rbc$DOahv38DaBYN1w^spViU>c}gfzDmGdSOX%Fjz-TRn~^MCrKUq*Wn-tP|j(4@-C`Mjv(+m}b5OPItK!6fJequb2G&Rp;Dz-j;C zYy7vKAwR5cm8}}(1@WWjWf&~r)o*0%Hzi*^8X)*w-bJ}dtVu_cCPMEZbq#2(N=kM6 zVwfaxoChfCVMkZRa`f?i+qbRFzbyZxPwesZ;K2Awyf^r>%-Qad<5xdQ{>lY=!a_WP zc9Uh^V(it0jZdv2_HL?~M=B7^YTbGj)qJ3ma5RziCX^80IY~UdJ>8`gQ^x@`Qmy35{_b*#DvociCT2I7`ZTfC~ zX=6=;K@00@Efq!RGbd-ukH~kt_w&EJ-f(F5L0q{_p)}j}jl6Yc^ziXsqN}CN7;k|) znnL;_bj{w%J@}Hb=QS5cI}aThuG_M=x~!lE*G6?wV3wNF_GAyL{?Mawx!j_OXc3wN z2Ha>`l`hI$Ebk&4O99;x+ybODmRW^#;9!j{~fWvq2^t zM!VyKeWUmyMe0XRh2n^u89TpE(=0ZU^`r?Ybrd>w0^2)l@h^ZzuKV_H@?!b!cULM4 zl;^f=FQr|gNE3XVKiG=$0e2)2XfP5i z!_hQKt2mxvQNb=+f>t;Z7Nj5%OlQ0qY=nV@vGS zTDYuEU$PAqW}VYrW8q`Nu~3>YM>Av(_N#w;TQFvVDzj_pD42HrmgRpE>^|Pl*0MC|iY!zX5SlopA z-LZQa&Dkz78YZ{{+_9^u{7col{8j!WzNVS8S_r3+F*KseUZp}VQVlV2dmP_X&sZ!3 zU&8NPhbli;t>D&jyYV;D?r@L@C&XAgDswWTI^OH%(}WUH=|!Wr1y?JfcNon%En?hO zw;Bn9^v0f$#%UEqGeJ8j8)pzrQGF{3Gr=eu$NUku5F|Q0X@5KfUp?giP`()*dqOqD z#T+qweI4BxM5_3Lt*G*q%BL(@>nZ18we4q&=6t6R4G_mvtz1VxDz8w5NXgz&duq#x zgezEy)n=YUG*;~Ip^B%(_I=p^Z3J8)H z4cG!rJZ{V-NuIj4lyTDzBRaf$YmH@0a!luXsfHb?l z7{5)!6w-U>0;96lpuFN#?uKcR@#9P)D-oHHkqun-i2u*dD;McngK|7ZR$gbw#X zei)AMK#8@hyhbV!+Zid0l!%T%gReHAO>azX441&EHeLj%Q}lbKST7=lJz*)5Je@k5 zI2j$uyq>(k+uc^)3R`)b73C}3B|;rc@)n-Nc@^&jMRuW4XyBW8gJ9%Mpqibx<2=bb zov6`c37YVq_6YF+L0_f5u>ZOTvB_$?e5*rNv+Zoy6@zQ`GjucIAc2qU;`wPDcL3p9 z@{KzTJFI0b1(q_hg=!-8w4Jdt4wuDcc3H``X*@SoVEh0r(P1V`N9ZsWW5QGd~OYE(@P)ri?`f*3?{ zE5f#;OuG#CeDGIJE`{Za#qyoX3T5dN%FT*FS#d?llONOJ!oDL9{K9tBbs=&&dNFb# zFv<)vU6zC;lyQq5zv!RVlF+u61>oxs%Bbe)wO;`5Yr};venX8f&7T&6HabE{&d%-Xl15o zl!l|Z?vDjSJFZye&os2d>^5UXGLs9`DWAa3jTA3kT31qIBh-r#H959u>=&8{qP&?~ zQfEOdDD7qiC(a}jTnv+Mx;~U^@RHw0uc5q zW6-!pV`3PFra`Yxj;c{|>y+GvnT|qoKUL`*NR8flZ+&EIY-4m&`nmMe(QTqtvI`W_ zPrU0Nu{Wf&7!$mai}Clvm*dX`3Sz6{YvZp(wh4AHN;@pLL_2U$_-Q=NvpfyEhidx- zJR^hx(E!g2!$|V*QF-~WJj>VX>+^LZDt^!NF?eI2lr<|AIzr}Ee>f)(RQ2TuZr zipI=Nx64JiNWf{pBu;a+=;k&)v|pJqovNejT#a_aB(&5 zY9wr6TU=x3XX1j$nZ<@7M$=SCE4tm|32&}Wn`8k+Lv~vM05)>l= zBHoud*nTL|o*W3Z`D6aLFPy9jHO4EvR>|tM@vOk`Y`P>>nka=Gi8vF(x`*A7C?UDs zoCjAwkEowR)OoUD{D7wTMKA-x<=_l9lvOZRS*_f*mD~Cns?1dFaqr?waUsZgFy`<_ zu8**^SMr}mjZ(9y=WV>r?c`f{tJ}i6B20!$!`ylIJMt3r8~$7VU7SDU{<`;{!av3_RQ=O41>e;hkWRHNFaer33&x@$L8~jXrlfm^|yId79{YxJSw9dU!<1 z8&w`Wpya`?$Ap-l;3c=mgQn;V*@@T4ldI%O)$-(;iFO41`;V>$u7m)B${ZD`(`iToVgV0Oe|yA&byK`-o%21*=QPmOiS9 zMBM2S^!n0Vi`lK;kE)eJa^BczC&Gj{fe|O!RA!_RzF^!?; zg#99R=xm_pFmd+4iI3#_fKhny^~d&ASX*jwd8z6clLUr*M-sU%sI-F7Z@~WmJe63w z+n>R%4DHWT=B~|uO>gi>Rzm&gdPHp>d+^ViB!9eh#C>1{mA_COW3vrxPa^lLd`6w8 z8r!j5SGb=j+*7x5KB~O%@#ep|@NTZt(;DtUf|B3sl_KU=V~5bow?=|dSmHbQC|*U1 z$K2@5^O|P2iL>EAw{slM^HW#+PeC z_PXFcY}um~3T7iHq=hCqlGRa^%e5EjY3oM3qR96mDW5kao`*SWr5*7MrAD!E)Q9*z z9*=-{VZU2)`Ed2Vv0LOfHT`6JO9#${y#PtKhf|$VI!yZsdo#xY44K=^)pAWhI(eXI z;E=W|T!zhjag8!{ZD#k8I)bKo&PAKd`WCYpcG_JeiSpGQb*D_QyVri4#=XFE!`ir3 zu9X?2gGPTd%($Hl$`@{bb+rLz3^kMvv)BbcOVG)oq5iI9f*Ndyyjn2pso)N{B6!?( z8tWhONS%ayd(Usb{T0ew$sbx9b_iyGcv|=tp@lCNr&WlCKmhgdUXKS)r>C;vFqg#s z@#fWk5X#E$G{-th*LjIoCByE$*v`g-*IdNU$nL>|nA&uS(Oe9>qe&tV7vgDDKC8;K z_N4}~-b~8|kr3=|ivd-fDudVp;PKC5Vgw-YVfTsisQjQx#%eA+P^Wc~2HZp%oh`Nz zM0JRXI%D{bN>@EtxF)`)5WRk@Y7w^zKpFKb$b$w%3CqM-t!8fp(-1@|_|k2t;!)Lc z)m#(KY;jc9Sz$+W6J=ysp2h6Bl#MYtP*E4Nw$U&X2y=1F8*uY};*ivz$@C|Bf^UeLI)Tb>sSdIcDuT~6AS*93ny>xtVX$T@iHw?~=CCznlcx+LykLoWj%P*rE%5Zt1>Z~gckp0do zvNnj+P37zOnsQY6r>c}wZsW@Fzsvur8SLmvq_DWpq<<~HV0#^*Rub~#f7XlzRA;z0 z%8Ok0sdPbI%7QyXF)E4yao`cPTrozZH1c0m3^O&8?n?GzMfW!6>5H zQQQowyu$9lPv@q>Zr)3T0W1%r>J^B(Y3$j@0XUN9D4R4w1T+A&jLMekpa!wQgZ|!O z;@shpPvkpLdG6w?4>uZUQitnEFdPdQjwNWo+%C71#hV=_lCr}-okv@RUN2B>cUOo; zoQX!+FxJr~Nq*v8*{^AA2*=EXJg7>fyAwUw#s0zvl?CgI3JA4h?BTInH3L$cFO8>n z?g(witd=ZT+({KSSJ%PY_g1|07^*z~>-E1M912I<@tvyG+-7bcej)d$=<{}=56Eea z&QxpDVq5bM%adgE(xnSToJ(>k04`meg9du{S-ElSUX3WaSs%gqgIov;xZQ$}h^9T= zF_eiJf;FOEY@&^%o@{pLEpU`_c$+br4j|5>h!?Chr!1bjlzW-mhQAr(!EOeC4+NI_ z9aSpT>g&SPwMe%JRd~ zk4Ss`5fQlWcjX@7`}xCoT(zFA*E_XbEmOtSQVhVTtc5nwCOhnOa_$;5Mkc5(;KDmp z&!*;o+kYSsYr}nB$t(C^-tP_t&;Wo4UR=EdQ7@C%%lkCW5&NiPgzO?a$yQf_jseFe zM#gL*cx{8DnIt^sL1l8VK|&5Fw{$LUeGWEDZMF<1NN!#v*j!!^+&mWZBt7Bq;NQCc z)b({}C^!@x>_L33o>&+a-5$!1d#YM^8)l|h5~y-5S?{c)^i(6&WX-qlvKPAaEqAn{pBieE?yAYG!~9BFh{F)}7|n zmRBvi=p^tP2VH5p)zxacYQADS2@HME7Sv1YO$-e?RGizzQn;(JO1X<_wCCFksRpu> z3tBI`+FV^!m;I3au;U=`1*3L9yspO4;vl@1*tXVZI_CD|C0BbU#ylbY9ag0ARu^PsiD@>g4NM3vz?$E zq=T}93*0Tu=2~kL%mHwLTGLd}+~EX+AVVgE6T(Jaw+XRk>@iaq-k+>; zpUr3WI=l|aE;(VUHN{4-4!%_gd0_P-ME$_2Wq`An$a}|IHLtX|O-ABB>r`(f+|dM3 zkR9?pa;K(vzrC6vxMsG7t#r*Nm7BSBaJ{7S8S%G|cv}U6{fv@v*LUm)H^6nk`~ILg zoX#9PY)TjIvexbAh!QTKJAtX?3uE7EDBv7nl+(>oL^W+A*=87+G0u*2cDJ3g!)tW& zHzv!>F;m%vOVK z+TA6q|2}x-L&3{&Ko@c$CP>4ql(q4`fFK2cAr)JFs54#_EA&mR0g=J!jddC1m!)lGEy8G0Nevkwo?OU5j@;`=Ii?a+9*B zVx3@1+QK9cAiRf$)gQ|P8q&^I3$^{?5ou6_m6dyt->bUCHlbS_5QPC|w1O$7%NfBh zhD0gU)p@e}WZ#8>tFghgNb!^TPZmFEYtwH;I(8>YLbW}}a;FXWDRy$YTKV~33&BoJ zmrst3YPcvHp(CsS`~X2X6h6^<5DwX`k{L6ZNQ;%IuddldZwLH!45?xfWj5c5uoJkH zC6W4?n*YM3k)}=>=R7{Qg!OhLGhRZi?UY%KQ?vooc6j|qd2RdQexYk9hz=}}?`&xW z2$17M58T@SsSJ0c{<^_x5x3|H)|bB4P^K-n8DV)jqoG7gz>0^BF_Fi_>O7|M54?5*5evD@5~Z9w+O7LAVy zxI#D?v`3ubeOK3gZRmw<^e$ZkTKOjhi|KHVqg^Zm*A_3%t3hi{>ii7u55?Nzt%onY z{&~6|R=dYE019!|W_Q!}*VxyvR(FC=H)Zw@;N;QUk}3al)&;@e@!r-UB*4mAU39%c9C>Z#GZ2XfC9)v8W&?Nkb90uHGnto2uh3PH<{F^HXM z%C@{0>2A*gY(byJ)CaL}jAfz(U(46+L)DS78FHznKasp75fY!`qWJZDa;q8*+c<)8 zYx(RO*3Mx$F!9-0e^8EVHni5h{B^qh7*!hTUwu)u3&xI%M&{l08Nbi^aMMa<)qhLHY=W_qk0bw4!Yy3ve0r%_`U903w%i>#&MIM%V(|uIAzd5)mW5V5pkE7L;y$}Xi7r^*Fju*$3auN_hekM;AW4^ZA*$a)x&z$`=#oT`m@W{igwBSN6Qi zt=GMp5YUmq?0J{lemL)PxC?QLE}$3@`BL`0OFE4h2S35(Gk1TS85tN%RV`l9R90E8 z2YLUz$%-fseI@5=_V2FPyBb|GvgvPU-bxH-@B@aVUcmL$4P-M`(U3ZPu&uYPpAbEQ zNA$#6W0_zp9LRW5DKyGPnF!wPY=|^r#wK7&2@>oYi36Bnv7)stPkJ=`Up!rWa^r@=JzMcpyrw(d+0&n@ zudUTLRMj@M^#T#x{fC-0h=Ry3j2&Kz$bTHq$SzI9?@sbV@IRO4*4NSICc@(23}xu} zi@BX0LLx(`Wyjcl`E`waUv6(Oe$7iL>vFk4c7W|?``v?FKm1MZP>&#Hh@{__;7`NK z>D(r+26%*7B6{bt%7Y=rtW)W?_*UfgLIPgvM>`2unDY73IrtpEJQCa zBoDTqKP#(X`H@_4t!KSwEl_eaPlC(AlQ7%=I{g-QdW7dYiB4ZA4iGR`vCdtHRn@tL zW`g;c{)qkrRvwd|(1g1qeZf|7uCPU%2P;!^GpEu=d#(ab{fV4guL;B=$#5F38#!yb zgvqNvmG71tex?7c;B_L=*Lp;X!cnfKO^exRnsPauHq!1e!QCZ0!<(_?o0|*riS?T* z9#y8H^W#q+TZ=6-Xjt6SMv87C4cRj`IU8{_6B_P+Noci z`fbPY&XaA2;Mjiu&W)&Y%A}DuU;X@xp>Dgj4Sz|wdzFH`q}&6W4p4_LqiXr}u~GRA zjf~~?h66wI5z4FbmcQo?q}@`Ii2M98;T){cV7<+xgSJxB8dW3vx5ZauYj&DS^NE)? zmprOGfGXwhDR;_i-@cSObP(66ct#|726hW1@4}hUx0&~_V_iHSBRajIM0Q1s*1HR^ zhB{u*5!W9C{K2c_&2N4B>8EeJs_axYuX|zPvui$-SK(^;$+57!U(+KFN5Vl^$_V{E z=%3hKKbwJV-)k(ZB8K#3iOtw^tE-zE3C6_%?l<~dLM?cdk4jNsz(O5;kv?oF(^4)G zyqz~&QLReUldK+lc*%>iA3am}E<=2DKGQu&l=Yo9pT$OpecmL|>hEj?<|WK#s5oq{ z+h^W~tu7e)2e2{&quc>3o#Od8q5M{vsi|tooVYg9dfZ3wUdipnqPd0Qc260u)z#Z- zF*zWAqd5^RT}E(r#tGm&=d`#jSapdYmJ@&Q4-XswGTkZPH&&%7tYVt&1Ygfr?a8{$ z>$nZu(TjZ4?F+(zs2FcU<=?9gb1^cCt2xnikhE{n&IHYJ%5 z6HSCu;pDXD?gP$~*oRZJ`e2)h7`=Mt7biY|y*g1_iz;^)?Ae7sAurd+w;w)zx-~)S zqqx=JY_S?jx0Y`y+Xhn!wyhH|ly2;ExkS?~jzmJC?56eouxSrq8((WGs34A3>`Je} zw(c=hRS>)PHm-aURbGC7g?#7b^PQvpc=7PLhKtz25ucO*8`0IqC%Gt-aV6lE>U?uP zwz_cWyX;1cxJR*c+MV5q|5P5;>?k@aqw*c2$4{Rd-lyEDp!s|E5b}WhSIzNQcKsa_ z>(AQlZWC5hEO;u2-}{FK4r1#5F+{#cQ{*kHFT!dY!hSPhwlM%AJ1s6V*8=M!N1ETj zzWw0*zn#_|EFU0jA$?pQqea?7izY;uY)QAG0|V_tZ8)17^-WF2;~fW3?)}|wJP?F= zyN$PEMTHK11+k;LVi&b4dxn8>n}CkyM4kl_<)kSJuJ_lZoA3g@s1Q|Pm5$RJ&`l~6 zYvV|qIe?&S)ZW|KCWfN87>&{AF~AKPdF$-uK%_5Bp(0;O9jw5a8GV zG308kFF<2+Q<6`}o1m<~`0DojRTLoVgyT$r7ufbNGW)ei?C*aoJC zt*7_UyP0CxD(}?zg@7dZVZqObLa4GT*JWdz9KpH3MZqQ&6Y~P0J*uqEB?YJG#7BJY zK!|8l0Vh6`eGjibgmin`&vhX3Bl4@7l8FBBsZhtD8A%9M-F`y- zT*p{Squg?JgFH#jdGkGRN7h|Ya+I7^uP9S-`N4nJ(5@`%q5+)4>@J1@eyOr$YJbS> z4-(gazwhR8Z*IUEG)ko~2An%y!UEpxJ0YFE-d=;PDIWd$tD}dmy*09Tagn3i z*hF|#&Ot+uEegj&VW2LC8(a4ZIPbK%?bwm^8WU}&7@SwJ6wT2XdoEW1yM~2*b~6Fy z*yHIq%Xc8^_V;C6!&^mrzy=#)Q7VOv96ES-WOTG(xa43yoJjg(QNri-1wFt3xdKjq z4`N6gQu>rJ*^)FQTiX(;Olxafdq=vJ3R>f)kO}cRogN2fwcDIdf?>9AGn>ptlWBVK z;nI=fBe1T;VYU(uD{W)#X!mNV!Y7=lSOzP$sYVM7{@?d*wSdL8>xR zg1xY+P?^ZU{FdOQ0?JP>em zKEemQBQMMG(-_IiNtnkgie5tWN(n!zeh4`fhg0*f7^s4jhTH&*wjiRiFO;Z z{djX6e);;rrx8$Z%UI8xY`w|jyEQ>iTXzs;0+g5Z!G{s2gCT6R-9kH#ATCs-y)GZ! zYU?(3nfeU9#sC8^z5WPj13WB_4`?==EV_8@_>s%!3QoPcX5a2j_=W3+r;&lN0G zds}C;v7yndZD=xPI&si#*I0Tc@-njYpWD_Ti)SHS<99E}#^=Yow`AkR<9p_0*BU zuOUar$G3p!_3<`MX~r--)YdxKW9%p{X>O>(m1plRZ8Z!HxBif~zX{}JkHPFjem-8M z3H`7dKFaU(nUJCV&s;WRR_8?TjNRmNW_$OzfUadS_I_i#so&6Jj8L%p8RX%y%`=g2 zWR3jEa-{I^f`5@ddD{-;#yJN?Bdg|sg3rfwnxL5I@q-D{66J-5_dj>pf{~O1EWX2L z0lfkANKvT3co>N}S^x2#b~~)>xxQd|_RPfn*|v6lHxpTm=#FRt8OiS@`sxs^H9XA{ zvBa7aCdB8o33kk&ZHywjTTQU~6M5j$d}QbNLko}}NNN1c4rCsp{A9dh4#;|IyjJ5A zya8_r_V`4fi1Kc?8)w1tQxZH}x)_X(qMQzzFj=bXM0QwS`eeh$yTPz3{<&~IqGn|4 z7@@JX>0d}(Y^>ZWS3H>|D&&u!Le?SvZAe&-Eks^Hs^!>YAkxV*HPwhR=kBaKQjZiJ z2mXCrzIUuvv#)GJw)*{Y4D^ZYkn3k6PayKFF>)b@|1m~B2BN)V>IpyQc&G}bM#jje zK=f1jKQvy~RDCjL?8PpPoH^20wp5j_ z1s;IdQQwM1M4!(?1X96pQ1oyT?RUcJTBJuGSc&MaEJYqc^zsJ_L9|i+;4u&>+vL(G zLHwBf!Bfa=q{v?AC_I2r98Fb{)l?(Q@oru~sdi^0^5>J9?yhrZhK41??~^1!g1wB_ z<;CMRL%azytX{aMieOzF&7q`7c_`d(_d8-t_C;um{LquglNXmE@~v+#MS#oa9ABc5 zCeDBa`K&3I5MZxG-UExGn-@`ml^7q+32uQCSPva!eS4XG&N>^sxv?^T3tAB`P88!j z%QHOd*ZH+RtzZ|-9LY7Bs-6T`isBqD)ZuYRPQXiHGRj?W`5OgLi3Qpm<~Wu^sU}Cg zvl%uN==T(%`%15f<1nZ-!F1JkFp+0bO>f~njQFIG!N(d8+(RP zKN!3ZAQ-SV-n1{{Z;uXy2fTf-7!tz~^rW_bUm8E2P7j<$hYg2J!#F2$U@m0BnWEBc zfQ|D3ArT!p5r)-Q=i3^h!*Q5!1UsKe^CG^0kz z>2={=fO8W5R4^4vg`)m=AQlLjLWWQS>@n#C16Hxtxp6a*zo%~fOX#a@8~gL|eZyxO zu3?wYb)Fa^22ZuW^Gj5|sYsr(^?lqY4Y#_{wx|&4LBovS>A{6u(r4q$6s)vdU!yq@ zK9=dmyIb3$16X@IrA-s8Fx6M%Dcz4Mw=})9zVMYn`Ht)fKKs~YOIZ zZ&v+w)vs3t$sikSMWn6}>Q*+$_?A&T=#x$IN;ev&+gu@Bt$Xh)jpPV8!%T<`c!GX7 z5|$J4E;kwoh~YRIb_VPLeBRr$-=6=r332-Df{n9rPKt8Cq{UThL)~(zGJBvAcTkE& z0nPFju~D+aYPbCU*l#r?<>pA@RccdV!>kYdpZnhTiG^%sC9Ujlp2oFsCIfI#FM8kd zUi69uAfMUkDst^~!Lu@#cYT2!JDZGk5Ppkf;w`M++0tTxjk?_}n=s}6M=#4u8VECI zWUPZoPz>>KuPV-kgcxq+T3E**5(0Svep=|AuFIo`;VT#3PQHiz>O{FNNN7oe-GV!E z?HncAB$m%!z;kY>m$aDrDS7w!^BTJYplredW=WFTihYmPZ*m%)Eez>1r|l^U_C@`H z>_56?jwTNU(*CqRCQ@#9u&Hf#|LT|?cKH2N2uo$!;~|0!8@rADg=+?7aRMt znyRJPSd4S*0fee$cf1L_j7V4v1c)yp=~x19ZV+AkN5{Yi6Vr1e;Dm(x0khvMlzYY2`SDx&#V zAzw<>nRtF$SxZqBwJG}r8z3j1VcZO6<#Z0S0oK(Ui*^A=_V>?y_A=hCYt?0RQB%+) zw!nhx7*bH$w5NP`AY6$vCfaC5EuuBx!c$~F)z86^+yg_tSU16OL$N~x(xvH~iudsz zKbmmGoe4bqy;mfT#nMC45G;iFXb_d5#c#d%Q(TwO#~a&P;w>?u7gnY!8qH|9Kh}Xu zxld9wpdMTMHjr}4Q0~T{5Np+Fp@k; zvma^5-OJev5amr(i@wTSBfy3A$u9}M7zrY!eDNlfS00(F#8j28CbAj#kqJk;4Q9RB zuW;?bSkg<#PpD#anr-Fa_O!>BCVz@wAkvZ>+0(2eapT;!j?cJEV3^GDx#<`45R2b>MY+s^UaJ?@hktI|8~p#&piM0<$2z| zm8x6!+o+?7T)^qHr6cJKPlu6M^;B3WQaVqYEgHb8jg|*u^?0ErS;x1?^ zD=Ob+T;-V$*9+x>k#K=dY@;0%828GB>ZG^F+vbJ+HDW~_MrER9sU4- zehKSy54pRE1HJ>|K_9H!E7y#Dr?;_LPR8kWVajynUaj5jraYvx+1yaP`iAEAegd%M`^EahqwQyeZ{b7Q1fNV{ zV`{C)i-O?o>B#qjAxp$0hv*=bS2fB+MfHaAr1HScIX{rY!d_{Q1nf#66LOALfhrcS zZ6YY6$z#H;CPAzxdCF(;wfexs8l_=RN4uvR9~h9j2T5JGd|m!R{}&dSrupb^Sp}Nb zjE&S>-B@9&0(Od~hH3;xnq01VL&?f|M3W9tWy#-bW-IEfZcT4jxGhSayqP5%SVY;b z(a9rY`^FyBmu+{N>d0nmLrt>*zF4(r|2+KF$y=42$NppapDrBk*uRHlwZULG777O= z`OAAwmtVqvbA9;B02$KyfN_f5o_>A`0VeQO`o@N4M=8F%Yd08ShASMZ$1fDjDp-kM)(7ZmTZeKFu3ePd?7H0OK^|_^&nVn*j^jlgwKAmJiY0Y*U}B_v$Y%f zx8`EXDmM~@>DD&HCAtHM+@k%S|Kk}v7__2}h=+K>1T&S2@s)}Wsl5n*5Ev9*Lq zcejE@wQ);TG+p%F{iJ5v14ija%HwetWgeun@FsriW=wg^jf7y@+=fsh6#zc&g3Dk~ z6|dxj8<<`^$c$f&YR z4@M#evpZTS3+~ahdL6-jY;Q-T*GKSTkoV!jR*i*oFfJnfSn|}EMgOca)MK0kH(2SS<8>c8mP)xW#Fa>lC-HPsc3eYXf**Qv1w^hOg^Q-;ZZPzp4@I4G{) zJHY!B>6O}09FwcG#Zs0|UM-*G^}ZWfbGtU)k{`q5rRRQk`^(ESl!eNRo2Sh*65<io8bdkEN z@-N0(_2z*co!i|Wmf`?dGwo(SQeMBV%=%n;Ce+l~)F)h&@_XfVJ#qo~g|BJ)qOnnd zD(**Hty{*IWx zqpGReN-k1nEl^gjQD(rKE&DdUhsz7bcIjFEu=4nO%GATk6O&k;;|L|JX|W}fe)5p~ z(5U7Qy1tVQq(svKnD|V!9-8cV z=0xRXFfA(ua;~D*pJ?xu$Dbr@0c*(ShLzk6#lsgusw(cJc7eZtVlIR z`!Gmq{eb#ub+y{4PN@&7Z;pF# z-12d2$9**JvvD`Zzcqev{I&5nvs77+WzEa_vu3j9Wz7fL4O&5aMfKbjxd(Dj z=U&del6x!nPq{zlY4YyPBk~^0o0|7}-rISr^0wra=QZZp^0>TkUMjCMuQzXB-r>A6 zc^~F|k#{ri%e-&$zRUY--j5U26XA(D6YrV$z(jl^d2H+%z1QYzj+xl_ z#CQ7|3(56hlyh+o++pKb2f4_lj4xGkuxCCfywcy}jdT-!Usw`-aBH;N+=UPDov~mP z_VZoo?f;Z8%BA?xE{xyL4e!SeX}_+0?=@h)w$MN`-(nV3HyU8Z&ARE#R6`N+3gxsO z3=}u0v9%w#dIBiCnkCn*dnl9GesJq_yp)24JZC=>b%CUBm^02Ij1S1E_ z;#S($>LKfLUbf$CtD z)F3qk8f~3-G@Reo$7jZ9gNYu|qxTXuM(=e<)M(M86DC9_QGSV@AToL{K@cru7z7Ey z=tlI23^52s4PJTg@80*f?t0I&*7NLjp7YszpYzu_Yn}Do4hJO-KiIWRu#x7M@RjBJ zOfQVc!a*tl^FLWQgHI1DIPxeS`tjGxVZki?ok4Xf`}B406{xD3NZzZBglyw6`r$y> z@pAOib^d$9st&=|m%8IOf~u)}=h}zgGx()l7INrM)iLUIO!rQ(Q$8+mzy(kEjtuuC z75ydJ_eSPoX@?EV-&3i)b~LL|+8{dL#-h^sV^&Q?aJ)PpuZtS;_3dGU3*tCU`=<{* z!z0W$?h@%yuqErU8jeK9PzGr_KLk!^Y9dJ!)a^P#oaA=&vzD)F0>E_5E(oJG3?%B^ z#-P`=5IO0y;DA*)<}Hea+E238mEw>NH+)r1PvcO(DE5-_3!~`D^Iu5j5ig!Go$Y@2 z5_{rJd#<1JFUK$Lk-pv|x)izKTLoU+Bft9w`(+M7Gbz?}bpI6OBJ6bYBrCD$~ajG2e-!?P8&6N5S5hAes!ki(KF?;Cz zu#{V^Tipbr?@|rJX3pP`J$!|j?1q2d`9<`qONo((B|aXGZ(yN>-UY=TRo)(A31IZg!%y``u7uM8EUW!LdIN}tcICuNrt) z@n={67o-e$^FAEt@4}1>OxBvA>kpGfx-{EYoX#(h5>baFt@!UBz$=Q z-YKjSj#Rv<5@ehIm0zQtdXJ~nA>V250Ptg@5n6m%!B2A4>D7^k$yi6 z`abGb@tO7Fc~PWqA9ktb#kQK;CD3qt3i{4x8tQZ}vx0%?p4RX7xa>yFj^Pds0q%?s zv*|z9e*DUrq1#DLs~UaM8iih~n)CXd@SEJJD7k@I@70)d<$zTg?5E78)=+p4zYBJ_ zxrJB0y1+hiokeP6ad?ZRd(Cc+$UBEyt>#SyendL2nL@(#cA+V<{3Q!Zo{&oN(y?Ns zOSWMI`)})biIbVR(c{Eu3^S>h+)J}}16CAd1Xq+H3Zsx(-#Te^9Uv68YMGD{)B`NI zpCMr!Hso6ZkQ=qNysHZeoBdwyAawTzfx2D#dbx4}&iVTNMfvQ=R}hIZYot-F^qqy) z&5wr>83iPr-19B(y69YF#lk(DZ`@9j5OLd-M6^BY?3~~~!oTF{{<+`FRz~!QlWwTx zzJ6XJ+iQ9ehFms^($FZo!s!_yq^49&8uJ0{$Evx<8gr((`NKjtspA%Zqqv2e&0jv43(f5bG|^*XoH4#vW}@C{lP+PWxpd z&$AxiD^|1!aj=N2<}%N(i4DOo55`QPemV-}ryTy=G=eAxY_Z{~E|{b8t#GSfs7i+n zTPK|%WecCWzAb1nE&9k?M=>GDT&K^(mo5$`ch=(edgA2DQEgkUu7Wp>)1xp$fst?K zOELrr;)0>Bb&c*Vx#F(|%XeJO^3{q*Ic@MZtbKI>Vb9*gZ$6yj#u>if8n8Tow1l1> zywLreFY3~R@xaxkTJ3AK;%IP{yiQa$%VRy7t~4z!XDQ}3vaN(nn%d;7=Zs%gN_v8N zCWAg6NQx~*5u`%qZ}b-RXt)@RV|;bH74fbi#hve*UKLGD`uZFl0HiKv9Y&vZD!(d8 zEECA% z`J9${8O{C@-wpc1YXlrD_iZXEKd zXe3ixIFuA?>*wiS*Vt;lS`E20@`CS1xG zlb)sL7b;`cVe8gdR_w&sPPws|Aq1I0e@F;akm&nBqc-44;A@@0VIi$8VNhB&W^9kfYPvZ#l9X z$D(<<>w*ev_Lakbon0zM+1T@m=Y$0cB`ChbX}K~8cBNU&yzV-=XIBAylvnnpVE%z< zxT6`(AhYxeC4;7I0`+}`XwUY;WJ5FKx9ij#>rOQC^!wThk|*Z~3CjmzegV`v$;nn_ zRaGzetOonqf1O$j&T2MVcMU^XA9eK4t$Is1Po+!6U)xn#l0O}w>0RtWxdb*z`X;r; z4m;WaeD&`(fTw4$EpqG?Rq`~4v9Be#8y))xdRDCJVb4RHGEs?|*}>SyEGYqd#2CUN z($Bj6g>8G`rpI@7!(Zpl?Y{@HCC^+~(*Gi%U6hsGOo5fuzmIJ8psTB7jJP?P_9`;y zJScjNSA~D2h-`@D6PPVDR8p1&2{bN#Bi4!cfJ$ zr&~p1cP>5&@cr;ey_Hv4F#_L(c81Fw)&#NKw^c02>!LI@sr_tRlt|+tlU-{kJBPMf zIoH8Y$s}hn@X;C$HPHovsU+j&+1sWf7+8ds-t%fQeIywHe4v&T>*qR04V*%+ejBQ)cn$x1IYNq|>8t9pvRZ>K!brx8H zM0;jx8?xwM_VJeasem%JjkWgLR+PG}U^Em7FT+{06v=~g>nF43gm%k@f(K)3lpe3< z>h2Yz$sac^;Pun;b+3x#&=?79kT$ALbSF25_wgCNMiJ=+;EIq21pZC<*U%MLBo%y4 zt;Of({P>;8&Sqzu0NAN4Ew9OuZv{{J-7x5u!c$o;pJ9AXdgc4xr={)QpW1tq$*o#> z7ZJy1O{{|Mo60G{i<6og&V=48L51y!I?hZ6vXcCP#R=eGP;)!A`(sGYe>b<%P zc0#O$XYcd#ro|kKe{JyrawtlJ3o*FXUC6o+khB zb+c3JmFFC~zDQ+Ej>E&=jtVlvWHqQEDKUd9WpWr{61EXW`QNeI6ut#s6Kc{Q{>cCE zzKA;ebWj88DU&s8b8E+0E2pDkqZC^jwTGGsWF52t`O99VGWstbE|eYK&LQ+KYfMya z+nRgMEFX7^nb)^1MJcoIri&JG>3P+o%g$!O1q)cv#i5_^MirIbP9d;Hn#mRF?`r=7J4-DYdTJ09|dJ{7c# zD%}%s2W6S!Jp}&xZ-xt=7;FFM<#y7~HeaPM7AS<_7TyiCvfK3B9D>Nz@hlB(y<3!- zwJ^2wseDu8xy8(hmkFR!s|mb%V9&%sg8{?S3PyTE@nlO%voxjnmaN6{x-7jU zxwXA9iT0nzRW0z~$!zIk$C1xtA}ekJNY`QA#O9X8=oIZi>I_3Ybi6!qNf>KKGRM@# zVhbreWj*bW*k2{xM;^XG#DX@zU(;H|E2fc$HZJqAxHhB-ye-R_J!Phx zvheRSMk})2(o`d6WD86r5Ixfmb6}3^&a?YC zhetdjt6B~AMw?_o)1RPkfi@nfD+=AIfA96ynflV;Yt%f~SV3>3D11_XuTo`RkY|OH zUotP2bjqgrgGN0?P(BUQv{*;wydtTV>9s6z{ey~NRS5XJ zQbjjFai9@AxWDWha=Wd+n6j^E&9as|F+-4PF2Yt8wC;sLeZ8;zpuZMQUoO}mqu}^C zIv|5oBYC4vhzz8kbA$erwa}>$+zmnqPLkk+Q^pBoyhT!>$fDoC zHAbs1pi0oZC!A9y)X;HK^RhJPT9j&^>f4wDwdl_PEWi}7ZHZ4{2MLlC8vhubrx*l$ zUwC@~MHWwMF#F?)r;NxfjX?+|Tu3HcsC~pWX}xpm11}uOLGJUmeaoqVg^|F z1Zmtuf~`rx&E3nd7;!U8vOlTB% z?h$ks$1HyVUW+0>&GZx&2xMRGHOF~mf^TlJQG*6C)xR>5h!%;O*rtMN>IhnmQ6kU;n0Z&x`mNe)J4KO8Xl;;oy(D6IwNx{4$%7gm3Yf38H zFx4>Sj0z<2aSK?;j2z=RCD3F>;4!PN2HbKgq@6-xxTa*?r`xJ9M4O2iFo6U&_D})+ zjS&YlIh3CA18}4gy%$t<>mmx!ABrWF9^Y6hdcTQXRsben7NlimJ||MVlmpyWf-bb} zE1~^>7isBZ2wV|mbj^0Tl{I`*5Yu#Jk9%lqU%oKKMX7?(kYffC!kHZ4#4x6A;Rew$ z^D@T`tSyPp>1BG>s5$44C>?8AkG8@vz%_D0DeRKUMOp9rhkaw5aqvY|J9sS=h^%W5 z`9aD+j9^~7b57K*6TzWLq+Cm~!9bWuyVisPI&imnD>%NTZ&$C~Dp zwS%lgoz+pG(_aG1GB_9@+Zxc`u{)AbeFo&yFb0Mwmz}a4P&|T!TAxc~)aH}A>;nsx zAemKAjUB-2PDsji&)`;!Fesa9mk*r_i;7sY(-qp>_COK22f_mgNDInUqBn)(Q?4`XxJ5G>mSK3pMPyv77dDy zTmH=z9SmS35&>F4>M0#g@{ECHG^!q86J}tBHsF8FCe6GPF~~s_(8K@;q6g4l>4TF2 z$_qa!Ja<)^h)4qfcG-Xc=vD8(8t32NJKm0gfgWCe^urMdw|}x9uUPNZCcVZG03dDs zUy9@Z$5vf={G0u^{`i0KNU=`}S~R?I+CTun0|Wq2{2dVp_!B!30t6u3U7cKgUBnQN z5&sNKQ`W1{3NTK>Tn&401z-OU_{TV<00Jzu?%w0nGcqy0Yo-P_0j&OG@Csv;SK;{htZ^e_HE*sDHID!T(|ZJ#)W5D@RQ7=Re8S N6Lgg{VDiuFe*lPvsYCz( diff --git a/artwork/scrapy-blog-logo.xcf b/artwork/scrapy-blog-logo.xcf deleted file mode 100644 index 320102604f4511d26094cd4c964ae767151d880c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 52428 zcmeFZ2XtK3)jxXgNS1BcR<&hGqcZBvy{K8eSDR{MTb3-@>dg%T1Y$@?LQ4bkg^&c2 z5C|b3ACM3rgc=ATKnTUgOd?wch&HdjGec;cCw8a?U>c z?6dbi`^-#pYv(b|%}1IvO|2b`f*^Qk1VKzgga7>Th$8+3;#pDfLqXt!e{1o0;!)!X zT4Xwd1&B^YL)}u+@iA~K%x;wg0 zXcCo|jVGEL>NQ(7ZBhPxNn`5*DhaF&U&6YG*L-z@tT&%LR)3_mt68&kaV!9D`7+=F zY~zhxt*11tT}N6PPm;g8sp(|nsU-o){huH=625@oqUNe3sr*hA-Z&1?A=M+@Umo}w6|3S^+(#9PjuhdrI?r(Kr~Ak&x(aM$yP43@dDA^G;FWBtiAfO_L|Gu zzG$7CmByBo(en%@U-E%70Mc8o>%a^kLMyD6P`KD*BehLo@6|kcq;H5z|)53Ry_AoAE*8P4}V}Kr!Rm0|M~pC zYfUd-8@B(ywmxg#6i)x2{CEurr!RYuE-=~nncG(i@8Pw2p-pEWztG->SEeWUC;k`y zFrw1uufF!kiTY!=XbL*IoAFjq@els}Z^npkqW{BqzAQ#CO+tDk?!&7)Jv!BR^Qny; z^|v&h07yxXb|n&~iM~wqXe1X+-CY1)-BEw?WNW*oy76f1jh!17UKjBjPqyCLh=k^{ zjD~Cq*qwtMYrVO#gXv34YZp@w%`HkYQ;#?G9j)~z7v6W&cQxOLyiF6`*tPM-lkC5t z{$%6Eqpc?zkDO}l?$UIeIsxFnCvM1wc5K<6v2EK@TC~()&ee9~$y2ROw`e*Vn@+LI z?wdDub{}m-+Ie#$GeH8r)_U|*i)P1`)WtN<4Q^>{ZEiWG*}8S-m2f2~TbvbDxbV;W zg1fO~4hX^Cv&bTTj^`;nFIN*anUZWWEk9Y4toQ-zEO<1YRv92-EAXzvsoX*4cS%LDx4Zj?Q0*5$3&H z-*w5$WDh|KcM|%d3i3L|b|WV1DYzcRi|bw?Qjo&_DtiMWh1&@_E`$rxHm$r$UM9Fs zD2h3ZbBMHM3`$;xNxeasbU+9JhY4iT)EF{YDO69mbHA`&kUZZ-dw%l4J>f|0q>LGb zqaG(2*ViG5({-O8@?q%)iEvjZ(mY8s#pzB|BiD5`LLApO;B;J1C^{%a>MXUpurRJ4 zuo*8=35_6-8VQviQIYKzguV@S4vq8)pm<-$6HkTARcM(9o&`Zg{%ItTE50@Gm)u7?TxEn)8vmJ65^<6^Uy zz*pHS$CiOzVIk&uKyv+wz&ZlIN$xPh9w+Q(!uArDNZ5UZJxW*}VOt10N!a^@We}E3 z*j0c@^Q$RN6@ifi#yS?>|}q2&_~%C=Dhz*{9Gg_u2RIA&B>UFqU9x-!LeL%4Tuo^WfKKnub8>*} z`VlngT=}pIE@R%Al=L|_hs499)1%EKua#XrRCPQyrSygkyGeO31^iVGqTymL7fJLR z64l5)vP;?Qw-NCO5!VrM2#7f$xad0<>6lW8tfzSMrU}eUy1;WRyv<5^uJ@(KfAkn8 zGoN=lded`)*WnlEVVqBT3$=7XhL+gvz^<>LkhV~xJ?IobgExj{jKbKOI;*d+k@yFu^;2S(vd z010$M;4@SYLiK>GbwdK6gkOWk>FjxjqXsR4Vf1if!{5(e1*hX$?*_yD#4q3Bs-=ENqGHJ8b0=7P$m_kgW{DqLyF}uvPjWd*w|*l;_6Yd$f8P{FzPM zcJm9pcF8{c*9Z5mfv<$M8uz0gXTp6WupsBmlUuMYusa?`Sz>PYKc@e-1<92&@=1wy z==!-I`d8w}D(LDXSm~V3Pw28EDoZ^Z;1J>Sr`N-UO6$4tq^~~*+m=SR!b0Kn?shxe zaEuuS`vhc!kSF8_S$a>$N% z0Hw1<+@oUK1Q?|;A8wgiQanhCtu*=hTQHO$Z6}567WT1fp3CtvDPln@>2@(<6>#aL z0g+vNOoD%sKv^3q10e|f`NTd)?0I5?Ih2BpBli2m{)*V)%dlA2cS(1K*q;(>xDtCi z2X?LD=6?7~(1q+e^Q*Ayh;8^%EKF2!wO?Pt>Mt7@*b5ZvAuRB`+D}l1)?G>P9tj=? z0lbfWg0hWVNuY$5r$K{FJh1JFSg;L37OW1#0#tUo? zu^X1Kf4dUfj-DlH)e`!xE73|~@F`70S%BBC#12!epD$tmav2sTxs3xiE@A(CCHA`< z*cHBne(^Fik|$3c`+k?f<8dnUg)7i-Zs_|MLd@waLhswcbjm%i^^VOtuMu`472-BW zD&U7u`nDbZfEx($)n}nU>R#LokB4H_ny++topp)iTA{%v`QJMh= z;;#G2{dagSB)_1nA?!WE?j|geu-|d>de0G9f$Yrn3Ax9YG36^@)9FK~l7$x#c$$w$PyM6-Wr%zDu-@*@^2h0*?~7 zgWTH)Q&=LHi478~A&i z8wtU=(@02}ut9M3q2>HsNW=?G0FWPoMq1ZQiYJ7Pf~%14ul_CHUx)Y7vT@S>&?tK# za;?JBTw(6wR$(UR{uj=h>@U#TJsu~qO=LFIz%^LM*3E2fgc+H|u>6KQAA9+uk+CUh zc4F}FPu)_qoJXB*Qe?sc=Dq&ZJ0trcN&S6=$V9A#n0(sCdL)X{r^*?Lxw5N-z+fC@ zrKflmfus?@ayKvpX-5!GI{OH$a05eS>62A7=4%P8bOY70>sN#^J9kyNVP3N1ldG~f zMQOHFfAb8FW4VmNd#<2>H&`x6d7mOn!uABZ`rpww`z`w&(Q!2Qs((i#;;P=J3%T)2 z(b!)JjJK{oekmFf|Akm3Zv02zz_tR*^cy<*GISb8X0Gz3SZ`tjzZAQg*^T#?qA)5m zTS*eX3az-cuM~MJC!{$4FOzsbu|N1KW&DiT-Cv4@CcPL8CCiGhz+=~e`@3Al{}>s| z6l7Mo{k{~>RaQcB4Xx-u*kL37e`fE$qBUNLw!fBETD50yL)YoY%pj5ewEW*`)+0Il zCJ*vgaITY`(z6>FIxg>TojgDLhx*lw6qkd8WvBHnCbCyRu@#03|6O1;mhR^NoTRh= z>m4m(%iPI3fBxPScf);Os>bCw2ww9d)6m8*U4AS>dN1rsnu(npHQB+Dk!n2NssKe(Z@ci8vlufqO-*tW029wheO zufnDi`?IgY?jZKRzY42V51#odtWs}y{Hw6nvge1t3foES?O%ocF|oB@g?*RUv@gXX zZCW!=P0#R}_&+s`b*dxxJ72L*2y4Ke{8B7z_$pwTv3+3z%dU(^ULOe)TaW+t1Ea$^ zY54Hzn|FC2iJJDzzw=M$#^z>VxX!@Sy*M~I4L^*;S=AYSp9PzHh2UI%hh6r?@&Ccq zIOaRO?wZC(*u2T-C=V_DT+hOH(g>#!qmVk(+Oc~x?xW1~r- zEYHkM{^eJ{dxxn(ybVEM#`cE??|K-gH=O_Uf!Jh{>P-yn9B*EU$w*$GVmPPn2yf%1 znBOpJJk6chM$e;*;fA`IQ-7D&?b5+Hv|U+%N^2|(xXx{C2k|T;o%QARJV1)K$afrF z!2AHai|Dxx-1bU6Y$G?Cl6|I;eT5Zxcv#H9Uuix-HE*H>qbl}vox&eVcXTgFhloGM z?v;f88N!~S|53`lTe$vD68qom-BYPV?xol2Az=s69}xW*`_5qBPJf~wBl-!VjYR*J zXyw@dG4lbXo~WD#rigu>ScT(JV)NO}l*rsosVICzv{DAViRegbC6rhPnQg=xIov+3 zua^>Uw-T)YW4JON^czS3Py`ugtAyx}*lOhTSu;Xv1MDDSdx?FDtv?W}RQIBYJxkaj zVp$rLoNCTb(Nh@F%E9ai(OcNzgTyLlmk?r=v%*neCGn!dy`7YsN%;kL4S7xiCh23C zd?pdCjLkw3ayg#sbpSf8M?INJMw37ho%V`}LEwY1@VU|7+}RmGWHtw%BvLVzM^QPF zMDA`>-l5GEq*+IU^!fv~lqBeh>Lf}zIE=&Sq-76dgGhpMzW+W^N%da}(~ST~7Li8r9C6oUy(G^kVDVG;bwB*#yYK&1$fgV_+=(7zSW?NQ zY>?&8fkyEn(#lzK-bAycak$>5yenTJ{x&YZ5}ZFGT1g?AxaQuk11)`~Li_ZxJDTzE zRXDA>Mwl~v`pecZNaP`pyZ(BO=o6K)vtZAm_FKMl7jz17&pNPqowz>aJq&K+@G{>Z z2(!p(XYs)VpN=uqiKLkTr2UfmBFmuP=fZ~Zd@*}lWnD{~vJi(knf{fEqv)lrc2i@g z*gC_OavDL-adu%CZj5pX)&LYpSbq}#WWa}li znv{V@*<`s3ew=;?MGV39I)M+c)lHz32rW8SeutMWNOF(`oPKt+td8!YfVD{k<5 z^EFsmSf`~NyzqRkr$RBDhwL1EWHVgK+#i3^jWYGs!s*p-Hsj$dYlJmi;G1Z?O1)l! z+e4;M6bgfp{QHPZq~X4fmZS^E5cA7NjC*MiD+t3m7EbOuy4`^dT<;nbQl0OVaVk>G zM|8A@Xc6MaT(+s(Rw>dj=D7<2{IC||54G!2K*CtfTQ2vd4T!7 z7X|iy3g!T_2>BP{i_>{ph3zL@H;8^wlV5qqB#P|Po7+)$onKb_8gDt42~NM`AEO_q z?KfGdQ969{kN@iL=X8$wQOs#9{O}L|e8%i>Iw!U7KYeQfjCtjgSEv_zj^#ZZGO@$H z!&VzBrtwR#R<@obENKarjg~YWf+ceXbbX)dQjY5ELt3faW4%cwI=q8ZuDkPXr##_7 z$F#-Gj}f1Q;LdYbFT-|m`t1EJ*C>$`+ZQRM4ZInfcYN}h1ygqg5SdEk*XfQP?muFxJ9?;RTbES(n(Qmz4drA82VVf=(gwIg*}%Q7dsq zdW;{?lWOJ(y@zUgtHykGXW1jHl}g(!KLsD^od32`nBnB5m*1l2dv`NdY^M5^iHU#RPuGEwkt{tdl>Cv? za?bs!g0-3jZHa>2r~KQGzxei_etC2LYF1np5GX4A@3oZ6j)z&eSjPoYT7gR~zonLP zd81&Ji=T&406{rf@s4Qquyq??Q`?^%mF)em+_qLY?b&yQ3BekuCd63$d0yDeyloW| z@6B*-$~Eq+udqdMy$QQSAz-WE>SU`HEh$*J)&_o?;QAd`;RRli$4*nc8h6zNHb1q7 z&F@i`rzzh**}4iaDVRkiX?!{HN;T~l#Q%Z#G~)edDbz;3%|=CUjD%|74StcV*J*^Mnq2Q&hjd5PU5*+LU*0c|m?f<74B5Ji{u~ zL(DZs0@*yr++sRa*hKBZ$If3xMcoTUNK?C^9jEwB<;3w3KVaa9y#3X|srkYGu2vyt zZ@WU<*szMxC}+*}TI%dsZjJvq4k9!JeMpN>`%*L#l*B%wpQ8OL+`AVo8LNO0=ySIc zGRD+P1l@}S_P(y&>GA5=th(P%atf|J+}=tjuefQHjfg0se72&4pRR@2h$N>OozuAf zzTEr4GKyT&hZswI=?7ejpH=E%?<6vxc@{ur;|IykvwN9ENwGX3xa{sO1Lb+sW4R(- zW$Fl*#ZUl*!83PX_ji8!^k3fltWQ5SEzOSgyz#TPZ6Xd!)85DbHLZK&m>*o>->%Rs zLR&M{nL%j+nOjRXUykxeF6fh(q2$k43C9hS|2|vy6U~~LG<_{Kpqxk=8I{v{`6Stt z4+@;Z9Ogebl}1-4MbfsVS>_W`jFM~K$n`0Lsmz-`dF2UKQtzY_ng7}{(pQ^E{||n? zSn{}tn2fX4vcIFpY10E$n>5+opMY4hYA9aL93nJar@Ei#IC}=8_8tcu7W&cjGQ3!G z5%Q$OH!&Jg#DJF=Ts}=i8saQ`jvtJGP_=pt;&gnAab_;C=Fzj9367l08~*_tcIxjA zMkt;3N-R6_wYwhpC5D)YJpAw*eG~KZV;BB-XRfDkTEvGZwy1?nVb0^WxnX}H9iK9J z7a0qDa+)BOTX^ezkB}-zr(MWV@aY8XsTHi%O7D4bPvKKOX#aT)+v`4I8=nI~d;J-< z9}(GJ^-s1RRoWhZWc!aE+|b`4zR1AJGzdGd6wE=c|B{@Or51W zLlMYl{I(zZ-dhftP%RYfoF3zKLozSW0{C*j@*??PL^_-zIQ@SQ7lqRvNVYv2n(^>e$QP!6bh>kY(dLk4upfjA*WlB~QXmMF`q5#Q zbkNzS8xi_^5P7R{`tgNt*a>^^Q_^@*n0xMxPYo{m)&roPU$&v)C5VabkEJi2!8~vl z#Fp!(5eE?{Du<>`1o9FaH#KM}pIvaRVl$e$=;4lJYJu}LTFO<-ZeXQl2I^9;e11mB z*}p+(sEyDup3fz(cgfbJ4D|~%r4N{1N*<50JsHck-*aqBSF@e1oGy>ix5PcA98+of z(ld(Vn>z{;1bl4{x`RbO!}?jDU}%T<39}FIWUzsM z{CY;1(h?9-a8g-lPvZ4eJR+Wz#rVp?sT(`$iThvtsH6BC`me$BWgm6K#~c^`ZKKfq zw}Pk@cOCDP)8v`g zI-8HT-*i&mDNl|6sHN-Jjkg|amUqaGk8eGCBA8$F)*45S2DQ}S{{(ijf#uHr~CmP$j+d7ZSo8|Fm>f1X`b|3HTXg}W7 zd8|d=C|d`=ex&L6iO%-!me$s`rk0kY@&TK@p>~2;y&*L+p9de>NcI<2IXsA7euk|#`@v?sWp@zm&-6y-6+uK{(4u9*z$;rRnw70GMh#V*DKIv>b zcC59nqpho{rK`EKM;@DzUu{0rbXbm+wa<5)=<4e1?C5B3?`Y{f{D3@ev(C&Osyoss zN6QY=ef5xo_;#^~i;c!cx93>CEdsF9$=5{xKTU%pi=fUptj=?GU zgPQ#fEpnth;C!^D>v(tPVGhvN*4omr_9c*m8P7aqX?>4nkjF#48?4+foxw-DBc+}q3baekU`|1zx zuRV+qZJ+!_L)X_jfQCX9bq+cb;m3(8oHC zG`f3hYC77`yub12!9$I;O$QGj*pKLLe*IK?cWb9xZVuUmmBXl0M;aOqAFe%gsOF&R zATq}FvTfogb*(oZZ*Np+QLGo6#U>YNoktq#>%=;#_TVA$(0t9o8?MJKn&Hz^hZ~M{ zb#!+uPDbhZ=n?UVtG-rMJ5_V=(4m9-_g%Xif$#JGIoff&<7msj(?W=P$WVKzrsmLr zz1Lp7y9yEUocV{1UHHybcWV<2a8cx=M@UR+(AFHd;rgqqtHi26dEou-!^dxeAovV! zN7KUm7Kk~)gW|#212^ow`YMtmAM%$co@{`no@i}vg*;d$v1zXHC^^K2iTcBb>*}DB z1N--0bJgxDRh3jxj(GgB@y-J$PIPr1X=-ZkYH4nBYjJ55g#{eDE32z2E6dBvkOx@a z?rQ1kJlS!)_3)9#Bh80fnj4QEQ5_K*q$BmUVy$>+7L&M6k!q%TPeo}(bp^5lljB#1 zI@&Os=Jq3X_dfo^x}yz8z$YG=!xCWfszYMU{PovCQL3t$@{00`s z?WGB;&FZ|+&{BW6uC}fY6Pv2pzxUcbRTY($(6_2`rhIo%X?0p2f}(b~ySBUiSX*l| zeM)W3$M%sav+J{KTfwTTbzo8VU9+dEs+vMNDk~}~iz@QA)%_I#OgsBZeM@I+M@M64 z)6t`?ZAWVEH^S9P_Z_Uq+$L*iW>@dVh*g!)3wg^cOSYH)ng^*K*Be!xEo~jG?NDk{ zBR<^ST=VUZKR^G-zM30gNr!6oUw=)tN3|1*tW;Hs6=L~(Uhci4$d^vjHJj<0KkA}u zex^E~uG#p_?2-z)=C@mN%SsC9njfpoE6yn|rE4CzExWLwthy+duK6!Dx#cCrC51&5 zbj`M36%>|~7F8A%SLD()UpSRjQdE*#Sy)<{S4`LZ%D!FYB_(C$S%sy!`9*ZiKPV_I zE-x-EDK03^%SE4X&7ZYoWtWu~6_n%`6cmCi7p{5w57l|a)w#um#bWVHVSZs@Nq!D} zv+@4y+>*Tf{L-SLoPzwKlA`>ALilFWM~&M`D+)_WO7lyLi+APb7v^W@7tlBVx-6@z zxUjUOw4k(XXIXVgPG(+y#xDBip_>cy%L~hj3(HFL3bI>YH5vYRU~6tcCVlhU2eOLM zQE6dues2D*iuYYp^RC|&?a0oOW8@LnkMpuhN{Wk%iXc{ALB`1m^VICTwJ2*>7M=6O zW4Wcp3cgS*9M3P#O+8^5v)QJ#C0Ti?bk5ITTToh1N$!Gz{QT_Vl8hTZni;jvznr}z zBbUy3?4FW>{KE1)H$h%rQAT!t`mO(*H2>|$&g>mobk1j+in7ZJ3Mz`(MP6QRZhl^N zQO5SGj(1mWOV7{Q4CmbUTxE7)X<2!GPLZ1=2Xo(*k+UmfOKL`L);4HCEl=vdn^#g* z1dSFI=jVxe=G>f|?CiqK{H%=BtX(--sacs@v*Dbb@3-v2bZq6t`Nc)K1soBCV)n$Y z!kzi~nY%K!Wo=E{nTo2N$@N(IuA;)~#rb_fnzd_J)~<~7^t9AeJoL}~cjV@kV+|GB zd@*0l6LVecb|!0A=B{0t8R@F@`Hb`(Tj`zuwl6=oytuHWcu_5i2-#U;mMT+~;gLR{ zx_$Eod~EQn^Cu+*<;8hiyG8AAQdt;36J2IxW@MzNZQs0cLo!^m_R^`Gy!_mfisD=_ zD5_i{7BlT22Yvd^Z5vaP#pFPF?&WJU3X7o$OMXsaQSQRz7W)<<#%x;Zj*S~ql2yt6 z^7zCLuzISR*Ik{jHv*vbXre|hSGVpHS zws`|)GLw*)keD1FN5A|*<4!0FwppB2`m1r5^UsyLva+*waYE^7Sg)Ckw4K{GLR}uo zGn*6QlM<5>;g|JKm8X`K7Z>Cf6qjb-ZkwK(k$#e{tcaL7m5~Njq^EA(l)OB7K0aZ? zhIla^e%W+uI=o9>K~W);SMY|zHsz2%DoBTEa86>zR9fn`%^Q;wlaeqsRnknthS-En zYJQ+n`~J0AWqD>J@aSnYE48`e59J5`RRh__?YMz zO-wR9v+n+QO;lWLN=#gG6g~6%HDNJPF_Fo!@$q3%^vqA^gv3V0BqoJL$A*Qe>6uT) z#6(1eM@7a*M~A7SBI%jm-5(wfQcu|^SOqEs02-1TugF8cx2=?&kPRzwm2v}Dh!T!?B&Yf=!Ce$ z^vRr7ZZb#qcoc6a7|d9aZH}KUyKV{A3@Lj!NHh> zXa%nkHRBP{k)id5DcLo5aZ_k`Fg^35J0s)bw#CM}`v{GWizxe0M(*)^Qb?;k(!`LrprruKSV#L&KIt zr;Z{cBO=1oVPcpAFX~XWnrj2U{8?AL21KVuYj)Q;X{P@UT#IxESsX z3+1QJ46kNHM<&L_B_}RU+1)cJCe)#!Ft5;XaD=S$hj;Fq`Szx;xL8emd`#q`2owVp zpbJrlg@J8Wy?$Mwzdzjb;77HQVUZE>8{;A(Bmb@YaFB~?M?`oSRO06g`T%+4>FiMG zJ0>nVJW><QY4jnDijQB zF#tJSe(=#}{&8J+oF+0QUK6g-sH0OhEzCE3VH!|UC|LY`{Cs`kH zsK}%nPSiwcpoR!&Xf8}0f&~-R;Da850{vC~Gd{jPzW!@g(@Vcn85kKC6BiQ}soDOZ zNuK@PW;KLVCIvQ?dL~>QvVQH_0DlkvnYF7|`G6H(dg8w}252G^VxuC$V-t=~Vsp*R zeJ2e1=a7ytEG5=)eW0(um;cPF)&BmgR;@w3$U4{>9uuPpi;j;A3y<9Jg6smL>Aj=? zNE*g{;E{4hI}-^}X3Rem8}est6??9wDe#6`!2$HYZwLgOBnt+NyIyJ}y^ zs*0EcZ^&BkhykAd4lID*s-S>XIZx0}_dOIA9TNfXk*J0Pi4G6HRxgjwj|6~a&8K16~Gf-4YNBJ!k#ipIO+TD)& z+csPwZAJNQ3rcUBP;SHL2~gk*M9puloFuH16NL40ybvtoPUmUt%@{!~M+;%HMu?Cj zg-98nJC26C2@|4awGboYqVaCHw_qV&UN0ob>jc+_a)P{8K>i}6$o|4c*-zLcqqf~7 zZ<5ytTjkZlHd(30Z9|=PC(GT87hXad9(1(>*Quvf0kYk8;g#S1>cx?6UR_t03}YZAnxjFEch-zBry6&OSh`qqs!RX)^THZXKN!4bdzrJM$y@~iXT$0l;j{|@z+ z+q=6_s!(<6ySolIwKcWvYiMg~Y;I_7XgJVZUzgSF;^|R;|KZkSUpvtyqD!$u*M8%{ z+uxs+tq;{5X=-RbjI(+}P3_TrseeFW`}2=(K63K-iH>IUx==;=WZE_>&wYP?{b8Os zYib(m>aQ(3I1W2M`$SvkiIb;}HFqdA8BC?&yQ5ai_{8kF{k1r$*Vi1rzNxnEhHDF- zA$9$+6DLocKxu&$9uT+IJ#5pO$EKzYH#8hYmFnQ3hJ7`M_EqP1fcng>$2yPmgxyU! z+E|6TbKGK^b2#6=s{YWC!!`R49;`cf;6Qa%4nMy$+d(~ac8DEw?vjnztgUZ)cHS{5 z>rOTtsI5csqNe7+{u{2{Q(5pDi@e96jyYwHSjqvVdeL@6>u)|f_u`4#12uJ^K8Wu3 z??bU5_m^Ne)4fo+QeWx zvEVRSTaS5C@Iwc0fH3oi_Ux`ISCu=VW>>fxTDn zE~~DDsO2TQOUueiE6Vb>-hKhMFRrF~v3>yMyMy~#>$@IBn5#iiRaU&axO`7ZS;4M@ z@AFcu?@)a`h#()!EsC0o%d5)vT#M>R`JUX8Uryum6!W!*Q3FyPFdSq7sSp@A-rNoQ_U@^!5G%%^y^> zVuh+)ESoB?tg0-_O|O0q&USJi3}>N2RiS99B(J)(xNJw!-F9Si`U3~|Uw0MiNDu(! zwenI-qPjTqG5$UJt>OED}yTMgadR1A~p3;{6r+xgzw^`P%7VNsoaT3BrWLk7U{pfeXAd68%gNy}vA8%lx40-XHS^lX z5Zdgoo-EBRDlaLmtST!Ki^RgI0u&wcv$nL4q4YcC`k$7<+^Xu*3e>?$U}JejC55O2 zWanhWH#m{B+Fn0goQwKUK{>F+`Ko+dQ7(%NI2Yz*MW0}F>-l%xO-u%>zYpOFXJ2NX| zS9WH`uGGv;iP?IFs~;2;pvYHN~_}bjC4hmtWbHmBXS>_O|@A^zB=s9tX9Q zS&W%qDk>~N)ki5_?YMK?JUTihf0UJ*gBn_9X7;wU^c^WtwV*yzURGRz5?!IYM1-R-<-Ix^pG@Wr-iC8kH)ba|%0w)VVVy;sVKYb9UvWrKV+MZ%@n2O5d59x()O@QE*8Ly9wlH$`=ZKd2ZsI?7VCU zo|dxFmH>AX*#GD8DOp&6%oNSfSzbaFg z4pGuTy@O&WB_}3pk~YO{j@Zp#&6q7zieV^HiMzy17t3clRAiv?ojbN+c`(c5#DvWo zHHnGg{{`c5737EuQe=vmb1XfkrHZLof09EhsKz)FlanLjlM^%O5jF>5A zxY8lKxD(d0W9!D`WY%t2@`;b$oSdSGRDLU?kXzARn-ep}**_GyZOf*V#Q0>Y24$&~ zgqY;8=x@Al;Ir?i6>Lwr<2U6BAHUN=!&dOp4!-99;M;H*2oDS}CSOgjCl~ ztc|p3Ly~8b4V5^|GCn?K{f6(D8Q542W2IEwsY_#hYC~LdN^;7E6jW|drAmnpkLdoC zku4ia7+|?E6~(sgTQ_e=+PonlaYJIV)7!;36UL9v-OGWulvH1*?R}v6;|^uBU>uQ1t>WoB^rOAvAEdK!rwEpp}!Pz zB_?c%PsZHhLR0S2F|s9z8^okZFvZ}co){I=`Vk`=7TuPVfI&7!$4A9#3ZG|WgN9KO zrNyD%wfRSkY#0rCOpJ?$d~vbKVb{LS$cFdviHXs%n}0dZ$d0kIw#bz48yMMYKff%p^}qV?H%>J+c6{UU4;k4E&)v0W zOI&!!+Q4-oYIVwWw+%3|Ip4Z9J6s*6(S)lbB7=i9QR@TO=RbhRru(0RaqC0kQYohRA`M(32eW-CQcn6uStxLiA{v<#3m-kMQb8aD_C{a468A(m&T(`6dt`X zIyN>s0`(%)jlwZd#Jb?HmGum4AJ&D(s$xB2j3`&eYBXWNQQ_ecu}Q(|*wDyO&u=lX z{jp3F6)VQ7V#H`M$`qlFj#?WNA0CVnKxBv}LLISI{SxDv_3rHnp>awvP8F?RuZi50 zx@lXaCR&X`O-yi1a73`iyV$@}+k5qKQCOQ8QRz{qi3qvox&D!lZ;FYE2#$%2R7Yuo z)$L#F* zjQmDyWN1W`MvWR_s5;nh1;2xI24xSFR^y-xl-Z(Ci}Sx4H~3>#>CuQtlmH{bLqfu# z-L-*n{Lsy8yjxx|YEJ1oQjFB9xBbK^&&+?eRTCT*j>QN?r79#Sz-OiMnf+Mk!%Z6{ zMh&=YeIeQRzV_BncZUUsqCyfH0*gVN#NWs38yMkCq+8k1Q41B+$e?g6byUpy(C|og za7f5H^dGW5(9g%C8RQoh6d0v(%Y$N1=-TzDw5ru8$OVU~*JFU-wWz^Jp8R5}Hg2)o zg~DTaj5-LFJJia;RADys971wGU%zGblqT6d-!Hn4iH!;kULO{PDqC1+(0X++ihisK zDde+ZP$IM_YL;r1li`tJA?gTJrNbe;7&0Hu5|ql%8L*6BHgT;-QPoXMu1GOrBs?-S z4CSk zFu%8AieD^iDy2A!HclxbD)lEZ+{H4h(>;bo z9U6p!;F>jlsO_x{Shi-3>VDXdDioKxziV45W@>Y*yNIduIxKR=%}D0lh#`+9i$ ztX{6U0Qy;9MIMwIQQ``7sY5+OP3Sqm-LpSBU%A%TZ-w{$Jh4SUB};YCab>ossIkhr z#%GPM=ahvHbAxQNrts#g%GoY9BNl6 z*JW)0XjFa@ikG7QG?sLo7<7W)Mw`{ZsIdhgaHtL~)B;!fU|q|?-&@_s2eeK8ny8C>--hTY-{~}*Q{9SztVU0nj`#CpCN1>Dm)|v2$&#> zjJ~T@tnl++5x6qvB}BI2$WTho%|g6ifPZixmc-Y0RqT%#*tF}_>JZdKS(Q}Kem?8H zSNM8Hw+}O}1&4(K=i^Ti{Q?90VJ3bnJS+apxE7`k@B`f6$A1M->jDE-`>afR0&z`T zYw>5n5=I`f#y@a%ppRGR*ZL6+ELep=>=kIm$JZMM;ZyrA1Dl`dZ}Y?I_^(^J%4_AS zoxfpV^T#If@x>ynS+#n(`Yr}GF#7lfVD8@DzIA69*!-1MUA@YGMbV!a*zg|HT)8Uf zK|KST|9aomt385l8e(7zSRJ&&tKuaFHg6ws(=Qm)hd`R}HF|#* zKc7ad!H+yNK^OSpoVzh?{19585H1G@8q_+YQRH-4A;>CJM7{B?{S`uz^57c|UZ{#L z|G)jX*;b&7l_;!wqq|i?9LlFLsHR5Cz8K9PCD}lfX4eU!{MO1kAp~{U^^k5I%Cl=B zYyc{@esUBFy%;?THQ!j&f8#No1k{6*P!~=?jd&wIaJUIIWAuO{Dn7~ihU}V^Gv#^X z$nY?4^|T|yBRYdY-j1&;>vVdZsGHXtj3(U6*=NTL`jKJ1#b`8{Rpw3doPKz4SZ5TC zQ|2)f4o()c1s82o&XIw^5xv=DG>hh0i`8TSMzkc#;-F(#tJ50{qd<+SMqOr$rzK9F z85`~+*r+m2S>SWy(4+YBrfIqgI%YT{k$S zH|UIJgNbs9mT9xaY8|y$MlH)MVe*6vLJw<=MuW*>T%eG^DvpX47fC|#NxqR0oldJa zm@Sajyu$1vngW=YnSO~m3cdzt5P zIAkv_i)ooDP@Wna85|g-CP62fW}qv~zzRaqW$zxvRF@k!L2N}>nPydisZ^S?@qe(Ow ztQK|)(fAi#Sj`mIWb`yTF|Ntegiiq>VHxSW*ss;=bifd88dD^)kXm5_zrtoT!490} z(J?4>xojJ}&@(tZJfwqoz^hF9aq@9OFfk4X&2hRE(qXoa;~nv`r+miPdtsojZ_qs` z2!I$jV|k2bEHv~2(gkJ%m}fC0nQ4UhVE22@^Pl(k_6-h?XmvXFXoP`|nZRx^j#^Cy zMa7Hkz~g^K6xjGOdAjFZZ*SiK)Gx@X)r=3S8|I6(Kti1;qWS>mu$Y! z)6+jVg!Rtee3A$&3~w$uR9@vU%)$Pj7$!z|hEu zmZT6{VHQpOMy#m9ZNcjTH#f~Fxv4ZSd1CO=#oqqD!I7a6Y&R1YS;E428YQrb#tHOi zzKmBi&YG=CkD_sfJlcPu2MQk=92(J^Oa{GP;lzGYlmj#8;1E{X>2N4|MQ8(Z0B!&I z-oCzpfx#iI(WKXr5ky7*?+5hn%_*Ix3yP2&V_l zLp`JL20FdPpcnO`!LIla{DHb~I5GB|>j&eYUN<~~xxjN{GEj`lIBh{ge{$aIg#mnS9vG?7 zK$nBlS>MHT7yA0K#fJ2HY%^?B&QH;`!36mbGd%QjbWR4HO6Mm}&i0+Z)Qg=lpwo?D zYpL|2$)>E0PH%>YpcnOax-&=#nYBods2_w1tnjH=BFxgH1&7W&0$HZDSUoLP0uv3C`}@!K4E79y zes~yRhK2&4Vb*Ce0H;l=DbOQ?VL0R)0~gNHJ?IC9xJ#8;>#&kqvjO^Eq&Iy*KRkQ> zTz}ty9$N$>XbsR3LWezHpT9wv5 zf}R&UM}9bb@j{j>qb8B)%ViDFfSmj)}qrIF#e(# zG;7kXmkno0+5>MgtV3D>L$v5epaLxFG?fImO5;-!lo=){hT)+hCMd&nAy5<&lo>77 zO&pnL`oIKb#)@q)gmu88sEkNZ5V(i5%HFoZ?imM^1Z5ce4D#vWK4xKt+|{B93Ce(D z2nvU70flIb21}COE&HRWIMla($_YJctlVO}0t2 zLLZi31VT^}B(Qvim)65j!3KW9@dqa2L}YGK}&B0~^I=HLeuFJ>53TmpPN#=D$b z)QS2TnvI(sU;R4I6k$Ldm>k0PH*&Xg`l4QC7=YTqZNSlhNfQ2C93b%;hi#1i)O?X{k=e9v4rg{y`PX;a^1oRFJGsyt6sFScXo_Yza zqJ9GXF@ad*#cnoQlpaMrQUyehe(*9%8Q_Ps3MZ^tQ4eAU2ZywfNJMZOMUsoRI_4(~ z-93E_-te3wWK`Tb>>Q-ndpfvHH!rQ#I7)qCX3S4!&VSz1hd7Kqic^Zh>Y-B^$FNdH zvle`Mb_O@c7=ieLd4du7iSGPGde2@&E_eqHsnYdh6n1Phj~Ee87C7Nj>Ep4?{3)EV zzVjC@^nwo#cUboYAL1Kg>t%chkQ5zfLGU3z>HYjt4-*aD$Pir>!hn+c(s~eDO(Te9 z2)ZJp4W@@bP#5!)OP}{$ggt3Ta7Z(Pjgx_5RQhQ|Oq2FXHs}o#XMSQmcj4ltOT9xl zHEF?y_>ay_qRxRSGWGxnh>Kt(CPwJTi7>41Po}7n*`v-LW14vMYm06EqA%~5+VJK&T-tYze@XWc-`+ECzNE$H$;w}vx z@x!2pAwe9wBF;khNKdrqE?w-?!PYncQ-k3VgC1*%Z8@wOwhy7_#m<%VB-q1T@M`$mYSKu$gpW-7>P4iND+fpONJ%!0^`DhDe1`^(-V4crYAU^kB&`DFg=+=tQ`}_=Eo;&lT1(M?4vm4 zSSDqPulFFebzBO=>SG_AU%Qcn?|i; zV^cs)sis_Zho=MSiQS~dxq58eCZd}Oo84iz*&N_?f|2-dD-0degsFbL2Tz`o3m6a^WXX%anhM0C4A;W6c9Ung5!x)GxVoY24| zk$CK0_W4P>G6uw5Hi-;nk{gSf#1zEI8R!ZVmJGi8HXZNs6jw$F@w8Mvu^y%zlW9S!Jb6|+edF+ckHc&6}AVV1$95GtQ zZ5YU9n-nJv(++lvx$-Z%n8xCAC{LRc<4$@`;x`wNp%{k;b=L8*31EoE6!?mTv|&%4XXb$!OwUqJz%xS`9qL2a zHDS;(LW!8QV|i@$Y1pSSU#ODI6JVajkR%gr_$>|8Po&;U4DGCfjj~7E#Dr}I#*6=_ zrojQW$whYH@jn-4(x$R4lV{*U=~q$jnHyJl#R+M0(i_i(Dh3!pNj|r81jBaLX9DFu*HqL!9g_%N^ zWJG#m?i*5aT+{f3)y=1x=*K8jnSE-^ZeQY@7>5e+2byMjGBPkSf~?;(2EGZ7=jL;e zZ~8Jm+XS7>#28nI>B&$JTs{)`iLp_-sj&&{XD%<)GX-ZbHtv{!0~9ChiVMLXsEg^z zU?1Fy-ZVaD8O2=SxG@JSs^u*4@OJ^7zvzW)hHm-E;5TQy;4TAtk ze@n0v>jV8b$0sJ4o|rB$D`#z7XSKmcVMWF;k#SfdD4+-%T!sfi0+$3x8}gHxAsmm8 z7@=nCoEWzvri_k{O=9+w&y;Y7#7?f5vN1~w`us3?145&|M( zc#O}{IZceI#*m=Q4fi7J$1r1KqgK(X8W-UdNk2Ajhm4>X$L(}y5EDXMnV^g^+eX4? zRapAR5Je`TH;6W7AGbOXtT;g|23f45Oi&CL25=TM8mtCX$6*{W8Ys{1%4#1)&x@TSLDBXMXsy&LPQcKD6*GVmZIEk;80vvoN`lfmVnsg3GvXMG z10mHhW}U$Ji(=5ONs9!fhooB8-Nq)yp%{x}d=x6cqE1su_@XDR4+#oBq>mqwzz6SD za+>>xG4LZJ(fs|KErRr+6}tyN;!^~RN#=c50IrL0-?a|6M^dyE14WTXE#ue5qme7% zH_PwF?Odo3i*En`62~tA+>I{*;76jZ$X@UZU?0jUxUfQ38+kkSAu=EQ_C~b46*pTL zW;^c3@M~clBNad1l7Sy}0ckp4qVa+s&S^p#1-upb<@lli$?-EjMfkD+I`|MBcndqw z<#y%E1^Bs|B7BQXxrW<}yRt19db9E^1pJiQhq&V7ml6Vm4ayf3_(g@a1Sdl3B;4pF zDqmyZ*BQPp&%0)gIt=2RMH(p%&$Gl0*?uoX7|94T>>zKFr(|M{VAevH0R?@vu5!q0sK6i`6oooM0x( zC6>pihgBT)#1jms!TcV_F_m&x3z##yH4m;Hb*#HR>hX zT^t7oDcq4@hLgNyvtY|H;T;WtxLy{OY;|c-r^h)7CX8`7lxS92W*~^tS-_~1O|Q`j z98egRP>Db|R_@yn+n_Qi83V2xok!OsbmCBoT8)I#l7|IqC8?K1g5Jb2+~0Xve3nSZ zl?oT#grWqcx6x%7oFZDy7B9>4(K%poJGf#LB|Hc==O)EvAT$k9d=~;}v&Gko+dr5)Zz~^(X!mUKnVC{gmZkut3;9ahGri# ztleT>Wu9S?l$Vk!vqS0C-S_eZDqg*!ylOHp$Fw}mHid*npbQDgp(FDevt;7MF)N|o zkaiizRYY>XD3*?u_Q5`4Arn`rbG*zP<@Q1C)GZZy6)h5IRpxo{S;iw8P^7$$`d3dO}MvL9UKEmC5KDZ86fH2y_V=SgC0i_fnWXI9Hhc*f%z) zXq;1oLNOJcL4FpOag}WHf+PVZbVV9~lo3$~Q!ZPT5;W(9>t-_f8F^97iY$aznLSL3 zR})!iMY)wlW(@=!xZ|LYf{R-0 z_zPF>Sj- zO;3gqeLxH+WOT8DtTLfI&igUk2=LO8_h@dhOh#W2Qq)=Y3$zGY&d`GQAY~!jjUdAs zg^W|;6!B+XnV**d=0RFq+p?&N-cg~2B25Ng!xTs3#UANBjLQuZD2n49)xz68RGi(t z_@JhXDS*_A1$g{L0)g`uhi4fblV;_TzSyt^sSYp3K&eJUS z3-`>T>2e4LAnz3?ak&Y8RG%Or{Vm!m{R~E6(Get07D81hRO21x0NzVar2;pKG=!ye zR&$m4Da=qK?=+O|MZG7=_M)B_4+i}TyK#DFS=xxsEK*8_A}G8qh0jD&9_#5O`D$MLn)Y z{*4WzfKAldpiNGTFQDgb{5(l@o?w#{kvw$#RKq1H-0_R1IX4$R4M0iyS}$7(pALNP!wb61YfG#Inde=Dyxz&Yk<7o!!~#=lh#o ziK(sS%scP%JKx{?JkRerzQ1j?BYOVCQMfnD3#B_~f1S$rL z#8ySD7@J$uqF2Z5`~rVFIgEP4Oq{&{PpxlOEnVYBh_fT6od6hjC`Kiu*WUps<6`Pm6!oW$f=1E03oBUAoZI$ z*=W^m9x}Z`7okSGqS4!xBa-`uZd(okTtcvffV{9q@Jqby#$V~{;)m1?W9O0BH4`yW zC#Ds+MK3Xk&TPzl0 zMyk)0=k{(Z{%ZI3mhq?>dAo>Q-v`NpWV<@6P%gTo5mTZxOQriix<5lD=pxiD5vH39 z6R91LzEIk&qfejdrY3RPPe<`#pq1T}b9ml#fNiyr_GD;3t*lt6frX%3hIKqjw3y0G^99B7uCWlO&1Xls2%)x4-t}GL~_d;^&|)l zt5mXI?=~gpN=s5V;K~SK(V4sCT*bWY9~y(%qH> zw@ZPdak3*Ok%30?n&2zAy!4~OAq2E|x6z(pBHo4n6%NG< z51ZPn_#8C=sNfq1D9}>@dEPX#S=57F;xbdy zG1SsKXmL@3s>BE-y6oZ{U1J(*l#?1Ffo{50EF6STT}2`qhH62p5faIT#~T()b>@Z& z957@Wk|Gl(GLH*n9${C|;$)|Bqb>yUQepB3^F6gDC#>iLh1q;_BCXXvtgwW*MsiZi zVjcqlZRV&Ct&V>goYV-_$hWuO#8r84xUm@QYb2)U0qw{m^?Z~s9tTA`Sg zZ84h~mOh}D#;ATh7bj^ihE{yn9x{T@>qWnQI zBPTDCRgT!*BC;z{!(5gy4Ye*eGp+bN4V2IU{Hp<puHqqX@U8kZTvxX42P^Y$$_VfhpiBHq^Os*>{x1>D(;I%c8vbqh8mEG|fvb3Qf z`vCQdYMEM=PypR{i>@05Ol7Hn2dNTPHo)yI6OUmPQJz(b$PFz8Z5q^ttvu?DDL^5y z6mHJ)M)7dB<)SySV)Sc<2?*lymCM^r$8m;JlgxZtGB&rSMX#RQ*#-V~WIfyT)3F3y zv+{kjZt0w7dr*5LrkD*9V?%EodUQI=%vDYAH7PWcikrdMG)&k+)?e^96~N1n0pId8 zgOq4`Q1Z?5h$o3Bj50Z_Zb+HLv|%IxLCjJnWqz`1+nCmI5?U>k$&N8WOuUAh-NELd z-Ocf z4&P3oYjuMBp-f`Zmw6qxp&YRLdRBi(wf)WhbRXz#lp*-LoER#OTLic6LT{&!UwG6X zK!Y}^Jtq7Wmlobe`M*2eK=;6hM7`-AR!MdZuwHE7^pOAe1_QOyJKVGTGd(GjMkT;l zS?e&wHXRst`-W)HTkZw=GhKV^?!ex0_)&{|P-?2z>-V9<8BAr%y={G+=`}ZU1Y+TT z#gi(xD^6@YKK(%7qPZizEh-jSuVkEq_>#M}yyf_3p{MhsUVmVbozBP>e|+XG6^bZ2`*{oe1*^{lqtvAe?wDKkDHr6F#2x(B;9U8VoC-2l&m5CD7c zmjDlrq1e5p?o4lP?>5hCQXtUXGOlmqvtR56d;1_+=$gs`-2r1Y9)GK+Cp=4~`#-ur zLnR}o%*Nwyo0|*r{M%4WL%mx^pFY!1GiwvNEmg^yn%$!gP~szMp4A@pzRrYpp0Niy z^SIV0_vDc`D?^XM#%azKuJlg#5M)+&wzEsf@AnW6bs=xNC3k#hvBP@rb{0AVf)n^% zkpOxQdqrktrxJMUS7Hb}t z=b@I$1gL`U*$hFg-a~5Zo~FCC+%f}zLYz{hxDMqsmAiGf;*Y4Qx7ok}ii=dS36zI) zrd2QweS!lN4kHgKK2)f11syw- zHG0J~$n(y3iXXrWDt|M?re=wedkZvFfZB|sQd4xWxe2GFw&pAyGu}%JPc`0iY7paA z??!yL8v|NEE*bZ_Joa`|^I-bH{h5X)73XIaw>l1TxzQ)%GliK(eAK$n{5OIGh**s+ z4Yo1YFprK3aiEG|I4LN~0N#8i&(v!-?M8rdn^dfFvk;mJ>;~eZi3@hXTyRr$&{0}L zSeg#cH1r(lDFdY#Q)K8(F zk?B&Pu3)9zNl}@KA_X8px>3YDwMKoeu6JR?uXcm7)H7tRp_r+caBS+~d{~4L$^AT6 zCqzW8DYY6CYMDNXt8b?S(8$< zZEE0As)SV!a2u#GUQVk$oRlKYMvDPvv7Yw_9|!+?=#ZR0L$ttI!TS9O zSU=$W-Ury#;P+huzYp*K_xJyJJoQq2T&k0_()hRE{*~{6!1t|R6Lgt@3#$Nh_Kh!p z&xJrfl+k}G^W7o-IuQmGwh{jzxH>;gp~*%%@-OUy!nsc1;+2Y-+uc&133S# z^=|X)Z*!6X&|#pM|NQ6PxMkg{-vqkpwbyRne)aZ^moI((&vaB0DG;c#jrebT=^wrQ zz3+VM*WUfcd+!-^FzcJ{J0N%oa9xOIfJwdn>Z`AV2=&VKn=f7d^uO2HzFFf?-55mw z>6gFr?eF~RyTAPQE=1cqz}#D3Edam;WIde)hw9avZvd=y1SU0&mE#QYK=;zg^#*e9{w3F4PQO?&nBYgga>TOvZ{YF05h{_|h@5?FP>u>m~y`J11+3C7qP z%*5@R*ZFq)^7TL1Ary!$$xJm{&NasW>&-8I^Xp%K`^~rB1{MFcS8l&?{nqQR-gxc0 zb$$58i_iVGLx{;-HHF==Gy1bHy!Gx^-}%N@zjXWN?a#gO#`V{3-FofSo40=cv(LTv z(>|d*bJfg7vydHj|K#RZzxkE7zWl|{LvrIfRbGD;psr_c{;6`<$=02zy0v<-k?(E( z@Qt^=2C(1t>%jjr39mD%&%F3M0%@76`t=GKLd|~vcW>VK#+SeFDnMfZfPL=vOD{eD zy$HG8^VK{qe^PC$oB!_m7k-hM1TgHz?OQLt^b0~F!xKMK$%~K5khj_TH`jmhI&*dl z7`w~2|3Da(OVviH)%<5K+yFT5#*OE%{Q5c}GfS0Lw$Z8n>QirAfBom4{ig89mMZN! z;dsfnpL^xXjUV=j(p{>MpPDPxW4B-Utqcj}OO<{%U-xs3JO7(kaETa(a;}_B?U2>J zRN369E4hoPo2Bw5T&h&?_&!&eD-W@0Ih$M@DsL5uSSHd%-%4(tZwD|FZxsufZNBf< zcdR5XFZKc+X53nZyjL_=vIDb8b;y*T!)$CWvpX4c*(D2s91^@vDZ+f_Hpp9%b8#c@ zWVxQ-=wVpJz?Dnxn9Dx)PAo>AOC-)*sb40Bg~00+p+Q_3KbfCa0>19aI_IWG<_gPV zc9ij8H}WoRZj`;Emt^BRja8AYzn@aR*EH75A=aW)iU^=-d5#KOIX_k4%2-ba*rW2< zM3&GaIf`_d^PCjSOLkQ(p=`h1)F#)mLdg7uYbi;9QCHThV%~MU7<=!oF_N*_Pt>@= zW2SoCOa>rR3<20o^2s!@N_=PQo=(*HZsmu5!3}4LWZJ}d;-xZ#HVt!bE*=$_wwK8y zihS8}+*p?2@om+1b!=sxACKY#u(vs^_WM(RIj)ZjwW9C1?_T}vYoGb-i!Z#qi-Bc5 zpL+Hx=vSAYyTaeIpS&n|AweyjJ$v%ZnNz3EoqFV#{u-9L<6#S_Zq$Bq^_Aye{ka!j z{_OJ;ynLXd(wDCQK@R-WbI*d)M0v22&R;ll>cWMSPo6#b@Rc8mrCMa_5F5Gj&ZQey zUi!>)FN4fJ0r#@5)xpmPt_%343ul2J0&&PXEm)z)&pdhRku$%kGQ`P`-;y0`e|jO&NjT(`9HsK z_2rkZy>RWt=M93_l`EgpxSzgY%AE(p3-q8fr_P^$;@FEnf}@g$3R`Iu>w~|z^)#4s zz{u~R)q^5`=@aLfB%#lPqI4S6wR4|%?4gVQrT|N|>exy0HKIAYfBx#TuU_I{2J33g z%+Hli2@&nwlK|-pJ*s#1hh;QdG5lgCoVjB=KN`}uufU0hUbny`Y+eTQOhZk zuf@&6h|#*ip>p@iT>kLm#zVv258nP zV6mw3%;|HFKk>k){_98_RnatV0W7>iz5E}Zy!71Dm!3X<0i=3igPl8j^0Cvu>4>A2 z{Sw@#h;H?pPo01M(#4Y(&zuB!>LRF2$KU)rIO-fpOl1@CRw@4TPdxR>^E@xQbK(5) z6QBNXVyR^uX~SxXrGm$$z$HMg>dV*vykN?-#^UOPhq7X5oymR02SVI4EWYVyUGq5>{NX zRDE-VmURGrge5j@SSsJQFts{jqH;}Rq8_;gb7ed#yF3&xjJfo8(qgH_F6m6HK2mIw zVyVThGU7<-$!2`#zN9GFm z27c+}wzWxy4SgHaLN?WA-B*aBAd@6#_O%!&ohB-fvD;u6F18Q!|_mX+>RJl9` zvS)g6Upy6isV;p2ZpBmKnbZWg%OnQksWQ-V+w+{bc&ZY4@_5M|@l@D07HwR~qqt$J z#@?R=A3#hM`!AN}L<&ZVjT_7Bb_y!EYIf-6oRDBjHnW~fCaSoqpYfv+aaBsk3*xG2 zC+dr!3|Dn_xoW3FK1=@d%csvD7bZTaR1@5#)KLJG?iYj={{aC7^7i4w zM}TPGe_&yL{@~*5=YK3kwa_YWw735D?BkD|KJl@W;CBmQOOWR2`yT}LN8mpX1E~Y* z5r8@O9X)tpd3hh8OfzSG$LOieZl%)M4uA2X(>rE^2lSyk3Ie&;t`lw*24wBk%XHg#Ga$T z<;?Cobl~vp{#$=3uG()8+Wldt{TEL^c=9BGX(x6YKK8&vAA9WB{fAAtWk7un-FImJ zzNPyP&VTZU(o?Z$^t!`(XZycjedzf8pkCbrvh>i=LxL3oFR3N`o~60PBMbA7{fo5p zR1AVNi6MID&o4dr*z$2rk!K*B9(_=a9Xbpgk>K)|!IWB@KJ?Dtz*DWx>Hr&VceuSX z`ooj+#~yxS0_gPc!}l*ATv`T>6}14kd@w1Y#3?>lmsC$qe)xjM3M?&|-{!BXdY4(5Y# z>jy`V9DC>ipq8LmI!J}UG0o2hPX3o|anvpiyZu3%Qyj|w3RJa+9(fRaDUDwMTnn?y zzZMfm?fL!gpxfW>SN{n!_Q=tFO9%IZcy#2x`Gtj7ek_jK%h4wiy*k6%_a0onPX`Db zWSK3`&0qXIanv66b`Gz;;21qt5y2)@_mc*`xh63 z2fp%Caa3$so2}%Hz>$Te{nJqjt$&X*Wt0an#P{u+<`vOdJ)P?Pd?NS#D_V413LX zr2r>g>hP5SUNKedeOt|1cEtDn`rc|(@?xWy9Iavf42%}6nC#Kr!ca+9Zr%~9bUN{UV zD65kz41IO>)H~)1yF-=G0x{Lrbdv+(0PK++YhN&v~pvh*SY!p=)dzwU? zc!sAM!JyKzwd5aah1%6N`UZ#~vRc{{8_H3LXOK>XE7AKv_Ksl+`i8a6Jm()d>Kvp5Tar zQ&UG(SRZH1A8*zBQ>Ved(m@7i!N8)fGkoZ5gL9m5pmPq+1B(S5);T_&1k#K0=lQq* zP!_d6$;U;`BBk5~K0b~`L{^bcfSsi?6Q1JiginH@^|T9;GZ_NotDMcCGa8>{9|;v`ik#IG6`@wvh3%4;ddxFl7A7aJrFlYoh2QcLmH}{n~0xvmuq{$4-s&89oV_dvEm6>$gNCh6LzH(dRbpDV^H4^N`DK+IpC>@Id#*=nQ@>NQDcCTY(J!eJ4 zaW@H|1ISB;$SryFo-T8fBzO6c=_LNnb~qGMr&5knaE+U|seJDbX|m9?{LmVxBurX5 z=g5~#A_$rbF6e^E3c-5}Z>+d?qaMM9sWeyvG=^_kfC>EqzFW-&%+gD&Bmn~gF2g0O z9PJJY#<)jtVq9AfrIN7(`Q4d3z2v^U$7G36gOz=JoG${J158gklVs;eFK{e-jKWb;y`cxdqIyt5jM#=srO zcvNkQs$rf2yt^5W=w2f3Qj{sT+yNQT3JKt5wY0JpO|h!TxZ@`Nb1fx;>R{(ZXDyTx zqCtXGX|g-WsM4B{qFha`CKBN|*imHH=;1(MduK^HiA6K%6j~yRv*Urj0Ctk=!QZRf z(To#M3r>PZ#=7*>lk?+pZGAl+VdB_pXVaNH8G~fLpC4CvaPd?m>Ski8L^|iIYjE>w z>P{pTi#Wcp9nf-p-~750U5&@1NjD8L4~tHZ?83OU8Rn9Nen(mTzU_Z}TraGJqtUd( z+M;XZ$ivSYnf0}Bif0?6N)pfE)TFhce zKBx==K=9mvOS}s85U`@4i=IAsV|eheq}XaA#(Tt*5bzKWPOO9XQjm8nYT9tcl5$J} zlbO!gnIdahWbQp=Dk7HTuZNN`8U=EUOr{+17#~up#=P%{CB@g1kqG!3u!;=m47^)= z9rMHyAQDUB`SY+rKgbkVrHo4Lxvm02%#wp8h1R3O)!?D$GQ9tmM?~>GG@e+Jx4xQ4 zr$9MK`e`wVEnWtfmzi8E%iR)N5KD6IMBH?gh3aP1nCtU8TV9`+l!5Rrz>?y3nNYY2 z6U7u63dXoc083h1PXN0`pHn2F>pl$(!Qh|<%Y!8qqS1I*&|z`5UFMYp--cI4w_!=y zNHoNllTi;)ExoWnQt7SXE-c9pMMA*Z#8SX#`TVMv8l;$Q9-3HEW+e(g13t>3YEx7V z^AyCALhz(y3SL1`rrgR5d^$zGs#sF-?rJ2-+w8<$Zt@0dDUE>kfF)&Cc~YS`Ym@#k z|41xxSh1wUN<5B0KsYi~9u5Sy14|l2Bg{7S`yPux%!dc4U43;sl7_noWQKLbx^&bN zv81)NXebeld&!uWgqgwf9kC=HTr|lmEzXeiNyv3FVoAvl!pUej<0g2jY?ocv{K`8R zi$xNdWSpl1Tndj2mNW<@DCPU4^TN7pS1f5Y#M%SMB~I7UnT%LcdTlk7NJS`5*HdY+ zq{M0{o(Sm|X#nCt>mMW%>k;?}V#EB9yBkhN*20l=47d%kB*a2;bv=}p>=sLkuf@Y$ z0y&rxOG+4``eSKwp zRV=9xTUmn>!IEQ#UIwgGEd@cy;>Jb%I6%gR!Ht=M^`b z(u8sPRYy20ZM>WK+)%QKu!kiTGrn+k2muF$R6a#KaU?d|)t)}^3Xug zRAaMN7E5yC1liMLs))prcri)X&gV8ZT4j2#BvR$Yff-6L7fIc) zq*Rh5eAG%zsT)ejR9f*pG)iY}72`zBp%e<2N_m!B{D#gg)&6lt$0 zo8MV_-LSE@xhX+;b;by0yYvU zmgJ2wIJ-6?n;({xi^j8=6p@!@`osLQyOi5WEXj>HUdqjFFy*Ay@Jfq6V&BvdQnNyE6(L~1pBKcoQL z6?jq!H7!#RiVLffVmr7-YKqN+#jCko3I}pAO5m%6jCsk0kytH`I#%dUwCOj#FiY8_R*qq@>$*_1* zarGXaq)hcEI2rGxNs8Agc@HZm{Wx3M6KrLVlR5vGIZN+RWw;+RN9!GDcdKLePKfd$ zbL!p`>~c?n>8!)}PU+{P=4?LgbSK+eyzV--&!p&|SEGm6(Jq?<{m9?{0d}y9^2{%? zjXh|#v6C$DD&I`|I_(VEEZVt17UV`fX_%Fr3-Qts25yq0*b1;h0ii;> zPb9`*l5kBdxk7kVI!IaeoUjuUfo&1-o8Y^)3EbuFRuD65IUd{GD{R7|+03fh0Kv-& zLkBF%?CK2Cmzv<6p$19zOcGBUN3`+-zfx{w{v}H+NO7tG4T$jcnd~-OV=1IWm9sH| zS$e5Rc5IKW9I?9s^qLf>pxW(1#F}S!ogp0|T#vmVI(>D>w z&;+s?Goj>a&d!2r$xKobuq~QJFG?mHo<-GkS36;8(V7jB*g{Ry;FKk%Qs~9nd8@Sa zUrgiiG;lRh&7G8CJOrk+xs+L6C1Fp(4TA!el34ahaqv^qsYFI-d<0$3WOvl(Hsn&i zO-y%oTEKu<2eN@|led<~Lm;)2j6`*+-<)qs3{Z(_Kr^w0n(p~QU%mt;*5@y8djm1L zR9N_mbAFv|0_r5&Gyb8kzBl6qfX(0wPhNdw^B;=}GH=HtP`W{9wE;jo7rE!p`Zids zng^6knnQ8q9S4bKLe6Z_1?Ep*Aei?}xOiJrM+L-y#{`Zy=WlCLiC5R!RIjC}+n(uj zPMLZvoMG0+t=KTi5w9aJEFf}ocS~(*gN-*CA@D(MOZYMyD*~xmdo|UUWk78if zpz)93b|y|*xwTk%`dA1+8cQazg9m-vtFe>7M58?y%iUxG*x7*J;E5VcGVyUg>F~}2 zfj$|Z*`Vo7;!mUV{As_zjIdc^D7atZ;WVs*Sas!dT5|A^_sJt&9g9%v%?6m6NubrcM$Me=Bp>0Ng|qtEdgmd<9C5~U5l7HCl^eWd|)@n+)5l3rI!_K#`4uIB;TlwM1;wEofO@x?~Q3nlw8H+DV{}Z$mo-T8_!J@5*q9indKYl_ zbezW;)geY!ioLGphD};9(_@nZOITmYB7D2-d3jc5>ql?W19sifyAc_>dhrDBpl;Dm zD{ZF+Np`sF$dOayQViU~$vrJzj1sX#XeEsvwt?p2YUhWAjGa)DP1N1h9NJcyw>cww zRGR%?JcqKG0cBSOG5CTQVbZ8iR-_S40nd$1GMRKR*eKMc9!VrHbZP6B zN=ELk<#8S~qOmAeGt3igpjRR#v?pu5vlfX*Lg8>|jR}2fY=ydOk%VkQku~krI)5L& z8&0svy0czGE$XH3tWo&RN*rZppk3ZNL%~zJ9k~-u=H?^{0}*Lct1 zHpULZ;?mNy{eTC$A@)pCm$?eTHh3ploZJ=z!1~IrJr$3S$EGw}00{-%4pJ5yjTSqW zMDxm%EJI-}X)yuucb;uDI@l{Lo@2xrMFTMKO|!ZuWzi%hKg5Go1Mi;0eESmN{2*qB z72EtvZcdQmNJVd(w*$$stp#quwP97faV*kg*p~>0rD9>t<|f4{sCK&$!OGYn$g*U) zs0ii!8V@8CV?#gn6 zSkhPwi7n_!4USje$Rsg57@A6pA<$|xDG{!@qab#tNojK_X`_;uNoZX~d7qJ8lj3^U zuter1!d>QLM}2NX4jDaAMbjRt2ONb#G~uH>9zu}PiEup`t{%TW6bPcq1lUHQ_olPQb6}3r}8s1YI}EJBbK);Mom23uA&s z8*;~;MFy}YA><_&Um>49>`H_ia%M{$*v!11>AZ_*xBWJQQ3b3@co!nv-PWYqzJVX-y#h)5dVBi@aPn6@6O*BnZJE(A`hbHQ zjv5?U2@OalPqV`S;J&=cuwW|_a9FyU@>J4FZpR}jiSR6ss0-(p$YN<|TAUFK+jf{* z*;$rjbo0-X23?6-)E@s&GsVDr{ zgPUg((Fo_uLFOZE0!{-076Y79aK6+RfVEpmD@`B6lHgeivsNtT4VkAWrE4j$xHU2f z_TT6Z>0~$~3C?2?u`N2r%0HszNBj+1<+KyW>tQ6g{3v2DL0&Ev2G1JN3pavH%5fwa zzMGT;XYB;sqGp4cp)mL$+dD@hIkXMEIma$n|QAo$5sm_#NylSQ*JcWYQ7g z%xVU;tBE9%JTl|q!%BDbGDLBp+zI6+!qco)2{bz$KrDNiV5WnNM@&W5R*5=9gyWY| zRdz~(5@E#*rR>xwor&A=ZA@|!;d-)|!?|Ong6Sq}CaGO~JQ7)7Ga_7*BZ9efi|4?4 zipLX?wU8?jo=%y2sr0^d56&D*L|3CmgkwFT5UVdp>s=7glL)tPBqaN=1^!@1QIo!XsgvfZRyNlZhx!lkKQVUutK z&P97*a~?$<_G0u7;upi!Fq^rS`C1S*QDkNx?BK& zy#SDK#Nn$X;2zARv19gV7EpcOdhNnZcV%hC0A^-G=0m%NQBSw9D-x0+iVpgPe6~(SeLhqzFL6dAMi%p4eMYr#$&uz$Mht-@` zfb1iCNO>cHY7-Db9*<68@RO;kM0mxN7+{U0fr)`uCR3o&lr;hx41aaI415ya^h8x6 zoJy!%*mVNpqY1^I1}0y4^6Deu22QFh5iT%2J$x*wq#_vdGFW-GzCnoLo92j1@RJ3J za6`^)$!aSzvw5cTpw+QN+nPFxl{fr!St7gwD5AnaJ6fCSwNBpB!CqPr!TOF;Z-^N! z#!3?57|f}FC#PqinwUosj(8}+dbh!G3k01b@+88mK+aj3|BAAKA*L^Sc}&g@NjLH$ z^fB(2t3cCBgo9I8!*-1eK$xD4SjK#oN}$4IX;nkaVKzlKD-jNchZ$c(#CntX)9A{e z=iseit~3-5Q*aZWM7SDP#+Fiyy55?hnun$;ErAj*}3EoJ1aPV=NTB{zc<@IjzsD-o(svwlazeEV zyKpf200@aiNR&t3jVI*!8J1YALZksE1^l5Z2zDJEqJ+wVVAYxMTb#$Gs_deUDllMh zI!g*r7Hq5AZZanct|u#mGd;cOa;wUDPmBZ|EFP8P7)ZR``@G350mK>55lpaoL^yH9 z<}S{J8DM&PSemkVA}&U3#L031S4~EG5Do2(E->L7_J{K0N{r{h^3n_m$tt)35DCr~ zCl=0YE6za>2K0?ZA^bwl6U=2oiIU_vM-WVIaYq9K=mJQE)KGHV^XVD}Tmx6YC!s)T zwtL8NW=-HMigIEmHA4|L$jEU#)TE3E*M(;##VVSvN{;7*dyI8iCksKlw40_N|NJQ6i7J|MrJGi{CM=< zl>7_tPwB&+x;hR_T_Kq6Iq+~Vk7vxEBI<;MtEjqX$imx;ta}RZTSeD>9OPR?*j)hf zmMFUuCd!VFLj>9p0k;g`Ezxg>!Mr`n36u8|b4S?QQQQ^}5_$*01IBalQ3M|GamP4v z@-bW$$ES!Z{SjwMa`NPRQz!6JoZO4Yd!Kl`N$}ky;tt019wBy!%vA#9el&$)V(R+`r^fgEO?`i%Grs3Ib+0xC{UQRB5-?0C)bjghxxOSaJ*KXXn^_ytN+qZAsqW?|z|BEhMzH;Rn-L<Fw%U7;G7P&`v z?QeZ62Di7I_mN+*qAv{GxfreAm3xTeyqKiE_r{8G8&=qEOlAL2vU$%Fm!oXt8Q=7T znfC!b?Z0RbxqS5+-37ou+4<)qAB$Z1Ti@-T)myA6gBbGb_g+o|_p2LIFRbs2GjeSj zaxKqUn!?m?uq)UI0HyLFY;FkHvcfSdY zUs4cP10B9)>KCSLAY$L3zj>>Mg_!@i5q73Q&6P!ATUmFLG9s5|(w-kDl90iQf-s2d z1k~s)P>$mwUO(t*LBgbMw7=G<3${nl>e-=U_qbTs^34ye*x^;BR|s#l+p@?$k!UkX zld<@1PT)vWc)Ufl@`$O+!{x;19x4vsT?|{}c-e$r{f`T!$}7L7pXJh>lZiqiEUy?? z*1BDgsjPYErkZ9+-54KLmarw6yP@-{*!gWU_}cpH^{E=mS^(g-t^Q~TC#qXeGfG^_aog^&}2=O>C;PKlW?l{`?7);f{F#e<4Kho z0wWGozds}}o-W%gd(xJ8n0ffz{=EZ93F32Hk2olZTI5%vc1<;XFg`i3Qi)#tQ=)lL zCplJ&WiZC8E*fDzc?C3hZKO2r&)6s^3z4D)wmS?ecJuGKwyU*b-nQQqSvazqiA$yH z53qIZxstuE;$Y}7`6Le~m55o7oQmLr>QaRI+D*w4 zpj8&#n7?*^0RU_Pgk^z1%b?RvP+9&VZ-5Q_2US@vrFTWShxa|NNKL@a!PuoSak$Q^ zx9T8jLm-snUj4@ChG?+q5ziQwETi^x)?Q2zQ5Qgj!q)$h0;+FoEn2ax=rHN#Z+)Ki zbrnuY`$YNnKPA=+p@$6vPtvF;o+cY#F`1Tl?EtoY$&MoIkg)6eU;&PJ9XB=Rx*YSt zEy;Bf>XPp8;U9ZtM#D)NC7_C#&t1 z>tq6k$G57w5~!lR)70-CddV!t^pbSCOONV;IlcUA$&4_Tavs(akJM6Y4E*BPUhNTn zc9YE8%#B*Q6|5km3b;oG-n*uw9z`k6M?YeR4fd(0-=bs`JN)s(uHr0pT(*Y!D|s5l zJGBte-0+siUu>{Zli@fd>Dukq&O{?A9Kf`$eJ=Pj6?9?b6zn(Kv}>|bGOZjmIM9}` zH5FadRKJ9S1^{oHih__OwSU6aQzy)R-16WkQE-4>&&T)mJwblwD^?{*Xo%j0TerK< z?iv4hQkk)&JUc!yR{etXKHs{a{-9v#m3BT@Fugix0{++|uGW8W+p>qJanQI$Oztw6dN|vg`R{pLw;NVGRwbE}mgxf>z}nlj%wlf90%J9}ko0^6 z^Le;|O!BHuo7yycgy}sAM^x4suo;X zwejp%r>;OBM4~p>o3uB=5|c}yV!2{W9J$>A_JSi`Cj=%1X?jLf-HotYRhq2(O)nOQbVSc2^MC@vf(0Q(k$ciX>b%fcS@xmAUsbzK zOV)v8I_doNj86k5qb0g3gjM+0*a@AC!V!AD%%>id3d?)zdCR`!iV{&hom7@(i|vpL zE^UJcV*;IBcD8wDZ7%6C2-H!#{wF<%TLVl_V&><}B=wW81JPH>vFRxGe?*T>PkzkV z$5Hcn+*1=*9}0vQ*~LW0)YkY1Js2?>nV{wv;&r4wj9H=fD<-g5&ku}q4YGKMcFkg^G#V6Z(^lU4io!V;a)lFOe!{usZGj6q9R%pGvLDR~r zEICLPQPud%j;<;)mo0qoy0+(@v9G1QEV0xo1p@0QNy?v36usNL&G z8;}x9;wH;e&H*eHSaK9#OX5yqzgZ<|V^uF4*J(m7y0Syx-GT|wU%)}`i{P!to=CXl zYZ-WzbSkjeC_CL|gbS9&ejl-Vm7(8X*>d$oIiIUBN}q9{w4;dXWVVH1pCtvxq4Vw5 zAL9QfRPomJZr!}sX3P`fXK`BY0K8LA9Z$* zt46vGMn61O$Qd~7GEU5rF2v-uQC&gh=Gu58|>Ng@H(I zX55K_aE+JC$&Urw)d%y@%bt0e9tjgdj45F|Tuf}Nw&wuB!Eu$t3psA7u3)E~HO>HD zt`cf_We_3Gy=fs8ZM4hezZKS9uH0N;xu5?}1pNPAE*JVbsGG?GUSzO{7h9rahm?+& zhZwRXT9O9{)DsjjA^VA#$%)qLC|fRW7g_tg5UJ{A|h265U; z%&z0Q)9-<^>Tt7XiPVfz2hGZx0y9slqSyqO`DM50HO{Q-8Q0YGv;11Ze|HX` zhVe+h?;_yrlUhrBLmS>L%lb7lelVH{KjLefJwoOr5kpUWVZl1mVxaW?s(1_Gt#d%a zgequ`d%%n+BfA+e$63yN4ru%vk3G$s4x!$wuG}qPcaJ1h;}6_J;WFl5XmYr~EHmBb zVdobTHt5}HOwgS~2^A)QMr~P;sHE@QE*i^RRRd(CeEUR95-ApXm^1ul=dl-9^S0^_ z8O6D-($ZdfM}hQ()GVzQ=xkNB+jq71w ztuK0_{`jBbr|XzoqI67v3GFFxr%ND>f|Pbv7aN1(dFhVu@Qfutv7F(Els1UgO1;4202@U6^y}G)zPjrC*PiKBGw?$$UjFmv0U5XrC?h$;MUEt083@}tvIIG(Z6CR6eM)p?~TN62){2Zkl%l> zETc_uTd9~uKu!A(*3Btu7EZVQ5Wx_&Iav9@!8o-`kg$X$CQKotl|7i!9A}kaqM*$Z zUG&4(llK>W`%HS0>y6wC{?d^6;L|-|I$S(tv&kwZ5u@Ag08~$#4xg7+DC&yy(R@8G zw8ADOX({7p(-(a>QFHj-NA?_$ggfJZ=ma5XZ@4e8Mc6Yf~3-P-aV#F_2JW0}tsYmIo#t@#|Vp z#}ZU0Y`I|{ZqzR{Y1kSatM|VL?g;24Ekn|o*C36s%?HV6kt#5r z#{Ry=JR`_PdIhmv4ySIMYZdKJYSP8{ZA4F>xc3?^mCBH_7@(WU=YW)?`l>U~WNmPY z`2&T@nu?eNR7yqD9Bg?N`zp3iGgxcdA207>D6@`X-APQ_2^8zFzZVl%ui+T%+LoNG zf1qRb$+(nhv&+4MoM*o5XAr^12{WQ)Fb$i|vdUDmuSgvG-@-R3RN0re@mI7h%u zItvd}qFl8*>CPoOyWYr<6rq_vJ<$DQz4>CwQUT61*+F z^*yHoX$ko@De4l42&(TlHFNVblqd!5*Wi`qTT@fyN5ob|r}k&wcg-0q);M(x1XYGS zO_FZO1Al1EHlJ7>Zy7Lhajf5}AzqdCx!>CDTQR!gKDTcZ7PzId3LX-|HN>iW+FvBy zbtZ6Cw@L30iLXytH$Mtu?_~=*Ef_~IRM$#yP2zgOuDs_3jU6GEb_QOnM(sR|shH9W z<$-+aJ+rsmhn&-lAP-lZ^Jw{n9{Fy7mkC(^M z3304YNHYiICsN15a=-(~I>ci(ONor4g!w;w`CLojwen$pfK8|*Y8JVKpe31#i;Gv3 zl`pFnR-k33dWWHhn@6&AZiOS460#yR#9Ywa9-xHCl2|pVnvRyCG6eAmD+7|yn_Ae% zLH)Llq)Vh_HV653OmMF9ST|0Tg`ZeDdR5s7agQjz%M#L4<$*dAR1`nG4ksyCzdFpz zS&;~(i%8s8$Bt3d3(H3>)XGgDItk8emgHh#C*CM-TTCkBYRgTlx;|1hWgd_-L6+$8 z&oLP=mL_588JvsX-rt3o*QzncXEtgsj+X4E-vK?*G#$0MRXmto;$kxEj}KsMqwLdH zO_F&(IA=BYVjRh28lQE(GP|uS`x!T1M^Ui0Ha<-YELLnU6!~3tG)~R|1_lc-iJ0 zv7;ncD4VFta*5zjhfR)Dmi&s_3LLjHqCA#a@f7tiDNUj&7K0!9JHKu zoRXq0<%uLcbis{+VJ_V~&|tzC4*DRY)Ia+nxx%UMfmyE!*-$p#f^k~JQq?1MyFw7D zDWB>;*pWytbfm|r-_Pfbe{Q<>Ikp)WK4WWdG1hY0lXabA^=>wlPBw_*vJ(9%lu-q54~@ndrs z+~A54EtKvR5lr6Fl~u_}mr&_syO}~IV%is0W{f8`w+tRd>L)SvjNV^4NveXs80JUc$pb*mAH!$uKwutnF+*hv+d`f=PKVvv$iUCsL-;3~M?Ae59lG?7A!> zLp|%rUePc=5n~?vN`y67Xx2yWH7oU--_H$`CBjt+F%zJd#iJyLjN9Hs=2Gg!;GRWQ zP}o&p5_}*iFbKXUkq(XbQu}l)1KWd`-&O0c8OeU45h+Cd1ewwEFUq! zqAdb%Yz-5^XJbn$@&p4^pe&vEsvdeGa9?YFgcFm zu&_9R&WZ#U*~53!;F88o$#V0a*TsuKTLk-nf9ar+b2eXTU(b68H+1Z9Z=JM>rcz7F z_RaiG;ct2$I4qW=SFhZ8v8BbjUVA#W<%e|pF)4v+A`4#*pQ}s<^H*546Y(gMK40!y z1v#H~1xK>A#0P*fP1U$rCUg2)=C`Y%+K5mt9uF&rM8s#i$CU}al5D@pnEkcY-D!D&aOFQ zK&z44P_<6S11PKivg3)hSEyu(nRuoFrGXcJ64*s0-!<+nsV9Vy+F$gfv8~*|Y7gi( z8B3T?wi^jly9a8j#NVE{`##ZW$5%Mc1Nq9MHb}#%0|pMrg>zVC;#x@_aIYJ@spP*k zczyv7OHC*{&^TwI0dWViNO$^p+VkAG5=#k$SwXC+5Kru!ADj5Lv~8jjD<)uUl@^%} z0)guWc+>)mBNaWzeSDa!^GR54^u8C?eDN$Uuu3v+yf%&FDn` zi28m%Ss^eK+Nzf?oH-i!Mf#j-wC$E2c zQ~DW4r1Vips&AZT94fYzLd6axqvre)jelxDJ=oWt=38SImsJeg5jLto9NF3iTZo3_ zm*QNDyPUKr$K5PjP+sZaIBWkLjs*{2^UPSZ{~=6d>_|ue8H;pW4U1B`I)1O`Yld~D zS=!45aD)>)q0(NrMZzL;)3J!PaA4gN?u;4ck!0@c2L_<6$O#QTm*<`U|GIf8<0$D+ zExwi@GxWNLNybqy5eHFNm7n1A&6!mZcW6jrB~>`YOxvS_1PvtW1XcHW6fTw@T|_CX z0LRjdSR~W>CvME>x!B!1>jg)o18Qmxdd40e5P2q#% zw7|kf4`V$vqOY=F(J?xhL$J|UiF4ItQl~Uxc{^>SUNW74sCOUIooH3P)gKvj+jsdE z6I6|g@tpgD_e!+vIy3r7aOM*l2wkD;e5+6%j+QmPGGRe@? zNAziOY->1JLPCMDjfL-$){?*^VCkf>D|ik%J-Jmh{#2U}N} z9b26d@VDYEo9Lbx3mTc<$Kz8P?L7BD>vZY3*hXP1&PG26(bw&)h!x8ni-jVrp|cA= zIih4yZYb5Ps>H0KL@V33Yh}>_nAIzNEU%L<46#bV`rK!zaOM;4wxJN%Tc5N_c=J8% z+twhnluwsxcEMPhDplHuK~86GQzsZ3wvrMXm5!-|(SB>cy(I#KA&J~Pyw(i9I_dV% zrY~pRTXJ-iSkV!@XCZ_90s#DNS#ek)WTVK;|L1nG_3I^4`V!V#CvYz(C&H<|46omj zWic81V!Z;YpuPh3SLi1}qU7e2-aSP|F;v8~LhLZBZoUL3VNv7q@`*k&)24<%x$sSs zfmI+-K0GDkF(RLzL9BzJAby->QCsjM(Ku!G#OI~vK)|K|va=P;m*>k&0+n>@4)JaH zNa5|vR#5clgO0q~rG4CD)JAIve9HkEY`8@v4~>24O;#N>h~k(cmtyP(O$LotKZ&n> zP+qIhUY@V&7YHfw*O?`wBGxLJ2)edxK-dJ^*$ZGWPG}~=aGO3*hk0bC%0BZ>r@Wzp zg2!Bp)_t%i*R6*HjXAQy{+meO*;-ssOp=?xZGZlm8JCU$W6sx`uaBZl7Ek;()O*0> zii76|`P}+yw_HM*aySu%ZNmfE5@=0YAi{aCqqD34G~Bi&7daqWPY;QS?bze7Y!zLS z_H|W)#+GlzCq<_5L{CgSM~7PK(_ zEb1o(%#d4#2F>#7TFH<+)?AxG#j{IqU_CIQ{D~knomb(`Yt@HiR(By zOQ=vb&&5j4KWxJ31>*g67@#99=wFx!jDn*46ShYJ=Kwkhm>eOHrM1NoZIjYQxEt9& z@~9KVUe@k(JGUo*+j*mg7FJOBVAE!dZF%{g;R#zKSh7T#vRagaV`ijuREGK*KIk+* zxx$N)ym&_3i>jJaoogE0=^(x*u(xclzgC)CevbNqlsx*HA-=Ct_ob<&xl>3@-aSix z$e15~5A(shosFT7DhvHx!;V+g#b%D-Fq@0syD{o=U%OtFqCLril#u7h8W;HLnh=k( zcErjdwu>9#!5Hk_Waer(x`OR|jo^u9M`u<>;10i*lG2=3N(;(s9g@6YT)#51K?s&7DM~5BZo>t#oDjMV7 z`_#2o#Wc4BJeI^6YnW&CCH9{2Mp@{^E9*(_3s8}fvyV+dKHsEK;D9X^c60?Gc@=+I$;a=UZ)g{Z! zS&;j@OLq%UXSxuj#AY2KEu~r8Ei^6Hw7u$JWX#Qg(x-P0FK*4|Wn^UJCvtdHrW=;H zn^itK7*mb=@$>sUi-{2?EYjtrq+h>-h~1nT-=^QK4@6fVCR7AT9#k4gE=(5{{)9xa zqb7L5@4PoxvxAVDzo^;Q~nwmusC%#8< zp{C{(IHf{@p3zS}JBy0yN`xKSP~LKL{@UB|&P)R>jWH#?<$q#*Mj<)aJVB!UkJ5l* zd4c5cb*HY7FpVlt$!$3Nc3vH*az*{uTSVlOCv(_{AK{rFUJTOb%x%kbW4`$Q+X4Nr z^CgsmYWGCpA-!eR-13-WOiQv`aw)4?8EC*EOHL8g;c1aFQDc{@TAS7(s#(T2}`|?{0*u-PX{^3I5E7 zmy;CRKBnd8!?_Af^g)ep76I}3{UyXCO%;B|=$DHxHxnbgdN~Atwb_=Bb+!M9m3lyJ zv3lNDm~>Bs&5h3&9}T4sdDc|g$W$Sf07bjXVISmFeL9V()-HZUoB1Y-m*;@L@kz8A zA^doj?#VjUl(%1Tq+oNHEOBG{G&@=+YT>RYEFg}fPlx{GUgN$uQy_;B@oqX?A?O@n zgoz9I{4eSl2_+RIpk#1|=K zWyfEQSpM0wGpcyvsy7a6CuXhbMP>{sz(Vv!pS{pP{bGs{7q`RwDSkCX!$LL2`$eBB zrN$5{G^6{RJ)>iUWl+MdAT!QbcSwAi1a7s(|5*~iitU(qXi_|c_sL1c8SOp;1wGch zq+GRN_>y>A1tp#~zAu$XLzvFz09+-~Do9>1Az|lUy~3D%@uYd{0e|pfuT*=2#ykJ& z-mI>`%r?k(4t6CSEq2Q*%BZ@+Wwsx@Ve$LY!R^iU1!?T{?a^^*Q}f{~Rd;L9TXl5{ zIcZIi8H<1I;)B(e*5{**7GX+U2WzwCcK9VSY8{kNm0;X#-t1FSvARCrdhd{YeD_C> z^HEUA8@QXqHn}wK1d>{r8Y~66EsuzlswCW--Uay_9mr+Vg#@`fjo-wmG*cTTPx@C> zkLVY>?%m8m@E@E5o^y3-HU;IUm1iCT4IQ#%0&tcMurgun6gdDvzOGQ&HBkjHw$aNWt z6%OT-hh0dMx7)TZ&YYaSG_Xc^Gdo!9510YUGg|J*-7i`|d;3eWUleM-U{(+b(A^z- zV^Hw*CF8|N+@+e2QC-!s)L<8rT>L&?y0yLj*~Im=6?vu@1}^h=JA(Jj>2KQ2;TIdGV&SpWM}f+;N;0>C$xxcVuDGMLoy68H$epyNYPR$Ve(T|w zhp`Qh(Vjhm8J-}>2p+@3?Le(nJrAK6xbxeF5nakri1t<@PzG7@Xt3Muz*XJ;6=F=( zZcR`mydh4YovWYRHWtRTN3qV*w8^#EcD3m(!4qE85*;t8cKh3XQ!6;+%6qK(x+}zW zCF93l&3>PZE%2mOLS7}#iPqy3cw98Qy#`yD+7TTgf1*v_RnDc4Gza(Xr~{=rI+`7_ z{h2kbA;C6F>l|R8MKcWKi^`)rk}4yk9WybECHz&ar}-0-v=by5dO={*MsdmQpELDL|R5h|Ws&@q#* z^(ko2>}amfd+-srn7RClJCNQHZDf%b!re4sFFS;;x35MqbaTo_`G;1XGH`t9*_N2k zp;h??V^3v|>NTq@&tf3MDY-uimnYA@dmW`Hs_5~?@ok5u56Tz#8&mm&iyU9F@MbAN z`UB4an>C7FRlG7GH41zV7KekUiY|HApI`uhfh^I@lE&>v#?~{s4_3vbuO(-6X(ge8 zdH8BL8=H6M88e9#9$;8a~JhGfX(D-;5U0ioJ?q`0_M3xfpRZjx^>zlkTNIm`oiS4`9?s2Cmb%CXk_&0$x14+Y z`fHPD9efiut~u=^Sf4{F#MW%VdkAC5^VG$C@{{5ii^ja=;Di1JEo-2uvqNfX%poB~ z2BV)-JwwvNs^%{d!w5bCAkVSj3rb}}?#j-J{AJ7T?QhPj7tGirYn;k9+=>=xVa#FM?r;k) z!&sf5Gp&C_>AT19C{G^l^^+k^;PM2OQtZ*WBun?^zZj!kTzFsmaq;+JSGv~r1IL+) zp*{ML>^g{5k{g!Mfa0|V`LcT$btv(?O4gCY(ps0iIR3(WZ)+66yR_42UR5cTJWBfH ztx89k732NVS0 zlMCtvwm);O#Qt#L)G%5#A;(HJ6M0o9g&XViY{G)@h?ZviFrq=yUQD4hgC4T+Q zwn3En5RbmTp+6r}Sl67m%2|zT28YnjZmQtD8S$YS**<>La{#^x@-pi?J3sT^r8O|c z(Y+sZKi^x3yT#eC5^ru@QXQ>&`bB|Jn1m}+n zC^cQBuW55*)m?NB_{vhTVitUOy|wPpAYoB))o^Gfv0?!<`QoR$3*;|x7(7m~&w}A! z@j^N+UijlMe}8^*SMxxs`Wpa%2&67e>MlT|5Yk_#qV%18fxTnn(xd}eNd(qxUIfbD z>w}F9&nPYZA*HZ~o3a!wbE%czYE9#v!VD?8vqD|r+q}sJ^^}5_^%rn7k9MX{Fhofb9g#Rni>}ckB!Wi$uV?l=c438~$ek_$A8|1MHh&I4H0*=jeF_5Nrt@~!Ga&tY9WU=W{{q}L@+gn$X7end%M~czV#B}Z3cYju?$?F9zl?53w?({^J6=N1NiWR;KCD?3mzE*P?GQ#6 zm_a|TA&t>)JGvlhY7I_5iRo{SI9iD4?qxB9XLk~Y39=9_nK=L5uM0VG)<+j|pmHf^agJHPDme&cm9R65z{lJ2 z`^U_uhO3aP`+oqgVFp{}gvHS)X=v_R20Xo8Y2JSQtC29c!qg z_vWDNn%fC!IWAn>C*am=)WxjXcMIVW%42_(`b#%n9%4{~!ynC6r*uU>uyL+m4U%dGmdY0z>9`VY-OUR%$VpsmO% ziypab?9!Q|V#y{`@@<>K*C1A?O;o$3DOez?nKuD0JL7P#mk;MU_?0e}2-a4!Ej1C8 zY6a^_hbM(8fxRPAeg;^qQ5JXh0>_+10+nNd7}odaq|<4$GAvKaX>flFsul?H+Fpl<+Ya0>Hk$2J3KHW_b6 zc6iKhyak%m3{+7QUk=dAhrFcplR}ZF*cI@72^mjy%TI1=gn6#gUcSy?&MgOF_qGEo zy?$>0Kxopy+vUW5ctOzDvht>EfLbXQ7qZR#Y&_$g0{^Or58IZs)G=<-O~(J}UVU&o zTSMzyE5_X?dv42NYtK^rICbZ%1w`COL|W?;yIG@@WASK-U8hu8jnw97+X%yqQvxiv zsjQxJH?_ybF{9oR3iX^Vm7Mrtk@{n<%Uao@6lWv9Nm@YJ=$XnBY0F)_)2Svzb+qSD z46~zrD)*V?V~|fTvgU4XNJdRn-bT6FoN23f%FVgbb)Pal`8MJm5I(b_0H&v){vqUL z*GA=Dm1}B{+#^jvy|n+ZQcuxgvMubu08wo}3>m8@{mR#uxv9#lz)QTYZ^{LVc3xLe z^3JK5bJGv#y{k5Y0>X1<{(5fgP-OUva)pKPatCOk<}19HN$2nXbH}Za5p7UkRCf`r zuK?hq4re~Pzu~5)@9fVY|dxqL_*qz}e$!t^^uFy27Id{D3=o&&6DSRn}YYhoHegO3uH zyE$f&rNFW9H_nh`v&a#Qa&$vL3@1Jo&d@C9E^fp7| z*U80}kb}{401ZPN!AV(+T$q!88nkZ$=#5;EX~uhPr%&D4HSUtEz=Yi`?$Sr#i@JFa}uePD2oy-WZfq4&p-gD^LD6?&Hwg zMti-V>`H0-2&{`y1qlqj`?f%aunEpRd2rxsl>9^Hzqp6cM~w0InD zb)cJGQ@2~<^Fk4ZKQI*qZ;u`v&6zBPi7F`*7oh5XNt7&$F}0O%ZDnL(qmgDF-fAPr zW%W3+_724@%|AhL>>)04Cg$Ou8YT5ICG$V5=u%^kb>fu#(TL)J$TNP!#fsyf-IYJV zU1_$^;klt1L(e_xY9t|b$m8Mo!|x7SPwS6P+% zXr)?!K|}rISVov!cTVMh&&JZF6^b1oXhp37>E-+ptIb}S08+uAKC{2_LJN20koUR1c>lB#61PP&>K{^3PW~AMG>OafC@j5Y+6qeBGDmL zd)rc#rd0Y|-;R4P-&%+>TLXRmA?~tXAaJYJC$?6#!&|^dj3Rh~wuk~Ra=cDB8DTN2 zOxT((QE(a*_1l+9UhiM{weB&ht!TUz1G0MMz&2U}@p2wtsu(FY7hz}yDR7u3&SeII zLWvnkQ?AckyMpRiDpk}trKR|QH_{eqYgmL|ubQaf%9>~$w!IUHULJb(ZpMr&Pc_HY zN{TK#G0lu|7bvKa~ybm`lWjWK(#v8enqqby$ z>}H6BpwRPY4N)Cl3Sy7k6FY-XeDRk!4qeu>13!xF)`an0QeIi`_O&35tGEp6DF}2B zRrfa9p?7Z_9T@)-V3Q2Fd{8)9%^57=a2!0Mc@Jm`7^wQQ)TVCesC12|#KYeGu66#B z;J`;kf#m>S($c8&;=i1;C(SwQSbm@seIR!ilvh=%M@@fjhs0+rX0FEfj9Ko=@d#c2 zs58`X8oIn|5FDAQ_XIe9TfR6zcL&p9FlpYaNmt7oqBPn6iI$tFI$L%%k@>x@Jld!0 z#w;G&@kRh`FN*CZb*jH+VYyA`AQ~)EG%;kMI=GjDsPdOys-K|cigb6T*Eo+t;evA5 z?w}~%7nH886Ah}K^OctvVgH-7mV)63Jguj|qF*v1PT4)|CftXmIgYexFPvPY#B-UTVHYJWL(YaMo z{ESk5le68KnUZc83P1QH*t~Y17}o0%sDo33Iq22ATTm;YHN0G*b$zet6yJU%v)HdD z6iL0B-m1fq^qCm83Er_d^V&KrjG89%KU@+7YX@uVvsZP=#s2yHD$Y2ut$&6vkMk`wX#`^XJoFZ_;$lhn4@IJ zfY|dj_~mU2-1_iEyx#qb5x6_*r#MI|0We*RhTOkcYJp#|_!5{Ue4|&A*7c}UjK4V3 zXkFhG<@P#(fK>c8F}t}gZbpeQ`S#y=p}&dl3X-VLXGq)=))U)u7oCDyt5$AyJUdNq{teV&_OG!w>rr1)HEng8t2&_e}mwz_< zZ1B;FVjAUBCV~*LuTXtVW+E(WDO;CN-~T;g?y$8K`FYH|Wm)U0EVg?l7gI0@z1LuR zC$L@>$r1L6us425u)sWyF(yBFmTyodM;0x%Uz=tyUb`k{dFA{wxxysfxVI9$GohjCpn#6wtZq7t!b4=~Ju6sgsaVRQ~81kWFkb zfYERmPL;m%=|KU{bgAIloaBN1_R{7GOmkx5k(MtSmW%{{YiYg(8z$2#X5@l*oJ_+A z4uRhrd%E;lBWX>o1V(0OzGurV7{6swN{wCTr_$S4{y&G{-b&YbGu6(W+vu)$gS|rF8b@O-yxLc|B(2XZwwOh5v$?Il9 zY_k9qR8y+c5oyPioMDA1VxrZ7tqrvj*&KeYm3(ROP^D-GgT$e1Y2^|2xYV@gvk?h_ zu9g#5Ylw)ZIrbb=gm6#2_JrrnrsYtt=*9Vyk^IrDlO#gKlPK_XZ&YP$&uqQCAd+SF z;LgIhb=J}cft{`aJyON@o?XLsx%;}F{%_s4tGtLp*IikOVJI-ik8O+nVbmxphgo$T zaui4u5J_BiJv-K)M&s5lB(F-WSFW~OKpML1VtehJH9y%pe?s^Iy*K5isti(aPOJ># zr}d1vU8`tZhB&p@o-+r@I*2lM=7XuI@ZN7>UaDW{R#}qC^PMeoXbiaCNIVo<+RLvM zfD$m65kc3VC}PaNYe$B0A$>uNw7~GR@uv4A)%XVo|KvFU8@H>64`jRZ#~ooCTbmqz zH<1Ojhp~J1iqRzl%Ce66!m2}7-@Xor6iLh+$B#7Mleb`q``!tvk>`;WluJt8*IEX< zV3Qc(Y-Lpo&hXxpT90{c^DhHKPASo*xJ($bHP+>NtH8Y z591pS!D!Ddt=rp*=v#7<#t%x3&9wYt6Ht#ls(MQ#e-1gYzHAspwvV(R;FPh~b4w!m z@kfTT3U8gVB%S(XbJkz9b8wd{IGJM-)%a%vs%pB;iZOXc?*~1@dt1UnBVfHuWsNp) z2?b=^xEkzpyA0)+b;SJBc3mP-+N7heR2d{27Q{cMy)%YnkVw$mzcWFOwB$0OX_!7E zYxo>m-#a^hiJ{nV>Yso!_utjCpnn0*JcWkefRnD@=cUP-LFKlNGnAK<@9v|pS98;7 zrn)l2WO&emEvS0%!DQO1R83Q2kI#dK+clb(igyUEp7p$&%1#CTF0X3PBnvIa(J>)Z zi}{b)V!UfkB4oTIdUj6R!b*>d2{pxzU3IO|(#r~AuKl?VJ#scA%fsW;Bl1y?d?%YuQ1LGJ zqNMKOq)3AL*9gMHTRK29kqE2;Bc<3_bT&OPO>03msQyozNW*x+C6=@uyWq@2L$@yZ zASle^Mrl%GP149ng8WBBX^HplQNQ#vz2a!g#tNtR)04fdy%vUkk6gq=*1;cw&hk4c zWNcvzC#`u$qKMWqj=gC(=llk6*?JkG9?bt`(~LNwz@H)SM>UsbfY(C$r1&uI>chl) z{Dm`Qq$@RR?<_>fsa0{1!#qUrrRqmNe_AuAMIEM3cqs5kf2ux9>HEh4ES9hHoSg0* zR$$H$(Ne!!=j(v&bzK{;DW|==dy>^HqLPx&DF{S30;>NsJOWGqM-|cFKhYxq@VxyI zk4+A;aA7R|x3RpnnZ<5dmvXQVffXt5RMu9V555vB3Ulw}AnA=Ojzi|fILwYUAb_Wl zIHQXRmtrw^%jUD6HzG4TVCyfRBK+4>E~Vm04c2eBkPBp>v5&j3Bl`ryA6OK8ZFIjQ zAykUDQ)Lyx@wp#**O&a?{Ttv9NXhtx!c}~_d83PPP2YZH;-0}6ma&W0yJ8;Ru7I+g zMp@K=iiioAT~<((-~ZLlbw)Lrb!+B2iZiwW1eAIQ5TuErgbw3KM?efUp^QXoh)4o~ zgr+keLTDL!5r{((2n3`g5J+GKDMADcp@ycRx6nZX=#Ag3yS{JknmcRVpLeY}zut4+ z_3m@dKKqn+@8>y3qODITCF~DwL?I1!V#oK)=NI_=PJJh4NK`Bdm1^p{Rn|~#1-dnOs5!PVj8lcck)u?qQ8M2Fvh^vt*u{%k z$LhZ1fx9lsW$oU=R1b&Vj1Ytj+@+*QmSG_A?Cv}dv!@lZzGk_*enO^bExImBcf%m8 zM)MWJbNP7NmCCOGhmcTk*KVbpw_lpc?!HIo8Z!D`O}b=-9$4Dyx5C_t#gf%a-we$> zzi|1v@R`KyY_6>88+Xs7L7ni)!;R7aw>R;GF;#GR{hd9VK4`RXQ4k2Ff=pw$AUKV} z6SMhQYuv>;@gIH^f&3;+4c%r#p>PyYEW;}MY-FCV>u{_d8KaR(=^{9%%t{Sye&TZ# z9&MG=i5h)sL@pL2FwSbJ}Q%=rN6 zRPAf9T(c1e!Ah`(*;+tKMp@s4KpU&CO%lCNj1ns6dAbp{sRm`u5i~!8@6_~^!aoo8 z*9jQ}B}aiChf@16rEsYy?~DIY;W9NH-{3L-%jy_>z~UCC+3pnF3azthvDB7&O6cfZ zmd;@9yTdc+(Q9Q)qtW93UfeVaZ9*3>7_0S%l}6voI1+jC{`mz7cX7#`yu@Xz!h~Xb z!T1c@IRl>FUu7NF4(BrmG)|YUhvy9k%*=V4T3cIWFZb@QbF1w9r{c^cfONPLdy2EQ zUa>&^U>-=epX5L?63*o>)T41ubg7v67uQyaU#4y?#F}71QTS zYB>Y*>JhEc>nGaz2CMzLs2;AX;60Q>r>xyG=N1Oj)0{=5}k+hfLygxvGhD;w1gSn5DM+iNy#Rnn1^@(M0Y4P$(if|?6 z9xWjVEBOzkqZnbA2xLASQkPhjmJS#lrhXmVSlf2-t*%N&z>N;~>Bl;ilNDgjSi>ot zDCSAKWw%HJMddlW$6Ndk~-px`YPKroZd`vG-O5b0gWjtst4XSbyfimQECYX zx0H$+-}Z;kTMDT;H28)2O$3T2Zvs4G9N3yGF zsNS;(9+h%Kj+7iJ;xD}6aHJYg5YByIPBZ>q*_fkmcRqy4G>9(AU~q*jmL$BqLY~T> zch6%y*oL@XLqsV)2=zD!r}bJvJKcgyp{BmUl%B%rz^vTN4mgr$cRYR$e~H#d?acyD z>`#h5nJ(0kL6`hCD@C#LeYHflr&T$XX_^fI%M%>LyT)gd4H@v^s7v>&<rE6Tcy z*1fZA__RBWT3OM09z<>~F+l0q9KTb?y*l8@ z>R#ZAlAkRhbycd;omZa0AW;b)0C!u_JZ^&=o(_vLf+X)Qr=K`+C3M?@2aVk2BV*ER zw`Vf)q%&O(Rdx&FvV$MRXH%5p(4*NB`b~2VHrTO|A}xJvpgzVcS@~=7MpQNu9%Lji ze(v4UqpE2?#CCU{p3$oLgCL)7mb>+r0~9h$Qbv`Em64HUfC3u*i;b?+JylvzkctWK zq*SptH*aIa81Zwn!>-;z0dlDFwo6$hF~_}Oy)mEomeoI6oxr-yLsEOA6jK}NZr$iY z8Y#m22}KvYihA8y=|0!-nl*1~Gr+bx6>a!b&z3p~Nuf~au2udng&VZOuRh;@#&@$( zoqK=25m4+?@$8)s8^(KDpPIZW^YXJ`!!xk?Wrir}@>(ae!gYkXVEQoTkvG*p#h|bd(IYW1C-r#YDR7$gkpXbE zKASdkHM3hQ)%r`7!&+ql9ce)npr?Lmv%rL*rtqE&g3UgE6 zdegNM>98-nFW;2Hz8W5*1O%GY{4(dTOJ8lj?^DAu&FWs|)H6#t?j#>`8uNXQ_++ldC<<*OE(e(O^rTRQ_ z5CR&sb@h65Vj7w_UBMphD95wa8*S(kFRsNJKU!bR2OHx=?=g!#g59i=n%rzkLMP<1 zaTpnlY-}%7QeW+tg65ASy6Cdu<8foNklRV*JbhwRx#!O?r|K?t4w<1>kWy$x1Few~ z*`c$`sZz%$XKCFW{X6~%%@|+YP&(JwYI#Z=8^>R|_0ntW?lCg zfi_3uQ5iCmzjf>@JR~kq4@=jsHazA*s(u|6OS>RlT;E*u{`J@v5HtmB$+8$DbbUmY zGd&=Oia<`B*~IyII@o;g&TWvOrRKH~J4|^ow?ECfdM{!WOJ8+V%xk3y2((K$dnr}V zvCtL8J|_lCaeg;IKLm#X4$;`!QWy7A9nZCDtYx{!4r8`DzrO%A5G;UD zIri+kgUt^LwLR#cC*(U5PIHx~H>6M5RNWAA`C}{oI>*nWBJYdW3|n%Q&<45A`C4CA ztm_Z9DVjw+Gv!4GYKqjB`y)OudC`H#wYzQEMoRn~WBg~-fKA%U%il%+2OzY#S{%!8 zS-${NRU-+v+o8IuhidB(h5-cw*-wPlDg=hMmn~{A!$)E8MY^tUrtB&&oI0eOxSv$; zTFRp(=eWOv?TdZTKBlXVjxDR_jP#qDsjW&8ac>DI_|0RtRLRsXdv{RiN2BSH6lA-B zFDG!8H$NbFl|Stq1cSTOg?r6v+|Y>`0vM&qFDfLUo+;mlcdBcNvl?brL66q9oD!$w zPYb_}D@rA^w|#!|>r4xs#!`f6&b8-#LPR~3`OoNYl<=H_t)`Q#eJ$e#25<<|^`C!Q z-~U#qckqet!rRg*E;Oyc}VT?etZRn0-yM7>u0*~M^&eBrN@*l9**lk zVc@fWRK6$F;dhbC^L1mBBI+9&Ih2$EQK(Q^OLK0xt3)90!n_<7kBN%b=uR1JaH_4M zyBNG%P#w@DajYFJQg9OU)^4Wt2c6Kb31$feDWOj0(V&d-hsa1sUu)bjH@Y9#yrO?q`9Gfc+qVcu)=#w;^>`-)N5bzGes->SGLa@xv6bP>a<~L4D>7al`F9gn z!$;2_?(tA`C5)Sd&jQaVt;pu++CmUrNBTczk`E`a>uZ2|N^+`fR@iX=JB+g9J)q&e z3MzZrC*NfOng=a`D^n&-;hbd+h6fUMCUfu%hx5e1-@){bnQNYy~Uz+Evd!%^9tYjXU(3i%cFq zDAn`cq-JyO(Fg&z zx<`n6)urN?NIkpR#$m}%e8?Y43wAzZqy8jG>&MG9$L{Ztq%ULJ7iths0>R7ItjkI# zR=ve51+}6o#M&S&9pl=fCOje$JVCx#*2|t4U@ve1n)4^p{;KG-)U-UM%~jaw8!hDm zrt7hY8HcHmK&ubYuo~DCVVXL?h6U_Bh6qaX+qg(n)mGg2md^{TjuD1Y<16XM1M{VU zSE_8*cL%^geVG3X1mOdKfChsni(EZb6~$N#DRsQN!cvrVk#1PTnTl?DA`ftmn7-{B zY?AvDeY0!Su$(}2bkZM34Gu83AX2(JW_k2Q$p|FQC_Epz5MQ0go#$D1$mbrYKD@sq zA*#MhTu2{M38PGIwkon6ET1>w0uf!64(xz~H3!0tPR)f^ZlrB`y?=A-gxrHN9q!55 z+f{ld3*bIOFvG{m~||T%y?JBV!!{;AGD Date: Sat, 7 Jun 2025 09:17:36 +0200 Subject: [PATCH 1770/2083] Remove the old artwork (#6874) --- artwork/README.rst | 20 -------------------- artwork/qlassik.zip | Bin 120204 -> 0 bytes artwork/scrapy-blog-logo.xcf | Bin 52428 -> 0 bytes artwork/scrapy-logo.jpg | Bin 23398 -> 0 bytes 4 files changed, 20 deletions(-) delete mode 100644 artwork/README.rst delete mode 100644 artwork/qlassik.zip delete mode 100644 artwork/scrapy-blog-logo.xcf delete mode 100644 artwork/scrapy-logo.jpg diff --git a/artwork/README.rst b/artwork/README.rst deleted file mode 100644 index c1880ef6c..000000000 --- a/artwork/README.rst +++ /dev/null @@ -1,20 +0,0 @@ -============== -Scrapy artwork -============== - -This folder contains the Scrapy artwork resources such as logos and fonts. - -scrapy-logo.jpg ---------------- - -The main Scrapy logo, in JPEG format. - -qlassik.zip ------------ - -The font used for the Scrapy logo. Homepage: https://www.dafont.com/qlassik.font - -scrapy-blog.logo.xcf --------------------- - -The logo used in the Scrapy blog, in Gimp format. diff --git a/artwork/qlassik.zip b/artwork/qlassik.zip deleted file mode 100644 index 2885c06ef4bab2fd9027bf748bd5ad2a69eb857f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 120204 zcmV(+K;6GkO9KQH000080H&bGI*MgI(|wQt0Pr0F01f~E08wmVb8~5HUsOUabaZCy zy?LA@MVU7q5qaO0m06W_SJvHKUENh(-Cfo9HQjUH1H%l*a1Q4TH;5M^iU%U#feN0i zB7(Z2;_HYgq9U&Ax*m8dBI1h2x+-|X)cZtaRZq`=``gd&^ZvO*=Tli(RT=R-@f_dh ziHHPYgir_wkI=g9+jf2LW50Mk!l@?^!s@nN_2!3ub^U%W*@qDl zpF~J_?979wUp)QD+@BEAzl@N)^~@_T_mK^nF7>3W^jL=`^&)t9eS^qWhid*3`9lqZ>7cPje;ywlMm*9Qz+=G{2m3?3R9(YgN zLl^A7^g`iD;VOjfJ#ejj;G#26-}jWi7Gd)dXztpBr(bn3w-f&nK06nF?_YTO!Tn#o z|K{H!9KHsAFI{}mWtR{C$Nm<=9`GaVz4+4o7au}s0Qz$Sq7tC8z-V#J)B2OATGM|- z3il-C=1V*Gd?0xJMXU3f4&v_P-UM$IL{RV#en)=W*^M&XkKyxW?ml*>RsZljy#n;* zNb3Cp{;BXgkPAEK@V)qHU=or0$@TF5hTs+d7&Xz~WJ{2G_qqm=k8DT&#@_pjUwA3I z!{14_4{yTAM>xWo`7n71UVLcgXWg&=EI9D}CdHx2_?{{Q4Wuz88`~~^Jo}9 zg&OFSNWn!^fY%bA70DoKkZHK*ooJlQ zqda~HK6@4P`zy#GeW;CpjSP-MQT%(P;jh8*E$HLf=mNYJ#o;xDKMB&Ne7z2?KZM5d z#mL7G!*L@V5@`J$NCIBl=qqRxeXa8ud;@Bse?e_>C#vHw!trU=Rs(+-zF&j)C&K#~ zXzyV(3Kn6LQ`tQ%{A#4)^FS6&bP0M84FI1T@SmZ-_ahCx1=`;Y#~0yq~t2dZ*B$n+AFhI_a1*HMYw3;q5*jM<|wPIOFefHvssQ)mlW z1Ub7%eZGT^6@EF4)2ai;sf7N$^E)`6fa4F{G4i`du6ItyiH_0ggN;#doL=`l$Lry^ z0)BhL(FgD`guc@GBOJei$`vo4`iO%x(g?Pf(dNnOuT$OlLtJ@V-fs$iN?k@jnUlgxVO@b$Y-# z`6lpAZF~@H4GwBwqBrKdYld4 ze+K%)gOB+*yr(wzMKr?fj@s*iZu<{|%&DzC%>2d#IT6}E8P4B}A|Uf|kjXgc_XNmt zg0*)kdK;dEdzC;|4}$(spNr`~q@%yc9V8>U^DOvq1*nQtpc>ME>Oh}C2GW6=$N*{~ z6KDuooxdX+g@A^U4b(wlpe}Mce?t-E0*#^wP!C0c#(@46#gPXzfnq?DC=N7*5}m(* zZ6<+cPzq=krGfe=)A=*>H48M4e4quC16l<7C$PCZ&@w6jt)L>%K2!pW=tpIsRa5~w zfciR5qZ;Z5I*6)3htL4fI;wT3H>Y$M4FYYVA)q6u-uVM)Q3L208V1@zO`vTw()m3a zN25R|&=}B3)B-vM^mk|)wSmr{aiFtk0_Yr??EDv+M^iu-&@|9RGy`-En(h1+EulG} z%V-|xTC@Ok9nk+o>(L_64QLI}jc5tzCbZo74cd&>0^Nev0o{t$1Kox;bbgJtqm4j! zpiMw`qRl{e0sR#^4s8K?JlYC$H`>yMjrMhZj?O`+0zDU< z2J}31I?(fhK7lSkX8=8b&IEc8odxtlw7>H+bP+lm=*8$9pqHR?fnJKv>pYGwL+1m% z99;nP3UmPIl|X-rUVsh)y$W3j^o8gmpf5rfcOFA8MwbA63Az;MOVMROuSS=5eu7?x zt^j%sx)SKi(F;0{qH7s_1-c68E71#qUWZ=P`7wGGdNI&fqn7}E4SFfi*P^RCKSI}| zmjQhpx(4VC=;c6PkFM?f5Z%b=P3RRsZ$_^K`UZ4e=LhJG=v6@9gkBBw&FD2iZ$Yo^ zd>>-n^+4Z>UI+AT=mwy-0{tGk4ZR-d+tH0c-+^ud`c8Cn=ey`#=nX*Mjot|KJ?Kq9 zZ%1$LdUHZ|C38L+BGgzl8n|(1+0{fqoh2R{?Dt0{Rv7DWLy~?gRQ&^y$vO zqJKm81N|EM3{YqX=-1H$ov)y8pw9vQCVCL)x6tQ1|AHQ2^xNnQK)-{&2=u$?q0X1l z_t2Msejhyy^atq6K!1q-rSmZQ5k!rL(T~x;0(}&H73fdUzjeNZ9%J;UfP=n-9*0Qt zB|wQ^?>vN_K;HoRbM#H1|AD>*^cU!n&KJ>>=-WVliM|8$SLnMye~rG^`2zY4qyLG% z5A?U_2SEP|{jl?S^gHw;pua~y2Koo|DA1?SPdX2xr_p0T{|K@8LG&l|IM6?%pLITm z{=(>A(Gx)bhJFt8@8~}|51?nzFMvLao&PG=wWO?dGPH;@abjn<$d78tKhq9;IoIoS2w^%H^DcLf=_ONFCGUUJPE#c z8gyj_e90Vm+6C~iYrwNEgGXHlo^%6v&`sbuw}8jo2A*;Uc*tGg8IK2#cmjCB2f+`1 z1@sGny}u4@`$n+TTfy$$2exzvSk+x%O?QL+{4-e3hrwPx2G;UXu#bDe7Cr%1aR@Bn zGob(XgNA(%{OGqq_s#*`x(0mAtH9@63|{O9;HzE)I`$*btw+JXLPiE!fIzz*1AY1# zXwXZ*UtI{k_zBRXpTZ~9=UxE*5`q7@0ldSD!ISL=zjrG5(^-hm*Mg2c0KVX4_{Qnr zlg|WS{9W+D=YrQg2>$CP@MJfGM|vZ8qBnyFdMo&xZ-TFRC-{~3fQP&U{Om75H!lP4 zdL`)PufenO9L-O!0vQS*Sp?%&0ol$%e@}znUkyIz4v@h|f#XjDAJ3u=9>$A!7d{$p4S&*K@;{Wz=gN68AI>N8 znfyThwEX_kSN_uZJLzG3)=V@&;@FB zpT!E6nSHziBGP*y)?5O*9BB5hLCb#wKK9SxZT}9Q@)_`%f5B<=Cy?GB@DzFqGS;WT zzf#MhR&_43uk*l?zY8|}pD+^tYZL!#6aQ-yN81E6hiaP+A!YlXk79h|-b1)^-66D; z{uD%f?$mvQhY+s${_=TC_u$ju4XMF}Y7Wl0n!olQuC#XNp2A-LdjI+jXI<~F_0K*1 ztb6zpdxa17U%$8R--CATIS=S@dvf>8?@g|p?%%t267Iv(`|#{O*YAZpoZr2}`RopG zgJqo_jvGl+5kj3*Qn~dJ3;EJulYn zvjlw>t8gL*ciFY)p84cGXz%sc2k#2Gd#<_u`sDS%NB8|9^tod$V07$-`R)af1id3y zT0VrY+0H(?rjSe03x!-E2mRc;1Z^v|O}q9iLr-&i2NA2qQeQ*>G3~^+l0HB$rvjcj zgx1y%p&f7};3(A_!|l;_d2F=R9+}1M!Wf*+lJ-ceJc~!75m%H}pQf?ocqfLcUy?1*4!}fcyDaP`PPA64vmXz~e;@Yv~-X@uv{1WpbRxO=3ae zCwZBNFYr8G3cjEVr-BDGRM|5HS=a*~@8c!mUoAo6i6>^JGLFP!MN>$qoQ?2OwCPzT zKgvnoa77Dv<5$$4PYlF^Ki%$II{b(7 zLOmJohqtGODr;^K6;1F?x$;%lUfO5NqAoi-55D}`7dK;{of)5)T^O6j{LJ{|?EKgi zvxP+GucV*c3f%97eBuzQfCTDrEY$A;-+wI^`ugAu3DYR;O#^p zm^VO{nxN_Bv88IGDVvVQ3*lU9sGO+Jm9y=^vNdpgEjLzly|7r&Oubx9Z$18mVtIPF zFD%n}5R7v8ujp3teaNjWkO&9T5a6(>UV!1#5s$c%$w{KbY`aY@pe<&@qG+3#Xdzdz zLb5H`VJu>fkbA8tA?c{hyM3js8sY>oyOblm?9#SoI?Fgp{s7q=)jtQ?k!d?rQXtDA zZUzUY2Jc=UDG#TGSSUQ%?k}}w;~|%u-hzu8u060j70W0~SHAxOy#L)-ZVq$l$nw|V z-i6LG`uobg#U2l%w9_LQE{*Bf!N|_y8T`SGvqH?WC&%H|GjZD`6WegPtWzvIO_!d%{6{N{}UwVz=;iQhxs2@%VI2w#D`Qi3)ldVdqwaTzQli(46DVX<8C zqR>#K+-gg$s3(b_C%};eQUl=~CYs}5ON&G=S9A&E>lBroA&MbSUE~FOt!(IvLddp- zGi6@AUg1SUBH#vv*Qv64Ig|WhDYYLAsu|U7lc*GS4a0K9x9*G<8SF9T`7E=kCf4^+M z{xmBj;d=`Hu5iVYILur2wfWp@85}}N=Vrhp2O&R~*jPd&p<0sf>Btz>p6DBi5lw~k zoaUy9j5$pLJQ-<5YSn@+tNthi2+dclXd<|00eu@I+=SepMk)jPZFTig4(?CmON&O9 zD5%Jf?CL-@ENY_UR8K2t8nJ)Q2?FTgGOr0lNcFLMmO6g|1a*K_7XV)L?llUXlc)ig zOH+9VW3Q0*nvSPm!J&3;4Bd5j6(3~@r_ z5@C*DjFp5T%Cab&^w^c0sw!9tDVD8-Lpl$BtoqU$-x!R4r-N^UKF)xZ9YQ*bPlr%H z95wLJY1EH&=Aj)XsIghD7pB;_aDW8BWsd}71e(Hh99%FBQzqFU6A=J}0&X>fYty9= z#zH<}<|>VJZbP}SXhMt)w+d^nXz+x{+UV|?++fuy=TrFSq7k-vD-qUW{o}r9h@}lh zx34i-#=Ku9gu|SwSy5~0oSo~6wVJG%l$ToP0emy?vIQ(8koPk1fB+*eGaeYq0E5gk zV*w=?^(e-eC6)4M03-r$#TXbO^gZPYKAOR^T%a$Kh?V(u<%K;XeGN?&K-I!6zcf{G zLE_@p3E7ch95_*k50@b9WTJ)XC=m$g=S z5#Tu1jjv7Myak^`fb;txBM#1}6}HA`Af=27RNtCWfKm~_nbc?F2#lndAHcx?7o658 zRY;K1EbidU%y>Oy=tR!+4G#8)%%Z7Az@xe)oj?q9L`5(ZTPxTCCf;CwUiUXvK_{)E zgF$2UjOom-+qBfOj1b{Th*G-*IZrQ#u4$7L~!Rr+^ z_JP$Aj)>9HSV;*hpr>bDj3Y`cstH)r;sevg_5{QX&=P;^mbEdexMM2gE8Tet?Cx!l z!BM`bUzi5Is%RQ%!8lE0=I}u6V1T4@U~d6L>TyFAl&Ud9iCkB;(?sac$J?OW$$mjt zJXrLyGAGWDMk^V+FT=}%nbK4(meGX#aMtu18=A-M|D}}+A9qd7uf|o)vBP|RIK88l z7)zhE&#g?9#>YVzBIkw^qK3JAb1v6BekOlq;Ik{8uj4n7Q&FCI*ib!)OMt@KU57wB z2t$yg-4~?1IE2@DvkP-xYR8m6XcJkqYjG@t;%k+m;aWk_GYLC8bIaRq8pm*%7qzS@ zkdSYd_g{S3c_$<>M==yf=N0kW$&IK6659+>EU@=996rb~3^t!;nm^DzTV?$eKZ4NO zD>(&-cqDMf(8*}Uje1dU7I%G^(-n6)0H}C(t{88u9c-;1bZjd-?&Xb8C_Xq}mJO-Y zD%xSwD8~Fx+hH)$`dBHPi--0)rL^t1v1Z@I9B!l*221meWOjHqn;mxh3ueZ0hGON} zff(lE{bQNTP}t7ck&k%3?Z&;y;^2T5$sZmy;^l;2bYiL4Vs&&PkbSE2S$qq~ewoRB z7;hOFFlBmV|)2+#V zU(`iC2I0d{5;mw$u9>&v!y6hS%T>L4GJvR=NSpBwUZ#vC+q335knnFdTvok;g4o(&37yT2i{$7cMn@Q*+{b zGB)`1Y$Ad$_QxXN_+VheZZw=u+mQWE_yu1#aU!WZy66^$Q}JrKaQGL=WFnu+j0b%l z=$t`5PR>Rbp_j2fOVDQ*`YccgDFuj&LNJacZ`33!EElOTTn5P>_%m>St+JGdZ-beR z2K-T12>-bek^V`b%PYRUoBfS2AylIDK69Aqek6lQ-5^>-JM;K8m+TmTC?m8_zIyX( zZp)1tKh4qHn4Kq@PG7jUg)dr6yU$tb5?xSKN}F z**n=cn{o!Z@nm!clW5zG%O>wvPr7NpY{@6S=A`-?Z#m)eT>Kk`$ z3jhMKgmR78qUcB*SO{+35AdC4)c(4i)rbz!-^4r@nVfFff*@#u zYzn4eSNh7KNXEdtk~fs_-|x`EYRHj{RK)1tI?~*mmn5E3ZF%m!ciz^DfN(fzZi|9-#^v)JHDOVii&6yt!2Jn>T;U{E+Z(1RL0v__YACM z22+JPg8G@K?*&a@qg!m6W(c8*rLk7C-K6#uTmh4s!eS&cl^I@Ji)6!6NRNf}q!-Fh z?HDT0miLH7ZFm_=^TV1iNo6w>cCuI)tth@gl=BpU0!6}meeLYnlm+iZRf^>5{>09) z1TZ*9xb<5_RcKepA~39s)&!Za0`K7C^PS)0&yYJHhoF{3u{ol75%t@&J|fb<6m;3- zg8B(0YJrodagJt8quEA2!pR~pm`WreTM%l6!t7A$v^~W&HegfNobR7GeYv!||1@%k zX4#S*m$AsJJXN>+(z)wj`Qoq!F3{Fpf6LYTK7UHh5A1vz#^FC<9Ac=C^^RtZVdm$! z6*05?(H?DK8bS+s$qr0*z2aA@&aMOV@&0DCF5*L8GBQyd=-fRuVvTHM#H!XL(THh1{Y)9mT{Rg6G;lcFY`#4e2 zl{c*ezktgrkPk;RN5enK&1Q&>iwS|lUyY?K;preg(|(-?{mL-uDBZF1@*t1$b6a<* zcRV|30Rl}qf8`WSOL^BkKm^rP-U^;4!O6tUjm4m$QjF*D_dRH6=NpzufG2&>lOK@x zp#@e`D%5+~0O-mnHDl6b6`4Q=RZwbfY$Xd`(WIUi1tp?M0$z`aq!mhe4dj58lo_2a z$+kSUXRgpH=1-qnrB{`qlh4VFEEbakvf|nX3E)K?$W-|pQw;*#D%40T@Sb$@|KJ9L`iI!D@ zAOV5VX0pWB%l2^fT0s`Gebjl&g_6ksR+WABPAYQ{J`IcyhvS+|iYK)2hWduI%qz*n zcfV2aB(8YE*x`E{YvB^~YohZkx&`_b)OXC4u>>iO(eT82!D^8-=D?qcmlWELSSg7d zeJyj&#GNNkTr}9lX2j>bz&kN<4A+^uZ02 z16G)Zo~W)H0*4rnf5YE^J}g3f2>hwdB6rXiItGJvTC=|onSgOz=3}S3XCRBAqeg=w z&?ZIJQCiGowStV`v2h!~Gs?6+JQWo5eB{8Gpo*#^$0c47oZ8S(Bvv1&htqE4fZq#zL8sT@yc6T}XgKO(cG^?(OSA(1v7)mXMg9n@M?#8#c@hDFV+C zo~y^AqpeXdk`umIosP*|V52au_*F;26<%nZ)D}V zNvDut$a<42vyh`^~rJS%&NltDxSxi5q;L5F-^ zj@KiE=gn#uunp+CtZ+i50=Xmn1XJHh|KzX}mh6Hp*lJ8Os!Ns@|gh_Kze6Znr>HydsY@^p9Qc8tveMr@WBH}W-~z3wgKGtkOI=Y8EZN-N&iKQVaoGu} ziWn|u+;FBpN^y-UMqM9d`XbSiMG&($ysh2ZP>*s%wk=+?3X=oHX4W8w@4qN+N?dLK zbbxj8om0r)K^84QoDvGWvcSxU)(lyBhQ?GDR#!U9?2tN-Ag>H$RtbVE`D84o(Qzy# z{q42d4nJK>n-e=mVz0b<{?wUFvbk;mi@F<9Y&9{SO%8inT*Hl$68U}=?3xY^qa@~#V&tAyjSHEG+ znTt8f6ZQAs0sJ9y1j(c-A!DnH-Ans=#r$`%l<&YJPS&%lDnM zV+JoO^V2gk1a~m*)lGg6KK~BLm#9`}>p?7cSfTBC=9JldoVdD}_1tOc?@Z52mLQ6{ z*}rN!+03TS*wM? z`QQ)+JMUAj@f<90MB)TZwySB}EDW6s}N5YaI z$cmQUe&VSE=iTTk3gP5zZt*R^&Tx+9}Aq~C6b10=?Z{Z(~KINDDq~>gO9<3IQ>yNiFFMt${6X! zZRd=g+-U3_J#go;q(W{4;6n&ondRE-8(y*Bku*X?kT?9-xL`6qH8-E}rWfW@QmHXk zF9@;Ol>nHxATm&cLwBE zXhDpPBAC(>0IB+2)ltoiZI{BKiW|=9f+|TVQ&xrY8BJ%f)@N}-s6Q^MvK~?GT*xlC z21N*#28kS>IIgj{8*l)Y2rr~3{ldmvdQ;h-ib*`CxNY5s_mp?-*p%T!3eRja(&v$5uO3w-o->WHH*de}cC;d+2S7(hQLc-td-%3i59V-2=^FO{Idw~W0WPBau(W_H zlxf)}kV_rYNp#2{@I|tz$Wa@HRO=t?Qv?Z+p&Cv{WysW>)M5hLnWh+xCu5F4-T)zi zKuQ*2kyD2D%IFW8hyuM{Yw?sE&d@ zQ#?Xz3VuDn<$bKqR6=5RG=no%;iV=<=Q~&$CQr2A4Qzo1NjQ`~pl_I1f$q@!dX3?WCSf;nCh zy?E_~uY2=af_ZE>W{lbee4@#~DJODk-gMJ-ZBZnYw+lK~;Eh}c`mA-wB^Ts5Ue_N` z;fs+JiAI>`fS5>87dV1Zmg>%SOSeByjKt)STeggd2CiDQ%dw0@cuAZ&Kidt$Jc;K^ zT!z=hq)A~v5rpiTFTHmDl%?T_pJW&8h%SP@JsTNUxEG(GH5axr0U&Vo(>C(m6R~eV?$M@SDUQv?25sf7N(~!2MI_vNB{yC zlR(AGII=c#L1s1ouj4f-VF9lgYm1M({eeQG$9vx8dYt3>#aCW2V$fn7V;%Y^b>4;V z>HHA3m zDP|coLrozmu!ZiV#3XfD6bM8_IBAP4kIt7XT`$*N%|&YkUKcCUn!nQ;ox$Zin}er? zO)r4a6PSc{Hg(k)uhJ^Iu46IjhV$UOWxMFc1`Cnm=%gnbl4FMa#6&c#N#cV>YkY@j~n1*PI^lW92Lf8$=57gYnK&_)hW#DJYtZdu>mw^^0(185 z)pHrr8d?#N`?CTUnJVU|Yw=LB;AGck`WMGlKO7k>dgH_HfNkn28}hWFu&rv@{CyUP zXQ-Ovi|tx}votAF{RI!h;&g8E;u6reXi zy-FcIMS@6b%IT~k)Mui_p;&w{M0l(#WGj2EIc4Mf)Re{(1M|C2Y%P^d zO_nV|i%P2QjPBdC{j6jvqUdHIJ6gNE17tUZmRN4ZR>!PT7Lh57xT5enC&#hwm9Zh(*V(tsiMDRBX$jfyN6lexAaYft+h- zo=G}drxiU~#iMfdR>FI^JNX;z zNQa1KO|&fOMrgS`2qj{^kitFg3P++yd5jgnTPxFm5RRD%> z(Z?Hv`d~qmGBerPYnHHm;VDJ8|*mI@K)>VS{>eT?am87wbyTF<`$dyG1RO1ER~gs=x;xMs-(O^bOENf9MZN|!1jP9xzMPbJMkI<6I>vSDU*cWtA(FzD%! zy$cX41xvL(Ox%Ip?TO~({Mc|S@K0^fwbz3mXt9;TYB%2JnHNP}FY3)6TV)PBQyQn( zBHg-jLtB@eX5RwmrPr=|SEAIdwBAC*rS`e>oyhUB})+1xtl zXXi7WY4mnzs|=a9gO1E{(Ta8dQEhhT2D;E?q#2}s&+W$z0>DGV$WDm55Y;mMt#U&_LNyU8(%q_?VeGzGmA&*XxH&`gJy>FqHGI5xtpGL3)Q{VhoZ#B z9n0m6D5hrfao>@1wL#r#O^@5MuGx+iu|+qLG6dVPc>-yWU6}ByStn%MgDz*Zroo9_ zm>6_ZVO`CZ@AaH;Djyn{tHpVBdfi&^k-VT9V$9E3g5!pHqH=H00nYwSP3J37+e@bc zoqig95Og}tP(hsK5%(bhbR{03f}=qZEt0I7=ecPmhA9qsJUS#az8J z6nzC!>>5!axKd%O&FJzdc%AGeqroW`*dU!apo2z#oW}TCkgw5C(&f}Z2<%L?<1Us2 zF;DY)E?fv@a=aqs`;BnP&SU|o<*IRoDA_8dH1T$3!*o;SiQBnE&~n|TM4X8*4Z8+x^@}H&xHKBH@9w3TokPG z3nGTFvTnJ|mM!CVrAOCRPdO_+x_;oZhks}#%Tc$O47CP{TTF$1JGXDnzx-NSUH;SP zx&g--SUY;gPG?|=ty$_c@dxqGKwGBSiVBM7gDiq#dJd(SZC1NyV4JhCAND8-)09!kQZx+%v0B0-;oExGOWTwTWmb8nr_2h6l+4jihM`@4`bnW_K*d&BKqI;V!Oh;9FtLefs1O zbfiiuuvNhVN;01C%9+|d11XPDJz!d}8Ui!kfb$3hZg4jYbFw8D`1Mw55aVz=o1Ys_ z*UH{xL2;A~%Yw9>V|R77fFDBx3>9nC!{?7E=vez01*N%q*CUqmj}2!yh-z$-h%T@6K{*ZH zF3<%=6<$(!K`=x?!8iJcA2b{p@=nmqsk=bHirJN~K=_E3GifcMS2uqO&x1}*vmCQm z)5&&g=Rpa9Cd?~~4!t7_uIYBSW)BJB00{vM0=eEVTjF3p2hc8RcmU?iuruZP$;8l1 zzP?tLWnK$~DiJ{ul3CYtYj&)59DW?|MM_bVG5`uC?HMuC>UR<&eqpo_jyZZtl_dqf z;>Yc*ZRX6#z{uo6z(W-5s!Dd#b+)Ki)1hneF~a??M6NmlI7Dbx{q4qape|HzT^y@FqcnmLyG4 z)J$VK^JhWOHId^KSvb-1NMSBpW)+;mvY(rbSI2l!3i>|Uc^bb3KLN3l`Y>9{ru)fS zpv?u;k6M8b15PCB7NV`Ty{ZgP3xz$zYO=+4N7UygK~z-;bBSStm%0;+$M~~a3;dZX zi%Oa2?GXOF-CzD>z8X_VB!p-Gk!K^ZftkK5hy+SEhIZ_bizQSvkt!S<+SxU?Sy-!UR<(SIxMx3+z z(wr4Cuu$j|6;84>*)p}W)W+P}TxGp4Kn@|RQR}1!FOekzdF=FcCn!3AP@Qz*@1t6&SZ1-bS{C$)Y*ElCVjbnq`tWmj^lxsBoe!8B*s?G_I1zb&Kkzg zP;VgwtJFt(#a*sV!|m+zP2;oKMXwo&_G+cwy9C8!PWq)F%3{iuAvD>JtJ)Dk=&PuL zWm!=fSR#h0gK~wUQBf3mIihKfpvO(sq>CLP61z@_B6MC$rqjmCtspL&p2q8zEu}*t zS_~GGMtpwlhQi!tO;eTV^xXP`>G0+;WpsK}Ap%x0Eno=YP-%Cgu{Bb7p2gcmi%xnr|ZGqpr4fJG|A;(-j$TxfbCa7x#V=xMG*63<* zCf&a78Q3#LQ>zQ=RBc$U#ZfFdg^w00RkA>m8JxxVHaTlMDLJf$ie|XsL$agAG$p&X zWabMwofC3qPH_w~Z)!E$fzYFZ6_BJxjhXHa?n`w!l~H! z#F(pas+^6)r#+|XSt(1Ecuq9Lf&Gn%t0r`2<7#IDUkFInBas|Y0#Uvq)x|L)l|EB?qyev zkFBeQT|-s`y4XVya=8EuV4FVG{}sEB{SL;9-7dOW>#&uMkgV-Sc;t3{Y_mZ6%3-6x;ik5w`r z_~lDFS@a-hf+qH23f=qXQPgbHWgrgy1UDyS6Oo~6GbiXKwqADqNNRav2Nhe#4#fBVB)G7&@kC`FY6C*?Acy_4m(F>L2np(33Ue-yt=z!be4)9)M+swkAiL_|vZ0g9E{PUgf;O~-KfD4LC z%&yI(SWTa+9KDUYkY5KGm5TMTwc9o?1^b~50^wlumJJz8ApJy=Rl2|Vx@#}%3(JD3 zIy(+tbL~r-@nEcb^ID56hSB&%t9!k9zc!l}YIk!xhixlcJ$dCkXQrzi6)k*Zx=Sht zOV>XQpzYY%tXCX8@x_Vw0FlCbyR%;x()krl+KG#VBgf8b;l$AsVWKN($ud#SSeXsG zPjiYQUH-%75}rX~+thQ>)K*J^MhN>EZLuS_E~A=bO#I z*_G)GxEGaWiFs$*?@B$$DQ4;c zW?@Upd8xOuoU4?db5&vyKd08}Rav$)$%-apHsN#SjHFqLlGgyHFNgEK8nAD*1?LRx5MVG_Pz7`&V< z_2*-pP@in+oM~#2>4Zy& zlSsuZ+G(X&9kA8k+w*H@O{_tpA?Nz45FofjaiHSMfFpHGH`~R+w9P(1K~FumO}g`y zIuyP=Bm@tD;JoNl+F)}~g%nW>#nTA~+KfS)nq_iI0oqiawC9(hO;Hcgg}wc(&C)<$ zPL@qg3fi=Lb?n!ZhtVE3a>`M$CfFNxq*|9Z@qb^phQ9z>w}Ls+ZV^1cx(ME$#Q3^y zna{2D`|@)u*b1bAK?ytE2!~7|D1i%6&8#^=3H(k-mya$A{s!ZJRL_Q7S|HtmHZeqo z?}Dnh1ktQp6emm;Pvbj57N@Yh@&wBN=>MVnh5C z$kFvL63z`rIR1ZF-#V8L$ky6HRaes#Weh_^Dy%SIT<2~<#D|_BMtM|M&(8tkN@d@2_sW)VK zK3zl@4G>+UGEh%KLKgOcZ7lZn_m!QjXY>WXw>ob|H=!3m-sG(ErnDj)W_FCb&yxsN z@Y0w*OZU2!2gIx+01h{!37r>tNQ?|O0&Y>*X?le7Bi?YWsL(b1s=w?L-le|V?%aY0 zpe+xLu=$bRYUO0tlISWcn$1-i=dE7;#u}$Bx7xuf`RD7<5dIfKg{YxhZkYB-(?dp? z={ z9Xhpg?DIRYAiL_32YVF#qaN%*z=!JipCA?$ARnb8Pvt+?<%!M(l#cbLbWu3N%Dt=H zijT$rqzyEx3W4AN(Iyrq#E!+IrUu!+llJxTPr zJ=}fd-N*(_X`nUeRCEElirFU3>zAlxyZdHWo*l93xjg?5AL~Qz4j0A>>?uBLG9yKI zgYTycel}BhcH-Cv{%k+yfj>X_ZyxyLuRQSQqbm>mx&5dI{EYOT<0Je(dkhd;H?k2@ z>z_lO(Pq*dsdt}+#fpJ5N0xvE*+sXmN>?Fp-TJF@Bn>*{zum?m1am4{fK-IMOjbvZY*#p1Y%2C*pM<1|7p>$lr+eYV;bf&d0)aY zQkAmBf<|~%*2CFq%_5xE%^l7s6-XMt-QC^1a$l++EFI{g+V9F?3!>apI7rolDppXj7{YgXCM1ofE@Nw`6iJv8AWWM0 z37RTlVtKw3kD0Pc&KGi;q{w)1IA%&pB&^D+N-RHNCrw!lWPC1}MGun?9TSIF?f5>o znFb)@W?~MxvXw|A{3O%QHfiG~xt;kQXO-s(yv|W~SnY9|DY%)Fj=Aq#IU@&0M(| z0|+aPF)#O}6UCSTX``faW(?wv!Yke911*pou8@BP?ds{5)77se=$8TC>FzxWDzeR$ zvf4ZPiZPI5i{avJ7Q+C})F4@vYm7sdTuj700k z9E)(ckPFrW5ya4!Ik1PwQeVE~t_h5B!hS`78Nn@)pS4f?Yk$`jHAW z%{%ETgY|$6nyk*Wv3tg51}Gf%R;q|WO|8JP_wM`PK`QXa9&&*PXmxQUc;(>A2oU^j zdnoBI$3dGzZmT@9DNdJ1i{!3|?}a2z4QZwdT{LOHu$(2wdf~3og3g%s$=K1QF+x0^!Ea8LOtJ zufkgz+sZiY%}|+esM>jql*zpi=ca(;O1CaVy0)8OYjg=pFp11{&p^M#m0gEJxcm5# zU_!Xf>Zr6D*PHb*vExQPhb_+%AjhB&45G6p=LnK2Ng9#TW@7cVzG_<=pZGlOxAv8y6lpt6WJgoX_S6pC-Q} z{{*OTE5n6eR~KABxp~HUseAVPx-hnSevtAH`NA<9drPaHF2&ZV5ziX*{Pk(NX;AMg zw037e8PX?>FYTL3jy3&__4e^8F>e0`G9@h=yW-PCQViuyhtH$is;C%Z0~5L2)IfYa zO?kRnAgDsUS{XXw>g@;LP))>(r8?1YA=%rJEUi6jE|Xb2Z{wD8<`Vw$`5Wl?r#nxP z9r$q=`x61>Q2U;$2lE+TH-5|j?^C^sI^}p}b>I;p1eMC}3Z}7On>Q`QcAXEngkVNE zSYRwPsn(S9Y{xq2NxK^l+TAH^BIqeAyO6N6wNzmuOxL@mOmjHhw?3NKG1Zt)49ym- zBp8&mkf=M3Dao9@XL48ETT?Byi?rY)_aA=CXpZCA{H9QD|1>$BFkD$7**`}<Cf{n_z7&Y)cFDaApR4oqA~DU zS=3vvaUW!w0;@05(Ghy9NFb6PF-e9mAuA6?X?6PUdh9DOK*3{eD$H81fY7UQT9j~! zP$V)twqtF-Waug}B1DKAkx1V2maFASB~;#2)_r$i`aHLk3WdUbeU*@rcjZV}BVj|T z#?re+U%D_=P+VIjj;Yq%^!CQ$NohY9dUDL4ymBgR9)I{rXJG9}yPTezor-%F<2BX! zJ$W1cDH;dgd@>qFJ^c%+YeQ`PR~l6q&xw_@f$kYyE1Lhuy4iaSLU(z>YE4{alJHxm ziK`TG^YbfWZEMooG&HnsIKCWTDBDquTneTH3){PA>RUKzwX?SdHE&K$> zbD~2PZZwFCh0fFXQSwp9m?&bQcDtq?>`&NN51!{S6zosyO}R76q(M0nb-kWau!2r^ zi79nn)2k-Mx=%&oo->`_V!Nf-1o*YY((Fn&k{<0>%z|&4f+m*B0^N&8LSPz>0%YGS(nvEXM4tr@h?H7;f2ihqL9WgqjTH z`sp3Tndy;<6W_bTifbYd>9ZS7Po$?`I7b%<$bw`@BkLlT0S2C^G?M9}rn%9$1(07$ znz6}palGtCjhL;Ars^aET#&}^#fQjS(N1&`+R?S#-Y&7NaBQ#B3fUoaVm+u8(8{4+ zKT@xzJGIZP2fMUs9b`XfYa6nh?o;!aJS$@@`cRu==5j~dO?s6sB=EW$_R6D`GJ6O~ zxB52F-81a62?IBn{SLOw)5!w5R(OovY>F@^^C+hZ7^`wf;3ZMe1zw?Bn-oVDI8F|U zbbl{TxRA@Um2ra3OQtLnTd-wL6ENTeQRcP1STJ0!Ey(O!JmK%+=+2Wn8W+=RY(mhs ztc<0lcrK-R^(Zah2}XV~-zf7$Rzq1^jT=Txu?lLq;&RqtM9!Oh)=mw_6p6!Mdn=3{ zk0tFhSQ8|k=X71N}}zXM3NHzP*tSdnfWD+e%Hdfq2NHLzmh7b~Hkjnb^o z@e}UK$uH>hny&&R9?K^1*5DP<|65)TpNuac^#5@dW6$U9;c{iA`gRN~XIq^oyZZn$ zbQ3o#UIi$i03XvqVG|>*k(S3`d?7*|N^bAAZM#m`w*9!qOlxFmx`oFU_sWJeV*|<+ zL2x|g6rK;sA`dTyO^cR<@H3VP($--tiGKV^C+yjA+6jA?=hrOFFD+y+ZaZ;F=}!`# zZKhB7D!-1y_tQ7yMhMSwa<{hl3y>#$0$OiR2&xBrM7yB5kPAe|9zg}7%IDedwxFH( z1Nc7pJ@qS8>qA{l?_+r|?wWrW=m)io-ZS`u?HiR|{TE`=!`bO9uZEn=OeWDxh#^7J zWLxkT{8T$7NSv&M!G*Q5rX)M8qAtU0?`^mAS2yxnx3Tq1K$+V!+XV~Nx ze19RQ>ar#K^Po3HbUR6tPeO0YV8gvR1vX!`QmY9b{MPP0KCkQcOK$He(iMOt6pwWC zQ^rWwJu+ldEU!6rV*KQVLTT}o@rhH`6sOKTvAS*~ofuwQtF0eP;{>EDoG3!dAsX`k zrS8gO>?*JO{l2@eZ-4jA+qZe!EN|8~i)ZoJ#ybU!V~P!!V1o%zT0$t4s!EfBL{ha$ znv?_;rB0i?xC;O{$Psmy@xzebLdWM7)H)zJC5dF~9%(#@e}qqxl2p z_FZ@=y?E2?^o>jD)WVIkzbS=90y+b4wZ5pU<5`6l4c^oQb?>p-QqgY`J<&Gb#|nRz zVm^I{-k%?>d!s!a4Y+7e>}kn)x(2i-Nc5=9;YmS}6hYBfhlYPjiYgcYMe!%DbovN8tbCAf+21^Y&5M+}M0N zMKU%FX~dPE zrMXvz78_r0E9$-f(%|B1p0yKHFV9h}Y_)OZyq$Ek@<>xFk2JON$hB!@A7$QaP*Goh z72+YZ_N87mWrF=OPBHl6&ejTsFxREh8X5qe?}uJh*;D;P_H{1c;=k`3%nvgRJ%}mB zoZgl#1if!@OZ7oe^x6W1K2i1Z-oZdmus!s-&c5H}>pbvvobokZ9~K?3t0V7|$BkZ2 zyC=7}+RI1t4$p^x=LY09cFWtG9QciTJKR25KH}kAdvb{`x89*Jo*v;65OsuFJn4jc zJJ;$b3HWW_T)s!$qKezxqI#h}PVv7`A1L!`(+3I)q_?Jis4noLi}n+g~FRZ0UG}u^eO3}Cy_cxPb9>ys?$oZ(o07= zQ_f%Lp&b0?+d%(+^49_VRgkhf$C7nUQFlqt(HSQvnAlb$XyxF^xQ$|V+_p!akH}A;b=M?wo@Y`&hOOcZ2k=Be1V?#MJi`<-jkl^JW9hsz8mGACB1Sq;xn5^TjL?% zyP<`0=Ln19))7_|7R9AHqgS%|v0tE{K>Tr>d$OAchY98N5 z@NsRxdaL~fAiIXYhd$mbF5wAo0#5RGQH%-JRYAO)O-@M{guaDVgYk(DkbNh;4M!&= zzglNE+@$parFEUsYS*3Fv2L_>GaQ}hexjEF>BPbJx>^r`;UR)wB|KCev_ir~jz-zt zIVpSk&`xNHiigweMD>F^U1d|&zX9=T)j^FX)?=pbl}%E7AHECW=%`1!sEFVe|Dgl% z_=jLbMl_lW`edCL5zsAggIDkDgaPT&e1gjMIx1O$HPPtjJonc9PABDPpGw_9@)x?Q zC}uc={DWMUyMsnyq$qm?A4Zr%nyfWh#8(Bt!Ex>&cUqL2SyWYuT1s{FQ|DNx&cuK@#nkc zA&kd!z~hM~k26E?*tMG2C%=1p8OQ-S=WD%l62`wcd*cc=jN$-Y-cG9btJ z&DQ*c@%IqKYu7g|?);X;UBlbMHK3OtjAOJ!t+0kcy(C_7LpWJfOZ_p8e?TKUNf1m+ zYesgWp%_(5?g5%HMS|F%}jKEK!2}Esvt=^ zQ%cKK%0CzY{qH4gC}7A{y+t8c2Sx?t9exji*Ix%KMdq&wmd3vN7M9@xi)RO6vBNnY z0O9aSkFz{5>LGfzC4`{;x>wM5`Hz6_Iz!RmTYc9?eX9mbFp(J+`7O><55Fd}8}Rf% zFp2YT_i||883IP}^)3$W(SacrnXRK3dH79=k@p}yE4S8nBIL2$=L12=<6GN_cMCEP zx3?wlB8nb<%)7&ROP+5~c|J<}6qziUTz`F=OfPZwe!$Z0+(x}ya-Qhkj=h^Qf3#a? zx8TVAB9;3&ILV|#@C-=&o|e;m<^z!a(roLQY|mB?9DA;H5;8)HX2J z3uYl=Q~uM7r2M1L~m<3pSOLG=d-aj;Ird~aktLsmlwxTbTn=)k5!jvQc-f_XeV~_ zkQQot0?A@%Y^gD6XR?t9**xc-({E6@ew=Fl-5}9lYmw~TAE0D=r}euf;(Pln1MH&E zmG0F&(w9E&V6`5h`2@M=SgsF-*2;IfpX?vT^;=%Ill@`4UM+SWE#VnPD%e4Sk$S2* z54BOhW|Y*najKzVQ+@3L&yYx}Xt6_Zvr`Udsr53P==zR2=?F2n4XR_;D3va8%h2*k zb`MmBmR0w0GqCwgW*4ce6eFT`xH)?{Mt66$#|rPp$1*`+vT)cALrw*3JyImso3Vw(d|p5Vi^N(?m4xXupKlKiR;~PE6(q-7l9BP7F@58BYW?oR zA74m?WL`}A)R=7Q==@ks)TOXVfOpW5gCatU-hlw-CO5Be{|IsjI=cu-pLWQ50;A1B zY#+&(_Q-O)rZf=hl&|2;{{~v#qO?p?jb^m4&1rc~*vwpVoRMcWrzk2T2fr(~XH+xyhq_n9Z% z*EGnW8XK4^wV%{aWyXMER7;Bnf1gKWdRg&@rwZSkyrl$LZ77dUW|PS15MsjFqS&Dj z3(az;X(U7@EsoW9B8akWdiCty zPkt=Be#h$cfw9nuyXFoOQV-#rr~13JeuL6_xG7n;hpPKY>8;(}#XR4iN_Uq0^}Tus zFvE1kf6J|aZ6g{%kSxi}Q>M4CybU9pO^BwgNJt}k zea{wPt0Li{0b6kd>$0r!64CCEuBd`SH2#1K<2p7P7o^0HFs^j&fXM11cZ561ou&Tq zc%4nm(v1{D%fuVjX_ruJBVKy48_2ewYtx=XS&vNOepEdDtHDrd-s zoiC4$3Rv`OvJU5`RFDkOFJrPVRX}GarlS^UEos=4{K**nC7i28RkGL3P|WC9C9;?z zUE%_KLp1`?gkQqL4T;w<&K34& zHnO2YKndw;&}U#|X{2Yv3`R9MsQB~2%u1rV5>O2+>b#_q-7SQ6F%${2*pA!$U-V=A z4A|*88fB9njV9t0%TnRk1PTNkfeM3Xs1U5W9H9_4ku-Y!<_GqURA-`7)AjK9k@5;s zt)if)*>Y_pyFQz_?m!}!j{Fu<{O5f|{LJLd5ADD3*{N{cN1%#V3c0^aCv*hFUHLP6 z{iCNp`hYI+FX!{Wz&yw5-2a7Mz~2P>aT`Us2-A#=AliqWiNQtZ34CgUx(pw~D2_M{ zGwz!t-N0-i6Wc~bp=nr&tM0<4)5E~CqWSzBEr*d0L67)FS+=4|0M{Csc>HkfbR(87 zW+Syy&M0QXwv{xKX$3?Kp(w-|F)_Kns01~iuQo9j&?CAY`EjgJ4b0YbeD~`>Mzv4rmtI*S^*MvYpAvnZKf|?e%Qx zyR}rF;+Rl+x%a@Ars>-yv*gh_*|Z^Z(E!8qU{8cjXK(Q;@*?ds5CWA8av7+ZbN1j| zPpDb{%iO-WA|ys%ivkVlx7cm&Dd($WnQffu_(= zH#>?R6Q%tP@B%i*c5&Qp?3c){Uj6F3lBOPA9ht2pcvCU(sG5-HDuH|=h;SgD4OFI- z9DEzAuZX4^)fXp2aoQn_?JXwKvI z_znud3!#vtX~$OLGu1e6sK!YxuG|JcKY$VW`Q>T&`6i-2Ux{f;li^YL`8w4zwE6z# zPw5OBLpdrq5sd_Twn5*+yz>NFMv-|KFttV2V%YT2IYiCjwK9RKZmgry-24h!aK3PF zo|?mNXs@V?^QS`ZU^)2D2%*1#-=FrMKDaf;Hg{^@gjxeLsrbNrmgqwuxTkvO?6w)W zB0i18<2S_;C2X5=@^h!WAV+Q*I|H}G!c54dG&(6aH z`a^UQzrdAfHIGyl6LmHRLhdy~a5`PgJkiNh#4^MCVx5-2>PwQHNGISZ#dJj%{h?66 z}Pi&$h;ucDgOc1=?X2PgCbEZ}~}z!^Di?d}AGLy9n6wflW`ihQ18m zED;SDsz_ZIk5h}0%2?DSt_~wLTFYTs^VbvL*@>gGqSF&+!Do?1EZGl!t3ckL4u1t) z`sipmZ-XO+&%yW0;YdiN&JFXB?(^VM4K=V*O{`>sdA|~rwP?^piUQ#R5_3U4qy!3~ z>_%pPA&0TXgYPA-jWm{Gxt0Ql>B|T0sW`aMO5L)iYw1;Rtfm~HvV^P*y@l?F7>Z1a zk>(D5*#fS zf{~1+!CM!VZ-aAN*jF@gCKf4_a-{TJrh%fk* zM|&S1YVAEeg&XK5h`|T@>{DLYg%9lfY{b2aUqQdmpM^|sM+DK@YW*Yc7Has%9ebh? z_Zj>;dX_&iJZ|KV`YVv}Mz1;;;y#9JD2y*V*qqw7NNH!9`h0udi7$I5l9;<4=g}yB zhqJ)Tx%e^$eIK2gMsD|N_P|eibZBt@&-MZ4HouKt#BXu4+&xql#KN7SlO1Vc+kzk3 z%wBRQ8rTYEXY1?24E*^5F>O4%sgz3OyA7hXeA6s8$jn5PSUnKuP1%{XJu_woGXAMp zcs5~1$1PP5!h4eC1{M;zlSgMGmL8RL-ikboqh{vVz6oAPW_i$dg%`Dvd?;9}77g|R zmDO}r*DN_&87Vi=<-*45!c^5)u8N9So7TZSE23BqDQcmBWg(u(?K^&Yd2~5w8b}E@ z?u@0(%<}%58i8PZLPr8`j97`K`AMHIo6Gtt6$mVAQ+fo$am{M*8Prd0uA!&VWyl*z ztVi}t5FBK(MR5;}p`u)g`tG3fMC#EkwWd-z;>5^hGB{Umt2=Gm5rsb8tTBahq{dW> z3#KLTff0ceURbFK>7mq)MN#+ftSRYVH3JCTg})F(+80}kOFMQ&QN3$yzK6b#zs{X( zR+b3vqyic<-@GNIVH!CRYj`0_LpCRjr7wJr_%kQ|09QuhDJKGv96v`7OPGY0G?IBk zfLu$eWN*mog>se51mb||kIrNpPaH$}vIy^Sg@qx!Al0Wo}46B`E^W(G+8`K(~`6u_TAx z%&5%5vw5QYL^LGPTSsqMIaA^ZB*0e$lA%fhmW^Vy;wPPb#UO-A5| zvV=BUn~iOZF~%egcFblBcAVHOAtVqUVT(fy0Ybp+;6OqWKc3@&V+@ZiJmOe0@0_~T zJtJ#D2EXsidoLsP?V8(n`c~CFbIu#!GMP{Pd0E5x<1?&veRA*H}RR@?gDi`{5!XhlFI;WF=0;MDI|dy52SuMr>Jcen~x< zZA>Mbmjy&6^wW) zSQ;(%jTgL=4PT$>c7Mhw2D06aqBk9}yjov6S+zWR&Yom6nIfC0LH`CQvX5DB5kqvh zIZYl3lZlKWRKptqnDAi867o1Er(_x%_~7-Mb~|D;mVK_R&{t_&p+`IrnmDT;AC#47z|Kfg#tuXs?fzy^?lbXlSq8=Wffx0N zX7!ln_qZd17;<}TuI69C)O`FNS*LVypZ#Qz^|_N{#8KBeot|W6Q&8<^opx+m3R{n@ zc2HJvvS(LLc1wcSXZQ4#ZN*e&(*!0A{gcb#SXMWbn?c3Y77Yda4cb_zZV$^N`b*(jVAqq3N9Q@lsB*c3`4@QYx!fB%n9Kezc*gqHJnXb=BoNuUOku zLDG1{t0S>|N|uMru+Q}62hwzG?wh?65iGs!je`;*sb)k|G~m2e%w=kH?lak_*V%JJ z=bxDGrR#)e`0l%Uo3ny2OmuwIynM}2N@S`;6L(}W`RJ%J2pqYMY*9V?fJFoEqFSUk zAMUeQK)UD)1(cM{Dxye~E z%Dz&dk>0s)SI+3^E$OTXy(a2duSYQ?W(USHH(H^m$j5gG#`o7`2OFRBY=E-PJF>G% z^mIn2EE?R|#eS5)JI_7V7ZvE?Lspc6$dSJ zyMSH--A{%7?nGufs;i1FsFnas!>fwAsFnJXg|IHrd=T?N$kt!Tme~hWFGZ3OKdet_T>9X?N`uoocYbczeF&gsS5> z@Ik0e*=6nz7LxWIufpy7cB&LS##s87oHbw(fWpZs33Y8?V33UP-Y2zB+;?*N%<889K3*fXM!>Ub(PrpkMBG#$@xE-BevGCjn%)BGd zGH|HC{d_RnCYQ4~%MD+t6>+bf@F{Z7<#jC|9v(S7 zAgJ=(zXQsoub2HcHB{kV3LMfCNqO(2p5G3}|t3Kwkv+WN1WT@jj8mjqSx

za(*i1v0_81P`mbzcx`NMq-0S_s zqh7C57iE?%7v2PG80Am97HB)uGRxyS$|Y#N1IHLy-(q zcq0IFA)dtL3fv?YrYxte&_(wNkHPBkaL(bYwryF=9@q|E;ua?Qo!S~dRSi?tB)d1C zFWcO#C$U1hh+ z^jCV~-r`h6^J(Q~!Z;F$SfT-VP;^JSV!=W#=aWn&kWQtY=XQxO0R4Xg=BSrxg*Z!r zs$&(v98rG9T(2rFiVAR$(~deoJ(FEK#w204oX}<9Ae9m7TM0IBPp+g|7<% zODm6Bm@`js7rVP$vM!$E9C7T|r|BW#KETars|a)Kfb(otk)|{fX4IACEugM5b8@x( zB_}IayNE_uObK=1(DWcnytS4m-g@klREybEadK}Re(W(dmQ1A{|AJr$?_hZdCU{TVzW8(HFHPF*Oo>iwgtvZ!Oo{Xlf9*7gFFn8E z|C{;XC>fbQP2MWpPZAxZ{VTJOy*N$R`EzppUy~>7zxKi`Y#N{>JO5L*XRg`Fu+oK` z5eEfwRdvAeLg2vSt#aYcwz@L?UU^viV?p)3RSf6*!jXy>?x|U|7;cQ~O*tFmynuWa zwu4So=mXNS>3ES=MzgMSZ-tYc{c;S)0bSiMJle@{AFN)?4vWRxDi+(|&#{@JV|D*e zx2J>Dfe+nIe+2k_4dX&yF2llACsw^xbt=y~!g9)q6|1xVfuIs#k(Bt1CMa%xWQip5 z$fY>vYH=$|Xiy8qa{)ON_C>mfGX5?tq{)6sQJE7XUXj(jE?TjWU)4eZ-RCmBAD3;- z2+4}y%Jd{cTJOk`q!{k1M01U-&Hf0TkDB1g27b}IB_k-N{XXjL?&heDYjX%?^?_Q*7Yr5(2&;Zl3{pGHRC*iSRW~y;R z2xSUh+s?Zv^>hbxLjnCOmGETZKKkhqP`@?F8yAXaDc_X#p0a|9tOhG9>z^N9SC2|k z029+FFt=)<)(F4XvR&1zWji+QI1%Yvgj>k6j`lL_*s^&@f$C_c!FFtpqoOLRqt`MX z+enG+tpJW@`*?QJ#3%x@n)JlAtf`pMp5)+YPeySmu82-mw;n^-maXUP=)me)W4i38 zG~MMJ6?V(Ok^)KF@WQ&ZhSyXFs%u+`jmrl|(T*?3zoFhR4xRjRbr0023GIz!MFn?+BX$2NrYu&mGhiA!sy z2fgJl#Y$WK*79grva}DZ{7bi>DK{L`48ir6@qyQT@9wbB6+8Ve^oN#BU8fHG$&3|7 zj3tt>`Tw8~3vUMvqDf|$j-_(cl4TljmKZ#i!)d|%qKRXu&E$Az91j{+r>3O^M_Ywu zDQ*qchtNC7&%gs1qe?J1mKvDp3#NUl*A?-(;$d%YY-6Q3S=_F4nFG^QT{2*1RJCY% zef~6+n2 zIxPc+VKK))qkjl^vILkV$azJYS%|$P?ToEGC>Bxd;FT8Ol?qm|3GTAkb?Y8Tyx2C|sW%YHpn9wSpeY zZ6nj7QrW4K{MMx?CS@u!IQVMS*Mxtm??h+M8`fIv1(-mU3|&cxT9M#Z!h6&lUpyv_|D( zQI@o{KTycV4cX-lx-EA|)_*!AgZy3RI(--+) zy6o?(ROsRGvem1nD!MF*f+SWWq2^#S9L&m}EssYuQ47F;Gno< z%|tSyzQIgktQ0_Gy?#XVYXBBSMUD;>!kRA?%8!QxkQE{bsATU@rhhc&4n^q^*ZQSH zTSk%|T??8zIwnjDF@vnjl^(QM%cfs5;<@gK6%V)q9+(v)Y|5^%ZAJ_IfxfO@NAG{0 zJ`c3Gj%B_@SzqCFs4+Q*R5rjfWUb*XmMP3JsWA8%VhP}o5wtVFO>(;|)8QQkhg(KR zog|P+W+p;V1lLj7idl)O?eQpqpe<2dE4r?@(!#Z)T}SS1kgm5b>KE)k~lK2N_6bheE7Op)iI{@zB=sGnlYJnk~LNR0&8?2hvm=&Z=GF2st| z^D?^?I0ffnEx`-$FUDBDdtPplxFAz<#%#_1QrHS(?(6Km?B=qRws#(~4J1r$6*;TT zT29_Xj;pEeDK2Z|%eW`R2wO(bh$cNK_wxn13DLb;ENDNXAG_vVr#pGI(;^hMR zFQREJ980)?Eh=sW1c>ZXHIEjp20^E>%05vNR1YW)qFgGOSYcE|^_wjP&FA;2cHWk4 zBVxMCA;D{yewodrDWdbya_b>Wlw2OK@M_tGc369KK<1*&FplJ+aBWCt)R4=Tg1E&Z8s4O@HO;nhgu9eDhMZo+*sFw|I+% zynD%wRm-nS5~ybr_r3f54lh1U|B=y)pL3o(UmF(Cj9f&a-P<++OrEhK8cw~^4(-sp zG<%?YyR0ecZX6qsiz5BV&@$Z)&4|FEKqAihqNXNvT{=itGiuS}v2TApA6CT$vSIOW zKM|6`yIFqYcJ^y#*}gz#@$2cM9(#8=blK^ zw@=eI3lH;b{K#EU->xxT8zUYYi>w6Bs_GPg1a&q^F?yY4JZgnNpcGc5vVeS)0 zqn0Z6?HzZpLXPjfsWQ%AZ1dv9`((I!ZJnn`6m4F7K`7zf7hU#=*(y)d&w#whF?rFp zRbo^H)(NHdd90J6dfRwGy^eL_wnf%PyPTwTX1Wm8FIXwgHX<@T{-^YW@D{R~-M@#+ zYNrpi-(dgn%d}Hic+0Q9D_HCAY?Q`4U8jX7V4gE9LkrSDtlEj-P#u#(ZlayB6Yv}x zDrWqb;SsPP=axlfSlhTG3&>S?;`i?bzE;rheXwW!(HZVny6^3$?>l^w(&I;k%#j=E zjfd%~n{HB)Syhw#lg6z#PF+5pk-|wColBN}`-v4BwBl^Jd$vp6HaWFfyT5m1LESzD z&js^cqYZmCSxOweVyf!k3FPUR^8hmvPE?(|2VSC*ejYv0ZVmDlaC~@R3y%9rJ9B;K z3_MuiTaKCcOzb;FS@t9z;76YZ5Y`hg{DzgX?Sju+3iz@v*-+Jlr5SQ@cirFL*XF{~+2kd~%vc1A6~dBU`Tp(2 z%^NRCi3-A4+X{C33oDYzHN|i>q^pu>xm$19aQtCs?TpSJ7KVg}c%9s~CmHKjC@AL~ zw;LNVJi2@qL6$?npnSqZ3}!owJZOi60pNR777|7DaV536*R>1O0N^$|IBZ3&yelue zBq?BrN0S29mZ6_SG*;MJsqC!&KyU*blN1KW=5BX!z40{tUE%jYYs9E2)ESFaM^7Zs zcbmt0un4~i0s0dP+9_smuX9<$i}E5xxv_ zI!fZiiT_yaA;OAZgU;Nt&@Sk7wmG;ktx2H)OJ(t|i>6rkt2T%|{by~2`?>ZUf37pb zImED-@D?6NXLcFZO)u+O^c(Pqg_+*U=|ME1?~|9sk=+Rwo#Mt_?R}#2awLUd!!D*z|8R6`xNx)46**p}EbS zfqq8TI)0STfZVAvYb+1sDTq@}+0R7aQFA-vmB#duztn z^R=@(Mj`V4BQnZMm{)ryg68E>`=IxLZ7ZMIR{&vRj+x~Wi+X$H% z9+Z(Y9PXx>>utoG{4Ic3CM! zi~IVAi1EAU{c@$Eei?I2J39H;{PT1J{UPA{ zCf`ri#`sQrC3<6UKP8T>O6_qvC9oj2Iz!nxB_ArG%a5t-+&Po?x07dp#u5bP?Kgrx z*FgW8L|@zhQU=QGi?1x?N&sM_yX2TDyHq!5b%lfy77O_SD);BLkSs;1rTAi9kqwn@ zic0zyB~UxD2461ST9Fn+4>PkB(wdZt3v+igh7Gxt5I%+p75*#lt&^(?_fNh+*V5-m z{*2h!9AD{CC(dV#89Xsqv6pFdK{nbF<{%wpDEkt3#Qjygo2AlsQYz`OlqzbHDB$yW z4)CWQFbzYP`$DqMRzyuytPs6d4=9Q%iVm-wn17hQkNz7-m0lM88bPEJR2vGj>Ew~Z z7Q!PMn8*bKB=TxIK!Uz+fTFGWJGy38QdvFq4jQrTm6lQU*@B_mvgQF(R771~HEpV@ zU_M|cFMIf?EqFcAxx48m!=-Oob5q^$s_mRI$AK3hZ6IZYsxM0{+^-$=_G?DCG1-WI z?j-#_xs6=IpD*t2zpUz1IoMfsa`GR8*R54YZ^t+z^tRQij?RYqSqXR|VOm+U)4I0P zI>lSS3&!X|A`<3_NITnaduhj`kfdRUd@9ba!9{Xm!tqovMZE3X9TU=&$rSDhZw+DG zf4J>}Lh=*n-+4{T1=M6DvN^9O6_1%#SQS;p6)?k5)vCs2AWT6?B{2Y8f@;WeMH5L+YH1WyciH7JJ+>%B>rq9KT?(dqi{=#pK5@&69IHoQ#=lQpif$OX zRqCw-z2*LDFxCUKtf_EW$xhE^WUq3=NM6t+N%QHCDqcA=o7Vld3(&U;JV5P<1*`pK zZ}*Oz4oqBDTtEfcRe4R*Y;9Ar;>RdlSqu5z0KaQmepObar0pADMxQ1dIyQg=kAs?Q z7xuH9&1{Sy6~2pikbms#-Djte6Eo#`icX59ir{RE##wKG)QOHI!@VB2A!H)iUP&}P zR&T6OjD!FNskpgSg?@SMNFW`T#H8Q$f>I5nC#bUM$;CaO==y_tBBXg$fF*xwSh8gZ zg4gHi@@ET6&%XMXbK=l0ia1hwNZU`fFUM2!-;}v z^9arhJ^)dMv<8biEjUtMg^3aef!2n+CUyI424tz3jOsG_kI)F@)lx+pH~MN}!3@CC zrDh^%ff^>fKT**^?_hD6etIKGhM68qR{r0-y?jLtS9@K6&N$;?K;OST>7%k0;8aF z`Nx05$=P^*2l>l2Yn#vI9(R1>*Ml6~$g=FA{OjU85d9B`b}>h1o7NBs&%(^QfgYV~ zrro?%8<2C{_kAX>LmjDAItTL_tdH?rI}S5k#5Q3cWPcmRT-y&AD5=O0AA*{m%Kgw)ld|;o2+SqAUQsD`e|5y6ATZwGlrnt z=&w4vRUOpCShW!3euf3mkXHg2sp+U@nAH;}p%JbRUaAL?Urdn?%>;n<1a4;E&3&0XsGnk5iu{ zYKMm8u8<4JO$ijOeUq>XHJ1wC4&Q35d~ElX)HHC;sT#O^oh~X;+`<}ylBi-}R3Pfz zm}rKf6MzSy!^7=5Pf8-7t8aw}NuIWb-1bH2}=dL7i@?!s#J_HyNeb*-UEV+M` zubMt|AusfKJ_C?z>@{kYfMkt z4)lcQKCOt7j;J=Xx|^zV_gO(UC$;%H{bTyiq@Rql=+Z>Bot*^rYmp>uCv&OOsttrK zCu)zImsknN#kCOH8!?zqW9ZC4T{&rk5=#|X40w?UEf4cW#_g`jYJ8v=9||X{?lpFI zF4U;SDqgeuuO(S@4;1M)BV8WZ(Au8GNORNZRj1M+kJosQEBIqyW#i<=>8LgLl;z~0 z%J4lSj_nC8oaO3EtFmJ#;f3jd@d?9Qj;l&!4Hr;iDJ0sg>=U za5O%Jj_-D~IZ)a*0JmYm?I=}4?quL;%-i(Lxlkc%dBRm6T@lFZ%JlNd*S%jH4ab|F_6td-40d20}(wKLe@&g5Kakuwn|mYH?LDw8E=#?K)!a&QJho4!L! z+x~>+GrV1vua=R0VKZXt>6wC+%V%ApoVBvL-)-eAGj8cVT{XRC#si=&xipa_$c6 zD2igypQL~aB(*ER&;m8Ht9$4QQ8bXb$Hud-dw?Er#WbCXg1P4yKPFA4okXjSjiHY}CzU-PnP=rG1Qr_xKq9tH0wAxMD|<|+54E#9acNJNQoSpv zilA}!#k@ma;7wXA1CpHTs>hcz71;=;Caw* zXo)yz{S3=XHb~gH%~X~HE_?#Q4OEu+!#E|;e>Oxm!A~fP#D%S3PJyKDJPQ!CbN?Rb z37vU@zQT$*G$YK?pOBSoJZ)<}?sdrXwZm$tD=(${LyJcbvLv9gM0-VnYHs;OBas8` zk;qyi!N>{_qUH;^WTYuJkmXLl)y(Q19LS4K(1WhHf|Iba7pTl-g(tcV8E#Qh^ zh>i!FvGjnAYHPplf#Fd!a)L0S{@&15-L$$wqnR#uFm{zE0WvagrCnZ+#|=PO)Pi2R z7o?5^;!%+`!GYkWuS;I58O8GYp2_}H(48CZ8?PG1Q<~#rq<;ZpDzHi#S-##!_{eb0 zPqP>=Hy@ebk(q7J8;;C=vBhkWV-dK3>|EK=O_|gM^+g4N11s){WWU{I1!JyIGnT5` zAo)@0=<(rlv-5^08FVM|p0ufks10a2DVvy(t}DjEhT!Kwz*nB_ndndHmItT6PSADX zv?U0BRn?-=jhZp{St~ozJ6_ez{~^g;up~7vy|EMp0=E@gQhe?myLZk65}HZ4FLE=V zk0_rFXC{8LS$GMoF5<|y> zs+962rnrpFG_NdlmAYqkwknlCNL7r1$;?n+Qu&dbK0OI4Fb%%@w6QGYXRGZ#@v`Cie^BM1dFAaV%>DZ z;bg`ap}MW>R!Glgx@O8vjk*AjD>{8d0Mz66=tjWhiduqb`iy8JyQEn18K&Ee>!$8< zr;~wfh`Kz6<_@{am1d?Ga*1NlPX!hmPhX(2@Ga1Mt4zYloG&1Mz=#Yd-UN5@i}@y~ zUW{Im5m;jQ>{B}hVA&FAgL>AZDVj_{m$~~K=LAgvVswu#9dQ9S7lM`+tXJ~txlR(S zFZsX7+l6~O`}Vb}<6YxLHc&Oh-525ZrukNLi6Ehsw9$t?AO&=rUcJ3>|I3A=H=Zn8 zhM5Rtc3rzO8T9U9oAy(~Wbw~EAY$$myuyrdLU=@YUhEUMiMNRlil3DvX`A#3xhcO# z@hPX252;P{LG^PQ(Tdt_`k?+nqiCEkK5k0psQHljtSjca!u2h8!2Mlok9EKGtY?eo zYu=joI`60Ln*C$n?Y{5&*Zbe)|5_jy7!4c=JRe*eygx)j`$GR1elYxtNMGdks1jWk zeLR+nT^9Rnyeobv{*^>u;th#slYfv(rQVeKWjc^vlm2-6N13ZL_hg>SZp+@AeLS}z z_nqAH`A~i^e<=TWSH0`Ot|tq36~0?^6$gtq79T5qr=*nzOZ!UiEj`}7rTbewTY8=^ z=gRl@HhaI-_h8>Q`d9Q{-T(1Qt@8Qmk(yL{WMFjQnfhh*&o!1b-q%btk2Js1%C&A9 zY!2Qv_}tKvp?3^R!}pIEBX^BFKYDER*|FW@WcPk527gl3VhorJE~{>+L~E50;6Jbmx<_hz=uJhSrZl|Ndwf7N$apIW1=dGL}Im)y6uvi8ok zpI>LJTeI$K>tpLbHEYZc&+eOj%j|bHNE@0P?%LS3@xD!&O?Pf~Z64qJrp-@o*|z18 zEiY`{zV*><&eH?|ww*Kfan`{UbR*iqhb(~hrPw(7DEUG~)FgO`8e z3hRm$SG?g$a^+oD{&;8K&PR66T{V8yhpxWu>R;}-@TnWSZ+yoq zrB}Y;mCxKXeA7cW&D~tS`Tm=qy(L2&PDW^sVC{rY0lwb?vTthsZa6Mw$7SreoE=xp zKL^KY*1m)t*Rta}c3jVnv+THm9XHP31;P)AP5yao5Q~0+w?(p+ew()?k`VH|EtB=a6mKgeCfv>2D%mIeA#ZCWC%SoChjx{> zP0}rH!MNX2pTS2TmM2kXgR#IJ{kda&M+y zs}1#KZn)uw3S5ZS!NnU7?K*z^z`@MAy?YLvJiKY|wI>hlI>!IB^OvKUGcOy(uNpb? z+_t^PjvqLBBr{MMs11y5*qo`=1}g9x+kf=LHAjz}f)8qy#xVaQ=WovZhRlKEnO&I^ z$9C=6dwAEegPEiI&K^VMtTShwnAtQvoym0XIeKK*3Fy!byN>Pc*?;21b)(hlnL}q6 zo_QVlBa$H}$bPaH7V2&|9wB?+%r0^eUKxOc2gqS~ogl~Hm;o}}MUL~|t|uqq`V6U) z8vGj~ebByv{j2bAY=58g%^S%f_SeVZZx6z?>)_lTXrF{TY=ZW+(84Q@z38rI-*XiH z`a<6}+UeEE1+Uu%?PKhdqtMR`sE!qQ)!;QoHh{vFVc&sgYy9kfxbg(KhFyP(|3wX6 z4P5$X+-dQBF4Tt%ydP&Xo?&C$1)uDN^EeN9?NPGtqMk$LBCh$>SIoeZn1+8D_}2|R z!Ex?_E1Vg+f%R%H>4AAT0spRpnXbaW3qAFPZ@kd=5rJ_W0z9!!&hbN+z)OIuMBpGQ za0{J$40wYJ_<%(`#0%Wa2WUS4s5t~EFap|P43J;~uvQ9godfN#3$(@}aLI1CU77R( zx%NXBtL(wn;Q=;*-wpy94+GJR!lTCQZj)q+EFsItM`7xwVJIufD!|Wc$R%VgJjC^6 z7RY2H3~)2Zx2-@u&$sObqMg;a`ytQIWi! ze3tw*`3QL*`D5}q@(4VNyUE{?zbF4l?jip~o+ST+{295Iypep6`~mqkd6@h+^4sJM zfEw>2|C9Wf{0{j7`5Tzt_X8<@fDz2+$>ZdUKr&w=Uxx+tP4cJYY4Qa5 zTX-zauazK$ckIZZR9oN z)#N$yDEU6wN&b~QOP(VCOwxe3C9;%U4G(bf&o{`O=p+Zbu3 zq#+un5gMg28m9@0s!Ez>XqM(^o_5g!Ez%P0raiO_N?af9r|5#MQBYoKgEnc44$>hy zOh@P_D1PHC3&0dzLYLBIbU9r?r|Aq`NmtR;bPc_PuBGefdOAxt(2aBx-AuOt-E9R4 zCdri}Cl4L6uG@R;z|lR|>^*W~@3Fmmr0J8#j>;?c9y+m0IlSwdV@HpuCyyMcH3w?W zdu@Qf5AydR=e@DgnRak-?pCd+sVbhIe%a0^HAsg ztn+@?d4KDCJayjhI`4O#_q)#fUFZF-^M2QPzw5l;b>8nf?{}TgUxVMj!SCB@|IY8< z;P-Fv`#1Rg8~pwae*Xr)e}mt@!SCPT_iym~H~9S<{QgaTza}4lllP;^@89J2Z}R&$ z`Td)1I^g$j^7}XW{hR#$O@99-zkie8zs0}5)nd#OUIy^NK0*)eI&lQJCss=&^YiQz zdT{SCJm*}8yRMxW2@PJ46Q_Q$1uH$k@H?V+#O{dP5w|01N6e0h9q~G%b;Rn3)Dfp6 zN=J;22p#b`qI1ONh|FUQnIkSoRF0S&5jo;Ur{;%!9Rh_w-EBhJQrP$;6w{{~P?0|XQR000O8#GuGJY~r*@vUvaikmdjY z5&!@IQEXvzb7^ZrZ){{=R6;IvbY|?lcbFqrl{b9vt(>c*3W}vHb*qEArIysWC-mg; z%y`E3csw|aJ>!HUIDie<#zY%1X2F0LmSisrOBe%Y2QR^7Ff1(DBx5jOSYUU-nC#Ja z?k%ai#|EC~ecpe*Ki;0Hqmopry7$E2IX57T5VC>r2yNN3d*6fad-UZ9{p1A*VRiSu zM)Mtiy8I$Ie+gW5&S}dhoqoyuGe3%ua1@S*&%5-{6+el6Z!1FDD-n{+^RBrnk6gGC zxd|ZT{3|ZFR6Ov8E<&+4!#$pP!6nZ=fBrq)+u{2RLZ8(yJaXvp|4iL{9KO@x`p$*$ zfp`;l9lT$L_vH&Oz3N%7dvErG@cw5Aai4qok!M^c{7iTjLf^OHv+5<6pLd8n@a14_Xs^whUT4p z#WRjvaSWjmXb){Qk0>uN+VBbW+n+t#ntvQA+|OulKC}1quSc(sUe>#}$8#Uypub2# z1Vw-FJM!PX(@@Ag3g2(%KEm#__U||Bli#4bkTm!O{HgFe;0rtF@KN$%B)}&hCbz)* zmqoAm`>2WjBwK}1FZy&tB;@QujcgX*l^xN=C^p`xs4tvb&(Bt$wc+4a4 zN{=s~3a+3e{xI_JXV5tL462e{Ahmx-3A_Qd;Yj08pmF>d8b%+7JCsoxUNgW^6Rr>N zOVJ#>X5gs6p~Eo*hmVhoxHH%V-i`2j^dbzP}C*2g&%o@VhUf3G{W) zfd_l{;yX|SeGPbW548JB(63JekG>A)Uxn*CsD=;2J+}ZKzJkW^tB^`ALMq%t!ncF; zo{2QvL>B^oi_p*Y1Vg(&13GmFJo7a8{ilF89=v+Mr|%*Ke+`bGfet-@%Ak=obRY2N zG-wmO_LHbUZbwb-QF!)Nu+H_sx83Lveg!%WA47Za2)Ynof*QcX^Kl&Q#7EG2ZW0CL z^GG2FQJFJ=$Je6(?%l=T0X}{dFDwCJNGwADMXfxRW--n3*YX_ATz7=@B z<^VYjqi^?q2FLf{_&Jl&Jj$~JWYvF7Q8|$z)MW?AioP-#4dnEizjC}5jvIhqcR>5r z93Y<|^!45!;P??7zh$!d0`%$02bIrPK+bCqDjRyd0L6fZAxZZB1Ts(Jn|qI;U-TZs z4}e@hi{|lndynH6f=aQ!jC}zzJjXwmmq`V zr~pS1pNCFm=fGF;7toI~@EU9fjud$X=-Z9^S=^xV3ALJgAk=%O%^hg1!A{D5HG@v@r zzaRtYKuu%-wU7zaMpp07$U!#H7;=EdQ4FYy;=MnC?YcmHiN2Z0_&PwV{@9YJRTJs+I~ z^a4gNL}&MYf-XYm0KFI;0{V1xF3?NRdA%Q_OVMGVm!Ts-FGuGCy#ig(djwsHE(H1v zMxTi;0(up?7_9SZ^mL%tpi6*Wi!KHFEOc4#hv?bpa-i3tD}Y{)t_1oV^o-ua=(&tO z4?Pp;4d|-g4q=~xc43O zYDQm!UIO$E^irU&MYr_6jb4Xt1^RmQGN5;&mjnGr^orht=nd#Lpl?L41o}_tcA#%U zuj+jZy_wOspnm}RR`hD1Z$q!?eG?-89YF6!uLb(g=ygEfj$YsU2KpCtC(w7Ge+2qY z^ah~+3iRvfUFeNK-;MqW=zGwcfW8;Kx%V}66ukxLJ?O1K--q4?^!@0r-dE8F(A_}q zMgI)+gN*(gdVB9H=otDJpdUi-0D2#KC(wUK|JwUc^kMWapdUf+2KrI-9-tpX@9ljV z-H(m}{W!V@=qDKcBzj-(OXySR{XjpBJ^=JH=w6_oMIY?_2l^cPH=v(K$ACV7J_Pg& z=)T?;(HGIb1N{$1zl1&v^vmcYy)Phm8qlwxj{*HEx*zD*(8qfZfRFzK&~Knm0{tfX z6wq&>Pxn5L9%S^}=rcgSgFXxNyXbSh&!O+3&jbBFdI0D{=nFu9fWFxKEP5FI2hbm) zF9H1#`ZCZ*(0}$mgMQ5DPtaF@{uF%`=%eUsy-%Z`K}7j9`Z@Xr(Emc;1o{i~t=^~5 zF98vJ3jGRV+^5igqwfIyAE2K^|BJp0^w;QnK!1b25A-qgQ127yx9A5zA4d-Z{T=!t z(BGpU^*)aNfF1$*NAzQ$e`55{=qJ7V(O=L{fj)sA1$rF)toJe0Lq7)!NH9A1>-s1$ z1Me`!w}22HU})}Pcu(L{9`K?Fi1IA(|0?)>9sIose%=QE9s|GbfZRj zvR(jRumqlRJ$T5C;2Af8N8AFQa2t5Q9pL#+0*|*FJl)CQ;r4-NI~6?Ie(+=;2S4@z z=m!FOy%lWp6<}$v2fMioY~gOOig$oDyc6{QJ)r&X27P}YX!|{&-yZ;7z8AFm7-;ZE zKz~0B8uB3c!Eb=>TmZUpBlw6HfzP-CywG>R*IWlW@;%UvhrmBl&W=SAQjUjlyR#o$$L2XFEZ;6+{oe&cnZ*>{3(_$TlN&jnxk4A8It0zY{* zc*b9Xeq9Ur>eXE+ku^20+ieYi2cuw(1<$*u+{!Ds3 zrZ-f|^B}pipdZ(RkGdQ9{vP1&ebB!ju&1oYd+<5<3V6zM;VGXX55iO8`9waG&$Fk@ zta*yF@|45uDY(}IzmC}P12`U{%G*1J*8KJE@9pf}%+7A?)q(D&=TGDY-#@(h;f05X z9!@-*`N7#g_}D{mr-#1y&_^D+_o3?^diHli>1)Ed%qGr-0JYIeF12} zC;GSg16URP`wKg0jwxap>)1w5`osK_KY_oq`j!&u`ao~b=5Gf)BEu3KDQizJ6znq!^P|laDyk{B4Kn#{wP=7bNb1rAH8uoeRSROfpnpe-*oi; zJ*OYNe>q(^Z~(3nS6T|M&%G$oZ?gn#7VB^#M|atG`q6diqv*gbw?ywsg`+p#a!dLa z=tuwkF?9cvKEUY7AFS(t0GyzA zWV4xan4f3QPvXb1LB0vtr3=R~WW)1gaE!t+4u=3o0ooHcxzbb>n$(6SHG>dmvB^on zU;Xp~;&`seL2C%sD~=|r3Km3}d-u^_7@C0NARL##aXIDV6mYUV)#^s(!7t`X#Vws?(A}9I&5K zK8^zwvv9Xy@E`BWhufEkK*lE2YFC#OUQ&g`QgL`PQc+pW2QaTym=MNJT6Zd`KwLNzs(1qN{% z<*yx@*~!4KS2B`24A(xBPpE2E=fUViIiKh3lo=0gUgESkf_Pr-JwomvZ-Ts_1gQNa zh#izaP2f)%_*0?U)eH$RsZ<;%_{6tyINFOsx5G>*Fa@b4mMX3v@eVkG*PdS!t9Glo zsj|CuYrb9ejvtLxrYpIrio10iymz02P1`iuoA>NmP8f<}Y9`Sp*6rN2BgD3WN7Csa zmrka_HQtT?`Ih^TM01ti@!> zP-A;9z2TK zPHYDB1!Uiv?Q2bmnOY=pYEI*X>Izi|U<*~0NMBk_YOE4d5zh??y?hb3=W&f|cDn2r zp6laxiem{$iJMA1X%jt_kW8^Wq;h85Rb)*wR3#R#?w@FEok+O(Aum@-jxJVOiH?8;Rd!<|#dKqR zHEfSpt@^3;LZ{*;U2({A%u+3}v}JoyleOj33X&ure`7jdJ++jWs|?M6EhX@8!1sR} zxMBgPxW2cIQE%IKr(Om;3MYvY*hVL^t6IAu6@5jF#WB%sS9Ri=FBib4VgkB(j}s6f z=W`N?4^{GNjFaVsjiAl4DbW!ijM}}eI3<e@XRI9z2io6F3fzJThB&^IESmnGv&I zY<{L*?JlK)BsaeUSGL~p^iwkFys~`FKVN*_o37p!6T)EA1MJ@E-WG7}zXrL}dy5Sk zP2k>O{xpnnOw7=MMQ#DVW!r)ny0N*LdS!YknMjh%WZA>j7kp)1I<0KD^34|@ z1@CLVC;{i_McqzFEG$-Q_i1s5GtI-2-j=l;Adm7LTwRWn%RHsa^cIp*PX6@Z5NJbB4x+Q(z85$jl$)?Ji_V76+crbDO zA3;Rp2w$yWUK6m$hjotiE7SWOpyaDq1qL7}dj^f{tpOOyEY|{zQ^TcHXE6-3m3+hx zv-df?i+l|9h3bk)<5U!(>3^=&iP83S0=z5JPP-$X6xRhk9ws_Z@9>(NERkbp9y@F6 zE4L=cG+rS1UVPaHb%Su2`MIF?8+9I@h%;!g#<_RFv|7zY@MPlzJXbJP z_oj36J4c*zFC5?2OkZjw3U0iRG>sr1A8zBJi(i;`V$Ho5O~2)>r5)GoUVnN!2&T{3 zK7Vr4dph)KuJ;7Kf*b_>FR(sEh%F9B0**HDy@C>`9eG)k^O#$ueiIr4|4m#hfu{8V z4EzVRpe70D9Eb)C45F!|%VvVtK<5SGw_1~N9J+!gf*Ucb)ld>-ESO4iW@=QI9kOfq zIk(=n04@@OZ7MEs7~`a5NE%$vku$z}HLof%R>Ao?iV}AWj!1o(_V8PPf9oLQI)+qe zdku~{9AnUjG^!(&^&yIlT%V!Riv0X0M8s~V*@|#6sMkQj7Qu%>U!~T1Sra*>+?>yk zud6y33#F7kr(%R}7VwIW;d8WddSSV` z@!(W_O2e8Y2Q%4XryK`C2wU26!T`h~3CTvq&sOu(z9cJ>z-wo1Y%G*b9lyw_#LZoc z-BV{mOI3Z%Nt=#eYlk3EnX5x@H6jX{E32tOEl3T=Z9!E7MMw=NYh#{2!fZkB{QSHu`WR;b_6p0S+v|w{gaS<^CD9;7*%?z8ZiZ=w#D}^ah+?7K6WHnNQ$* zz$LLb(ubENFt7GJ>9j#@eTuUm$J1AoER)E&+Su5zZB!jSE)f~}9;;{|DPHtE5!68U zv|(Ss*sT_`T4yVKbD0AElQ%;%zPM%Qa>q0*4$yeePMke<79cdRcv-enBDvgh0b=3{ zHCI)2P191d^Rr1VJ(rGr%n-=?4zR~a-dk9Jhr0GPXD6~J5qqojO+kW6(RUusNrXA`0>$Xe3U5?a`e6Yz7qCm-?PT-q znm=Ao&-zYL;}k^|IFY3Hp0+!|V_7CK4qR-0O9|6#LkudN3gibd!4z?GPd5ydv?r*B zX$JM#V!I6>8gk0S-27b1&S_NVX9bzCe*3-0zJeROzxc8f6*_@qI2@iBl(e}YIv%z@cBk&S4WTpe>6DfjO|-HQ=l1q zCZ3Tr%oSTp#n!0{rE8%t`QBIY8_0R!b0ns{6O%#2c(J%vG6k02Kmtg6QU{dLowq8( zBc5NfQ#+>$lQCYA5@P`d6XCIvuh!jc!?enIk0gc;o_pRzb8i*XtVAn05@&)d=5V<0 ztRoW}o;dOppP-=2*O=IwhkF)2Z0X*+~DapMg>4QOped z>{e?luz3zs&N2aj=1F}Gt44sCbNxP&j?#-WN|n^)w#M|1aW`hAr$>FqPBk_R@e*%V zXDYF{sg*s)m!IPllC~31)Jv_Is^zFzCqLB4=bE^k-7p3zL?+){%7sm@R0)%ABT-u% zOJFWl?}XXOn3=P*nDA>oQBH(KCy@*mtIan4S3k5gSM%G&`p^*5F|YSAkjpt}GmEe^ zvtOWb-|xq+sKL+$&>Wg)c6~SM+Nd_$3kKict4Dt4{L&bPeGBYwH?tEc#eYv%7u8`k!@s1;J>&DZ7)@0et>I6sm5yLY01At|`6*LoPxM2m& zI8O#P=~{911Yfh5%VsON<3H>}9l~%Qy=MXJkKWVjXPtio`PBxR;j;;q#} zwR)^vs#Y2A{NC^J82JaVj~Uj_8bdsAKc)m=0DT|;fs|Hb1{xyK&JGX+w^~4Zv;htu z3u(j2i-IA0n(SmjsVjjQ$hm4gUh5RB`lYm*7F2RceCfil;>dt*wxYe#X90V2L<=<_#y^^2TC&V`0RzrizJrK8`b~ zV9KSaK#&E@d6_V8dNL>oXKWd%P8mAoCH%>PkyLku4B{qE%UfIjBQB) z)8rtW3uY!rdPOs|&6FXVvIKfJp^IMYl%;1~@}|4@T)sI+G=Qk&u9!hvMEsc7@kq4| z;f~5f?EQt<3AmXLwU%*U=nUo!z-n?QiSX`Wm)i;MmQX{;`3zGIbEvdPcSR(Y}M(NZE9Oc_ny+1>~{d!bF zZM2=`cVgd)^9_n*^GJy>`us}QS5P{1K8ohh(S8|lo_4uZ2gK8%;2ro>q9!^3x>$Bp zZ$8`F+VBc-K^HAk&m`^A{Jx3m=IX^tN*)D|70;A?IU$P%Kmgv15hsQDv6AS^e+{jz8m;C@3{yJbKrrN;wJL41!y3GuB9r*3m#t6L zTmEEo!W%oOE+}}x&Q0V~In9%krEs>9CAVXaknw$2ZoJ@?6K={9M1gF~W{l(C=$_mO z1rZwUV=FHQqbBPrK8KMG+ z-=b0aO=isv_v8e}jnCA_c4q;u5(Q+dTYm)eS0sNdv+EcqX}132-2~V30O~ow()c_1 zbs;pFCp>o3w!-oF{&da}2JO2N+85PDjJ|9G#vp3C5Qe%1*0w<-vXGozXi_i5YgkS= z7;%QFWfNC@n25Tg-zr1UlZa86N)QPY3D+|`hrgK0JHlt5#;Np~NzkVslDp7G7X6Cw zOc#!T>J%#wERtr64VN&M)TwE-S4sx~9h4ARt$f9Y1HNg~((wXxQR4N=a9IWG&YAu~ zO?Kp|Gd5JF%cb+yG40b7QCBe}aFzOL=VTfS#h|39uC3!sAjc7CaLK>&RT7NZ%676zxw7c=!nDTm!|SKPl)-r4iMbXhbhm z+s;asMvBy~oqpMm*0Y#p00!|Kh||zre1l?*m#z^NUiS(Ks`To}h%9_lQ->z#J7uJn z$UrN87fTuf&y^0g@uu-@Syj}tsV_fJ4ivt8aO(Jn8k@432yL6_J%RoS+E!%pbynpI zA(G}9@WR^CgoMkbSOH#q%KBx6#Xh+{WQgpQQ_X8f54FO%?9+xCQd}*TpHtjaEUwR; zo2_alJdEml?|*wAMn7bJk@7-ka*cSxbe(4KE4_y?QLSmFXr2$z@eNyAG)Ow}@};}?u=ogK9tkqAOkGaad~(|^JbLK}9V zR=i6=vMl&2J~BG$di9~Q?G`lXooR&yO-<*^kV42< z8Xm@~XW;MD*QF%c$fUj$8;VUHZh-64B+1}3Rl=#c`PsBTyM9Ahp3^jm@ccx=pXxL{ zFD~4Zs5)bs5ar;LpqtNFk;kAOMk^2{>J_OFRuY627Zk*Qx;2`8d4sIrOYGH(v>`9;JfNQd>aMJj=$ClyBlxL@C!DetVp{jkdul0A9U8H!8IT+9-AF`>lNbSJ)a%{V8+4Fm}jY4Vvj zzGPE=L!=MGV57eWeb~*aVzgGf42RcPt<`dn1)BXjk=L)5)6q{_;RByZn#^ggC@hGs zfmdV%*Y!g^b+`SWaM2iI^dIkz6)l2Q#fWDU6jL}-sJUYaBV`V0;Aa#X3kWTCiKdrJ zP`FJ**Y`!ow;O5M01cP$-qPlL@s!f9iyi>Xv5V)9Zy_-vRCd*I&Cqyqy#w2Kw0Lksg<(2dA}&cisaqZ z@;L;>SL|KCX=hOD=GO0i^@c-Bx#z!d)42=T7vtsUKkvp{Hl4dJeE!$R_usJR?H`&r zaKoN=zjxIq+)3U7-qBivTO0IAwEn$XqzCUDJ!YjyPoYxZ=>cP#SWgD+^wi#&u+W-M zseW9y6uX#?+3B*U$7QT5VVDLsUSseIPdO1ZcTab=Hv*0*aYsP9;6y!z|jmGLl7eLTYc+PN_LEhs&|1^;=4_3CgDd-1aLG&uXj=LhenK zb&lg@qm~MH*GKmh!m+%KXLh%Om)uA>mzvx>0(pa@W7mk)ebJIq%|Hu|KWllXTWJ?I z?b^3d$^f)hZPg(}Q#Fg{tIeh(%^tXV!@J*0xxD+@Q<|ER2wX@TAp0YNNM3N`Et}6< z%Ho?}e$x#sPp7fz`=Ixm&_3|V(fIEu<9<{R?lxAvW15YEL41S=Lwj2*RtLpi#(s(u zqx6FM!6u!?F^8M}U{*||=iExF?*Vi0HznJ!Jkttdrl6*Cr5F)Ok(cCB(WP!i)TwF` z8PKGsOo*0v@Sq>&)$F9_<&^*Oi_Ia!mxr{tt3Y~$Z7-#JQ&TMue@BliDmQxJ96l*M z1|W+LRxvMzg)`2{%`}Q~{a~#+t>CTl{Os&3#=pgpF1mmPp8>L7W*T2?MEE353#0vd z{t2@lR>o{jC=0Jnqv*Q;THgBR(z5I*s-^;Hrqdius~*FgmDTg%`R7h;GA&z6F6DnYb8+d}Tn>p@BsB&AX@;yKG1azJyUi|4!TSx3Y?l!L=~Wilhrkq zAcp^bjeiGiX);@<_=jSEVX9eRDi9Rg@u<9FVhRBJ=^7hGSp(B(x{ecLV;Zk27&v3N zA$UZ?%s+j8ZqH;&0G$E2Gpg_qKUy0MnQL7)kF8h^0v3mBhq`av^Ze;*CO^BYk!)rE zePU-$lk8&l`2U36#KK8T52W57SUfm~gUH`qZvfJk$rYCi(A`CqF6ef2+g zLhP5;yrJ+d@J|>iW7!p!jD(_5IjVxa>;=P6eRDvT1M(C|Ho&!^cK0mRhXLUw6r= zzM=r{oZcVEAIa;`+t@5MntKevG17=;sl5=67dN7LB~RwpDzlcbH1M0WY}O2($|nYQ z`Vt8I0ZNQ2H7nP!3)-t#3HO0LgK4zV^As83OCXX;mM$rr?CAg&DE~xJ|2tiY@v+O3GK%=}CyYLlr6deaPYe!rHi}B@82D0aJm6dfWu%mlJ9%r%RS9 zg$7_K&Mxd95lq1y3A`bbmt*5uS(UXwaROVmWyzLQN9Mq(5UjXHEkQVv+Se#djg(@% z0IZYJoh*DN$CTtypahyT7R${9su&^Zg*$g@ajm*pzC-gL{eaxvzH zGI=GP+6s7Hwp2jyseSd*Mo;t2S0;v>aRG9TMDk~n#<_FF^h`Flm<7e%UYl4SEzbtO z@dWhgPKdoDYid)q6-06$N}y3z$D}pi(LqI%ik22FXy_Gaot&n`{YeJ^M8c@Uq}W#( zR%7TaU`UQiwCqr2xSHsio^|mAHr$16yQ?EtcG6XqXf&6Bd|39fLz#ouR|p-a9@@KW zQ~JR|njC~4NwO^3w3Mk6i3?IA&wA-?8))>=9NEf{=+>4DA_S*Qq>Zn*_4yrH=A!>$@F9RxXU<#-~WT&nqqfhKPxZ94>aptv(M0;9=EzDGzZYrtwj(%yas%weFs) zuUp!gwZapaM>rm5Hk4jDG~k{eagWXqdlUY*-cKOk(E9dG`{RLhDInui{7z0il*wtC zeANF0rgPK3hjNV9bk17j6X7 zViMW`=E8uJx!NvYRqTZgbbcZ&ZxP5HoZjR3YWx$l9i6u_Gl*7f>DUaN85E-OpgKZj zOQ$W((x z5^>S3(Hi6HAz2V)$pIIJ`K=p9Gcy?_k?F1=YbZjZ-*9unVoy*h0ktd!Nkb z9hv8~gnaxfN?>h(R%mYMmomhm<7X__n(RG>-$Oo$3TT4O#Gw2NS=BG>^QrsfTHi`G z8AUvnT?~c`r3LDjmC33gc&$QtaXe*b%JKZxTz%O;sHDvJaKUSi+EvRj(+>V)e%x_& zqfmUOZU?PB)2)*iMoix}&1qhUjqjY>vrgBj9`$~MkAO@knBI*v`Z)p}SLx6Bmgrz) zRODvExGOSSsfgnvvaHLtqGt?I811_Gx*tqJ5{gaDn#|UB%oN7LaCojXvqe3E6p|-iU zK<@|b)x3d@Dbp^PWY7bt)9y^Q@$G$`k#j*jrwW?vRK41UiF75G*;);92C%tV1w?g$ z3OK43#?4g8b&D#P_qkLnI1QYpjftz#F;E2nV6wA+{l@c4xlj}?9x(p+y;nG0oRZ}D z$nxa)V%0Wv4C)uyQQscJ*8$(h(1w+H0V!6Yrfg=ezBNES{iz}Y_sIkx)X717Fw>y0v&53iR!j6lMdV4Mz(GjAsJR0ez2tkhC^oD^q7!IbuC&F1jL)k zZZ>Q)Bg?j_ahwznvrk*xc4R4g(gpxc#iVQlCdGKi3_g`#zsyR0f~^2pRflTSAIT8Q zPbHs5r?Q-rRuky_dRph++lXesP=QGzAh<{(61W4}cnIBH`c<)%Sy`z7d3O}yXcE{4 zadMF_?|RXkCA^0^E5N*lCh&4xvC=-IQ6tWf0?1tjamd6h`CyWYT~Ta`fa`I>O8~JA z0J(f+Y9U=1lF3m)5e3L@AglvqmxEepf#Cp6K(fDJH)Y9m<`yRPL+71U6X@hK0LAj0 znX{i=SSS+Hicv&IIJa!$*7}NNMD~t*JiZ%bdL?=;v-dOl_I^3o`xy~BiwXzKq;jiE zDOL@>%4T>&C<>ygIGv9jMN1m>qughzJHWsL#i-J^rAX=F33Xb3tQE!R9`F5rJPaWAz|6(N#nX3?#+a(aY`A=?i9)TqNtk`YULWaP*Kes*ruaWyTI`>1RP zg5-^FZQVY+DJ_8WPNe?FjLW0Bgnj%QR#F#4j_J)v?+Nr4av$>hqoA}}qYXw!7`Sam zra)U=Tb&qmyU3ZG>J?JZJY}MI*5Y7w8GcaD9QwhhemCqJXDXFfvnl4<2wmQvo#}Q@|L1OV(3;1XIG~+ z-2_-`wC6e|z9=>7raeg$P6Yc0<3oYz74GotXPd9n#$R+*{_ zb}FkJ530trVodXWI$MtB`&4_k(MavG z$-xxDMT{GmQ}ju#6AfwxbR2DVkiWCjQpgR=F={vcb=SeSxYr4rHf^GRMv3b4oOE-uidkW;SK1ff1*VNcz^`9#nQ(&i%(QDZ3js`Y&{H|zj@e}y9_{XTj z@OPy~-=<6DHvr>eesy>>4mhq6~SiTLmEDBmqkcjCBR%qI}SYx)J zLFhKTLQ)Iyt9*w${zdRGkT>9=!!!R9HCKIi)8ay=HSZ)~2mD?YS=&M)k>@%!*k!R}_6Cr&j6V+!}78l6iUjdQa& zzoMAk4jnrP)~wKofB>B~*ImWHPKaMC=Hd>mQUDINWz19AhgLRLSwER+4yC74lUcIf zE{E=T-5;{F;>URa3e771kDzSHkOLTUTmw$E_syPlO+K)kI}BI-i!*l8{7D-;$8uW=@9V>J(j!mXY z>sz_;YA|0>;>z}&f^;D<)4Dg5bFEa_b*kDsKs1GY%}LElbWdNm_31@7D`1?n%1KF9 z!KwR=t25&Q^hFqEwKb&d7)9iFM`49g3!?hGSP^ydnfi$Ky2ORLtb~>G&s* z?1h>d(vlD1I3?xkz9HJO=BuerzT7TDJlAr%g6SMgk`0`!39wSn>h5@JegnfBCfHSz zoDMN*usWR1X{dy3Y2aWegW$W(NGj1QRVeJU8g@x z#0F~U%(_T@qHc_HwY6Gyry#0|leB_iou|uF{)$%{pULZxI*DS!6-DBC_!G`>^#l2# zKqipaOgo;ctoS$0B!;3{;NM7MbaD7gnR-mHR6^Idsl7kp=i-OZGFx*+D-qR3v{s4E zHJSlyC8&;OqL|p%{+0hLl~S5S_RFM2sU;3dhpoXX{}sjjcS$A^PiGW>nW%!SYjQde zpmVY%yCW$j$<`-}#Ln{0a0c_1WniJ+fRvFF>3D%;exaP!GG=a5-YJ@H%@KIWW59j_ z2dAb|8*baB(A>k+=AM6sGCnyh*rp8V+c%u5o0(<({Ta}OKJf9tEX3o&K!=$7N5Xx0UbRAGKp#|J63$* zV65>zWJ6RLiN-<(n63Z6&r((p8l-+T>IC~^ilq>Ij2BI14p_zW1Pxyii^iH7lwBq6S>WLyqm&IjrCu@XyhQ8k2|%5J1n`8KrUFW4a5@V zxLU7=4}<;BvoRC8{)o<5SdFc81yYgq{XRf0#Q_%}xX{{fe-%zNS^~j_f&gh{aBYP{ z5RK_N9jZv}SS|f|I`P>WYP8dCGvQYAJ`dU-cP1JW7gTwbPEnHMYVFk0nUMB*xsWqV z{Cyr$44E!&lwxLf`>AJ+V_N&x5^|o#Bu6Mz2Oz>v7}`~y8rNny&`&mGeCNS zcuG-fKywC%&bN9%v;d+|JfHYa04Ut|M4CVoRP60QZvw0~*r$b#IBJ7Y2QFZ>|TeX*pPRT>8P(3RX(Q>B}AL`(iVQ(d|vN)%f= zEj$gaPS@!)2wjsL@SJ3N%Fb3jznRihtw_rO1iWie@W(4bIM$)4j>H16bTC_TT2K!J zPZu(}#6eDpPu)E}ThJ|I3fctunUjy%CgCm-6O$*+E$*Mm$+9T}y0YmUfJg_^y>H=% z$Q__v6fZb^-7ECRU#KE+YvXIsADJ%|H&R#a5xui%ceT7R5OlB)QL~BGTHZ5MTuj@# zTm$U@+X~LS_C@O!udK?tz>Bu#?!WrQ8)u$AmO5c9bT?!Z(enqZM7FRo!!0Oe`ajh_ zV`GL(cy(65>X_$=FyO=iQ~1tjZ`X{R!nxA z(BtTNY_A|Hlfh0wLgR1k4s`zC*azru@ArdHv?tN;@dEiC%76!=vpwi+-bQ0}L^>Mn zXL`P76pCYuNKLx*#HZ8f{_X%3uC0tvVY-V(Ea6#>1MWj0cd~CdckS2}oiD}>#;$S! z2l#QQoY!1CyD>-jn2-PNm{tnu5SzaOk$DHJgUOAkvOzU!U4zvN+4B2nhYWFYuvG?E ztG4^zvj5TQH%!Ue895&}vdK7z+*VShqRffKat0uet`vucZTSQGcrqBKbN`iWr7Vi2 za#pcDT@2Epq4|m={N8A!;=|>PjCIT8w25vLz;tt_qXVkcvDuuP4QvBDOT}=;Aev?J zT5Ebjkte!S8pEAhp<0VsSwnI9b2O5OM~~qL!DHv@| zKb>-Uu~H2c%hRP)p=`^x#0!r@6N>9iqFWX(=PJYHv>>f&2hinrCxHpQ)WRk5O> z3J$VZU84gPt@(6*+yTQk4S@}LaH0^C6}rA<*s;<8nQ(1a)yXPPt@u6yclxWt7uX78 z6dKdPi{|>OMRbvt=P9C-MM0G%70@>Opg$r)Fyq&6@q2+^hggl_AmtHVr~H3BlJit% zQC2^tSV^zjFB^Ayu0 zR${h|2b2&;X#`yr> z3<*#w@ijd|hH_fKkL20!%jkae&v@}kd#J2gbzENALj|LiO;j-0@lo^*{IvdaxdG~- zbCUf&svVh_0<5>>sXkI4atjHg9@&7{y92!r-GD-71A|$=G@iyIzYy(nMEyah zB$ilVd=azN>$Q?p3>49gnO+(gZ3&zTS%C|=0OmP-x|t+=A@Cjx$|3 ziXn3EwYbHeyU1o+4xT*Hk6N9lKAkNbkG5H&Y7|fn>XukRr4=pA?Za4>9F?JQvgpQ5 zk1i9FR8IF}LsZ@%=r`E9NUI=VV~N1>0R`!-=`qObqjvjIyCHqqxWJP6nB%)?T7RKy zOUgme94*tG8|3_QRxnsh{0+p!Z-X3{*-jIKSV(6y(V37G1=9NAbfZ6esTmc#*X|%S z;A?e?4Nk1tU@EeSLDEG)bc7;HmjgPmuW^8(GOA$D>rn^d_FMO8&5vK(&6N840EUkgaV?|^tYjP|U= zOFGg^vk+GGH(8^Z1LwO@4JF#Yj{`?8t@)O)4y`zPi2HBX>IpsY9G-Jr3uKpw(<}lj zg?z$t~cr`iSBBbZmrf@fO({=%^3DZxyN18Bcfm6jhLxBUZC&8`@oebi98Tm`H!m>brs$TN*~#rVH%K}$ znj%H?6Y^kxKR2p}N$Oqt<@9J#C!6BNO6l(x%O0e!SW(ipkS`Y{q8N}+l;uKP4_!i+ zYsGW*F$-{&X(%){93ml(lW+9*qC-sAX*C73r*Ff3WdAM2i|GJM%zPoM+9A=4#Vq-T zYSW=L_%N4M`J@R6zcV(Rf%~3D(%>sU4Zc6y_a0s&x<9SkSV&%}Sg^#sxH7-H6cs2s z-Og8(Fqc)Ngr=x6~d5$FgW z1u0gLk0%qRtdfg`0$pm3$0ie|1bk9uRV7wF<)lqHTCa*OB#Y>?q7YaCKzuHb^&6|;$dTsb2bO{deTVm2GS zz`f5Wvv`CY2V9YSDzfOik7{>~M>xMb=9g0D%xG<9GV2BDbiz-QJBR0-bSYk+ZjT0G z-t!6t+TWeMQS`Ikk0I}pP;}Rz;}7CIEz@-AFwZzCE#kARf|Vah50A$+-KdxCWM~p0 zOH0T@*;F}UaKwkuZYEsS0>_<-__>}GaE*Kcv}>SW@xFegLBDAIP>%E~D(yE{ivS_p z{EmXwcPZI?MiD{5G`l1lDFf2rNWD~5(n(>UU-1d^1nnYx)P?~4fJ@=V8eFDEk+D>W zW~mjxAauLzKCzEiovKUTiMihxVk#?XQYstyC8hUA@_zDKh_X*(mA=8M{lQLbi_n`> z00AztO2--fvuK6llNR)^ls!reO>|GF_t8pCzhFFAX79U{DRlN34dyLofcUGNNbI2? zw>wEnF<=5wT1MRb+1JrQ7QV_wUKU zCyFhL3a;iv!X=uC#AK?Ge|=+1r8wQ4DLNxGUYC0hlTq>kh?C3cWL75{)Z4|rubGCc zvW$CM`)Bc!Xaew!#!LSi?ew?&AJPGW4mn z6JzTxu5Seqi@Yo=F-v1EIG@d}J8yYpM+Ra5-Vr#}cx6jio+wNY`Ntnrwax{dLYr070~xShF59kTV<8AG^#q$ir46jjb@&>I|SYI z4*1hvJK{}Q#Ht~!z6wP9+RbOA++22i|MT`- z@~V0|k<26j&Ela@=!7puJeO28QT3Q8bq!l`edcL3}3Lhr)-)+TkJbCy5_eZidjM|G^t*(ZAw6m zaFN!%D1$rQsVd?0jFp{9;@M_l(WTop<{Yyb*0uzxJu_3w>526f)1m9dI>A^xh|3zU z6h_z3n3lcmPhb|S|)$|UB$ReaCQIqJ*5fP zmLr@U_P&kZhyR2|!9KSj6hyNnPq0YlwwXl+kst=eh8oi~YL9-+9T+Vuu>Ybw*_fwZ|}fYzz>*7W)0hJB5wuHnnygFRKkB-aOKS{Li>%^ zI|MG*aPV40%WvKx1%(#bT2~z-H4_UYOR>yCgwWY!Ol`uco=!wh*RDKxBDPYwHX2*U zS2z_Uv3~PuGZQn$%AqV?e>4k*u`I==Z5%fv82>9QOXcaT9#plYf`y3_^G<5$@#jCi z8i^mxWT@B-2SoFW!Rqnx%cp~tV|CIc1TG(5!!fv^Q`t#wy$k*g{yy_b>cKSX*aVNq z>J1(zGEhqnU2wZUsB-aCcWfLV`|RXX-k^tA0>zMOcztJ_IFWMZ$E}!_W{JScU+N6k~HKd@Yq` z3n5F_VZ2J}nLMi}#CSSBaIbe$xWO5+Kqs0wB=Mw(Q3 zs6@orT(K})vf_Hg5_w)9qITZ~uYhmEFEaNtpJG;;dsTqu!= zPMIhzI(42a=6y<&;sTX^*H@XNgk8dSA z*f43>CdC32jk9!82XX)@JTIUaGDXUD0RfIrSpls9F_c6#q$uRPRqkupPVnG2oYlNl zA-!V=f3qCY1j5PB0+GydN;Wnw8ia%(ir2WzY9>+(3LG}Xq9KY{F0iYjuzFgDtUh8{ zMH8PRD*-N4(L^5%cdCdp^l_Qt>mz~k@_@ozfH zobfYR5k!SRR0XLsz(JD@O+Gl^97^gg)B|3GL&rOj`C{@PEiRtkO z{jPi%e^Yomc6npt{KbtsFZ`-#;2#O7Vg%n@D0h(!8@2FAKefQZoqihc71j9sZne{PLA2YwHHi&>zyi*!ws?m&_W zJM7g|MjE#k8A-(Iumq{a&Npm1fy$<(0^X$!Y$U%=I`fY1=s5a;ykI4r!;TVGCRpCU zQ94jeWaP|1!qlT$GQneMRl@jNQpwq*9#I7J+4QkgAZ3_YLkvif%#n-|Q>IuR%XL^Z zhfFhN>Tx}p!f5h_9LdaOQj2L>7X$R}Wxfpa@EdqXL)d2~@`mazPkG4C9Jw^6GnU=d zDjF^{a{Up~PDmTtBEF~OTWD{HrMPx=?#Sikp~C8wBXd_)3umw0MMiHfZ&X&t62Rsm z5?M(Aii9*FT4Yf^tqv&&VY4gcV#iKIN|Q;aiokC#Z5|)WuWTM$+*~Wi zJ9hQ4_JCUZ_0u>J?#(Q z66+QY-E_?aaUw_{>k^rG-3b$H3)AENAb(nnISBMRA=Q>uGb523Ei8$NR0`i9@BR92;z-Urlhah@?`1y7 zj5VFgMw8@k_>#ZjOa8{bPo`D!*|u4#eV)ZG@xN_682}*UM6K8Tr@f%QOZLAv8H>fz z@ud5S-k~nOPhfm6GLKT3zrDWATzUzkcss@Ovj6mu_q0D`ORW3D#m0D8qcfA_)ectH zpA^awt85aE9bJT~#AnMDowTdS3U~b=q#M2oZC#0qydcX$fEDDBlhLpbz-gKSZr0lH zp*H0DC5-zW%ry`BhFlYt|8p2+m*s!Mf4by7?HBo$vPz3_Z?2;35BQ%4lI3W$m^2nf z!^NbTZ^c@(&5&hTW_T~4v+;1z37E-(Rh!l0h4+5aD(wGmC7fU=m13IY{BCMTK2EL3 zMU3==O|nj`$VK0ZT=cET#eE~b>;CMWvEL{1yG)AEUj<+<2eHe_{CW2(0?FsqJbv7B zeimlAz88jEwo@*CaO&J>yX^gde7f%~cFFcbm)jQmZzIeJ>V?nOn;C8gVz%Aof(<}* zn`rM{^}~P*Uhd>QZSU_syuW+s{_few{ydKebn%D=*z&v>Pg3^-r8MH~1j0?ujuG>n zoqjNY)rK#lr`mfmZvCBLahmeWuhOm&!k@B_6L0T&41mEMZM=$p_6z=br`OOp=a1sO zou=o!wD(;#3c!ttBWeWmHuVAq*xOxVC2G}7o-{_0CVqOw0~HCfiu&5)m#`^N;v$Ew zB@zwRBsaJsvbv#q?^>vAy~q3z>u%1gyOTFa^vMj&w(9C}*`*2|_Nan~tL-(i7lSbT zZYOPoTR+6>eH-gyj#;PcUGIOr{pem}3)PD1vK=dZYFMv5v75cM3+va`Kj8hn%dX)4 zjd*il{562iER+l}tqc z@-wSf=9ACe)!qt$)#so6^yf}Ia4h}&Up8~&Ry(@_{#qZTYh9viB{idTueJJJGz~Jn zCZp47-OtIHTv|a9MU?|(B{ZY~(me;HcJPLf%%){6E@@Rcl9MT=b5O)) z#!-2MV>HSyjPG#5FI@K6f62T6sXf>`c9L)1NU7~@-HCm;&CT42k@)e}5Z=f71`{V4 zc&5S5L+l{o!F40-2`|E?W`pf=1Yn(XB?bxiqAuY37NtvNoscm~f6)Pd_^(_j8j}$x zi=rVYAt$Rz0;_7u&8zKkXXyK>-hnwBj}dHFDYB&Fv|9Jx!qXYqncf!Q#r81Y(Wg>( zK)zRV6m+?H*+=kx9;F^1=`p>d?!H6%GXA($A3YGJ-Wh-Vb)1$3)5NdJPd7?h)WOQ{ zna(#V#*#^TlRNs(n;c-b$-Bx`6czTx`&;9~@i_S{hQlF^T=QA?nwQBnukfUIcLwA6 zHhTp}>#h$&Di%w1mx*_=eFVfMT~#E25HAO?ssT-RuFuV$8I4h5UcOiqSe8=` z%k2%~tS(M`VhIE-N8>mMN|z3Q`P_4}`9x;+>_}|V`PSH(w20Jf>gIoDren)z;TO8a zOzs}<$9vrH_c*?XJ$5sk{qFZ@FGD)u&e>{L150feejCg#&^4Dq1s%2iMz29op)p}F!RiL%Jy@rzRmi2 z%Jm2v#(UQgV^7vsk~gND`>s0EA0=&E1GB+8q9fu6QP2X_a!!;2ng~@fSPcWaD7WW{ zjp)D9gC`bDum-+X6irLVO{kfcAV^5G6(z*EIY^AlX^g`OtnGxQC0}===XDRArG2B* zUI7NhCM7=IMiutTo(o=Vby}dyrHlQ;73M?qtuv(Ra0!20tv5635>7goRoAUOEO}3( z`()Lf>gx*KGxP2@TdvUU+FH{UN}}PtQ}liehso~!iNj=ke=3bI*}p$om;V&LMfp#k zqVnc4pMI0sth?-|n=Y2Ol>M|#5B|XtqAG1V`fmwID!kEQOu)zbpD5PP(Ei;`jPuj= zoout4REi&epO9xe&meeWmim?LIP#WYue2ESo$`N!^6e4%C-3keo!Sh>-j)5D_vEr&jChN+Tg1OwG8hc=SZ_q}dLCf%S%o zc%k8BjX^4IIX$HT=S(l|M5Mlu2bs!EJ67@5b15+AFiMHYS= z|G!kS_)t`{odCeN%coF0lU0GGC{6~)yHGZQ;AJtY;CR8zWX3RHW?|t_ej*z{63M3p zfhnT|MqHFM!Hz~Hd=9ncllVH84O0;D8KIkhjYRzGO_u{^GiD{{DYosKlr}LHsQi+H)wOP^8M1`8n6p-{m5|T$hc#Z zxt&Pj)(H4ebnx z*^$~iN>X>HX+MBGw~!kl*sP(IM!y#yQf(;AjWc9 zR9QabEPe3Q;zcJF5?EHz1EQGJ0~)xyRF?7Hr2wKc1FOe4u+rYEfP^QQ`Fo6o&5Za} zn~vPxZLG%U2~CXRb9{%}#bzDYz5&}Qykg+i|KfGMMb|Y&HJz+$vhK1-7x79X4WDxN zD!YV1EsW%Lk2t3D!n zzKhrOGR3sE^Q`G47Rfp~BwsR38IO01@@KMRr1%`qh84Ul{^2$r@%N>WHJba~=!m^R zRFJc?;Py;X1$+mm7P>su@axPpbBXqrxGdGKMxOQ7HW*uVskg5*2%+`)K0Tp9d9VMm z*X%u+$NLF*9iJEB#h&%&9dh*^%-C(T6ujPzEc^3ql(sX0Hh80>+P19uDAind%_Q$h z7#|UnbNeg1?x3>Aq^r4ApAnX?4BViFC-)cHPP$s^WIO zS=qa+|MV_>-+2}@{(KgdpT-!bI~DFOx-mQx+NJs6_51EWv_74>es%cNbn50?Ml=(w zj_UZoRyL{^b{m5KzVV6F;+3P%K9Me1$A`73Mm{w~h6Fm9BT)N0?ZH zgm5|ITq(My3RSCdAJRgq4^)9Dy zSOv&IIQEf1eIlqxycE!blA=H}n>1~m){K7&PQo{tHD;4)2g!qz_plTwUqYdg?LH9x z@tH0h5(9IS2W+34e6lP5#6FqJ=en_1kat$S3tqu8J;|J=5}mAjwY9$3zey3T!57Z9 zF_V?c)W)3Sn})xVMAK3yECVIz@fca9t?sv+9hEJ z+;?kwb;sGey%14P&I>vykvs<34RzbXuD+suaf_o;=epbI-Qi=d(*}6HRd+v9xpFKxC$=R-M0@BuJN-i zNbV8Dw)%2V4CL{4inMQ5@-OV_pMy+rYYRMr@8EgnUivm*HxJUxR_?NtA2%B;^nRq{$(EnrC~C=R6{3!T|mbulQa z5NYK794h2ko+Pv&joW{=^?%@}@Yk3Wj*=T}di2e_ZVkp$T9cN-T*yKms}Xh_t3V*{ zav26;`+IkP_*lF$VULZEg%6#s&H*hH=OsB;9*(Bw>P{d%nn|P-TYL)0kqZg<|D9b6 zcpTMrzW2^^cOJVlJF~Oew|2D;t+WrlW!aV=vWy?{Tb2#BF|sYNjV*yJ1K~{|#1J5a z0x2o4Q1T*y5DE#UltP-&gaQFVfDn?V2@OfgM_o!&`hCIbxp!t|8ym1IX|L|i&d%J& zx##@n%(>^>R#|uMz*#retY|D`-^kgETK?lhP4~6HBZ|qsqodhZIRKD0WP$gMQ5NMo`WoEOxRmN{ay{Eb+?mclh39R7Pjr5h&r+>GsoZJ^vcSPyauSiXH^S*$pqx`kAZs=>89{^G=J$(cn1(hG zWYT@t#yLB3*?d!DdbrkoYBn6zd5#Ok6-g82pe(}5)3l)N^U0D%LJdMR7EtY|4wFqp z8YfCn4h>c3HU-8u^;U?X+O|C(;pzXH{s5l~^WV<=4Cw6B_~|rmToFb$ih`)|sC-Da^E#e?`7(uXSW2B?rRbC8i?&(lmi z%y%eFLQ{aF;mdeoL1(xn3wS1-FIlxNN#GSZs1JO0k*x5-@;MFd%~4L)WWP$9|!_C zif(VaDdT>W(?a_e^E_VA6>iNqM2t7J*zI#b?gmT+4_vbd8nJ9nw5=E=LEWc~$O%7f zPSn7>6-x!1TIGyC;I&5gHK@JiP(EoB;6t=6)BipFpKR~u2%9(RskIp9O@wh<%;v4v ztHFw9F~g*Onw21cb$6!1-tuO|IJ1|NF6un(3ruT|R|m+^<2{*kw}e))UBP&s;1_Bu zLp|qr6JO*dj7z7Nppu)P3l7iA42I~Q&cMQ!u8oVzKE?47PSAkY`fLempr09YOQT)h z&QQ?GWigZ|Yq2me1P~?F!r7{$JFvXdE!6Hlv^1=^`#YWP4f8OE!$nahNi{?jSysn8 z=a%go z7~4;zNGzdg(6ofEgX(OBd@^tdRnp_KXbO5vARyWr8X`VH@yiDJhTK%^jzkJRH6DXHbg`c%MVx*~)%yQiz73VxA4Qhr~^o-8wvLL!_AH>#Z^&vx~e-}0nh2Cpx|_tW7&X$09ppJ zsW1(cM6m%OXNe&zD7egx6KQ-Nl}26ybfd~&#zc?c z-{Nync_uowQ+4;UvFA^e4mb)4-8IfESUv5`xSe6*oW--x&JWmrInk6i3v;40n{X%> zw88@C*IN`ZoNjikTrO)CEm_x9K@C3&sIhl2r;@ZoqlL#y*+58Qn-*( zsr)`beu=N=PC3b#tp|?dIm(4TyoYpzbmKMT?vuRs$Z>Q6vghb)WAqd9EWUx;iXum& zqYrNb`2F$S`{BzU>93E`=gH6T?cCb`e=OeLpV;#QPW`0k0T@k@61K=|jHA@^2fHZn zX#M_c$13dPwd1Pb1Z^Q1oC98<1CFRigJl^SKhno%%kjK&$Y+jqpkbIWI{i5QA^8R9 zrI#@YLOq~|f!pjwSv1Tf$R%Elj;0q-!Q(Mv)@3(cI?LS9Qr{6%-zsz1CE81`-qY?d z9~=-u4psA+YX=2_E}x=*Fg?tx-uu(>^TblkWFX$t8UtpN&Tzaa0c!|4F%8=t2`y6A z0oswsylA#ch^_=1%~XGTIvh}fEoTzFh%BMpObH))0upu{`P5RQA*eS?Olz7Cc(5> z3`!PhLaZy3w98Bm)A)ZLRDri0Kzc5bxe!J4AT^^@%XR~LhDE$3H7Ch~4k+f5r^J;^ zvDF4iM#4l1L^>LK4h-SO0!_%MXEeYlQb3jnFSa!-E;qI}g?XQ(gcZ@z{SjYNo(E`M zJhec4coipcW`_k5%`f$~0D%&o_s4ve?&AeP*XNo8Wq>iERtrR#-7LbW>MiD2UnToswz>3%Uk@^{=kbg za+;`Sa!oo7UD<4lAggJPCycN2Lz`7jmM;TIPS#N{;1@tEDq2~e=955K1CXYJY=_~- zg)Q5+S2*6u2?CZQ@uaqJywxt4jc|iPR1m*TLwP`7Gt(F2SIHW*i|yE>+$di?9AR>l z`GC+(@Ge7vc4zXmQ`Eb!o2CF}j$(AfL48=_%*g6m6&WeeEtc8(0XN{VJ4iq?o|5a~ zpKb*}3V~tZ{g-4HX9OPsm?|olm>e)1o+wejB5~9xc?%#Kj^@@O!39>@U~hR0@3A6k zY~*P2$(-W{whq;OG!>QA807fD3h~*=8EFLmb$vQ3Gi~o^)8gNX1DBTiKiv@ z+5h|$tM+fZ?B>YQ)39dk8hRDVxAdf@hN=U@@Y#-+^7+-t@j~w0=4`lZ2hPgpseQPN zHj?M?t*8Yw%Vlg_8@zEfP=hzFd9UV1<+0g#F$6=&cP5y&>l4Pny43H*F816kh5EG9 zFN8@fx1bisT(aDotPO$s5|E;U<)(p-RHl1DvSUM7OKG9@7yWU?o||p$Qk{$)O=Hwl zEaXBlP(zg#e8A3kr#X@Hrz^=orA3uP5{@VB;_Q66KNsw*2C_a~eM-)m>1-_ON%LMxLn8?S>4ZL!8Do+xi-U; z@P+vt2EGMaLb92MmCX?r6A9WT;}>Dv-S{GIR%6#Y$kUmRHA3`wx{?#-j&329!@uFh zNXSU=(0wy3GlPVd8b}JVuQBQr0Cz~{9Up2iv@{UJe=jHeMnH3Da%?SFw@x^14nR~# zjaWt!Kr`kJy>fIfcpxsCpN{t91=NK47^0wjfkut)X8m~#uw6G2Ty)pz=`x4y!gOsG zvK}*}T`O=O9{fTk-vw(eBn1?{;6xjFL6I&JgEZ(qFSs4vloA5L;%G)PBoYck&q^kM zUK!sg$NWJ~kolxxMMOjO>Cz^erOYSFS-+vHHn-@|j|+J^Ilu|MZgE}?$;9;gWG@i_ zC()dlIZsB=_^GsqHe1(1Jr=f?XT``G)O5xCkPZtC3LVh-s&qC?1`yI+hyB zwx@CXp*KqXjsP;%j^e8b5k<}i&=iOe8wpkMlS9uYoDhI9mD+Qjo37wt@|L$Vl}1!% z@zCjd_k0TK?M`w`nPEFHK$`bsmJ_X=g@mQTUzBqxBXW+crr3stP1b;Ju)-qTU{2Na!fk7ymwt1nWz4Pd;3y!;WfubRA z;{sf20nE^>4782Tx<+`*P%V*@L?hYQ70DI?Ce@AgMqL1PT)-C)PWD?dRpI?M49YqX z(mbcwn5&fHMw8Jk^9rX2h@@gr-g#asm6Q31ABZU)vS4gDDJVQH=>7uX!cjX&WEu&M z^?xZ~m(PIw4bSMfK6k$W>kTOE(veXR%S&eUFI>`h*~3Z4PR1MTB>v;-HOtnmU2%JV zwK})2t&Q>rv=={zuLQXgK!WxHrT75tSVClqYX;0}FL(eahi>$a*2 z1pla^pu4=Lj4+u1eWk?qQMuQlGWsz@AGrCMd^Wuri`_ii4La}PF3Ed@m@Rl)7+hZ= zsVo@Arsg@Hy`09x=r5fq8Zxo%mxWpVZmf)XojfiTnlC!-{%_N`7hA35Ab*ixw&HKUM`y&_RO9R_KcwgizI->`eRQ}8@j{S zWLITzAQ5;PLo1hBIji4F$+IRpcmr@J7pvkwtiFFWwcuTfg-d-toN2h_&}+9yq#-!< z=gZw}b_v&p07WO<--H&VJUNaS-6Ev0(P{K*wYCllLZMk85S_5;gvBh*7ex95OcKK# zjWyc9ouQN^3xzE;UoqO>J6`=4s4Hpue!NG&4Ezg)XTH{{OE<#;R= z$q!f3e0j7M$Qj|EwX7*=`hBJ0lBT(|wVSNM1LP(cE2{6*bIcqWy)D$|$=mD^53Zf> zrsqXy{^Ot#RGnjRCsDWVC-y(KZQHhO+qONiZQJ%Fnb@|IiEW>p_ujhao=@GqSM~m| zYIjxdy`J^_CN*DDHz$J~K&=ewwF(DSbs|jRZWbQx-MgvB4C#|-?M=mlh z#kGfT&Ra|gXn2!aN6tkthk{A5;Au!Sg~jfWA(4m4i9Swnz^vKLq}o zkp!-Ha#IN|3iD3xmY~t4dpu=`;x&~Fh}q{5DA zs6&p899EUR%Rt1YXZr8_naM(-YA(a3yS!GASXlYmAk*1wasf7usoAfvuBc0P|L^dn zbdbig#fQhUxFia3Dg|)jsw)!l?>>8$V|>1OllO$HgqacP(qA-8L1U89M4uUPSAVf@ zvrNc2|2{gIJGD7o-+F$*evy3veZdhD>Ck-WZIXRyt<(7Gqa9%%KKo+s;O*4(27igC zq>RHamUqhfO89DHFJH~+tsK1ZzFh-fgVIC#QNNfQx82rutaXokD4&>}t9dtk8nk14 zP2AkewKdIiB1npp5zDiwVA;ip71j-L86Py19Z(-YAbA;y??3%5ICQwXQej|C@uKXh zZm76jn{1pc%1CQcd+NMsk|ma1Ve84-$mpVfmisyjQaz!mrps1-QJbUE+n`~ixAbdk zZgjCc?VaDCy`-hqrnBp)J9=F9u=YQ%8}}ZEpk`Oat3s=SUv_JqvhT{j+;Z#r40;O2 zb$7qIigfFl_Z6OTn1SiJ_s+{*;ye%JYq;+2JvLu4!_RK=m|i*l@Vpw`B!72*>f^g^ z_MDm4XZa2VBZfH&m50UASnNCb?1whE$?H`MN{?Dd(=r@XDxs(tOZMmxQidw4hbK7t=_AM4^!#KpvF zu{Ye5uHWUDjZ%%4}1}89T^+qRDYV{_ksfQMY+MF-AoYdMD!K z9)}(A{A_sl)6lMkxBtA4s8fc1C3haR##vTREUnRzoHC)&k)6Z*cO11^XKY-xT4!pU zwOWV!Q<-9=;f;ZMrO7EI1zV#7QR)qO=o8ZWyl?`SU9IwFx&~Q538tG1V$>@_{dg~% zS5NnsiFcqnx}baKM3poH^s}ccCgw?+-WJ58dv_hjDF4BdCkkPyI{K`X(%l&mbP`N=XUH@OyN4^L>8Q8fRydQ+ zY85rb%%hhpYD)V;Pbc^Yybm#Ni*t3Ot&NYk*Tco!L!qv>yR7V$6nykmu>bKDzI5A) z6>rVy{S1E}F!}ub?jftU*z@IvtDD!}-&|blk?Hk5bC>z|{mJLIdGXxE5jy9?goK7M z-!8J@1>2NXt&NMN^Z6DnZ7{ticR3l`JPdw#zH(JAub#SD`^CN!D8r*3hy-8Dby~E_u&aI?edg@*rmA{r2!y zp>MAjZTq?ne!S1))&XLtZO{GgGhxo?g-{%gAT^;g(?QmA^5pz3BgWS&5IY~`)Q`Ax znQ>g^ptX!+v2p?%^W|__E8sg8!cBsyb-~1}Pqq7>d2qZL0cbYn$9c!TMc{RS&hfZT z=}$cT-e>6cV?gVp{%fLCc;J^%$ZkP! zz8hYRZ>nlnT$V%v2O4CK6fj6$*i~v^EHf0AHx8+Ml1fn7iP(tF>M94=k$xJei~ZU4#nXIE4_d}K}NgEW`*+=ebGLD zeIoDze>`WMSh~PZ4nCf+J~%&scLKfK)Xnx4_6f2V(&t0WZJnq-m_FdTz;yx`j#^Im zX9XWPAB-R589JXq;Q0QOABmq}{t);yA8@?zK0!~8z#fRI^YaSri`eHTPRtqT>+=1@ z{loo6PVC{=;fngu|Lx+4y&exlXG1hK;2-`in-x7Vav|&l{lxA>(T%MYNGoWYGd`i6 z1^x>7`t29&7o}8==uF@~9q`J1cTDQS`bP8y`^NId*NKEHkSESw&?mrNq&=%&ura4T zhkGLDLO+^UXMp!fi#AI?TQ}F2zfu32l*TyEZ+i8xHUf8JWVmdYY{cy*_??_QMAX>3 zRBWiHXoH-5xFsUzePjkJXS2_J=H!dgc1iJi8UNAlJK=Z)FLeyjCWRXJw!8th3 z%@mu@+a?jud1mki)9~BWgLiNmR1>_X1w*cYBGd>aQh|c6;3UKdIZ}p#uAn@`2q{v6 z0AfYOZhQP?iNE$NN%`Qj| z1+pvC3=`RWQg;(RGBTdcJ}%alWNc&jSAPY>l9N+lMWDh&(`MsPVwn#@_nJXRFM}ex zH&sC~L{o|NZP;|yN~5GVP9-I?Yhb|m#n}Y^)OnUqIHow$=j8zMYk^Y6(jQOea^1+0 zut`lZWKYT@PunR%y=M)xi8uQ?4Syk^uMl|5!>mkDp-dA<1GJb}OIV9;r-fc>_7KR~ z>@?r9yjC;gDGEmcTUyTiK>w3&#F9a-wqNqmvp;Dz|4BLe&s9xS zNs~ZQRzY4#MNm$KKu%so^gpSW@3$IIK2vVgkm_5jBV~o1( zx!D%Gz3wV_0$I=DotnOwHY=KB7{q75FnH|X@-h7E-N79{YW?e`UYidzF$M0spN`fy z&@y6efmEsqk)QyF%VKzPG~?fd_&IUM>?t|2vF@B*t}zeat2{Q~?X;r~*_|C=ao?_yTn z(NEi~^Ul6hlRS*^nh(J-RSN^D6Yn|HfH z(Y@r&;vmku#s)~FCLBSrX`aN??k0bK+Z#aL-97Q@eh2mGyn?4nk@Xv*dQ++Ru{1$n%6*Xf2qSnu+ zrNRqwS_IM38zPv)*otmVglbwP4?xSU8 zW_k7096g_1TnM52z2&TU;m@Yc!m&7@YbiRE@#@*&Y&M<2Yks?o+t6e_lL57`Py9+s z`RZv}Z}c?z0tVj+c>b7iG-8f2PBM~?OjKHMN{VbWzF+tzUt#_hA6vh)Uj94Gmy(&6 zrUHDLMvht{%d4cMRBy3d{hhs`F!*f~?jo#9BN$6yIVwYf$#);GqLCP4GKdmYEyk3f zlP(oc^b#G~C@&O=e+ZqRw91F3HE83(A1o8@@jAyVVyj zwM3{JMQIgq3;P?cmEz4_<(qs;wEp>quD7D1kFRi?iMg(1?Nc z2GC&greMDZ(t+ii313tSTTkOXVZ<)=FzNR%db7^_#^!H8K!dNbx|rojG?2xj!OuQ7 z*zh6w{I;qw|I%;PgjsQd>o(r|L@8$Gh$2}v^7uLuW?6`@a7ZP+p$o8Ph1vej+Wgu< z{BFAiuR|Q9Jez~qHX3CLzhjIXjXwqww*6Xq`hLeM&?wb_<{gnZAx-{~PXU8p`m;2< zvfeQ~$Qb9Be1pF+z%cF$L+A1vDE@<11WlND+aWU0L?g? z0KXSb6XfJuGZUD_LR*JWHv)PB?t<1!u_pq0LhppG(W-cVfFl7B5g`ys2#b*YW<)bF z?(DzLc)VhCPJsJ}u>}5nT(*E5QCa-LDCZ%p6B=injz}#*TKvSQ_b9}u-=XfItp#ph zFm?=B10fkj3(}#mNYqJuTzp)De7t;u$RV!8{vlPoumoBoikCL?CX}9tRy@ckwL<0CEU?lZUjl&DNa0xH zZ^SAh`o0*?Z}eM2zxJ6$0(-(13Euet7DBIbf6>vlQFjG=>4GhvxDO~R@drqs-$_Rb zK{G_b?^2ubC}_fw5%KuAhYhH;V~QN#_YT8(1pLH(6#ppHsN5(LQ9>i3MSpBuUAfPa z5LXI|3RC9d#1`h7ijKwXW4FXDEh&){{ED`cv&4WHV!YR|3H0Y{4UMq643>! zp!R#^4*Iy`yn~e)rSA(rT@Va2kh!lqPqXf`c^hI1S^BzT@v+1F>tBj^_~p*v>-_Eg z^6|Cu<74x_mN=99X7gbl_x&|{t^a`%`*~`er}z3WS=K4vg0DE*2%EZ+>B*B%>yyKB+RPN<#2`sl@ZSBOUL* zc3sa3*jVVZG2Q{`p$o+N{3!|4<=gqgyuM&qJ##CpED+Y|3;9k3uUmz$lf$khqG=^z zuO}f}6@CYj>4h(=au>(A&W>Ynw3QO(*n~8)3azyYLb(f9od2*Llz$WJVbFLI-9aKTQW2Gw|F#r4Y&dyryw1z|<}^J#Ft+3Wcv+2RY>B)6 ze%BxYc?#1*4&A;(JRuwruET)o*;P5YMicUH#eA#bodt(@E(dO}6>A?y(pg~0ww#G) z?S(vg;z}M8SK!+r@(*?XWhtcui*TtTs?=2P*heG$7wAL2*h7Z(+;C>$1({T&&TiNwlm<+s?RSXIa- zW*fDR>`Fn!OT}JEQpq%qGbGwk7^DcNc&R8}Bqy2^)rs|}PmIERFXil{W~;BPmF#bE zb>Yb{6;26MYHpzQYhj=`C@8zR`y;V=V7@qevio7OdlJc3PRsFDK~PLlOG#2|4nZ$I zBSSqgRWUiaiIkm|7%EIBgHn>SYQae=(do|B(i78isHCGqI8ez_@85-|vl)Egnsb=o z0E34a!5*0GUs5?q$UF_ z8Xy>T(BOzcgB4Z{KA3gT!1gO)o`<7uelxpxK9RO>`r^a2^nsr#Soat(<8sBI?i}u! z%{{rfX*hfgxaV+BAL$t6na2I+c*PL+e?kYFhIs!kB#q23L+_V9`vK8m=yf~zupfRp z1iui4UyA;palH?Qo|pd}V7E^n=-zvs!Y1RkPv7n~^tul_ox(0e1l}shaD$6R}BG)3BQa!GgQeLTAEv59Tno~JZ$zA!Zs$cb0`c`(9HdiDbhZz%> z%jMO7f?C8LyUpfxd=gv~7d?%s%g|}n?AUDF?CfIUa(+@&G!xy+@HT#uW+!H5EfTDJ z&Rn3^cL+V(U%OCn@K;lPPSC4Mi3K~LGw*#04s?&by)x&1qROCUcPTrtMIm}?Aj>nk zO5&e#L^H<9Gx7qz$O=w3?o zHz!_Gx!^jjT@k&*2W@9G)D!1Q7?2XR#eR#a_UR=p&jb$lN1Akx96u-?U$E+{U&IPy z^l#nmY4`PZQWPsNWQB=))o5DZKCexkrf*YJH;dXy45%-=P(QW^3#MKx@p9K5{ykOG zG=;mjj!lxt9!K6&ayBq#iRiLJDp_O15S5KvrS%=Lf~0vP#|7uwgS30K0;1+#%7|9C z?U&y5wDzd3`DZ-y((kXe=XH1Z{$C|GpW)67QmU94&m^UZAbqA~NtLMnv4`^gDl_pZ z8M&;*6WrVEG8#3TllEJ|J3kVqIQ+-Z;!ZRipCX>Y6)oXpsV2K%w+%V3jna?PM)0%} z{5&uydfoCW>wG0f75&aOD}KT_t#3jNI5)qSxfMOFfXIJ@L4LW<3y(HAHDOTeK5TIT zk$`9k;-&LZXs9LHovd6D?P|8F7{Lq!yOc*;w)CyCt*?zD={<1W%ot( z1pzSM-g*QfUYmeN<6(IuVWDweln zbP}@#pv0q_@p`tuN;$anJ*z9nZqkVq1#q4@CCOTc82w_&$J#ZjE~awR&`T@3LMDZ! zKi|LjMv@=-D9|xt1hb zGqTip0DE9!k#5*1{^9e|S?*n^{~ z)0Ce5y8>R+jEl9^VE^5lkN0IeWWk-j+xMuM<(tvPXB~VNs7<+s5T3 zm+yy>E&ajDE7cZ-c|+z@bFGGU;RG}5VIGCGaF1G565#{tkr&KXqMNs?HBcp6pk#t> zCRbNQZ!%LmZ{IHl4%UFkvCRzq* z@%4ad9a9|rZWN_k6DPZLT&9$Cx)V2beaP0hYL=sih(|Y|Upmjlm7VP= zjJtazyG+5n&*L93U|Z?QnUE|#LiCOWJhh^}te|_a|6rNHI~zJQgliw4QTwRe(`hpn zla1N>h_-OrJn@3U`;~S?cmGJPe@*6{R9?VJY$!nCV1#qRa05WZs)=xS;{In9{@3<5 zw|)=GZcllha~E8CH#J$$>>gFA>QX)vldf|Yp+6<<3N^K8&xXWSb&8K)CjXFrXKXO0 z6hq@-Z8cR528vNdi%49Cb#p{iPcodViPsC~JUr52;0ZBulorC2>hmS3YTqK-)9Bss*&d*EE|lN(T9fH*O-2;)hxK z^10Z;6^`#!p;gg}3W3KU=UF6;pk>jasY#~V`H<}Oll6RQ=AQWf##(>y_5*>P6s0r_p6cK zPz(v=6ot0?vYuONcF-F0Kd84RDacT&tD8Zm7R>=Q`WySxb2kDAhzU6odu6d$bxI?>_4e2oVA$9hYQ;j>INyJgvL)C zrO_3-*_gwIS5zkqCD5~MFuWe6F9x+>cszU!rSqDY)G5`hdGQ^UWF1d$cV~Ke!swLP zN=DGns)Nd!8+*Ik3k&-Bc`DlJSxUo~4AbAlr22~5bc7^mEt{>dQh%jd0TNU=$Bdh~O8X}EY~T!Pf2{i|=RF1zS45v5bUK!2ZQxbH zn&}q%IW=;F6d@<>JI@C83w3;DpKco&&RVrdsGBF0aE0B`Hn$cP11oErq5a^uCi1!i zgL%gTS~}Wkzt>ZvW?XFGOv4x6F3Kx24K$X==PRH10=Wn^umDJvp(%hG0KE%}!_Xpt zJ1J904hb%HpKq31KIs?W(|0(~SubfpY)ijX zXR6Zd~3~E1b4-Ru+zzz zEyd(a9mV(}XMpZ_Y9xC*Improiyqe{!WdOOt5Rvkz0dyqq=K9~ z-j>#~(wz3Z9-|;BnRG|8!n~^Q09i*iU;m)4!OmV?k(KT-*n;%m3Ti4@AlQKIIiJn# z9eF6mPuevo7?=g6B5a4zvD*$2`-!! zsXRgA4hcb0p1Cm0gjAgJp-(0J-c_0POqH=w>v3a*G4|D@czoQH4 zk!)6c7TwMsPXArJ!obWg0MZ_PWh!*UFlkB>pl-5SMglt?^L7KyHD~vuvn3F=5 zDLY7-%Vb5ngq6y1A;e0kN+qrM=w~Zw8Z{I$E=Yo-s*_wB(o{scX7lG?ir)Zvh{~J} z!{b<04#6w?ElO0}D`YMmyvUn!btlLw=-V&c9yT)_LsTWwSf?&4Zz^t?rXFHP1eB^R zb8_pu^_4lOHZ4G6K?(!uxL%Y)LC%2HH z1Pv&S*#bz2hxNdciUi`x5{N;`VqOuoWO@een(`KR zc8%xvkdQ7eg-7Wv^8B0UUjn19vDZj8j2S$k(;?KM*L|yl^@G^qoX9Se=Y!%^NoF$I z%x0tcH%V>s%Ul-=rQzZM$#uz6$yCWn62nCIM1Lg@rSsx#$uctC?1%Ycb+R7b(?Q8R z`8T)uxFmF0Jx0$1*V|-Hp}%XOFq{EV-ohikKooH=git4Uwxt)^McnW@HUFY8v%r;N3{DNRQ4%=?j!nVFVp z*U28X=ci5mebigxedGN(4;ns7K2ko~udnVP&z+eaM*m}RZ!W111J?C z(;&Y7cmuf;$rJhZcn8|Ol#ni{I?_#PkLQf7nG4_2s|%?^{##K02NiD`QMUZ2ams);(DoqG&x5@&Nm-rlA5uIo$V$VWdDro+Yp&4kNXL$z zDWa=G@CmT8!Goc;Ar9|Pac_q*VIxwiUPEIh;hAX1$Co(wJvzU78@<8%@al&!jAc72 zBS(x3DYnY}+(gNS5+=%%RE(}69l(TWmr0pNRxsm%0o7{$E1Ow;C@okH;(b7OC#(-J z7wh`l-FjkJcj`dt{J*>5GcRxCGZj6KIj^d+oCX%9>X9jKsImn@sb>B|%Zh&5!0Iqr zIevkWYx?Hnh0<4IbCnX?tw?hSr`S?uIMYj0-J!>nN7WY9laj^G0T&-}r?u42a3R^( zWJ&w!+JTzHDf5znx1_i8Y{^(W|B|IO$#;MLJ9yVLJKqrfrCf=tqQyTBbV_OxLH2hH z9Tw=|bXJnD=RQaj{l^B`{{*xup9mWGKKEnJam(l*=g6P7wr}QPPM&v3V=P=8$UXuL z-*d?;z;-4;e|zKiIW|5T$%nTT=0JyZZ|QZjQg&MohvnREtheTE$z4jUH6CI5jBlat z7I4=!cRQtMeIZ)g5o39MDCBebyIsGj&#m-Bj%f3?Z|v>ss%XoYj~)Z|$f47fa>tLD zY|sFz6TB-B%hpQuv9)P&26wOwh}kOvE$l#_b1TU`$~!9mMA$FFH%@6fFppF2pcuq* z{To88pxK|jPE4=u!I}H*bl|w@K+svbgAW`$&(S>R8$FXX5U6Y#it6-1i49C)=qLos zc+e&#Y)ZymJ87st$3D^yDets&bisHM%w#ckiMN(g6GxG8=J#nT>?Z1%VgphTWu0NxJ?tw{ujHdbXEn{z1V+^oe1$n9=SOI8m6&R@bIOOn)Z*^e}X36Z+l`{O1XsL+4j z0Ug`2{#a#VcM>N`4*`mC8DbSA{yCxtGE^6|zaoIRcwpxsNqpSHiCz9Dgj)cJsecjH zBRVG`QBhax*H9Ltd~Wo2!csuW@l_9SQEXP3+j!Jba0LCuYN8KJbhFB}6_skRM5uT!?T zzUhVEZ<-O0$|+zt>Jl*SIQBhG;xCuFIxOm3RT0pwf6AA5+hp;`)emVBsCZ2Gq!xdi zlb(nRf74Tq=;lig;{S7ZgqXv92R&ujdjG;j_#E%jNhFSt4+D-MNI1YsBj^B4yI=Qk z3+7pCO=Pg6NgcsFF8UJ7wJz{(cYfkS_UdZ99+mQW3u|dgWa=((8t)%eW^&PH1Fs7b zAD|lFy_(x5{zd5XSpY8+0JD4J8SKOGJIl}GYH_}kszRz7$+-kcykxicU)O$@gZQuC zE(7o#C%|4Y-7;R@oWIF_`Mt~k`er(~*W}Zx7i^PA9vrks)ipJ29#Vra9?B^oQ|H}t zyzP$SGrizz1opSunXa*h?*=BzcENSWEBml#?_2b}o43AZ^|2H#3D7|Pvr*kq3u`nH zv>)>L&~NK`kKqYAj>;`H;rO`c&55Pvt$Tv{_6xW)5S~83@(NtOsS7G)kWsTEs4JvZ zg@M#-lWr9(wvo-9(L7>D7dZs^;r>p=vzW#%H4oEeekU+1vl7cz)>GEr*;=?Ati~SB zHE)1l%Ez`coGA+o`Sx!noV*!CV^-vegQv7+8(R%WU41tKEj@z?`2h_5G= zJ-4}2VR8yi)QiPUz8%loyVLHP_qgKAnS#4>DeFhV=JvJK)fj(Q|01)AFJ=2p4)-f9 z`mHM4m1WgJT+#kJ5}&Q@$t{JkZ8H`JFfeRg+LKN2acy1e^Ib*#$qIL@+O0xH zfrA~Vp{FjY8NHJlL0VK7s7D?x^%P_lv|2y`r5 zSs`Mr>vHt73Nmrx6#>VZr+U(mr?W`rKuGJRU#WC8sX>*C^Er$JOj z(#vPw6%g+#J@prfeLVbFGLKlR%0hPpd!|E22GD^CXf>(bHWW*b*h;6>M+;DIYDF8U zA}74eECy*!+NFSQ_d=+@N9`?KWR?vzMHNl6Ji{PiwE|hx=;^A|#qFwJ#LdP@#&P=vfMvOpgO}MeaRum8IQ{Bwprs<~M-Cfq}M^ zcDL)G_z0H(od;_{_u3)sBz^+bK~Kz-vGg}Rd#0w!C>j&|-g8&748h+}s7KcuN6x;b&$PN)prhn;7zb)_W0%+glCStBu6L4H zAnf3ozCW7?rbTW81BeB5_%2~w<{r086=uVMlN!^yCRr3;t32uA*f?nbT0N23Q0`T> zIYGlVNFvNUt!uPCB`TP+tg8yk_=i?dc{_yQCukM@+UNrId3W6~Vh0HxM~r+Xa7y0N zt9LX^;v65qbl-xf2`kmErMKl?jA-yi5T({0cYaHJG)iR`$7dN@QoJzQ0+q)JXOAQp zj5nKHF(&0m4LTy&H?=$P~t8B_KV@FBdwFauXn#Qf@J^ZNAOOtxv zU>Y!P+h6+-Z{Alw)5z&;x4M3*NZI}_24nTinF%aY`vR$X{=ORI-#55O#b!Dyg$_*+ zG9|4euB`OzjbyuL+aAZ}3~OsD$H%h-V;`OH8nC@(r>y2%)}Tjr#KvTXq2g5Fhy{sM1EY4_#woZH_#v;nEn12lr8 z@kQ~9%MCAH<}X)TqI}kB`EN{}8+?M**MHu3s#{GLPJ?TNd3&H!D{wVP%|cU5aA^z- zb8t}GGB=QUw-9!4RXAnYIS+ANhD9E&AH#ORwY9Xl+M5KXZ}y@2E4ohOv4JV%q#ifB zaGIN5gb5fvBbS-JwO8{{^UpP^FiA(72;=I8<`BV}>&J?~@l-N81*i4+_~b_ao3NN8 z`Q%!VI|n+rJj;6X?$}Kz#HaR~@s3e15Iube7FKsaYns}vfuy&|^^+NvAB)mD=BUaR zO9&)Jcacyj!ng?muKnk15L{NZw^jNLy=>GgCsdp zW_9~dTmFaZ2sXph7DeDlZ;)atKD?)pl7>c^@urLAI+K zP@YUgAbNv?0RnQB2b%wa)_1}&k(XbbTyiBUlCotd`&amht2Xd@Z~fJ5y2e=zlC(Vj zdQ?`%#YXba5s6yv=uA;%jaOGJ{V_}jFo5=m^B?54HJ=%$o$yjRGUa>$38v+TzW9FC7GMv|(OziHqzWP`uIs1sL? zQ@z8MjgH>ibZp;hIiPoG4F|FvypQs|YC~Ep)Q4u{toz@eypCE*XO&y<@nKER*!k1@ z=#Ibtg3J1g0=j*Vj;6^ZVGSu{Bwm77?Ew?6AL)};B zJvs5IA-NR4DUe`wSDrGvTWwI%;T#T%^w4iA4T5X1>yDN+T6TvT=C~m~p5A?A{9RPd zIy{lBd7nm^?;PKskAz;dYS6~i8MUV6Dig=;DRp+M%(Xw#elc?!z|4|ArWONJW7u#! zp;)P#FLblTQx>v#REjUj7*6sX=^Sj)G56ekJ*0nTrM4v-3*ChDeCDJw_>)EXrHxu62!m0yudyu2kDQTsf5r;t8?W_s1WZWh8CO*Yn#+r|g}nqU``&J8W66=nJ>|YgmR*ua z6j^vLaGLEnbi=r5z8!>UU*11?4mzEYeiYL5c>mASLZvKK=@f0Lha*%9jzXTnB?cg> zidj9Y{7@u9dZ<2rwGM}qm=-T;N{F;1Yr?5Go>)9Ao&uLOUo?ykY+&ig z*D0XVgz^eRQvOZKV7(&tTT)Cg`7)RMo!%N+5U4;@@e-6q<_d85tEQ};irW$%-sG;ip@sL_p1+vbZCFEJ!?WeTGzEo zBY-=q%+M+x6%&2_S1^(Rkz01VR$5<3DtEHi*kQ8*yHFy%gtKq#y}H9q^&P!w-Hp5pja}jYO-e3V|qy`Q9|+xWg2(x@w37Tu<*ns z`lmzQi@;ww1oeU7X0gGysZG2y7a5LS1fFJf7tYUiFVYISm`PtGWqQJ?gouQ2mY!NP zdEzAphdzIN!vO8XK)yf4s)vUK-teY%epqQ~KCMUaW$)J7S{0Aua36MG1ghHS<)m@~ zpB96}%*mnSuXhr%lB#a3&1oRV0Z4zheP7wO{e;@AIE(FLI|_J_S)hq9*0zU(^&5OE|m-|KiG6{S%U`hqbhKw(N&*eS=`;{ z@lDf!=pL|Z>o8&Sf?geg$7%c_Ktf$X6WLjd1LPb~Z^~ueVLSaoWeuoe&qsr~p6t&s z$mvOo#~l~l^W1sBxd+9jpl3mxi-P70-wF5jq`xj*oRD~wa=4+#QkGwfGtE1r$C%FQ zx^$Hp-4}IF(QJXetmdKd`!(1Ql03oMflXdwS1TwXSZ zS=pZh_%RYHE~M%gz_8`nB#D-M^L^VIx-DTc8W&$80C4!vvh;2|nSZyev$jy5t+Y$? zg5?%UU2rJmf=qPz3+X;i|GE|@-5ygU@*XWAw{e58F4OE9oYt%npM>~vN;`0kqJGxo zuCiCzs_CrqU-MX7{tHf%D|O6AJW@zTdR>1{V@f@p>R8(D6yujxvg*=!7JPRtV=hxB zTecd%Wq+IB6;#IqUwDzt2K)Z#;_Kaf^9qdvV!Soh~B#68g-i$Y9HY&jIPAQC)@ojg{;QT~ zL)2v!0kK&Baiy~txy_%$k;8){bF?7YPYp}9=h7QnI={|7k#&vrlawejCIyXX5aLg*q;KaJgd2KOUF?S017an?4 zlrwm3xK(`67g$v$E*a&{GyVkBSdDXN1$*W2{A{=gr3goGU%!mJ9ppvjDL2!4C-}0SILxciG@;HGQU+`z3vR)Uw44^g*^iBzzm1Gx zha0|pQ!I2rqEa5jb^*J$XPQ8bnYZwsbluXEE@DMhP=hgYLl1>NI!i3GIsY8wl2J>O z6oWC+4E{1c1xK`&Zy?p(g?HU<&c9_Pa?;hypco3OIW-M~G$?{wHlDOhUp8nqc^WL6 zEA!8g8QRVs7yFG2_v_u9uvnpT&+3duXPS{ai6a;()nO1m3Hh-)33a zWEdq=PB1$DieO;t!zUz}Tdt^SxwBqZ-mNl(J^gosjRI67mGsR;-)Kl#CKG0EJgWjk ztZJu5WzDz!Zi8Jph`~WjKYHcvD96iaD7WRz_Z$>heAP&6D4wEgn?=$k6GQ4pwJA@h{ISt8y<4`K@e-U z7*55!+3Xp49Xv%Yf%+#0!~L(tk;|QbHQ~7+Koi8~sfoK~fntBGeC!J3P6Yk+u#{CS z>gdX*VD&R%;H!UI+9IA~S+X*SGwsaRDT-_FDP}`yzuWGLCeL zR73zebz25JRrt`AhtXGIqh)7Qw5invo?ps0+q80z+8{3M0H&XObRsOAynN)k{+*9+ zvJBv0(niIyavo6%;zL%+++Ha?;yS0uyMEh;WKB;TiNB+51Zhf+%!!IiZc5n@-oQFJE~U`^?CrAC+6D#$D>C-L0nBZGIfws^Dz}NI zJZd3>iNaIRliOMDO!V;cKS~&U2t%LG!}D=Yde!!%(fGdFdOH7H1S+ve!I+C|EYth9 zO+?o$m?y&CIkw0fJNUEXCP-O$E(4Mzrs&UY^ZQ;(v*#fvAA4t`$ze6Wd^dpMDVvE3 z5hyLy<-)r8A>K8tNcYBTB%}-HRq)I==4fk zY@WyvLrpn4X@acBK(X>OVGQ1yTvgbPl2kbO`QyZkwpv7SeaxywSfOvpJ}vNm+pzZ! z+*#Tg<(CZQ#)|8(UVR(a${B?`T3aV-X8o2h;BPn=(9|N!@oBd?5c09L@HWR%P1Vh{ zWtZ-(?@hhGiy^uf({}%v-KzhG}%jX(ezDA)QZu<4CAFVD-P+jXuoPX=!-KTu0U^EIwA~n@go<;A^&pE)XLwU2qY{sjtX1mpj z=6ed7%BW|AMbD=4X=%mwYCFycnAVm?(Ho$okR#lP^Z#%Fga<10q|hfGqud|0_Fsh| zxz{6j2)~c#eDoQnE$A|swtt=Q6`k3R?#Zjsdk^>a?*So*Twd7=ecw2dDP=@b1uqa6Ld@G|4kXOkeQ^EiLudX-fAxcAcz@?Oo`dYdC)@*+%^ru-MZcvS{8wkUlV>H` z9pkUADR&(~IhO~<*t{S*M8fNGz=ahAhbZEr($ZJnWeTEozM60we!RRfS)c^7*4yPOzjFQP~LMuZs zU?fhRJ9YNd+0%zlA3k*m4Fuh_HMGa$^0<6ADG1*O=84d@Bgd5DrzQ}Toyy~yBz;y8R zkADLT!OE#$%TLSGsF%kXF!1;%9|9DNdixn@agQ%rC`Zexd>8fztay&Vi{4Vm(nb>49G15H)y9X{0l$l8v z8avA0UyJlkZNEqEl&A|QvJ{>4` z4)R<|n$0Y2HdY(S!uJzfO{koBKmG|!y$sv)w>WG$*fo&1oow9oEH)Y2H!uax9div_ zg>%Q`@dJ4?urI*t1Nj<{==Qk-sM8~PBqIhnLi;in->N1U#$vY9Rn<8rdhnmv+&(hv z=Y&b5ngyu?2iB405~^FVH~lbewK9YM-tBPk4$3YvPAlayxlLXU?Ues87gIlY>nn%B zxEYuTDh4vQlk#)2Y6%vLu>;BkG8Ri5h?xd~yd@HP0>XrWnAxyWj*s+jBG<)|agksg zHfug9Z@!-mL$kFL$;;ri1QJ3wcx@UPkIl!XVfDbDKu*JofPWgf2Iijx{@K_{au-OC z!)B0RR>YJ*La2@5wf!b)x~C7&j@KR-q^s2t`b`U9T)?wD%i_?_IBk^2 z>^1v2bk?>fN&WLol|5MfO6Bc+q+f8*9rF&9#ZDv0mmkwj!MZvSp|z=5z$0nL3ckM5Iv%wayNQge62M- z!Cv<MLAbzc=dtW{EVtrHZb^KJtnaY!txzXHc3v)iMP6(jtJ0NplhGyt%AU*tf zP~M-R)Y{4p@bqo@jhJkVa^L@xX6W-E4Wz#)n+Dw*Esp_*VszNhr$b6z6&VEwse-;p zZ>}=y&3X%Z>d?{?$%N5jvoN%_$Xj}exln#0e}a+OknX^(Cab9guXI|SECo%J3%DEn zje&+h*xwLn@Yhy^ib9#Fx2)7zMC5F-Y=41HI#{@6CH`FVw%%mw(DBQ~w~6al+b$lX zk6mi{?Z-G6oeW}Lc#HD5&a^x5hM-u}i3d23O`_~hD{r!+vT*--&6S!9&0SP?Yin&k z(b;J!ZlXPT&U6^~j44{a?%=B9;4w-aFOsX0#)B6geB9$nd8cw`PNmsh){;EobjwIN zXCDScrCPR%59J7=Q5`4xU%4t&!1@ z3mUtZ@t9#i$FOEAsyE1u@<9s;I^@IUm}~*7OdiVb0amclf%|ODOpsEWKKQ-HY4KRS zgML=>deBf1G=hT;+~bh~b$GzywRkAG6CAd>;fBvou=0$}W~U~-nZt5LNByc79BW3@VXE4_x+4}fUNJX8e!6m`tZ2Hk=NsbEmD*4uiI+v+@r9&gwFFXz+(tw`HQKc<`52wmsM^ux{|D zR?vRuLi^?F4X3w3`_)1FwFry=1EIkmj7cazWJONNEqN)?DfvYAN$!-n(|}$p?H;sW zZL%(zG8LOkm@>}9=>@Y;P?z12T(a>Nl?xa&MUP``9+&tF@#)fFGvB0Az>=wJ3t48-Qx|~cRH@B-Zo6hV_ zYdIjIhqjl3ErLAgavv=C-qXVL4>isH(YeYiXm%vkhQJQdajw2~GTZ4MiY zCngu~PCxAN7f`k$CMRfZu=X-tJnC{foJ8MoPpE_Lt?ln|-9Vw=b$Rd(V`q6Awfl~o z4Mq!qLV*%Irnzvs^~IxYR#O+{tEp?(*PsD?ejtTN5i*$qeRfr9u5AY@kc<^C4CFT! zQavee9=McpCHZ24ewAIz=$J~*?qr?9nNawc>li8qg&lVsxcg`ZHmur@6AqH0rw z`iExu;IEQ&W$BffBDz1X;SaR11ry|HBjpBVwyo4!M!A@JGmIE`X)t`O_tu16G+3yt zw=~;3dGsLJ9Bf;Fsly-_e5e8AL;rfCx>PsP-pDpL;xH5!Je2GGuB+F4wNjX?*A zNm4+P3D>mix{A82ZusB2!Q1}D|9H20@O}Hg?|(FBs=E8u4JX`l zPSrW5J%~5T3~RibTPjPd%S>y%%kg@#($_|rSqJB2<8VzF;644957l5a&}|7Tpo)WAeI1ln^f4kiIMBB%K?99Qi|4`HZxT~z%*pJS zbN~(M>lLGUqFMwzBT~FqU?^!G)T^d0@{{Crql7Qv>l0&n0&*lohNnb1$yb`EIwZb! zh)6S0Z-f#V^cmUN>O~Wgy(KS)yI6+x64C<6pmrlii=kacS4$(o07m2x?&M5ccrT}v z^wSL()WGCwX&_kO&``(*Kv>mDInyEszg($5c%b{Fco`q%$9E7JtvbjD1i!C4l_#RT z#Ygz$@ZvI+^mCPzd&?qZ_TTjJ((mX)zJ1}T5dMT(Qr5h%s)cg3nY^u}wNvo5)5sTd zi0Bn`hgsa)+w1KkM@NJGV>EJMY{<3y4C<5+vlt#yz4s4gh02lD>G=;RCAX^0D~EQ$ zM%6+|$1+MusggrVIr>}GVrix_rt0g7cE#v7ey@afi$SVU>EQ9q?o6J(rov%vqK%z3 z6{aTq-G&thmyu82vqjR}^~IWRP95moIZ4OkecKZ${KfuLCFjVGKivIUkVbBGnE3qZ z)Av!5S$aj^++?(uk}I}O?LWGI&s!%B)fR52%?+)!9Tpr~8}$XHeS2%@RDaiCB8K<$ zbcaXDlVfFNB~{yArghR{sDu^z>8RJ&Pfy3iE;ww|tyGa8d zgj0GsM9G1aiJO&Gw?$IVLz?^gxow(<&u7*4S8oF%;^~!@wzT*hoU4~}g4{G4mUDs* zWpjcC#K?AGmq)aVE`GE78OK@=4!N1&tK|_FWp_Qi5)d|Deb7evodFM0_gqj21>p66 z%nJoWp%{^5qV6CCB=x1vIEAPsxRHH{41Wjiwt8CagbqHN`QZ&@yfVNQ!tKTucN1CB zkvKrZi+@(H)@->_yG(QJ%B`wvxmLbX1AHBpO7E%{46Ogd-u)BXMKtr<;eyO^eINgI ztAUNsAw~`-@cTvN{WTeaULQ3bdQhU7TmK+0y|~WsJr9#mN$8~YaDI;-y^|CB{y)ic zeW0(^@NYp1(|r|a&<6AG*;MNcI%&}dnZM|fd20)}aeqJ%5%s%}_nAE3MVbl1tbetqv3EQ5l8K=s2-wD8AiK@d5ev z4pLW~k1kYq0tf5LqU}(|rvF>%u_u1Dexrm**#3e~E*?AaPfA+B>3=smHbDuhU4zvN z=hfC$Zml2vlGaU}%G{`LZ}&9SAT#YN5T$HDS_7zIVK*;^aUmka;{>c#R|S+hSOoX; zdf)rXrTv?}fWIW*+Vfw2L-DFVy!W`YT3WdIk*AHcf0+^%srtpph&VZ&sD?`_zv|Q* zN57CWgDaoNMw>l09c`vNvha-jx=8EIswH^kCT7ZR`_X59Iwe^Ec)1RlSk;J`ruET%3}Yt5#h}C|NVtXhK|w zW%l0`^aq11{XE>Z2pSi`$~FGHng(N?on9_2e?eNgR$7V|TgHnHlJI#%=Z^3e(MiGE zuaqj0oyK@csaw_3-rF0bA(>gE6hon1X+r18+zT7^fruDO5CiUhYcEyyPSLr7Go`+~ z#%7zblP+De^`&hu;f{8X(L@;irbs)rclo{-4zHYYD_PcJcBPgs}Imkrg`mhJ2fpc=U~T5$5{>i{yk5%DAMY80Gfz;qkTqiS;L`OQOH0=fOFwHm z=A;q=GT?}iS9A!}ny!}kFoA}w`l0L|!$NrVqMj{X4@$_YS9k%rTQ9*~s6(3)Dpnh$ zuj zBJM3})|8f4HfeTD%8C2Ya3q}|f=rZ^^Y$wpRpbl#L-2YR4e#NV1H$M|hL)aIMK15# z|ArseM`Q@_OL^F>s71qiulPl!bu8holk?eWS{7G*`%&l!ts&6e5*7I&z? z3yE+PIUdldX995$CF5Oy576)#HSe_WCbFoeY4z&9#(oE_`w)8IRlRuh;9hc9j~wUI zE;x;Z^n{vQ#msANt2fs9+pIWXje?W129wS~G7{y*IK4k|{sa&25#XjgIGb;Je6KHY7k4&o+qld)my>4T z(gxw+0!WhZXYX8w#Gea3c{obD102l{qJx_UxA;vWA*EE(UT>2Pea6}oX9q%t+{>oYsm zJ8DKMknvH}tdwq1OLIDXn%O&S^1G=Y}b@6%Y}?U_q@=vAThkEjay8jrmh z=bQMELgGq~daGe$so7XZow@Rh3WN1z=6~t0oK?$N(9o;SUa+Nir3Uo^Bbi`Ep?^vv z-LHP`+eJ{Z^iSEKBB!O3vy*1diHvl&;PE?<&I{TCu47Bzn{#sUy>lLvnzvTqE}n~c zDJGa=(quxl)0f_z7$1J~@aHGa436{-M1#0=Ky4B%NMo)sBvgeOIjEs4le17WU!$9^ zIiH!eMDw@?Nl{m3gXrOvehX!HG_e~;u`IX@jgYJ(|^TU05 z^DBk~AB%Jv;C|RE+0+}rtLuqHUTu zc-nxhk%4;*&l#T4tS~J8>koI|rP0BIKUBcq>EYblzLqw|C?D|qUof~q8()n}FWM#5 zGt%RP^e|M+hifcUGeF$cDS86+nh+J{diS7BA_QlC3Lhn|{BK@=>4WodPMF%ruU$vH z4Aye%DQRwG$;m$unSFwO{<7z#M;sJ)1iW$TC^&wUfDhZ>Up1f5J*t6t=JacNKG6~` z2;rHZx-=HfYLm-nbT$xI`lKFcRdeUSg|oz2a9TL>vgQeH8JACGu2)6-cri_HsWw>P z1sKq`HqR3`aQH8|H{fjgKh$r4L%WDbBr2RG{Z^p^iR%t^n~?74Nu;RyiEhgPjHa>+E&dDFvPU-{8fuG(E8DJ>Ay}_L zoiEl;*CBux)#V0Sx7+}4h3!8q)UU`l*St#Cg_Nxz8G z(if_7CQ&;;!lNpQ=)C7vi6&RR)Vs5Q_Ba^3o!Hh@HD;wc_DnerkZ=v0i(YHs%hFtU zzqE$d!Qo5+8bx~O!JDNm4tgE0{FMLV;)5EQspJASZ*cGanmBar65;`B zh*$C{nexdv$0g}7=SQY*lnrn`2IobChu{Jl{y9FM+R)0y?6fPwWW&-6)h-XmdFWU2 z3+C77cUGCUlry;3i|7Yd3b7OgvtjAsG)R$M19CYKIV|076q?9YB_)qNzP)7JO6zXc z=GAF9sn?0ODn=U8lyob6RPXB%LgcPtbE%K!voTI3rbaW8 zHiwgQk)2!%aC7)B$)dNnh`fh}E7Stonxf>PiH=tSG}546H?g>^vVhXf(>$-y^=4*$ zsP6^v@5t|UCv6eR7U}SJa5$%u?l&u?8S0zu^V?QY?zYa378~wuK<>IUSAEDPyApoi z?0{bk1OYgaLYAef9Uybl<#YSIlponS0c6gojzpDkt{RksPHKr&FWsODg@aKsj`zJC z|Lm{C=jy{@WjyMzOa~*smD!YO*UNX=s|vSm*tTglFxGdpPG=)L`PrSFd!-qZ#zcoYraEeQ?+p}yOVJH+@g!} zx%@7lXJYf==l8BFEh<}5{gSDoy~g2Sm^?;p^D`j^X@@wA%n#L=m>7@rcki2cbLUhr z7>R{LcomTD2b7h!A_XCMm44NpEtv-9e5pcLT2h+~Bg*(EXX;%_4 zeKN_W$e}@hcz~XaZSU_M>lqL3_3rW4J{rUjAPnWfS1CWw$TWDmG^mQ3b%7t1snXK>CZA`4fy|I4#_ z#$`v6WapI*2Vfnvhh;rVj`3QZ1gqxM#=x+32MO^MTnOg_;kSI6PQ#g?i);hZ_q-;A zSVD+|Y0;)%55w@NozB+@-+w#)AgaugGnOX?)T zXcwU6=d!*q9}*(71zr>gaj*Z4;Cr~x1xAJlfIQ|9#mfnh-bihzUU1={`(;2SisM_t z^@1}mq>99OAxg)^V89p0Idx<>zB9f%J~=zwADxWuz*)et1mGJ1)yOySZIoc*TYRr5 zfmi`3-XgznT@5^*Y1F%|Y^M<*MQJ2V(oA3`y8*l&P(y6o6{WoL4pL#_Y%&cR`HIbi zWKpl@O12TUsCr*qe#8=q3yCxU@R!|0>MgY4jd$~Wl5Pdk!@MS_+LPv^(P^3>@*X0P z$%4o`N$^pnO;y&)S)BB>pmYn&i|a#CKHiNw`*SzZ1@jP}7=gq9Y8_b56;n=i5ba`- zULcUkM93eaeWEg;hVQGl^TWdg-v@eoh$!NkUKLxyxez&6&Z~?XJRIQ>n4pt#0_h;2 zUPeY`5rw9|D+N*!TK8l&kK|7HUS_@?JPu40V$B19+QQc%L&vQ~8>iiB;YvxjiU}y) zVK$mV^ZUyQIST3UR57tPAW$N)GwMofZT+#bqotZLfgs`iux%17~6_m#^<$v)|R zeQc5%+i{@lu;ZX(PyN6a{37>fE=moA2K-0J5YM;1+Q8lAAO`mA89o?396n$gtH8&$ zb+3JjkbYYJ)T<>_d2N}y+P@`OHDK<>FV5^S51B_C_`!~M_Q1!8uNw{*I_#|1PE}hg z8=4yAh;KW*D(E@8o%4{=L{!Be;E%jcz)#dZxt2?+RpV516`-`R2yEVr81EOxYqahG z@LNEoKsRzeW3d>&hm6`(ToJcvji_lO_*$`O9pUmgSuf>vyZDVkXAc*f&Gn&IKdyTa zemJa8aXs4!$f!^AhxbpjSg9Q#ed%&}o7>S)FWL!@vx99RO{xx+!xap1v;g3GG$G9|9wzGV0_qIR0tWO@I42`W7MVPk`zYdjcV3Y15eq zFJm3)ak<-W(A!VQ*Q0JfZa28H-LAE0E+K>L$B(a7*J;(bvfZwo?zT7EZA$D#-O7vE zh15G*Ofk3VB~fD{YJ?JD9p&^o+;+;zZwxql5gGbeL)xgj zYOu8`zLl(PblOey$LeIjKRQ4}A63_KHOLx4yD!TkedY16F5K&6I;;f47z9_4Q?ii= zh>>6nBq+3(d=02p(c-tsrKpsazDegka>ve4&j`J6yWZR``t3AqRmW3(!EW+!y6y=H zFRL!4b(4!VFjl3Hs-s-jFmX}Meart<`L#L?VqA#wb6mHngt&;94&l*(A^$LW>FreyC+KEr zE7}t{JKA(uhG_S9gdCJR81jZm0DlGgsfRn3uX>a$DT@zQ(e{wrrbNu_a1b@Ah9Ns; z+dk$VBax?}Mz34mtAA_bYxHXyRtzsA3kt2xTabHz!GC7F`ut6;rImE`R@2MgCAfIW<3@lW65Ub!(If>`e>Nu)Mm2nu`L2 znloy^9ZM5jPy|6zd^WXiWGh}g_FBt1^1_jU!xQxQfz*Z12v~Oefk&t->t-(BbQazS z4F}J@ec{MU5+M=AFD|2X&ud^cH0d8O+y@W*P==1uz0jC>RL^9?Vf}tVj;up!PZ^5K z=`Es~6O}1jdDqM2=B<{-Cc3rNR#Qn-Me6%&saSimv&$TI=7F#sP!UJi5p>`@ry8$~ z80(3$=9aRW+HlH9Il?ZR(rIaA`QM~jj&B31kD!FSx%XrOu*mR_B89o+o0Oqo7;Bi1Gt zXHsGU_P%0lR+O__`OaGhH`+-CLSFHS;ygH0v~0O<2>X*`fJF z^E=I7HInv0?Go)OZHd;e?bZ%zcW4i2FK9p1eyY7Z3!9ZY>(*Hh&w6s!>RE-eDrPxn zfmz{M!?X6yIyCFuS-+X})vUkHg1IwuiQKul&*ZMl-Im*&+n(#pWpabLvD{SdQ0|W0 z1Gy)1&*XlX`)TeMxnJl0E%$q!LN`lygO1Qqy1R69bPww0Exp;G)pw?%OpHtqi@q2w z1T`Yy1fDP-GoG#cZ85xYC0xB3{=(~biB17`+W~8%T~6fGA=}YuVd%}$ZIVJduts{^ zYh!Gj1^0H1FbT3R;17#*@?h_|{m&Z$E zg^2=OTBy?MBLS~HLL0lIEfeIi$s_v$w4X5zzHDf6**jWEhcg&<&|SvPo;q?vZAE?? z&4|4%Y0@wH10o&njVF>KuG2$C4#5tE&j)GgMOgIpYJ=q2tPx5^Ts6`E@| zjjQCF#@a98#Hrg55~OF}p7+gP-@F7@!_~_^eX!oJa-Ou7hL6C1>EWW^Nz%=fHOY(0zOVjhgdzl>F|*g zl%P7ZXM^-}i7F^qmtS-kQndC?L&r)_Cll^G-+p$3L3`7!2AyzAe?G)Lvo{<1V*?8` zi!@sOtp+VpRMq;DYPq&&17BT0NJ}3yXj|VD^~XFr*<&0IZ}{^&@ZL+Cc2sxM)}Y02 z_N4-mU^s{m^bGF0NPNETE$Qc_#SBwNSvUvhVeyE;%>afWZdHFJ*GLTyfh76Xp|-_g zns))WleleN!Sfw8qTr-5Lo@lnh!7)>?`~R?^|0M;Vs&Fhqm^Q!wO@|_EP9{xhy0nYOK z@g8uXhX|Mx#mNov;#isJ7@arvaC!}%Ov`t!V?x01i}zpXf2;p&_wntQd*8-+pdt7M zP%o77o54D9D{m5ve4DR8EEX&9Hn9WQ-)7MovQozfgjkUFo_GJ=e%S`Ex5Mil@CLSz ziF)IBQx(IQXpfh5F|N8rRrgmuU~jZ-v)4IVoJ}s1tIOTv9CQviMx6ulgKj2*SA16j z@s@*ltBGUHw2O1ezK5IP>~2D*Pn=s&Fi!J`DlE}%Hgz&o-SFDu8L^ETE@T%%(HUaKQ~+c0#hgqJej+g;by`kXo+M zF1y7b{R|e~VX(pr8Ka(MS*EkTWlQab*`Tc)P^Ky+Xet6!M_p%~wI1h3`H2w%KBMkt zVxAbqiHx8Ocep?q^0ifbA-{1gAt9p&nfrmHBS{Lqz!#x?f*^M9=-b`5XZG%QlwkYo zNHg?C2crEr@P&OLimTx&xpG{3LTwUEK?h}x#W-X`BI9L0ID8UW?FU0dAS8%BLI8eF zq&S~fPzJ4~$PRPb8Lxwu7f}k|dO$OtF-G;JzgzqFhvA&_|2p%}9T&xj5E8<;udYe# zByDcS%g|m&M{RqRxq5cP`leS}wzTePIn;c_GT1rXK4R~7%PEnSbzW~P?aEdp-~0Hl zrJo+VbGd{sm2~Up1d$gR<Nhdjb?PzVR;)Oi2CfvMr6!zX(83=|H&(qGV5+*8)Q z1$V{cY@8gL0O1JD1Hg*_ZK$;Fk1&zBMc+M<9_t$0b$;S>-xMB*M8zl=RoY|+D0xFWN-`sb6^8LxTaBUB6_Ju87 za$aj+Bp&n!a959~f1J?jcMlE@rbqEae`sioc)j`qN&D#Y)m0soN8Kr!gBFSr1Wq7D zIX#o^8J$WVB2V{Jl&xG|@N5I^wtG9hopRVZ@5cEk@;u3pPc^(4qIoxP^Dcb8u;#hD z$l|g{vWeypk{E(@uy!laDRjhblr=GKJwZb9``^MlFRefEXo}YH)CBKJYi;i``ZWfh zTpF}c^OWXF^R)*WI=!S{|KyWfqvaFDba&HG(`a3nsmq)+0XQm5zScw%SVyPJaTs%NA-rCg~_!ldVL zFJdS!@AVLZE$9?joIUyp3w7j2qrA{ZCxv)V5Sgza%0_U`UgIqxpL&27R@482FTz={ z{shGFP}-MDhPZ&=s*#SV8jOO_PTN?<#(EhG+sU-!x|1Ki_wBKZfnYG|_Zz~3KY%=# zfSIT4oxFu~x&TtwOYd7GDW02yBR-nslCC(lzR~M)(tHJ9yoS(z5A|}@@~70F@E7>k zskNMqx7u-sg|)O2TBPm%OW)Y!vbE5I;3S|X*n}rK?S3^?fd(70T$QYvH}P#Xgmg*W zCRqGdN=`0RTKXrQdy(xQ=^mvg2KOF^3gl<(Lw?3UOLd}*td-tF{(#T#pka$Tk?aff zkcZOsk4x&URVB0*z_ULre8|w&OjNcsmwPGB?c&^IU41azO8e~&(aFjgUv`VTwVh~; zoG_gu|M=12Z>HKOiYDor!Gi6LJ$O^MGnppFhI{vQr5I!crrk*{!Qi22us2S4V~qo= zd^j)H9wN2+^T)@&8KixTY3OCcy5`3E0&-L1@QG$R$S|Utw3r&uCNg;c@tO-9uDe$w zJq+)@!?05Gf1h<|U7AkLYX9+hPrtMQVLD&45fpMIl>A1OuX%!9mQQ3ZAzQ~~W7=3N zzUwu6U44sXjfr+3Vmb*&xGlj_$3mYCTqJf^jTS|yHh(i{<8bZ3sc+$&k$3Sx!XJqd z0vF(Xl+Y~K=C>jj+eT~e`zT}6GYn!i)Y0r|@9fI$>JHKBcT9V~d*gE1x$IpSuAdv( zgSObIKB5QShP)tN6&9jFF^=;p5wT>9h)Shub34zQX|0^E*W(W*{Kt?@4*;bFd<$)J z@ov)V^=vd~)k3FW5j$~a{5X4#JgE(L#l8?u;vu9hl6Er>udGpP&8_9_ z6#~AouKN#kL@TY)`vXBa4g3bU5!PmA?*E95I)YZe!RZoNlJ$73F8W^SA&I$i=lZ$4 zWzsAnD#+VSpp!Zi7o%Z%jN5mT&<4n4S5G2LH}BTg7{Wi6CZ-5@w_KW7Et& znPyHaC;R$r(+SCi)=kZBO$;fpNOVw}hy_zZ7Z*n+N>JMtGNx>lD;jsl$U056TRaX3 ztvAxgg~{&T1lsOowW8&FsY||nJqJl`hsM#{l1&pvfDY_1~&*OFMRDH2RGi zt^4>P?`iVHN3qvW({G<2{1Y?~@Y#HLlhmMXF*xOn5N#jP<=8yFfYeGKWK(-T{e#<3 zZ`icx4q7|JcSk}ov=Mjl2}*lpK!|AdJJa2}_FB6tO6-jl99_-@O?ydQWtVyT_U_)X zVeRv`5YnbUmO@NBoKJ3CZ{50u)_R;>G!uz1L2Wo|qIe_O;>aO)kPY&)1AM^m58=V6 zZ#YT-t*Va=+x^rw&cr#9E8E7`H4)ltx#T4^Omn57hTpb<&t?N z(tU*1Z;rP{oI`w%5aJ^~ArkD>`i|qg5BPk4O5 zM~hZirDa4`o6%VeS9Gd2uU7pj|1N);TBj|+oxvCrCAE9;kjo+3NiAn-0WG99v~TAD zl2@JR+VT|5Y5PE5xkg)VHg47OfY%};PO$i_l;~1w)&0Gr-SW_^nHco+yR`I`>s3at zoV}o>Y7E*sP*YAwm(@>b1wqi-J2^ACtw_`&HbRI346mgV0WFsOy*v5eSJR+4g8c%) zqd7;g`*N0G1oj$si(&!xB!b%%Z(t88{((J!9l)+r+>YIbj&~?FV|l1OrLbbZLdV+> z+=t*s1P>rED2B1S(D4?zU%7quTGR*XkIMNBo0EgQWAxqQii_B-IoD%Ppne`xv|*3s z;MmPMH(`%q?_vuTKf9Vyyort`Y++6uyFF(P_CU@PsLv5>LCzxVR>f)bod?bPJeHet zH#T4KXDoNd4D2SwcbHc34uW5!Ic~+O6w9#L==cl80Zg4;7eJq%#ug~*v6~fV5QqrW z*bRyyOoP@sA3Kjdg1vL~sDj5H#m-^#b4IYaic<()%Z~Mc;w)-^A9}w5y?+vocLsYz z!C*hhDa`h9y~2z7*@kfO5Y~Xbj@^saxm*F!e2-vSEQ-cohu{7h5pnGlb)((A*azx~#$~(Y)6w4r2L=d$9^dHCjK5 zZB^WetyGj@i)TEDkvYe)>vL9Pznn1>;j{rWpuXoT-ox(5*@x!+9irLq5uIe3@MsMA z_z&!boM#Z8e}-xPCy;5Sup>JC8z4H}fxUb6ZwS6d@b_ypy7?O1^5eNoCz(e74p|!g zNGI?=fPkO{UCaM>KyiNyDV*9N4)s|f=r(?h|d2GGHqlqV}xQUb|>OJ z4AJ~%h3V=Q?9$aMh`#rucF!m-Ui~kH8=E7yQT!8|jn>bgvDcyFyJ(F*G{0s91&V(n zKKTv8-x<{QOYBk9_HG3CC`#q;&^f|u&J`p-x1;$W*^wc~g^r7_exR6z;10w)ixK=C z;k^rcB0{`81(K0FvvM#zk_LvlR7l3R^e>!ycQFhpxX7$ztR5ct{===6r=@^z_w#Wa(Lj z<|5Cx5#i}Fnj4b4oS&lkRHL{e*4`=00me=ZQOkANf}vH5i56 zhbgLyn`!K>MC6jf!7@@Lb(FfCVm&&eKf1n09Yb1)rNg#8SA`l^5}M(_-R zUm;j>^)J{`bXj9e5v)e=($!z1k*3f{ zQ)r|qG}06rX$p-rg;tW2{e@oa{{T=+0|XQR000O8G%v_HX%^Im1Y7_B9BKdn5&!@I zQEXvzb7^ZrZ){{=R6;IqbY`Tx2bkQ{xj(FR*%1PUmH}G=X)(rhS1@2>1I9Ks#=YR~ zt=`^ecgjv5X;eniXnHT(d$HHOcVg2+O+rsk$R#%k(e9OV`OYlPz5MTezVG=z&o|nm z^PZwp-|~BpziUnmRcZa>R%ve@}aVlg5qyytB-)P0T33Jfs9=HWFw$| z4Z_T_YOOvpd>?4LWEH6Y)9M0!9rV`?peP3hx7?b7>JkWFlK|;D2%2K6t8LVNe+-%r z#?t}H^Prq;)#G0VAqw>|eAGZ;bYD*4fA#+q1l^!>pj$xd z2YM2uf?m8qPljeft8UO!epF{F^bk~agU*F+hWs~Z6|?|K-JoIU4(Pob^i4maZ-(xI zzP>@<0%^e5exPstQU2{e>g$e)@l1wtrrZQl6* zp7NtQQ=xS^U*Di}p}Qx|y+Ny>{gYPSpkZjnq=6gsO+TV_6|?m~ z<&9MZjg94%dCO|6iZ_*%HdPfg+<5T6o?Ddn<4TJrn%P#;&{$qulQ(bnyt(rhZ`hnS zckaB|AS^Dc)fUy(G=qYf`Md4T8Vg6nDMaX{-PuGu$Q{8xKFw7e1U zV)p4q@N6Zhy9}xYImOT>P*w`s%9b|#j~@Sbe~ZAA|8FfV`q5~%fw~P7HEO{a@}PNO z6!SpIJZLcx< zkO$(%iIvQo7=HmMDFOM}IcDqDLi_)()_3;*MV*yk4Of6G4|*8%k{y2mDF0#Rx``f3 zphv*`wBV`(t275(|6eo;uNw0XGesUO+J_RlRPvJ&zq4qD{pSz zqP*pK1NYCkf7XoL8F$V=XW%m)n~^`GFJN~Pe;)z~z zzW@IFub`(vfWH3%gwL}(32f_+e&3(_{?+fD6O`@yN5Or4mi{O4uktTfe!1k!hrgWm z<-K3*`{L~9pry~>{QUIihd(!dZuljxdVlJFTeZC)(Etb36O#F&<^Mt^aivO+68?Gy$^bF z0SlH7@P-rm1M~^>DUgW|pq~SIY=(XXWU~h1AR1yI9%3OkWPk+73yF{ilAs{u12!}O zU4TMR3<^ULC=Nv-3XD7hC4ogvgFUfXYli`U907B-K&QbDo`g<8XP|GPe*-&fg%Z$lz(wbPeCwb;0%>o8{sg4>8z7r{ zz-KU6vwdLA%At>-kAd#|Gg!CDKTQ1l!&xl!zkfM^Wucs$oLhcmXR_%B#%_nk=I4wp zm^8Lv^4L$Nyr4+`vGG44N zq_J@#qqL!xaVkSXC4kYD3{z-r<05d?HIe-PAzD3Y(2=EJF48T)>g#d3t1e8NVK$M}qIXO%& z;61=hKo+379Kb_h!*e`2fC+NEle6mB4FUE8{FW1(4B2u3a{>;|iB5t3Iw`xa02;jn zZOPd*37NEc()W|`$y+CjQ|_3ua>}MD)+zLq^HZUzHB*0^YgXN{Y^LDB;9oI zrZ;c;5?Ot7{mtfEZn@>QTejY^~6>rX#9`;!Ylxvbf!iQRqQ-Sh9>boaKqyYKG5XYM^u-a}1uPW#KW@22mX z{u}fr^bC3)eI0ul>$~^Xdo}mQ?oHme07voP;eW)xC31<)#2dXM8u~KOaCJ*!{q-XRe$1?#!D(mz3`QVsfB-d`kAMtXI4Jb@$Bd#^&-t8Y|)HGk1U$IXyKw~7C*B%`CRpqyrrt8 z3zqgSm7gb{KmWpgFGOGX+lvJ+URYMKEVk_8OZZFXm%d)UZaKC5=<;ut|9!>ND_&T! zZpF40dlg=t{p)x`09Zh$zuv1E9S-(;2tF-z3;oisee%uxRk*{ICX@K8e6oopglD}9 zF3k{1_)5MWKKevIaxxr8lvn;%h{zA{B}k3aVX8sYRbR@xHPn7`KUE3~0Zzo|c#H{R zy@3mdKT^>rJ zq=rmms=mXRv?k~bEUFIox1R0lpqy!(@%lM0=9S!%p9sbMtr64_sZ6hxg0gDb`U+PS z!PL_=Y(2a{pTA)vS{<$&D#jD~c3o$)ga|Ar2&_a09UeMj^N}GNe6l!RUxsck)otCo zFWy#(n+?|eCY1D%0Vf`}95kP%?aHKS*LtNsLhKSd#ZLJ1%=s%Hqi6I7^M`S@Y#l@7 zXPt=hYxy0Ga_^Tc0T_gF5h+Cs|ou3$q>Qi%4&JLTs=1FNEhM|U6{*($QR_T znvXMS#4z{?u6%7pWYyThu@^Oh&CNQ{eYS1+^KfgF5iC*ox!*h{PrZho9cVq&)$M5N z#bfEvfFJGJn|%4?!$##-&nT-Jsn6!c(|Vov7`1!S9@ur^+wLX;kh~H1ZR>+wr<)L#dPabm2MWLFFY> z-5{?U+oO4WRrH{?ljx%|j}d8V?Uu-MX6i*GI{ zS)kmH$~lZiUOf8d`7^_>DNicf3%Bjqw*R91BCdW`{^{7)8mDBD>|%vue%;FrR;%4c z*?6nh?6V|2KT946vdSD*d4N@Jb}~+uV&OWcmNGkdhXZX;o&AjGFB783doW*+7eYkT z%Lj%~pWdhU;x>^K9U?C;6l9wqPlDB-jWuex+<5!x*5lae&f2Z3H!fK=pQtIXEGpY; zYN9QSo(E5O>@a=pEBDvfiD6M_CsIPJ-*W<{v~JE!aM~KS7TaCq<_n3DQysmXy`BB7 z9jUItlbvV1$6?;j2Lk9 z?owF=qtAkqlEip1pGOcqM5gCp^e}cNUQxbo&AwIj1Zk(uj0t9Vpd4|Y&5E>;Nb*7K zy|dcQK7wc5G|lh~gDw_cTQdhMDU2km36~LYCTbE*F^e0q#0ISwF`1Baeknin*{*Yo zQp9jaTSu%7s7h9zq)`x6PaiKtiEc@X0@0W(s{B5;;3vBM;gmlEdpQr|#W{iVuo7&p z<=94o|Ec?p*U>+!`uUMQlp}ct6C4(?1#6;f>_+37O^QY_R4x$>{^qnEb{+qm`4F>k z4l63jE={wFE9DB=Qg~MZFO(8JZ=e3{xp!f%*3H*iII|g5k3nNynvfv*c(3R3H0BlM zDc4IFW2Uuqv-Rbc$DOahv38DaBYN1w^spViU>c}gfzDmGdSOX%Fjz-TRn~^MCrKUq*Wn-tP|j(4@-C`Mjv(+m}b5OPItK!6fJequb2G&Rp;Dz-j;C zYy7vKAwR5cm8}}(1@WWjWf&~r)o*0%Hzi*^8X)*w-bJ}dtVu_cCPMEZbq#2(N=kM6 zVwfaxoChfCVMkZRa`f?i+qbRFzbyZxPwesZ;K2Awyf^r>%-Qad<5xdQ{>lY=!a_WP zc9Uh^V(it0jZdv2_HL?~M=B7^YTbGj)qJ3ma5RziCX^80IY~UdJ>8`gQ^x@`Qmy35{_b*#DvociCT2I7`ZTfC~ zX=6=;K@00@Efq!RGbd-ukH~kt_w&EJ-f(F5L0q{_p)}j}jl6Yc^ziXsqN}CN7;k|) znnL;_bj{w%J@}Hb=QS5cI}aThuG_M=x~!lE*G6?wV3wNF_GAyL{?Mawx!j_OXc3wN z2Ha>`l`hI$Ebk&4O99;x+ybODmRW^#;9!j{~fWvq2^t zM!VyKeWUmyMe0XRh2n^u89TpE(=0ZU^`r?Ybrd>w0^2)l@h^ZzuKV_H@?!b!cULM4 zl;^f=FQr|gNE3XVKiG=$0e2)2XfP5i z!_hQKt2mxvQNb=+f>t;Z7Nj5%OlQ0qY=nV@vGS zTDYuEU$PAqW}VYrW8q`Nu~3>YM>Av(_N#w;TQFvVDzj_pD42HrmgRpE>^|Pl*0MC|iY!zX5SlopA z-LZQa&Dkz78YZ{{+_9^u{7col{8j!WzNVS8S_r3+F*KseUZp}VQVlV2dmP_X&sZ!3 zU&8NPhbli;t>D&jyYV;D?r@L@C&XAgDswWTI^OH%(}WUH=|!Wr1y?JfcNon%En?hO zw;Bn9^v0f$#%UEqGeJ8j8)pzrQGF{3Gr=eu$NUku5F|Q0X@5KfUp?giP`()*dqOqD z#T+qweI4BxM5_3Lt*G*q%BL(@>nZ18we4q&=6t6R4G_mvtz1VxDz8w5NXgz&duq#x zgezEy)n=YUG*;~Ip^B%(_I=p^Z3J8)H z4cG!rJZ{V-NuIj4lyTDzBRaf$YmH@0a!luXsfHb?l z7{5)!6w-U>0;96lpuFN#?uKcR@#9P)D-oHHkqun-i2u*dD;McngK|7ZR$gbw#X zei)AMK#8@hyhbV!+Zid0l!%T%gReHAO>azX441&EHeLj%Q}lbKST7=lJz*)5Je@k5 zI2j$uyq>(k+uc^)3R`)b73C}3B|;rc@)n-Nc@^&jMRuW4XyBW8gJ9%Mpqibx<2=bb zov6`c37YVq_6YF+L0_f5u>ZOTvB_$?e5*rNv+Zoy6@zQ`GjucIAc2qU;`wPDcL3p9 z@{KzTJFI0b1(q_hg=!-8w4Jdt4wuDcc3H``X*@SoVEh0r(P1V`N9ZsWW5QGd~OYE(@P)ri?`f*3?{ zE5f#;OuG#CeDGIJE`{Za#qyoX3T5dN%FT*FS#d?llONOJ!oDL9{K9tBbs=&&dNFb# zFv<)vU6zC;lyQq5zv!RVlF+u61>oxs%Bbe)wO;`5Yr};venX8f&7T&6HabE{&d%-Xl15o zl!l|Z?vDjSJFZye&os2d>^5UXGLs9`DWAa3jTA3kT31qIBh-r#H959u>=&8{qP&?~ zQfEOdDD7qiC(a}jTnv+Mx;~U^@RHw0uc5q zW6-!pV`3PFra`Yxj;c{|>y+GvnT|qoKUL`*NR8flZ+&EIY-4m&`nmMe(QTqtvI`W_ zPrU0Nu{Wf&7!$mai}Clvm*dX`3Sz6{YvZp(wh4AHN;@pLL_2U$_-Q=NvpfyEhidx- zJR^hx(E!g2!$|V*QF-~WJj>VX>+^LZDt^!NF?eI2lr<|AIzr}Ee>f)(RQ2TuZr zipI=Nx64JiNWf{pBu;a+=;k&)v|pJqovNejT#a_aB(&5 zY9wr6TU=x3XX1j$nZ<@7M$=SCE4tm|32&}Wn`8k+Lv~vM05)>l= zBHoud*nTL|o*W3Z`D6aLFPy9jHO4EvR>|tM@vOk`Y`P>>nka=Gi8vF(x`*A7C?UDs zoCjAwkEowR)OoUD{D7wTMKA-x<=_l9lvOZRS*_f*mD~Cns?1dFaqr?waUsZgFy`<_ zu8**^SMr}mjZ(9y=WV>r?c`f{tJ}i6B20!$!`ylIJMt3r8~$7VU7SDU{<`;{!av3_RQ=O41>e;hkWRHNFaer33&x@$L8~jXrlfm^|yId79{YxJSw9dU!<1 z8&w`Wpya`?$Ap-l;3c=mgQn;V*@@T4ldI%O)$-(;iFO41`;V>$u7m)B${ZD`(`iToVgV0Oe|yA&byK`-o%21*=QPmOiS9 zMBM2S^!n0Vi`lK;kE)eJa^BczC&Gj{fe|O!RA!_RzF^!?; zg#99R=xm_pFmd+4iI3#_fKhny^~d&ASX*jwd8z6clLUr*M-sU%sI-F7Z@~WmJe63w z+n>R%4DHWT=B~|uO>gi>Rzm&gdPHp>d+^ViB!9eh#C>1{mA_COW3vrxPa^lLd`6w8 z8r!j5SGb=j+*7x5KB~O%@#ep|@NTZt(;DtUf|B3sl_KU=V~5bow?=|dSmHbQC|*U1 z$K2@5^O|P2iL>EAw{slM^HW#+PeC z_PXFcY}um~3T7iHq=hCqlGRa^%e5EjY3oM3qR96mDW5kao`*SWr5*7MrAD!E)Q9*z z9*=-{VZU2)`Ed2Vv0LOfHT`6JO9#${y#PtKhf|$VI!yZsdo#xY44K=^)pAWhI(eXI z;E=W|T!zhjag8!{ZD#k8I)bKo&PAKd`WCYpcG_JeiSpGQb*D_QyVri4#=XFE!`ir3 zu9X?2gGPTd%($Hl$`@{bb+rLz3^kMvv)BbcOVG)oq5iI9f*Ndyyjn2pso)N{B6!?( z8tWhONS%ayd(Usb{T0ew$sbx9b_iyGcv|=tp@lCNr&WlCKmhgdUXKS)r>C;vFqg#s z@#fWk5X#E$G{-th*LjIoCByE$*v`g-*IdNU$nL>|nA&uS(Oe9>qe&tV7vgDDKC8;K z_N4}~-b~8|kr3=|ivd-fDudVp;PKC5Vgw-YVfTsisQjQx#%eA+P^Wc~2HZp%oh`Nz zM0JRXI%D{bN>@EtxF)`)5WRk@Y7w^zKpFKb$b$w%3CqM-t!8fp(-1@|_|k2t;!)Lc z)m#(KY;jc9Sz$+W6J=ysp2h6Bl#MYtP*E4Nw$U&X2y=1F8*uY};*ivz$@C|Bf^UeLI)Tb>sSdIcDuT~6AS*93ny>xtVX$T@iHw?~=CCznlcx+LykLoWj%P*rE%5Zt1>Z~gckp0do zvNnj+P37zOnsQY6r>c}wZsW@Fzsvur8SLmvq_DWpq<<~HV0#^*Rub~#f7XlzRA;z0 z%8Ok0sdPbI%7QyXF)E4yao`cPTrozZH1c0m3^O&8?n?GzMfW!6>5H zQQQowyu$9lPv@q>Zr)3T0W1%r>J^B(Y3$j@0XUN9D4R4w1T+A&jLMekpa!wQgZ|!O z;@shpPvkpLdG6w?4>uZUQitnEFdPdQjwNWo+%C71#hV=_lCr}-okv@RUN2B>cUOo; zoQX!+FxJr~Nq*v8*{^AA2*=EXJg7>fyAwUw#s0zvl?CgI3JA4h?BTInH3L$cFO8>n z?g(witd=ZT+({KSSJ%PY_g1|07^*z~>-E1M912I<@tvyG+-7bcej)d$=<{}=56Eea z&QxpDVq5bM%adgE(xnSToJ(>k04`meg9du{S-ElSUX3WaSs%gqgIov;xZQ$}h^9T= zF_eiJf;FOEY@&^%o@{pLEpU`_c$+br4j|5>h!?Chr!1bjlzW-mhQAr(!EOeC4+NI_ z9aSpT>g&SPwMe%JRd~ zk4Ss`5fQlWcjX@7`}xCoT(zFA*E_XbEmOtSQVhVTtc5nwCOhnOa_$;5Mkc5(;KDmp z&!*;o+kYSsYr}nB$t(C^-tP_t&;Wo4UR=EdQ7@C%%lkCW5&NiPgzO?a$yQf_jseFe zM#gL*cx{8DnIt^sL1l8VK|&5Fw{$LUeGWEDZMF<1NN!#v*j!!^+&mWZBt7Bq;NQCc z)b({}C^!@x>_L33o>&+a-5$!1d#YM^8)l|h5~y-5S?{c)^i(6&WX-qlvKPAaEqAn{pBieE?yAYG!~9BFh{F)}7|n zmRBvi=p^tP2VH5p)zxacYQADS2@HME7Sv1YO$-e?RGizzQn;(JO1X<_wCCFksRpu> z3tBI`+FV^!m;I3au;U=`1*3L9yspO4;vl@1*tXVZI_CD|C0BbU#ylbY9ag0ARu^PsiD@>g4NM3vz?$E zq=T}93*0Tu=2~kL%mHwLTGLd}+~EX+AVVgE6T(Jaw+XRk>@iaq-k+>; zpUr3WI=l|aE;(VUHN{4-4!%_gd0_P-ME$_2Wq`An$a}|IHLtX|O-ABB>r`(f+|dM3 zkR9?pa;K(vzrC6vxMsG7t#r*Nm7BSBaJ{7S8S%G|cv}U6{fv@v*LUm)H^6nk`~ILg zoX#9PY)TjIvexbAh!QTKJAtX?3uE7EDBv7nl+(>oL^W+A*=87+G0u*2cDJ3g!)tW& zHzv!>F;m%vOVK z+TA6q|2}x-L&3{&Ko@c$CP>4ql(q4`fFK2cAr)JFs54#_EA&mR0g=J!jddC1m!)lGEy8G0Nevkwo?OU5j@;`=Ii?a+9*B zVx3@1+QK9cAiRf$)gQ|P8q&^I3$^{?5ou6_m6dyt->bUCHlbS_5QPC|w1O$7%NfBh zhD0gU)p@e}WZ#8>tFghgNb!^TPZmFEYtwH;I(8>YLbW}}a;FXWDRy$YTKV~33&BoJ zmrst3YPcvHp(CsS`~X2X6h6^<5DwX`k{L6ZNQ;%IuddldZwLH!45?xfWj5c5uoJkH zC6W4?n*YM3k)}=>=R7{Qg!OhLGhRZi?UY%KQ?vooc6j|qd2RdQexYk9hz=}}?`&xW z2$17M58T@SsSJ0c{<^_x5x3|H)|bB4P^K-n8DV)jqoG7gz>0^BF_Fi_>O7|M54?5*5evD@5~Z9w+O7LAVy zxI#D?v`3ubeOK3gZRmw<^e$ZkTKOjhi|KHVqg^Zm*A_3%t3hi{>ii7u55?Nzt%onY z{&~6|R=dYE019!|W_Q!}*VxyvR(FC=H)Zw@;N;QUk}3al)&;@e@!r-UB*4mAU39%c9C>Z#GZ2XfC9)v8W&?Nkb90uHGnto2uh3PH<{F^HXM z%C@{0>2A*gY(byJ)CaL}jAfz(U(46+L)DS78FHznKasp75fY!`qWJZDa;q8*+c<)8 zYx(RO*3Mx$F!9-0e^8EVHni5h{B^qh7*!hTUwu)u3&xI%M&{l08Nbi^aMMa<)qhLHY=W_qk0bw4!Yy3ve0r%_`U903w%i>#&MIM%V(|uIAzd5)mW5V5pkE7L;y$}Xi7r^*Fju*$3auN_hekM;AW4^ZA*$a)x&z$`=#oT`m@W{igwBSN6Qi zt=GMp5YUmq?0J{lemL)PxC?QLE}$3@`BL`0OFE4h2S35(Gk1TS85tN%RV`l9R90E8 z2YLUz$%-fseI@5=_V2FPyBb|GvgvPU-bxH-@B@aVUcmL$4P-M`(U3ZPu&uYPpAbEQ zNA$#6W0_zp9LRW5DKyGPnF!wPY=|^r#wK7&2@>oYi36Bnv7)stPkJ=`Up!rWa^r@=JzMcpyrw(d+0&n@ zudUTLRMj@M^#T#x{fC-0h=Ry3j2&Kz$bTHq$SzI9?@sbV@IRO4*4NSICc@(23}xu} zi@BX0LLx(`Wyjcl`E`waUv6(Oe$7iL>vFk4c7W|?``v?FKm1MZP>&#Hh@{__;7`NK z>D(r+26%*7B6{bt%7Y=rtW)W?_*UfgLIPgvM>`2unDY73IrtpEJQCa zBoDTqKP#(X`H@_4t!KSwEl_eaPlC(AlQ7%=I{g-QdW7dYiB4ZA4iGR`vCdtHRn@tL zW`g;c{)qkrRvwd|(1g1qeZf|7uCPU%2P;!^GpEu=d#(ab{fV4guL;B=$#5F38#!yb zgvqNvmG71tex?7c;B_L=*Lp;X!cnfKO^exRnsPauHq!1e!QCZ0!<(_?o0|*riS?T* z9#y8H^W#q+TZ=6-Xjt6SMv87C4cRj`IU8{_6B_P+Noci z`fbPY&XaA2;Mjiu&W)&Y%A}DuU;X@xp>Dgj4Sz|wdzFH`q}&6W4p4_LqiXr}u~GRA zjf~~?h66wI5z4FbmcQo?q}@`Ii2M98;T){cV7<+xgSJxB8dW3vx5ZauYj&DS^NE)? zmprOGfGXwhDR;_i-@cSObP(66ct#|726hW1@4}hUx0&~_V_iHSBRajIM0Q1s*1HR^ zhB{u*5!W9C{K2c_&2N4B>8EeJs_axYuX|zPvui$-SK(^;$+57!U(+KFN5Vl^$_V{E z=%3hKKbwJV-)k(ZB8K#3iOtw^tE-zE3C6_%?l<~dLM?cdk4jNsz(O5;kv?oF(^4)G zyqz~&QLReUldK+lc*%>iA3am}E<=2DKGQu&l=Yo9pT$OpecmL|>hEj?<|WK#s5oq{ z+h^W~tu7e)2e2{&quc>3o#Od8q5M{vsi|tooVYg9dfZ3wUdipnqPd0Qc260u)z#Z- zF*zWAqd5^RT}E(r#tGm&=d`#jSapdYmJ@&Q4-XswGTkZPH&&%7tYVt&1Ygfr?a8{$ z>$nZu(TjZ4?F+(zs2FcU<=?9gb1^cCt2xnikhE{n&IHYJ%5 z6HSCu;pDXD?gP$~*oRZJ`e2)h7`=Mt7biY|y*g1_iz;^)?Ae7sAurd+w;w)zx-~)S zqqx=JY_S?jx0Y`y+Xhn!wyhH|ly2;ExkS?~jzmJC?56eouxSrq8((WGs34A3>`Je} zw(c=hRS>)PHm-aURbGC7g?#7b^PQvpc=7PLhKtz25ucO*8`0IqC%Gt-aV6lE>U?uP zwz_cWyX;1cxJR*c+MV5q|5P5;>?k@aqw*c2$4{Rd-lyEDp!s|E5b}WhSIzNQcKsa_ z>(AQlZWC5hEO;u2-}{FK4r1#5F+{#cQ{*kHFT!dY!hSPhwlM%AJ1s6V*8=M!N1ETj zzWw0*zn#_|EFU0jA$?pQqea?7izY;uY)QAG0|V_tZ8)17^-WF2;~fW3?)}|wJP?F= zyN$PEMTHK11+k;LVi&b4dxn8>n}CkyM4kl_<)kSJuJ_lZoA3g@s1Q|Pm5$RJ&`l~6 zYvV|qIe?&S)ZW|KCWfN87>&{AF~AKPdF$-uK%_5Bp(0;O9jw5a8GV zG308kFF<2+Q<6`}o1m<~`0DojRTLoVgyT$r7ufbNGW)ei?C*aoJC zt*7_UyP0CxD(}?zg@7dZVZqObLa4GT*JWdz9KpH3MZqQ&6Y~P0J*uqEB?YJG#7BJY zK!|8l0Vh6`eGjibgmin`&vhX3Bl4@7l8FBBsZhtD8A%9M-F`y- zT*p{Squg?JgFH#jdGkGRN7h|Ya+I7^uP9S-`N4nJ(5@`%q5+)4>@J1@eyOr$YJbS> z4-(gazwhR8Z*IUEG)ko~2An%y!UEpxJ0YFE-d=;PDIWd$tD}dmy*09Tagn3i z*hF|#&Ot+uEegj&VW2LC8(a4ZIPbK%?bwm^8WU}&7@SwJ6wT2XdoEW1yM~2*b~6Fy z*yHIq%Xc8^_V;C6!&^mrzy=#)Q7VOv96ES-WOTG(xa43yoJjg(QNri-1wFt3xdKjq z4`N6gQu>rJ*^)FQTiX(;Olxafdq=vJ3R>f)kO}cRogN2fwcDIdf?>9AGn>ptlWBVK z;nI=fBe1T;VYU(uD{W)#X!mNV!Y7=lSOzP$sYVM7{@?d*wSdL8>xR zg1xY+P?^ZU{FdOQ0?JP>em zKEemQBQMMG(-_IiNtnkgie5tWN(n!zeh4`fhg0*f7^s4jhTH&*wjiRiFO;Z z{djX6e);;rrx8$Z%UI8xY`w|jyEQ>iTXzs;0+g5Z!G{s2gCT6R-9kH#ATCs-y)GZ! zYU?(3nfeU9#sC8^z5WPj13WB_4`?==EV_8@_>s%!3QoPcX5a2j_=W3+r;&lN0G zds}C;v7yndZD=xPI&si#*I0Tc@-njYpWD_Ti)SHS<99E}#^=Yow`AkR<9p_0*BU zuOUar$G3p!_3<`MX~r--)YdxKW9%p{X>O>(m1plRZ8Z!HxBif~zX{}JkHPFjem-8M z3H`7dKFaU(nUJCV&s;WRR_8?TjNRmNW_$OzfUadS_I_i#so&6Jj8L%p8RX%y%`=g2 zWR3jEa-{I^f`5@ddD{-;#yJN?Bdg|sg3rfwnxL5I@q-D{66J-5_dj>pf{~O1EWX2L z0lfkANKvT3co>N}S^x2#b~~)>xxQd|_RPfn*|v6lHxpTm=#FRt8OiS@`sxs^H9XA{ zvBa7aCdB8o33kk&ZHywjTTQU~6M5j$d}QbNLko}}NNN1c4rCsp{A9dh4#;|IyjJ5A zya8_r_V`4fi1Kc?8)w1tQxZH}x)_X(qMQzzFj=bXM0QwS`eeh$yTPz3{<&~IqGn|4 z7@@JX>0d}(Y^>ZWS3H>|D&&u!Le?SvZAe&-Eks^Hs^!>YAkxV*HPwhR=kBaKQjZiJ z2mXCrzIUuvv#)GJw)*{Y4D^ZYkn3k6PayKFF>)b@|1m~B2BN)V>IpyQc&G}bM#jje zK=f1jKQvy~RDCjL?8PpPoH^20wp5j_ z1s;IdQQwM1M4!(?1X96pQ1oyT?RUcJTBJuGSc&MaEJYqc^zsJ_L9|i+;4u&>+vL(G zLHwBf!Bfa=q{v?AC_I2r98Fb{)l?(Q@oru~sdi^0^5>J9?yhrZhK41??~^1!g1wB_ z<;CMRL%azytX{aMieOzF&7q`7c_`d(_d8-t_C;um{LquglNXmE@~v+#MS#oa9ABc5 zCeDBa`K&3I5MZxG-UExGn-@`ml^7q+32uQCSPva!eS4XG&N>^sxv?^T3tAB`P88!j z%QHOd*ZH+RtzZ|-9LY7Bs-6T`isBqD)ZuYRPQXiHGRj?W`5OgLi3Qpm<~Wu^sU}Cg zvl%uN==T(%`%15f<1nZ-!F1JkFp+0bO>f~njQFIG!N(d8+(RP zKN!3ZAQ-SV-n1{{Z;uXy2fTf-7!tz~^rW_bUm8E2P7j<$hYg2J!#F2$U@m0BnWEBc zfQ|D3ArT!p5r)-Q=i3^h!*Q5!1UsKe^CG^0kz z>2={=fO8W5R4^4vg`)m=AQlLjLWWQS>@n#C16Hxtxp6a*zo%~fOX#a@8~gL|eZyxO zu3?wYb)Fa^22ZuW^Gj5|sYsr(^?lqY4Y#_{wx|&4LBovS>A{6u(r4q$6s)vdU!yq@ zK9=dmyIb3$16X@IrA-s8Fx6M%Dcz4Mw=})9zVMYn`Ht)fKKs~YOIZ zZ&v+w)vs3t$sikSMWn6}>Q*+$_?A&T=#x$IN;ev&+gu@Bt$Xh)jpPV8!%T<`c!GX7 z5|$J4E;kwoh~YRIb_VPLeBRr$-=6=r332-Df{n9rPKt8Cq{UThL)~(zGJBvAcTkE& z0nPFju~D+aYPbCU*l#r?<>pA@RccdV!>kYdpZnhTiG^%sC9Ujlp2oFsCIfI#FM8kd zUi69uAfMUkDst^~!Lu@#cYT2!JDZGk5Ppkf;w`M++0tTxjk?_}n=s}6M=#4u8VECI zWUPZoPz>>KuPV-kgcxq+T3E**5(0Svep=|AuFIo`;VT#3PQHiz>O{FNNN7oe-GV!E z?HncAB$m%!z;kY>m$aDrDS7w!^BTJYplredW=WFTihYmPZ*m%)Eez>1r|l^U_C@`H z>_56?jwTNU(*CqRCQ@#9u&Hf#|LT|?cKH2N2uo$!;~|0!8@rADg=+?7aRMt znyRJPSd4S*0fee$cf1L_j7V4v1c)yp=~x19ZV+AkN5{Yi6Vr1e;Dm(x0khvMlzYY2`SDx&#V zAzw<>nRtF$SxZqBwJG}r8z3j1VcZO6<#Z0S0oK(Ui*^A=_V>?y_A=hCYt?0RQB%+) zw!nhx7*bH$w5NP`AY6$vCfaC5EuuBx!c$~F)z86^+yg_tSU16OL$N~x(xvH~iudsz zKbmmGoe4bqy;mfT#nMC45G;iFXb_d5#c#d%Q(TwO#~a&P;w>?u7gnY!8qH|9Kh}Xu zxld9wpdMTMHjr}4Q0~T{5Np+Fp@k; zvma^5-OJev5amr(i@wTSBfy3A$u9}M7zrY!eDNlfS00(F#8j28CbAj#kqJk;4Q9RB zuW;?bSkg<#PpD#anr-Fa_O!>BCVz@wAkvZ>+0(2eapT;!j?cJEV3^GDx#<`45R2b>MY+s^UaJ?@hktI|8~p#&piM0<$2z| zm8x6!+o+?7T)^qHr6cJKPlu6M^;B3WQaVqYEgHb8jg|*u^?0ErS;x1?^ zD=Ob+T;-V$*9+x>k#K=dY@;0%828GB>ZG^F+vbJ+HDW~_MrER9sU4- zehKSy54pRE1HJ>|K_9H!E7y#Dr?;_LPR8kWVajynUaj5jraYvx+1yaP`iAEAegd%M`^EahqwQyeZ{b7Q1fNV{ zV`{C)i-O?o>B#qjAxp$0hv*=bS2fB+MfHaAr1HScIX{rY!d_{Q1nf#66LOALfhrcS zZ6YY6$z#H;CPAzxdCF(;wfexs8l_=RN4uvR9~h9j2T5JGd|m!R{}&dSrupb^Sp}Nb zjE&S>-B@9&0(Od~hH3;xnq01VL&?f|M3W9tWy#-bW-IEfZcT4jxGhSayqP5%SVY;b z(a9rY`^FyBmu+{N>d0nmLrt>*zF4(r|2+KF$y=42$NppapDrBk*uRHlwZULG777O= z`OAAwmtVqvbA9;B02$KyfN_f5o_>A`0VeQO`o@N4M=8F%Yd08ShASMZ$1fDjDp-kM)(7ZmTZeKFu3ePd?7H0OK^|_^&nVn*j^jlgwKAmJiY0Y*U}B_v$Y%f zx8`EXDmM~@>DD&HCAtHM+@k%S|Kk}v7__2}h=+K>1T&S2@s)}Wsl5n*5Ev9*Lq zcejE@wQ);TG+p%F{iJ5v14ija%HwetWgeun@FsriW=wg^jf7y@+=fsh6#zc&g3Dk~ z6|dxj8<<`^$c$f&YR z4@M#evpZTS3+~ahdL6-jY;Q-T*GKSTkoV!jR*i*oFfJnfSn|}EMgOca)MK0kH(2SS<8>c8mP)xW#Fa>lC-HPsc3eYXf**Qv1w^hOg^Q-;ZZPzp4@I4G{) zJHY!B>6O}09FwcG#Zs0|UM-*G^}ZWfbGtU)k{`q5rRRQk`^(ESl!eNRo2Sh*65<io8bdkEN z@-N0(_2z*co!i|Wmf`?dGwo(SQeMBV%=%n;Ce+l~)F)h&@_XfVJ#qo~g|BJ)qOnnd zD(**Hty{*IWx zqpGReN-k1nEl^gjQD(rKE&DdUhsz7bcIjFEu=4nO%GATk6O&k;;|L|JX|W}fe)5p~ z(5U7Qy1tVQq(svKnD|V!9-8cV z=0xRXFfA(ua;~D*pJ?xu$Dbr@0c*(ShLzk6#lsgusw(cJc7eZtVlIR z`!Gmq{eb#ub+y{4PN@&7Z;pF# z-12d2$9**JvvD`Zzcqev{I&5nvs77+WzEa_vu3j9Wz7fL4O&5aMfKbjxd(Dj z=U&del6x!nPq{zlY4YyPBk~^0o0|7}-rISr^0wra=QZZp^0>TkUMjCMuQzXB-r>A6 zc^~F|k#{ri%e-&$zRUY--j5U26XA(D6YrV$z(jl^d2H+%z1QYzj+xl_ z#CQ7|3(56hlyh+o++pKb2f4_lj4xGkuxCCfywcy}jdT-!Usw`-aBH;N+=UPDov~mP z_VZoo?f;Z8%BA?xE{xyL4e!SeX}_+0?=@h)w$MN`-(nV3HyU8Z&ARE#R6`N+3gxsO z3=}u0v9%w#dIBiCnkCn*dnl9GesJq_yp)24JZC=>b%CUBm^02Ij1S1E_ z;#S($>LKfLUbf$CtD z)F3qk8f~3-G@Reo$7jZ9gNYu|qxTXuM(=e<)M(M86DC9_QGSV@AToL{K@cru7z7Ey z=tlI23^52s4PJTg@80*f?t0I&*7NLjp7YszpYzu_Yn}Do4hJO-KiIWRu#x7M@RjBJ zOfQVc!a*tl^FLWQgHI1DIPxeS`tjGxVZki?ok4Xf`}B406{xD3NZzZBglyw6`r$y> z@pAOib^d$9st&=|m%8IOf~u)}=h}zgGx()l7INrM)iLUIO!rQ(Q$8+mzy(kEjtuuC z75ydJ_eSPoX@?EV-&3i)b~LL|+8{dL#-h^sV^&Q?aJ)PpuZtS;_3dGU3*tCU`=<{* z!z0W$?h@%yuqErU8jeK9PzGr_KLk!^Y9dJ!)a^P#oaA=&vzD)F0>E_5E(oJG3?%B^ z#-P`=5IO0y;DA*)<}Hea+E238mEw>NH+)r1PvcO(DE5-_3!~`D^Iu5j5ig!Go$Y@2 z5_{rJd#<1JFUK$Lk-pv|x)izKTLoU+Bft9w`(+M7Gbz?}bpI6OBJ6bYBrCD$~ajG2e-!?P8&6N5S5hAes!ki(KF?;Cz zu#{V^Tipbr?@|rJX3pP`J$!|j?1q2d`9<`qONo((B|aXGZ(yN>-UY=TRo)(A31IZg!%y``u7uM8EUW!LdIN}tcICuNrt) z@n={67o-e$^FAEt@4}1>OxBvA>kpGfx-{EYoX#(h5>baFt@!UBz$=Q z-YKjSj#Rv<5@ehIm0zQtdXJ~nA>V250Ptg@5n6m%!B2A4>D7^k$yi6 z`abGb@tO7Fc~PWqA9ktb#kQK;CD3qt3i{4x8tQZ}vx0%?p4RX7xa>yFj^Pds0q%?s zv*|z9e*DUrq1#DLs~UaM8iih~n)CXd@SEJJD7k@I@70)d<$zTg?5E78)=+p4zYBJ_ zxrJB0y1+hiokeP6ad?ZRd(Cc+$UBEyt>#SyendL2nL@(#cA+V<{3Q!Zo{&oN(y?Ns zOSWMI`)})biIbVR(c{Eu3^S>h+)J}}16CAd1Xq+H3Zsx(-#Te^9Uv68YMGD{)B`NI zpCMr!Hso6ZkQ=qNysHZeoBdwyAawTzfx2D#dbx4}&iVTNMfvQ=R}hIZYot-F^qqy) z&5wr>83iPr-19B(y69YF#lk(DZ`@9j5OLd-M6^BY?3~~~!oTF{{<+`FRz~!QlWwTx zzJ6XJ+iQ9ehFms^($FZo!s!_yq^49&8uJ0{$Evx<8gr((`NKjtspA%Zqqv2e&0jv43(f5bG|^*XoH4#vW}@C{lP+PWxpd z&$AxiD^|1!aj=N2<}%N(i4DOo55`QPemV-}ryTy=G=eAxY_Z{~E|{b8t#GSfs7i+n zTPK|%WecCWzAb1nE&9k?M=>GDT&K^(mo5$`ch=(edgA2DQEgkUu7Wp>)1xp$fst?K zOELrr;)0>Bb&c*Vx#F(|%XeJO^3{q*Ic@MZtbKI>Vb9*gZ$6yj#u>if8n8Tow1l1> zywLreFY3~R@xaxkTJ3AK;%IP{yiQa$%VRy7t~4z!XDQ}3vaN(nn%d;7=Zs%gN_v8N zCWAg6NQx~*5u`%qZ}b-RXt)@RV|;bH74fbi#hve*UKLGD`uZFl0HiKv9Y&vZD!(d8 zEECA% z`J9${8O{C@-wpc1YXlrD_iZXEKd zXe3ixIFuA?>*wiS*Vt;lS`E20@`CS1xG zlb)sL7b;`cVe8gdR_w&sPPws|Aq1I0e@F;akm&nBqc-44;A@@0VIi$8VNhB&W^9kfYPvZ#l9X z$D(<<>w*ev_Lakbon0zM+1T@m=Y$0cB`ChbX}K~8cBNU&yzV-=XIBAylvnnpVE%z< zxT6`(AhYxeC4;7I0`+}`XwUY;WJ5FKx9ij#>rOQC^!wThk|*Z~3CjmzegV`v$;nn_ zRaGzetOonqf1O$j&T2MVcMU^XA9eK4t$Is1Po+!6U)xn#l0O}w>0RtWxdb*z`X;r; z4m;WaeD&`(fTw4$EpqG?Rq`~4v9Be#8y))xdRDCJVb4RHGEs?|*}>SyEGYqd#2CUN z($Bj6g>8G`rpI@7!(Zpl?Y{@HCC^+~(*Gi%U6hsGOo5fuzmIJ8psTB7jJP?P_9`;y zJScjNSA~D2h-`@D6PPVDR8p1&2{bN#Bi4!cfJ$ zr&~p1cP>5&@cr;ey_Hv4F#_L(c81Fw)&#NKw^c02>!LI@sr_tRlt|+tlU-{kJBPMf zIoH8Y$s}hn@X;C$HPHovsU+j&+1sWf7+8ds-t%fQeIywHe4v&T>*qR04V*%+ejBQ)cn$x1IYNq|>8t9pvRZ>K!brx8H zM0;jx8?xwM_VJeasem%JjkWgLR+PG}U^Em7FT+{06v=~g>nF43gm%k@f(K)3lpe3< z>h2Yz$sac^;Pun;b+3x#&=?79kT$ALbSF25_wgCNMiJ=+;EIq21pZC<*U%MLBo%y4 zt;Of({P>;8&Sqzu0NAN4Ew9OuZv{{J-7x5u!c$o;pJ9AXdgc4xr={)QpW1tq$*o#> z7ZJy1O{{|Mo60G{i<6og&V=48L51y!I?hZ6vXcCP#R=eGP;)!A`(sGYe>b<%P zc0#O$XYcd#ro|kKe{JyrawtlJ3o*FXUC6o+khB zb+c3JmFFC~zDQ+Ej>E&=jtVlvWHqQEDKUd9WpWr{61EXW`QNeI6ut#s6Kc{Q{>cCE zzKA;ebWj88DU&s8b8E+0E2pDkqZC^jwTGGsWF52t`O99VGWstbE|eYK&LQ+KYfMya z+nRgMEFX7^nb)^1MJcoIri&JG>3P+o%g$!O1q)cv#i5_^MirIbP9d;Hn#mRF?`r=7J4-DYdTJ09|dJ{7c# zD%}%s2W6S!Jp}&xZ-xt=7;FFM<#y7~HeaPM7AS<_7TyiCvfK3B9D>Nz@hlB(y<3!- zwJ^2wseDu8xy8(hmkFR!s|mb%V9&%sg8{?S3PyTE@nlO%voxjnmaN6{x-7jU zxwXA9iT0nzRW0z~$!zIk$C1xtA}ekJNY`QA#O9X8=oIZi>I_3Ybi6!qNf>KKGRM@# zVhbreWj*bW*k2{xM;^XG#DX@zU(;H|E2fc$HZJqAxHhB-ye-R_J!Phx zvheRSMk})2(o`d6WD86r5Ixfmb6}3^&a?YC zhetdjt6B~AMw?_o)1RPkfi@nfD+=AIfA96ynflV;Yt%f~SV3>3D11_XuTo`RkY|OH zUotP2bjqgrgGN0?P(BUQv{*;wydtTV>9s6z{ey~NRS5XJ zQbjjFai9@AxWDWha=Wd+n6j^E&9as|F+-4PF2Yt8wC;sLeZ8;zpuZMQUoO}mqu}^C zIv|5oBYC4vhzz8kbA$erwa}>$+zmnqPLkk+Q^pBoyhT!>$fDoC zHAbs1pi0oZC!A9y)X;HK^RhJPT9j&^>f4wDwdl_PEWi}7ZHZ4{2MLlC8vhubrx*l$ zUwC@~MHWwMF#F?)r;NxfjX?+|Tu3HcsC~pWX}xpm11}uOLGJUmeaoqVg^|F z1Zmtuf~`rx&E3nd7;!U8vOlTB% z?h$ks$1HyVUW+0>&GZx&2xMRGHOF~mf^TlJQG*6C)xR>5h!%;O*rtMN>IhnmQ6kU;n0Z&x`mNe)J4KO8Xl;;oy(D6IwNx{4$%7gm3Yf38H zFx4>Sj0z<2aSK?;j2z=RCD3F>;4!PN2HbKgq@6-xxTa*?r`xJ9M4O2iFo6U&_D})+ zjS&YlIh3CA18}4gy%$t<>mmx!ABrWF9^Y6hdcTQXRsben7NlimJ||MVlmpyWf-bb} zE1~^>7isBZ2wV|mbj^0Tl{I`*5Yu#Jk9%lqU%oKKMX7?(kYffC!kHZ4#4x6A;Rew$ z^D@T`tSyPp>1BG>s5$44C>?8AkG8@vz%_D0DeRKUMOp9rhkaw5aqvY|J9sS=h^%W5 z`9aD+j9^~7b57K*6TzWLq+Cm~!9bWuyVisPI&imnD>%NTZ&$C~Dp zwS%lgoz+pG(_aG1GB_9@+Zxc`u{)AbeFo&yFb0Mwmz}a4P&|T!TAxc~)aH}A>;nsx zAemKAjUB-2PDsji&)`;!Fesa9mk*r_i;7sY(-qp>_COK22f_mgNDInUqBn)(Q?4`XxJ5G>mSK3pMPyv77dDy zTmH=z9SmS35&>F4>M0#g@{ECHG^!q86J}tBHsF8FCe6GPF~~s_(8K@;q6g4l>4TF2 z$_qa!Ja<)^h)4qfcG-Xc=vD8(8t32NJKm0gfgWCe^urMdw|}x9uUPNZCcVZG03dDs zUy9@Z$5vf={G0u^{`i0KNU=`}S~R?I+CTun0|Wq2{2dVp_!B!30t6u3U7cKgUBnQN z5&sNKQ`W1{3NTK>Tn&401z-OU_{TV<00Jzu?%w0nGcqy0Yo-P_0j&OG@Csv;SK;{htZ^e_HE*sDHID!T(|ZJ#)W5D@RQ7=Re8S N6Lgg{VDiuFe*lPvsYCz( diff --git a/artwork/scrapy-blog-logo.xcf b/artwork/scrapy-blog-logo.xcf deleted file mode 100644 index 320102604f4511d26094cd4c964ae767151d880c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 52428 zcmeFZ2XtK3)jxXgNS1BcR<&hGqcZBvy{K8eSDR{MTb3-@>dg%T1Y$@?LQ4bkg^&c2 z5C|b3ACM3rgc=ATKnTUgOd?wch&HdjGec;cCw8a?U>c z?6dbi`^-#pYv(b|%}1IvO|2b`f*^Qk1VKzgga7>Th$8+3;#pDfLqXt!e{1o0;!)!X zT4Xwd1&B^YL)}u+@iA~K%x;wg0 zXcCo|jVGEL>NQ(7ZBhPxNn`5*DhaF&U&6YG*L-z@tT&%LR)3_mt68&kaV!9D`7+=F zY~zhxt*11tT}N6PPm;g8sp(|nsU-o){huH=625@oqUNe3sr*hA-Z&1?A=M+@Umo}w6|3S^+(#9PjuhdrI?r(Kr~Ak&x(aM$yP43@dDA^G;FWBtiAfO_L|Gu zzG$7CmByBo(en%@U-E%70Mc8o>%a^kLMyD6P`KD*BehLo@6|kcq;H5z|)53Ry_AoAE*8P4}V}Kr!Rm0|M~pC zYfUd-8@B(ywmxg#6i)x2{CEurr!RYuE-=~nncG(i@8Pw2p-pEWztG->SEeWUC;k`y zFrw1uufF!kiTY!=XbL*IoAFjq@els}Z^npkqW{BqzAQ#CO+tDk?!&7)Jv!BR^Qny; z^|v&h07yxXb|n&~iM~wqXe1X+-CY1)-BEw?WNW*oy76f1jh!17UKjBjPqyCLh=k^{ zjD~Cq*qwtMYrVO#gXv34YZp@w%`HkYQ;#?G9j)~z7v6W&cQxOLyiF6`*tPM-lkC5t z{$%6Eqpc?zkDO}l?$UIeIsxFnCvM1wc5K<6v2EK@TC~()&ee9~$y2ROw`e*Vn@+LI z?wdDub{}m-+Ie#$GeH8r)_U|*i)P1`)WtN<4Q^>{ZEiWG*}8S-m2f2~TbvbDxbV;W zg1fO~4hX^Cv&bTTj^`;nFIN*anUZWWEk9Y4toQ-zEO<1YRv92-EAXzvsoX*4cS%LDx4Zj?Q0*5$3&H z-*w5$WDh|KcM|%d3i3L|b|WV1DYzcRi|bw?Qjo&_DtiMWh1&@_E`$rxHm$r$UM9Fs zD2h3ZbBMHM3`$;xNxeasbU+9JhY4iT)EF{YDO69mbHA`&kUZZ-dw%l4J>f|0q>LGb zqaG(2*ViG5({-O8@?q%)iEvjZ(mY8s#pzB|BiD5`LLApO;B;J1C^{%a>MXUpurRJ4 zuo*8=35_6-8VQviQIYKzguV@S4vq8)pm<-$6HkTARcM(9o&`Zg{%ItTE50@Gm)u7?TxEn)8vmJ65^<6^Uy zz*pHS$CiOzVIk&uKyv+wz&ZlIN$xPh9w+Q(!uArDNZ5UZJxW*}VOt10N!a^@We}E3 z*j0c@^Q$RN6@ifi#yS?>|}q2&_~%C=Dhz*{9Gg_u2RIA&B>UFqU9x-!LeL%4Tuo^WfKKnub8>*} z`VlngT=}pIE@R%Al=L|_hs499)1%EKua#XrRCPQyrSygkyGeO31^iVGqTymL7fJLR z64l5)vP;?Qw-NCO5!VrM2#7f$xad0<>6lW8tfzSMrU}eUy1;WRyv<5^uJ@(KfAkn8 zGoN=lded`)*WnlEVVqBT3$=7XhL+gvz^<>LkhV~xJ?IobgExj{jKbKOI;*d+k@yFu^;2S(vd z010$M;4@SYLiK>GbwdK6gkOWk>FjxjqXsR4Vf1if!{5(e1*hX$?*_yD#4q3Bs-=ENqGHJ8b0=7P$m_kgW{DqLyF}uvPjWd*w|*l;_6Yd$f8P{FzPM zcJm9pcF8{c*9Z5mfv<$M8uz0gXTp6WupsBmlUuMYusa?`Sz>PYKc@e-1<92&@=1wy z==!-I`d8w}D(LDXSm~V3Pw28EDoZ^Z;1J>Sr`N-UO6$4tq^~~*+m=SR!b0Kn?shxe zaEuuS`vhc!kSF8_S$a>$N% z0Hw1<+@oUK1Q?|;A8wgiQanhCtu*=hTQHO$Z6}567WT1fp3CtvDPln@>2@(<6>#aL z0g+vNOoD%sKv^3q10e|f`NTd)?0I5?Ih2BpBli2m{)*V)%dlA2cS(1K*q;(>xDtCi z2X?LD=6?7~(1q+e^Q*Ayh;8^%EKF2!wO?Pt>Mt7@*b5ZvAuRB`+D}l1)?G>P9tj=? z0lbfWg0hWVNuY$5r$K{FJh1JFSg;L37OW1#0#tUo? zu^X1Kf4dUfj-DlH)e`!xE73|~@F`70S%BBC#12!epD$tmav2sTxs3xiE@A(CCHA`< z*cHBne(^Fik|$3c`+k?f<8dnUg)7i-Zs_|MLd@waLhswcbjm%i^^VOtuMu`472-BW zD&U7u`nDbZfEx($)n}nU>R#LokB4H_ny++topp)iTA{%v`QJMh= z;;#G2{dagSB)_1nA?!WE?j|geu-|d>de0G9f$Yrn3Ax9YG36^@)9FK~l7$x#c$$w$PyM6-Wr%zDu-@*@^2h0*?~7 zgWTH)Q&=LHi478~A&i z8wtU=(@02}ut9M3q2>HsNW=?G0FWPoMq1ZQiYJ7Pf~%14ul_CHUx)Y7vT@S>&?tK# za;?JBTw(6wR$(UR{uj=h>@U#TJsu~qO=LFIz%^LM*3E2fgc+H|u>6KQAA9+uk+CUh zc4F}FPu)_qoJXB*Qe?sc=Dq&ZJ0trcN&S6=$V9A#n0(sCdL)X{r^*?Lxw5N-z+fC@ zrKflmfus?@ayKvpX-5!GI{OH$a05eS>62A7=4%P8bOY70>sN#^J9kyNVP3N1ldG~f zMQOHFfAb8FW4VmNd#<2>H&`x6d7mOn!uABZ`rpww`z`w&(Q!2Qs((i#;;P=J3%T)2 z(b!)JjJK{oekmFf|Akm3Zv02zz_tR*^cy<*GISb8X0Gz3SZ`tjzZAQg*^T#?qA)5m zTS*eX3az-cuM~MJC!{$4FOzsbu|N1KW&DiT-Cv4@CcPL8CCiGhz+=~e`@3Al{}>s| z6l7Mo{k{~>RaQcB4Xx-u*kL37e`fE$qBUNLw!fBETD50yL)YoY%pj5ewEW*`)+0Il zCJ*vgaITY`(z6>FIxg>TojgDLhx*lw6qkd8WvBHnCbCyRu@#03|6O1;mhR^NoTRh= z>m4m(%iPI3fBxPScf);Os>bCw2ww9d)6m8*U4AS>dN1rsnu(npHQB+Dk!n2NssKe(Z@ci8vlufqO-*tW029wheO zufnDi`?IgY?jZKRzY42V51#odtWs}y{Hw6nvge1t3foES?O%ocF|oB@g?*RUv@gXX zZCW!=P0#R}_&+s`b*dxxJ72L*2y4Ke{8B7z_$pwTv3+3z%dU(^ULOe)TaW+t1Ea$^ zY54Hzn|FC2iJJDzzw=M$#^z>VxX!@Sy*M~I4L^*;S=AYSp9PzHh2UI%hh6r?@&Ccq zIOaRO?wZC(*u2T-C=V_DT+hOH(g>#!qmVk(+Oc~x?xW1~r- zEYHkM{^eJ{dxxn(ybVEM#`cE??|K-gH=O_Uf!Jh{>P-yn9B*EU$w*$GVmPPn2yf%1 znBOpJJk6chM$e;*;fA`IQ-7D&?b5+Hv|U+%N^2|(xXx{C2k|T;o%QARJV1)K$afrF z!2AHai|Dxx-1bU6Y$G?Cl6|I;eT5Zxcv#H9Uuix-HE*H>qbl}vox&eVcXTgFhloGM z?v;f88N!~S|53`lTe$vD68qom-BYPV?xol2Az=s69}xW*`_5qBPJf~wBl-!VjYR*J zXyw@dG4lbXo~WD#rigu>ScT(JV)NO}l*rsosVICzv{DAViRegbC6rhPnQg=xIov+3 zua^>Uw-T)YW4JON^czS3Py`ugtAyx}*lOhTSu;Xv1MDDSdx?FDtv?W}RQIBYJxkaj zVp$rLoNCTb(Nh@F%E9ai(OcNzgTyLlmk?r=v%*neCGn!dy`7YsN%;kL4S7xiCh23C zd?pdCjLkw3ayg#sbpSf8M?INJMw37ho%V`}LEwY1@VU|7+}RmGWHtw%BvLVzM^QPF zMDA`>-l5GEq*+IU^!fv~lqBeh>Lf}zIE=&Sq-76dgGhpMzW+W^N%da}(~ST~7Li8r9C6oUy(G^kVDVG;bwB*#yYK&1$fgV_+=(7zSW?NQ zY>?&8fkyEn(#lzK-bAycak$>5yenTJ{x&YZ5}ZFGT1g?AxaQuk11)`~Li_ZxJDTzE zRXDA>Mwl~v`pecZNaP`pyZ(BO=o6K)vtZAm_FKMl7jz17&pNPqowz>aJq&K+@G{>Z z2(!p(XYs)VpN=uqiKLkTr2UfmBFmuP=fZ~Zd@*}lWnD{~vJi(knf{fEqv)lrc2i@g z*gC_OavDL-adu%CZj5pX)&LYpSbq}#WWa}li znv{V@*<`s3ew=;?MGV39I)M+c)lHz32rW8SeutMWNOF(`oPKt+td8!YfVD{k<5 z^EFsmSf`~NyzqRkr$RBDhwL1EWHVgK+#i3^jWYGs!s*p-Hsj$dYlJmi;G1Z?O1)l! z+e4;M6bgfp{QHPZq~X4fmZS^E5cA7NjC*MiD+t3m7EbOuy4`^dT<;nbQl0OVaVk>G zM|8A@Xc6MaT(+s(Rw>dj=D7<2{IC||54G!2K*CtfTQ2vd4T!7 z7X|iy3g!T_2>BP{i_>{ph3zL@H;8^wlV5qqB#P|Po7+)$onKb_8gDt42~NM`AEO_q z?KfGdQ969{kN@iL=X8$wQOs#9{O}L|e8%i>Iw!U7KYeQfjCtjgSEv_zj^#ZZGO@$H z!&VzBrtwR#R<@obENKarjg~YWf+ceXbbX)dQjY5ELt3faW4%cwI=q8ZuDkPXr##_7 z$F#-Gj}f1Q;LdYbFT-|m`t1EJ*C>$`+ZQRM4ZInfcYN}h1ygqg5SdEk*XfQP?muFxJ9?;RTbES(n(Qmz4drA82VVf=(gwIg*}%Q7dsq zdW;{?lWOJ(y@zUgtHykGXW1jHl}g(!KLsD^od32`nBnB5m*1l2dv`NdY^M5^iHU#RPuGEwkt{tdl>Cv? za?bs!g0-3jZHa>2r~KQGzxei_etC2LYF1np5GX4A@3oZ6j)z&eSjPoYT7gR~zonLP zd81&Ji=T&406{rf@s4Qquyq??Q`?^%mF)em+_qLY?b&yQ3BekuCd63$d0yDeyloW| z@6B*-$~Eq+udqdMy$QQSAz-WE>SU`HEh$*J)&_o?;QAd`;RRli$4*nc8h6zNHb1q7 z&F@i`rzzh**}4iaDVRkiX?!{HN;T~l#Q%Z#G~)edDbz;3%|=CUjD%|74StcV*J*^Mnq2Q&hjd5PU5*+LU*0c|m?f<74B5Ji{u~ zL(DZs0@*yr++sRa*hKBZ$If3xMcoTUNK?C^9jEwB<;3w3KVaa9y#3X|srkYGu2vyt zZ@WU<*szMxC}+*}TI%dsZjJvq4k9!JeMpN>`%*L#l*B%wpQ8OL+`AVo8LNO0=ySIc zGRD+P1l@}S_P(y&>GA5=th(P%atf|J+}=tjuefQHjfg0se72&4pRR@2h$N>OozuAf zzTEr4GKyT&hZswI=?7ejpH=E%?<6vxc@{ur;|IykvwN9ENwGX3xa{sO1Lb+sW4R(- zW$Fl*#ZUl*!83PX_ji8!^k3fltWQ5SEzOSgyz#TPZ6Xd!)85DbHLZK&m>*o>->%Rs zLR&M{nL%j+nOjRXUykxeF6fh(q2$k43C9hS|2|vy6U~~LG<_{Kpqxk=8I{v{`6Stt z4+@;Z9Ogebl}1-4MbfsVS>_W`jFM~K$n`0Lsmz-`dF2UKQtzY_ng7}{(pQ^E{||n? zSn{}tn2fX4vcIFpY10E$n>5+opMY4hYA9aL93nJar@Ei#IC}=8_8tcu7W&cjGQ3!G z5%Q$OH!&Jg#DJF=Ts}=i8saQ`jvtJGP_=pt;&gnAab_;C=Fzj9367l08~*_tcIxjA zMkt;3N-R6_wYwhpC5D)YJpAw*eG~KZV;BB-XRfDkTEvGZwy1?nVb0^WxnX}H9iK9J z7a0qDa+)BOTX^ezkB}-zr(MWV@aY8XsTHi%O7D4bPvKKOX#aT)+v`4I8=nI~d;J-< z9}(GJ^-s1RRoWhZWc!aE+|b`4zR1AJGzdGd6wE=c|B{@Or51W zLlMYl{I(zZ-dhftP%RYfoF3zKLozSW0{C*j@*??PL^_-zIQ@SQ7lqRvNVYv2n(^>e$QP!6bh>kY(dLk4upfjA*WlB~QXmMF`q5#Q zbkNzS8xi_^5P7R{`tgNt*a>^^Q_^@*n0xMxPYo{m)&roPU$&v)C5VabkEJi2!8~vl z#Fp!(5eE?{Du<>`1o9FaH#KM}pIvaRVl$e$=;4lJYJu}LTFO<-ZeXQl2I^9;e11mB z*}p+(sEyDup3fz(cgfbJ4D|~%r4N{1N*<50JsHck-*aqBSF@e1oGy>ix5PcA98+of z(ld(Vn>z{;1bl4{x`RbO!}?jDU}%T<39}FIWUzsM z{CY;1(h?9-a8g-lPvZ4eJR+Wz#rVp?sT(`$iThvtsH6BC`me$BWgm6K#~c^`ZKKfq zw}Pk@cOCDP)8v`g zI-8HT-*i&mDNl|6sHN-Jjkg|amUqaGk8eGCBA8$F)*45S2DQ}S{{(ijf#uHr~CmP$j+d7ZSo8|Fm>f1X`b|3HTXg}W7 zd8|d=C|d`=ex&L6iO%-!me$s`rk0kY@&TK@p>~2;y&*L+p9de>NcI<2IXsA7euk|#`@v?sWp@zm&-6y-6+uK{(4u9*z$;rRnw70GMh#V*DKIv>b zcC59nqpho{rK`EKM;@DzUu{0rbXbm+wa<5)=<4e1?C5B3?`Y{f{D3@ev(C&Osyoss zN6QY=ef5xo_;#^~i;c!cx93>CEdsF9$=5{xKTU%pi=fUptj=?GU zgPQ#fEpnth;C!^D>v(tPVGhvN*4omr_9c*m8P7aqX?>4nkjF#48?4+foxw-DBc+}q3baekU`|1zx zuRV+qZJ+!_L)X_jfQCX9bq+cb;m3(8oHC zG`f3hYC77`yub12!9$I;O$QGj*pKLLe*IK?cWb9xZVuUmmBXl0M;aOqAFe%gsOF&R zATq}FvTfogb*(oZZ*Np+QLGo6#U>YNoktq#>%=;#_TVA$(0t9o8?MJKn&Hz^hZ~M{ zb#!+uPDbhZ=n?UVtG-rMJ5_V=(4m9-_g%Xif$#JGIoff&<7msj(?W=P$WVKzrsmLr zz1Lp7y9yEUocV{1UHHybcWV<2a8cx=M@UR+(AFHd;rgqqtHi26dEou-!^dxeAovV! zN7KUm7Kk~)gW|#212^ow`YMtmAM%$co@{`no@i}vg*;d$v1zXHC^^K2iTcBb>*}DB z1N--0bJgxDRh3jxj(GgB@y-J$PIPr1X=-ZkYH4nBYjJ55g#{eDE32z2E6dBvkOx@a z?rQ1kJlS!)_3)9#Bh80fnj4QEQ5_K*q$BmUVy$>+7L&M6k!q%TPeo}(bp^5lljB#1 zI@&Os=Jq3X_dfo^x}yz8z$YG=!xCWfszYMU{PovCQL3t$@{00`s z?WGB;&FZ|+&{BW6uC}fY6Pv2pzxUcbRTY($(6_2`rhIo%X?0p2f}(b~ySBUiSX*l| zeM)W3$M%sav+J{KTfwTTbzo8VU9+dEs+vMNDk~}~iz@QA)%_I#OgsBZeM@I+M@M64 z)6t`?ZAWVEH^S9P_Z_Uq+$L*iW>@dVh*g!)3wg^cOSYH)ng^*K*Be!xEo~jG?NDk{ zBR<^ST=VUZKR^G-zM30gNr!6oUw=)tN3|1*tW;Hs6=L~(Uhci4$d^vjHJj<0KkA}u zex^E~uG#p_?2-z)=C@mN%SsC9njfpoE6yn|rE4CzExWLwthy+duK6!Dx#cCrC51&5 zbj`M36%>|~7F8A%SLD()UpSRjQdE*#Sy)<{S4`LZ%D!FYB_(C$S%sy!`9*ZiKPV_I zE-x-EDK03^%SE4X&7ZYoWtWu~6_n%`6cmCi7p{5w57l|a)w#um#bWVHVSZs@Nq!D} zv+@4y+>*Tf{L-SLoPzwKlA`>ALilFWM~&M`D+)_WO7lyLi+APb7v^W@7tlBVx-6@z zxUjUOw4k(XXIXVgPG(+y#xDBip_>cy%L~hj3(HFL3bI>YH5vYRU~6tcCVlhU2eOLM zQE6dues2D*iuYYp^RC|&?a0oOW8@LnkMpuhN{Wk%iXc{ALB`1m^VICTwJ2*>7M=6O zW4Wcp3cgS*9M3P#O+8^5v)QJ#C0Ti?bk5ITTToh1N$!Gz{QT_Vl8hTZni;jvznr}z zBbUy3?4FW>{KE1)H$h%rQAT!t`mO(*H2>|$&g>mobk1j+in7ZJ3Mz`(MP6QRZhl^N zQO5SGj(1mWOV7{Q4CmbUTxE7)X<2!GPLZ1=2Xo(*k+UmfOKL`L);4HCEl=vdn^#g* z1dSFI=jVxe=G>f|?CiqK{H%=BtX(--sacs@v*Dbb@3-v2bZq6t`Nc)K1soBCV)n$Y z!kzi~nY%K!Wo=E{nTo2N$@N(IuA;)~#rb_fnzd_J)~<~7^t9AeJoL}~cjV@kV+|GB zd@*0l6LVecb|!0A=B{0t8R@F@`Hb`(Tj`zuwl6=oytuHWcu_5i2-#U;mMT+~;gLR{ zx_$Eod~EQn^Cu+*<;8hiyG8AAQdt;36J2IxW@MzNZQs0cLo!^m_R^`Gy!_mfisD=_ zD5_i{7BlT22Yvd^Z5vaP#pFPF?&WJU3X7o$OMXsaQSQRz7W)<<#%x;Zj*S~ql2yt6 z^7zCLuzISR*Ik{jHv*vbXre|hSGVpHS zws`|)GLw*)keD1FN5A|*<4!0FwppB2`m1r5^UsyLva+*waYE^7Sg)Ckw4K{GLR}uo zGn*6QlM<5>;g|JKm8X`K7Z>Cf6qjb-ZkwK(k$#e{tcaL7m5~Njq^EA(l)OB7K0aZ? zhIla^e%W+uI=o9>K~W);SMY|zHsz2%DoBTEa86>zR9fn`%^Q;wlaeqsRnknthS-En zYJQ+n`~J0AWqD>J@aSnYE48`e59J5`RRh__?YMz zO-wR9v+n+QO;lWLN=#gG6g~6%HDNJPF_Fo!@$q3%^vqA^gv3V0BqoJL$A*Qe>6uT) z#6(1eM@7a*M~A7SBI%jm-5(wfQcu|^SOqEs02-1TugF8cx2=?&kPRzwm2v}Dh!T!?B&Yf=!Ce$ z^vRr7ZZb#qcoc6a7|d9aZH}KUyKV{A3@Lj!NHh> zXa%nkHRBP{k)id5DcLo5aZ_k`Fg^35J0s)bw#CM}`v{GWizxe0M(*)^Qb?;k(!`LrprruKSV#L&KIt zr;Z{cBO=1oVPcpAFX~XWnrj2U{8?AL21KVuYj)Q;X{P@UT#IxESsX z3+1QJ46kNHM<&L_B_}RU+1)cJCe)#!Ft5;XaD=S$hj;Fq`Szx;xL8emd`#q`2owVp zpbJrlg@J8Wy?$Mwzdzjb;77HQVUZE>8{;A(Bmb@YaFB~?M?`oSRO06g`T%+4>FiMG zJ0>nVJW><QY4jnDijQB zF#tJSe(=#}{&8J+oF+0QUK6g-sH0OhEzCE3VH!|UC|LY`{Cs`kH zsK}%nPSiwcpoR!&Xf8}0f&~-R;Da850{vC~Gd{jPzW!@g(@Vcn85kKC6BiQ}soDOZ zNuK@PW;KLVCIvQ?dL~>QvVQH_0DlkvnYF7|`G6H(dg8w}252G^VxuC$V-t=~Vsp*R zeJ2e1=a7ytEG5=)eW0(um;cPF)&BmgR;@w3$U4{>9uuPpi;j;A3y<9Jg6smL>Aj=? zNE*g{;E{4hI}-^}X3Rem8}est6??9wDe#6`!2$HYZwLgOBnt+NyIyJ}y^ zs*0EcZ^&BkhykAd4lID*s-S>XIZx0}_dOIA9TNfXk*J0Pi4G6HRxgjwj|6~a&8K16~Gf-4YNBJ!k#ipIO+TD)& z+csPwZAJNQ3rcUBP;SHL2~gk*M9puloFuH16NL40ybvtoPUmUt%@{!~M+;%HMu?Cj zg-98nJC26C2@|4awGboYqVaCHw_qV&UN0ob>jc+_a)P{8K>i}6$o|4c*-zLcqqf~7 zZ<5ytTjkZlHd(30Z9|=PC(GT87hXad9(1(>*Quvf0kYk8;g#S1>cx?6UR_t03}YZAnxjFEch-zBry6&OSh`qqs!RX)^THZXKN!4bdzrJM$y@~iXT$0l;j{|@z+ z+q=6_s!(<6ySolIwKcWvYiMg~Y;I_7XgJVZUzgSF;^|R;|KZkSUpvtyqD!$u*M8%{ z+uxs+tq;{5X=-RbjI(+}P3_TrseeFW`}2=(K63K-iH>IUx==;=WZE_>&wYP?{b8Os zYib(m>aQ(3I1W2M`$SvkiIb;}HFqdA8BC?&yQ5ai_{8kF{k1r$*Vi1rzNxnEhHDF- zA$9$+6DLocKxu&$9uT+IJ#5pO$EKzYH#8hYmFnQ3hJ7`M_EqP1fcng>$2yPmgxyU! z+E|6TbKGK^b2#6=s{YWC!!`R49;`cf;6Qa%4nMy$+d(~ac8DEw?vjnztgUZ)cHS{5 z>rOTtsI5csqNe7+{u{2{Q(5pDi@e96jyYwHSjqvVdeL@6>u)|f_u`4#12uJ^K8Wu3 z??bU5_m^Ne)4fo+QeWx zvEVRSTaS5C@Iwc0fH3oi_Ux`ISCu=VW>>fxTDn zE~~DDsO2TQOUueiE6Vb>-hKhMFRrF~v3>yMyMy~#>$@IBn5#iiRaU&axO`7ZS;4M@ z@AFcu?@)a`h#()!EsC0o%d5)vT#M>R`JUX8Uryum6!W!*Q3FyPFdSq7sSp@A-rNoQ_U@^!5G%%^y^> zVuh+)ESoB?tg0-_O|O0q&USJi3}>N2RiS99B(J)(xNJw!-F9Si`U3~|Uw0MiNDu(! zwenI-qPjTqG5$UJt>OED}yTMgadR1A~p3;{6r+xgzw^`P%7VNsoaT3BrWLk7U{pfeXAd68%gNy}vA8%lx40-XHS^lX z5Zdgoo-EBRDlaLmtST!Ki^RgI0u&wcv$nL4q4YcC`k$7<+^Xu*3e>?$U}JejC55O2 zWanhWH#m{B+Fn0goQwKUK{>F+`Ko+dQ7(%NI2Yz*MW0}F>-l%xO-u%>zYpOFXJ2NX| zS9WH`uGGv;iP?IFs~;2;pvYHN~_}bjC4hmtWbHmBXS>_O|@A^zB=s9tX9Q zS&W%qDk>~N)ki5_?YMK?JUTihf0UJ*gBn_9X7;wU^c^WtwV*yzURGRz5?!IYM1-R-<-Ix^pG@Wr-iC8kH)ba|%0w)VVVy;sVKYb9UvWrKV+MZ%@n2O5d59x()O@QE*8Ly9wlH$`=ZKd2ZsI?7VCU zo|dxFmH>AX*#GD8DOp&6%oNSfSzbaFg z4pGuTy@O&WB_}3pk~YO{j@Zp#&6q7zieV^HiMzy17t3clRAiv?ojbN+c`(c5#DvWo zHHnGg{{`c5737EuQe=vmb1XfkrHZLof09EhsKz)FlanLjlM^%O5jF>5A zxY8lKxD(d0W9!D`WY%t2@`;b$oSdSGRDLU?kXzARn-ep}**_GyZOf*V#Q0>Y24$&~ zgqY;8=x@Al;Ir?i6>Lwr<2U6BAHUN=!&dOp4!-99;M;H*2oDS}CSOgjCl~ ztc|p3Ly~8b4V5^|GCn?K{f6(D8Q542W2IEwsY_#hYC~LdN^;7E6jW|drAmnpkLdoC zku4ia7+|?E6~(sgTQ_e=+PonlaYJIV)7!;36UL9v-OGWulvH1*?R}v6;|^uBU>uQ1t>WoB^rOAvAEdK!rwEpp}!Pz zB_?c%PsZHhLR0S2F|s9z8^okZFvZ}co){I=`Vk`=7TuPVfI&7!$4A9#3ZG|WgN9KO zrNyD%wfRSkY#0rCOpJ?$d~vbKVb{LS$cFdviHXs%n}0dZ$d0kIw#bz48yMMYKff%p^}qV?H%>J+c6{UU4;k4E&)v0W zOI&!!+Q4-oYIVwWw+%3|Ip4Z9J6s*6(S)lbB7=i9QR@TO=RbhRru(0RaqC0kQYohRA`M(32eW-CQcn6uStxLiA{v<#3m-kMQb8aD_C{a468A(m&T(`6dt`X zIyN>s0`(%)jlwZd#Jb?HmGum4AJ&D(s$xB2j3`&eYBXWNQQ_ecu}Q(|*wDyO&u=lX z{jp3F6)VQ7V#H`M$`qlFj#?WNA0CVnKxBv}LLISI{SxDv_3rHnp>awvP8F?RuZi50 zx@lXaCR&X`O-yi1a73`iyV$@}+k5qKQCOQ8QRz{qi3qvox&D!lZ;FYE2#$%2R7Yuo z)$L#F* zjQmDyWN1W`MvWR_s5;nh1;2xI24xSFR^y-xl-Z(Ci}Sx4H~3>#>CuQtlmH{bLqfu# z-L-*n{Lsy8yjxx|YEJ1oQjFB9xBbK^&&+?eRTCT*j>QN?r79#Sz-OiMnf+Mk!%Z6{ zMh&=YeIeQRzV_BncZUUsqCyfH0*gVN#NWs38yMkCq+8k1Q41B+$e?g6byUpy(C|og za7f5H^dGW5(9g%C8RQoh6d0v(%Y$N1=-TzDw5ru8$OVU~*JFU-wWz^Jp8R5}Hg2)o zg~DTaj5-LFJJia;RADys971wGU%zGblqT6d-!Hn4iH!;kULO{PDqC1+(0X++ihisK zDde+ZP$IM_YL;r1li`tJA?gTJrNbe;7&0Hu5|ql%8L*6BHgT;-QPoXMu1GOrBs?-S z4CSk zFu%8AieD^iDy2A!HclxbD)lEZ+{H4h(>;bo z9U6p!;F>jlsO_x{Shi-3>VDXdDioKxziV45W@>Y*yNIduIxKR=%}D0lh#`+9i$ ztX{6U0Qy;9MIMwIQQ``7sY5+OP3Sqm-LpSBU%A%TZ-w{$Jh4SUB};YCab>ossIkhr z#%GPM=ahvHbAxQNrts#g%GoY9BNl6 z*JW)0XjFa@ikG7QG?sLo7<7W)Mw`{ZsIdhgaHtL~)B;!fU|q|?-&@_s2eeK8ny8C>--hTY-{~}*Q{9SztVU0nj`#CpCN1>Dm)|v2$&#> zjJ~T@tnl++5x6qvB}BI2$WTho%|g6ifPZixmc-Y0RqT%#*tF}_>JZdKS(Q}Kem?8H zSNM8Hw+}O}1&4(K=i^Ti{Q?90VJ3bnJS+apxE7`k@B`f6$A1M->jDE-`>afR0&z`T zYw>5n5=I`f#y@a%ppRGR*ZL6+ELep=>=kIm$JZMM;ZyrA1Dl`dZ}Y?I_^(^J%4_AS zoxfpV^T#If@x>ynS+#n(`Yr}GF#7lfVD8@DzIA69*!-1MUA@YGMbV!a*zg|HT)8Uf zK|KST|9aomt385l8e(7zSRJ&&tKuaFHg6ws(=Qm)hd`R}HF|#* zKc7ad!H+yNK^OSpoVzh?{19585H1G@8q_+YQRH-4A;>CJM7{B?{S`uz^57c|UZ{#L z|G)jX*;b&7l_;!wqq|i?9LlFLsHR5Cz8K9PCD}lfX4eU!{MO1kAp~{U^^k5I%Cl=B zYyc{@esUBFy%;?THQ!j&f8#No1k{6*P!~=?jd&wIaJUIIWAuO{Dn7~ihU}V^Gv#^X z$nY?4^|T|yBRYdY-j1&;>vVdZsGHXtj3(U6*=NTL`jKJ1#b`8{Rpw3doPKz4SZ5TC zQ|2)f4o()c1s82o&XIw^5xv=DG>hh0i`8TSMzkc#;-F(#tJ50{qd<+SMqOr$rzK9F z85`~+*r+m2S>SWy(4+YBrfIqgI%YT{k$S zH|UIJgNbs9mT9xaY8|y$MlH)MVe*6vLJw<=MuW*>T%eG^DvpX47fC|#NxqR0oldJa zm@Sajyu$1vngW=YnSO~m3cdzt5P zIAkv_i)ooDP@Wna85|g-CP62fW}qv~zzRaqW$zxvRF@k!L2N}>nPydisZ^S?@qe(Ow ztQK|)(fAi#Sj`mIWb`yTF|Ntegiiq>VHxSW*ss;=bifd88dD^)kXm5_zrtoT!490} z(J?4>xojJ}&@(tZJfwqoz^hF9aq@9OFfk4X&2hRE(qXoa;~nv`r+miPdtsojZ_qs` z2!I$jV|k2bEHv~2(gkJ%m}fC0nQ4UhVE22@^Pl(k_6-h?XmvXFXoP`|nZRx^j#^Cy zMa7Hkz~g^K6xjGOdAjFZZ*SiK)Gx@X)r=3S8|I6(Kti1;qWS>mu$Y! z)6+jVg!Rtee3A$&3~w$uR9@vU%)$Pj7$!z|hEu zmZT6{VHQpOMy#m9ZNcjTH#f~Fxv4ZSd1CO=#oqqD!I7a6Y&R1YS;E428YQrb#tHOi zzKmBi&YG=CkD_sfJlcPu2MQk=92(J^Oa{GP;lzGYlmj#8;1E{X>2N4|MQ8(Z0B!&I z-oCzpfx#iI(WKXr5ky7*?+5hn%_*Ix3yP2&V_l zLp`JL20FdPpcnO`!LIla{DHb~I5GB|>j&eYUN<~~xxjN{GEj`lIBh{ge{$aIg#mnS9vG?7 zK$nBlS>MHT7yA0K#fJ2HY%^?B&QH;`!36mbGd%QjbWR4HO6Mm}&i0+Z)Qg=lpwo?D zYpL|2$)>E0PH%>YpcnOax-&=#nYBods2_w1tnjH=BFxgH1&7W&0$HZDSUoLP0uv3C`}@!K4E79y zes~yRhK2&4Vb*Ce0H;l=DbOQ?VL0R)0~gNHJ?IC9xJ#8;>#&kqvjO^Eq&Iy*KRkQ> zTz}ty9$N$>XbsR3LWezHpT9wv5 zf}R&UM}9bb@j{j>qb8B)%ViDFfSmj)}qrIF#e(# zG;7kXmkno0+5>MgtV3D>L$v5epaLxFG?fImO5;-!lo=){hT)+hCMd&nAy5<&lo>77 zO&pnL`oIKb#)@q)gmu88sEkNZ5V(i5%HFoZ?imM^1Z5ce4D#vWK4xKt+|{B93Ce(D z2nvU70flIb21}COE&HRWIMla($_YJctlVO}0t2 zLLZi31VT^}B(Qvim)65j!3KW9@dqa2L}YGK}&B0~^I=HLeuFJ>53TmpPN#=D$b z)QS2TnvI(sU;R4I6k$Ldm>k0PH*&Xg`l4QC7=YTqZNSlhNfQ2C93b%;hi#1i)O?X{k=e9v4rg{y`PX;a^1oRFJGsyt6sFScXo_Yza zqJ9GXF@ad*#cnoQlpaMrQUyehe(*9%8Q_Ps3MZ^tQ4eAU2ZywfNJMZOMUsoRI_4(~ z-93E_-te3wWK`Tb>>Q-ndpfvHH!rQ#I7)qCX3S4!&VSz1hd7Kqic^Zh>Y-B^$FNdH zvle`Mb_O@c7=ieLd4du7iSGPGde2@&E_eqHsnYdh6n1Phj~Ee87C7Nj>Ep4?{3)EV zzVjC@^nwo#cUboYAL1Kg>t%chkQ5zfLGU3z>HYjt4-*aD$Pir>!hn+c(s~eDO(Te9 z2)ZJp4W@@bP#5!)OP}{$ggt3Ta7Z(Pjgx_5RQhQ|Oq2FXHs}o#XMSQmcj4ltOT9xl zHEF?y_>ay_qRxRSGWGxnh>Kt(CPwJTi7>41Po}7n*`v-LW14vMYm06EqA%~5+VJK&T-tYze@XWc-`+ECzNE$H$;w}vx z@x!2pAwe9wBF;khNKdrqE?w-?!PYncQ-k3VgC1*%Z8@wOwhy7_#m<%VB-q1T@M`$mYSKu$gpW-7>P4iND+fpONJ%!0^`DhDe1`^(-V4crYAU^kB&`DFg=+=tQ`}_=Eo;&lT1(M?4vm4 zSSDqPulFFebzBO=>SG_AU%Qcn?|i; zV^cs)sis_Zho=MSiQS~dxq58eCZd}Oo84iz*&N_?f|2-dD-0degsFbL2Tz`o3m6a^WXX%anhM0C4A;W6c9Ung5!x)GxVoY24| zk$CK0_W4P>G6uw5Hi-;nk{gSf#1zEI8R!ZVmJGi8HXZNs6jw$F@w8Mvu^y%zlW9S!Jb6|+edF+ckHc&6}AVV1$95GtQ zZ5YU9n-nJv(++lvx$-Z%n8xCAC{LRc<4$@`;x`wNp%{k;b=L8*31EoE6!?mTv|&%4XXb$!OwUqJz%xS`9qL2a zHDS;(LW!8QV|i@$Y1pSSU#ODI6JVajkR%gr_$>|8Po&;U4DGCfjj~7E#Dr}I#*6=_ zrojQW$whYH@jn-4(x$R4lV{*U=~q$jnHyJl#R+M0(i_i(Dh3!pNj|r81jBaLX9DFu*HqL!9g_%N^ zWJG#m?i*5aT+{f3)y=1x=*K8jnSE-^ZeQY@7>5e+2byMjGBPkSf~?;(2EGZ7=jL;e zZ~8Jm+XS7>#28nI>B&$JTs{)`iLp_-sj&&{XD%<)GX-ZbHtv{!0~9ChiVMLXsEg^z zU?1Fy-ZVaD8O2=SxG@JSs^u*4@OJ^7zvzW)hHm-E;5TQy;4TAtk ze@n0v>jV8b$0sJ4o|rB$D`#z7XSKmcVMWF;k#SfdD4+-%T!sfi0+$3x8}gHxAsmm8 z7@=nCoEWzvri_k{O=9+w&y;Y7#7?f5vN1~w`us3?145&|M( zc#O}{IZceI#*m=Q4fi7J$1r1KqgK(X8W-UdNk2Ajhm4>X$L(}y5EDXMnV^g^+eX4? zRapAR5Je`TH;6W7AGbOXtT;g|23f45Oi&CL25=TM8mtCX$6*{W8Ys{1%4#1)&x@TSLDBXMXsy&LPQcKD6*GVmZIEk;80vvoN`lfmVnsg3GvXMG z10mHhW}U$Ji(=5ONs9!fhooB8-Nq)yp%{x}d=x6cqE1su_@XDR4+#oBq>mqwzz6SD za+>>xG4LZJ(fs|KErRr+6}tyN;!^~RN#=c50IrL0-?a|6M^dyE14WTXE#ue5qme7% zH_PwF?Odo3i*En`62~tA+>I{*;76jZ$X@UZU?0jUxUfQ38+kkSAu=EQ_C~b46*pTL zW;^c3@M~clBNad1l7Sy}0ckp4qVa+s&S^p#1-upb<@lli$?-EjMfkD+I`|MBcndqw z<#y%E1^Bs|B7BQXxrW<}yRt19db9E^1pJiQhq&V7ml6Vm4ayf3_(g@a1Sdl3B;4pF zDqmyZ*BQPp&%0)gIt=2RMH(p%&$Gl0*?uoX7|94T>>zKFr(|M{VAevH0R?@vu5!q0sK6i`6oooM0x( zC6>pihgBT)#1jms!TcV_F_m&x3z##yH4m;Hb*#HR>hX zT^t7oDcq4@hLgNyvtY|H;T;WtxLy{OY;|c-r^h)7CX8`7lxS92W*~^tS-_~1O|Q`j z98egRP>Db|R_@yn+n_Qi83V2xok!OsbmCBoT8)I#l7|IqC8?K1g5Jb2+~0Xve3nSZ zl?oT#grWqcx6x%7oFZDy7B9>4(K%poJGf#LB|Hc==O)EvAT$k9d=~;}v&Gko+dr5)Zz~^(X!mUKnVC{gmZkut3;9ahGri# ztleT>Wu9S?l$Vk!vqS0C-S_eZDqg*!ylOHp$Fw}mHid*npbQDgp(FDevt;7MF)N|o zkaiizRYY>XD3*?u_Q5`4Arn`rbG*zP<@Q1C)GZZy6)h5IRpxo{S;iw8P^7$$`d3dO}MvL9UKEmC5KDZ86fH2y_V=SgC0i_fnWXI9Hhc*f%z) zXq;1oLNOJcL4FpOag}WHf+PVZbVV9~lo3$~Q!ZPT5;W(9>t-_f8F^97iY$aznLSL3 zR})!iMY)wlW(@=!xZ|LYf{R-0 z_zPF>Sj- zO;3gqeLxH+WOT8DtTLfI&igUk2=LO8_h@dhOh#W2Qq)=Y3$zGY&d`GQAY~!jjUdAs zg^W|;6!B+XnV**d=0RFq+p?&N-cg~2B25Ng!xTs3#UANBjLQuZD2n49)xz68RGi(t z_@JhXDS*_A1$g{L0)g`uhi4fblV;_TzSyt^sSYp3K&eJUS z3-`>T>2e4LAnz3?ak&Y8RG%Or{Vm!m{R~E6(Get07D81hRO21x0NzVar2;pKG=!ye zR&$m4Da=qK?=+O|MZG7=_M)B_4+i}TyK#DFS=xxsEK*8_A}G8qh0jD&9_#5O`D$MLn)Y z{*4WzfKAldpiNGTFQDgb{5(l@o?w#{kvw$#RKq1H-0_R1IX4$R4M0iyS}$7(pALNP!wb61YfG#Inde=Dyxz&Yk<7o!!~#=lh#o ziK(sS%scP%JKx{?JkRerzQ1j?BYOVCQMfnD3#B_~f1S$rL z#8ySD7@J$uqF2Z5`~rVFIgEP4Oq{&{PpxlOEnVYBh_fT6od6hjC`Kiu*WUps<6`Pm6!oW$f=1E03oBUAoZI$ z*=W^m9x}Z`7okSGqS4!xBa-`uZd(okTtcvffV{9q@Jqby#$V~{;)m1?W9O0BH4`yW zC#Ds+MK3Xk&TPzl0 zMyk)0=k{(Z{%ZI3mhq?>dAo>Q-v`NpWV<@6P%gTo5mTZxOQriix<5lD=pxiD5vH39 z6R91LzEIk&qfejdrY3RPPe<`#pq1T}b9ml#fNiyr_GD;3t*lt6frX%3hIKqjw3y0G^99B7uCWlO&1Xls2%)x4-t}GL~_d;^&|)l zt5mXI?=~gpN=s5V;K~SK(V4sCT*bWY9~y(%qH> zw@ZPdak3*Ok%30?n&2zAy!4~OAq2E|x6z(pBHo4n6%NG< z51ZPn_#8C=sNfq1D9}>@dEPX#S=57F;xbdy zG1SsKXmL@3s>BE-y6oZ{U1J(*l#?1Ffo{50EF6STT}2`qhH62p5faIT#~T()b>@Z& z957@Wk|Gl(GLH*n9${C|;$)|Bqb>yUQepB3^F6gDC#>iLh1q;_BCXXvtgwW*MsiZi zVjcqlZRV&Ct&V>goYV-_$hWuO#8r84xUm@QYb2)U0qw{m^?Z~s9tTA`Sg zZ84h~mOh}D#;ATh7bj^ihE{yn9x{T@>qWnQI zBPTDCRgT!*BC;z{!(5gy4Ye*eGp+bN4V2IU{Hp<puHqqX@U8kZTvxX42P^Y$$_VfhpiBHq^Os*>{x1>D(;I%c8vbqh8mEG|fvb3Qf z`vCQdYMEM=PypR{i>@05Ol7Hn2dNTPHo)yI6OUmPQJz(b$PFz8Z5q^ttvu?DDL^5y z6mHJ)M)7dB<)SySV)Sc<2?*lymCM^r$8m;JlgxZtGB&rSMX#RQ*#-V~WIfyT)3F3y zv+{kjZt0w7dr*5LrkD*9V?%EodUQI=%vDYAH7PWcikrdMG)&k+)?e^96~N1n0pId8 zgOq4`Q1Z?5h$o3Bj50Z_Zb+HLv|%IxLCjJnWqz`1+nCmI5?U>k$&N8WOuUAh-NELd z-Ocf z4&P3oYjuMBp-f`Zmw6qxp&YRLdRBi(wf)WhbRXz#lp*-LoER#OTLic6LT{&!UwG6X zK!Y}^Jtq7Wmlobe`M*2eK=;6hM7`-AR!MdZuwHE7^pOAe1_QOyJKVGTGd(GjMkT;l zS?e&wHXRst`-W)HTkZw=GhKV^?!ex0_)&{|P-?2z>-V9<8BAr%y={G+=`}ZU1Y+TT z#gi(xD^6@YKK(%7qPZizEh-jSuVkEq_>#M}yyf_3p{MhsUVmVbozBP>e|+XG6^bZ2`*{oe1*^{lqtvAe?wDKkDHr6F#2x(B;9U8VoC-2l&m5CD7c zmjDlrq1e5p?o4lP?>5hCQXtUXGOlmqvtR56d;1_+=$gs`-2r1Y9)GK+Cp=4~`#-ur zLnR}o%*Nwyo0|*r{M%4WL%mx^pFY!1GiwvNEmg^yn%$!gP~szMp4A@pzRrYpp0Niy z^SIV0_vDc`D?^XM#%azKuJlg#5M)+&wzEsf@AnW6bs=xNC3k#hvBP@rb{0AVf)n^% zkpOxQdqrktrxJMUS7Hb}t z=b@I$1gL`U*$hFg-a~5Zo~FCC+%f}zLYz{hxDMqsmAiGf;*Y4Qx7ok}ii=dS36zI) zrd2QweS!lN4kHgKK2)f11syw- zHG0J~$n(y3iXXrWDt|M?re=wedkZvFfZB|sQd4xWxe2GFw&pAyGu}%JPc`0iY7paA z??!yL8v|NEE*bZ_Joa`|^I-bH{h5X)73XIaw>l1TxzQ)%GliK(eAK$n{5OIGh**s+ z4Yo1YFprK3aiEG|I4LN~0N#8i&(v!-?M8rdn^dfFvk;mJ>;~eZi3@hXTyRr$&{0}L zSeg#cH1r(lDFdY#Q)K8(F zk?B&Pu3)9zNl}@KA_X8px>3YDwMKoeu6JR?uXcm7)H7tRp_r+caBS+~d{~4L$^AT6 zCqzW8DYY6CYMDNXt8b?S(8$< zZEE0As)SV!a2u#GUQVk$oRlKYMvDPvv7Yw_9|!+?=#ZR0L$ttI!TS9O zSU=$W-Ury#;P+huzYp*K_xJyJJoQq2T&k0_()hRE{*~{6!1t|R6Lgt@3#$Nh_Kh!p z&xJrfl+k}G^W7o-IuQmGwh{jzxH>;gp~*%%@-OUy!nsc1;+2Y-+uc&133S# z^=|X)Z*!6X&|#pM|NQ6PxMkg{-vqkpwbyRne)aZ^moI((&vaB0DG;c#jrebT=^wrQ zz3+VM*WUfcd+!-^FzcJ{J0N%oa9xOIfJwdn>Z`AV2=&VKn=f7d^uO2HzFFf?-55mw z>6gFr?eF~RyTAPQE=1cqz}#D3Edam;WIde)hw9avZvd=y1SU0&mE#QYK=;zg^#*e9{w3F4PQO?&nBYgga>TOvZ{YF05h{_|h@5?FP>u>m~y`J11+3C7qP z%*5@R*ZFq)^7TL1Ary!$$xJm{&NasW>&-8I^Xp%K`^~rB1{MFcS8l&?{nqQR-gxc0 zb$$58i_iVGLx{;-HHF==Gy1bHy!Gx^-}%N@zjXWN?a#gO#`V{3-FofSo40=cv(LTv z(>|d*bJfg7vydHj|K#RZzxkE7zWl|{LvrIfRbGD;psr_c{;6`<$=02zy0v<-k?(E( z@Qt^=2C(1t>%jjr39mD%&%F3M0%@76`t=GKLd|~vcW>VK#+SeFDnMfZfPL=vOD{eD zy$HG8^VK{qe^PC$oB!_m7k-hM1TgHz?OQLt^b0~F!xKMK$%~K5khj_TH`jmhI&*dl z7`w~2|3Da(OVviH)%<5K+yFT5#*OE%{Q5c}GfS0Lw$Z8n>QirAfBom4{ig89mMZN! z;dsfnpL^xXjUV=j(p{>MpPDPxW4B-Utqcj}OO<{%U-xs3JO7(kaETa(a;}_B?U2>J zRN369E4hoPo2Bw5T&h&?_&!&eD-W@0Ih$M@DsL5uSSHd%-%4(tZwD|FZxsufZNBf< zcdR5XFZKc+X53nZyjL_=vIDb8b;y*T!)$CWvpX4c*(D2s91^@vDZ+f_Hpp9%b8#c@ zWVxQ-=wVpJz?Dnxn9Dx)PAo>AOC-)*sb40Bg~00+p+Q_3KbfCa0>19aI_IWG<_gPV zc9ij8H}WoRZj`;Emt^BRja8AYzn@aR*EH75A=aW)iU^=-d5#KOIX_k4%2-ba*rW2< zM3&GaIf`_d^PCjSOLkQ(p=`h1)F#)mLdg7uYbi;9QCHThV%~MU7<=!oF_N*_Pt>@= zW2SoCOa>rR3<20o^2s!@N_=PQo=(*HZsmu5!3}4LWZJ}d;-xZ#HVt!bE*=$_wwK8y zihS8}+*p?2@om+1b!=sxACKY#u(vs^_WM(RIj)ZjwW9C1?_T}vYoGb-i!Z#qi-Bc5 zpL+Hx=vSAYyTaeIpS&n|AweyjJ$v%ZnNz3EoqFV#{u-9L<6#S_Zq$Bq^_Aye{ka!j z{_OJ;ynLXd(wDCQK@R-WbI*d)M0v22&R;ll>cWMSPo6#b@Rc8mrCMa_5F5Gj&ZQey zUi!>)FN4fJ0r#@5)xpmPt_%343ul2J0&&PXEm)z)&pdhRku$%kGQ`P`-;y0`e|jO&NjT(`9HsK z_2rkZy>RWt=M93_l`EgpxSzgY%AE(p3-q8fr_P^$;@FEnf}@g$3R`Iu>w~|z^)#4s zz{u~R)q^5`=@aLfB%#lPqI4S6wR4|%?4gVQrT|N|>exy0HKIAYfBx#TuU_I{2J33g z%+Hli2@&nwlK|-pJ*s#1hh;QdG5lgCoVjB=KN`}uufU0hUbny`Y+eTQOhZk zuf@&6h|#*ip>p@iT>kLm#zVv258nP zV6mw3%;|HFKk>k){_98_RnatV0W7>iz5E}Zy!71Dm!3X<0i=3igPl8j^0Cvu>4>A2 z{Sw@#h;H?pPo01M(#4Y(&zuB!>LRF2$KU)rIO-fpOl1@CRw@4TPdxR>^E@xQbK(5) z6QBNXVyR^uX~SxXrGm$$z$HMg>dV*vykN?-#^UOPhq7X5oymR02SVI4EWYVyUGq5>{NX zRDE-VmURGrge5j@SSsJQFts{jqH;}Rq8_;gb7ed#yF3&xjJfo8(qgH_F6m6HK2mIw zVyVThGU7<-$!2`#zN9GFm z27c+}wzWxy4SgHaLN?WA-B*aBAd@6#_O%!&ohB-fvD;u6F18Q!|_mX+>RJl9` zvS)g6Upy6isV;p2ZpBmKnbZWg%OnQksWQ-V+w+{bc&ZY4@_5M|@l@D07HwR~qqt$J z#@?R=A3#hM`!AN}L<&ZVjT_7Bb_y!EYIf-6oRDBjHnW~fCaSoqpYfv+aaBsk3*xG2 zC+dr!3|Dn_xoW3FK1=@d%csvD7bZTaR1@5#)KLJG?iYj={{aC7^7i4w zM}TPGe_&yL{@~*5=YK3kwa_YWw735D?BkD|KJl@W;CBmQOOWR2`yT}LN8mpX1E~Y* z5r8@O9X)tpd3hh8OfzSG$LOieZl%)M4uA2X(>rE^2lSyk3Ie&;t`lw*24wBk%XHg#Ga$T z<;?Cobl~vp{#$=3uG()8+Wldt{TEL^c=9BGX(x6YKK8&vAA9WB{fAAtWk7un-FImJ zzNPyP&VTZU(o?Z$^t!`(XZycjedzf8pkCbrvh>i=LxL3oFR3N`o~60PBMbA7{fo5p zR1AVNi6MID&o4dr*z$2rk!K*B9(_=a9Xbpgk>K)|!IWB@KJ?Dtz*DWx>Hr&VceuSX z`ooj+#~yxS0_gPc!}l*ATv`T>6}14kd@w1Y#3?>lmsC$qe)xjM3M?&|-{!BXdY4(5Y# z>jy`V9DC>ipq8LmI!J}UG0o2hPX3o|anvpiyZu3%Qyj|w3RJa+9(fRaDUDwMTnn?y zzZMfm?fL!gpxfW>SN{n!_Q=tFO9%IZcy#2x`Gtj7ek_jK%h4wiy*k6%_a0onPX`Db zWSK3`&0qXIanv66b`Gz;;21qt5y2)@_mc*`xh63 z2fp%Caa3$so2}%Hz>$Te{nJqjt$&X*Wt0an#P{u+<`vOdJ)P?Pd?NS#D_V413LX zr2r>g>hP5SUNKedeOt|1cEtDn`rc|(@?xWy9Iavf42%}6nC#Kr!ca+9Zr%~9bUN{UV zD65kz41IO>)H~)1yF-=G0x{Lrbdv+(0PK++YhN&v~pvh*SY!p=)dzwU? zc!sAM!JyKzwd5aah1%6N`UZ#~vRc{{8_H3LXOK>XE7AKv_Ksl+`i8a6Jm()d>Kvp5Tar zQ&UG(SRZH1A8*zBQ>Ved(m@7i!N8)fGkoZ5gL9m5pmPq+1B(S5);T_&1k#K0=lQq* zP!_d6$;U;`BBk5~K0b~`L{^bcfSsi?6Q1JiginH@^|T9;GZ_NotDMcCGa8>{9|;v`ik#IG6`@wvh3%4;ddxFl7A7aJrFlYoh2QcLmH}{n~0xvmuq{$4-s&89oV_dvEm6>$gNCh6LzH(dRbpDV^H4^N`DK+IpC>@Id#*=nQ@>NQDcCTY(J!eJ4 zaW@H|1ISB;$SryFo-T8fBzO6c=_LNnb~qGMr&5knaE+U|seJDbX|m9?{LmVxBurX5 z=g5~#A_$rbF6e^E3c-5}Z>+d?qaMM9sWeyvG=^_kfC>EqzFW-&%+gD&Bmn~gF2g0O z9PJJY#<)jtVq9AfrIN7(`Q4d3z2v^U$7G36gOz=JoG${J158gklVs;eFK{e-jKWb;y`cxdqIyt5jM#=srO zcvNkQs$rf2yt^5W=w2f3Qj{sT+yNQT3JKt5wY0JpO|h!TxZ@`Nb1fx;>R{(ZXDyTx zqCtXGX|g-WsM4B{qFha`CKBN|*imHH=;1(MduK^HiA6K%6j~yRv*Urj0Ctk=!QZRf z(To#M3r>PZ#=7*>lk?+pZGAl+VdB_pXVaNH8G~fLpC4CvaPd?m>Ski8L^|iIYjE>w z>P{pTi#Wcp9nf-p-~750U5&@1NjD8L4~tHZ?83OU8Rn9Nen(mTzU_Z}TraGJqtUd( z+M;XZ$ivSYnf0}Bif0?6N)pfE)TFhce zKBx==K=9mvOS}s85U`@4i=IAsV|eheq}XaA#(Tt*5bzKWPOO9XQjm8nYT9tcl5$J} zlbO!gnIdahWbQp=Dk7HTuZNN`8U=EUOr{+17#~up#=P%{CB@g1kqG!3u!;=m47^)= z9rMHyAQDUB`SY+rKgbkVrHo4Lxvm02%#wp8h1R3O)!?D$GQ9tmM?~>GG@e+Jx4xQ4 zr$9MK`e`wVEnWtfmzi8E%iR)N5KD6IMBH?gh3aP1nCtU8TV9`+l!5Rrz>?y3nNYY2 z6U7u63dXoc083h1PXN0`pHn2F>pl$(!Qh|<%Y!8qqS1I*&|z`5UFMYp--cI4w_!=y zNHoNllTi;)ExoWnQt7SXE-c9pMMA*Z#8SX#`TVMv8l;$Q9-3HEW+e(g13t>3YEx7V z^AyCALhz(y3SL1`rrgR5d^$zGs#sF-?rJ2-+w8<$Zt@0dDUE>kfF)&Cc~YS`Ym@#k z|41xxSh1wUN<5B0KsYi~9u5Sy14|l2Bg{7S`yPux%!dc4U43;sl7_noWQKLbx^&bN zv81)NXebeld&!uWgqgwf9kC=HTr|lmEzXeiNyv3FVoAvl!pUej<0g2jY?ocv{K`8R zi$xNdWSpl1Tndj2mNW<@DCPU4^TN7pS1f5Y#M%SMB~I7UnT%LcdTlk7NJS`5*HdY+ zq{M0{o(Sm|X#nCt>mMW%>k;?}V#EB9yBkhN*20l=47d%kB*a2;bv=}p>=sLkuf@Y$ z0y&rxOG+4``eSKwp zRV=9xTUmn>!IEQ#UIwgGEd@cy;>Jb%I6%gR!Ht=M^`b z(u8sPRYy20ZM>WK+)%QKu!kiTGrn+k2muF$R6a#KaU?d|)t)}^3Xug zRAaMN7E5yC1liMLs))prcri)X&gV8ZT4j2#BvR$Yff-6L7fIc) zq*Rh5eAG%zsT)ejR9f*pG)iY}72`zBp%e<2N_m!B{D#gg)&6lt$0 zo8MV_-LSE@xhX+;b;by0yYvU zmgJ2wIJ-6?n;({xi^j8=6p@!@`osLQyOi5WEXj>HUdqjFFy*Ay@Jfq6V&BvdQnNyE6(L~1pBKcoQL z6?jq!H7!#RiVLffVmr7-YKqN+#jCko3I}pAO5m%6jCsk0kytH`I#%dUwCOj#FiY8_R*qq@>$*_1* zarGXaq)hcEI2rGxNs8Agc@HZm{Wx3M6KrLVlR5vGIZN+RWw;+RN9!GDcdKLePKfd$ zbL!p`>~c?n>8!)}PU+{P=4?LgbSK+eyzV--&!p&|SEGm6(Jq?<{m9?{0d}y9^2{%? zjXh|#v6C$DD&I`|I_(VEEZVt17UV`fX_%Fr3-Qts25yq0*b1;h0ii;> zPb9`*l5kBdxk7kVI!IaeoUjuUfo&1-o8Y^)3EbuFRuD65IUd{GD{R7|+03fh0Kv-& zLkBF%?CK2Cmzv<6p$19zOcGBUN3`+-zfx{w{v}H+NO7tG4T$jcnd~-OV=1IWm9sH| zS$e5Rc5IKW9I?9s^qLf>pxW(1#F}S!ogp0|T#vmVI(>D>w z&;+s?Goj>a&d!2r$xKobuq~QJFG?mHo<-GkS36;8(V7jB*g{Ry;FKk%Qs~9nd8@Sa zUrgiiG;lRh&7G8CJOrk+xs+L6C1Fp(4TA!el34ahaqv^qsYFI-d<0$3WOvl(Hsn&i zO-y%oTEKu<2eN@|led<~Lm;)2j6`*+-<)qs3{Z(_Kr^w0n(p~QU%mt;*5@y8djm1L zR9N_mbAFv|0_r5&Gyb8kzBl6qfX(0wPhNdw^B;=}GH=HtP`W{9wE;jo7rE!p`Zids zng^6knnQ8q9S4bKLe6Z_1?Ep*Aei?}xOiJrM+L-y#{`Zy=WlCLiC5R!RIjC}+n(uj zPMLZvoMG0+t=KTi5w9aJEFf}ocS~(*gN-*CA@D(MOZYMyD*~xmdo|UUWk78if zpz)93b|y|*xwTk%`dA1+8cQazg9m-vtFe>7M58?y%iUxG*x7*J;E5VcGVyUg>F~}2 zfj$|Z*`Vo7;!mUV{As_zjIdc^D7atZ;WVs*Sas!dT5|A^_sJt&9g9%v%?6m6NubrcM$Me=Bp>0Ng|qtEdgmd<9C5~U5l7HCl^eWd|)@n+)5l3rI!_K#`4uIB;TlwM1;wEofO@x?~Q3nlw8H+DV{}Z$mo-T8_!J@5*q9indKYl_ zbezW;)geY!ioLGphD};9(_@nZOITmYB7D2-d3jc5>ql?W19sifyAc_>dhrDBpl;Dm zD{ZF+Np`sF$dOayQViU~$vrJzj1sX#XeEsvwt?p2YUhWAjGa)DP1N1h9NJcyw>cww zRGR%?JcqKG0cBSOG5CTQVbZ8iR-_S40nd$1GMRKR*eKMc9!VrHbZP6B zN=ELk<#8S~qOmAeGt3igpjRR#v?pu5vlfX*Lg8>|jR}2fY=ydOk%VkQku~krI)5L& z8&0svy0czGE$XH3tWo&RN*rZppk3ZNL%~zJ9k~-u=H?^{0}*Lct1 zHpULZ;?mNy{eTC$A@)pCm$?eTHh3ploZJ=z!1~IrJr$3S$EGw}00{-%4pJ5yjTSqW zMDxm%EJI-}X)yuucb;uDI@l{Lo@2xrMFTMKO|!ZuWzi%hKg5Go1Mi;0eESmN{2*qB z72EtvZcdQmNJVd(w*$$stp#quwP97faV*kg*p~>0rD9>t<|f4{sCK&$!OGYn$g*U) zs0ii!8V@8CV?#gn6 zSkhPwi7n_!4USje$Rsg57@A6pA<$|xDG{!@qab#tNojK_X`_;uNoZX~d7qJ8lj3^U zuter1!d>QLM}2NX4jDaAMbjRt2ONb#G~uH>9zu}PiEup`t{%TW6bPcq1lUHQ_olPQb6}3r}8s1YI}EJBbK);Mom23uA&s z8*;~;MFy}YA><_&Um>49>`H_ia%M{$*v!11>AZ_*xBWJQQ3b3@co!nv-PWYqzJVX-y#h)5dVBi@aPn6@6O*BnZJE(A`hbHQ zjv5?U2@OalPqV`S;J&=cuwW|_a9FyU@>J4FZpR}jiSR6ss0-(p$YN<|TAUFK+jf{* z*;$rjbo0-X23?6-)E@s&GsVDr{ zgPUg((Fo_uLFOZE0!{-076Y79aK6+RfVEpmD@`B6lHgeivsNtT4VkAWrE4j$xHU2f z_TT6Z>0~$~3C?2?u`N2r%0HszNBj+1<+KyW>tQ6g{3v2DL0&Ev2G1JN3pavH%5fwa zzMGT;XYB;sqGp4cp)mL$+dD@hIkXMEIma$n|QAo$5sm_#NylSQ*JcWYQ7g z%xVU;tBE9%JTl|q!%BDbGDLBp+zI6+!qco)2{bz$KrDNiV5WnNM@&W5R*5=9gyWY| zRdz~(5@E#*rR>xwor&A=ZA@|!;d-)|!?|Ong6Sq}CaGO~JQ7)7Ga_7*BZ9efi|4?4 zipLX?wU8?jo=%y2sr0^d56&D*L|3CmgkwFT5UVdp>s=7glL)tPBqaN=1^!@1QIo!XsgvfZRyNlZhx!lkKQVUutK z&P97*a~?$<_G0u7;upi!Fq^rS`C1S*QDkNx?BK& zy#SDK#Nn$X;2zARv19gV7EpcOdhNnZcV%hC0A^-G=0m%NQBSw9D-x0+iVpgPe6~(SeLhqzFL6dAMi%p4eMYr#$&uz$Mht-@` zfb1iCNO>cHY7-Db9*<68@RO;kM0mxN7+{U0fr)`uCR3o&lr;hx41aaI415ya^h8x6 zoJy!%*mVNpqY1^I1}0y4^6Deu22QFh5iT%2J$x*wq#_vdGFW-GzCnoLo92j1@RJ3J za6`^)$!aSzvw5cTpw+QN+nPFxl{fr!St7gwD5AnaJ6fCSwNBpB!CqPr!TOF;Z-^N! z#!3?57|f}FC#PqinwUosj(8}+dbh!G3k01b@+88mK+aj3|BAAKA*L^Sc}&g@NjLH$ z^fB(2t3cCBgo9I8!*-1eK$xD4SjK#oN}$4IX;nkaVKzlKD-jNchZ$c(#CntX)9A{e z=iseit~3-5Q*aZWM7SDP#+Fiyy55?hnun$;ErAj*}3EoJ1aPV=NTB{zc<@IjzsD-o(svwlazeEV zyKpf200@aiNR&t3jVI*!8J1YALZksE1^l5Z2zDJEqJ+wVVAYxMTb#$Gs_deUDllMh zI!g*r7Hq5AZZanct|u#mGd;cOa;wUDPmBZ|EFP8P7)ZR``@G350mK>55lpaoL^yH9 z<}S{J8DM&PSemkVA}&U3#L031S4~EG5Do2(E->L7_J{K0N{r{h^3n_m$tt)35DCr~ zCl=0YE6za>2K0?ZA^bwl6U=2oiIU_vM-WVIaYq9K=mJQE)KGHV^XVD}Tmx6YC!s)T zwtL8NW=-HMigIEmHA4|L$jEU#)TE3E*M(;##VVSvN{;7*dyI8iCksKlw40_N|NJQ6i7J|MrJGi{CM=< zl>7_tPwB&+x;hR_T_Kq6Iq+~Vk7vxEBI<;MtEjqX$imx;ta}RZTSeD>9OPR?*j)hf zmMFUuCd!VFLj>9p0k;g`Ezxg>!Mr`n36u8|b4S?QQQQ^}5_$*01IBalQ3M|GamP4v z@-bW$$ES!Z{SjwMa`NPRQz!6JoZO4Yd!Kl`N$}ky;tt019wBy!%vA#9el&$)V(R+`r^fgEO?`i%Grs3Ib+0xC{UQRB5-?0C)bjghxxOSaJ*KXXn^_ytN+qZAsqW?|z|BEhMzH;Rn-L<Fw%U7;G7P&`v z?QeZ62Di7I_mN+*qAv{GxfreAm3xTeyqKiE_r{8G8&=qEOlAL2vU$%Fm!oXt8Q=7T znfC!b?Z0RbxqS5+-37ou+4<)qAB$Z1Ti@-T)myA6gBbGb_g+o|_p2LIFRbs2GjeSj zaxKqUn!?m?uq)UI0HyLFY;FkHvcfSdY zUs4cP10B9)>KCSLAY$L3zj>>Mg_!@i5q73Q&6P!ATUmFLG9s5|(w-kDl90iQf-s2d z1k~s)P>$mwUO(t*LBgbMw7=G<3${nl>e-=U_qbTs^34ye*x^;BR|s#l+p@?$k!UkX zld<@1PT)vWc)Ufl@`$O+!{x;19x4vsT?|{}c-e$r{f`T!$}7L7pXJh>lZiqiEUy?? z*1BDgsjPYErkZ9+-54KLmarw6yP@-{*!gWU_}cpH^{E=mS^(g-t^Q~TC#qXeGfG^_aog^&}2=O>C;PKlW?l{`?7);f{F#e<4Kho z0wWGozds}}o-W%gd(xJ8n0ffz{=EZ93F32Hk2olZTI5%vc1<;XFg`i3Qi)#tQ=)lL zCplJ&WiZC8E*fDzc?C3hZKO2r&)6s^3z4D)wmS?ecJuGKwyU*b-nQQqSvazqiA$yH z53qIZxstuE;$Y}7`6Le~m55o7oQmLr>QaRI+D*w4 zpj8&#n7?*^0RU_Pgk^z1%b?RvP+9&VZ-5Q_2US@vrFTWShxa|NNKL@a!PuoSak$Q^ zx9T8jLm-snUj4@ChG?+q5ziQwETi^x)?Q2zQ5Qgj!q)$h0;+FoEn2ax=rHN#Z+)Ki zbrnuY`$YNnKPA=+p@$6vPtvF;o+cY#F`1Tl?EtoY$&MoIkg)6eU;&PJ9XB=Rx*YSt zEy;Bf>XPp8;U9ZtM#D)NC7_C#&t1 z>tq6k$G57w5~!lR)70-CddV!t^pbSCOONV;IlcUA$&4_Tavs(akJM6Y4E*BPUhNTn zc9YE8%#B*Q6|5km3b;oG-n*uw9z`k6M?YeR4fd(0-=bs`JN)s(uHr0pT(*Y!D|s5l zJGBte-0+siUu>{Zli@fd>Dukq&O{?A9Kf`$eJ=Pj6?9?b6zn(Kv}>|bGOZjmIM9}` zH5FadRKJ9S1^{oHih__OwSU6aQzy)R-16WkQE-4>&&T)mJwblwD^?{*Xo%j0TerK< z?iv4hQkk)&JUc!yR{etXKHs{a{-9v#m3BT@Fugix0{++|uGW8W+p>qJanQI$Oztw6dN|vg`R{pLw;NVGRwbE}mgxf>z}nlj%wlf90%J9}ko0^6 z^Le;|O!BHuo7yycgy}sAM^x4suo;X zwejp%r>;OBM4~p>o3uB=5|c}yV!2{W9J$>A_JSi`Cj=%1X?jLf-HotYRhq2(O)nOQbVSc2^MC@vf(0Q(k$ciX>b%fcS@xmAUsbzK zOV)v8I_doNj86k5qb0g3gjM+0*a@AC!V!AD%%>id3d?)zdCR`!iV{&hom7@(i|vpL zE^UJcV*;IBcD8wDZ7%6C2-H!#{wF<%TLVl_V&><}B=wW81JPH>vFRxGe?*T>PkzkV z$5Hcn+*1=*9}0vQ*~LW0)YkY1Js2?>nV{wv;&r4wj9H=fD<-g5&ku}q4YGKMcFkg^G#V6Z(^lU4io!V;a)lFOe!{usZGj6q9R%pGvLDR~r zEICLPQPud%j;<;)mo0qoy0+(@v9G1QEV0xo1p@0QNy?v36usNL&G z8;}x9;wH;e&H*eHSaK9#OX5yqzgZ<|V^uF4*J(m7y0Syx-GT|wU%)}`i{P!to=CXl zYZ-WzbSkjeC_CL|gbS9&ejl-Vm7(8X*>d$oIiIUBN}q9{w4;dXWVVH1pCtvxq4Vw5 zAL9QfRPomJZr!}sX3P`fXK`BY0K8LA9Z$* zt46vGMn61O$Qd~7GEU5rF2v-uQC&gh=Gu58|>Ng@H(I zX55K_aE+JC$&Urw)d%y@%bt0e9tjgdj45F|Tuf}Nw&wuB!Eu$t3psA7u3)E~HO>HD zt`cf_We_3Gy=fs8ZM4hezZKS9uH0N;xu5?}1pNPAE*JVbsGG?GUSzO{7h9rahm?+& zhZwRXT9O9{)DsjjA^VA#$%)qLC|fRW7g_tg5UJ{A|h265U; z%&z0Q)9-<^>Tt7XiPVfz2hGZx0y9slqSyqO`DM50HO{Q-8Q0YGv;11Ze|HX` zhVe+h?;_yrlUhrBLmS>L%lb7lelVH{KjLefJwoOr5kpUWVZl1mVxaW?s(1_Gt#d%a zgequ`d%%n+BfA+e$63yN4ru%vk3G$s4x!$wuG}qPcaJ1h;}6_J;WFl5XmYr~EHmBb zVdobTHt5}HOwgS~2^A)QMr~P;sHE@QE*i^RRRd(CeEUR95-ApXm^1ul=dl-9^S0^_ z8O6D-($ZdfM}hQ()GVzQ=xkNB+jq71w ztuK0_{`jBbr|XzoqI67v3GFFxr%ND>f|Pbv7aN1(dFhVu@Qfutv7F(Els1UgO1;4202@U6^y}G)zPjrC*PiKBGw?$$UjFmv0U5XrC?h$;MUEt083@}tvIIG(Z6CR6eM)p?~TN62){2Zkl%l> zETc_uTd9~uKu!A(*3Btu7EZVQ5Wx_&Iav9@!8o-`kg$X$CQKotl|7i!9A}kaqM*$Z zUG&4(llK>W`%HS0>y6wC{?d^6;L|-|I$S(tv&kwZ5u@Ag08~$#4xg7+DC&yy(R@8G zw8ADOX({7p(-(a>QFHj-NA?_$ggfJZ=ma5XZ@4e8Mc6Yf~3-P-aV#F_2JW0}tsYmIo#t@#|Vp z#}ZU0Y`I|{ZqzR{Y1kSatM|VL?g;24Ekn|o*C36s%?HV6kt#5r z#{Ry=JR`_PdIhmv4ySIMYZdKJYSP8{ZA4F>xc3?^mCBH_7@(WU=YW)?`l>U~WNmPY z`2&T@nu?eNR7yqD9Bg?N`zp3iGgxcdA207>D6@`X-APQ_2^8zFzZVl%ui+T%+LoNG zf1qRb$+(nhv&+4MoM*o5XAr^12{WQ)Fb$i|vdUDmuSgvG-@-R3RN0re@mI7h%u zItvd}qFl8*>CPoOyWYr<6rq_vJ<$DQz4>CwQUT61*+F z^*yHoX$ko@De4l42&(TlHFNVblqd!5*Wi`qTT@fyN5ob|r}k&wcg-0q);M(x1XYGS zO_FZO1Al1EHlJ7>Zy7Lhajf5}AzqdCx!>CDTQR!gKDTcZ7PzId3LX-|HN>iW+FvBy zbtZ6Cw@L30iLXytH$Mtu?_~=*Ef_~IRM$#yP2zgOuDs_3jU6GEb_QOnM(sR|shH9W z<$-+aJ+rsmhn&-lAP-lZ^Jw{n9{Fy7mkC(^M z3304YNHYiICsN15a=-(~I>ci(ONor4g!w;w`CLojwen$pfK8|*Y8JVKpe31#i;Gv3 zl`pFnR-k33dWWHhn@6&AZiOS460#yR#9Ywa9-xHCl2|pVnvRyCG6eAmD+7|yn_Ae% zLH)Llq)Vh_HV653OmMF9ST|0Tg`ZeDdR5s7agQjz%M#L4<$*dAR1`nG4ksyCzdFpz zS&;~(i%8s8$Bt3d3(H3>)XGgDItk8emgHh#C*CM-TTCkBYRgTlx;|1hWgd_-L6+$8 z&oLP=mL_588JvsX-rt3o*QzncXEtgsj+X4E-vK?*G#$0MRXmto;$kxEj}KsMqwLdH zO_F&(IA=BYVjRh28lQE(GP|uS`x!T1M^Ui0Ha<-YELLnU6!~3tG)~R|1_lc-iJ0 zv7;ncD4VFta*5zjhfR)Dmi&s_3LLjHqCA#a@f7tiDNUj&7K0!9JHKu zoRXq0<%uLcbis{+VJ_V~&|tzC4*DRY)Ia+nxx%UMfmyE!*-$p#f^k~JQq?1MyFw7D zDWB>;*pWytbfm|r-_Pfbe{Q<>Ikp)WK4WWdG1hY0lXabA^=>wlPBw_*vJ(9%lu-q54~@ndrs z+~A54EtKvR5lr6Fl~u_}mr&_syO}~IV%is0W{f8`w+tRd>L)SvjNV^4NveXs80JUc$pb*mAH!$uKwutnF+*hv+d`f=PKVvv$iUCsL-;3~M?Ae59lG?7A!> zLp|%rUePc=5n~?vN`y67Xx2yWH7oU--_H$`CBjt+F%zJd#iJyLjN9Hs=2Gg!;GRWQ zP}o&p5_}*iFbKXUkq(XbQu}l)1KWd`-&O0c8OeU45h+Cd1ewwEFUq! zqAdb%Yz-5^XJbn$@&p4^pe&vEsvdeGa9?YFgcFm zu&_9R&WZ#U*~53!;F88o$#V0a*TsuKTLk-nf9ar+b2eXTU(b68H+1Z9Z=JM>rcz7F z_RaiG;ct2$I4qW=SFhZ8v8BbjUVA#W<%e|pF)4v+A`4#*pQ}s<^H*546Y(gMK40!y z1v#H~1xK>A#0P*fP1U$rCUg2)=C`Y%+K5mt9uF&rM8s#i$CU}al5D@pnEkcY-D!D&aOFQ zK&z44P_<6S11PKivg3)hSEyu(nRuoFrGXcJ64*s0-!<+nsV9Vy+F$gfv8~*|Y7gi( z8B3T?wi^jly9a8j#NVE{`##ZW$5%Mc1Nq9MHb}#%0|pMrg>zVC;#x@_aIYJ@spP*k zczyv7OHC*{&^TwI0dWViNO$^p+VkAG5=#k$SwXC+5Kru!ADj5Lv~8jjD<)uUl@^%} z0)guWc+>)mBNaWzeSDa!^GR54^u8C?eDN$Uuu3v+yf%&FDn` zi28m%Ss^eK+Nzf?oH-i!Mf#j-wC$E2c zQ~DW4r1Vips&AZT94fYzLd6axqvre)jelxDJ=oWt=38SImsJeg5jLto9NF3iTZo3_ zm*QNDyPUKr$K5PjP+sZaIBWkLjs*{2^UPSZ{~=6d>_|ue8H;pW4U1B`I)1O`Yld~D zS=!45aD)>)q0(NrMZzL;)3J!PaA4gN?u;4ck!0@c2L_<6$O#QTm*<`U|GIf8<0$D+ zExwi@GxWNLNybqy5eHFNm7n1A&6!mZcW6jrB~>`YOxvS_1PvtW1XcHW6fTw@T|_CX z0LRjdSR~W>CvME>x!B!1>jg)o18Qmxdd40e5P2q#% zw7|kf4`V$vqOY=F(J?xhL$J|UiF4ItQl~Uxc{^>SUNW74sCOUIooH3P)gKvj+jsdE z6I6|g@tpgD_e!+vIy3r7aOM*l2wkD;e5+6%j+QmPGGRe@? zNAziOY->1JLPCMDjfL-$){?*^VCkf>D|ik%J-Jmh{#2U}N} z9b26d@VDYEo9Lbx3mTc<$Kz8P?L7BD>vZY3*hXP1&PG26(bw&)h!x8ni-jVrp|cA= zIih4yZYb5Ps>H0KL@V33Yh}>_nAIzNEU%L<46#bV`rK!zaOM;4wxJN%Tc5N_c=J8% z+twhnluwsxcEMPhDplHuK~86GQzsZ3wvrMXm5!-|(SB>cy(I#KA&J~Pyw(i9I_dV% zrY~pRTXJ-iSkV!@XCZ_90s#DNS#ek)WTVK;|L1nG_3I^4`V!V#CvYz(C&H<|46omj zWic81V!Z;YpuPh3SLi1}qU7e2-aSP|F;v8~LhLZBZoUL3VNv7q@`*k&)24<%x$sSs zfmI+-K0GDkF(RLzL9BzJAby->QCsjM(Ku!G#OI~vK)|K|va=P;m*>k&0+n>@4)JaH zNa5|vR#5clgO0q~rG4CD)JAIve9HkEY`8@v4~>24O;#N>h~k(cmtyP(O$LotKZ&n> zP+qIhUY@V&7YHfw*O?`wBGxLJ2)edxK-dJ^*$ZGWPG}~=aGO3*hk0bC%0BZ>r@Wzp zg2!Bp)_t%i*R6*HjXAQy{+meO*;-ssOp=?xZGZlm8JCU$W6sx`uaBZl7Ek;()O*0> zii76|`P}+yw_HM*aySu%ZNmfE5@=0YAi{aCqqD34G~Bi&7daqWPY;QS?bze7Y!zLS z_H|W)#+GlzCq<_5L{CgSM~7PK(_ zEb1o(%#d4#2F>#7TFH<+)?AxG#j{IqU_CIQ{D~knomb(`Yt@HiR(By zOQ=vb&&5j4KWxJ31>*g67@#99=wFx!jDn*46ShYJ=Kwkhm>eOHrM1NoZIjYQxEt9& z@~9KVUe@k(JGUo*+j*mg7FJOBVAE!dZF%{g;R#zKSh7T#vRagaV`ijuREGK*KIk+* zxx$N)ym&_3i>jJaoogE0=^(x*u(xclzgC)CevbNqlsx*HA-=Ct_ob<&xl>3@-aSix z$e15~5A(shosFT7DhvHx!;V+g#b%D-Fq@0syD{o=U%OtFqCLril#u7h8W;HLnh=k( zcErjdwu>9#!5Hk_Waer(x`OR|jo^u9M`u<>;10i*lG2=3N(;(s9g@6YT)#51K?s&7DM~5BZo>t#oDjMV7 z`_#2o#Wc4BJeI^6YnW&CCH9{2Mp@{^E9*(_3s8}fvyV+dKHsEK;D9X^c60?Gc@=+I$;a=UZ)g{Z! zS&;j@OLq%UXSxuj#AY2KEu~r8Ei^6Hw7u$JWX#Qg(x-P0FK*4|Wn^UJCvtdHrW=;H zn^itK7*mb=@$>sUi-{2?EYjtrq+h>-h~1nT-=^QK4@6fVCR7AT9#k4gE=(5{{)9xa zqb7L5@4PoxvxAVDzo^;Q~nwmusC%#8< zp{C{(IHf{@p3zS}JBy0yN`xKSP~LKL{@UB|&P)R>jWH#?<$q#*Mj<)aJVB!UkJ5l* zd4c5cb*HY7FpVlt$!$3Nc3vH*az*{uTSVlOCv(_{AK{rFUJTOb%x%kbW4`$Q+X4Nr z^CgsmYWGCpA-!eR-13-WOiQv`aw)4?8EC*EOHL8g;c1aFQDc{@TAS7(s#(T2}`|?{0*u-PX{^3I5E7 zmy;CRKBnd8!?_Af^g)ep76I}3{UyXCO%;B|=$DHxHxnbgdN~Atwb_=Bb+!M9m3lyJ zv3lNDm~>Bs&5h3&9}T4sdDc|g$W$Sf07bjXVISmFeL9V()-HZUoB1Y-m*;@L@kz8A zA^doj?#VjUl(%1Tq+oNHEOBG{G&@=+YT>RYEFg}fPlx{GUgN$uQy_;B@oqX?A?O@n zgoz9I{4eSl2_+RIpk#1|=K zWyfEQSpM0wGpcyvsy7a6CuXhbMP>{sz(Vv!pS{pP{bGs{7q`RwDSkCX!$LL2`$eBB zrN$5{G^6{RJ)>iUWl+MdAT!QbcSwAi1a7s(|5*~iitU(qXi_|c_sL1c8SOp;1wGch zq+GRN_>y>A1tp#~zAu$XLzvFz09+-~Do9>1Az|lUy~3D%@uYd{0e|pfuT*=2#ykJ& z-mI>`%r?k(4t6CSEq2Q*%BZ@+Wwsx@Ve$LY!R^iU1!?T{?a^^*Q}f{~Rd;L9TXl5{ zIcZIi8H<1I;)B(e*5{**7GX+U2WzwCcK9VSY8{kNm0;X#-t1FSvARCrdhd{YeD_C> z^HEUA8@QXqHn}wK1d>{r8Y~66EsuzlswCW--Uay_9mr+Vg#@`fjo-wmG*cTTPx@C> zkLVY>?%m8m@E@E5o^y3-HU;IUm1iCT4IQ#%0&tcMurgun6gdDvzOGQ&HBkjHw$aNWt z6%OT-hh0dMx7)TZ&YYaSG_Xc^Gdo!9510YUGg|J*-7i`|d;3eWUleM-U{(+b(A^z- zV^Hw*CF8|N+@+e2QC-!s)L<8rT>L&?y0yLj*~Im=6?vu@1}^h=JA(Jj>2KQ2;TIdGV&SpWM}f+;N;0>C$xxcVuDGMLoy68H$epyNYPR$Ve(T|w zhp`Qh(Vjhm8J-}>2p+@3?Le(nJrAK6xbxeF5nakri1t<@PzG7@Xt3Muz*XJ;6=F=( zZcR`mydh4YovWYRHWtRTN3qV*w8^#EcD3m(!4qE85*;t8cKh3XQ!6;+%6qK(x+}zW zCF93l&3>PZE%2mOLS7}#iPqy3cw98Qy#`yD+7TTgf1*v_RnDc4Gza(Xr~{=rI+`7_ z{h2kbA;C6F>l|R8MKcWKi^`)rk}4yk9WybECHz&ar}-0-v=by5dO={*MsdmQpELDL|R5h|Ws&@q#* z^(ko2>}amfd+-srn7RClJCNQHZDf%b!re4sFFS;;x35MqbaTo_`G;1XGH`t9*_N2k zp;h??V^3v|>NTq@&tf3MDY-uimnYA@dmW`Hs_5~?@ok5u56Tz#8&mm&iyU9F@MbAN z`UB4an>C7FRlG7GH41zV7KekUiY|HApI`uhfh^I@lE&>v#?~{s4_3vbuO(-6X(ge8 zdH8BL8=H6M88e9#9$;8a~JhGfX(D-;5U0ioJ?q`0_M3xfpRZjx^>zlkTNIm`oiS4`9?s2Cmb%CXk_&0$x14+Y z`fHPD9efiut~u=^Sf4{F#MW%VdkAC5^VG$C@{{5ii^ja=;Di1JEo-2uvqNfX%poB~ z2BV)-JwwvNs^%{d!w5bCAkVSj3rb}}?#j-J{AJ7T?QhPj7tGirYn;k9+=>=xVa#FM?r;k) z!&sf5Gp&C_>AT19C{G^l^^+k^;PM2OQtZ*WBun?^zZj!kTzFsmaq;+JSGv~r1IL+) zp*{ML>^g{5k{g!Mfa0|V`LcT$btv(?O4gCY(ps0iIR3(WZ)+66yR_42UR5cTJWBfH ztx89k732NVS0 zlMCtvwm);O#Qt#L)G%5#A;(HJ6M0o9g&XViY{G)@h?ZviFrq=yUQD4hgC4T+Q zwn3En5RbmTp+6r}Sl67m%2|zT28YnjZmQtD8S$YS**<>La{#^x@-pi?J3sT^r8O|c z(Y+sZKi^x3yT#eC5^ru@QXQ>&`bB|Jn1m}+n zC^cQBuW55*)m?NB_{vhTVitUOy|wPpAYoB))o^Gfv0?!<`QoR$3*;|x7(7m~&w}A! z@j^N+UijlMe}8^*SMxxs`Wpa%2&67e>MlT|5Yk_#qV%18fxTnn(xd}eNd(qxUIfbD z>w}F9&nPYZA*HZ~o3a!wbE%czYE9#v!VD?8vqD|r+q}sJ^^}5_^%rn7k9MX{Fhofb9g#Rni>}ckB!Wi$uV?l=c438~$ek_$A8|1MHh&I4H0*=jeF_5Nrt@~!Ga&tY9WU=W{{q}L@+gn$X7end%M~czV#B}Z3cYju?$?F9zl?53w?({^J6=N1NiWR;KCD?3mzE*P?GQ#6 zm_a|TA&t>)JGvlhY7I_5iRo{SI9iD4?qxB9XLk~Y39=9_nK=L5uM0VG)<+j|pmHf^agJHPDme&cm9R65z{lJ2 z`^U_uhO3aP`+oqgVFp{}gvHS)X=v_R20Xo8Y2JSQtC29c!qg z_vWDNn%fC!IWAn>C*am=)WxjXcMIVW%42_(`b#%n9%4{~!ynC6r*uU>uyL+m4U%dGmdY0z>9`VY-OUR%$VpsmO% ziypab?9!Q|V#y{`@@<>K*C1A?O;o$3DOez?nKuD0JL7P#mk;MU_?0e}2-a4!Ej1C8 zY6a^_hbM(8fxRPAeg;^qQ5JXh0>_+10+nNd7}odaq|<4$GAvKaX>flFsul?H+Fpl<+Ya0>Hk$2J3KHW_b6 zc6iKhyak%m3{+7QUk=dAhrFcplR}ZF*cI@72^mjy%TI1=gn6#gUcSy?&MgOF_qGEo zy?$>0Kxopy+vUW5ctOzDvht>EfLbXQ7qZR#Y&_$g0{^Or58IZs)G=<-O~(J}UVU&o zTSMzyE5_X?dv42NYtK^rICbZ%1w`COL|W?;yIG@@WASK-U8hu8jnw97+X%yqQvxiv zsjQxJH?_ybF{9oR3iX^Vm7Mrtk@{n<%Uao@6lWv9Nm@YJ=$XnBY0F)_)2Svzb+qSD z46~zrD)*V?V~|fTvgU4XNJdRn-bT6FoN23f%FVgbb)Pal`8MJm5I(b_0H&v){vqUL z*GA=Dm1}B{+#^jvy|n+ZQcuxgvMubu08wo}3>m8@{mR#uxv9#lz)QTYZ^{LVc3xLe z^3JK5bJGv#y{k5Y0>X1<{(5fgP-OUva)pKPatCOk<}19HN$2nXbH}Za5p7UkRCf`r zuK?hq4re~Pzu~5)@9fVY|dxqL_*qz}e$!t^^uFy27Id{D3=o&&6DSRn}YYhoHegO3uH zyE$f&rNFW9H_nh`v&a#Qa&$vL3@1Jo&d@C9E^fp7| z*U80}kb}{401ZPN!AV(+T$q!88nkZ$=#5;EX~uhPr%&D4HSUtEz=Yi`?$Sr#i@JFa}uePD2oy-WZfq4&p-gD^LD6?&Hwg zMti-V>`H0-2&{`y1qlqj`?f%aunEpRd2rxsl>9^Hzqp6cM~w0InD zb)cJGQ@2~<^Fk4ZKQI*qZ;u`v&6zBPi7F`*7oh5XNt7&$F}0O%ZDnL(qmgDF-fAPr zW%W3+_724@%|AhL>>)04Cg$Ou8YT5ICG$V5=u%^kb>fu#(TL)J$TNP!#fsyf-IYJV zU1_$^;klt1L(e_xY9t|b$m8Mo!|x7SPwS6P+% zXr)?!K|}rISVov!cTVMh&&JZF6^b1oXhp37>E-+ptIb}S08+uAKC{2_LJN20koUR1c>lB#61PP&>K{^3PW~AMG>OafC@j5Y+6qeBGDmL zd)rc#rd0Y|-;R4P-&%+>TLXRmA?~tXAaJYJC$?6#!&|^dj3Rh~wuk~Ra=cDB8DTN2 zOxT((QE(a*_1l+9UhiM{weB&ht!TUz1G0MMz&2U}@p2wtsu(FY7hz}yDR7u3&SeII zLWvnkQ?AckyMpRiDpk}trKR|QH_{eqYgmL|ubQaf%9>~$w!IUHULJb(ZpMr&Pc_HY zN{TK#G0lu|7bvKa~ybm`lWjWK(#v8enqqby$ z>}H6BpwRPY4N)Cl3Sy7k6FY-XeDRk!4qeu>13!xF)`an0QeIi`_O&35tGEp6DF}2B zRrfa9p?7Z_9T@)-V3Q2Fd{8)9%^57=a2!0Mc@Jm`7^wQQ)TVCesC12|#KYeGu66#B z;J`;kf#m>S($c8&;=i1;C(SwQSbm@seIR!ilvh=%M@@fjhs0+rX0FEfj9Ko=@d#c2 zs58`X8oIn|5FDAQ_XIe9TfR6zcL&p9FlpYaNmt7oqBPn6iI$tFI$L%%k@>x@Jld!0 z#w;G&@kRh`FN*CZb*jH+VYyA`AQ~)EG%;kMI=GjDsPdOys-K|cigb6T*Eo+t;evA5 z?w}~%7nH886Ah}K^OctvVgH-7mV)63Jguj|qF*v1PT4)|CftXmIgYexFPvPY#B-UTVHYJWL(YaMo z{ESk5le68KnUZc83P1QH*t~Y17}o0%sDo33Iq22ATTm;YHN0G*b$zet6yJU%v)HdD z6iL0B-m1fq^qCm83Er_d^V&KrjG89%KU@+7YX@uVvsZP=#s2yHD$Y2ut$&6vkMk`wX#`^XJoFZ_;$lhn4@IJ zfY|dj_~mU2-1_iEyx#qb5x6_*r#MI|0We*RhTOkcYJp#|_!5{Ue4|&A*7c}UjK4V3 zXkFhG<@P#(fK>c8F}t}gZbpeQ`S#y=p}&dl3X-VLXGq)=))U)u7oCDyt5$AyJUdNq{teV&_OG!w>rr1)HEng8t2&_e}mwz_< zZ1B;FVjAUBCV~*LuTXtVW+E(WDO;CN-~T;g?y$8K`FYH|Wm)U0EVg?l7gI0@z1LuR zC$L@>$r1L6us425u)sWyF(yBFmTyodM;0x%Uz=tyUb`k{dFA{wxxysfxVI9$GohjCpn#6wtZq7t!b4=~Ju6sgsaVRQ~81kWFkb zfYERmPL;m%=|KU{bgAIloaBN1_R{7GOmkx5k(MtSmW%{{YiYg(8z$2#X5@l*oJ_+A z4uRhrd%E;lBWX>o1V(0OzGurV7{6swN{wCTr_$S4{y&G{-b&YbGu6(W+vu)$gS|rF8b@O-yxLc|B(2XZwwOh5v$?Il9 zY_k9qR8y+c5oyPioMDA1VxrZ7tqrvj*&KeYm3(ROP^D-GgT$e1Y2^|2xYV@gvk?h_ zu9g#5Ylw)ZIrbb=gm6#2_JrrnrsYtt=*9Vyk^IrDlO#gKlPK_XZ&YP$&uqQCAd+SF z;LgIhb=J}cft{`aJyON@o?XLsx%;}F{%_s4tGtLp*IikOVJI-ik8O+nVbmxphgo$T zaui4u5J_BiJv-K)M&s5lB(F-WSFW~OKpML1VtehJH9y%pe?s^Iy*K5isti(aPOJ># zr}d1vU8`tZhB&p@o-+r@I*2lM=7XuI@ZN7>UaDW{R#}qC^PMeoXbiaCNIVo<+RLvM zfD$m65kc3VC}PaNYe$B0A$>uNw7~GR@uv4A)%XVo|KvFU8@H>64`jRZ#~ooCTbmqz zH<1Ojhp~J1iqRzl%Ce66!m2}7-@Xor6iLh+$B#7Mleb`q``!tvk>`;WluJt8*IEX< zV3Qc(Y-Lpo&hXxpT90{c^DhHKPASo*xJ($bHP+>NtH8Y z591pS!D!Ddt=rp*=v#7<#t%x3&9wYt6Ht#ls(MQ#e-1gYzHAspwvV(R;FPh~b4w!m z@kfTT3U8gVB%S(XbJkz9b8wd{IGJM-)%a%vs%pB;iZOXc?*~1@dt1UnBVfHuWsNp) z2?b=^xEkzpyA0)+b;SJBc3mP-+N7heR2d{27Q{cMy)%YnkVw$mzcWFOwB$0OX_!7E zYxo>m-#a^hiJ{nV>Yso!_utjCpnn0*JcWkefRnD@=cUP-LFKlNGnAK<@9v|pS98;7 zrn)l2WO&emEvS0%!DQO1R83Q2kI#dK+clb(igyUEp7p$&%1#CTF0X3PBnvIa(J>)Z zi}{b)V!UfkB4oTIdUj6R!b*>d2{pxzU3IO|(#r~AuKl?VJ#scA%fsW;Bl1y?d?%YuQ1LGJ zqNMKOq)3AL*9gMHTRK29kqE2;Bc<3_bT&OPO>03msQyozNW*x+C6=@uyWq@2L$@yZ zASle^Mrl%GP149ng8WBBX^HplQNQ#vz2a!g#tNtR)04fdy%vUkk6gq=*1;cw&hk4c zWNcvzC#`u$qKMWqj=gC(=llk6*?JkG9?bt`(~LNwz@H)SM>UsbfY(C$r1&uI>chl) z{Dm`Qq$@RR?<_>fsa0{1!#qUrrRqmNe_AuAMIEM3cqs5kf2ux9>HEh4ES9hHoSg0* zR$$H$(Ne!!=j(v&bzK{;DW|==dy>^HqLPx&DF{S30;>NsJOWGqM-|cFKhYxq@VxyI zk4+A;aA7R|x3RpnnZ<5dmvXQVffXt5RMu9V555vB3Ulw}AnA=Ojzi|fILwYUAb_Wl zIHQXRmtrw^%jUD6HzG4TVCyfRBK+4>E~Vm04c2eBkPBp>v5&j3Bl`ryA6OK8ZFIjQ zAykUDQ)Lyx@wp#**O&a?{Ttv9NXhtx!c}~_d83PPP2YZH;-0}6ma&W0yJ8;Ru7I+g zMp@K=iiioAT~<((-~ZLlbw)Lrb!+B2iZiwW1eAIQ5TuErgbw3KM?efUp^QXoh)4o~ zgr+keLTDL!5r{((2n3`g5J+GKDMADcp@ycRx6nZX=#Ag3yS{JknmcRVpLeY}zut4+ z_3m@dKKqn+@8>y3qODITCF~DwL?I1!V#oK)=NI_=PJJh4NK`Bdm1^p{Rn|~#1-dnOs5!PVj8lcck)u?qQ8M2Fvh^vt*u{%k z$LhZ1fx9lsW$oU=R1b&Vj1Ytj+@+*QmSG_A?Cv}dv!@lZzGk_*enO^bExImBcf%m8 zM)MWJbNP7NmCCOGhmcTk*KVbpw_lpc?!HIo8Z!D`O}b=-9$4Dyx5C_t#gf%a-we$> zzi|1v@R`KyY_6>88+Xs7L7ni)!;R7aw>R;GF;#GR{hd9VK4`RXQ4k2Ff=pw$AUKV} z6SMhQYuv>;@gIH^f&3;+4c%r#p>PyYEW;}MY-FCV>u{_d8KaR(=^{9%%t{Sye&TZ# z9&MG=i5h)sL@pL2FwSbJ}Q%=rN6 zRPAf9T(c1e!Ah`(*;+tKMp@s4KpU&CO%lCNj1ns6dAbp{sRm`u5i~!8@6_~^!aoo8 z*9jQ}B}aiChf@16rEsYy?~DIY;W9NH-{3L-%jy_>z~UCC+3pnF3azthvDB7&O6cfZ zmd;@9yTdc+(Q9Q)qtW93UfeVaZ9*3>7_0S%l}6voI1+jC{`mz7cX7#`yu@Xz!h~Xb z!T1c@IRl>FUu7NF4(BrmG)|YUhvy9k%*=V4T3cIWFZb@QbF1w9r{c^cfONPLdy2EQ zUa>&^U>-=epX5L?63*o>)T41ubg7v67uQyaU#4y?#F}71QTS zYB>Y*>JhEc>nGaz2CMzLs2;AX;60Q>r>xyG=N1Oj)0{=5}k+hfLygxvGhD;w1gSn5DM+iNy#Rnn1^@(M0Y4P$(if|?6 z9xWjVEBOzkqZnbA2xLASQkPhjmJS#lrhXmVSlf2-t*%N&z>N;~>Bl;ilNDgjSi>ot zDCSAKWw%HJMddlW$6Ndk~-px`YPKroZd`vG-O5b0gWjtst4XSbyfimQECYX zx0H$+-}Z;kTMDT;H28)2O$3T2Zvs4G9N3yGF zsNS;(9+h%Kj+7iJ;xD}6aHJYg5YByIPBZ>q*_fkmcRqy4G>9(AU~q*jmL$BqLY~T> zch6%y*oL@XLqsV)2=zD!r}bJvJKcgyp{BmUl%B%rz^vTN4mgr$cRYR$e~H#d?acyD z>`#h5nJ(0kL6`hCD@C#LeYHflr&T$XX_^fI%M%>LyT)gd4H@v^s7v>&<rE6Tcy z*1fZA__RBWT3OM09z<>~F+l0q9KTb?y*l8@ z>R#ZAlAkRhbycd;omZa0AW;b)0C!u_JZ^&=o(_vLf+X)Qr=K`+C3M?@2aVk2BV*ER zw`Vf)q%&O(Rdx&FvV$MRXH%5p(4*NB`b~2VHrTO|A}xJvpgzVcS@~=7MpQNu9%Lji ze(v4UqpE2?#CCU{p3$oLgCL)7mb>+r0~9h$Qbv`Em64HUfC3u*i;b?+JylvzkctWK zq*SptH*aIa81Zwn!>-;z0dlDFwo6$hF~_}Oy)mEomeoI6oxr-yLsEOA6jK}NZr$iY z8Y#m22}KvYihA8y=|0!-nl*1~Gr+bx6>a!b&z3p~Nuf~au2udng&VZOuRh;@#&@$( zoqK=25m4+?@$8)s8^(KDpPIZW^YXJ`!!xk?Wrir}@>(ae!gYkXVEQoTkvG*p#h|bd(IYW1C-r#YDR7$gkpXbE zKASdkHM3hQ)%r`7!&+ql9ce)npr?Lmv%rL*rtqE&g3UgE6 zdegNM>98-nFW;2Hz8W5*1O%GY{4(dTOJ8lj?^DAu&FWs|)H6#t?j#>`8uNXQ_++ldC<<*OE(e(O^rTRQ_ z5CR&sb@h65Vj7w_UBMphD95wa8*S(kFRsNJKU!bR2OHx=?=g!#g59i=n%rzkLMP<1 zaTpnlY-}%7QeW+tg65ASy6Cdu<8foNklRV*JbhwRx#!O?r|K?t4w<1>kWy$x1Few~ z*`c$`sZz%$XKCFW{X6~%%@|+YP&(JwYI#Z=8^>R|_0ntW?lCg zfi_3uQ5iCmzjf>@JR~kq4@=jsHazA*s(u|6OS>RlT;E*u{`J@v5HtmB$+8$DbbUmY zGd&=Oia<`B*~IyII@o;g&TWvOrRKH~J4|^ow?ECfdM{!WOJ8+V%xk3y2((K$dnr}V zvCtL8J|_lCaeg;IKLm#X4$;`!QWy7A9nZCDtYx{!4r8`DzrO%A5G;UD zIri+kgUt^LwLR#cC*(U5PIHx~H>6M5RNWAA`C}{oI>*nWBJYdW3|n%Q&<45A`C4CA ztm_Z9DVjw+Gv!4GYKqjB`y)OudC`H#wYzQEMoRn~WBg~-fKA%U%il%+2OzY#S{%!8 zS-${NRU-+v+o8IuhidB(h5-cw*-wPlDg=hMmn~{A!$)E8MY^tUrtB&&oI0eOxSv$; zTFRp(=eWOv?TdZTKBlXVjxDR_jP#qDsjW&8ac>DI_|0RtRLRsXdv{RiN2BSH6lA-B zFDG!8H$NbFl|Stq1cSTOg?r6v+|Y>`0vM&qFDfLUo+;mlcdBcNvl?brL66q9oD!$w zPYb_}D@rA^w|#!|>r4xs#!`f6&b8-#LPR~3`OoNYl<=H_t)`Q#eJ$e#25<<|^`C!Q z-~U#qckqet!rRg*E;Oyc}VT?etZRn0-yM7>u0*~M^&eBrN@*l9**lk zVc@fWRK6$F;dhbC^L1mBBI+9&Ih2$EQK(Q^OLK0xt3)90!n_<7kBN%b=uR1JaH_4M zyBNG%P#w@DajYFJQg9OU)^4Wt2c6Kb31$feDWOj0(V&d-hsa1sUu)bjH@Y9#yrO?q`9Gfc+qVcu)=#w;^>`-)N5bzGes->SGLa@xv6bP>a<~L4D>7al`F9gn z!$;2_?(tA`C5)Sd&jQaVt;pu++CmUrNBTczk`E`a>uZ2|N^+`fR@iX=JB+g9J)q&e z3MzZrC*NfOng=a`D^n&-;hbd+h6fUMCUfu%hx5e1-@){bnQNYy~Uz+Evd!%^9tYjXU(3i%cFq zDAn`cq-JyO(Fg&z zx<`n6)urN?NIkpR#$m}%e8?Y43wAzZqy8jG>&MG9$L{Ztq%ULJ7iths0>R7ItjkI# zR=ve51+}6o#M&S&9pl=fCOje$JVCx#*2|t4U@ve1n)4^p{;KG-)U-UM%~jaw8!hDm zrt7hY8HcHmK&ubYuo~DCVVXL?h6U_Bh6qaX+qg(n)mGg2md^{TjuD1Y<16XM1M{VU zSE_8*cL%^geVG3X1mOdKfChsni(EZb6~$N#DRsQN!cvrVk#1PTnTl?DA`ftmn7-{B zY?AvDeY0!Su$(}2bkZM34Gu83AX2(JW_k2Q$p|FQC_Epz5MQ0go#$D1$mbrYKD@sq zA*#MhTu2{M38PGIwkon6ET1>w0uf!64(xz~H3!0tPR)f^ZlrB`y?=A-gxrHN9q!55 z+f{ld3*bIOFvG{m~||T%y?JBV!!{;AGD Date: Mon, 9 Jun 2025 12:53:42 +0500 Subject: [PATCH 1771/2083] Pin older Twisted in tests, update type hints. (#6882) --- scrapy/core/downloader/handlers/http11.py | 19 +++++++++++-------- scrapy/utils/test.py | 4 ++-- tests/test_core_downloader.py | 14 +++++++++++--- tox.ini | 1 + 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 74a6e54ee..54fef48b6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, TypedDict, TypeVar +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast from urllib.parse import urldefrag, urlparse from twisted.internet import ssl @@ -27,7 +27,7 @@ from twisted.web.client import ( from twisted.web.client import Response as TxResponse from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponse from zope.interface import implementer from scrapy import Request, Spider, signals @@ -286,11 +286,11 @@ class TunnelingAgent(Agent): key: Any, endpoint: TCP4ClientEndpoint, method: bytes, - parsedURI: bytes, + parsedURI: URI, headers: TxHeaders | None, bodyProducer: IBodyProducer | None, requestPath: bytes, - ) -> Deferred[TxResponse]: + ) -> Deferred[IResponse]: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -329,14 +329,14 @@ class ScrapyProxyAgent(Agent): uri: bytes, headers: TxHeaders | None = None, bodyProducer: IBodyProducer | None = None, - ) -> Deferred[TxResponse]: + ) -> Deferred[IResponse]: """ Issue a new request via the configured proxy. """ # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: return self._requestWithEndpoint( - key=("http-proxy", self._proxyURI.host, self._proxyURI.port), + key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port), endpoint=self._getEndpoint(self._proxyURI), method=method, parsedURI=URI.fromBytes(uri), @@ -426,8 +426,11 @@ class ScrapyAgent: headers.removeHeader(b"Proxy-Authorization") bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() - d: Deferred[TxResponse] = agent.request( - method, to_bytes(url, encoding="ascii"), headers, bodyproducer + d: Deferred[IResponse] = agent.request( + method, + to_bytes(url, encoding="ascii"), + headers, + cast(IBodyProducer, bodyproducer), ) # set download latency d.addCallback(self._cb_latency, request, start_time) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 2da526cd8..94b1a1fc7 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -10,7 +10,7 @@ import warnings from importlib import import_module from pathlib import Path from posixpath import split -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar, cast from unittest import TestCase, mock from twisted.trial.unittest import SkipTest @@ -211,4 +211,4 @@ def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: from twisted.web.client import Agent # imports twisted.internet.reactor agent = Agent(reactor) - return agent.request(b"GET", url.encode("utf-8")) + return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8"))) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index ef77f7843..464320602 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -4,7 +4,7 @@ import shutil import warnings from pathlib import Path from tempfile import mkdtemp -from typing import Any +from typing import Any, cast import OpenSSL.SSL import pytest @@ -14,6 +14,7 @@ from twisted.trial import unittest from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse +from twisted.web.iweb import IBodyProducer from scrapy.core.downloader import Slot from scrapy.core.downloader.contextfactory import ( @@ -76,8 +77,15 @@ class TestContextFactoryBase(unittest.TestCase): agent = Agent(reactor, contextFactory=client_context_factory) body_producer = _RequestBodyProducer(body.encode()) if body else None - response: TxResponse = await maybe_deferred_to_future( - agent.request(b"GET", url.encode(), bodyProducer=body_producer) + response: TxResponse = cast( + TxResponse, + await maybe_deferred_to_future( + agent.request( + b"GET", + url.encode(), + bodyProducer=cast(IBodyProducer, body_producer), + ) + ), ) with warnings.catch_warnings(): # https://github.com/twisted/twisted/issues/8227 diff --git a/tox.ini b/tox.ini index 92cfc3794..5680d98d1 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ deps = sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures pywin32; sys_platform == "win32" + Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467 [testenv] deps = From 24a827c72e9f5b35ddcd12ccce2ce7c6611d2845 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 9 Jun 2025 12:53:42 +0500 Subject: [PATCH 1772/2083] Pin older Twisted in tests, update type hints. (#6882) --- scrapy/core/downloader/handlers/http11.py | 19 +++++++++++-------- scrapy/utils/test.py | 4 ++-- tests/test_core_downloader.py | 14 +++++++++++--- tox.ini | 1 + 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 74a6e54ee..54fef48b6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ import re from contextlib import suppress from io import BytesIO from time import time -from typing import TYPE_CHECKING, Any, TypedDict, TypeVar +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast from urllib.parse import urldefrag, urlparse from twisted.internet import ssl @@ -27,7 +27,7 @@ from twisted.web.client import ( from twisted.web.client import Response as TxResponse from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponse from zope.interface import implementer from scrapy import Request, Spider, signals @@ -286,11 +286,11 @@ class TunnelingAgent(Agent): key: Any, endpoint: TCP4ClientEndpoint, method: bytes, - parsedURI: bytes, + parsedURI: URI, headers: TxHeaders | None, bodyProducer: IBodyProducer | None, requestPath: bytes, - ) -> Deferred[TxResponse]: + ) -> Deferred[IResponse]: # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy @@ -329,14 +329,14 @@ class ScrapyProxyAgent(Agent): uri: bytes, headers: TxHeaders | None = None, bodyProducer: IBodyProducer | None = None, - ) -> Deferred[TxResponse]: + ) -> Deferred[IResponse]: """ Issue a new request via the configured proxy. """ # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: return self._requestWithEndpoint( - key=("http-proxy", self._proxyURI.host, self._proxyURI.port), + key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port), endpoint=self._getEndpoint(self._proxyURI), method=method, parsedURI=URI.fromBytes(uri), @@ -426,8 +426,11 @@ class ScrapyAgent: headers.removeHeader(b"Proxy-Authorization") bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() - d: Deferred[TxResponse] = agent.request( - method, to_bytes(url, encoding="ascii"), headers, bodyproducer + d: Deferred[IResponse] = agent.request( + method, + to_bytes(url, encoding="ascii"), + headers, + cast(IBodyProducer, bodyproducer), ) # set download latency d.addCallback(self._cb_latency, request, start_time) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4a732bd72..3780ad23e 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -10,7 +10,7 @@ import warnings from importlib import import_module from pathlib import Path from posixpath import split -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar, cast from unittest import TestCase, mock from twisted.trial.unittest import SkipTest @@ -216,4 +216,4 @@ def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: from twisted.web.client import Agent # imports twisted.internet.reactor agent = Agent(reactor) - return agent.request(b"GET", url.encode("utf-8")) + return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8"))) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 1bffd69ed..668a2cd1b 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -4,7 +4,7 @@ import shutil import warnings from pathlib import Path from tempfile import mkdtemp -from typing import Any +from typing import Any, cast import OpenSSL.SSL import pytest @@ -15,6 +15,7 @@ from twisted.trial import unittest from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse +from twisted.web.iweb import IBodyProducer from scrapy.core.downloader import Slot from scrapy.core.downloader.contextfactory import ( @@ -73,8 +74,15 @@ class TestContextFactoryBase(unittest.TestCase): ) -> bytes: agent = Agent(reactor, contextFactory=client_context_factory) body_producer = _RequestBodyProducer(body.encode()) if body else None - response: TxResponse = await maybe_deferred_to_future( - agent.request(b"GET", url.encode(), bodyProducer=body_producer) + response: TxResponse = cast( + TxResponse, + await maybe_deferred_to_future( + agent.request( + b"GET", + url.encode(), + bodyProducer=cast(IBodyProducer, body_producer), + ) + ), ) with warnings.catch_warnings(): # https://github.com/twisted/twisted/issues/8227 diff --git a/tox.ini b/tox.ini index 92cfc3794..5680d98d1 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ deps = sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures pywin32; sys_platform == "win32" + Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467 [testenv] deps = From 6b5a4a64173fc051063f01c05925519e45dbbfdd Mon Sep 17 00:00:00 2001 From: nakanoh Date: Mon, 9 Jun 2025 19:07:01 +0900 Subject: [PATCH 1773/2083] Minor improvement in cmdline.py (#6875) --- scrapy/cmdline.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 3d448532b..2b0204071 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -32,8 +32,9 @@ class ScrapyArgumentParser(argparse.ArgumentParser): def _parse_optional( self, arg_string: str ) -> tuple[argparse.Action | None, str, str | None] | None: - # if starts with -: it means that is a parameter not a argument - if arg_string[:2] == "-:": + # Support something like ‘-o -:json’, where ‘-:json’ is a value for + # ‘-o’, not another parameter. + if arg_string.startswith("-:"): return None return super()._parse_optional(arg_string) From 7400868ad5b80f6c91fd69dea476aa8f59ce9081 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 9 Jun 2025 15:19:02 +0500 Subject: [PATCH 1774/2083] Release notes for 2.13.2. (#6868) * Release notes for 2.13.2. * Update release notes. --- docs/news.rst | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 8b1d51674..8e7d80e26 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,52 @@ Release notes ============= +.. _release-2.13.2: + +Scrapy 2.13.2 (unreleased) +-------------------------- + +- Fixed a bug introduced in Scrapy 2.13.0 that caused results of request + errbacks to be ignored when the errback was called because of a downloader + error. + (:issue:`6861`, :issue:`6863`) + +- Added a note about the behavior change of + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to its docs and + to the "Backward-incompatible changes" section of :ref:`the Scrapy 2.13.0 + release notes `. + (:issue:`6866`) + +- Improved the message in the exception raised by + :func:`scrapy.utils.test.get_reactor_settings` when there is no reactor + installed. + (:issue:`6866`) + +- Updated the :class:`scrapy.crawler.CrawlerRunner` examples in + :ref:`topics-practices` to install the reactor explicitly, to fix + reactor-related errors with Scrapy 2.13.0 and later. + (:issue:`6865`) + +- Fixed ``scrapy fetch`` not working with scrapy-poet_. + (:issue:`6872`) + +- Fixed an exception produced by :class:`scrapy.core.engine.ExecutionEngine` + when it's closed before being fully initialized. + (:issue:`6857`, :issue:`6867`) + +- Improved the README, updated the Scrapy logo in it. + (:issue:`6831`, :issue:`6833`, :issue:`6839`) + +- Restricted the Twisted version used in tests to below 25.5.0, as some tests + fail with 25.5.0. + (:issue:`6878`, :issue:`6882`) + +- Updated type hints for Twisted 25.5.0 changes. + (:issue:`6882`) + +- Removed the old artwork. + (:issue:`6874`) + .. _release-2.13.1: Scrapy 2.13.1 (2025-05-28) From c6740604a405c51e92bedce8617b9151cd9766c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 9 Jun 2025 15:21:06 +0500 Subject: [PATCH 1775/2083] =?UTF-8?q?Bump=20version:=202.13.1=20=E2=86=92?= =?UTF-8?q?=202.13.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/news.rst | 2 +- pyproject.toml | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 8e7d80e26..b2fe78bc6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.13.2: -Scrapy 2.13.2 (unreleased) +Scrapy 2.13.2 (2025-06-09) -------------------------- - Fixed a bug introduced in Scrapy 2.13.0 that caused results of request diff --git a/pyproject.toml b/pyproject.toml index 68c1e07bb..d6aebf514 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,7 +115,7 @@ module = "twisted" implicit_reexport = true [tool.bumpversion] -current_version = "2.13.1" +current_version = "2.13.2" commit = true tag = true tag_name = "{new_version}" diff --git a/scrapy/VERSION b/scrapy/VERSION index 94f15e9cc..0e83a9a9c 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.13.1 +2.13.2 From ac956f8595354fde80bf64b3eaf95a22b3433f98 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 10 Jun 2025 13:02:27 +0500 Subject: [PATCH 1776/2083] Replace most of the @inlineCallbacks test helpers. (#6883) --- tests/test_crawl.py | 101 ++-- tests/test_feedexport.py | 619 +++++++++++--------- tests/test_spidermiddleware.py | 219 +++---- tests/test_spidermiddleware_output_chain.py | 75 +-- 4 files changed, 565 insertions(+), 449 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 4c1f6216b..42f9899f9 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -4,7 +4,7 @@ import json import logging from ipaddress import IPv4Address from socket import gethostbyname -from typing import Any +from typing import TYPE_CHECKING, Any from urllib.parse import urlparse import pytest @@ -14,11 +14,12 @@ from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase -from scrapy import signals +from scrapy import Spider, signals from scrapy.crawler import CrawlerRunner from scrapy.exceptions import CloseSpider, StopDownload from scrapy.http import Request from scrapy.http.response import Response +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler, get_reactor_settings from tests import NON_EXISTING_RESOLVABLE @@ -55,8 +56,13 @@ from tests.spiders import ( StartItemSpider, ) +if TYPE_CHECKING: + from scrapy.statscollectors import StatsCollector + class TestCrawl(TestCase): + mockserver: MockServer + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -72,16 +78,17 @@ class TestCrawl(TestCase): yield crawler.crawl(mockserver=self.mockserver) assert len(crawler.spider.urls_visited) == 11 # 10 + start_url - @inlineCallbacks - def test_fixed_delay(self): - yield self._test_delay(total=3, delay=0.2) + @deferred_f_from_coro_f + async def test_fixed_delay(self): + await self._test_delay(total=3, delay=0.2) - @inlineCallbacks - def test_randomized_delay(self): - yield self._test_delay(total=3, delay=0.1, randomize=True) + @deferred_f_from_coro_f + async def test_randomized_delay(self): + await self._test_delay(total=3, delay=0.1, randomize=True) - @inlineCallbacks - def _test_delay(self, total, delay, randomize=False): + async def _test_delay( + self, total: int, delay: float, randomize: bool = False + ) -> None: crawl_kwargs = { "maxlatency": delay * 2, "mockserver": self.mockserver, @@ -91,7 +98,9 @@ class TestCrawl(TestCase): settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize} crawler = get_crawler(FollowAllSpider, settings) - yield crawler.crawl(**crawl_kwargs) + await maybe_deferred_to_future(crawler.crawl(**crawl_kwargs)) + assert crawler.spider + assert isinstance(crawler.spider, FollowAllSpider) times = crawler.spider.times total_time = times[-1] - times[0] average = total_time / (len(times) - 1) @@ -103,7 +112,9 @@ class TestCrawl(TestCase): # code above to have any meaning. settings["DOWNLOAD_DELAY"] = 0 crawler = get_crawler(FollowAllSpider, settings) - yield crawler.crawl(**crawl_kwargs) + await maybe_deferred_to_future(crawler.crawl(**crawl_kwargs)) + assert crawler.spider + assert isinstance(crawler.spider, FollowAllSpider) times = crawler.spider.times total_time = times[-1] - times[0] average = total_time / (len(times) - 1) @@ -428,8 +439,9 @@ class TestCrawlSpider(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @inlineCallbacks - def _run_spider(self, spider_cls): + async def _run_spider( + self, spider_cls: type[Spider] + ) -> tuple[LogCapture, list[Any], StatsCollector]: items = [] def _on_item_scraped(item): @@ -438,9 +450,12 @@ class TestCrawlSpider(TestCase): crawler = get_crawler(spider_cls) crawler.signals.connect(_on_item_scraped, signals.item_scraped) with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + await maybe_deferred_to_future( + crawler.crawl( + self.mockserver.url("/status?n=200"), mockserver=self.mockserver + ) ) + assert crawler.stats return log, items, crawler.stats @inlineCallbacks @@ -521,9 +536,9 @@ class TestCrawlSpider(TestCase): assert "Got response 200" in str(log) @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_asyncio_parse_items_list(self): - log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) + @deferred_f_from_coro_f + async def test_async_def_asyncio_parse_items_list(self): + log, items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider) assert "Got response 200" in str(log) assert {"id": 1} in items assert {"id": 2} in items @@ -546,17 +561,17 @@ class TestCrawlSpider(TestCase): assert {"foo": 42} in items @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_asyncgen_parse(self): - log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) + @deferred_f_from_coro_f + async def test_async_def_asyncgen_parse(self): + log, _, stats = await self._run_spider(AsyncDefAsyncioGenSpider) assert "Got response 200" in str(log) itemcount = stats.get_value("item_scraped_count") assert itemcount == 1 @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_asyncgen_parse_loop(self): - log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) + @deferred_f_from_coro_f + async def test_async_def_asyncgen_parse_loop(self): + log, items, stats = await self._run_spider(AsyncDefAsyncioGenLoopSpider) assert "Got response 200" in str(log) itemcount = stats.get_value("item_scraped_count") assert itemcount == 10 @@ -564,9 +579,9 @@ class TestCrawlSpider(TestCase): assert {"foo": i} in items @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_asyncgen_parse_exc(self): - log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) + @deferred_f_from_coro_f + async def test_async_def_asyncgen_parse_exc(self): + log, items, stats = await self._run_spider(AsyncDefAsyncioGenExcSpider) log = str(log) assert "Spider error processing" in log assert "ValueError" in log @@ -576,9 +591,9 @@ class TestCrawlSpider(TestCase): assert {"foo": i} in items @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_asyncgen_parse_complex(self): - _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) + @deferred_f_from_coro_f + async def test_async_def_asyncgen_parse_complex(self): + _, items, stats = await self._run_spider(AsyncDefAsyncioGenComplexSpider) itemcount = stats.get_value("item_scraped_count") assert itemcount == 156 # some random items @@ -588,27 +603,27 @@ class TestCrawlSpider(TestCase): assert {"index2": i} in items @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_asyncio_parse_reqs_list(self): - log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) + @deferred_f_from_coro_f + async def test_async_def_asyncio_parse_reqs_list(self): + log, *_ = await self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): assert f"Got response 200, req_id {req_id}" in str(log) @pytest.mark.only_not_asyncio - @inlineCallbacks - def test_async_def_deferred_direct(self): - _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) + @deferred_f_from_coro_f + async def test_async_def_deferred_direct(self): + _, items, _ = await self._run_spider(AsyncDefDeferredDirectSpider) assert items == [{"code": 200}] @pytest.mark.only_asyncio - @inlineCallbacks - def test_async_def_deferred_wrapped(self): - log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) + @deferred_f_from_coro_f + async def test_async_def_deferred_wrapped(self): + log, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider) assert items == [{"code": 200}] - @inlineCallbacks - def test_async_def_deferred_maybe_wrapped(self): - _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider) + @deferred_f_from_coro_f + async def test_async_def_deferred_maybe_wrapped(self): + _, items, _ = await self._run_spider(AsyncDefDeferredMaybeWrappedSpider) assert items == [{"code": 200}] @inlineCallbacks diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index cdf03ca76..262c0b434 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -33,7 +33,7 @@ from zope.interface import implementer from zope.interface.verify import verifyObject import scrapy -from scrapy import signals +from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( @@ -48,12 +48,14 @@ from scrapy.extensions.feedexport import ( StdoutFeedStorage, ) from scrapy.settings import Settings +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler from tests.mockserver import MockFTPServer, MockServer from tests.spiders import ItemSpider if TYPE_CHECKING: + from collections.abc import Iterable from os import PathLike @@ -89,24 +91,25 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: return (client_mock, bucket_mock, blob_mock) +# TODO: replace self.mktemp() and drop the unittest.TestCase base class TestFileFeedStorage(unittest.TestCase): def test_store_file_uri(self): path = Path(self.mktemp()).resolve() uri = path_to_file_uri(str(path)) - return self._assert_stores(FileFeedStorage(uri), path) + self._assert_stores(FileFeedStorage(uri), path) def test_store_file_uri_makedirs(self): path = Path(self.mktemp()).resolve() / "more" / "paths" / "file.txt" uri = path_to_file_uri(str(path)) - return self._assert_stores(FileFeedStorage(uri), path) + self._assert_stores(FileFeedStorage(uri), path) def test_store_direct_path(self): path = Path(self.mktemp()).resolve() - return self._assert_stores(FileFeedStorage(str(path)), path) + self._assert_stores(FileFeedStorage(str(path)), path) def test_store_direct_path_relative(self): path = Path(self.mktemp()) - return self._assert_stores(FileFeedStorage(str(path)), path) + self._assert_stores(FileFeedStorage(str(path)), path) def test_interface(self): path = self.mktemp() @@ -124,20 +127,21 @@ class TestFileFeedStorage(unittest.TestCase): def test_append(self): path = self._store() - return self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") + self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") def test_overwrite(self): path = self._store({"overwrite": True}) - return self._assert_stores( + self._assert_stores( FileFeedStorage(str(path), feed_options={"overwrite": True}), path ) - @inlineCallbacks - def _assert_stores(self, storage, path: Path, expected_content=b"content"): + def _assert_stores( + self, storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" + ) -> None: spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") - yield storage.store(file) + storage.store(file) assert path.exists() try: assert path.read_bytes() == expected_content @@ -153,7 +157,7 @@ class TestFTPFeedStorage(unittest.TestCase): crawler = get_crawler(settings_dict=settings) return TestSpider.from_crawler(crawler) - def _store(self, uri, content, feed_options=None, settings=None): + async def _store(self, uri, content, feed_options=None, settings=None): crawler = get_crawler(settings_dict=settings or {}) storage = FTPFeedStorage.from_crawler( crawler, @@ -164,7 +168,7 @@ class TestFTPFeedStorage(unittest.TestCase): spider = self.get_test_spider() file = storage.open(spider) file.write(content) - return storage.store(file) + await maybe_deferred_to_future(storage.store(file)) def _assert_stored(self, path: Path, content): assert path.exists() @@ -173,44 +177,44 @@ class TestFTPFeedStorage(unittest.TestCase): finally: path.unlink() - @inlineCallbacks - def test_append(self): + @deferred_f_from_coro_f + async def test_append(self): with MockFTPServer() as ftp_server: filename = "file" url = ftp_server.url(filename) feed_options = {"overwrite": False} - yield self._store(url, b"foo", feed_options=feed_options) - yield self._store(url, b"bar", feed_options=feed_options) + await self._store(url, b"foo", feed_options=feed_options) + await self._store(url, b"bar", feed_options=feed_options) self._assert_stored(ftp_server.path / filename, b"foobar") - @inlineCallbacks - def test_overwrite(self): + @deferred_f_from_coro_f + async def test_overwrite(self): with MockFTPServer() as ftp_server: filename = "file" url = ftp_server.url(filename) - yield self._store(url, b"foo") - yield self._store(url, b"bar") + await self._store(url, b"foo") + await self._store(url, b"bar") self._assert_stored(ftp_server.path / filename, b"bar") - @inlineCallbacks - def test_append_active_mode(self): + @deferred_f_from_coro_f + async def test_append_active_mode(self): with MockFTPServer() as ftp_server: settings = {"FEED_STORAGE_FTP_ACTIVE": True} filename = "file" url = ftp_server.url(filename) feed_options = {"overwrite": False} - yield self._store(url, b"foo", feed_options=feed_options, settings=settings) - yield self._store(url, b"bar", feed_options=feed_options, settings=settings) + await self._store(url, b"foo", feed_options=feed_options, settings=settings) + await self._store(url, b"bar", feed_options=feed_options, settings=settings) self._assert_stored(ftp_server.path / filename, b"foobar") - @inlineCallbacks - def test_overwrite_active_mode(self): + @deferred_f_from_coro_f + async def test_overwrite_active_mode(self): with MockFTPServer() as ftp_server: settings = {"FEED_STORAGE_FTP_ACTIVE": True} filename = "file" url = ftp_server.url(filename) - yield self._store(url, b"foo", settings=settings) - yield self._store(url, b"bar", settings=settings) + await self._store(url, b"foo", settings=settings) + await self._store(url, b"bar", settings=settings) self._assert_stored(ftp_server.path / filename, b"bar") def test_uri_auth_quote(self): @@ -291,8 +295,8 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.access_key == "uri_key" assert storage.secret_key == "uri_secret" - @inlineCallbacks - def test_store(self): + @deferred_f_from_coro_f + async def test_store(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", "AWS_SECRET_ACCESS_KEY": "secret_key", @@ -306,7 +310,7 @@ class TestS3FeedStorage(unittest.TestCase): file = mock.MagicMock() storage.s3_client = mock.MagicMock() - yield storage.store(file) + await maybe_deferred_to_future(storage.store(file)) assert storage.s3_client.upload_fileobj.call_args == mock.call( Bucket=bucket, Key=key, Fileobj=file ) @@ -432,8 +436,8 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.region_name == region_name assert storage.s3_client._client_config.region_name == region_name - @inlineCallbacks - def test_store_without_acl(self): + @deferred_f_from_coro_f + async def test_store_without_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", @@ -444,7 +448,7 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.acl is None storage.s3_client = mock.MagicMock() - yield storage.store(BytesIO(b"test file")) + await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) acl = ( storage.s3_client.upload_fileobj.call_args[1] .get("ExtraArgs", {}) @@ -452,8 +456,8 @@ class TestS3FeedStorage(unittest.TestCase): ) assert acl is None - @inlineCallbacks - def test_store_with_acl(self): + @deferred_f_from_coro_f + async def test_store_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) @@ -462,7 +466,7 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.acl == "custom-acl" storage.s3_client = mock.MagicMock() - yield storage.store(BytesIO(b"test file")) + await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] assert acl == "custom-acl" @@ -516,8 +520,8 @@ class TestGCSFeedStorage(unittest.TestCase): storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") assert storage.acl is None - @inlineCallbacks - def test_store(self): + @deferred_f_from_coro_f + async def test_store(self): try: from google.cloud.storage import Client # noqa: F401 except ImportError: @@ -532,7 +536,7 @@ class TestGCSFeedStorage(unittest.TestCase): f = mock.Mock() storage = GCSFeedStorage(uri, project_id, acl) - yield storage.store(f) + await maybe_deferred_to_future(storage.store(f)) f.seek.assert_called_once_with(0) m.assert_called_once_with(project=project_id) @@ -556,14 +560,13 @@ class TestGCSFeedStorage(unittest.TestCase): assert "GCS does not support appending to files" in str(log) -class TestStdoutFeedStorage(unittest.TestCase): - @inlineCallbacks +class TestStdoutFeedStorage: def test_store(self): out = BytesIO() storage = StdoutFeedStorage("stdout:", _stdout=out) file = storage.open(scrapy.Spider("default")) file.write(b"content") - yield storage.store(file) + storage.store(file) assert out.getvalue() == b"content" def test_overwrite_default(self): @@ -641,6 +644,8 @@ class LogOnStoreFileStorage: class TestFeedExportBase(ABC, unittest.TestCase): + mockserver: MockServer + class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() @@ -670,8 +675,9 @@ class TestFeedExportBase(ABC, unittest.TestCase): def tearDown(self): shutil.rmtree(self.temp_dir, ignore_errors=True) - @inlineCallbacks - def exported_data(self, items, settings): + async def exported_data( + self, items: Iterable[Any], settings: dict[str, Any] + ) -> dict[str, Any]: """ Return exported data which a spider yielding ``items`` would return. """ @@ -682,11 +688,9 @@ class TestFeedExportBase(ABC, unittest.TestCase): def parse(self, response): yield from items - data = yield self.run_and_export(TestSpider, settings) - return data + return await self.run_and_export(TestSpider, settings) - @inlineCallbacks - def exported_no_data(self, settings): + async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]: """ Return exported data which a spider yielding no ``items`` would return. """ @@ -697,20 +701,75 @@ class TestFeedExportBase(ABC, unittest.TestCase): def parse(self, response): pass - data = yield self.run_and_export(TestSpider, settings) - return data + return await self.run_and_export(TestSpider, settings) - @inlineCallbacks - def assertExported(self, items, header, rows, settings=None): - yield self.assertExportedCsv(items, header, rows, settings) - yield self.assertExportedJsonLines(items, rows, settings) - yield self.assertExportedXml(items, rows, settings) - yield self.assertExportedPickle(items, rows, settings) - yield self.assertExportedMarshal(items, rows, settings) - yield self.assertExportedMultiple(items, rows, settings) + async def assertExported( + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + await self.assertExportedCsv(items, header, rows, settings) + await self.assertExportedJsonLines(items, rows, settings) + await self.assertExportedXml(items, rows, settings) + await self.assertExportedPickle(items, rows, settings) + await self.assertExportedMarshal(items, rows, settings) + await self.assertExportedMultiple(items, rows, settings) + + async def assertExportedCsv( + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedJsonLines( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedXml( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedMultiple( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedPickle( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedMarshal( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass @abstractmethod - def run_and_export(self, spider_cls, settings): + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, Any]: pass def _load_until_eof(self, data, load_func): @@ -771,8 +830,9 @@ class ExceptionJsonItemExporter(JsonItemExporter): class TestFeedExport(TestFeedExportBase): - @inlineCallbacks - def run_and_export(self, spider_cls, settings): + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, Any]: """Run spider with specified settings; return exported data.""" FEEDS = settings.get("FEEDS") or {} @@ -781,11 +841,11 @@ class TestFeedExport(TestFeedExportBase): for file_path, feed_options in FEEDS.items() } - content = {} + content: dict[str, Any] = {} try: spider_cls.start_urls = [self.mockserver.url("/")] crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + await maybe_deferred_to_future(crawler.crawl()) for file_path, feed_options in FEEDS.items(): content[feed_options["format"]] = ( @@ -801,8 +861,13 @@ class TestFeedExport(TestFeedExportBase): return content - @inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None): + async def assertExportedCsv( + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -811,13 +876,17 @@ class TestFeedExport(TestFeedExportBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) reader = csv.DictReader(to_unicode(data["csv"]).splitlines()) assert reader.fieldnames == list(header) assert rows == list(reader) - @inlineCallbacks - def assertExportedJsonLines(self, items, rows, settings=None): + async def assertExportedJsonLines( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -826,13 +895,17 @@ class TestFeedExport(TestFeedExportBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) parsed = [json.loads(to_unicode(line)) for line in data["jl"].splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] assert rows == parsed - @inlineCallbacks - def assertExportedXml(self, items, rows, settings=None): + async def assertExportedXml( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -841,14 +914,18 @@ class TestFeedExport(TestFeedExportBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) rows = [{k: v for k, v in row.items() if v} for row in rows] root = lxml.etree.fromstring(data["xml"]) got_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] assert rows == got_rows - @inlineCallbacks - def assertExportedMultiple(self, items, rows, settings=None): + async def assertExportedMultiple( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -858,7 +935,7 @@ class TestFeedExport(TestFeedExportBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) rows = [{k: v for k, v in row.items() if v} for row in rows] # XML root = lxml.etree.fromstring(data["xml"]) @@ -868,8 +945,12 @@ class TestFeedExport(TestFeedExportBase): json_rows = json.loads(to_unicode(data["json"])) assert rows == json_rows - @inlineCallbacks - def assertExportedPickle(self, items, rows, settings=None): + async def assertExportedPickle( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -878,15 +959,19 @@ class TestFeedExport(TestFeedExportBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) expected = [{k: v for k, v in row.items() if v} for row in rows] import pickle result = self._load_until_eof(data["pickle"], load_func=pickle.load) assert result == expected - @inlineCallbacks - def assertExportedMarshal(self, items, rows, settings=None): + async def assertExportedMarshal( + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: settings = settings or {} settings.update( { @@ -895,7 +980,7 @@ class TestFeedExport(TestFeedExportBase): }, } ) - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) expected = [{k: v for k, v in row.items() if v} for row in rows] import marshal @@ -956,8 +1041,8 @@ class TestFeedExport(TestFeedExportBase): crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1 ) - @inlineCallbacks - def test_export_items(self): + @deferred_f_from_coro_f + async def test_export_items(self): # feed exporters use field names from Item items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -968,10 +1053,10 @@ class TestFeedExport(TestFeedExportBase): {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, ] header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows) + await self.assertExported(items, header, rows) - @inlineCallbacks - def test_export_no_items_not_store_empty(self): + @deferred_f_from_coro_f + async def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { "FEEDS": { @@ -979,11 +1064,11 @@ class TestFeedExport(TestFeedExportBase): }, "FEED_STORE_EMPTY": False, } - data = yield self.exported_no_data(settings) + data = await self.exported_no_data(settings) assert data[fmt] is None - @inlineCallbacks - def test_start_finish_exporting_items(self): + @deferred_f_from_coro_f + async def test_start_finish_exporting_items(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), ] @@ -998,12 +1083,12 @@ class TestFeedExport(TestFeedExportBase): InstrumentedFeedSlot.subscribe__listener(listener) with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): - _ = yield self.exported_data(items, settings) + await self.exported_data(items, settings) assert not listener.start_without_finish assert not listener.finish_without_start - @inlineCallbacks - def test_start_finish_exporting_no_items(self): + @deferred_f_from_coro_f + async def test_start_finish_exporting_no_items(self): items = [] settings = { "FEEDS": { @@ -1016,12 +1101,12 @@ class TestFeedExport(TestFeedExportBase): InstrumentedFeedSlot.subscribe__listener(listener) with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): - _ = yield self.exported_data(items, settings) + await self.exported_data(items, settings) assert not listener.start_without_finish assert not listener.finish_without_start - @inlineCallbacks - def test_start_finish_exporting_items_exception(self): + @deferred_f_from_coro_f + async def test_start_finish_exporting_items_exception(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), ] @@ -1037,12 +1122,12 @@ class TestFeedExport(TestFeedExportBase): InstrumentedFeedSlot.subscribe__listener(listener) with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): - _ = yield self.exported_data(items, settings) + await self.exported_data(items, settings) assert not listener.start_without_finish assert not listener.finish_without_start - @inlineCallbacks - def test_start_finish_exporting_no_items_exception(self): + @deferred_f_from_coro_f + async def test_start_finish_exporting_no_items_exception(self): items = [] settings = { "FEEDS": { @@ -1056,12 +1141,12 @@ class TestFeedExport(TestFeedExportBase): InstrumentedFeedSlot.subscribe__listener(listener) with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): - _ = yield self.exported_data(items, settings) + await self.exported_data(items, settings) assert not listener.start_without_finish assert not listener.finish_without_start - @inlineCallbacks - def test_export_no_items_store_empty(self): + @deferred_f_from_coro_f + async def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), ("jsonlines", b""), @@ -1077,11 +1162,11 @@ class TestFeedExport(TestFeedExportBase): "FEED_STORE_EMPTY": True, "FEED_EXPORT_INDENT": None, } - data = yield self.exported_no_data(settings) + data = await self.exported_no_data(settings) assert expctd == data[fmt] - @inlineCallbacks - def test_export_no_items_multiple_feeds(self): + @deferred_f_from_coro_f + async def test_export_no_items_multiple_feeds(self): """Make sure that `storage.store` is called for every feed.""" settings = { "FEEDS": { @@ -1094,12 +1179,12 @@ class TestFeedExport(TestFeedExportBase): } with LogCapture() as log: - yield self.exported_no_data(settings) + await self.exported_no_data(settings) assert str(log).count("Storage.store is called") == 0 - @inlineCallbacks - def test_export_multiple_item_classes(self): + @deferred_f_from_coro_f + async def test_export_multiple_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem2({"hello": "world2", "foo": "bar2"}), @@ -1117,53 +1202,53 @@ class TestFeedExport(TestFeedExportBase): {"egg": "spam4", "foo": "", "baz": ""}, ] rows_jl = [dict(row) for row in items] - yield self.assertExportedCsv(items, header, rows_csv) - yield self.assertExportedJsonLines(items, rows_jl) + await self.assertExportedCsv(items, header, rows_csv) + await self.assertExportedJsonLines(items, rows_jl) - @inlineCallbacks - def test_export_items_empty_field_list(self): + @deferred_f_from_coro_f + async def test_export_items_empty_field_list(self): # FEED_EXPORT_FIELDS==[] means the same as default None items = [{"foo": "bar"}] header = ["foo"] rows = [{"foo": "bar"}] settings = {"FEED_EXPORT_FIELDS": []} - yield self.assertExportedCsv(items, header, rows) - yield self.assertExportedJsonLines(items, rows, settings) + await self.assertExportedCsv(items, header, rows) + await self.assertExportedJsonLines(items, rows, settings) - @inlineCallbacks - def test_export_items_field_list(self): + @deferred_f_from_coro_f + async def test_export_items_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] rows = [{"foo": "bar", "baz": ""}] settings = {"FEED_EXPORT_FIELDS": header} - yield self.assertExported(items, header, rows, settings=settings) + await self.assertExported(items, header, rows, settings=settings) - @inlineCallbacks - def test_export_items_comma_separated_field_list(self): + @deferred_f_from_coro_f + async def test_export_items_comma_separated_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] rows = [{"foo": "bar", "baz": ""}] settings = {"FEED_EXPORT_FIELDS": ",".join(header)} - yield self.assertExported(items, header, rows, settings=settings) + await self.assertExported(items, header, rows, settings=settings) - @inlineCallbacks - def test_export_items_json_field_list(self): + @deferred_f_from_coro_f + async def test_export_items_json_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] rows = [{"foo": "bar", "baz": ""}] settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} - yield self.assertExported(items, header, rows, settings=settings) + await self.assertExported(items, header, rows, settings=settings) - @inlineCallbacks - def test_export_items_field_names(self): + @deferred_f_from_coro_f + async def test_export_items_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} rows = [{"Foo": "bar"}] settings = {"FEED_EXPORT_FIELDS": header} - yield self.assertExported(items, list(header.values()), rows, settings=settings) + await self.assertExported(items, list(header.values()), rows, settings=settings) - @inlineCallbacks - def test_export_items_dict_field_names(self): + @deferred_f_from_coro_f + async def test_export_items_dict_field_names(self): items = [{"foo": "bar"}] header = { "baz": "Baz", @@ -1171,18 +1256,18 @@ class TestFeedExport(TestFeedExportBase): } rows = [{"Baz": "", "Foo": "bar"}] settings = {"FEED_EXPORT_FIELDS": header} - yield self.assertExported(items, ["Baz", "Foo"], rows, settings=settings) + await self.assertExported(items, ["Baz", "Foo"], rows, settings=settings) - @inlineCallbacks - def test_export_items_json_field_names(self): + @deferred_f_from_coro_f + async def test_export_items_json_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} rows = [{"Foo": "bar"}] settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} - yield self.assertExported(items, list(header.values()), rows, settings=settings) + await self.assertExported(items, list(header.values()), rows, settings=settings) - @inlineCallbacks - def test_export_based_on_item_classes(self): + @deferred_f_from_coro_f + async def test_export_based_on_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem2({"hello": "world2", "foo": "bar2"}), @@ -1223,12 +1308,12 @@ class TestFeedExport(TestFeedExportBase): }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): assert data[fmt] == expected - @inlineCallbacks - def test_export_based_on_custom_filters(self): + @deferred_f_from_coro_f + async def test_export_based_on_custom_filters(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem2({"hello": "world2", "foo": "bar2"}), @@ -1282,12 +1367,12 @@ class TestFeedExport(TestFeedExportBase): }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): assert data[fmt] == expected - @inlineCallbacks - def test_export_dicts(self): + @deferred_f_from_coro_f + async def test_export_dicts(self): # When dicts are used, only keys from the first row are used as # a header for CSV, and all fields are used for JSON Lines. items = [ @@ -1296,11 +1381,11 @@ class TestFeedExport(TestFeedExportBase): ] rows_csv = [{"egg": "spam", "foo": "bar"}, {"egg": "spam", "foo": "bar"}] rows_jl = items - yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv) - yield self.assertExportedJsonLines(items, rows_jl) + await self.assertExportedCsv(items, ["foo", "egg"], rows_csv) + await self.assertExportedJsonLines(items, rows_jl) - @inlineCallbacks - def test_export_tuple(self): + @deferred_f_from_coro_f + async def test_export_tuple(self): items = [ {"foo": "bar1", "egg": "spam1"}, {"foo": "bar2", "egg": "spam2", "baz": "quux"}, @@ -1308,10 +1393,10 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": ("foo", "baz")} rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}] - yield self.assertExported(items, ["foo", "baz"], rows, settings=settings) + await self.assertExported(items, ["foo", "baz"], rows, settings=settings) - @inlineCallbacks - def test_export_feed_export_fields(self): + @deferred_f_from_coro_f + async def test_export_feed_export_fields(self): # FEED_EXPORT_FIELDS option allows to order export fields # and to select a subset of fields to export, both for Items and dicts. @@ -1327,17 +1412,17 @@ class TestFeedExport(TestFeedExportBase): {"egg": "spam1", "foo": "bar1", "baz": ""}, {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, ] - yield self.assertExported( + await self.assertExported( items, ["foo", "baz", "egg"], rows, settings=settings ) # export a subset of columns settings = {"FEED_EXPORT_FIELDS": "egg,baz"} rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}] - yield self.assertExported(items, ["egg", "baz"], rows, settings=settings) + await self.assertExported(items, ["egg", "baz"], rows, settings=settings) - @inlineCallbacks - def test_export_encoding(self): + @deferred_f_from_coro_f + async def test_export_encoding(self): items = [{"foo": "Test\xd6"}] formats = { @@ -1357,7 +1442,7 @@ class TestFeedExport(TestFeedExportBase): }, "FEED_EXPORT_INDENT": None, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) assert data[fmt] == expected formats = { @@ -1378,11 +1463,11 @@ class TestFeedExport(TestFeedExportBase): "FEED_EXPORT_INDENT": None, "FEED_EXPORT_ENCODING": "latin-1", } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) assert data[fmt] == expected - @inlineCallbacks - def test_export_multiple_configs(self): + @deferred_f_from_coro_f + async def test_export_multiple_configs(self): items = [{"foo": "FOO", "bar": "BAR"}] formats = { @@ -1417,12 +1502,12 @@ class TestFeedExport(TestFeedExportBase): }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): assert data[fmt] == expected - @inlineCallbacks - def test_export_indentation(self): + @deferred_f_from_coro_f + async def test_export_indentation(self): items = [ {"foo": ["bar"]}, {"key": "value"}, @@ -1574,11 +1659,11 @@ class TestFeedExport(TestFeedExportBase): }, }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) assert data[row["format"]] == row["expected"] - @inlineCallbacks - def test_init_exporters_storages_with_crawler(self): + @deferred_f_from_coro_f + async def test_init_exporters_storages_with_crawler(self): settings = { "FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter}, "FEED_STORAGES": {"file": FromCrawlerFileFeedStorage}, @@ -1586,21 +1671,21 @@ class TestFeedExport(TestFeedExportBase): self._random_temp_filename(): {"format": "csv"}, }, } - yield self.exported_data(items=[], settings=settings) + await self.exported_data(items=[], settings=settings) assert FromCrawlerCsvItemExporter.init_with_crawler assert FromCrawlerFileFeedStorage.init_with_crawler - @inlineCallbacks - def test_str_uri(self): + @deferred_f_from_coro_f + async def test_str_uri(self): settings = { "FEED_STORE_EMPTY": True, "FEEDS": {str(self._random_temp_filename()): {"format": "csv"}}, } - data = yield self.exported_no_data(settings) + data = await self.exported_no_data(settings) assert data["csv"] == b"" - @inlineCallbacks - def test_multiple_feeds_success_logs_blocking_feed_storage(self): + @deferred_f_from_coro_f + async def test_multiple_feeds_success_logs_blocking_feed_storage(self): settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, @@ -1614,14 +1699,14 @@ class TestFeedExport(TestFeedExportBase): {"foo": "bar2", "baz": "quux"}, ] with LogCapture() as log: - yield self.exported_data(items, settings) + await self.exported_data(items, settings) print(log) for fmt in ["json", "xml", "csv"]: assert f"Stored {fmt} feed (2 items)" in str(log) - @inlineCallbacks - def test_multiple_feeds_failing_logs_blocking_feed_storage(self): + @deferred_f_from_coro_f + async def test_multiple_feeds_failing_logs_blocking_feed_storage(self): settings = { "FEEDS": { self._random_temp_filename(): {"format": "json"}, @@ -1635,14 +1720,14 @@ class TestFeedExport(TestFeedExportBase): {"foo": "bar2", "baz": "quux"}, ] with LogCapture() as log: - yield self.exported_data(items, settings) + await self.exported_data(items, settings) print(log) for fmt in ["json", "xml", "csv"]: assert f"Error storing {fmt} feed (2 items)" in str(log) - @inlineCallbacks - def test_extend_kwargs(self): + @deferred_f_from_coro_f + async def test_extend_kwargs(self): items = [{"foo": "FOO", "bar": "BAR"}] expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n" @@ -1675,11 +1760,11 @@ class TestFeedExport(TestFeedExportBase): "FEED_EXPORT_INDENT": None, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) assert data[feed_options["format"]] == row["expected"] - @inlineCallbacks - def test_storage_file_no_postprocessing(self): + @deferred_f_from_coro_f + async def test_storage_file_no_postprocessing(self): @implementer(IFeedStorage) class Storage: def __init__(self, uri, *, feed_options=None): @@ -1697,11 +1782,11 @@ class TestFeedExport(TestFeedExportBase): "FEEDS": {self._random_temp_filename(): {"format": "jsonlines"}}, "FEED_STORAGES": {"file": Storage}, } - yield self.exported_no_data(settings) + await self.exported_no_data(settings) assert Storage.open_file is Storage.store_file - @inlineCallbacks - def test_storage_file_postprocessing(self): + @deferred_f_from_coro_f + async def test_storage_file_postprocessing(self): @implementer(IFeedStorage) class Storage: def __init__(self, uri, *, feed_options=None): @@ -1727,7 +1812,7 @@ class TestFeedExport(TestFeedExportBase): }, "FEED_STORAGES": {"file": Storage}, } - yield self.exported_no_data(settings) + await self.exported_no_data(settings) assert Storage.open_file is Storage.store_file assert not Storage.file_was_closed @@ -1753,8 +1838,9 @@ class TestFeedPostProcessedExports(TestFeedExportBase): def _named_tempfile(self, name) -> str: return str(Path(self.temp_dir, name)) - @inlineCallbacks - def run_and_export(self, spider_cls, settings): + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, bytes | None]: """Run spider with specified settings; return exported data with filename.""" FEEDS = settings.get("FEEDS") or {} @@ -1763,11 +1849,11 @@ class TestFeedPostProcessedExports(TestFeedExportBase): for file_path, feed_options in FEEDS.items() } - content = {} + content: dict[str, bytes | None] = {} try: spider_cls.start_urls = [self.mockserver.url("/")] crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + await maybe_deferred_to_future(crawler.crawl()) for file_path in FEEDS: content[str(file_path)] = ( @@ -1797,8 +1883,8 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data_stream.seek(0) return data_stream.read() - @inlineCallbacks - def test_gzip_plugin(self): + @deferred_f_from_coro_f + async def test_gzip_plugin(self): filename = self._named_tempfile("gzip_file") settings = { @@ -1810,14 +1896,14 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) try: gzip.decompress(data[filename]) except OSError: pytest.fail("Received invalid gzip data.") - @inlineCallbacks - def test_gzip_plugin_compresslevel(self): + @deferred_f_from_coro_f + async def test_gzip_plugin_compresslevel(self): filename_to_compressed = { self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( self.expected, compresslevel=0 @@ -1846,15 +1932,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = gzip.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_gzip_plugin_mtime(self): + @deferred_f_from_coro_f + async def test_gzip_plugin_mtime(self): filename_to_compressed = { self._named_tempfile("mtime_123"): self.get_gzip_compressed( self.expected, mtime=123 @@ -1881,15 +1967,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = gzip.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_gzip_plugin_filename(self): + @deferred_f_from_coro_f + async def test_gzip_plugin_filename(self): filename_to_compressed = { self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( self.expected, filename="FILE1" @@ -1916,15 +2002,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = gzip.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_lzma_plugin(self): + @deferred_f_from_coro_f + async def test_lzma_plugin(self): filename = self._named_tempfile("lzma_file") settings = { @@ -1936,14 +2022,14 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) try: lzma.decompress(data[filename]) except lzma.LZMAError: pytest.fail("Received invalid lzma data.") - @inlineCallbacks - def test_lzma_plugin_format(self): + @deferred_f_from_coro_f + async def test_lzma_plugin_format(self): filename_to_compressed = { self._named_tempfile("format_FORMAT_XZ"): lzma.compress( self.expected, format=lzma.FORMAT_XZ @@ -1968,15 +2054,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = lzma.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_lzma_plugin_check(self): + @deferred_f_from_coro_f + async def test_lzma_plugin_check(self): filename_to_compressed = { self._named_tempfile("check_CHECK_NONE"): lzma.compress( self.expected, check=lzma.CHECK_NONE @@ -2001,15 +2087,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = lzma.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_lzma_plugin_preset(self): + @deferred_f_from_coro_f + async def test_lzma_plugin_preset(self): filename_to_compressed = { self._named_tempfile("preset_PRESET_0"): lzma.compress( self.expected, preset=0 @@ -2034,15 +2120,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = lzma.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_lzma_plugin_filters(self): + @deferred_f_from_coro_f + async def test_lzma_plugin_filters(self): if "PyPy" in sys.version: # https://foss.heptapod.net/pypy/pypy/-/issues/3527 pytest.skip("lzma filters doesn't work in PyPy") @@ -2061,13 +2147,13 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) assert compressed == data[filename] result = lzma.decompress(data[filename]) assert result == self.expected - @inlineCallbacks - def test_bz2_plugin(self): + @deferred_f_from_coro_f + async def test_bz2_plugin(self): filename = self._named_tempfile("bz2_file") settings = { @@ -2079,14 +2165,14 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) try: bz2.decompress(data[filename]) except OSError: pytest.fail("Received invalid bz2 data.") - @inlineCallbacks - def test_bz2_plugin_compresslevel(self): + @deferred_f_from_coro_f + async def test_bz2_plugin_compresslevel(self): filename_to_compressed = { self._named_tempfile("compresslevel_1"): bz2.compress( self.expected, compresslevel=1 @@ -2111,15 +2197,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, compressed in filename_to_compressed.items(): result = bz2.decompress(data[filename]) assert compressed == data[filename] assert result == self.expected - @inlineCallbacks - def test_custom_plugin(self): + @deferred_f_from_coro_f + async def test_custom_plugin(self): filename = self._named_tempfile("csv_file") settings = { @@ -2131,11 +2217,11 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) assert data[filename] == self.expected - @inlineCallbacks - def test_custom_plugin_with_parameter(self): + @deferred_f_from_coro_f + async def test_custom_plugin_with_parameter(self): expected = b"foo\r\n\nbar\r\n\n" filename = self._named_tempfile("newline") @@ -2149,11 +2235,11 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) assert data[filename] == expected - @inlineCallbacks - def test_custom_plugin_with_compression(self): + @deferred_f_from_coro_f + async def test_custom_plugin_with_compression(self): expected = b"foo\r\n\nbar\r\n\n" filename_to_decompressor = { @@ -2191,14 +2277,14 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, decompressor in filename_to_decompressor.items(): result = decompressor(data[filename]) assert result == expected - @inlineCallbacks - def test_exports_compatibility_with_postproc(self): + @deferred_f_from_coro_f + async def test_exports_compatibility_with_postproc(self): import marshal import pickle @@ -2240,7 +2326,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): }, } - data = yield self.exported_data(self.items, settings) + data = await self.exported_data(self.items, settings) for filename, result in data.items(): if "pickle" in filename: @@ -2255,18 +2341,19 @@ class TestFeedPostProcessedExports(TestFeedExportBase): class TestBatchDeliveries(TestFeedExportBase): _file_mark = "_%(batch_time)s_#%(batch_id)02d_" - @inlineCallbacks - def run_and_export(self, spider_cls, settings): + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, list[bytes]]: """Run spider with specified settings; return exported data.""" FEEDS = settings.get("FEEDS") or {} settings["FEEDS"] = { build_url(file_path): feed for file_path, feed in FEEDS.items() } - content = defaultdict(list) + content: defaultdict[str, list[bytes]] = defaultdict(list) spider_cls.start_urls = [self.mockserver.url("/")] crawler = get_crawler(spider_cls, settings) - yield crawler.crawl() + await maybe_deferred_to_future(crawler.crawl()) for path, feed in FEEDS.items(): dir_name = Path(path).parent @@ -2277,8 +2364,7 @@ class TestBatchDeliveries(TestFeedExportBase): content[feed["format"]].append(file.read_bytes()) return content - @inlineCallbacks - def assertExportedJsonLines(self, items, rows, settings=None): + async def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} settings.update( { @@ -2291,7 +2377,7 @@ class TestBatchDeliveries(TestFeedExportBase): ) batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for batch in data["jl"]: got_batch = [ json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines() @@ -2299,8 +2385,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None): + async def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update( { @@ -2312,15 +2397,14 @@ class TestBatchDeliveries(TestFeedExportBase): } ) batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for batch in data["csv"]: got_batch = csv.DictReader(to_unicode(batch).splitlines()) assert list(header) == got_batch.fieldnames expected_batch, rows = rows[:batch_size], rows[batch_size:] assert list(got_batch) == expected_batch - @inlineCallbacks - def assertExportedXml(self, items, rows, settings=None): + async def assertExportedXml(self, items, rows, settings=None): settings = settings or {} settings.update( { @@ -2333,15 +2417,14 @@ class TestBatchDeliveries(TestFeedExportBase): ) batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for batch in data["xml"]: root = lxml.etree.fromstring(batch) got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @inlineCallbacks - def assertExportedMultiple(self, items, rows, settings=None): + async def assertExportedMultiple(self, items, rows, settings=None): settings = settings or {} settings.update( { @@ -2357,7 +2440,7 @@ class TestBatchDeliveries(TestFeedExportBase): ) batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) # XML xml_rows = rows.copy() for batch in data["xml"]: @@ -2372,8 +2455,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] assert got_batch == expected_batch - @inlineCallbacks - def assertExportedPickle(self, items, rows, settings=None): + async def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} settings.update( { @@ -2386,7 +2468,7 @@ class TestBatchDeliveries(TestFeedExportBase): ) batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) import pickle for batch in data["pickle"]: @@ -2394,8 +2476,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @inlineCallbacks - def assertExportedMarshal(self, items, rows, settings=None): + async def assertExportedMarshal(self, items, rows, settings=None): settings = settings or {} settings.update( { @@ -2408,7 +2489,7 @@ class TestBatchDeliveries(TestFeedExportBase): ) batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") rows = [{k: v for k, v in row.items() if v} for row in rows] - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) import marshal for batch in data["marshal"]: @@ -2416,8 +2497,8 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @inlineCallbacks - def test_export_items(self): + @deferred_f_from_coro_f + async def test_export_items(self): """Test partial deliveries in all supported formats""" items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -2431,7 +2512,7 @@ class TestBatchDeliveries(TestFeedExportBase): ] settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2} header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, settings=settings) + await self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): """If path is without %(batch_time)s and %(batch_id) an exception must be raised""" @@ -2445,8 +2526,8 @@ class TestBatchDeliveries(TestFeedExportBase): with pytest.raises(NotConfigured): FeedExporter(crawler) - @inlineCallbacks - def test_export_no_items_not_store_empty(self): + @deferred_f_from_coro_f + async def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { "FEEDS": { @@ -2457,12 +2538,12 @@ class TestBatchDeliveries(TestFeedExportBase): "FEED_EXPORT_BATCH_ITEM_COUNT": 1, "FEED_STORE_EMPTY": False, } - data = yield self.exported_no_data(settings) + data = await self.exported_no_data(settings) data = dict(data) assert len(data[fmt]) == 0 - @inlineCallbacks - def test_export_no_items_store_empty(self): + @deferred_f_from_coro_f + async def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), ("jsonlines", b""), @@ -2481,12 +2562,12 @@ class TestBatchDeliveries(TestFeedExportBase): "FEED_EXPORT_INDENT": None, "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } - data = yield self.exported_no_data(settings) + data = await self.exported_no_data(settings) data = dict(data) assert data[fmt][0] == expctd - @inlineCallbacks - def test_export_multiple_configs(self): + @deferred_f_from_coro_f + async def test_export_multiple_configs(self): items = [ {"foo": "FOO", "bar": "BAR"}, {"foo": "FOO1", "bar": "BAR1"}, @@ -2536,13 +2617,13 @@ class TestBatchDeliveries(TestFeedExportBase): }, "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): for expected_batch, got_batch in zip(expected, data[fmt]): assert got_batch == expected_batch - @inlineCallbacks - def test_batch_item_count_feeds_setting(self): + @deferred_f_from_coro_f + async def test_batch_item_count_feeds_setting(self): items = [{"foo": "FOO"}, {"foo": "FOO1"}] formats = { "json": [ @@ -2560,13 +2641,13 @@ class TestBatchDeliveries(TestFeedExportBase): }, }, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) for fmt, expected in formats.items(): for expected_batch, got_batch in zip(expected, data[fmt]): assert got_batch == expected_batch - @inlineCallbacks - def test_batch_path_differ(self): + @deferred_f_from_coro_f + async def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. So %(batch_id)d replaced with the current id. @@ -2584,7 +2665,7 @@ class TestBatchDeliveries(TestFeedExportBase): }, "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } - data = yield self.exported_data(items, settings) + data = await self.exported_data(items, settings) assert len(items) == len(data["json"]) @inlineCallbacks diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 6ebaa19ce..28ffbe767 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -8,7 +8,6 @@ from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase from scrapy.core.spidermw import SpiderMiddlewareManager @@ -18,7 +17,6 @@ from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import ( deferred_f_from_coro_f, - deferred_from_coro, maybe_deferred_to_future, ) from scrapy.utils.test import get_crawler @@ -130,25 +128,22 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): yield {"foo": 2} yield {"foo": 3} - @inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: int | None = None): + async def _get_middleware_result(self, *mw_classes, start_index: int | None = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) self.crawler = get_crawler( Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting} ) self.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - result = yield self.mwman.scrape_response( + return await self.mwman.scrape_response_async( self._scrape_func, self.response, self.request, self.spider ) - return result - @inlineCallbacks - def _test_simple_base( + async def _test_simple_base( self, *mw_classes, downgrade: bool = False, start_index: int | None = None ): with LogCapture() as log: - result = yield self._get_middleware_result( + result = await self._get_middleware_result( *mw_classes, start_index=start_index ) assert isinstance(result, Iterable) @@ -160,16 +155,15 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): ProcessSpiderOutputSimpleMiddleware in mw_classes ) - @inlineCallbacks - def _test_asyncgen_base( + async def _test_asyncgen_base( self, *mw_classes, downgrade: bool = False, start_index: int | None = None ): with LogCapture() as log: - result = yield self._get_middleware_result( + result = await self._get_middleware_result( *mw_classes, start_index=start_index ) assert isinstance(result, AsyncIterator) - result_list = yield deferred_from_coro(collect_asyncgen(result)) + result_list = await collect_asyncgen(result) assert len(result_list) == self.RESULT_COUNT assert isinstance(result_list[0], self.ITEM_TYPE) assert ("downgraded to a non-async" in str(log)) == downgrade @@ -222,41 +216,50 @@ class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware - def test_simple(self): + @deferred_f_from_coro_f + async def test_simple(self): """Simple mw""" - return self._test_simple_base(self.MW_SIMPLE) + await self._test_simple_base(self.MW_SIMPLE) - def test_asyncgen(self): + @deferred_f_from_coro_f + async def test_asyncgen(self): """Asyncgen mw; upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN) + await self._test_asyncgen_base(self.MW_ASYNCGEN) - def test_simple_asyncgen(self): + @deferred_f_from_coro_f + async def test_simple_asyncgen(self): """Simple mw -> asyncgen mw; upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE) + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE) - def test_asyncgen_simple(self): + @deferred_f_from_coro_f + async def test_asyncgen_simple(self): """Asyncgen mw -> simple mw; upgrade then downgrade""" - return self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True) + await self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True) - def test_universal(self): + @deferred_f_from_coro_f + async def test_universal(self): """Universal mw""" - return self._test_simple_base(self.MW_UNIVERSAL) + await self._test_simple_base(self.MW_UNIVERSAL) - def test_universal_simple(self): + @deferred_f_from_coro_f + async def test_universal_simple(self): """Universal mw -> simple mw""" - return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL) + await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL) - def test_simple_universal(self): + @deferred_f_from_coro_f + async def test_simple_universal(self): """Simple mw -> universal mw""" - return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE) + await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE) - def test_universal_asyncgen(self): + @deferred_f_from_coro_f + async def test_universal_asyncgen(self): """Universal mw -> asyncgen mw; upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) - def test_asyncgen_universal(self): + @deferred_f_from_coro_f + async def test_asyncgen_universal(self): """Asyncgen mw -> universal mw; upgrade""" - return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) + await self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): @@ -266,27 +269,30 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): for item in super()._scrape_func(): yield item - def test_simple(self): + @deferred_f_from_coro_f + async def test_simple(self): """Simple mw; downgrade""" - return self._test_simple_base(self.MW_SIMPLE, downgrade=True) + await self._test_simple_base(self.MW_SIMPLE, downgrade=True) - def test_simple_asyncgen(self): + @deferred_f_from_coro_f + async def test_simple_asyncgen(self): """Simple mw -> asyncgen mw; downgrade then upgrade""" - return self._test_asyncgen_base( - self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True - ) + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True) - def test_universal(self): + @deferred_f_from_coro_f + async def test_universal(self): """Universal mw""" - return self._test_asyncgen_base(self.MW_UNIVERSAL) + await self._test_asyncgen_base(self.MW_UNIVERSAL) - def test_universal_simple(self): + @deferred_f_from_coro_f + async def test_universal_simple(self): """Universal mw -> simple mw; downgrade""" - return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True) + await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True) - def test_simple_universal(self): + @deferred_f_from_coro_f + async def test_simple_universal(self): """Simple mw -> universal mw; downgrade""" - return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True) + await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True) class ProcessSpiderOutputNonIterableMiddleware: @@ -300,25 +306,21 @@ class ProcessSpiderOutputCoroutineMiddleware: class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): - @inlineCallbacks - def test_non_iterable(self): + @deferred_f_from_coro_f + async def test_non_iterable(self): with pytest.raises( _InvalidOutput, match=r"\.process_spider_output must return an iterable, got ", ): - yield self._get_middleware_result( - ProcessSpiderOutputNonIterableMiddleware, - ) + await self._get_middleware_result(ProcessSpiderOutputNonIterableMiddleware) - @inlineCallbacks - def test_coroutine(self): + @deferred_f_from_coro_f + async def test_coroutine(self): with pytest.raises( _InvalidOutput, match=r"\.process_spider_output must be an asynchronous generator", ): - yield self._get_middleware_result( - ProcessSpiderOutputCoroutineMiddleware, - ) + await self._get_middleware_result(ProcessSpiderOutputCoroutineMiddleware) class ProcessStartSimpleMiddleware: @@ -445,39 +447,44 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware - @inlineCallbacks - def _get_middleware_result(self, *mw_classes, start_index: int | None = None): + async def _get_middleware_result(self, *mw_classes, start_index: int | None = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting}) self.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - result = yield self.mwman.scrape_response( + return await self.mwman.scrape_response_async( self._scrape_func, self.response, self.request, self.spider ) - return result - def test_just_builtin(self): - return self._test_simple_base() + @deferred_f_from_coro_f + async def test_just_builtin(self): + await self._test_simple_base() - def test_builtin_simple(self): - return self._test_simple_base(self.MW_SIMPLE, start_index=1000) + @deferred_f_from_coro_f + async def test_builtin_simple(self): + await self._test_simple_base(self.MW_SIMPLE, start_index=1000) - def test_builtin_async(self): + @deferred_f_from_coro_f + async def test_builtin_async(self): """Upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) + await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) - def test_builtin_universal(self): - return self._test_simple_base(self.MW_UNIVERSAL, start_index=1000) + @deferred_f_from_coro_f + async def test_builtin_universal(self): + await self._test_simple_base(self.MW_UNIVERSAL, start_index=1000) - def test_simple_builtin(self): - return self._test_simple_base(self.MW_SIMPLE) + @deferred_f_from_coro_f + async def test_simple_builtin(self): + await self._test_simple_base(self.MW_SIMPLE) - def test_async_builtin(self): + @deferred_f_from_coro_f + async def test_async_builtin(self): """Upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN) + await self._test_asyncgen_base(self.MW_ASYNCGEN) - def test_universal_builtin(self): - return self._test_simple_base(self.MW_UNIVERSAL) + @deferred_f_from_coro_f + async def test_universal_builtin(self): + await self._test_simple_base(self.MW_UNIVERSAL) class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): @@ -485,28 +492,35 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): for item in super()._scrape_func(): yield item - def test_just_builtin(self): - return self._test_asyncgen_base() + @deferred_f_from_coro_f + async def test_just_builtin(self): + await self._test_asyncgen_base() - def test_builtin_simple(self): + @deferred_f_from_coro_f + async def test_builtin_simple(self): """Downgrade""" - return self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000) + await self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000) - def test_builtin_async(self): - return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) + @deferred_f_from_coro_f + async def test_builtin_async(self): + await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) - def test_builtin_universal(self): - return self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) + @deferred_f_from_coro_f + async def test_builtin_universal(self): + await self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) - def test_simple_builtin(self): + @deferred_f_from_coro_f + async def test_simple_builtin(self): """Downgrade""" - return self._test_simple_base(self.MW_SIMPLE, downgrade=True) + await self._test_simple_base(self.MW_SIMPLE, downgrade=True) - def test_async_builtin(self): - return self._test_asyncgen_base(self.MW_ASYNCGEN) + @deferred_f_from_coro_f + async def test_async_builtin(self): + await self._test_asyncgen_base(self.MW_ASYNCGEN) - def test_universal_builtin(self): - return self._test_asyncgen_base(self.MW_UNIVERSAL) + @deferred_f_from_coro_f + async def test_universal_builtin(self): + await self._test_asyncgen_base(self.MW_UNIVERSAL) class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): @@ -520,33 +534,38 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): def _scrape_func(self, *args, **kwargs): 1 / 0 - @inlineCallbacks - def _test_asyncgen_nodowngrade(self, *mw_classes): + async def _test_asyncgen_nodowngrade(self, *mw_classes): with pytest.raises( _InvalidOutput, match="Async iterable returned from .+ cannot be downgraded" ): - yield self._get_middleware_result(*mw_classes) + await self._get_middleware_result(*mw_classes) - def test_exc_simple(self): + @deferred_f_from_coro_f + async def test_exc_simple(self): """Simple exc mw""" - return self._test_simple_base(self.MW_EXC_SIMPLE) + await self._test_simple_base(self.MW_EXC_SIMPLE) - def test_exc_async(self): + @deferred_f_from_coro_f + async def test_exc_async(self): """Async exc mw""" - return self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) + await self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) - def test_exc_simple_simple(self): + @deferred_f_from_coro_f + async def test_exc_simple_simple(self): """Simple exc mw -> simple output mw""" - return self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE) + await self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE) - def test_exc_async_async(self): + @deferred_f_from_coro_f + async def test_exc_async_async(self): """Async exc mw -> async output mw""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN) + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN) - def test_exc_simple_async(self): + @deferred_f_from_coro_f + async def test_exc_simple_async(self): """Simple exc mw -> async output mw; upgrade""" - return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE) + await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE) - def test_exc_async_simple(self): + @deferred_f_from_coro_f + async def test_exc_async_simple(self): """Async exc mw -> simple output mw; cannot work as downgrading is not supported""" - return self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN) + await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 62ec1a624..60464d696 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -1,8 +1,8 @@ from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase from scrapy import Request, Spider +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.mockserver import MockServer @@ -299,6 +299,8 @@ class NotGeneratorOutputChainSpider(Spider): # ================================================================================ class TestSpiderMiddleware(TestCase): + mockserver: MockServer + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -308,53 +310,52 @@ class TestSpiderMiddleware(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @inlineCallbacks - def crawl_log(self, spider): + async def crawl_log(self, spider: type[Spider]) -> LogCapture: crawler = get_crawler(spider) with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + await maybe_deferred_to_future(crawler.crawl(mockserver=self.mockserver)) return log - @inlineCallbacks - def test_recovery(self): + @deferred_f_from_coro_f + async def test_recovery(self): """ (0) Recover from an exception in a spider callback. The final item count should be 3 (one yielded from the callback method before the exception is raised, one directly from the recovery middleware and one from the spider when processing the request that was enqueued from the recovery middleware) """ - log = yield self.crawl_log(RecoverySpider) + log = await self.crawl_log(RecoverySpider) assert "Middleware: TabError exception caught" in str(log) assert str(log).count("Middleware: TabError exception caught") == 1 assert "'item_scraped_count': 3" in str(log) - @inlineCallbacks - def test_recovery_asyncgen(self): + @deferred_f_from_coro_f + async def test_recovery_asyncgen(self): """ Same as test_recovery but with an async callback. """ - log = yield self.crawl_log(RecoveryAsyncGenSpider) + log = await self.crawl_log(RecoveryAsyncGenSpider) assert "Middleware: TabError exception caught" in str(log) assert str(log).count("Middleware: TabError exception caught") == 1 assert "'item_scraped_count': 3" in str(log) - @inlineCallbacks - def test_process_spider_input_without_errback(self): + @deferred_f_from_coro_f + async def test_process_spider_input_without_errback(self): """ (1.1) An exception from the process_spider_input chain should be caught by the process_spider_exception chain from the start if the Request has no errback """ - log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) + log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) assert "Middleware: will raise IndexError" in str(log1) assert "Middleware: IndexError exception caught" in str(log1) - @inlineCallbacks - def test_process_spider_input_with_errback(self): + @deferred_f_from_coro_f + async def test_process_spider_input_with_errback(self): """ (1.2) An exception from the process_spider_input chain should not be caught by the process_spider_exception chain if the Request has an errback """ - log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback) + log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback) assert "Middleware: IndexError exception caught" not in str(log1) assert "Middleware: will raise IndexError" in str(log1) assert "Got a Failure on the Request errback" in str(log1) @@ -362,60 +363,60 @@ class TestSpiderMiddleware(TestCase): assert "{'from': 'callback'}" not in str(log1) assert "'item_scraped_count': 1" in str(log1) - @inlineCallbacks - def test_generator_callback(self): + @deferred_f_from_coro_f + async def test_generator_callback(self): """ (2) An exception from a spider callback (returning a generator) should be caught by the process_spider_exception chain. Items yielded before the exception is raised should be processed normally. """ - log2 = yield self.crawl_log(GeneratorCallbackSpider) + log2 = await self.crawl_log(GeneratorCallbackSpider) assert "Middleware: ImportError exception caught" in str(log2) assert "'item_scraped_count': 2" in str(log2) - @inlineCallbacks - def test_async_generator_callback(self): + @deferred_f_from_coro_f + async def test_async_generator_callback(self): """ Same as test_generator_callback but with an async callback. """ - log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider) + log2 = await self.crawl_log(AsyncGeneratorCallbackSpider) assert "Middleware: ImportError exception caught" in str(log2) assert "'item_scraped_count': 2" in str(log2) - @inlineCallbacks - def test_generator_callback_right_after_callback(self): + @deferred_f_from_coro_f + async def test_generator_callback_right_after_callback(self): """ (2.1) Special case of (2): Exceptions should be caught even if the middleware is placed right after the spider """ - log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) + log21 = await self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) assert "Middleware: ImportError exception caught" in str(log21) assert "'item_scraped_count': 2" in str(log21) - @inlineCallbacks - def test_not_a_generator_callback(self): + @deferred_f_from_coro_f + async def test_not_a_generator_callback(self): """ (3) An exception from a spider callback (returning a list) should be caught by the process_spider_exception chain. No items should be processed. """ - log3 = yield self.crawl_log(NotGeneratorCallbackSpider) + log3 = await self.crawl_log(NotGeneratorCallbackSpider) assert "Middleware: ZeroDivisionError exception caught" in str(log3) assert "item_scraped_count" not in str(log3) - @inlineCallbacks - def test_not_a_generator_callback_right_after_callback(self): + @deferred_f_from_coro_f + async def test_not_a_generator_callback_right_after_callback(self): """ (3.1) Special case of (3): Exceptions should be caught even if the middleware is placed right after the spider """ - log31 = yield self.crawl_log( + log31 = await self.crawl_log( NotGeneratorCallbackSpiderMiddlewareRightAfterSpider ) assert "Middleware: ZeroDivisionError exception caught" in str(log31) assert "item_scraped_count" not in str(log31) - @inlineCallbacks - def test_generator_output_chain(self): + @deferred_f_from_coro_f + async def test_generator_output_chain(self): """ (4) An exception from a middleware's process_spider_output method should be sent to the process_spider_exception method from the next middleware in the chain. @@ -424,7 +425,7 @@ class TestSpiderMiddleware(TestCase): The final item count should be 2 (one from the spider callback and one from the process_spider_exception chain) """ - log4 = yield self.crawl_log(GeneratorOutputChainSpider) + log4 = await self.crawl_log(GeneratorOutputChainSpider) assert "'item_scraped_count': 2" in str(log4) assert ( "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught" @@ -461,8 +462,8 @@ class TestSpiderMiddleware(TestCase): assert str(item_recovered) in str(log4) assert "parse-second-item" not in str(log4) - @inlineCallbacks - def test_not_a_generator_output_chain(self): + @deferred_f_from_coro_f + async def test_not_a_generator_output_chain(self): """ (5) An exception from a middleware's process_spider_output method should be sent to the process_spider_exception method from the next middleware in the chain. @@ -471,7 +472,7 @@ class TestSpiderMiddleware(TestCase): The final item count should be 1 (from the process_spider_exception chain, the items from the spider callback are lost) """ - log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) + log5 = await self.crawl_log(NotGeneratorOutputChainSpider) assert "'item_scraped_count': 1" in str(log5) assert ( "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught" From b4d11b8b2565b5686c9f395ca1a8dd085609aafc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 11 Jun 2025 04:28:09 +0500 Subject: [PATCH 1777/2083] Further reduce deps on unittest. (#6884) --- .../test_downloader_handler_twisted_http11.py | 10 +- .../test_downloader_handler_twisted_http2.py | 14 +- tests/test_downloader_handlers.py | 7 +- tests/test_downloadermiddleware_robotstxt.py | 4 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_engine_loop.py | 4 +- tests/test_extension_telnet.py | 2 +- tests/test_feedexport.py | 43 +- tests/test_http2_client_protocol.py | 480 ++++++++---------- tests/test_pipeline_crawl.py | 15 +- tests/test_pipeline_files.py | 16 +- tests/test_pipeline_images.py | 17 +- tests/test_pipeline_media.py | 2 +- tests/test_scheduler_base.py | 6 +- tests/test_signals.py | 4 +- tests/test_spider_start.py | 2 +- tests/test_spidermiddleware_process_start.py | 4 +- tests/test_utils_defer.py | 6 +- tests/test_utils_signal.py | 16 +- tests/test_webclient.py | 89 ++-- 20 files changed, 355 insertions(+), 388 deletions(-) diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index 70f55e787..7b26ce03f 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -39,23 +39,21 @@ class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase): pass -class Https11WrongHostnameTestCase( - HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase -): +class TestHttps11WrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class Https11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttps11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class Https11InvalidDNSPattern( +class TestHttps11InvalidDNSPattern( HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase ): pass -class Https11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttps11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): pass diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index e058cedae..3e685bb28 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -163,23 +163,25 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): assert json.loads(response.text)["headers"][header] == [value1, value2] -class Https2WrongHostnameTestCase(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): +class TestHttps2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class Https2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttps2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): pass -class Https2InvalidDNSPattern(H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase): +class TestHttps2InvalidDNSPattern( + H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase +): pass -class Https2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttps2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): pass -class Http2MockServerTestCase(TestHttpMockServerBase): +class TestHttp2MockServer(TestHttpMockServerBase): """HTTP 2.0 test case with MockServer""" @property @@ -193,7 +195,7 @@ class Http2MockServerTestCase(TestHttpMockServerBase): is_secure = True -class Https2ProxyTestCase(H2DownloadHandlerMixin, TestHttpProxyBase): +class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): # only used for HTTPS tests keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2c8e96040..518dc6b24 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -12,6 +12,7 @@ from unittest import mock import pytest from twisted.cred import checkers, credentials, portal +from twisted.internet.defer import inlineCallbacks from twisted.protocols.ftp import FTPFactory, FTPRealm from twisted.trial import unittest from w3lib.url import path_to_file_uri @@ -340,9 +341,10 @@ class TestFTPBase(unittest.TestCase): self.portNum = self.port.getHost().port crawler = get_crawler() self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) - self.addCleanup(self.port.stopListening) + @inlineCallbacks def tearDown(self): + yield self.port.stopListening() shutil.rmtree(self.directory) def _add_test_callbacks(self, deferred, callback=None, errback=None): @@ -478,9 +480,10 @@ class TestAnonymousFTP(TestFTPBase): self.portNum = self.port.getHost().port crawler = get_crawler() self.download_handler = build_from_crawler(FTPDownloadHandler, crawler) - self.addCleanup(self.port.stopListening) + @inlineCallbacks def tearDown(self): + yield self.port.stopListening() shutil.rmtree(self.directory) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 146b0057e..dd5d47cab 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -247,10 +247,8 @@ Disallow: /some/randome/page.html assert request.callback == NO_CALLBACK +@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed") class TestRobotsTxtMiddlewareWithRerp(TestRobotsTxtMiddleware): - if not rerp_available(): - skip = "Rerp parser is not installed" - def setUp(self): super().setUp() self.crawler.settings.set( diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9b7c09448..ddac95edf 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -49,7 +49,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): self.times[slot].append(time.time()) -class CrawlTestCase(TestCase): +class TestCrawl(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index c7dbc82d4..bfb8eeced 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -27,7 +27,7 @@ async def sleep(seconds: float = 0.001) -> None: await maybe_deferred_to_future(deferred) -class MainTestCase(TestCase): +class TestMain(TestCase): @deferred_f_from_coro_f async def test_sleep(self): """Neither asynchronous sleeps on Spider.start() nor the equivalent on @@ -119,7 +119,7 @@ class MainTestCase(TestCase): assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" -class RequestSendOrderTestCase(TestCase): +class TestRequestSendOrder(TestCase): seconds = 0.1 # increase if flaky @classmethod diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 2ac4d7830..f9e54cb28 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -8,7 +8,7 @@ from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler -class TelnetExtensionTest(unittest.TestCase): +class TestTelnetExtension(unittest.TestCase): def _get_console_and_portal(self, settings=None): crawler = get_crawler(settings_dict=settings) console = TelnetConsole(crawler) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 262c0b434..01797fd20 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -91,52 +91,53 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]: return (client_mock, bucket_mock, blob_mock) -# TODO: replace self.mktemp() and drop the unittest.TestCase base -class TestFileFeedStorage(unittest.TestCase): - def test_store_file_uri(self): - path = Path(self.mktemp()).resolve() +class TestFileFeedStorage: + def test_store_file_uri(self, tmp_path): + path = tmp_path / "file.txt" uri = path_to_file_uri(str(path)) self._assert_stores(FileFeedStorage(uri), path) - def test_store_file_uri_makedirs(self): - path = Path(self.mktemp()).resolve() / "more" / "paths" / "file.txt" + def test_store_file_uri_makedirs(self, tmp_path): + path = tmp_path / "more" / "paths" / "file.txt" uri = path_to_file_uri(str(path)) self._assert_stores(FileFeedStorage(uri), path) - def test_store_direct_path(self): - path = Path(self.mktemp()).resolve() + def test_store_direct_path(self, tmp_path): + path = tmp_path / "file.txt" self._assert_stores(FileFeedStorage(str(path)), path) - def test_store_direct_path_relative(self): - path = Path(self.mktemp()) + def test_store_direct_path_relative(self, tmp_path): + path = (tmp_path / "foo" / "bar").relative_to(Path.cwd()) self._assert_stores(FileFeedStorage(str(path)), path) - def test_interface(self): - path = self.mktemp() - st = FileFeedStorage(path) + def test_interface(self, tmp_path): + path = tmp_path / "file.txt" + st = FileFeedStorage(str(path)) verifyObject(IFeedStorage, st) - def _store(self, feed_options=None) -> Path: - path = Path(self.mktemp()).resolve() + @staticmethod + def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None: storage = FileFeedStorage(str(path), feed_options=feed_options) spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") storage.store(file) - return path - def test_append(self): - path = self._store() + def test_append(self, tmp_path): + path = tmp_path / "file.txt" + self._store(path) self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") - def test_overwrite(self): - path = self._store({"overwrite": True}) + def test_overwrite(self, tmp_path): + path = tmp_path / "file.txt" + self._store(path, {"overwrite": True}) self._assert_stores( FileFeedStorage(str(path), feed_options={"overwrite": True}), path ) + @staticmethod def _assert_stores( - self, storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" + storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" ) -> None: spider = scrapy.Spider("default") file = storage.open(spider) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index ef1806cc0..80edd50d6 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -8,7 +8,7 @@ import string from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Callable from unittest import mock from urllib.parse import urlencode @@ -32,17 +32,22 @@ from twisted.web.static import File from scrapy.http import JsonRequest, Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_from_coro, + maybe_deferred_to_future, +) from tests.mockserver import LeafResource, Status, ssl_context_factory if TYPE_CHECKING: - from twisted.python.failure import Failure + from collections.abc import Coroutine -def generate_random_string(size): +def generate_random_string(size: int) -> str: return "".join(random.choices(string.ascii_uppercase + string.digits, k=size)) -def make_html_body(val): +def make_html_body(val: str) -> bytes: response = f"""

Hello from HTTP2

{val}

@@ -92,7 +97,7 @@ class GetDataHtmlLarge(LeafResource): class PostDataJsonMixin: @staticmethod - def make_response(request: TxRequest, extra_data: str): + def make_response(request: TxRequest, extra_data: str) -> bytes: assert request.content is not None response = { "request-headers": {}, @@ -179,7 +184,6 @@ def get_client_certificate( pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text( encoding="utf-8" ) - return PrivateCertificate.loadPEM(pem) @@ -238,6 +242,7 @@ class TestHttps2ClientProtocol(TestCase): uri = URI.fromBytes(bytes(self.get_url("/"), "utf-8")) self.conn_closed_deferred = Deferred() + from scrapy.core.http2.protocol import H2ClientFactory h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) @@ -255,7 +260,7 @@ class TestHttps2ClientProtocol(TestCase): shutil.rmtree(self.temp_directory) self.conn_closed_deferred = None - def get_url(self, path): + def get_url(self, path: str) -> str: """ :param path: Should have / at the starting compulsorily if not empty :return: Complete url @@ -264,143 +269,146 @@ class TestHttps2ClientProtocol(TestCase): assert path[0] == "/" or path[0] == "&" return f"{self.scheme}://{self.hostname}:{self.port_number}{path}" - def make_request(self, request: Request) -> Deferred: + async def make_request(self, request: Request) -> Response: + return await maybe_deferred_to_future(self.make_request_dfd(request)) + + def make_request_dfd(self, request: Request) -> Deferred[Response]: return self.client.request(request, DummySpider()) @staticmethod - def _check_repeat(get_deferred, count): + async def _check_repeat( + get_coro: Callable[[], Coroutine[Any, Any, None]], count: int + ) -> None: d_list = [] for _ in range(count): - d = get_deferred() + d = deferred_from_coro(get_coro()) d_list.append(d) - return DeferredList(d_list, fireOnOneErrback=True) + await maybe_deferred_to_future(DeferredList(d_list, fireOnOneErrback=True)) - def _check_GET(self, request: Request, expected_body, expected_status): - def check_response(response: Response): - assert response.status == expected_status - assert response.body == expected_body - assert response.request == request + async def _check_GET( + self, request: Request, expected_body: bytes, expected_status: int + ) -> None: + response = await self.make_request(request) + assert response.status == expected_status + assert response.body == expected_body + assert response.request == request - content_length_header = response.headers.get("Content-Length") - assert content_length_header is not None - content_length = int(content_length_header) - assert len(response.body) == content_length + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) + assert len(response.body) == content_length - d = self.make_request(request) - d.addCallback(check_response) - d.addErrback(self.fail) - return d - - def test_GET_small_body(self): + @deferred_f_from_coro_f + async def test_GET_small_body(self): request = Request(self.get_url("/get-data-html-small")) - return self._check_GET(request, Data.HTML_SMALL, 200) + await self._check_GET(request, Data.HTML_SMALL, 200) - def test_GET_large_body(self): + @deferred_f_from_coro_f + async def test_GET_large_body(self): request = Request(self.get_url("/get-data-html-large")) - return self._check_GET(request, Data.HTML_LARGE, 200) + await self._check_GET(request, Data.HTML_LARGE, 200) - def _check_GET_x10(self, *args, **kwargs): - def get_deferred(): - return self._check_GET(*args, **kwargs) + async def _check_GET_x10( + self, request: Request, expected_body: bytes, expected_status: int + ) -> None: + async def get_coro() -> None: + await self._check_GET(request, expected_body, expected_status) - return self._check_repeat(get_deferred, 10) + await self._check_repeat(get_coro, 10) - def test_GET_small_body_x10(self): - return self._check_GET_x10( + @deferred_f_from_coro_f + async def test_GET_small_body_x10(self): + await self._check_GET_x10( Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200 ) - def test_GET_large_body_x10(self): - return self._check_GET_x10( + @deferred_f_from_coro_f + async def test_GET_large_body_x10(self): + await self._check_GET_x10( Request(self.get_url("/get-data-html-large")), Data.HTML_LARGE, 200 ) - def _check_POST_json( + async def _check_POST_json( self, request: Request, - expected_request_body, - expected_extra_data, + expected_request_body: dict[str, str], + expected_extra_data: str, expected_status: int, - ): - d = self.make_request(request) + ) -> None: + response = await self.make_request(request) - def assert_response(response: Response): - assert response.status == expected_status - assert response.request == request + assert response.status == expected_status + assert response.request == request - content_length_header = response.headers.get("Content-Length") - assert content_length_header is not None - content_length = int(content_length_header) - assert len(response.body) == content_length + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) + assert len(response.body) == content_length - # Parse the body - content_encoding_header = response.headers[b"Content-Encoding"] - assert content_encoding_header is not None - content_encoding = str(content_encoding_header, "utf-8") - body = json.loads(str(response.body, content_encoding)) - assert "request-body" in body - assert "extra-data" in body - assert "request-headers" in body + # Parse the body + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") + body = json.loads(str(response.body, content_encoding)) + assert "request-body" in body + assert "extra-data" in body + assert "request-headers" in body - request_body = body["request-body"] - assert request_body == expected_request_body + request_body = body["request-body"] + assert request_body == expected_request_body - extra_data = body["extra-data"] - assert extra_data == expected_extra_data + extra_data = body["extra-data"] + assert extra_data == expected_extra_data - # Check if headers were sent successfully - request_headers = body["request-headers"] - for k, v in request.headers.items(): - k_str = str(k, "utf-8") - assert k_str in request_headers - assert request_headers[k_str] == str(v[0], "utf-8") + # Check if headers were sent successfully + request_headers = body["request-headers"] + for k, v in request.headers.items(): + k_str = str(k, "utf-8") + assert k_str in request_headers + assert request_headers[k_str] == str(v[0], "utf-8") - d.addCallback(assert_response) - d.addErrback(self.fail) - return d - - def test_POST_small_json(self): + @deferred_f_from_coro_f + async def test_POST_small_json(self): request = JsonRequest( url=self.get_url("/post-data-json-small"), method="POST", data=Data.JSON_SMALL, ) - return self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200) + await self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200) - def test_POST_large_json(self): + @deferred_f_from_coro_f + async def test_POST_large_json(self): request = JsonRequest( url=self.get_url("/post-data-json-large"), method="POST", data=Data.JSON_LARGE, ) - return self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200) + await self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200) - def _check_POST_json_x10(self, *args, **kwargs): - def get_deferred(): - return self._check_POST_json(*args, **kwargs) + async def _check_POST_json_x10(self, *args, **kwargs): + async def get_coro() -> None: + await self._check_POST_json(*args, **kwargs) - return self._check_repeat(get_deferred, 10) + await self._check_repeat(get_coro, 10) - def test_POST_small_json_x10(self): + @deferred_f_from_coro_f + async def test_POST_small_json_x10(self): request = JsonRequest( url=self.get_url("/post-data-json-small"), method="POST", data=Data.JSON_SMALL, ) - return self._check_POST_json_x10( - request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200 - ) + await self._check_POST_json_x10(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200) - def test_POST_large_json_x10(self): + @deferred_f_from_coro_f + async def test_POST_large_json_x10(self): request = JsonRequest( url=self.get_url("/post-data-json-large"), method="POST", data=Data.JSON_LARGE, ) - return self._check_POST_json_x10( - request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200 - ) + await self._check_POST_json_x10(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200) @inlineCallbacks def test_invalid_negotiated_protocol(self): @@ -409,77 +417,59 @@ class TestHttps2ClientProtocol(TestCase): ): request = Request(url=self.get_url("/status?n=200")) with pytest.raises(ResponseFailed): - yield self.make_request(request) + yield self.make_request_dfd(request) + @inlineCallbacks def test_cancel_request(self): request = Request(url=self.get_url("/get-data-html-large")) - - def assert_response(response: Response): - assert response.status == 499 - assert response.request == request - - d = self.make_request(request) - d.addCallback(assert_response) - d.addErrback(self.fail) + d = self.make_request_dfd(request) d.cancel() + response = yield d + assert response.status == 499 + assert response.request == request - return d - - def test_download_maxsize_exceeded(self): + @deferred_f_from_coro_f + async def test_download_maxsize_exceeded(self): request = Request( url=self.get_url("/get-data-html-large"), meta={"download_maxsize": 1000} ) + with pytest.raises(CancelledError) as exc_info: + await self.make_request(request) + error_pattern = re.compile( + rf"Cancelling download of {request.url}: received response " + rf"size \(\d*\) larger than download max size \(1000\)" + ) + assert len(re.findall(error_pattern, str(exc_info.value))) == 1 - def assert_cancelled_error(failure): - assert isinstance(failure.value, CancelledError) - error_pattern = re.compile( - rf"Cancelling download of {request.url}: received response " - rf"size \(\d*\) larger than download max size \(1000\)" - ) - assert len(re.findall(error_pattern, str(failure.value))) == 1 - - d = self.make_request(request) - d.addCallback(self.fail) - d.addErrback(assert_cancelled_error) - return d - + @inlineCallbacks def test_received_dataloss_response(self): """In case when value of Header Content-Length != len(Received Data) ProtocolError is raised""" + from h2.exceptions import InvalidBodyLengthError + request = Request(url=self.get_url("/dataloss")) + with pytest.raises(ResponseFailed) as exc_info: + yield self.make_request_dfd(request) + assert len(exc_info.value.reasons) > 0 + assert any( + isinstance(error, InvalidBodyLengthError) + for error in exc_info.value.reasons + ) - def assert_failure(failure: Failure): - assert len(failure.value.reasons) > 0 - from h2.exceptions import InvalidBodyLengthError - - assert any( - isinstance(error, InvalidBodyLengthError) - for error in failure.value.reasons - ) - - d = self.make_request(request) - d.addCallback(self.fail) - d.addErrback(assert_failure) - return d - - def test_missing_content_length_header(self): + @deferred_f_from_coro_f + async def test_missing_content_length_header(self): request = Request(url=self.get_url("/no-content-length-header")) + response = await self.make_request(request) + assert response.status == 200 + assert response.body == Data.NO_CONTENT_LENGTH + assert response.request == request + assert "Content-Length" not in response.headers - def assert_content_length(response: Response): - assert response.status == 200 - assert response.body == Data.NO_CONTENT_LENGTH - assert response.request == request - assert "Content-Length" not in response.headers - - d = self.make_request(request) - d.addCallback(assert_content_length) - d.addErrback(self.fail) - return d - - @inlineCallbacks - def _check_log_warnsize(self, request, warn_pattern, expected_body): + async def _check_log_warnsize( + self, request: Request, warn_pattern: re.Pattern[str], expected_body: bytes + ) -> None: with self.assertLogs("scrapy.core.http2.stream", level="WARNING") as cm: - response = yield self.make_request(request) + response = await self.make_request(request) assert response.status == 200 assert response.request == request assert response.body == expected_body @@ -487,8 +477,8 @@ class TestHttps2ClientProtocol(TestCase): # Check the warning is raised only once for this request assert sum(len(re.findall(warn_pattern, log)) for log in cm.output) == 1 - @inlineCallbacks - def test_log_expected_warnsize(self): + @deferred_f_from_coro_f + async def test_log_expected_warnsize(self): request = Request( url=self.get_url("/get-data-html-large"), meta={"download_warnsize": 1000} ) @@ -497,10 +487,10 @@ class TestHttps2ClientProtocol(TestCase): rf"download warn size \(1000\) in request {request}" ) - yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) + await self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) - @inlineCallbacks - def test_log_received_warnsize(self): + @deferred_f_from_coro_f + async def test_log_received_warnsize(self): request = Request( url=self.get_url("/no-content-length-header"), meta={"download_warnsize": 10}, @@ -510,20 +500,22 @@ class TestHttps2ClientProtocol(TestCase): rf"warn size \(10\) in request {request}" ) - yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) + await self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) - def test_max_concurrent_streams(self): + @deferred_f_from_coro_f + async def test_max_concurrent_streams(self): """Send 500 requests at one to check if we can handle very large number of request. """ - def get_deferred(): - return self._check_GET( + async def get_coro() -> None: + await self._check_GET( Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200 ) - return self._check_repeat(get_deferred, 500) + await self._check_repeat(get_coro, 500) + @inlineCallbacks def test_inactive_stream(self): """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS by default is 100. After sending the first 100 requests we close the @@ -532,6 +524,7 @@ class TestHttps2ClientProtocol(TestCase): def assert_inactive_stream(failure): assert failure.check(ResponseFailed) is not None + from scrapy.core.http2.stream import InactiveStreamClosed assert any( @@ -540,14 +533,14 @@ class TestHttps2ClientProtocol(TestCase): # Send 100 request (we do not check the result) for _ in range(100): - d = self.make_request(Request(self.get_url("/get-data-html-small"))) + d = self.make_request_dfd(Request(self.get_url("/get-data-html-small"))) d.addBoth(lambda _: None) d_list.append(d) # Now send 10 extra request and save the response deferred in a list for _ in range(10): - d = self.make_request(Request(self.get_url("/get-data-html-small"))) - d.addCallback(self.fail) + d = self.make_request_dfd(Request(self.get_url("/get-data-html-small"))) + d.addCallback(lambda _: pytest.fail("This request should have failed")) d.addErrback(assert_inactive_stream) d_list.append(d) @@ -555,13 +548,15 @@ class TestHttps2ClientProtocol(TestCase): # with InactiveStreamClosed self.client.transport.loseConnection() - return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) + yield DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) - def test_invalid_request_type(self): + @deferred_f_from_coro_f + async def test_invalid_request_type(self): with pytest.raises(TypeError): - self.make_request("https://InvalidDataTypePassed.com") + await self.make_request("https://InvalidDataTypePassed.com") - def test_query_parameters(self): + @deferred_f_from_coro_f + async def test_query_parameters(self): params = { "a": generate_random_string(20), "b": generate_random_string(20), @@ -569,133 +564,96 @@ class TestHttps2ClientProtocol(TestCase): "d": generate_random_string(20), } request = Request(self.get_url(f"/query-params?{urlencode(params)}")) + response = await self.make_request(request) + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") + data = json.loads(str(response.body, content_encoding)) + assert data == params - def assert_query_params(response: Response): - content_encoding_header = response.headers[b"Content-Encoding"] - assert content_encoding_header is not None - content_encoding = str(content_encoding_header, "utf-8") - data = json.loads(str(response.body, content_encoding)) - assert data == params - - d = self.make_request(request) - d.addCallback(assert_query_params) - d.addErrback(self.fail) - - return d - - def test_status_codes(self): - def assert_response_status(response: Response, expected_status: int): - assert response.status == expected_status - - d_list = [] + @deferred_f_from_coro_f + async def test_status_codes(self): for status in [200, 404]: request = Request(self.get_url(f"/status?n={status}")) - d = self.make_request(request) - d.addCallback(assert_response_status, status) - d.addErrback(self.fail) - d_list.append(d) + response = await self.make_request(request) + assert response.status == status - return DeferredList(d_list, fireOnOneErrback=True) - - def test_response_has_correct_certificate_ip_address(self): + @deferred_f_from_coro_f + async def test_response_has_correct_certificate_ip_address(self): request = Request(self.get_url("/status?n=200")) + response = await self.make_request(request) + assert response.request == request + assert isinstance(response.certificate, Certificate) + assert response.certificate.original is not None + assert response.certificate.getIssuer() == self.client_certificate.getIssuer() + assert response.certificate.getPublicKey().matches( + self.client_certificate.getPublicKey() + ) + assert isinstance(response.ip_address, IPv4Address) + assert str(response.ip_address) == "127.0.0.1" - def assert_metadata(response: Response): - assert response.request == request - assert isinstance(response.certificate, Certificate) - assert response.certificate.original is not None - assert ( - response.certificate.getIssuer() == self.client_certificate.getIssuer() - ) - assert response.certificate.getPublicKey().matches( - self.client_certificate.getPublicKey() - ) + async def _check_invalid_netloc(self, url: str) -> None: + from scrapy.core.http2.stream import InvalidHostname - assert isinstance(response.ip_address, IPv4Address) - assert str(response.ip_address) == "127.0.0.1" - - d = self.make_request(request) - d.addCallback(assert_metadata) - d.addErrback(self.fail) - - return d - - def _check_invalid_netloc(self, url): request = Request(url) + with pytest.raises(InvalidHostname) as exc_info: + await self.make_request(request) + error_msg = str(exc_info.value) + assert "localhost" in error_msg + assert "127.0.0.1" in error_msg + assert str(request) in error_msg - def assert_invalid_hostname(failure: Failure): - from scrapy.core.http2.stream import InvalidHostname + @deferred_f_from_coro_f + async def test_invalid_hostname(self): + await self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain") - assert failure.check(InvalidHostname) is not None - error_msg = str(failure.value) - assert "localhost" in error_msg - assert "127.0.0.1" in error_msg - assert str(request) in error_msg - - d = self.make_request(request) - d.addCallback(self.fail) - d.addErrback(assert_invalid_hostname) - return d - - def test_invalid_hostname(self): - return self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain") - - def test_invalid_host_port(self): + @deferred_f_from_coro_f + async def test_invalid_host_port(self): port = self.port_number + 1 - return self._check_invalid_netloc(f"https://127.0.0.1:{port}") + await self._check_invalid_netloc(f"https://127.0.0.1:{port}") - def test_connection_stays_with_invalid_requests(self): - d_list = [ - self.test_invalid_hostname(), - self.test_invalid_host_port(), - self.test_GET_small_body(), - self.test_POST_small_json(), - ] - - return DeferredList(d_list, fireOnOneErrback=True) + @deferred_f_from_coro_f + async def test_connection_stays_with_invalid_requests(self): + await maybe_deferred_to_future(self.test_invalid_hostname()) + await maybe_deferred_to_future(self.test_invalid_host_port()) + await maybe_deferred_to_future(self.test_GET_small_body()) + await maybe_deferred_to_future(self.test_POST_small_json()) + @inlineCallbacks def test_connection_timeout(self): request = Request(self.get_url("/timeout")) - d = self.make_request(request) # Update the timer to 1s to test connection timeout self.client.setTimeout(1) - def assert_timeout_error(failure: Failure): - for err in failure.value.reasons: - from scrapy.core.http2.protocol import H2ClientProtocol + with pytest.raises(ResponseFailed) as exc_info: + yield self.make_request_dfd(request) - if isinstance(err, TimeoutError): - assert ( - f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s" - in str(err) - ) - break - else: - pytest.fail("No TimeoutError raised.") + for err in exc_info.value.reasons: + from scrapy.core.http2.protocol import H2ClientProtocol - d.addCallback(self.fail) - d.addErrback(assert_timeout_error) - return d + if isinstance(err, TimeoutError): + assert ( + f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s" + in str(err) + ) + break + else: + pytest.fail("No TimeoutError raised.") - def test_request_headers_received(self): + @deferred_f_from_coro_f + async def test_request_headers_received(self): request = Request( self.get_url("/request-headers"), headers={"header-1": "header value 1", "header-2": "header value 2"}, ) - d = self.make_request(request) + response = await self.make_request(request) + assert response.status == 200 + assert response.request == request - def assert_request_headers(response: Response): - assert response.status == 200 - assert response.request == request - - response_headers = json.loads(str(response.body, "utf-8")) - assert isinstance(response_headers, dict) - for k, v in request.headers.items(): - k, v = str(k, "utf-8"), str(v[0], "utf-8") - assert k in response_headers - assert v == response_headers[k] - - d.addErrback(self.fail) - d.addCallback(assert_request_headers) - return d + response_headers = json.loads(str(response.body, "utf-8")) + assert isinstance(response_headers, dict) + for k, v in request.headers.items(): + k, v = str(k, "utf-8"), str(v[0], "utf-8") + assert k in response_headers + assert v == response_headers[k] diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 00e534c4b..cf827e481 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -5,6 +5,7 @@ from pathlib import Path from tempfile import mkdtemp from typing import TYPE_CHECKING, Any +import pytest from testfixtures import LogCapture from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase @@ -218,18 +219,20 @@ class TestFileDownloadCrawl(TestCase): assert "ZeroDivisionError" in str(log) -skip_pillow: str | None +pillow_available: bool try: from PIL import Image # noqa: F401 except ImportError: - skip_pillow = "Missing Python Imaging Library, install https://pypi.org/pypi/Pillow" + pillow_available = False else: - skip_pillow = None + pillow_available = True -class ImageDownloadCrawlTestCase(TestFileDownloadCrawl): - skip = skip_pillow - +@pytest.mark.skipif( + not pillow_available, + reason="Missing Python Imaging Library, install https://pypi.org/pypi/Pillow", +) +class TestImageDownloadCrawl(TestFileDownloadCrawl): pipeline_class = "scrapy.pipelines.images.ImagesPipeline" store_setting_key = "IMAGES_STORE" media_key = "images" diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 492409d02..b4eae108f 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -3,6 +3,7 @@ import os import random import time import warnings +from abc import ABC, abstractmethod from datetime import datetime from io import BytesIO from pathlib import Path @@ -265,7 +266,12 @@ class TestFilesPipeline(unittest.TestCase): assert file_path(request, item=item) == "full/path-to-store-file" -class FilesPipelineTestCaseFieldsMixin: +class TestFilesPipelineFieldsMixin(ABC): + @property + @abstractmethod + def item_class(self) -> Any: + raise NotImplementedError + def test_item_fields_default(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) @@ -302,7 +308,7 @@ class FilesPipelineTestCaseFieldsMixin: assert isinstance(item, self.item_class) -class TestFilesPipelineFieldsDict(FilesPipelineTestCaseFieldsMixin): +class TestFilesPipelineFieldsDict(TestFilesPipelineFieldsMixin): item_class = dict @@ -316,7 +322,7 @@ class FilesPipelineTestItem(Item): custom_files = Field() -class TestFilesPipelineFieldsItem(FilesPipelineTestCaseFieldsMixin): +class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin): item_class = FilesPipelineTestItem @@ -331,7 +337,7 @@ class FilesPipelineTestDataClass: custom_files: list = dataclasses.field(default_factory=list) -class TestFilesPipelineFieldsDataClass(FilesPipelineTestCaseFieldsMixin): +class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin): item_class = FilesPipelineTestDataClass @@ -346,7 +352,7 @@ class FilesPipelineTestAttrsItem: custom_files: list[dict[str, str]] = attr.ib(default=list) -class TestFilesPipelineFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin): +class TestFilesPipelineFieldsAttrsItem(TestFilesPipelineFieldsMixin): item_class = FilesPipelineTestAttrsItem diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index f2ee18bd9..74b4495ad 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -3,8 +3,10 @@ from __future__ import annotations import dataclasses import io import random +from abc import ABC, abstractmethod from shutil import rmtree from tempfile import mkdtemp +from typing import Any import attr import pytest @@ -208,7 +210,12 @@ class TestImagesPipeline: assert converted.getcolors() == [(10000, (205, 230, 255))] -class ImagesPipelineTestCaseFieldsMixin: +class TestImagesPipelineFieldsMixin(ABC): + @property + @abstractmethod + def item_class(self) -> Any: + raise NotImplementedError + def test_item_fields_default(self): url = "http://www.example.com/images/1.jpg" item = self.item_class(name="item1", image_urls=[url]) @@ -245,7 +252,7 @@ class ImagesPipelineTestCaseFieldsMixin: assert isinstance(item, self.item_class) -class TestImagesPipelineFieldsDict(ImagesPipelineTestCaseFieldsMixin): +class TestImagesPipelineFieldsDict(TestImagesPipelineFieldsMixin): item_class = dict @@ -259,7 +266,7 @@ class ImagesPipelineTestItem(Item): custom_images = Field() -class TestImagesPipelineFieldsItem(ImagesPipelineTestCaseFieldsMixin): +class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin): item_class = ImagesPipelineTestItem @@ -274,7 +281,7 @@ class ImagesPipelineTestDataClass: custom_images: list = dataclasses.field(default_factory=list) -class TestImagesPipelineFieldsDataClass(ImagesPipelineTestCaseFieldsMixin): +class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin): item_class = ImagesPipelineTestDataClass @@ -289,7 +296,7 @@ class ImagesPipelineTestAttrsItem: custom_images: list[dict[str, str]] = attr.ib(default=list) -class TestImagesPipelineFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin): +class TestImagesPipelineFieldsAttrsItem(TestImagesPipelineFieldsMixin): item_class = ImagesPipelineTestAttrsItem diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 2d0db6e25..40149f184 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -345,7 +345,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): @inlineCallbacks def test_use_media_to_download_result(self): - req = Request("http://url", meta={"result": "ITSME", "response": self.fail}) + req = Request("http://url", meta={"result": "ITSME"}) item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) assert new_item["results"] == [(True, "ITSME")] diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 5c2772c30..26482fc8d 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -115,7 +115,7 @@ class TestMinimalScheduler(InterfaceCheckMixin): assert not self.scheduler.has_pending_requests() -class SimpleSchedulerTest(TestCase, InterfaceCheckMixin): +class TestSimpleScheduler(TestCase, InterfaceCheckMixin): def setUp(self): self.scheduler = SimpleScheduler() @@ -145,7 +145,7 @@ class SimpleSchedulerTest(TestCase, InterfaceCheckMixin): assert close_result == "close" -class MinimalSchedulerCrawlTest(TestCase): +class TestMinimalSchedulerCrawl(TestCase): scheduler_cls = MinimalScheduler @inlineCallbacks @@ -162,5 +162,5 @@ class MinimalSchedulerCrawlTest(TestCase): assert f"'item_scraped_count': {len(PATHS)}" in str(log) -class SimpleSchedulerCrawlTest(MinimalSchedulerCrawlTest): +class TestSimpleSchedulerCrawl(TestMinimalSchedulerCrawl): scheduler_cls = SimpleScheduler diff --git a/tests/test_signals.py b/tests/test_signals.py index 5a536896e..b20a949e8 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -21,7 +21,7 @@ class ItemSpider(Spider): return {"index": response.meta["index"]} -class MainTestCase(TestCase): +class TestMain(TestCase): @deferred_f_from_coro_f async def test_scheduler_empty(self): crawler = get_crawler() @@ -35,7 +35,7 @@ class MainTestCase(TestCase): assert len(calls) >= 1 -class MockServerTestCase(TestCase): +class TestMockServer(TestCase): @classmethod def setUpClass(cls): cls.mockserver = MockServer() diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 1815aad76..3c7fc65d5 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -18,7 +18,7 @@ ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} -class MainTestCase(TestCase): +class TestMain(TestCase): async def _test_spider(self, spider, expected_items=None): actual_items = [] expected_items = [] if expected_items is None else expected_items diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 725833a49..e1c8b5fec 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -47,7 +47,7 @@ class UniversalSpiderMiddleware: raise NotImplementedError -# Spiders and spider middlewares for MainTestCase._test_wrap +# Spiders and spider middlewares for TestMain._test_wrap class ModernWrapSpider(Spider): @@ -106,7 +106,7 @@ class DeprecatedWrapSpiderMiddleware: yield ITEM_C -class MainTestCase(TestCase): +class TestMain(TestCase): async def _test(self, spider_middlewares, spider_cls, expected_items): actual_items = [] diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 372213319..1cfaf70fa 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -92,10 +92,10 @@ class TestDeferUtils(unittest.TestCase): x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"] + @inlineCallbacks def test_process_parallel_failure(self): - d = process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2") - self.failUnlessFailure(d, TypeError) - return d + with pytest.raises(TypeError): + yield process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2") class TestIterErrback: diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 97dade26e..79bac8bc5 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -59,12 +59,12 @@ class TestSendCatchLog(unittest.TestCase): return "OK" -class SendCatchLogDeferredTest(TestSendCatchLog): +class TestSendCatchLogDeferred(TestSendCatchLog): def _get_result(self, signal, *a, **kw): return send_catch_log_deferred(signal, *a, **kw) -class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): +class TestSendCatchLogDeferred2(TestSendCatchLogDeferred): def ok_handler(self, arg, handlers_called): from twisted.internet import reactor @@ -76,7 +76,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): @pytest.mark.usefixtures("reactor_pytest") -class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): +class TestSendCatchLogDeferredAsyncDef(TestSendCatchLogDeferred): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" @@ -85,7 +85,7 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): @pytest.mark.only_asyncio -class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): +class TestSendCatchLogDeferredAsyncio(TestSendCatchLogDeferred): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" @@ -93,12 +93,12 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): return await get_from_asyncio_queue("OK") -class SendCatchLogAsyncTest(TestSendCatchLog): +class TestSendCatchLogAsync(TestSendCatchLog): def _get_result(self, signal, *a, **kw): return deferred_from_coro(send_catch_log_async(signal, *a, **kw)) -class SendCatchLogAsyncTest2(SendCatchLogAsyncTest): +class TestSendCatchLogAsync2(TestSendCatchLogAsync): def ok_handler(self, arg, handlers_called): from twisted.internet import reactor @@ -110,7 +110,7 @@ class SendCatchLogAsyncTest2(SendCatchLogAsyncTest): @pytest.mark.usefixtures("reactor_pytest") -class SendCatchLogAsyncAsyncDefTest(SendCatchLogAsyncTest): +class TestSendCatchLogAsyncAsyncDef(TestSendCatchLogAsync): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" @@ -119,7 +119,7 @@ class SendCatchLogAsyncAsyncDefTest(SendCatchLogAsyncTest): @pytest.mark.only_asyncio -class SendCatchLogAsyncAsyncioTest(SendCatchLogAsyncTest): +class TestSendCatchLogAsyncAsyncio(TestSendCatchLogAsync): async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) assert arg == "test" diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 8b32e40bb..569f4f639 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -234,35 +234,31 @@ class TestWebClient(unittest.TestCase): def getURL(self, path): return f"http://127.0.0.1:{self.portno}/{path}" + @inlineCallbacks def testPayload(self): s = "0123456789" * 10 - return getPage(self.getURL("payload"), body=s).addCallback( - self.assertEqual, to_bytes(s) - ) + body = yield getPage(self.getURL("payload"), body=s) + assert body == to_bytes(s) + @inlineCallbacks def testHostHeader(self): # if we pass Host header explicitly, it should be used, otherwise # it should extract from url - return defer.gatherResults( - [ - getPage(self.getURL("host")).addCallback( - self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}") - ), - getPage( - self.getURL("host"), headers={"Host": "www.example.com"} - ).addCallback(self.assertEqual, to_bytes("www.example.com")), - ] - ) + body = yield getPage(self.getURL("host")) + assert body == to_bytes(f"127.0.0.1:{self.portno}") + body = yield getPage(self.getURL("host"), headers={"Host": "www.example.com"}) + assert body == to_bytes("www.example.com") + @inlineCallbacks def test_getPage(self): """ L{client.getPage} returns a L{Deferred} which is called back with the body of the response if the default method B{GET} is used. """ - d = getPage(self.getURL("file")) - d.addCallback(self.assertEqual, b"0123456789") - return d + body = yield getPage(self.getURL("file")) + assert body == b"0123456789" + @inlineCallbacks def test_getPageHead(self): """ L{client.getPage} returns a L{Deferred} which is called back with @@ -273,22 +269,20 @@ class TestWebClient(unittest.TestCase): def _getPage(method): return getPage(self.getURL("file"), method=method) - return defer.gatherResults( - [ - _getPage("head").addCallback(self.assertEqual, b""), - _getPage("HEAD").addCallback(self.assertEqual, b""), - ] - ) + body = yield _getPage("head") + assert body == b"" + body = yield _getPage("HEAD") + assert body == b"" + @inlineCallbacks def test_timeoutNotTriggering(self): """ When a non-zero timeout is passed to L{getPage} and the page is retrieved before the timeout period elapses, the L{Deferred} is called back with the contents of the page. """ - d = getPage(self.getURL("host"), timeout=100) - d.addCallback(self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")) - return d + body = yield getPage(self.getURL("host"), timeout=100) + assert body == to_bytes(f"127.0.0.1:{self.portno}") @inlineCallbacks def test_timeoutTriggering(self): @@ -307,12 +301,12 @@ class TestWebClient(unittest.TestCase): if connected: connected[0].transport.loseConnection() + @inlineCallbacks def testNotFound(self): - return getPage(self.getURL("notsuchfile")).addCallback(self._cbNoSuchFile) - - def _cbNoSuchFile(self, pageData): - assert b"404 - No Such Resource" in pageData + body = yield getPage(self.getURL("notsuchfile")) + assert b"404 - No Such Resource" in body + @inlineCallbacks def testFactoryInfo(self): from twisted.internet import reactor @@ -320,63 +314,60 @@ class TestWebClient(unittest.TestCase): parsed = urlparse(url) factory = client.ScrapyHTTPClientFactory(Request(url)) reactor.connectTCP(parsed.hostname, parsed.port, factory) - return factory.deferred.addCallback(self._cbFactoryInfo, factory) - - def _cbFactoryInfo(self, ignoredResult, factory): + yield factory.deferred assert factory.status == b"200" assert factory.version.startswith(b"HTTP/") assert factory.message == b"OK" assert factory.response_headers[b"content-length"] == b"10" + @inlineCallbacks def testRedirect(self): - return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) - - def _cbRedirect(self, pageData): + body = yield getPage(self.getURL("redirect")) assert ( - pageData + body == b'\n\n \n \n' b' \n \n ' b'click here\n \n\n' ) + @inlineCallbacks def test_encoding(self): """Test that non-standart body encoding matches Content-Encoding header""" - body = b"\xd0\x81\xd1\x8e\xd0\xaf" - dfd = getPage( - self.getURL("encoding"), body=body, response_transform=lambda r: r + original_body = b"\xd0\x81\xd1\x8e\xd0\xaf" + response = yield getPage( + self.getURL("encoding"), body=original_body, response_transform=lambda r: r ) - return dfd.addCallback(self._check_Encoding, body) - - def _check_Encoding(self, response, original_body): content_encoding = to_unicode(response.headers[b"Content-Encoding"]) assert content_encoding == EncodingResource.out_encoding assert response.body.decode(content_encoding) == to_unicode(original_body) @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class WebClientSSLTestCase(TestContextFactoryBase): +class TestWebClientSSL(TestContextFactoryBase): + @inlineCallbacks def testPayload(self): s = "0123456789" * 10 - return getPage(self.getURL("payload"), body=s).addCallback( - self.assertEqual, to_bytes(s) - ) + body = yield getPage(self.getURL("payload"), body=s) + assert body == to_bytes(s) -class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): +class TestWebClientCustomCiphersSSL(TestWebClientSSL): # we try to use a cipher that is not enabled by default in OpenSSL custom_ciphers = "CAMELLIA256-SHA" context_factory = ssl_context_factory(cipher_string=custom_ciphers) + @inlineCallbacks def testPayload(self): s = "0123456789" * 10 crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers} ) client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) - return getPage( + body = yield getPage( self.getURL("payload"), body=s, contextFactory=client_context_factory - ).addCallback(self.assertEqual, to_bytes(s)) + ) + assert body == to_bytes(s) @inlineCallbacks def testPayloadDisabledCipher(self): From 92c18d15b4dbed7c98dc3d5ac329c90abb23950f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 13 Jun 2025 22:18:27 +0500 Subject: [PATCH 1778/2083] Remove ProcessTest and SiteTest. (#6885) * Remove ProcessTest and SiteTest. * Restore the support for Windows line endings in TestParseCommand. * Add a test for running a scrapy command in a project subdir. * Remove pywin32 from test deps. --- tests/mockserver.py | 43 +++- tests/test_command_check.py | 6 +- tests/test_command_fetch.py | 46 ++-- tests/test_command_parse.py | 379 ++++++++++++++--------------- tests/test_command_runspider.py | 17 +- tests/test_command_shell.py | 153 ++++++------ tests/test_command_startproject.py | 4 +- tests/test_command_version.py | 26 +- tests/test_commands.py | 49 ++-- tests/utils/testproc.py | 67 ----- tests/utils/testsite.py | 47 ---- tox.ini | 1 - 12 files changed, 360 insertions(+), 478 deletions(-) delete mode 100644 tests/utils/testproc.py delete mode 100644 tests/utils/testsite.py diff --git a/tests/mockserver.py b/tests/mockserver.py index e0ac127f2..841a2cfe4 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -18,11 +18,12 @@ from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory from twisted.web import resource, server -from twisted.web.server import NOT_DONE_YET, GzipEncoderFactory, Site -from twisted.web.static import File -from twisted.web.util import redirectTo +from twisted.web.server import NOT_DONE_YET, Site +from twisted.web.static import Data, File +from twisted.web.util import Redirect, redirectTo from scrapy.utils.python import to_bytes, to_unicode +from tests import tests_datadir if TYPE_CHECKING: from twisted.internet.protocol import ServerFactory @@ -245,6 +246,14 @@ class ArbitraryLengthPayloadResource(LeafResource): return request.content.read() +class NoMetaRefreshRedirect(Redirect): + def render(self, request: server.Request) -> bytes: + content = Redirect.render(self, request) + return content.replace( + b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' + ) + + class Root(resource.Resource): def __init__(self): resource.Resource.__init__(self) @@ -256,18 +265,26 @@ class Root(resource.Resource): self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) self.putChild(b"payload", PayloadResource()) - self.putChild( - b"xpayload", - resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]), - ) self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) - try: - from tests import tests_datadir - - self.putChild(b"files", File(str(Path(tests_datadir, "test_site/files/")))) - except Exception: - pass + self.putChild(b"files", File(str(Path(tests_datadir, "test_site/files/")))) self.putChild(b"redirect-to", RedirectTo()) + self.putChild(b"text", Data(b"Works", "text/plain")) + self.putChild( + b"html", + Data( + b"

Works

World

", + "text/html", + ), + ) + self.putChild( + b"enc-gb18030", + Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), + ) + self.putChild(b"redirect", Redirect(b"/redirected")) + self.putChild( + b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") + ) + self.putChild(b"redirected", Data(b"Redirected here", "text/plain")) def getChild(self, name, request): return self diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 975f31dfe..97bd9d726 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -7,10 +7,8 @@ from tests.test_commands import TestCommandBase class TestCheckCommand(TestCommandBase): - command = "check" - - def setUp(self): - super().setUp() + def setup_method(self): + super().setup_method() self.spider_name = "check_spider" self.spider = (self.proj_mod_path / "spiders" / "checkspider.py").resolve() diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 89f664336..c83594361 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -1,35 +1,35 @@ -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest - -from tests.utils.testproc import ProcessTest -from tests.utils.testsite import SiteTest +from tests.mockserver import MockServer +from tests.test_commands import TestProjectBase -class TestFetchCommand(ProcessTest, SiteTest, unittest.TestCase): - command = "fetch" +class TestFetchCommand(TestProjectBase): + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) - @inlineCallbacks def test_output(self): - _, out, _ = yield self.execute([self.url("/text")]) - assert out.strip() == b"Works" + _, out, _ = self.proc("fetch", self.mockserver.url("/text")) + assert out.strip() == "Works" - @inlineCallbacks def test_redirect_default(self): - _, out, _ = yield self.execute([self.url("/redirect")]) - assert out.strip() == b"Redirected here" + _, out, _ = self.proc("fetch", self.mockserver.url("/redirect")) + assert out.strip() == "Redirected here" - @inlineCallbacks def test_redirect_disabled(self): - _, out, err = yield self.execute( - ["--no-redirect", self.url("/redirect-no-meta-refresh")] + _, _, err = self.proc( + "fetch", "--no-redirect", self.mockserver.url("/redirect-no-meta-refresh") ) err = err.strip() - assert b"downloader/response_status_count/302" in err, err - assert b"downloader/response_status_count/200" not in err, err + assert "downloader/response_status_count/302" in err, err + assert "downloader/response_status_count/200" not in err, err - @inlineCallbacks def test_headers(self): - _, out, _ = yield self.execute([self.url("/text"), "--headers"]) - out = out.replace(b"\r", b"") # required on win32 - assert b"Server: TwistedWeb" in out, out - assert b"Content-Type: text/plain" in out + _, out, _ = self.proc("fetch", self.mockserver.url("/text"), "--headers") + out = out.replace("\r", "") # required on win32 + assert "Server: TwistedWeb" in out, out + assert "Content-Type: text/plain" in out diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 6681aba17..5c3120c21 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,29 +1,25 @@ import argparse -import os import re from pathlib import Path -from twisted.internet.defer import inlineCallbacks - from scrapy.commands import parse from scrapy.settings import Settings -from scrapy.utils.python import to_unicode +from tests.mockserver import MockServer from tests.test_commands import TestCommandBase -from tests.utils.testproc import ProcessTest -from tests.utils.testsite import SiteTest -def _textmode(bstr: bytes) -> str: - """Normalize input the same as writing to a file - and reading from it in text mode""" - return to_unicode(bstr).replace(os.linesep, "\n") +class TestParseCommand(TestCommandBase): + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) -class TestParseCommand(ProcessTest, SiteTest, TestCommandBase): - command = "parse" - - def setUp(self): - super().setUp() + def setup_method(self): + super().setup_method() self.spider_name = "parse_spider" (self.proj_mod_path / "spiders" / "myspider.py").write_text( f""" @@ -171,260 +167,253 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """ ) - @inlineCallbacks def test_spider_arguments(self): - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "-a", - "test_arg=1", - "-c", - "parse", - "--verbose", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "-a", + "test_arg=1", + "-c", + "parse", + "--verbose", + self.mockserver.url("/html"), ) - assert "DEBUG: It Works!" in _textmode(stderr) + assert "DEBUG: It Works!" in stderr - @inlineCallbacks def test_request_with_meta(self): raw_json_string = '{"foo" : "baz"}' - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "--meta", - raw_json_string, - "-c", - "parse_request_with_meta", - "--verbose", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "--meta", + raw_json_string, + "-c", + "parse_request_with_meta", + "--verbose", + self.mockserver.url("/html"), ) - assert "DEBUG: It Works!" in _textmode(stderr) + assert "DEBUG: It Works!" in stderr - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "-m", - raw_json_string, - "-c", - "parse_request_with_meta", - "--verbose", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "-m", + raw_json_string, + "-c", + "parse_request_with_meta", + "--verbose", + self.mockserver.url("/html"), ) - assert "DEBUG: It Works!" in _textmode(stderr) + assert "DEBUG: It Works!" in stderr - @inlineCallbacks def test_request_with_cb_kwargs(self): raw_json_string = '{"foo" : "bar", "key": "value"}' - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "--cbkwargs", - raw_json_string, - "-c", - "parse_request_with_cb_kwargs", - "--verbose", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "--cbkwargs", + raw_json_string, + "-c", + "parse_request_with_cb_kwargs", + "--verbose", + self.mockserver.url("/html"), ) - log = _textmode(stderr) - assert "DEBUG: It Works!" in log + assert "DEBUG: It Works!" in stderr assert ( - "DEBUG: request.callback signature: (response, foo=None, key=None)" in log + "DEBUG: request.callback signature: (response, foo=None, key=None)" + in stderr ) - @inlineCallbacks def test_request_without_meta(self): - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "-c", - "parse_request_without_meta", - "--nolinks", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse_request_without_meta", + "--nolinks", + self.mockserver.url("/html"), ) - assert "DEBUG: It Works!" in _textmode(stderr) + assert "DEBUG: It Works!" in stderr - @inlineCallbacks def test_pipelines(self): - _, _, stderr = yield self.execute( - [ - "--spider", - self.spider_name, - "--pipelines", - "-c", - "parse", - "--verbose", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "--pipelines", + "-c", + "parse", + "--verbose", + self.mockserver.url("/html"), ) - assert "INFO: It Works!" in _textmode(stderr) + assert "INFO: It Works!" in stderr - @inlineCallbacks def test_async_def_asyncio_parse_items_list(self): - status, out, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio_return", - "-c", - "parse", - self.url("/html"), - ] + _, out, stderr = self.proc( + "parse", + "--spider", + "asyncdef_asyncio_return", + "-c", + "parse", + self.mockserver.url("/html"), ) - assert "INFO: Got response 200" in _textmode(stderr) - assert "{'id': 1}" in _textmode(out) - assert "{'id': 2}" in _textmode(out) + assert "INFO: Got response 200" in stderr + assert "{'id': 1}" in out + assert "{'id': 2}" in out - @inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): - status, out, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio_return_single_element", - "-c", - "parse", - self.url("/html"), - ] + _, out, stderr = self.proc( + "parse", + "--spider", + "asyncdef_asyncio_return_single_element", + "-c", + "parse", + self.mockserver.url("/html"), ) - assert "INFO: Got response 200" in _textmode(stderr) - assert "{'foo': 42}" in _textmode(out) + assert "INFO: Got response 200" in stderr + assert "{'foo': 42}" in out - @inlineCallbacks def test_async_def_asyncgen_parse_loop(self): - status, out, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio_gen_loop", - "-c", - "parse", - self.url("/html"), - ] + _, out, stderr = self.proc( + "parse", + "--spider", + "asyncdef_asyncio_gen_loop", + "-c", + "parse", + self.mockserver.url("/html"), ) - assert "INFO: Got response 200" in _textmode(stderr) + assert "INFO: Got response 200" in stderr for i in range(10): - assert f"{{'foo': {i}}}" in _textmode(out) + assert f"{{'foo': {i}}}" in out - @inlineCallbacks def test_async_def_asyncgen_parse_exc(self): - status, out, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio_gen_exc", - "-c", - "parse", - self.url("/html"), - ] + _, out, stderr = self.proc( + "parse", + "--spider", + "asyncdef_asyncio_gen_exc", + "-c", + "parse", + self.mockserver.url("/html"), ) - assert "ValueError" in _textmode(stderr) + assert "ValueError" in stderr for i in range(7): - assert f"{{'foo': {i}}}" in _textmode(out) + assert f"{{'foo': {i}}}" in out - @inlineCallbacks def test_async_def_asyncio_parse(self): - _, _, stderr = yield self.execute( - [ - "--spider", - "asyncdef_asyncio", - "-c", - "parse", - self.url("/html"), - ] + _, _, stderr = self.proc( + "parse", + "--spider", + "asyncdef_asyncio", + "-c", + "parse", + self.mockserver.url("/html"), ) - assert "DEBUG: Got response 200" in _textmode(stderr) + assert "DEBUG: Got response 200" in stderr - @inlineCallbacks def test_parse_items(self): - status, out, stderr = yield self.execute( - ["--spider", self.spider_name, "-c", "parse", self.url("/html")] + _, out, _ = self.proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse", + self.mockserver.url("/html"), ) - assert "[{}, {'foo': 'bar'}]" in _textmode(out) + assert "[{}, {'foo': 'bar'}]" in out - @inlineCallbacks def test_parse_items_no_callback_passed(self): - status, out, stderr = yield self.execute( - ["--spider", self.spider_name, self.url("/html")] + _, out, _ = self.proc( + "parse", "--spider", self.spider_name, self.mockserver.url("/html") ) - assert "[{}, {'foo': 'bar'}]" in _textmode(out) + assert "[{}, {'foo': 'bar'}]" in out - @inlineCallbacks def test_wrong_callback_passed(self): - status, out, stderr = yield self.execute( - ["--spider", self.spider_name, "-c", "dummy", self.url("/html")] + _, out, stderr = self.proc( + "parse", + "--spider", + self.spider_name, + "-c", + "dummy", + self.mockserver.url("/html"), ) - assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) - assert "Cannot find callback" in _textmode(stderr) + assert re.search(r"# Scraped Items -+\r?\n\[\]", out) + assert "Cannot find callback" in stderr - @inlineCallbacks def test_crawlspider_matching_rule_callback_set(self): """If a rule matches the URL, use it's defined callback.""" - status, out, stderr = yield self.execute( - ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/html")] + _, out, _ = self.proc( + "parse", + "--spider", + "goodcrawl" + self.spider_name, + "-r", + self.mockserver.url("/html"), ) - assert "[{}, {'foo': 'bar'}]" in _textmode(out) + assert "[{}, {'foo': 'bar'}]" in out - @inlineCallbacks def test_crawlspider_matching_rule_default_callback(self): """If a rule match but it has no callback set, use the 'parse' callback.""" - status, out, stderr = yield self.execute( - ["--spider", "goodcrawl" + self.spider_name, "-r", self.url("/text")] + _, out, _ = self.proc( + "parse", + "--spider", + "goodcrawl" + self.spider_name, + "-r", + self.mockserver.url("/text"), ) - assert "[{}, {'nomatch': 'default'}]" in _textmode(out) + assert "[{}, {'nomatch': 'default'}]" in out - @inlineCallbacks def test_spider_with_no_rules_attribute(self): """Using -r with a spider with no rule should not produce items.""" - status, out, stderr = yield self.execute( - ["--spider", self.spider_name, "-r", self.url("/html")] + _, out, stderr = self.proc( + "parse", "--spider", self.spider_name, "-r", self.mockserver.url("/html") ) - assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) - assert "No CrawlSpider rules found" in _textmode(stderr) + assert re.search(r"# Scraped Items -+\r?\n\[\]", out) + assert "No CrawlSpider rules found" in stderr - @inlineCallbacks def test_crawlspider_missing_callback(self): - status, out, stderr = yield self.execute( - ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/html")] + _, out, _ = self.proc( + "parse", + "--spider", + "badcrawl" + self.spider_name, + "-r", + self.mockserver.url("/html"), ) - assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) + assert re.search(r"# Scraped Items -+\r?\n\[\]", out) - @inlineCallbacks def test_crawlspider_no_matching_rule(self): """The requested URL has no matching rule, so no items should be scraped""" - status, out, stderr = yield self.execute( - ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/enc-gb18030")] + _, out, stderr = self.proc( + "parse", + "--spider", + "badcrawl" + self.spider_name, + "-r", + self.mockserver.url("/enc-gb18030"), ) - assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) - assert "Cannot find a rule that matches" in _textmode(stderr) + assert re.search(r"# Scraped Items -+\r?\n\[\]", out) + assert "Cannot find a rule that matches" in stderr - @inlineCallbacks def test_crawlspider_not_exists_with_not_matched_url(self): - status, out, stderr = yield self.execute([self.url("/invalid_url")]) - assert status == 0 + assert self.call("parse", self.mockserver.url("/invalid_url")) == 0 - @inlineCallbacks def test_output_flag(self): """Checks if a file was created successfully having correct format containing correct data in it. """ file_name = "data.json" file_path = Path(self.proj_path, file_name) - yield self.execute( - [ - "--spider", - self.spider_name, - "-c", - "parse", - "-o", - file_name, - self.url("/html"), - ] + self.proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse", + "-o", + file_name, + self.mockserver.url("/html"), ) assert file_path.exists() diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 7f8d9fb61..c1a6d9b18 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -7,7 +7,6 @@ from contextlib import contextmanager from pathlib import Path from tempfile import TemporaryDirectory, mkdtemp from typing import TYPE_CHECKING -from unittest import skipIf import pytest @@ -58,7 +57,7 @@ class BadSpider(scrapy.Spider): return self.proc("runspider", fname, *args) def get_log(self, code, name=None, args=()): - p, stdout, stderr = self.runspider(code, name, args=args) + _, _, stderr = self.runspider(code, name, args=args) return stderr def test_runspider(self): @@ -288,7 +287,7 @@ class MySpider(scrapy.Spider): log = self.get_log(spider_code, args=args) assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log - @skipIf(platform.system() == "Windows", reason="Linux only") + @pytest.mark.skipif(platform.system() == "Windows", reason="Linux only") def test_absolute_path_linux(self): spider_code = """ import scrapy @@ -317,7 +316,7 @@ class MySpider(scrapy.Spider): in log ) - @skipIf(platform.system() != "Windows", reason="Windows only") + @pytest.mark.skipif(platform.system() != "Windows", reason="Windows only") def test_absolute_path_windows(self): spider_code = """ import scrapy @@ -370,18 +369,16 @@ class MySpider(scrapy.Spider): assert "The value of FOO is 42" in log +@pytest.mark.skipif( + platform.system() != "Windows", reason="Windows required for .pyw files" +) class TestWindowsRunSpiderCommand(TestRunSpiderCommand): spider_filename = "myspider.pyw" - def setUp(self): - if platform.system() != "Windows": - pytest.skip("Windows required for .pyw files") - return super().setUp() - def test_start_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") assert "start" in log assert "badspider.pyw" in log def test_runspider_unable_to_load(self): - pytest.skip("Already Tested in 'RunSpiderCommandTest' ") + pytest.skip("Already Tested in 'RunSpiderCommandTest'") diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index d9f17d76b..76c1eb663 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -5,140 +5,137 @@ from pathlib import Path import pytest from pexpect.popen_spawn import PopenSpawn -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest from scrapy.utils.reactor import _asyncio_reactor_path from tests import NON_EXISTING_RESOLVABLE, tests_datadir from tests.mockserver import MockServer -from tests.utils.testproc import ProcessTest -from tests.utils.testsite import SiteTest +from tests.test_commands import TestProjectBase -class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): - command = "shell" +class TestShellCommand(TestProjectBase): + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) - @inlineCallbacks def test_empty(self): - _, out, _ = yield self.execute(["-c", "item"]) - assert b"{}" in out + _, out, _ = self.proc("shell", "-c", "item") + assert "{}" in out - @inlineCallbacks def test_response_body(self): - _, out, _ = yield self.execute([self.url("/text"), "-c", "response.body"]) - assert b"Works" in out + _, out, _ = self.proc( + "shell", self.mockserver.url("/text"), "-c", "response.body" + ) + assert "Works" in out - @inlineCallbacks def test_response_type_text(self): - _, out, _ = yield self.execute([self.url("/text"), "-c", "type(response)"]) - assert b"TextResponse" in out + _, out, _ = self.proc( + "shell", self.mockserver.url("/text"), "-c", "type(response)" + ) + assert "TextResponse" in out - @inlineCallbacks def test_response_type_html(self): - _, out, _ = yield self.execute([self.url("/html"), "-c", "type(response)"]) - assert b"HtmlResponse" in out + _, out, _ = self.proc( + "shell", self.mockserver.url("/html"), "-c", "type(response)" + ) + assert "HtmlResponse" in out - @inlineCallbacks def test_response_selector_html(self): xpath = "response.xpath(\"//p[@class='one']/text()\").get()" - _, out, _ = yield self.execute([self.url("/html"), "-c", xpath]) - assert out.strip() == b"Works" + _, out, _ = self.proc("shell", self.mockserver.url("/html"), "-c", xpath) + assert out.strip() == "Works" - @inlineCallbacks def test_response_encoding_gb18030(self): - _, out, _ = yield self.execute( - [self.url("/enc-gb18030"), "-c", "response.encoding"] + _, out, _ = self.proc( + "shell", self.mockserver.url("/enc-gb18030"), "-c", "response.encoding" ) - assert out.strip() == b"gb18030" + assert out.strip() == "gb18030" - @inlineCallbacks def test_redirect(self): - _, out, _ = yield self.execute([self.url("/redirect"), "-c", "response.url"]) - assert out.strip().endswith(b"/redirected") + _, out, _ = self.proc( + "shell", self.mockserver.url("/redirect"), "-c", "response.url" + ) + assert out.strip().endswith("/redirected") - @inlineCallbacks def test_redirect_follow_302(self): - _, out, _ = yield self.execute( - [self.url("/redirect-no-meta-refresh"), "-c", "response.status"] + _, out, _ = self.proc( + "shell", + self.mockserver.url("/redirect-no-meta-refresh"), + "-c", + "response.status", ) - assert out.strip().endswith(b"200") + assert out.strip().endswith("200") - @inlineCallbacks def test_redirect_not_follow_302(self): - _, out, _ = yield self.execute( - [ - "--no-redirect", - self.url("/redirect-no-meta-refresh"), - "-c", - "response.status", - ] + _, out, _ = self.proc( + "shell", + "--no-redirect", + self.mockserver.url("/redirect-no-meta-refresh"), + "-c", + "response.status", ) - assert out.strip().endswith(b"302") + assert out.strip().endswith("302") - @inlineCallbacks def test_fetch_redirect_follow_302(self): """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" - url = self.url("/redirect-no-meta-refresh") + url = self.mockserver.url("/redirect-no-meta-refresh") code = f"fetch('{url}')" - errcode, out, errout = yield self.execute(["-c", code]) - assert errcode == 0, out - assert b"Redirecting (302)" in errout - assert b"Crawled (200)" in errout + p, out, errout = self.proc("shell", "-c", code) + assert p.returncode == 0, out + assert "Redirecting (302)" in errout + assert "Crawled (200)" in errout - @inlineCallbacks def test_fetch_redirect_not_follow_302(self): """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" - url = self.url("/redirect-no-meta-refresh") + url = self.mockserver.url("/redirect-no-meta-refresh") code = f"fetch('{url}', redirect=False)" - errcode, out, errout = yield self.execute(["-c", code]) - assert errcode == 0, out - assert b"Crawled (302)" in errout + p, out, errout = self.proc("shell", "-c", code) + assert p.returncode == 0, out + assert "Crawled (302)" in errout - @inlineCallbacks def test_request_replace(self): - url = self.url("/text") + url = self.mockserver.url("/text") code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" - errcode, out, _ = yield self.execute(["-c", code]) - assert errcode == 0, out + p, out, _ = self.proc("shell", "-c", code) + assert p.returncode == 0, out - @inlineCallbacks def test_scrapy_import(self): - url = self.url("/text") + url = self.mockserver.url("/text") code = f"fetch(scrapy.Request('{url}'))" - errcode, out, _ = yield self.execute(["-c", code]) - assert errcode == 0, out + p, out, _ = self.proc("shell", "-c", code) + assert p.returncode == 0, out - @inlineCallbacks def test_local_file(self): filepath = Path(tests_datadir, "test_site", "index.html") - _, out, _ = yield self.execute([str(filepath), "-c", "item"]) - assert b"{}" in out + _, out, _ = self.proc("shell", str(filepath), "-c", "item") + assert "{}" in out - @inlineCallbacks def test_local_nofile(self): filepath = "file:///tests/sample_data/test_site/nothinghere.html" - errcode, out, err = yield self.execute( - [filepath, "-c", "item"], check_code=False - ) - assert errcode == 1, out or err - assert b"No such file or directory" in err + p, out, err = self.proc("shell", filepath, "-c", "item") + assert p.returncode == 1, out or err + assert "No such file or directory" in err - @inlineCallbacks def test_dns_failures(self): if NON_EXISTING_RESOLVABLE: pytest.skip("Non-existing hosts are resolvable") url = "www.somedomainthatdoesntexi.st" - errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False) - assert errcode == 1, out or err - assert b"DNS lookup failed" in err + p, out, err = self.proc("shell", url, "-c", "item") + assert p.returncode == 1, out or err + assert "DNS lookup failed" in err - @inlineCallbacks def test_shell_fetch_async(self): - url = self.url("/html") + url = self.mockserver.url("/html") code = f"fetch('{url}')" - args = ["-c", code, "--set", f"TWISTED_REACTOR={_asyncio_reactor_path}"] - _, _, err = yield self.execute(args, check_code=True) - assert b"RuntimeError: There is no current event loop in thread" not in err + p, _, err = self.proc( + "shell", "-c", code, "--set", f"TWISTED_REACTOR={_asyncio_reactor_path}" + ) + assert p.returncode == 0, err + assert "RuntimeError: There is no current event loop in thread" not in err class TestInteractiveShell: diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 08bf9b0fd..988ad50b9 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -108,8 +108,8 @@ def get_permissions_dict( class TestStartprojectTemplates(TestProjectBase): maxDiff = None - def setUp(self): - super().setUp() + def setup_method(self): + super().setup_method() self.tmpl = str(Path(self.temp_path, "templates")) self.tmpl_proj = str(Path(self.tmpl, "project")) diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 87dfb16df..de58203fc 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -1,29 +1,15 @@ -import sys - -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest - import scrapy -from tests.utils.testproc import ProcessTest +from tests.test_commands import TestProjectBase -class TestVersionCommand(ProcessTest, unittest.TestCase): - command = "version" - - @inlineCallbacks +class TestVersionCommand(TestProjectBase): def test_output(self): - encoding = sys.stdout.encoding or "utf-8" - _, out, _ = yield self.execute([]) - assert out.strip().decode(encoding) == f"Scrapy {scrapy.__version__}" + _, out, _ = self.proc("version") + assert out.strip() == f"Scrapy {scrapy.__version__}" - @inlineCallbacks def test_verbose_output(self): - encoding = sys.stdout.encoding or "utf-8" - _, out, _ = yield self.execute(["-v"]) - headers = [ - line.partition(":")[0].strip() - for line in out.strip().decode(encoding).splitlines() - ] + _, out, _ = self.proc("version", "-v") + headers = [line.partition(":")[0].strip() for line in out.strip().splitlines()] assert headers == [ "Scrapy", "lxml", diff --git a/tests/test_commands.py b/tests/test_commands.py index 8ca5d51e5..851c92db4 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -10,11 +10,9 @@ from pathlib import Path from shutil import rmtree from tempfile import TemporaryFile, mkdtemp from threading import Timer -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from unittest import mock -from twisted.trial import unittest - import scrapy from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view @@ -61,28 +59,30 @@ class TestCommandSettings: ) -class TestProjectBase(unittest.TestCase): +class TestProjectBase: project_name = "testproject" - def setUp(self): + def setup_method(self): self.temp_path = mkdtemp() self.cwd = self.temp_path self.proj_path = Path(self.temp_path, self.project_name) self.proj_mod_path = self.proj_path / self.project_name self.env = get_testenv() - def tearDown(self): + def teardown_method(self): rmtree(self.temp_path) - def call(self, *new_args, **kwargs): + def call(self, *args: str, **popen_kwargs: Any) -> int: with TemporaryFile() as out: - args = (sys.executable, "-m", "scrapy.cmdline", *new_args) + args = (sys.executable, "-m", "scrapy.cmdline", *args) return subprocess.call( - args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs + args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **popen_kwargs ) - def proc(self, *new_args, **popen_kwargs): - args = (sys.executable, "-m", "scrapy.cmdline", *new_args) + def proc( + self, *args: str, **popen_kwargs: Any + ) -> tuple[subprocess.Popen[bytes], str, str]: + args = (sys.executable, "-m", "scrapy.cmdline", *args) p = subprocess.Popen( args, cwd=popen_kwargs.pop("cwd", self.cwd), @@ -118,10 +118,10 @@ class TestProjectBase(unittest.TestCase): class TestCommandBase(TestProjectBase): - def setUp(self): - super().setUp() + def setup_method(self): + super().setup_method() self.call("startproject", self.project_name) - self.cwd = Path(self.temp_path, self.project_name) + self.cwd = self.proj_path self.env["SCRAPY_SETTINGS_MODULE"] = f"{self.project_name}.settings" @@ -136,8 +136,8 @@ class TestCommandCrawlerProcess(TestCommandBase): "Type of self.crawler_process: " ) - def setUp(self): - super().setUp() + def setup_method(self): + super().setup_method() (self.cwd / self.project_name / "commands").mkdir(exist_ok=True) (self.cwd / self.project_name / "commands" / "__init__.py").touch() (self.cwd / self.project_name / "commands" / f"{self.name}.py").write_text(""" @@ -363,6 +363,19 @@ Unknown command: abc assert out.getvalue().strip() == message.strip() +class TestProjectSubdir(TestProjectBase): + """Test that commands work in a subdirectory of the project.""" + + def setup_method(self): + super().setup_method() + self.call("startproject", self.project_name) + self.cwd = self.proj_path / "subdir" + self.cwd.mkdir(exist_ok=True) + + def test_list(self): + assert self.call("list") == 0 + + class TestBenchCommand(TestCommandBase): def test_run(self): _, _, log = self.proc( @@ -389,8 +402,8 @@ class TestViewCommand(TestCommandBase): class TestHelpMessage(TestCommandBase): - def setUp(self): - super().setUp() + def setup_method(self): + super().setup_method() self.commands = [ "parse", "startproject", diff --git a/tests/utils/testproc.py b/tests/utils/testproc.py deleted file mode 100644 index 85d7c940f..000000000 --- a/tests/utils/testproc.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -import os -import sys -from typing import TYPE_CHECKING, cast - -from twisted.internet.defer import Deferred -from twisted.internet.error import ProcessTerminated -from twisted.internet.protocol import ProcessProtocol - -if TYPE_CHECKING: - from collections.abc import Iterable - - from twisted.python.failure import Failure - - -class ProcessTest: - command: str | None = None - prefix = [sys.executable, "-m", "scrapy.cmdline"] - cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109 - - def execute( - self, - args: Iterable[str], - check_code: bool = True, - settings: str | None = None, - ) -> Deferred[TestProcessProtocol]: - from twisted.internet import reactor - - env = os.environ.copy() - if settings is not None: - env["SCRAPY_SETTINGS_MODULE"] = settings - assert self.command - cmd = [*self.prefix, self.command, *args] - pp = TestProcessProtocol() - pp.deferred.addCallback(self._process_finished, cmd, check_code) - reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) - return pp.deferred - - def _process_finished( - self, pp: TestProcessProtocol, cmd: list[str], check_code: bool - ) -> tuple[int, bytes, bytes]: - if pp.exitcode and check_code: - msg = f"process {cmd} exit with code {pp.exitcode}" - msg += f"\n>>> stdout <<<\n{pp.out.decode()}" - msg += "\n" - msg += f"\n>>> stderr <<<\n{pp.err.decode()}" - raise RuntimeError(msg) - return cast(int, pp.exitcode), pp.out, pp.err - - -class TestProcessProtocol(ProcessProtocol): - def __init__(self) -> None: - self.deferred: Deferred[TestProcessProtocol] = Deferred() - self.out: bytes = b"" - self.err: bytes = b"" - self.exitcode: int | None = None - - def outReceived(self, data: bytes) -> None: - self.out += data - - def errReceived(self, data: bytes) -> None: - self.err += data - - def processEnded(self, status: Failure) -> None: - self.exitcode = cast(ProcessTerminated, status.value).exitCode - self.deferred.callback(self) diff --git a/tests/utils/testsite.py b/tests/utils/testsite.py deleted file mode 100644 index 473738773..000000000 --- a/tests/utils/testsite.py +++ /dev/null @@ -1,47 +0,0 @@ -from urllib.parse import urljoin - -from twisted.web import resource, server, static, util - - -class SiteTest: - def setUp(self): - from twisted.internet import reactor - - super().setUp() - self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - self.baseurl = f"http://localhost:{self.site.getHost().port}/" - - def tearDown(self): - super().tearDown() - self.site.stopListening() - - def url(self, path: str) -> str: - return urljoin(self.baseurl, path) - - -class NoMetaRefreshRedirect(util.Redirect): - def render(self, request: server.Request) -> bytes: - content = util.Redirect.render(self, request) - return content.replace( - b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' - ) - - -def test_site(): - r = resource.Resource() - r.putChild(b"text", static.Data(b"Works", "text/plain")) - r.putChild( - b"html", - static.Data( - b"

Works

World

", - "text/html", - ), - ) - r.putChild( - b"enc-gb18030", - static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), - ) - r.putChild(b"redirect", util.Redirect(b"/redirected")) - r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) - r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) - return server.Site(r) diff --git a/tox.ini b/tox.ini index 5680d98d1..f28467ec1 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,6 @@ deps = pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures - pywin32; sys_platform == "win32" Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467 [testenv] From daa1a7d0b6549f901a002bf4c8bb7c4aed23e068 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 14 Jun 2025 14:01:20 +0500 Subject: [PATCH 1779/2083] Remove the chdir fixture, re-enable fancy pytest asserts (#6888) * Remove the chdir fixture. * Re-enable fancy pytest asserts. * Remove doc files from pytest ignores. * Restore docs/_ext in test collection ignores. * Skip a doctest that fails on Windows. * Fix tests that were writing to the current dir. --- conftest.py | 8 +--- docs/news.rst | 4 ++ docs/topics/dynamic-content.rst | 9 +++++ docs/topics/items.rst | 4 ++ docs/topics/leaks.rst | 7 ++++ docs/topics/loaders.rst | 13 ++++++ docs/topics/selectors.rst | 3 +- docs/topics/shell.rst | 8 ++++ docs/topics/stats.rst | 4 ++ docs/topics/telnetconsole.rst | 4 ++ pyproject.toml | 16 -------- tests/test_feedexport.py | 32 ++++++++------- tests/test_pipeline_files.py | 11 +++--- tests/test_squeues_request.py | 70 ++++++++++++++------------------- 14 files changed, 110 insertions(+), 83 deletions(-) diff --git a/conftest.py b/conftest.py index ed7d14166..f952127b9 100644 --- a/conftest.py +++ b/conftest.py @@ -12,6 +12,8 @@ def _py_files(folder): collect_ignore = [ + # may need extra deps + "docs/_ext", # not a test, but looks like a test "scrapy/utils/testproc.py", "scrapy/utils/testsite.py", @@ -46,12 +48,6 @@ if not H2_ENABLED: ) -@pytest.fixture -def chdir(tmpdir): - """Change to pytest-provided temporary directory""" - tmpdir.chdir() - - def pytest_addoption(parser): parser.addoption( "--reactor", diff --git a/docs/news.rst b/docs/news.rst index 36d229760..05ad611ef 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -4454,6 +4454,8 @@ Highlights: Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. skip: start + * Python 3.4 is no longer supported, and some of the minimum requirements of Scrapy have also changed: @@ -4494,6 +4496,8 @@ Backward-incompatible changes (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`, :issue:`4036`) +.. skip: end + See also :ref:`1.8-deprecation-removals` below. diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 65270433f..6c57a88f1 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -111,6 +111,8 @@ you may use `curl2scrapy `_. Handling different response formats =================================== +.. skip: start + Once you have a response with the desired data, how you extract the desired data from it depends on the type of response: @@ -157,11 +159,15 @@ data from it depends on the type of response: Otherwise, you might need to convert the SVG code into a raster image, and :ref:`handle that raster image `. +.. skip: end + .. _topics-parsing-javascript: Parsing JavaScript code ======================= +.. skip: start + If the desired data is hardcoded in JavaScript, you first need to get the JavaScript code: @@ -220,6 +226,8 @@ data from it: >>> selector.css('var[name="data"]').get() 'value' +.. skip: end + .. _topics-headless-browsing: Using a headless browser @@ -242,6 +250,7 @@ it is possible to integrate ``asyncio``-based libraries which handle headless br One such library is `playwright-python`_ (an official Python port of `playwright`_). The following is a simple snippet to illustrate its usage within a Scrapy spider: +.. skip: next .. code-block:: python import scrapy diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 0365c95b3..3588d033e 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -214,6 +214,8 @@ the :attr:`~scrapy.Item.fields` attribute. Working with Item objects ------------------------- +.. skip: start + Here are some examples of common tasks performed with items, using the ``Product`` item :ref:`declared above `. You will notice the API is very similar to the :class:`dict` API. @@ -375,6 +377,8 @@ appending more values, or changing existing values, like this: That adds (or replaces) the ``serializer`` metadata key for the ``name`` field, keeping all the previously existing metadata values. +.. skip: end + .. _supporting-item-types: diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index cd8914644..bbe1f3dd4 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -60,6 +60,8 @@ in control. Debugging memory leaks with ``trackref`` ======================================== +.. skip: start + :mod:`trackref` is a module provided by Scrapy to debug the most common cases of memory leaks. It basically tracks the references to all live Request, Response, Item, Spider and Selector objects. @@ -203,6 +205,8 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. ``None`` if none is found. Use :func:`print_live_refs` first to get a list of all tracked live objects per class name. +.. skip: end + .. _topics-leaks-muppy: Debugging memory leaks with muppy @@ -226,6 +230,7 @@ If you use ``pip``, you can install muppy with the following command:: Here's an example to view all Python objects available in the heap using muppy: +.. skip: start .. code-block:: pycon >>> from pympler import muppy @@ -253,6 +258,8 @@ the heap using muppy: `, using the :ref:`Product item ` declared in the :ref:`Items chapter `: +.. skip: next .. code-block:: python from scrapy.loader import ItemLoader @@ -130,6 +131,7 @@ assigned to the item. Let's see an example to illustrate how the input and output processors are called for a particular field (the same applies for any other field): +.. skip: next .. code-block:: python l = ItemLoader(Product(), some_selector) @@ -250,6 +252,7 @@ metadata. Here is an example: ) +.. skip: start .. code-block:: pycon >>> from scrapy.loader import ItemLoader @@ -259,6 +262,8 @@ metadata. Here is an example: >>> il.load_item() {'name': 'Welcome to my website', 'price': '1000'} +.. skip: end + The precedence order, for both input and output processors, is as follows: 1. Item Loader field-specific attributes: ``field_in`` and ``field_out`` (most @@ -294,6 +299,8 @@ the Item Loader that it's able to receive an Item Loader context, so the Item Loader passes the currently active context when calling it, and the processor function (``parse_length`` in this case) can thus use them. +.. skip: start + There are several ways to modify Item Loader context values: 1. By modifying the currently active Item Loader context @@ -320,6 +327,8 @@ There are several ways to modify Item Loader context values: class ProductLoader(ItemLoader): length_out = MapCompose(parse_length, unit="cm") +.. skip: end + ItemLoader objects ================== @@ -350,6 +359,7 @@ that you wish to extract. Example: +.. skip: next .. code-block:: python loader = ItemLoader(item=Item()) @@ -364,6 +374,7 @@ the footer selector. Example: +.. skip: next .. code-block:: python loader = ItemLoader(item=Item()) @@ -401,6 +412,7 @@ those dashes in the final product names. Here's how you can remove those dashes by reusing and extending the default Product Item Loader (``ProductLoader``): +.. skip: next .. code-block:: python from itemloaders.processors import MapCompose @@ -418,6 +430,7 @@ Another case where extending Item Loaders can be very helpful is when you have multiple source formats, for example XML and HTML. In the XML version you may want to remove ``CDATA`` occurrences. Here's an example of how to do it: +.. skip: next .. code-block:: python from itemloaders.processors import MapCompose diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index dbef07b73..40a85201a 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -308,6 +308,7 @@ Examples: * ``*::text`` selects all descendant text nodes of the current selector context: +..skip: next .. code-block:: pycon >>> response.css("#images *::text").getall() @@ -878,7 +879,7 @@ Example selecting links in list item with a "class" attribute ending with a digi >>> sel = Selector(text=doc, type="html") >>> sel.xpath("//li//@href").getall() ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html'] - >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall() + >>> sel.xpath(r'//li[re:test(@class, "item-\d$")]//@href').getall() ['link1.html', 'link2.html', 'link4.html', 'link5.html'] .. warning:: C library ``libxslt`` doesn't natively support EXSLT regular diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 4898843e4..85a08cebd 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -142,6 +142,8 @@ Those objects are: Example of shell session ======================== +.. skip: start + Here's an example of a typical shell session where we start by scraping the https://scrapy.org page, and then proceed to scrape the https://old.reddit.com/ page. Finally, we modify the (Reddit) request method to POST and re-fetch it @@ -232,6 +234,8 @@ After that, we can start playing with the objects: 'X-Ua-Compatible': ['IE=edge'], 'X-Xss-Protection': ['1; mode=block']} +.. skip: end + .. _topics-shell-inspect-response: @@ -268,6 +272,8 @@ Here's an example of how you would call it from your spider: # Rest of parsing code. +.. skip: start + When you run the spider, you will get something similar to this:: 2014-01-23 17:48:31-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) @@ -301,6 +307,8 @@ crawling:: 2014-01-23 17:50:03-0400 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) ... +.. skip: end + Note that you can't use the ``fetch`` shortcut here since the Scrapy engine is blocked by the shell. However, after you leave the shell, the spider will continue crawling where it stopped, as shown above. diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 9572a3785..e34999b58 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -42,6 +42,8 @@ attribute. Here is an example of an extension that access stats: def from_crawler(cls, crawler): return cls(crawler.stats) +.. skip: start + Set stat value: .. code-block:: python @@ -80,6 +82,8 @@ Get all stats: >>> stats.get_stats() {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} +.. skip: end + Available Stats Collectors ========================== diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 3e9bbe56e..ae9cb634c 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -97,6 +97,8 @@ convenience: Telnet console usage examples ============================= +.. skip: start + Here are some example tasks you can do with the telnet console: View engine status @@ -146,6 +148,8 @@ To stop:: >>> engine.stop() Connection closed by foreign host. +.. skip: end + Telnet Console signals ====================== diff --git a/pyproject.toml b/pyproject.toml index 0742991db..bc809a7b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -224,23 +224,7 @@ disable = [ [tool.pytest.ini_options] xfail_strict = true -usefixtures = "chdir" python_files = ["test_*.py", "test_*/__init__.py"] -addopts = [ - "--assert=plain", - "--ignore=docs/_ext", - "--ignore=docs/conf.py", - "--ignore=docs/news.rst", - "--ignore=docs/topics/dynamic-content.rst", - "--ignore=docs/topics/items.rst", - "--ignore=docs/topics/leaks.rst", - "--ignore=docs/topics/loaders.rst", - "--ignore=docs/topics/selectors.rst", - "--ignore=docs/topics/shell.rst", - "--ignore=docs/topics/stats.rst", - "--ignore=docs/topics/telnetconsole.rst", - "--ignore=docs/utils", -] markers = [ "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed", "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed", diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 01797fd20..f8f3eb22a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,6 +5,7 @@ import csv import gzip import json import lzma +import os import random import shutil import string @@ -107,8 +108,13 @@ class TestFileFeedStorage: self._assert_stores(FileFeedStorage(str(path)), path) def test_store_direct_path_relative(self, tmp_path): - path = (tmp_path / "foo" / "bar").relative_to(Path.cwd()) - self._assert_stores(FileFeedStorage(str(path)), path) + old_cwd = Path.cwd() + try: + os.chdir(tmp_path) + path = Path("foo", "bar") + self._assert_stores(FileFeedStorage(str(path)), path) + finally: + os.chdir(old_cwd) def test_interface(self, tmp_path): path = tmp_path / "file.txt" @@ -236,24 +242,22 @@ class TestBlockingFeedStorage: def test_default_temp_dir(self): b = BlockingFeedStorage() - tmp = b.open(self.get_test_spider()) - tmp_path = Path(tmp.name).parent - assert str(tmp_path) == tempfile.gettempdir() + storage_file = b.open(self.get_test_spider()) + storage_dir = Path(storage_file.name).parent + assert str(storage_dir) == tempfile.gettempdir() - def test_temp_file(self): + def test_temp_file(self, tmp_path): b = BlockingFeedStorage() - tests_path = Path(__file__).resolve().parent - spider = self.get_test_spider({"FEED_TEMPDIR": str(tests_path)}) - tmp = b.open(spider) - tmp_path = Path(tmp.name).parent - assert tmp_path == tests_path + spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)}) + storage_file = b.open(spider) + storage_dir = Path(storage_file.name).parent + assert storage_dir == tmp_path - def test_invalid_folder(self): + def test_invalid_folder(self, tmp_path): b = BlockingFeedStorage() - tests_path = Path(__file__).resolve().parent - invalid_path = tests_path / "invalid_path" + invalid_path = tmp_path / "invalid_path" spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) with pytest.raises(OSError, match="Not a Directory:"): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index b4eae108f..808fde23d 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -524,21 +524,20 @@ class TestFilesPipelineCustomSettings: expected_value = settings.get(settings_attr) assert getattr(pipeline_cls, pipe_inst_attr) == expected_value - def test_file_pipeline_using_pathlike_objects(self): + def test_file_pipeline_using_pathlike_objects(self, tmp_path): class CustomFilesPipelineWithPathLikeDir(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): return Path("subdir") / Path(request.url).name pipeline = CustomFilesPipelineWithPathLikeDir.from_crawler( - get_crawler(None, {"FILES_STORE": Path("./Temp")}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) request = Request("http://example.com/image01.jpg") assert pipeline.file_path(request) == Path("subdir/image01.jpg") - def test_files_store_constructor_with_pathlike_object(self): - path = Path("./FileDir") - fs_store = FSFilesStore(path) - assert fs_store.basedir == str(path) + def test_files_store_constructor_with_pathlike_object(self, tmp_path): + fs_store = FSFilesStore(tmp_path) + assert fs_store.basedir == str(tmp_path) @pytest.mark.requires_botocore diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index 68bd6df68..8353ad73c 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -2,8 +2,7 @@ Queues that handle requests """ -import shutil -import tempfile +from pathlib import Path import pytest import queuelib @@ -23,30 +22,17 @@ from scrapy.utils.test import get_crawler class TestBaseQueue: def setup_method(self): - self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") - self.qpath = self.tempfilename() - self.qdir = tempfile.mkdtemp() self.crawler = get_crawler(Spider) - def teardown_method(self): - shutil.rmtree(self.tmpdir) - - def tempfilename(self): - with tempfile.NamedTemporaryFile(dir=self.tmpdir) as nf: - return nf.name - - def mkdtemp(self): - return tempfile.mkdtemp(dir=self.tmpdir) - class RequestQueueTestMixin: - def queue(self): + def queue(self, base_path: Path): raise NotImplementedError - def test_one_element_with_peek(self): + def test_one_element_with_peek(self, tmp_path): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): pytest.skip("The queuelib queues do not define peek") - q = self.queue() + q = self.queue(tmp_path) assert len(q) == 0 assert q.peek() is None assert q.pop() is None @@ -60,10 +46,10 @@ class RequestQueueTestMixin: assert q.pop() is None q.close() - def test_one_element_without_peek(self): + def test_one_element_without_peek(self, tmp_path): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): pytest.skip("The queuelib queues define peek") - q = self.queue() + q = self.queue(tmp_path) assert len(q) == 0 assert q.pop() is None req = Request("http://www.example.com") @@ -81,10 +67,10 @@ class RequestQueueTestMixin: class FifoQueueMixin(RequestQueueTestMixin): - def test_fifo_with_peek(self): + def test_fifo_with_peek(self, tmp_path): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): pytest.skip("The queuelib queues do not define peek") - q = self.queue() + q = self.queue(tmp_path) assert len(q) == 0 assert q.peek() is None assert q.pop() is None @@ -108,10 +94,10 @@ class FifoQueueMixin(RequestQueueTestMixin): assert q.pop() is None q.close() - def test_fifo_without_peek(self): + def test_fifo_without_peek(self, tmp_path): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - pytest.skip("The queuelib queues do not define peek") - q = self.queue() + pytest.skip("The queuelib queues define peek") + q = self.queue(tmp_path) assert len(q) == 0 assert q.pop() is None req1 = Request("http://www.example.com/1") @@ -137,10 +123,10 @@ class FifoQueueMixin(RequestQueueTestMixin): class LifoQueueMixin(RequestQueueTestMixin): - def test_lifo_with_peek(self): + def test_lifo_with_peek(self, tmp_path): if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): pytest.skip("The queuelib queues do not define peek") - q = self.queue() + q = self.queue(tmp_path) assert len(q) == 0 assert q.peek() is None assert q.pop() is None @@ -164,10 +150,10 @@ class LifoQueueMixin(RequestQueueTestMixin): assert q.pop() is None q.close() - def test_lifo_without_peek(self): + def test_lifo_without_peek(self, tmp_path): if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): - pytest.skip("The queuelib queues do not define peek") - q = self.queue() + pytest.skip("The queuelib queues define peek") + q = self.queue(tmp_path) assert len(q) == 0 assert q.pop() is None req1 = Request("http://www.example.com/1") @@ -193,34 +179,38 @@ class LifoQueueMixin(RequestQueueTestMixin): class TestPickleFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): - def queue(self): - return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo") + def queue(self, base_path): + return PickleFifoDiskQueue.from_crawler( + crawler=self.crawler, key=str(base_path / "pickle" / "fifo") + ) class TestPickleLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): - def queue(self): - return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo") + def queue(self, base_path): + return PickleLifoDiskQueue.from_crawler( + crawler=self.crawler, key=str(base_path / "pickle" / "lifo") + ) class TestMarshalFifoDiskQueueRequest(FifoQueueMixin, TestBaseQueue): - def queue(self): + def queue(self, base_path): return MarshalFifoDiskQueue.from_crawler( - crawler=self.crawler, key="marshal/fifo" + crawler=self.crawler, key=str(base_path / "marshal" / "fifo") ) class TestMarshalLifoDiskQueueRequest(LifoQueueMixin, TestBaseQueue): - def queue(self): + def queue(self, base_path): return MarshalLifoDiskQueue.from_crawler( - crawler=self.crawler, key="marshal/lifo" + crawler=self.crawler, key=str(base_path / "marshal" / "lifo") ) class TestFifoMemoryQueueRequest(FifoQueueMixin, TestBaseQueue): - def queue(self): + def queue(self, base_path): return FifoMemoryQueue.from_crawler(crawler=self.crawler) class TestLifoMemoryQueueRequest(LifoQueueMixin, TestBaseQueue): - def queue(self): + def queue(self, base_path): return LifoMemoryQueue.from_crawler(crawler=self.crawler) From 85aeda365db01939f70d0888593e7808380c8514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 16 Jun 2025 09:28:06 +0200 Subject: [PATCH 1780/2083] Clean up setting getter defaults (#6892) --- scrapy/core/downloader/__init__.py | 2 +- scrapy/downloadermiddlewares/ajaxcrawl.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 6 ++---- scrapy/extensions/feedexport.py | 2 +- scrapy/extensions/periodic_log.py | 2 +- scrapy/settings/default_settings.py | 4 ++++ scrapy/spiders/crawl.py | 4 +--- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 9293d7b78..4b5654826 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -124,7 +124,7 @@ class Downloader: ) self._slot_gc_loop.start(60) self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( - "DOWNLOAD_SLOTS", {} + "DOWNLOAD_SLOTS" ) @inlineCallbacks diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index e7a8962a1..a23deaa45 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -43,7 +43,7 @@ class AjaxCrawlMiddleware: # middleware parses first 4k. 4k turns out to be insufficient # for this middleware, and parsing 100k could be slow. # We use something in between (32K) by default. - self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE", 32768) + self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE") @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index aba455bdd..fbd737970 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -38,10 +38,8 @@ class RobotsTxtMiddleware: def __init__(self, crawler: Crawler): if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured - self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy") - self._robotstxt_useragent: str | None = crawler.settings.get( - "ROBOTSTXT_USER_AGENT", None - ) + self._default_useragent: str = crawler.settings["USER_AGENT"] + self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"] self.crawler: Crawler = crawler self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {} self._parserimpl: RobotParser = load_object( diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c39a9c92e..d9e9ea775 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -479,7 +479,7 @@ class FeedExporter: uri = self.settings["FEED_URI"] # handle pathlib.Path objects uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() - feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")} + feed_options = {"format": self.settings["FEED_FORMAT"]} self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings ) diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 9158482fa..98210990a 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -78,7 +78,7 @@ class PeriodicLog: ) ext_timing_enabled: bool = crawler.settings.getbool( - "PERIODIC_LOG_TIMING_ENABLED", False + "PERIODIC_LOG_TIMING_ENABLED" ) if not (ext_stats or ext_delta or ext_timing_enabled): raise NotConfigured diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7cd470f11..b6f47f1c3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -20,6 +20,7 @@ from pathlib import Path ADDONS = {} AJAXCRAWL_ENABLED = False +AJAXCRAWL_MAXSIZE = 32768 ASYNCIO_EVENT_LOOP = None @@ -49,6 +50,8 @@ CONCURRENT_REQUESTS_PER_IP = 0 COOKIES_ENABLED = True COOKIES_DEBUG = False +CRAWLSPIDER_FOLLOW_LINKS = True + DEFAULT_DROPITEM_LOG_LEVEL = "WARNING" DEFAULT_ITEM_CLASS = "scrapy.item.Item" @@ -158,6 +161,7 @@ FEED_EXPORTERS_BASE = { "marshal": "scrapy.exporters.MarshalItemExporter", "pickle": "scrapy.exporters.PickleItemExporter", } +FEED_FORMAT = "jsonlines" FEED_STORE_EMPTY = True FEED_STORAGES = {} FEED_STORAGES_BASE = { diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index f44f70e40..98e7b23c0 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -213,7 +213,5 @@ class CrawlSpider(Spider): @classmethod def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) - spider._follow_links = crawler.settings.getbool( - "CRAWLSPIDER_FOLLOW_LINKS", True - ) + spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS") return spider From 91b186cf1868038569ef05a334650eae2a74b5b1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 20:42:11 +0500 Subject: [PATCH 1781/2083] Use new pytest for new Twisted. (#6893) --- tox.ini | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index f28467ec1..85935b011 100644 --- a/tox.ini +++ b/tox.ini @@ -14,16 +14,16 @@ deps = pexpect >= 4.8.0 pyftpdlib >= 2.0.1 pygments - pytest != 8.2.* # https://github.com/pytest-dev/pytest/issues/12275 + pytest pytest-cov >= 4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures - Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467 [testenv] deps = {[test-requirements]deps} + pytest >= 8.4.1 # https://github.com/pytest-dev/pytest/pull/13502 # mitmproxy does not support PyPy mitmproxy; implementation_name != "pypy" @@ -96,6 +96,8 @@ commands = [pinned] basepython = python3.9 deps = + # pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0 + pytest==8.4.0 Protego==0.1.15 Twisted==21.7.0 cryptography==37.0.0 @@ -195,6 +197,7 @@ basepython = pypy3.10 deps = PyPyDispatcher==2.1.0 {[test-requirements]deps} + pytest==8.4.0 Protego==0.1.15 Twisted==21.7.0 cryptography==41.0.5 From d1575220efc605a4e617aee4d4d9948de56fb529 Mon Sep 17 00:00:00 2001 From: Thalison Fernandes Date: Mon, 23 Jun 2025 12:49:57 -0300 Subject: [PATCH 1782/2083] Add .venv folder to .gitignore (#6901) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0a3f0ac1c..e02c2241d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ docs/build *egg-info .tox venv +.venv build dist .idea From 712e965dbd2d58bcfdcab13e289d1e1d1cd0abae Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 20:56:48 +0500 Subject: [PATCH 1783/2083] Replace Black with Ruff in contributing.rst. (#6903) --- docs/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 0172887d6..3976d34c2 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -251,10 +251,10 @@ Coding style Please follow these coding conventions when writing code for inclusion in Scrapy: -* We use `black `_ for code formatting. +* We use `Ruff `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also - run black manually with ``tox -e pre-commit``. + run Ruff manually with ``tox -e pre-commit``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. From 0d86fb69dcfbc51383e2e6fb926b9f166fee4395 Mon Sep 17 00:00:00 2001 From: Thalison Fernandes Date: Mon, 23 Jun 2025 13:56:29 -0300 Subject: [PATCH 1784/2083] Fix FileFeedStorage handling of Windows paths without file:// scheme (#6897) --- scrapy/extensions/feedexport.py | 2 +- tests/test_feedexport.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index d9e9ea775..f7bf50a5c 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -185,7 +185,7 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): - self.path: str = file_uri_to_path(uri) + self.path: str = file_uri_to_path(uri) if uri.startswith("file://") else uri feed_options = feed_options or {} self.write_mode: OpenBinaryMode = ( "wb" if feed_options.get("overwrite", False) else "ab" diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f8f3eb22a..7073d5a35 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -155,6 +155,11 @@ class TestFileFeedStorage: finally: path.unlink() + def test_preserves_windows_path_without_file_scheme(self): + path = r"C:\Users\user\Desktop\test.txt" + storage = FileFeedStorage(path) + assert storage.path == path + class TestFTPFeedStorage(unittest.TestCase): def get_test_spider(self, settings=None): From 9d324ebd1303613adde27dd28a2233803f1bcf14 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 21:58:54 +0500 Subject: [PATCH 1785/2083] Add .vscode to .gitignore. (#6907) --- .gitignore | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index e02c2241d..4100bcd97 100644 --- a/.gitignore +++ b/.gitignore @@ -5,15 +5,16 @@ _trial_temp* dropin.cache docs/build *egg-info -.tox -venv -.venv -build -dist -.idea +.tox/ +venv/ +.venv/ +build/ +dist/ +.idea/ +.vscode/ htmlcov/ -.coverage .pytest_cache/ +.coverage .coverage.* coverage.* *.junit.xml @@ -27,4 +28,4 @@ test-output.* Thumbs.db # OSX miscellaneous -.DS_Store \ No newline at end of file +.DS_Store From 9149b6e7fc47743cdbcb5d3471a9650a2163b09b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 20:42:11 +0500 Subject: [PATCH 1786/2083] Use new pytest for new Twisted. (#6893) --- tox.ini | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 5680d98d1..d3dc7a3db 100644 --- a/tox.ini +++ b/tox.ini @@ -14,17 +14,17 @@ deps = pexpect >= 4.8.0 pyftpdlib >= 2.0.1 pygments - pytest != 8.2.* # https://github.com/pytest-dev/pytest/issues/12275 + pytest pytest-cov >= 4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures pywin32; sys_platform == "win32" - Twisted < 25.5.0 # https://github.com/twisted/twisted/issues/12467 [testenv] deps = {[test-requirements]deps} + pytest >= 8.4.1 # https://github.com/pytest-dev/pytest/pull/13502 # mitmproxy does not support PyPy mitmproxy; implementation_name != "pypy" @@ -97,6 +97,8 @@ commands = [pinned] basepython = python3.9 deps = + # pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0 + pytest==8.4.0 Protego==0.1.15 Twisted==21.7.0 cryptography==37.0.0 @@ -196,6 +198,7 @@ basepython = pypy3.10 deps = PyPyDispatcher==2.1.0 {[test-requirements]deps} + pytest==8.4.0 Protego==0.1.15 Twisted==21.7.0 cryptography==41.0.5 From 020bfa7e5fd98c53db17e0c72db88d43d6fbf1f6 Mon Sep 17 00:00:00 2001 From: Thalison Fernandes Date: Mon, 23 Jun 2025 12:49:57 -0300 Subject: [PATCH 1787/2083] Add .venv folder to .gitignore (#6901) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0a3f0ac1c..e02c2241d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ docs/build *egg-info .tox venv +.venv build dist .idea From 843ad1afb1383a263b8c8da9004a899a71ea864d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 20:56:48 +0500 Subject: [PATCH 1788/2083] Replace Black with Ruff in contributing.rst. (#6903) --- docs/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 0172887d6..3976d34c2 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -251,10 +251,10 @@ Coding style Please follow these coding conventions when writing code for inclusion in Scrapy: -* We use `black `_ for code formatting. +* We use `Ruff `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also - run black manually with ``tox -e pre-commit``. + run Ruff manually with ``tox -e pre-commit``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. From 5e20b46e35c3affc5df7b5f63778e4fb924b28a8 Mon Sep 17 00:00:00 2001 From: Thalison Fernandes Date: Mon, 23 Jun 2025 13:56:29 -0300 Subject: [PATCH 1789/2083] Fix FileFeedStorage handling of Windows paths without file:// scheme (#6897) --- scrapy/extensions/feedexport.py | 2 +- tests/test_feedexport.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 8bcd4e40d..4ac546765 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -185,7 +185,7 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): - self.path: str = file_uri_to_path(uri) + self.path: str = file_uri_to_path(uri) if uri.startswith("file://") else uri feed_options = feed_options or {} self.write_mode: OpenBinaryMode = ( "wb" if feed_options.get("overwrite", False) else "ab" diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 44cd10ec3..7c6425f19 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -143,6 +143,11 @@ class TestFileFeedStorage(unittest.TestCase): finally: path.unlink() + def test_preserves_windows_path_without_file_scheme(self): + path = r"C:\Users\user\Desktop\test.txt" + storage = FileFeedStorage(path) + assert storage.path == path + class TestFTPFeedStorage(unittest.TestCase): def get_test_spider(self, settings=None): From b53faacfcd64f6717645bf4cd33159cef87cd5d9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 21:58:54 +0500 Subject: [PATCH 1790/2083] Add .vscode to .gitignore. (#6907) --- .gitignore | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index e02c2241d..4100bcd97 100644 --- a/.gitignore +++ b/.gitignore @@ -5,15 +5,16 @@ _trial_temp* dropin.cache docs/build *egg-info -.tox -venv -.venv -build -dist -.idea +.tox/ +venv/ +.venv/ +build/ +dist/ +.idea/ +.vscode/ htmlcov/ -.coverage .pytest_cache/ +.coverage .coverage.* coverage.* *.junit.xml @@ -27,4 +28,4 @@ test-output.* Thumbs.db # OSX miscellaneous -.DS_Store \ No newline at end of file +.DS_Store From 0d75355b41a84896d4ab1c19ed1b88f65206fb9f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 22:36:54 +0500 Subject: [PATCH 1791/2083] Handle exceptions in _start_request_processing(), cancel it on engine stop (#6900) --- scrapy/core/engine.py | 48 ++++++++++++++++++++++++++++++------------ scrapy/core/scraper.py | 1 - tests/test_engine.py | 23 +++++++++++++++++++- 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index fe635dc82..d6c171247 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -7,12 +7,13 @@ For more information see docs/topics/architecture.rst from __future__ import annotations +import asyncio import logging from time import time from traceback import format_exc from typing import TYPE_CHECKING, Any, cast -from twisted.internet.defer import Deferred, inlineCallbacks, succeed +from twisted.internet.defer import CancelledError, Deferred, inlineCallbacks, succeed from twisted.python.failure import Failure from scrapy import signals @@ -108,6 +109,8 @@ class ExecutionEngine: ) self.start_time: float | None = None self._start: AsyncIterator[Any] | None = None + self._closewait: Deferred[None] | None = None + self._start_request_processing_dfd: Deferred[None] | None = None downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -139,9 +142,9 @@ class ExecutionEngine: self.start_time = time() await self.signals.send_catch_log_async(signal=signals.engine_started) self.running = True - self._closewait: Deferred[None] = Deferred() + self._closewait = Deferred() if _start_request_processing: - self._start_request_processing() + self._start_request_processing_dfd = self._start_request_processing() await maybe_deferred_to_future(self._closewait) def stop(self) -> Deferred[None]: @@ -150,12 +153,16 @@ class ExecutionEngine: @deferred_f_from_coro_f async def _finish_stopping_engine(_: Any) -> None: await self.signals.send_catch_log_async(signal=signals.engine_stopped) - self._closewait.callback(None) + if self._closewait: + self._closewait.callback(None) if not self.running: raise RuntimeError("Engine not running") self.running = False + if self._start_request_processing_dfd is not None: + self._start_request_processing_dfd.cancel() + self._start_request_processing_dfd = None dfd = ( self.close_spider(self.spider, reason="shutdown") if self.spider is not None @@ -217,17 +224,30 @@ class ExecutionEngine: # Starts the processing of scheduled requests, as well as a periodic # call to that processing method for scenarios where the scheduler # reports having pending requests but returns none. - assert self._slot is not None # typing - self._slot.nextcall.schedule() - self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) + try: + assert self._slot is not None # typing + self._slot.nextcall.schedule() + self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) - while self._start and self.spider: - await self._process_start_next() - if not self.needs_backout(): - # Give room for the outcome of self._process_start_next() to be - # processed before continuing with the next iteration. - self._slot.nextcall.schedule() - await self._slot.nextcall.wait() + while self._start and self.spider: + await self._process_start_next() + if not self.needs_backout(): + # Give room for the outcome of self._process_start_next() to be + # processed before continuing with the next iteration. + self._slot.nextcall.schedule() + await self._slot.nextcall.wait() + except (asyncio.exceptions.CancelledError, CancelledError): + # self.stop() has cancelled us, nothing to do + return + except Exception: + # an error happened, log it and stop the engine + self._start_request_processing_dfd = None + logger.error( + "Error while processing requests from start()", + exc_info=True, + extra={"spider": self.spider}, + ) + await maybe_deferred_to_future(self.stop()) def _start_scheduled_requests(self) -> None: if self._slot is None or self._slot.closing is not None or self.paused: diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 1f0d57c63..dc3a287b4 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -140,7 +140,6 @@ class Scraper: def _check_if_closing(self) -> None: assert self.slot is not None # typing - assert self.crawler.spider if self.slot.closing and self.slot.is_idle(): assert self.crawler.spider self.slot.closing.callback(self.crawler.spider) diff --git a/tests/test_engine.py b/tests/test_engine.py index e181a36cf..d9d25c240 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -25,6 +25,7 @@ import attr import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher +from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest @@ -451,12 +452,32 @@ class TestEngine(TestEngineBase): @inlineCallbacks def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) - yield e.open_spider(MySpider(), []) + yield e.open_spider(MySpider()) e.start() with pytest.raises(RuntimeError, match="Engine already running"): yield e.start() yield e.stop() + @inlineCallbacks + def test_start_request_processing_exception(self): + class BadRequestFingerprinter: + def fingerprint(self, request): + raise ValueError # to make Scheduler.enqueue_request() fail + + class SimpleSpider(Spider): + name = "simple" + + async def start(self): + yield Request("data:,") + + crawler = get_crawler( + SimpleSpider, {"REQUEST_FINGERPRINTER_CLASS": BadRequestFingerprinter} + ) + with LogCapture() as log: + yield crawler.crawl() + assert "Error while processing requests from start()" in str(log) + assert "Spider closed (shutdown)" in str(log) + def test_short_timeout(self): args = ( sys.executable, From 7fbd56bc9baa33a4eb874f4337d740f5f25cfad3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 22:36:54 +0500 Subject: [PATCH 1792/2083] Handle exceptions in _start_request_processing(), cancel it on engine stop (#6900) --- scrapy/core/engine.py | 48 ++++++++++++++++++++++++++++++------------ scrapy/core/scraper.py | 1 - tests/test_engine.py | 23 +++++++++++++++++++- 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 0df9ad2b2..6c6d24d5c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -7,12 +7,13 @@ For more information see docs/topics/architecture.rst from __future__ import annotations +import asyncio import logging from time import time from traceback import format_exc from typing import TYPE_CHECKING, Any, TypeVar, cast -from twisted.internet.defer import Deferred, inlineCallbacks, succeed +from twisted.internet.defer import CancelledError, Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -102,6 +103,8 @@ class ExecutionEngine: ) self.start_time: float | None = None self._start: AsyncIterator[Any] | None = None + self._closewait: Deferred[None] | None = None + self._start_request_processing_dfd: Deferred[None] | None = None downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -133,9 +136,9 @@ class ExecutionEngine: self.signals.send_catch_log_deferred(signal=signals.engine_started) ) self.running = True - self._closewait: Deferred[None] = Deferred() + self._closewait = Deferred() if _start_request_processing: - self._start_request_processing() + self._start_request_processing_dfd = self._start_request_processing() await maybe_deferred_to_future(self._closewait) def stop(self) -> Deferred[None]: @@ -146,12 +149,16 @@ class ExecutionEngine: await maybe_deferred_to_future( self.signals.send_catch_log_deferred(signal=signals.engine_stopped) ) - self._closewait.callback(None) + if self._closewait: + self._closewait.callback(None) if not self.running: raise RuntimeError("Engine not running") self.running = False + if self._start_request_processing_dfd is not None: + self._start_request_processing_dfd.cancel() + self._start_request_processing_dfd = None dfd = ( self.close_spider(self.spider, reason="shutdown") if self.spider is not None @@ -213,17 +220,30 @@ class ExecutionEngine: # Starts the processing of scheduled requests, as well as a periodic # call to that processing method for scenarios where the scheduler # reports having pending requests but returns none. - assert self._slot is not None # typing - self._slot.nextcall.schedule() - self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) + try: + assert self._slot is not None # typing + self._slot.nextcall.schedule() + self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) - while self._start and self.spider: - await self._process_start_next() - if not self.needs_backout(): - # Give room for the outcome of self._process_start_next() to be - # processed before continuing with the next iteration. - self._slot.nextcall.schedule() - await self._slot.nextcall.wait() + while self._start and self.spider: + await self._process_start_next() + if not self.needs_backout(): + # Give room for the outcome of self._process_start_next() to be + # processed before continuing with the next iteration. + self._slot.nextcall.schedule() + await self._slot.nextcall.wait() + except (asyncio.exceptions.CancelledError, CancelledError): + # self.stop() has cancelled us, nothing to do + return + except Exception: + # an error happened, log it and stop the engine + self._start_request_processing_dfd = None + logger.error( + "Error while processing requests from start()", + exc_info=True, + extra={"spider": self.spider}, + ) + await maybe_deferred_to_future(self.stop()) def _start_scheduled_requests(self) -> None: if self._slot is None or self._slot.closing is not None or self.paused: diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index ac720e03f..a2fe281e0 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -139,7 +139,6 @@ class Scraper: def _check_if_closing(self) -> None: assert self.slot is not None # typing - assert self.crawler.spider if self.slot.closing and self.slot.is_idle(): assert self.crawler.spider self.slot.closing.callback(self.crawler.spider) diff --git a/tests/test_engine.py b/tests/test_engine.py index b2e436425..a3c5243d5 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -25,6 +25,7 @@ import attr import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher +from testfixtures import LogCapture from twisted.internet import defer, reactor from twisted.trial import unittest from twisted.web import server, static, util @@ -448,7 +449,7 @@ class TestEngine(TestEngineBase): @defer.inlineCallbacks def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) - yield e.open_spider(MySpider(), []) + yield e.open_spider(MySpider()) e.start() def cb(exc: BaseException) -> None: @@ -459,6 +460,26 @@ class TestEngine(TestEngineBase): finally: yield e.stop() + @defer.inlineCallbacks + def test_start_request_processing_exception(self): + class BadRequestFingerprinter: + def fingerprint(self, request): + raise ValueError # to make Scheduler.enqueue_request() fail + + class SimpleSpider(Spider): + name = "simple" + + async def start(self): + yield Request("data:,") + + crawler = get_crawler( + SimpleSpider, {"REQUEST_FINGERPRINTER_CLASS": BadRequestFingerprinter} + ) + with LogCapture() as log: + yield crawler.crawl() + assert "Error while processing requests from start()" in str(log) + assert "Spider closed (shutdown)" in str(log) + def test_short_timeout(self): args = ( sys.executable, From d70f8a3f14252715fbc9b9541364c5b5d142335f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 23 Jun 2025 23:39:24 +0500 Subject: [PATCH 1793/2083] Refactoring of test_utils_*. (#6905) --- tests/test_command_startproject.py | 2 - tests/test_request_cb_kwargs.py | 2 - tests/test_toplevel.py | 42 ++- tests/test_urlparse_monkeypatches.py | 10 - tests/test_utils_conf.py | 11 +- tests/test_utils_console.py | 35 +- tests/test_utils_curl.py | 6 +- tests/test_utils_datatypes.py | 12 +- tests/test_utils_deprecate.py | 5 +- tests/test_utils_display.py | 146 +++---- tests/test_utils_gz.py | 78 ++-- tests/test_utils_httpobj.py | 27 +- tests/test_utils_iterators.py | 80 ++-- tests/test_utils_log.py | 185 +++++---- tests/test_utils_project.py | 33 +- tests/test_utils_python.py | 283 +++++++------- tests/test_utils_request.py | 100 ++--- tests/test_utils_response.py | 370 +++++++++--------- tests/test_utils_serialize.py | 30 +- tests/test_utils_sitemap.py | 421 ++++++++++----------- tests/test_utils_spider.py | 26 +- tests/test_utils_template.py | 27 +- tests/test_utils_trackref.py | 95 ++--- tests/test_utils_url.py | 543 +++++++++++---------------- 24 files changed, 1251 insertions(+), 1318 deletions(-) delete mode 100644 tests/test_urlparse_monkeypatches.py diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 988ad50b9..1edef0b4a 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -106,8 +106,6 @@ def get_permissions_dict( class TestStartprojectTemplates(TestProjectBase): - maxDiff = None - def setup_method(self): super().setup_method() self.tmpl = str(Path(self.temp_path, "templates")) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 9d2e5f997..1714bd4db 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -150,8 +150,6 @@ class KeywordArgumentsSpider(MockServerSpider): class TestCallbackKeywordArguments(TestCase): - maxDiff = None - @classmethod def setUpClass(cls): cls.mockserver = MockServer() diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index a4f31096e..66a6f5318 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -1,31 +1,35 @@ import scrapy -class TestToplevel: - def test_version(self): - assert isinstance(scrapy.__version__, str) +def test_version(): + assert isinstance(scrapy.__version__, str) - def test_version_info(self): - assert isinstance(scrapy.version_info, tuple) - def test_request_shortcut(self): - from scrapy.http import FormRequest, Request +def test_version_info(): + assert isinstance(scrapy.version_info, tuple) - assert scrapy.Request is Request - assert scrapy.FormRequest is FormRequest - def test_spider_shortcut(self): - from scrapy.spiders import Spider +def test_request_shortcut(): + from scrapy.http import FormRequest, Request - assert scrapy.Spider is Spider + assert scrapy.Request is Request + assert scrapy.FormRequest is FormRequest - def test_selector_shortcut(self): - from scrapy.selector import Selector - assert scrapy.Selector is Selector +def test_spider_shortcut(): + from scrapy.spiders import Spider - def test_item_shortcut(self): - from scrapy.item import Field, Item + assert scrapy.Spider is Spider - assert scrapy.Item is Item - assert scrapy.Field is Field + +def test_selector_shortcut(): + from scrapy.selector import Selector + + assert scrapy.Selector is Selector + + +def test_item_shortcut(): + from scrapy.item import Field, Item + + assert scrapy.Item is Item + assert scrapy.Field is Field diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py deleted file mode 100644 index 0e1e89e81..000000000 --- a/tests/test_urlparse_monkeypatches.py +++ /dev/null @@ -1,10 +0,0 @@ -from urllib.parse import urlparse - - -class TestUrlparse: - def test_s3_url(self): - p = urlparse("s3://bucket/key/name?param=value") - assert p.scheme == "s3" - assert p.hostname == "bucket" - assert p.path == "/key/name" - assert p.query == "param=value" diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 26f158380..ed7dda18d 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -47,12 +47,11 @@ class TestBuildComponentList: assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"] -class TestUtilsConf: - def test_arglist_to_dict(self): - assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == { - "arg1": "val1", - "arg2": "val2", - } +def test_arglist_to_dict(): + assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == { + "arg1": "val1", + "arg2": "val2", + } class TestFeedExportConfig: diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index 6598bdce7..dc1d96f66 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -18,23 +18,24 @@ except ImportError: ipy = False -class TestUtilsConsole: - def test_get_shell_embed_func(self): - shell = get_shell_embed_func(["invalid"]) - assert shell is None +def test_get_shell_embed_func(): + shell = get_shell_embed_func(["invalid"]) + assert shell is None - shell = get_shell_embed_func(["invalid", "python"]) - assert callable(shell) - assert shell.__name__ == "_embed_standard_shell" + shell = get_shell_embed_func(["invalid", "python"]) + assert callable(shell) + assert shell.__name__ == "_embed_standard_shell" - @pytest.mark.skipif(not bpy, reason="bpython not available in testenv") - def test_get_shell_embed_func2(self): - shell = get_shell_embed_func(["bpython"]) - assert callable(shell) - assert shell.__name__ == "_embed_bpython_shell" - @pytest.mark.skipif(not ipy, reason="IPython not available in testenv") - def test_get_shell_embed_func3(self): - # default shell should be 'ipython' - shell = get_shell_embed_func() - assert shell.__name__ == "_embed_ipython_shell" +@pytest.mark.skipif(not bpy, reason="bpython not available in testenv") +def test_get_shell_embed_func_bpython(): + shell = get_shell_embed_func(["bpython"]) + assert callable(shell) + assert shell.__name__ == "_embed_bpython_shell" + + +@pytest.mark.skipif(not ipy, reason="IPython not available in testenv") +def test_get_shell_embed_func_ipython(): + # default shell should be 'ipython' + shell = get_shell_embed_func() + assert shell.__name__ == "_embed_ipython_shell" diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index e8dd88049..02362693a 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,4 +1,5 @@ import warnings +from typing import Any import pytest from w3lib.http import basic_auth_header @@ -8,9 +9,8 @@ from scrapy.utils.curl import curl_to_request_kwargs class TestCurlToRequestKwargs: - maxDiff = 5000 - - def _test_command(self, curl_command, expected_result): + @staticmethod + def _test_command(curl_command: str, expected_result: dict[str, Any]) -> None: result = curl_to_request_kwargs(curl_command) assert result == expected_result try: diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 75b6b0e99..352e49165 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,5 +1,6 @@ import copy import warnings +from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, MutableMapping import pytest @@ -16,7 +17,12 @@ from scrapy.utils.datatypes import ( from scrapy.utils.python import garbage_collect -class CaseInsensitiveDictBase: +class TestCaseInsensitiveDictBase(ABC): + @property + @abstractmethod + def dict_class(self) -> type[MutableMapping]: + raise NotImplementedError + def test_init_dict(self): seq = {"red": 1, "black": 3} d = self.dict_class(seq) @@ -199,7 +205,7 @@ class CaseInsensitiveDictBase: assert h1.get("header1") == h3.get("HEADER1") -class TestCaseInsensitiveDict(CaseInsensitiveDictBase): +class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): dict_class = CaseInsensitiveDict def test_repr(self): @@ -216,7 +222,7 @@ class TestCaseInsensitiveDict(CaseInsensitiveDictBase): @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestCaselessDict(CaseInsensitiveDictBase): +class TestCaselessDict(TestCaseInsensitiveDictBase): dict_class = CaselessDict def test_deprecation_message(self): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 662de0dc3..a88b5e008 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,6 +1,7 @@ import inspect import warnings from unittest import mock +from warnings import WarningMessage import pytest @@ -21,7 +22,9 @@ class NewName(SomeBaseClass): class TestWarnWhenSubclassed: - def _mywarnings(self, w, category=MyWarning): + def _mywarnings( + self, w: list[WarningMessage], category: type[Warning] = MyWarning + ) -> list[WarningMessage]: return [x for x in w if x.category is MyWarning] def test_no_warning_on_definition(self): diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index cea564653..20251ca59 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -3,88 +3,92 @@ from unittest import mock from scrapy.utils.display import pformat, pprint - -class TestDisplay: - object = {"a": 1} - colorized_strings = { +value = {"a": 1} +colorized_strings = { + ( ( - ( - "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" - "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}" - ) - + suffix + "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" + "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}" ) - for suffix in ( - # https://github.com/pygments/pygments/issues/2313 - "\n", # pygments ≤ 2.13 - "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14 - ) - } - plain_string = "{'a': 1}" + + suffix + ) + for suffix in ( + # https://github.com/pygments/pygments/issues/2313 + "\n", # pygments ≤ 2.13 + "\x1b[37m\x1b[39;49;00m\n", # pygments ≥ 2.14 + ) +} +plain_string = "{'a': 1}" - @mock.patch("sys.platform", "linux") - @mock.patch("sys.stdout.isatty") - def test_pformat(self, isatty): - isatty.return_value = True - assert pformat(self.object) in self.colorized_strings - @mock.patch("sys.stdout.isatty") - def test_pformat_dont_colorize(self, isatty): - isatty.return_value = True - assert pformat(self.object, colorize=False) == self.plain_string +@mock.patch("sys.platform", "linux") +@mock.patch("sys.stdout.isatty") +def test_pformat(isatty): + isatty.return_value = True + assert pformat(value) in colorized_strings - def test_pformat_not_tty(self): - assert pformat(self.object) == self.plain_string - @mock.patch("sys.platform", "win32") - @mock.patch("platform.version") - @mock.patch("sys.stdout.isatty") - def test_pformat_old_windows(self, isatty, version): - isatty.return_value = True - version.return_value = "10.0.14392" - assert pformat(self.object) in self.colorized_strings +@mock.patch("sys.stdout.isatty") +def test_pformat_dont_colorize(isatty): + isatty.return_value = True + assert pformat(value, colorize=False) == plain_string - @mock.patch("sys.platform", "win32") - @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") - @mock.patch("platform.version") - @mock.patch("sys.stdout.isatty") - def test_pformat_windows_no_terminal_processing( - self, isatty, version, terminal_processing - ): - isatty.return_value = True - version.return_value = "10.0.14393" - terminal_processing.return_value = False - assert pformat(self.object) == self.plain_string - @mock.patch("sys.platform", "win32") - @mock.patch("scrapy.utils.display._enable_windows_terminal_processing") - @mock.patch("platform.version") - @mock.patch("sys.stdout.isatty") - def test_pformat_windows(self, isatty, version, terminal_processing): - isatty.return_value = True - version.return_value = "10.0.14393" - terminal_processing.return_value = True - assert pformat(self.object) in self.colorized_strings +def test_pformat_not_tty(): + assert pformat(value) == plain_string - @mock.patch("sys.platform", "linux") - @mock.patch("sys.stdout.isatty") - def test_pformat_no_pygments(self, isatty): - isatty.return_value = True - import builtins +@mock.patch("sys.platform", "win32") +@mock.patch("platform.version") +@mock.patch("sys.stdout.isatty") +def test_pformat_old_windows(isatty, version): + isatty.return_value = True + version.return_value = "10.0.14392" + assert pformat(value) in colorized_strings - real_import = builtins.__import__ - def mock_import(name, globals, locals, fromlist, level): - if "pygments" in name: - raise ImportError - return real_import(name, globals, locals, fromlist, level) +@mock.patch("sys.platform", "win32") +@mock.patch("scrapy.utils.display._enable_windows_terminal_processing") +@mock.patch("platform.version") +@mock.patch("sys.stdout.isatty") +def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing): + isatty.return_value = True + version.return_value = "10.0.14393" + terminal_processing.return_value = False + assert pformat(value) == plain_string - builtins.__import__ = mock_import - assert pformat(self.object) == self.plain_string - builtins.__import__ = real_import - def test_pprint(self): - with mock.patch("sys.stdout", new=StringIO()) as mock_out: - pprint(self.object) - assert mock_out.getvalue() == "{'a': 1}\n" +@mock.patch("sys.platform", "win32") +@mock.patch("scrapy.utils.display._enable_windows_terminal_processing") +@mock.patch("platform.version") +@mock.patch("sys.stdout.isatty") +def test_pformat_windows(isatty, version, terminal_processing): + isatty.return_value = True + version.return_value = "10.0.14393" + terminal_processing.return_value = True + assert pformat(value) in colorized_strings + + +@mock.patch("sys.platform", "linux") +@mock.patch("sys.stdout.isatty") +def test_pformat_no_pygments(isatty): + isatty.return_value = True + + import builtins + + real_import = builtins.__import__ + + def mock_import(name, globals, locals, fromlist, level): + if "pygments" in name: + raise ImportError + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = mock_import + assert pformat(value) == plain_string + builtins.__import__ = real_import + + +def test_pprint(): + with mock.patch("sys.stdout", new=StringIO()) as mock_out: + pprint(value) + assert mock_out.getvalue() == "{'a': 1}\n" diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index c43ed152b..06fdf9cba 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -11,47 +11,51 @@ from tests import tests_datadir SAMPLEDIR = Path(tests_datadir, "compressed") -class TestGunzip: - def test_gunzip_basic(self): - r1 = Response( - "http://www.example.com", - body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(), - ) - assert gzip_magic_number(r1) +def test_gunzip_basic(): + r1 = Response( + "http://www.example.com", + body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(), + ) + assert gzip_magic_number(r1) - r2 = Response("http://www.example.com", body=gunzip(r1.body)) - assert not gzip_magic_number(r2) - assert len(r2.body) == 9950 + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert not gzip_magic_number(r2) + assert len(r2.body) == 9950 - def test_gunzip_truncated(self): - text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes()) - assert text.endswith(b"") - assert not gzip_magic_number(r2) +def test_gunzip_no_gzip_file_raises(): + with pytest.raises(BadGzipFile): + gunzip((SAMPLEDIR / "feed-sample1.xml").read_bytes()) - def test_is_gzipped_empty(self): - r1 = Response("http://www.example.com") - assert not gzip_magic_number(r1) - def test_gunzip_illegal_eof(self): - text = html_to_unicode( - "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes()) - )[1] - expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text( - encoding="utf-8" - ) - assert len(text) == len(expected_text) - assert text == expected_text +def test_gunzip_truncated_short(): + r1 = Response( + "http://www.example.com", + body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(), + ) + assert gzip_magic_number(r1) + + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert r2.body.endswith(b"") + assert not gzip_magic_number(r2) + + +def test_is_gzipped_empty(): + r1 = Response("http://www.example.com") + assert not gzip_magic_number(r1) + + +def test_gunzip_illegal_eof(): + text = html_to_unicode( + "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes()) + )[1] + expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text( + encoding="utf-8" + ) + assert len(text) == len(expected_text) + assert text == expected_text diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 0c05ef7d6..9bd86f7fb 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -4,18 +4,17 @@ from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached -class TestHttpobjUtils: - def test_urlparse_cached(self): - url = "http://www.example.com/index.html" - request1 = Request(url) - request2 = Request(url) - req1a = urlparse_cached(request1) - req1b = urlparse_cached(request1) - req2 = urlparse_cached(request2) - urlp = urlparse(url) +def test_urlparse_cached(): + url = "http://www.example.com/index.html" + request1 = Request(url) + request2 = Request(url) + req1a = urlparse_cached(request1) + req1b = urlparse_cached(request1) + req2 = urlparse_cached(request2) + urlp = urlparse(url) - assert req1a == req2 - assert req1a == urlp - assert req1a is req1b - assert req1a is not req2 - assert req1a is not req2 + assert req1a == req2 + assert req1a == urlp + assert req1a is req1b + assert req1a is not req2 + assert req1a is not req2 diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index fa0d37866..ac32fff2c 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,3 +1,8 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + import pytest from scrapy.exceptions import ScrapyDeprecationWarning @@ -5,9 +10,19 @@ from scrapy.http import Response, TextResponse, XmlResponse from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata +if TYPE_CHECKING: + from collections.abc import Iterator + + from scrapy import Selector + + +class TestXmliterBase(ABC): + @abstractmethod + def xmliter( + self, obj: Response | str | bytes, nodename: str, *args: Any + ) -> Iterator[Selector]: + raise NotImplementedError -class XmliterBase: - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter(self): body = b""" @@ -39,7 +54,6 @@ class XmliterBase: ("002", ["Name 2"], ["Type 2"]), ] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unusual_node(self): body = b""" @@ -53,7 +67,6 @@ class XmliterBase: ] assert nodenames == [["matchme..."]] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 body = """ @@ -113,7 +126,6 @@ class XmliterBase: ("27", ["A"], ["27"]), ] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_text(self): body = ( '' @@ -125,7 +137,6 @@ class XmliterBase: ["two"], ] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaces(self): body = b""" @@ -163,7 +174,6 @@ class XmliterBase: assert node.xpath("id/text()").getall() == [] assert node.xpath("price/text()").getall() == [] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename(self): body = b""" @@ -191,7 +201,6 @@ class XmliterBase: "http://www.mydummycompany.com/images/item1.jpg" ] - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename_missing(self): body = b""" @@ -216,7 +225,6 @@ class XmliterBase: with pytest.raises(StopIteration): next(my_iter) - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_exception(self): body = ( '' @@ -229,13 +237,11 @@ class XmliterBase: with pytest.raises(StopIteration): next(iter) - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") with pytest.raises(TypeError): next(i) - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_encoding(self): body = ( b'\n' @@ -250,8 +256,12 @@ class XmliterBase: ) -class TestXmliter(XmliterBase): - xmliter = staticmethod(xmliter) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestXmliter(TestXmliterBase): + def xmliter( + self, obj: Response | str | bytes, nodename: str, *args: Any + ) -> Iterator[Selector]: + return xmliter(obj, nodename) def test_deprecation(self): body = b""" @@ -267,8 +277,11 @@ class TestXmliter(XmliterBase): next(self.xmliter(body, "product")) -class TestLxmlXmliter(XmliterBase): - xmliter = staticmethod(xmliter_lxml) +class TestLxmlXmliter(TestXmliterBase): + def xmliter( + self, obj: Response | str | bytes, nodename: str, *args: Any + ) -> Iterator[Selector]: + return xmliter_lxml(obj, nodename, *args) def test_xmliter_iterate_namespace(self): body = b""" @@ -493,23 +506,32 @@ class TestUtilsCsv: ] -class TestHelper: +class TestBodyOrStr: bbody = b"utf8-body" ubody = bbody.decode("utf8") - txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8") - response = Response(url="http://example.org/", body=bbody) - def test_body_or_str(self): - for obj in (self.bbody, self.ubody, self.txtresponse, self.response): - r1 = _body_or_str(obj) - self._assert_type_and_value(r1, self.ubody, obj) - r2 = _body_or_str(obj, unicode=True) - self._assert_type_and_value(r2, self.ubody, obj) - r3 = _body_or_str(obj, unicode=False) - self._assert_type_and_value(r3, self.bbody, obj) - assert type(r1) is type(r2) - assert type(r1) is not type(r3) + @pytest.mark.parametrize( + "obj", + [ + bbody, + ubody, + TextResponse(url="http://example.org/", body=bbody, encoding="utf-8"), + Response(url="http://example.org/", body=bbody), + ], + ) + def test_body_or_str(self, obj: Response | str | bytes) -> None: + r1 = _body_or_str(obj) + self._assert_type_and_value(r1, self.ubody, obj) + r2 = _body_or_str(obj, unicode=True) + self._assert_type_and_value(r2, self.ubody, obj) + r3 = _body_or_str(obj, unicode=False) + self._assert_type_and_value(r3, self.bbody, obj) + assert type(r1) is type(r2) + assert type(r1) is not type(r3) - def _assert_type_and_value(self, a, b, obj): + @staticmethod + def _assert_type_and_value( + a: str | bytes, b: str | bytes, obj: Response | str | bytes + ) -> None: assert type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}" assert a == b diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 56375606c..f40e424ff 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -22,7 +22,9 @@ from scrapy.utils.test import get_crawler from tests.spiders import LogSpider if TYPE_CHECKING: - from collections.abc import Mapping, MutableMapping + from collections.abc import Generator, Mapping, MutableMapping + + from scrapy.crawler import Crawler class TestFailureToExcInfo: @@ -70,33 +72,41 @@ class TestTopLevelFormatter: class TestLogCounterHandler: - def setup_method(self): + @pytest.fixture + def crawler(self) -> Crawler: settings = {"LOG_LEVEL": "WARNING"} - self.logger = logging.getLogger("test") - self.logger.setLevel(logging.NOTSET) - self.logger.propagate = False - self.crawler = get_crawler(settings_dict=settings) - self.handler = LogCounterHandler(self.crawler) - self.logger.addHandler(self.handler) + return get_crawler(settings_dict=settings) - def teardown_method(self): - self.logger.propagate = True - self.logger.removeHandler(self.handler) + @pytest.fixture + def logger(self, crawler: Crawler) -> Generator[logging.Logger]: + logger = logging.getLogger("test") + logger.setLevel(logging.NOTSET) + logger.propagate = False + handler = LogCounterHandler(crawler) + logger.addHandler(handler) - def test_init(self): - assert self.crawler.stats.get_value("log_count/DEBUG") is None - assert self.crawler.stats.get_value("log_count/INFO") is None - assert self.crawler.stats.get_value("log_count/WARNING") is None - assert self.crawler.stats.get_value("log_count/ERROR") is None - assert self.crawler.stats.get_value("log_count/CRITICAL") is None + yield logger - def test_accepted_level(self): - self.logger.error("test log msg") - assert self.crawler.stats.get_value("log_count/ERROR") == 1 + logger.propagate = True + logger.removeHandler(handler) - def test_filtered_out_level(self): - self.logger.debug("test log msg") - assert self.crawler.stats.get_value("log_count/INFO") is None + def test_init(self, crawler: Crawler, logger: logging.Logger) -> None: + assert crawler.stats + assert crawler.stats.get_value("log_count/DEBUG") is None + assert crawler.stats.get_value("log_count/INFO") is None + assert crawler.stats.get_value("log_count/WARNING") is None + assert crawler.stats.get_value("log_count/ERROR") is None + assert crawler.stats.get_value("log_count/CRITICAL") is None + + def test_accepted_level(self, crawler: Crawler, logger: logging.Logger) -> None: + logger.error("test log msg") + assert crawler.stats + assert crawler.stats.get_value("log_count/ERROR") == 1 + + def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None: + logger.debug("test log msg") + assert crawler.stats + assert crawler.stats.get_value("log_count/INFO") is None class TestStreamLogger: @@ -135,7 +145,7 @@ class TestStreamLogger: ) def test_spider_logger_adapter_process( base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict -): +) -> None: logger = logging.getLogger("test") spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) @@ -149,59 +159,75 @@ def test_spider_logger_adapter_process( class TestLogging: - def setup_method(self): - self.log_stream = StringIO() - handler = logging.StreamHandler(self.log_stream) + @pytest.fixture + def log_stream(self) -> StringIO: + return StringIO() + + @pytest.fixture + def spider(self) -> LogSpider: + return LogSpider() + + @pytest.fixture(autouse=True) + def logger(self, log_stream: StringIO) -> Generator[logging.Logger]: + handler = logging.StreamHandler(log_stream) logger = logging.getLogger("log_spider") logger.addHandler(handler) logger.setLevel(logging.DEBUG) - self.handler = handler - self.logger = logger - self.spider = LogSpider() - def teardown_method(self): - self.logger.removeHandler(self.handler) + yield logger - def test_debug_logging(self): + logger.removeHandler(handler) + + def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo message" - self.spider.log_debug(log_message) - log_contents = self.log_stream.getvalue() + spider.log_debug(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" - def test_info_logging(self): + def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Bar message" - self.spider.log_info(log_message) - log_contents = self.log_stream.getvalue() + spider.log_info(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" - def test_warning_logging(self): + def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Baz message" - self.spider.log_warning(log_message) - log_contents = self.log_stream.getvalue() + spider.log_warning(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" - def test_error_logging(self): + def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo bar message" - self.spider.log_error(log_message) - log_contents = self.log_stream.getvalue() + spider.log_error(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" - def test_critical_logging(self): + def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo bar baz message" - self.spider.log_critical(log_message) - log_contents = self.log_stream.getvalue() + spider.log_critical(log_message) + log_contents = log_stream.getvalue() assert log_contents == f"{log_message}\n" class TestLoggingWithExtra: - def setup_method(self): - self.log_stream = StringIO() - handler = logging.StreamHandler(self.log_stream) + regex_pattern = re.compile(r"^]+>$") + + @pytest.fixture + def log_stream(self) -> StringIO: + return StringIO() + + @pytest.fixture + def spider(self) -> LogSpider: + return LogSpider() + + @pytest.fixture(autouse=True) + def logger(self, log_stream: StringIO) -> Generator[logging.Logger]: + handler = logging.StreamHandler(log_stream) formatter = logging.Formatter( '{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}' ) @@ -209,80 +235,79 @@ class TestLoggingWithExtra: logger = logging.getLogger("log_spider") logger.addHandler(handler) logger.setLevel(logging.DEBUG) - self.handler = handler - self.logger = logger - self.spider = LogSpider() - self.regex_pattern = re.compile(r"^]+>$") - def teardown_method(self): - self.logger.removeHandler(self.handler) + yield logger - def test_debug_logging(self): + logger.removeHandler(handler) + + def test_debug_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo message" extra = {"important_info": "foo"} - self.spider.log_debug(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_debug(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "DEBUG" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_info_logging(self): + def test_info_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Bar message" extra = {"important_info": "bar"} - self.spider.log_info(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_info(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "INFO" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_warning_logging(self): + def test_warning_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Baz message" extra = {"important_info": "baz"} - self.spider.log_warning(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_warning(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "WARNING" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_error_logging(self): + def test_error_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo bar message" extra = {"important_info": "foo bar"} - self.spider.log_error(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_error(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "ERROR" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_critical_logging(self): + def test_critical_logging(self, log_stream: StringIO, spider: LogSpider) -> None: log_message = "Foo bar baz message" extra = {"important_info": "foo bar baz"} - self.spider.log_critical(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_critical(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "CRITICAL" assert log_contents["message"] == log_message assert self.regex_pattern.match(log_contents["spider"]) assert log_contents["important_info"] == extra["important_info"] - def test_overwrite_spider_extra(self): + def test_overwrite_spider_extra( + self, log_stream: StringIO, spider: LogSpider + ) -> None: log_message = "Foo message" extra = {"important_info": "foo", "spider": "shouldn't change"} - self.spider.log_error(log_message, extra) - log_contents = self.log_stream.getvalue() - log_contents = json.loads(log_contents) + spider.log_error(log_message, extra) + log_contents_str = log_stream.getvalue() + log_contents = json.loads(log_contents_str) assert log_contents["levelname"] == "ERROR" assert log_contents["message"] == log_message diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index aa250be69..20a3d940c 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -1,18 +1,17 @@ -import contextlib import os -import shutil -import tempfile import warnings from pathlib import Path +import pytest + from scrapy.utils.misc import set_environ from scrapy.utils.project import data_path, get_project_settings -@contextlib.contextmanager -def inside_a_project(): +@pytest.fixture +def proj_path(tmp_path): prev_dir = Path.cwd() - project_dir = tempfile.mkdtemp() + project_dir = tmp_path try: os.chdir(project_dir) @@ -21,21 +20,19 @@ def inside_a_project(): yield project_dir finally: os.chdir(prev_dir) - shutil.rmtree(project_dir) -class TestProjectUtils: - def test_data_path_outside_project(self): - assert str(Path(".scrapy", "somepath")) == data_path("somepath") - abspath = str(Path(os.path.sep, "absolute", "path")) - assert abspath == data_path(abspath) +def test_data_path_outside_project(): + assert str(Path(".scrapy", "somepath")) == data_path("somepath") + abspath = str(Path(os.path.sep, "absolute", "path")) + assert abspath == data_path(abspath) - def test_data_path_inside_project(self): - with inside_a_project() as proj_path: - expected = Path(proj_path, ".scrapy", "somepath") - assert expected.resolve() == Path(data_path("somepath")).resolve() - abspath = str(Path(os.path.sep, "absolute", "path").resolve()) - assert abspath == data_path(abspath) + +def test_data_path_inside_project(proj_path: Path) -> None: + expected = proj_path / ".scrapy" / "somepath" + assert expected.resolve() == Path(data_path("somepath")).resolve() + abspath = str(Path(os.path.sep, "absolute", "path").resolve()) + assert abspath == data_path(abspath) class TestGetProjectSettings: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 291646ad7..c933e0ac9 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,7 +1,10 @@ +from __future__ import annotations + import functools import operator import platform import sys +from typing import TYPE_CHECKING, TypeVar import pytest from twisted.trial import unittest @@ -20,16 +23,22 @@ from scrapy.utils.python import ( without_none_values, ) +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping -class TestMutableChain: - def test_mutablechain(self): - m = MutableChain(range(2), [2, 3], (4, 5)) - m.extend(range(6, 7)) - m.extend([7, 8]) - m.extend([9, 10], (11, 12)) - assert next(m) == 0 - assert m.__next__() == 1 - assert list(m) == list(range(2, 13)) + +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") + + +def test_mutablechain(): + m = MutableChain(range(2), [2, 3], (4, 5)) + m.extend(range(6, 7)) + m.extend([7, 8]) + m.extend([9, 10], (11, 12)) + assert next(m) == 0 + assert m.__next__() == 1 + assert list(m) == list(range(2, 13)) class TestMutableAsyncChain(unittest.TestCase): @@ -112,144 +121,150 @@ class TestToBytes: assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b" -class TestMemoizedMethod: - def test_memoizemethod_noargs(self): - class A: - @memoizemethod_noargs - def cached(self): - return object() +def test_memoizemethod_noargs(): + class A: + @memoizemethod_noargs + def cached(self): + return object() - def noncached(self): - return object() + def noncached(self): + return object() - a = A() - one = a.cached() - two = a.cached() - three = a.noncached() - assert one is two - assert one is not three + a = A() + one = a.cached() + two = a.cached() + three = a.noncached() + assert one is two + assert one is not three -class TestBinaryIsText: - def test_binaryistext(self): - assert binary_is_text(b"hello") - - def test_utf_16_strings_contain_null_bytes(self): - assert binary_is_text("hello".encode("utf-16")) - - def test_one_with_encoding(self): - assert binary_is_text(b"
Price \xa3
") - - def test_real_binary_bytes(self): - assert not binary_is_text(b"\x02\xa3") +@pytest.mark.parametrize( + ("value", "expected"), + [ + (b"hello", True), + ("hello".encode("utf-16"), True), + (b"
Price \xa3
", True), + (b"\x02\xa3", False), + ], +) +def test_binaryistext(value: bytes, expected: bool) -> None: + assert binary_is_text(value) is expected -class TestUtilsPython: - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_equal_attributes(self): - class Obj: +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +def test_equal_attributes(): + class Obj: + pass + + a = Obj() + b = Obj() + # no attributes given return False + assert not equal_attributes(a, b, []) + # nonexistent attributes + assert not equal_attributes(a, b, ["x", "y"]) + + a.x = 1 + b.x = 1 + # equal attribute + assert equal_attributes(a, b, ["x"]) + + b.y = 2 + # obj1 has no attribute y + assert not equal_attributes(a, b, ["x", "y"]) + + a.y = 2 + # equal attributes + assert equal_attributes(a, b, ["x", "y"]) + + a.y = 1 + # different attributes + assert not equal_attributes(a, b, ["x", "y"]) + + # test callable + a.meta = {} + b.meta = {} + assert equal_attributes(a, b, ["meta"]) + + # compare ['meta']['a'] + a.meta["z"] = 1 + b.meta["z"] = 1 + + get_z = operator.itemgetter("z") + get_meta = operator.attrgetter("meta") + + def compare_z(obj): + return get_z(get_meta(obj)) + + assert equal_attributes(a, b, [compare_z, "x"]) + # fail z equality + a.meta["z"] = 2 + assert not equal_attributes(a, b, [compare_z, "x"]) + + +def test_get_func_args(): + def f1(a, b, c): + pass + + def f2(a, b=None, c=None): + pass + + def f3(a, b=None, *, c=None): + pass + + class A: + def __init__(self, a, b, c): pass - a = Obj() - b = Obj() - # no attributes given return False - assert not equal_attributes(a, b, []) - # nonexistent attributes - assert not equal_attributes(a, b, ["x", "y"]) - - a.x = 1 - b.x = 1 - # equal attribute - assert equal_attributes(a, b, ["x"]) - - b.y = 2 - # obj1 has no attribute y - assert not equal_attributes(a, b, ["x", "y"]) - - a.y = 2 - # equal attributes - assert equal_attributes(a, b, ["x", "y"]) - - a.y = 1 - # different attributes - assert not equal_attributes(a, b, ["x", "y"]) - - # test callable - a.meta = {} - b.meta = {} - assert equal_attributes(a, b, ["meta"]) - - # compare ['meta']['a'] - a.meta["z"] = 1 - b.meta["z"] = 1 - - get_z = operator.itemgetter("z") - get_meta = operator.attrgetter("meta") - - def compare_z(obj): - return get_z(get_meta(obj)) - - assert equal_attributes(a, b, [compare_z, "x"]) - # fail z equality - a.meta["z"] = 2 - assert not equal_attributes(a, b, [compare_z, "x"]) - - def test_get_func_args(self): - def f1(a, b, c): + def method(self, a, b, c): pass - def f2(a, b=None, c=None): + class Callable: + def __call__(self, a, b, c): pass - def f3(a, b=None, *, c=None): - pass + a = A(1, 2, 3) + cal = Callable() + partial_f1 = functools.partial(f1, None) + partial_f2 = functools.partial(f1, b=None) + partial_f3 = functools.partial(partial_f2, None) - class A: - def __init__(self, a, b, c): - pass + assert get_func_args(f1) == ["a", "b", "c"] + assert get_func_args(f2) == ["a", "b", "c"] + assert get_func_args(f3) == ["a", "b", "c"] + assert get_func_args(A) == ["a", "b", "c"] + assert get_func_args(a.method) == ["a", "b", "c"] + assert get_func_args(partial_f1) == ["b", "c"] + assert get_func_args(partial_f2) == ["a", "c"] + assert get_func_args(partial_f3) == ["c"] + assert get_func_args(cal) == ["a", "b", "c"] + assert get_func_args(object) == [] + assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"] + assert get_func_args(" ".join, stripself=True) == ["iterable"] - def method(self, a, b, c): - pass + if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy": + # the correct and correctly extracted signature + assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"] + elif platform.python_implementation() == "CPython": + # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature + # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951) + assert get_func_args(operator.itemgetter(2), stripself=True) in [ + [], + ["args", "kwargs"], + ] - class Callable: - def __call__(self, a, b, c): - pass - a = A(1, 2, 3) - cal = Callable() - partial_f1 = functools.partial(f1, None) - partial_f2 = functools.partial(f1, b=None) - partial_f3 = functools.partial(partial_f2, None) - - assert get_func_args(f1) == ["a", "b", "c"] - assert get_func_args(f2) == ["a", "b", "c"] - assert get_func_args(f3) == ["a", "b", "c"] - assert get_func_args(A) == ["a", "b", "c"] - assert get_func_args(a.method) == ["a", "b", "c"] - assert get_func_args(partial_f1) == ["b", "c"] - assert get_func_args(partial_f2) == ["a", "c"] - assert get_func_args(partial_f3) == ["c"] - assert get_func_args(cal) == ["a", "b", "c"] - assert get_func_args(object) == [] - assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"] - assert get_func_args(" ".join, stripself=True) == ["iterable"] - - if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy": - # the correct and correctly extracted signature - assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"] - elif platform.python_implementation() == "CPython": - # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature - # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951) - assert get_func_args(operator.itemgetter(2), stripself=True) in [ - [], - ["args", "kwargs"], - ] - - def test_without_none_values(self): - assert without_none_values([1, None, 3, 4]) == [1, 3, 4] - assert without_none_values((1, None, 3, 4)) == (1, 3, 4) - assert without_none_values({"one": 1, "none": None, "three": 3, "four": 4}) == { - "one": 1, - "three": 3, - "four": 4, - } +@pytest.mark.parametrize( + ("value", "expected"), + [ + ([1, None, 3, 4], [1, 3, 4]), + ((1, None, 3, 4), (1, 3, 4)), + ( + {"one": 1, "none": None, "three": 3, "four": 4}, + {"one": 1, "three": 3, "four": 4}, + ), + ], +) +def test_without_none_values( + value: Mapping[_KT, _VT] | Iterable[_KT], expected: dict[_KT, _VT] | Iterable[_KT] +) -> None: + assert without_none_values(value) == expected diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 5b8509753..9c4cb7159 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -20,45 +20,52 @@ from scrapy.utils.request import ( from scrapy.utils.test import get_crawler -class TestUtilsRequest: - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_request_authenticate(self): - r = Request("http://www.example.com") - request_authenticate(r, "someuser", "somepass") - assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M=" +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +def test_request_authenticate(): + r = Request("http://www.example.com") + request_authenticate(r, "someuser", "somepass") + assert r.headers["Authorization"] == b"Basic c29tZXVzZXI6c29tZXBhc3M=" - def test_request_httprepr(self): - r1 = Request("http://www.example.com") - assert ( - request_httprepr(r1) == b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n" - ) - r1 = Request("http://www.example.com/some/page.html?arg=1") - assert ( - request_httprepr(r1) - == b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n" - ) +@pytest.mark.parametrize( + ("r", "expected"), + [ + ( + Request("http://www.example.com"), + b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + ), + ( + Request("http://www.example.com/some/page.html?arg=1"), + b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n", + ), + ( + Request( + "http://www.example.com", + method="POST", + headers={"Content-type": b"text/html"}, + body=b"Some body", + ), + b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body", + ), + ], +) +def test_request_httprepr(r: Request, expected: bytes) -> None: + assert request_httprepr(r) == expected - r1 = Request( - "http://www.example.com", - method="POST", - headers={"Content-type": b"text/html"}, - body=b"Some body", - ) - assert ( - request_httprepr(r1) - == b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body" - ) - def test_request_httprepr_for_non_http_request(self): - # the representation is not important but it must not fail. - request_httprepr(Request("file:///tmp/foo.txt")) - request_httprepr(Request("ftp://localhost/tmp/foo.txt")) +@pytest.mark.parametrize( + "r", + [ + Request("file:///tmp/foo.txt"), + Request("ftp://localhost/tmp/foo.txt"), + ], +) +def test_request_httprepr_for_non_http_request(r: Request) -> None: + # the representation is not important but it must not fail. + request_httprepr(r) class TestFingerprint: - maxDiff = None - function: staticmethod = staticmethod(fingerprint) cache: ( WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]] @@ -229,35 +236,6 @@ class TestFingerprint: assert actual == expected -REQUEST_OBJECTS_TO_TEST = ( - Request("http://www.example.com/"), - Request("http://www.example.com/query?id=111&cat=222"), - Request("http://www.example.com/query?cat=222&id=111"), - Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"), - Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"), - Request("http://www.example.com/members/offers.html"), - Request( - "http://www.example.com/members/offers.html", - headers={"SESSIONID": b"somehash"}, - ), - Request( - "http://www.example.com/", - headers={"Accept-Language": b"en"}, - ), - Request( - "http://www.example.com/", - headers={ - "Accept-Language": b"en", - "SESSIONID": b"somehash", - }, - ), - Request("http://www.example.com/test.html"), - Request("http://www.example.com/test.html#fragment"), - Request("http://www.example.com", method="POST"), - Request("http://www.example.com", method="POST", body=b"request body"), -) - - class TestRequestFingerprinter: def test_default_implementation(self): crawler = get_crawler() diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 80f2f25d5..179ca49e4 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse import pytest -from scrapy.http import HtmlResponse, Response, TextResponse +from scrapy.http import HtmlResponse, Response from scrapy.utils.python import to_bytes from scrapy.utils.response import ( _remove_html_comments, @@ -15,229 +15,203 @@ from scrapy.utils.response import ( ) -class TestResponseUtils: - dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") +def test_open_in_browser(): + url = "http:///www.example.com/some/page.html" + body = ( + b" test page test body " + ) - def test_open_in_browser(self): - url = "http:///www.example.com/some/page.html" - body = b" test page test body " + def browser_open(burl: str) -> bool: + path = urlparse(burl).path + if not path or not Path(path).exists(): + path = burl.replace("file://", "") + bbody = Path(path).read_bytes() + assert b'' in bbody + return True - def browser_open(burl): - path = urlparse(burl).path - if not path or not Path(path).exists(): - path = burl.replace("file://", "") - bbody = Path(path).read_bytes() - assert b'' in bbody - return True + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=browser_open), "Browser not called" - response = HtmlResponse(url, body=body) - assert open_in_browser(response, _openfunc=browser_open), "Browser not called" + resp = Response(url, body=body) + with pytest.raises(TypeError): + open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg - resp = Response(url, body=body) - with pytest.raises(TypeError): - open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg - def test_get_meta_refresh(self): - r1 = HtmlResponse( - "http://www.example.com", - body=b""" - - Dummy - blahablsdfsal& - """, - ) - r2 = HtmlResponse( - "http://www.example.com", - body=b""" - - Dummy - blahablsdfsal& - """, - ) - r3 = HtmlResponse( - "http://www.example.com", - body=b""" -